From a3f20567cca3e4ba3d3aed21396236fa76372038 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Tue, 14 Mar 2023 10:11:43 -0400 Subject: [PATCH 001/121] add suite for obs filter tests --- src/swell/suites/test_obs_filters/flow.cylc | 126 ++++++ .../suites-test_obs_filters.yaml | 69 +++ .../ufo/eva_observations.yaml | 408 ++++++++++++++++++ .../ufo/suites-ufo_tests-geos_atmosphere.yaml | 109 +++++ 4 files changed, 712 insertions(+) create mode 100644 src/swell/suites/test_obs_filters/flow.cylc create mode 100644 src/swell/suites/test_obs_filters/suites-test_obs_filters.yaml create mode 100644 src/swell/suites/test_obs_filters/ufo/eva_observations.yaml create mode 100644 src/swell/suites/test_obs_filters/ufo/suites-ufo_tests-geos_atmosphere.yaml diff --git a/src/swell/suites/test_obs_filters/flow.cylc b/src/swell/suites/test_obs_filters/flow.cylc new file mode 100644 index 00000000..d9958f0a --- /dev/null +++ b/src/swell/suites/test_obs_filters/flow.cylc @@ -0,0 +1,126 @@ +# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + +# Cylc suite for executing UFO ObsFilters tests + +# -------------------------------------------------------------------------------------------------- + +[scheduler] + UTC mode = True + allow implicit tasks = False + +# -------------------------------------------------------------------------------------------------- + +[scheduling] + + initial cycle point = {{start_cycle_point}} + final cycle point = {{final_cycle_point}} + runahead limit = {{runahead_limit}} + + [[graph]] + R1 = """ + # Triggers for non cycle time dependent tasks + # ------------------------------------------- + # Clone JEDI source code + CloneJedi + + # Build JEDI source code by linking + CloneJedi => BuildJediByLinking? + + # If not able to link to build create the build + BuildJediByLinking:fail? => BuildJedi + """ + + {% for cycle_time in cycle_times %} + {{cycle_time.cycle_time}} = """ + # Task triggers + # ------------- + # Get GeoVaLs + GetGeovals + + # Get observations + GetObservations + + # Run Jedi hofx executable + BuildJediByLinking[^]? | BuildJedi[^] => RunJediTestObsFiltersExecutable + GetGeovals => RunJediTestObsFiltersExecutable + GetObservations => RunJediTestObsFiltersExecutable + + # EvaObservations + RunJediTestObsFiltersExecutable => EvaObservations + + # Save observations + RunJediTestObsFiltersExecutable => SaveObsDiags + + # Clean up large files + EvaObservations & SaveObsDiags => CleanCycle + + """ + {% endfor %} + +# -------------------------------------------------------------------------------------------------- + +[runtime] + + # Task defaults + # ------------- + [[root]] + pre-script = "source $CYLC_SUITE_DEF_PATH/modules" + + [[[environment]]] + datetime = $CYLC_TASK_CYCLE_POINT + config = $CYLC_SUITE_DEF_PATH/experiment.yaml + + # Tasks + # ----- + [[CloneJedi]] + script = "swell_task CloneJedi $config" + + [[BuildJediByLinking]] + script = "swell_task BuildJediByLinking $config" + + [[BuildJedi]] + script = "swell_task BuildJedi $config" + platform = {{platform}} + execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} + [[[directives]]] + --account = {{scheduling["BuildJedi"]["account"]}} + --qos = {{scheduling["BuildJedi"]["qos"]}} + --job-name = BuildJedi + --nodes={{scheduling["BuildJedi"]["nodes"]}} + --ntasks-per-node={{scheduling["BuildJedi"]["ntasks_per_node"]}} + --constraint={{scheduling["BuildJedi"]["constraint"]}} + + [[ GetGeovals ]] + script = "swell_task GetGeovals $config -d $datetime -m ufo" + + [[GetObservations]] + script = "swell_task GetObservations $config -d $datetime -m ufo" + + [[RunJediTestObsFiltersExecutable]] + script = "swell_task RunJediTestObsFiltersExecutable $config -d $datetime -m ufo" + platform = {{platform}} + execution time limit = {{scheduling["RunJediTestObsFiltersExecutable"]["execution_time_limit"]}} + [[[directives]]] + --account = {{scheduling["RunJediTestObsFiltersExecutable"]["account"]}} + --qos = {{scheduling["RunJediTestObsFiltersExecutable"]["qos"]}} + --job-name = RunJediTestObsFiltersExecutable + --nodes={{scheduling["RunJediTestObsFiltersExecutable"]["nodes"]}} + --ntasks-per-node={{scheduling["RunJediTestObsFiltersExecutable"]["ntasks_per_node"]}} + --constraint={{scheduling["RunJediTestObsFiltersExecutable"]["constraint"]}} + + [[EvaObservations]] + script = "swell_task EvaObservations $config -d $datetime -m ufo" + + [[SaveObsDiags]] + script = "swell_task SaveObsDiags $config -d $datetime -m ufo" + + [[CleanCycle]] + script = "swell_task CleanCycle $config -d $datetime -m ufo" + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/test_obs_filters/suites-test_obs_filters.yaml b/src/swell/suites/test_obs_filters/suites-test_obs_filters.yaml new file mode 100644 index 00000000..c599a16f --- /dev/null +++ b/src/swell/suites/test_obs_filters/suites-test_obs_filters.yaml @@ -0,0 +1,69 @@ +# Cycle start point +start_cycle_point: + default_value: '2020-12-15T00:00:00Z' + prompt: What is the time of the first cycle (middle of the window)? + type: iso-datetime + +# Cycle final point +final_cycle_point: + default_value: '2020-12-15T06:00:00Z' + prompt: What is the time of the final cycle (middle of the window)? + type: iso-datetime + +# Run ahead limit for workflow +runahead_limit: + default_value: 'P4' + prompt: Since this suite is non-cycling choose how many hours the workflow can run ahead? + type: string + +# Jedi build system +jedi_build_method: + default_value: use_existing + prompt: Create a new JEDI build or use an existing build? + options: ['create', 'use_existing'] + type: string-drop-list + +# Existing JEDI bundle directory +existing_source_directory: + default_value: {{existing_source_directory}} + prompt: Provide the path to an existing JEDI bundle directory containing source code repos + type: string + depends: + key: jedi_build_method + value: use_existing + +# Existing JEDI build +existing_build_directory: + default_value: {{existing_build_directory}} + prompt: Provide the path to an existing JEDI build directory + type: string + depends: + key: jedi_build_method + value: use_existing + +# R2D2 Configuration +r2d2_local_path: + default_value: {{r2d2_local_path}} + prompt: Enter the path where R2D2 will store experiment output + type: string + +# What kind of coupling is needed +coupling_style: + default_value: uncoupled + prompt: What kind of coupling would you like to run with? + options: ['uncoupled', 'weakly coupled', 'strongly coupled'] + type: string-drop-list + +# Models to use +model_components: + default_value: ['geos_atmosphere', 'geos_ocean'] + prompt: Select models to use (choose at least one) + options: use_method + type: file-check-list + +# Fixed options +fixed_options: + data_assimilation_run: + default_value: true + prompt: Does this workflow include data assimilation? + diff --git a/src/swell/suites/test_obs_filters/ufo/eva_observations.yaml b/src/swell/suites/test_obs_filters/ufo/eva_observations.yaml new file mode 100644 index 00000000..220f8744 --- /dev/null +++ b/src/swell/suites/test_obs_filters/ufo/eva_observations.yaml @@ -0,0 +1,408 @@ +diagnostics: + +- data: + type: IodaObsSpace + datasets: + + - name: experiment + filenames: + - {{obs_path_file}} + channels: &channels {{channels}} + groups: + - name: ObsValue + variables: &variables {{simulated_variables}} + - name: GsiHofXBc + #- name: GsiEffectiveQC + - name: hofx + - name: EffectiveQC + - name: MetaData + + transforms: + + # Generate omb for GSI + - transform: arithmetic + new name: experiment::ObsValueMinusGsiHofXBc::${variable} + equals: experiment::ObsValue::${variable}-experiment::GsiHofXBc::${variable} + for: + variable: *variables + + # Generate omb for JEDI + - transform: arithmetic + new name: experiment::ObsValueMinusHofx::${variable} + equals: experiment::ObsValue::${variable}-experiment::hofx::${variable} + for: + variable: *variables + + # Generate hofx difference + - transform: arithmetic + new name: experiment::HofxMinusGsiHofXBc::${variable} + equals: experiment::hofx::${variable}-experiment::GsiHofXBc::${variable} + for: + variable: *variables + + # Generate hofx that passed QC for JEDI + - transform: accept where + new name: experiment::hofxPassedQc::${variable} + starting field: experiment::hofx::${variable} + where: + - experiment::EffectiveQC::${variable} == 0 + for: + variable: *variables + + # Generate GSI hofx that passed JEDI QC + - transform: accept where + new name: experiment::GsiHofXBcPassedQc::${variable} + starting field: experiment::GsiHofXBc::${variable} + where: + - experiment::EffectiveQC::${variable} == 0 + for: + variable: *variables + + # Generate omb that passed QC for JEDI + - transform: accept where + new name: experiment::ObsValueMinushofxPassedQc::${variable} + starting field: experiment::ObsValueMinusHofx::${variable} + where: + - experiment::EffectiveQC::${variable} == 0 + for: + variable: *variables + + # Generate omb that passed QC for GSI + - transform: accept where + new name: experiment::ObsValueMinusGsiHofXBcPassedQc::${variable} + starting field: experiment::ObsValueMinusGsiHofXBc::${variable} + where: + - experiment::EffectiveQC::${variable} == 0 + for: + variable: *variables + + graphics: + + # Correlation scatter plots + # ------------------------- + + # JEDI h(x) vs Observations + - batch figure: + variables: *variables + channels: *channels + figure: + layout: [1,1] + title: 'Observations vs. JEDI h(x) | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/correlation_scatter/${variable}${channel}/jedi_hofx_vs_obs_{{instrument}}_${variable}${channel}.png' + plots: + - add_xlabel: 'Observation Value' + add_ylabel: 'JEDI h(x)' + add_grid: + add_legend: + loc: 'upper left' + layers: + - type: Scatter + x: + variable: experiment::ObsValue::${variable} + y: + variable: experiment::hofx::${variable} + channel: ${channel} + markersize: 5 + color: 'black' + label: 'JEDI h(x) versus obs (all obs)' + - type: Scatter + x: + variable: experiment::ObsValue::${variable} + y: + variable: experiment::hofxPassedQc::${variable} + channel: ${channel} + markersize: 5 + color: 'red' + label: 'JEDI h(x) versus obs (passed QC in JEDI)' + + # GSI h(x) vs Observations + - batch figure: + variables: *variables + channels: *channels + figure: + layout: [1,1] + title: 'Observations vs. GSI h(x) | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/correlation_scatter/${variable}${channel}/gsi_hofx_vs_obs_{{instrument}}_${variable}${channel}.png' + plots: + - add_xlabel: 'Observation Value' + add_ylabel: 'GSI h(x)' + add_grid: + add_legend: + loc: 'upper left' + layers: + - type: Scatter + x: + variable: experiment::ObsValue::${variable} + y: + variable: experiment::GsiHofXBc::${variable} + channel: ${channel} + markersize: 5 + color: 'black' + label: 'GSI h(x) versus obs (all obs)' + - type: Scatter + x: + variable: experiment::ObsValue::${variable} + y: + variable: experiment::GsiHofXBcPassedQc::${variable} + channel: ${channel} + markersize: 5 + color: 'red' + label: 'GSI h(x) versus obs (passed QC in JEDI)' + + # JEDI h(x) vs GSI h(x) + - batch figure: + variables: *variables + channels: *channels + figure: + layout: [1,1] + title: 'JEDI h(x) vs. GSI h(x) | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/correlation_scatter/${variable}${channel}/gsi_hofx_vs_jedi_hofx_{{instrument}}_${variable}${channel}.png' + plots: + - add_xlabel: 'GSI h(x)' + add_ylabel: 'JEDI h(x)' + add_grid: + add_legend: + loc: 'upper left' + layers: + - type: Scatter + x: + variable: experiment::GsiHofXBc::${variable} + y: + variable: experiment::hofx::${variable} + channel: ${channel} + markersize: 5 + color: 'black' + label: 'JEDI h(x) versus GSI h(x)' + - type: Scatter + x: + variable: experiment::GsiHofXBcPassedQc::${variable} + y: + variable: experiment::hofxPassedQc::${variable} + channel: ${channel} + markersize: 5 + color: 'red' + label: 'JEDI h(x) versus GSI h(x) (passed QC in JEDI)' + + # JEDI omb vs GSI omb + - batch figure: + variables: *variables + channels: *channels + figure: + layout: [1,1] + title: 'JEDI omb vs. GSI omb | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/correlation_scatter/${variable}${channel}/gsi_omb_vs_jedi_omb_{{instrument}}_${variable}${channel}.png' + plots: + - add_xlabel: 'GSI observation minus h(x)' + add_ylabel: 'JEDI observation minus h(x)' + add_grid: + add_legend: + loc: 'upper left' + layers: + - type: Scatter + x: + variable: experiment::ObsValueMinusGsiHofXBc::${variable} + y: + variable: experiment::ObsValueMinusHofx::${variable} + channel: ${channel} + markersize: 5 + color: 'black' + label: 'GSI omb vs JEDI omb (all obs)' + - type: Scatter + x: + variable: experiment::ObsValueMinusGsiHofXBcPassedQc::${variable} + y: + variable: experiment::ObsValueMinushofxPassedQc::${variable} + channel: ${channel} + markersize: 5 + color: 'red' + label: 'GSI omb vs JEDI omb (passed QC in JEDI)' + +# Map plots +# --------- + + # Observations + - batch figure: + variables: *variables + channels: *channels + dynamic options: + - type: vminvmaxcmap + channel: ${channel} + data variable: experiment::ObsValue::${variable} + figure: + figure size: [20,10] + layout: [1,1] + title: 'Observations | {{instrument_title}} | Obs Value' + output name: '{{cycle_dir}}/eva/{{instrument}}/map_plots/${variable}${channel}/observations_{{instrument}}_${variable}${channel}.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: ObsValue + add_grid: + layers: + - type: MapScatter + longitude: + variable: experiment::MetaData::longitude + latitude: + variable: experiment::MetaData::latitude + data: + variable: experiment::ObsValue::${variable} + channel: ${channel} + markersize: 2 + label: ObsValue + colorbar: true + cmap: ${dynamic_cmap} + vmin: ${dynamic_vmin} + vmax: ${dynamic_vmax} + + # omb jedi + - batch figure: + variables: *variables + channels: *channels + dynamic options: + - type: vminvmaxcmap + channel: ${channel} + data variable: experiment::ObsValueMinusHofx::${variable} + figure: + figure size: [20,10] + layout: [1,1] + title: 'JEDI OmB | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/map_plots/${variable}${channel}/omb_jedi_{{instrument}}_${variable}${channel}.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: '${variable}' + add_grid: + layers: + - type: MapScatter + longitude: + variable: experiment::MetaData::longitude + latitude: + variable: experiment::MetaData::latitude + data: + variable: experiment::ObsValueMinusHofx::${variable} + channel: ${channel} + markersize: 2 + label: '${variable}' + colorbar: true + cmap: ${dynamic_cmap} + vmin: ${dynamic_vmin} + vmax: ${dynamic_vmax} + + # omb gsi + - batch figure: + variables: *variables + channels: *channels + dynamic options: + - type: vminvmaxcmap + channel: ${channel} + data variable: experiment::ObsValueMinusGsiHofXBc::${variable} + figure: + figure size: [20,10] + layout: [1,1] + title: 'GSI OmB | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/map_plots/${variable}${channel}/omb_gsi_{{instrument}}_${variable}${channel}.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: '${variable}' + add_grid: + layers: + - type: MapScatter + longitude: + variable: experiment::MetaData::longitude + latitude: + variable: experiment::MetaData::latitude + data: + variable: experiment::ObsValueMinusGsiHofXBc::${variable} + channel: ${channel} + markersize: 2 + label: '${variable}' + colorbar: true + cmap: ${dynamic_cmap} + vmin: ${dynamic_vmin} + vmax: ${dynamic_vmax} + + # hofx difference + - batch figure: + variables: *variables + channels: *channels + dynamic options: + - type: vminvmaxcmap + channel: ${channel} + data variable: experiment::HofxMinusGsiHofXBc::${variable} + figure: + figure size: [20,10] + layout: [1,1] + title: 'Hofx Difference | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/map_plots/${variable}${channel}/hofx_difference_{{instrument}}_${variable}${channel}.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: '${variable}' + add_grid: + layers: + - type: MapScatter + longitude: + variable: experiment::MetaData::longitude + latitude: + variable: experiment::MetaData::latitude + data: + variable: experiment::HofxMinusGsiHofXBc::${variable} + channel: ${channel} + markersize: 2 + label: '${variable}' + colorbar: true + cmap: ${dynamic_cmap} + vmin: ${dynamic_vmin} + vmax: ${dynamic_vmax} + +# Histogram plots +# --------------- + + # omb vs omb + - batch figure: + variables: *variables + channels: *channels + dynamic options: + - type: histogram_bins + channel: ${channel} + number of bins rule: sturges + data variable: experiment::ObsValueMinusHofx::${variable} + figure: + layout: [1,1] + title: 'JEDI omb vs. GSI omb | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/histograms/${variable}${channel}/gsi_omb_vs_jedi_omb_{{instrument}}_${variable}${channel}.png' + plots: + - add_xlabel: 'Observation minus h(x)' + add_ylabel: 'Count' + add_legend: + loc: 'upper left' + layers: + - type: Histogram + data: + variable: experiment::ObsValueMinusGsiHofXBc::${variable} + channel: ${channel} + color: 'blue' + label: 'GSI omb (all obs)' + bins: ${dynamic_bins} + alpha: 0.5 + - type: Histogram + data: + variable: experiment::ObsValueMinusHofx::${variable} + channel: ${channel} + color: 'red' + label: 'JEDI omb (all obs)' + bins: ${dynamic_bins} + alpha: 0.5 diff --git a/src/swell/suites/test_obs_filters/ufo/suites-ufo_tests-geos_atmosphere.yaml b/src/swell/suites/test_obs_filters/ufo/suites-ufo_tests-geos_atmosphere.yaml new file mode 100644 index 00000000..6d4a7a89 --- /dev/null +++ b/src/swell/suites/test_obs_filters/ufo/suites-ufo_tests-geos_atmosphere.yaml @@ -0,0 +1,109 @@ +# Cycle frequency +cycle_times: + default_value: ['T00', 'T06', 'T12', 'T18'] + prompt: List all cycle times, by middle of the window? + options: ['T00', 'T06', 'T12', 'T18'] + type: string-check-list + +# Window length +window_length: + default_value: 'PT6H' + prompt: Window length + type: iso-duration + +# Horizontal resolution +horizontal_resolution: + default_value: '361' + prompt: What resolution for the atmospheric background (For c360 choose 361)? + options: ['181', '361', '721'] + type: string-drop-list + +# Vertical resolution +vertical_resolution: + default_value: '72' + prompt: What vertical resolution for the atmospheric background? + options: ['72'] + type: string-drop-list + +# Processor layout +npx_proc: + default_value: '2' + prompt: What processor layout (x-direction) per cube face? + type: string +npy_proc: + default_value: '2' + prompt: What processor layout (y-direction) per cube face? + type: string + +# Backgrounds +background_experiment: + default_value: x0044 + prompt: Experiment providing the background files? + type: string + +# Observation experiment +obs_experiment: + default_value: x0044_v3 + prompt: Experiment providing the observations? + type: string + +# Observations +observations: + default_value: + # Conventional + - aircraft + # Radiances + - amsua_n19 + - amsua_aqua + - amsua_metop-a + - amsua_metop-b + - amsua_metop-c + - amsua_n15 + - amsua_n18 + - amsua_n19 + - gmi_gpm + - amsr2_gcom-w1 + # Ozone + - mls55_aura + - omi_aura + - ompsnm_npp + prompt: Select atmospheric observations. + options: use_method + type: string-check-list + +# Fixed options (user not prompted for these) +# ------------------------------------------- +fixed_options: + window_type: + default_value: 4D + prompt: Window type (3D or 4D) + background_frequency: + default_value: PT1H + prompt: 'Frequency of 4D backgrounds' + background_source: + default_value: file + prompt: 'Source of 4D background files (file or from model)' + crtm_coeff_dir: + default_value: {{crtm_coeff_dir}} + prompt: 'Directory containing the CRTM coefficient files' + window_offset: + default_value: PT3H + prompt: Time from beginning to middle of the window + analysis_forecast_window_offset: + default_value: -PT3H + prompt: Time from the middle of the window when forecasts start + total_processors: + default_value: 'npx_proc * npy_proc * 6' + prompt: Equation to compute total number of processors + model: + default_value: pseudo-model + prompt: Type of model to use for 4D runs. + background_time_offset: + default_value: PT9H + prompt: Time before the middle of the window that the background providing forecast began + clean_patterns: + default_value: ['*.nc4','*.txt','logfile.*.out'] + prompt: 'Patterns for the files to remove after completing a cycle' + obs_provider: + default_value: ncdiag + prompt: Database providing the observations. From f8bf7ddff920b40c4edb52f86d90fe51f8c73753 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 22 Mar 2023 21:19:19 -0400 Subject: [PATCH 002/121] add ncdiag tasks --- .../platforms/nccs_discover/modules | 3 + src/swell/deployment/prep_suite.py | 8 ++ src/swell/suites/test_obs_filters/flow.cylc | 17 ++- .../suites-test_obs_filters.yaml | 2 +- .../ufo/suites-test_obs_filters-ufo.yaml | 52 +++++++++ .../ufo/suites-ufo_tests-geos_atmosphere.yaml | 109 ------------------ src/swell/tasks/base/task_registry.py | 2 + src/swell/tasks/get_gsi_ncdiag.py | 52 +++++++++ src/swell/tasks/gsi_ncdiag_to_ioda.py | 52 +++++++++ 9 files changed, 177 insertions(+), 120 deletions(-) create mode 100644 src/swell/suites/test_obs_filters/ufo/suites-test_obs_filters-ufo.yaml delete mode 100644 src/swell/suites/test_obs_filters/ufo/suites-ufo_tests-geos_atmosphere.yaml create mode 100644 src/swell/tasks/get_gsi_ncdiag.py create mode 100644 src/swell/tasks/gsi_ncdiag_to_ioda.py diff --git a/src/swell/deployment/platforms/nccs_discover/modules b/src/swell/deployment/platforms/nccs_discover/modules index 5cb86416..f32eef25 100644 --- a/src/swell/deployment/platforms/nccs_discover/modules +++ b/src/swell/deployment/platforms/nccs_discover/modules @@ -30,6 +30,9 @@ module load sp/2.3.3 # ------------------------------------------- PYTHONPATH={{experiment_root}}/{{experiment_id}}/jedi_bundle/build/lib/python3.9:$PYTHONPATH PYTHONPATH={{experiment_root}}/{{experiment_id}}/jedi_bundle/build/lib/python3.9/pyioda:$PYTHONPATH +PYTHONPATH={{experiment_root}}/{{experiment_id}}/jedi_bundle/source/iodaconv/src:$PYTHONPATH + +export PYTHONPATH=$PYTHONPATH:/path/to/clone/iodaconv/src # Load GMAO modules # ----------------- diff --git a/src/swell/deployment/prep_suite.py b/src/swell/deployment/prep_suite.py index 4bfae12e..bb9623af 100644 --- a/src/swell/deployment/prep_suite.py +++ b/src/swell/deployment/prep_suite.py @@ -101,6 +101,14 @@ def prepare_cylc_suite_jinja2(logger, swell_suite_path, exp_suite_path, experime render_dictionary['scheduling']['RunGeosExecutable']['ntasks_per_node'] = 24 render_dictionary['scheduling']['RunGeosExecutable']['constraint'] = 'cas' + render_dictionary['scheduling']['RunJediTestObsFiltersExecutable'] = {} + render_dictionary['scheduling']['RunJediTestObsFiltersExecutable']['execution_time_limit'] = 'PT30M' + render_dictionary['scheduling']['RunJediTestObsFiltersExecutable']['account'] = 'g0613' + render_dictionary['scheduling']['RunJediTestObsFiltersExecutable']['qos'] = 'allnccs' + render_dictionary['scheduling']['RunJediTestObsFiltersExecutable']['nodes'] = 1 + render_dictionary['scheduling']['RunJediTestObsFiltersExecutable']['ntasks_per_node'] = 1 + render_dictionary['scheduling']['RunJediTestObsFiltersExecutable']['constraint'] = 'cas' + # Render the template # ------------------- new_suite_file = template_string_jinja2(logger, suite_file, render_dictionary) diff --git a/src/swell/suites/test_obs_filters/flow.cylc b/src/swell/suites/test_obs_filters/flow.cylc index d9958f0a..20c3d4ac 100644 --- a/src/swell/suites/test_obs_filters/flow.cylc +++ b/src/swell/suites/test_obs_filters/flow.cylc @@ -40,16 +40,13 @@ {{cycle_time.cycle_time}} = """ # Task triggers # ------------- - # Get GeoVaLs - GetGeovals + GetGsiNcdiags - # Get observations - GetObservations + GetGsiNcdiags => GsiNcdiagsToIoda # Run Jedi hofx executable BuildJediByLinking[^]? | BuildJedi[^] => RunJediTestObsFiltersExecutable - GetGeovals => RunJediTestObsFiltersExecutable - GetObservations => RunJediTestObsFiltersExecutable + GsiNcdiagsToIoda => RunJediTestObsFiltersExecutable # EvaObservations RunJediTestObsFiltersExecutable => EvaObservations @@ -96,11 +93,11 @@ --ntasks-per-node={{scheduling["BuildJedi"]["ntasks_per_node"]}} --constraint={{scheduling["BuildJedi"]["constraint"]}} - [[ GetGeovals ]] - script = "swell_task GetGeovals $config -d $datetime -m ufo" + [[ GetGsiNcdiags ]] + script = "swell_task GetGsiNcdiags $config -d $datetime -m ufo" - [[GetObservations]] - script = "swell_task GetObservations $config -d $datetime -m ufo" + [[ GsiNcdiagsToIoda ]] + script = "swell_task GsiNcdiagsToIoda $config -d $datetime -m ufo" [[RunJediTestObsFiltersExecutable]] script = "swell_task RunJediTestObsFiltersExecutable $config -d $datetime -m ufo" diff --git a/src/swell/suites/test_obs_filters/suites-test_obs_filters.yaml b/src/swell/suites/test_obs_filters/suites-test_obs_filters.yaml index c599a16f..7b9d862d 100644 --- a/src/swell/suites/test_obs_filters/suites-test_obs_filters.yaml +++ b/src/swell/suites/test_obs_filters/suites-test_obs_filters.yaml @@ -56,7 +56,7 @@ coupling_style: # Models to use model_components: - default_value: ['geos_atmosphere', 'geos_ocean'] + default_value: ['ufo'] prompt: Select models to use (choose at least one) options: use_method type: file-check-list diff --git a/src/swell/suites/test_obs_filters/ufo/suites-test_obs_filters-ufo.yaml b/src/swell/suites/test_obs_filters/ufo/suites-test_obs_filters-ufo.yaml new file mode 100644 index 00000000..46dd6f9d --- /dev/null +++ b/src/swell/suites/test_obs_filters/ufo/suites-test_obs_filters-ufo.yaml @@ -0,0 +1,52 @@ +# Cycle frequency +cycle_times: + default_value: ['T00', 'T06', 'T12', 'T18'] + prompt: List all cycle times, by middle of the window? + options: ['T00', 'T06', 'T12', 'T18'] + type: string-check-list + +# Window length +window_length: + default_value: 'PT6H' + prompt: Window length + type: iso-duration + +# Observations +observations: + default_value: + - aircraft + - amsua_n19 + prompt: Select observations to run with. + options: use_method + type: string-check-list + +# Path to GSI ncdiags +path_to_gsi_diags: + default_value: '/discover/nobackup/drholdaw/SwellTestData/ncdiag/' + prompt: Path to where the ncdiags will be held + type: string + +# Fixed options (user not prompted for these) +# ------------------------------------------- +fixed_options: + crtm_coeff_dir: + default_value: {{crtm_coeff_dir}} + prompt: 'Directory containing the CRTM coefficient files' + total_processors: + default_value: 1 + prompt: Equation to compute total number of processors + clean_patterns: + default_value: ['*.nc4','*.txt','logfile.*.out'] + prompt: 'Patterns for the files to remove after completing a cycle' + + + # Needed because of config gen + window_type: + default_value: 4D + prompt: Window type (3D or 4D) + window_offset: + default_value: PT3H + prompt: Time from beginning to middle of the window + background_time_offset: + default_value: PT9H + prompt: Time before the middle of the window that the background providing forecast began diff --git a/src/swell/suites/test_obs_filters/ufo/suites-ufo_tests-geos_atmosphere.yaml b/src/swell/suites/test_obs_filters/ufo/suites-ufo_tests-geos_atmosphere.yaml deleted file mode 100644 index 6d4a7a89..00000000 --- a/src/swell/suites/test_obs_filters/ufo/suites-ufo_tests-geos_atmosphere.yaml +++ /dev/null @@ -1,109 +0,0 @@ -# Cycle frequency -cycle_times: - default_value: ['T00', 'T06', 'T12', 'T18'] - prompt: List all cycle times, by middle of the window? - options: ['T00', 'T06', 'T12', 'T18'] - type: string-check-list - -# Window length -window_length: - default_value: 'PT6H' - prompt: Window length - type: iso-duration - -# Horizontal resolution -horizontal_resolution: - default_value: '361' - prompt: What resolution for the atmospheric background (For c360 choose 361)? - options: ['181', '361', '721'] - type: string-drop-list - -# Vertical resolution -vertical_resolution: - default_value: '72' - prompt: What vertical resolution for the atmospheric background? - options: ['72'] - type: string-drop-list - -# Processor layout -npx_proc: - default_value: '2' - prompt: What processor layout (x-direction) per cube face? - type: string -npy_proc: - default_value: '2' - prompt: What processor layout (y-direction) per cube face? - type: string - -# Backgrounds -background_experiment: - default_value: x0044 - prompt: Experiment providing the background files? - type: string - -# Observation experiment -obs_experiment: - default_value: x0044_v3 - prompt: Experiment providing the observations? - type: string - -# Observations -observations: - default_value: - # Conventional - - aircraft - # Radiances - - amsua_n19 - - amsua_aqua - - amsua_metop-a - - amsua_metop-b - - amsua_metop-c - - amsua_n15 - - amsua_n18 - - amsua_n19 - - gmi_gpm - - amsr2_gcom-w1 - # Ozone - - mls55_aura - - omi_aura - - ompsnm_npp - prompt: Select atmospheric observations. - options: use_method - type: string-check-list - -# Fixed options (user not prompted for these) -# ------------------------------------------- -fixed_options: - window_type: - default_value: 4D - prompt: Window type (3D or 4D) - background_frequency: - default_value: PT1H - prompt: 'Frequency of 4D backgrounds' - background_source: - default_value: file - prompt: 'Source of 4D background files (file or from model)' - crtm_coeff_dir: - default_value: {{crtm_coeff_dir}} - prompt: 'Directory containing the CRTM coefficient files' - window_offset: - default_value: PT3H - prompt: Time from beginning to middle of the window - analysis_forecast_window_offset: - default_value: -PT3H - prompt: Time from the middle of the window when forecasts start - total_processors: - default_value: 'npx_proc * npy_proc * 6' - prompt: Equation to compute total number of processors - model: - default_value: pseudo-model - prompt: Type of model to use for 4D runs. - background_time_offset: - default_value: PT9H - prompt: Time before the middle of the window that the background providing forecast began - clean_patterns: - default_value: ['*.nc4','*.txt','logfile.*.out'] - prompt: 'Patterns for the files to remove after completing a cycle' - obs_provider: - default_value: ncdiag - prompt: Database providing the observations. diff --git a/src/swell/tasks/base/task_registry.py b/src/swell/tasks/base/task_registry.py index 267c24fa..9a70429d 100644 --- a/src/swell/tasks/base/task_registry.py +++ b/src/swell/tasks/base/task_registry.py @@ -18,8 +18,10 @@ 'GenerateBClimatology', 'GetBackgroundGeosExperiment', 'GetBackground', + 'GetGsiNcdiag', 'GetObservations', 'GetRestart', + 'GsiNcdiagToIoda', 'ObsProcessSetup', 'PrepGeosRunDir', 'RunGeosExecutable', diff --git a/src/swell/tasks/get_gsi_ncdiag.py b/src/swell/tasks/get_gsi_ncdiag.py new file mode 100644 index 00000000..75bc6963 --- /dev/null +++ b/src/swell/tasks/get_gsi_ncdiag.py @@ -0,0 +1,52 @@ +# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import glob +import os +import shutil + +from swell.tasks.base.task_base import taskBase + + +# -------------------------------------------------------------------------------------------------- + + +class GetGsiNcdiag(taskBase): + + def execute(self): + + # Get the build method + # -------------------- + gsi_diag_path = self.config_get('path_to_gsi_diags') + + # Get list of ncdiags to test with + # -------------------------------- + gsi_diag_path_files_pattern = os.path.join(gsi_diag_path, '*ges*.nc*') + gsi_diag_path_files = glob.glob(gsi_diag_path_files_pattern) + + # Get cycle dir and create if needed + # ---------------------------------- + cycle_dir = self.config_get('cycle_dir') + os.makedirs(cycle_dir, 0o755, exist_ok=True) + + # Copy all the files into the cycle directory + # ------------------------------------------- + for gsi_diag_path_file in gsi_diag_path_files: + + # File name + gsi_diag_file = os.path.basename(gsi_diag_path_file) + + self.logger.info(f'Copying {gsi_diag_file}') + + # Copy file + shutil.copyfile(gsi_diag_path_file, os.path.join(cycle_dir, gsi_diag_file)) + + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/gsi_ncdiag_to_ioda.py b/src/swell/tasks/gsi_ncdiag_to_ioda.py new file mode 100644 index 00000000..1deeeb93 --- /dev/null +++ b/src/swell/tasks/gsi_ncdiag_to_ioda.py @@ -0,0 +1,52 @@ +# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import glob +import os +import shutil + +import gsi_ncdiag.gsi_ncdiag as gsid + +from swell.tasks.base.task_base import taskBase + + +# -------------------------------------------------------------------------------------------------- + + +class GsiNcdiagToIoda(taskBase): + + def execute(self): + + # Parse configuration + # ------------------- + cycle_dir = self.config_get('cycle_dir') + observations = self.config_get('observations') + + # Assemble all conventional types from ioda + # ----------------------------------------- + conv_platforms = gsid.conv_platforms + all_conv = [] + print(conv_platforms) + for key in conv_platforms: + all_conv.append(conv_platforms[key].items()) + + print(all_conv) + + # List out the conventional types + # ------------------------------- + conv_types = ['aircraft', '', '', '', ''] + + # Copy all the files into the cycle directory + # ------------------------------------------- + for observation in observations: + + # File name + self.logger.info(f'{observation}') +# -------------------------------------------------------------------------------------------------- From 84f29573cee740a71d14744d3fdbaf0d6d6e3407 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Fri, 31 Mar 2023 12:53:30 -0400 Subject: [PATCH 003/121] converter working (no geovals) --- .../jedi/oops/TestObsFilters.yaml | 3 + .../platforms/nccs_discover/modules | 9 +- .../ufo/suites-test_obs_filters-ufo.yaml | 41 ++++ src/swell/tasks/get_gsi_ncdiag.py | 5 +- src/swell/tasks/gsi_ncdiag_to_ioda.py | 183 ++++++++++++++++-- .../run_jedi_test_obs_filters_executable.py | 85 ++++++++ 6 files changed, 306 insertions(+), 20 deletions(-) create mode 100644 src/swell/configuration/jedi/oops/TestObsFilters.yaml create mode 100644 src/swell/tasks/run_jedi_test_obs_filters_executable.py diff --git a/src/swell/configuration/jedi/oops/TestObsFilters.yaml b/src/swell/configuration/jedi/oops/TestObsFilters.yaml new file mode 100644 index 00000000..2bdeaf7b --- /dev/null +++ b/src/swell/configuration/jedi/oops/TestObsFilters.yaml @@ -0,0 +1,3 @@ +window begin: '{{window_begin_iso}}' +window end: '{{window_end_iso}}' +observations: SPECIALobservations diff --git a/src/swell/deployment/platforms/nccs_discover/modules b/src/swell/deployment/platforms/nccs_discover/modules index f32eef25..5ef9c23a 100644 --- a/src/swell/deployment/platforms/nccs_discover/modules +++ b/src/swell/deployment/platforms/nccs_discover/modules @@ -26,13 +26,14 @@ module load jedi-fv3-env/1.0.0 module load nco/5.0.6 module load sp/2.3.3 -# Add IODA to PYTHONPATH in case it is needed -# ------------------------------------------- +# IODA from experiement JEDI build +# -------------------------------- PYTHONPATH={{experiment_root}}/{{experiment_id}}/jedi_bundle/build/lib/python3.9:$PYTHONPATH PYTHONPATH={{experiment_root}}/{{experiment_id}}/jedi_bundle/build/lib/python3.9/pyioda:$PYTHONPATH -PYTHONPATH={{experiment_root}}/{{experiment_id}}/jedi_bundle/source/iodaconv/src:$PYTHONPATH -export PYTHONPATH=$PYTHONPATH:/path/to/clone/iodaconv/src +# IODA-Converters from experiement JEDI source +# -------------------------------------------- +PYTHONPATH=/discover/nobackup/drholdaw/JediSwell/iodaconv-v3/src:$PYTHONPATH # Load GMAO modules # ----------------- diff --git a/src/swell/suites/test_obs_filters/ufo/suites-test_obs_filters-ufo.yaml b/src/swell/suites/test_obs_filters/ufo/suites-test_obs_filters-ufo.yaml index 46dd6f9d..168eeea5 100644 --- a/src/swell/suites/test_obs_filters/ufo/suites-test_obs_filters-ufo.yaml +++ b/src/swell/suites/test_obs_filters/ufo/suites-test_obs_filters-ufo.yaml @@ -15,7 +15,40 @@ window_length: observations: default_value: - aircraft + - airs_aqua + - amsr2_gcom-w1 + - amsua_aqua + - amsua_metop-a + - amsua_metop-b + - amsua_metop-c + - amsua_n15 + - amsua_n18 - amsua_n19 + - atms_n20 + - atms_npp + - avhrr_metop-a + - avhrr_n18 + - cris-fsr_n20 + - cris-fsr_npp + - gmi_gpm + - gps + - iasi_metop-a + - iasi_metop-b + - mhs_metop-b + - mhs_metop-c + - mhs_n19 + - mls55_aura + - omi_aura + - ompsnm_npp + - satwind + - scatwind + - seviri_m08 + - sfcship + - sfc + - sondes + - ssmis_f17 + - vadwind + prompt: Select observations to run with. options: use_method type: string-check-list @@ -26,6 +59,12 @@ path_to_gsi_diags: prompt: Path to where the ncdiags will be held type: string +# Compute GeoVaLs and observations +produce_geovals: + default_value: true + prompt: Compute GeoVaLs as well as observations? + type: boolean + # Fixed options (user not prompted for these) # ------------------------------------------- fixed_options: @@ -50,3 +89,5 @@ fixed_options: background_time_offset: default_value: PT9H prompt: Time before the middle of the window that the background providing forecast began + + diff --git a/src/swell/tasks/get_gsi_ncdiag.py b/src/swell/tasks/get_gsi_ncdiag.py index 75bc6963..88a85afe 100644 --- a/src/swell/tasks/get_gsi_ncdiag.py +++ b/src/swell/tasks/get_gsi_ncdiag.py @@ -34,7 +34,8 @@ def execute(self): # Get cycle dir and create if needed # ---------------------------------- cycle_dir = self.config_get('cycle_dir') - os.makedirs(cycle_dir, 0o755, exist_ok=True) + gsi_diag_dir = os.path.join(cycle_dir, 'gsi_ncdiags') + os.makedirs(gsi_diag_dir, 0o755, exist_ok=True) # Copy all the files into the cycle directory # ------------------------------------------- @@ -46,7 +47,7 @@ def execute(self): self.logger.info(f'Copying {gsi_diag_file}') # Copy file - shutil.copyfile(gsi_diag_path_file, os.path.join(cycle_dir, gsi_diag_file)) + shutil.copyfile(gsi_diag_path_file, os.path.join(gsi_diag_dir, gsi_diag_file)) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/gsi_ncdiag_to_ioda.py b/src/swell/tasks/gsi_ncdiag_to_ioda.py index 1deeeb93..c7c79d0b 100644 --- a/src/swell/tasks/gsi_ncdiag_to_ioda.py +++ b/src/swell/tasks/gsi_ncdiag_to_ioda.py @@ -11,11 +11,14 @@ import glob import os import shutil +import netCDF4 as nc +# Ioda converters import gsi_ncdiag.gsi_ncdiag as gsid +from gsi_ncdiag.combine_obsspace import combine_obsspace from swell.tasks.base.task_base import taskBase - +from swell.utilities.shell_commands import run_track_log_subprocess # -------------------------------------------------------------------------------------------------- @@ -26,27 +29,179 @@ def execute(self): # Parse configuration # ------------------- + experiment_root = self.config_get('experiment_root') + experiment_id = self.config_get('experiment_id') cycle_dir = self.config_get('cycle_dir') observations = self.config_get('observations') + produce_geovals = self.config_get('produce_geovals') + + # Directory containing the ncdiags + gsi_diag_dir = os.path.join(cycle_dir, 'gsi_ncdiags') + + # Assemble all conventional types that ioda considers + # --------------------------------------------------- + gsi_to_ioda_dict = gsid.conv_platforms + ioda_types = [] + for key in gsi_to_ioda_dict: + ioda_types += gsi_to_ioda_dict[key] + + # Remove duplicates + ioda_types = list(set(ioda_types)) + + # Invert the dictionary so for each ioda type we know the gsi files that provide data + ioda_to_gsi_dict = {} + for ioda_type in ioda_types: + ioda_to_gsi_dict[ioda_type] = [] + for key in gsi_to_ioda_dict: + if ioda_type in gsi_to_ioda_dict[key]: + ioda_to_gsi_dict[ioda_type].append(key) + + # Get all the elements that are in both observations and ioda_types + needed_ioda_types = [elem for elem in observations if elem in ioda_types] + + # Determine which gsi files need to be processed + gsi_types_to_process = [] + for needed_ioda_type in needed_ioda_types: + gsi_types_to_process += ioda_to_gsi_dict[needed_ioda_type] + gsi_types_to_process = list(set(gsi_types_to_process)) # Unique values only + + # Remove conv_platforms_to_process from the total observations list, leaving rad and ozn + for needed_ioda_type in needed_ioda_types: + observations.remove(needed_ioda_type) + +# # First process the conventional data (if needed) +# # ----------------------------------------------- +# for gsi_type_to_process in gsi_types_to_process: +# +# log_str = f'Processing GSI file {gsi_type_to_process}' +# self.logger.info('', wrap=False) +# self.logger.info(log_str) +# self.logger.info('-'*len(log_str)) +# +# gsi_conv_file = glob.glob(os.path.join(gsi_diag_dir, f'*{gsi_type_to_process}*'))[0] +# +# # Open the file +# Diag = gsid.Conv(gsi_conv_file) +# Diag.read() +# +# # Assemble list of needed platforms +# needed_platforms = [] +# for platform in gsid.conv_platforms[gsi_type_to_process]: +# if platform in needed_ioda_types: +# needed_platforms.append(platform) +# +# # Extract data +# Diag.toIODAobs(cycle_dir, platforms=needed_platforms) +# +# if produce_geovals: +# self.logger.info('', wrap=False) +# self.logger.info(f'Processing GeoVaLs from {gsi_type_to_process}') +# Diag.toGeovals(cycle_dir) +# +# Diag.close() + + # Combine the conventional data + # ----------------------------- + for needed_ioda_type in needed_ioda_types: + + # Logging + log_str = f'Combining IODA files for {needed_ioda_type}' + self.logger.info('', wrap=False) + self.logger.info(log_str) + self.logger.info('-'*len(log_str)) + + # Check dictionary for number of gsi files that were needed + gsi_sources = ioda_to_gsi_dict[needed_ioda_type] + + # Check the number of files that are found + ioda_type_pattern = f'*{needed_ioda_type}*_obs_*' # Pattern, e.g.: *aircraft*_obs_* + + # List of files for that instrument + ioda_path_files = glob.glob(os.path.join(cycle_dir, ioda_type_pattern)) + + # Get last file (first could be type_obs_ if the code already ran) + ioda_file_0 = os.path.basename(ioda_path_files[-1]) + + # Split by underscore + ioda_file_0_ = ioda_file_0.split('_') + + # Logic fails for gps so skip + if needed_ioda_type == 'gps': + ioda_file_0_ = ['gps', 'is', 'skipped'] - # Assemble all conventional types from ioda - # ----------------------------------------- - conv_platforms = gsid.conv_platforms - all_conv = [] - print(conv_platforms) - for key in conv_platforms: - all_conv.append(conv_platforms[key].items()) + # Only combine if the split name has 4 elements + if len(ioda_file_0_) == 4: - print(all_conv) + # If we are here there should be more than one file associated with the ioda type. + # i.e. files with *_uv_*, *_tsen_* for aircraft. + if len(ioda_path_files) == 1: + self.logger.abort(f'Combine issue for {needed_ioda_type}, not multiple files.') - # List out the conventional types - # ------------------------------- - conv_types = ['aircraft', '', '', '', ''] + # Create new file name + new_name_split = ioda_file_0_ + del new_name_split[1] + new_name = os.path.join(cycle_dir, '_'.join(new_name_split)) + + # Check if new file already exists and remove if so + if os.path.exists(new_name): + os.remove(new_name) + if new_name in ioda_path_files: + ioda_path_files.remove(new_name) + + # Run the combine step + geo_dir = None + if produce_geovals: + geo_dir = cycle_dir + combine_obsspace(ioda_path_files, new_name, geo_dir) + + # Remove input files + for ioda_path_file in ioda_path_files: + os.remove(ioda_path_file) + + elif len(ioda_file_0_) == 3: + self.logger.info(f'Skipping combine for {needed_ioda_type}, single file already.') + else: + self.logger.abort(f'Combine failed for {needed_ioda_type}, file name issue.') + + + # Get list of the observations that are ozone observations + # -------------------------------------------------------- + ozone_sensors = gsid.oz_lay_sensors + gsid.oz_lev_sensors + ozone_observations = [] + for observation in observations: + for ozone_sensor in ozone_sensors: + if ozone_sensor in observation: + ozone_observations.append(observation) # Copy all the files into the cycle directory # ------------------------------------------- for observation in observations: - # File name - self.logger.info(f'{observation}') + self.logger.info(f'Converting {observation} to IODA format') + + gsi_obs_file = glob.glob(os.path.join(gsi_diag_dir, f'*{observation}*')) + + print(gsi_obs_file) + + if observation not in ozone_observations: + + # Radiances + Diag = gsid.Radiances(gsi_obs_file[0]) + Diag.read() + Diag.toIODAobs(cycle_dir, False, False, False) + + else: + + # Ozone + Diag = gsid.Ozone(gsi_obs_file[0]) + Diag.read() + Diag.toIODAobs(cycle_dir) + + # GeoVaLs call + if produce_geovals: + Diag.toGeovals(cycle_dir) + + if observation not in ozone_observations: + Diag.close() + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/run_jedi_test_obs_filters_executable.py b/src/swell/tasks/run_jedi_test_obs_filters_executable.py new file mode 100644 index 00000000..9e7b0d48 --- /dev/null +++ b/src/swell/tasks/run_jedi_test_obs_filters_executable.py @@ -0,0 +1,85 @@ +# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import os +import yaml + +from swell.tasks.base.run_jedi_executable_base import RunJediExecutableBase + + +# -------------------------------------------------------------------------------------------------- + + +interface_executable = { + 'fv3-jedi-4D': 'fv3jedi_hofx.x', + 'fv3-jedi-3D': 'fv3jedi_hofx_nomodel.x', + 'soca-4D': 'soca_hofx.x', + 'soca-3D': 'soca_hofx3d.x', +} + + +# -------------------------------------------------------------------------------------------------- + + +class RunJediHofxExecutable(RunJediExecutableBase): + + # ---------------------------------------------------------------------------------------------- + + def execute(self): + + # Path to executable being run + # ---------------------------- + cycle_dir = self.config_get('cycle_dir') + experiment_dir = self.config_get('experiment_dir') + window_type = self.config_get('window_type') + model = self.config_get('window_type') + suite_to_run = self.config_get('suite_to_run') + npx_proc = self.config_get('npx_proc') # Used in eval(total_processors) + npy_proc = self.config_get('npy_proc') # Used in eval(total_processors) + total_processors = self.config_get('total_processors') + + # Jedi configuration file + # ----------------------- + jedi_config_file = os.path.join(cycle_dir, 'jedi_hofx_config.yaml') + + # Output log file + # --------------- + output_log_file = os.path.join(cycle_dir, 'jedi_hofx_log.log') + + # Generate the JEDI configuration file for running the executable + # --------------------------------------------------------------- + jedi_config_dict = self.generate_jedi_config(suite_to_run, window_type) + + with open(jedi_config_file, 'w') as jedi_config_file_open: + yaml.dump(jedi_config_dict, jedi_config_file_open, default_flow_style=False) + + # Get the JEDI interface for this model component + # ----------------------------------------------- + model_component_meta = self.open_jedi_interface_meta_config_file() + jedi_interface = model_component_meta['jedi_interface'] + + # Jedi executable name + # -------------------- + jedi_executable = interface_executable[jedi_interface + '-' + window_type] + jedi_executable_path = os.path.join(experiment_dir, 'jedi_bundle', 'build', 'bin', + jedi_executable) + + # Compute number of processors + # ---------------------------- + total_processors = total_processors.replace('npx_proc', str(npx_proc)) + total_processors = total_processors.replace('npy_proc', str(npy_proc)) + np = eval(total_processors) + + # Run the JEDI executable + # ----------------------- + self.run_executable(cycle_dir, np, jedi_executable_path, jedi_config_file, output_log_file) + self.logger.info('Running '+jedi_executable_path+' with '+str(np)+' processors.') + +# -------------------------------------------------------------------------------------------------- From f7d02f4c97818943e9cfea1c03f7dbc2ad60ec61 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Tue, 11 Apr 2023 13:02:10 -0400 Subject: [PATCH 004/121] executable runs --- src/swell/suites/test_obs_filters/flow.cylc | 30 ++++--- .../eva_observations.yaml | 0 ...tes-test_obs_filters-geos_atmosphere.yaml} | 77 ++++++++++-------- .../suites-test_obs_filters.yaml | 6 +- src/swell/tasks/base/config.py | 5 ++ .../tasks/base/run_jedi_executable_base.py | 4 +- src/swell/tasks/base/task_registry.py | 2 + src/swell/tasks/get_geovals.py | 58 ++++++++++++++ src/swell/tasks/gsi_ncdiag_to_ioda.py | 78 +++++++++++-------- .../run_jedi_test_obs_filters_executable.py | 73 ++++++++--------- 10 files changed, 214 insertions(+), 119 deletions(-) rename src/swell/suites/test_obs_filters/{ufo => geos_atmosphere}/eva_observations.yaml (100%) rename src/swell/suites/test_obs_filters/{ufo/suites-test_obs_filters-ufo.yaml => geos_atmosphere/suites-test_obs_filters-geos_atmosphere.yaml} (67%) create mode 100644 src/swell/tasks/get_geovals.py diff --git a/src/swell/suites/test_obs_filters/flow.cylc b/src/swell/suites/test_obs_filters/flow.cylc index 20c3d4ac..245fdd9e 100644 --- a/src/swell/suites/test_obs_filters/flow.cylc +++ b/src/swell/suites/test_obs_filters/flow.cylc @@ -6,7 +6,7 @@ # -------------------------------------------------------------------------------------------------- -# Cylc suite for executing UFO ObsFilters tests +# Cylc suite for executing geos_atmosphere ObsFilters tests # -------------------------------------------------------------------------------------------------- @@ -40,13 +40,16 @@ {{cycle_time.cycle_time}} = """ # Task triggers # ------------- - GetGsiNcdiags + GetGsiNcdiag - GetGsiNcdiags => GsiNcdiagsToIoda + GetGsiNcdiag => GsiNcdiagToIoda + + GetGeovals # Run Jedi hofx executable BuildJediByLinking[^]? | BuildJedi[^] => RunJediTestObsFiltersExecutable - GsiNcdiagsToIoda => RunJediTestObsFiltersExecutable + GsiNcdiagToIoda => RunJediTestObsFiltersExecutable + GetGeovals => RunJediTestObsFiltersExecutable # EvaObservations RunJediTestObsFiltersExecutable => EvaObservations @@ -93,14 +96,17 @@ --ntasks-per-node={{scheduling["BuildJedi"]["ntasks_per_node"]}} --constraint={{scheduling["BuildJedi"]["constraint"]}} - [[ GetGsiNcdiags ]] - script = "swell_task GetGsiNcdiags $config -d $datetime -m ufo" + [[ GetGsiNcdiag ]] + script = "swell_task GetGsiNcdiag $config -d $datetime -m geos_atmosphere" + + [[ GsiNcdiagToIoda ]] + script = "swell_task GsiNcdiagToIoda $config -d $datetime -m geos_atmosphere" - [[ GsiNcdiagsToIoda ]] - script = "swell_task GsiNcdiagsToIoda $config -d $datetime -m ufo" + [[ GetGeovals ]] + script = "swell_task GetGeovals $config -d $datetime -m geos_atmosphere" [[RunJediTestObsFiltersExecutable]] - script = "swell_task RunJediTestObsFiltersExecutable $config -d $datetime -m ufo" + script = "swell_task RunJediTestObsFiltersExecutable $config -d $datetime -m geos_atmosphere" platform = {{platform}} execution time limit = {{scheduling["RunJediTestObsFiltersExecutable"]["execution_time_limit"]}} [[[directives]]] @@ -112,12 +118,12 @@ --constraint={{scheduling["RunJediTestObsFiltersExecutable"]["constraint"]}} [[EvaObservations]] - script = "swell_task EvaObservations $config -d $datetime -m ufo" + script = "swell_task EvaObservations $config -d $datetime -m geos_atmosphere" [[SaveObsDiags]] - script = "swell_task SaveObsDiags $config -d $datetime -m ufo" + script = "swell_task SaveObsDiags $config -d $datetime -m geos_atmosphere" [[CleanCycle]] - script = "swell_task CleanCycle $config -d $datetime -m ufo" + script = "swell_task CleanCycle $config -d $datetime -m geos_atmosphere" # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/test_obs_filters/ufo/eva_observations.yaml b/src/swell/suites/test_obs_filters/geos_atmosphere/eva_observations.yaml similarity index 100% rename from src/swell/suites/test_obs_filters/ufo/eva_observations.yaml rename to src/swell/suites/test_obs_filters/geos_atmosphere/eva_observations.yaml diff --git a/src/swell/suites/test_obs_filters/ufo/suites-test_obs_filters-ufo.yaml b/src/swell/suites/test_obs_filters/geos_atmosphere/suites-test_obs_filters-geos_atmosphere.yaml similarity index 67% rename from src/swell/suites/test_obs_filters/ufo/suites-test_obs_filters-ufo.yaml rename to src/swell/suites/test_obs_filters/geos_atmosphere/suites-test_obs_filters-geos_atmosphere.yaml index 168eeea5..35c5352a 100644 --- a/src/swell/suites/test_obs_filters/ufo/suites-test_obs_filters-ufo.yaml +++ b/src/swell/suites/test_obs_filters/geos_atmosphere/suites-test_obs_filters-geos_atmosphere.yaml @@ -15,39 +15,39 @@ window_length: observations: default_value: - aircraft - - airs_aqua - - amsr2_gcom-w1 - - amsua_aqua - - amsua_metop-a - - amsua_metop-b - - amsua_metop-c - - amsua_n15 - - amsua_n18 - - amsua_n19 - - atms_n20 - - atms_npp - - avhrr_metop-a - - avhrr_n18 - - cris-fsr_n20 - - cris-fsr_npp - - gmi_gpm - - gps - - iasi_metop-a - - iasi_metop-b - - mhs_metop-b - - mhs_metop-c - - mhs_n19 - - mls55_aura - - omi_aura - - ompsnm_npp - - satwind - - scatwind - - seviri_m08 - - sfcship - - sfc - - sondes - - ssmis_f17 - - vadwind +# - airs_aqua +# - amsr2_gcom-w1 +# - amsua_aqua +# - amsua_metop-a +# - amsua_metop-b +# - amsua_metop-c +# - amsua_n15 +# - amsua_n18 +# - amsua_n19 +# - atms_n20 +# - atms_npp +# - avhrr_metop-a +# - avhrr_n18 +# - cris-fsr_n20 +# - cris-fsr_npp +# - gmi_gpm +# - gps +# - iasi_metop-a +# - iasi_metop-b +# - mhs_metop-b +# - mhs_metop-c +# - mhs_n19 +# - mls55_aura +# - omi_aura +# - ompsnm_npp +# - satwind +# - scatwind +# - seviri_m08 +# - sfcship +# - sfc +# - sondes +# - ssmis_f17 +# - vadwind prompt: Select observations to run with. options: use_method @@ -61,10 +61,16 @@ path_to_gsi_diags: # Compute GeoVaLs and observations produce_geovals: - default_value: true + default_value: false prompt: Compute GeoVaLs as well as observations? type: boolean +# Experiment that provide geovals +geovals_experiment: + default_value: x0044_v3_geovals + prompt: Experiment providing the GeoVaLs + type: string + # Fixed options (user not prompted for these) # ------------------------------------------- fixed_options: @@ -89,5 +95,8 @@ fixed_options: background_time_offset: default_value: PT9H prompt: Time before the middle of the window that the background providing forecast began + geovals_provider: + default_value: ncdiag + prompt: Database providing the GeoVaLs diff --git a/src/swell/suites/test_obs_filters/suites-test_obs_filters.yaml b/src/swell/suites/test_obs_filters/suites-test_obs_filters.yaml index 7b9d862d..1901480f 100644 --- a/src/swell/suites/test_obs_filters/suites-test_obs_filters.yaml +++ b/src/swell/suites/test_obs_filters/suites-test_obs_filters.yaml @@ -25,7 +25,7 @@ jedi_build_method: # Existing JEDI bundle directory existing_source_directory: - default_value: {{existing_source_directory}} + default_value: '/discover/nobackup/drholdaw/JediDev/swell-ufo' prompt: Provide the path to an existing JEDI bundle directory containing source code repos type: string depends: @@ -34,7 +34,7 @@ existing_source_directory: # Existing JEDI build existing_build_directory: - default_value: {{existing_build_directory}} + default_value: '/discover/nobackup/drholdaw/JediDev/swell-ufo/build-intel-release' prompt: Provide the path to an existing JEDI build directory type: string depends: @@ -56,7 +56,7 @@ coupling_style: # Models to use model_components: - default_value: ['ufo'] + default_value: ['geos_atmosphere'] prompt: Select models to use (choose at least one) options: use_method type: file-check-list diff --git a/src/swell/tasks/base/config.py b/src/swell/tasks/base/config.py index 34261925..3e53d823 100644 --- a/src/swell/tasks/base/config.py +++ b/src/swell/tasks/base/config.py @@ -175,6 +175,9 @@ def add_data_assimilation_window_parameters(self): # Compute window beginning time window_begin_dto = current_cycle_dto - window_offset_dur + # Compute window end time + window_end_dto = current_cycle_dto + window_offset_dur + # Background time for satbias files background_time_offset = self.get('background_time_offset') background_time_offset_dur = isodate.parse_duration(background_time_offset) @@ -192,6 +195,7 @@ def add_data_assimilation_window_parameters(self): window_begin = window_begin_dto.strftime(self.__datetime_swl_format__) window_begin_iso = window_begin_dto.strftime(self.__datetime_iso_format__) + window_end_iso = window_end_dto.strftime(self.__datetime_iso_format__) background_time = background_time_dto.strftime(self.__datetime_swl_format__) local_background_time_iso = local_background_time.strftime(self.__datetime_iso_format__) local_background_time = local_background_time.strftime(self.__datetime_swl_format__) @@ -199,6 +203,7 @@ def add_data_assimilation_window_parameters(self): # Create new dictionary with these items self.put('window_begin', window_begin) self.put('window_begin_iso', window_begin_iso) + self.put('window_end_iso', window_end_iso) self.put('background_time', background_time) self.put('local_background_time', local_background_time) self.put('local_background_time_iso', local_background_time_iso) diff --git a/src/swell/tasks/base/run_jedi_executable_base.py b/src/swell/tasks/base/run_jedi_executable_base.py index 243be687..ab9b9e3b 100644 --- a/src/swell/tasks/base/run_jedi_executable_base.py +++ b/src/swell/tasks/base/run_jedi_executable_base.py @@ -30,7 +30,7 @@ def execute(self): # ---------------------------------------------------------------------------------------------- - def jedi_dictionary_iterator(self, jedi_config_dict, window_type): + def jedi_dictionary_iterator(self, jedi_config_dict, window_type = ''): # Loop over dictionary and replace if value is a dictionary, meanwhile # inquire list objects for dictionary items. @@ -68,7 +68,7 @@ def jedi_dictionary_iterator(self, jedi_config_dict, window_type): # ---------------------------------------------------------------------------------------------- - def generate_jedi_config(self, jedi_application, window_type): + def generate_jedi_config(self, jedi_application, window_type = ''): # Var suite names are handled in variational executable # ----------------------------------------------------- diff --git a/src/swell/tasks/base/task_registry.py b/src/swell/tasks/base/task_registry.py index 9a70429d..9ac4811d 100644 --- a/src/swell/tasks/base/task_registry.py +++ b/src/swell/tasks/base/task_registry.py @@ -18,6 +18,7 @@ 'GenerateBClimatology', 'GetBackgroundGeosExperiment', 'GetBackground', + 'GetGeovals', 'GetGsiNcdiag', 'GetObservations', 'GetRestart', @@ -26,6 +27,7 @@ 'PrepGeosRunDir', 'RunGeosExecutable', 'RunJediHofxExecutable', + 'RunJediTestObsFiltersExecutable', 'RunJediVariationalExecutable', 'SaveObsDiags', 'SaveRestart', diff --git a/src/swell/tasks/get_geovals.py b/src/swell/tasks/get_geovals.py new file mode 100644 index 00000000..73c7f434 --- /dev/null +++ b/src/swell/tasks/get_geovals.py @@ -0,0 +1,58 @@ +# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import os + +from swell.tasks.base.task_base import taskBase +from r2d2 import fetch + + +# -------------------------------------------------------------------------------------------------- + + +class GetGeovals(taskBase): + + def execute(self): + + # Parse config + # ------------ + cycle_dir = self.config_get('cycle_dir') + geovals_experiment = self.config_get('geovals_experiment') + geovals_provider = self.config_get('geovals_provider') + window_begin = self.config_get('window_begin') + observations = self.config_get('observations') + window_length = self.config_get('window_length') + + # Loop over observation operators + # ------------------------------- + for observation in observations: + + # Open the observation operator dictionary + # ---------------------------------------- + observation_dict = self.open_jedi_interface_obs_config_file(observation) + + # Fetch observation files + # ----------------------- + target_file = os.path.join(cycle_dir, f'{observation}_geovals.{window_begin}.nc4') + self.logger.info("Processing observation file "+target_file) + + fetch(date=window_begin, + target_file=target_file, + provider=geovals_provider, + obs_type=observation, + time_window=window_length, + type='ob', + experiment=geovals_experiment) + + # Change permission + os.chmod(target_file, 0o644) + + + # ---------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/gsi_ncdiag_to_ioda.py b/src/swell/tasks/gsi_ncdiag_to_ioda.py index c7c79d0b..4c316898 100644 --- a/src/swell/tasks/gsi_ncdiag_to_ioda.py +++ b/src/swell/tasks/gsi_ncdiag_to_ioda.py @@ -34,6 +34,10 @@ def execute(self): cycle_dir = self.config_get('cycle_dir') observations = self.config_get('observations') produce_geovals = self.config_get('produce_geovals') + window_begin = self.config_get('window_begin') + + # Keep copy of the + observations_orig = observations.copy() # Directory containing the ncdiags gsi_diag_dir = os.path.join(cycle_dir, 'gsi_ncdiags') @@ -69,36 +73,36 @@ def execute(self): for needed_ioda_type in needed_ioda_types: observations.remove(needed_ioda_type) -# # First process the conventional data (if needed) -# # ----------------------------------------------- -# for gsi_type_to_process in gsi_types_to_process: -# -# log_str = f'Processing GSI file {gsi_type_to_process}' -# self.logger.info('', wrap=False) -# self.logger.info(log_str) -# self.logger.info('-'*len(log_str)) -# -# gsi_conv_file = glob.glob(os.path.join(gsi_diag_dir, f'*{gsi_type_to_process}*'))[0] -# -# # Open the file -# Diag = gsid.Conv(gsi_conv_file) -# Diag.read() -# -# # Assemble list of needed platforms -# needed_platforms = [] -# for platform in gsid.conv_platforms[gsi_type_to_process]: -# if platform in needed_ioda_types: -# needed_platforms.append(platform) -# -# # Extract data -# Diag.toIODAobs(cycle_dir, platforms=needed_platforms) -# -# if produce_geovals: -# self.logger.info('', wrap=False) -# self.logger.info(f'Processing GeoVaLs from {gsi_type_to_process}') -# Diag.toGeovals(cycle_dir) -# -# Diag.close() + # First process the conventional data (if needed) + # ----------------------------------------------- + for gsi_type_to_process in gsi_types_to_process: + + log_str = f'Processing GSI file {gsi_type_to_process}' + self.logger.info('', wrap=False) + self.logger.info(log_str) + self.logger.info('-'*len(log_str)) + + gsi_conv_file = glob.glob(os.path.join(gsi_diag_dir, f'*{gsi_type_to_process}*'))[0] + + # Open the file + Diag = gsid.Conv(gsi_conv_file) + Diag.read() + + # Assemble list of needed platforms + needed_platforms = [] + for platform in gsid.conv_platforms[gsi_type_to_process]: + if platform in needed_ioda_types: + needed_platforms.append(platform) + + # Extract data + Diag.toIODAobs(cycle_dir, platforms=needed_platforms) + + if produce_geovals: + self.logger.info('', wrap=False) + self.logger.info(f'Processing GeoVaLs from {gsi_type_to_process}') + Diag.toGeovals(cycle_dir) + + Diag.close() # Combine the conventional data # ----------------------------- @@ -181,8 +185,6 @@ def execute(self): gsi_obs_file = glob.glob(os.path.join(gsi_diag_dir, f'*{observation}*')) - print(gsi_obs_file) - if observation not in ozone_observations: # Radiances @@ -204,4 +206,16 @@ def execute(self): if observation not in ozone_observations: Diag.close() + # Rename files to be swell compliant + # ---------------------------------- + for observation in observations_orig: + + # Input filename + ioda_file_in_pattern = f'{observation}_obs_*nc*' + ioda_file_in = glob.glob(os.path.join(cycle_dir, ioda_file_in_pattern))[0] + + ioda_file_out = f'{observation}.{window_begin}.nc4' + + os.rename(ioda_file_in, os.path.join(cycle_dir, ioda_file_out)) + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/run_jedi_test_obs_filters_executable.py b/src/swell/tasks/run_jedi_test_obs_filters_executable.py index 9e7b0d48..b2905976 100644 --- a/src/swell/tasks/run_jedi_test_obs_filters_executable.py +++ b/src/swell/tasks/run_jedi_test_obs_filters_executable.py @@ -17,18 +17,7 @@ # -------------------------------------------------------------------------------------------------- -interface_executable = { - 'fv3-jedi-4D': 'fv3jedi_hofx.x', - 'fv3-jedi-3D': 'fv3jedi_hofx_nomodel.x', - 'soca-4D': 'soca_hofx.x', - 'soca-3D': 'soca_hofx3d.x', -} - - -# -------------------------------------------------------------------------------------------------- - - -class RunJediHofxExecutable(RunJediExecutableBase): +class RunJediTestObsFiltersExecutable(RunJediExecutableBase): # ---------------------------------------------------------------------------------------------- @@ -38,48 +27,60 @@ def execute(self): # ---------------------------- cycle_dir = self.config_get('cycle_dir') experiment_dir = self.config_get('experiment_dir') - window_type = self.config_get('window_type') - model = self.config_get('window_type') - suite_to_run = self.config_get('suite_to_run') - npx_proc = self.config_get('npx_proc') # Used in eval(total_processors) - npy_proc = self.config_get('npy_proc') # Used in eval(total_processors) - total_processors = self.config_get('total_processors') + observations = self.config_get('observations') + window_begin = self.config_get('window_begin') + + # Make cycle dir + # -------------- + os.makedirs(cycle_dir, 0o755, exist_ok=True) # Jedi configuration file # ----------------------- - jedi_config_file = os.path.join(cycle_dir, 'jedi_hofx_config.yaml') + jedi_config_file = os.path.join(cycle_dir, 'jedi_test_obs_filters.yaml') # Output log file # --------------- - output_log_file = os.path.join(cycle_dir, 'jedi_hofx_log.log') + output_log_file = os.path.join(cycle_dir, 'jedi_test_obs_filters.log') # Generate the JEDI configuration file for running the executable # --------------------------------------------------------------- - jedi_config_dict = self.generate_jedi_config(suite_to_run, window_type) + jedi_config_dict = self.generate_jedi_config('TestObsFilters') + + # Make modifications needed for testing + # ------------------------------------- + + conventional_types = ['aircraft'] + # Loop over the observations + for index in range(len(observations)): + + # Remove GetValues if present + if 'get values' in jedi_config_dict['observations'][index]: + del jedi_config_dict['observations'][index]['get values'] + + # Create GeoVaLs dictionary + geovals = {} + geovals['filename'] = os.path.join(cycle_dir, + f'{observations[index]}_geovals.{window_begin}.nc4') + # For conventional add the GeoVaLs flip + if observations[index] in conventional_types: + geovals['levels_are_top_down'] = False + + jedi_config_dict['observations'][index]['geovals'] = geovals + jedi_config_dict['observations'][index]['passedBenchmark'] = 317688 + + # Write executable configuration to file + # -------------------------------------- with open(jedi_config_file, 'w') as jedi_config_file_open: yaml.dump(jedi_config_dict, jedi_config_file_open, default_flow_style=False) - # Get the JEDI interface for this model component - # ----------------------------------------------- - model_component_meta = self.open_jedi_interface_meta_config_file() - jedi_interface = model_component_meta['jedi_interface'] - # Jedi executable name # -------------------- - jedi_executable = interface_executable[jedi_interface + '-' + window_type] jedi_executable_path = os.path.join(experiment_dir, 'jedi_bundle', 'build', 'bin', - jedi_executable) - - # Compute number of processors - # ---------------------------- - total_processors = total_processors.replace('npx_proc', str(npx_proc)) - total_processors = total_processors.replace('npy_proc', str(npy_proc)) - np = eval(total_processors) + 'test_ObsFilters.x') # Run the JEDI executable # ----------------------- - self.run_executable(cycle_dir, np, jedi_executable_path, jedi_config_file, output_log_file) - self.logger.info('Running '+jedi_executable_path+' with '+str(np)+' processors.') + self.run_executable(cycle_dir, 1, jedi_executable_path, jedi_config_file, output_log_file) # -------------------------------------------------------------------------------------------------- From e25bae1a5b391768c19ad689ed8b1aae0a2ea22d Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 12 Apr 2023 13:37:05 -0400 Subject: [PATCH 005/121] sat bias correction conversion added --- src/swell/deployment/prep_suite.py | 2 +- src/swell/suites/test_obs_filters/flow.cylc | 9 ++ ...ites-test_obs_filters-geos_atmosphere.yaml | 5 + .../tasks/base/run_jedi_executable_base.py | 6 +- src/swell/tasks/base/task_registry.py | 2 + src/swell/tasks/get_geovals.py | 2 - src/swell/tasks/get_gsi_bc.py | 80 +++++++++++ src/swell/tasks/gsi_bc_to_ioda.py | 128 ++++++++++++++++++ src/swell/tasks/gsi_ncdiag_to_ioda.py | 3 +- src/swell/utilities/dictionary.py | 16 +++ 10 files changed, 245 insertions(+), 8 deletions(-) create mode 100644 src/swell/tasks/get_gsi_bc.py create mode 100644 src/swell/tasks/gsi_bc_to_ioda.py diff --git a/src/swell/deployment/prep_suite.py b/src/swell/deployment/prep_suite.py index 4d867aa8..1c4f4d70 100644 --- a/src/swell/deployment/prep_suite.py +++ b/src/swell/deployment/prep_suite.py @@ -102,7 +102,7 @@ def prepare_cylc_suite_jinja2(logger, swell_suite_path, exp_suite_path, experime render_dictionary['scheduling']['RunGeosExecutable']['constraint'] = 'cas|sky|hasw' render_dictionary['scheduling']['RunJediTestObsFiltersExecutable'] = {} - render_dictionary['scheduling']['RunJediTestObsFiltersExecutable']['execution_time_limit'] = 'PT30M' + render_dictionary['scheduling']['RunJediTestObsFiltersExecutable']['execution_time_limit'] = 'PT30M' # noqa render_dictionary['scheduling']['RunJediTestObsFiltersExecutable']['account'] = 'g0613' render_dictionary['scheduling']['RunJediTestObsFiltersExecutable']['qos'] = 'allnccs' render_dictionary['scheduling']['RunJediTestObsFiltersExecutable']['nodes'] = 1 diff --git a/src/swell/suites/test_obs_filters/flow.cylc b/src/swell/suites/test_obs_filters/flow.cylc index 245fdd9e..0063f251 100644 --- a/src/swell/suites/test_obs_filters/flow.cylc +++ b/src/swell/suites/test_obs_filters/flow.cylc @@ -41,14 +41,17 @@ # Task triggers # ------------- GetGsiNcdiag + GetGsiBc GetGsiNcdiag => GsiNcdiagToIoda + GetGsiBc => GsiBcToIoda GetGeovals # Run Jedi hofx executable BuildJediByLinking[^]? | BuildJedi[^] => RunJediTestObsFiltersExecutable GsiNcdiagToIoda => RunJediTestObsFiltersExecutable + GsiBcToIoda => RunJediTestObsFiltersExecutable GetGeovals => RunJediTestObsFiltersExecutable # EvaObservations @@ -99,9 +102,15 @@ [[ GetGsiNcdiag ]] script = "swell_task GetGsiNcdiag $config -d $datetime -m geos_atmosphere" + [[ GetGsiBc ]] + script = "swell_task GetGsiNcdiag $config -d $datetime -m geos_atmosphere" + [[ GsiNcdiagToIoda ]] script = "swell_task GsiNcdiagToIoda $config -d $datetime -m geos_atmosphere" + [[ GsiBcToIoda ]] + script = "swell_task GsiNcdiagToIoda $config -d $datetime -m geos_atmosphere" + [[ GetGeovals ]] script = "swell_task GetGeovals $config -d $datetime -m geos_atmosphere" diff --git a/src/swell/suites/test_obs_filters/geos_atmosphere/suites-test_obs_filters-geos_atmosphere.yaml b/src/swell/suites/test_obs_filters/geos_atmosphere/suites-test_obs_filters-geos_atmosphere.yaml index 35c5352a..9332d1b1 100644 --- a/src/swell/suites/test_obs_filters/geos_atmosphere/suites-test_obs_filters-geos_atmosphere.yaml +++ b/src/swell/suites/test_obs_filters/geos_atmosphere/suites-test_obs_filters-geos_atmosphere.yaml @@ -71,6 +71,11 @@ geovals_experiment: prompt: Experiment providing the GeoVaLs type: string +gsi_bc_location: + default_value: /discover/nobackup/projects/gmao/dadev/rtodling/archive/x0044/rs/Y2020/M12/x0044.rst.20201214_21z.tar + prompt: Tar or directory containing BC files + type: string + # Fixed options (user not prompted for these) # ------------------------------------------- fixed_options: diff --git a/src/swell/tasks/base/run_jedi_executable_base.py b/src/swell/tasks/base/run_jedi_executable_base.py index ab9b9e3b..f80d7cac 100644 --- a/src/swell/tasks/base/run_jedi_executable_base.py +++ b/src/swell/tasks/base/run_jedi_executable_base.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 @@ -30,7 +30,7 @@ def execute(self): # ---------------------------------------------------------------------------------------------- - def jedi_dictionary_iterator(self, jedi_config_dict, window_type = ''): + def jedi_dictionary_iterator(self, jedi_config_dict, window_type=''): # Loop over dictionary and replace if value is a dictionary, meanwhile # inquire list objects for dictionary items. @@ -68,7 +68,7 @@ def jedi_dictionary_iterator(self, jedi_config_dict, window_type = ''): # ---------------------------------------------------------------------------------------------- - def generate_jedi_config(self, jedi_application, window_type = ''): + def generate_jedi_config(self, jedi_application, window_type=''): # Var suite names are handled in variational executable # ----------------------------------------------------- diff --git a/src/swell/tasks/base/task_registry.py b/src/swell/tasks/base/task_registry.py index 9ac4811d..d62c8ec7 100644 --- a/src/swell/tasks/base/task_registry.py +++ b/src/swell/tasks/base/task_registry.py @@ -19,9 +19,11 @@ 'GetBackgroundGeosExperiment', 'GetBackground', 'GetGeovals', + 'GetGsiBc', 'GetGsiNcdiag', 'GetObservations', 'GetRestart', + 'GsiBcToIoda', 'GsiNcdiagToIoda', 'ObsProcessSetup', 'PrepGeosRunDir', diff --git a/src/swell/tasks/get_geovals.py b/src/swell/tasks/get_geovals.py index 73c7f434..db781631 100644 --- a/src/swell/tasks/get_geovals.py +++ b/src/swell/tasks/get_geovals.py @@ -16,7 +16,6 @@ # -------------------------------------------------------------------------------------------------- - class GetGeovals(taskBase): def execute(self): @@ -54,5 +53,4 @@ def execute(self): # Change permission os.chmod(target_file, 0o644) - # ---------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/get_gsi_bc.py b/src/swell/tasks/get_gsi_bc.py new file mode 100644 index 00000000..48454b5e --- /dev/null +++ b/src/swell/tasks/get_gsi_bc.py @@ -0,0 +1,80 @@ +# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import glob +import os +import shutil +import tarfile + +from swell.tasks.base.task_base import taskBase + + +# -------------------------------------------------------------------------------------------------- + + +class GetGsiBc(taskBase): + + def execute(self): + + # Get the build method + # -------------------- + gsi_bc_location = self.config_get('gsi_bc_location') + cycle_dir = self.config_get('cycle_dir') + + # Holding directory + gsi_bc_dir = os.path.join(cycle_dir, 'gsi_bcs') + os.makedirs(gsi_bc_dir, 0o755, exist_ok=True) + + # Get list of bias correction files to copy + # ----------------------------------------- + files_found = False + + if os.path.isdir(gsi_bc_location): + + # Get list of matching files in the directory + ana_satbias_rst = glob.glob(os.path.join(gsi_bc_location, '*ana_satbias_rst*')) + ana_satbiaspc_rst = glob.glob(os.path.join(gsi_bc_location, '*ana_satbiaspc_rst*')) + + # Record that files were found + if len(ana_satbias_rst) == 1 and len(ana_satbiaspc_rst) == 1: + files_found = True + + # Report if too many files were found + if len(ana_satbias_rst) > 1 or len(ana_satbiaspc_rst) > 1: + self.logger.abort(f'In GetGsiBc too many files were found in the directory.') + + # Copy files + shutil.copy(ana_satbias_rst[0], gsi_bc_dir) + shutil.copy(ana_satbiaspc_rst[0], gsi_bc_dir) + + elif tarfile.is_tarfile(gsi_bc_location): + + bc_tar = tarfile.open(gsi_bc_location) + found_satbias = False + found_satbiaspc = False + for bc_tar_file in bc_tar.getnames(): + if 'ana_satbias_rst' in bc_tar_file: + bc_tar.extract(bc_tar_file, gsi_bc_dir) + found_satbias = True + if 'ana_satbiaspc_rst' in bc_tar_file: + bc_tar.extract(bc_tar_file, gsi_bc_dir) + found_satbiaspc = True + bc_tar.close() + + # Record that files were found + if found_satbias and found_satbiaspc: + files_found = True + + self.logger.assert_abort(files_found, f'When passing \'gsi_bc_location\' it should point ' + f'to either a tar file or directory containing *ana_satbias_rst* ' + f'and *ana_satbiaspc_rst*. gsi_bc_location = {gsi_bc_location}.') + + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/gsi_bc_to_ioda.py b/src/swell/tasks/gsi_bc_to_ioda.py new file mode 100644 index 00000000..9d2857df --- /dev/null +++ b/src/swell/tasks/gsi_bc_to_ioda.py @@ -0,0 +1,128 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import glob +import os +import shutil +import netCDF4 as nc + + +from swell.tasks.base.task_base import taskBase +from swell.utilities.dictionary import write_dict_to_yaml +from swell.utilities.shell_commands import run_track_log_subprocess + + +# -------------------------------------------------------------------------------------------------- + + +class GsiBcToIoda(taskBase): + + def execute(self): + + # Parse configuration + # ------------------- + experiment_dir = self.config_get('experiment_dir') + cycle_dir = self.config_get('cycle_dir') + observations = self.config_get('observations') + produce_geovals = self.config_get('produce_geovals') + window_begin = self.config_get('window_begin') + current_cycle = self.config_get('current_cycle') + background_time = self.config_get('background_time') + + # Assemble list of needed sensors + # ------------------------------- + sensors = [] + sensors_satbias = [] + sensors_tlapse = [] + for observation in observations: + # Open configuration file for observation + observation_dict = self.open_jedi_interface_obs_config_file(observation) + + # Check for sensor key + try: + sensor = observation_dict['obs operator']['obs options']['Sensor_ID'] + except Exception: + continue + + sensors.append(sensor) + sensors_satbias.append(f'{sensor}.{background_time}.satbias.nc4') + sensors_tlapse.append(f'{sensor}.{background_time}.tlapse.txt') + + # If there are no sensors then there is nothing to do + if not sensors: + self.logger.info('In GsiBcToIoda no radiance observations were found so nothing to do.') + return + + # Holding directory + gsi_bc_dir = os.path.join(cycle_dir, 'gsi_bcs') + + # Get list of files from holding directory + bc_files = glob.glob(os.path.join(gsi_bc_dir, '*.txt')) + + # Create dictionary that will be passed to converter + satbias_converter_dict = {} + + # Add the files + for bc_file in bc_files: + if 'ana_satbias_rst' in bc_file: + satbias_converter_dict['input coeff file'] = bc_file + if 'ana_satbiaspc_rst' in bc_file: + satbias_converter_dict['input err file'] = bc_file + + # Add the default predictors + default_predictors = [] + default_predictors.append('constant') + default_predictors.append('zenith_angle') + default_predictors.append('cloud_liquid_water') + default_predictors.append('lapse_rate_order_2') + default_predictors.append('lapse_rate') + default_predictors.append('cosine_of_latitude_times_orbit_node') + default_predictors.append('sine_of_latitude') + default_predictors.append('emissivity') + default_predictors.append('scan_angle_order_4') + default_predictors.append('scan_angle_order_3') + default_predictors.append('scan_angle_order_2') + default_predictors.append('scan_angle') + + satbias_converter_dict['default predictors'] = default_predictors + + satbias_converter_dict_output = [] + for sensor, sensor_satbias in zip(sensors, sensors_satbias): + output_dict = {} + output_dict['sensor'] = sensor + output_dict['output file'] = os.path.join(cycle_dir, sensor_satbias) + output_dict['predictors'] = default_predictors + satbias_converter_dict_output.append(output_dict) + + satbias_converter_dict['output'] = satbias_converter_dict_output + + # Write to YAML file + satbias_converter_yaml = os.path.join(gsi_bc_dir, 'satbias.yaml') + write_dict_to_yaml(satbias_converter_dict, satbias_converter_yaml) + + # Run IODA satbias converter + satbias_converter_exe = os.path.join(experiment_dir, 'jedi_bundle', 'build', 'bin', + 'satbias2ioda.x') + + run_track_log_subprocess(self.logger, [satbias_converter_exe, satbias_converter_yaml]) + + # Run tlapse converter (just a grep) + for sensor, sensor_tlapse in zip(sensors, sensors_tlapse): + sensor_tlapse_file = '' + with open(satbias_converter_dict['input err file'], 'r') as f: + for line in f.readlines(): + if sensor in line: + sensor_tlapse_file = sensor_tlapse_file + ' '.join(line.split()[1:]) + '\n' + # Write to tlapse file + with open(os.path.join(cycle_dir, sensor_tlapse), 'w') as file_open: + file_open.write(sensor_tlapse_file) + + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/gsi_ncdiag_to_ioda.py b/src/swell/tasks/gsi_ncdiag_to_ioda.py index 4c316898..fa74b35b 100644 --- a/src/swell/tasks/gsi_ncdiag_to_ioda.py +++ b/src/swell/tasks/gsi_ncdiag_to_ioda.py @@ -18,7 +18,7 @@ from gsi_ncdiag.combine_obsspace import combine_obsspace from swell.tasks.base.task_base import taskBase -from swell.utilities.shell_commands import run_track_log_subprocess + # -------------------------------------------------------------------------------------------------- @@ -167,7 +167,6 @@ def execute(self): else: self.logger.abort(f'Combine failed for {needed_ioda_type}, file name issue.') - # Get list of the observations that are ozone observations # -------------------------------------------------------- ozone_sensors = gsid.oz_lay_sensors + gsid.oz_lev_sensors diff --git a/src/swell/utilities/dictionary.py b/src/swell/utilities/dictionary.py index 7bfeed7a..f31b9dab 100644 --- a/src/swell/utilities/dictionary.py +++ b/src/swell/utilities/dictionary.py @@ -116,3 +116,19 @@ def replace_string_in_dictionary(dictionary, string_in, string_out): # Convert back to dictionary return yaml.safe_load(dictionary_string) + + +# -------------------------------------------------------------------------------------------------- + + +def write_dict_to_yaml(dictionary, file): + + # Convert dictionary to YAML string + dictionary_string = yaml.dump(dictionary, default_flow_style=False, sort_keys=False) + + # Write string to file + with open(file, 'w') as file_open: + file_open.write(dictionary_string) + + +# -------------------------------------------------------------------------------------------------- From adbe94a95173dd0da825ec1947f7754402beb98a Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 12 Apr 2023 16:36:23 -0400 Subject: [PATCH 006/121] add aircraft yaml --- .../observations/aircraft.yaml | 944 ++++++++++-------- src/swell/suites/test_obs_filters/flow.cylc | 4 +- .../geos_atmosphere/eva_observations.yaml | 398 ++------ .../run_jedi_test_obs_filters_executable.py | 8 +- 4 files changed, 671 insertions(+), 683 deletions(-) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/aircraft.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/aircraft.yaml index 38971ea5..3bfa9b61 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/aircraft.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/aircraft.yaml @@ -1,6 +1,9 @@ + obs operator: name: VertInterp - observation alias file: '{{experiment_root}}/{{experiment_id}}/configuration/jedi/interfaces/{{model_component}}/observations/obsop_name_map.yaml' + observation alias file: /discover/nobackup/msienkie/jedi/JediUfoTests/Config/obsop_name_map.yaml + apply near surface wind scaling: true + obs space: name: Aircraft obsdatain: @@ -8,397 +11,560 @@ obs space: type: H5File obsfile: '{{cycle_dir}}/aircraft.{{window_begin}}.nc4' obsgrouping: - group variables: ["stationIdentification"] + group variables: ["stationIdentification", "releaseTime"] sort variable: "pressure" sort order: "descending" obsdataout: engine: type: H5File obsfile: '{{cycle_dir}}/{{experiment_id}}.aircraft.{{window_begin}}.nc4' - simulated variables: [windEastward, windNorthward, airTemperature] -#obs filters: -##-------------------------------------------------------------------------------------------------------------------- -## WINDS -##-------------------------------------------------------------------------------------------------------------------- -## -## Begin by assigning all ObsError to a constant value. These will get overwritten (as needed) for specific types. -#- filter: BlackList -# filter variables: -# - name: windEastward -# - name: windNorthward -# action: -# name: assign error -# error parameter: 2.0 # 2.0 m/s -## -## Assign intial ObsError specific to AIREP/ACARS -#- filter: BlackList -# filter variables: -# - name: windEastward -# - name: windNorthward -# action: -# name: assign error -# error parameter: 3.6 # 3.6 m/s -# where: -# - variable: -# name: ObsType/windEastward -# is_in: 230 -## -## Assign intial ObsError specific to AMDAR -#- filter: BlackList -# filter variables: -# - name: windEastward -# - name: windNorthward -# action: -# name: assign error -# error parameter: 3.0 # 3.0 m/s -# where: -# - variable: -# name: ObsType/windEastward -# is_in: 231 -## -## Assign intial ObsError specific to MDCRS -#- filter: BlackList -# filter variables: -# - name: windEastward -# - name: windNorthward -# action: -# name: assign error -# error parameter: 2.5 # 2.5 m/s -# where: -# - variable: -# name: ObsType/windEastward -# is_in: 233 -## -## Assign the initial ObsError, based on height/pressure for RECON aircraft -#- filter: Bounds Check -# filter variables: -# - name: windEastward -# - name: windNorthward -# minvalue: -135 -# maxvalue: 135 -# action: -# name: assign error -# error function: -# name: ObsFunction/ObsErrorModelStepwiseLinear -# options: -# xvar: -# name: MetaData/pressure -# xvals: [70000, 65000, 60000, 55000, 50000, 45000, 40000, 35000, 30000, 25000, 20000, 15000, 10000, 7500, 5000] -# errors: [2.4, 2.5, 2.6, 2.7, 2.8, 2.95, 3.1, 3.25, 3.4, 3.175, 2.95, 2.725, 2.5, 2.6, 2.7] -# where: -# - variable: -# name: ObsType/windEastward -# is_in: 232 -## -## Reject all obs with PreQC mark already set above 3 -#- filter: PreQC -# maxvalue: 3 -# action: -# name: reject -## -## Observation Range Sanity Check: either wind component or velocity exceeds 135 m/s -#- filter: Bounds Check -# filter variables: -# - name: windEastward -# - name: windNorthward -# minvalue: -135 -# maxvalue: 135 -# action: -# name: reject -#- filter: Bounds Check -# filter variables: -# - name: windEastward -# - name: windNorthward -# test variables: -# - name: ObsFunction/Velocity -# maxvalue: 135.0 -# action: -# name: reject -## -## Reject when pressure is less than 126 mb. -#- filter: Bounds Check -# filter variables: -# - name: windEastward -# - name: windNorthward -# test variables: -# - name: MetaData/pressure -# minvalue: 12600 -# action: -# name: reject -## -## Reject when difference of wind direction is more than 50 degrees. -#- filter: Bounds Check -# filter variables: -# - name: windEastward -# - name: windNorthward -# test variables: -# - name: ObsFunction/WindDirAngleDiff -# maxvalue: 50.0 -# action: -# name: reject -## -## When multiple obs exist within a single vertical model level, inflate ObsError -#- filter: BlackList -# filter variables: -# - name: windEastward -# action: -# name: inflate error -# inflation variable: -# name: ObsFunction/ObsErrorFactorConventional -# options: -# test QCflag: PreQC -# inflate variables: [eastward_wind] -# pressure: MetaData/pressure -# defer to post: true -## -#- filter: BlackList -# filter variables: -# - name: windNorthward -# action: -# name: inflate error -# inflation variable: -# name: ObsFunction/ObsErrorFactorConventional -# options: -# test QCflag: PreQC -# inflate variables: [northward_wind] -# pressure: MetaData/pressure -# defer to post: true -## -## If background check is largely different than obs, inflate ObsError -#- filter: Background Check -# filter variables: -# - name: windEastward -# - name: windNorthward -# absolute threshold: 7.5 -# action: -# name: inflate error -# inflation factor: 3.0 -# defer to post: true -## -#- filter: Bounds Check -# filter variables: -# - name: windEastward -# action: -# name: reject -# maxvalue: 7.0 -# test variables: -# - name: ObsFunction/ObsErrorFactorQuotient -# options: -# numerator: -# name: ObsErrorData/windEastward # After inflation step -# denominator: -# name: ObsError/windEastward -# defer to post: true -## -## If ObsError inflation factor is larger than threshold, reject obs -#- filter: Bounds Check -# filter variables: -# - name: windNorthward -# action: -# name: reject -# maxvalue: 7.0 -# test variables: -# - name: ObsFunction/ObsErrorFactorQuotient -# options: -# numerator: -# name: ObsErrorData/windNorthward # After inflation step -# denominator: -# name: ObsError/windNorthward -# defer to post: true -##-------------------------------------------------------------------------------------------------------------------- -## TEMPERATURE -##-------------------------------------------------------------------------------------------------------------------- -## -## Begin by assigning all ObsError to a constant value. These will get overwritten for specific types. -#- filter: BlackList -# filter variables: -# - name: airTemperature -# action: -# name: assign error -# error parameter: 2.0 # 2.0 K -## -## Assign the initial observation error, based on pressure (for AIREP/ACARS; itype=130) -#- filter: Bounds Check -# filter variables: -# - name: airTemperature -# minvalue: 195 -# maxvalue: 327 -# action: -# name: assign error -# error function: -# name: ObsFunction/ObsErrorModelStepwiseLinear -# options: -# xvar: -# name: MetaData/pressure -# xvals: [100000, 95000, 90000, 85000, 80000] -# errors: [2.5, 2.3, 2.1, 1.9, 1.7] -# where: -# - variable: -# name: ObsType/airTemperature -# is_in: 130 -## -## Assign the initial observation error, based on pressure (for AMDAR and MDCRS; itype=131,133) -#- filter: Bounds Check -# filter variables: -# - name: airTemperature -# minvalue: 195 -# maxvalue: 327 -# action: -# name: assign error -# error function: -# name: ObsFunction/ObsErrorModelStepwiseLinear -# options: -# xvar: -# name: MetaData/pressure -# xvals: [100000, 95000, 90000, 85000, 80000] -# errors: [1.4706, 1.3529, 1.2353, 1.1176, 1.0] -# where: -# - variable: -# name: ObsType/airTemperature -# is_in: 131,133 -## -## Assign the initial observation error, based on pressure (for RECON aircraft; itype=132) -#- filter: Bounds Check -# filter variables: -# - name: airTemperature -# minvalue: 195 -# maxvalue: 327 -# action: -# name: assign error -# error function: -# name: ObsFunction/ObsErrorModelStepwiseLinear -# options: -# xvar: -# name: MetaData/pressure -# xvals: [100000, 95000, 90000, 85000, 35000, 30000, 25000, 20000, 15000, 10000, 7500, 5000, 4000, 3200, 2000, 1000] -# errors: [1.2, 1.1, 0.9, 0.8, 0.8, 0.9, 1.2, 1.2, 1.0, 0.8, 0.8, 0.9, 0.95, 1.0, 1.25, 1.5] -# where: -# - variable: -# name: ObsType/airTemperature -# is_in: 132 -## -## Observation Range Sanity Check -#- filter: Bounds Check -# filter variables: -# - name: airTemperature -# minvalue: 195 -# maxvalue: 327 -# action: -# name: reject -## -## Reject all obs with PreQC mark already set above 3 -#- filter: PreQC -# maxvalue: 3 -# action: -# name: reject -## -## When multiple obs exist within a single vertical model level, inflate ObsError -#- filter: BlackList -# filter variables: -# - name: airTemperature -# action: -# name: inflate error -# inflation variable: -# name: ObsFunction/ObsErrorFactorConventional -# options: -## test QCflag: PreQC -# inflate variables: [air_temperature] -# pressure: MetaData/pressure -# defer to post: true -## -## If background check is largely different than obs, inflate ObsError -#- filter: Background Check -# filter variables: -# - name: airTemperature -# absolute threshold: 4.0 -# action: -# name: inflate error -# inflation factor: 3.0 -# defer to post: true -## -## If ObsError inflation factor is larger than threshold, reject obs -#- filter: Bounds Check -# filter variables: -# - name: airTemperature -# action: -# name: reject -# maxvalue: 7.0 -# test variables: -# - name: ObsFunction/ObsErrorFactorQuotient -# options: -# numerator: -# name: ObsErrorData/airTemperature # After inflation step -# denominator: -# name: ObsError/airTemperature -# defer to post: true -# #-------------------------------------------------------------------------------------------------------------------- -# # MOISTURE -# #-------------------------------------------------------------------------------------------------------------------- -# # -# # Assign the initial observation error, based on height/pressure ONLY MDCRS -# - filter: Bounds Check -# filter variables: -# - name: specificHumidity -# minvalue: 1.0E-7 -# maxvalue: 0.34999999 -# action: -# name: assign error -# error function: -# name: ObsFunction/ObsErrorModelStepwiseLinear -# options: -# xvar: -# name: MetaData/pressure -# xvals: [110000, 105000, 100000, 95000, 90000, 85000, 80000, 75000, 70000, 65000, 60000, 55000, -# 50000, 45000, 40000, 35000, 30000, 25000, 20000, 15000, 10000, 7500, 5000, 4000, 3000] -# errors: [.19455, .19062, .18488, .17877, .17342, .16976, .16777, .16696, .16605, .16522, .16637, .17086, -# .17791, .18492, .18996, .19294, .19447, .19597, .19748, .19866, .19941, .19979, .19994, .19999, .2] -# scale_factor_var: specific_humidity@ObsValue -# where: -# - variable: -# name: ObsType/specificHumidity -# is_in: 133 -# # -# # Observation Range Sanity Check -# - filter: Bounds Check -# filter variables: -# - name: specificHumidity -# minvalue: 1.0E-7 -# maxvalue: 0.34999999 -# action: -# name: reject -# # -# # Reject all obs with PreQC mark already set above 3 -# - filter: PreQC -# maxvalue: 3 -# action: -# name: reject -# # -# # When multiple obs exist within a single vertical model level, inflate ObsError -# - filter: BlackList -# filter variables: -# - name: specificHumidity -# action: -# name: inflate error -# inflation variable: -# name: ObsFunction/ObsErrorFactorConventional -# options: -# test QCflag: PreQC -# inflate variables: [specific_humidity] -# pressure: MetaData/pressure -# defer to post: true -# # -# # If ObsError inflation factor is larger than threshold, reject obs -# - filter: Bounds Check -# filter variables: -# - name: specificHumidity -# action: -# name: reject -# maxvalue: 8.0 -# test variables: -# - name: ObsFunction/ObsErrorFactorQuotient -# options: -# numerator: -# name: ObsErrorData/specificHumidity # After inflation step -# denominator: -# name: ObsError/specificHumidity -# defer to post: true + simulated variables: [airTemperature, windEastward, windNorthward] + +obs filters: + +# airTemperature +# -------------- + +# Reject all obs with PreQC mark already set above 3 - checked, GMAO has noiqc=false so QC limit=4 +# Input prepbufr without OIQC would not have QC marks 4-7 though ("global" assignment of obs error) +- filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error parameter: 2.0 + +# Assign the initial observation error, based on pressure (for AIREP/ACARS; itype=130) +- filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [100000, 95000, 90000, 85000, 80000] + errors: [2.5, 2.3, 2.1, 1.9, 1.7] + where: + - variable: + name: ObsType/airTemperature + is_in: 130 + +# Assign the initial observation error, based on pressure (for AMDAR and MDCRS; itype=131,133) +- filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [100000, 95000, 90000, 85000, 80000] + errors: [1.4706, 1.3529, 1.2353, 1.1176, 1.0] + where: + - variable: + name: ObsType/airTemperature + is_in: 131,133 + +# Assign the initial observation error, based on pressure (for RECON aircraft; itype=132) +- filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [105000, 100000, 95000, 90000, 85000, 70000, 65000, 60000, 40000, 35000, 30000, 25000, 20000, 5000] + errors: [1.5, 1.3, 1.1, 0.9, 0.8, 0.8, 0.75, 0.7, 0.7, 0.75, 0.85, 1.3, 1.5, 1.5] + where: + - variable: + name: ObsType/airTemperature + is_in: 132 + +# Assign the initial observation error, based on pressure (for TAMDAR itype=134) +- filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [100000, 95000, 90000, 85000, 80000, 60000, 40000] + errors: [1.5, 1.35, 1.25, 1.10, 1.0, 1.3, 1.7] + where: + - variable: + name: ObsType/airTemperature + is_in: 134 + +# Assign the initial observation error, based on pressure (for Canadian AMDAR itype=135) +- filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [100000, 95000, 90000, 85000, 80000] + errors: [1.4706, 1.3529, 1.2353, 1.1176, 1000000000.0 ] + where: + - variable: + name: ObsType/airTemperature + is_in: 135 + +# When multiple obs exist within a single vertical model level, inflate ObsError +- filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorConventional + options: + test QCflag: PreQC + inflate variables: [airTemperature] + pressure: MetaData/pressure + distance threshold: 60000.0 + defer to post: true + +- filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: PreQC/airTemperature + is_in: 4-15 + action: + name: passivate + defer to post: true + +# TAMDAR and Canadian AMDAR are passive in GSI +- filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: + name: ObsType/airTemperature + is_in: 134, 135 + action: + name: passivate + defer to post: true + +# Inflate kx130 error for obs below 500 if (aircraft_t_bc .and. kx==130 .and. ppb>=500.0_r_kind) +# toe=toe*r10 +- filter: Perform Action + action: + name: inflate error + inflation factor: 10. + filter variables: + - name: airTemperature + where: + - variable: + name: MetaData/pressure + maxvalue: 110000 + minvalue: 50000 + - variable: + name: ObsType/airTemperature + is_in: 130 + defer to post: true + +# A temporary solution: Replace error by GsiAdjustObsError overwrite above error assignments and +# inflation. +- filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: GsiAdjustObsError/airTemperature + where: + - variable: + name: GsiAdjustObsError/airTemperature + is_defined: + defer to post: true + +# error inflation based on pressure check +- filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: airTemperature + inflation factor: 8.0 + defer to post: true + +# assign TempObsErrorData/airTemperature <--- ObsErrorData before changing its Min and Max +- filter: Variable Assignment + assignments: + - name: TempObsErrorData/airTemperature + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/airTemperature + defer to post: true + +# set ObsErrorData 1.3 K if it < 1.3 K +- filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error parameter: 1.3 + where: + - variable: + name: TempObsErrorData/airTemperature + maxvalue: 1.3 + - variable: + name: TempObsErrorData/airTemperature + is_defined: + defer to post: true + +# set ObsErrorData 5.6 K if it > 5.6 K. +- filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error parameter: 5.6 + where: + - variable: + name: TempObsErrorData/airTemperature + minvalue: 5.6 + - variable: + name: TempObsErrorData/airTemperature + is_defined: + defer to post: true + +# Screen data by PreUseFlag. PreUseFlag is assigned in reading programs in GSI. +- filter: Perform Action + filter variables: + - name: airTemperature + where: + - variable: PreUseFlag/airTemperature + # PreUseFlag should =< ijter ( # of outloop(0,1,2) + 1) in GSI + minvalue: 1 + action: + name: reject + +# Background check +# ratio threshold 7.0 +- filter: Background Check + filter variables: + - name: airTemperature + threshold: 7.0 + action: + name: reject + where: + - variable: PreQC/airTemperature + is_not_in: [3] + defer to post: true + +# ratio threshold * 0.7 if PreQC=3 +- filter: Background Check + filter variables: + - name: airTemperature + threshold: 4.9 + action: + name: reject + where: + - variable: PreQC/airTemperature + is_in: [3] + defer to post: true + +# Re-assign error ObsErrorData/airTemperature <--- TempObsErrorData/airTemperature +- filter: Perform Action + filter variables: + - name: airTemperature + action: + name: assign error + error function: TempObsErrorData/airTemperature + where: + - variable: + name: TempObsErrorData/airTemperature + is_defined: + defer to post: true + +## Duplicate factor +- filter: Perform Action + filter variables: + - name: airTemperature + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: airTemperature + defer to post: true + +# Note: have not implemented error inflation for new tail. One tail 'RPXTOZBA' had uninitialized +# errors and its error was inflated by a factor of 1.2 + + +# Eastward and northward wind +# --------------------------- + +# Reject all obs with PreQC mark already set above 3 - checked, GMAO has noiqc=false so QC limit=4 +# Input prepbufr without OIQC would not have QC marks 4-7 though +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: PreQC/windEastward + is_in: 4-15 + action: + name: passivate + defer to post: true + +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + where: + - variable: + name: ObsType/windEastward + is_in: 234, 235 + action: + name: passivate + defer to post: true + +# Begin by assigning all ObsError as a function of pressure. These will get overwritten (as needed) +# for specific types. (this is a little strange, but whatever?) +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [100000, 95000, 80000, 65000, 60000, 55000, 50000, 45000, 40000, 35000, 30000, 25000, 20000, 15000, 10000] #Pressure (Pa) + errors: [1.4, 1.5, 1.6, 1.8, 1.9, 2.0, 2.1, 2.3, 2.6, 2.8, 3.0, 3.2, 2.7, 2.4, 2.1] + +# Assign intial ObsError specific to AIREP/ACARS +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error parameter: 3.6 + where: + - variable: + name: ObsType/windEastward + is_in: 230 + +# Assign intial ObsError specific to AMDAR +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error parameter: 3.0 + where: + - variable: + name: ObsType/windEastward + is_in: 231 + +# Assign intial ObsError specific to MDCRS +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error parameter: 2.5 + where: + - variable: + name: ObsType/windEastward + is_in: 233 + +# Assign intial ObsError specific to TAMDAR, Canadian AMDAR +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error parameter: 3.0 + where: + - variable: + name: ObsType/windEastward + is_in: 234, 235 + +# Assign the initial ObsError, based on height/pressure for RECON aircraft +- filter: Perform Action + filter variables: + - name: windEastward + - name: windNorthward + action: + name: assign error + error function: + name: ObsErrorModelStepwiseLinear@ObsFunction + options: + xvar: + name: MetaData/pressure + xvals: [80000, 75000, 70000, 65000, 60000, 55000, 50000, 45000, 40000, 35000, 30000, 25000, 4000, 3000, 2000, 1000, 500] + errors: [1.5, 1.6, 1.7, 1.8, 1.9, 2.0, 2.1, 2.2, 2.2, 2.3, 2.3, 2.4, 2.4, 2.5, 2.7, 2.9, 3.1] + where: + - variable: + name: ObsType/windEastward + is_in: 232 + +# When multiple obs exist within a single vertical model level, inflate ObsError +- filter: Perform Action + filter variables: + - name: windEastward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorConventional + options: + test QCflag: PreQC + inflate variables: [windEastward] + pressure: MetaData/pressure + distance threshold: 60000.0 + defer to post: true + +- filter: Perform Action + filter variables: + - name: windNorthward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorConventional + options: + test QCflag: PreQC + inflate variables: [windNorthward] + pressure: MetaData/pressure + distance threshold: 60000.0 + defer to post: true + +# A temporary solution: Replace error by GsiAdjustObsError overwrite above error assignments and +# inflation. +- filter: Perform Action + filter variables: + - name: windEastward + action: + name: assign error + error function: GsiAdjustObsError/windEastward + where: + - variable: + name: GsiAdjustObsError/windEastward + is_defined: + defer to post: true + +- filter: Perform Action + filter variables: + - name: windNorthward + action: + name: assign error + error function: GsiAdjustObsError/windNorthward + where: + - variable: + name: GsiAdjustObsError/windNorthward + is_defined: + defer to post: true + +# Error inflation based on pressure check +- filter: Perform Action + filter variables: + - name: windEastward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windEastward + inflation factor: 4.0 + defer to post: true + +# Error inflation based on pressure check +- filter: Perform Action + filter variables: + - name: windNorthward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4.0 + defer to post: true + + # Gross Check +- filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/SatWindsSPDBCheck + options: + wndtype: [ 230, 231, 232, 233, 234, 235 ] + cgross: [ 6.0, 6.5, 7.0, 7.5, 7.5, 7.5 ] + error_min: [ 1.4, 1.4, 1.4, 1.4, 1.4, 1.4 ] + error_max: [ 6.1, 6.1, 6.1, 6.1, 6.1, 6.1 ] + variable: windEastward + action: + name: reject + defer to post: true + + # Gross Check +- filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/SatWindsSPDBCheck + options: + wndtype: [ 230, 231, 232, 233, 234, 235 ] + cgross: [ 6.0, 6.5, 7.0, 7.5, 7.5, 7.5 ] + error_min: [ 1.4, 1.4, 1.4, 1.4, 1.4, 1.4 ] + error_max: [ 6.1, 6.1, 6.1, 6.1, 6.1, 6.1 ] + variable: windNorthward + action: + name: reject + defer to post: true + +# Duplicate factor +- filter: Perform Action + filter variables: + - name: windEastward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windEastward + defer to post: true + +- filter: Perform Action + filter variables: + - name: windNorthward + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windNorthward + defer to post: true diff --git a/src/swell/suites/test_obs_filters/flow.cylc b/src/swell/suites/test_obs_filters/flow.cylc index 0063f251..e700c0ee 100644 --- a/src/swell/suites/test_obs_filters/flow.cylc +++ b/src/swell/suites/test_obs_filters/flow.cylc @@ -103,13 +103,13 @@ script = "swell_task GetGsiNcdiag $config -d $datetime -m geos_atmosphere" [[ GetGsiBc ]] - script = "swell_task GetGsiNcdiag $config -d $datetime -m geos_atmosphere" + script = "swell_task GetGsiBc $config -d $datetime -m geos_atmosphere" [[ GsiNcdiagToIoda ]] script = "swell_task GsiNcdiagToIoda $config -d $datetime -m geos_atmosphere" [[ GsiBcToIoda ]] - script = "swell_task GsiNcdiagToIoda $config -d $datetime -m geos_atmosphere" + script = "swell_task GsiBcToIoda $config -d $datetime -m geos_atmosphere" [[ GetGeovals ]] script = "swell_task GetGeovals $config -d $datetime -m geos_atmosphere" diff --git a/src/swell/suites/test_obs_filters/geos_atmosphere/eva_observations.yaml b/src/swell/suites/test_obs_filters/geos_atmosphere/eva_observations.yaml index 220f8744..8b725b40 100644 --- a/src/swell/suites/test_obs_filters/geos_atmosphere/eva_observations.yaml +++ b/src/swell/suites/test_obs_filters/geos_atmosphere/eva_observations.yaml @@ -11,152 +11,177 @@ diagnostics: groups: - name: ObsValue variables: &variables {{simulated_variables}} - - name: GsiHofXBc - #- name: GsiEffectiveQC + - name: MetaData + # Jedi - name: hofx - name: EffectiveQC - - name: MetaData + - name: EffectiveError + # Gsi + - name: GsiHofX + - name: GsiHofXBc + - name: GsiEffectiveQC + - name: GsiFinalObsError transforms: - # Generate omb for GSI + # Hofx differences + # ---------------- + - transform: arithmetic - new name: experiment::ObsValueMinusGsiHofXBc::${variable} - equals: experiment::ObsValue::${variable}-experiment::GsiHofXBc::${variable} + new name: experiment::hofxDiff::${variable} + equals: experiment::hofx::${variable}-experiment::GsiHofX::${variable} for: variable: *variables - # Generate omb for JEDI - transform: arithmetic - new name: experiment::ObsValueMinusHofx::${variable} - equals: experiment::ObsValue::${variable}-experiment::hofx::${variable} + new name: experiment::hofxDiffBc::${variable} + equals: experiment::hofx::${variable}-experiment::GsiHofXBc::${variable} for: variable: *variables - # Generate hofx difference + # Observation minus Hofx + # ---------------------- + - transform: arithmetic - new name: experiment::HofxMinusGsiHofXBc::${variable} - equals: experiment::hofx::${variable}-experiment::GsiHofXBc::${variable} + new name: experiment::ObsValueMinusHofx::${variable} + equals: experiment::ObsValue::${variable}-experiment::hofx::${variable} for: variable: *variables - # Generate hofx that passed QC for JEDI - - transform: accept where - new name: experiment::hofxPassedQc::${variable} - starting field: experiment::hofx::${variable} - where: - - experiment::EffectiveQC::${variable} == 0 + - transform: arithmetic + new name: experiment::ObsValueMinusGsiHofX::${variable} + equals: experiment::ObsValue::${variable}-experiment::GsiHofX::${variable} for: variable: *variables - # Generate GSI hofx that passed JEDI QC - - transform: accept where - new name: experiment::GsiHofXBcPassedQc::${variable} - starting field: experiment::GsiHofXBc::${variable} - where: - - experiment::EffectiveQC::${variable} == 0 + - transform: arithmetic + new name: experiment::ObsValueMinusGsiHofXBc::${variable} + equals: experiment::ObsValue::${variable}-experiment::GsiHofXBc::${variable} for: variable: *variables - # Generate omb that passed QC for JEDI + # Effective error that passed QC + # ------------------------------ + - transform: accept where - new name: experiment::ObsValueMinushofxPassedQc::${variable} - starting field: experiment::ObsValueMinusHofx::${variable} + new name: experiment::EffectiveErrorPassedQc::${variable} + starting field: experiment::EffectiveError::${variable} where: - experiment::EffectiveQC::${variable} == 0 for: variable: *variables - # Generate omb that passed QC for GSI - transform: accept where - new name: experiment::ObsValueMinusGsiHofXBcPassedQc::${variable} - starting field: experiment::ObsValueMinusGsiHofXBc::${variable} + new name: experiment::FinalObsErrorPassedQc::${variable} + starting field: experiment::GsiFinalObsError::${variable} where: - - experiment::EffectiveQC::${variable} == 0 + - experiment::GsiFinalObsError::${variable} > 0 for: variable: *variables - graphics: - # Correlation scatter plots - # ------------------------- - # JEDI h(x) vs Observations +# # hofx that passed qz +# +# # Generate hofx that passed QC for JEDI +# - transform: accept where +# new name: experiment::hofxPassedQc::${variable} +# starting field: experiment::hofx::${variable} +# where: +# - experiment::EffectiveQC::${variable} == 0 +# for: +# variable: *variables +# +# # Generate GSI hofx that passed JEDI QC +# - transform: accept where +# new name: experiment::GsiHofXBcPassedQc::${variable} +# starting field: experiment::GsiHofXBc::${variable} +# where: +# - experiment::EffectiveQC::${variable} == 0 +# for: +# variable: *variables +# +# # Generate omb that passed QC for JEDI +# - transform: accept where +# new name: experiment::ObsValueMinushofxPassedQc::${variable} +# starting field: experiment::ObsValueMinusHofx::${variable} +# where: +# - experiment::EffectiveQC::${variable} == 0 +# for: +# variable: *variables +# +# # Generate omb that passed QC for GSI +# - transform: accept where +# new name: experiment::ObsValueMinusGsiHofXBcPassedQc::${variable} +# starting field: experiment::ObsValueMinusGsiHofXBc::${variable} +# where: +# - experiment::EffectiveQC::${variable} == 0 +# for: +# variable: *variables + + graphics: + + # Correlation scatter JEDI h(x) vs GSI h(x) + # ----------------------------------------- - batch figure: variables: *variables channels: *channels figure: layout: [1,1] - title: 'Observations vs. JEDI h(x) | {{instrument_title}} | ${variable_title}' - output name: '{{cycle_dir}}/eva/{{instrument}}/correlation_scatter/${variable}${channel}/jedi_hofx_vs_obs_{{instrument}}_${variable}${channel}.png' + title: 'JEDI h(x) vs. GSI h(x) | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/${variable}${channel}_a_hofx_vs_gsihofx.png' plots: - - add_xlabel: 'Observation Value' + - add_xlabel: 'GSI h(x)' add_ylabel: 'JEDI h(x)' add_grid: add_legend: loc: 'upper left' layers: - type: Scatter - x: - variable: experiment::ObsValue::${variable} y: variable: experiment::hofx::${variable} - channel: ${channel} - markersize: 5 - color: 'black' - label: 'JEDI h(x) versus obs (all obs)' - - type: Scatter x: - variable: experiment::ObsValue::${variable} - y: - variable: experiment::hofxPassedQc::${variable} + variable: experiment::GsiHofX::${variable} channel: ${channel} markersize: 5 - color: 'red' - label: 'JEDI h(x) versus obs (passed QC in JEDI)' + color: 'blue' + label: 'JEDI h(x) versus GSI h(x)' - # GSI h(x) vs Observations + # Correlation scatter JEDI h(x) vs GSI h(x) BC + # -------------------------------------------- - batch figure: variables: *variables channels: *channels figure: layout: [1,1] - title: 'Observations vs. GSI h(x) | {{instrument_title}} | ${variable_title}' - output name: '{{cycle_dir}}/eva/{{instrument}}/correlation_scatter/${variable}${channel}/gsi_hofx_vs_obs_{{instrument}}_${variable}${channel}.png' + title: 'JEDI h(x) vs. GSI h(x) | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/${variable}${channel}_b_hofx_vs_gsihofxbc.png' plots: - - add_xlabel: 'Observation Value' - add_ylabel: 'GSI h(x)' + - add_xlabel: 'GSI h(x)' + add_ylabel: 'JEDI h(x)' add_grid: add_legend: loc: 'upper left' layers: - type: Scatter - x: - variable: experiment::ObsValue::${variable} y: - variable: experiment::GsiHofXBc::${variable} - channel: ${channel} - markersize: 5 - color: 'black' - label: 'GSI h(x) versus obs (all obs)' - - type: Scatter + variable: experiment::hofx::${variable} x: - variable: experiment::ObsValue::${variable} - y: - variable: experiment::GsiHofXBcPassedQc::${variable} + variable: experiment::GsiHofXBc::${variable} channel: ${channel} markersize: 5 - color: 'red' - label: 'GSI h(x) versus obs (passed QC in JEDI)' + color: 'blue' + label: 'JEDI h(x) versus GSI h(x)' - # JEDI h(x) vs GSI h(x) + # Correlation scatter JEDI h(x) - GSI H(x) vs GSI h(x) + # ---------------------------------------------------- - batch figure: variables: *variables channels: *channels figure: layout: [1,1] title: 'JEDI h(x) vs. GSI h(x) | {{instrument_title}} | ${variable_title}' - output name: '{{cycle_dir}}/eva/{{instrument}}/correlation_scatter/${variable}${channel}/gsi_hofx_vs_jedi_hofx_{{instrument}}_${variable}${channel}.png' + output name: '{{cycle_dir}}/eva/{{instrument}}/${variable}${channel}_c_hofxdiff_vs_gsihofx.png' plots: - add_xlabel: 'GSI h(x)' add_ylabel: 'JEDI h(x)' @@ -165,244 +190,37 @@ diagnostics: loc: 'upper left' layers: - type: Scatter - x: - variable: experiment::GsiHofXBc::${variable} y: - variable: experiment::hofx::${variable} - channel: ${channel} - markersize: 5 - color: 'black' - label: 'JEDI h(x) versus GSI h(x)' - - type: Scatter + variable: experiment::hofxDiff::${variable} x: - variable: experiment::GsiHofXBcPassedQc::${variable} - y: - variable: experiment::hofxPassedQc::${variable} + variable: experiment::GsiHofX::${variable} channel: ${channel} markersize: 5 - color: 'red' - label: 'JEDI h(x) versus GSI h(x) (passed QC in JEDI)' + color: 'blue' + label: 'JEDI h(x) versus GSI h(x)' - # JEDI omb vs GSI omb + # Correlation scatter JEDI h(x) - GSI H(x) BC vs GSI h(x) BC + # ---------------------------------------------------------- - batch figure: variables: *variables channels: *channels figure: layout: [1,1] - title: 'JEDI omb vs. GSI omb | {{instrument_title}} | ${variable_title}' - output name: '{{cycle_dir}}/eva/{{instrument}}/correlation_scatter/${variable}${channel}/gsi_omb_vs_jedi_omb_{{instrument}}_${variable}${channel}.png' + title: 'JEDI h(x) vs. GSI h(x) | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/${variable}${channel}_d_hofxdiffbc_vs_gsihofxbc.png' plots: - - add_xlabel: 'GSI observation minus h(x)' - add_ylabel: 'JEDI observation minus h(x)' + - add_xlabel: 'GSI h(x)' + add_ylabel: 'JEDI h(x)' add_grid: add_legend: loc: 'upper left' layers: - type: Scatter - x: - variable: experiment::ObsValueMinusGsiHofXBc::${variable} y: - variable: experiment::ObsValueMinusHofx::${variable} - channel: ${channel} - markersize: 5 - color: 'black' - label: 'GSI omb vs JEDI omb (all obs)' - - type: Scatter + variable: experiment::hofxDiffBc::${variable} x: - variable: experiment::ObsValueMinusGsiHofXBcPassedQc::${variable} - y: - variable: experiment::ObsValueMinushofxPassedQc::${variable} + variable: experiment::GsiHofXBc::${variable} channel: ${channel} markersize: 5 - color: 'red' - label: 'GSI omb vs JEDI omb (passed QC in JEDI)' - -# Map plots -# --------- - - # Observations - - batch figure: - variables: *variables - channels: *channels - dynamic options: - - type: vminvmaxcmap - channel: ${channel} - data variable: experiment::ObsValue::${variable} - figure: - figure size: [20,10] - layout: [1,1] - title: 'Observations | {{instrument_title}} | Obs Value' - output name: '{{cycle_dir}}/eva/{{instrument}}/map_plots/${variable}${channel}/observations_{{instrument}}_${variable}${channel}.png' - plots: - - mapping: - projection: plcarr - domain: global - add_map_features: ['coastline'] - add_colorbar: - label: ObsValue - add_grid: - layers: - - type: MapScatter - longitude: - variable: experiment::MetaData::longitude - latitude: - variable: experiment::MetaData::latitude - data: - variable: experiment::ObsValue::${variable} - channel: ${channel} - markersize: 2 - label: ObsValue - colorbar: true - cmap: ${dynamic_cmap} - vmin: ${dynamic_vmin} - vmax: ${dynamic_vmax} - - # omb jedi - - batch figure: - variables: *variables - channels: *channels - dynamic options: - - type: vminvmaxcmap - channel: ${channel} - data variable: experiment::ObsValueMinusHofx::${variable} - figure: - figure size: [20,10] - layout: [1,1] - title: 'JEDI OmB | {{instrument_title}} | ${variable_title}' - output name: '{{cycle_dir}}/eva/{{instrument}}/map_plots/${variable}${channel}/omb_jedi_{{instrument}}_${variable}${channel}.png' - plots: - - mapping: - projection: plcarr - domain: global - add_map_features: ['coastline'] - add_colorbar: - label: '${variable}' - add_grid: - layers: - - type: MapScatter - longitude: - variable: experiment::MetaData::longitude - latitude: - variable: experiment::MetaData::latitude - data: - variable: experiment::ObsValueMinusHofx::${variable} - channel: ${channel} - markersize: 2 - label: '${variable}' - colorbar: true - cmap: ${dynamic_cmap} - vmin: ${dynamic_vmin} - vmax: ${dynamic_vmax} - - # omb gsi - - batch figure: - variables: *variables - channels: *channels - dynamic options: - - type: vminvmaxcmap - channel: ${channel} - data variable: experiment::ObsValueMinusGsiHofXBc::${variable} - figure: - figure size: [20,10] - layout: [1,1] - title: 'GSI OmB | {{instrument_title}} | ${variable_title}' - output name: '{{cycle_dir}}/eva/{{instrument}}/map_plots/${variable}${channel}/omb_gsi_{{instrument}}_${variable}${channel}.png' - plots: - - mapping: - projection: plcarr - domain: global - add_map_features: ['coastline'] - add_colorbar: - label: '${variable}' - add_grid: - layers: - - type: MapScatter - longitude: - variable: experiment::MetaData::longitude - latitude: - variable: experiment::MetaData::latitude - data: - variable: experiment::ObsValueMinusGsiHofXBc::${variable} - channel: ${channel} - markersize: 2 - label: '${variable}' - colorbar: true - cmap: ${dynamic_cmap} - vmin: ${dynamic_vmin} - vmax: ${dynamic_vmax} - - # hofx difference - - batch figure: - variables: *variables - channels: *channels - dynamic options: - - type: vminvmaxcmap - channel: ${channel} - data variable: experiment::HofxMinusGsiHofXBc::${variable} - figure: - figure size: [20,10] - layout: [1,1] - title: 'Hofx Difference | {{instrument_title}} | ${variable_title}' - output name: '{{cycle_dir}}/eva/{{instrument}}/map_plots/${variable}${channel}/hofx_difference_{{instrument}}_${variable}${channel}.png' - plots: - - mapping: - projection: plcarr - domain: global - add_map_features: ['coastline'] - add_colorbar: - label: '${variable}' - add_grid: - layers: - - type: MapScatter - longitude: - variable: experiment::MetaData::longitude - latitude: - variable: experiment::MetaData::latitude - data: - variable: experiment::HofxMinusGsiHofXBc::${variable} - channel: ${channel} - markersize: 2 - label: '${variable}' - colorbar: true - cmap: ${dynamic_cmap} - vmin: ${dynamic_vmin} - vmax: ${dynamic_vmax} - -# Histogram plots -# --------------- - - # omb vs omb - - batch figure: - variables: *variables - channels: *channels - dynamic options: - - type: histogram_bins - channel: ${channel} - number of bins rule: sturges - data variable: experiment::ObsValueMinusHofx::${variable} - figure: - layout: [1,1] - title: 'JEDI omb vs. GSI omb | {{instrument_title}} | ${variable_title}' - output name: '{{cycle_dir}}/eva/{{instrument}}/histograms/${variable}${channel}/gsi_omb_vs_jedi_omb_{{instrument}}_${variable}${channel}.png' - plots: - - add_xlabel: 'Observation minus h(x)' - add_ylabel: 'Count' - add_legend: - loc: 'upper left' - layers: - - type: Histogram - data: - variable: experiment::ObsValueMinusGsiHofXBc::${variable} - channel: ${channel} color: 'blue' - label: 'GSI omb (all obs)' - bins: ${dynamic_bins} - alpha: 0.5 - - type: Histogram - data: - variable: experiment::ObsValueMinusHofx::${variable} - channel: ${channel} - color: 'red' - label: 'JEDI omb (all obs)' - bins: ${dynamic_bins} - alpha: 0.5 + label: 'JEDI h(x) versus GSI h(x)' diff --git a/src/swell/tasks/run_jedi_test_obs_filters_executable.py b/src/swell/tasks/run_jedi_test_obs_filters_executable.py index b2905976..3e7acd6e 100644 --- a/src/swell/tasks/run_jedi_test_obs_filters_executable.py +++ b/src/swell/tasks/run_jedi_test_obs_filters_executable.py @@ -64,10 +64,14 @@ def execute(self): f'{observations[index]}_geovals.{window_begin}.nc4') # For conventional add the GeoVaLs flip if observations[index] in conventional_types: - geovals['levels_are_top_down'] = False + geovals['levels_are_top_down'] = True jedi_config_dict['observations'][index]['geovals'] = geovals - jedi_config_dict['observations'][index]['passedBenchmark'] = 317688 + + # Need to insert at least one benchmark, but we do not really want to check anything + # so check that some made up variable does not exist + jedi_config_dict['observations'][index]['expectVariablesNotToExist'] = \ + [{'name': 'Fake/Group/Var'}] # Write executable configuration to file # -------------------------------------- From e846b81fa014a23f9bd315f35991e4612bb52932 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Thu, 13 Apr 2023 16:52:51 -0400 Subject: [PATCH 007/121] move to R2D2 for getting the observations --- src/swell/suites/test_obs_filters/flow.cylc | 3 +- .../geos_atmosphere/eva_observations.yaml | 242 ++++++++++++++---- ...ites-test_obs_filters-geos_atmosphere.yaml | 28 +- src/swell/tasks/clean_cycle.py | 68 +++-- 4 files changed, 275 insertions(+), 66 deletions(-) diff --git a/src/swell/suites/test_obs_filters/flow.cylc b/src/swell/suites/test_obs_filters/flow.cylc index e700c0ee..0aae9e6c 100644 --- a/src/swell/suites/test_obs_filters/flow.cylc +++ b/src/swell/suites/test_obs_filters/flow.cylc @@ -106,7 +106,8 @@ script = "swell_task GetGsiBc $config -d $datetime -m geos_atmosphere" [[ GsiNcdiagToIoda ]] - script = "swell_task GsiNcdiagToIoda $config -d $datetime -m geos_atmosphere" + #script = "swell_task GsiNcdiagToIoda $config -d $datetime -m geos_atmosphere" + script = "swell_task GetObservations $config -d $datetime -m geos_atmosphere" [[ GsiBcToIoda ]] script = "swell_task GsiBcToIoda $config -d $datetime -m geos_atmosphere" diff --git a/src/swell/suites/test_obs_filters/geos_atmosphere/eva_observations.yaml b/src/swell/suites/test_obs_filters/geos_atmosphere/eva_observations.yaml index 8b725b40..8ac0670a 100644 --- a/src/swell/suites/test_obs_filters/geos_atmosphere/eva_observations.yaml +++ b/src/swell/suites/test_obs_filters/geos_atmosphere/eva_observations.yaml @@ -7,6 +7,7 @@ diagnostics: - name: experiment filenames: - {{obs_path_file}} + missing_value_threshold: 1.0e06 channels: &channels {{channels}} groups: - name: ObsValue @@ -79,45 +80,40 @@ diagnostics: for: variable: *variables + # Error Difference + # ---------------- + - transform: arithmetic + new name: experiment::EffectiveErrorPassedQcDiff::${variable} + equals: experiment::FinalObsErrorPassedQc::${variable}-experiment::EffectiveErrorPassedQc::${variable} + for: + variable: *variables + + # Observation minus Hofx passing QC + # --------------------------------- + - transform: accept where + new name: experiment::ObsValueMinusHofxPassedQc::${variable} + starting field: experiment::ObsValueMinusHofx::${variable} + where: + - experiment::EffectiveQC::${variable} == 0 + for: + variable: *variables + + - transform: accept where + new name: experiment::ObsValueMinusGsiHofXPassedQc::${variable} + starting field: experiment::ObsValueMinusGsiHofX::${variable} + where: + - experiment::GsiEffectiveQC::${variable} == 0 + for: + variable: *variables + - transform: accept where + new name: experiment::ObsValueMinusGsiHofXBcPassedQc::${variable} + starting field: experiment::ObsValueMinusGsiHofXBc::${variable} + where: + - experiment::GsiEffectiveQC::${variable} == 0 + for: + variable: *variables -# # hofx that passed qz -# -# # Generate hofx that passed QC for JEDI -# - transform: accept where -# new name: experiment::hofxPassedQc::${variable} -# starting field: experiment::hofx::${variable} -# where: -# - experiment::EffectiveQC::${variable} == 0 -# for: -# variable: *variables -# -# # Generate GSI hofx that passed JEDI QC -# - transform: accept where -# new name: experiment::GsiHofXBcPassedQc::${variable} -# starting field: experiment::GsiHofXBc::${variable} -# where: -# - experiment::EffectiveQC::${variable} == 0 -# for: -# variable: *variables -# -# # Generate omb that passed QC for JEDI -# - transform: accept where -# new name: experiment::ObsValueMinushofxPassedQc::${variable} -# starting field: experiment::ObsValueMinusHofx::${variable} -# where: -# - experiment::EffectiveQC::${variable} == 0 -# for: -# variable: *variables -# -# # Generate omb that passed QC for GSI -# - transform: accept where -# new name: experiment::ObsValueMinusGsiHofXBcPassedQc::${variable} -# starting field: experiment::ObsValueMinusGsiHofXBc::${variable} -# where: -# - experiment::EffectiveQC::${variable} == 0 -# for: -# variable: *variables graphics: @@ -154,10 +150,10 @@ diagnostics: channels: *channels figure: layout: [1,1] - title: 'JEDI h(x) vs. GSI h(x) | {{instrument_title}} | ${variable_title}' + title: 'JEDI h(x) vs. GSI h(x) BC | {{instrument_title}} | ${variable_title}' output name: '{{cycle_dir}}/eva/{{instrument}}/${variable}${channel}_b_hofx_vs_gsihofxbc.png' plots: - - add_xlabel: 'GSI h(x)' + - add_xlabel: 'GSI h(x) BC' add_ylabel: 'JEDI h(x)' add_grid: add_legend: @@ -180,11 +176,11 @@ diagnostics: channels: *channels figure: layout: [1,1] - title: 'JEDI h(x) vs. GSI h(x) | {{instrument_title}} | ${variable_title}' + title: 'JEDI h(x) - GSI h(x) vs. GSI h(x) | {{instrument_title}} | ${variable_title}' output name: '{{cycle_dir}}/eva/{{instrument}}/${variable}${channel}_c_hofxdiff_vs_gsihofx.png' plots: - add_xlabel: 'GSI h(x)' - add_ylabel: 'JEDI h(x)' + add_ylabel: 'JEDI h(x) - GSI h(x)' add_grid: add_legend: loc: 'upper left' @@ -206,11 +202,11 @@ diagnostics: channels: *channels figure: layout: [1,1] - title: 'JEDI h(x) vs. GSI h(x) | {{instrument_title}} | ${variable_title}' + title: 'JEDI h(x) - GSI h(x) BC vs. GSI h(x) BC | {{instrument_title}} | ${variable_title}' output name: '{{cycle_dir}}/eva/{{instrument}}/${variable}${channel}_d_hofxdiffbc_vs_gsihofxbc.png' plots: - - add_xlabel: 'GSI h(x)' - add_ylabel: 'JEDI h(x)' + - add_xlabel: 'GSI h(x) BC' + add_ylabel: 'JEDI h(x) - GSI h(x) BC' add_grid: add_legend: loc: 'upper left' @@ -224,3 +220,161 @@ diagnostics: markersize: 5 color: 'blue' label: 'JEDI h(x) versus GSI h(x)' + + + # Effective Error vs GSI Final Error (Passed QC) + # ---------------------------------------------- + - batch figure: + variables: *variables + channels: *channels + figure: + layout: [1,1] + title: 'JEDI Effective Error vs. GSI FinalObsError (Passing QC) | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/${variable}${channel}_e_effectiveerror_vs_gsifinalerror.png' + plots: + - add_xlabel: 'GSI FinalObsError (Passed QC)' + add_ylabel: 'JEDI Effective Error (Passed QC)' + add_grid: + add_legend: + loc: 'upper left' + layers: + - type: Scatter + y: + variable: experiment::EffectiveErrorPassedQc::${variable} + x: + variable: experiment::FinalObsErrorPassedQc::${variable} + channel: ${channel} + markersize: 5 + color: 'blue' + label: 'JEDI h(x) versus GSI h(x)' + + + # Effective Error - GSI Final Error vs GSI Final Error (Passed QC) + # ---------------------------------------------------------------- + - batch figure: + variables: *variables + channels: *channels + figure: + layout: [1,1] + title: 'JEDI Effective Error vs. GSI FinalObsError (Passing QC) | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/${variable}${channel}_f_effectiveerrordiff_vs_gsifinalerror.png' + plots: + - add_xlabel: 'GSI FinalObsError (Passed QC)' + add_ylabel: 'JEDI Effective Error (Passed QC) - GSI FinalObsError (Passed QC)' + add_grid: + add_legend: + loc: 'upper left' + layers: + - type: Scatter + y: + variable: experiment::EffectiveErrorPassedQcDiff::${variable} + x: + variable: experiment::FinalObsErrorPassedQc::${variable} + channel: ${channel} + markersize: 5 + color: 'blue' + label: 'JEDI h(x) versus GSI h(x)' + + + # Density plot for observation minus background passed QC + # ------------------------------------------------------- + - batch figure: + variables: *variables + figure: + layout: [1,1] + title: 'Observation minus Background Density (Passing QC) | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/${variable}${channel}_g_omb_density.png' + plots: + - add_xlabel: 'Observation minus h(x)' + add_ylabel: 'Density' + add_legend: + loc: 'upper left' + statistics: + fields: + - field_name: experiment::ObsValueMinusGsiHofXPassedQc::${variable} + xloc: 0.5 + yloc: -0.10 + kwargs: + color: 'blue' + fontsize: 8 + fontfamily: monospace + - field_name: experiment::ObsValueMinusHofxPassedQc::${variable} + xloc: 0.5 + yloc: -0.13 + kwargs: + color: 'red' + fontsize: 8 + fontfamily: monospace + statistics_variables: + - n + - min + - mean + - max + - std + layers: + - type: Density + data: + variable: experiment::ObsValueMinusGsiHofXPassedQc::${variable} + color: 'blue' + label: 'GSI omb (Passed QC)' + alpha: 0.5 + bw_adjust: 0.1 # Reduce this value to fit the data more closely + - type: Density + data: + variable: experiment::ObsValueMinusHofxPassedQc::${variable} + color: 'red' + label: 'JEDI omb (Passed QC)' + alpha: 0.5 + bw_adjust: 0.1 # Reduce this value to fit the data more closely + + + # Density plot for observation minus background passed QC (bias corrected) + # ------------------------------------------------------------------------ + - batch figure: + variables: *variables + figure: + layout: [1,1] + title: 'Observation minus Background Density (Passing QC) | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/${variable}${channel}_h_ombbc_density.png' + plots: + - add_xlabel: 'Observation minus h(x)' + add_ylabel: 'Density' + add_legend: + loc: 'upper left' + statistics: + fields: + - field_name: experiment::ObsValueMinusGsiHofXBcPassedQc::${variable} + xloc: 0.5 + yloc: -0.10 + kwargs: + color: 'blue' + fontsize: 8 + fontfamily: monospace + - field_name: experiment::ObsValueMinusHofxPassedQc::${variable} + xloc: 0.5 + yloc: -0.13 + kwargs: + color: 'red' + fontsize: 8 + fontfamily: monospace + statistics_variables: + - n + - min + - mean + - max + - std + layers: + - type: Density + data: + variable: experiment::ObsValueMinusGsiHofXBcPassedQc::${variable} + color: 'blue' + label: 'GSI omb BC (Passed QC)' + alpha: 0.5 + bw_adjust: 0.1 # Reduce this value to fit the data more closely + - type: Density + data: + variable: experiment::ObsValueMinusHofxPassedQc::${variable} + color: 'red' + label: 'JEDI omb (Passed QC)' + alpha: 0.5 + bw_adjust: 0.1 # Reduce this value to fit the data more closely diff --git a/src/swell/suites/test_obs_filters/geos_atmosphere/suites-test_obs_filters-geos_atmosphere.yaml b/src/swell/suites/test_obs_filters/geos_atmosphere/suites-test_obs_filters-geos_atmosphere.yaml index 9332d1b1..2ff61a30 100644 --- a/src/swell/suites/test_obs_filters/geos_atmosphere/suites-test_obs_filters-geos_atmosphere.yaml +++ b/src/swell/suites/test_obs_filters/geos_atmosphere/suites-test_obs_filters-geos_atmosphere.yaml @@ -48,14 +48,13 @@ observations: # - sondes # - ssmis_f17 # - vadwind - prompt: Select observations to run with. options: use_method type: string-check-list # Path to GSI ncdiags path_to_gsi_diags: - default_value: '/discover/nobackup/drholdaw/SwellTestData/ncdiag/' + default_value: '/archive/u/jjin3/x0044.jj_20230201/obs/Y2020/M12/D15/H00' prompt: Path to where the ncdiags will be held type: string @@ -65,9 +64,15 @@ produce_geovals: prompt: Compute GeoVaLs as well as observations? type: boolean +# Experiment that provide observations +obs_experiment: + default_value: x0044_test_obs_filters + prompt: Experiment providing the Observations + type: string + # Experiment that provide geovals geovals_experiment: - default_value: x0044_v3_geovals + default_value: x0044_test_obs_filters_geovals prompt: Experiment providing the GeoVaLs type: string @@ -86,10 +91,18 @@ fixed_options: default_value: 1 prompt: Equation to compute total number of processors clean_patterns: - default_value: ['*.nc4','*.txt','logfile.*.out'] + default_value: + - '*.nc4' + - '*.txt' + - '*.log' + - '*.yaml' + - gsi_bcs/*.nc4 + - gsi_bcs/*.txt + - gsi_bcs/*.yaml + - gsi_bcs + - gsi_ncdiags/*.nc4 + - gsi_ncdiags prompt: 'Patterns for the files to remove after completing a cycle' - - # Needed because of config gen window_type: default_value: 4D @@ -100,6 +113,9 @@ fixed_options: background_time_offset: default_value: PT9H prompt: Time before the middle of the window that the background providing forecast began + obs_provider: + default_value: ncdiag + prompt: Database providing the Observations geovals_provider: default_value: ncdiag prompt: Database providing the GeoVaLs diff --git a/src/swell/tasks/clean_cycle.py b/src/swell/tasks/clean_cycle.py index 79ac400d..d1d6106d 100644 --- a/src/swell/tasks/clean_cycle.py +++ b/src/swell/tasks/clean_cycle.py @@ -27,6 +27,7 @@ class CleanCycle(taskBase): def execute(self): + # Parse config cycle_dir = self.config_get("cycle_dir") clean_list = self.config_get('clean_patterns', None) @@ -34,20 +35,57 @@ def execute(self): if clean_list is None: return - if os.path.isdir(cycle_dir): - os.chdir(cycle_dir) - # Remove all specified files - for pattern in clean_list: - if pattern == '*': - continue - path_parts = os.path.split(pattern) - if path_parts[1] == '*': - continue + # Move to the cycle directory + os.chdir(cycle_dir) + + # Remove all specified files + for pattern in clean_list: + + # --------------------------- + # Perform some safety checks: + # --------------------------- + + # 1. Check that path is not absolute. Things can only be deleted relative to the cycle + # directory. + if os.path.isabs(pattern): + self.logger.abort(f'Absolute paths are forbidden. Offending entry: {pattern}') + + # 2. Check that the pattern does not begin with a / or ./ + if pattern[0] == '/' or pattern[0] == '.': + self.logger.abort(f'Patterns beginning with \'/\' or \'.\' are forbidden. ' + f'Offending entry: {pattern}') + + # 3. Check that the pattern is not a blanket removal of all files + if any(ele == '*' for ele in os.path.split(pattern)): + self.logger.abort(f'Deleting all files from any directory is forbidden. Offending ' + f'entry: {pattern}') + + # --------------------------- + # --------------------------- + + # Assemble list of files to remove + items_to_remove = glob.glob(pattern) + + # Loop over files and remove + for item_to_remove in items_to_remove: + + # Print info about what will be removed + self.logger.info(f'Removing item {item_to_remove}') + + # Only allow removing of empty directories + if os.path.isdir(item_to_remove): + + if len(os.listdir(item_to_remove)) == 0: + os.rmdir(item_to_remove) + else: + self.logger.info(f'Trying to remove directory {item_to_remove} but code ' + f'can only remove empty directories. Reorder removal ' + f'to empty directory first.') + else: - for file_to_delete in glob.glob(path_parts[1]): - os.remove(file_to_delete) - # Save cycle_done file to cycle_dir - filename = 'cycle_done' - cmd = 'touch ' + os.path.join(cycle_dir, filename) - os.system(cmd) + os.remove(item_to_remove) + + # Save cycle_done file to cycle_dir + with open(os.path.join(cycle_dir, 'cycle_done'), 'w') as fp: + pass From 3ac880a20b7c8a38d68f6176aa0c86abf12376bc Mon Sep 17 00:00:00 2001 From: danholdaway Date: Thu, 13 Apr 2023 16:56:04 -0400 Subject: [PATCH 008/121] convert to using GetObservations --- src/swell/suites/test_obs_filters/flow.cylc | 32 ++++++++++++--------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/src/swell/suites/test_obs_filters/flow.cylc b/src/swell/suites/test_obs_filters/flow.cylc index 0aae9e6c..df423b34 100644 --- a/src/swell/suites/test_obs_filters/flow.cylc +++ b/src/swell/suites/test_obs_filters/flow.cylc @@ -38,19 +38,23 @@ {% for cycle_time in cycle_times %} {{cycle_time.cycle_time}} = """ - # Task triggers - # ------------- - GetGsiNcdiag - GetGsiBc - GetGsiNcdiag => GsiNcdiagToIoda + # Convert ncdiags to ioda + #GetGsiNcdiag + #GetGsiNcdiag => GsiNcdiagToIoda + GetObservations + + # Convert ncdiags to ioda + GetGsiBc GetGsiBc => GsiBcToIoda + # Get GeoVaLs GetGeovals # Run Jedi hofx executable BuildJediByLinking[^]? | BuildJedi[^] => RunJediTestObsFiltersExecutable - GsiNcdiagToIoda => RunJediTestObsFiltersExecutable + #GsiNcdiagToIoda => RunJediTestObsFiltersExecutable + GetObservations => RunJediTestObsFiltersExecutable GsiBcToIoda => RunJediTestObsFiltersExecutable GetGeovals => RunJediTestObsFiltersExecutable @@ -99,19 +103,21 @@ --ntasks-per-node={{scheduling["BuildJedi"]["ntasks_per_node"]}} --constraint={{scheduling["BuildJedi"]["constraint"]}} - [[ GetGsiNcdiag ]] - script = "swell_task GetGsiNcdiag $config -d $datetime -m geos_atmosphere" - [[ GetGsiBc ]] script = "swell_task GetGsiBc $config -d $datetime -m geos_atmosphere" - [[ GsiNcdiagToIoda ]] - #script = "swell_task GsiNcdiagToIoda $config -d $datetime -m geos_atmosphere" - script = "swell_task GetObservations $config -d $datetime -m geos_atmosphere" - [[ GsiBcToIoda ]] script = "swell_task GsiBcToIoda $config -d $datetime -m geos_atmosphere" +# [[ GetGsiNcdiag ]] +# script = "swell_task GetGsiNcdiag $config -d $datetime -m geos_atmosphere" +# +# [[ GsiNcdiagToIoda ]] +# script = "swell_task GsiNcdiagToIoda $config -d $datetime -m geos_atmosphere" + + [[ GetObservations ]] + script = "swell_task GetObservations $config -d $datetime -m geos_atmosphere" + [[ GetGeovals ]] script = "swell_task GetGeovals $config -d $datetime -m geos_atmosphere" From 28c857741b7c0f59d6831c068830f393dc93ccf1 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Tue, 2 May 2023 13:11:01 -0400 Subject: [PATCH 009/121] append --- src/swell/tasks/gsi_ncdiag_to_ioda.py | 154 +++++++++++++------------- 1 file changed, 77 insertions(+), 77 deletions(-) diff --git a/src/swell/tasks/gsi_ncdiag_to_ioda.py b/src/swell/tasks/gsi_ncdiag_to_ioda.py index fa74b35b..f12a993c 100644 --- a/src/swell/tasks/gsi_ncdiag_to_ioda.py +++ b/src/swell/tasks/gsi_ncdiag_to_ioda.py @@ -75,34 +75,34 @@ def execute(self): # First process the conventional data (if needed) # ----------------------------------------------- - for gsi_type_to_process in gsi_types_to_process: - - log_str = f'Processing GSI file {gsi_type_to_process}' - self.logger.info('', wrap=False) - self.logger.info(log_str) - self.logger.info('-'*len(log_str)) - - gsi_conv_file = glob.glob(os.path.join(gsi_diag_dir, f'*{gsi_type_to_process}*'))[0] - - # Open the file - Diag = gsid.Conv(gsi_conv_file) - Diag.read() - - # Assemble list of needed platforms - needed_platforms = [] - for platform in gsid.conv_platforms[gsi_type_to_process]: - if platform in needed_ioda_types: - needed_platforms.append(platform) - - # Extract data - Diag.toIODAobs(cycle_dir, platforms=needed_platforms) - - if produce_geovals: - self.logger.info('', wrap=False) - self.logger.info(f'Processing GeoVaLs from {gsi_type_to_process}') - Diag.toGeovals(cycle_dir) - - Diag.close() +# for gsi_type_to_process in gsi_types_to_process: +# +# log_str = f'Processing GSI file {gsi_type_to_process}' +# self.logger.info('', wrap=False) +# self.logger.info(log_str) +# self.logger.info('-'*len(log_str)) +# +# gsi_conv_file = glob.glob(os.path.join(gsi_diag_dir, f'*{gsi_type_to_process}*'))[0] +# +# # Open the file +# Diag = gsid.Conv(gsi_conv_file) +# Diag.read() +# +# # Assemble list of needed platforms +# needed_platforms = [] +# for platform in gsid.conv_platforms[gsi_type_to_process]: +# if platform in needed_ioda_types: +# needed_platforms.append(platform) +# +# # Extract data +# Diag.toIODAobs(cycle_dir, platforms=needed_platforms) +# +# if produce_geovals: +# self.logger.info('', wrap=False) +# self.logger.info(f'Processing GeoVaLs from {gsi_type_to_process}') +# Diag.toGeovals(cycle_dir) +# +# Diag.close() # Combine the conventional data # ----------------------------- @@ -167,54 +167,54 @@ def execute(self): else: self.logger.abort(f'Combine failed for {needed_ioda_type}, file name issue.') - # Get list of the observations that are ozone observations - # -------------------------------------------------------- - ozone_sensors = gsid.oz_lay_sensors + gsid.oz_lev_sensors - ozone_observations = [] - for observation in observations: - for ozone_sensor in ozone_sensors: - if ozone_sensor in observation: - ozone_observations.append(observation) - - # Copy all the files into the cycle directory - # ------------------------------------------- - for observation in observations: - - self.logger.info(f'Converting {observation} to IODA format') - - gsi_obs_file = glob.glob(os.path.join(gsi_diag_dir, f'*{observation}*')) - - if observation not in ozone_observations: - - # Radiances - Diag = gsid.Radiances(gsi_obs_file[0]) - Diag.read() - Diag.toIODAobs(cycle_dir, False, False, False) - - else: - - # Ozone - Diag = gsid.Ozone(gsi_obs_file[0]) - Diag.read() - Diag.toIODAobs(cycle_dir) - - # GeoVaLs call - if produce_geovals: - Diag.toGeovals(cycle_dir) - - if observation not in ozone_observations: - Diag.close() - - # Rename files to be swell compliant - # ---------------------------------- - for observation in observations_orig: - - # Input filename - ioda_file_in_pattern = f'{observation}_obs_*nc*' - ioda_file_in = glob.glob(os.path.join(cycle_dir, ioda_file_in_pattern))[0] - - ioda_file_out = f'{observation}.{window_begin}.nc4' - - os.rename(ioda_file_in, os.path.join(cycle_dir, ioda_file_out)) +# # Get list of the observations that are ozone observations +# # -------------------------------------------------------- +# ozone_sensors = gsid.oz_lay_sensors + gsid.oz_lev_sensors +# ozone_observations = [] +# for observation in observations: +# for ozone_sensor in ozone_sensors: +# if ozone_sensor in observation: +# ozone_observations.append(observation) +# +# # Copy all the files into the cycle directory +# # ------------------------------------------- +# for observation in observations: +# +# self.logger.info(f'Converting {observation} to IODA format') +# +# gsi_obs_file = glob.glob(os.path.join(gsi_diag_dir, f'*{observation}*')) +# +# if observation not in ozone_observations: +# +# # Radiances +# Diag = gsid.Radiances(gsi_obs_file[0]) +# Diag.read() +# Diag.toIODAobs(cycle_dir, False, False, False) +# +# else: +# +# # Ozone +# Diag = gsid.Ozone(gsi_obs_file[0]) +# Diag.read() +# Diag.toIODAobs(cycle_dir) +# +# # GeoVaLs call +# if produce_geovals: +# Diag.toGeovals(cycle_dir) +# +# if observation not in ozone_observations: +# Diag.close() +# +# # Rename files to be swell compliant +# # ---------------------------------- +# for observation in observations_orig: +# +# # Input filename +# ioda_file_in_pattern = f'{observation}_obs_*nc*' +# ioda_file_in = glob.glob(os.path.join(cycle_dir, ioda_file_in_pattern))[0] +# +# ioda_file_out = f'{observation}.{window_begin}.nc4' +# +# os.rename(ioda_file_in, os.path.join(cycle_dir, ioda_file_out)) # -------------------------------------------------------------------------------------------------- From 66791dafcb10675155ce999cd6f88531830274c2 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 10 May 2023 13:22:51 -0400 Subject: [PATCH 010/121] uncomment --- src/swell/tasks/gsi_ncdiag_to_ioda.py | 177 ++++++++++++++------------ 1 file changed, 99 insertions(+), 78 deletions(-) diff --git a/src/swell/tasks/gsi_ncdiag_to_ioda.py b/src/swell/tasks/gsi_ncdiag_to_ioda.py index f12a993c..b9205831 100644 --- a/src/swell/tasks/gsi_ncdiag_to_ioda.py +++ b/src/swell/tasks/gsi_ncdiag_to_ioda.py @@ -18,7 +18,7 @@ from gsi_ncdiag.combine_obsspace import combine_obsspace from swell.tasks.base.task_base import taskBase - +from swell.utilities.shell_commands import run_track_log_subprocess # -------------------------------------------------------------------------------------------------- @@ -75,34 +75,34 @@ def execute(self): # First process the conventional data (if needed) # ----------------------------------------------- -# for gsi_type_to_process in gsi_types_to_process: -# -# log_str = f'Processing GSI file {gsi_type_to_process}' -# self.logger.info('', wrap=False) -# self.logger.info(log_str) -# self.logger.info('-'*len(log_str)) -# -# gsi_conv_file = glob.glob(os.path.join(gsi_diag_dir, f'*{gsi_type_to_process}*'))[0] -# -# # Open the file -# Diag = gsid.Conv(gsi_conv_file) -# Diag.read() -# -# # Assemble list of needed platforms -# needed_platforms = [] -# for platform in gsid.conv_platforms[gsi_type_to_process]: -# if platform in needed_ioda_types: -# needed_platforms.append(platform) -# -# # Extract data -# Diag.toIODAobs(cycle_dir, platforms=needed_platforms) -# -# if produce_geovals: -# self.logger.info('', wrap=False) -# self.logger.info(f'Processing GeoVaLs from {gsi_type_to_process}') -# Diag.toGeovals(cycle_dir) -# -# Diag.close() + for gsi_type_to_process in gsi_types_to_process: + + log_str = f'Processing GSI file {gsi_type_to_process}' + self.logger.info('', wrap=False) + self.logger.info(log_str) + self.logger.info('-'*len(log_str)) + + gsi_conv_file = glob.glob(os.path.join(gsi_diag_dir, f'*{gsi_type_to_process}*'))[0] + + # Open the file + Diag = gsid.Conv(gsi_conv_file) + Diag.read() + + # Assemble list of needed platforms + needed_platforms = [] + for platform in gsid.conv_platforms[gsi_type_to_process]: + if platform in needed_ioda_types: + needed_platforms.append(platform) + + # Extract data + Diag.toIODAobs(cycle_dir, platforms=needed_platforms) + + if produce_geovals: + self.logger.info('', wrap=False) + self.logger.info(f'Processing GeoVaLs from {gsi_type_to_process}') + Diag.toGeovals(cycle_dir) + + Diag.close() # Combine the conventional data # ----------------------------- @@ -156,6 +156,17 @@ def execute(self): geo_dir = None if produce_geovals: geo_dir = cycle_dir + + # Remove wind_reduction_factor_at_10m from non-uv geoval files + geoval_files = glob.glob(os.path.join(cycle_dir, + f'{needed_ioda_type}_*_geoval_*.nc4')) + for geoval_file in geoval_files: + if f'{needed_ioda_type}_uv_geoval_' not in geoval_file: + var_remove_command = ['ncks', '-O', '-x', '-v', + 'wind_reduction_factor_at_10m', + geoval_file, geoval_file] + run_track_log_subprocess(self.logger, var_remove_command) + combine_obsspace(ioda_path_files, new_name, geo_dir) # Remove input files @@ -167,54 +178,64 @@ def execute(self): else: self.logger.abort(f'Combine failed for {needed_ioda_type}, file name issue.') -# # Get list of the observations that are ozone observations -# # -------------------------------------------------------- -# ozone_sensors = gsid.oz_lay_sensors + gsid.oz_lev_sensors -# ozone_observations = [] -# for observation in observations: -# for ozone_sensor in ozone_sensors: -# if ozone_sensor in observation: -# ozone_observations.append(observation) -# -# # Copy all the files into the cycle directory -# # ------------------------------------------- -# for observation in observations: -# -# self.logger.info(f'Converting {observation} to IODA format') -# -# gsi_obs_file = glob.glob(os.path.join(gsi_diag_dir, f'*{observation}*')) -# -# if observation not in ozone_observations: -# -# # Radiances -# Diag = gsid.Radiances(gsi_obs_file[0]) -# Diag.read() -# Diag.toIODAobs(cycle_dir, False, False, False) -# -# else: -# -# # Ozone -# Diag = gsid.Ozone(gsi_obs_file[0]) -# Diag.read() -# Diag.toIODAobs(cycle_dir) -# -# # GeoVaLs call -# if produce_geovals: -# Diag.toGeovals(cycle_dir) -# -# if observation not in ozone_observations: -# Diag.close() -# -# # Rename files to be swell compliant -# # ---------------------------------- -# for observation in observations_orig: -# -# # Input filename -# ioda_file_in_pattern = f'{observation}_obs_*nc*' -# ioda_file_in = glob.glob(os.path.join(cycle_dir, ioda_file_in_pattern))[0] -# -# ioda_file_out = f'{observation}.{window_begin}.nc4' -# -# os.rename(ioda_file_in, os.path.join(cycle_dir, ioda_file_out)) + # Get list of the observations that are ozone observations + # -------------------------------------------------------- + ozone_sensors = gsid.oz_lay_sensors + gsid.oz_lev_sensors + ozone_observations = [] + for observation in observations: + for ozone_sensor in ozone_sensors: + if ozone_sensor in observation: + ozone_observations.append(observation) + + # Copy all the files into the cycle directory + # ------------------------------------------- + for observation in observations: + + self.logger.info(f'Converting {observation} to IODA format') + + gsi_obs_file = glob.glob(os.path.join(gsi_diag_dir, f'*{observation}*')) + + if observation not in ozone_observations: + + # Radiances + Diag = gsid.Radiances(gsi_obs_file[0]) + Diag.read() + Diag.toIODAobs(cycle_dir, False, False, False) + + else: + + # Ozone + Diag = gsid.Ozone(gsi_obs_file[0]) + Diag.read() + Diag.toIODAobs(cycle_dir) + + # GeoVaLs call + if produce_geovals: + Diag.toGeovals(cycle_dir) + + if observation not in ozone_observations: + Diag.close() + + # Rename files to be swell compliant + # ---------------------------------- + for observation in observations_orig: + + # Input filename + ioda_obs_in_pattern = f'{observation}_obs_*nc*' + ioda_obs_in = glob.glob(os.path.join(cycle_dir, ioda_obs_in_pattern))[0] + + ioda_obs_out = f'{observation}.{window_begin}.nc4' + + os.rename(ioda_obs_in, os.path.join(cycle_dir, ioda_obs_out)) + + # Rename GeoVaLs file if need be + if produce_geovals: + ioda_geoval_in_pattern = f'{observation}_geoval_*.nc*' + ioda_geoval_in = glob.glob(os.path.join(cycle_dir, ioda_geoval_in_pattern))[0] + + ioda_geoval_out = f'{observation}_geovals.{window_begin}.nc4' + + os.rename(ioda_geoval_in, os.path.join(cycle_dir, ioda_geoval_out)) + # -------------------------------------------------------------------------------------------------- From 399e5ac6ee74605521113e86bab981097cc70e79 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Mon, 15 May 2023 10:50:59 -0400 Subject: [PATCH 011/121] clean up hidden aspects of the config --- src/swell/tasks/base/config.py | 123 ++---------------------------- src/swell/tasks/base/task_base.py | 62 ++++++++++----- src/swell/tasks/get_background.py | 3 +- src/swell/utilities/date_time.py | 98 ++++++++++++++++++++++++ src/swell/utilities/jinja2.py | 4 +- 5 files changed, 154 insertions(+), 136 deletions(-) create mode 100644 src/swell/utilities/date_time.py diff --git a/src/swell/tasks/base/config.py b/src/swell/tasks/base/config.py index 34261925..443591e3 100644 --- a/src/swell/tasks/base/config.py +++ b/src/swell/tasks/base/config.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 @@ -6,9 +6,6 @@ # -------------------------------------------------------------------------------------------------- -import datetime -import isodate -import os import yaml from swell.utilities.jinja2 import template_string_jinja2 @@ -44,29 +41,25 @@ class Config(): # ---------------------------------------------------------------------------------------------- - def __init__(self, input_file, logger, **kwargs): - - # Keep track of the input config file - self.__input_file__ = input_file + def __init__(self, input_file, logger, model): # Keep copy of owner's logger self.__logger__ = logger # Read the configuration yaml file - with open(self.__input_file__, 'r') as ymlfile: + with open(input_file, 'r') as ymlfile: self.__config__ = yaml.safe_load(ymlfile) # Get model part of the config - self.__model__ = kwargs['model'] - if self.__model__ is not None: + if model is not None: # Assert the model name is found in the config - if self.__model__ not in self.__config__['models'].keys(): - self.__logger__.abort(f'Did not find the model \'{self.__model__}\' in the ' + + if model not in self.__config__['models'].keys(): + self.__logger__.abort(f'Did not find the model \'{model}\' in the ' + f'experiment configuration') # Extract the model specific part of the config - model_config = self.__config__['models'][self.__model__] + model_config = self.__config__['models'][model] # Add model component to config - self.__config__['model_component'] = self.__model__ + self.__config__['model_component'] = model else: model_config = {} @@ -85,25 +78,6 @@ def __init__(self, input_file, logger, **kwargs): # supposed to act upon. self.__config__.update(model_config) - # Add the experiment directory to the configuration - experiment_root = self.get('experiment_root') - experiment_id = self.get('experiment_id') - experiment_dir = os.path.join(experiment_root, experiment_id) - self.__config__['experiment_dir'] = experiment_dir - - # Swell datetime format (avoid colons in paths and filenames) - self.__datetime_swl_format__ = "%Y%m%dT%H%M%SZ" - - # ISO datetime format - self.__datetime_iso_format__ = "%Y-%m-%dT%H:%M:%SZ" - - # If datetime passed add some extra datetime parameters to config - if 'datetime_in' in kwargs and kwargs['datetime_in'] is not None: - self.add_cycle_time_parameter(kwargs['datetime_in'].datetime) - - if self.get('data_assimilation_run', False): - self.add_data_assimilation_window_parameters() - # ---------------------------------------------------------------------------------------------- def get(self, key, default='NODEFAULT'): @@ -119,89 +93,8 @@ def get(self, key, default='NODEFAULT'): # ---------------------------------------------------------------------------------------------- - def put(self, key, value): - self.__config__[key] = value - - # ---------------------------------------------------------------------------------------------- - def use_config_to_template_string(self, string_in): return template_string_jinja2(self.__logger__, string_in, self.__config__) - # ---------------------------------------------------------------------------------------------- - - def get_datetime_format(self): - return self.__datetime_swl_format__ - - # ---------------------------------------------------------------------------------------------- - - def add_cycle_time_parameter(self, cycle_dt): - """ - Defines cycle time parameter and adds to config - """ - - # Add current cycle to the config - # ------------------------------- - current_cycle = cycle_dt.strftime(self.__datetime_swl_format__) - self.put('current_cycle', current_cycle) - - # Add cycle directory to config - # ----------------------------- - cycle_dir = current_cycle - if self.__model__ is not None: - cycle_dir = cycle_dir + '-' + self.__model__ - cycle_dir = os.path.join(self.__config__['experiment_dir'], 'run', cycle_dir) - - self.put('cycle_dir', cycle_dir) - - # ---------------------------------------------------------------------------------------------- - - def add_data_assimilation_window_parameters(self): - """ - Defines cycle dependent parameters for the data assimilation window and adds to config - """ - - # Current cycle datetime object - current_cycle_dto = datetime.datetime.strptime(self.get('current_cycle'), - self.__datetime_swl_format__) - - # Type of data assimilation window (3D or 4D) - window_type = self.get('window_type') - - # Time from beginning of the window to the middle of the window - window_offset = self.get('window_offset') - window_offset_dur = isodate.parse_duration(window_offset) - - # Compute window beginning time - window_begin_dto = current_cycle_dto - window_offset_dur - - # Background time for satbias files - background_time_offset = self.get('background_time_offset') - background_time_offset_dur = isodate.parse_duration(background_time_offset) - - background_time_dto = current_cycle_dto - background_time_offset_dur - - # Background time for the window - if window_type == '4D': - local_background_time = window_begin_dto - elif window_type == '3D': - local_background_time = current_cycle_dto - else: - self.__logger__.abort('add_data_assimilation_window_parameters: window type must be ' + - 'either 4D or 3D') - - window_begin = window_begin_dto.strftime(self.__datetime_swl_format__) - window_begin_iso = window_begin_dto.strftime(self.__datetime_iso_format__) - background_time = background_time_dto.strftime(self.__datetime_swl_format__) - local_background_time_iso = local_background_time.strftime(self.__datetime_iso_format__) - local_background_time = local_background_time.strftime(self.__datetime_swl_format__) - - # Create new dictionary with these items - self.put('window_begin', window_begin) - self.put('window_begin_iso', window_begin_iso) - self.put('background_time', background_time) - self.put('local_background_time', local_background_time) - self.put('local_background_time_iso', local_background_time_iso) - - # ---------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/base/task_base.py b/src/swell/tasks/base/task_base.py index 013a48de..9fb9c882 100644 --- a/src/swell/tasks/base/task_base.py +++ b/src/swell/tasks/base/task_base.py @@ -21,6 +21,7 @@ # local imports from swell.tasks.base.config import Config from swell.tasks.base.datetime import Datetime +from swell.utilities.date_time import datetime_formats from swell.utilities.logger import Logger from swell.tasks.base.task_registry import valid_tasks from swell.tasks.base.utils import camelcase_to_underscore @@ -58,8 +59,13 @@ def __init__(self, config_input, datetime_input, model, task_name): # Create a configuration object # ----------------------------- - self.__config__ = Config(config_input, self.logger, datetime_in=self.__datetime__, - model=self.__model__) + self.__config__ = Config(config_input, self.logger, self.__model__) + + # Ensure that the cycle directory is present + # ------------------------------------------ + if self.__datetime__ is not None: + cycle_dir = self.get_cycle_dir() + os.makedirs(cycle_dir, 0o755, exist_ok=True) # ---------------------------------------------------------------------------------------------- @@ -77,6 +83,16 @@ def config_get(self, key, default='NODEFAULT'): # ---------------------------------------------------------------------------------------------- + # Method to return the experiment directory + def experiment_dir(self): + + experiment_root = self.config_get('experiment_root') + experiment_id = self.config_get('experiment_id') + experiment_dir = os.path.join(experiment_root, experiment_id) + return experiment_dir + + # ---------------------------------------------------------------------------------------------- + # Method to get the Swell experiment path def get_swell_exp_path(self): experiment_root = self.config_get('experiment_root') @@ -92,12 +108,6 @@ def get_swell_exp_config_path(self): # ---------------------------------------------------------------------------------------------- - # Method to get the Swell experiment configuration path - def get_datetime_format(self): - return self.__config__.get_datetime_format() - - # ---------------------------------------------------------------------------------------------- - # Method to open a specific configuration file def __open_jedi_interface_config_file(self, model_or_obs, config_name): @@ -167,10 +177,7 @@ def open_jedi_interface_meta_config_file(self, model=None): # Open file as a string with open(config_file, 'r') as config_file_open: - config_file_str_templated = config_file_open.read() - - # Fill templates in the configuration file using the config - config_file_str = self.__config__.use_config_to_template_string(config_file_str_templated) + config_file_str = config_file_open.read() # Convert string to dictionary return yaml.safe_load(config_file_str) @@ -184,7 +191,7 @@ def open_jedi_interface_model_config_file(self, config_name): # ---------------------------------------------------------------------------------------------- # Method to open a specific observation configuration file - def open_jedi_interface_obs_config_file(self, config_name): + def open_jedi_interface_obs_config_file(self, config_name, window_type): obs_dict = self.__open_jedi_interface_config_file('observations', config_name) # Check that a config file was opened @@ -192,7 +199,7 @@ def open_jedi_interface_obs_config_file(self, config_name): return None # If 4D window then add time interpolation to the dictionary - if self.config_get('window_type') == '4D': + if window_type == '4D': obs_dict['get values'] = {} obs_dict['get values']['time interpolation'] = 'linear' @@ -222,12 +229,31 @@ def is_datetime_dependent(self): # ---------------------------------------------------------------------------------------------- + def get_cycle(self): + + # Check that datetime is set + self.logger.assert_abort(self.__datetime__ is not None, 'In get_cycle_dir but this ' + + 'should not be called if the task does not receive datetime.') + + # Return + return self.__datetime__ + + # ---------------------------------------------------------------------------------------------- + def get_cycle_dir(self): - cycle_dir = self.__config__.get('cycle_dir', None) - if cycle_dir is None: - self.logger.abort('Do not call get_cycle_dir when the task is run without time') - return cycle_dir + # Check that model is set + self.logger.assert_abort(self.__model__ is not None, 'In get_cycle_dir but this ' + + 'should not be called if the task does not receive model.') + + # Get the current cycle in directory format + current_cycle_dir_format = self.get_cycle().strftime(datetime_formats['dir_format']) + + # Combine with the model + cycle_dir = os.path.join(current_cycle_dir_format, self.__model__) + + # Return + return cycle_dir # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/get_background.py b/src/swell/tasks/get_background.py index 59ac4efb..9d327720 100644 --- a/src/swell/tasks/get_background.py +++ b/src/swell/tasks/get_background.py @@ -9,6 +9,7 @@ from swell.tasks.base.task_base import taskBase +from swell.utilities.date_time import datetime_formats from datetime import datetime as dt import isodate @@ -42,7 +43,7 @@ def execute(self): # Current cycle time object # ------------------------- current_cycle = self.config_get('current_cycle') - current_cycle_dto = dt.strptime(current_cycle, self.get_datetime_format()) + current_cycle_dto = dt.strptime(current_cycle, datetime_formats['dir_format']) # Get duration into forecast for first background file # ---------------------------------------------------- diff --git a/src/swell/utilities/date_time.py b/src/swell/utilities/date_time.py new file mode 100644 index 00000000..43d52218 --- /dev/null +++ b/src/swell/utilities/date_time.py @@ -0,0 +1,98 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + + +import isodate + + +# -------------------------------------------------------------------------------------------------- + + +# Swell datetime format (yyyymmddThhMMssZ) + +datetime_formats = { + 'dir_format': "%Y%m%dT%H%M%SZ" # yyyymmddThhMMssZ for directory formats + 'iso_format': "%Y-%m-%dT%H:%M:%SZ" # yyyy-mm-ddThh:MM:ssZ ISO format +} + +# -------------------------------------------------------------------------------------------------- + + +class DataAssimilationWindowParams(): + + def __init__(self, cycle_time, window_offset): + + """ + Defines cycle dependent parameters for the data assimilation window and adds to config + """ + + # Current cycle datetime object + self.__current_cycle_dto__ = datetime.datetime.strptime(cycle_time, + datetime_formats['dir_format']) + + # Save the window offset + self.__window_offset__ = window_offset + + # ---------------------------------------------------------------------------------------------- + + def __get_window_begin_dto__(self): + + window_offset_dur = isodate.parse_duration(self.__window_offset__) + return self.__current_cycle_dto__ - window_offset_dur + + # ---------------------------------------------------------------------------------------------- + + def __get_local_background_time__(self, window_type): + + # Background time for the window + if window_type == '4D': + local_background_time = self.__get_window_begin_dto__(self.__window_offset__) + elif window_type == '3D': + local_background_time = self.__current_cycle_dto__ + + return local_background_time + + # ---------------------------------------------------------------------------------------------- + + def window_begin(self): + + window_begin_dto = self.__get_window_begin_dto__() + return window_begin_dto.strftime(datetime_formats['dir_format']) + + # ---------------------------------------------------------------------------------------------- + + def window_begin_iso(self): + + window_begin_dto = self.__get_window_begin_dto__() + return window_begin_dto.strftime(datetime_formats['iso_format']) + + # ---------------------------------------------------------------------------------------------- + + def background_time(self, background_time_offset): + + background_time_offset_dur = isodate.parse_duration(background_time_offset) + background_time_dto = self.__current_cycle_dto__ - background_time_offset_dur + return background_time_dto.strftime(datetime_formats['dir_format']) + + # ---------------------------------------------------------------------------------------------- + + def local_background_time_iso(self, window_type): + + local_background_time = self.__get_local_background_time__(window_type) + return local_background_time.strftime(datetime_formats['iso_format']) + + # ---------------------------------------------------------------------------------------------- + + def local_background_time(self, window_type): + + local_background_time = self.__get_local_background_time__(window_type) + return local_background_time.strftime(datetime_formats['dir_format']) + + # ---------------------------------------------------------------------------------------------- + + diff --git a/src/swell/utilities/jinja2.py b/src/swell/utilities/jinja2.py index e4e29a4c..6e5e3bb9 100644 --- a/src/swell/utilities/jinja2.py +++ b/src/swell/utilities/jinja2.py @@ -23,11 +23,11 @@ def template_string_jinja2(logger, templated_string, dictionary_of_templates): # Render the templates using the dictionary string_rendered = t.render(dictionary_of_templates) - logger.assert_abort('{{' not in string_rendered, f'In use_config_to_template_string ' + + logger.assert_abort('{{' not in string_rendered, f'In template_string_jinja2 ' + f'the output string still contains template directives. ' + f'{string_rendered}') - logger.assert_abort('}}' not in string_rendered, f'In use_config_to_template_string ' + + logger.assert_abort('}}' not in string_rendered, f'In template_string_jinja2 ' + f'the output string still contains template directives. ' + f'{string_rendered}') From 78c171df6e2566a77dd28064ba5f76ec393691a0 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Mon, 15 May 2023 13:54:41 -0400 Subject: [PATCH 012/121] move the opening of jedi interface files --- setup.py | 2 + src/swell/tasks/base/config.py | 7 +- src/swell/tasks/base/datetime.py | 2 +- .../tasks/base/run_jedi_executable_base.py | 2 +- src/swell/tasks/base/task_base.py | 133 +-------------- src/swell/tasks/base/task_registry.py | 2 +- src/swell/tasks/generate_b_climatology.py | 3 +- .../bin/check_jedi_interface_templates.py | 87 ++++++++++ src/swell/utilities/date_time.py | 2 - .../utilities/render_jedi_interface_files.py | 160 ++++++++++++++++++ 10 files changed, 256 insertions(+), 144 deletions(-) create mode 100644 src/swell/utilities/bin/check_jedi_interface_templates.py create mode 100644 src/swell/utilities/render_jedi_interface_files.py diff --git a/setup.py b/setup.py index 1e4ebf0b..bc4d8452 100644 --- a/setup.py +++ b/setup.py @@ -63,6 +63,8 @@ 'swell_prepare_experiment_config = swell.deployment.bin.swell_prepare_config:main', 'swell_launch_experiment = swell.deployment.bin.swell_launch_experiment:main', 'swell_sat_db_processing = swell.deployment.bin.swell_sat_db_processing:main', + 'util_check_jedi_interface_templates = \ + swell.utilities.bin.check_jedi_interface_templates:main', ], }, ) diff --git a/src/swell/tasks/base/config.py b/src/swell/tasks/base/config.py index 443591e3..eb73b420 100644 --- a/src/swell/tasks/base/config.py +++ b/src/swell/tasks/base/config.py @@ -6,9 +6,9 @@ # -------------------------------------------------------------------------------------------------- + import yaml -from swell.utilities.jinja2 import template_string_jinja2 # -------------------------------------------------------------------------------------------------- # @package configuration @@ -91,10 +91,5 @@ def get(self, key, default='NODEFAULT'): else: return default - # ---------------------------------------------------------------------------------------------- - - def use_config_to_template_string(self, string_in): - - return template_string_jinja2(self.__logger__, string_in, self.__config__) # ---------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/base/datetime.py b/src/swell/tasks/base/datetime.py index 48b53482..594cadca 100644 --- a/src/swell/tasks/base/datetime.py +++ b/src/swell/tasks/base/datetime.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/tasks/base/run_jedi_executable_base.py b/src/swell/tasks/base/run_jedi_executable_base.py index 243be687..72ebea91 100644 --- a/src/swell/tasks/base/run_jedi_executable_base.py +++ b/src/swell/tasks/base/run_jedi_executable_base.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/tasks/base/task_base.py b/src/swell/tasks/base/task_base.py index 9fb9c882..ce4875f2 100644 --- a/src/swell/tasks/base/task_base.py +++ b/src/swell/tasks/base/task_base.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 @@ -83,16 +83,6 @@ def config_get(self, key, default='NODEFAULT'): # ---------------------------------------------------------------------------------------------- - # Method to return the experiment directory - def experiment_dir(self): - - experiment_root = self.config_get('experiment_root') - experiment_id = self.config_get('experiment_id') - experiment_dir = os.path.join(experiment_root, experiment_id) - return experiment_dir - - # ---------------------------------------------------------------------------------------------- - # Method to get the Swell experiment path def get_swell_exp_path(self): experiment_root = self.config_get('experiment_root') @@ -108,114 +98,6 @@ def get_swell_exp_config_path(self): # ---------------------------------------------------------------------------------------------- - # Method to open a specific configuration file - def __open_jedi_interface_config_file(self, model_or_obs, config_name): - - # Assert that the task has a model associated with it - self.logger.assert_abort(self.__model__ is not None, - 'Task must have a model associated with it.') - - # Get experiment configuration path - swell_exp_config_path = self.get_swell_exp_config_path() - - # Path to configuration file - config_file = os.path.join(swell_exp_config_path, 'jedi', 'interfaces', - self.__model__, model_or_obs, config_name + '.yaml') - - # Check that config file exists - if not os.path.exists(config_file): - return None - - # Open file as a string - with open(config_file, 'r') as config_file_open: - config_file_str_templated = config_file_open.read() - - # Fill templates in the configuration file using the config - config_file_str = self.__config__.use_config_to_template_string(config_file_str_templated) - - # Convert string to dictionary - return yaml.safe_load(config_file_str) - - # ---------------------------------------------------------------------------------------------- - - # Method to open a specific model configuration file - def open_jedi_oops_config_file(self, config_name): - - # Get experiment configuration path - swell_exp_config_path = self.get_swell_exp_config_path() - - # Path to configuration file - config_file = os.path.join(swell_exp_config_path, 'jedi', 'oops', config_name + '.yaml') - - # Open file as a string - with open(config_file, 'r') as config_file_open: - config_file_str_templated = config_file_open.read() - - # Fill templates in the configuration file using the config - config_file_str = self.__config__.use_config_to_template_string(config_file_str_templated) - - # Convert string to dictionary - return yaml.safe_load(config_file_str) - - # ---------------------------------------------------------------------------------------------- - - # Method to open a specific model configuration file metadata - def open_jedi_interface_meta_config_file(self, model=None): - - # Set model to the actual model if needed - if model is None: - self.logger.assert_abort(self.__model__ is not None, - 'Task must have a model associated with it.') - model = self.__model__ - - # Get experiment configuration path - swell_exp_config_path = self.get_swell_exp_config_path() - - # Path to configuration file - config_file = os.path.join(swell_exp_config_path, 'jedi', 'interfaces', - model, model + '.yaml') - - # Open file as a string - with open(config_file, 'r') as config_file_open: - config_file_str = config_file_open.read() - - # Convert string to dictionary - return yaml.safe_load(config_file_str) - - # ---------------------------------------------------------------------------------------------- - - # Method to open a specific model configuration file - def open_jedi_interface_model_config_file(self, config_name): - return self.__open_jedi_interface_config_file('model', config_name) - - # ---------------------------------------------------------------------------------------------- - - # Method to open a specific observation configuration file - def open_jedi_interface_obs_config_file(self, config_name, window_type): - obs_dict = self.__open_jedi_interface_config_file('observations', config_name) - - # Check that a config file was opened - if obs_dict is None: - return None - - # If 4D window then add time interpolation to the dictionary - if window_type == '4D': - obs_dict['get values'] = {} - obs_dict['get values']['time interpolation'] = 'linear' - - # Placeholder to add GeoVaLs saver filter - - # Placeholder for IO pool things - - return obs_dict - - # ---------------------------------------------------------------------------------------------- - - def use_config_to_template_string(self, string_in): - return self.__config__.use_config_to_template_string(string_in) - - # ---------------------------------------------------------------------------------------------- - def get_model(self): return self.__model__ @@ -229,17 +111,6 @@ def is_datetime_dependent(self): # ---------------------------------------------------------------------------------------------- - def get_cycle(self): - - # Check that datetime is set - self.logger.assert_abort(self.__datetime__ is not None, 'In get_cycle_dir but this ' + - 'should not be called if the task does not receive datetime.') - - # Return - return self.__datetime__ - - # ---------------------------------------------------------------------------------------------- - def get_cycle_dir(self): # Check that model is set @@ -247,7 +118,7 @@ def get_cycle_dir(self): 'should not be called if the task does not receive model.') # Get the current cycle in directory format - current_cycle_dir_format = self.get_cycle().strftime(datetime_formats['dir_format']) + current_cycle_dir_format = self.__datetime__.strftime(datetime_formats['dir_format']) # Combine with the model cycle_dir = os.path.join(current_cycle_dir_format, self.__model__) diff --git a/src/swell/tasks/base/task_registry.py b/src/swell/tasks/base/task_registry.py index 267c24fa..353291a4 100644 --- a/src/swell/tasks/base/task_registry.py +++ b/src/swell/tasks/base/task_registry.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2023 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/tasks/generate_b_climatology.py b/src/swell/tasks/generate_b_climatology.py index c4fcbcab..7e881f77 100644 --- a/src/swell/tasks/generate_b_climatology.py +++ b/src/swell/tasks/generate_b_climatology.py @@ -101,7 +101,7 @@ def generate_bump(self): # Jedi executable name # -------------------- jedi_executable = interface_executable[self.jedi_interface] - jedi_executable_path = os.path.join(self.experiment_dir, 'jedi_bundle', + jedi_executable_path = os.path.join(self.get_swell_exp_path(), 'jedi_bundle', 'build', 'bin', jedi_executable) # Run the JEDI executable @@ -144,7 +144,6 @@ def execute(self): self.horizontal_resolution = self.config_get('horizontal_resolution') self.vertical_resolution = self.config_get('vertical_resolution') self.cycle_dir = self.config_get('cycle_dir') - self.experiment_dir = self.config_get('experiment_dir') self.npx_proc = self.config_get('npx_proc') # Used in eval(total_processors) self.npy_proc = self.config_get('npy_proc') # Used in eval(total_processors) diff --git a/src/swell/utilities/bin/check_jedi_interface_templates.py b/src/swell/utilities/bin/check_jedi_interface_templates.py new file mode 100644 index 00000000..d6f9e393 --- /dev/null +++ b/src/swell/utilities/bin/check_jedi_interface_templates.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python + +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + + +# standard imports +import os +import re +import glob + +# swell imports +from swell.swell_path import get_swell_path +from swell.utilities.logger import Logger + + +# -------------------------------------------------------------------------------------------------- + + +def main(): + + # Create a logger + logger = Logger('CheckJediInterfaceTemplates') + + config_types = ['oops', 'interfaces/*/model', 'interfaces/*/observations'] + + # Path to JEDI interface code + swell_path = get_swell_path() + jedi_interfaces_path = os.path.join(swell_path, 'configuration', 'jedi') + + # Loop over config types + for config_type in config_types: + + # Path to the files for this config type + config_yaml_path = os.path.join(jedi_interfaces_path, config_type, '*yaml') + + config_yaml_files = glob.glob(config_yaml_path) + + # String to hold contents of all the files + all_config_files_str = '' + + for config_yaml_file in config_yaml_files: + + with open(config_yaml_file, 'r') as file: + file_lines = file.read().split('\n') + + # Rejoin with space between lines to eliminate new line deliminator + all_config_files_str = all_config_files_str + ' '.join(file_lines) + + # Split again using space or slash + all_config_files_lines = re.split(' |/', all_config_files_str) + + # Keep only elements with templates + all_config_files_lines = [x for x in all_config_files_lines if "{{" in x] + + # Loop and find all instances of templates and add to list + templates = [] + for all_config_files_line in all_config_files_lines: + + res = re.findall(r'\{\{.*?\}\}', all_config_files_line) + templates = templates + res + + # Remove duplicates & sort + templates = sorted(list(set(templates))) + + # Print the templates + logger.info(config_type) + logger.info('-'*len(config_type)) + for template in templates: + logger.info(template.strip('{').strip('}')) + if config_type != config_types[-1]: + logger.info(' ', False) + + +# -------------------------------------------------------------------------------------------------- + + +if __name__ == '__main__': + main() + + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/date_time.py b/src/swell/utilities/date_time.py index 43d52218..c5af0183 100644 --- a/src/swell/utilities/date_time.py +++ b/src/swell/utilities/date_time.py @@ -94,5 +94,3 @@ def local_background_time(self, window_type): return local_background_time.strftime(datetime_formats['dir_format']) # ---------------------------------------------------------------------------------------------- - - diff --git a/src/swell/utilities/render_jedi_interface_files.py b/src/swell/utilities/render_jedi_interface_files.py new file mode 100644 index 00000000..b10f0c62 --- /dev/null +++ b/src/swell/utilities/render_jedi_interface_files.py @@ -0,0 +1,160 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + + +import os +import shutil + +from jedi_bundle.bin.jedi_bundle import get_default_config, get_bundles +from swell.utilities.jinja2 import template_string_jinja2 + + +# -------------------------------------------------------------------------------------------------- + + +def template_dictionary_oops_config(analysis_variables, gradient_norm_reduction, minimizer, + number_of_iterations, window_begin_iso, window_length): + +dictionary_oops_config = {} +dictionary_oops_config['analysis_variables'] = analysis_variables +dictionary_oops_config['gradient_norm_reduction'] = gradient_norm_reduction +dictionary_oops_config['minimizer'] = minimizer +dictionary_oops_config['number_of_iterations'] = number_of_iterations +dictionary_oops_config['window_begin_iso'] = window_begin_iso +dictionary_oops_config['window_length'] = window_length +return dictionary_oops_config + + +# -------------------------------------------------------------------------------------------------- + + +def template_dictionary_model_config(analysis_variables, background_frequency, cycle_dir, + experiment_dir, experiment_id, horizontal_resolution, + local_background_time_iso, local_background_time, + npx_proc, npy_proc, swell_static_files, vertical_resolution): + +dictionary_model_config = {} +dictionary_model_config['analysis_variables'] = analysis_variables +dictionary_model_config['background_frequency'] = background_frequency +dictionary_model_config['cycle_dir'] = cycle_dir +dictionary_model_config['experiment_dir'] = experiment_dir +dictionary_model_config['experiment_id'] = experiment_id +dictionary_model_config['horizontal_resolution'] = horizontal_resolution +dictionary_model_config['local_background_time_iso'] = local_background_time_iso +dictionary_model_config['local_background_time'] = local_background_time +dictionary_model_config['npx_proc'] = npx_proc +dictionary_model_config['npy_proc'] = npy_proc +dictionary_model_config['swell_static_files'] = swell_static_files +dictionary_model_config['vertical_resolution'] = vertical_resolution +return dictionary_model_config + + +# -------------------------------------------------------------------------------------------------- + + +def template_dictionary_observation_config(background_time, crtm_coeff_dir, cycle_dir, + experiment_id, experiment_root, model_component, + window_begin): + +dictionary_observation_config = {} +dictionary_observation_config['background_time'] = background_time +dictionary_observation_config['crtm_coeff_dir'] = crtm_coeff_dir +dictionary_observation_config['cycle_dir'] = cycle_dir +dictionary_observation_config['experiment_id'] = experiment_id +dictionary_observation_config['experiment_root'] = experiment_root +dictionary_observation_config['model_component'] = model_component +dictionary_observation_config['window_begin'] = window_begin +return dictionary_observation_config + + +# -------------------------------------------------------------------------------------------------- + + +def __open_file_render_to_dict(logger, config_file, template_dictionary): + + # Check that config file exists + logger.assert_abort(os.path.exists(config_file), f'In open_file_and_render failed ' + f'to find file \'{config_file}\'') + + # Open file as a string + with open(config_file, 'r') as config_file_open: + config_file_str_templated = config_file_open.read() + + # Fill templates in the configuration file using the config + config_file_str = template_string_jinja2(logger, config_file_str_templated, template_dictionary) + + # Convert string to dictionary + return yaml.safe_load(config_file_str) + + +# -------------------------------------------------------------------------------------------------- + + +def read_render_jedi_interface_oops(logger, exp_config_path, config_name, + template_dictionary): + + # Path to configuration file + config_file = os.path.join(exp_config_path, 'jedi', 'oops', f'{config_name}.yaml') + + # Render templates in file and return dictionary + return __open_file_render_to_dict(config_file, template_dictionary) + + +# -------------------------------------------------------------------------------------------------- + + +# Method to open a specific configuration file +def read_render_jedi_interface_model(logger, interface, exp_config_path, config_name, + template_dictionary): + + # Path to configuration file + config_file = os.path.join(exp_config_path, 'jedi', 'interfaces', interface, 'model', + f'{config_name}.yaml') + + # Render templates in file and return dictionary + return __open_file_render_to_dict(config_file, template_dictionary) + + +# -------------------------------------------------------------------------------------------------- + + +def read_render_jedi_interface_obs(logger, interface, exp_config_path, config_name, + template_dictionary): + + # Path to configuration file + config_file = os.path.join(exp_config_path, 'jedi', 'interfaces', interface, 'observations', + f'{config_name}.yaml') + + # Render templates in file and return dictionary + obs_dict = __open_file_render_to_dict(config_file, template_dictionary) + + # If 4D window then add time interpolation to the dictionary + if window_type == '4D': + obs_dict['get values'] = {} + obs_dict['get values']['time interpolation'] = 'linear' + + return obs_dict + + +# -------------------------------------------------------------------------------------------------- + + +def read_render_jedi_interface_meta(logger, interface, exp_config_path): + + # Path to configuration file + config_file = os.path.join(exp_config_path, 'jedi', 'interfaces', interface, + f'{interface}.yaml') + + # Currently no templates in these kinds of files + template_dictionary = {} + + # Render templates in file and return dictionary + return __open_file_render_to_dict(config_file, template_dictionary) + + +# -------------------------------------------------------------------------------------------------- From b118ee66ea6d59445a8ab8602eaf976b6dfc397e Mon Sep 17 00:00:00 2001 From: danholdaway Date: Fri, 19 May 2023 14:53:10 -0400 Subject: [PATCH 013/121] add class to handle jedi file templating --- pycodestyle.cfg | 3 +- pycodestyle_run.py | 2 +- src/swell/__init__.py | 2 +- .../geos_atmosphere/model/background.yaml | 2 +- .../geos_atmosphere/model/geometry.yaml | 8 +- .../geos_atmosphere/model/pseudo-model.yaml | 2 +- .../geos_atmosphere/model/stage.yaml | 6 +- .../tasks/base/run_jedi_executable_base.py | 61 ++++- src/swell/tasks/base/task_base.py | 63 +++++- src/swell/tasks/build_geos.py | 2 +- src/swell/tasks/build_geos_by_linking.py | 2 +- src/swell/tasks/build_jedi.py | 5 +- src/swell/tasks/build_jedi_by_linking.py | 2 +- src/swell/tasks/clean_cycle.py | 9 +- src/swell/tasks/clone_geos.py | 2 +- src/swell/tasks/clone_jedi.py | 4 +- src/swell/tasks/eva_jedi_log.py | 13 +- src/swell/tasks/eva_observations.py | 24 +- src/swell/tasks/generate_b_climatology.py | 19 +- src/swell/tasks/get_background.py | 23 +- .../tasks/get_background_geos_experiment.py | 31 ++- src/swell/tasks/get_observations.py | 12 +- src/swell/tasks/obs_process_setup.py | 203 ----------------- src/swell/tasks/run_jedi_hofx_executable.py | 17 +- .../tasks/run_jedi_variational_executable.py | 18 +- src/swell/tasks/save_obs_diags.py | 17 +- src/swell/tasks/stage_jedi.py | 21 +- src/swell/tasks/store_background.py | 36 ++- src/swell/utilities/bin/__init__.py | 9 + .../bin/check_jedi_interface_templates.py | 25 ++- src/swell/utilities/date_time.py | 35 +-- .../utilities/render_jedi_interface_files.py | 209 +++++++++--------- 32 files changed, 413 insertions(+), 474 deletions(-) delete mode 100644 src/swell/tasks/obs_process_setup.py create mode 100644 src/swell/utilities/bin/__init__.py diff --git a/pycodestyle.cfg b/pycodestyle.cfg index 25b0307f..1a12154b 100644 --- a/pycodestyle.cfg +++ b/pycodestyle.cfg @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 @@ -8,3 +8,4 @@ max-line-length = 100 indent-size = 4 statistics = True +#ignore = build diff --git a/pycodestyle_run.py b/pycodestyle_run.py index f69a20a2..dd35e149 100644 --- a/pycodestyle_run.py +++ b/pycodestyle_run.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/__init__.py b/src/swell/__init__.py index 9f04c64e..439f968b 100644 --- a/src/swell/__init__.py +++ b/src/swell/__init__.py @@ -9,4 +9,4 @@ repo_directory = os.path.dirname(__file__) # Set the version for swell -__version__ = '1.2.4' +__version__ = '1.3.0' diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/background.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/background.yaml index 20d65d2b..4d10032f 100755 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/background.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/background.yaml @@ -3,7 +3,7 @@ filetype: cube sphere history provider: geos datapath: '' filenames: ['{{cycle_dir}}/bkg.%yyyy%mm%ddT%hh%MM%ssZ.nc4', - '{{experiment_dir}}/stage/fv3-jedi/geos_atmosphere/bkg/geos.crtmsrf.{{horizontal_resolution}}.nc4'] + '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/bkg/geos.crtmsrf.{{horizontal_resolution}}.nc4'] state variables: [u,v,ua,va,t,delp,q,qi,ql,qr,qs,o3ppmv,phis, qls,qcn,cfcn,frocean,frland,varflt,ustar,bstar, zpbl,cm,ct,cq,kcbl,tsm,khl,khu,frlake,frseaice,vtype, diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/geometry.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/geometry.yaml index 9e3e1e1d..3116dada 100755 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/geometry.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/geometry.yaml @@ -1,9 +1,9 @@ fms initialization: - namelist filename: '{{experiment_dir}}/stage/fv3-jedi/geos_atmosphere/fv3files/fmsmpp.nml' - field table filename: '{{experiment_dir}}/stage/fv3-jedi/geos_atmosphere/fv3files/field_table_gmao' -akbk: '{{experiment_dir}}/stage/fv3-jedi/geos_atmosphere/fv3files/akbk{{vertical_resolution}}.nc4' + namelist filename: '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/fv3files/fmsmpp.nml' + field table filename: '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/fv3files/field_table_gmao' +akbk: '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/fv3files/akbk{{vertical_resolution}}.nc4' layout: [{{npx_proc}},{{npy_proc}}] npx: {{horizontal_resolution}} npy: {{horizontal_resolution}} npz: {{vertical_resolution}} -field metadata override: '{{experiment_dir}}/stage/fv3-jedi/geos_atmosphere/fieldmetadata/geos.yaml' +field metadata override: '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/fieldmetadata/geos.yaml' diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/pseudo-model.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/pseudo-model.yaml index 5d53e997..075af1c1 100755 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/pseudo-model.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/pseudo-model.yaml @@ -4,7 +4,7 @@ filetype: cube sphere history provider: geos datapath: '' filenames: ['{{cycle_dir}}/bkg.%yyyy%mm%ddT%hh%MM%ssZ.nc4', - '{{experiment_dir}}/stage/fv3-jedi/geos_atmosphere/bkg/geos.crtmsrf.{{horizontal_resolution}}.nc4'] + '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/bkg/geos.crtmsrf.{{horizontal_resolution}}.nc4'] model variables: [u,v,ua,va,t,delp,q,qi,ql,qr,qs,o3ppmv,phis,qls,qcn,cfcn,frocean,frland,varflt,ustar, bstar,zpbl,cm,ct,cq,kcbl,tsm,khl,khu,frlake,frseaice,vtype,stype,vfrac,sheleg, ts,soilt,soilm,u10m,v10m] diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/stage.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/stage.yaml index 15b3e606..efc0957e 100755 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/stage.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/stage.yaml @@ -1,5 +1,5 @@ - link_files: directories: - - ['{{swell_static_files}}/jedi/interfaces/geos_atmosphere/GEOS_CRTM_Surface/geos.crtmsrf.{{horizontal_resolution}}.nc4', '{{experiment_dir}}/stage/fv3-jedi/geos_atmosphere/bkg/'] - - ['{{experiment_dir}}/jedi_bundle/source/fv3-jedi/test/Data/fieldmetadata/*', '{{experiment_dir}}/stage/fv3-jedi/geos_atmosphere/fieldmetadata/'] - - ['{{experiment_dir}}/jedi_bundle/source/fv3-jedi/test/Data/fv3files/*', '{{experiment_dir}}/stage/fv3-jedi/geos_atmosphere/fv3files/'] + - ['{{swell_static_files}}/jedi/interfaces/geos_atmosphere/GEOS_CRTM_Surface/geos.crtmsrf.{{horizontal_resolution}}.nc4', '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/bkg/'] + - ['{{experiment_root}}/{{experiment_id}}/jedi_bundle/source/fv3-jedi/test/Data/fieldmetadata/*', '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/fieldmetadata/'] + - ['{{experiment_root}}/{{experiment_id}}/jedi_bundle/source/fv3-jedi/test/Data/fv3files/*', '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/fv3files/'] diff --git a/src/swell/tasks/base/run_jedi_executable_base.py b/src/swell/tasks/base/run_jedi_executable_base.py index 72ebea91..f69f978b 100644 --- a/src/swell/tasks/base/run_jedi_executable_base.py +++ b/src/swell/tasks/base/run_jedi_executable_base.py @@ -32,10 +32,8 @@ def execute(self): def jedi_dictionary_iterator(self, jedi_config_dict, window_type): - # Loop over dictionary and replace if value is a dictionary, meanwhile - # inquire list objects for dictionary items. - # ----------------------------------------------------------- - + # Assemble configuration YAML file + # -------------------------------- for key, value in jedi_config_dict.items(): if isinstance(value, dict): self.jedi_dictionary_iterator(value, window_type) @@ -48,7 +46,8 @@ def jedi_dictionary_iterator(self, jedi_config_dict, window_type): else: if 'TASKFILL' in value: value_file = value.replace('TASKFILL', '') - value_dict = self.open_jedi_interface_model_config_file(value_file) + value_dict = self.jedi_rendering.render_interface_model(value_file) + jedi_config_dict[key] = value_dict elif 'SPECIAL' in value: @@ -58,12 +57,13 @@ def jedi_dictionary_iterator(self, jedi_config_dict, window_type): obs = self.config_get('observations') for ob in obs: # Get observation dictionary - observations.append(self.open_jedi_interface_obs_config_file(ob)) + obs_dict = self.jedi_rendering.render_interface_observations(ob) + observations.append(obs_dict) jedi_config_dict[key] = observations elif value_special == 'model' and window_type == '4D': model = self.config_get('model') - model_dict = self.open_jedi_interface_model_config_file(model) + model_dict = self.jedi_rendering.render_interface_model(model) jedi_config_dict[key] = model_dict # ---------------------------------------------------------------------------------------------- @@ -75,9 +75,54 @@ def generate_jedi_config(self, jedi_application, window_type): if 'var' not in jedi_application: jedi_application = jedi_application + window_type + # Build dictionary for rendering JEDI configuration files + analysis_variables = self.config_get('analysis_variables', None) + gradient_norm_reduction = self.config_get('gradient_norm_reduction', None) + minimizer = self.config_get('minimizer', None) + number_of_iterations = self.config_get('number_of_iterations', None) + window_length = self.config_get('window_length', None) + background_frequency = self.config_get('background_frequency', None) + horizontal_resolution = self.config_get('horizontal_resolution', None) + npx_proc = self.config_get('npx_proc', None) + npy_proc = self.config_get('npy_proc', None) + swell_static_files = self.config_get('swell_static_files', None) + vertical_resolution = self.config_get('vertical_resolution', None) + crtm_coeff_dir = self.config_get('crtm_coeff_dir', None) + window_offset = self.config_get('window_offset', None) + background_time_offset = self.config_get('background_time_offset', None) + + # Compute data assimilation window parameters + background_time = self.da_window_params.background_time(window_offset, + background_time_offset) + local_background_time = self.da_window_params.local_background_time(window_offset, + window_type) + local_background_time_iso = self.da_window_params.local_background_time_iso(window_offset, + window_type) + window_begin = self.da_window_params.window_begin(window_offset) + window_begin_iso = self.da_window_params.window_begin(window_offset) + + # Add config to template rendering dictionary + self.jedi_rendering.add_key('analysis_variables', analysis_variables) + self.jedi_rendering.add_key('gradient_norm_reduction', gradient_norm_reduction) + self.jedi_rendering.add_key('minimizer', minimizer) + self.jedi_rendering.add_key('number_of_iterations', number_of_iterations) + self.jedi_rendering.add_key('window_begin_iso', window_begin_iso) + self.jedi_rendering.add_key('window_length', window_length) + self.jedi_rendering.add_key('background_frequency', background_frequency) + self.jedi_rendering.add_key('horizontal_resolution', horizontal_resolution) + self.jedi_rendering.add_key('local_background_time', local_background_time) + self.jedi_rendering.add_key('local_background_time_iso', local_background_time_iso) + self.jedi_rendering.add_key('npx_proc', npx_proc) + self.jedi_rendering.add_key('npy_proc', npy_proc) + self.jedi_rendering.add_key('swell_static_files', swell_static_files) + self.jedi_rendering.add_key('vertical_resolution', vertical_resolution) + self.jedi_rendering.add_key('background_time', background_time) + self.jedi_rendering.add_key('crtm_coeff_dir', crtm_coeff_dir) + self.jedi_rendering.add_key('window_begin', window_begin) + # Create dictionary from the templated JEDI config file # ----------------------------------------------------- - jedi_config_dict = self.open_jedi_oops_config_file(jedi_application) + jedi_config_dict = self.jedi_rendering.render_oops_file(jedi_application) # Read configs for the rest of the dictionary # ------------------------------------------- diff --git a/src/swell/tasks/base/task_base.py b/src/swell/tasks/base/task_base.py index ce4875f2..38f8c4d3 100644 --- a/src/swell/tasks/base/task_base.py +++ b/src/swell/tasks/base/task_base.py @@ -25,6 +25,8 @@ from swell.utilities.logger import Logger from swell.tasks.base.task_registry import valid_tasks from swell.tasks.base.utils import camelcase_to_underscore +from swell.utilities.date_time import DataAssimilationWindowParams +from swell.utilities.render_jedi_interface_files import JediConfigRendering # -------------------------------------------------------------------------------------------------- @@ -61,12 +63,28 @@ def __init__(self, config_input, datetime_input, model, task_name): # ----------------------------- self.__config__ = Config(config_input, self.logger, self.__model__) - # Ensure that the cycle directory is present - # ------------------------------------------ + # All experiment have the experiment root and id and suite + # -------------------------------------------------------- + self.__experiment_root__ = self.config_get('experiment_root') + self.__experiment_id__ = self.config_get('experiment_id') + self.__suite__ = self.config_get('suite_to_run') + + # Create some extra helpers available when the datetime is present + # ---------------------------------------------------------------- if self.__datetime__ is not None: + + # Create the cycle directory cycle_dir = self.get_cycle_dir() os.makedirs(cycle_dir, 0o755, exist_ok=True) + # Jedi config file rendering + self.jedi_rendering = JediConfigRendering(self.logger, self.__experiment_root__, + self.__experiment_id__, cycle_dir, + self.__model__) + + # Object for computing data assimilation window parameters + self.da_window_params = DataAssimilationWindowParams(self.logger, self.__datetime__) + # ---------------------------------------------------------------------------------------------- # Execute is the place where a task does its work. It's defined as abstract in the base class @@ -83,17 +101,33 @@ def config_get(self, key, default='NODEFAULT'): # ---------------------------------------------------------------------------------------------- - # Method to get the Swell experiment path - def get_swell_exp_path(self): - experiment_root = self.config_get('experiment_root') - experiment_id = self.config_get('experiment_id') - return os.path.join(experiment_root, experiment_id) + # Method to get the experiment root + def experiment_root(self): + return self.__experiment_root__ + + # ---------------------------------------------------------------------------------------------- + + # Method to get the experiment ID + def experiment_id(self): + return self.__experiment_id__ + + # ---------------------------------------------------------------------------------------------- + + # Method to get the suite type + def suite(self): + return self.__suite__ + + # ---------------------------------------------------------------------------------------------- + + # Method to get the experiment directory + def experiment_path(self): + return os.path.join(self.__experiment_root__, self.__experiment_id__) # ---------------------------------------------------------------------------------------------- - # Method to get the Swell experiment configuration path - def get_swell_exp_config_path(self): - swell_exp_path = self.get_swell_exp_path() + # Method to get the experiment configuration directory + def experiment_config_path(self): + swell_exp_path = self.experiment_path() return os.path.join(swell_exp_path, 'configuration') # ---------------------------------------------------------------------------------------------- @@ -111,7 +145,14 @@ def is_datetime_dependent(self): # ---------------------------------------------------------------------------------------------- - def get_cycle_dir(self): + def cycle_time(self): + self.logger.assert_abort(self.__datetime__ is not None, f'Cannot obtain the cycle time' + + f'when the task was not initialized with the cycle time.') + return self.__datetime__ + + # ---------------------------------------------------------------------------------------------- + + def cycle_dir(self): # Check that model is set self.logger.assert_abort(self.__model__ is not None, 'In get_cycle_dir but this ' + diff --git a/src/swell/tasks/build_geos.py b/src/swell/tasks/build_geos.py index 436aa2b4..186ece23 100644 --- a/src/swell/tasks/build_geos.py +++ b/src/swell/tasks/build_geos.py @@ -28,7 +28,7 @@ def execute(self): # Get the experiment/geos directory # --------------------------------- - swell_exp_path = self.get_swell_exp_path() + swell_exp_path = self.experiment_path() geos_gcm_path = os.path.join(swell_exp_path, 'GEOSgcm') # Get paths to build and source diff --git a/src/swell/tasks/build_geos_by_linking.py b/src/swell/tasks/build_geos_by_linking.py index 1e1611c1..deb08a10 100644 --- a/src/swell/tasks/build_geos_by_linking.py +++ b/src/swell/tasks/build_geos_by_linking.py @@ -27,7 +27,7 @@ def execute(self): # Get the experiment/geos directory # --------------------------------- - swell_exp_path = self.get_swell_exp_path() + swell_exp_path = self.experiment_path() geos_gcm_path = os.path.join(swell_exp_path, 'GEOSgcm') # Get paths to build and source diff --git a/src/swell/tasks/build_jedi.py b/src/swell/tasks/build_jedi.py index c2445155..575e8f75 100644 --- a/src/swell/tasks/build_jedi.py +++ b/src/swell/tasks/build_jedi.py @@ -15,7 +15,6 @@ from swell.tasks.base.task_base import taskBase from swell.utilities.build import set_jedi_bundle_config, get_bundles, build_and_source_dirs - # -------------------------------------------------------------------------------------------------- @@ -29,7 +28,7 @@ def execute(self): # Get the experiment/jedi_bundle directory # ---------------------------------------- - swell_exp_path = self.get_swell_exp_path() + swell_exp_path = self.experiment_path() jedi_bundle_path = os.path.join(swell_exp_path, 'jedi_bundle') # Get paths to build and source @@ -46,7 +45,7 @@ def execute(self): bundles = [] for model_component in model_components: # Open the metadata config for interface - meta = self.open_jedi_interface_meta_config_file(model_component) + meta = self.jedi_rendering.render_interface_meta() bundles.append(meta['jedi_interface']) else: bundles_default = get_bundles() diff --git a/src/swell/tasks/build_jedi_by_linking.py b/src/swell/tasks/build_jedi_by_linking.py index 8df8617a..74dc2ffd 100644 --- a/src/swell/tasks/build_jedi_by_linking.py +++ b/src/swell/tasks/build_jedi_by_linking.py @@ -29,7 +29,7 @@ def execute(self): # Get the experiment/jedi_bundle directory # ---------------------------------------- - swell_exp_path = self.get_swell_exp_path() + swell_exp_path = self.experiment_path() jedi_bundle_path = os.path.join(swell_exp_path, 'jedi_bundle') # Get paths to build and source diff --git a/src/swell/tasks/clean_cycle.py b/src/swell/tasks/clean_cycle.py index 79ac400d..6a4b2fce 100644 --- a/src/swell/tasks/clean_cycle.py +++ b/src/swell/tasks/clean_cycle.py @@ -27,15 +27,14 @@ class CleanCycle(taskBase): def execute(self): - cycle_dir = self.config_get("cycle_dir") clean_list = self.config_get('clean_patterns', None) # If no cleaning requested then exit if clean_list is None: return - if os.path.isdir(cycle_dir): - os.chdir(cycle_dir) + if os.path.isdir(self.cycle_dir()): + os.chdir(self.cycle_dir()) # Remove all specified files for pattern in clean_list: if pattern == '*': @@ -47,7 +46,7 @@ def execute(self): for file_to_delete in glob.glob(path_parts[1]): os.remove(file_to_delete) - # Save cycle_done file to cycle_dir + # Save cycle_done file to cycle directory filename = 'cycle_done' - cmd = 'touch ' + os.path.join(cycle_dir, filename) + cmd = 'touch ' + os.path.join(self.cycle_dir(), filename) os.system(cmd) diff --git a/src/swell/tasks/clone_geos.py b/src/swell/tasks/clone_geos.py index f4f7c484..5aed293f 100644 --- a/src/swell/tasks/clone_geos.py +++ b/src/swell/tasks/clone_geos.py @@ -30,7 +30,7 @@ def execute(self): # Get the experiment/geos directory # --------------------------------- - swell_exp_path = self.get_swell_exp_path() + swell_exp_path = self.experiment_path() geos_gcm_path = os.path.join(swell_exp_path, 'GEOSgcm') # Get paths to build and source diff --git a/src/swell/tasks/clone_jedi.py b/src/swell/tasks/clone_jedi.py index 3df92c34..0d122276 100644 --- a/src/swell/tasks/clone_jedi.py +++ b/src/swell/tasks/clone_jedi.py @@ -30,7 +30,7 @@ def execute(self): # Get the experiment/jedi_bundle directory # ---------------------------------------- - swell_exp_path = self.get_swell_exp_path() + swell_exp_path = self.experiment_path() jedi_bundle_path = os.path.join(swell_exp_path, 'jedi_bundle') # Get paths to build and source @@ -55,7 +55,7 @@ def execute(self): bundles = [] for model_component in model_components: # Open the metadata config for interface - meta = self.open_jedi_interface_meta_config_file(model_component) + meta = self.jedi_rendering.render_interface_meta() bundles.append(meta['jedi_interface']) else: bundles_default = get_bundles() diff --git a/src/swell/tasks/eva_jedi_log.py b/src/swell/tasks/eva_jedi_log.py index 2457dbc7..d618e0f0 100644 --- a/src/swell/tasks/eva_jedi_log.py +++ b/src/swell/tasks/eva_jedi_log.py @@ -24,20 +24,13 @@ class EvaJediLog(taskBase): def execute(self): - # Parse config for jedi_config - # ---------------------------- - cycle_dir = self.config_get('cycle_dir') - experiment_root = self.config_get('experiment_root') - experiment_id = self.config_get('experiment_id') - # Get the model # ------------- model = self.get_model() # Read Eva template file into dictionary # -------------------------------------- - exp_path = os.path.join(experiment_root, experiment_id) - exp_suite_path = os.path.join(exp_path, experiment_id+'-suite') + exp_suite_path = os.path.join(self.experiment_path(), self.experiment_id()+'-suite') eva_config_file = os.path.join(exp_suite_path, f'eva_jedi_log-{model}.yaml') with open(eva_config_file, 'r') as eva_config_file_open: eva_str_template = eva_config_file_open.read() @@ -50,7 +43,7 @@ def execute(self): # Create dictionary used to override the eva config eva_override = {} - eva_override['cycle_dir'] = cycle_dir + eva_override['cycle_dir'] = self.cycle_dir() # Override the eva dictionary eva_str = template_string_jinja2(self.logger, eva_str_template, eva_override) @@ -58,7 +51,7 @@ def execute(self): # Write eva dictionary to file # ---------------------------- - conf_output = os.path.join(cycle_dir, 'eva', 'jedi_log', 'jedi_log_eva.yaml') + conf_output = os.path.join(self.cycle_dir(), 'eva', 'jedi_log', 'jedi_log_eva.yaml') os.makedirs(os.path.dirname(conf_output), exist_ok=True) with open(conf_output, 'w') as outfile: yaml.dump(eva_dict, outfile, default_flow_style=False) diff --git a/src/swell/tasks/eva_observations.py b/src/swell/tasks/eva_observations.py index 4f3fe9a1..69c84b0d 100644 --- a/src/swell/tasks/eva_observations.py +++ b/src/swell/tasks/eva_observations.py @@ -18,7 +18,6 @@ from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.observations import ioda_name_to_long_name - # -------------------------------------------------------------------------------------------------- @@ -28,10 +27,18 @@ def execute(self): # Parse config for jedi_config # ---------------------------- - cycle_dir = self.config_get('cycle_dir') - experiment_root = self.config_get('experiment_root') - experiment_id = self.config_get('experiment_id') + background_time = self.config_get('background_time') + crtm_coeff_dir = self.config_get('crtm_coeff_dir', None) observations = self.config_get('observations') + window_offset = self.config_get('window_offset') + + # Compute window beginning time + window_begin = self.da_window_params.window_begin(window_offset) + + # Create JEDI interface config templates dictionary + self.jedi_rendering.add_key('background_time', background_time) + self.jedi_rendering.add_key('crtm_coeff_dir', crtm_coeff_dir) + self.jedi_rendering.add_key('window_begin', window_begin) # Get the model # ------------- @@ -39,8 +46,7 @@ def execute(self): # Read Eva template file into dictionary # -------------------------------------- - exp_path = os.path.join(experiment_root, experiment_id) - exp_suite_path = os.path.join(exp_path, experiment_id+'-suite') + exp_suite_path = os.path.join(self.experiment_path(), self.experiment_id()+'-suite') eva_config_file = os.path.join(exp_suite_path, f'eva_observations-{model}.yaml') with open(eva_config_file, 'r') as eva_config_file_open: eva_str_template = eva_config_file_open.read() @@ -50,7 +56,7 @@ def execute(self): for observation in observations: # Load the observation dictionary - observation_dict = self.open_jedi_interface_obs_config_file(observation) + observation_dict = self.jedi_rendering.render_interface_observations(observation) # Split the full path into path and filename obs_path_file = observation_dict['obs space']['obsdataout']['engine']['obsfile'] @@ -78,7 +84,7 @@ def execute(self): # Create dictionary used to override the eva config eva_override = {} - eva_override['cycle_dir'] = cycle_dir + eva_override['cycle_dir'] = self.cycle_dir() eva_override['obs_path_file'] = obs_path_file eva_override['instrument'] = ioda_name eva_override['instrument_title'] = full_name @@ -105,7 +111,7 @@ def execute(self): # Write eva dictionary to file # ---------------------------- - conf_output = os.path.join(cycle_dir, 'eva', ioda_name, ioda_name+'_eva.yaml') + conf_output = os.path.join(self.cycle_dir(), 'eva', ioda_name, ioda_name+'_eva.yaml') os.makedirs(os.path.dirname(conf_output), exist_ok=True) with open(conf_output, 'w') as outfile: yaml.dump(eva_dict, outfile, default_flow_style=False) diff --git a/src/swell/tasks/generate_b_climatology.py b/src/swell/tasks/generate_b_climatology.py index 7e881f77..93ecc23f 100644 --- a/src/swell/tasks/generate_b_climatology.py +++ b/src/swell/tasks/generate_b_climatology.py @@ -12,6 +12,8 @@ from swell.tasks.base.task_base import taskBase from swell.utilities.shell_commands import run_track_log_subprocess +from swell.utilities.render_jedi_interface_files import template_dictionary_oops_config, \ + read_render_jedi_interface_oops, # -------------------------------------------------------------------------------------------------- @@ -42,9 +44,9 @@ def jedi_dictionary_iterator(self, jedi_config_dict): def generate_jedi_config(self): - # Create dictionary from the templated JEDI config file - # --------------------------------------------------------- - jedi_config_dict = self.open_jedi_oops_config_file('StaticBInit') + # Render StaticBInit (no templates needed) + # ---------------------------------------- + jedi_config_dict = self.jedi_rendering.render_oops_file('StaticBInit') # Read configs for the rest of the dictionary # ------------------------------------------- @@ -78,7 +80,7 @@ def generate_bump(self): model_component, self.background_error_model, 'climatological', resolution, str(self.np)) - d_dir = os.path.join(self.cycle_dir, 'background_error_model') + d_dir = os.path.join(self.cycle_dir(), 'background_error_model') try: self.logger.info(' Copying BUMP files from: '+b_dir) @@ -89,7 +91,7 @@ def generate_bump(self): # Jedi configuration file # ----------------------- - jedi_config_file = os.path.join(self.cycle_dir, 'jedi_bump_config.yaml') + jedi_config_file = os.path.join(self.cycle_dir(), 'jedi_bump_config.yaml') # Generate the JEDI configuration file for running the executable # --------------------------------------------------------------- @@ -101,7 +103,7 @@ def generate_bump(self): # Jedi executable name # -------------------- jedi_executable = interface_executable[self.jedi_interface] - jedi_executable_path = os.path.join(self.get_swell_exp_path(), 'jedi_bundle', + jedi_executable_path = os.path.join(self.experiment_path(), 'jedi_bundle', 'build', 'bin', jedi_executable) # Run the JEDI executable @@ -112,7 +114,7 @@ def generate_bump(self): # Move to the cycle directory # --------------------------- - os.chdir(self.cycle_dir) + os.chdir(self.cycle_dir()) if not os.path.exists('background_error_model'): os.mkdir('background_error_model') @@ -143,13 +145,12 @@ def execute(self): self.swell_static_files = self.config_get('swell_static_files') self.horizontal_resolution = self.config_get('horizontal_resolution') self.vertical_resolution = self.config_get('vertical_resolution') - self.cycle_dir = self.config_get('cycle_dir') self.npx_proc = self.config_get('npx_proc') # Used in eval(total_processors) self.npy_proc = self.config_get('npy_proc') # Used in eval(total_processors) # Get the JEDI interface for this model component # ----------------------------------------------- - model_component_meta = self.open_jedi_interface_meta_config_file() + model_component_meta = self.jedi_rendering.render_interface_meta() self.jedi_interface = model_component_meta['jedi_interface'] # Compute number of processors diff --git a/src/swell/tasks/get_background.py b/src/swell/tasks/get_background.py index 9d327720..7f67548b 100644 --- a/src/swell/tasks/get_background.py +++ b/src/swell/tasks/get_background.py @@ -50,13 +50,21 @@ def execute(self): bkg_steps = [] # Parse config - window_type = self.config_get('window_type') - window_length = self.config_get('window_length') - window_offset = self.config_get('window_offset') - background_source = self.config_get('background_source', 'file') background_experiment = self.config_get('background_experiment') - horizontal_resolution = self.config_get('horizontal_resolution') + background_frequency = self.config_get('background_frequency', None) + background_source = self.config_get('background_source', 'file') forecast_offset = self.config_get('analysis_forecast_window_offset') + horizontal_resolution = self.config_get('horizontal_resolution') + window_length = self.config_get('window_length') + window_offset = self.config_get('window_offset') + window_type = self.config_get('window_type') + + # Get window parameters + local_background_time = self.da_window_params.local_background_time(window_offset, + window_type) + + # Add to jedi config rendering dictionary + self.jedi_rendering.add_key('local_background_time', local_background_time) # Convert to datetime durations window_length_dur = isodate.parse_duration(window_length) @@ -79,8 +87,7 @@ def execute(self): # ---------------------------------------------------------- if window_type == "4D" and background_source == 'file': - bkg_freq = self.config_get('background_frequency') - bkg_freq_dur = isodate.parse_duration(bkg_freq) + bkg_freq_dur = isodate.parse_duration(background_frequency) # Check for a sensible frequency if (window_length_dur/bkg_freq_dur) % 2: @@ -110,7 +117,7 @@ def execute(self): self.logger.info('Background steps being fetched: '+' '.join(str(e) for e in bkg_steps)) # Get r2d2 dictionary - r2d2_dict = self.open_jedi_interface_model_config_file('r2d2') + r2d2_dict = self.jedi_rendering.render_interface_model('r2d2') # Loop over fc for fc in r2d2_dict['fetch']['fc']: diff --git a/src/swell/tasks/get_background_geos_experiment.py b/src/swell/tasks/get_background_geos_experiment.py index 23d63ec3..44143e87 100644 --- a/src/swell/tasks/get_background_geos_experiment.py +++ b/src/swell/tasks/get_background_geos_experiment.py @@ -14,6 +14,7 @@ import tarfile from swell.tasks.base.task_base import taskBase +from swell.utilities.date_time import datetime_formats # -------------------------------------------------------------------------------------------------- @@ -31,28 +32,26 @@ def execute(self): See the taskBase constructor for more information. """ + # Parse config + # ------------ + geos_background_restart_offset = self.config_get('geos_background_restart_offset') + tar_filename_template_geos = self.config_get('geos_bkg_tar_filename_template') + bkg_filename_template_geos = self.config_get('geos_bkg_filename_template') + bkg_filename_template_jedi = self.config_get('jedi_bkg_filename_template') + # Current cycle time (middle of the window) # ----------------------------------------- - current_cycle = self.config.get('current_cycle') - current_cycle_dt = dt.strptime(current_cycle, self.config.dt_format) + current_cycle_dt = dt.strptime(self.cycle_time(), datetime_formats['iso_format']) # Hours before middle of window that forecast began # ------------------------------------------------- - forecast_offset = isodate.parse_duration(self.config.get('geos_background_restart_offset')) + forecast_offset = isodate.parse_duration(geos_background_restart_offset) forecast_rst_dt = current_cycle_dt - forecast_offset # Create cycle directory if needed # -------------------------------- - cycle_dir = self.config.get('cycle_dir') - if not os.path.exists(cycle_dir): - os.makedirs(cycle_dir, 0o755, exist_ok=True) - - # Geos experiment settings - # ------------------------ - geos_experiment = self.config.get('geos_experiment') - tar_filename_template_geos = self.config.get('geos_bkg_tar_filename_template') - bkg_filename_template_geos = self.config.get('geos_bkg_filename_template') - bkg_filename_template_jedi = self.config.get('jedi_bkg_filename_template') + if not os.path.exists(self.cycle_dir()): + os.makedirs(self.cycle_dir(), 0o755, exist_ok=True) # Path to restarts # ---------------- @@ -67,14 +66,14 @@ def execute(self): # Extract files # ------------- member.name = os.path.basename(member.name) - traj_tar_file.extract(member, cycle_dir) + traj_tar_file.extract(member, self.cycle_dir()) # Rename the files # ---------------- bkg_filename_dt = dt.strptime(member.name, bkg_filename_template_geos) bkg_filename_jedi = bkg_filename_dt.strftime(bkg_filename_template_jedi) - os.rename(os.path.join(cycle_dir, member.name), - os.path.join(cycle_dir, bkg_filename_jedi)) + os.rename(os.path.join(self.cycle_dir(), member.name), + os.path.join(self.cycle_dir(), bkg_filename_jedi)) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/get_observations.py b/src/swell/tasks/get_observations.py index 07aba05e..771312b1 100644 --- a/src/swell/tasks/get_observations.py +++ b/src/swell/tasks/get_observations.py @@ -41,6 +41,16 @@ def execute(self): background_time = self.config_get('background_time') observations = self.config_get('observations') window_length = self.config_get('window_length') + crtm_coeff_dir = self.config_get('crtm_coeff_dir', None) + window_offset = self.config_get('window_offset') + + # Get window begin time + window_begin = self.da_window_params.window_begin(window_offset) + + # Add to JEDI template rendering dictionary + self.jedi_rendering.add_key('background_time', background_time) + self.jedi_rendering.add_key('crtm_coeff_dir', crtm_coeff_dir) + self.jedi_rendering.add_key('window_begin', window_begin) # Loop over observation operators # ------------------------------- @@ -48,7 +58,7 @@ def execute(self): # Open the observation operator dictionary # ---------------------------------------- - observation_dict = self.open_jedi_interface_obs_config_file(observation) + observation_dict = self.jedi_rendering.render_interface_observations(observation) # Fetch observation files # ----------------------- diff --git a/src/swell/tasks/obs_process_setup.py b/src/swell/tasks/obs_process_setup.py deleted file mode 100644 index be7012e3..00000000 --- a/src/swell/tasks/obs_process_setup.py +++ /dev/null @@ -1,203 +0,0 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - - -# -------------------------------------------------------------------------------------------------- - - -from datetime import datetime as dt -import isodate -import os -import tarfile -import glob -from swell.tasks.base.task_base import taskBase -from r2d2 import store - -# -------------------------------------------------------------------------------------------------- - - -class ObsProcessSetup(taskBase): - - def execute(self): - - """Acquires background files from a GEOS experiment. Expects traj files stored in a tarfile - - Parameters - ---------- - All inputs are extracted from the JEDI experiment file configuration. - See the taskBase constructor for more information. - """ - - # Path to ioda-converters install - # ------------------------------- - # define iodabin in experiment file - iodabin = self.config.get('iodabin') - - # Current cycle time (middle of the window) - # ----------------------------------------- - current_cycle = self.config.get('current_cycle') - current_cycle_dt = dt.strptime(current_cycle, self.config.dt_format) - - # Set time windows for R2D2 - # ------------------------ - obs_offset = isodate.parse_duration(self.config.get('geos_obs_offset')) - obs_r2d2_dt = current_cycle_dt - obs_offset - - satbias_offset = isodate.parse_duration(self.config.get('geos_satbias_offset')) - satbias_r2d2_dt = current_cycle_dt - satbias_offset - - # Create cycle directory if needed - # -------------------------------- - cycle_dir = self.config.get('cycle_dir') - print(cycle_dir) - if not os.path.exists(cycle_dir): - os.makedirs(cycle_dir, 0o755, exist_ok=True) - out_dir = cycle_dir + '/out/' - os.makedirs(out_dir, 0o755, exist_ok=True) - - # Geos experiment settings - # ------------------------ - geos_experiment = self.config.get('geos_experiment') - obs_dir_template = self.config.get('geos_obs_dir_template') - combined_obs_template = self.config.get('combined_obs_template') - sondes_obs_template = self.config.get('sondes_obs_template') - sondes_obs_template_rename = self.config.get('sondes_obs_template_rename') - sfcship_obs_template = self.config.get('sfcship_obs_template') - sfcship_obs_template_rename = self.config.get('sfcship_obs_template_rename') - r2d2_obs_standard_template = self.config.get('r2d2_obs_standard_template') - satbias_org_template = self.config.get('satbias_org_template') - satbias_template = self.config.get('satbias_template') - satbiaspc_template = self.config.get('satbiaspc_template') - satbias_dir_template = self.config.get('satbias_dir_template') - tlapse_template = self.config.get('tlapse_template') - r2d2_satbias_template = self.config.get('r2d2_satbias_template') - - obs_dir = current_cycle_dt.strftime(obs_dir_template) - satbias_dir = obs_r2d2_dt.strftime(satbias_dir_template) - combined_obs = current_cycle_dt.strftime(combined_obs_template) - sondes_obs = current_cycle_dt.strftime(sondes_obs_template) - sondes_obs_rename = current_cycle_dt.strftime(sondes_obs_template_rename) - sfcship_obs = current_cycle_dt.strftime(sfcship_obs_template) - sfcship_obs_rename = current_cycle_dt.strftime(sfcship_obs_template_rename) - r2d2_obs_standard = obs_r2d2_dt.strftime(r2d2_obs_standard_template) - satbias_org = obs_r2d2_dt.strftime(satbias_org_template) - satbias = obs_r2d2_dt.strftime(satbias_template) - satbiaspc = obs_r2d2_dt.strftime(satbiaspc_template) - tlapse_name = satbias_r2d2_dt.strftime(tlapse_template) - r2d2_satbias = satbias_r2d2_dt.strftime(r2d2_satbias_template) - - # Copy obs files to cycle directory - # --------------------------------- - for filepath in list(glob.glob(obs_dir + '/*ges*nc4')): - filename = os.path.basename(filepath) - os.system('ln -sf ' + filepath + ' ' + cycle_dir + '/' + filename) - - # Run proc_gsi_ncdiag - # --------------------------------- - # try these without the python calls - os.system('python ' + iodabin + '/proc_gsi_ncdiag.py -o ' + out_dir + ' ' + cycle_dir) - - # Combine conventional types - # -------------------------- - conv_types = ['aircraft_', 'rass_', 'sfc_', 'sfcship_', 'sondes_'] - for conv_type in conv_types: - file_list_str = '' - for file_name in glob.glob(out_dir + conv_type + '*'): - file_list_str = file_list_str + file_name + ' ' - os.system('python ' + iodabin + '/combine_obsspace.py -i ' - + file_list_str + ' -o ' + out_dir + '/' + conv_type + combined_obs) - # remove files - os.system('rm ' + file_list_str) - - # Rename some files - # ----------------- - os.system('mv ' + out_dir + '/' + sondes_obs + ' ' + out_dir + '/' + sondes_obs_rename) - os.system('mv ' + out_dir + '/' + sfcship_obs + ' ' + out_dir + '/' + sfcship_obs_rename) - - # Rename the files with R2D2 standard - # ----------------------------------- - os.system('rename _' + combined_obs + ' ' + r2d2_obs_standard + ' ' + out_dir + '/*') - - # Handle satbias - # -------------- - satbias_out_dir = out_dir + '/satbias/' - os.system('mkdir -p ' + satbias_out_dir) - - os.system('tar -xvf ' + satbias_dir + satbias_org + ' -C ' - + satbias_out_dir + ' ' + satbias + ' ' + satbiaspc) - os.system('ln -sf ' + satbias_out_dir + satbias + ' ' - + satbias_out_dir + 'ana_satbias_rst.txt') - os.system('ln -sf ' + satbias_out_dir + satbiaspc + ' ' - + satbias_out_dir + 'ana_satbiaspc_rst.txt') - - sensors = ['airs_aqua', 'amsua_aqua', 'amsua_metop-a', 'amsua_metop-b', 'amsua_metop-c', - 'amsua_n15', 'amsua_n18', 'amsua_n19', 'atms_n20', 'atms_npp', 'avhrr3_metop-a', - 'avhrr3_metop-b', 'avhrr3_n18', 'avhrr3_n19', 'cris-fsr_npp', 'cris-fsr_n20', - 'gmi_gpm', 'hirs4_metop-a', 'hirs4_n18', 'hirs4_n19', 'iasi_metop-a', - 'iasi_metop-b', 'mhs_metop-a', 'mhs_metop-b', 'mhs_metop-c', 'mhs_n19', - 'seviri_m08', 'ssmis_f17', 'ssmis_f18'] - for sensor in sensors: - os.system('grep -i ' + sensor + ' ' + satbias_out_dir - + """ana_satbias_rst.txt | awk '{print $2" "$3" "$4}' > """ - + satbias_out_dir + 'gsi.' + geos_experiment + '.bc.' + sensor + tlapse_name) - - os.system('module unload core/anaconda/3.8') - # Make satbias converter part of swell - os.system('cp ' + '/discover/nobackup/asewnath/jedi_scripts/satbias_converter.yaml ' - + satbias_out_dir) - os.chdir(satbias_out_dir) - print(iodabin + '/satbias2ioda.x satbias_converter.yaml') - os.system(iodabin + '/satbias2ioda.x satbias_converter.yaml') - - os.system('rename satbias_ gsi.' + geos_experiment + '.bc. *nc4') - os.system('rename .nc4 ' + r2d2_satbias + ' *nc4') - - os.system('mv ' + satbias_out_dir + '*satbias ' + out_dir) - os.system('mv ' + satbias_out_dir + '*tlapse ' + out_dir) - os.chdir(out_dir) - os.system('rm -rf ' + satbias_out_dir) - - # Call R2D2 to store files - # ------------------------ - - # Perform store of observations - for filepath in list(glob.glob(out_dir + '/*nc4')): - source_file = os.path.basename(filepath) - filename_parts = source_file.split('.') - name = filename_parts[0] - store(date=obs_r2d2_dt, - source_file=source_file, - provider='ncdiag', - obs_type=name, - type='ob', - experiment=geos_experiment) - - # Perform store of satbias - for filepath in list(glob.glob(out_dir + '/*satbias')): - source_file = os.path.basename(filepath) - filename_parts = source_file.split('.') - name = filename_parts[3] - store(date=satbias_r2d2_dt, - source_file=source_file, - provider='gsi', - obs_type=name, - type='bc', - experiment=geos_experiment, - file_type='satbias') - - # Perform store of tlapse - for filepath in list(glob.glob(out_dir + '/*tlapse')): - source_file = os.path.basename(filepath) - filename_parts = source_file.split('.') - name = filename_parts[3] - store(date=satbias_r2d2_dt, - source_file=source_file, - provider='gsi', - obs_type=name, - type='bc', - experiment=geos_experiment, - file_type='tlapse') -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/run_jedi_hofx_executable.py b/src/swell/tasks/run_jedi_hofx_executable.py index 9e7b0d48..0db49b69 100644 --- a/src/swell/tasks/run_jedi_hofx_executable.py +++ b/src/swell/tasks/run_jedi_hofx_executable.py @@ -36,39 +36,35 @@ def execute(self): # Path to executable being run # ---------------------------- - cycle_dir = self.config_get('cycle_dir') - experiment_dir = self.config_get('experiment_dir') window_type = self.config_get('window_type') - model = self.config_get('window_type') - suite_to_run = self.config_get('suite_to_run') npx_proc = self.config_get('npx_proc') # Used in eval(total_processors) npy_proc = self.config_get('npy_proc') # Used in eval(total_processors) total_processors = self.config_get('total_processors') # Jedi configuration file # ----------------------- - jedi_config_file = os.path.join(cycle_dir, 'jedi_hofx_config.yaml') + jedi_config_file = os.path.join(self.cycle_dir(), 'jedi_hofx_config.yaml') # Output log file # --------------- - output_log_file = os.path.join(cycle_dir, 'jedi_hofx_log.log') + output_log_file = os.path.join(self.cycle_dir(), 'jedi_hofx_log.log') # Generate the JEDI configuration file for running the executable # --------------------------------------------------------------- - jedi_config_dict = self.generate_jedi_config(suite_to_run, window_type) + jedi_config_dict = self.generate_jedi_config(self.suite(), window_type) with open(jedi_config_file, 'w') as jedi_config_file_open: yaml.dump(jedi_config_dict, jedi_config_file_open, default_flow_style=False) # Get the JEDI interface for this model component # ----------------------------------------------- - model_component_meta = self.open_jedi_interface_meta_config_file() + model_component_meta = self.jedi_rendering.render_interface_meta() jedi_interface = model_component_meta['jedi_interface'] # Jedi executable name # -------------------- jedi_executable = interface_executable[jedi_interface + '-' + window_type] - jedi_executable_path = os.path.join(experiment_dir, 'jedi_bundle', 'build', 'bin', + jedi_executable_path = os.path.join(self.experiment_path(), 'jedi_bundle', 'build', 'bin', jedi_executable) # Compute number of processors @@ -79,7 +75,8 @@ def execute(self): # Run the JEDI executable # ----------------------- - self.run_executable(cycle_dir, np, jedi_executable_path, jedi_config_file, output_log_file) + self.run_executable(self.cycle_dir(), np, jedi_executable_path, jedi_config_file, + output_log_file) self.logger.info('Running '+jedi_executable_path+' with '+str(np)+' processors.') # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/run_jedi_variational_executable.py b/src/swell/tasks/run_jedi_variational_executable.py index 6db59778..4749502c 100644 --- a/src/swell/tasks/run_jedi_variational_executable.py +++ b/src/swell/tasks/run_jedi_variational_executable.py @@ -29,30 +29,27 @@ def execute(self): # Path to executable being run # ---------------------------- - cycle_dir = self.config_get('cycle_dir') - experiment_dir = self.config_get('experiment_dir') window_type = self.config_get('window_type') - suite_to_run = self.config_get('suite_to_run') npx_proc = self.config_get('npx_proc') # Used in eval(total_processors) npy_proc = self.config_get('npy_proc') # Used in eval(total_processors) total_processors = self.config_get('total_processors') # Jedi configuration file # ----------------------- - jedi_config_file = os.path.join(cycle_dir, 'jedi_variational_config.yaml') + jedi_config_file = os.path.join(self.cycle_dir(), 'jedi_variational_config.yaml') # Output log file # --------------- - output_log_file = os.path.join(cycle_dir, 'jedi_variational_log.log') + output_log_file = os.path.join(self.cycle_dir(), 'jedi_variational_log.log') # Get the JEDI interface for this model component # ----------------------------------------------- - model_component_meta = self.open_jedi_interface_meta_config_file() + model_component_meta = self.jedi_rendering.render_interface_meta() jedi_interface = model_component_meta['jedi_interface'] # Generate the JEDI configuration file for running the executable # --------------------------------------------------------------- - jedi_config_dict = self.generate_jedi_config(suite_to_run, window_type) + jedi_config_dict = self.generate_jedi_config(self.suite(), window_type) with open(jedi_config_file, 'w') as jedi_config_file_open: yaml.dump(jedi_config_dict, jedi_config_file_open, default_flow_style=False) @@ -60,8 +57,8 @@ def execute(self): # Jedi executable name # -------------------- jedi_executable = interface_executable[jedi_interface + '-' + window_type] - jedi_executable_path = os.path.join(experiment_dir, 'jedi_bundle', 'build', 'bin', - jedi_executable) + jedi_executable_path = os.path.join(self.experiment_path(), 'jedi_bundle', 'build', + 'bin', jedi_executable) # Compute number of processors # ---------------------------- total_processors = total_processors.replace('npx_proc', str(npx_proc)) @@ -70,7 +67,8 @@ def execute(self): # Run the JEDI executable # ----------------------- - self.run_executable(cycle_dir, np, jedi_executable_path, jedi_config_file, output_log_file) + self.run_executable(self.cycle_dir(), np, jedi_executable_path, jedi_config_file, + output_log_file) self.logger.info('Running '+jedi_executable_path+' with '+str(np)+' processors.') diff --git a/src/swell/tasks/save_obs_diags.py b/src/swell/tasks/save_obs_diags.py index 066f7e9a..fb9724ce 100644 --- a/src/swell/tasks/save_obs_diags.py +++ b/src/swell/tasks/save_obs_diags.py @@ -24,16 +24,25 @@ def execute(self): # Parse config # ------------ - experiment_id = self.config_get('experiment_id') - window_begin = self.config_get('window_begin') + background_time = self.config_get('background_time') + crtm_coeff_dir = self.config_get('crtm_coeff_dir', None) observations = self.config_get('observations') + window_offset = self.config_get('window_offset') + + # Get window beginning + window_begin = self.da_window_params.window_begin(window_offset) + + # Create templates dictionary + self.jedi_rendering.add_key('background_time', background_time) + self.jedi_rendering.add_key('crtm_coeff_dir', crtm_coeff_dir) + self.jedi_rendering.add_key('window_begin', window_begin) # Loop over observation operators # ------------------------------- for observation in observations: # Load the observation dictionary - observation_dict = self.open_jedi_interface_obs_config_file(observation) + observation_dict = self.jedi_rendering.render_interface_observations(observation) # Store observation files # ----------------------- @@ -54,4 +63,4 @@ def execute(self): source_file=obs_path_file, obs_type=name, type='ob', - experiment=experiment_id) + experiment=self.experiment_id()) diff --git a/src/swell/tasks/stage_jedi.py b/src/swell/tasks/stage_jedi.py index e832ba27..9c3041d8 100644 --- a/src/swell/tasks/stage_jedi.py +++ b/src/swell/tasks/stage_jedi.py @@ -33,19 +33,32 @@ def execute(self): See the taskBase constructor for more information. """ + # Extract potential template variables from config + horizontal_resolution = self.config_get('horizontal_resolution') + swell_static_files = self.config_get('swell_static_files') + vertical_resolution = self.config_get('vertical_resolution') + + # Add jedi interface template keys + self.jedi_rendering.add_key('horizontal_resolution', horizontal_resolution) + self.jedi_rendering.add_key('swell_static_files', swell_static_files) + self.jedi_rendering.add_key('vertical_resolution', vertical_resolution) + # Open the stage configuration file # --------------------------------- stage_file = 'stage' if self.is_datetime_dependent(): stage_file = 'stage_cycle' - stage_dict = self.open_jedi_interface_model_config_file(stage_file) - - # Check that the passed configuration had a stage component - if stage_dict is None: + # Check for presence of stage file + # -------------------------------- + stage_file = os.path.join(self.experiment_config_path(), self.get_model(), stage_file) + if not os.path.exists(stage_file): self.logger.info('No stage dictionary was found for this configuration') exit(0) + # Open file and template it + stage_dict = self.jedi_rendering.render_interface_model(stage_file) + # Run the file handler # -------------------- try: diff --git a/src/swell/tasks/store_background.py b/src/swell/tasks/store_background.py index 74ab0bc2..945b9b6a 100644 --- a/src/swell/tasks/store_background.py +++ b/src/swell/tasks/store_background.py @@ -8,8 +8,6 @@ # -------------------------------------------------------------------------------------------------- -from swell.tasks.base.task_base import taskBase - from datetime import datetime as dt import isodate import os @@ -17,6 +15,10 @@ from r2d2 import store +from swell.tasks.base.task_base import taskBase +from swell.utilities.date_time import datetime_formats + + # -------------------------------------------------------------------------------------------------- @@ -32,27 +34,21 @@ def execute(self): See the taskBase constructor for more information. """ - # Shortcuts to base objects - # ------------------------- - cfg = self.config - logger = self.logger - # Current cycle time object # ------------------------- - current_cycle = cfg.get('current_cycle') - current_cycle_dto = dt.strptime(current_cycle, cfg.dt_format) + current_cycle_dto = dt.strptime(self.cycle_time(), datetime_formats['iso_format']) # Get duration into forecast for first background file # ---------------------------------------------------- bkg_steps = [] # Parse config - window_type = cfg.get('window_type') - window_length = cfg.get('window_length') - window_offset = cfg.get('window_offset') + window_type = self.config_get('window_type') + window_length = self.config_get('window_length') + window_offset = self.config_get('window_offset') # Position relative to center of the window where forecast starts - forecast_offset = cfg.get('analysis_forecast_window_offset') + forecast_offset = self.config_get('analysis_forecast_window_offset') # Convert to datetime durations window_length_dur = isodate.parse_duration(window_length) @@ -75,7 +71,7 @@ def execute(self): # If background is provided though files get all backgrounds # ---------------------------------------------------------- - bkg_info = cfg.get('backgrounds') + bkg_info = self.config_get('backgrounds') if window_type == "4D" and bkg_info['background source'] == 'file': @@ -84,7 +80,7 @@ def execute(self): # Check for a sensible frequency if (window_length_dur/bkg_freq_dur) % 2: - logger.abort('Window length not divisible by background frequency') + self.logger.abort('Window length not divisible by background frequency') # Loop over window start_date = current_cycle_dto - window_offset_dur @@ -99,13 +95,13 @@ def execute(self): # Loop over background files in the R2D2 config and store # ------------------------------------------------------- - logger.info('Background steps being fetched: '+' '.join(str(e) for e in bkg_steps)) + self.logger.info('Background steps being fetched: '+' '.join(str(e) for e in bkg_steps)) # Background dictionary from config - background_dict = cfg.get('BACKGROUND') + background_dict = self.jedi_rendering.render_interface_model('background') # Get r2d2 dictionary - r2d2_dict = cfg.get('R2D2') + r2d2_dict = self.jedi_rendering.render_interface_model('r2d2') # Loop over fc for fc in r2d2_dict['store']['fc']: @@ -131,7 +127,7 @@ def execute(self): # Set the target file name target_file = target_file_type_template.replace("$(valid_date)", valid_time_str) - target_file = os.path.join(cfg.get('cycle_dir'), target_file) + target_file = os.path.join(self.cycle_dir(), target_file) # Perform the store store(date=forecast_start_time, @@ -140,6 +136,6 @@ def execute(self): file_type='bkg', fc_date_rendering='analysis', step=bkg_step, - resolution=cfg.get('horizontal_resolution'), + resolution=self.config_get('horizontal_resolution'), type='fc', experiment=bkg_info['background experiment']) diff --git a/src/swell/utilities/bin/__init__.py b/src/swell/utilities/bin/__init__.py new file mode 100644 index 00000000..ac1c0bc4 --- /dev/null +++ b/src/swell/utilities/bin/__init__.py @@ -0,0 +1,9 @@ +# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +import os + +repo_directory = os.path.dirname(__file__) diff --git a/src/swell/utilities/bin/check_jedi_interface_templates.py b/src/swell/utilities/bin/check_jedi_interface_templates.py index d6f9e393..89f77cfa 100644 --- a/src/swell/utilities/bin/check_jedi_interface_templates.py +++ b/src/swell/utilities/bin/check_jedi_interface_templates.py @@ -27,17 +27,27 @@ def main(): # Create a logger logger = Logger('CheckJediInterfaceTemplates') - config_types = ['oops', 'interfaces/*/model', 'interfaces/*/observations'] + config_types = ['oops/*yaml', + 'interfaces/*/model/*yaml', + 'interfaces/*/observations/*yaml', + 'interfaces/*/model/r2d2.yaml', + 'interfaces/*/model/stage*.yaml', + 'interfaces/*/model/background.yaml', + 'interfaces/*/model/StaticBInit.yaml', + ] # Path to JEDI interface code swell_path = get_swell_path() jedi_interfaces_path = os.path.join(swell_path, 'configuration', 'jedi') + # Default keys + def_keys = ['experiment_id', 'experiment_root', 'model_component', 'cycle_dir'] + # Loop over config types for config_type in config_types: # Path to the files for this config type - config_yaml_path = os.path.join(jedi_interfaces_path, config_type, '*yaml') + config_yaml_path = os.path.join(jedi_interfaces_path, config_type) config_yaml_files = glob.glob(config_yaml_path) @@ -68,11 +78,20 @@ def main(): # Remove duplicates & sort templates = sorted(list(set(templates))) + # Remove some keys + for ind, template in enumerate(templates): + templates[ind] = templates[ind].strip('{').strip('}') + if templates[ind] in def_keys: + templates[ind] = '*** ' + templates[ind] + + # Resort to gather *** + templates = sorted(list(set(templates))) + # Print the templates logger.info(config_type) logger.info('-'*len(config_type)) for template in templates: - logger.info(template.strip('{').strip('}')) + logger.info(template) if config_type != config_types[-1]: logger.info(' ', False) diff --git a/src/swell/utilities/date_time.py b/src/swell/utilities/date_time.py index c5af0183..76d0cf28 100644 --- a/src/swell/utilities/date_time.py +++ b/src/swell/utilities/date_time.py @@ -7,6 +7,7 @@ # -------------------------------------------------------------------------------------------------- +import datetime import isodate @@ -25,33 +26,33 @@ class DataAssimilationWindowParams(): - def __init__(self, cycle_time, window_offset): + def __init__(self, logger, cycle_time): """ Defines cycle dependent parameters for the data assimilation window and adds to config """ + # Keep logger + self.logger = logger + # Current cycle datetime object self.__current_cycle_dto__ = datetime.datetime.strptime(cycle_time, datetime_formats['dir_format']) - # Save the window offset - self.__window_offset__ = window_offset - # ---------------------------------------------------------------------------------------------- - def __get_window_begin_dto__(self): + def __get_window_begin_dto__(self, window_offset): - window_offset_dur = isodate.parse_duration(self.__window_offset__) + window_offset_dur = isodate.parse_duration(window_offset) return self.__current_cycle_dto__ - window_offset_dur # ---------------------------------------------------------------------------------------------- - def __get_local_background_time__(self, window_type): + def __get_local_background_time__(self, window_type, window_offset): # Background time for the window if window_type == '4D': - local_background_time = self.__get_window_begin_dto__(self.__window_offset__) + local_background_time = self.__get_window_begin_dto__(window_offset) elif window_type == '3D': local_background_time = self.__current_cycle_dto__ @@ -59,21 +60,21 @@ def __get_local_background_time__(self, window_type): # ---------------------------------------------------------------------------------------------- - def window_begin(self): + def window_begin(self, window_offset): - window_begin_dto = self.__get_window_begin_dto__() + window_begin_dto = self.__get_window_begin_dto__(window_offset) return window_begin_dto.strftime(datetime_formats['dir_format']) # ---------------------------------------------------------------------------------------------- - def window_begin_iso(self): + def window_begin_iso(self, window_offset): - window_begin_dto = self.__get_window_begin_dto__() + window_begin_dto = self.__get_window_begin_dto__(window_offset) return window_begin_dto.strftime(datetime_formats['iso_format']) # ---------------------------------------------------------------------------------------------- - def background_time(self, background_time_offset): + def background_time(self, window_offset, background_time_offset): background_time_offset_dur = isodate.parse_duration(background_time_offset) background_time_dto = self.__current_cycle_dto__ - background_time_offset_dur @@ -81,16 +82,16 @@ def background_time(self, background_time_offset): # ---------------------------------------------------------------------------------------------- - def local_background_time_iso(self, window_type): + def local_background_time_iso(self, window_offset, window_type): - local_background_time = self.__get_local_background_time__(window_type) + local_background_time = self.__get_local_background_time__(window_type, window_offset) return local_background_time.strftime(datetime_formats['iso_format']) # ---------------------------------------------------------------------------------------------- - def local_background_time(self, window_type): + def local_background_time(self, window_offset, window_type): - local_background_time = self.__get_local_background_time__(window_type) + local_background_time = self.__get_local_background_time__(window_type, window_offset) return local_background_time.strftime(datetime_formats['dir_format']) # ---------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/render_jedi_interface_files.py b/src/swell/utilities/render_jedi_interface_files.py index b10f0c62..3d005c1f 100644 --- a/src/swell/utilities/render_jedi_interface_files.py +++ b/src/swell/utilities/render_jedi_interface_files.py @@ -8,153 +8,152 @@ import os -import shutil +import yaml -from jedi_bundle.bin.jedi_bundle import get_default_config, get_bundles from swell.utilities.jinja2 import template_string_jinja2 # -------------------------------------------------------------------------------------------------- +class JediConfigRendering(): -def template_dictionary_oops_config(analysis_variables, gradient_norm_reduction, minimizer, - number_of_iterations, window_begin_iso, window_length): + def __init__(self, logger, experiment_root, experiment_id, cycle_dir, jedi_interface=None): -dictionary_oops_config = {} -dictionary_oops_config['analysis_variables'] = analysis_variables -dictionary_oops_config['gradient_norm_reduction'] = gradient_norm_reduction -dictionary_oops_config['minimizer'] = minimizer -dictionary_oops_config['number_of_iterations'] = number_of_iterations -dictionary_oops_config['window_begin_iso'] = window_begin_iso -dictionary_oops_config['window_length'] = window_length -return dictionary_oops_config + # Keep a copy of the logger + self.logger = logger + # Copy the experiment configuration path + self.jedi_config_path = os.path.join(experiment_root, experiment_id, 'configuration', + 'jedi') -# -------------------------------------------------------------------------------------------------- - - -def template_dictionary_model_config(analysis_variables, background_frequency, cycle_dir, - experiment_dir, experiment_id, horizontal_resolution, - local_background_time_iso, local_background_time, - npx_proc, npy_proc, swell_static_files, vertical_resolution): - -dictionary_model_config = {} -dictionary_model_config['analysis_variables'] = analysis_variables -dictionary_model_config['background_frequency'] = background_frequency -dictionary_model_config['cycle_dir'] = cycle_dir -dictionary_model_config['experiment_dir'] = experiment_dir -dictionary_model_config['experiment_id'] = experiment_id -dictionary_model_config['horizontal_resolution'] = horizontal_resolution -dictionary_model_config['local_background_time_iso'] = local_background_time_iso -dictionary_model_config['local_background_time'] = local_background_time -dictionary_model_config['npx_proc'] = npx_proc -dictionary_model_config['npy_proc'] = npy_proc -dictionary_model_config['swell_static_files'] = swell_static_files -dictionary_model_config['vertical_resolution'] = vertical_resolution -return dictionary_model_config - - -# -------------------------------------------------------------------------------------------------- - - -def template_dictionary_observation_config(background_time, crtm_coeff_dir, cycle_dir, - experiment_id, experiment_root, model_component, - window_begin): + # Dictionary to hold things that can be templated + self.template_dict = {} -dictionary_observation_config = {} -dictionary_observation_config['background_time'] = background_time -dictionary_observation_config['crtm_coeff_dir'] = crtm_coeff_dir -dictionary_observation_config['cycle_dir'] = cycle_dir -dictionary_observation_config['experiment_id'] = experiment_id -dictionary_observation_config['experiment_root'] = experiment_root -dictionary_observation_config['model_component'] = model_component -dictionary_observation_config['window_begin'] = window_begin -return dictionary_observation_config + # Always store the cycle directory in the dictionary + self.template_dict['cycle_dir'] = cycle_dir + # Add the jedi interface to the dictionary + self.jedi_interface = jedi_interface + self.template_dict['model_component'] = jedi_interface -# -------------------------------------------------------------------------------------------------- + # Add experiment info to dictionary + self.template_dict['experiment_id'] = experiment_id + self.template_dict['experiment_root'] = experiment_root + # List of all potential valid keys that can be used in templates + self.valid_template_keys = [ + 'analysis_variables', + 'background_frequency', + 'background_time', + 'crtm_coeff_dir', + 'gradient_norm_reduction', + 'horizontal_resolution', + 'local_background_time', + 'local_background_time_iso', + 'minimizer', + 'npx_proc', + 'npy_proc', + 'number_of_iterations', + 'swell_static_files', + 'vertical_resolution', + 'window_begin', + 'window_begin_iso', + 'window_length', + ] -def __open_file_render_to_dict(logger, config_file, template_dictionary): + # ---------------------------------------------------------------------------------------------- - # Check that config file exists - logger.assert_abort(os.path.exists(config_file), f'In open_file_and_render failed ' - f'to find file \'{config_file}\'') + # Function to add key to the template dictionary + def add_key(self, key, element): - # Open file as a string - with open(config_file, 'r') as config_file_open: - config_file_str_templated = config_file_open.read() + # First assert that key is allowed + self.logger.assert_abort(key in self.valid_template_keys, f'Trying to add key \'{key}\' ' + + f'to jedi config rendering dictionary. But the key is not part ' + + f'of the valid keys: \'{self.valid_template_keys}\'') - # Fill templates in the configuration file using the config - config_file_str = template_string_jinja2(logger, config_file_str_templated, template_dictionary) + # Add element to dictionary + self.template_dict[key] = element - # Convert string to dictionary - return yaml.safe_load(config_file_str) + # ---------------------------------------------------------------------------------------------- + # Open the file at the provided path, use dictionary to complete templates and return dictionary + def __open_file_render_to_dict__(self, config_file): -# -------------------------------------------------------------------------------------------------- + # Check that config file exists + self.logger.assert_abort(os.path.exists(config_file), f'In open_file_and_render failed ' + f'to find file \'{config_file}\'') + # Open file as a string + with open(config_file, 'r') as config_file_open: + config_file_str_templated = config_file_open.read() -def read_render_jedi_interface_oops(logger, exp_config_path, config_name, - template_dictionary): + # Fill templates in the configuration file using the config + config_file_str = template_string_jinja2(self.logger, config_file_str_templated, + self.template_dictionary) - # Path to configuration file - config_file = os.path.join(exp_config_path, 'jedi', 'oops', f'{config_name}.yaml') + # Convert string to dictionary + return yaml.safe_load(config_file_str) - # Render templates in file and return dictionary - return __open_file_render_to_dict(config_file, template_dictionary) + # ---------------------------------------------------------------------------------------------- + # Prepare path to oops file and call rendering + def render_oops_file(self, config_name): -# -------------------------------------------------------------------------------------------------- + # Path to configuration file + config_file = os.path.join(self.jedi_config_path, 'oops', f'{config_name}.yaml') + # Render templates in file and return dictionary + return self.__open_file_render_to_dict__(config_file) -# Method to open a specific configuration file -def read_render_jedi_interface_model(logger, interface, exp_config_path, config_name, - template_dictionary): + # ---------------------------------------------------------------------------------------------- - # Path to configuration file - config_file = os.path.join(exp_config_path, 'jedi', 'interfaces', interface, 'model', - f'{config_name}.yaml') + # Prepare path to interface model file and call rendering + def render_interface_model(self, config_name): - # Render templates in file and return dictionary - return __open_file_render_to_dict(config_file, template_dictionary) + # Assert that there is a jedi interface associated with the task + self.logger.assert_abort(self.jedi_interface is not None, f'In order to render a ' + f'jedi interface config file the task must have an associated' + f'jedi interface.') + # Path to configuration file + config_file = os.path.join(self.jedi_config_path, 'interfaces', self.jedi_interface, + 'model', f'{config_name}.yaml') -# -------------------------------------------------------------------------------------------------- - + # Render templates in file and return dictionary + return self.__open_file_render_to_dict__(config_file) -def read_render_jedi_interface_obs(logger, interface, exp_config_path, config_name, - template_dictionary): + # ---------------------------------------------------------------------------------------------- - # Path to configuration file - config_file = os.path.join(exp_config_path, 'jedi', 'interfaces', interface, 'observations', - f'{config_name}.yaml') + # Prepare path to interface observations file and call rendering + def render_interface_observations(self, config_name): - # Render templates in file and return dictionary - obs_dict = __open_file_render_to_dict(config_file, template_dictionary) + # Assert that there is a jedi interface associated with the task + self.logger.assert_abort(self.jedi_interface is not None, f'In order to render a ' + f'jedi interface config file the task must have an associated' + f'jedi interface.') - # If 4D window then add time interpolation to the dictionary - if window_type == '4D': - obs_dict['get values'] = {} - obs_dict['get values']['time interpolation'] = 'linear' - - return obs_dict - - -# -------------------------------------------------------------------------------------------------- + # Path to configuration file + config_file = os.path.join(self.jedi_config_path, 'interfaces', self.jedi_interface, + 'observations', f'{config_name}.yaml') + # Render templates in file and return dictionary + return self.__open_file_render_to_dict__(config_file) -def read_render_jedi_interface_meta(logger, interface, exp_config_path): + # ---------------------------------------------------------------------------------------------- - # Path to configuration file - config_file = os.path.join(exp_config_path, 'jedi', 'interfaces', interface, - f'{interface}.yaml') + # Prepare path to interface metadata file and call rendering + def render_interface_meta(self): - # Currently no templates in these kinds of files - template_dictionary = {} + # Assert that there is a jedi interface associated with the task + self.logger.assert_abort(self.jedi_interface is not None, f'In order to render a ' + f'jedi interface config file the task must have an associated' + f'jedi interface.') - # Render templates in file and return dictionary - return __open_file_render_to_dict(config_file, template_dictionary) + # Path to configuration file + config_file = os.path.join(self.jedi_config_path, 'interfaces', self.jedi_interface, + f'{self.jedi_interface}.yaml') + # Render templates in file and return dictionary + return self.__open_file_render_to_dict__(config_file) # -------------------------------------------------------------------------------------------------- From 23d90e9c7e4328323dd81251622326c2e39c1f75 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Fri, 19 May 2023 16:50:43 -0400 Subject: [PATCH 014/121] stage tasks working --- src/swell/tasks/base/datetime.py | 34 ++++++++++++++++--- src/swell/tasks/base/task_base.py | 32 +++++++++-------- src/swell/tasks/stage_jedi.py | 6 ++-- src/swell/utilities/date_time.py | 6 ++-- .../utilities/render_jedi_interface_files.py | 2 +- 5 files changed, 54 insertions(+), 26 deletions(-) diff --git a/src/swell/tasks/base/datetime.py b/src/swell/tasks/base/datetime.py index 594cadca..656ad5b5 100644 --- a/src/swell/tasks/base/datetime.py +++ b/src/swell/tasks/base/datetime.py @@ -4,16 +4,22 @@ # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + + import re import datetime as pydatetime + # -------------------------------------------------------------------------------------------------- -# @package datetime -# -# Class containing the datetime -# -# -------------------------------------------------------------------------------------------------- +datetime_formats = { + 'directory_format': '%Y%m%dT%H%M%SZ', # yyyymmddThhMMssZ for directory formats + 'iso_format': '%Y-%m-%dT%H:%M:%SZ' # yyyy-mm-ddThh:MM:ssZ ISO format +} + +# -------------------------------------------------------------------------------------------------- class Datetime: @@ -25,5 +31,23 @@ def __init__(self, datetime_input): # Convert string to datetime object self.datetime = pydatetime.datetime.strptime(datetime_str, '%Y%m%d%H%M%S') + # Datetime formats + self.directory_format = datetime_formats['directory_format'] + self.iso_format = datetime_formats['iso_format'] + + # ---------------------------------------------------------------------------------------------- + + def string_iso(self): + + return self.datetime.strftime(self.iso_format) + + # ---------------------------------------------------------------------------------------------- + + def string_directory(self): + + return self.datetime.strftime(self.directory_format) + + # ---------------------------------------------------------------------------------------------- + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/base/task_base.py b/src/swell/tasks/base/task_base.py index 38f8c4d3..417d2ce5 100644 --- a/src/swell/tasks/base/task_base.py +++ b/src/swell/tasks/base/task_base.py @@ -69,21 +69,25 @@ def __init__(self, config_input, datetime_input, model, task_name): self.__experiment_id__ = self.config_get('experiment_id') self.__suite__ = self.config_get('suite_to_run') + # Create cycle directory + # ---------------------- + cycle_dir = None + if datetime_input is not None: + cycle_dir = self.cycle_dir() + os.makedirs(cycle_dir, 0o755, exist_ok=True) + + # Add JEDI config rendering helper + # -------------------------------- + self.jedi_rendering = JediConfigRendering(self.logger, self.__experiment_root__, + self.__experiment_id__, cycle_dir, self.__model__) + # Create some extra helpers available when the datetime is present # ---------------------------------------------------------------- if self.__datetime__ is not None: - # Create the cycle directory - cycle_dir = self.get_cycle_dir() - os.makedirs(cycle_dir, 0o755, exist_ok=True) - - # Jedi config file rendering - self.jedi_rendering = JediConfigRendering(self.logger, self.__experiment_root__, - self.__experiment_id__, cycle_dir, - self.__model__) - # Object for computing data assimilation window parameters - self.da_window_params = DataAssimilationWindowParams(self.logger, self.__datetime__) + self.da_window_params = DataAssimilationWindowParams(self.logger, + self.__datetime__.string_iso()) # ---------------------------------------------------------------------------------------------- @@ -158,11 +162,9 @@ def cycle_dir(self): self.logger.assert_abort(self.__model__ is not None, 'In get_cycle_dir but this ' + 'should not be called if the task does not receive model.') - # Get the current cycle in directory format - current_cycle_dir_format = self.__datetime__.strftime(datetime_formats['dir_format']) - - # Combine with the model - cycle_dir = os.path.join(current_cycle_dir_format, self.__model__) + # Combine datetime string (directory format) with the model + cycle_dir = os.path.join(self.experiment_path(), 'run', self.__datetime__.string_directory(), + self.__model__) # Return return cycle_dir diff --git a/src/swell/tasks/stage_jedi.py b/src/swell/tasks/stage_jedi.py index 9c3041d8..975d76c6 100644 --- a/src/swell/tasks/stage_jedi.py +++ b/src/swell/tasks/stage_jedi.py @@ -51,8 +51,10 @@ def execute(self): # Check for presence of stage file # -------------------------------- - stage_file = os.path.join(self.experiment_config_path(), self.get_model(), stage_file) - if not os.path.exists(stage_file): + stage_pathfile = os.path.join(self.experiment_config_path(), 'jedi', 'interfaces', + self.get_model(), 'model', stage_file + '.yaml') + print(stage_file) + if not os.path.exists(stage_pathfile): self.logger.info('No stage dictionary was found for this configuration') exit(0) diff --git a/src/swell/utilities/date_time.py b/src/swell/utilities/date_time.py index 76d0cf28..8d6ee2d5 100644 --- a/src/swell/utilities/date_time.py +++ b/src/swell/utilities/date_time.py @@ -17,8 +17,8 @@ # Swell datetime format (yyyymmddThhMMssZ) datetime_formats = { - 'dir_format': "%Y%m%dT%H%M%SZ" # yyyymmddThhMMssZ for directory formats - 'iso_format': "%Y-%m-%dT%H:%M:%SZ" # yyyy-mm-ddThh:MM:ssZ ISO format + 'dir_format': "%Y%m%dT%H%M%SZ", # yyyymmddThhMMssZ for directory formats + 'iso_format': "%Y-%m-%dT%H:%M:%SZ" # yyyy-mm-ddThh:MM:ssZ ISO format } # -------------------------------------------------------------------------------------------------- @@ -37,7 +37,7 @@ def __init__(self, logger, cycle_time): # Current cycle datetime object self.__current_cycle_dto__ = datetime.datetime.strptime(cycle_time, - datetime_formats['dir_format']) + datetime_formats['iso_format']) # ---------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/render_jedi_interface_files.py b/src/swell/utilities/render_jedi_interface_files.py index 3d005c1f..c7e4d709 100644 --- a/src/swell/utilities/render_jedi_interface_files.py +++ b/src/swell/utilities/render_jedi_interface_files.py @@ -89,7 +89,7 @@ def __open_file_render_to_dict__(self, config_file): # Fill templates in the configuration file using the config config_file_str = template_string_jinja2(self.logger, config_file_str_templated, - self.template_dictionary) + self.template_dict) # Convert string to dictionary return yaml.safe_load(config_file_str) From eda29db45f5221aa640a20ac407a782b0c9bb7d1 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Mon, 22 May 2023 17:10:44 -0400 Subject: [PATCH 015/121] unify copyright date --- src/swell/__init__.py | 2 +- src/swell/configuration/__init__.py | 2 +- src/swell/deployment/__init__.py | 2 +- src/swell/deployment/bin/__init__.py | 2 +- .../deployment/bin/swell_create_experiment.py | 2 +- .../deployment/bin/swell_launch_experiment.py | 2 +- .../deployment/bin/swell_prepare_config.py | 2 +- .../deployment/bin/swell_sat_db_processing.py | 2 +- src/swell/deployment/platforms/__init__.py | 2 +- .../deployment/platforms/generic/__init__.py | 2 +- .../platforms/nccs_discover/__init__.py | 2 +- src/swell/deployment/prep_config.py | 2 +- src/swell/deployment/prep_config_base.py | 2 +- src/swell/deployment/prep_config_cli.py | 2 +- src/swell/deployment/prep_config_defaults.py | 2 +- src/swell/deployment/prep_exp_dirs.py | 2 +- src/swell/deployment/prep_suite.py | 2 +- src/swell/suites/3dvar/flow.cylc | 2 +- src/swell/suites/__init__.py | 2 +- src/swell/suites/build_geos/flow.cylc | 2 +- src/swell/suites/build_jedi/flow.cylc | 2 +- src/swell/suites/forecast_geos/flow.cylc | 2 +- src/swell/suites/hofx/flow.cylc | 2 +- src/swell/swell_path.py | 2 +- src/swell/tasks/__init__.py | 2 +- src/swell/tasks/base/__init__.py | 2 +- .../tasks/base/run_jedi_executable_base.py | 2 +- src/swell/tasks/base/task_base.py | 42 +++++++++++-------- src/swell/tasks/build_geos.py | 2 +- src/swell/tasks/build_geos_by_linking.py | 2 +- src/swell/tasks/build_jedi.py | 2 +- src/swell/tasks/build_jedi_by_linking.py | 2 +- src/swell/tasks/clean_cycle.py | 2 +- src/swell/tasks/clone_geos.py | 2 +- src/swell/tasks/clone_jedi.py | 2 +- src/swell/tasks/eva_jedi_log.py | 2 +- src/swell/tasks/eva_observations.py | 6 ++- src/swell/tasks/generate_b_climatology.py | 2 +- .../generate_b_climatology_by_linking.py | 2 +- src/swell/tasks/get_background.py | 17 ++++---- .../tasks/get_background_geos_experiment.py | 4 +- src/swell/tasks/get_observations.py | 19 +++++---- src/swell/tasks/get_restart.py | 2 +- src/swell/tasks/prep_geos_run_dir.py | 2 +- src/swell/tasks/run_geos_executable.py | 2 +- src/swell/tasks/run_jedi_hofx_executable.py | 2 +- .../tasks/run_jedi_variational_executable.py | 2 +- src/swell/tasks/save_obs_diags.py | 6 ++- src/swell/tasks/save_restart.py | 2 +- src/swell/tasks/stage_jedi.py | 4 +- src/swell/tasks/store_background.py | 4 +- src/swell/utilities/__init__.py | 2 +- src/swell/utilities/bin/__init__.py | 2 +- src/swell/utilities/build.py | 2 +- .../utils.py => utilities/case_switching.py} | 14 +++---- src/swell/{tasks/base => utilities}/config.py | 0 ....py => data_assimilation_window_params.py} | 16 ++----- .../{tasks/base => utilities}/datetime.py | 14 ++++--- src/swell/utilities/dictionary.py | 2 +- src/swell/utilities/exceptions.py | 2 +- src/swell/utilities/file_system_operations.py | 2 +- src/swell/utilities/filehandler.py | 2 +- src/swell/utilities/git_utils.py | 2 +- src/swell/utilities/instr_state_machine.py | 2 +- src/swell/utilities/jinja2.py | 2 +- src/swell/utilities/logger.py | 2 +- src/swell/utilities/observations.py | 2 +- src/swell/utilities/sat_db_utils.py | 3 +- src/swell/utilities/shell_commands.py | 2 +- src/swell/utilities/welcome_message.py | 2 +- 70 files changed, 133 insertions(+), 130 deletions(-) rename src/swell/{tasks/base/utils.py => utilities/case_switching.py} (74%) rename src/swell/{tasks/base => utilities}/config.py (100%) rename src/swell/utilities/{date_time.py => data_assimilation_window_params.py} (88%) rename src/swell/{tasks/base => utilities}/datetime.py (79%) diff --git a/src/swell/__init__.py b/src/swell/__init__.py index 439f968b..9563e872 100644 --- a/src/swell/__init__.py +++ b/src/swell/__init__.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/configuration/__init__.py b/src/swell/configuration/__init__.py index ac1c0bc4..f82722ea 100644 --- a/src/swell/configuration/__init__.py +++ b/src/swell/configuration/__init__.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/deployment/__init__.py b/src/swell/deployment/__init__.py index ac1c0bc4..f82722ea 100644 --- a/src/swell/deployment/__init__.py +++ b/src/swell/deployment/__init__.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/deployment/bin/__init__.py b/src/swell/deployment/bin/__init__.py index ac1c0bc4..f82722ea 100644 --- a/src/swell/deployment/bin/__init__.py +++ b/src/swell/deployment/bin/__init__.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/deployment/bin/swell_create_experiment.py b/src/swell/deployment/bin/swell_create_experiment.py index dbb5da67..3dcb24c5 100644 --- a/src/swell/deployment/bin/swell_create_experiment.py +++ b/src/swell/deployment/bin/swell_create_experiment.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/deployment/bin/swell_launch_experiment.py b/src/swell/deployment/bin/swell_launch_experiment.py index ed8ea7a9..3f36bae6 100644 --- a/src/swell/deployment/bin/swell_launch_experiment.py +++ b/src/swell/deployment/bin/swell_launch_experiment.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/deployment/bin/swell_prepare_config.py b/src/swell/deployment/bin/swell_prepare_config.py index c29970be..d919e425 100644 --- a/src/swell/deployment/bin/swell_prepare_config.py +++ b/src/swell/deployment/bin/swell_prepare_config.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -# (C) Copyright 2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/deployment/bin/swell_sat_db_processing.py b/src/swell/deployment/bin/swell_sat_db_processing.py index 0547372b..46ce3767 100644 --- a/src/swell/deployment/bin/swell_sat_db_processing.py +++ b/src/swell/deployment/bin/swell_sat_db_processing.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/deployment/platforms/__init__.py b/src/swell/deployment/platforms/__init__.py index ac1c0bc4..f82722ea 100644 --- a/src/swell/deployment/platforms/__init__.py +++ b/src/swell/deployment/platforms/__init__.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/deployment/platforms/generic/__init__.py b/src/swell/deployment/platforms/generic/__init__.py index ac1c0bc4..f82722ea 100644 --- a/src/swell/deployment/platforms/generic/__init__.py +++ b/src/swell/deployment/platforms/generic/__init__.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/deployment/platforms/nccs_discover/__init__.py b/src/swell/deployment/platforms/nccs_discover/__init__.py index ac1c0bc4..f82722ea 100644 --- a/src/swell/deployment/platforms/nccs_discover/__init__.py +++ b/src/swell/deployment/platforms/nccs_discover/__init__.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/deployment/prep_config.py b/src/swell/deployment/prep_config.py index 5b9d8671..f6581f0f 100644 --- a/src/swell/deployment/prep_config.py +++ b/src/swell/deployment/prep_config.py @@ -1,4 +1,4 @@ -# (C) Copyright 2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/deployment/prep_config_base.py b/src/swell/deployment/prep_config_base.py index b2900264..bc3f1fa3 100644 --- a/src/swell/deployment/prep_config_base.py +++ b/src/swell/deployment/prep_config_base.py @@ -1,4 +1,4 @@ -# (C) Copyright 2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/deployment/prep_config_cli.py b/src/swell/deployment/prep_config_cli.py index 6900f577..c3b266d6 100644 --- a/src/swell/deployment/prep_config_cli.py +++ b/src/swell/deployment/prep_config_cli.py @@ -1,4 +1,4 @@ -# (C) Copyright 2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/deployment/prep_config_defaults.py b/src/swell/deployment/prep_config_defaults.py index 74650933..ca1eb136 100644 --- a/src/swell/deployment/prep_config_defaults.py +++ b/src/swell/deployment/prep_config_defaults.py @@ -1,4 +1,4 @@ -# (C) Copyright 2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/deployment/prep_exp_dirs.py b/src/swell/deployment/prep_exp_dirs.py index f10275c2..9489a8af 100644 --- a/src/swell/deployment/prep_exp_dirs.py +++ b/src/swell/deployment/prep_exp_dirs.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/deployment/prep_suite.py b/src/swell/deployment/prep_suite.py index 686a3514..157abcd6 100644 --- a/src/swell/deployment/prep_suite.py +++ b/src/swell/deployment/prep_suite.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/suites/3dvar/flow.cylc b/src/swell/suites/3dvar/flow.cylc index f556ffcf..9c103cd4 100644 --- a/src/swell/suites/3dvar/flow.cylc +++ b/src/swell/suites/3dvar/flow.cylc @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/suites/__init__.py b/src/swell/suites/__init__.py index ac1c0bc4..f82722ea 100644 --- a/src/swell/suites/__init__.py +++ b/src/swell/suites/__init__.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/suites/build_geos/flow.cylc b/src/swell/suites/build_geos/flow.cylc index 8418af3e..0556d737 100644 --- a/src/swell/suites/build_geos/flow.cylc +++ b/src/swell/suites/build_geos/flow.cylc @@ -1,4 +1,4 @@ -# (C) Copyright 2023 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/suites/build_jedi/flow.cylc b/src/swell/suites/build_jedi/flow.cylc index 4612aaa7..a3c3cef2 100644 --- a/src/swell/suites/build_jedi/flow.cylc +++ b/src/swell/suites/build_jedi/flow.cylc @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/suites/forecast_geos/flow.cylc b/src/swell/suites/forecast_geos/flow.cylc index 04472efe..22a8ec33 100644 --- a/src/swell/suites/forecast_geos/flow.cylc +++ b/src/swell/suites/forecast_geos/flow.cylc @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/suites/hofx/flow.cylc b/src/swell/suites/hofx/flow.cylc index 12613e41..41369ddc 100644 --- a/src/swell/suites/hofx/flow.cylc +++ b/src/swell/suites/hofx/flow.cylc @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/swell_path.py b/src/swell/swell_path.py index d1adeb09..357237ff 100644 --- a/src/swell/swell_path.py +++ b/src/swell/swell_path.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/tasks/__init__.py b/src/swell/tasks/__init__.py index ac1c0bc4..f82722ea 100644 --- a/src/swell/tasks/__init__.py +++ b/src/swell/tasks/__init__.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/tasks/base/__init__.py b/src/swell/tasks/base/__init__.py index ac1c0bc4..f82722ea 100644 --- a/src/swell/tasks/base/__init__.py +++ b/src/swell/tasks/base/__init__.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/tasks/base/run_jedi_executable_base.py b/src/swell/tasks/base/run_jedi_executable_base.py index f69f978b..fe2d2b52 100644 --- a/src/swell/tasks/base/run_jedi_executable_base.py +++ b/src/swell/tasks/base/run_jedi_executable_base.py @@ -99,7 +99,7 @@ def generate_jedi_config(self, jedi_application, window_type): local_background_time_iso = self.da_window_params.local_background_time_iso(window_offset, window_type) window_begin = self.da_window_params.window_begin(window_offset) - window_begin_iso = self.da_window_params.window_begin(window_offset) + window_begin_iso = self.da_window_params.window_begin_iso(window_offset) # Add config to template rendering dictionary self.jedi_rendering.add_key('analysis_variables', analysis_variables) diff --git a/src/swell/tasks/base/task_base.py b/src/swell/tasks/base/task_base.py index 417d2ce5..8575cb4a 100644 --- a/src/swell/tasks/base/task_base.py +++ b/src/swell/tasks/base/task_base.py @@ -18,14 +18,13 @@ import time import yaml -# local imports -from swell.tasks.base.config import Config -from swell.tasks.base.datetime import Datetime -from swell.utilities.date_time import datetime_formats -from swell.utilities.logger import Logger +# swell imports from swell.tasks.base.task_registry import valid_tasks -from swell.tasks.base.utils import camelcase_to_underscore -from swell.utilities.date_time import DataAssimilationWindowParams +from swell.utilities.case_switching import camel_case_to_snake_case +from swell.utilities.config import Config +from swell.utilities.data_assimilation_window_params import DataAssimilationWindowParams +from swell.utilities.datetime import Datetime +from swell.utilities.logger import Logger from swell.utilities.render_jedi_interface_files import JediConfigRendering @@ -149,13 +148,6 @@ def is_datetime_dependent(self): # ---------------------------------------------------------------------------------------------- - def cycle_time(self): - self.logger.assert_abort(self.__datetime__ is not None, f'Cannot obtain the cycle time' + - f'when the task was not initialized with the cycle time.') - return self.__datetime__ - - # ---------------------------------------------------------------------------------------------- - def cycle_dir(self): # Check that model is set @@ -163,12 +155,26 @@ def cycle_dir(self): 'should not be called if the task does not receive model.') # Combine datetime string (directory format) with the model - cycle_dir = os.path.join(self.experiment_path(), 'run', self.__datetime__.string_directory(), - self.__model__) + cycle_dir = os.path.join(self.experiment_path(), 'run', + self.__datetime__.string_directory(), self.__model__) # Return return cycle_dir + # ---------------------------------------------------------------------------------------------- + + def cycle_time_dto(self): + + return self.__datetime__.dto() + + # ---------------------------------------------------------------------------------------------- + + def cycle_time(self): + + return self.__datetime__.string_iso() + + # ---------------------------------------------------------------------------------------------- + # -------------------------------------------------------------------------------------------------- @@ -176,8 +182,8 @@ class taskFactory(): def create_task(self, task, config, datetime, model): - # Convert capitilized string to one with underscores - task_lower = camelcase_to_underscore(task) + # Convert camel case string to snake case + task_lower = camel_case_to_snake_case(task) # Import class based on user selected task task_class = getattr(importlib.import_module('swell.tasks.'+task_lower), task) diff --git a/src/swell/tasks/build_geos.py b/src/swell/tasks/build_geos.py index 186ece23..653eb498 100644 --- a/src/swell/tasks/build_geos.py +++ b/src/swell/tasks/build_geos.py @@ -1,4 +1,4 @@ -# (C) Copyright 2023 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/tasks/build_geos_by_linking.py b/src/swell/tasks/build_geos_by_linking.py index deb08a10..801b0820 100644 --- a/src/swell/tasks/build_geos_by_linking.py +++ b/src/swell/tasks/build_geos_by_linking.py @@ -1,4 +1,4 @@ -# (C) Copyright 2023 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/tasks/build_jedi.py b/src/swell/tasks/build_jedi.py index 575e8f75..d688e77f 100644 --- a/src/swell/tasks/build_jedi.py +++ b/src/swell/tasks/build_jedi.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/tasks/build_jedi_by_linking.py b/src/swell/tasks/build_jedi_by_linking.py index 74dc2ffd..1f789c7b 100644 --- a/src/swell/tasks/build_jedi_by_linking.py +++ b/src/swell/tasks/build_jedi_by_linking.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/tasks/clean_cycle.py b/src/swell/tasks/clean_cycle.py index 6a4b2fce..e8207901 100644 --- a/src/swell/tasks/clean_cycle.py +++ b/src/swell/tasks/clean_cycle.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/tasks/clone_geos.py b/src/swell/tasks/clone_geos.py index 5aed293f..4c2f715d 100644 --- a/src/swell/tasks/clone_geos.py +++ b/src/swell/tasks/clone_geos.py @@ -1,4 +1,4 @@ -# (C) Copyright 2023 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/tasks/clone_jedi.py b/src/swell/tasks/clone_jedi.py index 0d122276..818f266e 100644 --- a/src/swell/tasks/clone_jedi.py +++ b/src/swell/tasks/clone_jedi.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/tasks/eva_jedi_log.py b/src/swell/tasks/eva_jedi_log.py index d618e0f0..84563b90 100644 --- a/src/swell/tasks/eva_jedi_log.py +++ b/src/swell/tasks/eva_jedi_log.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2023 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/tasks/eva_observations.py b/src/swell/tasks/eva_observations.py index 69c84b0d..322c2d88 100644 --- a/src/swell/tasks/eva_observations.py +++ b/src/swell/tasks/eva_observations.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # This software is licensed under the terms of the Apache Licence Version 2.0 @@ -27,13 +27,15 @@ def execute(self): # Parse config for jedi_config # ---------------------------- - background_time = self.config_get('background_time') + background_time_offset = self.config_get('background_time_offset') crtm_coeff_dir = self.config_get('crtm_coeff_dir', None) observations = self.config_get('observations') window_offset = self.config_get('window_offset') # Compute window beginning time window_begin = self.da_window_params.window_begin(window_offset) + background_time = self.da_window_params.background_time(window_offset, + background_time_offset) # Create JEDI interface config templates dictionary self.jedi_rendering.add_key('background_time', background_time) diff --git a/src/swell/tasks/generate_b_climatology.py b/src/swell/tasks/generate_b_climatology.py index 93ecc23f..fca199ba 100644 --- a/src/swell/tasks/generate_b_climatology.py +++ b/src/swell/tasks/generate_b_climatology.py @@ -1,4 +1,4 @@ -# (C) Copyright 2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/tasks/generate_b_climatology_by_linking.py b/src/swell/tasks/generate_b_climatology_by_linking.py index 1c1dd5e4..29f49d1b 100644 --- a/src/swell/tasks/generate_b_climatology_by_linking.py +++ b/src/swell/tasks/generate_b_climatology_by_linking.py @@ -1,4 +1,4 @@ -# (C) Copyright 2023 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/tasks/get_background.py b/src/swell/tasks/get_background.py index 7f67548b..cc2e0cf0 100644 --- a/src/swell/tasks/get_background.py +++ b/src/swell/tasks/get_background.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 @@ -9,7 +9,6 @@ from swell.tasks.base.task_base import taskBase -from swell.utilities.date_time import datetime_formats from datetime import datetime as dt import isodate @@ -40,11 +39,6 @@ def execute(self): See the taskBase constructor for more information. """ - # Current cycle time object - # ------------------------- - current_cycle = self.config_get('current_cycle') - current_cycle_dto = dt.strptime(current_cycle, datetime_formats['dir_format']) - # Get duration into forecast for first background file # ---------------------------------------------------- bkg_steps = [] @@ -94,8 +88,11 @@ def execute(self): self.logger.abort('Window length not divisible by background frequency') # Loop over window - start_date = current_cycle_dto - window_offset_dur - final_date = current_cycle_dto + window_offset_dur + print('self.cycle_time_dto()', self.cycle_time_dto()) + print('window_offset_dur', window_offset_dur) + + start_date = self.cycle_time_dto() - window_offset_dur + final_date = self.cycle_time_dto() + window_offset_dur loop_date = start_date + bkg_freq_dur @@ -106,7 +103,7 @@ def execute(self): # Get the forecast start time # --------------------------- - forecast_start_time = current_cycle_dto - window_length_dur + forecast_offset_dur + forecast_start_time = self.cycle_time_dto() - window_length_dur + forecast_offset_dur # Get name of this model component # -------------------------------- diff --git a/src/swell/tasks/get_background_geos_experiment.py b/src/swell/tasks/get_background_geos_experiment.py index 44143e87..ce6159d8 100644 --- a/src/swell/tasks/get_background_geos_experiment.py +++ b/src/swell/tasks/get_background_geos_experiment.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 @@ -14,7 +14,7 @@ import tarfile from swell.tasks.base.task_base import taskBase -from swell.utilities.date_time import datetime_formats +from swell.tasks.base.datetime import datetime_formats # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/get_observations.py b/src/swell/tasks/get_observations.py index 771312b1..55b66c40 100644 --- a/src/swell/tasks/get_observations.py +++ b/src/swell/tasks/get_observations.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 @@ -35,10 +35,9 @@ def execute(self): # Parse config # ------------ - experiment = self.config_get('obs_experiment') - provider = self.config_get('obs_provider') - window_begin = self.config_get('window_begin') - background_time = self.config_get('background_time') + obs_experiment = self.config_get('obs_experiment') + obs_provider = self.config_get('obs_provider') + background_time_offset = self.config_get('background_time_offset') observations = self.config_get('observations') window_length = self.config_get('window_length') crtm_coeff_dir = self.config_get('crtm_coeff_dir', None) @@ -46,6 +45,8 @@ def execute(self): # Get window begin time window_begin = self.da_window_params.window_begin(window_offset) + background_time = self.da_window_params.background_time(window_offset, + background_time_offset) # Add to JEDI template rendering dictionary self.jedi_rendering.add_key('background_time', background_time) @@ -67,11 +68,11 @@ def execute(self): fetch(date=window_begin, target_file=target_file, - provider=provider, + provider=obs_provider, obs_type=observation, time_window=window_length, type='ob', - experiment=experiment) + experiment=obs_experiment) # Change permission os.chmod(target_file, 0o644) @@ -90,7 +91,7 @@ def execute(self): provider='gsi', obs_type=observation, type='bc', - experiment=experiment, + experiment=obs_experiment, file_type='satbias') # Change permission @@ -106,7 +107,7 @@ def execute(self): provider='gsi', obs_type=observation, type='bc', - experiment=experiment, + experiment=obs_experiment, file_type='tlapse') # Change permission diff --git a/src/swell/tasks/get_restart.py b/src/swell/tasks/get_restart.py index b83be850..7c0710bf 100644 --- a/src/swell/tasks/get_restart.py +++ b/src/swell/tasks/get_restart.py @@ -1,4 +1,4 @@ -# (C) Copyright 2023 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/tasks/prep_geos_run_dir.py b/src/swell/tasks/prep_geos_run_dir.py index 0ab1cd8e..8e706e29 100644 --- a/src/swell/tasks/prep_geos_run_dir.py +++ b/src/swell/tasks/prep_geos_run_dir.py @@ -1,4 +1,4 @@ -# (C) Copyright 2023 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/tasks/run_geos_executable.py b/src/swell/tasks/run_geos_executable.py index 04745d35..441033f8 100644 --- a/src/swell/tasks/run_geos_executable.py +++ b/src/swell/tasks/run_geos_executable.py @@ -1,4 +1,4 @@ -# (C) Copyright 2023 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/tasks/run_jedi_hofx_executable.py b/src/swell/tasks/run_jedi_hofx_executable.py index 0db49b69..6b67a658 100644 --- a/src/swell/tasks/run_jedi_hofx_executable.py +++ b/src/swell/tasks/run_jedi_hofx_executable.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/tasks/run_jedi_variational_executable.py b/src/swell/tasks/run_jedi_variational_executable.py index 4749502c..e062a074 100644 --- a/src/swell/tasks/run_jedi_variational_executable.py +++ b/src/swell/tasks/run_jedi_variational_executable.py @@ -1,4 +1,4 @@ -# (C) Copyright 2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/tasks/save_obs_diags.py b/src/swell/tasks/save_obs_diags.py index fb9724ce..644a625b 100644 --- a/src/swell/tasks/save_obs_diags.py +++ b/src/swell/tasks/save_obs_diags.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # This software is licensed under the terms of the Apache Licence Version 2.0 @@ -24,13 +24,15 @@ def execute(self): # Parse config # ------------ - background_time = self.config_get('background_time') + background_time_offset = self.config_get('background_time_offset') crtm_coeff_dir = self.config_get('crtm_coeff_dir', None) observations = self.config_get('observations') window_offset = self.config_get('window_offset') # Get window beginning window_begin = self.da_window_params.window_begin(window_offset) + background_time = self.da_window_params.background_time(window_offset, + background_time_offset) # Create templates dictionary self.jedi_rendering.add_key('background_time', background_time) diff --git a/src/swell/tasks/save_restart.py b/src/swell/tasks/save_restart.py index 0ab016eb..8a880a83 100644 --- a/src/swell/tasks/save_restart.py +++ b/src/swell/tasks/save_restart.py @@ -1,4 +1,4 @@ -# (C) Copyright 2023 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/tasks/stage_jedi.py b/src/swell/tasks/stage_jedi.py index 975d76c6..bc5d0b82 100644 --- a/src/swell/tasks/stage_jedi.py +++ b/src/swell/tasks/stage_jedi.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 @@ -53,7 +53,7 @@ def execute(self): # -------------------------------- stage_pathfile = os.path.join(self.experiment_config_path(), 'jedi', 'interfaces', self.get_model(), 'model', stage_file + '.yaml') - print(stage_file) + if not os.path.exists(stage_pathfile): self.logger.info('No stage dictionary was found for this configuration') exit(0) diff --git a/src/swell/tasks/store_background.py b/src/swell/tasks/store_background.py index 945b9b6a..3a751d4f 100644 --- a/src/swell/tasks/store_background.py +++ b/src/swell/tasks/store_background.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 @@ -16,7 +16,7 @@ from swell.tasks.base.task_base import taskBase -from swell.utilities.date_time import datetime_formats +from swell.tasks.base.datetime import datetime_formats # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/__init__.py b/src/swell/utilities/__init__.py index ac1c0bc4..f82722ea 100644 --- a/src/swell/utilities/__init__.py +++ b/src/swell/utilities/__init__.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/utilities/bin/__init__.py b/src/swell/utilities/bin/__init__.py index ac1c0bc4..f82722ea 100644 --- a/src/swell/utilities/bin/__init__.py +++ b/src/swell/utilities/bin/__init__.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/utilities/build.py b/src/swell/utilities/build.py index 84308c69..532a6d05 100644 --- a/src/swell/utilities/build.py +++ b/src/swell/utilities/build.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/tasks/base/utils.py b/src/swell/utilities/case_switching.py similarity index 74% rename from src/swell/tasks/base/utils.py rename to src/swell/utilities/case_switching.py index 1aaf8b32..c74eb600 100644 --- a/src/swell/tasks/base/utils.py +++ b/src/swell/utilities/case_switching.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 @@ -8,13 +8,13 @@ # -------------------------------------------------------------------------------------------------- -def camelcase_to_underscore(CamelCaseString): +def camel_case_to_snake_case(CamelCaseString): # Convert a string that looks like e.g. ThisIsAString to this_is_a_string # ----------------------------------------------------------------------- # Create empty output string - underscore_string = '' + snake_case_string = '' # Loop over the elements in the string for element in CamelCaseString: @@ -26,13 +26,13 @@ def camelcase_to_underscore(CamelCaseString): new_element = element # Add new element to the output string - underscore_string = underscore_string+new_element + snake_case_string = snake_case_string+new_element # If this results in leading underscore then remove it - if underscore_string[0] == "_": - underscore_string = underscore_string[1:] + if snake_case_string[0] == "_": + snake_case_string = snake_case_string[1:] - return underscore_string + return snake_case_string # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/base/config.py b/src/swell/utilities/config.py similarity index 100% rename from src/swell/tasks/base/config.py rename to src/swell/utilities/config.py diff --git a/src/swell/utilities/date_time.py b/src/swell/utilities/data_assimilation_window_params.py similarity index 88% rename from src/swell/utilities/date_time.py rename to src/swell/utilities/data_assimilation_window_params.py index 8d6ee2d5..13ab1ec8 100644 --- a/src/swell/utilities/date_time.py +++ b/src/swell/utilities/data_assimilation_window_params.py @@ -10,16 +10,8 @@ import datetime import isodate +from swell.utilities.datetime import datetime_formats -# -------------------------------------------------------------------------------------------------- - - -# Swell datetime format (yyyymmddThhMMssZ) - -datetime_formats = { - 'dir_format': "%Y%m%dT%H%M%SZ", # yyyymmddThhMMssZ for directory formats - 'iso_format': "%Y-%m-%dT%H:%M:%SZ" # yyyy-mm-ddThh:MM:ssZ ISO format -} # -------------------------------------------------------------------------------------------------- @@ -63,7 +55,7 @@ def __get_local_background_time__(self, window_type, window_offset): def window_begin(self, window_offset): window_begin_dto = self.__get_window_begin_dto__(window_offset) - return window_begin_dto.strftime(datetime_formats['dir_format']) + return window_begin_dto.strftime(datetime_formats['directory_format']) # ---------------------------------------------------------------------------------------------- @@ -78,7 +70,7 @@ def background_time(self, window_offset, background_time_offset): background_time_offset_dur = isodate.parse_duration(background_time_offset) background_time_dto = self.__current_cycle_dto__ - background_time_offset_dur - return background_time_dto.strftime(datetime_formats['dir_format']) + return background_time_dto.strftime(datetime_formats['directory_format']) # ---------------------------------------------------------------------------------------------- @@ -92,6 +84,6 @@ def local_background_time_iso(self, window_offset, window_type): def local_background_time(self, window_offset, window_type): local_background_time = self.__get_local_background_time__(window_type, window_offset) - return local_background_time.strftime(datetime_formats['dir_format']) + return local_background_time.strftime(datetime_formats['directory_format']) # ---------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/base/datetime.py b/src/swell/utilities/datetime.py similarity index 79% rename from src/swell/tasks/base/datetime.py rename to src/swell/utilities/datetime.py index 656ad5b5..bb8ffdd0 100644 --- a/src/swell/tasks/base/datetime.py +++ b/src/swell/utilities/datetime.py @@ -29,23 +29,25 @@ def __init__(self, datetime_input): datetime_str = re.sub('[^0-9]', '', datetime_input+'000000')[0:14] # Convert string to datetime object - self.datetime = pydatetime.datetime.strptime(datetime_str, '%Y%m%d%H%M%S') + self.__datetime__ = pydatetime.datetime.strptime(datetime_str, '%Y%m%d%H%M%S') - # Datetime formats - self.directory_format = datetime_formats['directory_format'] - self.iso_format = datetime_formats['iso_format'] + # ---------------------------------------------------------------------------------------------- + + def dto(self): + + return self.__datetime__ # ---------------------------------------------------------------------------------------------- def string_iso(self): - return self.datetime.strftime(self.iso_format) + return self.__datetime__.strftime(datetime_formats['iso_format']) # ---------------------------------------------------------------------------------------------- def string_directory(self): - return self.datetime.strftime(self.directory_format) + return self.__datetime__.strftime(datetime_formats['directory_format']) # ---------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/dictionary.py b/src/swell/utilities/dictionary.py index 7bfeed7a..9b827705 100644 --- a/src/swell/utilities/dictionary.py +++ b/src/swell/utilities/dictionary.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/utilities/exceptions.py b/src/swell/utilities/exceptions.py index c2f766b6..b0031d08 100644 --- a/src/swell/utilities/exceptions.py +++ b/src/swell/utilities/exceptions.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/utilities/file_system_operations.py b/src/swell/utilities/file_system_operations.py index 516f14ad..830665c6 100644 --- a/src/swell/utilities/file_system_operations.py +++ b/src/swell/utilities/file_system_operations.py @@ -1,4 +1,4 @@ -# (C) Copyright 2023 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/utilities/filehandler.py b/src/swell/utilities/filehandler.py index 40a3eb88..6d32c5d1 100755 --- a/src/swell/utilities/filehandler.py +++ b/src/swell/utilities/filehandler.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/utilities/git_utils.py b/src/swell/utilities/git_utils.py index 9245da28..950073ac 100644 --- a/src/swell/utilities/git_utils.py +++ b/src/swell/utilities/git_utils.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/utilities/instr_state_machine.py b/src/swell/utilities/instr_state_machine.py index 3453e6ee..75586926 100644 --- a/src/swell/utilities/instr_state_machine.py +++ b/src/swell/utilities/instr_state_machine.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/utilities/jinja2.py b/src/swell/utilities/jinja2.py index 6e5e3bb9..7dcbe6fb 100644 --- a/src/swell/utilities/jinja2.py +++ b/src/swell/utilities/jinja2.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/utilities/logger.py b/src/swell/utilities/logger.py index 061d66a1..1d516da1 100644 --- a/src/swell/utilities/logger.py +++ b/src/swell/utilities/logger.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/utilities/observations.py b/src/swell/utilities/observations.py index 961353bd..c03f8505 100644 --- a/src/swell/utilities/observations.py +++ b/src/swell/utilities/observations.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/utilities/sat_db_utils.py b/src/swell/utilities/sat_db_utils.py index 399f7d60..2bb8344d 100644 --- a/src/swell/utilities/sat_db_utils.py +++ b/src/swell/utilities/sat_db_utils.py @@ -1,4 +1,5 @@ -# (C) Copyright 2021 NASA Global Modeling and Assimilation Office +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. diff --git a/src/swell/utilities/shell_commands.py b/src/swell/utilities/shell_commands.py index fe541798..c5ac871e 100644 --- a/src/swell/utilities/shell_commands.py +++ b/src/swell/utilities/shell_commands.py @@ -1,4 +1,4 @@ -# (C) Copyright 2023 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/utilities/welcome_message.py b/src/swell/utilities/welcome_message.py index 62ad354d..2dfc5593 100644 --- a/src/swell/utilities/welcome_message.py +++ b/src/swell/utilities/welcome_message.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 From a28793fcf447571c3ec46cb9129b178308d4387d Mon Sep 17 00:00:00 2001 From: danholdaway Date: Mon, 22 May 2023 17:20:19 -0400 Subject: [PATCH 016/121] pep8 --- src/swell/utilities/datetime.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/swell/utilities/datetime.py b/src/swell/utilities/datetime.py index bb8ffdd0..bc3c9188 100644 --- a/src/swell/utilities/datetime.py +++ b/src/swell/utilities/datetime.py @@ -21,6 +21,7 @@ # -------------------------------------------------------------------------------------------------- + class Datetime: def __init__(self, datetime_input): From 35e574df23092d12db83e07b04c3dd00fe14bb8b Mon Sep 17 00:00:00 2001 From: danholdaway Date: Mon, 22 May 2023 23:23:56 -0400 Subject: [PATCH 017/121] add utility for setting the list of tasks for each question --- setup.py | 3 + src/swell/tasks/questions.yaml | 377 ++++++++++++++++++ .../utilities/bin/list_of_task_questions.py | 124 ++++++ 3 files changed, 504 insertions(+) create mode 100644 src/swell/tasks/questions.yaml create mode 100644 src/swell/utilities/bin/list_of_task_questions.py diff --git a/setup.py b/setup.py index bc4d8452..442bef1d 100644 --- a/setup.py +++ b/setup.py @@ -65,6 +65,9 @@ 'swell_sat_db_processing = swell.deployment.bin.swell_sat_db_processing:main', 'util_check_jedi_interface_templates = \ swell.utilities.bin.check_jedi_interface_templates:main', + 'util_list_of_task_questions = \ + swell.utilities.bin.list_of_task_questions:main', + ], }, ) diff --git a/src/swell/tasks/questions.yaml b/src/swell/tasks/questions.yaml new file mode 100644 index 00000000..54996a31 --- /dev/null +++ b/src/swell/tasks/questions.yaml @@ -0,0 +1,377 @@ +analysis_forecast_window_offset: + default_value: null + models: null + prompt: This is an actual question + tasks: + - store_background + - get_background + type: string + +analysis_variables: + default_value: null + models: null + prompt: Question + tasks: + - run_jedi_executable_base + type: string + +background_error_model: + default_value: null + models: null + prompt: Question + tasks: + - generate_b_climatology + type: string + +background_experiment: + default_value: null + models: null + prompt: Question + tasks: + - get_background + type: string + +background_frequency: + default_value: null + models: null + prompt: Question + tasks: + - get_background + - run_jedi_executable_base + type: string + +background_source: + default_value: null + models: null + prompt: Question + tasks: + - get_background + type: string + +background_time_offset: + default_value: null + models: null + prompt: Question + tasks: + - save_obs_diags + - get_observations + - eva_observations + - run_jedi_executable_base + type: string + +backgrounds: + default_value: null + models: null + prompt: Question + tasks: + - store_background + type: string + +bundles: + default_value: null + models: null + prompt: Question + tasks: + - build_jedi + - clone_jedi + type: string + +clean_patterns: + default_value: null + models: null + prompt: Question + tasks: + - clean_cycle + type: string + +crtm_coeff_dir: + default_value: null + models: null + prompt: Question + tasks: + - save_obs_diags + - get_observations + - eva_observations + - run_jedi_executable_base + type: string + +existing_build_directory: + default_value: null + models: null + prompt: Question + tasks: + - build_jedi_by_linking + type: string + +existing_geos_gcm_build_path: + default_value: null + models: null + prompt: Question + tasks: + - build_geos_by_linking + type: string + +existing_geos_gcm_source_path: + default_value: null + models: null + prompt: Question + tasks: + - clone_geos + type: string + +existing_source_directory: + default_value: null + models: null + prompt: Question + tasks: + - clone_jedi + type: string + +geos_background_restart_offset: + default_value: null + models: null + prompt: Question + tasks: + - get_background_geos_experiment + type: string + +geos_bkg_filename_template: + default_value: null + models: null + prompt: Question + tasks: + - get_background_geos_experiment + type: string + +geos_bkg_tar_filename_template: + default_value: null + models: null + prompt: Question + tasks: + - get_background_geos_experiment + type: string + +geos_build_method: + default_value: null + models: null + prompt: Question + tasks: + - build_geos + - clone_geos + - build_geos_by_linking + type: string + +geos_gcm_tag: + default_value: null + models: null + prompt: Question + tasks: + - clone_geos + type: string + +gradient_norm_reduction: + default_value: null + models: null + prompt: Question + tasks: + - run_jedi_executable_base + type: string + +horizontal_resolution: + default_value: null + models: null + prompt: Question + tasks: + - generate_b_climatology_by_linking + - stage_jedi + - store_background + - generate_b_climatology + - get_background + - run_jedi_executable_base + type: string + +jedi_bkg_filename_template: + default_value: null + models: null + prompt: Question + tasks: + - get_background_geos_experiment + type: string + +jedi_build_method: + default_value: null + models: null + prompt: Question + tasks: + - build_jedi + - clone_jedi + - build_jedi_by_linking + type: string + +minimizer: + default_value: null + models: null + prompt: Question + tasks: + - run_jedi_executable_base + type: string + +model: + default_value: null + models: null + prompt: Question + tasks: + - run_jedi_executable_base + type: string + +model_components: + default_value: null + models: null + prompt: Question + tasks: + - build_jedi + - clone_jedi + type: string + +npx_proc: + default_value: null + models: null + prompt: Question + tasks: + - generate_b_climatology_by_linking + - generate_b_climatology + - run_jedi_hofx_executable + - run_jedi_variational_executable + - run_jedi_executable_base + type: string + +npy_proc: + default_value: null + models: null + prompt: Question + tasks: + - generate_b_climatology_by_linking + - generate_b_climatology + - run_jedi_hofx_executable + - run_jedi_variational_executable + - run_jedi_executable_base + type: string + +number_of_iterations: + default_value: null + models: null + prompt: Question + tasks: + - run_jedi_executable_base + type: string + +obs_experiment: + default_value: null + models: null + prompt: Question + tasks: + - get_observations + type: string + +obs_provider: + default_value: null + models: null + prompt: Question + tasks: + - get_observations + type: string + +observations: + default_value: null + models: null + prompt: Question + tasks: + - save_obs_diags + - get_observations + - eva_observations + - run_jedi_executable_base + type: string + +static_background_error_model: + default_value: null + models: null + prompt: Question + tasks: + - generate_b_climatology_by_linking + type: string + +swell_static_files: + default_value: null + models: null + prompt: Question + tasks: + - generate_b_climatology_by_linking + - stage_jedi + - generate_b_climatology + - run_jedi_executable_base + type: string + +swell_static_files_user: + default_value: null + models: null + prompt: Question + tasks: + - generate_b_climatology_by_linking + type: string + +total_processors: + default_value: null + models: null + prompt: Question + tasks: + - generate_b_climatology + - run_jedi_hofx_executable + - run_jedi_variational_executable + type: string + +vertical_resolution: + default_value: null + models: null + prompt: Question + tasks: + - generate_b_climatology_by_linking + - stage_jedi + - generate_b_climatology + - run_jedi_executable_base + type: string + +window_length: + default_value: null + models: null + prompt: Question + tasks: + - store_background + - get_background + - get_observations + - run_jedi_executable_base + type: string + +window_offset: + default_value: null + models: null + prompt: Question + tasks: + - store_background + - save_obs_diags + - get_background + - get_observations + - eva_observations + - run_jedi_executable_base + type: string + +window_type: + default_value: null + models: null + prompt: Question + tasks: + - store_background + - get_background + - run_jedi_hofx_executable + - run_jedi_variational_executable + type: string + diff --git a/src/swell/utilities/bin/list_of_task_questions.py b/src/swell/utilities/bin/list_of_task_questions.py new file mode 100644 index 00000000..8781a1c7 --- /dev/null +++ b/src/swell/utilities/bin/list_of_task_questions.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python + +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + + +# standard imports +from collections import OrderedDict +import os +import pathlib +import yaml + +# swell imports +from swell.swell_path import get_swell_path +from swell.utilities.logger import Logger + + +# -------------------------------------------------------------------------------------------------- + + +def main(): + + # Create a logger + logger = Logger('ListOfTaskQuestions') + + # Path to JEDI interface code + swell_path = get_swell_path() + jedi_tasks_path = pathlib.Path(os.path.join(swell_path, 'tasks')) + + # All python files + task_codes = jedi_tasks_path.rglob("*py") + + # All lines of all task files + raw_task_code_lines = [] + task_names = [] + + # Output file + outfile_yaml = os.path.join(swell_path, 'tasks', 'questions.yaml') + + # Read input file into dictionary + if os.path.exists(outfile_yaml): + with open(outfile_yaml, 'r') as ymlfile: + question_dict = yaml.safe_load(ymlfile) + else: + question_dict = {} + + for task_code in task_codes: + + if '__init__.py' not in str(task_code) and 'task_base.py' not in str(task_code): + + with open(task_code, 'r') as file: + file_lines = file.read().split('\n') + + for file_line in file_lines: + if 'self.config_get' in file_line: + raw_task_code_lines.append(file_line.split('=')[1].strip()) + task_names.append(os.path.basename(str(task_code)).split('.')[0]) + + # Extract just the config key from the string + config_keys = [] + for raw_task_code_line in raw_task_code_lines: + + config_key = raw_task_code_line.replace('self.config_get(', '') + config_key = config_key.replace(')', '') + config_key = config_key.split(',')[0].strip() + config_key = config_key.split('#')[0].strip() + config_key = config_key.replace('\'', '') + + config_keys.append(config_key) + + # For each key create lists of tasks + unique_keys = sorted(list(set(config_keys))) + + # question to task dictionary + question_to_tasks = {} + + # Output file + outfile_yaml = os.path.join(swell_path, 'tasks', 'questions.yaml') + outfile = open(outfile_yaml, 'w') + + # Task for each key + for unique_key in unique_keys: + + tasks = [] + + for task_name, config_key in zip(task_names, config_keys): + + if unique_key == config_key: + + tasks.append(task_name) + + question_to_tasks = {} + + if unique_key in question_dict: + question_to_tasks[unique_key] = question_dict[unique_key] + else: + question_to_tasks[unique_key] = {} + question_to_tasks[unique_key]['default_value'] = None + question_to_tasks[unique_key]['prompt'] = 'Question' + question_to_tasks[unique_key]['type'] = 'string' + question_to_tasks[unique_key]['models'] = None + + # Regardless of whether question was already in dictionary + question_to_tasks[unique_key]['tasks'] = tasks + + outfile.write(yaml.dump(question_to_tasks, default_flow_style=False)) + outfile.write('\n') + + outfile.close() + + +# -------------------------------------------------------------------------------------------------- + + +if __name__ == '__main__': + main() + + +# -------------------------------------------------------------------------------------------------- From dbb762551191cfbf175489ec5a05d84daa700081 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Tue, 23 May 2023 22:41:24 -0400 Subject: [PATCH 018/121] add question prompts --- src/swell/tasks/base/task_base.py | 9 + src/swell/tasks/build_jedi.py | 2 +- src/swell/tasks/build_jedi_by_linking.py | 8 +- src/swell/tasks/clone_jedi.py | 6 +- src/swell/tasks/get_background.py | 3 +- src/swell/tasks/questions.yaml | 372 +++++++++++------- src/swell/tasks/store_background.py | 11 +- .../utilities/bin/list_of_task_questions.py | 26 +- .../utilities/render_jedi_interface_files.py | 6 +- 9 files changed, 285 insertions(+), 158 deletions(-) diff --git a/src/swell/tasks/base/task_base.py b/src/swell/tasks/base/task_base.py index 8575cb4a..283d81e0 100644 --- a/src/swell/tasks/base/task_base.py +++ b/src/swell/tasks/base/task_base.py @@ -68,6 +68,10 @@ def __init__(self, config_input, datetime_input, model, task_name): self.__experiment_id__ = self.config_get('experiment_id') self.__suite__ = self.config_get('suite_to_run') + # Save the model components + # ------------------------- + self.__model_components__ = self.config_get('model_components', None) + # Create cycle directory # ---------------------- cycle_dir = None @@ -140,6 +144,11 @@ def get_model(self): # ---------------------------------------------------------------------------------------------- + def get_model_components(self): + return self.__model_components__ + + # ---------------------------------------------------------------------------------------------- + def is_datetime_dependent(self): if self.__datetime__ is None: return False diff --git a/src/swell/tasks/build_jedi.py b/src/swell/tasks/build_jedi.py index d688e77f..4558a8c2 100644 --- a/src/swell/tasks/build_jedi.py +++ b/src/swell/tasks/build_jedi.py @@ -40,7 +40,7 @@ def execute(self): if jedi_build_method == 'create': # Determine which bundles need to be build - model_components = self.config_get('model_components', None) + model_components = self.get_model_components() if model_components is not None: bundles = [] for model_component in model_components: diff --git a/src/swell/tasks/build_jedi_by_linking.py b/src/swell/tasks/build_jedi_by_linking.py index 1f789c7b..6089547f 100644 --- a/src/swell/tasks/build_jedi_by_linking.py +++ b/src/swell/tasks/build_jedi_by_linking.py @@ -41,13 +41,13 @@ def execute(self): if jedi_build_method == 'use_existing': # Get the existing build directory from the dictionary - existing_build_directory = self.config_get('existing_build_directory') + existing_jedi_build_directory = self.config_get('existing_jedi_build_directory') # Assert that the existing build directory contains a bin directory - if not os.path.exists(os.path.join(existing_build_directory, 'bin')): + if not os.path.exists(os.path.join(existing_jedi_build_directory, 'bin')): self.logger.abort(f'Existing JEDI build directory is provided but a bin ' + f'directory is not found in the path ' + - f'\'{existing_build_directory}\'') + f'\'{existing_jedi_build_directory}\'') # Write warning to user self.logger.info('Suitable JEDI build found, linking build directory. Warning: ' + @@ -56,7 +56,7 @@ def execute(self): 'this experiment may not be reproducible if the build changes.') # Link the source code directory - link_path(existing_build_directory, jedi_bundle_build_path) + link_path(existing_jedi_build_directory, jedi_bundle_build_path) else: diff --git a/src/swell/tasks/clone_jedi.py b/src/swell/tasks/clone_jedi.py index 818f266e..94572816 100644 --- a/src/swell/tasks/clone_jedi.py +++ b/src/swell/tasks/clone_jedi.py @@ -42,15 +42,15 @@ def execute(self): if jedi_build_method == 'use_existing': # Get the existing bundle directory to get the source code - existing_source_directory = self.config_get('existing_source_directory') + existing_jedi_source_directory = self.config_get('existing_jedi_source_directory') # Link the source code directory - link_path(existing_source_directory, jedi_bundle_source_path) + link_path(existing_jedi_source_directory, jedi_bundle_source_path) elif jedi_build_method == 'create': # Determine which bundles need to be build - model_components = self.config_get('model_components', None) + model_components = self.get_model_components() if model_components is not None: bundles = [] for model_component in model_components: diff --git a/src/swell/tasks/get_background.py b/src/swell/tasks/get_background.py index cc2e0cf0..340cf62f 100644 --- a/src/swell/tasks/get_background.py +++ b/src/swell/tasks/get_background.py @@ -46,7 +46,6 @@ def execute(self): # Parse config background_experiment = self.config_get('background_experiment') background_frequency = self.config_get('background_frequency', None) - background_source = self.config_get('background_source', 'file') forecast_offset = self.config_get('analysis_forecast_window_offset') horizontal_resolution = self.config_get('horizontal_resolution') window_length = self.config_get('window_length') @@ -79,7 +78,7 @@ def execute(self): # If background is provided though files get all backgrounds # ---------------------------------------------------------- - if window_type == "4D" and background_source == 'file': + if window_type == "4D": bkg_freq_dur = isodate.parse_duration(background_frequency) diff --git a/src/swell/tasks/questions.yaml b/src/swell/tasks/questions.yaml index 54996a31..2527bfd4 100644 --- a/src/swell/tasks/questions.yaml +++ b/src/swell/tasks/questions.yaml @@ -1,93 +1,105 @@ analysis_forecast_window_offset: - default_value: null - models: null - prompt: This is an actual question - tasks: - - store_background + ask_question: false + default_value: model() + models: + - all + options: model() + prompt: What is the duration from the middle of the window when forecasts start? + tasks: + - store_backgr:qound - get_background - type: string + type: duration analysis_variables: - default_value: null - models: null - prompt: Question + ask_question: false + default_value: model() + models: + - all + options: model() + prompt: What are the analysis variables? tasks: - run_jedi_executable_base - type: string + type: string-check-list background_error_model: - default_value: null - models: null - prompt: Question + ask_question: false + default_value: model() + models: + - all + options: model() + prompt: Which background error model do you want to use? tasks: - generate_b_climatology - type: string + type: string-drop-list background_experiment: + ask_question: true default_value: null - models: null - prompt: Question + models: + - all + prompt: What is the name of the name of the experiment providing the backgrounds? tasks: - get_background type: string background_frequency: + ask_question: false default_value: null - models: null - prompt: Question + models: + - all + options: null + prompt: What is the frequency of the background files? tasks: - get_background - run_jedi_executable_base - type: string - -background_source: - default_value: null - models: null - prompt: Question - tasks: - - get_background - type: string + type: duration + depends: + key: window_type + value: 4D background_time_offset: + ask_question: false default_value: null - models: null - prompt: Question + models: + - null + options: null + prompt: How long before the middle of the analysis window did the background providing forecast begin? tasks: - save_obs_diags - get_observations - eva_observations - run_jedi_executable_base - type: string - -backgrounds: - default_value: null - models: null - prompt: Question - tasks: - - store_background - type: string + type: duration bundles: + ask_question: true default_value: null - models: null - prompt: Question + models: + - null + options: null + prompt: Which JEDI bundles do you wish to build? tasks: - build_jedi - clone_jedi - type: string + type: string-check-list clean_patterns: + ask_question: false default_value: null - models: null - prompt: Question + models: + - null + options: null + prompt: Provide a list of patterns that you wish to remove from the cycle directory. tasks: - clean_cycle - type: string + type: string-list crtm_coeff_dir: - default_value: null - models: null - prompt: Question + ask_question: false + default_value: platform() + models: + - null + prompt: What is the path to the CRTM coefficient files? tasks: - save_obs_diags - get_observations @@ -95,92 +107,131 @@ crtm_coeff_dir: - run_jedi_executable_base type: string -existing_build_directory: +existing_geos_gcm_build_path: + ask_question: true default_value: null - models: null - prompt: Question + models: + - null + prompt: What is the path to the existing GEOS build directory? tasks: - - build_jedi_by_linking + - build_geos_by_linking type: string + depends: + key: geos_build_method + value: use_existing -existing_geos_gcm_build_path: +existing_jedi_build_directory: + ask_question: true default_value: null - models: null - prompt: Question + models: + - null + prompt: What is the path to the existing JEDI build directory? tasks: - - build_geos_by_linking + - build_jedi_by_linking type: string + depends: + key: jedi_build_method + value: use_existing existing_geos_gcm_source_path: + ask_question: true default_value: null - models: null - prompt: Question + models: + - null + prompt: What is the path to the existing GEOS source code directory? tasks: - clone_geos type: string + depends: + key: geos_build_method + value: use_existing -existing_source_directory: +existing_jedi_source_directory: + ask_question: true default_value: null - models: null - prompt: Question + models: + - null + prompt: What is the path to the existing JEDI source code directory? tasks: - clone_jedi type: string + depends: + key: jedi_build_method + value: use_existing geos_background_restart_offset: + ask_question: true default_value: null - models: null - prompt: Question + models: + - null + options: null + prompt: Duration before the middle of the window used in GEOS restart tar file. tasks: - get_background_geos_experiment - type: string + type: duration geos_bkg_filename_template: + ask_question: true default_value: null - models: null - prompt: Question + models: + - null + prompt: Name of the background files as output by GEOS (with date and time templated). tasks: - get_background_geos_experiment type: string geos_bkg_tar_filename_template: + ask_question: true default_value: null - models: null - prompt: Question + models: + - null + prompt: Name of the GEOS backgrounds tar file (with date and time templated tasks: - get_background_geos_experiment type: string geos_build_method: - default_value: null - models: null - prompt: Question + ask_question: true + default_value: use_existing + models: + - null + options: ['use_existing', 'create'] + prompt: Do you want to use an existing GEOS build or create a new build? tasks: - build_geos - clone_geos - build_geos_by_linking - type: string + type: string-drop-list geos_gcm_tag: - default_value: null - models: null - prompt: Question + ask_question: true + default_value: v11.0.2 + models: + - null + options: None + prompt: Which GEOS tag do you wish to clone? tasks: - clone_geos type: string gradient_norm_reduction: - default_value: null - models: null - prompt: Question + ask_question: false + default_value: model() + models: + - null + options: null + prompt: What value of gradient norm reduction for convergence? tasks: - run_jedi_executable_base - type: string + type: float horizontal_resolution: + ask_question: true default_value: null - models: null - prompt: Question + models: + - null + options: null + prompt: What is the horizontal resolution for the forecast model and backgrounds? tasks: - generate_b_climatology_by_linking - stage_jedi @@ -188,20 +239,26 @@ horizontal_resolution: - generate_b_climatology - get_background - run_jedi_executable_base - type: string + type: string-drop-list jedi_bkg_filename_template: + ask_question: true default_value: null - models: null - prompt: Question + models: + - null + options: null + prompt: What is the name of the background as used by JEDI (with date and time templated) tasks: - get_background_geos_experiment type: string jedi_build_method: - default_value: null - models: null - prompt: Question + ask_question: true + default_value: use_existing + models: + - null + options: ['use_existing', 'create'] + prompt: Do you want to use an existing JEDI build or create a new build? tasks: - build_jedi - clone_jedi @@ -209,101 +266,123 @@ jedi_build_method: type: string minimizer: + ask_question: false default_value: null - models: null - prompt: Question + models: + - null + options: null + prompt: Which data assimilation minimizer do you wish to use? tasks: - run_jedi_executable_base - type: string + type: string-drop-list model: + ask_question: true default_value: null - models: null - prompt: Question + models: + - null + options: null + prompt: What forecast model should be used within JEDI for 4D window propagation? tasks: - run_jedi_executable_base - type: string - -model_components: - default_value: null - models: null - prompt: Question - tasks: - - build_jedi - - clone_jedi - type: string + type: string-drop-list + depends: + key: window_type + value: 4D npx_proc: + ask_question: true default_value: null - models: null - prompt: Question + models: + - null + options: null + prompt: What number of processors do you wish to use in the x-direction? tasks: - generate_b_climatology_by_linking - generate_b_climatology - run_jedi_hofx_executable - run_jedi_variational_executable - run_jedi_executable_base - type: string + type: integer npy_proc: + ask_question: true default_value: null - models: null - prompt: Question + models: + - null + options: null + prompt: What number of processors do you wish to use in the y-direction? tasks: - generate_b_climatology_by_linking - generate_b_climatology - run_jedi_hofx_executable - run_jedi_variational_executable - run_jedi_executable_base - type: string + type: integer number_of_iterations: + ask_question: false default_value: null - models: null - prompt: Question + models: + - null + options: null + prompt: What number of iterations do you wish to use for each outer loop? Provide a list of integers the same length as the number of outer loops. tasks: - run_jedi_executable_base - type: string + type: integer-list obs_experiment: + ask_question: true default_value: null - models: null - prompt: Question + models: + - null + prompt: What is the database providing the observations? tasks: - get_observations type: string obs_provider: + ask_question: true default_value: null - models: null - prompt: Question + models: + - null + options: null + prompt: What is the group providing the observations? tasks: - get_observations type: string observations: + ask_question: true default_value: null - models: null - prompt: Question + models: + - null + options: null + prompt: Which observations do you want to include? tasks: - save_obs_diags - get_observations - eva_observations - run_jedi_executable_base - type: string + type: string-check-list static_background_error_model: + ask_question: true default_value: null - models: null - prompt: Question + models: + - null + options: null + prompt: Which static background error model do you want to use? tasks: - generate_b_climatology_by_linking - type: string + type: string-drop-list swell_static_files: + ask_question: false default_value: null - models: null - prompt: Question + models: + - null + prompt: What is the path to the Swell Static files directory? tasks: - generate_b_climatology_by_linking - stage_jedi @@ -312,17 +391,22 @@ swell_static_files: type: string swell_static_files_user: + ask_question: false default_value: null - models: null - prompt: Question + models: + - null + prompt: What is the path to the user provided Swell Static files directory? tasks: - generate_b_climatology_by_linking type: string total_processors: + ask_question: true default_value: null - models: null - prompt: Question + models: + - null + options: null + prompt: What is the total number of processors for JEDI? tasks: - generate_b_climatology - run_jedi_hofx_executable @@ -330,20 +414,26 @@ total_processors: type: string vertical_resolution: + ask_question: true default_value: null - models: null - prompt: Question + models: + - null + options: null + prompt: What is the vertical resolution for the forecast model and background? tasks: - generate_b_climatology_by_linking - stage_jedi - generate_b_climatology - run_jedi_executable_base - type: string + type: string-drop-list window_length: + ask_question: false default_value: null - models: null - prompt: Question + models: + - null + options: null + prompt: What is the duration for the data assimilation window? tasks: - store_background - get_background @@ -352,9 +442,12 @@ window_length: type: string window_offset: + ask_question: true default_value: null - models: null - prompt: Question + models: + - null + options: null + prompt: What is the duration between the middle of the window and the beginning? tasks: - store_background - save_obs_diags @@ -362,16 +455,19 @@ window_offset: - get_observations - eva_observations - run_jedi_executable_base - type: string + type: duration window_type: - default_value: null - models: null - prompt: Question + ask_question: true + default_value: 4D + models: + - null + options: ['3D', '4D'] + prompt: Do you want to use a 3D or 4D (including FGAT) window? tasks: - store_background - get_background - run_jedi_hofx_executable - run_jedi_variational_executable - type: string + type: string-drop-list diff --git a/src/swell/tasks/store_background.py b/src/swell/tasks/store_background.py index 3a751d4f..6eb01f27 100644 --- a/src/swell/tasks/store_background.py +++ b/src/swell/tasks/store_background.py @@ -46,6 +46,8 @@ def execute(self): window_type = self.config_get('window_type') window_length = self.config_get('window_length') window_offset = self.config_get('window_offset') + background_experiment = self.config_get('background_experiment') + background_frequency = self.config_get('background_frequency') # Position relative to center of the window where forecast starts forecast_offset = self.config_get('analysis_forecast_window_offset') @@ -71,12 +73,9 @@ def execute(self): # If background is provided though files get all backgrounds # ---------------------------------------------------------- - bkg_info = self.config_get('backgrounds') + if window_type == "4D": - if window_type == "4D" and bkg_info['background source'] == 'file': - - bkg_freq = bkg_info['background frequency'] - bkg_freq_dur = isodate.parse_duration(bkg_freq) + bkg_freq_dur = isodate.parse_duration(background_frequency) # Check for a sensible frequency if (window_length_dur/bkg_freq_dur) % 2: @@ -138,4 +137,4 @@ def execute(self): step=bkg_step, resolution=self.config_get('horizontal_resolution'), type='fc', - experiment=bkg_info['background experiment']) + experiment=background_experiment) diff --git a/src/swell/utilities/bin/list_of_task_questions.py b/src/swell/utilities/bin/list_of_task_questions.py index 8781a1c7..39836808 100644 --- a/src/swell/utilities/bin/list_of_task_questions.py +++ b/src/swell/utilities/bin/list_of_task_questions.py @@ -98,12 +98,36 @@ def main(): if unique_key in question_dict: question_to_tasks[unique_key] = question_dict[unique_key] + + question_dict_key = question_dict[unique_key] + + if 'default_value' not in question_dict_key: + question_to_tasks[unique_key]['default_value'] = None + + if 'options' not in question_dict_key: + question_to_tasks[unique_key]['options'] = None + + if 'prompt' not in question_dict_key: + question_to_tasks[unique_key]['prompt'] = 'Question' + + if 'type' not in question_dict_key: + question_to_tasks[unique_key]['type'] = 'string' + + if 'models' not in question_dict_key: + question_to_tasks[unique_key]['models'] = [None] + + if 'ask_question' not in question_dict_key: + question_to_tasks[unique_key]['ask_question'] = True + + else: question_to_tasks[unique_key] = {} question_to_tasks[unique_key]['default_value'] = None + question_to_tasks[unique_key]['options'] = None question_to_tasks[unique_key]['prompt'] = 'Question' question_to_tasks[unique_key]['type'] = 'string' - question_to_tasks[unique_key]['models'] = None + question_to_tasks[unique_key]['models'] = [None] + question_to_tasks[unique_key]['ask_question'] = True # Regardless of whether question was already in dictionary question_to_tasks[unique_key]['tasks'] = tasks diff --git a/src/swell/utilities/render_jedi_interface_files.py b/src/swell/utilities/render_jedi_interface_files.py index c7e4d709..df472347 100644 --- a/src/swell/utilities/render_jedi_interface_files.py +++ b/src/swell/utilities/render_jedi_interface_files.py @@ -142,7 +142,7 @@ def render_interface_observations(self, config_name): # ---------------------------------------------------------------------------------------------- # Prepare path to interface metadata file and call rendering - def render_interface_meta(self): + def render_interface_meta(self, model_component = self.jedi_interface): # Assert that there is a jedi interface associated with the task self.logger.assert_abort(self.jedi_interface is not None, f'In order to render a ' @@ -150,8 +150,8 @@ def render_interface_meta(self): f'jedi interface.') # Path to configuration file - config_file = os.path.join(self.jedi_config_path, 'interfaces', self.jedi_interface, - f'{self.jedi_interface}.yaml') + config_file = os.path.join(self.jedi_config_path, 'interfaces', model_component, + f'{model_component}.yaml') # Render templates in file and return dictionary return self.__open_file_render_to_dict__(config_file) From c54565cb38f89d43142250752b15a03bcb3d6006 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 24 May 2023 12:11:11 -0400 Subject: [PATCH 019/121] add more questions and defaults --- .../oops/{3dvar.yaml => variational3D.yaml} | 0 .../tasks/base/run_jedi_executable_base.py | 70 +------ src/swell/tasks/questions.yaml | 177 +++++++----------- src/swell/tasks/run_jedi_hofx_executable.py | 55 +++++- .../tasks/run_jedi_variational_executable.py | 63 ++++++- .../bin/check_jedi_interface_templates.py | 4 +- 6 files changed, 183 insertions(+), 186 deletions(-) rename src/swell/configuration/jedi/oops/{3dvar.yaml => variational3D.yaml} (100%) diff --git a/src/swell/configuration/jedi/oops/3dvar.yaml b/src/swell/configuration/jedi/oops/variational3D.yaml similarity index 100% rename from src/swell/configuration/jedi/oops/3dvar.yaml rename to src/swell/configuration/jedi/oops/variational3D.yaml diff --git a/src/swell/tasks/base/run_jedi_executable_base.py b/src/swell/tasks/base/run_jedi_executable_base.py index fe2d2b52..73ed5ef6 100644 --- a/src/swell/tasks/base/run_jedi_executable_base.py +++ b/src/swell/tasks/base/run_jedi_executable_base.py @@ -30,7 +30,7 @@ def execute(self): # ---------------------------------------------------------------------------------------------- - def jedi_dictionary_iterator(self, jedi_config_dict, window_type): + def jedi_dictionary_iterator(self, jedi_config_dict, window_type, obs, jedi_forecast_mode): # Assemble configuration YAML file # -------------------------------- @@ -54,7 +54,6 @@ def jedi_dictionary_iterator(self, jedi_config_dict, window_type): value_special = value.replace('SPECIAL', '') if value_special == 'observations': observations = [] - obs = self.config_get('observations') for ob in obs: # Get observation dictionary obs_dict = self.jedi_rendering.render_interface_observations(ob) @@ -62,76 +61,11 @@ def jedi_dictionary_iterator(self, jedi_config_dict, window_type): jedi_config_dict[key] = observations elif value_special == 'model' and window_type == '4D': - model = self.config_get('model') - model_dict = self.jedi_rendering.render_interface_model(model) + model_dict = self.jedi_rendering.render_interface_model(jedi_forecast_model) jedi_config_dict[key] = model_dict # ---------------------------------------------------------------------------------------------- - def generate_jedi_config(self, jedi_application, window_type): - - # Var suite names are handled in variational executable - # ----------------------------------------------------- - if 'var' not in jedi_application: - jedi_application = jedi_application + window_type - - # Build dictionary for rendering JEDI configuration files - analysis_variables = self.config_get('analysis_variables', None) - gradient_norm_reduction = self.config_get('gradient_norm_reduction', None) - minimizer = self.config_get('minimizer', None) - number_of_iterations = self.config_get('number_of_iterations', None) - window_length = self.config_get('window_length', None) - background_frequency = self.config_get('background_frequency', None) - horizontal_resolution = self.config_get('horizontal_resolution', None) - npx_proc = self.config_get('npx_proc', None) - npy_proc = self.config_get('npy_proc', None) - swell_static_files = self.config_get('swell_static_files', None) - vertical_resolution = self.config_get('vertical_resolution', None) - crtm_coeff_dir = self.config_get('crtm_coeff_dir', None) - window_offset = self.config_get('window_offset', None) - background_time_offset = self.config_get('background_time_offset', None) - - # Compute data assimilation window parameters - background_time = self.da_window_params.background_time(window_offset, - background_time_offset) - local_background_time = self.da_window_params.local_background_time(window_offset, - window_type) - local_background_time_iso = self.da_window_params.local_background_time_iso(window_offset, - window_type) - window_begin = self.da_window_params.window_begin(window_offset) - window_begin_iso = self.da_window_params.window_begin_iso(window_offset) - - # Add config to template rendering dictionary - self.jedi_rendering.add_key('analysis_variables', analysis_variables) - self.jedi_rendering.add_key('gradient_norm_reduction', gradient_norm_reduction) - self.jedi_rendering.add_key('minimizer', minimizer) - self.jedi_rendering.add_key('number_of_iterations', number_of_iterations) - self.jedi_rendering.add_key('window_begin_iso', window_begin_iso) - self.jedi_rendering.add_key('window_length', window_length) - self.jedi_rendering.add_key('background_frequency', background_frequency) - self.jedi_rendering.add_key('horizontal_resolution', horizontal_resolution) - self.jedi_rendering.add_key('local_background_time', local_background_time) - self.jedi_rendering.add_key('local_background_time_iso', local_background_time_iso) - self.jedi_rendering.add_key('npx_proc', npx_proc) - self.jedi_rendering.add_key('npy_proc', npy_proc) - self.jedi_rendering.add_key('swell_static_files', swell_static_files) - self.jedi_rendering.add_key('vertical_resolution', vertical_resolution) - self.jedi_rendering.add_key('background_time', background_time) - self.jedi_rendering.add_key('crtm_coeff_dir', crtm_coeff_dir) - self.jedi_rendering.add_key('window_begin', window_begin) - - # Create dictionary from the templated JEDI config file - # ----------------------------------------------------- - jedi_config_dict = self.jedi_rendering.render_oops_file(jedi_application) - - # Read configs for the rest of the dictionary - # ------------------------------------------- - self.jedi_dictionary_iterator(jedi_config_dict, window_type) - - return jedi_config_dict - - # ---------------------------------------------------------------------------------------------- - def run_executable(self, cycle_dir, np, jedi_executable_path, jedi_config_file, output_log): # Run the JEDI executable diff --git a/src/swell/tasks/questions.yaml b/src/swell/tasks/questions.yaml index 2527bfd4..0693f9dd 100644 --- a/src/swell/tasks/questions.yaml +++ b/src/swell/tasks/questions.yaml @@ -1,9 +1,8 @@ analysis_forecast_window_offset: ask_question: false - default_value: model() + default_value: defer_to_model models: - all - options: model() prompt: What is the duration from the middle of the window when forecasts start? tasks: - store_backgr:qound @@ -12,10 +11,9 @@ analysis_forecast_window_offset: analysis_variables: ask_question: false - default_value: model() + default_value: defer_to_model models: - all - options: model() prompt: What are the analysis variables? tasks: - run_jedi_executable_base @@ -23,10 +21,10 @@ analysis_variables: background_error_model: ask_question: false - default_value: model() + default_value: defer_to_model models: - all - options: model() + options: defer_to_model prompt: Which background error model do you want to use? tasks: - generate_b_climatology @@ -34,7 +32,7 @@ background_error_model: background_experiment: ask_question: true - default_value: null + default_value: defer_to_model models: - all prompt: What is the name of the name of the experiment providing the backgrounds? @@ -44,10 +42,9 @@ background_experiment: background_frequency: ask_question: false - default_value: null + default_value: defer_to_model models: - all - options: null prompt: What is the frequency of the background files? tasks: - get_background @@ -59,10 +56,9 @@ background_frequency: background_time_offset: ask_question: false - default_value: null + default_value: defer_to_model models: - - null - options: null + - all prompt: How long before the middle of the analysis window did the background providing forecast begin? tasks: - save_obs_diags @@ -73,10 +69,8 @@ background_time_offset: bundles: ask_question: true - default_value: null - models: - - null - options: null + default_value: ['fv3-jedi', 'soca', 'iodaconv'] + options: ['fv3-jedi', 'soca', 'iodaconv', 'ufo', 'ioda', 'oops', 'saber'] prompt: Which JEDI bundles do you wish to build? tasks: - build_jedi @@ -85,10 +79,9 @@ bundles: clean_patterns: ask_question: false - default_value: null + default_value: defer_to_model models: - - null - options: null + - all prompt: Provide a list of patterns that you wish to remove from the cycle directory. tasks: - clean_cycle @@ -96,9 +89,9 @@ clean_patterns: crtm_coeff_dir: ask_question: false - default_value: platform() + default_value: defer_to_platform models: - - null + - geos_atmosphere prompt: What is the path to the CRTM coefficient files? tasks: - save_obs_diags @@ -109,9 +102,7 @@ crtm_coeff_dir: existing_geos_gcm_build_path: ask_question: true - default_value: null - models: - - null + default_value: defer_to_platform prompt: What is the path to the existing GEOS build directory? tasks: - build_geos_by_linking @@ -122,9 +113,7 @@ existing_geos_gcm_build_path: existing_jedi_build_directory: ask_question: true - default_value: null - models: - - null + default_value: defer_to_platform prompt: What is the path to the existing JEDI build directory? tasks: - build_jedi_by_linking @@ -135,9 +124,7 @@ existing_jedi_build_directory: existing_geos_gcm_source_path: ask_question: true - default_value: null - models: - - null + default_value: defer_to_platform prompt: What is the path to the existing GEOS source code directory? tasks: - clone_geos @@ -148,9 +135,7 @@ existing_geos_gcm_source_path: existing_jedi_source_directory: ask_question: true - default_value: null - models: - - null + default_value: defer_to_platform prompt: What is the path to the existing JEDI source code directory? tasks: - clone_jedi @@ -161,10 +146,9 @@ existing_jedi_source_directory: geos_background_restart_offset: ask_question: true - default_value: null + default_value: defer_to_model models: - - null - options: null + - geos_atmosphere prompt: Duration before the middle of the window used in GEOS restart tar file. tasks: - get_background_geos_experiment @@ -172,9 +156,9 @@ geos_background_restart_offset: geos_bkg_filename_template: ask_question: true - default_value: null + default_value: defer_to_model models: - - null + - geos_atmosphere prompt: Name of the background files as output by GEOS (with date and time templated). tasks: - get_background_geos_experiment @@ -182,9 +166,9 @@ geos_bkg_filename_template: geos_bkg_tar_filename_template: ask_question: true - default_value: null + default_value: defer_to_model models: - - null + - geos_atmosphere prompt: Name of the GEOS backgrounds tar file (with date and time templated tasks: - get_background_geos_experiment @@ -193,8 +177,6 @@ geos_bkg_tar_filename_template: geos_build_method: ask_question: true default_value: use_existing - models: - - null options: ['use_existing', 'create'] prompt: Do you want to use an existing GEOS build or create a new build? tasks: @@ -206,9 +188,6 @@ geos_build_method: geos_gcm_tag: ask_question: true default_value: v11.0.2 - models: - - null - options: None prompt: Which GEOS tag do you wish to clone? tasks: - clone_geos @@ -216,10 +195,9 @@ geos_gcm_tag: gradient_norm_reduction: ask_question: false - default_value: model() + default_value: defer_to_model models: - - null - options: null + - all prompt: What value of gradient norm reduction for convergence? tasks: - run_jedi_executable_base @@ -227,10 +205,10 @@ gradient_norm_reduction: horizontal_resolution: ask_question: true - default_value: null + default_value: defer_to_model models: - - null - options: null + - all + options: defer_to_model prompt: What is the horizontal resolution for the forecast model and backgrounds? tasks: - generate_b_climatology_by_linking @@ -243,10 +221,9 @@ horizontal_resolution: jedi_bkg_filename_template: ask_question: true - default_value: null + default_value: defer_to_model models: - - null - options: null + - geos_atmosphere prompt: What is the name of the background as used by JEDI (with date and time templated) tasks: - get_background_geos_experiment @@ -255,8 +232,6 @@ jedi_bkg_filename_template: jedi_build_method: ask_question: true default_value: use_existing - models: - - null options: ['use_existing', 'create'] prompt: Do you want to use an existing JEDI build or create a new build? tasks: @@ -267,21 +242,21 @@ jedi_build_method: minimizer: ask_question: false - default_value: null + default_value: defer_to_model models: - - null - options: null + - all + options: defer_to_model prompt: Which data assimilation minimizer do you wish to use? tasks: - run_jedi_executable_base type: string-drop-list -model: +jedi_forecast_model: ask_question: true - default_value: null + default_value: defer_to_model models: - - null - options: null + - all + options: defer_to_model prompt: What forecast model should be used within JEDI for 4D window propagation? tasks: - run_jedi_executable_base @@ -292,10 +267,9 @@ model: npx_proc: ask_question: true - default_value: null + default_value: defer_to_model models: - - null - options: null + - geos_atmosphere prompt: What number of processors do you wish to use in the x-direction? tasks: - generate_b_climatology_by_linking @@ -307,10 +281,9 @@ npx_proc: npy_proc: ask_question: true - default_value: null + default_value: defer_to_model models: - - null - options: null + - geos_atmosphere prompt: What number of processors do you wish to use in the y-direction? tasks: - generate_b_climatology_by_linking @@ -322,10 +295,9 @@ npy_proc: number_of_iterations: ask_question: false - default_value: null + default_value: defer_to_model models: - - null - options: null + - all prompt: What number of iterations do you wish to use for each outer loop? Provide a list of integers the same length as the number of outer loops. tasks: - run_jedi_executable_base @@ -333,9 +305,9 @@ number_of_iterations: obs_experiment: ask_question: true - default_value: null + default_value: defer_to_model models: - - null + - all prompt: What is the database providing the observations? tasks: - get_observations @@ -343,10 +315,9 @@ obs_experiment: obs_provider: ask_question: true - default_value: null + default_value: defer_to_model models: - - null - options: null + - all prompt: What is the group providing the observations? tasks: - get_observations @@ -354,10 +325,10 @@ obs_provider: observations: ask_question: true - default_value: null + default_value: defer_to_model models: - - null - options: null + - all + options: defer_to_model prompt: Which observations do you want to include? tasks: - save_obs_diags @@ -368,10 +339,10 @@ observations: static_background_error_model: ask_question: true - default_value: null + default_value: defer_to_model models: - - null - options: null + - all + options: defer_to_model prompt: Which static background error model do you want to use? tasks: - generate_b_climatology_by_linking @@ -379,9 +350,9 @@ static_background_error_model: swell_static_files: ask_question: false - default_value: null + default_value: defer_to_platform models: - - null + - all prompt: What is the path to the Swell Static files directory? tasks: - generate_b_climatology_by_linking @@ -392,33 +363,32 @@ swell_static_files: swell_static_files_user: ask_question: false - default_value: null + default_value: None models: - - null - prompt: What is the path to the user provided Swell Static files directory? + - all + prompt: What is the path to the user provided Swell Static Files directory? tasks: - generate_b_climatology_by_linking type: string total_processors: ask_question: true - default_value: null + default_value: defer_to_model models: - - null - options: null - prompt: What is the total number of processors for JEDI? + - geos_ocean + prompt: What is the number of processors for JEDI? tasks: - generate_b_climatology - run_jedi_hofx_executable - run_jedi_variational_executable - type: string + type: integer vertical_resolution: ask_question: true - default_value: null + default_value: defer_to_model models: - - null - options: null + - all + options: defer_to_model prompt: What is the vertical resolution for the forecast model and background? tasks: - generate_b_climatology_by_linking @@ -429,24 +399,22 @@ vertical_resolution: window_length: ask_question: false - default_value: null + default_value: defer_to_model models: - - null - options: null + - all prompt: What is the duration for the data assimilation window? tasks: - store_background - get_background - get_observations - run_jedi_executable_base - type: string + type: duration window_offset: ask_question: true - default_value: null + default_value: defer_to_model models: - - null - options: null + - all prompt: What is the duration between the middle of the window and the beginning? tasks: - store_background @@ -461,7 +429,7 @@ window_type: ask_question: true default_value: 4D models: - - null + - all options: ['3D', '4D'] prompt: Do you want to use a 3D or 4D (including FGAT) window? tasks: @@ -470,4 +438,3 @@ window_type: - run_jedi_hofx_executable - run_jedi_variational_executable type: string-drop-list - diff --git a/src/swell/tasks/run_jedi_hofx_executable.py b/src/swell/tasks/run_jedi_hofx_executable.py index 6b67a658..f01357d9 100644 --- a/src/swell/tasks/run_jedi_hofx_executable.py +++ b/src/swell/tasks/run_jedi_hofx_executable.py @@ -37,9 +37,46 @@ def execute(self): # Path to executable being run # ---------------------------- window_type = self.config_get('window_type') - npx_proc = self.config_get('npx_proc') # Used in eval(total_processors) - npy_proc = self.config_get('npy_proc') # Used in eval(total_processors) - total_processors = self.config_get('total_processors') + npx_proc = self.config_get('npx_proc', None) + npy_proc = self.config_get('npy_proc', None) + total_processors = self.config_get('total_processors', None) + window_length = self.config_get('window_length') + horizontal_resolution = self.config_get('horizontal_resolution') + vertical_resolution = self.config_get('vertical_resolution') + crtm_coeff_dir = self.config_get('crtm_coeff_dir', None) + window_offset = self.config_get('window_offset') + background_time_offset = self.config_get('background_time_offset') + observations = self.config_get('observations') + + # Compute data assimilation window parameters + background_time = self.da_window_params.background_time(window_offset, + background_time_offset) + local_background_time = self.da_window_params.local_background_time(window_offset, + window_type) + local_background_time_iso = self.da_window_params.local_background_time_iso(window_offset, + window_type) + window_begin = self.da_window_params.window_begin(window_offset) + window_begin_iso = self.da_window_params.window_begin_iso(window_offset) + + # Populate jedi interface templates dictionary + # -------------------------------------------- + self.jedi_rendering.add_key('window_begin_iso', window_begin_iso) + self.jedi_rendering.add_key('window_length', window_length) + + # Background + self.jedi_rendering.add_key('horizontal_resolution', horizontal_resolution) + self.jedi_rendering.add_key('local_background_time', local_background_time) + self.jedi_rendering.add_key('local_background_time_iso', local_background_time_iso) + + # Geometry + self.jedi_rendering.add_key('npx_proc', npx_proc) + self.jedi_rendering.add_key('npy_proc', npy_proc) + self.jedi_rendering.add_key('vertical_resolution', vertical_resolution) + + # Observations + self.jedi_rendering.add_key('background_time', background_time) + self.jedi_rendering.add_key('crtm_coeff_dir', crtm_coeff_dir) + self.jedi_rendering.add_key('window_begin', window_begin) # Jedi configuration file # ----------------------- @@ -49,10 +86,16 @@ def execute(self): # --------------- output_log_file = os.path.join(self.cycle_dir(), 'jedi_hofx_log.log') - # Generate the JEDI configuration file for running the executable - # --------------------------------------------------------------- - jedi_config_dict = self.generate_jedi_config(self.suite(), window_type) + # Open the JEDI config file and fill initial templates + # ---------------------------------------------------- + jedi_config_dict = self.jedi_rendering.render_oops_file(f'hofx{window_type}') + # Perform complete template rendering + # ----------------------------------- + self.jedi_dictionary_iterator(jedi_config_dict, window_type) + + # Write the expanded dictionary to YAML file + # ------------------------------------------ with open(jedi_config_file, 'w') as jedi_config_file_open: yaml.dump(jedi_config_dict, jedi_config_file_open, default_flow_style=False) diff --git a/src/swell/tasks/run_jedi_variational_executable.py b/src/swell/tasks/run_jedi_variational_executable.py index e062a074..05790948 100644 --- a/src/swell/tasks/run_jedi_variational_executable.py +++ b/src/swell/tasks/run_jedi_variational_executable.py @@ -27,12 +27,57 @@ class RunJediVariationalExecutable(RunJediExecutableBase): def execute(self): - # Path to executable being run - # ---------------------------- + # Parse configuration + # ------------------- window_type = self.config_get('window_type') - npx_proc = self.config_get('npx_proc') # Used in eval(total_processors) - npy_proc = self.config_get('npy_proc') # Used in eval(total_processors) - total_processors = self.config_get('total_processors') + npx_proc = self.config_get('npx_proc', None) + npy_proc = self.config_get('npy_proc', None) + total_processors = self.config_get('total_processors', None) + window_length = self.config_get('window_length') + horizontal_resolution = self.config_get('horizontal_resolution') + vertical_resolution = self.config_get('vertical_resolution') + crtm_coeff_dir = self.config_get('crtm_coeff_dir', None) + window_offset = self.config_get('window_offset') + background_time_offset = self.config_get('background_time_offset') + number_of_iterations = self.config_get('number_of_iterations') + minimizer = self.config_get('minimizer') + analysis_variables = self.config_get('analysis_variables') + gradient_norm_reduction = self.config_get('gradient_norm_reduction') + observations = self.config_get('observations') + + # Compute data assimilation window parameters + background_time = self.da_window_params.background_time(window_offset, + background_time_offset) + local_background_time = self.da_window_params.local_background_time(window_offset, + window_type) + local_background_time_iso = self.da_window_params.local_background_time_iso(window_offset, + window_type) + window_begin = self.da_window_params.window_begin(window_offset) + window_begin_iso = self.da_window_params.window_begin_iso(window_offset) + + # Populate jedi interface templates dictionary + # -------------------------------------------- + self.jedi_rendering.add_key('window_begin_iso', window_begin_iso) + self.jedi_rendering.add_key('window_length', window_length) + self.jedi_rendering.add_key('minimizer', minimizer) + self.jedi_rendering.add_key('number_of_iterations', number_of_iterations[0]) + self.jedi_rendering.add_key('analysis_variables', analysis_variables) + self.jedi_rendering.add_key('gradient_norm_reduction', gradient_norm_reduction) + + # Background + self.jedi_rendering.add_key('horizontal_resolution', horizontal_resolution) + self.jedi_rendering.add_key('local_background_time', local_background_time) + self.jedi_rendering.add_key('local_background_time_iso', local_background_time_iso) + + # Geometry + self.jedi_rendering.add_key('npx_proc', npx_proc) + self.jedi_rendering.add_key('npy_proc', npy_proc) + self.jedi_rendering.add_key('vertical_resolution', vertical_resolution) + + # Observations + self.jedi_rendering.add_key('background_time', background_time) + self.jedi_rendering.add_key('crtm_coeff_dir', crtm_coeff_dir) + self.jedi_rendering.add_key('window_begin', window_begin) # Jedi configuration file # ----------------------- @@ -49,8 +94,14 @@ def execute(self): # Generate the JEDI configuration file for running the executable # --------------------------------------------------------------- - jedi_config_dict = self.generate_jedi_config(self.suite(), window_type) + jedi_config_dict = self.jedi_rendering.render_oops_file(f'variational{window_type}') + + # Perform complete template rendering + # ----------------------------------- + self.jedi_dictionary_iterator(jedi_config_dict, window_type) + # Write the expanded dictionary to YAML file + # ------------------------------------------ with open(jedi_config_file, 'w') as jedi_config_file_open: yaml.dump(jedi_config_dict, jedi_config_file_open, default_flow_style=False) diff --git a/src/swell/utilities/bin/check_jedi_interface_templates.py b/src/swell/utilities/bin/check_jedi_interface_templates.py index 89f77cfa..fa4ccf03 100644 --- a/src/swell/utilities/bin/check_jedi_interface_templates.py +++ b/src/swell/utilities/bin/check_jedi_interface_templates.py @@ -30,9 +30,11 @@ def main(): config_types = ['oops/*yaml', 'interfaces/*/model/*yaml', 'interfaces/*/observations/*yaml', + 'interfaces/*/model/geometry.yaml', + 'interfaces/*/model/background.yaml', 'interfaces/*/model/r2d2.yaml', 'interfaces/*/model/stage*.yaml', - 'interfaces/*/model/background.yaml', + 'interfaces/*/model/background_error.yaml', 'interfaces/*/model/StaticBInit.yaml', ] From c8ccdd5954372a75796c5ec48a0abdf83d95f438 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 24 May 2023 12:12:22 -0400 Subject: [PATCH 020/121] move jedi exe base --- .../run_jedi_executables.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/swell/{tasks/base/run_jedi_executable_base.py => utilities/run_jedi_executables.py} (100%) diff --git a/src/swell/tasks/base/run_jedi_executable_base.py b/src/swell/utilities/run_jedi_executables.py similarity index 100% rename from src/swell/tasks/base/run_jedi_executable_base.py rename to src/swell/utilities/run_jedi_executables.py From 0abbd51bcff1b208a8e7124cd87802ce72a12eb2 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 24 May 2023 12:27:16 -0400 Subject: [PATCH 021/121] refactor run jedi exe base --- .../geos_atmosphere/geos_atmosphere.yaml | 9 +- .../interfaces/geos_ocean/geos_ocean.yaml | 8 +- src/swell/tasks/base/task_base.py | 7 -- src/swell/tasks/generate_b_climatology.py | 2 - src/swell/tasks/run_jedi_hofx_executable.py | 29 ++---- .../tasks/run_jedi_variational_executable.py | 19 ++-- src/swell/utilities/run_jedi_executables.py | 92 +++++++++---------- 7 files changed, 68 insertions(+), 98 deletions(-) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/geos_atmosphere.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/geos_atmosphere.yaml index 9232ba04..3756e6c1 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/geos_atmosphere.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/geos_atmosphere.yaml @@ -1,5 +1,8 @@ jedi_interface: fv3-jedi executables: - hofx3d: fv3jedi_hofx_nomodel.x - hofx: fv3jedi_hofx.x - var: fv3jedi_var.x + hofx3D: fv3jedi_hofx_nomodel.x + hofx4D: fv3jedi_hofx.x + variational3D: fv3jedi_var.x + variationalFGAT: fv3jedi_var.x + variational4D: fv3jedi_var.x + variational4DEnsVar: fv3jedi_var.x diff --git a/src/swell/configuration/jedi/interfaces/geos_ocean/geos_ocean.yaml b/src/swell/configuration/jedi/interfaces/geos_ocean/geos_ocean.yaml index f6f192c9..11c2e024 100644 --- a/src/swell/configuration/jedi/interfaces/geos_ocean/geos_ocean.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_ocean/geos_ocean.yaml @@ -1,6 +1,8 @@ jedi_interface: soca executables: - hofx3d: soca_hofx3d.x - hofx: soca_hofx.x - var: soca_var.x + hofx3D: soca_hofx3d.x + hofx4D: soca_hofx.x + variational3D: soca_var.x + variational4D: soca_var.x + variational4DEnsVar: soca_var.x bump: soca_staticbinit.x diff --git a/src/swell/tasks/base/task_base.py b/src/swell/tasks/base/task_base.py index 283d81e0..283b2369 100644 --- a/src/swell/tasks/base/task_base.py +++ b/src/swell/tasks/base/task_base.py @@ -66,7 +66,6 @@ def __init__(self, config_input, datetime_input, model, task_name): # -------------------------------------------------------- self.__experiment_root__ = self.config_get('experiment_root') self.__experiment_id__ = self.config_get('experiment_id') - self.__suite__ = self.config_get('suite_to_run') # Save the model components # ------------------------- @@ -120,12 +119,6 @@ def experiment_id(self): # ---------------------------------------------------------------------------------------------- - # Method to get the suite type - def suite(self): - return self.__suite__ - - # ---------------------------------------------------------------------------------------------- - # Method to get the experiment directory def experiment_path(self): return os.path.join(self.__experiment_root__, self.__experiment_id__) diff --git a/src/swell/tasks/generate_b_climatology.py b/src/swell/tasks/generate_b_climatology.py index fca199ba..9513a152 100644 --- a/src/swell/tasks/generate_b_climatology.py +++ b/src/swell/tasks/generate_b_climatology.py @@ -12,8 +12,6 @@ from swell.tasks.base.task_base import taskBase from swell.utilities.shell_commands import run_track_log_subprocess -from swell.utilities.render_jedi_interface_files import template_dictionary_oops_config, \ - read_render_jedi_interface_oops, # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/run_jedi_hofx_executable.py b/src/swell/tasks/run_jedi_hofx_executable.py index f01357d9..fe88ddeb 100644 --- a/src/swell/tasks/run_jedi_hofx_executable.py +++ b/src/swell/tasks/run_jedi_hofx_executable.py @@ -11,24 +11,14 @@ import os import yaml -from swell.tasks.base.run_jedi_executable_base import RunJediExecutableBase +from swell.tasks.base.task_base import taskBase +from swell.utilities.run_jedi_executables import jedi_dictionary_iterator, run_executable # -------------------------------------------------------------------------------------------------- -interface_executable = { - 'fv3-jedi-4D': 'fv3jedi_hofx.x', - 'fv3-jedi-3D': 'fv3jedi_hofx_nomodel.x', - 'soca-4D': 'soca_hofx.x', - 'soca-3D': 'soca_hofx3d.x', -} - - -# -------------------------------------------------------------------------------------------------- - - -class RunJediHofxExecutable(RunJediExecutableBase): +class RunJediHofxExecutable(taskBase): # ---------------------------------------------------------------------------------------------- @@ -92,21 +82,20 @@ def execute(self): # Perform complete template rendering # ----------------------------------- - self.jedi_dictionary_iterator(jedi_config_dict, window_type) + jedi_dictionary_iterator(jedi_config_dict, window_type) # Write the expanded dictionary to YAML file # ------------------------------------------ with open(jedi_config_file, 'w') as jedi_config_file_open: yaml.dump(jedi_config_dict, jedi_config_file_open, default_flow_style=False) - # Get the JEDI interface for this model component - # ----------------------------------------------- + # Get the JEDI interface metadata + # ------------------------------- model_component_meta = self.jedi_rendering.render_interface_meta() - jedi_interface = model_component_meta['jedi_interface'] # Jedi executable name # -------------------- - jedi_executable = interface_executable[jedi_interface + '-' + window_type] + jedi_executable = model_component_meta['executables'][f'hofx{window_type}'] jedi_executable_path = os.path.join(self.experiment_path(), 'jedi_bundle', 'build', 'bin', jedi_executable) @@ -118,8 +107,8 @@ def execute(self): # Run the JEDI executable # ----------------------- - self.run_executable(self.cycle_dir(), np, jedi_executable_path, jedi_config_file, - output_log_file) + run_executable(self.cycle_dir(), np, jedi_executable_path, jedi_config_file, + output_log_file) self.logger.info('Running '+jedi_executable_path+' with '+str(np)+' processors.') # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/run_jedi_variational_executable.py b/src/swell/tasks/run_jedi_variational_executable.py index 05790948..ef826cb7 100644 --- a/src/swell/tasks/run_jedi_variational_executable.py +++ b/src/swell/tasks/run_jedi_variational_executable.py @@ -7,23 +7,18 @@ # -------------------------------------------------------------------------------------------------- + import os import yaml -from swell.tasks.base.run_jedi_executable_base import RunJediExecutableBase - -# -------------------------------------------------------------------------------------------------- - - -interface_executable = { - 'soca-3D': 'soca_var.x', -} +from swell.tasks.base.task_base import taskBase +from swell.utilities.run_jedi_executables import jedi_dictionary_iterator, run_executable # -------------------------------------------------------------------------------------------------- -class RunJediVariationalExecutable(RunJediExecutableBase): +class RunJediVariationalExecutable(taskBase): def execute(self): @@ -98,7 +93,7 @@ def execute(self): # Perform complete template rendering # ----------------------------------- - self.jedi_dictionary_iterator(jedi_config_dict, window_type) + jedi_dictionary_iterator(jedi_config_dict, window_type) # Write the expanded dictionary to YAML file # ------------------------------------------ @@ -107,7 +102,7 @@ def execute(self): # Jedi executable name # -------------------- - jedi_executable = interface_executable[jedi_interface + '-' + window_type] + jedi_executable = model_component_meta['executables'][f'variational{window_type}'] jedi_executable_path = os.path.join(self.experiment_path(), 'jedi_bundle', 'build', 'bin', jedi_executable) # Compute number of processors @@ -118,7 +113,7 @@ def execute(self): # Run the JEDI executable # ----------------------- - self.run_executable(self.cycle_dir(), np, jedi_executable_path, jedi_config_file, + run_executable(self.cycle_dir(), np, jedi_executable_path, jedi_config_file, output_log_file) self.logger.info('Running '+jedi_executable_path+' with '+str(np)+' processors.') diff --git a/src/swell/utilities/run_jedi_executables.py b/src/swell/utilities/run_jedi_executables.py index 73ed5ef6..e09dd636 100644 --- a/src/swell/utilities/run_jedi_executables.py +++ b/src/swell/utilities/run_jedi_executables.py @@ -8,79 +8,69 @@ # -------------------------------------------------------------------------------------------------- -from abc import ABC, abstractmethod import os -from swell.tasks.base.task_base import taskBase from swell.utilities.shell_commands import run_track_log_subprocess # -------------------------------------------------------------------------------------------------- -class RunJediExecutableBase(taskBase): +def jedi_dictionary_iterator(jedi_config_dict, jedi_rendering, window_type, obs, + jedi_forecast_mode): - # ---------------------------------------------------------------------------------------------- + # Assemble configuration YAML file + # -------------------------------- + for key, value in jedi_config_dict.items(): + if isinstance(value, dict): + jedi_dictionary_iterator(value, jedi_rendering, window_type, obs, jedi_forecast_mode) - @abstractmethod - def execute(self): - # This class does not execute, it provides helper function for the children - # ------------------------------------------------------------------------ - pass + elif isinstance(value, list): + for item in value: + if isinstance(item, dict): + jedi_dictionary_iterator(item, jedi_rendering, window_type, obs, + jedi_forecast_mode) - # ---------------------------------------------------------------------------------------------- + else: + if 'TASKFILL' in value: + value_file = value.replace('TASKFILL', '') + value_dict = jedi_rendering.render_interface_model(value_file) - def jedi_dictionary_iterator(self, jedi_config_dict, window_type, obs, jedi_forecast_mode): + jedi_config_dict[key] = value_dict - # Assemble configuration YAML file - # -------------------------------- - for key, value in jedi_config_dict.items(): - if isinstance(value, dict): - self.jedi_dictionary_iterator(value, window_type) + elif 'SPECIAL' in value: + value_special = value.replace('SPECIAL', '') + if value_special == 'observations': + observations = [] + for ob in obs: + # Get observation dictionary + obs_dict = jedi_rendering.render_interface_observations(ob) + observations.append(obs_dict) + jedi_config_dict[key] = observations - elif isinstance(value, list): - for item in value: - if isinstance(item, dict): - self.jedi_dictionary_iterator(item, window_type) + elif value_special == 'model' and window_type == '4D': + model_dict = jedi_rendering.render_interface_model(jedi_forecast_model) + jedi_config_dict[key] = model_dict - else: - if 'TASKFILL' in value: - value_file = value.replace('TASKFILL', '') - value_dict = self.jedi_rendering.render_interface_model(value_file) - jedi_config_dict[key] = value_dict +# ---------------------------------------------------------------------------------------------- - elif 'SPECIAL' in value: - value_special = value.replace('SPECIAL', '') - if value_special == 'observations': - observations = [] - for ob in obs: - # Get observation dictionary - obs_dict = self.jedi_rendering.render_interface_observations(ob) - observations.append(obs_dict) - jedi_config_dict[key] = observations - elif value_special == 'model' and window_type == '4D': - model_dict = self.jedi_rendering.render_interface_model(jedi_forecast_model) - jedi_config_dict[key] = model_dict +def run_executable(logger, cycle_dir, np, jedi_executable_path, jedi_config_file, output_log): - # ---------------------------------------------------------------------------------------------- + # Run the JEDI executable + # ----------------------- + logger.info('Running '+jedi_executable_path+' with '+str(np)+' processors.') - def run_executable(self, cycle_dir, np, jedi_executable_path, jedi_config_file, output_log): + command = ['mpirun', '-np', str(np), jedi_executable_path, jedi_config_file] - # Run the JEDI executable - # ----------------------- - self.logger.info('Running '+jedi_executable_path+' with '+str(np)+' processors.') + # Move to the cycle directory + # --------------------------- + os.chdir(cycle_dir) - command = ['mpirun', '-np', str(np), jedi_executable_path, jedi_config_file] - - # Move to the cycle directory - # --------------------------- - os.chdir(cycle_dir) - - # Run command - # ----------- - run_track_log_subprocess(self.logger, command, output_log=output_log) + # Run command + # ----------- + run_track_log_subprocess(logger, command, output_log=output_log) # -------------------------------------------------------------------------------------------------- From c6fe46f637db2ef46ca8244ba4bdafe50cf31346 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 24 May 2023 12:37:00 -0400 Subject: [PATCH 022/121] remove fgat as not needed --- .../jedi/interfaces/geos_atmosphere/geos_atmosphere.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/geos_atmosphere.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/geos_atmosphere.yaml index 3756e6c1..fe58d95e 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/geos_atmosphere.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/geos_atmosphere.yaml @@ -3,6 +3,5 @@ executables: hofx3D: fv3jedi_hofx_nomodel.x hofx4D: fv3jedi_hofx.x variational3D: fv3jedi_var.x - variationalFGAT: fv3jedi_var.x variational4D: fv3jedi_var.x variational4DEnsVar: fv3jedi_var.x From 92c884b5dc116429c30ffe3d7a83e761504d8496 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 24 May 2023 12:43:01 -0400 Subject: [PATCH 023/121] move depends --- src/swell/tasks/questions.yaml | 36 +++++++++---------- .../utilities/bin/list_of_task_questions.py | 13 +++---- 2 files changed, 23 insertions(+), 26 deletions(-) diff --git a/src/swell/tasks/questions.yaml b/src/swell/tasks/questions.yaml index 0693f9dd..01c356ce 100644 --- a/src/swell/tasks/questions.yaml +++ b/src/swell/tasks/questions.yaml @@ -43,6 +43,9 @@ background_experiment: background_frequency: ask_question: false default_value: defer_to_model + depends: + key: window_type + value: 4D models: - all prompt: What is the frequency of the background files? @@ -50,9 +53,6 @@ background_frequency: - get_background - run_jedi_executable_base type: duration - depends: - key: window_type - value: 4D background_time_offset: ask_question: false @@ -103,46 +103,46 @@ crtm_coeff_dir: existing_geos_gcm_build_path: ask_question: true default_value: defer_to_platform + depends: + key: geos_build_method + value: use_existing prompt: What is the path to the existing GEOS build directory? tasks: - build_geos_by_linking type: string - depends: - key: geos_build_method - value: use_existing existing_jedi_build_directory: ask_question: true default_value: defer_to_platform + depends: + key: jedi_build_method + value: use_existing prompt: What is the path to the existing JEDI build directory? tasks: - build_jedi_by_linking type: string - depends: - key: jedi_build_method - value: use_existing existing_geos_gcm_source_path: ask_question: true default_value: defer_to_platform + depends: + key: geos_build_method + value: use_existing prompt: What is the path to the existing GEOS source code directory? tasks: - clone_geos type: string - depends: - key: geos_build_method - value: use_existing existing_jedi_source_directory: ask_question: true default_value: defer_to_platform + depends: + key: jedi_build_method + value: use_existing prompt: What is the path to the existing JEDI source code directory? tasks: - clone_jedi type: string - depends: - key: jedi_build_method - value: use_existing geos_background_restart_offset: ask_question: true @@ -254,6 +254,9 @@ minimizer: jedi_forecast_model: ask_question: true default_value: defer_to_model + depends: + key: window_type + value: 4D models: - all options: defer_to_model @@ -261,9 +264,6 @@ jedi_forecast_model: tasks: - run_jedi_executable_base type: string-drop-list - depends: - key: window_type - value: 4D npx_proc: ask_question: true diff --git a/src/swell/utilities/bin/list_of_task_questions.py b/src/swell/utilities/bin/list_of_task_questions.py index 39836808..73042e3e 100644 --- a/src/swell/utilities/bin/list_of_task_questions.py +++ b/src/swell/utilities/bin/list_of_task_questions.py @@ -102,10 +102,7 @@ def main(): question_dict_key = question_dict[unique_key] if 'default_value' not in question_dict_key: - question_to_tasks[unique_key]['default_value'] = None - - if 'options' not in question_dict_key: - question_to_tasks[unique_key]['options'] = None + question_to_tasks[unique_key]['default_value'] = defer_to_model if 'prompt' not in question_dict_key: question_to_tasks[unique_key]['prompt'] = 'Question' @@ -114,7 +111,7 @@ def main(): question_to_tasks[unique_key]['type'] = 'string' if 'models' not in question_dict_key: - question_to_tasks[unique_key]['models'] = [None] + question_to_tasks[unique_key]['models'] = ['all'] if 'ask_question' not in question_dict_key: question_to_tasks[unique_key]['ask_question'] = True @@ -122,11 +119,11 @@ def main(): else: question_to_tasks[unique_key] = {} - question_to_tasks[unique_key]['default_value'] = None - question_to_tasks[unique_key]['options'] = None + question_to_tasks[unique_key]['default_value'] = defer_to_model + question_to_tasks[unique_key]['options'] = defer_to_model question_to_tasks[unique_key]['prompt'] = 'Question' question_to_tasks[unique_key]['type'] = 'string' - question_to_tasks[unique_key]['models'] = [None] + question_to_tasks[unique_key]['models'] = ['all'] question_to_tasks[unique_key]['ask_question'] = True # Regardless of whether question was already in dictionary From f80c852bf0abc8f3cfbf12ad23394790347838ef Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 24 May 2023 12:47:21 -0400 Subject: [PATCH 024/121] write lists the way python does --- src/swell/tasks/questions.yaml | 26 +++++++++++++++---- .../utilities/bin/list_of_task_questions.py | 4 +-- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/swell/tasks/questions.yaml b/src/swell/tasks/questions.yaml index 01c356ce..f68ba5a0 100644 --- a/src/swell/tasks/questions.yaml +++ b/src/swell/tasks/questions.yaml @@ -69,8 +69,18 @@ background_time_offset: bundles: ask_question: true - default_value: ['fv3-jedi', 'soca', 'iodaconv'] - options: ['fv3-jedi', 'soca', 'iodaconv', 'ufo', 'ioda', 'oops', 'saber'] + default_value: + - 'fv3-jedi' + - 'soca' + - 'iodaconv' + options: + - 'fv3-jedi' + - 'soca' + - 'iodaconv' + - 'ufo' + - 'ioda' + - 'oops' + - 'saber' prompt: Which JEDI bundles do you wish to build? tasks: - build_jedi @@ -177,7 +187,9 @@ geos_bkg_tar_filename_template: geos_build_method: ask_question: true default_value: use_existing - options: ['use_existing', 'create'] + options: + - 'use_existing' + - 'create' prompt: Do you want to use an existing GEOS build or create a new build? tasks: - build_geos @@ -232,7 +244,9 @@ jedi_bkg_filename_template: jedi_build_method: ask_question: true default_value: use_existing - options: ['use_existing', 'create'] + options: + - 'use_existing' + - 'create' prompt: Do you want to use an existing JEDI build or create a new build? tasks: - build_jedi @@ -430,7 +444,9 @@ window_type: default_value: 4D models: - all - options: ['3D', '4D'] + options: + - '3D' + - '4D' prompt: Do you want to use a 3D or 4D (including FGAT) window? tasks: - store_background diff --git a/src/swell/utilities/bin/list_of_task_questions.py b/src/swell/utilities/bin/list_of_task_questions.py index 73042e3e..e8364b6b 100644 --- a/src/swell/utilities/bin/list_of_task_questions.py +++ b/src/swell/utilities/bin/list_of_task_questions.py @@ -101,6 +101,7 @@ def main(): question_dict_key = question_dict[unique_key] + # Make sure minimal things are in the question's dictionary if 'default_value' not in question_dict_key: question_to_tasks[unique_key]['default_value'] = defer_to_model @@ -110,9 +111,6 @@ def main(): if 'type' not in question_dict_key: question_to_tasks[unique_key]['type'] = 'string' - if 'models' not in question_dict_key: - question_to_tasks[unique_key]['models'] = ['all'] - if 'ask_question' not in question_dict_key: question_to_tasks[unique_key]['ask_question'] = True From 7a532526cbab151e2fd7f2271218f37d7b23ebe6 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 24 May 2023 12:54:33 -0400 Subject: [PATCH 025/121] make sure jedi_forecast model is used --- src/swell/tasks/questions.yaml | 32 +++++++++---------- src/swell/tasks/run_jedi_hofx_executable.py | 6 ++-- .../tasks/run_jedi_variational_executable.py | 9 ++++-- src/swell/utilities/run_jedi_executables.py | 6 ++-- 4 files changed, 29 insertions(+), 24 deletions(-) diff --git a/src/swell/tasks/questions.yaml b/src/swell/tasks/questions.yaml index f68ba5a0..f82d57fa 100644 --- a/src/swell/tasks/questions.yaml +++ b/src/swell/tasks/questions.yaml @@ -70,17 +70,17 @@ background_time_offset: bundles: ask_question: true default_value: - - 'fv3-jedi' - - 'soca' - - 'iodaconv' + - fv3-jedi + - soca + - iodaconv options: - - 'fv3-jedi' - - 'soca' - - 'iodaconv' - - 'ufo' - - 'ioda' - - 'oops' - - 'saber' + - fv3-jedi + - soca + - iodaconv + - ufo + - ioda + - oops + - saber prompt: Which JEDI bundles do you wish to build? tasks: - build_jedi @@ -188,8 +188,8 @@ geos_build_method: ask_question: true default_value: use_existing options: - - 'use_existing' - - 'create' + - use_existing + - create prompt: Do you want to use an existing GEOS build or create a new build? tasks: - build_geos @@ -245,8 +245,8 @@ jedi_build_method: ask_question: true default_value: use_existing options: - - 'use_existing' - - 'create' + - use_existing + - create prompt: Do you want to use an existing JEDI build or create a new build? tasks: - build_jedi @@ -445,8 +445,8 @@ window_type: models: - all options: - - '3D' - - '4D' + - 3D + - 4D prompt: Do you want to use a 3D or 4D (including FGAT) window? tasks: - store_background diff --git a/src/swell/tasks/run_jedi_hofx_executable.py b/src/swell/tasks/run_jedi_hofx_executable.py index fe88ddeb..04011838 100644 --- a/src/swell/tasks/run_jedi_hofx_executable.py +++ b/src/swell/tasks/run_jedi_hofx_executable.py @@ -37,6 +37,7 @@ def execute(self): window_offset = self.config_get('window_offset') background_time_offset = self.config_get('background_time_offset') observations = self.config_get('observations') + jedi_forecast_model = self.config_get('jedi_forecast_model', None) # Compute data assimilation window parameters background_time = self.da_window_params.background_time(window_offset, @@ -82,7 +83,8 @@ def execute(self): # Perform complete template rendering # ----------------------------------- - jedi_dictionary_iterator(jedi_config_dict, window_type) + jedi_dictionary_iterator(jedi_config_dict, self.jedi_rendering, window_type, observations, + jedi_forecast_model) # Write the expanded dictionary to YAML file # ------------------------------------------ @@ -107,7 +109,7 @@ def execute(self): # Run the JEDI executable # ----------------------- - run_executable(self.cycle_dir(), np, jedi_executable_path, jedi_config_file, + run_executable(self.logger, self.cycle_dir(), np, jedi_executable_path, jedi_config_file, output_log_file) self.logger.info('Running '+jedi_executable_path+' with '+str(np)+' processors.') diff --git a/src/swell/tasks/run_jedi_variational_executable.py b/src/swell/tasks/run_jedi_variational_executable.py index ef826cb7..5855e110 100644 --- a/src/swell/tasks/run_jedi_variational_executable.py +++ b/src/swell/tasks/run_jedi_variational_executable.py @@ -39,6 +39,7 @@ def execute(self): analysis_variables = self.config_get('analysis_variables') gradient_norm_reduction = self.config_get('gradient_norm_reduction') observations = self.config_get('observations') + jedi_forecast_model = self.config_get('jedi_forecast_model', None) # Compute data assimilation window parameters background_time = self.da_window_params.background_time(window_offset, @@ -93,7 +94,9 @@ def execute(self): # Perform complete template rendering # ----------------------------------- - jedi_dictionary_iterator(jedi_config_dict, window_type) + jedi_dictionary_iterator(jedi_config_dict, self.jedi_rendering, window_type, observations, + jedi_forecast_model) + # Write the expanded dictionary to YAML file # ------------------------------------------ @@ -113,8 +116,8 @@ def execute(self): # Run the JEDI executable # ----------------------- - run_executable(self.cycle_dir(), np, jedi_executable_path, jedi_config_file, - output_log_file) + run_executable(self.logger, self.logger, self.cycle_dir(), np, jedi_executable_path, jedi_config_file, + output_log_file) self.logger.info('Running '+jedi_executable_path+' with '+str(np)+' processors.') diff --git a/src/swell/utilities/run_jedi_executables.py b/src/swell/utilities/run_jedi_executables.py index e09dd636..48d2ebe8 100644 --- a/src/swell/utilities/run_jedi_executables.py +++ b/src/swell/utilities/run_jedi_executables.py @@ -17,19 +17,19 @@ def jedi_dictionary_iterator(jedi_config_dict, jedi_rendering, window_type, obs, - jedi_forecast_mode): + jedi_forecast_model): # Assemble configuration YAML file # -------------------------------- for key, value in jedi_config_dict.items(): if isinstance(value, dict): - jedi_dictionary_iterator(value, jedi_rendering, window_type, obs, jedi_forecast_mode) + jedi_dictionary_iterator(value, jedi_rendering, window_type, obs, jedi_forecast_model) elif isinstance(value, list): for item in value: if isinstance(item, dict): jedi_dictionary_iterator(item, jedi_rendering, window_type, obs, - jedi_forecast_mode) + jedi_forecast_model) else: if 'TASKFILL' in value: From 0f670721c7f3bee5275fc10df0b7aae714977bbd Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 24 May 2023 12:57:26 -0400 Subject: [PATCH 026/121] more q clean up --- src/swell/tasks/questions.yaml | 40 ++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/src/swell/tasks/questions.yaml b/src/swell/tasks/questions.yaml index f82d57fa..c58e2d8f 100644 --- a/src/swell/tasks/questions.yaml +++ b/src/swell/tasks/questions.yaml @@ -59,7 +59,8 @@ background_time_offset: default_value: defer_to_model models: - all - prompt: How long before the middle of the analysis window did the background providing forecast begin? + prompt: How long before the middle of the analysis window did the background providing + forecast begin? tasks: - save_obs_diags - get_observations @@ -121,26 +122,26 @@ existing_geos_gcm_build_path: - build_geos_by_linking type: string -existing_jedi_build_directory: +existing_geos_gcm_source_path: ask_question: true default_value: defer_to_platform depends: - key: jedi_build_method + key: geos_build_method value: use_existing - prompt: What is the path to the existing JEDI build directory? + prompt: What is the path to the existing GEOS source code directory? tasks: - - build_jedi_by_linking + - clone_geos type: string -existing_geos_gcm_source_path: +existing_jedi_build_directory: ask_question: true default_value: defer_to_platform depends: - key: geos_build_method + key: jedi_build_method value: use_existing - prompt: What is the path to the existing GEOS source code directory? + prompt: What is the path to the existing JEDI build directory? tasks: - - clone_geos + - build_jedi_by_linking type: string existing_jedi_source_directory: @@ -254,27 +255,27 @@ jedi_build_method: - build_jedi_by_linking type: string -minimizer: - ask_question: false +jedi_forecast_model: + ask_question: true default_value: defer_to_model + depends: + key: window_type + value: 4D models: - all options: defer_to_model - prompt: Which data assimilation minimizer do you wish to use? + prompt: What forecast model should be used within JEDI for 4D window propagation? tasks: - run_jedi_executable_base type: string-drop-list -jedi_forecast_model: - ask_question: true +minimizer: + ask_question: false default_value: defer_to_model - depends: - key: window_type - value: 4D models: - all options: defer_to_model - prompt: What forecast model should be used within JEDI for 4D window propagation? + prompt: Which data assimilation minimizer do you wish to use? tasks: - run_jedi_executable_base type: string-drop-list @@ -312,7 +313,8 @@ number_of_iterations: default_value: defer_to_model models: - all - prompt: What number of iterations do you wish to use for each outer loop? Provide a list of integers the same length as the number of outer loops. + prompt: What number of iterations do you wish to use for each outer loop? Provide + a list of integers the same length as the number of outer loops. tasks: - run_jedi_executable_base type: integer-list From 8aacda5164c6f023f41713d43a41dcc90541fd52 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 24 May 2023 12:58:23 -0400 Subject: [PATCH 027/121] remove jedi exe base class from questions --- src/swell/tasks/questions.yaml | 41 ++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/src/swell/tasks/questions.yaml b/src/swell/tasks/questions.yaml index c58e2d8f..cec64436 100644 --- a/src/swell/tasks/questions.yaml +++ b/src/swell/tasks/questions.yaml @@ -5,7 +5,7 @@ analysis_forecast_window_offset: - all prompt: What is the duration from the middle of the window when forecasts start? tasks: - - store_backgr:qound + - store_background - get_background type: duration @@ -16,7 +16,7 @@ analysis_variables: - all prompt: What are the analysis variables? tasks: - - run_jedi_executable_base + - run_jedi_variational_executable type: string-check-list background_error_model: @@ -37,6 +37,7 @@ background_experiment: - all prompt: What is the name of the name of the experiment providing the backgrounds? tasks: + - store_background - get_background type: string @@ -50,8 +51,8 @@ background_frequency: - all prompt: What is the frequency of the background files? tasks: + - store_background - get_background - - run_jedi_executable_base type: duration background_time_offset: @@ -63,9 +64,10 @@ background_time_offset: forecast begin? tasks: - save_obs_diags + - run_jedi_hofx_executable + - run_jedi_variational_executable - get_observations - eva_observations - - run_jedi_executable_base type: duration bundles: @@ -106,9 +108,10 @@ crtm_coeff_dir: prompt: What is the path to the CRTM coefficient files? tasks: - save_obs_diags + - run_jedi_hofx_executable + - run_jedi_variational_executable - get_observations - eva_observations - - run_jedi_executable_base type: string existing_geos_gcm_build_path: @@ -213,7 +216,7 @@ gradient_norm_reduction: - all prompt: What value of gradient norm reduction for convergence? tasks: - - run_jedi_executable_base + - run_jedi_variational_executable type: float horizontal_resolution: @@ -229,7 +232,8 @@ horizontal_resolution: - store_background - generate_b_climatology - get_background - - run_jedi_executable_base + - run_jedi_hofx_executable + - run_jedi_variational_executable type: string-drop-list jedi_bkg_filename_template: @@ -266,7 +270,8 @@ jedi_forecast_model: options: defer_to_model prompt: What forecast model should be used within JEDI for 4D window propagation? tasks: - - run_jedi_executable_base + - run_jedi_hofx_executable + - run_jedi_variational_executable type: string-drop-list minimizer: @@ -277,7 +282,7 @@ minimizer: options: defer_to_model prompt: Which data assimilation minimizer do you wish to use? tasks: - - run_jedi_executable_base + - run_jedi_variational_executable type: string-drop-list npx_proc: @@ -291,7 +296,6 @@ npx_proc: - generate_b_climatology - run_jedi_hofx_executable - run_jedi_variational_executable - - run_jedi_executable_base type: integer npy_proc: @@ -305,7 +309,6 @@ npy_proc: - generate_b_climatology - run_jedi_hofx_executable - run_jedi_variational_executable - - run_jedi_executable_base type: integer number_of_iterations: @@ -316,7 +319,7 @@ number_of_iterations: prompt: What number of iterations do you wish to use for each outer loop? Provide a list of integers the same length as the number of outer loops. tasks: - - run_jedi_executable_base + - run_jedi_variational_executable type: integer-list obs_experiment: @@ -348,9 +351,10 @@ observations: prompt: Which observations do you want to include? tasks: - save_obs_diags + - run_jedi_hofx_executable + - run_jedi_variational_executable - get_observations - eva_observations - - run_jedi_executable_base type: string-check-list static_background_error_model: @@ -374,7 +378,6 @@ swell_static_files: - generate_b_climatology_by_linking - stage_jedi - generate_b_climatology - - run_jedi_executable_base type: string swell_static_files_user: @@ -410,7 +413,8 @@ vertical_resolution: - generate_b_climatology_by_linking - stage_jedi - generate_b_climatology - - run_jedi_executable_base + - run_jedi_hofx_executable + - run_jedi_variational_executable type: string-drop-list window_length: @@ -422,8 +426,9 @@ window_length: tasks: - store_background - get_background + - run_jedi_hofx_executable + - run_jedi_variational_executable - get_observations - - run_jedi_executable_base type: duration window_offset: @@ -436,9 +441,10 @@ window_offset: - store_background - save_obs_diags - get_background + - run_jedi_hofx_executable + - run_jedi_variational_executable - get_observations - eva_observations - - run_jedi_executable_base type: duration window_type: @@ -456,3 +462,4 @@ window_type: - run_jedi_hofx_executable - run_jedi_variational_executable type: string-drop-list + From ff0c3f9f72586b9f15b935ceedd9ef09fdd4439b Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 24 May 2023 13:05:15 -0400 Subject: [PATCH 028/121] convert task names to CamelCase in the questions --- src/swell/tasks/questions.yaml | 184 +++++++++--------- .../utilities/bin/list_of_task_questions.py | 3 +- src/swell/utilities/case_switching.py | 30 ++- 3 files changed, 108 insertions(+), 109 deletions(-) diff --git a/src/swell/tasks/questions.yaml b/src/swell/tasks/questions.yaml index cec64436..c7786033 100644 --- a/src/swell/tasks/questions.yaml +++ b/src/swell/tasks/questions.yaml @@ -5,8 +5,8 @@ analysis_forecast_window_offset: - all prompt: What is the duration from the middle of the window when forecasts start? tasks: - - store_background - - get_background + - StoreBackground + - GetBackground type: duration analysis_variables: @@ -16,7 +16,7 @@ analysis_variables: - all prompt: What are the analysis variables? tasks: - - run_jedi_variational_executable + - RunJediVariationalExecutable type: string-check-list background_error_model: @@ -27,7 +27,7 @@ background_error_model: options: defer_to_model prompt: Which background error model do you want to use? tasks: - - generate_b_climatology + - GenerateBClimatology type: string-drop-list background_experiment: @@ -37,8 +37,8 @@ background_experiment: - all prompt: What is the name of the name of the experiment providing the backgrounds? tasks: - - store_background - - get_background + - StoreBackground + - GetBackground type: string background_frequency: @@ -51,8 +51,8 @@ background_frequency: - all prompt: What is the frequency of the background files? tasks: - - store_background - - get_background + - StoreBackground + - GetBackground type: duration background_time_offset: @@ -63,11 +63,11 @@ background_time_offset: prompt: How long before the middle of the analysis window did the background providing forecast begin? tasks: - - save_obs_diags - - run_jedi_hofx_executable - - run_jedi_variational_executable - - get_observations - - eva_observations + - SaveObsDiags + - RunJediHofxExecutable + - RunJediVariationalExecutable + - GetObservations + - EvaObservations type: duration bundles: @@ -86,8 +86,8 @@ bundles: - saber prompt: Which JEDI bundles do you wish to build? tasks: - - build_jedi - - clone_jedi + - BuildJedi + - CloneJedi type: string-check-list clean_patterns: @@ -97,7 +97,7 @@ clean_patterns: - all prompt: Provide a list of patterns that you wish to remove from the cycle directory. tasks: - - clean_cycle + - CleanCycle type: string-list crtm_coeff_dir: @@ -107,11 +107,11 @@ crtm_coeff_dir: - geos_atmosphere prompt: What is the path to the CRTM coefficient files? tasks: - - save_obs_diags - - run_jedi_hofx_executable - - run_jedi_variational_executable - - get_observations - - eva_observations + - SaveObsDiags + - RunJediHofxExecutable + - RunJediVariationalExecutable + - GetObservations + - EvaObservations type: string existing_geos_gcm_build_path: @@ -122,7 +122,7 @@ existing_geos_gcm_build_path: value: use_existing prompt: What is the path to the existing GEOS build directory? tasks: - - build_geos_by_linking + - BuildGeosByLinking type: string existing_geos_gcm_source_path: @@ -133,7 +133,7 @@ existing_geos_gcm_source_path: value: use_existing prompt: What is the path to the existing GEOS source code directory? tasks: - - clone_geos + - CloneGeos type: string existing_jedi_build_directory: @@ -144,7 +144,7 @@ existing_jedi_build_directory: value: use_existing prompt: What is the path to the existing JEDI build directory? tasks: - - build_jedi_by_linking + - BuildJediByLinking type: string existing_jedi_source_directory: @@ -155,7 +155,7 @@ existing_jedi_source_directory: value: use_existing prompt: What is the path to the existing JEDI source code directory? tasks: - - clone_jedi + - CloneJedi type: string geos_background_restart_offset: @@ -165,7 +165,7 @@ geos_background_restart_offset: - geos_atmosphere prompt: Duration before the middle of the window used in GEOS restart tar file. tasks: - - get_background_geos_experiment + - GetBackgroundGeosExperiment type: duration geos_bkg_filename_template: @@ -175,7 +175,7 @@ geos_bkg_filename_template: - geos_atmosphere prompt: Name of the background files as output by GEOS (with date and time templated). tasks: - - get_background_geos_experiment + - GetBackgroundGeosExperiment type: string geos_bkg_tar_filename_template: @@ -185,7 +185,7 @@ geos_bkg_tar_filename_template: - geos_atmosphere prompt: Name of the GEOS backgrounds tar file (with date and time templated tasks: - - get_background_geos_experiment + - GetBackgroundGeosExperiment type: string geos_build_method: @@ -196,9 +196,9 @@ geos_build_method: - create prompt: Do you want to use an existing GEOS build or create a new build? tasks: - - build_geos - - clone_geos - - build_geos_by_linking + - BuildGeos + - CloneGeos + - BuildGeosByLinking type: string-drop-list geos_gcm_tag: @@ -206,7 +206,7 @@ geos_gcm_tag: default_value: v11.0.2 prompt: Which GEOS tag do you wish to clone? tasks: - - clone_geos + - CloneGeos type: string gradient_norm_reduction: @@ -216,7 +216,7 @@ gradient_norm_reduction: - all prompt: What value of gradient norm reduction for convergence? tasks: - - run_jedi_variational_executable + - RunJediVariationalExecutable type: float horizontal_resolution: @@ -227,13 +227,13 @@ horizontal_resolution: options: defer_to_model prompt: What is the horizontal resolution for the forecast model and backgrounds? tasks: - - generate_b_climatology_by_linking - - stage_jedi - - store_background - - generate_b_climatology - - get_background - - run_jedi_hofx_executable - - run_jedi_variational_executable + - GenerateBClimatologyByLinking + - StageJedi + - StoreBackground + - GenerateBClimatology + - GetBackground + - RunJediHofxExecutable + - RunJediVariationalExecutable type: string-drop-list jedi_bkg_filename_template: @@ -243,7 +243,7 @@ jedi_bkg_filename_template: - geos_atmosphere prompt: What is the name of the background as used by JEDI (with date and time templated) tasks: - - get_background_geos_experiment + - GetBackgroundGeosExperiment type: string jedi_build_method: @@ -254,9 +254,9 @@ jedi_build_method: - create prompt: Do you want to use an existing JEDI build or create a new build? tasks: - - build_jedi - - clone_jedi - - build_jedi_by_linking + - BuildJedi + - CloneJedi + - BuildJediByLinking type: string jedi_forecast_model: @@ -270,8 +270,8 @@ jedi_forecast_model: options: defer_to_model prompt: What forecast model should be used within JEDI for 4D window propagation? tasks: - - run_jedi_hofx_executable - - run_jedi_variational_executable + - RunJediHofxExecutable + - RunJediVariationalExecutable type: string-drop-list minimizer: @@ -282,7 +282,7 @@ minimizer: options: defer_to_model prompt: Which data assimilation minimizer do you wish to use? tasks: - - run_jedi_variational_executable + - RunJediVariationalExecutable type: string-drop-list npx_proc: @@ -292,10 +292,10 @@ npx_proc: - geos_atmosphere prompt: What number of processors do you wish to use in the x-direction? tasks: - - generate_b_climatology_by_linking - - generate_b_climatology - - run_jedi_hofx_executable - - run_jedi_variational_executable + - GenerateBClimatologyByLinking + - GenerateBClimatology + - RunJediHofxExecutable + - RunJediVariationalExecutable type: integer npy_proc: @@ -305,10 +305,10 @@ npy_proc: - geos_atmosphere prompt: What number of processors do you wish to use in the y-direction? tasks: - - generate_b_climatology_by_linking - - generate_b_climatology - - run_jedi_hofx_executable - - run_jedi_variational_executable + - GenerateBClimatologyByLinking + - GenerateBClimatology + - RunJediHofxExecutable + - RunJediVariationalExecutable type: integer number_of_iterations: @@ -319,7 +319,7 @@ number_of_iterations: prompt: What number of iterations do you wish to use for each outer loop? Provide a list of integers the same length as the number of outer loops. tasks: - - run_jedi_variational_executable + - RunJediVariationalExecutable type: integer-list obs_experiment: @@ -329,7 +329,7 @@ obs_experiment: - all prompt: What is the database providing the observations? tasks: - - get_observations + - GetObservations type: string obs_provider: @@ -339,7 +339,7 @@ obs_provider: - all prompt: What is the group providing the observations? tasks: - - get_observations + - GetObservations type: string observations: @@ -350,11 +350,11 @@ observations: options: defer_to_model prompt: Which observations do you want to include? tasks: - - save_obs_diags - - run_jedi_hofx_executable - - run_jedi_variational_executable - - get_observations - - eva_observations + - SaveObsDiags + - RunJediHofxExecutable + - RunJediVariationalExecutable + - GetObservations + - EvaObservations type: string-check-list static_background_error_model: @@ -365,7 +365,7 @@ static_background_error_model: options: defer_to_model prompt: Which static background error model do you want to use? tasks: - - generate_b_climatology_by_linking + - GenerateBClimatologyByLinking type: string-drop-list swell_static_files: @@ -375,9 +375,9 @@ swell_static_files: - all prompt: What is the path to the Swell Static files directory? tasks: - - generate_b_climatology_by_linking - - stage_jedi - - generate_b_climatology + - GenerateBClimatologyByLinking + - StageJedi + - GenerateBClimatology type: string swell_static_files_user: @@ -387,7 +387,7 @@ swell_static_files_user: - all prompt: What is the path to the user provided Swell Static Files directory? tasks: - - generate_b_climatology_by_linking + - GenerateBClimatologyByLinking type: string total_processors: @@ -397,9 +397,9 @@ total_processors: - geos_ocean prompt: What is the number of processors for JEDI? tasks: - - generate_b_climatology - - run_jedi_hofx_executable - - run_jedi_variational_executable + - GenerateBClimatology + - RunJediHofxExecutable + - RunJediVariationalExecutable type: integer vertical_resolution: @@ -410,11 +410,11 @@ vertical_resolution: options: defer_to_model prompt: What is the vertical resolution for the forecast model and background? tasks: - - generate_b_climatology_by_linking - - stage_jedi - - generate_b_climatology - - run_jedi_hofx_executable - - run_jedi_variational_executable + - GenerateBClimatologyByLinking + - StageJedi + - GenerateBClimatology + - RunJediHofxExecutable + - RunJediVariationalExecutable type: string-drop-list window_length: @@ -424,11 +424,11 @@ window_length: - all prompt: What is the duration for the data assimilation window? tasks: - - store_background - - get_background - - run_jedi_hofx_executable - - run_jedi_variational_executable - - get_observations + - StoreBackground + - GetBackground + - RunJediHofxExecutable + - RunJediVariationalExecutable + - GetObservations type: duration window_offset: @@ -438,13 +438,13 @@ window_offset: - all prompt: What is the duration between the middle of the window and the beginning? tasks: - - store_background - - save_obs_diags - - get_background - - run_jedi_hofx_executable - - run_jedi_variational_executable - - get_observations - - eva_observations + - StoreBackground + - SaveObsDiags + - GetBackground + - RunJediHofxExecutable + - RunJediVariationalExecutable + - GetObservations + - EvaObservations type: duration window_type: @@ -457,9 +457,9 @@ window_type: - 4D prompt: Do you want to use a 3D or 4D (including FGAT) window? tasks: - - store_background - - get_background - - run_jedi_hofx_executable - - run_jedi_variational_executable + - StoreBackground + - GetBackground + - RunJediHofxExecutable + - RunJediVariationalExecutable type: string-drop-list diff --git a/src/swell/utilities/bin/list_of_task_questions.py b/src/swell/utilities/bin/list_of_task_questions.py index e8364b6b..e1f5ca4a 100644 --- a/src/swell/utilities/bin/list_of_task_questions.py +++ b/src/swell/utilities/bin/list_of_task_questions.py @@ -18,6 +18,7 @@ # swell imports from swell.swell_path import get_swell_path from swell.utilities.logger import Logger +from swell.utilities.case_switching import snake_case_to_camel_case # -------------------------------------------------------------------------------------------------- @@ -92,7 +93,7 @@ def main(): if unique_key == config_key: - tasks.append(task_name) + tasks.append(snake_case_to_camel_case(task_name)) question_to_tasks = {} diff --git a/src/swell/utilities/case_switching.py b/src/swell/utilities/case_switching.py index c74eb600..900962ed 100644 --- a/src/swell/utilities/case_switching.py +++ b/src/swell/utilities/case_switching.py @@ -12,27 +12,25 @@ def camel_case_to_snake_case(CamelCaseString): # Convert a string that looks like e.g. ThisIsAString to this_is_a_string # ----------------------------------------------------------------------- + snake_case_string = "" + for char in CamelCaseString: + if char.isupper(): + snake_case_string += "_" + char.lower() + else: + snake_case_string += char + return snake_case_string.lstrip("_") - # Create empty output string - snake_case_string = '' - - # Loop over the elements in the string - for element in CamelCaseString: - # Check if element is upper case and if so prepend with underscore - if element.isupper(): - new_element = '_'+element.lower() - else: - new_element = element +# -------------------------------------------------------------------------------------------------- - # Add new element to the output string - snake_case_string = snake_case_string+new_element - # If this results in leading underscore then remove it - if snake_case_string[0] == "_": - snake_case_string = snake_case_string[1:] +def snake_case_to_camel_case(snake_case_string): - return snake_case_string + # Convert a string that looks like e.g. this_is_a_string to ThisIsAString + # ----------------------------------------------------------------------- + words = snake_case_string.split('_') + CamelCaseString = words[0].capitalize() + ''.join(word.capitalize() for word in words[1:]) + return CamelCaseString # -------------------------------------------------------------------------------------------------- From 6711c2e92d852f2bc218934ab622dd61b0d91ebb Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 24 May 2023 13:43:35 -0400 Subject: [PATCH 029/121] modify config class to create variables and methods to get them --- src/swell/tasks/base/task_base.py | 8 +--- src/swell/utilities/config.py | 67 ++++++++++++++++++++++--------- 2 files changed, 48 insertions(+), 27 deletions(-) diff --git a/src/swell/tasks/base/task_base.py b/src/swell/tasks/base/task_base.py index 283b2369..172bcfc7 100644 --- a/src/swell/tasks/base/task_base.py +++ b/src/swell/tasks/base/task_base.py @@ -60,7 +60,7 @@ def __init__(self, config_input, datetime_input, model, task_name): # Create a configuration object # ----------------------------- - self.__config__ = Config(config_input, self.logger, self.__model__) + self.config = Config(config_input, self.logger, task_name, self.__model__) # All experiment have the experiment root and id and suite # -------------------------------------------------------- @@ -101,12 +101,6 @@ def execute(self): # ---------------------------------------------------------------------------------------------- - # Method to get something from config (with fail if not existing) - def config_get(self, key, default='NODEFAULT'): - return self.__config__.get(key, default) - - # ---------------------------------------------------------------------------------------------- - # Method to get the experiment root def experiment_root(self): return self.__experiment_root__ diff --git a/src/swell/utilities/config.py b/src/swell/utilities/config.py index eb73b420..28368def 100644 --- a/src/swell/utilities/config.py +++ b/src/swell/utilities/config.py @@ -9,6 +9,8 @@ import yaml +from swell.swell_path import get_swell_path + # -------------------------------------------------------------------------------------------------- # @package configuration @@ -41,34 +43,37 @@ class Config(): # ---------------------------------------------------------------------------------------------- - def __init__(self, input_file, logger, model): + def __init__(self, input_file, logger, task_name, model): # Keep copy of owner's logger self.__logger__ = logger # Read the configuration yaml file with open(input_file, 'r') as ymlfile: - self.__config__ = yaml.safe_load(ymlfile) + experiment_dict = yaml.safe_load(ymlfile) + + # Step1: flatten the dictionary based on the model + # ------------------------------------------------ - # Get model part of the config + # Extract the model config if model is not None: # Assert the model name is found in the config - if model not in self.__config__['models'].keys(): + if model not in experiment_dict['models'].keys(): self.__logger__.abort(f'Did not find the model \'{model}\' in the ' + f'experiment configuration') # Extract the model specific part of the config - model_config = self.__config__['models'][model] + model_config = experiment_dict['models'][model] # Add model component to config - self.__config__['model_component'] = model + experiment_dict['model_component'] = model else: model_config = {} # Remove the model specific part from the full config - if 'models' in self.__config__.keys(): - del self.__config__['models'] + if 'models' in experiment_dict.keys(): + del experiment_dict['models'] # Assert that the full and model level configs have only unique keys - for key in self.__config__.keys(): + for key in experiment_dict.keys(): if key in model_config.keys(): self.__logger__.abort(f'Model config contains the key \'{key}\'. Which is ' + f'also contained in the top level config.') @@ -76,20 +81,42 @@ def __init__(self, input_file, logger, model): # Now merge the top level config and the model specific parts of the config. This prevents # tasks from accessing the config associated with any model other than the one they are # supposed to act upon. - self.__config__.update(model_config) + experiment_dict.update(model_config) - # ---------------------------------------------------------------------------------------------- + # Step 2: create variables in the object with the keys/values in the config + # ------------------------------------------------------------------------- - def get(self, key, default='NODEFAULT'): + # Open the question dictionary + with open(os.path.join(get_swell_path(), 'tasks', 'questions.yaml'), 'r') as ymlfile: + question_dict = yaml.safe_load(ymlfile) - if key in self.__config__.keys(): - return self.__config__[key] - else: - if default == 'NODEFAULT': - self.__logger__.abort(f'In config.get the key \'{key}\' was not found in the ' + - f'configuration and no default was provided.') - else: - return default + # Loop through the dictionary + for experiment_key, experiment_value in experiment_dict.items(): + + # Loop through the question dictionary + for question_key, question_value in question_dict.items(): + + # List of valid tasks + if task_name in question_value['tasks']: + + # Add this variable to the object + setattr(self, f'__{experiment_key}__', experiment_value) + + # Add a method to get the variable + setattr(self, f'{experiment_key}', self.get(experiment_key)) + + # ---------------------------------------------------------------------------------------------- + def get(self, experiment_key): + def getter(): + try: + var = getattr(self, f'__{experiment_key}__') + except Exception e: + self.logger.abort(f'In config class trying to get variable {experiment_key} but ' + + f'it was not created. Ensure that the variable is in the ' + + f'experiment configuration and that the task can access that ' + + f'key based on the rules in tasks/questions.yaml.') + return getattr(self, f'__{experiment_key}__') + return getter # ---------------------------------------------------------------------------------------------- From b9fc302d12898bbb00e9928bf3c6063c7073d838 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Doruk=20Arda=C4=9F?= <38666458+Dooruk@users.noreply.github.com> Date: Wed, 24 May 2023 13:59:53 -0400 Subject: [PATCH 030/121] Update generate_b_climatology_by_linking.py quick fix --- src/swell/tasks/generate_b_climatology_by_linking.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swell/tasks/generate_b_climatology_by_linking.py b/src/swell/tasks/generate_b_climatology_by_linking.py index 29f49d1b..5cb6efa9 100644 --- a/src/swell/tasks/generate_b_climatology_by_linking.py +++ b/src/swell/tasks/generate_b_climatology_by_linking.py @@ -41,7 +41,7 @@ def execute(self): swell_static_files_user = self.config_get('swell_static_files_user', None) # Set the destination directory - target_path = os.path.join(self.get_cycle_dir(), 'background_error_model') + target_path = os.path.join(self.cycle_dir(), 'background_error_model') os.makedirs(target_path, mode=0o777, exist_ok=True) # Source path base the part that looks like /path/to/static_background_error_model/ From 0bde1166ee52de97463dad529addbffa32fbf851 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 24 May 2023 14:08:44 -0400 Subject: [PATCH 031/121] common config availability --- src/swell/tasks/base/task_base.py | 6 +++--- src/swell/utilities/config.py | 21 ++++++++++++++------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/swell/tasks/base/task_base.py b/src/swell/tasks/base/task_base.py index 172bcfc7..c6718f75 100644 --- a/src/swell/tasks/base/task_base.py +++ b/src/swell/tasks/base/task_base.py @@ -64,12 +64,12 @@ def __init__(self, config_input, datetime_input, model, task_name): # All experiment have the experiment root and id and suite # -------------------------------------------------------- - self.__experiment_root__ = self.config_get('experiment_root') - self.__experiment_id__ = self.config_get('experiment_id') + self.__experiment_root__ = self.config.__experiment_root__ + self.__experiment_id__ = self.config.__experiment_id__ # Save the model components # ------------------------- - self.__model_components__ = self.config_get('model_components', None) + self.__model_components__ = self.config.__model_components__ # Create cycle directory # ---------------------- diff --git a/src/swell/utilities/config.py b/src/swell/utilities/config.py index 28368def..c11097fc 100644 --- a/src/swell/utilities/config.py +++ b/src/swell/utilities/config.py @@ -52,6 +52,11 @@ def __init__(self, input_file, logger, task_name, model): with open(input_file, 'r') as ymlfile: experiment_dict = yaml.safe_load(ymlfile) + # Save some things that all tasks can use + self.__experiment_root__ = experiment_dict.get('experiment_root') + self.__experiment_id__ = experiment_dict.get('experiment_id') + self.__model_components__ = experiment_dict.get('model_components') + # Step1: flatten the dictionary based on the model # ------------------------------------------------ @@ -108,15 +113,17 @@ def __init__(self, input_file, logger, task_name, model): # ---------------------------------------------------------------------------------------------- def get(self, experiment_key): - def getter(): + def getter(default=None): try: - var = getattr(self, f'__{experiment_key}__') + return getattr(self, f'__{experiment_key}__') except Exception e: - self.logger.abort(f'In config class trying to get variable {experiment_key} but ' + - f'it was not created. Ensure that the variable is in the ' + - f'experiment configuration and that the task can access that ' + - f'key based on the rules in tasks/questions.yaml.') - return getattr(self, f'__{experiment_key}__') + if default is None: + self.logger.abort(f'In config class, trying to get variable {experiment_key} ' + + f'but it was not created. Ensure that the variable is in ' + + f'the experiment configuration and that the task can ' + + f'access that key based on the rules in ' + f'tasks/questions.yaml.') + return default return getter # ---------------------------------------------------------------------------------------------- From e66856f03a4620d67fe3b7f6b558e6d437fe2159 Mon Sep 17 00:00:00 2001 From: dooruk Date: Wed, 24 May 2023 15:00:12 -0400 Subject: [PATCH 032/121] eva obs fix --- .../3dvar/geos_ocean/eva_observations.yaml | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/src/swell/suites/3dvar/geos_ocean/eva_observations.yaml b/src/swell/suites/3dvar/geos_ocean/eva_observations.yaml index 0a27c621..c3a6d007 100644 --- a/src/swell/suites/3dvar/geos_ocean/eva_observations.yaml +++ b/src/swell/suites/3dvar/geos_ocean/eva_observations.yaml @@ -129,17 +129,28 @@ diagnostics: add_legend: loc: 'upper left' statistics: - data: - variable: experiment::omanPassedQc::${variable} - statistic list: + fields: + - field_name: experiment::omanPassedQc::${variable} + xloc: 0.5 + yloc: -0.10 + kwargs: + color: 'blue' + fontsize: 8 + fontfamily: monospace + - field_name: experiment::ombgPassedQc::${variable} + xloc: 0.5 + yloc: -0.13 + kwargs: + color: 'red' + fontsize: 8 + fontfamily: monospace + statistics_variables: - n - min - mean - max - std - - name - kwargs: - fontsize: 6 + layers: - type: Histogram data: From b3d175ba2e7ba4bfd2a6c61d6c730d1bbc0472d8 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 24 May 2023 15:17:27 -0400 Subject: [PATCH 033/121] use new variable functions for config instead of config_get --- .../geos_atmosphere/geos_atmosphere.yaml | 1 + .../interfaces/geos_ocean/geos_ocean.yaml | 1 + src/swell/suites/3dvar/suites-3dvar.yaml | 4 +- .../suites/build_jedi/suites-build_jedi.yaml | 4 +- src/swell/suites/hofx/suites-hofx.yaml | 4 +- src/swell/tasks/build_geos.py | 6 +-- src/swell/tasks/build_geos_by_linking.py | 20 +++----- src/swell/tasks/build_jedi.py | 15 +++--- src/swell/tasks/build_jedi_by_linking.py | 13 ++--- src/swell/tasks/clean_cycle.py | 6 +-- src/swell/tasks/clone_geos.py | 15 ++---- src/swell/tasks/clone_jedi.py | 22 +++----- src/swell/tasks/eva_observations.py | 17 ++----- src/swell/tasks/generate_b_climatology.py | 18 +++---- .../generate_b_climatology_by_linking.py | 14 +++--- src/swell/tasks/get_background.py | 14 +++--- .../tasks/get_background_geos_experiment.py | 8 +-- src/swell/tasks/get_observations.py | 14 +++--- src/swell/tasks/run_jedi_hofx_executable.py | 36 ++++++------- .../tasks/run_jedi_variational_executable.py | 50 ++++++++----------- src/swell/tasks/save_obs_diags.py | 8 +-- src/swell/tasks/stage_jedi.py | 6 +-- src/swell/tasks/store_background.py | 14 +++--- src/swell/utilities/config.py | 19 ++++--- .../utilities/render_jedi_interface_files.py | 23 +++++---- 25 files changed, 152 insertions(+), 200 deletions(-) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/geos_atmosphere.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/geos_atmosphere.yaml index fe58d95e..a8415ea8 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/geos_atmosphere.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/geos_atmosphere.yaml @@ -1,4 +1,5 @@ jedi_interface: fv3-jedi +total_processors: {{npx_proc}}x{{npy_proc}} executables: hofx3D: fv3jedi_hofx_nomodel.x hofx4D: fv3jedi_hofx.x diff --git a/src/swell/configuration/jedi/interfaces/geos_ocean/geos_ocean.yaml b/src/swell/configuration/jedi/interfaces/geos_ocean/geos_ocean.yaml index 11c2e024..a3338c76 100644 --- a/src/swell/configuration/jedi/interfaces/geos_ocean/geos_ocean.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_ocean/geos_ocean.yaml @@ -1,4 +1,5 @@ jedi_interface: soca +total_processors: {{total_processors}} executables: hofx3D: soca_hofx3d.x hofx4D: soca_hofx.x diff --git a/src/swell/suites/3dvar/suites-3dvar.yaml b/src/swell/suites/3dvar/suites-3dvar.yaml index e7b7bde3..b6b4aa90 100644 --- a/src/swell/suites/3dvar/suites-3dvar.yaml +++ b/src/swell/suites/3dvar/suites-3dvar.yaml @@ -24,7 +24,7 @@ jedi_build_method: type: string-drop-list # Existing JEDI bundle directory -existing_source_directory: +existing_jedi_source_directory: default_value: {{existing_source_directory}} prompt: Provide the path to an existing JEDI bundle directory containing source code repos type: string @@ -33,7 +33,7 @@ existing_source_directory: value: use_existing # Existing JEDI build -existing_build_directory: +existing_jedi_build_directory: default_value: {{existing_build_directory}} prompt: Provide the path to an existing JEDI build directory type: string diff --git a/src/swell/suites/build_jedi/suites-build_jedi.yaml b/src/swell/suites/build_jedi/suites-build_jedi.yaml index cfbecf91..a5b1ab37 100644 --- a/src/swell/suites/build_jedi/suites-build_jedi.yaml +++ b/src/swell/suites/build_jedi/suites-build_jedi.yaml @@ -6,7 +6,7 @@ jedi_build_method: type: string-drop-list # Existing JEDI bundle directory -existing_source_directory: +existing_jedi_source_directory: default_value: {{existing_source_directory}} prompt: Provide the path to an existing JEDI bundle directory containing source code repos type: string @@ -15,7 +15,7 @@ existing_source_directory: value: use_existing # Existing JEDI build -existing_build_directory: +existing_jedi_build_directory: default_value: {{existing_build_directory}} prompt: Provide the path to an existing JEDI build directory type: string diff --git a/src/swell/suites/hofx/suites-hofx.yaml b/src/swell/suites/hofx/suites-hofx.yaml index c599a16f..18e228a3 100644 --- a/src/swell/suites/hofx/suites-hofx.yaml +++ b/src/swell/suites/hofx/suites-hofx.yaml @@ -24,7 +24,7 @@ jedi_build_method: type: string-drop-list # Existing JEDI bundle directory -existing_source_directory: +existing_jedi_source_directory: default_value: {{existing_source_directory}} prompt: Provide the path to an existing JEDI bundle directory containing source code repos type: string @@ -33,7 +33,7 @@ existing_source_directory: value: use_existing # Existing JEDI build -existing_build_directory: +existing_jedi_build_directory: default_value: {{existing_build_directory}} prompt: Provide the path to an existing JEDI build directory type: string diff --git a/src/swell/tasks/build_geos.py b/src/swell/tasks/build_geos.py index 653eb498..27c138b6 100644 --- a/src/swell/tasks/build_geos.py +++ b/src/swell/tasks/build_geos.py @@ -22,10 +22,6 @@ class BuildGeos(taskBase): def execute(self): - # Get the build method - # -------------------- - geos_build_method = self.config_get('geos_build_method') - # Get the experiment/geos directory # --------------------------------- swell_exp_path = self.experiment_path() @@ -38,7 +34,7 @@ def execute(self): # Check that the choice is to create build # ---------------------------------------- - if not geos_build_method == 'create': + if not self.config.geos_build_method() == 'create': self.logger.abort(f'Found \'{jedi_build_method}\' for jedi_build_method in the ' f'experiment dictionary. Must be \'create\'.') diff --git a/src/swell/tasks/build_geos_by_linking.py b/src/swell/tasks/build_geos_by_linking.py index 801b0820..1bc9610e 100644 --- a/src/swell/tasks/build_geos_by_linking.py +++ b/src/swell/tasks/build_geos_by_linking.py @@ -21,10 +21,6 @@ class BuildGeosByLinking(taskBase): def execute(self): - # Get the build method - # -------------------- - geos_build_method = self.config_get('geos_build_method') - # Get the experiment/geos directory # --------------------------------- swell_exp_path = self.experiment_path() @@ -36,18 +32,16 @@ def execute(self): # Choice to link to existing build or build GEOS # ---------------------------------------------- - if not geos_build_method == 'use_existing': - self.logger.abort(f'Found \'{geos_build_method}\' for geos_build_method in the ' - f'experiment dictionary. Must be \'use_existing\'.') - - # Get the existing build directory from the dictionary - existing_geos_gcm_build_path = self.config_get('existing_geos_gcm_build_path') + if not self.config.geos_build_method() == 'use_existing': + self.logger.abort(f'Found \'{self.config.geos_build_method()}\' for ' + + f'geos_build_method in the experiment dictionary. Must be ' + + f'\'use_existing\'.') # Assert that the existing build directory contains a bin directory - if not os.path.exists(os.path.join(existing_geos_gcm_build_path, 'bin')): + if not os.path.exists(os.path.join(self.config.existing_geos_gcm_build_path(), 'bin')): self.logger.abort(f'Existing GEOS build directory is provided but a bin ' + f'directory is not found in the path ' + - f'\'{existing_geos_gcm_build_path}\'') + f'\'{self.config.existing_geos_gcm_build_path()}\'') # Write warning to user self.logger.info('Suitable GEOS build found, linking build directory. Warning: ' + @@ -56,7 +50,7 @@ def execute(self): 'this experiment may not be reproducible if the build changes.') # Link the source code directory - link_path(existing_geos_gcm_build_path, geos_gcm_build_path) + link_path(self.config.existing_geos_gcm_build_path(), geos_gcm_build_path) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/build_jedi.py b/src/swell/tasks/build_jedi.py index 4558a8c2..472dd4a4 100644 --- a/src/swell/tasks/build_jedi.py +++ b/src/swell/tasks/build_jedi.py @@ -22,10 +22,6 @@ class BuildJedi(taskBase): def execute(self): - # Get the build method - # -------------------- - jedi_build_method = self.config_get('jedi_build_method') - # Get the experiment/jedi_bundle directory # ---------------------------------------- swell_exp_path = self.experiment_path() @@ -37,7 +33,7 @@ def execute(self): # Choice to link to existing build or build JEDI using jedi_bundle # ---------------------------------------------------------------- - if jedi_build_method == 'create': + if self.config.jedi_build_method() == 'create': # Determine which bundles need to be build model_components = self.get_model_components() @@ -49,10 +45,10 @@ def execute(self): bundles.append(meta['jedi_interface']) else: bundles_default = get_bundles() - bundles = self.config_get('bundles', bundles_default) # Generate the build dictionary - jedi_bundle_dict = set_jedi_bundle_config(bundles, jedi_bundle_source_path, + jedi_bundle_dict = set_jedi_bundle_config(self.config.bundles(bundles_default), + jedi_bundle_source_path, jedi_bundle_build_path, 24) # Perform the clone of JEDI repos @@ -63,8 +59,9 @@ def execute(self): else: - self.logger.abort(f'Found \'{jedi_build_method}\' for jedi_build_method in the ' - f'experiment dictionary. Must be \'create\'.') + self.logger.abort(f'Found \'{self.config.jedi_build_method()}\' for ' + + f'jedi_build_method in the experiment dictionary. Must be ' + + f'\'create\'.') # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/build_jedi_by_linking.py b/src/swell/tasks/build_jedi_by_linking.py index 6089547f..02d77d23 100644 --- a/src/swell/tasks/build_jedi_by_linking.py +++ b/src/swell/tasks/build_jedi_by_linking.py @@ -23,10 +23,6 @@ class BuildJediByLinking(taskBase): def execute(self): - # Get the build method - # -------------------- - jedi_build_method = self.config_get('jedi_build_method') - # Get the experiment/jedi_bundle directory # ---------------------------------------- swell_exp_path = self.experiment_path() @@ -38,10 +34,10 @@ def execute(self): # Choice to link to existing build or build JEDI using jedi_bundle # ---------------------------------------------------------------- - if jedi_build_method == 'use_existing': + if self.config.jedi_build_method() == 'use_existing': # Get the existing build directory from the dictionary - existing_jedi_build_directory = self.config_get('existing_jedi_build_directory') + existing_jedi_build_directory = self.config.existing_jedi_build_directory() # Assert that the existing build directory contains a bin directory if not os.path.exists(os.path.join(existing_jedi_build_directory, 'bin')): @@ -60,8 +56,9 @@ def execute(self): else: - self.logger.abort(f'Found \'{jedi_build_method}\' for jedi_build_method in the ' - f'experiment dictionary. Must be \'use_existing\'.') + self.logger.abort(f'Found \'{self.config.jedi_build_method()}\' for ' + + f'jedi_build_method in the experiment dictionary. Must be ' + + f'\'use_existing\'.') # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/clean_cycle.py b/src/swell/tasks/clean_cycle.py index e8207901..fd7a76d2 100644 --- a/src/swell/tasks/clean_cycle.py +++ b/src/swell/tasks/clean_cycle.py @@ -27,16 +27,14 @@ class CleanCycle(taskBase): def execute(self): - clean_list = self.config_get('clean_patterns', None) - # If no cleaning requested then exit - if clean_list is None: + if self.config.clean_list() is None: return if os.path.isdir(self.cycle_dir()): os.chdir(self.cycle_dir()) # Remove all specified files - for pattern in clean_list: + for pattern in self.config.clean_list(): if pattern == '*': continue path_parts = os.path.split(pattern) diff --git a/src/swell/tasks/clone_geos.py b/src/swell/tasks/clone_geos.py index 4c2f715d..79c5806f 100644 --- a/src/swell/tasks/clone_geos.py +++ b/src/swell/tasks/clone_geos.py @@ -24,10 +24,6 @@ class CloneGeos(taskBase): def execute(self): - # Get the build method - # -------------------- - geos_build_method = self.config_get('geos_build_method') - # Get the experiment/geos directory # --------------------------------- swell_exp_path = self.experiment_path() @@ -39,18 +35,15 @@ def execute(self): # Choice to link to existing build or build GEOS # ---------------------------------------------- - if geos_build_method == 'use_existing': - - # Get the existing bundle directory to get the source code - existing_geos_gcm_source_path = self.config_get('existing_geos_gcm_source_path') + if self.config.geos_build_method() == 'use_existing': # Link the source code directory - link_path(existing_geos_gcm_source_path, geos_gcm_source_path) + link_path(self.config.existing_geos_gcm_source_path(), geos_gcm_source_path) - elif geos_build_method == 'create': + elif self.config.geos_build_method() == 'create': # Get tag to build - geos_gcm_tag = self.config_get('geos_gcm_tag') + geos_gcm_tag = self.config.geos_gcm_tag() # Make sure tag is prepended with 'v' if geos_gcm_tag[0] != 'v': diff --git a/src/swell/tasks/clone_jedi.py b/src/swell/tasks/clone_jedi.py index 94572816..77f1f41b 100644 --- a/src/swell/tasks/clone_jedi.py +++ b/src/swell/tasks/clone_jedi.py @@ -24,10 +24,6 @@ class CloneJedi(taskBase): def execute(self): - # Get the build method - # -------------------- - jedi_build_method = self.config_get('jedi_build_method') - # Get the experiment/jedi_bundle directory # ---------------------------------------- swell_exp_path = self.experiment_path() @@ -39,15 +35,12 @@ def execute(self): # Choice to link to existing build or build JEDI using jedi_bundle # ---------------------------------------------------------------- - if jedi_build_method == 'use_existing': - - # Get the existing bundle directory to get the source code - existing_jedi_source_directory = self.config_get('existing_jedi_source_directory') + if self.config.jedi_build_method() == 'use_existing': # Link the source code directory - link_path(existing_jedi_source_directory, jedi_bundle_source_path) + link_path(self.config.existing_jedi_source_directory(), jedi_bundle_source_path) - elif jedi_build_method == 'create': + elif self.config.jedi_build_method() == 'create': # Determine which bundles need to be build model_components = self.get_model_components() @@ -59,10 +52,10 @@ def execute(self): bundles.append(meta['jedi_interface']) else: bundles_default = get_bundles() - bundles = self.config_get('bundles', bundles_default) # Generate the build dictionary - jedi_bundle_dict = set_jedi_bundle_config(bundles, jedi_bundle_source_path, + jedi_bundle_dict = set_jedi_bundle_config(self.config.bundles(bundles_default), + jedi_bundle_source_path, jedi_bundle_build_path) # Perform the clone of JEDI repos @@ -73,8 +66,9 @@ def execute(self): else: - self.logger.abort(f'Found \'{jedi_build_method}\' for jedi_build_method in the ' - f'experiment dictionary. Must be \'use_existing\' or \'create\'.') + self.logger.abort(f'Found \'{self.config.jedi_build_method()}\' for ' + + f'jedi_build_method in the experiment dictionary. Must be ' + + f'\'use_existing\' or \'create\'.') # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/eva_observations.py b/src/swell/tasks/eva_observations.py index 322c2d88..c67c6862 100644 --- a/src/swell/tasks/eva_observations.py +++ b/src/swell/tasks/eva_observations.py @@ -25,21 +25,14 @@ class EvaObservations(taskBase): def execute(self): - # Parse config for jedi_config - # ---------------------------- - background_time_offset = self.config_get('background_time_offset') - crtm_coeff_dir = self.config_get('crtm_coeff_dir', None) - observations = self.config_get('observations') - window_offset = self.config_get('window_offset') - # Compute window beginning time - window_begin = self.da_window_params.window_begin(window_offset) - background_time = self.da_window_params.background_time(window_offset, - background_time_offset) + window_begin = self.da_window_params.window_begin(self.config.window_offset()) + background_time = self.da_window_params.background_time(self.config.window_offset(), + self.config.background_time_offset()) # Create JEDI interface config templates dictionary self.jedi_rendering.add_key('background_time', background_time) - self.jedi_rendering.add_key('crtm_coeff_dir', crtm_coeff_dir) + self.jedi_rendering.add_key('crtm_coeff_dir', self.config.crtm_coeff_dir(None)) self.jedi_rendering.add_key('window_begin', window_begin) # Get the model @@ -55,7 +48,7 @@ def execute(self): # Loop over observations # ------------------- - for observation in observations: + for observation in self.config.observations(): # Load the observation dictionary observation_dict = self.jedi_rendering.render_interface_observations(observation) diff --git a/src/swell/tasks/generate_b_climatology.py b/src/swell/tasks/generate_b_climatology.py index 9513a152..786f7935 100644 --- a/src/swell/tasks/generate_b_climatology.py +++ b/src/swell/tasks/generate_b_climatology.py @@ -139,27 +139,25 @@ def execute(self): See the taskBase constructor for more information. """ - self.total_processors = self.config_get('total_processors') - self.swell_static_files = self.config_get('swell_static_files') - self.horizontal_resolution = self.config_get('horizontal_resolution') - self.vertical_resolution = self.config_get('vertical_resolution') - self.npx_proc = self.config_get('npx_proc') # Used in eval(total_processors) - self.npy_proc = self.config_get('npy_proc') # Used in eval(total_processors) + self.swell_static_files = self.config.swell_static_files() + self.horizontal_resolution = self.config.horizontal_resolution() + self.vertical_resolution = self.config.vertical_resolution() # Get the JEDI interface for this model component # ----------------------------------------------- + self.jedi_rendering.add_key('npx_proc', self.config.npx_proc(None)) + self.jedi_rendering.add_key('npy_proc', self.config.npy_proc(None)) + self.jedi_rendering.add_key('total_processors', self.config.total_processors(None)) model_component_meta = self.jedi_rendering.render_interface_meta() self.jedi_interface = model_component_meta['jedi_interface'] # Compute number of processors # ---------------------------- - self.total_processors = self.total_processors.replace('npx_proc', str(self.npx_proc)) - self.total_processors = self.total_processors.replace('npy_proc', str(self.npy_proc)) - self.np = eval(self.total_processors) + np = eval(model_component_meta['total_processors']) # Obtain and initialize proper error model # ----------------------------------------------- - self.background_error_model = self.config_get('background_error_model') + self.background_error_model = self.config.background_error_model() self.initialize_background() # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/generate_b_climatology_by_linking.py b/src/swell/tasks/generate_b_climatology_by_linking.py index 29f49d1b..746ab780 100644 --- a/src/swell/tasks/generate_b_climatology_by_linking.py +++ b/src/swell/tasks/generate_b_climatology_by_linking.py @@ -34,11 +34,11 @@ def execute(self): """ # Get the flavor of static background error model - static_background_error_model = self.config_get('static_background_error_model') + static_background_error_model = self.config.static_background_error_model() # Extract general parts of the config - swell_static_files_main = self.config_get('swell_static_files') - swell_static_files_user = self.config_get('swell_static_files_user', None) + swell_static_files_main = self.config.swell_static_files() + swell_static_files_user = self.config.swell_static_files_user(None) # Set the destination directory target_path = os.path.join(self.get_cycle_dir(), 'background_error_model') @@ -79,13 +79,13 @@ def execute(self): def append_source_path_bump(self): # First part of bump path is the model resolution - horizontal_resolution = self.config_get('horizontal_resolution') - vertical_resolution = self.config_get('vertical_resolution') + horizontal_resolution = self.config.horizontal_resolution() + vertical_resolution = self.config.vertical_resolution() res_path = horizontal_resolution + 'x' + vertical_resolution # Second part of bump path is the number of processors - npx_proc = self.config_get('npx_proc') - npy_proc = self.config_get('npy_proc') + npx_proc = self.config.npx_proc() + npy_proc = self.config.npy_proc() proc_path = str(npx_proc) + 'x' + str(npy_proc) return os.path.join(res_path, proc_path) diff --git a/src/swell/tasks/get_background.py b/src/swell/tasks/get_background.py index 340cf62f..b3a409aa 100644 --- a/src/swell/tasks/get_background.py +++ b/src/swell/tasks/get_background.py @@ -44,13 +44,13 @@ def execute(self): bkg_steps = [] # Parse config - background_experiment = self.config_get('background_experiment') - background_frequency = self.config_get('background_frequency', None) - forecast_offset = self.config_get('analysis_forecast_window_offset') - horizontal_resolution = self.config_get('horizontal_resolution') - window_length = self.config_get('window_length') - window_offset = self.config_get('window_offset') - window_type = self.config_get('window_type') + background_experiment = self.config.background_experiment() + background_frequency = self.config.background_frequency(None) + forecast_offset = self.config.analysis_forecast_window_offset() + horizontal_resolution = self.config.horizontal_resolution() + window_length = self.config.window_length() + window_offset = self.config.window_offset() + window_type = self.config.window_type() # Get window parameters local_background_time = self.da_window_params.local_background_time(window_offset, diff --git a/src/swell/tasks/get_background_geos_experiment.py b/src/swell/tasks/get_background_geos_experiment.py index ce6159d8..f188e7d7 100644 --- a/src/swell/tasks/get_background_geos_experiment.py +++ b/src/swell/tasks/get_background_geos_experiment.py @@ -34,10 +34,10 @@ def execute(self): # Parse config # ------------ - geos_background_restart_offset = self.config_get('geos_background_restart_offset') - tar_filename_template_geos = self.config_get('geos_bkg_tar_filename_template') - bkg_filename_template_geos = self.config_get('geos_bkg_filename_template') - bkg_filename_template_jedi = self.config_get('jedi_bkg_filename_template') + geos_background_restart_offset = self.config.geos_background_restart_offset() + tar_filename_template_geos = self.config.geos_bkg_tar_filename_template() + bkg_filename_template_geos = self.config.geos_bkg_filename_template() + bkg_filename_template_jedi = self.config.jedi_bkg_filename_template() # Current cycle time (middle of the window) # ----------------------------------------- diff --git a/src/swell/tasks/get_observations.py b/src/swell/tasks/get_observations.py index 55b66c40..380dfe7a 100644 --- a/src/swell/tasks/get_observations.py +++ b/src/swell/tasks/get_observations.py @@ -35,13 +35,13 @@ def execute(self): # Parse config # ------------ - obs_experiment = self.config_get('obs_experiment') - obs_provider = self.config_get('obs_provider') - background_time_offset = self.config_get('background_time_offset') - observations = self.config_get('observations') - window_length = self.config_get('window_length') - crtm_coeff_dir = self.config_get('crtm_coeff_dir', None) - window_offset = self.config_get('window_offset') + obs_experiment = self.config.obs_experiment() + obs_provider = self.config.obs_provider() + background_time_offset = self.config.background_time_offset() + observations = self.config.observations() + window_length = self.config.window_length() + crtm_coeff_dir = self.config.crtm_coeff_dir(None) + window_offset = self.config.window_offset() # Get window begin time window_begin = self.da_window_params.window_begin(window_offset) diff --git a/src/swell/tasks/run_jedi_hofx_executable.py b/src/swell/tasks/run_jedi_hofx_executable.py index 04011838..09248d71 100644 --- a/src/swell/tasks/run_jedi_hofx_executable.py +++ b/src/swell/tasks/run_jedi_hofx_executable.py @@ -26,18 +26,11 @@ def execute(self): # Path to executable being run # ---------------------------- - window_type = self.config_get('window_type') - npx_proc = self.config_get('npx_proc', None) - npy_proc = self.config_get('npy_proc', None) - total_processors = self.config_get('total_processors', None) - window_length = self.config_get('window_length') - horizontal_resolution = self.config_get('horizontal_resolution') - vertical_resolution = self.config_get('vertical_resolution') - crtm_coeff_dir = self.config_get('crtm_coeff_dir', None) - window_offset = self.config_get('window_offset') - background_time_offset = self.config_get('background_time_offset') - observations = self.config_get('observations') - jedi_forecast_model = self.config_get('jedi_forecast_model', None) + window_type = self.config.window_type() + window_offset = self.config.window_offset() + background_time_offset = self.config.background_time_offset() + number_of_iterations = self.config.number_of_iterations() + observations = self.config.observations() # Compute data assimilation window parameters background_time = self.da_window_params.background_time(window_offset, @@ -52,21 +45,22 @@ def execute(self): # Populate jedi interface templates dictionary # -------------------------------------------- self.jedi_rendering.add_key('window_begin_iso', window_begin_iso) - self.jedi_rendering.add_key('window_length', window_length) + self.jedi_rendering.add_key('window_length', self.config.window_length()) # Background - self.jedi_rendering.add_key('horizontal_resolution', horizontal_resolution) + self.jedi_rendering.add_key('horizontal_resolution', self.config.horizontal_resolution()) self.jedi_rendering.add_key('local_background_time', local_background_time) self.jedi_rendering.add_key('local_background_time_iso', local_background_time_iso) # Geometry - self.jedi_rendering.add_key('npx_proc', npx_proc) - self.jedi_rendering.add_key('npy_proc', npy_proc) - self.jedi_rendering.add_key('vertical_resolution', vertical_resolution) + self.jedi_rendering.add_key('npx_proc', self.config.npx_proc(None)) + self.jedi_rendering.add_key('npy_proc', self.config.npy_proc(None)) + self.jedi_rendering.add_key('total_processors', self.config.total_processors(None)) + self.jedi_rendering.add_key('vertical_resolution', self.config.vertical_resolution()) # Observations self.jedi_rendering.add_key('background_time', background_time) - self.jedi_rendering.add_key('crtm_coeff_dir', crtm_coeff_dir) + self.jedi_rendering.add_key('crtm_coeff_dir', self.config.crtm_coeff_dir(None)) self.jedi_rendering.add_key('window_begin', window_begin) # Jedi configuration file @@ -84,7 +78,7 @@ def execute(self): # Perform complete template rendering # ----------------------------------- jedi_dictionary_iterator(jedi_config_dict, self.jedi_rendering, window_type, observations, - jedi_forecast_model) + self.config.jedi_forecast_model(None)) # Write the expanded dictionary to YAML file # ------------------------------------------ @@ -103,9 +97,7 @@ def execute(self): # Compute number of processors # ---------------------------- - total_processors = total_processors.replace('npx_proc', str(npx_proc)) - total_processors = total_processors.replace('npy_proc', str(npy_proc)) - np = eval(total_processors) + np = eval(model_component_meta['total_processors']) # Run the JEDI executable # ----------------------- diff --git a/src/swell/tasks/run_jedi_variational_executable.py b/src/swell/tasks/run_jedi_variational_executable.py index 5855e110..a0235b07 100644 --- a/src/swell/tasks/run_jedi_variational_executable.py +++ b/src/swell/tasks/run_jedi_variational_executable.py @@ -24,22 +24,11 @@ def execute(self): # Parse configuration # ------------------- - window_type = self.config_get('window_type') - npx_proc = self.config_get('npx_proc', None) - npy_proc = self.config_get('npy_proc', None) - total_processors = self.config_get('total_processors', None) - window_length = self.config_get('window_length') - horizontal_resolution = self.config_get('horizontal_resolution') - vertical_resolution = self.config_get('vertical_resolution') - crtm_coeff_dir = self.config_get('crtm_coeff_dir', None) - window_offset = self.config_get('window_offset') - background_time_offset = self.config_get('background_time_offset') - number_of_iterations = self.config_get('number_of_iterations') - minimizer = self.config_get('minimizer') - analysis_variables = self.config_get('analysis_variables') - gradient_norm_reduction = self.config_get('gradient_norm_reduction') - observations = self.config_get('observations') - jedi_forecast_model = self.config_get('jedi_forecast_model', None) + window_type = self.config.window_type() + window_offset = self.config.window_offset() + background_time_offset = self.config.background_time_offset() + number_of_iterations = self.config.number_of_iterations() + observations = self.config.observations() # Compute data assimilation window parameters background_time = self.da_window_params.background_time(window_offset, @@ -54,25 +43,27 @@ def execute(self): # Populate jedi interface templates dictionary # -------------------------------------------- self.jedi_rendering.add_key('window_begin_iso', window_begin_iso) - self.jedi_rendering.add_key('window_length', window_length) - self.jedi_rendering.add_key('minimizer', minimizer) + self.jedi_rendering.add_key('window_length', self.config.window_length()) + self.jedi_rendering.add_key('minimizer', self.config.minimizer()) self.jedi_rendering.add_key('number_of_iterations', number_of_iterations[0]) - self.jedi_rendering.add_key('analysis_variables', analysis_variables) - self.jedi_rendering.add_key('gradient_norm_reduction', gradient_norm_reduction) + self.jedi_rendering.add_key('analysis_variables', self.config.analysis_variables()) + self.jedi_rendering.add_key('gradient_norm_reduction', + self.config.gradient_norm_reduction()) # Background - self.jedi_rendering.add_key('horizontal_resolution', horizontal_resolution) + self.jedi_rendering.add_key('horizontal_resolution', self.config.horizontal_resolution()) self.jedi_rendering.add_key('local_background_time', local_background_time) self.jedi_rendering.add_key('local_background_time_iso', local_background_time_iso) # Geometry - self.jedi_rendering.add_key('npx_proc', npx_proc) - self.jedi_rendering.add_key('npy_proc', npy_proc) - self.jedi_rendering.add_key('vertical_resolution', vertical_resolution) + self.jedi_rendering.add_key('npx_proc', self.config.npx_proc(None)) + self.jedi_rendering.add_key('npy_proc', self.config.npy_proc(None)) + self.jedi_rendering.add_key('total_processors', self.config.total_processors(None)) + self.jedi_rendering.add_key('vertical_resolution', self.config.vertical_resolution()) # Observations self.jedi_rendering.add_key('background_time', background_time) - self.jedi_rendering.add_key('crtm_coeff_dir', crtm_coeff_dir) + self.jedi_rendering.add_key('crtm_coeff_dir', self.config.crtm_coeff_dir(None)) self.jedi_rendering.add_key('window_begin', window_begin) # Jedi configuration file @@ -95,7 +86,7 @@ def execute(self): # Perform complete template rendering # ----------------------------------- jedi_dictionary_iterator(jedi_config_dict, self.jedi_rendering, window_type, observations, - jedi_forecast_model) + self.config.jedi_forecast_model(None)) # Write the expanded dictionary to YAML file @@ -108,15 +99,14 @@ def execute(self): jedi_executable = model_component_meta['executables'][f'variational{window_type}'] jedi_executable_path = os.path.join(self.experiment_path(), 'jedi_bundle', 'build', 'bin', jedi_executable) + # Compute number of processors # ---------------------------- - total_processors = total_processors.replace('npx_proc', str(npx_proc)) - total_processors = total_processors.replace('npy_proc', str(npy_proc)) - np = eval(total_processors) + np = eval(model_component_meta['total_processors']) # Run the JEDI executable # ----------------------- - run_executable(self.logger, self.logger, self.cycle_dir(), np, jedi_executable_path, jedi_config_file, + run_executable(self.logger, self.cycle_dir(), np, jedi_executable_path, jedi_config_file, output_log_file) self.logger.info('Running '+jedi_executable_path+' with '+str(np)+' processors.') diff --git a/src/swell/tasks/save_obs_diags.py b/src/swell/tasks/save_obs_diags.py index 644a625b..b3b15b90 100644 --- a/src/swell/tasks/save_obs_diags.py +++ b/src/swell/tasks/save_obs_diags.py @@ -24,10 +24,10 @@ def execute(self): # Parse config # ------------ - background_time_offset = self.config_get('background_time_offset') - crtm_coeff_dir = self.config_get('crtm_coeff_dir', None) - observations = self.config_get('observations') - window_offset = self.config_get('window_offset') + background_time_offset = self.config.background_time_offset() + crtm_coeff_dir = self.config.crtm_coeff_dir(None) + observations = self.config.observations() + window_offset = self.config.window_offset() # Get window beginning window_begin = self.da_window_params.window_begin(window_offset) diff --git a/src/swell/tasks/stage_jedi.py b/src/swell/tasks/stage_jedi.py index bc5d0b82..479cdd04 100644 --- a/src/swell/tasks/stage_jedi.py +++ b/src/swell/tasks/stage_jedi.py @@ -34,9 +34,9 @@ def execute(self): """ # Extract potential template variables from config - horizontal_resolution = self.config_get('horizontal_resolution') - swell_static_files = self.config_get('swell_static_files') - vertical_resolution = self.config_get('vertical_resolution') + horizontal_resolution = self.config.horizontal_resolution() + swell_static_files = self.config.swell_static_files() + vertical_resolution = self.config.vertical_resolution() # Add jedi interface template keys self.jedi_rendering.add_key('horizontal_resolution', horizontal_resolution) diff --git a/src/swell/tasks/store_background.py b/src/swell/tasks/store_background.py index 6eb01f27..4172c770 100644 --- a/src/swell/tasks/store_background.py +++ b/src/swell/tasks/store_background.py @@ -43,14 +43,14 @@ def execute(self): bkg_steps = [] # Parse config - window_type = self.config_get('window_type') - window_length = self.config_get('window_length') - window_offset = self.config_get('window_offset') - background_experiment = self.config_get('background_experiment') - background_frequency = self.config_get('background_frequency') + window_type = self.config.window_type() + window_length = self.config.window_length() + window_offset = self.config.window_offset() + background_experiment = self.config.background_experiment() + background_frequency = self.config.background_frequency() # Position relative to center of the window where forecast starts - forecast_offset = self.config_get('analysis_forecast_window_offset') + forecast_offset = self.config.analysis_forecast_window_offset() # Convert to datetime durations window_length_dur = isodate.parse_duration(window_length) @@ -135,6 +135,6 @@ def execute(self): file_type='bkg', fc_date_rendering='analysis', step=bkg_step, - resolution=self.config_get('horizontal_resolution'), + resolution=self.config.horizontal_resolution(), type='fc', experiment=background_experiment) diff --git a/src/swell/utilities/config.py b/src/swell/utilities/config.py index c11097fc..8a85b614 100644 --- a/src/swell/utilities/config.py +++ b/src/swell/utilities/config.py @@ -7,6 +7,7 @@ # -------------------------------------------------------------------------------------------------- +import os import yaml from swell.swell_path import get_swell_path @@ -92,17 +93,19 @@ def __init__(self, input_file, logger, task_name, model): # ------------------------------------------------------------------------- # Open the question dictionary - with open(os.path.join(get_swell_path(), 'tasks', 'questions.yaml'), 'r') as ymlfile: + with open(os.path.join(get_swell_path(), 'tasks', 'questions.yaml'), 'r') as ymlfile: question_dict = yaml.safe_load(ymlfile) # Loop through the dictionary for experiment_key, experiment_value in experiment_dict.items(): - # Loop through the question dictionary - for question_key, question_value in question_dict.items(): + key_question_dict = question_dict.get(experiment_key) - # List of valid tasks - if task_name in question_value['tasks']: + if key_question_dict is not None: + + if task_name in key_question_dict['tasks']: + + print(f'Adding {experiment_key} for {task_name}') # Add this variable to the object setattr(self, f'__{experiment_key}__', experiment_value) @@ -113,11 +116,11 @@ def __init__(self, input_file, logger, task_name, model): # ---------------------------------------------------------------------------------------------- def get(self, experiment_key): - def getter(default=None): + def getter(default='None'): try: return getattr(self, f'__{experiment_key}__') - except Exception e: - if default is None: + except Exception: + if default == 'None': self.logger.abort(f'In config class, trying to get variable {experiment_key} ' + f'but it was not created. Ensure that the variable is in ' + f'the experiment configuration and that the task can ' + diff --git a/src/swell/utilities/render_jedi_interface_files.py b/src/swell/utilities/render_jedi_interface_files.py index df472347..fdd0b322 100644 --- a/src/swell/utilities/render_jedi_interface_files.py +++ b/src/swell/utilities/render_jedi_interface_files.py @@ -80,7 +80,7 @@ def add_key(self, key, element): def __open_file_render_to_dict__(self, config_file): # Check that config file exists - self.logger.assert_abort(os.path.exists(config_file), f'In open_file_and_render failed ' + self.logger.assert_abort(os.path.exists(config_file), f'In open_file_and_render failed ' + f'to find file \'{config_file}\'') # Open file as a string @@ -111,8 +111,8 @@ def render_oops_file(self, config_name): def render_interface_model(self, config_name): # Assert that there is a jedi interface associated with the task - self.logger.assert_abort(self.jedi_interface is not None, f'In order to render a ' - f'jedi interface config file the task must have an associated' + self.logger.assert_abort(self.jedi_interface is not None, f'In order to render a ' + + f'jedi interface config file the task must have an associated' + f'jedi interface.') # Path to configuration file @@ -128,8 +128,8 @@ def render_interface_model(self, config_name): def render_interface_observations(self, config_name): # Assert that there is a jedi interface associated with the task - self.logger.assert_abort(self.jedi_interface is not None, f'In order to render a ' - f'jedi interface config file the task must have an associated' + self.logger.assert_abort(self.jedi_interface is not None, f'In order to render a ' + + f'jedi interface config file the task must have an associated' + f'jedi interface.') # Path to configuration file @@ -142,12 +142,17 @@ def render_interface_observations(self, config_name): # ---------------------------------------------------------------------------------------------- # Prepare path to interface metadata file and call rendering - def render_interface_meta(self, model_component = self.jedi_interface): + def render_interface_meta(self, model_component_in = None): + + # Optionally open a different model interface + model_component = self.jedi_interface + if model_component_in is not None: + model_component = model_component_in # Assert that there is a jedi interface associated with the task - self.logger.assert_abort(self.jedi_interface is not None, f'In order to render a ' - f'jedi interface config file the task must have an associated' - f'jedi interface.') + self.logger.assert_abort(model_component is not None, f'In order to render a jedi ' + + f'interface config file the function or object must have an ' + + f'associated jedi interface.') # Path to configuration file config_file = os.path.join(self.jedi_config_path, 'interfaces', model_component, From d0dba9fa8bbfb62d1bca9219506d74e2dc752910 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 24 May 2023 15:25:50 -0400 Subject: [PATCH 034/121] fix clean patterns --- src/swell/tasks/clean_cycle.py | 4 +-- .../utilities/bin/list_of_task_questions.py | 29 +++++-------------- 2 files changed, 10 insertions(+), 23 deletions(-) diff --git a/src/swell/tasks/clean_cycle.py b/src/swell/tasks/clean_cycle.py index fd7a76d2..5a0466ea 100644 --- a/src/swell/tasks/clean_cycle.py +++ b/src/swell/tasks/clean_cycle.py @@ -28,13 +28,13 @@ class CleanCycle(taskBase): def execute(self): # If no cleaning requested then exit - if self.config.clean_list() is None: + if self.config.clean_patterns() is None: return if os.path.isdir(self.cycle_dir()): os.chdir(self.cycle_dir()) # Remove all specified files - for pattern in self.config.clean_list(): + for pattern in self.config.clean_patterns(): if pattern == '*': continue path_parts = os.path.split(pattern) diff --git a/src/swell/utilities/bin/list_of_task_questions.py b/src/swell/utilities/bin/list_of_task_questions.py index e1f5ca4a..071ccd73 100644 --- a/src/swell/utilities/bin/list_of_task_questions.py +++ b/src/swell/utilities/bin/list_of_task_questions.py @@ -36,10 +36,6 @@ def main(): # All python files task_codes = jedi_tasks_path.rglob("*py") - # All lines of all task files - raw_task_code_lines = [] - task_names = [] - # Output file outfile_yaml = os.path.join(swell_path, 'tasks', 'questions.yaml') @@ -50,6 +46,9 @@ def main(): else: question_dict = {} + + config_keys = [] + task_names = [] for task_code in task_codes: if '__init__.py' not in str(task_code) and 'task_base.py' not in str(task_code): @@ -58,22 +57,10 @@ def main(): file_lines = file.read().split('\n') for file_line in file_lines: - if 'self.config_get' in file_line: - raw_task_code_lines.append(file_line.split('=')[1].strip()) + if 'self.config.' in file_line: + config_keys.append(file_line.split('self.config.')[1].split('(')[0].strip()) task_names.append(os.path.basename(str(task_code)).split('.')[0]) - # Extract just the config key from the string - config_keys = [] - for raw_task_code_line in raw_task_code_lines: - - config_key = raw_task_code_line.replace('self.config_get(', '') - config_key = config_key.replace(')', '') - config_key = config_key.split(',')[0].strip() - config_key = config_key.split('#')[0].strip() - config_key = config_key.replace('\'', '') - - config_keys.append(config_key) - # For each key create lists of tasks unique_keys = sorted(list(set(config_keys))) @@ -104,7 +91,7 @@ def main(): # Make sure minimal things are in the question's dictionary if 'default_value' not in question_dict_key: - question_to_tasks[unique_key]['default_value'] = defer_to_model + question_to_tasks[unique_key]['default_value'] = 'defer_to_model' if 'prompt' not in question_dict_key: question_to_tasks[unique_key]['prompt'] = 'Question' @@ -118,8 +105,8 @@ def main(): else: question_to_tasks[unique_key] = {} - question_to_tasks[unique_key]['default_value'] = defer_to_model - question_to_tasks[unique_key]['options'] = defer_to_model + question_to_tasks[unique_key]['default_value'] = 'defer_to_model' + question_to_tasks[unique_key]['options'] = 'defer_to_model' question_to_tasks[unique_key]['prompt'] = 'Question' question_to_tasks[unique_key]['type'] = 'string' question_to_tasks[unique_key]['models'] = ['all'] From a9fc04969f4bd701ed8941c2fefde015aac79d3b Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 24 May 2023 15:38:17 -0400 Subject: [PATCH 035/121] update question yaml generation for now not using config_get --- src/swell/tasks/questions.yaml | 79 ++++++++++--------- .../utilities/bin/list_of_task_questions.py | 43 +++++----- 2 files changed, 65 insertions(+), 57 deletions(-) diff --git a/src/swell/tasks/questions.yaml b/src/swell/tasks/questions.yaml index c7786033..5c382f15 100644 --- a/src/swell/tasks/questions.yaml +++ b/src/swell/tasks/questions.yaml @@ -5,8 +5,8 @@ analysis_forecast_window_offset: - all prompt: What is the duration from the middle of the window when forecasts start? tasks: - - StoreBackground - GetBackground + - StoreBackground type: duration analysis_variables: @@ -37,8 +37,8 @@ background_experiment: - all prompt: What is the name of the name of the experiment providing the backgrounds? tasks: - - StoreBackground - GetBackground + - StoreBackground type: string background_frequency: @@ -51,8 +51,8 @@ background_frequency: - all prompt: What is the frequency of the background files? tasks: - - StoreBackground - GetBackground + - StoreBackground type: duration background_time_offset: @@ -63,11 +63,11 @@ background_time_offset: prompt: How long before the middle of the analysis window did the background providing forecast begin? tasks: - - SaveObsDiags - - RunJediHofxExecutable - RunJediVariationalExecutable - - GetObservations - EvaObservations + - RunJediHofxExecutable + - SaveObsDiags + - GetObservations type: duration bundles: @@ -86,8 +86,8 @@ bundles: - saber prompt: Which JEDI bundles do you wish to build? tasks: - - BuildJedi - CloneJedi + - BuildJedi type: string-check-list clean_patterns: @@ -107,11 +107,11 @@ crtm_coeff_dir: - geos_atmosphere prompt: What is the path to the CRTM coefficient files? tasks: - - SaveObsDiags - - RunJediHofxExecutable - RunJediVariationalExecutable - - GetObservations - EvaObservations + - RunJediHofxExecutable + - SaveObsDiags + - GetObservations type: string existing_geos_gcm_build_path: @@ -196,9 +196,9 @@ geos_build_method: - create prompt: Do you want to use an existing GEOS build or create a new build? tasks: - - BuildGeos - - CloneGeos - BuildGeosByLinking + - CloneGeos + - BuildGeos type: string-drop-list geos_gcm_tag: @@ -227,13 +227,13 @@ horizontal_resolution: options: defer_to_model prompt: What is the horizontal resolution for the forecast model and backgrounds? tasks: - - GenerateBClimatologyByLinking - - StageJedi - - StoreBackground - GenerateBClimatology - - GetBackground - - RunJediHofxExecutable + - StoreBackground - RunJediVariationalExecutable + - StageJedi + - GenerateBClimatologyByLinking + - RunJediHofxExecutable + - GetBackground type: string-drop-list jedi_bkg_filename_template: @@ -254,9 +254,9 @@ jedi_build_method: - create prompt: Do you want to use an existing JEDI build or create a new build? tasks: - - BuildJedi - - CloneJedi - BuildJediByLinking + - CloneJedi + - BuildJedi type: string jedi_forecast_model: @@ -292,9 +292,9 @@ npx_proc: - geos_atmosphere prompt: What number of processors do you wish to use in the x-direction? tasks: - - GenerateBClimatologyByLinking - - GenerateBClimatology - RunJediHofxExecutable + - GenerateBClimatology + - GenerateBClimatologyByLinking - RunJediVariationalExecutable type: integer @@ -305,9 +305,9 @@ npy_proc: - geos_atmosphere prompt: What number of processors do you wish to use in the y-direction? tasks: - - GenerateBClimatologyByLinking - - GenerateBClimatology - RunJediHofxExecutable + - GenerateBClimatology + - GenerateBClimatologyByLinking - RunJediVariationalExecutable type: integer @@ -319,6 +319,7 @@ number_of_iterations: prompt: What number of iterations do you wish to use for each outer loop? Provide a list of integers the same length as the number of outer loops. tasks: + - RunJediHofxExecutable - RunJediVariationalExecutable type: integer-list @@ -350,11 +351,11 @@ observations: options: defer_to_model prompt: Which observations do you want to include? tasks: - - SaveObsDiags - - RunJediHofxExecutable - RunJediVariationalExecutable - - GetObservations - EvaObservations + - RunJediHofxExecutable + - SaveObsDiags + - GetObservations type: string-check-list static_background_error_model: @@ -375,9 +376,9 @@ swell_static_files: - all prompt: What is the path to the Swell Static files directory? tasks: - - GenerateBClimatologyByLinking - - StageJedi - GenerateBClimatology + - StageJedi + - GenerateBClimatologyByLinking type: string swell_static_files_user: @@ -410,11 +411,11 @@ vertical_resolution: options: defer_to_model prompt: What is the vertical resolution for the forecast model and background? tasks: - - GenerateBClimatologyByLinking - - StageJedi - GenerateBClimatology - - RunJediHofxExecutable - RunJediVariationalExecutable + - StageJedi + - GenerateBClimatologyByLinking + - RunJediHofxExecutable type: string-drop-list window_length: @@ -424,10 +425,10 @@ window_length: - all prompt: What is the duration for the data assimilation window? tasks: + - RunJediVariationalExecutable - StoreBackground - - GetBackground - RunJediHofxExecutable - - RunJediVariationalExecutable + - GetBackground - GetObservations type: duration @@ -438,13 +439,13 @@ window_offset: - all prompt: What is the duration between the middle of the window and the beginning? tasks: - - StoreBackground - SaveObsDiags - - GetBackground - - RunJediHofxExecutable - RunJediVariationalExecutable - - GetObservations + - StoreBackground - EvaObservations + - RunJediHofxExecutable + - GetBackground + - GetObservations type: duration window_type: @@ -457,9 +458,9 @@ window_type: - 4D prompt: Do you want to use a 3D or 4D (including FGAT) window? tasks: - - StoreBackground - GetBackground - - RunJediHofxExecutable - RunJediVariationalExecutable + - RunJediHofxExecutable + - StoreBackground type: string-drop-list diff --git a/src/swell/utilities/bin/list_of_task_questions.py b/src/swell/utilities/bin/list_of_task_questions.py index 071ccd73..c50b96c3 100644 --- a/src/swell/utilities/bin/list_of_task_questions.py +++ b/src/swell/utilities/bin/list_of_task_questions.py @@ -10,9 +10,8 @@ # standard imports -from collections import OrderedDict +import glob import os -import pathlib import yaml # swell imports @@ -31,13 +30,13 @@ def main(): # Path to JEDI interface code swell_path = get_swell_path() - jedi_tasks_path = pathlib.Path(os.path.join(swell_path, 'tasks')) # All python files - task_codes = jedi_tasks_path.rglob("*py") + task_codes = glob.glob(os.path.join(get_swell_path(), 'tasks', '*.py')) + task_codes = list(filter(lambda task_code: task_code != '__init__.py', task_codes)) # Output file - outfile_yaml = os.path.join(swell_path, 'tasks', 'questions.yaml') + outfile_yaml = os.path.join(get_swell_path(), 'tasks', 'questions.yaml') # Read input file into dictionary if os.path.exists(outfile_yaml): @@ -46,20 +45,27 @@ def main(): else: question_dict = {} + # Now safe to overwrite file + outfile = open(outfile_yaml, 'w') + # Loop through task code and accumulate all lines containing a use of config config_keys = [] task_names = [] for task_code in task_codes: - if '__init__.py' not in str(task_code) and 'task_base.py' not in str(task_code): + # Open code for this task + with open(task_code, 'r') as file: + + # Loop over lines and append if line contains + for file_line in file.read().split('\n'): + if 'self.config.' in file_line: - with open(task_code, 'r') as file: - file_lines = file.read().split('\n') + config_key = file_line.split('self.config.')[1].split('(')[0].strip() + task_name = os.path.basename(str(task_code)).split('.')[0] + task_name = snake_case_to_camel_case(task_name) - for file_line in file_lines: - if 'self.config.' in file_line: - config_keys.append(file_line.split('self.config.')[1].split('(')[0].strip()) - task_names.append(os.path.basename(str(task_code)).split('.')[0]) + config_keys.append(config_key) + task_names.append(task_name) # For each key create lists of tasks unique_keys = sorted(list(set(config_keys))) @@ -67,10 +73,6 @@ def main(): # question to task dictionary question_to_tasks = {} - # Output file - outfile_yaml = os.path.join(swell_path, 'tasks', 'questions.yaml') - outfile = open(outfile_yaml, 'w') - # Task for each key for unique_key in unique_keys: @@ -80,11 +82,16 @@ def main(): if unique_key == config_key: - tasks.append(snake_case_to_camel_case(task_name)) + tasks.append(task_name) + # Make sure tasks are unique + tasks = list(set(tasks)) + + # Create dictionary to hold question components question_to_tasks = {} if unique_key in question_dict: + question_to_tasks[unique_key] = question_dict[unique_key] question_dict_key = question_dict[unique_key] @@ -102,8 +109,8 @@ def main(): if 'ask_question' not in question_dict_key: question_to_tasks[unique_key]['ask_question'] = True - else: + question_to_tasks[unique_key] = {} question_to_tasks[unique_key]['default_value'] = 'defer_to_model' question_to_tasks[unique_key]['options'] = 'defer_to_model' From 2f0d602d8e8d107c0dcb0f9fea9460498bd324eb Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 24 May 2023 15:53:26 -0400 Subject: [PATCH 036/121] change jedi build/source in platform --- src/swell/deployment/platforms/nccs_discover/experiment.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/swell/deployment/platforms/nccs_discover/experiment.yaml b/src/swell/deployment/platforms/nccs_discover/experiment.yaml index 0259e196..b0433e14 100644 --- a/src/swell/deployment/platforms/nccs_discover/experiment.yaml +++ b/src/swell/deployment/platforms/nccs_discover/experiment.yaml @@ -3,6 +3,6 @@ experiment_root: /discover/nobackup/${USER}/SwellExperiments platform: nccs_discover swell_static_files: /discover/nobackup/drholdaw/SwellStaticFiles r2d2_local_path: /discover/nobackup/${USER}/R2D2DataStore/Local -existing_source_directory: /discover/nobackup/gmao_ci/swell/nightly/latest/jedi_build/ -existing_build_directory: /discover/nobackup/gmao_ci/swell/nightly/latest/jedi_build/build-intel-release/ +existing_jedi_source_directory: /discover/nobackup/gmao_ci/swell/nightly/latest/jedi_build/ +existing_jedi_build_directory: /discover/nobackup/gmao_ci/swell/nightly/latest/jedi_build/build-intel-release/ crtm_coeff_dir: /discover/nobackup/drholdaw/SwellStaticFiles/jedi/crtm_coefficients/2.4/ From ed4751841d4f0ed5ef1d813e1b03382d8df4b029 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Thu, 25 May 2023 09:29:51 -0400 Subject: [PATCH 037/121] add code for genrating default dictionaries --- setup.py | 9 +- .../generate_task_question_default_dicts.py | 141 ++++++++++++++++++ ...ons.py => generate_task_questions_dict.py} | 0 3 files changed, 146 insertions(+), 4 deletions(-) create mode 100644 src/swell/utilities/bin/generate_task_question_default_dicts.py rename src/swell/utilities/bin/{list_of_task_questions.py => generate_task_questions_dict.py} (100%) diff --git a/setup.py b/setup.py index 442bef1d..579a72da 100644 --- a/setup.py +++ b/setup.py @@ -63,11 +63,12 @@ 'swell_prepare_experiment_config = swell.deployment.bin.swell_prepare_config:main', 'swell_launch_experiment = swell.deployment.bin.swell_launch_experiment:main', 'swell_sat_db_processing = swell.deployment.bin.swell_sat_db_processing:main', - 'util_check_jedi_interface_templates = \ + 'swell_util_check_jedi_interface_templates = \ swell.utilities.bin.check_jedi_interface_templates:main', - 'util_list_of_task_questions = \ - swell.utilities.bin.list_of_task_questions:main', - + 'swell_util_generate_task_questions_dict = \ + swell.utilities.bin.generate_task_questions_dict.py:main', + 'swell_util_generate_task_question_default_dicts = \ + swell.utilities.bin.generate_task_question_default_dicts.py:main', ], }, ) diff --git a/src/swell/utilities/bin/generate_task_question_default_dicts.py b/src/swell/utilities/bin/generate_task_question_default_dicts.py new file mode 100644 index 00000000..6f84fe5b --- /dev/null +++ b/src/swell/utilities/bin/generate_task_question_default_dicts.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python + +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + + +# standard imports +import glob +import os +import yaml + +# swell imports +from swell.swell_path import get_swell_path +from swell.utilities.logger import Logger +from swell.utilities.case_switching import snake_case_to_camel_case + + +# -------------------------------------------------------------------------------------------------- + + +def main(): + + # Create a logger + logger = Logger('ListOfTaskQuestions') + + # Path to JEDI interface code + swell_path = get_swell_path() + + # Output file + task_questions_config = os.path.join(get_swell_path(), 'tasks', 'questions.yaml') + + # Read input file into dictionary + if os.path.exists(task_questions_config): + with open(task_questions_config, 'r') as ymlfile: + question_dict = yaml.safe_load(ymlfile) + else: + logger.abort(f'Did not fine the task questions dictionary at {task_questions_config}') + + # Generate list of jedi interfaces + jedi_interfaces = glob.glob(os.path.join(get_swell_path(), 'configuration', 'jedi', + 'interfaces')) + + + print(jedi_interfaces) + + exit() + + + # Loop through task code and accumulate all lines containing a use of config + config_keys = [] + task_names = [] + for task_code in task_codes: + + # Open code for this task + with open(task_code, 'r') as file: + + # Loop over lines and append if line contains + for file_line in file.read().split('\n'): + if 'self.config.' in file_line: + + config_key = file_line.split('self.config.')[1].split('(')[0].strip() + task_name = os.path.basename(str(task_code)).split('.')[0] + task_name = snake_case_to_camel_case(task_name) + + config_keys.append(config_key) + task_names.append(task_name) + + # For each key create lists of tasks + unique_keys = sorted(list(set(config_keys))) + + # question to task dictionary + question_to_tasks = {} + + # Task for each key + for unique_key in unique_keys: + + tasks = [] + + for task_name, config_key in zip(task_names, config_keys): + + if unique_key == config_key: + + tasks.append(task_name) + + # Make sure tasks are unique + tasks = list(set(tasks)) + + # Create dictionary to hold question components + question_to_tasks = {} + + if unique_key in question_dict: + + question_to_tasks[unique_key] = question_dict[unique_key] + + question_dict_key = question_dict[unique_key] + + # Make sure minimal things are in the question's dictionary + if 'default_value' not in question_dict_key: + question_to_tasks[unique_key]['default_value'] = 'defer_to_model' + + if 'prompt' not in question_dict_key: + question_to_tasks[unique_key]['prompt'] = 'Question' + + if 'type' not in question_dict_key: + question_to_tasks[unique_key]['type'] = 'string' + + if 'ask_question' not in question_dict_key: + question_to_tasks[unique_key]['ask_question'] = True + + else: + + question_to_tasks[unique_key] = {} + question_to_tasks[unique_key]['default_value'] = 'defer_to_model' + question_to_tasks[unique_key]['options'] = 'defer_to_model' + question_to_tasks[unique_key]['prompt'] = 'Question' + question_to_tasks[unique_key]['type'] = 'string' + question_to_tasks[unique_key]['models'] = ['all'] + question_to_tasks[unique_key]['ask_question'] = True + + # Regardless of whether question was already in dictionary + question_to_tasks[unique_key]['tasks'] = tasks + + outfile.write(yaml.dump(question_to_tasks, default_flow_style=False)) + outfile.write('\n') + + outfile.close() + + +# -------------------------------------------------------------------------------------------------- + + +if __name__ == '__main__': + main() + + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/bin/list_of_task_questions.py b/src/swell/utilities/bin/generate_task_questions_dict.py similarity index 100% rename from src/swell/utilities/bin/list_of_task_questions.py rename to src/swell/utilities/bin/generate_task_questions_dict.py From 9412c674ec8cd87b30bb0b824573b4a53577a3e7 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Thu, 25 May 2023 09:35:45 -0400 Subject: [PATCH 038/121] fix the setup py paths --- setup.py | 4 +- src/swell/tasks/questions.yaml | 52 +++++++++---------- .../bin/generate_task_questions_dict.py | 2 +- 3 files changed, 29 insertions(+), 29 deletions(-) diff --git a/setup.py b/setup.py index 579a72da..5ca168b1 100644 --- a/setup.py +++ b/setup.py @@ -66,9 +66,9 @@ 'swell_util_check_jedi_interface_templates = \ swell.utilities.bin.check_jedi_interface_templates:main', 'swell_util_generate_task_questions_dict = \ - swell.utilities.bin.generate_task_questions_dict.py:main', + swell.utilities.bin.generate_task_questions_dict:main', 'swell_util_generate_task_question_default_dicts = \ - swell.utilities.bin.generate_task_question_default_dicts.py:main', + swell.utilities.bin.generate_task_question_default_dicts:main', ], }, ) diff --git a/src/swell/tasks/questions.yaml b/src/swell/tasks/questions.yaml index 5c382f15..1a3954d9 100644 --- a/src/swell/tasks/questions.yaml +++ b/src/swell/tasks/questions.yaml @@ -63,11 +63,11 @@ background_time_offset: prompt: How long before the middle of the analysis window did the background providing forecast begin? tasks: - - RunJediVariationalExecutable - EvaObservations + - GetObservations - RunJediHofxExecutable + - RunJediVariationalExecutable - SaveObsDiags - - GetObservations type: duration bundles: @@ -86,8 +86,8 @@ bundles: - saber prompt: Which JEDI bundles do you wish to build? tasks: - - CloneJedi - BuildJedi + - CloneJedi type: string-check-list clean_patterns: @@ -107,11 +107,11 @@ crtm_coeff_dir: - geos_atmosphere prompt: What is the path to the CRTM coefficient files? tasks: - - RunJediVariationalExecutable - EvaObservations + - GetObservations - RunJediHofxExecutable + - RunJediVariationalExecutable - SaveObsDiags - - GetObservations type: string existing_geos_gcm_build_path: @@ -196,9 +196,9 @@ geos_build_method: - create prompt: Do you want to use an existing GEOS build or create a new build? tasks: + - BuildGeos - BuildGeosByLinking - CloneGeos - - BuildGeos type: string-drop-list geos_gcm_tag: @@ -228,12 +228,12 @@ horizontal_resolution: prompt: What is the horizontal resolution for the forecast model and backgrounds? tasks: - GenerateBClimatology - - StoreBackground - - RunJediVariationalExecutable - - StageJedi - GenerateBClimatologyByLinking - - RunJediHofxExecutable - GetBackground + - RunJediHofxExecutable + - RunJediVariationalExecutable + - StageJedi + - StoreBackground type: string-drop-list jedi_bkg_filename_template: @@ -254,9 +254,9 @@ jedi_build_method: - create prompt: Do you want to use an existing JEDI build or create a new build? tasks: + - BuildJedi - BuildJediByLinking - CloneJedi - - BuildJedi type: string jedi_forecast_model: @@ -292,9 +292,9 @@ npx_proc: - geos_atmosphere prompt: What number of processors do you wish to use in the x-direction? tasks: - - RunJediHofxExecutable - GenerateBClimatology - GenerateBClimatologyByLinking + - RunJediHofxExecutable - RunJediVariationalExecutable type: integer @@ -305,9 +305,9 @@ npy_proc: - geos_atmosphere prompt: What number of processors do you wish to use in the y-direction? tasks: - - RunJediHofxExecutable - GenerateBClimatology - GenerateBClimatologyByLinking + - RunJediHofxExecutable - RunJediVariationalExecutable type: integer @@ -351,11 +351,11 @@ observations: options: defer_to_model prompt: Which observations do you want to include? tasks: - - RunJediVariationalExecutable - EvaObservations + - GetObservations - RunJediHofxExecutable + - RunJediVariationalExecutable - SaveObsDiags - - GetObservations type: string-check-list static_background_error_model: @@ -377,8 +377,8 @@ swell_static_files: prompt: What is the path to the Swell Static files directory? tasks: - GenerateBClimatology - - StageJedi - GenerateBClimatologyByLinking + - StageJedi type: string swell_static_files_user: @@ -412,10 +412,10 @@ vertical_resolution: prompt: What is the vertical resolution for the forecast model and background? tasks: - GenerateBClimatology - - RunJediVariationalExecutable - - StageJedi - GenerateBClimatologyByLinking - RunJediHofxExecutable + - RunJediVariationalExecutable + - StageJedi type: string-drop-list window_length: @@ -425,11 +425,11 @@ window_length: - all prompt: What is the duration for the data assimilation window? tasks: - - RunJediVariationalExecutable - - StoreBackground - - RunJediHofxExecutable - GetBackground - GetObservations + - RunJediHofxExecutable + - RunJediVariationalExecutable + - StoreBackground type: duration window_offset: @@ -439,13 +439,13 @@ window_offset: - all prompt: What is the duration between the middle of the window and the beginning? tasks: - - SaveObsDiags - - RunJediVariationalExecutable - - StoreBackground - EvaObservations - - RunJediHofxExecutable - GetBackground - GetObservations + - RunJediHofxExecutable + - RunJediVariationalExecutable + - SaveObsDiags + - StoreBackground type: duration window_type: @@ -459,8 +459,8 @@ window_type: prompt: Do you want to use a 3D or 4D (including FGAT) window? tasks: - GetBackground - - RunJediVariationalExecutable - RunJediHofxExecutable + - RunJediVariationalExecutable - StoreBackground type: string-drop-list diff --git a/src/swell/utilities/bin/generate_task_questions_dict.py b/src/swell/utilities/bin/generate_task_questions_dict.py index c50b96c3..61003cb0 100644 --- a/src/swell/utilities/bin/generate_task_questions_dict.py +++ b/src/swell/utilities/bin/generate_task_questions_dict.py @@ -85,7 +85,7 @@ def main(): tasks.append(task_name) # Make sure tasks are unique - tasks = list(set(tasks)) + tasks = sorted(list(set(tasks))) # Create dictionary to hold question components question_to_tasks = {} From ea33459314b50887a5fa656d8c7c1456259870d1 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Thu, 25 May 2023 10:53:11 -0400 Subject: [PATCH 039/121] move task questions --- .../{questions.yaml => task_questions.yaml} | 0 .../generate_task_question_default_dicts.py | 129 +++++++++--------- .../bin/generate_task_questions_dict.py | 2 +- 3 files changed, 62 insertions(+), 69 deletions(-) rename src/swell/tasks/{questions.yaml => task_questions.yaml} (100%) diff --git a/src/swell/tasks/questions.yaml b/src/swell/tasks/task_questions.yaml similarity index 100% rename from src/swell/tasks/questions.yaml rename to src/swell/tasks/task_questions.yaml diff --git a/src/swell/utilities/bin/generate_task_question_default_dicts.py b/src/swell/utilities/bin/generate_task_question_default_dicts.py index 6f84fe5b..8db6deb0 100644 --- a/src/swell/utilities/bin/generate_task_question_default_dicts.py +++ b/src/swell/utilities/bin/generate_task_question_default_dicts.py @@ -37,98 +37,91 @@ def main(): # Read input file into dictionary if os.path.exists(task_questions_config): with open(task_questions_config, 'r') as ymlfile: - question_dict = yaml.safe_load(ymlfile) + questions_dict = yaml.safe_load(ymlfile) else: logger.abort(f'Did not fine the task questions dictionary at {task_questions_config}') # Generate list of jedi interfaces - jedi_interfaces = glob.glob(os.path.join(get_swell_path(), 'configuration', 'jedi', - 'interfaces')) + jedi_interfaces_path = os.path.join(get_swell_path(), 'configuration', 'jedi', 'interfaces') + jedi_interfaces = [ f.path for f in os.scandir(jedi_interfaces_path) if f.is_dir() ] + jedi_interfaces = list(filter(lambda jedi_interface: '__' not in jedi_interface, + jedi_interfaces)) + jedi_interface_names = [] + for jedi_interface in jedi_interfaces: + jedi_interface_names.append(os.path.basename(jedi_interface)) + # Loop over jedi interfaces + for jedi_interface_name in jedi_interface_names: - print(jedi_interfaces) + question_defaults_dict = os.path.join(jedi_interfaces_path, jedi_interface_name, + 'question_defaults.yaml') - exit() + # Open file ready to overwrite + outfile = open(question_defaults_dict, 'w') + # Loop over main question list + for question_key, question_dict in questions_dict.items(): - # Loop through task code and accumulate all lines containing a use of config - config_keys = [] - task_names = [] - for task_code in task_codes: + # Check if question defers the default value to the jedi interface + if question_dict['default_value'] == 'defer_to_model': - # Open code for this task - with open(task_code, 'r') as file: + # If default is deferred to model the dict must contain the jedi interface list + logger.assert_abort('models' in question_dict, f'If the default for the config ' + + f'is defer to model then the question dictionary must ' + + f'contain models. Offending key: {question_key}') - # Loop over lines and append if line contains - for file_line in file.read().split('\n'): - if 'self.config.' in file_line: + # Set the required jedi interfaces for this question + jedi_interfaces_needed = question_dict['models'] + if jedi_interfaces_needed[0] == 'all' or jedi_interface_name in jedi_interfaces_needed: - config_key = file_line.split('self.config.')[1].split('(')[0].strip() - task_name = os.path.basename(str(task_code)).split('.')[0] - task_name = snake_case_to_camel_case(task_name) + # Create defaults dictionary for the question + question_dict_defaults = {question_key: { + 'default_value': 'defer_to_model' + }} - config_keys.append(config_key) - task_names.append(task_name) + if 'options' in question_dict: + question_dict_defaults[question_key]['options'] = ['defer_to_model'] - # For each key create lists of tasks - unique_keys = sorted(list(set(config_keys))) + # Write to the YAML file + outfile.write(yaml.dump(question_dict_defaults, default_flow_style=False)) + outfile.write('\n') - # question to task dictionary - question_to_tasks = {} + # Generate list of platforms + platforms_path = os.path.join(get_swell_path(), 'deployment', 'platforms') + platforms = [ f.path for f in os.scandir(platforms_path) if f.is_dir() ] + platforms = list(filter(lambda platform: '__' not in platform, platforms)) + platform_names = [] + for platform in platforms: + platform_names.append(os.path.basename(platform)) - # Task for each key - for unique_key in unique_keys: + # Loop over platforms + for platform_name in platform_names: - tasks = [] + print(platform_name) - for task_name, config_key in zip(task_names, config_keys): + question_defaults_dict = os.path.join(platforms_path, platform_name, + 'question_defaults.yaml') - if unique_key == config_key: + # Open file ready to overwrite + outfile = open(question_defaults_dict, 'w') - tasks.append(task_name) + # Loop over main question list + for question_key, question_dict in questions_dict.items(): - # Make sure tasks are unique - tasks = list(set(tasks)) + # Check if question defers the default value to the platform + if question_dict['default_value'] == 'defer_to_platform': - # Create dictionary to hold question components - question_to_tasks = {} + # Create defaults dictionary for the question + question_dict_defaults = {question_key: { + 'default_value': 'defer_to_model' + }} - if unique_key in question_dict: + if 'options' in question_dict: + question_dict_defaults[question_key]['options'] = ['defer_to_model'] - question_to_tasks[unique_key] = question_dict[unique_key] - - question_dict_key = question_dict[unique_key] - - # Make sure minimal things are in the question's dictionary - if 'default_value' not in question_dict_key: - question_to_tasks[unique_key]['default_value'] = 'defer_to_model' - - if 'prompt' not in question_dict_key: - question_to_tasks[unique_key]['prompt'] = 'Question' - - if 'type' not in question_dict_key: - question_to_tasks[unique_key]['type'] = 'string' - - if 'ask_question' not in question_dict_key: - question_to_tasks[unique_key]['ask_question'] = True - - else: - - question_to_tasks[unique_key] = {} - question_to_tasks[unique_key]['default_value'] = 'defer_to_model' - question_to_tasks[unique_key]['options'] = 'defer_to_model' - question_to_tasks[unique_key]['prompt'] = 'Question' - question_to_tasks[unique_key]['type'] = 'string' - question_to_tasks[unique_key]['models'] = ['all'] - question_to_tasks[unique_key]['ask_question'] = True - - # Regardless of whether question was already in dictionary - question_to_tasks[unique_key]['tasks'] = tasks - - outfile.write(yaml.dump(question_to_tasks, default_flow_style=False)) - outfile.write('\n') - - outfile.close() + # Write to the YAML file + outfile.write(yaml.dump(question_dict_defaults, default_flow_style=False)) + outfile.write('\n') # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/bin/generate_task_questions_dict.py b/src/swell/utilities/bin/generate_task_questions_dict.py index 61003cb0..a3c990d6 100644 --- a/src/swell/utilities/bin/generate_task_questions_dict.py +++ b/src/swell/utilities/bin/generate_task_questions_dict.py @@ -36,7 +36,7 @@ def main(): task_codes = list(filter(lambda task_code: task_code != '__init__.py', task_codes)) # Output file - outfile_yaml = os.path.join(get_swell_path(), 'tasks', 'questions.yaml') + outfile_yaml = os.path.join(get_swell_path(), 'tasks', 'task_questions.yaml') # Read input file into dictionary if os.path.exists(outfile_yaml): From c0fa630f43a1e56c87a8b392005334a49706b6a2 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Thu, 25 May 2023 11:11:43 -0400 Subject: [PATCH 040/121] initialize the task questions for components --- .../geos_atmosphere/task_questions.yaml | 0 .../interfaces/geos_ocean/task_questions.yaml | 74 +++++++++++++++++++ .../platforms/generic/task_questions.yaml | 18 +++++ .../nccs_discover/task_questions.yaml | 18 +++++ .../generate_task_question_default_dicts.py | 32 +++++--- 5 files changed, 131 insertions(+), 11 deletions(-) create mode 100644 src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml create mode 100644 src/swell/configuration/jedi/interfaces/geos_ocean/task_questions.yaml create mode 100644 src/swell/deployment/platforms/generic/task_questions.yaml create mode 100644 src/swell/deployment/platforms/nccs_discover/task_questions.yaml diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml new file mode 100644 index 00000000..e69de29b diff --git a/src/swell/configuration/jedi/interfaces/geos_ocean/task_questions.yaml b/src/swell/configuration/jedi/interfaces/geos_ocean/task_questions.yaml new file mode 100644 index 00000000..4fbafd04 --- /dev/null +++ b/src/swell/configuration/jedi/interfaces/geos_ocean/task_questions.yaml @@ -0,0 +1,74 @@ +analysis_forecast_window_offset: + default_value: defer_to_model + +analysis_variables: + default_value: defer_to_model + +background_error_model: + default_value: defer_to_model + options: + - defer_to_model + +background_experiment: + default_value: defer_to_model + +background_frequency: + default_value: defer_to_model + +background_time_offset: + default_value: defer_to_model + +clean_patterns: + default_value: defer_to_model + +gradient_norm_reduction: + default_value: defer_to_model + +horizontal_resolution: + default_value: defer_to_model + options: + - defer_to_model + +jedi_forecast_model: + default_value: defer_to_model + options: + - defer_to_model + +minimizer: + default_value: defer_to_model + options: + - defer_to_model + +number_of_iterations: + default_value: defer_to_model + +obs_experiment: + default_value: defer_to_model + +obs_provider: + default_value: defer_to_model + +observations: + default_value: defer_to_model + options: + - defer_to_model + +static_background_error_model: + default_value: defer_to_model + options: + - defer_to_model + +total_processors: + default_value: defer_to_model + +vertical_resolution: + default_value: defer_to_model + options: + - defer_to_model + +window_length: + default_value: defer_to_model + +window_offset: + default_value: defer_to_model + diff --git a/src/swell/deployment/platforms/generic/task_questions.yaml b/src/swell/deployment/platforms/generic/task_questions.yaml new file mode 100644 index 00000000..f9c8dfc0 --- /dev/null +++ b/src/swell/deployment/platforms/generic/task_questions.yaml @@ -0,0 +1,18 @@ +crtm_coeff_dir: + default_value: defer_to_model + +existing_geos_gcm_build_path: + default_value: defer_to_model + +existing_geos_gcm_source_path: + default_value: defer_to_model + +existing_jedi_build_directory: + default_value: defer_to_model + +existing_jedi_source_directory: + default_value: defer_to_model + +swell_static_files: + default_value: defer_to_model + diff --git a/src/swell/deployment/platforms/nccs_discover/task_questions.yaml b/src/swell/deployment/platforms/nccs_discover/task_questions.yaml new file mode 100644 index 00000000..f9c8dfc0 --- /dev/null +++ b/src/swell/deployment/platforms/nccs_discover/task_questions.yaml @@ -0,0 +1,18 @@ +crtm_coeff_dir: + default_value: defer_to_model + +existing_geos_gcm_build_path: + default_value: defer_to_model + +existing_geos_gcm_source_path: + default_value: defer_to_model + +existing_jedi_build_directory: + default_value: defer_to_model + +existing_jedi_source_directory: + default_value: defer_to_model + +swell_static_files: + default_value: defer_to_model + diff --git a/src/swell/utilities/bin/generate_task_question_default_dicts.py b/src/swell/utilities/bin/generate_task_question_default_dicts.py index 8db6deb0..927a4922 100644 --- a/src/swell/utilities/bin/generate_task_question_default_dicts.py +++ b/src/swell/utilities/bin/generate_task_question_default_dicts.py @@ -32,7 +32,7 @@ def main(): swell_path = get_swell_path() # Output file - task_questions_config = os.path.join(get_swell_path(), 'tasks', 'questions.yaml') + task_questions_config = os.path.join(get_swell_path(), 'tasks', 'task_questions.yaml') # Read input file into dictionary if os.path.exists(task_questions_config): @@ -54,7 +54,11 @@ def main(): for jedi_interface_name in jedi_interface_names: question_defaults_dict = os.path.join(jedi_interfaces_path, jedi_interface_name, - 'question_defaults.yaml') + 'task_questions.yaml') + + # Open the existing file + with open(question_defaults_dict, 'r') as ymlfile: + question_dict_defaults_exist = yaml.safe_load(ymlfile) # Open file ready to overwrite outfile = open(question_defaults_dict, 'w') @@ -74,13 +78,21 @@ def main(): jedi_interfaces_needed = question_dict['models'] if jedi_interfaces_needed[0] == 'all' or jedi_interface_name in jedi_interfaces_needed: - # Create defaults dictionary for the question - question_dict_defaults = {question_key: { - 'default_value': 'defer_to_model' - }} + # Create dictionary if it does not already exist + if question_key not in question_dict_defaults_exist: + + # Create defaults dictionary for the question + question_dict_defaults = {question_key: { + 'default_value': 'defer_to_model' + }} - if 'options' in question_dict: - question_dict_defaults[question_key]['options'] = ['defer_to_model'] + if 'options' in question_dict: + question_dict_defaults[question_key]['options'] = ['defer_to_model'] + + else: + + # Copy from the existing dictionary + question_dict_defaults = {question_key: question_dict_defaults_exist[question_key]} # Write to the YAML file outfile.write(yaml.dump(question_dict_defaults, default_flow_style=False)) @@ -97,10 +109,8 @@ def main(): # Loop over platforms for platform_name in platform_names: - print(platform_name) - question_defaults_dict = os.path.join(platforms_path, platform_name, - 'question_defaults.yaml') + 'task_questions.yaml') # Open file ready to overwrite outfile = open(question_defaults_dict, 'w') From a190ff35a7806699acd3d7404d970456d9efbeeb Mon Sep 17 00:00:00 2001 From: danholdaway Date: Thu, 25 May 2023 11:18:39 -0400 Subject: [PATCH 041/121] init def dict atmos --- .../geos_atmosphere/task_questions.yaml | 89 +++++++++++++++++++ .../generate_task_question_default_dicts.py | 3 + 2 files changed, 92 insertions(+) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml index e69de29b..52b3671e 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml @@ -0,0 +1,89 @@ +analysis_forecast_window_offset: + default_value: defer_to_model + +analysis_variables: + default_value: defer_to_model + +background_error_model: + default_value: defer_to_model + options: + - defer_to_model + +background_experiment: + default_value: defer_to_model + +background_frequency: + default_value: defer_to_model + +background_time_offset: + default_value: defer_to_model + +clean_patterns: + default_value: defer_to_model + +geos_background_restart_offset: + default_value: defer_to_model + +geos_bkg_filename_template: + default_value: defer_to_model + +geos_bkg_tar_filename_template: + default_value: defer_to_model + +gradient_norm_reduction: + default_value: defer_to_model + +horizontal_resolution: + default_value: defer_to_model + options: + - defer_to_model + +jedi_bkg_filename_template: + default_value: defer_to_model + +jedi_forecast_model: + default_value: defer_to_model + options: + - defer_to_model + +minimizer: + default_value: defer_to_model + options: + - defer_to_model + +npx_proc: + default_value: defer_to_model + +npy_proc: + default_value: defer_to_model + +number_of_iterations: + default_value: defer_to_model + +obs_experiment: + default_value: defer_to_model + +obs_provider: + default_value: defer_to_model + +observations: + default_value: defer_to_model + options: + - defer_to_model + +static_background_error_model: + default_value: defer_to_model + options: + - defer_to_model + +vertical_resolution: + default_value: defer_to_model + options: + - defer_to_model + +window_length: + default_value: defer_to_model + +window_offset: + default_value: defer_to_model + diff --git a/src/swell/utilities/bin/generate_task_question_default_dicts.py b/src/swell/utilities/bin/generate_task_question_default_dicts.py index 927a4922..ba84e1d6 100644 --- a/src/swell/utilities/bin/generate_task_question_default_dicts.py +++ b/src/swell/utilities/bin/generate_task_question_default_dicts.py @@ -60,6 +60,9 @@ def main(): with open(question_defaults_dict, 'r') as ymlfile: question_dict_defaults_exist = yaml.safe_load(ymlfile) + if question_dict_defaults_exist is None: + question_dict_defaults_exist = {} + # Open file ready to overwrite outfile = open(question_defaults_dict, 'w') From c368a52b4e5e15fee4366b69120b94a02623b027 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Thu, 25 May 2023 11:20:38 -0400 Subject: [PATCH 042/121] use existing for platform --- .../geos_atmosphere/task_questions.yaml | 2 +- .../interfaces/geos_ocean/task_questions.yaml | 2 +- .../nccs_discover/task_questions.yaml | 2 +- .../generate_task_question_default_dicts.py | 27 ++++++++++++++----- 4 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml index 52b3671e..86108058 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml @@ -1,5 +1,5 @@ analysis_forecast_window_offset: - default_value: defer_to_model + default_value: Now this is something analysis_variables: default_value: defer_to_model diff --git a/src/swell/configuration/jedi/interfaces/geos_ocean/task_questions.yaml b/src/swell/configuration/jedi/interfaces/geos_ocean/task_questions.yaml index 4fbafd04..2ecbe815 100644 --- a/src/swell/configuration/jedi/interfaces/geos_ocean/task_questions.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_ocean/task_questions.yaml @@ -1,5 +1,5 @@ analysis_forecast_window_offset: - default_value: defer_to_model + default_value: This is something analysis_variables: default_value: defer_to_model diff --git a/src/swell/deployment/platforms/nccs_discover/task_questions.yaml b/src/swell/deployment/platforms/nccs_discover/task_questions.yaml index f9c8dfc0..60904437 100644 --- a/src/swell/deployment/platforms/nccs_discover/task_questions.yaml +++ b/src/swell/deployment/platforms/nccs_discover/task_questions.yaml @@ -1,5 +1,5 @@ crtm_coeff_dir: - default_value: defer_to_model + default_value: this is something as well existing_geos_gcm_build_path: default_value: defer_to_model diff --git a/src/swell/utilities/bin/generate_task_question_default_dicts.py b/src/swell/utilities/bin/generate_task_question_default_dicts.py index ba84e1d6..bb43e124 100644 --- a/src/swell/utilities/bin/generate_task_question_default_dicts.py +++ b/src/swell/utilities/bin/generate_task_question_default_dicts.py @@ -115,6 +115,13 @@ def main(): question_defaults_dict = os.path.join(platforms_path, platform_name, 'task_questions.yaml') + # Open the existing file + with open(question_defaults_dict, 'r') as ymlfile: + question_dict_defaults_exist = yaml.safe_load(ymlfile) + + if question_dict_defaults_exist is None: + question_dict_defaults_exist = {} + # Open file ready to overwrite outfile = open(question_defaults_dict, 'w') @@ -124,13 +131,21 @@ def main(): # Check if question defers the default value to the platform if question_dict['default_value'] == 'defer_to_platform': - # Create defaults dictionary for the question - question_dict_defaults = {question_key: { - 'default_value': 'defer_to_model' - }} + # Create dictionary if it does not already exist + if question_key not in question_dict_defaults_exist: + + # Create defaults dictionary for the question + question_dict_defaults = {question_key: { + 'default_value': 'defer_to_model' + }} + + if 'options' in question_dict: + question_dict_defaults[question_key]['options'] = ['defer_to_model'] + + else: - if 'options' in question_dict: - question_dict_defaults[question_key]['options'] = ['defer_to_model'] + # Copy from the existing dictionary + question_dict_defaults = {question_key: question_dict_defaults_exist[question_key]} # Write to the YAML file outfile.write(yaml.dump(question_dict_defaults, default_flow_style=False)) From d3785ec045df1fe9566512326882a38626191290 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Thu, 25 May 2023 11:46:29 -0400 Subject: [PATCH 043/121] set the defaults --- .../geos_atmosphere/task_questions.yaml | 102 ++++++++++++------ .../interfaces/geos_ocean/task_questions.yaml | 68 +++++++----- .../platforms/generic/experiment.yaml | 8 -- .../platforms/generic/suite_questions.yaml | 8 ++ .../platforms/generic/task_questions.yaml | 13 ++- .../platforms/nccs_discover/experiment.yaml | 8 -- .../nccs_discover/suite_questions.yaml | 8 ++ .../nccs_discover/task_questions.yaml | 8 +- 8 files changed, 137 insertions(+), 86 deletions(-) delete mode 100644 src/swell/deployment/platforms/generic/experiment.yaml create mode 100644 src/swell/deployment/platforms/generic/suite_questions.yaml delete mode 100644 src/swell/deployment/platforms/nccs_discover/experiment.yaml create mode 100644 src/swell/deployment/platforms/nccs_discover/suite_questions.yaml diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml index 86108058..fa40e839 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml @@ -1,89 +1,127 @@ analysis_forecast_window_offset: - default_value: Now this is something + default_value: -PT3H analysis_variables: - default_value: defer_to_model + default_value: + - ua + - va + - t + - q + - ps + - qi + - ql + - o3 background_error_model: - default_value: defer_to_model + default_value: GSIbec options: - - defer_to_model + - GSIbec background_experiment: - default_value: defer_to_model + default_value: x0044 background_frequency: - default_value: defer_to_model + default_value: PT1H background_time_offset: - default_value: defer_to_model + default_value: PT9H clean_patterns: - default_value: defer_to_model + default_value: + - '*.nc4' + - '*.txt' + - 'logfile.*.out' geos_background_restart_offset: - default_value: defer_to_model + default_value: TODO geos_bkg_filename_template: - default_value: defer_to_model + default_value: TODO geos_bkg_tar_filename_template: - default_value: defer_to_model + default_value: TODO gradient_norm_reduction: - default_value: defer_to_model + default_value: 10e-5 horizontal_resolution: - default_value: defer_to_model + default_value: C361 options: - - defer_to_model + - C361 jedi_bkg_filename_template: - default_value: defer_to_model + default_value: TODO jedi_forecast_model: - default_value: defer_to_model + default_value: pseudo-model options: - - defer_to_model + - pseudo-model minimizer: - default_value: defer_to_model + default_value: DRIPCG options: - - defer_to_model + - DRIPCG npx_proc: - default_value: defer_to_model + default_value: 2 npy_proc: - default_value: defer_to_model + default_value: 2 number_of_iterations: - default_value: defer_to_model + default_value: 50 obs_experiment: - default_value: defer_to_model + default_value: x0044_v3 obs_provider: - default_value: defer_to_model + default_value: ncdiag observations: - default_value: defer_to_model + default_value: + - aircraft + - amsua_n19 + - amsua_aqua + - amsua_metop-a + - amsua_metop-b + - amsua_metop-c + - amsua_n15 + - amsua_n18 + - amsua_n19 + - gmi_gpm + - amsr2_gcom-w1 + - mls55_aura + - omi_aura + - ompsnm_npp options: - - defer_to_model + - aircraft + - amsua_n19 + - amsua_aqua + - amsua_metop-a + - amsua_metop-b + - amsua_metop-c + - amsua_n15 + - amsua_n18 + - amsua_n19 + - gmi_gpm + - amsr2_gcom-w1 + - mls55_aura + - omi_aura + - ompsnm_npp static_background_error_model: - default_value: defer_to_model + default_value: GSIbec options: - - defer_to_model + - GSIbec vertical_resolution: - default_value: defer_to_model + default_value: 72 options: - - defer_to_model + - 72 window_length: - default_value: defer_to_model + default_value: PT6H window_offset: - default_value: defer_to_model + default_value: PT3H diff --git a/src/swell/configuration/jedi/interfaces/geos_ocean/task_questions.yaml b/src/swell/configuration/jedi/interfaces/geos_ocean/task_questions.yaml index 2ecbe815..c27c7c07 100644 --- a/src/swell/configuration/jedi/interfaces/geos_ocean/task_questions.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_ocean/task_questions.yaml @@ -1,74 +1,88 @@ analysis_forecast_window_offset: - default_value: This is something + default_value: -PT6H analysis_variables: - default_value: defer_to_model + default_value: + - socn + - tocn + - ssh + - hocn background_error_model: - default_value: defer_to_model + default_value: bump options: - - defer_to_model + - bump background_experiment: - default_value: defer_to_model + default_value: s2s background_frequency: - default_value: defer_to_model + default_value: PT1H background_time_offset: - default_value: defer_to_model + default_value: PT9H clean_patterns: - default_value: defer_to_model + default_value: + - '*.nc4' + - '*.txt' + - '*MOM*' + - 'logfile.*.out' gradient_norm_reduction: - default_value: defer_to_model + default_value: 1e-10 horizontal_resolution: - default_value: defer_to_model + default_value: '1440x1080' options: - - defer_to_model + - '360x320' + - '1440x1080' jedi_forecast_model: - default_value: defer_to_model + default_value: NA options: - - defer_to_model + - NA minimizer: - default_value: defer_to_model + default_value: RPCG options: - - defer_to_model + - RPCG number_of_iterations: - default_value: defer_to_model + default_value: 5 obs_experiment: - default_value: defer_to_model + default_value: s2s obs_provider: - default_value: defer_to_model + default_value: odas observations: - default_value: defer_to_model + default_value: + - adt + - insitus + - insitut options: - - defer_to_model + - adt + - insitus + - insitut static_background_error_model: - default_value: defer_to_model + default_value: bump options: - - defer_to_model + - bump total_processors: - default_value: defer_to_model + default_value: 24 vertical_resolution: - default_value: defer_to_model + default_value: 75 options: - - defer_to_model + - 75 window_length: - default_value: defer_to_model + default_value: PT12H window_offset: - default_value: defer_to_model + default_value: PT6H diff --git a/src/swell/deployment/platforms/generic/experiment.yaml b/src/swell/deployment/platforms/generic/experiment.yaml deleted file mode 100644 index 7fcbf75b..00000000 --- a/src/swell/deployment/platforms/generic/experiment.yaml +++ /dev/null @@ -1,8 +0,0 @@ -experiment_id: swell-{{suite_to_run}} -experiment_root: /home/$USER/SwellExperiments -platform: generic -swell_static_files: /home/$USER/SwellStaticFiles -r2d2_local_path: /home/${USER}/R2D2DataStore/Local -existing_source_directory: /home/$USER/latest/jedi_build/ -existing_build_directory: /home/$USER/build-intel-release/ -crtm_coeff_dir: /home/$USER/CRTM_Coeffs/2.4/Little_Endian/ diff --git a/src/swell/deployment/platforms/generic/suite_questions.yaml b/src/swell/deployment/platforms/generic/suite_questions.yaml new file mode 100644 index 00000000..2793cbf0 --- /dev/null +++ b/src/swell/deployment/platforms/generic/suite_questions.yaml @@ -0,0 +1,8 @@ +experiment_id: + default_value: swell-{{suite_to_run}} + +experiment_root: + default_value: /home/${USER}/SwellExperiments + +r2d2_local_path: + default_value: /home/${USER}/R2D2DataStore/Local diff --git a/src/swell/deployment/platforms/generic/task_questions.yaml b/src/swell/deployment/platforms/generic/task_questions.yaml index f9c8dfc0..0fe4800d 100644 --- a/src/swell/deployment/platforms/generic/task_questions.yaml +++ b/src/swell/deployment/platforms/generic/task_questions.yaml @@ -1,18 +1,17 @@ crtm_coeff_dir: - default_value: defer_to_model + default_value: This would need to be provided by user existing_geos_gcm_build_path: - default_value: defer_to_model + default_value: This would need to be provided by user existing_geos_gcm_source_path: - default_value: defer_to_model + default_value: This would need to be provided by user existing_jedi_build_directory: - default_value: defer_to_model + default_value: This would need to be provided by user existing_jedi_source_directory: - default_value: defer_to_model + default_value: This would need to be provided by user swell_static_files: - default_value: defer_to_model - + default_value: This would need to be provided by user diff --git a/src/swell/deployment/platforms/nccs_discover/experiment.yaml b/src/swell/deployment/platforms/nccs_discover/experiment.yaml deleted file mode 100644 index b0433e14..00000000 --- a/src/swell/deployment/platforms/nccs_discover/experiment.yaml +++ /dev/null @@ -1,8 +0,0 @@ -experiment_id: swell-{{suite_to_run}} -experiment_root: /discover/nobackup/${USER}/SwellExperiments -platform: nccs_discover -swell_static_files: /discover/nobackup/drholdaw/SwellStaticFiles -r2d2_local_path: /discover/nobackup/${USER}/R2D2DataStore/Local -existing_jedi_source_directory: /discover/nobackup/gmao_ci/swell/nightly/latest/jedi_build/ -existing_jedi_build_directory: /discover/nobackup/gmao_ci/swell/nightly/latest/jedi_build/build-intel-release/ -crtm_coeff_dir: /discover/nobackup/drholdaw/SwellStaticFiles/jedi/crtm_coefficients/2.4/ diff --git a/src/swell/deployment/platforms/nccs_discover/suite_questions.yaml b/src/swell/deployment/platforms/nccs_discover/suite_questions.yaml new file mode 100644 index 00000000..efad7fc8 --- /dev/null +++ b/src/swell/deployment/platforms/nccs_discover/suite_questions.yaml @@ -0,0 +1,8 @@ +experiment_id: + default_value: swell-{{suite_to_run}} + +experiment_root: + default_value: /discover/nobackup/${USER}/SwellExperiments + +r2d2_local_path: + default_value: /discover/nobackup/${USER}/R2D2DataStore/Local diff --git a/src/swell/deployment/platforms/nccs_discover/task_questions.yaml b/src/swell/deployment/platforms/nccs_discover/task_questions.yaml index 60904437..6f645222 100644 --- a/src/swell/deployment/platforms/nccs_discover/task_questions.yaml +++ b/src/swell/deployment/platforms/nccs_discover/task_questions.yaml @@ -1,5 +1,5 @@ crtm_coeff_dir: - default_value: this is something as well + default_value: /discover/nobackup/drholdaw/SwellStaticFiles/jedi/crtm_coefficients/2.4/ existing_geos_gcm_build_path: default_value: defer_to_model @@ -8,11 +8,11 @@ existing_geos_gcm_source_path: default_value: defer_to_model existing_jedi_build_directory: - default_value: defer_to_model + default_value: /discover/nobackup/gmao_ci/swell/nightly/latest/jedi_build/build-intel-release/ existing_jedi_source_directory: - default_value: defer_to_model + default_value: /discover/nobackup/gmao_ci/swell/nightly/latest/jedi_build/ swell_static_files: - default_value: defer_to_model + default_value: /discover/nobackup/drholdaw/SwellStaticFiles From fd5aa838580acbd9032f2f36403205af81680d9d Mon Sep 17 00:00:00 2001 From: danholdaway Date: Thu, 25 May 2023 11:47:39 -0400 Subject: [PATCH 044/121] standardize the yamls --- .../jedi/interfaces/geos_atmosphere/task_questions.yaml | 2 +- .../jedi/interfaces/geos_ocean/task_questions.yaml | 8 ++++---- .../deployment/platforms/generic/task_questions.yaml | 1 + 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml index fa40e839..57b6eb88 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml @@ -30,7 +30,7 @@ clean_patterns: default_value: - '*.nc4' - '*.txt' - - 'logfile.*.out' + - logfile.*.out geos_background_restart_offset: default_value: TODO diff --git a/src/swell/configuration/jedi/interfaces/geos_ocean/task_questions.yaml b/src/swell/configuration/jedi/interfaces/geos_ocean/task_questions.yaml index c27c7c07..8e02881e 100644 --- a/src/swell/configuration/jedi/interfaces/geos_ocean/task_questions.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_ocean/task_questions.yaml @@ -27,16 +27,16 @@ clean_patterns: - '*.nc4' - '*.txt' - '*MOM*' - - 'logfile.*.out' + - logfile.*.out gradient_norm_reduction: default_value: 1e-10 horizontal_resolution: - default_value: '1440x1080' + default_value: 1440x1080 options: - - '360x320' - - '1440x1080' + - 360x320 + - 1440x1080 jedi_forecast_model: default_value: NA diff --git a/src/swell/deployment/platforms/generic/task_questions.yaml b/src/swell/deployment/platforms/generic/task_questions.yaml index 0fe4800d..e9fa7727 100644 --- a/src/swell/deployment/platforms/generic/task_questions.yaml +++ b/src/swell/deployment/platforms/generic/task_questions.yaml @@ -15,3 +15,4 @@ existing_jedi_source_directory: swell_static_files: default_value: This would need to be provided by user + From 41949ada970ffdf120d5ec46a116de5f633d65bb Mon Sep 17 00:00:00 2001 From: danholdaway Date: Thu, 25 May 2023 12:30:45 -0400 Subject: [PATCH 045/121] add geos directories --- .../deployment/platforms/nccs_discover/task_questions.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/swell/deployment/platforms/nccs_discover/task_questions.yaml b/src/swell/deployment/platforms/nccs_discover/task_questions.yaml index 6f645222..6bd82470 100644 --- a/src/swell/deployment/platforms/nccs_discover/task_questions.yaml +++ b/src/swell/deployment/platforms/nccs_discover/task_questions.yaml @@ -2,10 +2,10 @@ crtm_coeff_dir: default_value: /discover/nobackup/drholdaw/SwellStaticFiles/jedi/crtm_coefficients/2.4/ existing_geos_gcm_build_path: - default_value: defer_to_model + default_value: /discover/nobackup/projects/gmao/SIteam/Models/GEOSgcm-v11.0.2/GEOSgcm/install-Release existing_geos_gcm_source_path: - default_value: defer_to_model + default_value: /discover/nobackup/projects/gmao/SIteam/Models/GEOSgcm-v11.0.2/GEOSgcm existing_jedi_build_directory: default_value: /discover/nobackup/gmao_ci/swell/nightly/latest/jedi_build/build-intel-release/ From cdeabcab01831a4b67e352a51df21ef9d08fcaba Mon Sep 17 00:00:00 2001 From: danholdaway Date: Thu, 25 May 2023 13:44:31 -0400 Subject: [PATCH 046/121] add new suites --- src/swell/suites_new/hofx/flow.cylc | 149 ++++++++++++++++++ .../suites_new/hofx/suite_questions.yaml | 20 +++ src/swell/suites_new/suite_questions.yaml | 10 ++ 3 files changed, 179 insertions(+) create mode 100644 src/swell/suites_new/hofx/flow.cylc create mode 100644 src/swell/suites_new/hofx/suite_questions.yaml create mode 100644 src/swell/suites_new/suite_questions.yaml diff --git a/src/swell/suites_new/hofx/flow.cylc b/src/swell/suites_new/hofx/flow.cylc new file mode 100644 index 00000000..41369ddc --- /dev/null +++ b/src/swell/suites_new/hofx/flow.cylc @@ -0,0 +1,149 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + +# Cylc suite for executing JEDI-based h(x) + +# -------------------------------------------------------------------------------------------------- + +[scheduler] + UTC mode = True + allow implicit tasks = False + +# -------------------------------------------------------------------------------------------------- + +[scheduling] + + initial cycle point = {{start_cycle_point}} + final cycle point = {{final_cycle_point}} + runahead limit = {{runahead_limit}} + + [[graph]] + R1 = """ + # Triggers for non cycle time dependent tasks + # ------------------------------------------- + # Clone JEDI source code + CloneJedi + + # Build JEDI source code by linking + CloneJedi => BuildJediByLinking? + + # If not able to link to build create the build + BuildJediByLinking:fail? => BuildJedi + + {% for model_component in model_components %} + # Stage JEDI static files + CloneJedi => StageJedi-{{model_component}} + {% endfor %} + """ + + {% for cycle_time in cycle_times %} + {{cycle_time.cycle_time}} = """ + {% for model_component in model_components %} + {% if cycle_time[model_component] %} + # Task triggers for: {{model_component}} + # ------------------ + # Get background + GetBackground-{{model_component}} + + # Get observations + GetObservations-{{model_component}} + + # Perform staging that is cycle dependent + StageJediCycle-{{model_component}} + + # Run Jedi hofx executable + BuildJediByLinking[^]? | BuildJedi[^] => RunJediHofxExecutable-{{model_component}} + StageJedi-{{model_component}}[^] => RunJediHofxExecutable-{{model_component}} + StageJediCycle-{{model_component}} => RunJediHofxExecutable-{{model_component}} + GetBackground-{{model_component}} => RunJediHofxExecutable-{{model_component}} + GetObservations-{{model_component}} => RunJediHofxExecutable-{{model_component}} + + # EvaObservations + RunJediHofxExecutable-{{model_component}} => EvaObservations-{{model_component}} + + # Save observations + RunJediHofxExecutable-{{model_component}} => SaveObsDiags-{{model_component}} + + # Clean up large files + EvaObservations-{{model_component}} & SaveObsDiags-{{model_component}} => + CleanCycle-{{model_component}} + + {% endif %} + {% endfor %} + """ + {% endfor %} + +# -------------------------------------------------------------------------------------------------- + +[runtime] + + # Task defaults + # ------------- + [[root]] + pre-script = "source $CYLC_SUITE_DEF_PATH/modules" + + [[[environment]]] + datetime = $CYLC_TASK_CYCLE_POINT + config = $CYLC_SUITE_DEF_PATH/experiment.yaml + + # Tasks + # ----- + [[CloneJedi]] + script = "swell_task CloneJedi $config" + + [[BuildJediByLinking]] + script = "swell_task BuildJediByLinking $config" + + [[BuildJedi]] + script = "swell_task BuildJedi $config" + platform = {{platform}} + execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} + [[[directives]]] + --account = {{scheduling["BuildJedi"]["account"]}} + --qos = {{scheduling["BuildJedi"]["qos"]}} + --job-name = BuildJedi + --nodes={{scheduling["BuildJedi"]["nodes"]}} + --ntasks-per-node={{scheduling["BuildJedi"]["ntasks_per_node"]}} + --constraint={{scheduling["BuildJedi"]["constraint"]}} + + {% for model_component in model_components %} + [[StageJedi-{{model_component}}]] + script = "swell_task StageJedi $config -m {{model_component}}" + + [[StageJediCycle-{{model_component}}]] + script = "swell_task StageJedi $config -d $datetime -m {{model_component}}" + + [[ GetBackground-{{model_component}} ]] + script = "swell_task GetBackground $config -d $datetime -m {{model_component}}" + + [[GetObservations-{{model_component}}]] + script = "swell_task GetObservations $config -d $datetime -m {{model_component}}" + + [[RunJediHofxExecutable-{{model_component}}]] + script = "swell_task RunJediHofxExecutable $config -d $datetime -m {{model_component}}" + platform = {{platform}} + execution time limit = {{scheduling["RunJediHofxExecutable"]["execution_time_limit"]}} + [[[directives]]] + --account = {{scheduling["RunJediHofxExecutable"]["account"]}} + --qos = {{scheduling["RunJediHofxExecutable"]["qos"]}} + --job-name = RunJediHofxExecutable + --nodes={{scheduling["RunJediHofxExecutable"]["nodes"]}} + --ntasks-per-node={{scheduling["RunJediHofxExecutable"]["ntasks_per_node"]}} + --constraint={{scheduling["RunJediHofxExecutable"]["constraint"]}} + + [[EvaObservations-{{model_component}}]] + script = "swell_task EvaObservations $config -d $datetime -m {{model_component}}" + + [[SaveObsDiags-{{model_component}}]] + script = "swell_task SaveObsDiags $config -d $datetime -m {{model_component}}" + + [[CleanCycle-{{model_component}}]] + script = "swell_task CleanCycle $config -d $datetime -m {{model_component}}" + {% endfor %} + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites_new/hofx/suite_questions.yaml b/src/swell/suites_new/hofx/suite_questions.yaml new file mode 100644 index 00000000..d0f6825b --- /dev/null +++ b/src/swell/suites_new/hofx/suite_questions.yaml @@ -0,0 +1,20 @@ +start_cycle_point: + default_value: '2020-12-15T00:00:00Z' + prompt: What is the time of the first cycle (middle of the window)? + type: iso-datetime + +final_cycle_point: + default_value: '2020-12-15T06:00:00Z' + prompt: What is the time of the final cycle (middle of the window)? + type: iso-datetime + +runahead_limit: + default_value: 'P4' + prompt: Since this suite is non-cycling choose how many hours the workflow can run ahead? + type: string + +r2d2_local_path: + default_value: {{r2d2_local_path}} + prompt: Enter the path where R2D2 will store experiment output + type: string + diff --git a/src/swell/suites_new/suite_questions.yaml b/src/swell/suites_new/suite_questions.yaml new file mode 100644 index 00000000..0b664dd5 --- /dev/null +++ b/src/swell/suites_new/suite_questions.yaml @@ -0,0 +1,10 @@ +experiment_id: + ask_question: true + default_value: swell-{{suite_to_run}} + prompt: Enter the experiment ID + type: string + +experiment_root: + default_value: {{experiment_root}} + prompt: Enter the path where experiment will be staged + type: string From f83cbcdaf886de83b97f332646177819f33c1e9d Mon Sep 17 00:00:00 2001 From: danholdaway Date: Thu, 25 May 2023 14:00:56 -0400 Subject: [PATCH 047/121] name change in config --- src/swell/utilities/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swell/utilities/config.py b/src/swell/utilities/config.py index 8a85b614..faa5c43b 100644 --- a/src/swell/utilities/config.py +++ b/src/swell/utilities/config.py @@ -93,7 +93,7 @@ def __init__(self, input_file, logger, task_name, model): # ------------------------------------------------------------------------- # Open the question dictionary - with open(os.path.join(get_swell_path(), 'tasks', 'questions.yaml'), 'r') as ymlfile: + with open(os.path.join(get_swell_path(), 'tasks', 'task_questions.yaml'), 'r') as ymlfile: question_dict = yaml.safe_load(ymlfile) # Loop through the dictionary From 0cf378a6202a40037d2bc0f62c841dccdb98dcee Mon Sep 17 00:00:00 2001 From: danholdaway Date: Fri, 2 Jun 2023 12:36:12 -0400 Subject: [PATCH 048/121] fixes to get hofx and 3dvar running --- .../geos_atmosphere/geos_atmosphere.yaml | 2 +- src/swell/deployment/prep_config_base.py | 18 +++++-- .../3dvar/geos_ocean/eva_observations.yaml | 22 ++++++--- .../geos_ocean/suites-3dvar-geos_ocean.yaml | 14 ++---- src/swell/suites/3dvar/suites-3dvar.yaml | 4 +- .../suites-hofx-geos_atmosphere.yaml | 5 +- .../geos_ocean/suites-hofx-geos_ocean.yaml | 11 +---- src/swell/suites/hofx/suites-hofx.yaml | 4 +- .../generate_b_climatology_by_linking.py | 11 +++-- src/swell/tasks/run_jedi_hofx_executable.py | 34 ++++++++----- .../tasks/run_jedi_variational_executable.py | 48 +++++++++++-------- src/swell/tasks/task_questions.yaml | 3 ++ src/swell/utilities/config.py | 26 ++++++---- src/swell/utilities/logger.py | 14 +++++- .../utilities/render_jedi_interface_files.py | 1 + 15 files changed, 132 insertions(+), 85 deletions(-) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/geos_atmosphere.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/geos_atmosphere.yaml index a8415ea8..402d2544 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/geos_atmosphere.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/geos_atmosphere.yaml @@ -1,5 +1,5 @@ jedi_interface: fv3-jedi -total_processors: {{npx_proc}}x{{npy_proc}} +total_processors: 6*{{npx_proc}}*{{npy_proc}} executables: hofx3D: fv3jedi_hofx_nomodel.x hofx4D: fv3jedi_hofx.x diff --git a/src/swell/deployment/prep_config_base.py b/src/swell/deployment/prep_config_base.py index bc3f1fa3..437b91c9 100644 --- a/src/swell/deployment/prep_config_base.py +++ b/src/swell/deployment/prep_config_base.py @@ -56,16 +56,26 @@ def __init__(self, logger, dictionary_file, suite, platform): user_inputs_dict['platform'] = platform # Open the platform specific defaults - platform_dict_file = os.path.join(swell_path, 'deployment', 'platforms', platform, - 'experiment.yaml') - with open(platform_dict_file, 'r') as platform_dict_file_open: + platform_suite_file = os.path.join(swell_path, 'deployment', 'platforms', platform, + 'suite_questions.yaml') + platform_task_file = os.path.join(swell_path, 'deployment', 'platforms', platform, + 'task_questions.yaml') + with open(platform_suite_file, 'r') as platform_dict_file_open: platform_dict_str = platform_dict_file_open.read() + with open(platform_task_file, 'r') as platform_dict_file_open: + platform_dict_str = platform_dict_str + platform_dict_file_open.read() # Render the templates in the platform dictionary using user inputs platform_dict_str = template_string_jinja2(self.logger, platform_dict_str, user_inputs_dict) # Dictionary of templates to use whenever opening a file - self.template_dictionary = yaml.safe_load(platform_dict_str) + platform_dictionary = yaml.safe_load(platform_dict_str) + + # Flatten dictionary + self.template_dictionary = {} + for key, value in platform_dictionary.items(): + self.template_dictionary[key] = value['default_value'] + self.template_dictionary.update(user_inputs_dict) # Starting dictionary diff --git a/src/swell/suites/3dvar/geos_ocean/eva_observations.yaml b/src/swell/suites/3dvar/geos_ocean/eva_observations.yaml index 0a27c621..bbaff4e6 100644 --- a/src/swell/suites/3dvar/geos_ocean/eva_observations.yaml +++ b/src/swell/suites/3dvar/geos_ocean/eva_observations.yaml @@ -129,17 +129,27 @@ diagnostics: add_legend: loc: 'upper left' statistics: - data: - variable: experiment::omanPassedQc::${variable} - statistic list: + fields: + - field_name: experiment::ombgPassedQc::${variable} + xloc: 0.5 + yloc: -0.10 + kwargs: + color: 'black' + fontsize: 8 + fontfamily: monospace + - field_name: experiment::omanPassedQc::${variable} + xloc: 0.5 + yloc: -0.13 + kwargs: + color: 'red' + fontsize: 8 + fontfamily: monospace + statistics_variables: - n - min - mean - max - std - - name - kwargs: - fontsize: 6 layers: - type: Histogram data: diff --git a/src/swell/suites/3dvar/geos_ocean/suites-3dvar-geos_ocean.yaml b/src/swell/suites/3dvar/geos_ocean/suites-3dvar-geos_ocean.yaml index 1e90ed78..e232703c 100644 --- a/src/swell/suites/3dvar/geos_ocean/suites-3dvar-geos_ocean.yaml +++ b/src/swell/suites/3dvar/geos_ocean/suites-3dvar-geos_ocean.yaml @@ -72,15 +72,12 @@ observations: type: string-check-list # Processor layout -npx_proc: +total_processors: default_value: '24' - prompt: What processor layout (x-direction) per cube face? - type: string -npy_proc: - default_value: '1' - prompt: What processor layout (y-direction) per cube face? + prompt: What is the total number of required processors for JEDI? type: string + # Fixed options (user not prompted for these) # ------------------------------------------- fixed_options: @@ -91,14 +88,11 @@ fixed_options: default_value: RPCG prompt: Minimizer to use for gradient descent number_of_iterations: - default_value: 5 + default_value: [5] prompt: Number of iterations to use in the minimization gradient_norm_reduction: default_value: 1e-10 prompt: Threshold for convergence of the minimization - total_processors: - default_value: 'npx_proc * npy_proc' - prompt: Equation to compute total number of processors window_offset: default_value: PT12H prompt: Time from beginning to middle of the window diff --git a/src/swell/suites/3dvar/suites-3dvar.yaml b/src/swell/suites/3dvar/suites-3dvar.yaml index b6b4aa90..c068c7a7 100644 --- a/src/swell/suites/3dvar/suites-3dvar.yaml +++ b/src/swell/suites/3dvar/suites-3dvar.yaml @@ -25,7 +25,7 @@ jedi_build_method: # Existing JEDI bundle directory existing_jedi_source_directory: - default_value: {{existing_source_directory}} + default_value: {{existing_jedi_source_directory}} prompt: Provide the path to an existing JEDI bundle directory containing source code repos type: string depends: @@ -34,7 +34,7 @@ existing_jedi_source_directory: # Existing JEDI build existing_jedi_build_directory: - default_value: {{existing_build_directory}} + default_value: {{existing_jedi_build_directory}} prompt: Provide the path to an existing JEDI build directory type: string depends: diff --git a/src/swell/suites/hofx/geos_atmosphere/suites-hofx-geos_atmosphere.yaml b/src/swell/suites/hofx/geos_atmosphere/suites-hofx-geos_atmosphere.yaml index 6d4a7a89..19d2e828 100644 --- a/src/swell/suites/hofx/geos_atmosphere/suites-hofx-geos_atmosphere.yaml +++ b/src/swell/suites/hofx/geos_atmosphere/suites-hofx-geos_atmosphere.yaml @@ -92,10 +92,7 @@ fixed_options: analysis_forecast_window_offset: default_value: -PT3H prompt: Time from the middle of the window when forecasts start - total_processors: - default_value: 'npx_proc * npy_proc * 6' - prompt: Equation to compute total number of processors - model: + jedi_forecast_model: default_value: pseudo-model prompt: Type of model to use for 4D runs. background_time_offset: diff --git a/src/swell/suites/hofx/geos_ocean/suites-hofx-geos_ocean.yaml b/src/swell/suites/hofx/geos_ocean/suites-hofx-geos_ocean.yaml index 5fbe5642..9d785e55 100644 --- a/src/swell/suites/hofx/geos_ocean/suites-hofx-geos_ocean.yaml +++ b/src/swell/suites/hofx/geos_ocean/suites-hofx-geos_ocean.yaml @@ -46,13 +46,9 @@ observations: type: string-check-list # Processor layout -npx_proc: +total_processors: default_value: '24' - prompt: What processor layout (x-direction) per cube face? - type: string -npy_proc: - default_value: '1' - prompt: What processor layout (y-direction) per cube face? + prompt: What is the number of processors needed for soca? type: string # Fixed options (user not prompted for these) @@ -61,9 +57,6 @@ fixed_options: window_type: default_value: 3D prompt: Window type (3D or 4D) - total_processors: - default_value: 'npx_proc * npy_proc' - prompt: Equation to compute total number of processors window_offset: default_value: PT6H prompt: Time from beginning to middle of the window diff --git a/src/swell/suites/hofx/suites-hofx.yaml b/src/swell/suites/hofx/suites-hofx.yaml index 18e228a3..7114e936 100644 --- a/src/swell/suites/hofx/suites-hofx.yaml +++ b/src/swell/suites/hofx/suites-hofx.yaml @@ -25,7 +25,7 @@ jedi_build_method: # Existing JEDI bundle directory existing_jedi_source_directory: - default_value: {{existing_source_directory}} + default_value: {{existing_jedi_source_directory}} prompt: Provide the path to an existing JEDI bundle directory containing source code repos type: string depends: @@ -34,7 +34,7 @@ existing_jedi_source_directory: # Existing JEDI build existing_jedi_build_directory: - default_value: {{existing_build_directory}} + default_value: {{existing_jedi_build_directory}} prompt: Provide the path to an existing JEDI build directory type: string depends: diff --git a/src/swell/tasks/generate_b_climatology_by_linking.py b/src/swell/tasks/generate_b_climatology_by_linking.py index 746ab780..8e9ad8ba 100644 --- a/src/swell/tasks/generate_b_climatology_by_linking.py +++ b/src/swell/tasks/generate_b_climatology_by_linking.py @@ -41,7 +41,7 @@ def execute(self): swell_static_files_user = self.config.swell_static_files_user(None) # Set the destination directory - target_path = os.path.join(self.get_cycle_dir(), 'background_error_model') + target_path = os.path.join(self.cycle_dir(), 'background_error_model') os.makedirs(target_path, mode=0o777, exist_ok=True) # Source path base the part that looks like /path/to/static_background_error_model/ @@ -84,8 +84,13 @@ def append_source_path_bump(self): res_path = horizontal_resolution + 'x' + vertical_resolution # Second part of bump path is the number of processors - npx_proc = self.config.npx_proc() - npy_proc = self.config.npy_proc() + npx_proc = self.config.npx_proc(None) + npy_proc = self.config.npy_proc(None) + + if npx_proc is None: + npx_proc = self.config.total_processors() + npy_proc = 1 + proc_path = str(npx_proc) + 'x' + str(npy_proc) return os.path.join(res_path, proc_path) diff --git a/src/swell/tasks/run_jedi_hofx_executable.py b/src/swell/tasks/run_jedi_hofx_executable.py index 09248d71..354f12f8 100644 --- a/src/swell/tasks/run_jedi_hofx_executable.py +++ b/src/swell/tasks/run_jedi_hofx_executable.py @@ -24,13 +24,17 @@ class RunJediHofxExecutable(taskBase): def execute(self): - # Path to executable being run - # ---------------------------- + # Jedi application name + # --------------------- + jedi_application = 'hofx' + + # Parse configuration + # ------------------- window_type = self.config.window_type() window_offset = self.config.window_offset() background_time_offset = self.config.background_time_offset() - number_of_iterations = self.config.number_of_iterations() observations = self.config.observations() + jedi_forecast_model = self.config.jedi_forecast_model(None) # Compute data assimilation window parameters background_time = self.da_window_params.background_time(window_offset, @@ -53,32 +57,36 @@ def execute(self): self.jedi_rendering.add_key('local_background_time_iso', local_background_time_iso) # Geometry + self.jedi_rendering.add_key('vertical_resolution', self.config.vertical_resolution()) self.jedi_rendering.add_key('npx_proc', self.config.npx_proc(None)) self.jedi_rendering.add_key('npy_proc', self.config.npy_proc(None)) self.jedi_rendering.add_key('total_processors', self.config.total_processors(None)) - self.jedi_rendering.add_key('vertical_resolution', self.config.vertical_resolution()) # Observations self.jedi_rendering.add_key('background_time', background_time) self.jedi_rendering.add_key('crtm_coeff_dir', self.config.crtm_coeff_dir(None)) self.jedi_rendering.add_key('window_begin', window_begin) + # Model + if window_type == '4D': + self.jedi_rendering.add_key('background_frequency', self.config.background_frequency()) + # Jedi configuration file # ----------------------- - jedi_config_file = os.path.join(self.cycle_dir(), 'jedi_hofx_config.yaml') + jedi_config_file = os.path.join(self.cycle_dir(), f'jedi_{jedi_application}_config.yaml') # Output log file # --------------- - output_log_file = os.path.join(self.cycle_dir(), 'jedi_hofx_log.log') + output_log_file = os.path.join(self.cycle_dir(), f'jedi_{jedi_application}_log.log') # Open the JEDI config file and fill initial templates # ---------------------------------------------------- - jedi_config_dict = self.jedi_rendering.render_oops_file(f'hofx{window_type}') + jedi_config_dict = self.jedi_rendering.render_oops_file(f'{jedi_application}{window_type}') # Perform complete template rendering # ----------------------------------- jedi_dictionary_iterator(jedi_config_dict, self.jedi_rendering, window_type, observations, - self.config.jedi_forecast_model(None)) + jedi_forecast_model) # Write the expanded dictionary to YAML file # ------------------------------------------ @@ -89,16 +97,16 @@ def execute(self): # ------------------------------- model_component_meta = self.jedi_rendering.render_interface_meta() + # Compute number of processors + # ---------------------------- + np = eval(str(model_component_meta['total_processors'])) + # Jedi executable name # -------------------- - jedi_executable = model_component_meta['executables'][f'hofx{window_type}'] + jedi_executable = model_component_meta['executables'][f'{jedi_application}{window_type}'] jedi_executable_path = os.path.join(self.experiment_path(), 'jedi_bundle', 'build', 'bin', jedi_executable) - # Compute number of processors - # ---------------------------- - np = eval(model_component_meta['total_processors']) - # Run the JEDI executable # ----------------------- run_executable(self.logger, self.cycle_dir(), np, jedi_executable_path, jedi_config_file, diff --git a/src/swell/tasks/run_jedi_variational_executable.py b/src/swell/tasks/run_jedi_variational_executable.py index a0235b07..7936d49d 100644 --- a/src/swell/tasks/run_jedi_variational_executable.py +++ b/src/swell/tasks/run_jedi_variational_executable.py @@ -20,8 +20,14 @@ class RunJediVariationalExecutable(taskBase): + # ---------------------------------------------------------------------------------------------- + def execute(self): + # Jedi application name + # --------------------- + jedi_application = 'variational' + # Parse configuration # ------------------- window_type = self.config.window_type() @@ -29,6 +35,7 @@ def execute(self): background_time_offset = self.config.background_time_offset() number_of_iterations = self.config.number_of_iterations() observations = self.config.observations() + jedi_forecast_model = self.config.jedi_forecast_model(None) # Compute data assimilation window parameters background_time = self.da_window_params.background_time(window_offset, @@ -56,53 +63,55 @@ def execute(self): self.jedi_rendering.add_key('local_background_time_iso', local_background_time_iso) # Geometry + self.jedi_rendering.add_key('vertical_resolution', self.config.vertical_resolution()) self.jedi_rendering.add_key('npx_proc', self.config.npx_proc(None)) self.jedi_rendering.add_key('npy_proc', self.config.npy_proc(None)) self.jedi_rendering.add_key('total_processors', self.config.total_processors(None)) - self.jedi_rendering.add_key('vertical_resolution', self.config.vertical_resolution()) # Observations self.jedi_rendering.add_key('background_time', background_time) self.jedi_rendering.add_key('crtm_coeff_dir', self.config.crtm_coeff_dir(None)) self.jedi_rendering.add_key('window_begin', window_begin) + # Model + if window_type == '4D': + self.jedi_rendering.add_key('background_frequency', self.config.background_frequency()) + # Jedi configuration file # ----------------------- - jedi_config_file = os.path.join(self.cycle_dir(), 'jedi_variational_config.yaml') + jedi_config_file = os.path.join(self.cycle_dir(), f'jedi_{jedi_application}_config.yaml') # Output log file # --------------- - output_log_file = os.path.join(self.cycle_dir(), 'jedi_variational_log.log') - - # Get the JEDI interface for this model component - # ----------------------------------------------- - model_component_meta = self.jedi_rendering.render_interface_meta() - jedi_interface = model_component_meta['jedi_interface'] + output_log_file = os.path.join(self.cycle_dir(), f'jedi_{jedi_application}_log.log') - # Generate the JEDI configuration file for running the executable - # --------------------------------------------------------------- - jedi_config_dict = self.jedi_rendering.render_oops_file(f'variational{window_type}') + # Open the JEDI config file and fill initial templates + # ---------------------------------------------------- + jedi_config_dict = self.jedi_rendering.render_oops_file(f'{jedi_application}{window_type}') # Perform complete template rendering # ----------------------------------- jedi_dictionary_iterator(jedi_config_dict, self.jedi_rendering, window_type, observations, - self.config.jedi_forecast_model(None)) - + jedi_forecast_model) # Write the expanded dictionary to YAML file # ------------------------------------------ with open(jedi_config_file, 'w') as jedi_config_file_open: yaml.dump(jedi_config_dict, jedi_config_file_open, default_flow_style=False) - # Jedi executable name - # -------------------- - jedi_executable = model_component_meta['executables'][f'variational{window_type}'] - jedi_executable_path = os.path.join(self.experiment_path(), 'jedi_bundle', 'build', - 'bin', jedi_executable) + # Get the JEDI interface metadata + # ------------------------------- + model_component_meta = self.jedi_rendering.render_interface_meta() # Compute number of processors # ---------------------------- - np = eval(model_component_meta['total_processors']) + np = eval(str(model_component_meta['total_processors'])) + + # Jedi executable name + # -------------------- + jedi_executable = model_component_meta['executables'][f'{jedi_application}{window_type}'] + jedi_executable_path = os.path.join(self.experiment_path(), 'jedi_bundle', 'build', 'bin', + jedi_executable) # Run the JEDI executable # ----------------------- @@ -110,5 +119,4 @@ def execute(self): output_log_file) self.logger.info('Running '+jedi_executable_path+' with '+str(np)+' processors.') - # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/task_questions.yaml b/src/swell/tasks/task_questions.yaml index 1a3954d9..c2bdcd43 100644 --- a/src/swell/tasks/task_questions.yaml +++ b/src/swell/tasks/task_questions.yaml @@ -52,6 +52,8 @@ background_frequency: prompt: What is the frequency of the background files? tasks: - GetBackground + - RunJediHofxExecutable + - RunJediVariationalExecutable - StoreBackground type: duration @@ -399,6 +401,7 @@ total_processors: prompt: What is the number of processors for JEDI? tasks: - GenerateBClimatology + - GenerateBClimatologyByLinking - RunJediHofxExecutable - RunJediVariationalExecutable type: integer diff --git a/src/swell/utilities/config.py b/src/swell/utilities/config.py index faa5c43b..e6ee44e3 100644 --- a/src/swell/utilities/config.py +++ b/src/swell/utilities/config.py @@ -105,28 +105,34 @@ def __init__(self, input_file, logger, task_name, model): if task_name in key_question_dict['tasks']: - print(f'Adding {experiment_key} for {task_name}') - # Add this variable to the object setattr(self, f'__{experiment_key}__', experiment_value) # Add a method to get the variable + print('adding method for ', experiment_key) setattr(self, f'{experiment_key}', self.get(experiment_key)) # ---------------------------------------------------------------------------------------------- def get(self, experiment_key): def getter(default='None'): - try: - return getattr(self, f'__{experiment_key}__') - except Exception: - if default == 'None': - self.logger.abort(f'In config class, trying to get variable {experiment_key} ' + - f'but it was not created. Ensure that the variable is in ' + - f'the experiment configuration and that the task can ' + + return getattr(self, f'__{experiment_key}__') + return getter + + # ---------------------------------------------------------------------------------------------- + + # Implementation of __getattr__ to ensure there is no crash when a task requests a variable that + # does not exist. This is valid so long as the task provides a default value. + def __getattr__(self, name): + def variable_not_found(default='LrZRExPGcQ'): + if default=='LrZRExPGcQ': + self.__logger__.abort(f'In config class, trying to get variable \'{name}\' but ' + + f'this variable was not created. Ensure that the variable ' + + f'is in the experiment configuration and that the task can ' + f'access that key based on the rules in ' f'tasks/questions.yaml.') + else: return default - return getter + return variable_not_found # ---------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/logger.py b/src/swell/utilities/logger.py index 1d516da1..8b958340 100644 --- a/src/swell/utilities/logger.py +++ b/src/swell/utilities/logger.py @@ -10,6 +10,7 @@ import os import sys import textwrap +import traceback # -------------------------------------------------------------------------------------------------- @@ -62,7 +63,12 @@ def send_message(self, level, message, wrap): if level != 'BLANK': level_show = level_show+' '+self.task_name+': ' + if level == 'ABORT': + level_show = 'ABORT IN '+self.task_name+': ' + if level == 'ABORT' or self.loggerdict[level]: + if level == 'ABORT': + print('\n') first_line = True for message_item in message_items: if not first_line: @@ -99,7 +105,13 @@ def blank(self, message, wrap=True): def abort(self, message, wrap=True): self.send_message('ABORT', message, wrap) - sys.exit('ABORTING\n') + + # Get traceback stack (without logger.py lines) + filtered_stack = [line for line in traceback.format_stack() if 'logger.py' not in line] + traceback_str = '\n'.join(filtered_stack) + + # Exit with traceback + sys.exit('\nHERE IS THE TRACEBACK: \n----------------------\n\n' + traceback_str) # ---------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/render_jedi_interface_files.py b/src/swell/utilities/render_jedi_interface_files.py index fdd0b322..5913afd4 100644 --- a/src/swell/utilities/render_jedi_interface_files.py +++ b/src/swell/utilities/render_jedi_interface_files.py @@ -55,6 +55,7 @@ def __init__(self, logger, experiment_root, experiment_id, cycle_dir, jedi_inter 'npy_proc', 'number_of_iterations', 'swell_static_files', + 'total_processors', 'vertical_resolution', 'window_begin', 'window_begin_iso', From d8f12976603dd3e6eb52df6f6cce50d0de849bf5 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Fri, 2 Jun 2023 12:39:06 -0400 Subject: [PATCH 049/121] remove num iterations from RunJediHofx task --- src/swell/tasks/task_questions.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/src/swell/tasks/task_questions.yaml b/src/swell/tasks/task_questions.yaml index c2bdcd43..9600bf14 100644 --- a/src/swell/tasks/task_questions.yaml +++ b/src/swell/tasks/task_questions.yaml @@ -321,7 +321,6 @@ number_of_iterations: prompt: What number of iterations do you wish to use for each outer loop? Provide a list of integers the same length as the number of outer loops. tasks: - - RunJediHofxExecutable - RunJediVariationalExecutable type: integer-list From 1fd3ddb6cfe3b5413dd07b3469637473cde04710 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Fri, 2 Jun 2023 14:02:28 -0400 Subject: [PATCH 050/121] coding norms --- setup.py | 5 ++- src/swell/deployment/prep_config_base.py | 2 +- src/swell/tasks/eva_observations.py | 3 +- .../question_dictionary_comparison_test.py | 42 +++++++++++++++++++ src/swell/test/test_suite.py | 32 ++++++++++++++ .../generate_task_question_default_dicts.py | 16 ++++--- .../bin/generate_task_questions_dict.py | 14 ++++--- src/swell/utilities/config.py | 2 +- .../utilities/render_jedi_interface_files.py | 2 +- 9 files changed, 100 insertions(+), 18 deletions(-) create mode 100644 src/swell/test/question_dictionary_comparison_test.py create mode 100644 src/swell/test/test_suite.py diff --git a/setup.py b/setup.py index 5ca168b1..57b04dcc 100644 --- a/setup.py +++ b/setup.py @@ -63,12 +63,13 @@ 'swell_prepare_experiment_config = swell.deployment.bin.swell_prepare_config:main', 'swell_launch_experiment = swell.deployment.bin.swell_launch_experiment:main', 'swell_sat_db_processing = swell.deployment.bin.swell_sat_db_processing:main', + 'swell_util_generate_task_question_default_dicts = \ + swell.utilities.bin.generate_task_question_default_dicts:main', 'swell_util_check_jedi_interface_templates = \ swell.utilities.bin.check_jedi_interface_templates:main', 'swell_util_generate_task_questions_dict = \ swell.utilities.bin.generate_task_questions_dict:main', - 'swell_util_generate_task_question_default_dicts = \ - swell.utilities.bin.generate_task_question_default_dicts:main', + 'swell_test_suite = swell.test.test_suite:main', ], }, ) diff --git a/src/swell/deployment/prep_config_base.py b/src/swell/deployment/prep_config_base.py index 437b91c9..b48146e1 100644 --- a/src/swell/deployment/prep_config_base.py +++ b/src/swell/deployment/prep_config_base.py @@ -57,7 +57,7 @@ def __init__(self, logger, dictionary_file, suite, platform): # Open the platform specific defaults platform_suite_file = os.path.join(swell_path, 'deployment', 'platforms', platform, - 'suite_questions.yaml') + 'suite_questions.yaml') platform_task_file = os.path.join(swell_path, 'deployment', 'platforms', platform, 'task_questions.yaml') with open(platform_suite_file, 'r') as platform_dict_file_open: diff --git a/src/swell/tasks/eva_observations.py b/src/swell/tasks/eva_observations.py index c67c6862..5be0be46 100644 --- a/src/swell/tasks/eva_observations.py +++ b/src/swell/tasks/eva_observations.py @@ -28,7 +28,8 @@ def execute(self): # Compute window beginning time window_begin = self.da_window_params.window_begin(self.config.window_offset()) background_time = self.da_window_params.background_time(self.config.window_offset(), - self.config.background_time_offset()) + self.config.background_time_offset() + ) # Create JEDI interface config templates dictionary self.jedi_rendering.add_key('background_time', background_time) diff --git a/src/swell/test/question_dictionary_comparison_test.py b/src/swell/test/question_dictionary_comparison_test.py new file mode 100644 index 00000000..1935b327 --- /dev/null +++ b/src/swell/test/question_dictionary_comparison_test.py @@ -0,0 +1,42 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import unittest +import json + +from swell.swell_path import get_swell_path +from swell.utilities.bin.generate_task_questions_dict import main as generate_task_questions_dict + + +# -------------------------------------------------------------------------------------------------- + +class QuestionDictionaryTest(unittest.TestCase): + + def test_dictionary_comparison(self): + + # First test the + destination_yaml = os.path.join(get_swell_path(), 'tasks', 'task_questions.yaml') + + # Read input file into dictionary + with open(destination_yaml, 'r') as ymlfile: + question_dict = yaml.safe_load(ymlfile) + question_dict_in = question_dict.copy() + + # Run dictionary generation + generate_task_questions_dict() + + # Read new dictionary + with open(destination_yaml, 'r') as ymlfile: + question_dict = yaml.safe_load(ymlfile) + + # Assert that dictionaries are equal + self.assertDictEqual(question_dict_in, question_dict) + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/test/test_suite.py b/src/swell/test/test_suite.py new file mode 100644 index 00000000..0a18440e --- /dev/null +++ b/src/swell/test/test_suite.py @@ -0,0 +1,32 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import unittest + +from swell.test.question_dictionary_comparison_test import QuestionDictionaryTest + + +# -------------------------------------------------------------------------------------------------- + +def main(): + + # Create a test suite + test_suite = unittest.TestSuite() + + # Load tests from QuestionDictionaryTest + test_suite.addTests(unittest.TestLoader().loadTestsFromTestCase(QuestionDictionaryTest)) + + # Create a test runner + test_runner = unittest.TextTestRunner() + + # Run the tests + test_runner.run(test_suite) + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/bin/generate_task_question_default_dicts.py b/src/swell/utilities/bin/generate_task_question_default_dicts.py index bb43e124..71d00794 100644 --- a/src/swell/utilities/bin/generate_task_question_default_dicts.py +++ b/src/swell/utilities/bin/generate_task_question_default_dicts.py @@ -43,7 +43,7 @@ def main(): # Generate list of jedi interfaces jedi_interfaces_path = os.path.join(get_swell_path(), 'configuration', 'jedi', 'interfaces') - jedi_interfaces = [ f.path for f in os.scandir(jedi_interfaces_path) if f.is_dir() ] + jedi_interfaces = [f.path for f in os.scandir(jedi_interfaces_path) if f.is_dir()] jedi_interfaces = list(filter(lambda jedi_interface: '__' not in jedi_interface, jedi_interfaces)) jedi_interface_names = [] @@ -78,8 +78,8 @@ def main(): f'contain models. Offending key: {question_key}') # Set the required jedi interfaces for this question - jedi_interfaces_needed = question_dict['models'] - if jedi_interfaces_needed[0] == 'all' or jedi_interface_name in jedi_interfaces_needed: + jedi_int_needed = question_dict['models'] + if jedi_int_needed[0] == 'all' or jedi_interface_name in jedi_int_needed: # Create dictionary if it does not already exist if question_key not in question_dict_defaults_exist: @@ -95,7 +95,9 @@ def main(): else: # Copy from the existing dictionary - question_dict_defaults = {question_key: question_dict_defaults_exist[question_key]} + question_dict_defaults = { + question_key: question_dict_defaults_exist[question_key] + } # Write to the YAML file outfile.write(yaml.dump(question_dict_defaults, default_flow_style=False)) @@ -103,7 +105,7 @@ def main(): # Generate list of platforms platforms_path = os.path.join(get_swell_path(), 'deployment', 'platforms') - platforms = [ f.path for f in os.scandir(platforms_path) if f.is_dir() ] + platforms = [f.path for f in os.scandir(platforms_path) if f.is_dir()] platforms = list(filter(lambda platform: '__' not in platform, platforms)) platform_names = [] for platform in platforms: @@ -145,7 +147,9 @@ def main(): else: # Copy from the existing dictionary - question_dict_defaults = {question_key: question_dict_defaults_exist[question_key]} + question_dict_defaults = { + question_key: question_dict_defaults_exist[question_key] + } # Write to the YAML file outfile.write(yaml.dump(question_dict_defaults, default_flow_style=False)) diff --git a/src/swell/utilities/bin/generate_task_questions_dict.py b/src/swell/utilities/bin/generate_task_questions_dict.py index a3c990d6..cf673a04 100644 --- a/src/swell/utilities/bin/generate_task_questions_dict.py +++ b/src/swell/utilities/bin/generate_task_questions_dict.py @@ -23,7 +23,7 @@ # -------------------------------------------------------------------------------------------------- -def main(): +def main(test_mode): # Create a logger logger = Logger('ListOfTaskQuestions') @@ -35,18 +35,20 @@ def main(): task_codes = glob.glob(os.path.join(get_swell_path(), 'tasks', '*.py')) task_codes = list(filter(lambda task_code: task_code != '__init__.py', task_codes)) - # Output file - outfile_yaml = os.path.join(get_swell_path(), 'tasks', 'task_questions.yaml') + # Target YAML file + destination_yaml = os.path.join(get_swell_path(), 'tasks', 'task_questions.yaml') # Read input file into dictionary - if os.path.exists(outfile_yaml): - with open(outfile_yaml, 'r') as ymlfile: + if os.path.exists(destination_yaml): + with open(destination_yaml, 'r') as ymlfile: question_dict = yaml.safe_load(ymlfile) else: question_dict = {} + question_dict_in = question_dict.copy() + # Now safe to overwrite file - outfile = open(outfile_yaml, 'w') + outfile = open(destination_yaml, 'w') # Loop through task code and accumulate all lines containing a use of config config_keys = [] diff --git a/src/swell/utilities/config.py b/src/swell/utilities/config.py index e6ee44e3..62e76dcc 100644 --- a/src/swell/utilities/config.py +++ b/src/swell/utilities/config.py @@ -125,7 +125,7 @@ def getter(default='None'): # does not exist. This is valid so long as the task provides a default value. def __getattr__(self, name): def variable_not_found(default='LrZRExPGcQ'): - if default=='LrZRExPGcQ': + if default == 'LrZRExPGcQ': self.__logger__.abort(f'In config class, trying to get variable \'{name}\' but ' + f'this variable was not created. Ensure that the variable ' + f'is in the experiment configuration and that the task can ' + diff --git a/src/swell/utilities/render_jedi_interface_files.py b/src/swell/utilities/render_jedi_interface_files.py index 5913afd4..e0319cca 100644 --- a/src/swell/utilities/render_jedi_interface_files.py +++ b/src/swell/utilities/render_jedi_interface_files.py @@ -143,7 +143,7 @@ def render_interface_observations(self, config_name): # ---------------------------------------------------------------------------------------------- # Prepare path to interface metadata file and call rendering - def render_interface_meta(self, model_component_in = None): + def render_interface_meta(self, model_component_in=None): # Optionally open a different model interface model_component = self.jedi_interface From af993dec33a1e190052eb12eb11c9edcfe085970 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Fri, 2 Jun 2023 14:03:51 -0400 Subject: [PATCH 051/121] add dictionary gneeration test --- src/swell/test/question_dictionary_comparison_test.py | 3 ++- src/swell/utilities/bin/generate_task_questions_dict.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/swell/test/question_dictionary_comparison_test.py b/src/swell/test/question_dictionary_comparison_test.py index 1935b327..bf0229d1 100644 --- a/src/swell/test/question_dictionary_comparison_test.py +++ b/src/swell/test/question_dictionary_comparison_test.py @@ -8,8 +8,9 @@ # -------------------------------------------------------------------------------------------------- +import os import unittest -import json +import yaml from swell.swell_path import get_swell_path from swell.utilities.bin.generate_task_questions_dict import main as generate_task_questions_dict diff --git a/src/swell/utilities/bin/generate_task_questions_dict.py b/src/swell/utilities/bin/generate_task_questions_dict.py index cf673a04..d897a75d 100644 --- a/src/swell/utilities/bin/generate_task_questions_dict.py +++ b/src/swell/utilities/bin/generate_task_questions_dict.py @@ -23,7 +23,7 @@ # -------------------------------------------------------------------------------------------------- -def main(test_mode): +def main(): # Create a logger logger = Logger('ListOfTaskQuestions') From 2c65996cc6b7f6a37aaac4f3aca7737dd33ace78 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Fri, 2 Jun 2023 14:07:05 -0400 Subject: [PATCH 052/121] add test_suite action --- .github/workflows/test_suite.yml | 33 ++++++++++++++++++++++++++++++++ README.md | 1 + 2 files changed, 34 insertions(+) create mode 100644 .github/workflows/test_suite.yml diff --git a/.github/workflows/test_suite.yml b/.github/workflows/test_suite.yml new file mode 100644 index 00000000..bff589a2 --- /dev/null +++ b/.github/workflows/test_suite.yml @@ -0,0 +1,33 @@ +name: Swell Test Suite + +on: + pull_request: + types: [opened, synchronize, reopened] + +jobs: + pycodestyle: + name: Check Python Coding Norms + runs-on: ubuntu-latest + steps: + + # Setup Python + - name: Set up Python 3.9 + uses: actions/setup-python@v2 + with: + python-version: 3.9 + + # Update conda + - name: Update conda + run: conda update -n base -c defaults conda + + # Install pycodestyle + - name: Install pycodestyle + run: conda install -c conda-forge pycodestyle + + # Clone the swell code + - name: Clone swell repo + uses: actions/checkout@v2 + + # Run python codestyle + - name: Run python codestyle + run: $CONDA/bin/python3 pycodestyle_run.py diff --git a/README.md b/README.md index 3a95ebd9..553babb7 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ width="50%" /> +![Swell test suite](https://github.com/GEOS-ESM/swell/actions/workflows/test_suite.yml/badge.svg) ![Python coding norms](https://github.com/GEOS-ESM/swell/actions/workflows/python_coding_norms.yml/badge.svg) ![YAML coding norms](https://github.com/GEOS-ESM/swell/actions/workflows/yaml_coding_norms.yml/badge.svg) ![NCCS Discover Nightly](https://github.com/GEOS-ESM/swell/actions/workflows/discover_nightly.yml/badge.svg) From 08b05037fdc4cf50afc596d7dc445d98ca1e1faf Mon Sep 17 00:00:00 2001 From: danholdaway Date: Fri, 2 Jun 2023 14:23:40 -0400 Subject: [PATCH 053/121] action for test suite --- .github/workflows/test_suite.yml | 43 ++++++++++++++++++++++++-------- requirements-github.txt | 8 ++++++ requirements.txt | 4 ++- 3 files changed, 43 insertions(+), 12 deletions(-) create mode 100644 requirements-github.txt diff --git a/.github/workflows/test_suite.yml b/.github/workflows/test_suite.yml index bff589a2..39291806 100644 --- a/.github/workflows/test_suite.yml +++ b/.github/workflows/test_suite.yml @@ -1,33 +1,54 @@ name: Swell Test Suite on: + push: + branches: + - develop pull_request: - types: [opened, synchronize, reopened] + types: + - opened + - synchronize + - reopened jobs: - pycodestyle: - name: Check Python Coding Norms + testsuite: + name: Run swell test suite runs-on: ubuntu-latest + steps: # Setup Python - - name: Set up Python 3.9 + - name: Set up Python 3.10 uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: 3.10.10 # Update conda - name: Update conda run: conda update -n base -c defaults conda - # Install pycodestyle - - name: Install pycodestyle - run: conda install -c conda-forge pycodestyle + # Install pip + - name: Install pip + run: conda install pip + + # Upgrade pip + - name: Upgrade pip + run: $CONDA/bin/pip3 install --upgrade pip # Clone the swell code - name: Clone swell repo uses: actions/checkout@v2 + with: + lfs: true + + # Install swell + - name: Install swell and dependencies + run: $CONDA/bin/pip3 install --use-deprecated=legacy-resolver -r requirements-github.txt --user . + + # Put swell executables in the path + - name: Put swell in the path + run: echo "$HOME/.local/bin" >> $GITHUB_PATH - # Run python codestyle - - name: Run python codestyle - run: $CONDA/bin/python3 pycodestyle_run.py + # Run the swell test suite + - name: Run swell applications tests + run: swell_test_suite diff --git a/requirements-github.txt b/requirements-github.txt new file mode 100644 index 00000000..512f0333 --- /dev/null +++ b/requirements-github.txt @@ -0,0 +1,8 @@ +click +jinja2==3.1.2 +pyyaml==6.0 +pycodestyle==2.10.0 +pandas==2.0.2 +isodate==0.6.1 +xarray==2023.5.0 +questionary==1.10.0 diff --git a/requirements.txt b/requirements.txt index d9144e1e..7a5a6b40 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,8 @@ click -PyYAML>=6.0 +jinja2>=3.0.3 +pyyaml>=6.0 pycodestyle>=2.8.0 pandas>=1.4.0 isodate>=0.5.4 xarray>=0.11.3 +questionary>=1.10.0 From c5b2218bbf81e4d03214d72272c1de045da148da Mon Sep 17 00:00:00 2001 From: danholdaway Date: Fri, 2 Jun 2023 16:11:51 -0400 Subject: [PATCH 054/121] add init to test path --- src/swell/test/__init__.py | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 src/swell/test/__init__.py diff --git a/src/swell/test/__init__.py b/src/swell/test/__init__.py new file mode 100644 index 00000000..f82722ea --- /dev/null +++ b/src/swell/test/__init__.py @@ -0,0 +1,9 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +import os + +repo_directory = os.path.dirname(__file__) From f313213e681c989c910c248865f5cc032888fd77 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Fri, 2 Jun 2023 16:14:41 -0400 Subject: [PATCH 055/121] copy task qustions to install dir --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 57b04dcc..cd34ca64 100644 --- a/setup.py +++ b/setup.py @@ -48,6 +48,7 @@ 'suites/*', 'suites/*/*', 'suites/*/*/*', + 'tasks/task_questions.yaml', 'configuration/*', 'configuration/*/*', 'configuration/*/*/*', From 89832577e475c64f447eb0b6659dbcdf5baac00c Mon Sep 17 00:00:00 2001 From: danholdaway Date: Fri, 2 Jun 2023 16:17:07 -0400 Subject: [PATCH 056/121] action job name --- .github/workflows/test_suite.yml | 2 +- src/swell/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_suite.yml b/.github/workflows/test_suite.yml index 39291806..b8bb4a72 100644 --- a/.github/workflows/test_suite.yml +++ b/.github/workflows/test_suite.yml @@ -50,5 +50,5 @@ jobs: run: echo "$HOME/.local/bin" >> $GITHUB_PATH # Run the swell test suite - - name: Run swell applications tests + - name: Run swell test suite run: swell_test_suite diff --git a/src/swell/__init__.py b/src/swell/__init__.py index 9563e872..f62ac6b5 100644 --- a/src/swell/__init__.py +++ b/src/swell/__init__.py @@ -9,4 +9,4 @@ repo_directory = os.path.dirname(__file__) # Set the version for swell -__version__ = '1.3.0' +__version__ = '1.4.0' From 35a527e715805589dd2310a7608d5b160631c80f Mon Sep 17 00:00:00 2001 From: danholdaway Date: Mon, 5 Jun 2023 10:02:31 -0400 Subject: [PATCH 057/121] build jedi paths --- src/swell/suites/build_jedi/suites-build_jedi.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/swell/suites/build_jedi/suites-build_jedi.yaml b/src/swell/suites/build_jedi/suites-build_jedi.yaml index a5b1ab37..eab54fb3 100644 --- a/src/swell/suites/build_jedi/suites-build_jedi.yaml +++ b/src/swell/suites/build_jedi/suites-build_jedi.yaml @@ -7,7 +7,7 @@ jedi_build_method: # Existing JEDI bundle directory existing_jedi_source_directory: - default_value: {{existing_source_directory}} + default_value: {{existing_jedi_source_directory}} prompt: Provide the path to an existing JEDI bundle directory containing source code repos type: string depends: @@ -16,7 +16,7 @@ existing_jedi_source_directory: # Existing JEDI build existing_jedi_build_directory: - default_value: {{existing_build_directory}} + default_value: {{existing_jedi_build_directory}} prompt: Provide the path to an existing JEDI build directory type: string depends: From 090196715906af8c49e953883f932125a248aa65 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Mon, 5 Jun 2023 12:24:25 -0400 Subject: [PATCH 058/121] make tasks complianct with new config and base delgations --- requirements-github.txt | 1 + requirements.txt | 1 + setup.py | 2 + ...tObsFilters.yaml => test_obs_filters.yaml} | 0 .../deployment/bin/swell_prepare_config.py | 2 +- .../deployment/bin/swell_sat_db_processing.py | 2 - src/swell/deployment/prep_config.py | 8 +- src/swell/deployment/prep_config_base.py | 2 - src/swell/deployment/prep_config_cli.py | 2 - src/swell/deployment/prep_config_defaults.py | 3 - src/swell/deployment/prep_exp_dirs.py | 2 - src/swell/tasks/base/task_base.py | 2 - src/swell/tasks/build_jedi_by_linking.py | 2 - src/swell/tasks/clean_cycle.py | 4 +- src/swell/tasks/clone_geos.py | 2 - src/swell/tasks/generate_b_climatology.py | 2 +- src/swell/tasks/get_background.py | 2 - src/swell/tasks/get_geovals.py | 30 ++++--- src/swell/tasks/get_gsi_bc.py | 5 +- src/swell/tasks/get_gsi_ncdiag.py | 5 +- src/swell/tasks/gsi_bc_to_ioda.py | 34 +++---- src/swell/tasks/gsi_ncdiag_to_ioda.py | 46 +++++----- .../run_jedi_test_obs_filters_executable.py | 86 +++++++++++------- ...un_jedi_test_obs_filters_executable_old.py | 88 +++++++++++++++++++ src/swell/tasks/stage_jedi.py | 4 - src/swell/tasks/store_background.py | 1 - .../question_dictionary_comparison_test.py | 1 + src/swell/test/test_suite.py | 6 +- src/swell/test/unused_variables_test.py | 44 ++++++++++ .../generate_task_question_default_dicts.py | 5 -- .../bin/generate_task_questions_dict.py | 9 +- src/swell/utilities/build.py | 2 +- .../data_assimilation_window_params.py | 12 +++ src/swell/utilities/dictionary.py | 2 - src/swell/utilities/jinja2.py | 1 - src/swell/utilities/run_jedi_executables.py | 2 +- src/swell/utilities/shell_commands.py | 3 +- 37 files changed, 287 insertions(+), 138 deletions(-) rename src/swell/configuration/jedi/oops/{TestObsFilters.yaml => test_obs_filters.yaml} (100%) create mode 100644 src/swell/tasks/run_jedi_test_obs_filters_executable_old.py create mode 100644 src/swell/test/unused_variables_test.py diff --git a/requirements-github.txt b/requirements-github.txt index 512f0333..d03df4c3 100644 --- a/requirements-github.txt +++ b/requirements-github.txt @@ -6,3 +6,4 @@ pandas==2.0.2 isodate==0.6.1 xarray==2023.5.0 questionary==1.10.0 +flake8==6.0.0 diff --git a/requirements.txt b/requirements.txt index 7a5a6b40..62cfecc0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ pandas>=1.4.0 isodate>=0.5.4 xarray>=0.11.3 questionary>=1.10.0 +flake8>=6.0.0 diff --git a/setup.py b/setup.py index cd34ca64..e1402a9b 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,7 @@ 'Operating System :: OS Independent'], python_requires='>=3.6', install_requires=[ + 'tomlkit', 'click', 'jinja2>=3.0.3', 'pyyaml>=6.0', @@ -40,6 +41,7 @@ 'isodate>=0.5.4', 'xarray>=0.11.3', 'questionary>=1.10.0', + 'flake8>=6.0.0', ], package_data={ '': [ diff --git a/src/swell/configuration/jedi/oops/TestObsFilters.yaml b/src/swell/configuration/jedi/oops/test_obs_filters.yaml similarity index 100% rename from src/swell/configuration/jedi/oops/TestObsFilters.yaml rename to src/swell/configuration/jedi/oops/test_obs_filters.yaml diff --git a/src/swell/deployment/bin/swell_prepare_config.py b/src/swell/deployment/bin/swell_prepare_config.py index d919e425..bf491bcb 100644 --- a/src/swell/deployment/bin/swell_prepare_config.py +++ b/src/swell/deployment/bin/swell_prepare_config.py @@ -37,7 +37,7 @@ def main(input_method, suite, platform, override): # Create suites object # -------------------- - config_file = prepare_config(input_method, suite, platform, override) + prepare_config(input_method, suite, platform, override) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/deployment/bin/swell_sat_db_processing.py b/src/swell/deployment/bin/swell_sat_db_processing.py index 46ce3767..d1e17c18 100644 --- a/src/swell/deployment/bin/swell_sat_db_processing.py +++ b/src/swell/deployment/bin/swell_sat_db_processing.py @@ -9,11 +9,9 @@ import yaml import click import numpy as np -import pandas as pd from datetime import datetime as dt -from swell.utilities.git_utils import git_got from swell.utilities.sat_db_utils import run_sat_db_process diff --git a/src/swell/deployment/prep_config.py b/src/swell/deployment/prep_config.py index f6581f0f..64dee423 100644 --- a/src/swell/deployment/prep_config.py +++ b/src/swell/deployment/prep_config.py @@ -11,14 +11,11 @@ import datetime import os import importlib -import ruamel.yaml as ry -import sys import yaml from swell.utilities.logger import Logger from swell.swell_path import get_swell_path from swell.utilities.dictionary import dict_get, add_comments_to_dictionary -from swell.utilities.jinja2 import template_string_jinja2 # -------------------------------------------------------------------------------------------------- @@ -96,9 +93,8 @@ def prepare_config(method, suite, platform, override=None): # Write dictionary to YAML file # ----------------------------- - exp_dict_file_open = open(exp_dict_file, "w") - n = exp_dict_file_open.write(experiment_dict_string_comments) - exp_dict_file_open.close() + with open(exp_dict_file, 'w') as file: + file.write(experiment_dict_string_comments) logger.info(f'Prepared configuration file written to {exp_dict_file}', False) # Return path to dictionary file diff --git a/src/swell/deployment/prep_config_base.py b/src/swell/deployment/prep_config_base.py index b48146e1..8073d142 100644 --- a/src/swell/deployment/prep_config_base.py +++ b/src/swell/deployment/prep_config_base.py @@ -11,7 +11,6 @@ from abc import ABC, abstractmethod import datetime import os -import pathlib import yaml from swell.swell_path import get_swell_path @@ -180,7 +179,6 @@ def add_to_experiment_dictionary(self, key, element_dict): # Check for disallowed element types for element_item in element_items: - element_item_type = type(element_item) for dis_elem_type in self.dis_elem_types: if isinstance(element_item, dis_elem_type): self.logger.abort(f'Element \'{element}\' has a type that is not permitted. ' + diff --git a/src/swell/deployment/prep_config_cli.py b/src/swell/deployment/prep_config_cli.py index c3b266d6..9e6fb4c4 100644 --- a/src/swell/deployment/prep_config_cli.py +++ b/src/swell/deployment/prep_config_cli.py @@ -16,7 +16,6 @@ import questionary from swell.deployment.prep_config_base import PrepConfigBase -from swell.swell_path import get_swell_path # -------------------------------------------------------------------------------------------------- @@ -248,7 +247,6 @@ def validate(self, document): def make_check_widget(self, quest, options, default, prompt): if options == 'file': - dir_list = os.listdir(self.directory) new_path = os.path.join(self.directory, '*/') suite_list = [x.split('/')[-2] for x in glob.glob(new_path)] choices = suite_list diff --git a/src/swell/deployment/prep_config_defaults.py b/src/swell/deployment/prep_config_defaults.py index ca1eb136..5e4c77fe 100644 --- a/src/swell/deployment/prep_config_defaults.py +++ b/src/swell/deployment/prep_config_defaults.py @@ -8,9 +8,6 @@ # -------------------------------------------------------------------------------------------------- -import os -import yaml - from swell.deployment.prep_config_base import PrepConfigBase diff --git a/src/swell/deployment/prep_exp_dirs.py b/src/swell/deployment/prep_exp_dirs.py index 9489a8af..962582d8 100644 --- a/src/swell/deployment/prep_exp_dirs.py +++ b/src/swell/deployment/prep_exp_dirs.py @@ -10,9 +10,7 @@ import copy import glob -import importlib import os -import pathlib import shutil from swell.swell_path import get_swell_path diff --git a/src/swell/tasks/base/task_base.py b/src/swell/tasks/base/task_base.py index c6718f75..81270266 100644 --- a/src/swell/tasks/base/task_base.py +++ b/src/swell/tasks/base/task_base.py @@ -14,9 +14,7 @@ import click import importlib import os -import sys import time -import yaml # swell imports from swell.tasks.base.task_registry import valid_tasks diff --git a/src/swell/tasks/build_jedi_by_linking.py b/src/swell/tasks/build_jedi_by_linking.py index 02d77d23..cbd7bb9d 100644 --- a/src/swell/tasks/build_jedi_by_linking.py +++ b/src/swell/tasks/build_jedi_by_linking.py @@ -10,8 +10,6 @@ import os -from jedi_bundle.bin.jedi_bundle import execute_tasks - from swell.tasks.base.task_base import taskBase from swell.utilities.build import build_and_source_dirs, link_path diff --git a/src/swell/tasks/clean_cycle.py b/src/swell/tasks/clean_cycle.py index 2cf815fc..8fba3a84 100644 --- a/src/swell/tasks/clean_cycle.py +++ b/src/swell/tasks/clean_cycle.py @@ -86,5 +86,5 @@ def execute(self): os.remove(item_to_remove) # Save cycle_done file to cycle_dir - with open(os.path.join(self.cycle_dir(), 'cycle_done'), 'w') as fp: - pass + with open(os.path.join(self.cycle_dir(), 'cycle_done'), 'w') as file: + file.write('cycle complete') diff --git a/src/swell/tasks/clone_geos.py b/src/swell/tasks/clone_geos.py index 79c5806f..36396022 100644 --- a/src/swell/tasks/clone_geos.py +++ b/src/swell/tasks/clone_geos.py @@ -9,8 +9,6 @@ import os -import tarfile -import urllib.request from swell.tasks.base.task_base import taskBase from swell.utilities.build import build_and_source_dirs, link_path diff --git a/src/swell/tasks/generate_b_climatology.py b/src/swell/tasks/generate_b_climatology.py index 786f7935..a18c94a9 100644 --- a/src/swell/tasks/generate_b_climatology.py +++ b/src/swell/tasks/generate_b_climatology.py @@ -153,7 +153,7 @@ def execute(self): # Compute number of processors # ---------------------------- - np = eval(model_component_meta['total_processors']) + self.np = eval(model_component_meta['total_processors']) # Obtain and initialize proper error model # ----------------------------------------------- diff --git a/src/swell/tasks/get_background.py b/src/swell/tasks/get_background.py index b3a409aa..c4cc5a3b 100644 --- a/src/swell/tasks/get_background.py +++ b/src/swell/tasks/get_background.py @@ -10,10 +10,8 @@ from swell.tasks.base.task_base import taskBase -from datetime import datetime as dt import isodate import os -import re from r2d2 import fetch diff --git a/src/swell/tasks/get_geovals.py b/src/swell/tasks/get_geovals.py index db781631..9669f91c 100644 --- a/src/swell/tasks/get_geovals.py +++ b/src/swell/tasks/get_geovals.py @@ -22,24 +22,32 @@ def execute(self): # Parse config # ------------ - cycle_dir = self.config_get('cycle_dir') - geovals_experiment = self.config_get('geovals_experiment') - geovals_provider = self.config_get('geovals_provider') - window_begin = self.config_get('window_begin') - observations = self.config_get('observations') - window_length = self.config_get('window_length') + geovals_experiment = self.config.geovals_experiment() + geovals_provider = self.config.geovals_provider() + window_offset = self.config.window_offset() + background_time_offset = self.config.background_time_offset() + observations = self.config.observations() + window_length = self.config.window_length() + crtm_coeff_dir = self.config.crtm_coeff_dir(None) + + # Get window begin time + window_begin = self.da_window_params.window_begin(window_offset) + background_time = self.da_window_params.background_time(window_offset, + background_time_offset) + + # Add to JEDI template rendering dictionary + self.jedi_rendering.add_key('background_time', background_time) + self.jedi_rendering.add_key('crtm_coeff_dir', crtm_coeff_dir) + self.jedi_rendering.add_key('window_begin', window_begin) # Loop over observation operators # ------------------------------- for observation in observations: - # Open the observation operator dictionary - # ---------------------------------------- - observation_dict = self.open_jedi_interface_obs_config_file(observation) - # Fetch observation files # ----------------------- - target_file = os.path.join(cycle_dir, f'{observation}_geovals.{window_begin}.nc4') + target_file = os.path.join(self.cycle_dir(), + f'{observation}_geovals.{window_begin}.nc4') self.logger.info("Processing observation file "+target_file) fetch(date=window_begin, diff --git a/src/swell/tasks/get_gsi_bc.py b/src/swell/tasks/get_gsi_bc.py index 48454b5e..3bd64881 100644 --- a/src/swell/tasks/get_gsi_bc.py +++ b/src/swell/tasks/get_gsi_bc.py @@ -25,11 +25,10 @@ def execute(self): # Get the build method # -------------------- - gsi_bc_location = self.config_get('gsi_bc_location') - cycle_dir = self.config_get('cycle_dir') + gsi_bc_location = self.config.gsi_bc_location() # Holding directory - gsi_bc_dir = os.path.join(cycle_dir, 'gsi_bcs') + gsi_bc_dir = os.path.join(self.cycle_dir(), 'gsi_bcs') os.makedirs(gsi_bc_dir, 0o755, exist_ok=True) # Get list of bias correction files to copy diff --git a/src/swell/tasks/get_gsi_ncdiag.py b/src/swell/tasks/get_gsi_ncdiag.py index 88a85afe..dbfa7845 100644 --- a/src/swell/tasks/get_gsi_ncdiag.py +++ b/src/swell/tasks/get_gsi_ncdiag.py @@ -24,7 +24,7 @@ def execute(self): # Get the build method # -------------------- - gsi_diag_path = self.config_get('path_to_gsi_diags') + gsi_diag_path = self.config.path_to_gsi_diags() # Get list of ncdiags to test with # -------------------------------- @@ -33,8 +33,7 @@ def execute(self): # Get cycle dir and create if needed # ---------------------------------- - cycle_dir = self.config_get('cycle_dir') - gsi_diag_dir = os.path.join(cycle_dir, 'gsi_ncdiags') + gsi_diag_dir = os.path.join(self.cycle_dir(), 'gsi_ncdiags') os.makedirs(gsi_diag_dir, 0o755, exist_ok=True) # Copy all the files into the cycle directory diff --git a/src/swell/tasks/gsi_bc_to_ioda.py b/src/swell/tasks/gsi_bc_to_ioda.py index 9d2857df..723ff292 100644 --- a/src/swell/tasks/gsi_bc_to_ioda.py +++ b/src/swell/tasks/gsi_bc_to_ioda.py @@ -10,9 +10,6 @@ import glob import os -import shutil -import netCDF4 as nc - from swell.tasks.base.task_base import taskBase from swell.utilities.dictionary import write_dict_to_yaml @@ -28,13 +25,20 @@ def execute(self): # Parse configuration # ------------------- - experiment_dir = self.config_get('experiment_dir') - cycle_dir = self.config_get('cycle_dir') - observations = self.config_get('observations') - produce_geovals = self.config_get('produce_geovals') - window_begin = self.config_get('window_begin') - current_cycle = self.config_get('current_cycle') - background_time = self.config_get('background_time') + observations = self.config.observations() + window_offset = self.config.window_offset() + background_time_offset = self.config.background_time_offset() + crtm_coeff_dir = self.config.crtm_coeff_dir(None) + + # Get window beginning time + window_begin = self.da_window_params.window_begin(window_offset) + background_time = self.da_window_params.background_time(window_offset, + background_time_offset) + + # Prepare dictionary for rendering jedi interface files + self.jedi_rendering.add_key('background_time', background_time) + self.jedi_rendering.add_key('crtm_coeff_dir', crtm_coeff_dir) + self.jedi_rendering.add_key('window_begin', window_begin) # Assemble list of needed sensors # ------------------------------- @@ -43,7 +47,7 @@ def execute(self): sensors_tlapse = [] for observation in observations: # Open configuration file for observation - observation_dict = self.open_jedi_interface_obs_config_file(observation) + observation_dict = self.jedi_rendering.render_interface_observations(observation) # Check for sensor key try: @@ -61,7 +65,7 @@ def execute(self): return # Holding directory - gsi_bc_dir = os.path.join(cycle_dir, 'gsi_bcs') + gsi_bc_dir = os.path.join(self.cycle_dir(), 'gsi_bcs') # Get list of files from holding directory bc_files = glob.glob(os.path.join(gsi_bc_dir, '*.txt')) @@ -97,7 +101,7 @@ def execute(self): for sensor, sensor_satbias in zip(sensors, sensors_satbias): output_dict = {} output_dict['sensor'] = sensor - output_dict['output file'] = os.path.join(cycle_dir, sensor_satbias) + output_dict['output file'] = os.path.join(self.cycle_dir(), sensor_satbias) output_dict['predictors'] = default_predictors satbias_converter_dict_output.append(output_dict) @@ -108,7 +112,7 @@ def execute(self): write_dict_to_yaml(satbias_converter_dict, satbias_converter_yaml) # Run IODA satbias converter - satbias_converter_exe = os.path.join(experiment_dir, 'jedi_bundle', 'build', 'bin', + satbias_converter_exe = os.path.join(self.experiment_path(), 'jedi_bundle', 'build', 'bin', 'satbias2ioda.x') run_track_log_subprocess(self.logger, [satbias_converter_exe, satbias_converter_yaml]) @@ -121,7 +125,7 @@ def execute(self): if sensor in line: sensor_tlapse_file = sensor_tlapse_file + ' '.join(line.split()[1:]) + '\n' # Write to tlapse file - with open(os.path.join(cycle_dir, sensor_tlapse), 'w') as file_open: + with open(os.path.join(self.cycle_dir(), sensor_tlapse), 'w') as file_open: file_open.write(sensor_tlapse_file) diff --git a/src/swell/tasks/gsi_ncdiag_to_ioda.py b/src/swell/tasks/gsi_ncdiag_to_ioda.py index b9205831..e190f69d 100644 --- a/src/swell/tasks/gsi_ncdiag_to_ioda.py +++ b/src/swell/tasks/gsi_ncdiag_to_ioda.py @@ -10,8 +10,6 @@ import glob import os -import shutil -import netCDF4 as nc # Ioda converters import gsi_ncdiag.gsi_ncdiag as gsid @@ -29,18 +27,18 @@ def execute(self): # Parse configuration # ------------------- - experiment_root = self.config_get('experiment_root') - experiment_id = self.config_get('experiment_id') - cycle_dir = self.config_get('cycle_dir') - observations = self.config_get('observations') - produce_geovals = self.config_get('produce_geovals') - window_begin = self.config_get('window_begin') + observations = self.config.observations() + produce_geovals = self.config.produce_geovals() + window_offset = self.config.window_offset() + + # Get window beginning time + window_begin = self.da_window_params.window_begin(window_offset) # Keep copy of the observations_orig = observations.copy() # Directory containing the ncdiags - gsi_diag_dir = os.path.join(cycle_dir, 'gsi_ncdiags') + gsi_diag_dir = os.path.join(self.cycle_dir(), 'gsi_ncdiags') # Assemble all conventional types that ioda considers # --------------------------------------------------- @@ -95,12 +93,12 @@ def execute(self): needed_platforms.append(platform) # Extract data - Diag.toIODAobs(cycle_dir, platforms=needed_platforms) + Diag.toIODAobs(self.cycle_dir(), platforms=needed_platforms) if produce_geovals: self.logger.info('', wrap=False) self.logger.info(f'Processing GeoVaLs from {gsi_type_to_process}') - Diag.toGeovals(cycle_dir) + Diag.toGeovals(self.cycle_dir()) Diag.close() @@ -114,14 +112,11 @@ def execute(self): self.logger.info(log_str) self.logger.info('-'*len(log_str)) - # Check dictionary for number of gsi files that were needed - gsi_sources = ioda_to_gsi_dict[needed_ioda_type] - # Check the number of files that are found ioda_type_pattern = f'*{needed_ioda_type}*_obs_*' # Pattern, e.g.: *aircraft*_obs_* # List of files for that instrument - ioda_path_files = glob.glob(os.path.join(cycle_dir, ioda_type_pattern)) + ioda_path_files = glob.glob(os.path.join(self.cycle_dir(), ioda_type_pattern)) # Get last file (first could be type_obs_ if the code already ran) ioda_file_0 = os.path.basename(ioda_path_files[-1]) @@ -144,7 +139,7 @@ def execute(self): # Create new file name new_name_split = ioda_file_0_ del new_name_split[1] - new_name = os.path.join(cycle_dir, '_'.join(new_name_split)) + new_name = os.path.join(self.cycle_dir(), '_'.join(new_name_split)) # Check if new file already exists and remove if so if os.path.exists(new_name): @@ -155,10 +150,10 @@ def execute(self): # Run the combine step geo_dir = None if produce_geovals: - geo_dir = cycle_dir + geo_dir = self.cycle_dir() # Remove wind_reduction_factor_at_10m from non-uv geoval files - geoval_files = glob.glob(os.path.join(cycle_dir, + geoval_files = glob.glob(os.path.join(self.cycle_dir(), f'{needed_ioda_type}_*_geoval_*.nc4')) for geoval_file in geoval_files: if f'{needed_ioda_type}_uv_geoval_' not in geoval_file: @@ -200,18 +195,18 @@ def execute(self): # Radiances Diag = gsid.Radiances(gsi_obs_file[0]) Diag.read() - Diag.toIODAobs(cycle_dir, False, False, False) + Diag.toIODAobs(self.cycle_dir(), False, False, False) else: # Ozone Diag = gsid.Ozone(gsi_obs_file[0]) Diag.read() - Diag.toIODAobs(cycle_dir) + Diag.toIODAobs(self.cycle_dir()) # GeoVaLs call if produce_geovals: - Diag.toGeovals(cycle_dir) + Diag.toGeovals(self.cycle_dir()) if observation not in ozone_observations: Diag.close() @@ -222,20 +217,21 @@ def execute(self): # Input filename ioda_obs_in_pattern = f'{observation}_obs_*nc*' - ioda_obs_in = glob.glob(os.path.join(cycle_dir, ioda_obs_in_pattern))[0] + ioda_obs_in = glob.glob(os.path.join(self.cycle_dir(), ioda_obs_in_pattern))[0] ioda_obs_out = f'{observation}.{window_begin}.nc4' - os.rename(ioda_obs_in, os.path.join(cycle_dir, ioda_obs_out)) + os.rename(ioda_obs_in, os.path.join(self.cycle_dir(), ioda_obs_out)) # Rename GeoVaLs file if need be if produce_geovals: ioda_geoval_in_pattern = f'{observation}_geoval_*.nc*' - ioda_geoval_in = glob.glob(os.path.join(cycle_dir, ioda_geoval_in_pattern))[0] + ioda_geoval_in = glob.glob(os.path.join(self.cycle_dir(), + ioda_geoval_in_pattern))[0] ioda_geoval_out = f'{observation}_geovals.{window_begin}.nc4' - os.rename(ioda_geoval_in, os.path.join(cycle_dir, ioda_geoval_out)) + os.rename(ioda_geoval_in, os.path.join(self.cycle_dir(), ioda_geoval_out)) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/run_jedi_test_obs_filters_executable.py b/src/swell/tasks/run_jedi_test_obs_filters_executable.py index 3e7acd6e..5f2ba270 100644 --- a/src/swell/tasks/run_jedi_test_obs_filters_executable.py +++ b/src/swell/tasks/run_jedi_test_obs_filters_executable.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 @@ -11,44 +11,64 @@ import os import yaml -from swell.tasks.base.run_jedi_executable_base import RunJediExecutableBase +from swell.tasks.base.task_base import taskBase +from swell.utilities.run_jedi_executables import jedi_dictionary_iterator, run_executable # -------------------------------------------------------------------------------------------------- -class RunJediTestObsFiltersExecutable(RunJediExecutableBase): +class RunJediHofxExecutable(taskBase): # ---------------------------------------------------------------------------------------------- def execute(self): - # Path to executable being run - # ---------------------------- - cycle_dir = self.config_get('cycle_dir') - experiment_dir = self.config_get('experiment_dir') - observations = self.config_get('observations') - window_begin = self.config_get('window_begin') - - # Make cycle dir - # -------------- - os.makedirs(cycle_dir, 0o755, exist_ok=True) + # Jedi application name + # --------------------- + jedi_application = 'test_obs_filters' + + # Parse configuration + # ------------------- + window_offset = self.config.window_offset() + window_length = self.config.window_length() + bkg_time_offset = self.config.background_time_offset() + observations = self.config.observations() + + # Compute data assimilation window parameters + window_begin = self.da_window_params.window_begin(window_offset) + window_begin_iso = self.da_window_params.window_begin_iso(window_offset) + window_end_iso = self.da_window_params.window_end_iso(window_offset, window_length) + + # Populate jedi interface templates dictionary + # -------------------------------------------- + background_time = self.da_window_params.background_time(window_offset, bkg_time_offset) + self.jedi_rendering.add_key('window_begin_iso', window_begin_iso) + self.jedi_rendering.add_key('window_end_iso', window_end_iso) + + # Observations + self.jedi_rendering.add_key('background_time', background_time) + self.jedi_rendering.add_key('crtm_coeff_dir', self.config.crtm_coeff_dir(None)) + self.jedi_rendering.add_key('window_begin', window_begin) # Jedi configuration file # ----------------------- - jedi_config_file = os.path.join(cycle_dir, 'jedi_test_obs_filters.yaml') + jedi_config_file = os.path.join(self.cycle_dir(), f'jedi_{jedi_application}_config.yaml') # Output log file # --------------- - output_log_file = os.path.join(cycle_dir, 'jedi_test_obs_filters.log') + output_log_file = os.path.join(self.cycle_dir(), f'jedi_{jedi_application}_log.log') + + # Open the JEDI config file and fill initial templates + # ---------------------------------------------------- + jedi_config_dict = self.jedi_rendering.render_oops_file(f'{jedi_application}') - # Generate the JEDI configuration file for running the executable - # --------------------------------------------------------------- - jedi_config_dict = self.generate_jedi_config('TestObsFilters') + # Perform complete template rendering + # ----------------------------------- + jedi_dictionary_iterator(jedi_config_dict, self.jedi_rendering, '3D', observations) # Make modifications needed for testing # ------------------------------------- - conventional_types = ['aircraft'] # Loop over the observations @@ -58,33 +78,39 @@ def execute(self): if 'get values' in jedi_config_dict['observations'][index]: del jedi_config_dict['observations'][index]['get values'] + # GeoVaLs filename + geo_va_ls_fname = os.path.join(self.cycle_dir(), + f'{observations[index]}_geovals.{window_begin}.nc4') + # Create GeoVaLs dictionary - geovals = {} - geovals['filename'] = os.path.join(cycle_dir, - f'{observations[index]}_geovals.{window_begin}.nc4') + geo_va_ls_dict = {} + geo_va_ls_dict['filename'] = geo_va_ls_fname + # For conventional add the GeoVaLs flip if observations[index] in conventional_types: - geovals['levels_are_top_down'] = True + geo_va_ls_dict['levels_are_top_down'] = True - jedi_config_dict['observations'][index]['geovals'] = geovals + jedi_config_dict['observations'][index]['geovals'] = geo_va_ls_dict # Need to insert at least one benchmark, but we do not really want to check anything # so check that some made up variable does not exist - jedi_config_dict['observations'][index]['expectVariablesNotToExist'] = \ - [{'name': 'Fake/Group/Var'}] + dummy_search = [{'name': 'Dummy/Group/Var'}] + jedi_config_dict['observations'][index]['expectVariablesNotToExist'] = dummy_search - # Write executable configuration to file - # -------------------------------------- + # Write the expanded dictionary to YAML file + # ------------------------------------------ with open(jedi_config_file, 'w') as jedi_config_file_open: yaml.dump(jedi_config_dict, jedi_config_file_open, default_flow_style=False) # Jedi executable name # -------------------- - jedi_executable_path = os.path.join(experiment_dir, 'jedi_bundle', 'build', 'bin', + jedi_executable_path = os.path.join(self.experiment_path(), 'jedi_bundle', 'build', 'bin', 'test_ObsFilters.x') # Run the JEDI executable # ----------------------- - self.run_executable(cycle_dir, 1, jedi_executable_path, jedi_config_file, output_log_file) + run_executable(self.logger, self.cycle_dir(), 1, jedi_executable_path, jedi_config_file, + output_log_file) + self.logger.info('Running '+jedi_executable_path+' with '+str(np)+' processors.') # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/run_jedi_test_obs_filters_executable_old.py b/src/swell/tasks/run_jedi_test_obs_filters_executable_old.py new file mode 100644 index 00000000..0e3237a5 --- /dev/null +++ b/src/swell/tasks/run_jedi_test_obs_filters_executable_old.py @@ -0,0 +1,88 @@ +# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import os +import yaml + +from swell.tasks.base.run_jedi_executable_base import RunJediExecutableBase + + +# -------------------------------------------------------------------------------------------------- + + +class RunJediTestObsFiltersExecutable(RunJediExecutableBase): + + # ---------------------------------------------------------------------------------------------- + + def execute(self): + + # Path to executable being run + # ---------------------------- + cycle_dir = self.config_get('cycle_dir') + experiment_dir = self.config_get('experiment_dir') + observations = self.config_get('observations') + window_begin = self.config_get('window_begin') + + # Make cycle dir + # -------------- + os.makedirs(cycle_dir, 0o755, exist_ok=True) + + # Jedi configuration file + # ----------------------- + jedi_config_file = os.path.join(cycle_dir, 'jedi_test_obs_filters.yaml') + + # Output log file + # --------------- + output_log_file = os.path.join(cycle_dir, 'jedi_test_obs_filters.log') + + # Generate the JEDI configuration file for running the executable + # --------------------------------------------------------------- + jedi_config_dict = self.generate_jedi_config('TestObsFilters') + + # Make modifications needed for testing + # ------------------------------------- + + conventional_types = ['aircraft'] + + # Loop over the observations + for index in range(len(observations)): + + # Remove GetValues if present + if 'get values' in jedi_config_dict['observations'][index]: + del jedi_config_dict['observations'][index]['get values'] + + # Create GeoVaLs dictionary + geovals = {} + geovals['filename'] = os.path.join(cycle_dir, + f'{observations[index]}_geovals.{window_begin}.nc4') + # For conventional add the GeoVaLs flip + if observations[index] in conventional_types: + geovals['levels_are_top_down'] = True + + jedi_config_dict['observations'][index]['geovals'] = geovals + + # Need to insert at least one benchmark, but we do not really want to check anything + # so check that some made up variable does not exist + + # Write executable configuration to file + # -------------------------------------- + with open(jedi_config_file, 'w') as jedi_config_file_open: + yaml.dump(jedi_config_dict, jedi_config_file_open, default_flow_style=False) + + # Jedi executable name + # -------------------- + jedi_executable_path = os.path.join(experiment_dir, 'jedi_bundle', 'build', 'bin', + 'test_ObsFilters.x') + + # Run the JEDI executable + # ----------------------- + self.run_executable(cycle_dir, 1, jedi_executable_path, jedi_config_file, output_log_file) + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/stage_jedi.py b/src/swell/tasks/stage_jedi.py index 479cdd04..8e1d74cd 100644 --- a/src/swell/tasks/stage_jedi.py +++ b/src/swell/tasks/stage_jedi.py @@ -9,14 +9,10 @@ import os -import re -import glob -from shutil import copyfile from swell.tasks.base.task_base import taskBase from swell.utilities.filehandler import * from swell.utilities.exceptions import * -from r2d2 import fetch # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/store_background.py b/src/swell/tasks/store_background.py index 4172c770..338289ea 100644 --- a/src/swell/tasks/store_background.py +++ b/src/swell/tasks/store_background.py @@ -11,7 +11,6 @@ from datetime import datetime as dt import isodate import os -import re from r2d2 import store diff --git a/src/swell/test/question_dictionary_comparison_test.py b/src/swell/test/question_dictionary_comparison_test.py index bf0229d1..49b023b4 100644 --- a/src/swell/test/question_dictionary_comparison_test.py +++ b/src/swell/test/question_dictionary_comparison_test.py @@ -40,4 +40,5 @@ def test_dictionary_comparison(self): # Assert that dictionaries are equal self.assertDictEqual(question_dict_in, question_dict) + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/test/test_suite.py b/src/swell/test/test_suite.py index 0a18440e..e521576a 100644 --- a/src/swell/test/test_suite.py +++ b/src/swell/test/test_suite.py @@ -11,6 +11,7 @@ import unittest from swell.test.question_dictionary_comparison_test import QuestionDictionaryTest +from swell.test.unused_variables_test import UnusedVariablesTest # -------------------------------------------------------------------------------------------------- @@ -20,7 +21,10 @@ def main(): # Create a test suite test_suite = unittest.TestSuite() - # Load tests from QuestionDictionaryTest + # Load unused variable test + test_suite.addTests(unittest.TestLoader().loadTestsFromTestCase(UnusedVariablesTest)) + + # Load tests from UnusedVariablesTest test_suite.addTests(unittest.TestLoader().loadTestsFromTestCase(QuestionDictionaryTest)) # Create a test runner diff --git a/src/swell/test/unused_variables_test.py b/src/swell/test/unused_variables_test.py new file mode 100644 index 00000000..865b1bb7 --- /dev/null +++ b/src/swell/test/unused_variables_test.py @@ -0,0 +1,44 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import unittest +import subprocess +import os + +from swell.swell_path import get_swell_path + + +# -------------------------------------------------------------------------------------------------- + + +def run_flake8(file_path): + flake8_cmd = ['flake8', '--select', 'F401,F841', file_path] + result = subprocess.run(flake8_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) + return result.stdout.strip() + + +# -------------------------------------------------------------------------------------------------- + + +class UnusedVariablesTest(unittest.TestCase): + + def test_unused_variables(self): + + for root, _, files in os.walk(get_swell_path()): + for filename in files: + if filename.endswith('.py'): # Only process Python files + file_path = os.path.join(root, filename) + flake8_output = run_flake8(file_path) + + self.assertEqual(flake8_output, '', f"Unused variables found in {file_path}") + + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/bin/generate_task_question_default_dicts.py b/src/swell/utilities/bin/generate_task_question_default_dicts.py index 71d00794..53db80dc 100644 --- a/src/swell/utilities/bin/generate_task_question_default_dicts.py +++ b/src/swell/utilities/bin/generate_task_question_default_dicts.py @@ -10,14 +10,12 @@ # standard imports -import glob import os import yaml # swell imports from swell.swell_path import get_swell_path from swell.utilities.logger import Logger -from swell.utilities.case_switching import snake_case_to_camel_case # -------------------------------------------------------------------------------------------------- @@ -28,9 +26,6 @@ def main(): # Create a logger logger = Logger('ListOfTaskQuestions') - # Path to JEDI interface code - swell_path = get_swell_path() - # Output file task_questions_config = os.path.join(get_swell_path(), 'tasks', 'task_questions.yaml') diff --git a/src/swell/utilities/bin/generate_task_questions_dict.py b/src/swell/utilities/bin/generate_task_questions_dict.py index d897a75d..66abc94e 100644 --- a/src/swell/utilities/bin/generate_task_questions_dict.py +++ b/src/swell/utilities/bin/generate_task_questions_dict.py @@ -28,9 +28,6 @@ def main(): # Create a logger logger = Logger('ListOfTaskQuestions') - # Path to JEDI interface code - swell_path = get_swell_path() - # All python files task_codes = glob.glob(os.path.join(get_swell_path(), 'tasks', '*.py')) task_codes = list(filter(lambda task_code: task_code != '__init__.py', task_codes)) @@ -38,6 +35,10 @@ def main(): # Target YAML file destination_yaml = os.path.join(get_swell_path(), 'tasks', 'task_questions.yaml') + # Print message + logger.info(f'Running tool to regenerate the task questions dictionary. Output dictionary ' + + f'be written to {destination_yaml}') + # Read input file into dictionary if os.path.exists(destination_yaml): with open(destination_yaml, 'r') as ymlfile: @@ -45,8 +46,6 @@ def main(): else: question_dict = {} - question_dict_in = question_dict.copy() - # Now safe to overwrite file outfile = open(destination_yaml, 'w') diff --git a/src/swell/utilities/build.py b/src/swell/utilities/build.py index 532a6d05..64780271 100644 --- a/src/swell/utilities/build.py +++ b/src/swell/utilities/build.py @@ -10,7 +10,7 @@ import os import shutil -from jedi_bundle.bin.jedi_bundle import get_default_config, get_bundles +from jedi_bundle.bin.jedi_bundle import get_default_config # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/data_assimilation_window_params.py b/src/swell/utilities/data_assimilation_window_params.py index 13ab1ec8..a343a349 100644 --- a/src/swell/utilities/data_assimilation_window_params.py +++ b/src/swell/utilities/data_assimilation_window_params.py @@ -66,6 +66,18 @@ def window_begin_iso(self, window_offset): # ---------------------------------------------------------------------------------------------- + def window_end_iso(self, window_offset, window_length): + + # Compute window length duration + window_length_dur = isodate.parse_duration(window_length) + + # Get window beginning time + window_begin_dto = self.__get_window_begin_dto__(window_offset) + + return window_begin_dto + window_length_dur + + # ---------------------------------------------------------------------------------------------- + def background_time(self, window_offset, background_time_offset): background_time_offset_dur = isodate.parse_duration(background_time_offset) diff --git a/src/swell/utilities/dictionary.py b/src/swell/utilities/dictionary.py index dff0b39f..192a69f1 100644 --- a/src/swell/utilities/dictionary.py +++ b/src/swell/utilities/dictionary.py @@ -7,8 +7,6 @@ # -------------------------------------------------------------------------------------------------- -import re -import string import yaml from collections.abc import Hashable diff --git a/src/swell/utilities/jinja2.py b/src/swell/utilities/jinja2.py index 7dcbe6fb..fee239c3 100644 --- a/src/swell/utilities/jinja2.py +++ b/src/swell/utilities/jinja2.py @@ -8,7 +8,6 @@ import jinja2 -import yaml # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/run_jedi_executables.py b/src/swell/utilities/run_jedi_executables.py index 48d2ebe8..ca6247f9 100644 --- a/src/swell/utilities/run_jedi_executables.py +++ b/src/swell/utilities/run_jedi_executables.py @@ -17,7 +17,7 @@ def jedi_dictionary_iterator(jedi_config_dict, jedi_rendering, window_type, obs, - jedi_forecast_model): + jedi_forecast_model=None): # Assemble configuration YAML file # -------------------------------- diff --git a/src/swell/utilities/shell_commands.py b/src/swell/utilities/shell_commands.py index c5ac871e..28cced3d 100644 --- a/src/swell/utilities/shell_commands.py +++ b/src/swell/utilities/shell_commands.py @@ -70,7 +70,8 @@ def run_subprocess(logger, command, stdout=None, stderr=None): try: subprocess.run(command, check=True, stdout=stdout, stderr=stderr) except subprocess.CalledProcessError as e: - logger.abort('Subprocess with command {command} failed, throwing error \n{e}') + print(e) + logger.abort('Subprocess with command {command} failed, throwing error {error}') # -------------------------------------------------------------------------------------------------- From 23c80dbb3759dcf52af96f9a450e3ad02eaa1f98 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Mon, 5 Jun 2023 12:42:12 -0400 Subject: [PATCH 059/121] Add questions to task_questions --- src/swell/tasks/task_questions.yaml | 70 +++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/src/swell/tasks/task_questions.yaml b/src/swell/tasks/task_questions.yaml index 9600bf14..fc469b86 100644 --- a/src/swell/tasks/task_questions.yaml +++ b/src/swell/tasks/task_questions.yaml @@ -66,8 +66,11 @@ background_time_offset: forecast begin? tasks: - EvaObservations + - GetGeovals - GetObservations + - GsiBcToIoda - RunJediHofxExecutable + - RunJediTestObsFiltersExecutable - RunJediVariationalExecutable - SaveObsDiags type: duration @@ -110,8 +113,11 @@ crtm_coeff_dir: prompt: What is the path to the CRTM coefficient files? tasks: - EvaObservations + - GetGeovals - GetObservations + - GsiBcToIoda - RunJediHofxExecutable + - RunJediTestObsFiltersExecutable - RunJediVariationalExecutable - SaveObsDiags type: string @@ -211,6 +217,26 @@ geos_gcm_tag: - CloneGeos type: string +geovals_experiment: + ask_question: true + default_value: defer_to_model + models: + - geos_atmosphere + prompt: What is the name of the R2D2 experiment providing the GeoVaLs? + tasks: + - GetGeovals + type: string + +geovals_provider: + ask_question: true + default_value: defer_to_model + models: + - geos_atmosphere + prompt: What is the name of the R2D2 database providing the GeoVaLs? + tasks: + - GetGeovals + type: string + gradient_norm_reduction: ask_question: false default_value: defer_to_model @@ -221,6 +247,17 @@ gradient_norm_reduction: - RunJediVariationalExecutable type: float +gsi_bc_location: + ask_question: true + default_value: defer_to_model + models: + - geos_atmosphere + options: defer_to_model + prompt: What is the location where GSI bias correction files can be found? + tasks: + - GetGsiBc + type: string + horizontal_resolution: ask_question: true default_value: defer_to_model @@ -353,12 +390,39 @@ observations: prompt: Which observations do you want to include? tasks: - EvaObservations + - GetGeovals - GetObservations + - GsiBcToIoda + - GsiNcdiagToIoda - RunJediHofxExecutable + - RunJediTestObsFiltersExecutable - RunJediVariationalExecutable - SaveObsDiags type: string-check-list +path_to_gsi_diags: + ask_question: true + default_value: defer_to_model + models: + - geos_atmosphere + prompt: What is the path to where the GSI ncdiags are stored? + tasks: + - GetGsiNcdiag + type: string + +produce_geovals: + ask_question: true + default_value: defer_to_model + models: + - geos_atmosphere + options: + - true + - false + prompt: When running the ncdiag to ioda converted do you want to produce GeoVaLs files? + tasks: + - GsiNcdiagToIoda + type: boolean + static_background_error_model: ask_question: true default_value: defer_to_model @@ -428,8 +492,10 @@ window_length: prompt: What is the duration for the data assimilation window? tasks: - GetBackground + - GetGeovals - GetObservations - RunJediHofxExecutable + - RunJediTestObsFiltersExecutable - RunJediVariationalExecutable - StoreBackground type: duration @@ -443,8 +509,12 @@ window_offset: tasks: - EvaObservations - GetBackground + - GetGeovals - GetObservations + - GsiBcToIoda + - GsiNcdiagToIoda - RunJediHofxExecutable + - RunJediTestObsFiltersExecutable - RunJediVariationalExecutable - SaveObsDiags - StoreBackground From bf07cf97b787bbb5ad98ba957337f2f81d378309 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Mon, 5 Jun 2023 12:42:41 -0400 Subject: [PATCH 060/121] Add questions to task_questions formatting --- src/swell/tasks/task_questions.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/swell/tasks/task_questions.yaml b/src/swell/tasks/task_questions.yaml index fc469b86..fa9e6921 100644 --- a/src/swell/tasks/task_questions.yaml +++ b/src/swell/tasks/task_questions.yaml @@ -418,7 +418,8 @@ produce_geovals: options: - true - false - prompt: When running the ncdiag to ioda converted do you want to produce GeoVaLs files? + prompt: When running the ncdiag to ioda converted do you want to produce GeoVaLs + files? tasks: - GsiNcdiagToIoda type: boolean From f049afe5a29b7f3718792b5480e6735190325fa6 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Mon, 5 Jun 2023 12:47:10 -0400 Subject: [PATCH 061/121] add question answers to model interface --- .../geos_atmosphere/task_questions.yaml | 18 ++++++++++++++++++ .../suites-test_obs_filters.yaml | 18 ++---------------- src/swell/tasks/task_questions.yaml | 3 +-- 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml index 57b6eb88..6facbf95 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml @@ -41,9 +41,18 @@ geos_bkg_filename_template: geos_bkg_tar_filename_template: default_value: TODO +geovals_experiment: + default_value: x0044_test_obs_filters_geovals + +geovals_provider: + default_value: ncdiag + gradient_norm_reduction: default_value: 10e-5 +gsi_bc_location: + default_value: /discover/nobackup/projects/gmao/dadev/rtodling/archive/x0044/rs/Y2020/M12/x0044.rst.20201214_21z.tar + horizontal_resolution: default_value: C361 options: @@ -109,6 +118,15 @@ observations: - omi_aura - ompsnm_npp +path_to_gsi_diags: + default_value: /archive/u/jjin3/x0044.jj_20230201/obs/Y2020/M12/D15/H00 + +produce_geovals: + default_value: true + options: + - true + - false + static_background_error_model: default_value: GSIbec options: diff --git a/src/swell/suites/test_obs_filters/suites-test_obs_filters.yaml b/src/swell/suites/test_obs_filters/suites-test_obs_filters.yaml index 1901480f..0e7bd3b6 100644 --- a/src/swell/suites/test_obs_filters/suites-test_obs_filters.yaml +++ b/src/swell/suites/test_obs_filters/suites-test_obs_filters.yaml @@ -24,7 +24,7 @@ jedi_build_method: type: string-drop-list # Existing JEDI bundle directory -existing_source_directory: +existing_jedi_source_directory: default_value: '/discover/nobackup/drholdaw/JediDev/swell-ufo' prompt: Provide the path to an existing JEDI bundle directory containing source code repos type: string @@ -33,7 +33,7 @@ existing_source_directory: value: use_existing # Existing JEDI build -existing_build_directory: +existing_jedi_build_directory: default_value: '/discover/nobackup/drholdaw/JediDev/swell-ufo/build-intel-release' prompt: Provide the path to an existing JEDI build directory type: string @@ -47,23 +47,9 @@ r2d2_local_path: prompt: Enter the path where R2D2 will store experiment output type: string -# What kind of coupling is needed -coupling_style: - default_value: uncoupled - prompt: What kind of coupling would you like to run with? - options: ['uncoupled', 'weakly coupled', 'strongly coupled'] - type: string-drop-list - # Models to use model_components: default_value: ['geos_atmosphere'] prompt: Select models to use (choose at least one) options: use_method type: file-check-list - -# Fixed options -fixed_options: - data_assimilation_run: - default_value: true - prompt: Does this workflow include data assimilation? - diff --git a/src/swell/tasks/task_questions.yaml b/src/swell/tasks/task_questions.yaml index fa9e6921..7ed9cfe5 100644 --- a/src/swell/tasks/task_questions.yaml +++ b/src/swell/tasks/task_questions.yaml @@ -228,7 +228,7 @@ geovals_experiment: type: string geovals_provider: - ask_question: true + ask_question: false default_value: defer_to_model models: - geos_atmosphere @@ -252,7 +252,6 @@ gsi_bc_location: default_value: defer_to_model models: - geos_atmosphere - options: defer_to_model prompt: What is the location where GSI bias correction files can be found? tasks: - GetGsiBc From ac93f14d86930919902bff602fe275977683d3c5 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Mon, 5 Jun 2023 22:48:26 -0400 Subject: [PATCH 062/121] Do not overwrite the dictionary in the source code --- .../question_dictionary_comparison_test.py | 16 +------- .../bin/generate_task_questions_dict.py | 39 +++++++++++++++---- 2 files changed, 33 insertions(+), 22 deletions(-) diff --git a/src/swell/test/question_dictionary_comparison_test.py b/src/swell/test/question_dictionary_comparison_test.py index bf0229d1..b912ca48 100644 --- a/src/swell/test/question_dictionary_comparison_test.py +++ b/src/swell/test/question_dictionary_comparison_test.py @@ -22,22 +22,10 @@ class QuestionDictionaryTest(unittest.TestCase): def test_dictionary_comparison(self): - # First test the - destination_yaml = os.path.join(get_swell_path(), 'tasks', 'task_questions.yaml') - - # Read input file into dictionary - with open(destination_yaml, 'r') as ymlfile: - question_dict = yaml.safe_load(ymlfile) - question_dict_in = question_dict.copy() - # Run dictionary generation - generate_task_questions_dict() - - # Read new dictionary - with open(destination_yaml, 'r') as ymlfile: - question_dict = yaml.safe_load(ymlfile) + return_code = generate_task_questions_dict() # Assert that dictionaries are equal - self.assertDictEqual(question_dict_in, question_dict) + assert return_code == 0 # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/bin/generate_task_questions_dict.py b/src/swell/utilities/bin/generate_task_questions_dict.py index d897a75d..6e30919f 100644 --- a/src/swell/utilities/bin/generate_task_questions_dict.py +++ b/src/swell/utilities/bin/generate_task_questions_dict.py @@ -12,6 +12,8 @@ # standard imports import glob import os +import random +import string import yaml # swell imports @@ -37,6 +39,7 @@ def main(): # Target YAML file destination_yaml = os.path.join(get_swell_path(), 'tasks', 'task_questions.yaml') + destination_yaml = os.path.join(get_swell_path(), 'tasks', 'task_questions.yaml') # Read input file into dictionary if os.path.exists(destination_yaml): @@ -47,9 +50,6 @@ def main(): question_dict_in = question_dict.copy() - # Now safe to overwrite file - outfile = open(destination_yaml, 'w') - # Loop through task code and accumulate all lines containing a use of config config_keys = [] task_names = [] @@ -90,8 +90,6 @@ def main(): tasks = sorted(list(set(tasks))) # Create dictionary to hold question components - question_to_tasks = {} - if unique_key in question_dict: question_to_tasks[unique_key] = question_dict[unique_key] @@ -124,10 +122,35 @@ def main(): # Regardless of whether question was already in dictionary question_to_tasks[unique_key]['tasks'] = tasks - outfile.write(yaml.dump(question_to_tasks, default_flow_style=False)) - outfile.write('\n') + if question_to_tasks == question_dict_in: + + logger.info(f'The code will make no change to the task questions dictionary. Clean exit') + return 0 + + else: - outfile.close() + # Create a file in the tmp directory with some random characters + random_string = ''.join(random.choices(string.ascii_letters + string.digits, k=8)) + destination_yaml_temp = os.path.join('/tmp', f'task_questions_{random_string}.yaml') + + logger.info(f'The configuration elements that are being accessed by the tasks are out of ' + + f'sync with what is described in \'{destination_yaml}\'. The expected ' + + f'dictionary will be written to a temporary file {destination_yaml_temp}. ' + + f'Compare this file with the one in the tasks directory and resolve the ' + + f'differences.') + + # Changes to the dictionary. + outfile = open(destination_yaml_temp, 'w') + for key, value in question_to_tasks.items(): + # Create dictionary one at a time and write + dict_to_write = {} + dict_to_write[key] = value + outfile.write(yaml.dump(dict_to_write, default_flow_style=False)) + # Add a gap between dictionaries to make it more human readable. + outfile.write('\n') + outfile.close() + + return 1 # -------------------------------------------------------------------------------------------------- From 84201205c39653b3d77b3dfa2e64ca28fd5fb4ed Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 7 Jun 2023 11:25:58 -0400 Subject: [PATCH 063/121] more checking --- .../generate_task_question_default_dicts.py | 270 ++++++++++++------ .../bin/generate_task_questions_dict.py | 52 ++-- 2 files changed, 206 insertions(+), 116 deletions(-) diff --git a/src/swell/utilities/bin/generate_task_question_default_dicts.py b/src/swell/utilities/bin/generate_task_question_default_dicts.py index 71d00794..296eb70c 100644 --- a/src/swell/utilities/bin/generate_task_question_default_dicts.py +++ b/src/swell/utilities/bin/generate_task_question_default_dicts.py @@ -12,6 +12,8 @@ # standard imports import glob import os +import random +import string import yaml # swell imports @@ -23,6 +25,109 @@ # -------------------------------------------------------------------------------------------------- +def create_jedi_tq_dicts(logger, jedi_interface_name, tq_dicts, jedi_tq_dicts_str_in): + + # Convert string read from file to dictionary + if jedi_tq_dicts_str_in == '': + jedi_tq_dicts = {} + else: + jedi_tq_dicts = yaml.safe_load(jedi_tq_dicts_str_in) + + # Create string that will hold the new dictionaries + jedi_tq_dicts_str = '' + + # Loop over main question list + for tq, tq_dict in tq_dicts.items(): + + # Check if question defers the default value to the jedi interface + if tq_dict['default_value'] == 'defer_to_model': + + # If default is deferred to model the dict must contain the jedi interface list + logger.assert_abort('models' in tq_dict, f'If the default for the config is ' + + f'\'defer_to_model\' then the question dictionary must ' + + f'contain \'models\'. Offending task question: {tq}') + + # Set the required jedi interfaces for this question + jedi_int_needed = tq_dict['models'] + + # This tq is only added if this model in question can provide that config + if jedi_int_needed[0] == 'all' or jedi_interface_name in jedi_int_needed: + + # Create dictionary to hold this task question + jedi_tq_dict = {tq: {}} + + # Create dictionary if it does not already exist + if tq not in jedi_tq_dicts: + + # Create defaults dictionary for the question + jedi_tq_dict[tq]['default_value'] = ['defer_to_model'] + + if 'options' in tq_dict: + jedi_tq_dict[tq]['options'] = ['defer_to_model'] + + else: + + # Copy from the existing dictionary + jedi_tq_dict[tq] = jedi_tq_dicts[tq] + + # Add dictionary to the output dictionary + jedi_tq_dicts_str = jedi_tq_dicts_str + yaml.dump(jedi_tq_dict, + default_flow_style=False) + jedi_tq_dicts_str = jedi_tq_dicts_str + '\n' + + # Return dictionary in string format + return jedi_tq_dicts_str + + +# -------------------------------------------------------------------------------------------------- + + +def create_platform_tq_dicts(logger, platform_name, tq_dicts, platform_tq_dicts_str_in): + + # Convert string read from file to dictionary + if platform_tq_dicts_str_in == '': + platform_tq_dicts = {} + else: + platform_tq_dicts = yaml.safe_load(platform_tq_dicts_str_in) + + # Create string that will hold the new dictionaries + platform_tq_dicts_str = '' + + # Loop over main question list + for tq, tq_dict in tq_dicts.items(): + + # Check if question defers the default value to the platform + if tq_dict['default_value'] == 'defer_to_platform': + + # Create dictionary to hold this task question + platform_tq_dict = {tq: {}} + + # Create dictionary if it does not already exist + if tq not in platform_tq_dicts: + + # Create defaults dictionary for the question + platform_tq_dict[tq]['default_value'] = ['defer_to_model'] + + if 'options' in tq_dict: + platform_tq_dict[tq]['options'] = ['defer_to_model'] + + else: + + # Copy from the existing dictionary + platform_tq_dict[tq] = platform_tq_dicts[tq] + + # Add dictionary to the output dictionary + platform_tq_dicts_str = platform_tq_dicts_str + yaml.dump(platform_tq_dict, + default_flow_style=False) + platform_tq_dicts_str = platform_tq_dicts_str + '\n' + + # Return dictionary in string format + return platform_tq_dicts_str + + +# -------------------------------------------------------------------------------------------------- + + def main(): # Create a logger @@ -37,123 +142,100 @@ def main(): # Read input file into dictionary if os.path.exists(task_questions_config): with open(task_questions_config, 'r') as ymlfile: - questions_dict = yaml.safe_load(ymlfile) + tq_dicts = yaml.safe_load(ymlfile) else: logger.abort(f'Did not fine the task questions dictionary at {task_questions_config}') - # Generate list of jedi interfaces + # Generate list of JEDI interfaces + # -------------------------------- jedi_interfaces_path = os.path.join(get_swell_path(), 'configuration', 'jedi', 'interfaces') jedi_interfaces = [f.path for f in os.scandir(jedi_interfaces_path) if f.is_dir()] jedi_interfaces = list(filter(lambda jedi_interface: '__' not in jedi_interface, jedi_interfaces)) jedi_interface_names = [] + jedi_interface_paths = [] for jedi_interface in jedi_interfaces: jedi_interface_names.append(os.path.basename(jedi_interface)) - - # Loop over jedi interfaces - for jedi_interface_name in jedi_interface_names: - - question_defaults_dict = os.path.join(jedi_interfaces_path, jedi_interface_name, - 'task_questions.yaml') - - # Open the existing file - with open(question_defaults_dict, 'r') as ymlfile: - question_dict_defaults_exist = yaml.safe_load(ymlfile) - - if question_dict_defaults_exist is None: - question_dict_defaults_exist = {} - - # Open file ready to overwrite - outfile = open(question_defaults_dict, 'w') - - # Loop over main question list - for question_key, question_dict in questions_dict.items(): - - # Check if question defers the default value to the jedi interface - if question_dict['default_value'] == 'defer_to_model': - - # If default is deferred to model the dict must contain the jedi interface list - logger.assert_abort('models' in question_dict, f'If the default for the config ' + - f'is defer to model then the question dictionary must ' + - f'contain models. Offending key: {question_key}') - - # Set the required jedi interfaces for this question - jedi_int_needed = question_dict['models'] - if jedi_int_needed[0] == 'all' or jedi_interface_name in jedi_int_needed: - - # Create dictionary if it does not already exist - if question_key not in question_dict_defaults_exist: - - # Create defaults dictionary for the question - question_dict_defaults = {question_key: { - 'default_value': 'defer_to_model' - }} - - if 'options' in question_dict: - question_dict_defaults[question_key]['options'] = ['defer_to_model'] - - else: - - # Copy from the existing dictionary - question_dict_defaults = { - question_key: question_dict_defaults_exist[question_key] - } - - # Write to the YAML file - outfile.write(yaml.dump(question_dict_defaults, default_flow_style=False)) - outfile.write('\n') + jedi_interface_paths.append(os.path.join(jedi_interfaces_path, jedi_interface_names[-1], + 'task_questions.yaml')) # Generate list of platforms + # -------------------------- platforms_path = os.path.join(get_swell_path(), 'deployment', 'platforms') platforms = [f.path for f in os.scandir(platforms_path) if f.is_dir()] platforms = list(filter(lambda platform: '__' not in platform, platforms)) platform_names = [] + platform_paths = [] for platform in platforms: platform_names.append(os.path.basename(platform)) + platform_paths.append(os.path.join(platforms_path, platform_names[-1], + 'task_questions.yaml')) - # Loop over platforms - for platform_name in platform_names: - - question_defaults_dict = os.path.join(platforms_path, platform_name, - 'task_questions.yaml') - - # Open the existing file - with open(question_defaults_dict, 'r') as ymlfile: - question_dict_defaults_exist = yaml.safe_load(ymlfile) - - if question_dict_defaults_exist is None: - question_dict_defaults_exist = {} - - # Open file ready to overwrite - outfile = open(question_defaults_dict, 'w') + # Loop over jedi interfaces and platforms and create dictionaries + # --------------------------------------------------------------- + # Combine jedi and platforms + tq_dicts_names = jedi_interface_names + platform_names + tq_dicts_paths = jedi_interface_paths + platform_paths - # Loop over main question list - for question_key, question_dict in questions_dict.items(): + # Record whether anything failed + failure = 0 - # Check if question defers the default value to the platform - if question_dict['default_value'] == 'defer_to_platform': + for tq_dicts_name, tq_dicts_path in zip(tq_dicts_names, tq_dicts_paths): - # Create dictionary if it does not already exist - if question_key not in question_dict_defaults_exist: - - # Create defaults dictionary for the question - question_dict_defaults = {question_key: { - 'default_value': 'defer_to_model' - }} + logger.info(f'Processing defaults dictionary for \'{tq_dicts_name}\'') - if 'options' in question_dict: - question_dict_defaults[question_key]['options'] = ['defer_to_model'] - - else: - - # Copy from the existing dictionary - question_dict_defaults = { - question_key: question_dict_defaults_exist[question_key] - } - - # Write to the YAML file - outfile.write(yaml.dump(question_dict_defaults, default_flow_style=False)) - outfile.write('\n') + # Open the existing file + if os.path.exists(tq_dicts_path): + with open(tq_dicts_path, 'r') as file: + dicts_str_in = file.read() + else: + dicts_str_in = '' + + # Create a new dictionary for this interface + if tq_dicts_name in jedi_interface_names: + tq_dicts_str = create_jedi_tq_dicts(logger, tq_dicts_name, tq_dicts, dicts_str_in) + elif tq_dicts_name in platform_names: + tq_dicts_str = create_platform_tq_dicts(logger, tq_dicts_name, tq_dicts, dicts_str_in) + + # Perform comparison of input and output and write temporaries if needed + + # Check whether the string of the new dictionary matches the existing one. + if tq_dicts_str == dicts_str_in: + + logger.info(f'The code will make no change to the task questions dictionary.') + + else: + + # Write the new dictionary to a temporary file + random_string = ''.join(random.choices(string.ascii_letters + string.digits, k=8)) + temp_tq_file_name = f'{tq_dicts_name}_task_questions_{random_string}.yaml' + destination_yaml_temp = os.path.join('/tmp', temp_tq_file_name) + + with open(destination_yaml_temp, 'w') as file: + file.write(tq_dicts_str) + + logger.info(f'If a new \'task_questions.yaml\' is generated for \'{tq_dicts_name}\' ' + + f'using this utility the resulting file will be different. This could ' + + f'be for a number of reasons:') + logger.info(f' ', False) + logger.info(f' - Comments were added to the original file.' ) + logger.info(f' - A new key is accessed from a task.' ) + logger.info(f' - Referencing of a particular key has been removed from a task.') + logger.info(f' ', False) + logger.info(f'Please compare the new (temporary) file \'{destination_yaml_temp}\' ' + + f'with the existing file: \'{tq_dicts_name}/task_questions.yaml\' and ' + + f'resolve the differences') + + failure = failure + 1 + + # Crete a space between loop elements + logger.info(' ', False) + + # Return code + if failure > 0: + return 1 + else: + return 0 # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/bin/generate_task_questions_dict.py b/src/swell/utilities/bin/generate_task_questions_dict.py index 6e30919f..add049c1 100644 --- a/src/swell/utilities/bin/generate_task_questions_dict.py +++ b/src/swell/utilities/bin/generate_task_questions_dict.py @@ -25,7 +25,7 @@ # -------------------------------------------------------------------------------------------------- -def main(): +def generate_tq_dict_defaults(): # Create a logger logger = Logger('ListOfTaskQuestions') @@ -39,16 +39,18 @@ def main(): # Target YAML file destination_yaml = os.path.join(get_swell_path(), 'tasks', 'task_questions.yaml') - destination_yaml = os.path.join(get_swell_path(), 'tasks', 'task_questions.yaml') # Read input file into dictionary if os.path.exists(destination_yaml): with open(destination_yaml, 'r') as ymlfile: - question_dict = yaml.safe_load(ymlfile) + question_dict_str = ymlfile.read() + question_dict = yaml.safe_load(question_dict_str) else: question_dict = {} + question_dict_str = '' question_dict_in = question_dict.copy() + question_dict_str_in = question_dict_str # Loop through task code and accumulate all lines containing a use of config config_keys = [] @@ -122,33 +124,39 @@ def main(): # Regardless of whether question was already in dictionary question_to_tasks[unique_key]['tasks'] = tasks - if question_to_tasks == question_dict_in: + # Create string with new dictionary + dict_to_write_str = '' + for key, value in question_to_tasks.items(): + # Create dictionary one at a time and write + dict_to_write = {} + dict_to_write[key] = value + dict_to_write_str = dict_to_write_str + yaml.dump(dict_to_write, default_flow_style=False) + dict_to_write_str = dict_to_write_str + '\n' + + # Check whether the string of the new dictionary matches the existing one. + if dict_to_write_str == question_dict_str_in: logger.info(f'The code will make no change to the task questions dictionary. Clean exit') return 0 else: - # Create a file in the tmp directory with some random characters + # Write the new dictionary to a temporary file random_string = ''.join(random.choices(string.ascii_letters + string.digits, k=8)) destination_yaml_temp = os.path.join('/tmp', f'task_questions_{random_string}.yaml') - logger.info(f'The configuration elements that are being accessed by the tasks are out of ' + - f'sync with what is described in \'{destination_yaml}\'. The expected ' + - f'dictionary will be written to a temporary file {destination_yaml_temp}. ' + - f'Compare this file with the one in the tasks directory and resolve the ' + - f'differences.') - - # Changes to the dictionary. - outfile = open(destination_yaml_temp, 'w') - for key, value in question_to_tasks.items(): - # Create dictionary one at a time and write - dict_to_write = {} - dict_to_write[key] = value - outfile.write(yaml.dump(dict_to_write, default_flow_style=False)) - # Add a gap between dictionaries to make it more human readable. - outfile.write('\n') - outfile.close() + with open(destination_yaml_temp, 'w') as file: + file.write(dict_to_write_str) + + logger.info(f'If a new \'task_questions.yaml\' is generated using this utility the ' + + f'resulting file will be different. This could be for a number of reasons:') + logger.info(f' ', False) + logger.info(f' - Comments were added to the original file.' ) + logger.info(f' - A new key is accessed from a task.' ) + logger.info(f' - Referencing of a particular key has been removed from a task.') + logger.info(f' ', False) + logger.info(f'Please compare the new (temporary) file \'{destination_yaml_temp}\' with ' + + f'the existing file: \'tasks/task_questions.yaml\' and resolve differences.') return 1 @@ -157,7 +165,7 @@ def main(): if __name__ == '__main__': - main() + generate_tq_dict_defaults() # -------------------------------------------------------------------------------------------------- From a4db01cf64fe5d6a3d4e7f44801d424cb9440f16 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 7 Jun 2023 12:40:16 -0400 Subject: [PATCH 064/121] improve testing of dictionary creation --- setup.py | 8 ++++---- .../test/question_dictionary_comparison_test.py | 14 +++++++++----- ...sk_questions_dict.py => task_question_dicts.py} | 4 ++-- ...lt_dicts.py => task_question_dicts_defaults.py} | 7 +++++-- 4 files changed, 20 insertions(+), 13 deletions(-) rename src/swell/utilities/bin/{generate_task_questions_dict.py => task_question_dicts.py} (98%) rename src/swell/utilities/bin/{generate_task_question_default_dicts.py => task_question_dicts_defaults.py} (98%) diff --git a/setup.py b/setup.py index cd34ca64..b0285c24 100644 --- a/setup.py +++ b/setup.py @@ -64,12 +64,12 @@ 'swell_prepare_experiment_config = swell.deployment.bin.swell_prepare_config:main', 'swell_launch_experiment = swell.deployment.bin.swell_launch_experiment:main', 'swell_sat_db_processing = swell.deployment.bin.swell_sat_db_processing:main', - 'swell_util_generate_task_question_default_dicts = \ - swell.utilities.bin.generate_task_question_default_dicts:main', + # Utilities + 'swell_util_task_question_dicts = swell.utilities.bin.task_question_dicts:tq_dicts', 'swell_util_check_jedi_interface_templates = \ swell.utilities.bin.check_jedi_interface_templates:main', - 'swell_util_generate_task_questions_dict = \ - swell.utilities.bin.generate_task_questions_dict:main', + 'swell_util_task_question_dicts_defaults = \ + swell.utilities.bin.swell_util_task_question_dicts_defaults:tq_dicts_defaults', 'swell_test_suite = swell.test.test_suite:main', ], }, diff --git a/src/swell/test/question_dictionary_comparison_test.py b/src/swell/test/question_dictionary_comparison_test.py index b912ca48..6eab93be 100644 --- a/src/swell/test/question_dictionary_comparison_test.py +++ b/src/swell/test/question_dictionary_comparison_test.py @@ -13,7 +13,8 @@ import yaml from swell.swell_path import get_swell_path -from swell.utilities.bin.generate_task_questions_dict import main as generate_task_questions_dict +from swell.utilities.bin.task_question_dicts import tq_dicts +from swell.utilities.bin.task_question_dicts_defaults import tq_dicts_defaults # -------------------------------------------------------------------------------------------------- @@ -22,10 +23,13 @@ class QuestionDictionaryTest(unittest.TestCase): def test_dictionary_comparison(self): - # Run dictionary generation - return_code = generate_task_questions_dict() + # Run main task question dictionary generation + tq_dicts_rc = tq_dicts() + assert tq_dicts_rc == 0 + + # Run generation for defaults + tq_dicts_defaults_rc = tq_dicts_defaults() + assert tq_dicts_defaults_rc == 0 - # Assert that dictionaries are equal - assert return_code == 0 # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/bin/generate_task_questions_dict.py b/src/swell/utilities/bin/task_question_dicts.py similarity index 98% rename from src/swell/utilities/bin/generate_task_questions_dict.py rename to src/swell/utilities/bin/task_question_dicts.py index add049c1..dd7ca234 100644 --- a/src/swell/utilities/bin/generate_task_questions_dict.py +++ b/src/swell/utilities/bin/task_question_dicts.py @@ -25,7 +25,7 @@ # -------------------------------------------------------------------------------------------------- -def generate_tq_dict_defaults(): +def tq_dicts(): # Create a logger logger = Logger('ListOfTaskQuestions') @@ -165,7 +165,7 @@ def generate_tq_dict_defaults(): if __name__ == '__main__': - generate_tq_dict_defaults() + tq_dicts() # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/bin/generate_task_question_default_dicts.py b/src/swell/utilities/bin/task_question_dicts_defaults.py similarity index 98% rename from src/swell/utilities/bin/generate_task_question_default_dicts.py rename to src/swell/utilities/bin/task_question_dicts_defaults.py index 296eb70c..a40437f8 100644 --- a/src/swell/utilities/bin/generate_task_question_default_dicts.py +++ b/src/swell/utilities/bin/task_question_dicts_defaults.py @@ -128,7 +128,7 @@ def create_platform_tq_dicts(logger, platform_name, tq_dicts, platform_tq_dicts_ # -------------------------------------------------------------------------------------------------- -def main(): +def tq_dicts_defaults(): # Create a logger logger = Logger('ListOfTaskQuestions') @@ -228,6 +228,9 @@ def main(): failure = failure + 1 + # Check wither 'defer_to' remains anywhere in the file. + + # Crete a space between loop elements logger.info(' ', False) @@ -242,7 +245,7 @@ def main(): if __name__ == '__main__': - main() + tq_dicts_defaults() # -------------------------------------------------------------------------------------------------- From 23a66cf50d800782ef008951fbf89de542e2453d Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 7 Jun 2023 12:46:27 -0400 Subject: [PATCH 065/121] Use spack-stack 1.4 modules to maintain compatibility with JEDI --- src/swell/deployment/platforms/nccs_discover/modules | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/swell/deployment/platforms/nccs_discover/modules b/src/swell/deployment/platforms/nccs_discover/modules index fc6f7cd4..582af6e9 100644 --- a/src/swell/deployment/platforms/nccs_discover/modules +++ b/src/swell/deployment/platforms/nccs_discover/modules @@ -18,12 +18,12 @@ module use /discover/swdev/jcsda/spack-stack/modulefiles module load miniconda/3.9.7 module load ecflow/5.8.4 module load mysql/8.0.31 -module use /gpfsm/dswdev/jcsda/spack-stack/spack-stack-1.3.0/envs/unified-env/install/modulefiles/Core +module use /gpfsm/dswdev/jcsda/spack-stack/spack-stack-1.4.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2022.0.1 module load stack-intel-oneapi-mpi/2021.5.0 module load stack-python/3.9.7 module load jedi-fv3-env/unified-dev -module load jedi-ewok-env/unified-dev +module load ewok-env/unified-dev module load soca-env/unified-dev module unload gsibec @@ -43,7 +43,7 @@ module use -a /discover/nobackup/drholdaw/JediOpt/modulefiles/core module load solo/swell-1.0.0 module load r2d2/swell-1.0.5 module load eva/1.3.5 -module load jedi_bundle/1.0.10 +module load jedi_bundle/1.0.11 # Set the swell paths # ------------------- From e14ee13f40fc0b5cd4c7dec733ccf03510d85040 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 7 Jun 2023 15:30:31 -0400 Subject: [PATCH 066/121] check for defer_to_ in the files --- setup.py | 4 +- .../bin/task_question_dicts_defaults.py | 53 +++++++++++-------- 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/setup.py b/setup.py index b0285c24..0a4be5ee 100644 --- a/setup.py +++ b/setup.py @@ -65,11 +65,11 @@ 'swell_launch_experiment = swell.deployment.bin.swell_launch_experiment:main', 'swell_sat_db_processing = swell.deployment.bin.swell_sat_db_processing:main', # Utilities - 'swell_util_task_question_dicts = swell.utilities.bin.task_question_dicts:tq_dicts', 'swell_util_check_jedi_interface_templates = \ swell.utilities.bin.check_jedi_interface_templates:main', + 'swell_util_task_question_dicts = swell.utilities.bin.task_question_dicts:tq_dicts', 'swell_util_task_question_dicts_defaults = \ - swell.utilities.bin.swell_util_task_question_dicts_defaults:tq_dicts_defaults', + swell.utilities.bin.task_question_dicts_defaults:tq_dicts_defaults', 'swell_test_suite = swell.test.test_suite:main', ], }, diff --git a/src/swell/utilities/bin/task_question_dicts_defaults.py b/src/swell/utilities/bin/task_question_dicts_defaults.py index a40437f8..c3113847 100644 --- a/src/swell/utilities/bin/task_question_dicts_defaults.py +++ b/src/swell/utilities/bin/task_question_dicts_defaults.py @@ -204,7 +204,7 @@ def tq_dicts_defaults(): logger.info(f'The code will make no change to the task questions dictionary.') - else: + if tq_dicts_str != dicts_str_in or 'defer_to_' in tq_dicts_str: # Write the new dictionary to a temporary file random_string = ''.join(random.choices(string.ascii_letters + string.digits, k=8)) @@ -214,31 +214,42 @@ def tq_dicts_defaults(): with open(destination_yaml_temp, 'w') as file: file.write(tq_dicts_str) - logger.info(f'If a new \'task_questions.yaml\' is generated for \'{tq_dicts_name}\' ' + - f'using this utility the resulting file will be different. This could ' + - f'be for a number of reasons:') - logger.info(f' ', False) - logger.info(f' - Comments were added to the original file.' ) - logger.info(f' - A new key is accessed from a task.' ) - logger.info(f' - Referencing of a particular key has been removed from a task.') - logger.info(f' ', False) - logger.info(f'Please compare the new (temporary) file \'{destination_yaml_temp}\' ' + - f'with the existing file: \'{tq_dicts_name}/task_questions.yaml\' and ' + - f'resolve the differences') - + # If there is a dictionary mismatch inform the user + if tq_dicts_str != dicts_str_in: + + logger.info(f'If a new \'task_questions.yaml\' is generated for ' + + f'\'{tq_dicts_name}\' using this utility the resulting file will be ' + + f'different. This could be for a number of reasons:') + logger.info(f' ', False) + logger.info(f' - Comments were added to the original file.' ) + logger.info(f' - A new key is accessed from a task.' ) + logger.info(f' - Referencing of a particular key has been removed from a task.') + logger.info(f' ', False) + logger.info(f'Please compare the new (temporary) file \'{destination_yaml_temp}\' ' + + f'with the existing file: \'{tq_dicts_name}/task_questions.yaml\' and ' + + f'resolve the differences') + + # If there is a defer to in the file still then inform user + if 'defer_to_' in tq_dicts_str: + + logger.info(f'The dictionary that was created for \'{tq_dicts_name}\' contains ' + + f'\'defer_to_\'. Ensure these instances are resolved when comparing ' + + f'the new (temporary) file \'{destination_yaml_temp}\' with the ' + + f'existing file: \'{tq_dicts_name}/task_questions.yaml\' and resolve ' + + f'the differences. Once resolved rerun this utility to ensure tests ' + + f'pass.') + + # Uptick failure count failure = failure + 1 - # Check wither 'defer_to' remains anywhere in the file. - - # Crete a space between loop elements logger.info(' ', False) - # Return code - if failure > 0: - return 1 - else: - return 0 + # Return code + if failure > 0: + return 1 + else: + return 0 # -------------------------------------------------------------------------------------------------- From a483a092fb820585f8fed72c298931f1d82c942d Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 7 Jun 2023 16:08:22 -0400 Subject: [PATCH 067/121] reduce prints in tests --- src/swell/test/test_suite.py | 10 ++++++++++ src/swell/utilities/logger.py | 7 +++++++ 2 files changed, 17 insertions(+) diff --git a/src/swell/test/test_suite.py b/src/swell/test/test_suite.py index 0a18440e..2e2ed8b1 100644 --- a/src/swell/test/test_suite.py +++ b/src/swell/test/test_suite.py @@ -8,15 +8,24 @@ # -------------------------------------------------------------------------------------------------- +import os import unittest from swell.test.question_dictionary_comparison_test import QuestionDictionaryTest +from swell.utilities.logger import Logger # -------------------------------------------------------------------------------------------------- def main(): + # Create a logger + logger = Logger('TestSuite') + logger.test('Running Swell Test Suite') + + # Turn off the regular info testing + os.environ["LOG_INFO"] = "0" + # Create a test suite test_suite = unittest.TestSuite() @@ -29,4 +38,5 @@ def main(): # Run the tests test_runner.run(test_suite) + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/logger.py b/src/swell/utilities/logger.py index 8b958340..ab43ad26 100644 --- a/src/swell/utilities/logger.py +++ b/src/swell/utilities/logger.py @@ -32,6 +32,7 @@ def __init__(self, task_name): # Set default logging levels self.loggerdict = {'BLANK': True, 'INFO': True, + 'TEST': True, 'TRACE': False, 'DEBUG': False, } @@ -84,6 +85,12 @@ def info(self, message, wrap=True): # ---------------------------------------------------------------------------------------------- + def test(self, message, wrap=True): + + self.send_message('TEST', message, wrap) + + # ---------------------------------------------------------------------------------------------- + def trace(self, message, wrap=True): self.send_message('TRACE', message, wrap) From d5151534131b41ffff545e5ea3e844eff4ff86eb Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 7 Jun 2023 16:11:30 -0400 Subject: [PATCH 068/121] py coding norms --- src/swell/utilities/bin/task_question_dicts.py | 4 ++-- .../utilities/bin/task_question_dicts_defaults.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/swell/utilities/bin/task_question_dicts.py b/src/swell/utilities/bin/task_question_dicts.py index dd7ca234..e4264187 100644 --- a/src/swell/utilities/bin/task_question_dicts.py +++ b/src/swell/utilities/bin/task_question_dicts.py @@ -151,8 +151,8 @@ def tq_dicts(): logger.info(f'If a new \'task_questions.yaml\' is generated using this utility the ' + f'resulting file will be different. This could be for a number of reasons:') logger.info(f' ', False) - logger.info(f' - Comments were added to the original file.' ) - logger.info(f' - A new key is accessed from a task.' ) + logger.info(f' - Comments were added to the original file.') + logger.info(f' - A new key is accessed from a task.') logger.info(f' - Referencing of a particular key has been removed from a task.') logger.info(f' ', False) logger.info(f'Please compare the new (temporary) file \'{destination_yaml_temp}\' with ' + diff --git a/src/swell/utilities/bin/task_question_dicts_defaults.py b/src/swell/utilities/bin/task_question_dicts_defaults.py index c3113847..23afbb60 100644 --- a/src/swell/utilities/bin/task_question_dicts_defaults.py +++ b/src/swell/utilities/bin/task_question_dicts_defaults.py @@ -118,7 +118,7 @@ def create_platform_tq_dicts(logger, platform_name, tq_dicts, platform_tq_dicts_ # Add dictionary to the output dictionary platform_tq_dicts_str = platform_tq_dicts_str + yaml.dump(platform_tq_dict, - default_flow_style=False) + default_flow_style=False) platform_tq_dicts_str = platform_tq_dicts_str + '\n' # Return dictionary in string format @@ -221,13 +221,13 @@ def tq_dicts_defaults(): f'\'{tq_dicts_name}\' using this utility the resulting file will be ' + f'different. This could be for a number of reasons:') logger.info(f' ', False) - logger.info(f' - Comments were added to the original file.' ) - logger.info(f' - A new key is accessed from a task.' ) + logger.info(f' - Comments were added to the original file.') + logger.info(f' - A new key is accessed from a task.') logger.info(f' - Referencing of a particular key has been removed from a task.') logger.info(f' ', False) - logger.info(f'Please compare the new (temporary) file \'{destination_yaml_temp}\' ' + - f'with the existing file: \'{tq_dicts_name}/task_questions.yaml\' and ' + - f'resolve the differences') + logger.info(f'Please compare the new (temporary) file \'{destination_yaml_temp}\'' + + f' with the existing file: \'{tq_dicts_name}/task_questions.yaml\' ' + + f'and resolve the differences') # If there is a defer to in the file still then inform user if 'defer_to_' in tq_dicts_str: From 8453c27e5b10e71eadb5bae51a34909afa4fc958 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Fri, 9 Jun 2023 10:27:19 -0400 Subject: [PATCH 069/121] unload crtm and update jedi_bundle --- src/swell/deployment/platforms/nccs_discover/modules | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/swell/deployment/platforms/nccs_discover/modules b/src/swell/deployment/platforms/nccs_discover/modules index 582af6e9..321b239d 100644 --- a/src/swell/deployment/platforms/nccs_discover/modules +++ b/src/swell/deployment/platforms/nccs_discover/modules @@ -26,6 +26,7 @@ module load jedi-fv3-env/unified-dev module load ewok-env/unified-dev module load soca-env/unified-dev module unload gsibec +module load crtm/2.4.0 # Add IODA to PYTHONPATH in case it is needed # ------------------------------------------- @@ -43,7 +44,7 @@ module use -a /discover/nobackup/drholdaw/JediOpt/modulefiles/core module load solo/swell-1.0.0 module load r2d2/swell-1.0.5 module load eva/1.3.5 -module load jedi_bundle/1.0.11 +module load jedi_bundle/1.0.12-swell # Set the swell paths # ------------------- From 69158fb5ee366671fd312aec4d70559fc800c3dd Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 14 Jun 2023 15:43:45 -0400 Subject: [PATCH 070/121] unused variables --- src/swell/test/question_dictionary_comparison_test.py | 5 +---- src/swell/utilities/bin/task_question_dicts.py | 1 - src/swell/utilities/bin/task_question_dicts_defaults.py | 5 ----- 3 files changed, 1 insertion(+), 10 deletions(-) diff --git a/src/swell/test/question_dictionary_comparison_test.py b/src/swell/test/question_dictionary_comparison_test.py index 76cf2873..066c99a6 100644 --- a/src/swell/test/question_dictionary_comparison_test.py +++ b/src/swell/test/question_dictionary_comparison_test.py @@ -8,17 +8,15 @@ # -------------------------------------------------------------------------------------------------- -import os import unittest -import yaml -from swell.swell_path import get_swell_path from swell.utilities.bin.task_question_dicts import tq_dicts from swell.utilities.bin.task_question_dicts_defaults import tq_dicts_defaults # -------------------------------------------------------------------------------------------------- + class QuestionDictionaryTest(unittest.TestCase): def test_dictionary_comparison(self): @@ -32,5 +30,4 @@ def test_dictionary_comparison(self): assert tq_dicts_defaults_rc == 0 - # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/bin/task_question_dicts.py b/src/swell/utilities/bin/task_question_dicts.py index 47388ea4..b2e94c8c 100644 --- a/src/swell/utilities/bin/task_question_dicts.py +++ b/src/swell/utilities/bin/task_question_dicts.py @@ -50,7 +50,6 @@ def tq_dicts(): question_dict = {} question_dict_str = '' - question_dict_in = question_dict.copy() question_dict_str_in = question_dict_str # Loop through task code and accumulate all lines containing a use of config diff --git a/src/swell/utilities/bin/task_question_dicts_defaults.py b/src/swell/utilities/bin/task_question_dicts_defaults.py index 23afbb60..8c065263 100644 --- a/src/swell/utilities/bin/task_question_dicts_defaults.py +++ b/src/swell/utilities/bin/task_question_dicts_defaults.py @@ -10,7 +10,6 @@ # standard imports -import glob import os import random import string @@ -19,7 +18,6 @@ # swell imports from swell.swell_path import get_swell_path from swell.utilities.logger import Logger -from swell.utilities.case_switching import snake_case_to_camel_case # -------------------------------------------------------------------------------------------------- @@ -133,9 +131,6 @@ def tq_dicts_defaults(): # Create a logger logger = Logger('ListOfTaskQuestions') - # Path to JEDI interface code - swell_path = get_swell_path() - # Output file task_questions_config = os.path.join(get_swell_path(), 'tasks', 'task_questions.yaml') From ddfc102157f9e6987cc067c733d00b07777904e6 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 14 Jun 2023 15:45:55 -0400 Subject: [PATCH 071/121] rename suite --- src/swell/suites/{test_obs_filters => ufo_testing}/flow.cylc | 0 .../geos_atmosphere/eva_observations.yaml | 0 .../geos_atmosphere/suites-ufo_testing-geos_atmosphere.yaml} | 0 .../suites-ufo_testing.yaml} | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename src/swell/suites/{test_obs_filters => ufo_testing}/flow.cylc (100%) rename src/swell/suites/{test_obs_filters => ufo_testing}/geos_atmosphere/eva_observations.yaml (100%) rename src/swell/suites/{test_obs_filters/geos_atmosphere/suites-test_obs_filters-geos_atmosphere.yaml => ufo_testing/geos_atmosphere/suites-ufo_testing-geos_atmosphere.yaml} (100%) rename src/swell/suites/{test_obs_filters/suites-test_obs_filters.yaml => ufo_testing/suites-ufo_testing.yaml} (100%) diff --git a/src/swell/suites/test_obs_filters/flow.cylc b/src/swell/suites/ufo_testing/flow.cylc similarity index 100% rename from src/swell/suites/test_obs_filters/flow.cylc rename to src/swell/suites/ufo_testing/flow.cylc diff --git a/src/swell/suites/test_obs_filters/geos_atmosphere/eva_observations.yaml b/src/swell/suites/ufo_testing/geos_atmosphere/eva_observations.yaml similarity index 100% rename from src/swell/suites/test_obs_filters/geos_atmosphere/eva_observations.yaml rename to src/swell/suites/ufo_testing/geos_atmosphere/eva_observations.yaml diff --git a/src/swell/suites/test_obs_filters/geos_atmosphere/suites-test_obs_filters-geos_atmosphere.yaml b/src/swell/suites/ufo_testing/geos_atmosphere/suites-ufo_testing-geos_atmosphere.yaml similarity index 100% rename from src/swell/suites/test_obs_filters/geos_atmosphere/suites-test_obs_filters-geos_atmosphere.yaml rename to src/swell/suites/ufo_testing/geos_atmosphere/suites-ufo_testing-geos_atmosphere.yaml diff --git a/src/swell/suites/test_obs_filters/suites-test_obs_filters.yaml b/src/swell/suites/ufo_testing/suites-ufo_testing.yaml similarity index 100% rename from src/swell/suites/test_obs_filters/suites-test_obs_filters.yaml rename to src/swell/suites/ufo_testing/suites-ufo_testing.yaml From 09abaf7848340f04c01e607ca7f4f9eef75f6191 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Thu, 15 Jun 2023 12:06:22 -0400 Subject: [PATCH 072/121] Bug fix and remove print --- src/swell/utilities/build.py | 2 +- src/swell/utilities/config.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/swell/utilities/build.py b/src/swell/utilities/build.py index 64780271..532a6d05 100644 --- a/src/swell/utilities/build.py +++ b/src/swell/utilities/build.py @@ -10,7 +10,7 @@ import os import shutil -from jedi_bundle.bin.jedi_bundle import get_default_config +from jedi_bundle.bin.jedi_bundle import get_default_config, get_bundles # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/config.py b/src/swell/utilities/config.py index 62e76dcc..1df11841 100644 --- a/src/swell/utilities/config.py +++ b/src/swell/utilities/config.py @@ -109,7 +109,6 @@ def __init__(self, input_file, logger, task_name, model): setattr(self, f'__{experiment_key}__', experiment_value) # Add a method to get the variable - print('adding method for ', experiment_key) setattr(self, f'{experiment_key}', self.get(experiment_key)) # ---------------------------------------------------------------------------------------------- From d40fbb2d5ca0425a61cd379abd251652aa50f165 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Fri, 16 Jun 2023 23:43:57 -0400 Subject: [PATCH 073/121] add operator test capability --- .../observations/aircraft.yaml | 6 ++ .../observations/ufo_tests.yaml | 9 ++ .../platforms/nccs_discover/modules | 2 +- .../ufo_testing/suites-ufo_testing.yaml | 4 +- ...le.py => run_jedi_ufo_tests_executable.py} | 93 +++++++++++++------ .../data_assimilation_window_params.py | 8 +- .../utilities/render_jedi_interface_files.py | 1 + 7 files changed, 93 insertions(+), 30 deletions(-) create mode 100644 src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ufo_tests.yaml rename src/swell/tasks/{run_jedi_test_obs_filters_executable.py => run_jedi_ufo_tests_executable.py} (56%) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/aircraft.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/aircraft.yaml index 3bfa9b61..1f93cf2f 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/aircraft.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/aircraft.yaml @@ -204,6 +204,8 @@ obs filters: options: variable: airTemperature inflation factor: 8.0 + surface_obs: false + adjusted_error_name: GsiAdjustObsError defer to post: true # assign TempObsErrorData/airTemperature <--- ObsErrorData before changing its Min and Max @@ -497,6 +499,8 @@ obs filters: options: variable: windEastward inflation factor: 4.0 + surface_obs: false + adjusted_error_name: GsiAdjustObsError defer to post: true # Error inflation based on pressure check @@ -510,6 +514,8 @@ obs filters: options: variable: windNorthward inflation factor: 4.0 + surface_obs: false + adjusted_error_name: GsiAdjustObsError defer to post: true # Gross Check diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ufo_tests.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ufo_tests.yaml new file mode 100644 index 00000000..3478661d --- /dev/null +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ufo_tests.yaml @@ -0,0 +1,9 @@ +default: + linear obs operator test: + coef TL: 0.1 + tolerance TL: 1.0e-11 + tolerance AD: 1.0e-13 + tolerance: 1.0e-06 + +aircraft: + rms ref: 6.736593278810636 diff --git a/src/swell/deployment/platforms/nccs_discover/modules b/src/swell/deployment/platforms/nccs_discover/modules index ee9db367..57449e7b 100644 --- a/src/swell/deployment/platforms/nccs_discover/modules +++ b/src/swell/deployment/platforms/nccs_discover/modules @@ -36,7 +36,7 @@ PYTHONPATH={{experiment_root}}/{{experiment_id}}/jedi_bundle/build/lib/python3.9 # IODA-Converters from experiement JEDI source # -------------------------------------------- -PYTHONPATH=/discover/nobackup/drholdaw/JediSwell/iodaconv-v3/src:$PYTHONPATH +PYTHONPATH={{experiment_root}}/{{experiment_id}}/jedi_bundle/source/iodaconv/src:$PYTHONPATH # Load GMAO modules # ----------------- diff --git a/src/swell/suites/ufo_testing/suites-ufo_testing.yaml b/src/swell/suites/ufo_testing/suites-ufo_testing.yaml index 0e7bd3b6..030ee5a8 100644 --- a/src/swell/suites/ufo_testing/suites-ufo_testing.yaml +++ b/src/swell/suites/ufo_testing/suites-ufo_testing.yaml @@ -25,7 +25,7 @@ jedi_build_method: # Existing JEDI bundle directory existing_jedi_source_directory: - default_value: '/discover/nobackup/drholdaw/JediDev/swell-ufo' + default_value: '/discover/nobackup/drholdaw/JediDev/ufo-work' prompt: Provide the path to an existing JEDI bundle directory containing source code repos type: string depends: @@ -34,7 +34,7 @@ existing_jedi_source_directory: # Existing JEDI build existing_jedi_build_directory: - default_value: '/discover/nobackup/drholdaw/JediDev/swell-ufo/build-intel-release' + default_value: '/discover/nobackup/drholdaw/JediDev/ufo-work/build-intel-release' prompt: Provide the path to an existing JEDI build directory type: string depends: diff --git a/src/swell/tasks/run_jedi_test_obs_filters_executable.py b/src/swell/tasks/run_jedi_ufo_tests_executable.py similarity index 56% rename from src/swell/tasks/run_jedi_test_obs_filters_executable.py rename to src/swell/tasks/run_jedi_ufo_tests_executable.py index 5f2ba270..830d5d77 100644 --- a/src/swell/tasks/run_jedi_test_obs_filters_executable.py +++ b/src/swell/tasks/run_jedi_ufo_tests_executable.py @@ -8,6 +8,7 @@ # -------------------------------------------------------------------------------------------------- +import copy import os import yaml @@ -18,7 +19,7 @@ # -------------------------------------------------------------------------------------------------- -class RunJediHofxExecutable(taskBase): +class RunJediTestObsFiltersExecutable(taskBase): # ---------------------------------------------------------------------------------------------- @@ -26,7 +27,7 @@ def execute(self): # Jedi application name # --------------------- - jedi_application = 'test_obs_filters' + jedi_application = 'ufo_tests' # Parse configuration # ------------------- @@ -51,14 +52,6 @@ def execute(self): self.jedi_rendering.add_key('crtm_coeff_dir', self.config.crtm_coeff_dir(None)) self.jedi_rendering.add_key('window_begin', window_begin) - # Jedi configuration file - # ----------------------- - jedi_config_file = os.path.join(self.cycle_dir(), f'jedi_{jedi_application}_config.yaml') - - # Output log file - # --------------- - output_log_file = os.path.join(self.cycle_dir(), f'jedi_{jedi_application}_log.log') - # Open the JEDI config file and fill initial templates # ---------------------------------------------------- jedi_config_dict = self.jedi_rendering.render_oops_file(f'{jedi_application}') @@ -71,6 +64,11 @@ def execute(self): # ------------------------------------- conventional_types = ['aircraft'] + # Open the ufo_tests config file + # ------------------------------ + ufo_tests_dict = self.jedi_rendering.render_obs_file(f'ufo_tests') + ufo_tests_default = ufo_tests_dict['default'] + # Loop over the observations for index in range(len(observations)): @@ -88,7 +86,7 @@ def execute(self): # For conventional add the GeoVaLs flip if observations[index] in conventional_types: - geo_va_ls_dict['levels_are_top_down'] = True + geo_va_ls_dict['levels_are_top_down'] = False jedi_config_dict['observations'][index]['geovals'] = geo_va_ls_dict @@ -97,20 +95,63 @@ def execute(self): dummy_search = [{'name': 'Dummy/Group/Var'}] jedi_config_dict['observations'][index]['expectVariablesNotToExist'] = dummy_search - # Write the expanded dictionary to YAML file - # ------------------------------------------ - with open(jedi_config_file, 'w') as jedi_config_file_open: - yaml.dump(jedi_config_dict, jedi_config_file_open, default_flow_style=False) - - # Jedi executable name - # -------------------- - jedi_executable_path = os.path.join(self.experiment_path(), 'jedi_bundle', 'build', 'bin', - 'test_ObsFilters.x') - - # Run the JEDI executable - # ----------------------- - run_executable(self.logger, self.cycle_dir(), 1, jedi_executable_path, jedi_config_file, - output_log_file) - self.logger.info('Running '+jedi_executable_path+' with '+str(np)+' processors.') + # Make a copy of the jedi config dictionary + jedi_config_dict_operator_test = copy.deepcopy(jedi_config_dict) + jedi_config_dict_filter_test = copy.deepcopy(jedi_config_dict) + + # Add the ufo_tests dictionary to the jedi config dictionary + for index in range(len(observations)): + + # Get the test values for this observation + ufo_tests_obs = ufo_tests[observations[index]] + + # Overwrite the defaults with the values in ufo_tests_obs + ufo_tests_obs_dict = {**ufo_tests_default, **ufo_tests_obs} + + # Add the ufo_tests_obs_dict to the observation dictionary + jedi_config_dict_operator_test['observations'][index].update(ufo_tests_obs_dict) + + # If the observation has filters remove them + if 'filters' in jedi_config_dict_operator_test['observations'][index]: + del jedi_config_dict_operator_test['observations'][index]['filters'] + + # Tests to run + # ------------ + tests = ['test_ObsFilters', 'test_ObsOperator', 'test_ObsOperatorTLAD'] + + # Loop over the tests + # ------------------- + for test in tests: + + # Executable dictionary + if test == 'test_ObsFilters': + jedi_config_dict = jedi_config_dict_operator_test + else: + jedi_config_dict = jedi_config_dict_filter_test + + # Jedi configuration file + # ----------------------- + jedi_config_file = os.path.join(self.cycle_dir(), f'jedi_{test}_config.yaml') + + # Write the expanded dictionary to YAML file + # ------------------------------------------ + with open(jedi_config_file, 'w') as jedi_config_file_open: + yaml.dump(jedi_config_dict, jedi_config_file_open, default_flow_style=False) + + # Output log file + # --------------- + output_log_file = os.path.join(self.cycle_dir(), f'jedi_{test}_log.log') + + # Jedi executable name + # -------------------- + jedi_executable_path = os.path.join(self.experiment_path(), 'jedi_bundle', 'build', + 'bin', f'{test}.x') + + # Run the Test Obs Filters executable + # ----------------------------------- + self.logger.info('Running '+jedi_executable_path+' with 1 processor.') + run_executable(self.logger, self.cycle_dir(), 1, jedi_executable_path, jedi_config_file, + output_log_file) + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/data_assimilation_window_params.py b/src/swell/utilities/data_assimilation_window_params.py index a343a349..6a037862 100644 --- a/src/swell/utilities/data_assimilation_window_params.py +++ b/src/swell/utilities/data_assimilation_window_params.py @@ -74,7 +74,13 @@ def window_end_iso(self, window_offset, window_length): # Get window beginning time window_begin_dto = self.__get_window_begin_dto__(window_offset) - return window_begin_dto + window_length_dur + print(f'window_begin_dto: {window_begin_dto}') + print(f'window_length_dur: {window_length_dur}') + print(f'window_end_iso: {window_begin_dto + window_length_dur}') + + window_end_dto = window_begin_dto + window_length_dur + + return window_end_dto.strftime(datetime_formats['iso_format']) # ---------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/render_jedi_interface_files.py b/src/swell/utilities/render_jedi_interface_files.py index e0319cca..c465ccb6 100644 --- a/src/swell/utilities/render_jedi_interface_files.py +++ b/src/swell/utilities/render_jedi_interface_files.py @@ -59,6 +59,7 @@ def __init__(self, logger, experiment_root, experiment_id, cycle_dir, jedi_inter 'vertical_resolution', 'window_begin', 'window_begin_iso', + 'window_end_iso', 'window_length', ] From 205689e859cd143458ce308d1db2163239e5caee Mon Sep 17 00:00:00 2001 From: danholdaway Date: Sun, 18 Jun 2023 11:26:19 -0400 Subject: [PATCH 074/121] rename some things --- .../observations/ufo_tests.yaml | 1 + .../{test_obs_filters.yaml => ufo_tests.yaml} | 0 src/swell/tasks/base/task_registry.py | 2 +- .../tasks/run_jedi_ufo_tests_executable.py | 65 +++++++++++-------- src/swell/tasks/task_questions.yaml | 10 +-- 5 files changed, 45 insertions(+), 33 deletions(-) rename src/swell/configuration/jedi/oops/{test_obs_filters.yaml => ufo_tests.yaml} (100%) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ufo_tests.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ufo_tests.yaml index 3478661d..1cd0a521 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ufo_tests.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ufo_tests.yaml @@ -7,3 +7,4 @@ default: aircraft: rms ref: 6.736593278810636 + passedBenchmark: 1 diff --git a/src/swell/configuration/jedi/oops/test_obs_filters.yaml b/src/swell/configuration/jedi/oops/ufo_tests.yaml similarity index 100% rename from src/swell/configuration/jedi/oops/test_obs_filters.yaml rename to src/swell/configuration/jedi/oops/ufo_tests.yaml diff --git a/src/swell/tasks/base/task_registry.py b/src/swell/tasks/base/task_registry.py index 06cec6cb..360943c5 100644 --- a/src/swell/tasks/base/task_registry.py +++ b/src/swell/tasks/base/task_registry.py @@ -29,7 +29,7 @@ 'PrepGeosRunDir', 'RunGeosExecutable', 'RunJediHofxExecutable', - 'RunJediTestObsFiltersExecutable', + 'RunJediUfoTestsExecutable', 'RunJediVariationalExecutable', 'SaveObsDiags', 'SaveRestart', diff --git a/src/swell/tasks/run_jedi_ufo_tests_executable.py b/src/swell/tasks/run_jedi_ufo_tests_executable.py index 830d5d77..52376136 100644 --- a/src/swell/tasks/run_jedi_ufo_tests_executable.py +++ b/src/swell/tasks/run_jedi_ufo_tests_executable.py @@ -19,7 +19,7 @@ # -------------------------------------------------------------------------------------------------- -class RunJediTestObsFiltersExecutable(taskBase): +class RunJediUfoTestsExecutable(taskBase): # ---------------------------------------------------------------------------------------------- @@ -54,11 +54,11 @@ def execute(self): # Open the JEDI config file and fill initial templates # ---------------------------------------------------- - jedi_config_dict = self.jedi_rendering.render_oops_file(f'{jedi_application}') + jedi_config_filter_dict = self.jedi_rendering.render_oops_file(f'{jedi_application}') # Perform complete template rendering # ----------------------------------- - jedi_dictionary_iterator(jedi_config_dict, self.jedi_rendering, '3D', observations) + jedi_dictionary_iterator(jedi_config_filter_dict, self.jedi_rendering, '3D', observations) # Make modifications needed for testing # ------------------------------------- @@ -66,15 +66,18 @@ def execute(self): # Open the ufo_tests config file # ------------------------------ - ufo_tests_dict = self.jedi_rendering.render_obs_file(f'ufo_tests') + ufo_tests_dict = self.jedi_rendering.render_interface_observations(f'ufo_tests') ufo_tests_default = ufo_tests_dict['default'] + # Insert the GeoVaLs section + # -------------------------- + # Loop over the observations for index in range(len(observations)): # Remove GetValues if present - if 'get values' in jedi_config_dict['observations'][index]: - del jedi_config_dict['observations'][index]['get values'] + if 'get values' in jedi_config_filter_dict['observations'][index]: + del jedi_config_filter_dict['observations'][index]['get values'] # GeoVaLs filename geo_va_ls_fname = os.path.join(self.cycle_dir(), @@ -88,10 +91,14 @@ def execute(self): if observations[index] in conventional_types: geo_va_ls_dict['levels_are_top_down'] = False - jedi_config_dict['observations'][index]['geovals'] = geo_va_ls_dict + jedi_config_filter_dict['observations'][index]['geovals'] = geo_va_ls_dict + + # Make a copy for doing the operator test + # --------------------------------------- + jedi_config_operator_dict = copy.deepcopy(jedi_config_filter_dict) + - # Need to insert at least one benchmark, but we do not really want to check anything - # so check that some made up variable does not exist + # For filter test there needs to be at least one benchmark dummy_search = [{'name': 'Dummy/Group/Var'}] jedi_config_dict['observations'][index]['expectVariablesNotToExist'] = dummy_search @@ -103,7 +110,7 @@ def execute(self): for index in range(len(observations)): # Get the test values for this observation - ufo_tests_obs = ufo_tests[observations[index]] + ufo_tests_obs = ufo_tests_dict[observations[index]] # Overwrite the defaults with the values in ufo_tests_obs ufo_tests_obs_dict = {**ufo_tests_default, **ufo_tests_obs} @@ -115,31 +122,36 @@ def execute(self): if 'filters' in jedi_config_dict_operator_test['observations'][index]: del jedi_config_dict_operator_test['observations'][index]['filters'] + print(yaml.dump(jedi_config_dict_operator_test)) + + # Jedi configuration file + # ----------------------- + file = os.path.join(self.cycle_dir(), 'jedi_test_ObsOperator_config.yaml') + with open(file, 'w') as jedi_config_file_open: + yaml.dump(jedi_config_dict_operator_test, jedi_config_file_open, + default_flow_style=False) + + file = os.path.join(self.cycle_dir(), 'jedi_test_ObsOperatorTLAD_config.yaml') + with open(file, 'w') as jedi_config_file_open: + yaml.dump(jedi_config_dict_operator_test, jedi_config_file_open, + default_flow_style=False) + + file = os.path.join(self.cycle_dir(), 'jedi_test_ObsFilters_config.yaml') + with open(file, 'w') as jedi_config_file_open: + yaml.dump(jedi_config_dict_filter_test, jedi_config_file_open, + default_flow_style=False) + # Tests to run # ------------ - tests = ['test_ObsFilters', 'test_ObsOperator', 'test_ObsOperatorTLAD'] + tests = ['test_ObsOperator', 'test_ObsOperatorTLAD', 'test_ObsFilters'] # Loop over the tests # ------------------- for test in tests: - # Executable dictionary - if test == 'test_ObsFilters': - jedi_config_dict = jedi_config_dict_operator_test - else: - jedi_config_dict = jedi_config_dict_filter_test - - # Jedi configuration file - # ----------------------- - jedi_config_file = os.path.join(self.cycle_dir(), f'jedi_{test}_config.yaml') - - # Write the expanded dictionary to YAML file - # ------------------------------------------ - with open(jedi_config_file, 'w') as jedi_config_file_open: - yaml.dump(jedi_config_dict, jedi_config_file_open, default_flow_style=False) - # Output log file # --------------- + jedi_config_file = os.path.join(self.cycle_dir(), f'jedi_{test}_config.yaml') output_log_file = os.path.join(self.cycle_dir(), f'jedi_{test}_log.log') # Jedi executable name @@ -149,7 +161,6 @@ def execute(self): # Run the Test Obs Filters executable # ----------------------------------- - self.logger.info('Running '+jedi_executable_path+' with 1 processor.') run_executable(self.logger, self.cycle_dir(), 1, jedi_executable_path, jedi_config_file, output_log_file) diff --git a/src/swell/tasks/task_questions.yaml b/src/swell/tasks/task_questions.yaml index 7ed9cfe5..9f0d186f 100644 --- a/src/swell/tasks/task_questions.yaml +++ b/src/swell/tasks/task_questions.yaml @@ -70,7 +70,7 @@ background_time_offset: - GetObservations - GsiBcToIoda - RunJediHofxExecutable - - RunJediTestObsFiltersExecutable + - RunJediUfoTestsExecutable - RunJediVariationalExecutable - SaveObsDiags type: duration @@ -117,7 +117,7 @@ crtm_coeff_dir: - GetObservations - GsiBcToIoda - RunJediHofxExecutable - - RunJediTestObsFiltersExecutable + - RunJediUfoTestsExecutable - RunJediVariationalExecutable - SaveObsDiags type: string @@ -394,7 +394,7 @@ observations: - GsiBcToIoda - GsiNcdiagToIoda - RunJediHofxExecutable - - RunJediTestObsFiltersExecutable + - RunJediUfoTestsExecutable - RunJediVariationalExecutable - SaveObsDiags type: string-check-list @@ -495,7 +495,7 @@ window_length: - GetGeovals - GetObservations - RunJediHofxExecutable - - RunJediTestObsFiltersExecutable + - RunJediUfoTestsExecutable - RunJediVariationalExecutable - StoreBackground type: duration @@ -514,7 +514,7 @@ window_offset: - GsiBcToIoda - GsiNcdiagToIoda - RunJediHofxExecutable - - RunJediTestObsFiltersExecutable + - RunJediUfoTestsExecutable - RunJediVariationalExecutable - SaveObsDiags - StoreBackground From 36a25b3d0c9125bafbe90205dcd40d0fee7d87bc Mon Sep 17 00:00:00 2001 From: danholdaway Date: Tue, 20 Jun 2023 10:01:57 -0400 Subject: [PATCH 075/121] temp comment out --- src/swell/tasks/run_jedi_ufo_tests_executable.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/swell/tasks/run_jedi_ufo_tests_executable.py b/src/swell/tasks/run_jedi_ufo_tests_executable.py index 52376136..7f24d146 100644 --- a/src/swell/tasks/run_jedi_ufo_tests_executable.py +++ b/src/swell/tasks/run_jedi_ufo_tests_executable.py @@ -161,8 +161,8 @@ def execute(self): # Run the Test Obs Filters executable # ----------------------------------- - run_executable(self.logger, self.cycle_dir(), 1, jedi_executable_path, jedi_config_file, - output_log_file) + #run_executable(self.logger, self.cycle_dir(), 1, jedi_executable_path, jedi_config_file, + # output_log_file) # -------------------------------------------------------------------------------------------------- From 9d0a80a07e8170c6c6002a48ddfd612be8d26e5e Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 21 Jun 2023 09:19:12 -0400 Subject: [PATCH 076/121] YAML generation working --- .../observations/ufo_tests.yaml | 20 ++++-- .../tasks/run_jedi_ufo_tests_executable.py | 67 +++++++++---------- .../data_assimilation_window_params.py | 5 +- src/swell/utilities/dictionary.py | 17 +++++ 4 files changed, 61 insertions(+), 48 deletions(-) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ufo_tests.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ufo_tests.yaml index 1cd0a521..4f53906b 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ufo_tests.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ufo_tests.yaml @@ -1,10 +1,16 @@ default: - linear obs operator test: - coef TL: 0.1 - tolerance TL: 1.0e-11 - tolerance AD: 1.0e-13 - tolerance: 1.0e-06 + operator_test: + linear obs operator test: + coef TL: 0.1 + tolerance TL: 1.0e-11 + tolerance AD: 1.0e-13 + tolerance: 1.0e-06 + rms ref: 1.0 + filter_test: + passedBenchmark: 1 aircraft: - rms ref: 6.736593278810636 - passedBenchmark: 1 + operator_test: + rms ref: 1.9724833875646487e+38 + filter_test: + passedBenchmark: 297273 diff --git a/src/swell/tasks/run_jedi_ufo_tests_executable.py b/src/swell/tasks/run_jedi_ufo_tests_executable.py index 7f24d146..e7eed0da 100644 --- a/src/swell/tasks/run_jedi_ufo_tests_executable.py +++ b/src/swell/tasks/run_jedi_ufo_tests_executable.py @@ -13,6 +13,7 @@ import yaml from swell.tasks.base.task_base import taskBase +from swell.utilities.dictionary import update_dict from swell.utilities.run_jedi_executables import jedi_dictionary_iterator, run_executable @@ -54,11 +55,11 @@ def execute(self): # Open the JEDI config file and fill initial templates # ---------------------------------------------------- - jedi_config_filter_dict = self.jedi_rendering.render_oops_file(f'{jedi_application}') + jedi_config_dict = self.jedi_rendering.render_oops_file(f'{jedi_application}') # Perform complete template rendering # ----------------------------------- - jedi_dictionary_iterator(jedi_config_filter_dict, self.jedi_rendering, '3D', observations) + jedi_dictionary_iterator(jedi_config_dict, self.jedi_rendering, '3D', observations) # Make modifications needed for testing # ------------------------------------- @@ -76,8 +77,8 @@ def execute(self): for index in range(len(observations)): # Remove GetValues if present - if 'get values' in jedi_config_filter_dict['observations'][index]: - del jedi_config_filter_dict['observations'][index]['get values'] + if 'get values' in jedi_config_dict['observations'][index]: + del jedi_config_dict['observations'][index]['get values'] # GeoVaLs filename geo_va_ls_fname = os.path.join(self.cycle_dir(), @@ -91,59 +92,51 @@ def execute(self): if observations[index] in conventional_types: geo_va_ls_dict['levels_are_top_down'] = False - jedi_config_filter_dict['observations'][index]['geovals'] = geo_va_ls_dict + jedi_config_dict['observations'][index]['geovals'] = geo_va_ls_dict - # Make a copy for doing the operator test - # --------------------------------------- - jedi_config_operator_dict = copy.deepcopy(jedi_config_filter_dict) - - - # For filter test there needs to be at least one benchmark - dummy_search = [{'name': 'Dummy/Group/Var'}] - jedi_config_dict['observations'][index]['expectVariablesNotToExist'] = dummy_search + # Copies for each kind of test + # ---------------------------- + jedi_operator_dict = copy.deepcopy(jedi_config_dict) + jedi_filter_dict = copy.deepcopy(jedi_config_dict) - # Make a copy of the jedi config dictionary - jedi_config_dict_operator_test = copy.deepcopy(jedi_config_dict) - jedi_config_dict_filter_test = copy.deepcopy(jedi_config_dict) - - # Add the ufo_tests dictionary to the jedi config dictionary + # Loop through observations and moderate based on test needs + # ---------------------------------------------------------- for index in range(len(observations)): - # Get the test values for this observation - ufo_tests_obs = ufo_tests_dict[observations[index]] - # Overwrite the defaults with the values in ufo_tests_obs - ufo_tests_obs_dict = {**ufo_tests_default, **ufo_tests_obs} - - # Add the ufo_tests_obs_dict to the observation dictionary - jedi_config_dict_operator_test['observations'][index].update(ufo_tests_obs_dict) + ufo_tests_obs = ufo_tests_dict[observations[index]] + ufo_tests_obs = update_dict(ufo_tests_default, ufo_tests_obs) - # If the observation has filters remove them - if 'filters' in jedi_config_dict_operator_test['observations'][index]: - del jedi_config_dict_operator_test['observations'][index]['filters'] + # Merge the ufo_tests_obs dictionary with the observation dictionary + jedi_operator_dict['observations'][index].update(ufo_tests_obs['operator_test']) + jedi_filter_dict['observations'][index].update(ufo_tests_obs['filter_test']) - print(yaml.dump(jedi_config_dict_operator_test)) + # Remove filters from operator test + if 'obs filters' in jedi_operator_dict['observations'][index]: + del jedi_operator_dict['observations'][index]['obs filters'] - # Jedi configuration file - # ----------------------- + # Write configuration files for the tests + # --------------------------------------- file = os.path.join(self.cycle_dir(), 'jedi_test_ObsOperator_config.yaml') with open(file, 'w') as jedi_config_file_open: - yaml.dump(jedi_config_dict_operator_test, jedi_config_file_open, + yaml.dump(jedi_operator_dict, jedi_config_file_open, default_flow_style=False) file = os.path.join(self.cycle_dir(), 'jedi_test_ObsOperatorTLAD_config.yaml') with open(file, 'w') as jedi_config_file_open: - yaml.dump(jedi_config_dict_operator_test, jedi_config_file_open, + yaml.dump(jedi_operator_dict, jedi_config_file_open, default_flow_style=False) file = os.path.join(self.cycle_dir(), 'jedi_test_ObsFilters_config.yaml') with open(file, 'w') as jedi_config_file_open: - yaml.dump(jedi_config_dict_filter_test, jedi_config_file_open, + yaml.dump(jedi_filter_dict, jedi_config_file_open, default_flow_style=False) # Tests to run # ------------ - tests = ['test_ObsOperator', 'test_ObsOperatorTLAD', 'test_ObsFilters'] + #tests = ['test_ObsOperator'] #, 'test_ObsOperatorTLAD', 'test_ObsFilters'] + #tests = ['test_ObsOperatorTLAD'] + tests = ['test_ObsFilters'] # Loop over the tests # ------------------- @@ -161,8 +154,8 @@ def execute(self): # Run the Test Obs Filters executable # ----------------------------------- - #run_executable(self.logger, self.cycle_dir(), 1, jedi_executable_path, jedi_config_file, - # output_log_file) + run_executable(self.logger, self.cycle_dir(), 1, jedi_executable_path, jedi_config_file, + output_log_file) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/data_assimilation_window_params.py b/src/swell/utilities/data_assimilation_window_params.py index 6a037862..97f5bf87 100644 --- a/src/swell/utilities/data_assimilation_window_params.py +++ b/src/swell/utilities/data_assimilation_window_params.py @@ -74,10 +74,7 @@ def window_end_iso(self, window_offset, window_length): # Get window beginning time window_begin_dto = self.__get_window_begin_dto__(window_offset) - print(f'window_begin_dto: {window_begin_dto}') - print(f'window_length_dur: {window_length_dur}') - print(f'window_end_iso: {window_begin_dto + window_length_dur}') - + # Window end time window_end_dto = window_begin_dto + window_length_dur return window_end_dto.strftime(datetime_formats['iso_format']) diff --git a/src/swell/utilities/dictionary.py b/src/swell/utilities/dictionary.py index 192a69f1..ee58d8d2 100644 --- a/src/swell/utilities/dictionary.py +++ b/src/swell/utilities/dictionary.py @@ -130,3 +130,20 @@ def write_dict_to_yaml(dictionary, file): # -------------------------------------------------------------------------------------------------- + + +def update_dict(original_dict, overwrite_dict): + + # Create output dictionary from original dictionary + output_dict = original_dict.copy() + + for key, value in overwrite_dict.items(): + if isinstance(value, dict) and key in output_dict and isinstance(output_dict[key], dict): + output_dict[key] = update_dict(output_dict[key], value) + else: + output_dict[key] = value + + return output_dict + + +# -------------------------------------------------------------------------------------------------- From 4f8be990fc3c4c9d09ae7e831213e3e796c44aa6 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 21 Jun 2023 16:26:29 -0400 Subject: [PATCH 077/121] add geos adas suite --- .../geos_atmosphere/geos_atmosphere.yaml | 2 + .../geos_atmosphere/model/analysis.yaml | 6 + .../geos_atmosphere/model/background.yaml | 2 +- .../model/background_error.yaml | 28 +++++ .../geos_atmosphere/model/geometry_inner.yaml | 9 ++ .../geos_atmosphere/model/stage.yaml | 2 + src/swell/suites/geosadas/flow.cylc | 111 ++++++++++++++++++ .../suites-geosadas-geos_atmosphere.yaml | 106 +++++++++++++++++ .../suites/geosadas/suites-geosadas.yaml | 31 +++++ src/swell/tasks/get_geos_adas_background.py | 70 +++++++++++ src/swell/tasks/get_gsi_bc.py | 2 +- src/swell/tasks/get_gsi_ncdiag.py | 24 +++- .../tasks/run_jedi_variational_executable.py | 12 +- 13 files changed, 395 insertions(+), 10 deletions(-) create mode 100644 src/swell/configuration/jedi/interfaces/geos_atmosphere/model/analysis.yaml create mode 100644 src/swell/configuration/jedi/interfaces/geos_atmosphere/model/background_error.yaml create mode 100755 src/swell/configuration/jedi/interfaces/geos_atmosphere/model/geometry_inner.yaml create mode 100644 src/swell/suites/geosadas/flow.cylc create mode 100644 src/swell/suites/geosadas/geos_atmosphere/suites-geosadas-geos_atmosphere.yaml create mode 100644 src/swell/suites/geosadas/suites-geosadas.yaml create mode 100644 src/swell/tasks/get_geos_adas_background.py diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/geos_atmosphere.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/geos_atmosphere.yaml index 402d2544..def16162 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/geos_atmosphere.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/geos_atmosphere.yaml @@ -1,5 +1,7 @@ jedi_interface: fv3-jedi total_processors: 6*{{npx_proc}}*{{npy_proc}} +gsibec_npx_proc: {{npx_proc}} +gsibec_npx_proc: 6*{{npy_proc}} executables: hofx3D: fv3jedi_hofx_nomodel.x hofx4D: fv3jedi_hofx.x diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/analysis.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/analysis.yaml new file mode 100644 index 00000000..f7699578 --- /dev/null +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/analysis.yaml @@ -0,0 +1,6 @@ +filetype: cube sphere history +provider: geos +datapath: {{cycle_dir}} +filename: {{experiment_id}}.analysis.%yyyy%mm%dd_%hh%MM%ssz.nc4 +first: PT0H +frequency: PT1H diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/background.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/background.yaml index 4d10032f..b85df25a 100755 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/background.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/background.yaml @@ -4,7 +4,7 @@ provider: geos datapath: '' filenames: ['{{cycle_dir}}/bkg.%yyyy%mm%ddT%hh%MM%ssZ.nc4', '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/bkg/geos.crtmsrf.{{horizontal_resolution}}.nc4'] -state variables: [u,v,ua,va,t,delp,q,qi,ql,qr,qs,o3ppmv,phis, +state variables: [u,v,ua,va,t,delp,ps,q,qi,ql,qr,qs,o3ppmv,phis, qls,qcn,cfcn,frocean,frland,varflt,ustar,bstar, zpbl,cm,ct,cq,kcbl,tsm,khl,khu,frlake,frseaice,vtype, stype,vfrac,sheleg,ts,soilt,soilm,u10m,v10m] diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/background_error.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/background_error.yaml new file mode 100644 index 00000000..0eebfd21 --- /dev/null +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/background_error.yaml @@ -0,0 +1,28 @@ +covariance model: SABER +saber central block: + saber block name: gsi covariance + read: + gsi akbk: '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/fv3files/akbk{{vertical_resolution}}.nc4' + gsi error covariance file: '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/gsibec/coefficients.nc4' + gsi berror namelist file: '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/gsibec/namelist.nml' + processor layout x direction: {{gsibec_npx_proc}} + processor layout y direction: {{gsibec_npy_proc}} + debugging mode: false +saber outer blocks: +- saber block name: gsi interpolation to model grid + state variables to inverse: &bvars [eastward_wind,northward_wind,air_temperature,surface_pressure, + specific_humidity,cloud_liquid_ice,cloud_liquid_water, + mole_fraction_of_ozone_in_air, + skin_temperature, + fraction_of_ocean,fraction_of_lake,fraction_of_ice, + sfc_geopotential_height_times_grav] + gsi akbk: '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/fv3files/akbk{{vertical_resolution}}.nc4' + gsi error covariance file: '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/gsibec/coefficients.nc4' + gsi berror namelist file: '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/gsibec/namelist.nml' + processor layout x direction: {{gsibec_npx_proc}} + processor layout y direction: {{gsibec_npy_proc}} + debugging mode: false +linear variable change: + linear variable change name: Control2Analysis + input variables: *bvars + output variables: {{analysis_variables}} diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/geometry_inner.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/geometry_inner.yaml new file mode 100755 index 00000000..3116dada --- /dev/null +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/geometry_inner.yaml @@ -0,0 +1,9 @@ +fms initialization: + namelist filename: '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/fv3files/fmsmpp.nml' + field table filename: '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/fv3files/field_table_gmao' +akbk: '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/fv3files/akbk{{vertical_resolution}}.nc4' +layout: [{{npx_proc}},{{npy_proc}}] +npx: {{horizontal_resolution}} +npy: {{horizontal_resolution}} +npz: {{vertical_resolution}} +field metadata override: '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/fieldmetadata/geos.yaml' diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/stage.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/stage.yaml index efc0957e..bea18c7e 100755 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/stage.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/stage.yaml @@ -3,3 +3,5 @@ - ['{{swell_static_files}}/jedi/interfaces/geos_atmosphere/GEOS_CRTM_Surface/geos.crtmsrf.{{horizontal_resolution}}.nc4', '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/bkg/'] - ['{{experiment_root}}/{{experiment_id}}/jedi_bundle/source/fv3-jedi/test/Data/fieldmetadata/*', '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/fieldmetadata/'] - ['{{experiment_root}}/{{experiment_id}}/jedi_bundle/source/fv3-jedi/test/Data/fv3files/*', '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/fv3files/'] + - ['{{swell_static_files}}/jedi/interfaces/geos_atmosphere/gsibec/coefficients_{{horizontal_resolution}}.nc4', '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/gsibec/coefficients.nc4'] + - ['{{swell_static_files}}/jedi/interfaces/geos_atmosphere/gsibec/namelist_{{horizontal_resolution}}.nc4', '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/gsibec/namelist.nml'] diff --git a/src/swell/suites/geosadas/flow.cylc b/src/swell/suites/geosadas/flow.cylc new file mode 100644 index 00000000..f697e15c --- /dev/null +++ b/src/swell/suites/geosadas/flow.cylc @@ -0,0 +1,111 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + +# Cylc suite for executing JEDI-based non-cycling variational data assimilation + +# -------------------------------------------------------------------------------------------------- + +[scheduler] + UTC mode = True + allow implicit tasks = False + +# -------------------------------------------------------------------------------------------------- + +[scheduling] + + initial cycle point = 2020-12-15T00:00:00Z + final cycle point = 2020-12-15T00:00:00Z + + [[graph]] + R1 = """ + # Clone JEDI source code + CloneJedi + + # Build JEDI source code by linking + BuildJediByLinking? + + # Stage JEDI static files + CloneJedi => StageJedi + """ + + T00 = """ + # Get and convert bias correction coefficients + GetGsiBc => GsiBcToIoda + + # Get and convert ncdiags + GetGsiNcdiag => GsiNcdiagToIoda + + # Get background + GetGeosAdasBackground + + # Run Jedi variational executable + BuildJediByLinking[^] => RunJediVariationalExecutable + StageJedi[^] => RunJediVariationalExecutable + GsiBcToIoda => RunJediVariationalExecutable + GsiNcdiagToIoda => RunJediVariationalExecutable + GetGeosAdasBackground => RunJediVariationalExecutable + + # Clean cycle + RunJediVariationalExecutable => CleanCycle + """ + +# -------------------------------------------------------------------------------------------------- + +[runtime] + + # Task defaults + # ------------- + [[root]] + pre-script = "source $CYLC_SUITE_DEF_PATH/modules" + + [[[environment]]] + datetime = $CYLC_TASK_CYCLE_POINT + config = $CYLC_SUITE_DEF_PATH/experiment.yaml + + # Tasks + # ----- + [[CloneJedi]] + script = "swell_task CloneJedi $config" + + [[BuildJediByLinking]] + script = "swell_task BuildJediByLinking $config" + + [[StageJedi]] + script = "swell_task StageJedi $config -m geos_atmosphere" + + [[ GetGsiBc ]] + script = "swell_task GetGsiBc $config -d $datetime -m geos_atmosphere" + + [[ GsiBcToIoda ]] + script = "swell_task GsiBcToIoda $config -d $datetime -m geos_atmosphere" + + [[ GetGsiNcdiag ]] + script = "swell_task GetGsiNcdiag $config -d $datetime -m geos_atmosphere" + + [[ GsiNcdiagToIoda ]] + script = "swell_task GsiNcdiagToIoda $config -d $datetime -m geos_atmosphere" + + [[ GetGeosAdasBackground ]] + script = "swell_task GetGeosAdasBackground $config -d $datetime -m geos_atmosphere" + + [[RunJediVariationalExecutable]] + script = "swell_task RunJediVariationalExecutable $config -d $datetime -m geos_atmosphere" + platform = {{platform}} + execution time limit = {{scheduling["RunJediVariationalExecutable"]["execution_time_limit"]}} + [[[directives]]] + --account = {{scheduling["RunJediVariationalExecutable"]["account"]}} + --qos = {{scheduling["RunJediVariationalExecutable"]["qos"]}} + --job-name = RunJediVariationalExecutable + --nodes={{scheduling["RunJediVariationalExecutable"]["nodes"]}} + --ntasks-per-node={{scheduling["RunJediVariationalExecutable"]["ntasks_per_node"]}} + --constraint={{scheduling["RunJediVariationalExecutable"]["constraint"]}} + + [[CleanCycle]] + script = "swell_task CleanCycle $config -d $datetime -m geos_atmosphere" + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/geosadas/geos_atmosphere/suites-geosadas-geos_atmosphere.yaml b/src/swell/suites/geosadas/geos_atmosphere/suites-geosadas-geos_atmosphere.yaml new file mode 100644 index 00000000..201a0234 --- /dev/null +++ b/src/swell/suites/geosadas/geos_atmosphere/suites-geosadas-geos_atmosphere.yaml @@ -0,0 +1,106 @@ +# Window length +window_length: + default_value: 'PT6H' + prompt: Window length for ocean data assimilation + type: iso-duration + +# Horizontal resolution +horizontal_resolution: + default_value: '361' + prompt: What resolution for the atmospheric background (For c360 choose 361)? + options: ['181', '361', '721'] + type: string-drop-list + +# Vertical resolution +vertical_resolution: + default_value: '72' + prompt: What vertical resolution for the atmospheric background? + options: ['72'] + type: string-drop-list + +# Processor layout x direction +npx_proc: + default_value: '4' + prompt: What processor layout (x-direction) per cube face? + type: string + +# Processor layout y direction +npy_proc: + default_value: '1' + prompt: What processor layout (y-direction) per cube face? + type: string + +# Analysis +analysis_variables: + default_value: [ua,va,t,ps,q,qi,ql,o3ppmv,ts,phis,frocean,frlake,frseaice] + prompt: Variables to be assimilated? + type: string + +# Observations +observations: + default_value: + - aircraft + prompt: Select observations. + options: use_method + type: string-check-list + +# Path to GSI bias correction coefficients +path_to_gsi_bc_coefficients: + default_value: /discover/nobackup/projects/gmao/dadev/rtodling/archive/x0044/rs/Y2020/M12/x0044.rst.20201214_21z.tar + prompt: Tar or directory containing BC files + type: string + +# Path to GSI ncdiags +path_to_gsi_nc_diags: + default_value: '/archive/u/jjin3/x0044.jj_20230201/obs/Y2020/M12/D15/H00' + prompt: Path to where the ncdiags will be held + type: string + +# Path to GEOS adas background files +path_to_geos_adas_background: + default_value: /discover/nobackup/projects/gmao/dadev/rtodling/archive/x0044/rs/Y2020/M12/x0044.rst.20201214_21z.tar + prompt: Tar or directory containing BC files + type: string + +# Compute GeoVaLs and observations +produce_geovals: + default_value: false + prompt: Compute GeoVaLs as well as observations? + type: boolean + +# Fixed options (user not prompted for these) +# ------------------------------------------- +fixed_options: + window_type: + default_value: 3D + prompt: Window type (3D or 4D) + minimizer: + default_value: DRIPCG + prompt: Minimizer to use for gradient descent + number_of_iterations: + default_value: [2] + prompt: Number of iterations to use in the minimization + gradient_norm_reduction: + default_value: 1e-6 + prompt: Threshold for convergence of the minimization + window_offset: + default_value: PT3H + prompt: Time from beginning to middle of the window + analysis_forecast_window_offset: + default_value: -PT3H + prompt: Time from the middle of the window when forecasts start + background_time_offset: + default_value: PT1H + prompt: Time before the middle of the window that the background providing forecast began + background_frequency: + default_value: PT1H + prompt: 'Frequency of 4D backgrounds' + background_source: + default_value: file + prompt: 'Source of the background files (file or from model)' + clean_patterns: + default_value: ['*.nc4','*.txt'] + prompt: 'Patterns for the files to remove after completing a cycle' + crtm_coeff_dir: + default_value: {{crtm_coeff_dir}} + prompt: 'Directory containing the CRTM coefficient files' diff --git a/src/swell/suites/geosadas/suites-geosadas.yaml b/src/swell/suites/geosadas/suites-geosadas.yaml new file mode 100644 index 00000000..66701f14 --- /dev/null +++ b/src/swell/suites/geosadas/suites-geosadas.yaml @@ -0,0 +1,31 @@ +# Jedi build system +jedi_build_method: + default_value: use_existing + prompt: Create a new JEDI build or use an existing build? + options: ['create', 'use_existing'] + type: string-drop-list + +# Existing JEDI bundle directory +existing_jedi_source_directory: + default_value: {{existing_jedi_source_directory}} + prompt: Provide the path to an existing JEDI bundle directory containing source code repos + type: string + depends: + key: jedi_build_method + value: use_existing + +# Existing JEDI build +existing_jedi_build_directory: + default_value: {{existing_jedi_build_directory}} + prompt: Provide the path to an existing JEDI build directory + type: string + depends: + key: jedi_build_method + value: use_existing + +# Models to use +model_components: + default_value: ['geos_atmosphere'] + prompt: Select models to use + options: use_method + type: file-check-list diff --git a/src/swell/tasks/get_geos_adas_background.py b/src/swell/tasks/get_geos_adas_background.py new file mode 100644 index 00000000..2f57c721 --- /dev/null +++ b/src/swell/tasks/get_geos_adas_background.py @@ -0,0 +1,70 @@ +# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import glob +import os +import shutil + +from swell.tasks.base.task_base import taskBase + + +# -------------------------------------------------------------------------------------------------- + + +class GetGeosAdasBackground(taskBase): + + def execute(self): + + # Get the build method + # -------------------- + background_path = self.config.path_to_geos_adas_background() + + # Get list of ncdiags to test with + # -------------------------------- + search_pattern = '*traj*.nc*' + background_path_files_pattern = os.path.join(background_path, search_pattern) + background_path_files = glob.glob(background_path_files_pattern) + + # Filename format + source_filename_format = "%Y-%m-%d" + target_filename_format = 'bkg.%Y%m%dT%H%M%SZ.nc4' + + # Assert that some files were found + self.logger.assert_abort(len(background_path_files) != 0 is not None, f'No background ' + + f'files found in the source directory ' + + f'\'{background_path}/{search_pattern}\'') + + # Loop over all the files + # ----------------------- + for background_path_file in background_path_files: + + # Get filename from full path + background_file = os.path.basename(background_path_file) + + # Get datetime for the file from the filename + background_file_datetime = datetime.strptime(background_file, filename_format) + + # Create target filename using the datetime format + background_file_target = background_file_datetime.strftime(target_filename_format) + + # Target path and filename + background_path_file_target = os.path.join(self.cycle_dir(), background_file_target) + + # Remove target file if it exists (might be a link) + if os.path.exists(background_file_target): + os.remove(background_file_target) + + # Create symlink from target to source + self.logger.info(f'Creating sym link from {background_path_file} to ' + f'{background_path_file_target}') + os.symlink(background_path_file, background_path_file_target) + + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/get_gsi_bc.py b/src/swell/tasks/get_gsi_bc.py index 3bd64881..d0d12d02 100644 --- a/src/swell/tasks/get_gsi_bc.py +++ b/src/swell/tasks/get_gsi_bc.py @@ -25,7 +25,7 @@ def execute(self): # Get the build method # -------------------- - gsi_bc_location = self.config.gsi_bc_location() + gsi_bc_location = self.config.path_to_gsi_bc_coefficients() # Holding directory gsi_bc_dir = os.path.join(self.cycle_dir(), 'gsi_bcs') diff --git a/src/swell/tasks/get_gsi_ncdiag.py b/src/swell/tasks/get_gsi_ncdiag.py index dbfa7845..f3dbf5a5 100644 --- a/src/swell/tasks/get_gsi_ncdiag.py +++ b/src/swell/tasks/get_gsi_ncdiag.py @@ -24,7 +24,7 @@ def execute(self): # Get the build method # -------------------- - gsi_diag_path = self.config.path_to_gsi_diags() + gsi_diag_path = self.config.path_to_gsi_nc_diags() # Get list of ncdiags to test with # -------------------------------- @@ -36,17 +36,29 @@ def execute(self): gsi_diag_dir = os.path.join(self.cycle_dir(), 'gsi_ncdiags') os.makedirs(gsi_diag_dir, 0o755, exist_ok=True) + # Assert that some files were found + self.logger.assert_abort(len(gsi_diag_path_files) != 0 is not None, f'No ncdiag ' + + f'files found in the source directory ' + + f'\'{gsi_diag_path_files_pattern}\'') + # Copy all the files into the cycle directory # ------------------------------------------- for gsi_diag_path_file in gsi_diag_path_files: - # File name - gsi_diag_file = os.path.basename(gsi_diag_path_file) + # Source file + gsi_diag_file_source = os.path.basename(gsi_diag_path_file) + + # Target file + gsi_diag_file_target = os.path.join(gsi_diag_dir, gsi_diag_file_source) - self.logger.info(f'Copying {gsi_diag_file}') + # Remove target file if it exists (might be a link) + if os.path.exists(gsi_diag_file_target): + os.remove(gsi_diag_file_target) - # Copy file - shutil.copyfile(gsi_diag_path_file, os.path.join(gsi_diag_dir, gsi_diag_file)) + # Create symlink from target to source + self.logger.info(f'Creating sym link from {gsi_diag_path_file} to ' + f'{gsi_diag_file_target}') + os.symlink(gsi_diag_path_file, gsi_diag_file_target) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/run_jedi_variational_executable.py b/src/swell/tasks/run_jedi_variational_executable.py index 7936d49d..600d7f83 100644 --- a/src/swell/tasks/run_jedi_variational_executable.py +++ b/src/swell/tasks/run_jedi_variational_executable.py @@ -37,6 +37,9 @@ def execute(self): observations = self.config.observations() jedi_forecast_model = self.config.jedi_forecast_model(None) + npx_proc = self.config.npx_proc(None) + npy_proc = self.config.npy_proc(None) + # Compute data assimilation window parameters background_time = self.da_window_params.background_time(window_offset, background_time_offset) @@ -64,8 +67,8 @@ def execute(self): # Geometry self.jedi_rendering.add_key('vertical_resolution', self.config.vertical_resolution()) - self.jedi_rendering.add_key('npx_proc', self.config.npx_proc(None)) - self.jedi_rendering.add_key('npy_proc', self.config.npy_proc(None)) + self.jedi_rendering.add_key('npx_proc', npx_proc) + self.jedi_rendering.add_key('npy_proc', npy_proc) self.jedi_rendering.add_key('total_processors', self.config.total_processors(None)) # Observations @@ -73,6 +76,11 @@ def execute(self): self.jedi_rendering.add_key('crtm_coeff_dir', self.config.crtm_coeff_dir(None)) self.jedi_rendering.add_key('window_begin', window_begin) + # Atmosphere background error model + if npx_proc is not None and npy_proc is not None: + self.jedi_rendering.add_key('gsi_npx_proc', npx_proc) + self.jedi_rendering.add_key('gsi_npy_proc', 6*npy_proc) + # Model if window_type == '4D': self.jedi_rendering.add_key('background_frequency', self.config.background_frequency()) From 2ed822a18956ea65090cfd94606fe3ee4f1d6b3b Mon Sep 17 00:00:00 2001 From: danholdaway Date: Mon, 26 Jun 2023 22:17:58 -0400 Subject: [PATCH 078/121] Tasks for geosadas running --- setup.py | 1 + .../geos_atmosphere/geos_atmosphere.yaml | 2 -- .../model/background_error.yaml | 8 ++--- .../geos_atmosphere/model/stage.yaml | 4 +-- .../geos_atmosphere/task_questions.yaml | 3 ++ .../suites-geosadas-geos_atmosphere.yaml | 7 ++-- .../suites/geosadas/suites-geosadas.yaml | 4 +-- src/swell/tasks/base/task_registry.py | 1 + src/swell/tasks/get_geos_adas_background.py | 33 +++++++++---------- src/swell/tasks/get_gsi_bc.py | 5 +++ src/swell/tasks/gsi_ncdiag_to_ioda.py | 20 +++++++++-- .../tasks/run_jedi_ufo_tests_executable.py | 12 +++---- .../tasks/run_jedi_variational_executable.py | 6 ++-- src/swell/tasks/task_questions.yaml | 32 +++++++++++------- src/swell/utilities/datetime.py | 5 +-- .../utilities/render_jedi_interface_files.py | 2 ++ 16 files changed, 89 insertions(+), 56 deletions(-) diff --git a/setup.py b/setup.py index ce986cb3..36437d48 100644 --- a/setup.py +++ b/setup.py @@ -43,6 +43,7 @@ 'f90nml>=1.4.3', 'questionary>=1.10.0', 'flake8>=6.0.0', + 'netCDF4', ], package_data={ '': [ diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/geos_atmosphere.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/geos_atmosphere.yaml index def16162..402d2544 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/geos_atmosphere.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/geos_atmosphere.yaml @@ -1,7 +1,5 @@ jedi_interface: fv3-jedi total_processors: 6*{{npx_proc}}*{{npy_proc}} -gsibec_npx_proc: {{npx_proc}} -gsibec_npx_proc: 6*{{npy_proc}} executables: hofx3D: fv3jedi_hofx_nomodel.x hofx4D: fv3jedi_hofx.x diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/background_error.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/background_error.yaml index 0eebfd21..fe2f0b7e 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/background_error.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/background_error.yaml @@ -3,8 +3,8 @@ saber central block: saber block name: gsi covariance read: gsi akbk: '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/fv3files/akbk{{vertical_resolution}}.nc4' - gsi error covariance file: '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/gsibec/coefficients.nc4' - gsi berror namelist file: '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/gsibec/namelist.nml' + gsi error covariance file: '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/gsibec/gsibec_coefficients_c{{horizontal_resolution}}.nc4' + gsi berror namelist file: '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/gsibec/gsibec_configuration_c{{horizontal_resolution}}.nml' processor layout x direction: {{gsibec_npx_proc}} processor layout y direction: {{gsibec_npy_proc}} debugging mode: false @@ -17,8 +17,8 @@ saber outer blocks: fraction_of_ocean,fraction_of_lake,fraction_of_ice, sfc_geopotential_height_times_grav] gsi akbk: '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/fv3files/akbk{{vertical_resolution}}.nc4' - gsi error covariance file: '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/gsibec/coefficients.nc4' - gsi berror namelist file: '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/gsibec/namelist.nml' + gsi error covariance file: '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/gsibec/gsibec_coefficients_c{{horizontal_resolution}}.nc4' + gsi berror namelist file: '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/gsibec/gsibec_configuration_c{{horizontal_resolution}}.nml' processor layout x direction: {{gsibec_npx_proc}} processor layout y direction: {{gsibec_npy_proc}} debugging mode: false diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/stage.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/stage.yaml index bea18c7e..70c9f00d 100755 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/stage.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/stage.yaml @@ -3,5 +3,5 @@ - ['{{swell_static_files}}/jedi/interfaces/geos_atmosphere/GEOS_CRTM_Surface/geos.crtmsrf.{{horizontal_resolution}}.nc4', '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/bkg/'] - ['{{experiment_root}}/{{experiment_id}}/jedi_bundle/source/fv3-jedi/test/Data/fieldmetadata/*', '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/fieldmetadata/'] - ['{{experiment_root}}/{{experiment_id}}/jedi_bundle/source/fv3-jedi/test/Data/fv3files/*', '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/fv3files/'] - - ['{{swell_static_files}}/jedi/interfaces/geos_atmosphere/gsibec/coefficients_{{horizontal_resolution}}.nc4', '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/gsibec/coefficients.nc4'] - - ['{{swell_static_files}}/jedi/interfaces/geos_atmosphere/gsibec/namelist_{{horizontal_resolution}}.nc4', '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/gsibec/namelist.nml'] + - ['{{swell_static_files}}/jedi/interfaces/geos_atmosphere/gsibec/gsibec_coefficients_c{{horizontal_resolution}}.nc4', '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/gsibec/'] + - ['{{swell_static_files}}/jedi/interfaces/geos_atmosphere/gsibec/gsibec_configuration_c{{horizontal_resolution}}.nml', '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/gsibec/'] diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml index 6facbf95..9ba66753 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml @@ -118,6 +118,9 @@ observations: - omi_aura - ompsnm_npp +path_to_geos_adas_background: + default_value: /discover/nobackup/drholdaw/SwellTestData/geosadas/bkg/*bkg_clcv_rst* + path_to_gsi_diags: default_value: /archive/u/jjin3/x0044.jj_20230201/obs/Y2020/M12/D15/H00 diff --git a/src/swell/suites/geosadas/geos_atmosphere/suites-geosadas-geos_atmosphere.yaml b/src/swell/suites/geosadas/geos_atmosphere/suites-geosadas-geos_atmosphere.yaml index 201a0234..755b8e1a 100644 --- a/src/swell/suites/geosadas/geos_atmosphere/suites-geosadas-geos_atmosphere.yaml +++ b/src/swell/suites/geosadas/geos_atmosphere/suites-geosadas-geos_atmosphere.yaml @@ -40,25 +40,26 @@ analysis_variables: observations: default_value: - aircraft + - amsua_n19 prompt: Select observations. options: use_method type: string-check-list # Path to GSI bias correction coefficients path_to_gsi_bc_coefficients: - default_value: /discover/nobackup/projects/gmao/dadev/rtodling/archive/x0044/rs/Y2020/M12/x0044.rst.20201214_21z.tar + default_value: /discover/nobackup/drholdaw/SwellTestData/geosadas/bc prompt: Tar or directory containing BC files type: string # Path to GSI ncdiags path_to_gsi_nc_diags: - default_value: '/archive/u/jjin3/x0044.jj_20230201/obs/Y2020/M12/D15/H00' + default_value: /discover/nobackup/drholdaw/SwellTestData/geosadas/obs prompt: Path to where the ncdiags will be held type: string # Path to GEOS adas background files path_to_geos_adas_background: - default_value: /discover/nobackup/projects/gmao/dadev/rtodling/archive/x0044/rs/Y2020/M12/x0044.rst.20201214_21z.tar + default_value: /discover/nobackup/drholdaw/SwellTestData/geosadas/bkg prompt: Tar or directory containing BC files type: string diff --git a/src/swell/suites/geosadas/suites-geosadas.yaml b/src/swell/suites/geosadas/suites-geosadas.yaml index 66701f14..de1d7349 100644 --- a/src/swell/suites/geosadas/suites-geosadas.yaml +++ b/src/swell/suites/geosadas/suites-geosadas.yaml @@ -7,7 +7,7 @@ jedi_build_method: # Existing JEDI bundle directory existing_jedi_source_directory: - default_value: {{existing_jedi_source_directory}} + default_value: /discover/nobackup/drholdaw/JediDev/gsibec prompt: Provide the path to an existing JEDI bundle directory containing source code repos type: string depends: @@ -16,7 +16,7 @@ existing_jedi_source_directory: # Existing JEDI build existing_jedi_build_directory: - default_value: {{existing_jedi_build_directory}} + default_value: /discover/nobackup/drholdaw/JediDev/gsibec/build-intel-release/ prompt: Provide the path to an existing JEDI build directory type: string depends: diff --git a/src/swell/tasks/base/task_registry.py b/src/swell/tasks/base/task_registry.py index 7b3fa16c..ff5a6219 100644 --- a/src/swell/tasks/base/task_registry.py +++ b/src/swell/tasks/base/task_registry.py @@ -18,6 +18,7 @@ 'GenerateBClimatology', 'GetBackgroundGeosExperiment', 'GetBackground', + 'GetGeosAdasBackground', 'GetGeosRestart', 'GetGeovals', 'GetGsiBc', diff --git a/src/swell/tasks/get_geos_adas_background.py b/src/swell/tasks/get_geos_adas_background.py index 2f57c721..3a7f6e6e 100644 --- a/src/swell/tasks/get_geos_adas_background.py +++ b/src/swell/tasks/get_geos_adas_background.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 @@ -8,9 +8,10 @@ # -------------------------------------------------------------------------------------------------- +import datetime import glob import os -import shutil +import re from swell.tasks.base.task_base import taskBase @@ -22,24 +23,19 @@ class GetGeosAdasBackground(taskBase): def execute(self): - # Get the build method - # -------------------- + # Get the path and pattern for the background files + # ------------------------------------------------- background_path = self.config.path_to_geos_adas_background() # Get list of ncdiags to test with # -------------------------------- - search_pattern = '*traj*.nc*' - background_path_files_pattern = os.path.join(background_path, search_pattern) - background_path_files = glob.glob(background_path_files_pattern) - - # Filename format - source_filename_format = "%Y-%m-%d" - target_filename_format = 'bkg.%Y%m%dT%H%M%SZ.nc4' + background_path_files = glob.glob(background_path) # Assert that some files were found - self.logger.assert_abort(len(background_path_files) != 0 is not None, f'No background ' + + # --------------------------------- + self.logger.assert_abort(len(background_path_files) != 0, f'No background ' + f'files found in the source directory ' + - f'\'{background_path}/{search_pattern}\'') + f'\'{background_path}\'') # Loop over all the files # ----------------------- @@ -48,18 +44,21 @@ def execute(self): # Get filename from full path background_file = os.path.basename(background_path_file) + # Extract the datetime part from the string + datetime_part = re.search(r"\d{8}_\d{4}\w", background_file).group() + # Get datetime for the file from the filename - background_file_datetime = datetime.strptime(background_file, filename_format) + background_file_datetime = datetime.datetime.strptime(datetime_part, '%Y%m%d_%H%Mz') # Create target filename using the datetime format - background_file_target = background_file_datetime.strftime(target_filename_format) + background_file_target = background_file_datetime.strftime('bkg.%Y%m%dT%H%M%SZ.nc4') # Target path and filename background_path_file_target = os.path.join(self.cycle_dir(), background_file_target) # Remove target file if it exists (might be a link) - if os.path.exists(background_file_target): - os.remove(background_file_target) + if os.path.exists(background_path_file_target): + os.remove(background_path_file_target) # Create symlink from target to source self.logger.info(f'Creating sym link from {background_path_file} to ' diff --git a/src/swell/tasks/get_gsi_bc.py b/src/swell/tasks/get_gsi_bc.py index d0d12d02..92364eae 100644 --- a/src/swell/tasks/get_gsi_bc.py +++ b/src/swell/tasks/get_gsi_bc.py @@ -37,10 +37,15 @@ def execute(self): if os.path.isdir(gsi_bc_location): + print(gsi_bc_location) + # Get list of matching files in the directory ana_satbias_rst = glob.glob(os.path.join(gsi_bc_location, '*ana_satbias_rst*')) ana_satbiaspc_rst = glob.glob(os.path.join(gsi_bc_location, '*ana_satbiaspc_rst*')) + print(ana_satbias_rst) + print(ana_satbiaspc_rst) + # Record that files were found if len(ana_satbias_rst) == 1 and len(ana_satbiaspc_rst) == 1: files_found = True diff --git a/src/swell/tasks/gsi_ncdiag_to_ioda.py b/src/swell/tasks/gsi_ncdiag_to_ioda.py index e190f69d..99f966b1 100644 --- a/src/swell/tasks/gsi_ncdiag_to_ioda.py +++ b/src/swell/tasks/gsi_ncdiag_to_ioda.py @@ -8,6 +8,7 @@ # -------------------------------------------------------------------------------------------------- +import datetime import glob import os @@ -17,6 +18,7 @@ from swell.tasks.base.task_base import taskBase from swell.utilities.shell_commands import run_track_log_subprocess +from swell.utilities.datetime import datetime_formats # -------------------------------------------------------------------------------------------------- @@ -66,11 +68,16 @@ def execute(self): for needed_ioda_type in needed_ioda_types: gsi_types_to_process += ioda_to_gsi_dict[needed_ioda_type] gsi_types_to_process = list(set(gsi_types_to_process)) # Unique values only + gsi_types_to_process = sorted(gsi_types_to_process) # Remove conv_platforms_to_process from the total observations list, leaving rad and ozn for needed_ioda_type in needed_ioda_types: observations.remove(needed_ioda_type) + # Convert cycle time datetime object to string with format yyyymmdd_hhz + gsi_datetime_str = datetime.datetime.strftime(self.cycle_time_dto(), + datetime_formats['gsi_nc_diag_format']) + # First process the conventional data (if needed) # ----------------------------------------------- for gsi_type_to_process in gsi_types_to_process: @@ -80,10 +87,19 @@ def execute(self): self.logger.info(log_str) self.logger.info('-'*len(log_str)) - gsi_conv_file = glob.glob(os.path.join(gsi_diag_dir, f'*{gsi_type_to_process}*'))[0] + # Path to search to GSI ncdiag files + path_to_search = os.path.join(gsi_diag_dir, + f'*{gsi_type_to_process}_*{gsi_datetime_str}*') + + # Get the list of files + gsi_conv_file = glob.glob(path_to_search) + + # Check that at least one file was found + self.logger.assert_abort(len(gsi_conv_file) == 1, 'The search for GSI ncdiags files ' + + f'returned more than one file. Search: \'{gsi_conv_file}\'') # Open the file - Diag = gsid.Conv(gsi_conv_file) + Diag = gsid.Conv(gsi_conv_file[0]) Diag.read() # Assemble list of needed platforms diff --git a/src/swell/tasks/run_jedi_ufo_tests_executable.py b/src/swell/tasks/run_jedi_ufo_tests_executable.py index e7eed0da..ce853a02 100644 --- a/src/swell/tasks/run_jedi_ufo_tests_executable.py +++ b/src/swell/tasks/run_jedi_ufo_tests_executable.py @@ -119,23 +119,19 @@ def execute(self): # --------------------------------------- file = os.path.join(self.cycle_dir(), 'jedi_test_ObsOperator_config.yaml') with open(file, 'w') as jedi_config_file_open: - yaml.dump(jedi_operator_dict, jedi_config_file_open, - default_flow_style=False) + yaml.dump(jedi_operator_dict, jedi_config_file_open, default_flow_style=False) file = os.path.join(self.cycle_dir(), 'jedi_test_ObsOperatorTLAD_config.yaml') with open(file, 'w') as jedi_config_file_open: - yaml.dump(jedi_operator_dict, jedi_config_file_open, - default_flow_style=False) + yaml.dump(jedi_operator_dict, jedi_config_file_open, default_flow_style=False) file = os.path.join(self.cycle_dir(), 'jedi_test_ObsFilters_config.yaml') with open(file, 'w') as jedi_config_file_open: - yaml.dump(jedi_filter_dict, jedi_config_file_open, - default_flow_style=False) + yaml.dump(jedi_filter_dict, jedi_config_file_open, default_flow_style=False) # Tests to run # ------------ - #tests = ['test_ObsOperator'] #, 'test_ObsOperatorTLAD', 'test_ObsFilters'] - #tests = ['test_ObsOperatorTLAD'] + # tests = ['test_ObsOperator', 'test_ObsOperatorTLAD', 'test_ObsFilters'] tests = ['test_ObsFilters'] # Loop over the tests diff --git a/src/swell/tasks/run_jedi_variational_executable.py b/src/swell/tasks/run_jedi_variational_executable.py index 600d7f83..679bd3d6 100644 --- a/src/swell/tasks/run_jedi_variational_executable.py +++ b/src/swell/tasks/run_jedi_variational_executable.py @@ -78,8 +78,8 @@ def execute(self): # Atmosphere background error model if npx_proc is not None and npy_proc is not None: - self.jedi_rendering.add_key('gsi_npx_proc', npx_proc) - self.jedi_rendering.add_key('gsi_npy_proc', 6*npy_proc) + self.jedi_rendering.add_key('gsibec_npx_proc', npx_proc) + self.jedi_rendering.add_key('gsibec_npy_proc', 6*npy_proc) # Model if window_type == '4D': @@ -123,8 +123,8 @@ def execute(self): # Run the JEDI executable # ----------------------- + self.logger.info('Running '+jedi_executable_path+' with '+str(np)+' processors.') run_executable(self.logger, self.cycle_dir(), np, jedi_executable_path, jedi_config_file, output_log_file) - self.logger.info('Running '+jedi_executable_path+' with '+str(np)+' processors.') # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/task_questions.yaml b/src/swell/tasks/task_questions.yaml index 1966ed8e..6ed19036 100644 --- a/src/swell/tasks/task_questions.yaml +++ b/src/swell/tasks/task_questions.yaml @@ -283,16 +283,6 @@ gradient_norm_reduction: - RunJediVariationalExecutable type: float -gsi_bc_location: - ask_question: true - default_value: defer_to_model - models: - - geos_atmosphere - prompt: What is the location where GSI bias correction files can be found? - tasks: - - GetGsiBc - type: string - horizontal_resolution: ask_question: true default_value: defer_to_model @@ -435,7 +425,27 @@ observations: - SaveObsDiags type: string-check-list -path_to_gsi_diags: +path_to_geos_adas_background: + ask_question: true + default_value: defer_to_model + models: + - geos_atmosphere + prompt: What is the path to where the cubed sphere backgrounds are in the GEOSadas run? + tasks: + - GetGeosAdasBackground + type: string + +path_to_gsi_bc_coefficients: + ask_question: true + default_value: defer_to_model + models: + - geos_atmosphere + prompt: What is the location where GSI bias correction files can be found? + tasks: + - GetGsiBc + type: string + +path_to_gsi_nc_diags: ask_question: true default_value: defer_to_model models: diff --git a/src/swell/utilities/datetime.py b/src/swell/utilities/datetime.py index bc3c9188..956cb7e7 100644 --- a/src/swell/utilities/datetime.py +++ b/src/swell/utilities/datetime.py @@ -15,8 +15,9 @@ # -------------------------------------------------------------------------------------------------- datetime_formats = { - 'directory_format': '%Y%m%dT%H%M%SZ', # yyyymmddThhMMssZ for directory formats - 'iso_format': '%Y-%m-%dT%H:%M:%SZ' # yyyy-mm-ddThh:MM:ssZ ISO format + 'directory_format': '%Y%m%dT%H%M%SZ', # yyyymmddThhMMssZ for directory formats + 'iso_format': '%Y-%m-%dT%H:%M:%SZ', # yyyy-mm-ddThh:MM:ssZ ISO format + 'gsi_nc_diag_format': '%Y%m%d_%Hz', # yyyymmdd_hhz Format for GSI netcdf diagnostic files } # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/render_jedi_interface_files.py b/src/swell/utilities/render_jedi_interface_files.py index c465ccb6..cb6e8b0a 100644 --- a/src/swell/utilities/render_jedi_interface_files.py +++ b/src/swell/utilities/render_jedi_interface_files.py @@ -51,6 +51,8 @@ def __init__(self, logger, experiment_root, experiment_id, cycle_dir, jedi_inter 'local_background_time', 'local_background_time_iso', 'minimizer', + 'gsibec_npx_proc', + 'gsibec_npy_proc', 'npx_proc', 'npy_proc', 'number_of_iterations', From 70b8fec3e5ab40f024d14dec7c39eea71be3752b Mon Sep 17 00:00:00 2001 From: danholdaway Date: Mon, 26 Jun 2023 22:23:44 -0400 Subject: [PATCH 079/121] dates --- README.md | 2 +- setup.py | 2 +- src/swell/suites/ufo_testing/flow.cylc | 2 +- src/swell/tasks/get_geovals.py | 2 +- src/swell/tasks/get_gsi_bc.py | 2 +- src/swell/tasks/get_gsi_ncdiag.py | 2 +- src/swell/tasks/gsi_ncdiag_to_ioda.py | 2 +- src/swell/tasks/link_geos_output.py | 2 - ...un_jedi_test_obs_filters_executable_old.py | 88 ------------------- 9 files changed, 7 insertions(+), 97 deletions(-) delete mode 100644 src/swell/tasks/run_jedi_test_obs_filters_executable_old.py diff --git a/README.md b/README.md index ced5ac67..5fb5ed9a 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ ### Licence: -(C) Copyright 2021-2022 United States Government as represented by the Administrator of the National +(C) Copyright 2021- United States Government as represented by the Administrator of the National Aeronautics and Space Administration. All Rights Reserved. [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) diff --git a/setup.py b/setup.py index 36437d48..ba79e501 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/suites/ufo_testing/flow.cylc b/src/swell/suites/ufo_testing/flow.cylc index df423b34..fe98aef5 100644 --- a/src/swell/suites/ufo_testing/flow.cylc +++ b/src/swell/suites/ufo_testing/flow.cylc @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/tasks/get_geovals.py b/src/swell/tasks/get_geovals.py index 9669f91c..e215852c 100644 --- a/src/swell/tasks/get_geovals.py +++ b/src/swell/tasks/get_geovals.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/tasks/get_gsi_bc.py b/src/swell/tasks/get_gsi_bc.py index 92364eae..7792b415 100644 --- a/src/swell/tasks/get_gsi_bc.py +++ b/src/swell/tasks/get_gsi_bc.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/tasks/get_gsi_ncdiag.py b/src/swell/tasks/get_gsi_ncdiag.py index f3dbf5a5..1734f6c9 100644 --- a/src/swell/tasks/get_gsi_ncdiag.py +++ b/src/swell/tasks/get_gsi_ncdiag.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/tasks/gsi_ncdiag_to_ioda.py b/src/swell/tasks/gsi_ncdiag_to_ioda.py index 99f966b1..7440bae1 100644 --- a/src/swell/tasks/gsi_ncdiag_to_ioda.py +++ b/src/swell/tasks/gsi_ncdiag_to_ioda.py @@ -1,4 +1,4 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 diff --git a/src/swell/tasks/link_geos_output.py b/src/swell/tasks/link_geos_output.py index 962e1135..6718dd23 100644 --- a/src/swell/tasks/link_geos_output.py +++ b/src/swell/tasks/link_geos_output.py @@ -7,11 +7,9 @@ # -------------------------------------------------------------------------------------------------- -from datetime import datetime as dt import os from swell.tasks.base.task_base import taskBase -from swell.utilities.datetime import datetime_formats # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/run_jedi_test_obs_filters_executable_old.py b/src/swell/tasks/run_jedi_test_obs_filters_executable_old.py deleted file mode 100644 index 0e3237a5..00000000 --- a/src/swell/tasks/run_jedi_test_obs_filters_executable_old.py +++ /dev/null @@ -1,88 +0,0 @@ -# (C) Copyright 2021-2022 United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - - -# -------------------------------------------------------------------------------------------------- - - -import os -import yaml - -from swell.tasks.base.run_jedi_executable_base import RunJediExecutableBase - - -# -------------------------------------------------------------------------------------------------- - - -class RunJediTestObsFiltersExecutable(RunJediExecutableBase): - - # ---------------------------------------------------------------------------------------------- - - def execute(self): - - # Path to executable being run - # ---------------------------- - cycle_dir = self.config_get('cycle_dir') - experiment_dir = self.config_get('experiment_dir') - observations = self.config_get('observations') - window_begin = self.config_get('window_begin') - - # Make cycle dir - # -------------- - os.makedirs(cycle_dir, 0o755, exist_ok=True) - - # Jedi configuration file - # ----------------------- - jedi_config_file = os.path.join(cycle_dir, 'jedi_test_obs_filters.yaml') - - # Output log file - # --------------- - output_log_file = os.path.join(cycle_dir, 'jedi_test_obs_filters.log') - - # Generate the JEDI configuration file for running the executable - # --------------------------------------------------------------- - jedi_config_dict = self.generate_jedi_config('TestObsFilters') - - # Make modifications needed for testing - # ------------------------------------- - - conventional_types = ['aircraft'] - - # Loop over the observations - for index in range(len(observations)): - - # Remove GetValues if present - if 'get values' in jedi_config_dict['observations'][index]: - del jedi_config_dict['observations'][index]['get values'] - - # Create GeoVaLs dictionary - geovals = {} - geovals['filename'] = os.path.join(cycle_dir, - f'{observations[index]}_geovals.{window_begin}.nc4') - # For conventional add the GeoVaLs flip - if observations[index] in conventional_types: - geovals['levels_are_top_down'] = True - - jedi_config_dict['observations'][index]['geovals'] = geovals - - # Need to insert at least one benchmark, but we do not really want to check anything - # so check that some made up variable does not exist - - # Write executable configuration to file - # -------------------------------------- - with open(jedi_config_file, 'w') as jedi_config_file_open: - yaml.dump(jedi_config_dict, jedi_config_file_open, default_flow_style=False) - - # Jedi executable name - # -------------------- - jedi_executable_path = os.path.join(experiment_dir, 'jedi_bundle', 'build', 'bin', - 'test_ObsFilters.x') - - # Run the JEDI executable - # ----------------------- - self.run_executable(cycle_dir, 1, jedi_executable_path, jedi_config_file, output_log_file) - -# -------------------------------------------------------------------------------------------------- From fc7cd3cb4a87755ff8f83bddf65034432a3cc948 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Mon, 26 Jun 2023 22:36:00 -0400 Subject: [PATCH 080/121] unused variables --- src/swell/tasks/get_geos_restart.py | 2 -- src/swell/tasks/get_gsi_ncdiag.py | 1 - src/swell/tasks/move_da_restart.py | 3 +-- src/swell/tasks/move_forecast_restart.py | 3 +-- src/swell/tasks/prep_geos_run_dir.py | 7 ++----- src/swell/tasks/prepare_analysis.py | 5 +---- src/swell/tasks/run_geos_executable.py | 2 +- src/swell/utilities/build.py | 2 +- src/swell/utilities/geos.py | 7 ++----- 9 files changed, 9 insertions(+), 23 deletions(-) diff --git a/src/swell/tasks/get_geos_restart.py b/src/swell/tasks/get_geos_restart.py index 1a0c4e3f..efa8be80 100644 --- a/src/swell/tasks/get_geos_restart.py +++ b/src/swell/tasks/get_geos_restart.py @@ -13,8 +13,6 @@ from swell.tasks.base.task_base import taskBase from swell.utilities.file_system_operations import copy_to_dst_dir -from datetime import datetime as dt - # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/get_gsi_ncdiag.py b/src/swell/tasks/get_gsi_ncdiag.py index 1734f6c9..38e622a7 100644 --- a/src/swell/tasks/get_gsi_ncdiag.py +++ b/src/swell/tasks/get_gsi_ncdiag.py @@ -10,7 +10,6 @@ import glob import os -import shutil from swell.tasks.base.task_base import taskBase diff --git a/src/swell/tasks/move_da_restart.py b/src/swell/tasks/move_da_restart.py index da0619f3..0b4e3057 100644 --- a/src/swell/tasks/move_da_restart.py +++ b/src/swell/tasks/move_da_restart.py @@ -1,4 +1,4 @@ -# (C) Copyright 2023 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 @@ -12,7 +12,6 @@ from swell.tasks.base.task_base import taskBase from swell.utilities.file_system_operations import move_files -from datetime import datetime as dt # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/move_forecast_restart.py b/src/swell/tasks/move_forecast_restart.py index ab975afd..d1e99ba7 100644 --- a/src/swell/tasks/move_forecast_restart.py +++ b/src/swell/tasks/move_forecast_restart.py @@ -1,4 +1,4 @@ -# (C) Copyright 2023 United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 @@ -12,7 +12,6 @@ from swell.tasks.base.task_base import taskBase from swell.utilities.file_system_operations import move_files -from datetime import datetime as dt # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/prep_geos_run_dir.py b/src/swell/tasks/prep_geos_run_dir.py index 810e53b4..606411d2 100644 --- a/src/swell/tasks/prep_geos_run_dir.py +++ b/src/swell/tasks/prep_geos_run_dir.py @@ -12,7 +12,6 @@ import yaml from datetime import datetime as dt -import isodate from swell.tasks.base.task_base import taskBase from swell.utilities.file_system_operations import copy_to_dst_dir @@ -300,12 +299,12 @@ def get_bcs(self): self.logger.info(' OBTAINING EXTRA WOA13 files') rst_path = self.config.geos_restarts_directory() - src = os.path.join(self.swell_static_files, 'geos', 'restarts', self.rst_path, + src = os.path.join(self.swell_static_files, 'geos', 'restarts', rst_path, 'woa13_ptemp_monthly.nc') copy_to_dst_dir(self.logger, src, self.forecast_dir(['INPUT', 'woa13_ptemp_monthly.nc'])) - src = os.path.join(self.swell_static_files, 'geos', 'restarts', self.rst_path, + src = os.path.join(self.swell_static_files, 'geos', 'restarts', rst_path, 'woa13_s_monthly.nc') copy_to_dst_dir(self.logger, src, self.forecast_dir(['INPUT', 'woa13_s_monthly.nc'])) @@ -367,9 +366,7 @@ def link_replay(self): # --------------------------------------------------------- if self.agcm_dict['REPLAY_MODE'] == 'Exact' or self.agcm_dict['REPLAY_MODE'] == 'Regular': - ANA_EXPID = self.agcm_dict['REPLAY_ANA_EXPID'] ANA_LOCATION = self.agcm_dict['REPLAY_ANA_LOCATION'] - REPLAY_FILE = self.agcm_dict['REPLAY_FILE'] rply_dict = { os.path.join(ANA_LOCATION, 'aod'): '', diff --git a/src/swell/tasks/prepare_analysis.py b/src/swell/tasks/prepare_analysis.py index 8fe4d080..48d8aa2e 100644 --- a/src/swell/tasks/prepare_analysis.py +++ b/src/swell/tasks/prepare_analysis.py @@ -1,4 +1,4 @@ -# (C) Copyright 2023- United States Government as represented by the Administrator of the +# (C) Copyright 2021- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 @@ -7,14 +7,12 @@ # -------------------------------------------------------------------------------------------------- -from datetime import datetime as dt import glob import netCDF4 as nc import os import shutil from swell.tasks.base.task_base import taskBase -from swell.utilities.datetime import datetime_formats # -------------------------------------------------------------------------------------------------- @@ -41,7 +39,6 @@ def execute(self): # -------------------------------- self.current_cycle = os.path.basename(self.forecast_dir()) self.cc_dto = self.cycle_time_dto() - ana_path = self.at_cycledir(['ocn.*' + self.cc_dto.strftime('.an.%Y-%m-%dT%H:%M:%SZ.nc')]) # GEOS restarts have seconds in their filename # -------------------------------------------- diff --git a/src/swell/tasks/run_geos_executable.py b/src/swell/tasks/run_geos_executable.py index aed9e4dc..36dd4bd2 100644 --- a/src/swell/tasks/run_geos_executable.py +++ b/src/swell/tasks/run_geos_executable.py @@ -10,7 +10,7 @@ import os from swell.tasks.base.task_base import taskBase -from swell.utilities.shell_commands import run_subprocess, run_track_log_subprocess +from swell.utilities.shell_commands import run_track_log_subprocess # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/build.py b/src/swell/utilities/build.py index 532a6d05..64780271 100644 --- a/src/swell/utilities/build.py +++ b/src/swell/utilities/build.py @@ -10,7 +10,7 @@ import os import shutil -from jedi_bundle.bin.jedi_bundle import get_default_config, get_bundles +from jedi_bundle.bin.jedi_bundle import get_default_config # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/geos.py b/src/swell/utilities/geos.py index 57b2c7a8..e8017a00 100644 --- a/src/swell/utilities/geos.py +++ b/src/swell/utilities/geos.py @@ -1,4 +1,4 @@ -# (C) Copyright 2023 United States Government as represented by the Administrator of the +# (C) Copyright 2023- United States Government as represented by the Administrator of the # National Aeronautics and Space Administration. All Rights Reserved. # # This software is licensed under the terms of the Apache Licence Version 2.0 @@ -14,11 +14,8 @@ import netCDF4 import os import re -import shutil -from abc import ABC, abstractmethod - -from swell.utilities.shell_commands import run_subprocess, run_track_log_subprocess +from swell.utilities.shell_commands import run_subprocess from swell.utilities.datetime import datetime_formats # -------------------------------------------------------------------------------------------------- From 9abd3b747da5ac3dd11c7ebefc5bfb433635e596 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Mon, 26 Jun 2023 22:37:42 -0400 Subject: [PATCH 081/121] standardise task_questions --- src/swell/tasks/task_questions.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/swell/tasks/task_questions.yaml b/src/swell/tasks/task_questions.yaml index 6ed19036..ad326ad6 100644 --- a/src/swell/tasks/task_questions.yaml +++ b/src/swell/tasks/task_questions.yaml @@ -178,8 +178,8 @@ forecast_duration: - all prompt: GEOS forecast duration tasks: - - PrepGeosRunDir - MoveForecastRestart + - PrepGeosRunDir type: duration geos_background_restart_offset: @@ -272,6 +272,7 @@ geovals_provider: prompt: What is the name of the R2D2 database providing the GeoVaLs? tasks: - GetGeovals + type: string gradient_norm_reduction: ask_question: false @@ -430,7 +431,8 @@ path_to_geos_adas_background: default_value: defer_to_model models: - geos_atmosphere - prompt: What is the path to where the cubed sphere backgrounds are in the GEOSadas run? + prompt: What is the path to where the cubed sphere backgrounds are in the GEOSadas + run? tasks: - GetGeosAdasBackground type: string @@ -513,7 +515,7 @@ total_processors: tasks: - GenerateBClimatology - GenerateBClimatologyByLinking - - RunGeosExecutable + - PrepareAnalysis - RunJediHofxExecutable - RunJediVariationalExecutable type: integer From 87d43fa2a342d781dc4a5c82d9c4c5a9d6665b11 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Mon, 26 Jun 2023 22:44:12 -0400 Subject: [PATCH 082/121] update default dictionaries --- .../geos_atmosphere/task_questions.yaml | 16 +++++++++++----- .../interfaces/geos_ocean/task_questions.yaml | 4 ++++ .../platforms/generic/task_questions.yaml | 8 ++++++++ .../platforms/nccs_discover/task_questions.yaml | 8 ++++++++ 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml index 9ba66753..86d315c2 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml @@ -32,6 +32,10 @@ clean_patterns: - '*.txt' - logfile.*.out +forecast_duration: + default_value: + - PT6H + geos_background_restart_offset: default_value: TODO @@ -50,9 +54,6 @@ geovals_provider: gradient_norm_reduction: default_value: 10e-5 -gsi_bc_location: - default_value: /discover/nobackup/projects/gmao/dadev/rtodling/archive/x0044/rs/Y2020/M12/x0044.rst.20201214_21z.tar - horizontal_resolution: default_value: C361 options: @@ -121,8 +122,13 @@ observations: path_to_geos_adas_background: default_value: /discover/nobackup/drholdaw/SwellTestData/geosadas/bkg/*bkg_clcv_rst* -path_to_gsi_diags: - default_value: /archive/u/jjin3/x0044.jj_20230201/obs/Y2020/M12/D15/H00 +path_to_gsi_bc_coefficients: + default_value: + - /discover/nobackup/drholdaw/SwellTestData/geosadas/bc + +path_to_gsi_nc_diags: + default_value: + - /discover/nobackup/drholdaw/SwellTestData/geosadas/obs produce_geovals: default_value: true diff --git a/src/swell/configuration/jedi/interfaces/geos_ocean/task_questions.yaml b/src/swell/configuration/jedi/interfaces/geos_ocean/task_questions.yaml index 8e02881e..4739701f 100644 --- a/src/swell/configuration/jedi/interfaces/geos_ocean/task_questions.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_ocean/task_questions.yaml @@ -29,6 +29,10 @@ clean_patterns: - '*MOM*' - logfile.*.out +forecast_duration: + default_value: + - P1D + gradient_norm_reduction: default_value: 1e-10 diff --git a/src/swell/deployment/platforms/generic/task_questions.yaml b/src/swell/deployment/platforms/generic/task_questions.yaml index e9fa7727..6bb85dac 100644 --- a/src/swell/deployment/platforms/generic/task_questions.yaml +++ b/src/swell/deployment/platforms/generic/task_questions.yaml @@ -13,6 +13,14 @@ existing_jedi_build_directory: existing_jedi_source_directory: default_value: This would need to be provided by user +geos_experiment_directory: + default_value: + - /home/geos/ + +geos_restarts_directory: + default_value: + - /home/geos/restarts + swell_static_files: default_value: This would need to be provided by user diff --git a/src/swell/deployment/platforms/nccs_discover/task_questions.yaml b/src/swell/deployment/platforms/nccs_discover/task_questions.yaml index 6bd82470..f8207772 100644 --- a/src/swell/deployment/platforms/nccs_discover/task_questions.yaml +++ b/src/swell/deployment/platforms/nccs_discover/task_questions.yaml @@ -13,6 +13,14 @@ existing_jedi_build_directory: existing_jedi_source_directory: default_value: /discover/nobackup/gmao_ci/swell/nightly/latest/jedi_build/ +geos_experiment_directory: + default_value: + - /discover/nobackup/drholdaw/SwellStaticFiles/geos/run_dirs/5deg_v11 + +geos_restarts_directory: + default_value: + - /discover/nobackup/drholdaw/SwellStaticFiles/geos/restarts/restarts_20210620_000000 + swell_static_files: default_value: /discover/nobackup/drholdaw/SwellStaticFiles From e89fae894a72fb100515dd8f589aca3e75f2f53c Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 28 Jun 2023 16:53:07 -0400 Subject: [PATCH 083/121] config changes --- .../suites-geosadas-geos_atmosphere.yaml | 4 ++-- src/swell/suites/geosadas/suites-geosadas.yaml | 5 +++++ src/swell/tasks/clone_jedi.py | 4 ++-- src/swell/tasks/gsi_ncdiag_to_ioda.py | 5 +++++ src/swell/tasks/run_jedi_hofx_executable.py | 10 +++++++--- src/swell/tasks/run_jedi_ufo_tests_executable.py | 9 ++++++--- src/swell/tasks/run_jedi_variational_executable.py | 10 +++++++--- src/swell/tasks/task_questions.yaml | 10 ++++++++++ 8 files changed, 44 insertions(+), 13 deletions(-) diff --git a/src/swell/suites/geosadas/geos_atmosphere/suites-geosadas-geos_atmosphere.yaml b/src/swell/suites/geosadas/geos_atmosphere/suites-geosadas-geos_atmosphere.yaml index 755b8e1a..cc93920f 100644 --- a/src/swell/suites/geosadas/geos_atmosphere/suites-geosadas-geos_atmosphere.yaml +++ b/src/swell/suites/geosadas/geos_atmosphere/suites-geosadas-geos_atmosphere.yaml @@ -39,7 +39,7 @@ analysis_variables: # Observations observations: default_value: - - aircraft + - sondes - amsua_n19 prompt: Select observations. options: use_method @@ -59,7 +59,7 @@ path_to_gsi_nc_diags: # Path to GEOS adas background files path_to_geos_adas_background: - default_value: /discover/nobackup/drholdaw/SwellTestData/geosadas/bkg + default_value: /discover/nobackup/drholdaw/SwellTestData/geosadas/bkg/*bkg_clcv_rst* prompt: Tar or directory containing BC files type: string diff --git a/src/swell/suites/geosadas/suites-geosadas.yaml b/src/swell/suites/geosadas/suites-geosadas.yaml index de1d7349..63a0b853 100644 --- a/src/swell/suites/geosadas/suites-geosadas.yaml +++ b/src/swell/suites/geosadas/suites-geosadas.yaml @@ -29,3 +29,8 @@ model_components: prompt: Select models to use options: use_method type: file-check-list + +generate_yaml_and_exit: + default_value: True + prompt: Generate YAML and exit? + type: boolean diff --git a/src/swell/tasks/clone_jedi.py b/src/swell/tasks/clone_jedi.py index 77f1f41b..6f72d821 100644 --- a/src/swell/tasks/clone_jedi.py +++ b/src/swell/tasks/clone_jedi.py @@ -10,10 +10,10 @@ import os -from jedi_bundle.bin.jedi_bundle import execute_tasks +from jedi_bundle.bin.jedi_bundle import execute_tasks, get_bundles from swell.tasks.base.task_base import taskBase -from swell.utilities.build import set_jedi_bundle_config, get_bundles, build_and_source_dirs +from swell.utilities.build import set_jedi_bundle_config, build_and_source_dirs from swell.utilities.build import link_path diff --git a/src/swell/tasks/gsi_ncdiag_to_ioda.py b/src/swell/tasks/gsi_ncdiag_to_ioda.py index 7440bae1..0f78b7de 100644 --- a/src/swell/tasks/gsi_ncdiag_to_ioda.py +++ b/src/swell/tasks/gsi_ncdiag_to_ioda.py @@ -134,6 +134,11 @@ def execute(self): # List of files for that instrument ioda_path_files = glob.glob(os.path.join(self.cycle_dir(), ioda_type_pattern)) + # Check that there are some files to combine + self.logger.assert_abort(len(ioda_path_files)> 0, f'In combine of {needed_ioda_type} ' + + f'no files where found. Ensure that the converter worked as ' + + f'expected.') + # Get last file (first could be type_obs_ if the code already ran) ioda_file_0 = os.path.basename(ioda_path_files[-1]) diff --git a/src/swell/tasks/run_jedi_hofx_executable.py b/src/swell/tasks/run_jedi_hofx_executable.py index 354f12f8..9c3684b3 100644 --- a/src/swell/tasks/run_jedi_hofx_executable.py +++ b/src/swell/tasks/run_jedi_hofx_executable.py @@ -35,6 +35,7 @@ def execute(self): background_time_offset = self.config.background_time_offset() observations = self.config.observations() jedi_forecast_model = self.config.jedi_forecast_model(None) + generate_yaml_and_exit = self.config.generate_yaml_and_exit(False) # Compute data assimilation window parameters background_time = self.da_window_params.background_time(window_offset, @@ -109,8 +110,11 @@ def execute(self): # Run the JEDI executable # ----------------------- - run_executable(self.logger, self.cycle_dir(), np, jedi_executable_path, jedi_config_file, - output_log_file) - self.logger.info('Running '+jedi_executable_path+' with '+str(np)+' processors.') + if not generate_yaml_and_exit: + self.logger.info('Running '+jedi_executable_path+' with '+str(np)+' processors.') + run_executable(self.logger, self.cycle_dir(), np, jedi_executable_path, + jedi_config_file, output_log_file) + else: + self.logger.info('YAML generated, now exiting.') # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/run_jedi_ufo_tests_executable.py b/src/swell/tasks/run_jedi_ufo_tests_executable.py index ce853a02..665bdc4c 100644 --- a/src/swell/tasks/run_jedi_ufo_tests_executable.py +++ b/src/swell/tasks/run_jedi_ufo_tests_executable.py @@ -36,6 +36,7 @@ def execute(self): window_length = self.config.window_length() bkg_time_offset = self.config.background_time_offset() observations = self.config.observations() + generate_yaml_and_exit = self.config.generate_yaml_and_exit(False) # Compute data assimilation window parameters window_begin = self.da_window_params.window_begin(window_offset) @@ -150,8 +151,10 @@ def execute(self): # Run the Test Obs Filters executable # ----------------------------------- - run_executable(self.logger, self.cycle_dir(), 1, jedi_executable_path, jedi_config_file, - output_log_file) - + if not generate_yaml_and_exit: + run_executable(self.logger, self.cycle_dir(), 1, jedi_executable_path, + jedi_config_file, output_log_file) + else: + self.logger.info('YAML generated, now exiting.') # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/run_jedi_variational_executable.py b/src/swell/tasks/run_jedi_variational_executable.py index 679bd3d6..0b9e6727 100644 --- a/src/swell/tasks/run_jedi_variational_executable.py +++ b/src/swell/tasks/run_jedi_variational_executable.py @@ -36,6 +36,7 @@ def execute(self): number_of_iterations = self.config.number_of_iterations() observations = self.config.observations() jedi_forecast_model = self.config.jedi_forecast_model(None) + generate_yaml_and_exit = self.config.generate_yaml_and_exit(False) npx_proc = self.config.npx_proc(None) npy_proc = self.config.npy_proc(None) @@ -123,8 +124,11 @@ def execute(self): # Run the JEDI executable # ----------------------- - self.logger.info('Running '+jedi_executable_path+' with '+str(np)+' processors.') - run_executable(self.logger, self.cycle_dir(), np, jedi_executable_path, jedi_config_file, - output_log_file) + if not generate_yaml_and_exit: + self.logger.info('Running '+jedi_executable_path+' with '+str(np)+' processors.') + run_executable(self.logger, self.cycle_dir(), np, jedi_executable_path, + jedi_config_file, output_log_file) + else: + self.logger.info('YAML generated, now exiting.') # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/task_questions.yaml b/src/swell/tasks/task_questions.yaml index ad326ad6..53118a55 100644 --- a/src/swell/tasks/task_questions.yaml +++ b/src/swell/tasks/task_questions.yaml @@ -182,6 +182,16 @@ forecast_duration: - PrepGeosRunDir type: duration +generate_yaml_and_exit: + ask_question: false + default_value: false + prompt: Generate JEDI executable YAML and exit? + tasks: + - RunJediHofxExecutable + - RunJediUfoTestsExecutable + - RunJediVariationalExecutable + type: boolean + geos_background_restart_offset: ask_question: true default_value: defer_to_model From 6471d38d83fc40cbb87dbaa57ad30ee879898b57 Mon Sep 17 00:00:00 2001 From: Jianjun Jin <61251050+gmao-jjin3@users.noreply.github.com> Date: Wed, 12 Jul 2023 10:18:48 -0400 Subject: [PATCH 084/121] GMI_GPM full configuration for IODA v3 and x0048. (#187) * GMI_GPM full configuration for IODA v3 and x0048. * Code norm. * Code norm. --- .../geos_atmosphere/observations/gmi_gpm.yaml | 200 ++++++++++++++++-- 1 file changed, 177 insertions(+), 23 deletions(-) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/gmi_gpm.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/gmi_gpm.yaml index 26f162fb..51b4e9e6 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/gmi_gpm.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/gmi_gpm.yaml @@ -34,8 +34,17 @@ obs bias: - name: lapse_rate tlapse: *gmi_gpm_tlapse - name: emissivity -## - name: cosine_of_latitude_times_orbit_node -## - name: sine_of_latitude + - name: cloud_liquid_water + sensor: GMI_GPM + ch37v: 6 + ch37h: 7 + order: 2 + tlapse: *gmi_gpm_tlapse + - name: cloud_liquid_water + sensor: GMI_GPM + ch37v: 6 + ch37h: 7 + tlapse: *gmi_gpm_tlapse - name: scan_angle var_name: sensorScanPosition order: 4 @@ -57,7 +66,7 @@ obs filters: - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: 10,11,12,13 + channels: 10-13 minvalue: 70.0 maxvalue: 320.0 - filter: Domain Check @@ -74,28 +83,39 @@ obs filters: minvalue: -55.0 maxvalue: 55.0 - variable: - name: MetaData/height + name: MetaData/heightOfSurface maxvalue: 2000 - variable: name: GeoVaLs/water_area_fraction minvalue: 0.99 - variable: - name: GeoVaLs/surface_temperature_where_sea + name: GeoVaLs/average_surface_temperature_within_field_of_view minvalue: 275 - filter: BlackList filter variables: - name: brightnessTemperature - channels: 5,6,7,10,12,13 + channels: 1-13 where: - variable: name: MetaData/latitude - minvalue: -25.0 + minvalue: -20.0 maxvalue: 0.0 - variable: name: MetaData/longitude minvalue: 25.0 maxvalue: 40.0 -# CLW Retrieval Check from ObsValue +# Save CLW retrievals from ObsValue +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/clw_obs + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch37v: 6 + clwret_ch37h: 7 + clwret_types: [ObsValue] +# Check CLW retrievals from ObsValue - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -106,10 +126,8 @@ obs filters: clwret_ch37v: 6 clwret_ch37h: 7 clwret_types: [ObsValue] - maxvalue: 999.0 - action: - name: reject -# Ckeck CLW retrievals from Hofx + maxvalue: 900 +# Ckeck CLW retrievals from Hofx - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -120,42 +138,178 @@ obs filters: clwret_ch37v: 6 clwret_ch37h: 7 clwret_types: [HofX] - maxvalue: 999.0 + maxvalue: 900 +# Save CLW retrievals from HofX +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/clw_hofx + type: float + function: + name: ObsFunction/CLWRetMW + options: + clwret_ch37v: 6 + clwret_ch37h: 7 + clwret_types: [HofX] +# emiss_qc +- filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 1-13 + where: + - variable: + name: ObsFunction/CLWRetMW + options: + clwret_ch37v: 6 + clwret_ch37h: 7 + clwret_types: [ObsValue] + minvalue: 0.0 + maxvalue: 0.05 + - variable: + name: ObsFunction/Emissivity_Diff_GMI + options: + channel: 2 + regression_constant_1: 0.13290 + regression_constant_2: 0.42468 + regression_coeff_1: [-0.00548, 0.00772, 0.00530, -0.00425, + 0.00053, 0.00008, -0.00003, -0.00144, + 0.00059, -0.00016, 0.00003, -0.00011, + 0.00017] + regression_coeff_2: [0.00289, -0.00142] + minvalue: 0.01 + action: + name: reject +- filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 1-13 + where: + - variable: + name: ObsFunction/CLWRetMW + options: + clwret_ch37v: 6 + clwret_ch37h: 7 + clwret_types: [ObsValue] + minvalue: 0.0 + maxvalue: 0.05 + - variable: + name: ObsFunction/Emissivity_Diff_GMI + options: + channel: 4 + regression_constant_1: 0.15627 + regression_constant_2: 0.83807 + regression_coeff_1: [-0.01084, 0.01194, 0.01111, -0.00784, + 0.00060, 0.00008, -0.00003, -0.00248, + 0.00105, -0.00008, 0.00000, -0.00013, + 0.00016] + regression_coeff_2: [0.00048, -0.00207] + minvalue: 0.035 + action: + name: reject +- filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 1-13 + where: + - variable: + name: ObsFunction/CLWRetMW + options: + clwret_ch37v: 6 + clwret_ch37h: 7 + clwret_types: [ObsValue] + minvalue: 0.0 + maxvalue: 0.05 + - variable: + name: ObsFunction/Emissivity_Diff_GMI + options: + channel: 7 + regression_constant_1: 0.30306 + regression_constant_2: 1.24071 + regression_coeff_1: [-0.01793, 0.01730, 0.01784, -0.01199, + 0.00067, 0.00013, -0.00004, -0.00365, + 0.00154, -0.00004, -0.00001, -0.00015, + 0.00017] + regression_coeff_2: [0.00068, -0.00342] + minvalue: 0.05 action: name: reject # Assign observational error in all-sky DA. - filter: BlackList filter variables: - name: brightnessTemperature - channels: 5,6,7,10,12,13 + channels: 1-13 action: name: assign error error function: name: ObsFunction/ObsErrorModelRamp - channels: 5,6,7,10,12,13 + channels: 1-13 options: - channels: 5,6,7,10,12,13 + channels: 1-13 xvar: name: ObsFunction/CLWRetSymmetricMW options: clwret_ch37v: 6 clwret_ch37h: 7 clwret_types: [ObsValue, HofX] - x0: [ 0.050, 0.050, 0.050, + x0: [ 0.050, 0.050, 0.050, 0.050, 0.050, + 0.050, 0.050, 0.050, 0.050, 0.050, 0.050, 0.050, 0.050] - x1: [ 0.200, 0.200, 0.200, + x1: [ 0.200, 0.200, 0.200, 0.200, 0.200, + 0.200, 0.200, 0.200, 0.200, 0.300, 0.200, 0.300, 0.300] - x2: [ 0.500, 0.500, 0.500, + x2: [ 0.500, 0.500, 0.500, 0.500, 0.500, + 0.500, 0.500, 0.500, 0.500, 0.500, 0.500, 0.500, 0.500] - err0: [ 4.000, 3.800,300.000, + err0: [ 2.700, 3.700, 3.500, 4.500, 4.000, + 3.800,300.000, 5.000, 11.500, 5.000, 5.000, 2.500, 3.000] - err1: [11.000, 13.300, 23.000, + err1: [17.000, 23.000, 13.000, 25.000, 11.000, + 13.000, 23.000, 10.000, 20.000, 15.000, 20.000, 8.000, 13.000] - err2: [35.000, 25.300,500.000, + err2: [25.000, 40.000, 40.000, 55.000, 35.000, + 25.000,500.000, 50.000, 50.000, 50.000, 50.000, 30.000, 40.000] - filter: Background Check apply at iterations: 0, 1 filter variables: - name: brightnessTemperature - channels: 5,6,7,10,12,13 + channels: 1,2,4,6 + threshold: 2.0 + absolute threshold: 30.0 + action: + name: reject +- filter: Background Check + apply at iterations: 0, 1 + filter variables: + - name: brightnessTemperature + channels: 9,10,11 threshold: 2.0 + absolute threshold: 20.0 + action: + name: reject +- filter: Background Check + apply at iterations: 0, 1 + filter variables: + - name: brightnessTemperature + channels: 3,5,8 + threshold: 2.0 + absolute threshold: 15.0 + action: + name: reject +- filter: Background Check + apply at iterations: 0, 1 + filter variables: + - name: brightnessTemperature + channels: 12,13 + threshold: 2.0 + absolute threshold: 10.0 + action: + name: reject +- filter: Background Check + apply at iterations: 0, 1 + filter variables: + - name: brightnessTemperature + channels: 7 + threshold: 2.0 + absolute threshold: 5.0 + action: + name: reject From 3f17cd7eb3bf334b7022f08b28ea0c4d7c615ce7 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 12 Jul 2023 10:43:47 -0400 Subject: [PATCH 085/121] update data paths --- .../geos_atmosphere/observations/rass_tv.yaml | 14 -- .../observations/seviri_m11.yaml | 143 ------------ .../observations/ufo_tests.yaml | 208 +++++++++++++++++- src/swell/suites/ufo_testing/flow.cylc | 14 +- .../suites-ufo_testing-geos_atmosphere.yaml | 36 +-- .../ufo_testing/suites-ufo_testing.yaml | 10 +- 6 files changed, 223 insertions(+), 202 deletions(-) delete mode 100644 src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/rass_tv.yaml delete mode 100644 src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/seviri_m11.yaml diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/rass_tv.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/rass_tv.yaml deleted file mode 100644 index 4954bef3..00000000 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/rass_tv.yaml +++ /dev/null @@ -1,14 +0,0 @@ -obs operator: - name: VertInterp - observation alias file: '{{experiment_root}}/{{experiment_id}}/configuration/jedi/interfaces/{{model_component}}/observations/obsop_name_map.yaml' -obs space: - name: rass_tv - obsdatain: - engine: - type: H5File - obsfile: '{{cycle_dir}}/rass_tv.{{window_begin}}.nc4' - obsdataout: - engine: - type: H5File - obsfile: '{{cycle_dir}}/{{experiment_id}}.rass_tv.{{window_begin}}.nc4' - simulated variables: [virtualTemperature] diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/seviri_m11.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/seviri_m11.yaml deleted file mode 100644 index 3ff177cf..00000000 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/seviri_m11.yaml +++ /dev/null @@ -1,143 +0,0 @@ -obs space: - name: seviri_m11 - obsdatain: - engine: - type: H5File - obsfile: '{{cycle_dir}}/seviri_m11.{{window_begin}}.nc4' - obsdataout: - engine: - type: H5File - obsfile: '{{cycle_dir}}/{{experiment_id}}.seviri_m11.{{window_begin}}.nc4' - simulated variables: [brightnessTemperature] - channels: &seviri_m11_channels 4-11 -obs operator: - name: CRTM - Absorbers: [H2O,O3,CO2] - obs options: - Sensor_ID: &Sensor_ID seviri_m11 - EndianType: little_endian - CoefficientPath: '{{crtm_coeff_dir}}' -obs bias: - input file: '{{cycle_dir}}/seviri_m11.{{background_time}}.satbias.nc4' - variational bc: - predictors: - - name: constant - - name: lapse_rate - order: 2 - tlapse: &seviri_m11_tlapse '{{cycle_dir}}/seviri_m11.{{background_time}}.tlapse.txt' - - name: lapse_rate - tlapse: *seviri_m11_tlapse - - name: emissivity - - name: scan_angle - var_name: sensorScanPosition - order: 4 - - name: scan_angle - var_name: sensorScanPosition - order: 3 - - name: scan_angle - var_name: sensorScanPosition - order: 2 - - name: scan_angle - var_name: sensorScanPosition -obs filters: -# Observation Range Sanity Check -- filter: Bounds Check - filter variables: - - name: brightnessTemperature - channels: *seviri_m11_channels - minvalue: 50.00001 - maxvalue: 449.99999 - action: - name: reject -# Surface Check:use chn 2 and 3 over both sea and land while other IR chns only over sea -# ch2 and ch3 in GSI should be the original seviri ch5 and ch6 -- filter: BlackList - filter variables: - - name: brightnessTemperature - channels: 4,7-11 - where: - - variable: - name: GeoVaLs/water_area_fraction - maxvalue: 0.99 -# Do not use ch5,6 over snow -- filter: BlackList - filter variables: - - name: brightnessTemperature - channels: *seviri_m11_channels - where: - - variable: - name: GeoVaLs/surface_snow_area_fraction - minvalue: 0.01 -# Do not use ch5,6 over ice -- filter: BlackList - filter variables: - - name: brightnessTemperature - channels: *seviri_m11_channels - where: - - variable: - name: GeoVaLs/ice_area_fraction - minvalue: 0.01 -# Do not use over mixed surface -- filter: BlackList - filter variables: - - name: brightnessTemperature - channels: *seviri_m11_channels - where: - - variable: - name: GeoVaLs/land_area_fraction - maxvalue: 0.99 - - variable: - name: GeoVaLs/water_area_fraction - maxvalue: 0.99 - - variable: - name: GeoVaLs/ice_area_fraction - maxvalue: 0.99 - - variable: - name: GeoVaLs/surface_snow_area_fraction - maxvalue: 0.99 -# QC_terrain: If seviri and terrain height > 1km. do not use -- filter: Domain Check - filter variables: - - name: brightnessTemperature - channels: *seviri_m11_channels - where: - - variable: - name: GeoVaLs/surface_geopotential_height - maxvalue: 1000.0 -# Gross check -- filter: Background Check - filter variables: - - name: brightnessTemperature - channels: *seviri_m11_channels - absolute threshold: 2.0 - action: - name: reject -# Surface Jacobians Check -- filter: BlackList - filter variables: - - name: brightnessTemperature - channels: *seviri_m11_channels - action: - name: inflate error - inflation variable: - name: ObsFunction/ObsErrorFactorSurfJacobianRad - channels: *seviri_m11_channels - options: - channels: *seviri_m11_channels - sensor: *Sensor_ID - obserr_demisf: [0.01, 0.02, 0.02, 0.02, 0.02] - obserr_dtempf: [0.50, 2.00, 3.00, 3.00, 5.00] -# Useflag Check -- filter: Bounds Check - filter variables: - - name: brightnessTemperature - channels: *seviri_m11_channels - test variables: - - name: ObsFunction/ChannelUseflagCheckRad - channels: *seviri_m11_channels - options: - channels: *seviri_m11_channels - use_flag: [ -1, 1, 1, -1, -1, -1, -1, -1 ] - minvalue: 1.0e-12 - action: - name: reject diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ufo_tests.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ufo_tests.yaml index 4f53906b..8be05e62 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ufo_tests.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ufo_tests.yaml @@ -1,3 +1,6 @@ +# Default values applied to all observation types if not overridden below +# ----------------------------------------------------------------------- + default: operator_test: linear obs operator test: @@ -9,8 +12,209 @@ default: filter_test: passedBenchmark: 1 +# Values for individual observation types +# --------------------------------------- + aircraft: operator_test: - rms ref: 1.9724833875646487e+38 + rms ref: 1.0e-1 + filter_test: + passedBenchmark: 1 + +airs_aqua: + operator_test: + rms ref: 1.0e-1 + filter_test: + passedBenchmark: 1 + +amsr2_gcom-w1: + operator_test: + rms ref: 1.0e-1 + filter_test: + passedBenchmark: 1 + +amsua_aqua: + operator_test: + rms ref: 1.0e-1 + filter_test: + passedBenchmark: 1 + +amsua_metop-a: + operator_test: + rms ref: 1.0e-1 + filter_test: + passedBenchmark: 1 + +amsua_metop-b: + operator_test: + rms ref: 1.0e-1 + filter_test: + passedBenchmark: 1 + +amsua_metop-c: + operator_test: + rms ref: 1.0e-1 + filter_test: + passedBenchmark: 1 + +amsua_n15: + operator_test: + rms ref: 1.0e-1 + filter_test: + passedBenchmark: 1 + +amsua_n18: + operator_test: + rms ref: 1.0e-1 filter_test: - passedBenchmark: 297273 + passedBenchmark: 1 + +amsua_n19: + operator_test: + rms ref: 1.0e-1 + filter_test: + passedBenchmark: 1 + +atms_n20: + operator_test: + rms ref: 1.0e-1 + filter_test: + passedBenchmark: 1 + +atms_npp: + operator_test: + rms ref: 1.0e-1 + filter_test: + passedBenchmark: 1 + +avhrr3_metop-a: + operator_test: + rms ref: 1.0e-1 + filter_test: + passedBenchmark: 1 + +avhrr3_n18: + operator_test: + rms ref: 1.0e-1 + filter_test: + passedBenchmark: 1 + +cris-fsr_n20: + operator_test: + rms ref: 1.0e-1 + filter_test: + passedBenchmark: 1 + +cris-fsr_npp: + operator_test: + rms ref: 1.0e-1 + filter_test: + passedBenchmark: 1 + +gmi_gpm: + operator_test: + rms ref: 1.0e-1 + filter_test: + passedBenchmark: 1 + +gnssrobndnbam: + operator_test: + rms ref: 1.0e-1 + filter_test: + passedBenchmark: 1 + +iasi_metop-a: + operator_test: + rms ref: 1.0e-1 + filter_test: + passedBenchmark: 1 + +iasi_metop-b: + operator_test: + rms ref: 1.0e-1 + filter_test: + passedBenchmark: 1 + +mhs_metop-b: + operator_test: + rms ref: 1.0e-1 + filter_test: + passedBenchmark: 1 + +mhs_metop-c: + operator_test: + rms ref: 1.0e-1 + filter_test: + passedBenchmark: 1 + +mhs_n19: + operator_test: + rms ref: 1.0e-1 + filter_test: + passedBenchmark: 1 + +mls55_aura: + operator_test: + rms ref: 1.0e-1 + filter_test: + passedBenchmark: 1 + +omi_aura: + operator_test: + rms ref: 1.0e-1 + filter_test: + passedBenchmark: 1 + +ompslpnc_npp: + operator_test: + rms ref: 1.0e-1 + filter_test: + passedBenchmark: 1 + +ompsnm_npp: + operator_test: + rms ref: 1.0e-1 + filter_test: + passedBenchmark: 1 + +satwind: + operator_test: + rms ref: 1.0e-1 + filter_test: + passedBenchmark: 1 + +scatwind: + operator_test: + rms ref: 1.0e-1 + filter_test: + passedBenchmark: 1 + +sfcship: + operator_test: + rms ref: 1.0e-1 + filter_test: + passedBenchmark: 1 + +sfc: + operator_test: + rms ref: 1.0e-1 + filter_test: + passedBenchmark: 1 + +sondes: + operator_test: + rms ref: 1.0e-1 + filter_test: + passedBenchmark: 1 + +ssmis_f17: + operator_test: + rms ref: 1.0e-1 + filter_test: + passedBenchmark: 1 + +vadwind: + operator_test: + rms ref: 1.0e-1 + filter_test: + passedBenchmark: 1 diff --git a/src/swell/suites/ufo_testing/flow.cylc b/src/swell/suites/ufo_testing/flow.cylc index fe98aef5..1bec5134 100644 --- a/src/swell/suites/ufo_testing/flow.cylc +++ b/src/swell/suites/ufo_testing/flow.cylc @@ -109,17 +109,11 @@ [[ GsiBcToIoda ]] script = "swell_task GsiBcToIoda $config -d $datetime -m geos_atmosphere" -# [[ GetGsiNcdiag ]] -# script = "swell_task GetGsiNcdiag $config -d $datetime -m geos_atmosphere" -# -# [[ GsiNcdiagToIoda ]] -# script = "swell_task GsiNcdiagToIoda $config -d $datetime -m geos_atmosphere" - - [[ GetObservations ]] - script = "swell_task GetObservations $config -d $datetime -m geos_atmosphere" + [[ GetGsiNcdiag ]] + script = "swell_task GetGsiNcdiag $config -d $datetime -m geos_atmosphere" - [[ GetGeovals ]] - script = "swell_task GetGeovals $config -d $datetime -m geos_atmosphere" + [[ GsiNcdiagToIoda ]] + script = "swell_task GsiNcdiagToIoda $config -d $datetime -m geos_atmosphere" [[RunJediTestObsFiltersExecutable]] script = "swell_task RunJediTestObsFiltersExecutable $config -d $datetime -m geos_atmosphere" diff --git a/src/swell/suites/ufo_testing/geos_atmosphere/suites-ufo_testing-geos_atmosphere.yaml b/src/swell/suites/ufo_testing/geos_atmosphere/suites-ufo_testing-geos_atmosphere.yaml index 2ff61a30..c04efbbf 100644 --- a/src/swell/suites/ufo_testing/geos_atmosphere/suites-ufo_testing-geos_atmosphere.yaml +++ b/src/swell/suites/ufo_testing/geos_atmosphere/suites-ufo_testing-geos_atmosphere.yaml @@ -1,8 +1,8 @@ # Cycle frequency cycle_times: - default_value: ['T00', 'T06', 'T12', 'T18'] + default_value: ['T00'] prompt: List all cycle times, by middle of the window? - options: ['T00', 'T06', 'T12', 'T18'] + options: ['T00'] type: string-check-list # Window length @@ -14,7 +14,10 @@ window_length: # Observations observations: default_value: - - aircraft + - gmi_gpm + - sondes + +# - aircraft # - airs_aqua # - amsr2_gcom-w1 # - amsua_aqua @@ -45,7 +48,6 @@ observations: # - seviri_m08 # - sfcship # - sfc -# - sondes # - ssmis_f17 # - vadwind prompt: Select observations to run with. @@ -54,30 +56,19 @@ observations: # Path to GSI ncdiags path_to_gsi_diags: - default_value: '/archive/u/jjin3/x0044.jj_20230201/obs/Y2020/M12/D15/H00' + default_value: '/discover/nobackup/drholdaw/SwellTestData/ufo_testing/ncdiag' prompt: Path to where the ncdiags will be held type: string # Compute GeoVaLs and observations produce_geovals: - default_value: false + default_value: true prompt: Compute GeoVaLs as well as observations? type: boolean -# Experiment that provide observations -obs_experiment: - default_value: x0044_test_obs_filters - prompt: Experiment providing the Observations - type: string - -# Experiment that provide geovals -geovals_experiment: - default_value: x0044_test_obs_filters_geovals - prompt: Experiment providing the GeoVaLs - type: string - +# Location for bias correction coefficients gsi_bc_location: - default_value: /discover/nobackup/projects/gmao/dadev/rtodling/archive/x0044/rs/Y2020/M12/x0044.rst.20201214_21z.tar + default_value: '/discover/nobackup/drholdaw/SwellTestData/ufo_testing/bc' prompt: Tar or directory containing BC files type: string @@ -113,11 +104,6 @@ fixed_options: background_time_offset: default_value: PT9H prompt: Time before the middle of the window that the background providing forecast began - obs_provider: - default_value: ncdiag - prompt: Database providing the Observations - geovals_provider: - default_value: ncdiag - prompt: Database providing the GeoVaLs + diff --git a/src/swell/suites/ufo_testing/suites-ufo_testing.yaml b/src/swell/suites/ufo_testing/suites-ufo_testing.yaml index 030ee5a8..b428d7c1 100644 --- a/src/swell/suites/ufo_testing/suites-ufo_testing.yaml +++ b/src/swell/suites/ufo_testing/suites-ufo_testing.yaml @@ -25,7 +25,7 @@ jedi_build_method: # Existing JEDI bundle directory existing_jedi_source_directory: - default_value: '/discover/nobackup/drholdaw/JediDev/ufo-work' + default_value: {{existing_jedi_source_directory}} prompt: Provide the path to an existing JEDI bundle directory containing source code repos type: string depends: @@ -34,19 +34,13 @@ existing_jedi_source_directory: # Existing JEDI build existing_jedi_build_directory: - default_value: '/discover/nobackup/drholdaw/JediDev/ufo-work/build-intel-release' + default_value: {{existing_jedi_build_directory}} prompt: Provide the path to an existing JEDI build directory type: string depends: key: jedi_build_method value: use_existing -# R2D2 Configuration -r2d2_local_path: - default_value: {{r2d2_local_path}} - prompt: Enter the path where R2D2 will store experiment output - type: string - # Models to use model_components: default_value: ['geos_atmosphere'] From b05fe3652618f3b5dea49d079bb11f9a4b78786e Mon Sep 17 00:00:00 2001 From: danholdaway Date: Fri, 14 Jul 2023 13:48:39 -0400 Subject: [PATCH 086/121] all tasks running smoothly --- .github/workflows/discover_nightly.yml | 2 +- .../observations/aircraft.yaml | 4 +-- .../observations/ufo_tests.yaml | 4 +-- src/swell/deployment/prep_suite.py | 14 ++++---- .../suites/build_jedi/suites-build_jedi.yaml | 2 +- src/swell/suites/ufo_testing/flow.cylc | 18 +++++----- .../geos_atmosphere/eva_observations.yaml | 10 ++++++ .../suites-ufo_testing-geos_atmosphere.yaml | 11 +++---- src/swell/tasks/build_jedi.py | 4 +-- src/swell/tasks/get_gsi_bc.py | 9 ++--- src/swell/tasks/get_gsi_ncdiag.py | 14 ++++++++ src/swell/tasks/gsi_bc_to_ioda.py | 27 ++++++++++++--- src/swell/tasks/gsi_ncdiag_to_ioda.py | 33 ++++++++++++++++--- 13 files changed, 105 insertions(+), 47 deletions(-) diff --git a/.github/workflows/discover_nightly.yml b/.github/workflows/discover_nightly.yml index 92abfca8..b9eaec20 100644 --- a/.github/workflows/discover_nightly.yml +++ b/.github/workflows/discover_nightly.yml @@ -15,4 +15,4 @@ defaults: jobs: run-discover-nightly: - uses: GEOS-ESM/CI-workflows/.github/workflows/swell-discover-nightly.yml@main + uses: GEOS-ESM/CI-workflows/.github/workflows/swell-discover-nightly.yml@feature/ufo_tests diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/aircraft.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/aircraft.yaml index 1f93cf2f..26f86c12 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/aircraft.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/aircraft.yaml @@ -523,7 +523,7 @@ obs filters: filter variables: - name: windEastward function absolute threshold: - - name: ObsFunction/SatWindsSPDBCheck + - name: ObsFunction/WindsSPDBCheck options: wndtype: [ 230, 231, 232, 233, 234, 235 ] cgross: [ 6.0, 6.5, 7.0, 7.5, 7.5, 7.5 ] @@ -539,7 +539,7 @@ obs filters: filter variables: - name: windNorthward function absolute threshold: - - name: ObsFunction/SatWindsSPDBCheck + - name: ObsFunction/WindsSPDBCheck options: wndtype: [ 230, 231, 232, 233, 234, 235 ] cgross: [ 6.0, 6.5, 7.0, 7.5, 7.5, 7.5 ] diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ufo_tests.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ufo_tests.yaml index 8be05e62..12d8f64b 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ufo_tests.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ufo_tests.yaml @@ -19,7 +19,7 @@ aircraft: operator_test: rms ref: 1.0e-1 filter_test: - passedBenchmark: 1 + passedBenchmark: 341802 airs_aqua: operator_test: @@ -115,7 +115,7 @@ gmi_gpm: operator_test: rms ref: 1.0e-1 filter_test: - passedBenchmark: 1 + passedBenchmark: 41937 gnssrobndnbam: operator_test: diff --git a/src/swell/deployment/prep_suite.py b/src/swell/deployment/prep_suite.py index e7a10e41..f13a5e74 100644 --- a/src/swell/deployment/prep_suite.py +++ b/src/swell/deployment/prep_suite.py @@ -139,13 +139,13 @@ def prepare_cylc_suite_jinja2(logger, swell_suite_path, exp_suite_path, experime render_dictionary['scheduling']['RunGeosExecutable']['ntasks_per_node'] = 24 render_dictionary['scheduling']['RunGeosExecutable']['constraint'] = 'cas|sky|hasw' - render_dictionary['scheduling']['RunJediTestObsFiltersExecutable'] = {} - render_dictionary['scheduling']['RunJediTestObsFiltersExecutable']['execution_time_limit'] = 'PT30M' # noqa - render_dictionary['scheduling']['RunJediTestObsFiltersExecutable']['account'] = 'g0613' - render_dictionary['scheduling']['RunJediTestObsFiltersExecutable']['qos'] = 'allnccs' - render_dictionary['scheduling']['RunJediTestObsFiltersExecutable']['nodes'] = 1 - render_dictionary['scheduling']['RunJediTestObsFiltersExecutable']['ntasks_per_node'] = 1 - render_dictionary['scheduling']['RunJediTestObsFiltersExecutable']['constraint'] = 'cas' + render_dictionary['scheduling']['RunJediUfoTestsExecutable'] = {} + render_dictionary['scheduling']['RunJediUfoTestsExecutable']['execution_time_limit'] = 'PT30M' # noqa + render_dictionary['scheduling']['RunJediUfoTestsExecutable']['account'] = 'g0613' + render_dictionary['scheduling']['RunJediUfoTestsExecutable']['qos'] = 'allnccs' + render_dictionary['scheduling']['RunJediUfoTestsExecutable']['nodes'] = 1 + render_dictionary['scheduling']['RunJediUfoTestsExecutable']['ntasks_per_node'] = 1 + render_dictionary['scheduling']['RunJediUfoTestsExecutable']['constraint'] = 'cas' # Render the template # ------------------- diff --git a/src/swell/suites/build_jedi/suites-build_jedi.yaml b/src/swell/suites/build_jedi/suites-build_jedi.yaml index eab54fb3..051cd7e9 100644 --- a/src/swell/suites/build_jedi/suites-build_jedi.yaml +++ b/src/swell/suites/build_jedi/suites-build_jedi.yaml @@ -25,7 +25,7 @@ existing_jedi_build_directory: # Bundles to build bundles: - default_value: ['fv3-jedi', 'soca', 'iodaconv'] + default_value: ['fv3-jedi', 'soca', 'iodaconv', 'ufo'] prompt: List the bundles that you wish to build type: string-check-list options: use_method # This method would ask jedi_bundle for the default bundles diff --git a/src/swell/suites/ufo_testing/flow.cylc b/src/swell/suites/ufo_testing/flow.cylc index 1bec5134..11b51358 100644 --- a/src/swell/suites/ufo_testing/flow.cylc +++ b/src/swell/suites/ufo_testing/flow.cylc @@ -115,17 +115,17 @@ [[ GsiNcdiagToIoda ]] script = "swell_task GsiNcdiagToIoda $config -d $datetime -m geos_atmosphere" - [[RunJediTestObsFiltersExecutable]] - script = "swell_task RunJediTestObsFiltersExecutable $config -d $datetime -m geos_atmosphere" + [[RunJediUfoTestsExecutable]] + script = "swell_task RunJediUfoTestsExecutable $config -d $datetime -m geos_atmosphere" platform = {{platform}} - execution time limit = {{scheduling["RunJediTestObsFiltersExecutable"]["execution_time_limit"]}} + execution time limit = {{scheduling["RunJediUfoTestsExecutable"]["execution_time_limit"]}} [[[directives]]] - --account = {{scheduling["RunJediTestObsFiltersExecutable"]["account"]}} - --qos = {{scheduling["RunJediTestObsFiltersExecutable"]["qos"]}} - --job-name = RunJediTestObsFiltersExecutable - --nodes={{scheduling["RunJediTestObsFiltersExecutable"]["nodes"]}} - --ntasks-per-node={{scheduling["RunJediTestObsFiltersExecutable"]["ntasks_per_node"]}} - --constraint={{scheduling["RunJediTestObsFiltersExecutable"]["constraint"]}} + --account = {{scheduling["RunJediUfoTestsExecutable"]["account"]}} + --qos = {{scheduling["RunJediUfoTestsExecutable"]["qos"]}} + --job-name = RunJediUfoTestsExecutable + --nodes={{scheduling["RunJediUfoTestsExecutable"]["nodes"]}} + --ntasks-per-node={{scheduling["RunJediUfoTestsExecutable"]["ntasks_per_node"]}} + --constraint={{scheduling["RunJediUfoTestsExecutable"]["constraint"]}} [[EvaObservations]] script = "swell_task EvaObservations $config -d $datetime -m geos_atmosphere" diff --git a/src/swell/suites/ufo_testing/geos_atmosphere/eva_observations.yaml b/src/swell/suites/ufo_testing/geos_atmosphere/eva_observations.yaml index 8ac0670a..904bd387 100644 --- a/src/swell/suites/ufo_testing/geos_atmosphere/eva_observations.yaml +++ b/src/swell/suites/ufo_testing/geos_atmosphere/eva_observations.yaml @@ -280,6 +280,7 @@ diagnostics: # ------------------------------------------------------- - batch figure: variables: *variables + channels: *channels figure: layout: [1,1] title: 'Observation minus Background Density (Passing QC) | {{instrument_title}} | ${variable_title}' @@ -292,6 +293,7 @@ diagnostics: statistics: fields: - field_name: experiment::ObsValueMinusGsiHofXPassedQc::${variable} + channel: ${channel} xloc: 0.5 yloc: -0.10 kwargs: @@ -299,6 +301,7 @@ diagnostics: fontsize: 8 fontfamily: monospace - field_name: experiment::ObsValueMinusHofxPassedQc::${variable} + channel: ${channel} xloc: 0.5 yloc: -0.13 kwargs: @@ -315,6 +318,7 @@ diagnostics: - type: Density data: variable: experiment::ObsValueMinusGsiHofXPassedQc::${variable} + channel: ${channel} color: 'blue' label: 'GSI omb (Passed QC)' alpha: 0.5 @@ -322,6 +326,7 @@ diagnostics: - type: Density data: variable: experiment::ObsValueMinusHofxPassedQc::${variable} + channel: ${channel} color: 'red' label: 'JEDI omb (Passed QC)' alpha: 0.5 @@ -332,6 +337,7 @@ diagnostics: # ------------------------------------------------------------------------ - batch figure: variables: *variables + channels: *channels figure: layout: [1,1] title: 'Observation minus Background Density (Passing QC) | {{instrument_title}} | ${variable_title}' @@ -344,6 +350,7 @@ diagnostics: statistics: fields: - field_name: experiment::ObsValueMinusGsiHofXBcPassedQc::${variable} + channel: ${channel} xloc: 0.5 yloc: -0.10 kwargs: @@ -351,6 +358,7 @@ diagnostics: fontsize: 8 fontfamily: monospace - field_name: experiment::ObsValueMinusHofxPassedQc::${variable} + channel: ${channel} xloc: 0.5 yloc: -0.13 kwargs: @@ -367,6 +375,7 @@ diagnostics: - type: Density data: variable: experiment::ObsValueMinusGsiHofXBcPassedQc::${variable} + channel: ${channel} color: 'blue' label: 'GSI omb BC (Passed QC)' alpha: 0.5 @@ -374,6 +383,7 @@ diagnostics: - type: Density data: variable: experiment::ObsValueMinusHofxPassedQc::${variable} + channel: ${channel} color: 'red' label: 'JEDI omb (Passed QC)' alpha: 0.5 diff --git a/src/swell/suites/ufo_testing/geos_atmosphere/suites-ufo_testing-geos_atmosphere.yaml b/src/swell/suites/ufo_testing/geos_atmosphere/suites-ufo_testing-geos_atmosphere.yaml index c04efbbf..3c93986d 100644 --- a/src/swell/suites/ufo_testing/geos_atmosphere/suites-ufo_testing-geos_atmosphere.yaml +++ b/src/swell/suites/ufo_testing/geos_atmosphere/suites-ufo_testing-geos_atmosphere.yaml @@ -14,10 +14,7 @@ window_length: # Observations observations: default_value: - - gmi_gpm - - sondes - -# - aircraft + - aircraft # - airs_aqua # - amsr2_gcom-w1 # - amsua_aqua @@ -55,7 +52,7 @@ observations: type: string-check-list # Path to GSI ncdiags -path_to_gsi_diags: +path_to_gsi_nc_diags: default_value: '/discover/nobackup/drholdaw/SwellTestData/ufo_testing/ncdiag' prompt: Path to where the ncdiags will be held type: string @@ -67,7 +64,7 @@ produce_geovals: type: boolean # Location for bias correction coefficients -gsi_bc_location: +path_to_gsi_bc_coefficients: default_value: '/discover/nobackup/drholdaw/SwellTestData/ufo_testing/bc' prompt: Tar or directory containing BC files type: string @@ -92,6 +89,8 @@ fixed_options: - gsi_bcs/*.yaml - gsi_bcs - gsi_ncdiags/*.nc4 + - gsi_ncdiags/aircraft/*.nc4 + - gsi_ncdiags/aircraft - gsi_ncdiags prompt: 'Patterns for the files to remove after completing a cycle' # Needed because of config gen diff --git a/src/swell/tasks/build_jedi.py b/src/swell/tasks/build_jedi.py index 472dd4a4..7a7ad40b 100644 --- a/src/swell/tasks/build_jedi.py +++ b/src/swell/tasks/build_jedi.py @@ -10,10 +10,10 @@ import os -from jedi_bundle.bin.jedi_bundle import execute_tasks +from jedi_bundle.bin.jedi_bundle import execute_tasks, get_bundles from swell.tasks.base.task_base import taskBase -from swell.utilities.build import set_jedi_bundle_config, get_bundles, build_and_source_dirs +from swell.utilities.build import set_jedi_bundle_config, build_and_source_dirs # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/get_gsi_bc.py b/src/swell/tasks/get_gsi_bc.py index 7792b415..a5b31ec6 100644 --- a/src/swell/tasks/get_gsi_bc.py +++ b/src/swell/tasks/get_gsi_bc.py @@ -37,14 +37,9 @@ def execute(self): if os.path.isdir(gsi_bc_location): - print(gsi_bc_location) - # Get list of matching files in the directory - ana_satbias_rst = glob.glob(os.path.join(gsi_bc_location, '*ana_satbias_rst*')) - ana_satbiaspc_rst = glob.glob(os.path.join(gsi_bc_location, '*ana_satbiaspc_rst*')) - - print(ana_satbias_rst) - print(ana_satbiaspc_rst) + ana_satbias_rst = glob.glob(os.path.join(gsi_bc_location, '*.ana.satbias.*')) + ana_satbiaspc_rst = glob.glob(os.path.join(gsi_bc_location, '*.ana.satbiaspc.*')) # Record that files were found if len(ana_satbias_rst) == 1 and len(ana_satbiaspc_rst) == 1: diff --git a/src/swell/tasks/get_gsi_ncdiag.py b/src/swell/tasks/get_gsi_ncdiag.py index 38e622a7..0cdeb19e 100644 --- a/src/swell/tasks/get_gsi_ncdiag.py +++ b/src/swell/tasks/get_gsi_ncdiag.py @@ -59,5 +59,19 @@ def execute(self): f'{gsi_diag_file_target}') os.symlink(gsi_diag_path_file, gsi_diag_file_target) + # Create another directory to hold the aircraft data + # -------------------------------------------------- + prof_files = glob.glob(os.path.join(gsi_diag_dir, '*prof*.nc4')) + + for prof_file in prof_files: + os.makedirs(os.path.join(gsi_diag_dir, 'aircraft'), 0o755, exist_ok=True) + + # Replace _prof with nothing + file_name = os.path.basename(prof_file) + file_name = file_name.replace('_prof', '') + + # Move the file into the prof directory + os.rename(prof_file, os.path.join(gsi_diag_dir, 'aircraft', file_name)) + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/gsi_bc_to_ioda.py b/src/swell/tasks/gsi_bc_to_ioda.py index 723ff292..d0ad95a6 100644 --- a/src/swell/tasks/gsi_bc_to_ioda.py +++ b/src/swell/tasks/gsi_bc_to_ioda.py @@ -75,9 +75,9 @@ def execute(self): # Add the files for bc_file in bc_files: - if 'ana_satbias_rst' in bc_file: + if '.ana.satbias.' in bc_file: satbias_converter_dict['input coeff file'] = bc_file - if 'ana_satbiaspc_rst' in bc_file: + if '.ana.satbiaspc.' in bc_file: satbias_converter_dict['input err file'] = bc_file # Add the default predictors @@ -95,6 +95,20 @@ def execute(self): default_predictors.append('scan_angle_order_2') default_predictors.append('scan_angle') + gmi_gpm_predictors = [] + gmi_gpm_predictors.append('constant') + gmi_gpm_predictors.append('zenith_angle') + gmi_gpm_predictors.append('cosine_of_latitude_times_orbit_node') + gmi_gpm_predictors.append('lapse_rate_order_2') + gmi_gpm_predictors.append('lapse_rate') + gmi_gpm_predictors.append('cloud_liquid_water_order_2') + gmi_gpm_predictors.append('cloud_liquid_water') + gmi_gpm_predictors.append('emissivity') + gmi_gpm_predictors.append('scan_angle_order_4') + gmi_gpm_predictors.append('scan_angle_order_3') + gmi_gpm_predictors.append('scan_angle_order_2') + gmi_gpm_predictors.append('scan_angle') + satbias_converter_dict['default predictors'] = default_predictors satbias_converter_dict_output = [] @@ -102,7 +116,10 @@ def execute(self): output_dict = {} output_dict['sensor'] = sensor output_dict['output file'] = os.path.join(self.cycle_dir(), sensor_satbias) - output_dict['predictors'] = default_predictors + if sensor == 'gmi_gpm': + output_dict['predictors'] = gmi_gpm_predictors + else: + output_dict['predictors'] = default_predictors satbias_converter_dict_output.append(output_dict) satbias_converter_dict['output'] = satbias_converter_dict_output @@ -120,10 +137,10 @@ def execute(self): # Run tlapse converter (just a grep) for sensor, sensor_tlapse in zip(sensors, sensors_tlapse): sensor_tlapse_file = '' - with open(satbias_converter_dict['input err file'], 'r') as f: + with open(satbias_converter_dict['input coeff file'], 'r') as f: for line in f.readlines(): if sensor in line: - sensor_tlapse_file = sensor_tlapse_file + ' '.join(line.split()[1:]) + '\n' + sensor_tlapse_file = sensor_tlapse_file + ' '.join(line.split()[1:4]) + '\n' # Write to tlapse file with open(os.path.join(self.cycle_dir(), sensor_tlapse), 'w') as file_open: file_open.write(sensor_tlapse_file) diff --git a/src/swell/tasks/gsi_ncdiag_to_ioda.py b/src/swell/tasks/gsi_ncdiag_to_ioda.py index 0f78b7de..1cb68f93 100644 --- a/src/swell/tasks/gsi_ncdiag_to_ioda.py +++ b/src/swell/tasks/gsi_ncdiag_to_ioda.py @@ -8,6 +8,7 @@ # -------------------------------------------------------------------------------------------------- +import copy import datetime import glob import os @@ -44,7 +45,20 @@ def execute(self): # Assemble all conventional types that ioda considers # --------------------------------------------------- - gsi_to_ioda_dict = gsid.conv_platforms + gsi_to_ioda_dict = copy.copy(gsid.conv_platforms) + + # Remove aircraft from the lists and add new dictionaries for aircraft + # -------------------------------------------------------------------- + aircraft_is_in_prof_files = True + + if aircraft_is_in_prof_files: + for key, value in gsi_to_ioda_dict.items(): + gsi_to_ioda_dict[key] = [x for x in value if x not in ['aircraft']] + + if 'aircraft' in observations: + gsi_to_ioda_dict['conv_prof_t'] = ['aircraft'] + gsi_to_ioda_dict['conv_prof_uv'] = ['aircraft'] + ioda_types = [] for key in gsi_to_ioda_dict: ioda_types += gsi_to_ioda_dict[key] @@ -87,9 +101,18 @@ def execute(self): self.logger.info(log_str) self.logger.info('-'*len(log_str)) + # If prof in the name then it is aircraft data. Adjust path and rename + if 'prof' in gsi_type_to_process: + gsi_type_to_process_actual = gsi_type_to_process.replace('_prof', '') + extra_path = 'aircraft' + else: + gsi_type_to_process_actual = gsi_type_to_process + extra_path = '' + + # Path to search to GSI ncdiag files - path_to_search = os.path.join(gsi_diag_dir, - f'*{gsi_type_to_process}_*{gsi_datetime_str}*') + path_to_search = os.path.join(gsi_diag_dir, extra_path, + f'*{gsi_type_to_process_actual}*{gsi_datetime_str}*') # Get the list of files gsi_conv_file = glob.glob(path_to_search) @@ -104,7 +127,7 @@ def execute(self): # Assemble list of needed platforms needed_platforms = [] - for platform in gsid.conv_platforms[gsi_type_to_process]: + for platform in gsid.conv_platforms[gsi_type_to_process_actual]: if platform in needed_ioda_types: needed_platforms.append(platform) @@ -113,7 +136,7 @@ def execute(self): if produce_geovals: self.logger.info('', wrap=False) - self.logger.info(f'Processing GeoVaLs from {gsi_type_to_process}') + self.logger.info(f'Processing GeoVaLs from {gsi_type_to_process_actual}') Diag.toGeovals(self.cycle_dir()) Diag.close() From 6cd4b5766726bec46a2867ff5e093abf6a427a05 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Fri, 14 Jul 2023 15:49:36 -0400 Subject: [PATCH 087/121] rename workflows --- .github/workflows/discover-tier1-hofx.yml | 14 -------------- .github/workflows/discover_nightly.yml | 18 ------------------ .../workflows/tier1_application_discover.yml | 12 ++++++++++++ .../workflows/tier2_application_discover.yml | 16 ++++++++++++++++ 4 files changed, 28 insertions(+), 32 deletions(-) delete mode 100644 .github/workflows/discover-tier1-hofx.yml delete mode 100644 .github/workflows/discover_nightly.yml create mode 100644 .github/workflows/tier1_application_discover.yml create mode 100644 .github/workflows/tier2_application_discover.yml diff --git a/.github/workflows/discover-tier1-hofx.yml b/.github/workflows/discover-tier1-hofx.yml deleted file mode 100644 index 83c841b9..00000000 --- a/.github/workflows/discover-tier1-hofx.yml +++ /dev/null @@ -1,14 +0,0 @@ -# SWELL Discover Tier1 HOFX - -name: swell-discover-tier1-hofx - -on: workflow_dispatch - -defaults: - run: - shell: bash - -jobs: - - swell-discover-tier1-hofx: - uses: GEOS-ESM/CI-workflows/.github/workflows/swell-discover-tier1-hofx.yml@main diff --git a/.github/workflows/discover_nightly.yml b/.github/workflows/discover_nightly.yml deleted file mode 100644 index b9eaec20..00000000 --- a/.github/workflows/discover_nightly.yml +++ /dev/null @@ -1,18 +0,0 @@ -# Discover Nightly - -name: discover-nightly - -on: - workflow_dispatch: - - schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 0 * * *' - -defaults: - run: - shell: bash - -jobs: - run-discover-nightly: - uses: GEOS-ESM/CI-workflows/.github/workflows/swell-discover-nightly.yml@feature/ufo_tests diff --git a/.github/workflows/tier1_application_discover.yml b/.github/workflows/tier1_application_discover.yml new file mode 100644 index 00000000..e3b416fa --- /dev/null +++ b/.github/workflows/tier1_application_discover.yml @@ -0,0 +1,12 @@ +name: Tier 1 Applications Tests (Discover) + +on: workflow_dispatch + +defaults: + run: + shell: bash + +jobs: + + tier1-application-discover: + uses: GEOS-ESM/CI-workflows/.github/workflows/swell-tier1_application_discover.yml@feature/ufo_tests diff --git a/.github/workflows/tier2_application_discover.yml b/.github/workflows/tier2_application_discover.yml new file mode 100644 index 00000000..972b9e27 --- /dev/null +++ b/.github/workflows/tier2_application_discover.yml @@ -0,0 +1,16 @@ +wname: Tier 2 Applications Tests (Discover) + +on: + workflow_dispatch: + + schedule: + - cron: '0 0 * * *' + +defaults: + run: + shell: bash + +jobs: + tier2-application-discover: + uses: GEOS-ESM/CI-workflows/.github/workflows/swell-tier2_application_discover.yml@feature/ufo_tests + From 61f124b42532d6c2ba6a6830802df37671f5a35e Mon Sep 17 00:00:00 2001 From: danholdaway Date: Fri, 14 Jul 2023 17:24:33 -0400 Subject: [PATCH 088/121] coding norms --- src/swell/tasks/gsi_ncdiag_to_ioda.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/swell/tasks/gsi_ncdiag_to_ioda.py b/src/swell/tasks/gsi_ncdiag_to_ioda.py index 1cb68f93..934ff238 100644 --- a/src/swell/tasks/gsi_ncdiag_to_ioda.py +++ b/src/swell/tasks/gsi_ncdiag_to_ioda.py @@ -109,7 +109,6 @@ def execute(self): gsi_type_to_process_actual = gsi_type_to_process extra_path = '' - # Path to search to GSI ncdiag files path_to_search = os.path.join(gsi_diag_dir, extra_path, f'*{gsi_type_to_process_actual}*{gsi_datetime_str}*') @@ -158,9 +157,9 @@ def execute(self): ioda_path_files = glob.glob(os.path.join(self.cycle_dir(), ioda_type_pattern)) # Check that there are some files to combine - self.logger.assert_abort(len(ioda_path_files)> 0, f'In combine of {needed_ioda_type} ' + - f'no files where found. Ensure that the converter worked as ' + - f'expected.') + self.logger.assert_abort(len(ioda_path_files) > 0, f'In combine of ' + + f'{needed_ioda_type} no files where found. Ensure that ' + + f'the converter worked as expected.') # Get last file (first could be type_obs_ if the code already ran) ioda_file_0 = os.path.basename(ioda_path_files[-1]) From 82ef57052cf34b3e4b020454a256159818c49e63 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Fri, 14 Jul 2023 17:25:57 -0400 Subject: [PATCH 089/121] yaml norms --- .../jedi/interfaces/geos_atmosphere/model/analysis.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/analysis.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/analysis.yaml index f7699578..1fce43da 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/analysis.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/analysis.yaml @@ -1,6 +1,6 @@ filetype: cube sphere history provider: geos -datapath: {{cycle_dir}} -filename: {{experiment_id}}.analysis.%yyyy%mm%dd_%hh%MM%ssz.nc4 +datapath: '{{cycle_dir}}' +filename: '{{experiment_id}}.analysis.%yyyy%mm%dd_%hh%MM%ssz.nc4' first: PT0H frequency: PT1H From 12cdf58aedee8e6652df55b710ec3ab1ef5dec19 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Sun, 16 Jul 2023 22:00:22 -0400 Subject: [PATCH 090/121] test suite --- .github/workflows/test_suite_1.yml | 54 ++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 .github/workflows/test_suite_1.yml diff --git a/.github/workflows/test_suite_1.yml b/.github/workflows/test_suite_1.yml new file mode 100644 index 00000000..0b43df8c --- /dev/null +++ b/.github/workflows/test_suite_1.yml @@ -0,0 +1,54 @@ +name: Swell Test Suite One + +on: + push: + branches: + - develop + pull_request: + types: + - opened + - synchronize + - reopened + +jobs: + testsuite: + name: Run swell test suite + runs-on: ubuntu-latest + + steps: + + # Setup Python + - name: Set up Python 3.10 + uses: actions/setup-python@v2 + with: + python-version: 3.10.10 + + # Update conda + - name: Update conda + run: conda update -n base -c defaults conda + + # Install pip + - name: Install pip + run: conda install pip + + # Upgrade pip + - name: Upgrade pip + run: $CONDA/bin/pip3 install --upgrade pip + + # Clone the swell code + - name: Clone swell repo + uses: actions/checkout@v2 + with: + lfs: true + + # Install swell + - name: Install swell and dependencies + run: $CONDA/bin/pip3 install --use-deprecated=legacy-resolver -r requirements-github.txt --user . + + # Put swell executables in the path + - name: Put swell in the path + run: echo "$HOME/.local/bin" >> $GITHUB_PATH + + # Run the swell test suite + - name: Run swell test suite + run: swell_test_suite From 815637ef1e84203640f56f4ffdfdd647c8e9257d Mon Sep 17 00:00:00 2001 From: danholdaway Date: Mon, 17 Jul 2023 09:32:44 -0400 Subject: [PATCH 091/121] replace tier2 --- .github/workflows/tier2_application_discover.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tier2_application_discover.yml b/.github/workflows/tier2_application_discover.yml index 972b9e27..92abfca8 100644 --- a/.github/workflows/tier2_application_discover.yml +++ b/.github/workflows/tier2_application_discover.yml @@ -1,9 +1,12 @@ -wname: Tier 2 Applications Tests (Discover) +# Discover Nightly + +name: discover-nightly on: workflow_dispatch: schedule: + # * is a special character in YAML so you have to quote this string - cron: '0 0 * * *' defaults: @@ -11,6 +14,5 @@ defaults: shell: bash jobs: - tier2-application-discover: - uses: GEOS-ESM/CI-workflows/.github/workflows/swell-tier2_application_discover.yml@feature/ufo_tests - + run-discover-nightly: + uses: GEOS-ESM/CI-workflows/.github/workflows/swell-discover-nightly.yml@main From 0c8d2f40e1e42256a998265c4500ed0f5d577587 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Mon, 17 Jul 2023 09:34:10 -0400 Subject: [PATCH 092/121] tier2 name --- .github/workflows/tier2_application_discover.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tier2_application_discover.yml b/.github/workflows/tier2_application_discover.yml index 92abfca8..cd7ea8fd 100644 --- a/.github/workflows/tier2_application_discover.yml +++ b/.github/workflows/tier2_application_discover.yml @@ -1,6 +1,6 @@ # Discover Nightly -name: discover-nightly +name: Tier 2 Applications Tests (Discover) on: workflow_dispatch: From 71cb165d3789bb77076b365dfb097a88fe590227 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Mon, 17 Jul 2023 10:00:54 -0400 Subject: [PATCH 093/121] tier 2 take 2 --- .../workflows/tier2_application_discover.yml | 2 +- .github/workflows/tier_2.yml | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/tier_2.yml diff --git a/.github/workflows/tier2_application_discover.yml b/.github/workflows/tier2_application_discover.yml index cd7ea8fd..9e5c9e62 100644 --- a/.github/workflows/tier2_application_discover.yml +++ b/.github/workflows/tier2_application_discover.yml @@ -1,6 +1,6 @@ # Discover Nightly -name: Tier 2 Applications Tests (Discover) +name: Tier2 on: workflow_dispatch: diff --git a/.github/workflows/tier_2.yml b/.github/workflows/tier_2.yml new file mode 100644 index 00000000..ebd7a202 --- /dev/null +++ b/.github/workflows/tier_2.yml @@ -0,0 +1,18 @@ +# Discover Nightly + +name: Tier 2 Discover + +on: + workflow_dispatch: + + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 0 * * *' + +defaults: + run: + shell: bash + +jobs: + run-discover-nightly: + uses: GEOS-ESM/CI-workflows/.github/workflows/swell-discover-nightly.yml@main From 303b8c65257be39edf026a224810dbd1a0342dbf Mon Sep 17 00:00:00 2001 From: danholdaway Date: Mon, 17 Jul 2023 11:42:34 -0400 Subject: [PATCH 094/121] remove trigger for tier1 --- .github/workflows/tier1_application_discover.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tier1_application_discover.yml b/.github/workflows/tier1_application_discover.yml index e3b416fa..8ee91a5a 100644 --- a/.github/workflows/tier1_application_discover.yml +++ b/.github/workflows/tier1_application_discover.yml @@ -1,6 +1,6 @@ name: Tier 1 Applications Tests (Discover) -on: workflow_dispatch +on: [] defaults: run: From 6ddf8cf6d6c14d57500488cd9c67580189337aef Mon Sep 17 00:00:00 2001 From: danholdaway Date: Mon, 17 Jul 2023 13:56:50 -0400 Subject: [PATCH 095/121] update target workflows --- .github/workflows/tier1_application_discover.yml | 2 +- .github/workflows/tier2_application_discover.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tier1_application_discover.yml b/.github/workflows/tier1_application_discover.yml index 1546ef11..e65bf4f9 100644 --- a/.github/workflows/tier1_application_discover.yml +++ b/.github/workflows/tier1_application_discover.yml @@ -9,4 +9,4 @@ defaults: jobs: run-discover-nightly: - uses: GEOS-ESM/CI-workflows/.github/workflows/swell-discover-nightly.yml@main + uses: GEOS-ESM/CI-workflows/.github/workflows/swell-tier1_application_discover.yml@main diff --git a/.github/workflows/tier2_application_discover.yml b/.github/workflows/tier2_application_discover.yml index 67e92091..a0c19662 100644 --- a/.github/workflows/tier2_application_discover.yml +++ b/.github/workflows/tier2_application_discover.yml @@ -13,4 +13,4 @@ defaults: jobs: run-discover-nightly: - uses: GEOS-ESM/CI-workflows/.github/workflows/swell-discover-nightly.yml@main + uses: GEOS-ESM/CI-workflows/.github/workflows/swell-tier1_application_discover.yml@main From 72e4a7151e1a863923899c54373a7d73cf5205ca Mon Sep 17 00:00:00 2001 From: danholdaway Date: Mon, 17 Jul 2023 13:58:03 -0400 Subject: [PATCH 096/121] update target workflows branch --- .github/workflows/tier1_application_discover.yml | 2 +- .github/workflows/tier2_application_discover.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tier1_application_discover.yml b/.github/workflows/tier1_application_discover.yml index e65bf4f9..3113371d 100644 --- a/.github/workflows/tier1_application_discover.yml +++ b/.github/workflows/tier1_application_discover.yml @@ -9,4 +9,4 @@ defaults: jobs: run-discover-nightly: - uses: GEOS-ESM/CI-workflows/.github/workflows/swell-tier1_application_discover.yml@main + uses: GEOS-ESM/CI-workflows/.github/workflows/swell-tier1_application_discover.yml@feature/ufo_tests diff --git a/.github/workflows/tier2_application_discover.yml b/.github/workflows/tier2_application_discover.yml index a0c19662..7b0b3cee 100644 --- a/.github/workflows/tier2_application_discover.yml +++ b/.github/workflows/tier2_application_discover.yml @@ -13,4 +13,4 @@ defaults: jobs: run-discover-nightly: - uses: GEOS-ESM/CI-workflows/.github/workflows/swell-tier1_application_discover.yml@main + uses: GEOS-ESM/CI-workflows/.github/workflows/swell-tier1_application_discover.yml@feature/ufo_tests From de6aa445de5ca38b7a160802d33a7fdb42166110 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Mon, 17 Jul 2023 16:30:56 -0400 Subject: [PATCH 097/121] some renaming --- .github/workflows/tier1_application_discover.yml | 2 +- .github/workflows/tier2_application_discover.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tier1_application_discover.yml b/.github/workflows/tier1_application_discover.yml index 3113371d..350a2b9f 100644 --- a/.github/workflows/tier1_application_discover.yml +++ b/.github/workflows/tier1_application_discover.yml @@ -8,5 +8,5 @@ defaults: shell: bash jobs: - run-discover-nightly: + run-tier1-apps-discover: uses: GEOS-ESM/CI-workflows/.github/workflows/swell-tier1_application_discover.yml@feature/ufo_tests diff --git a/.github/workflows/tier2_application_discover.yml b/.github/workflows/tier2_application_discover.yml index 7b0b3cee..979cd877 100644 --- a/.github/workflows/tier2_application_discover.yml +++ b/.github/workflows/tier2_application_discover.yml @@ -12,5 +12,5 @@ defaults: shell: bash jobs: - run-discover-nightly: - uses: GEOS-ESM/CI-workflows/.github/workflows/swell-tier1_application_discover.yml@feature/ufo_tests + run-tier2-apps-discover: + uses: GEOS-ESM/CI-workflows/.github/workflows/swell-tier2_application_discover.yml@feature/ufo_tests From 736c868fdedae238c34e791507b988d1c6a1d8f0 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Mon, 17 Jul 2023 21:45:27 -0400 Subject: [PATCH 098/121] action testing --- .github/workflows/tier2_application_discover.yml | 6 +++--- requirements.txt | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tier2_application_discover.yml b/.github/workflows/tier2_application_discover.yml index 979cd877..80da1b72 100644 --- a/.github/workflows/tier2_application_discover.yml +++ b/.github/workflows/tier2_application_discover.yml @@ -3,9 +3,9 @@ name: Tier 2 Applications Tests (Discover) on: workflow_dispatch: - schedule: - # * is a special character in YAML so you have to quote this string - - cron: '0 0 * * *' +# schedule: +# # * is a special character in YAML so you have to quote this string +# - cron: '0 0 * * *' defaults: run: diff --git a/requirements.txt b/requirements.txt index e89a94b6..f92d3660 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ click>=8.1.5 jinja2>=3.1.2 pyyaml>=6.0 -pycodestyle>=2.8.0 +pycodestyle==2.8.0 pandas>=1.4.0 isodate>=0.5.4 f90nml>=1.4.3 From 3bb21f94e0d34cb89bfe87b9acf367c833bb28cc Mon Sep 17 00:00:00 2001 From: danholdaway Date: Mon, 17 Jul 2023 21:53:01 -0400 Subject: [PATCH 099/121] action testing --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f92d3660..5608a996 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ click>=8.1.5 jinja2>=3.1.2 pyyaml>=6.0 -pycodestyle==2.8.0 +pycodestyle>=2.10.0 pandas>=1.4.0 isodate>=0.5.4 f90nml>=1.4.3 From 0c3e4ff1b93ed478c2b683f4b4380c094f1cca19 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 19 Jul 2023 09:59:37 -0400 Subject: [PATCH 100/121] bug fix in run_command print --- src/swell/utilities/shell_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swell/utilities/shell_commands.py b/src/swell/utilities/shell_commands.py index 28cced3d..12fc8f59 100644 --- a/src/swell/utilities/shell_commands.py +++ b/src/swell/utilities/shell_commands.py @@ -71,7 +71,7 @@ def run_subprocess(logger, command, stdout=None, stderr=None): subprocess.run(command, check=True, stdout=stdout, stderr=stderr) except subprocess.CalledProcessError as e: print(e) - logger.abort('Subprocess with command {command} failed, throwing error {error}') + logger.abort(f'Subprocess with command {command} failed, throwing error {error}') # -------------------------------------------------------------------------------------------------- From dd13a66e3c6ff4cdc0f58d7f02f375b84f0ca66f Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 19 Jul 2023 10:02:48 -0400 Subject: [PATCH 101/121] bug fix in run_command print --- src/swell/utilities/shell_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swell/utilities/shell_commands.py b/src/swell/utilities/shell_commands.py index 12fc8f59..f48a39af 100644 --- a/src/swell/utilities/shell_commands.py +++ b/src/swell/utilities/shell_commands.py @@ -71,7 +71,7 @@ def run_subprocess(logger, command, stdout=None, stderr=None): subprocess.run(command, check=True, stdout=stdout, stderr=stderr) except subprocess.CalledProcessError as e: print(e) - logger.abort(f'Subprocess with command {command} failed, throwing error {error}') + logger.abort(f'Subprocess with command {command} failed, throwing error {e}') # -------------------------------------------------------------------------------------------------- From 4a3ab6b54b00e863820069185acd848a11c0408a Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 19 Jul 2023 10:09:06 -0400 Subject: [PATCH 102/121] action testing --- src/swell/deployment/bin/swell_launch_experiment.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/swell/deployment/bin/swell_launch_experiment.py b/src/swell/deployment/bin/swell_launch_experiment.py index 3f36bae6..6e75825a 100644 --- a/src/swell/deployment/bin/swell_launch_experiment.py +++ b/src/swell/deployment/bin/swell_launch_experiment.py @@ -52,6 +52,7 @@ def cylc_run_experiment(self): # NB: Could be a factory based on workflow_manag if self.log_path: # Add optional path for workflow engine logging. option = '--symlink-dirs=run=' + self.log_path + print(['cylc', 'install', option]) run_subprocess(self.logger, ['cylc', 'install', option]) else: run_subprocess(self.logger, ['cylc', 'install']) From 7a1aefbb0295862765dc3d7439811caa619aec71 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 19 Jul 2023 10:17:58 -0400 Subject: [PATCH 103/121] action testing --- src/swell/suites/ufo_testing/flow.cylc | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/src/swell/suites/ufo_testing/flow.cylc b/src/swell/suites/ufo_testing/flow.cylc index 11b51358..9697b5a6 100644 --- a/src/swell/suites/ufo_testing/flow.cylc +++ b/src/swell/suites/ufo_testing/flow.cylc @@ -39,33 +39,24 @@ {% for cycle_time in cycle_times %} {{cycle_time.cycle_time}} = """ - # Convert ncdiags to ioda - #GetGsiNcdiag - #GetGsiNcdiag => GsiNcdiagToIoda - GetObservations - - # Convert ncdiags to ioda + # Convert bias correction to ioda GetGsiBc GetGsiBc => GsiBcToIoda - # Get GeoVaLs - GetGeovals + # Convert ncdiags to ioda + GetGsiNcdiag + GetGsiNcdiag => GsiNcdiagToIoda # Run Jedi hofx executable BuildJediByLinking[^]? | BuildJedi[^] => RunJediTestObsFiltersExecutable - #GsiNcdiagToIoda => RunJediTestObsFiltersExecutable - GetObservations => RunJediTestObsFiltersExecutable + GsiNcdiagToIoda => RunJediTestObsFiltersExecutable GsiBcToIoda => RunJediTestObsFiltersExecutable - GetGeovals => RunJediTestObsFiltersExecutable # EvaObservations RunJediTestObsFiltersExecutable => EvaObservations - # Save observations - RunJediTestObsFiltersExecutable => SaveObsDiags - # Clean up large files - EvaObservations & SaveObsDiags => CleanCycle + EvaObservations => CleanCycle """ {% endfor %} @@ -130,9 +121,6 @@ [[EvaObservations]] script = "swell_task EvaObservations $config -d $datetime -m geos_atmosphere" - [[SaveObsDiags]] - script = "swell_task SaveObsDiags $config -d $datetime -m geos_atmosphere" - [[CleanCycle]] script = "swell_task CleanCycle $config -d $datetime -m geos_atmosphere" From 60b117279dd3f378a266503d7abcbcfe3879ce7e Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 19 Jul 2023 12:35:11 -0400 Subject: [PATCH 104/121] flow bug --- .../geos_atmosphere/observations/vadwind.yaml | 159 ------------------ src/swell/suites/ufo_testing/flow.cylc | 8 +- src/swell/tasks/gsi_ncdiag_to_ioda.py | 2 +- 3 files changed, 5 insertions(+), 164 deletions(-) delete mode 100644 src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/vadwind.yaml diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/vadwind.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/vadwind.yaml deleted file mode 100644 index 98484999..00000000 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/vadwind.yaml +++ /dev/null @@ -1,159 +0,0 @@ -obs operator: - name: VertInterp - observation alias file: '{{experiment_root}}/{{experiment_id}}/configuration/jedi/interfaces/{{model_component}}/observations/obsop_name_map.yaml' -obs space: - name: vadwind - obsdatain: - engine: - type: H5File - obsfile: '{{cycle_dir}}/vadwind.{{window_begin}}.nc4' - obsgrouping: - group variables: ["stationIdentification", "datetime"] - sort variable: "pressure" - sort order: "descending" - obsdataout: - engine: - type: H5File - obsfile: '{{cycle_dir}}/{{experiment_id}}.vadwind.{{window_begin}}.nc4' - simulated variables: [windEastward, windNorthward] -#-------------------------------------------------------------------------------------------------------------------- -obs filters: -# Begin by assigning all ObsError to a constant value. These might get overwritten later. -- filter: BlackList - filter variables: - - name: windEastward - - name: windNorthward - action: - name: assign error - error parameter: 2.0 # 2.0 m/s -# -# Assign the initial ObsError, based on height/pressure -- filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - minvalue: -135 - maxvalue: 135 - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelStepwiseLinear - options: - xvar: - name: MetaData/pressure - xvals: [100000, 95000, 85000, 80000, 70000, 65000, 60000, 55000, 50000, 45000, 40000, 35000, 30000, 25000, 20000, 15000, 10000] - errors: [1.4, 1.5, 1.5, 1.6, 1.6, 1.8, 1.9, 2.0, 2.1, 2.3, 2.6, 2.8, 3.0, 3.2, 2.7, 2.4, 2.1] -# -# Reject all obs with PreQC mark already set above 3 -- filter: PreQC - maxvalue: 3 - action: - name: reject -# -# Reject when pressure is less than 226 mb. -- filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - test variables: - - name: MetaData/pressure - minvalue: 22600 - action: - name: reject -# -# Observation Range Sanity Check: either wind component or velocity exceeds 135 m/s -- filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - minvalue: -135 - maxvalue: 135 - action: - name: reject -- filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - test variables: - - name: ObsFunction/Velocity - maxvalue: 135.0 - action: - name: reject -# -# Reject when difference of wind direction is more than 50 degrees. -- filter: Bounds Check - filter variables: - - name: windEastward - - name: windNorthward - test variables: - - name: ObsFunction/WindDirAngleDiff - maxvalue: 50.0 - action: - name: reject - defer to post: true -# -# Inflate obserror when multiple obs exist inside vertical model layers. -- filter: BlackList - filter variables: - - name: windEastward - action: - name: inflate error - inflation variable: - name: ObsFunction/ObsErrorFactorConventional - options: - test QCflag: PreQC - inflate variables: [eastward_wind] - defer to post: true -# -- filter: BlackList - filter variables: - - name: windNorthward - action: - name: inflate error - inflation variable: - name: ObsFunction/ObsErrorFactorConventional - options: - test QCflag: PreQC - inflate variables: [northward_wind] - defer to post: true -# -- filter: Background Check - filter variables: - - name: windEastward - - name: windNorthward - absolute threshold: 7.5 - action: - name: inflate error - inflation factor: 2.5 - defer to post: true -# -# If the total inflation factor is too big, reject. -- filter: Bounds Check - filter variables: - - name: windEastward - action: - name: reject - maxvalue: 6.5 - test variables: - - name: ObsFunction/ObsErrorFactorQuotient - options: - numerator: - name: ObsErrorData/windEastward # After inflation step - denominator: - name: ObsError/windEastward - defer to post: true -# -- filter: Bounds Check - filter variables: - - name: windNorthward - action: - name: reject - maxvalue: 6.5 - test variables: - - name: ObsFunction/ObsErrorFactorQuotient - options: - numerator: - name: ObsErrorData/windNorthward # After inflation step - denominator: - name: ObsError/windNorthward - defer to post: true diff --git a/src/swell/suites/ufo_testing/flow.cylc b/src/swell/suites/ufo_testing/flow.cylc index 9697b5a6..1142db8c 100644 --- a/src/swell/suites/ufo_testing/flow.cylc +++ b/src/swell/suites/ufo_testing/flow.cylc @@ -48,12 +48,12 @@ GetGsiNcdiag => GsiNcdiagToIoda # Run Jedi hofx executable - BuildJediByLinking[^]? | BuildJedi[^] => RunJediTestObsFiltersExecutable - GsiNcdiagToIoda => RunJediTestObsFiltersExecutable - GsiBcToIoda => RunJediTestObsFiltersExecutable + BuildJediByLinking[^]? | BuildJedi[^] => RunJediUfoTestsExecutable + GsiNcdiagToIoda => RunJediUfoTestsExecutable + GsiBcToIoda => RunJediUfoTestsExecutable # EvaObservations - RunJediTestObsFiltersExecutable => EvaObservations + RunJediUfoTestsExecutable => EvaObservations # Clean up large files EvaObservations => CleanCycle diff --git a/src/swell/tasks/gsi_ncdiag_to_ioda.py b/src/swell/tasks/gsi_ncdiag_to_ioda.py index 934ff238..dd782b8e 100644 --- a/src/swell/tasks/gsi_ncdiag_to_ioda.py +++ b/src/swell/tasks/gsi_ncdiag_to_ioda.py @@ -111,7 +111,7 @@ def execute(self): # Path to search to GSI ncdiag files path_to_search = os.path.join(gsi_diag_dir, extra_path, - f'*{gsi_type_to_process_actual}*{gsi_datetime_str}*') + f'*{gsi_type_to_process_actual}_*{gsi_datetime_str}*') # Get the list of files gsi_conv_file = glob.glob(path_to_search) From 5005029d2675434032ec9d1c4ac0d6066b930ced Mon Sep 17 00:00:00 2001 From: danholdaway Date: Fri, 21 Jul 2023 10:39:05 -0400 Subject: [PATCH 105/121] add alternative workflow that generates the geovals as well as obs --- requirements.txt | 4 + src/swell/__init__.py | 2 +- .../{gnssrobndnbam.yaml => gps.yaml} | 0 src/swell/suites/convert_ncdiags/flow.cylc | 97 +++++++++++++++++++ ...uites-convert_ncdiags-geos_atmosphere.yaml | 71 ++++++++++++++ .../suites-convert_ncdiags.yaml | 49 ++++++++++ src/swell/suites/ufo_testing/flow.cylc | 6 ++ .../suites-ufo_testing-geos_atmosphere.yaml | 16 ++- src/swell/tasks/gsi_ncdiag_to_ioda.py | 32 ++++-- 9 files changed, 268 insertions(+), 9 deletions(-) rename src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/{gnssrobndnbam.yaml => gps.yaml} (100%) create mode 100644 src/swell/suites/convert_ncdiags/flow.cylc create mode 100644 src/swell/suites/convert_ncdiags/geos_atmosphere/suites-convert_ncdiags-geos_atmosphere.yaml create mode 100644 src/swell/suites/convert_ncdiags/suites-convert_ncdiags.yaml diff --git a/requirements.txt b/requirements.txt index 5608a996..615c49e1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,7 @@ isodate>=0.5.4 f90nml>=1.4.3 questionary>=1.10.0 flake8>=6.0.0 +netCDF4 +xarray +matplotlib +cartopy diff --git a/src/swell/__init__.py b/src/swell/__init__.py index b8651c71..e1371654 100644 --- a/src/swell/__init__.py +++ b/src/swell/__init__.py @@ -9,4 +9,4 @@ repo_directory = os.path.dirname(__file__) # Set the version for swell -__version__ = '1.5.0' +__version__ = '1.6.0' diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/gnssrobndnbam.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/gps.yaml similarity index 100% rename from src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/gnssrobndnbam.yaml rename to src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/gps.yaml diff --git a/src/swell/suites/convert_ncdiags/flow.cylc b/src/swell/suites/convert_ncdiags/flow.cylc new file mode 100644 index 00000000..81fee698 --- /dev/null +++ b/src/swell/suites/convert_ncdiags/flow.cylc @@ -0,0 +1,97 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + +# Cylc suite for executing geos_atmosphere ObsFilters tests + +# -------------------------------------------------------------------------------------------------- + +[scheduler] + UTC mode = True + allow implicit tasks = False + +# -------------------------------------------------------------------------------------------------- + +[scheduling] + + initial cycle point = {{start_cycle_point}} + final cycle point = {{final_cycle_point}} + runahead limit = {{runahead_limit}} + + [[graph]] + R1 = """ + # Triggers for non cycle time dependent tasks + # ------------------------------------------- + # Clone JEDI source code + CloneJedi + + # Build JEDI source code by linking + CloneJedi => BuildJediByLinking? + + # If not able to link to build create the build + BuildJediByLinking:fail? => BuildJedi + """ + + {% for cycle_time in cycle_times %} + {{cycle_time.cycle_time}} = """ + + # Convert bias correction to ioda + GetGsiBc + GetGsiBc => GsiBcToIoda + + # Convert ncdiags to ioda + GetGsiNcdiag + GetGsiNcdiag => GsiNcdiagToIoda + """ + {% endfor %} + +# -------------------------------------------------------------------------------------------------- + +[runtime] + + # Task defaults + # ------------- + [[root]] + pre-script = "source $CYLC_SUITE_DEF_PATH/modules" + + [[[environment]]] + datetime = $CYLC_TASK_CYCLE_POINT + config = $CYLC_SUITE_DEF_PATH/experiment.yaml + + # Tasks + # ----- + [[CloneJedi]] + script = "swell_task CloneJedi $config" + + [[BuildJediByLinking]] + script = "swell_task BuildJediByLinking $config" + + [[BuildJedi]] + script = "swell_task BuildJedi $config" + platform = {{platform}} + execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} + [[[directives]]] + --account = {{scheduling["BuildJedi"]["account"]}} + --qos = {{scheduling["BuildJedi"]["qos"]}} + --job-name = BuildJedi + --nodes={{scheduling["BuildJedi"]["nodes"]}} + --ntasks-per-node={{scheduling["BuildJedi"]["ntasks_per_node"]}} + --constraint={{scheduling["BuildJedi"]["constraint"]}} + + [[ GetGsiBc ]] + script = "swell_task GetGsiBc $config -d $datetime -m geos_atmosphere" + + [[ GsiBcToIoda ]] + script = "swell_task GsiBcToIoda $config -d $datetime -m geos_atmosphere" + + [[ GetGsiNcdiag ]] + script = "swell_task GetGsiNcdiag $config -d $datetime -m geos_atmosphere" + + [[ GsiNcdiagToIoda ]] + script = "swell_task GsiNcdiagToIoda $config -d $datetime -m geos_atmosphere" + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/convert_ncdiags/geos_atmosphere/suites-convert_ncdiags-geos_atmosphere.yaml b/src/swell/suites/convert_ncdiags/geos_atmosphere/suites-convert_ncdiags-geos_atmosphere.yaml new file mode 100644 index 00000000..f1b22537 --- /dev/null +++ b/src/swell/suites/convert_ncdiags/geos_atmosphere/suites-convert_ncdiags-geos_atmosphere.yaml @@ -0,0 +1,71 @@ +# Cycle frequency +cycle_times: + default_value: ['T00'] + prompt: List all cycle times, by middle of the window? + options: ['T00'] + type: string-check-list + +# Observations +observations: + default_value: + # Conventional + - aircraft + - gps + - satwind + - scatwind + - sfcship + - sfc + - sondes + # Radiances + - airs_aqua + - amsr2_gcom-w1 + - amsua_aqua + - amsua_metop-b + - amsua_metop-c + - amsua_n15 + - amsua_n18 + - amsua_n19 + - atms_n20 + - atms_npp + - avhrr_metop-b + - avhrr_n18 + - avhrr_n19 + - cris-fsr_n20 + - cris-fsr_npp + - gmi_gpm + - hirs4_n18 + - hirs4_n19 + - iasi_metop-b + - iasi_metop-c + - mhs_metop-b + - mhs_metop-c + - mhs_n19 + - seviri_m08 + - ssmis_f17 + - ssmis_f18 + # Ozone + - mls55_aura + - omi_aura + - ompsnm_npp + prompt: Select observations to run with. + options: use_method + type: string-check-list + +# Path to GSI ncdiags +path_to_gsi_nc_diags: + default_value: '/discover/nobackup/drholdaw/SwellTestData/ufo_testing/ncdiag' + prompt: Path to where the ncdiags will be held + type: string + +# Compute GeoVaLs and observations +produce_geovals: + default_value: true + prompt: Compute GeoVaLs as well as observations? + type: boolean + +# Location for bias correction coefficients +path_to_gsi_bc_coefficients: + default_value: '/discover/nobackup/drholdaw/SwellTestData/ufo_testing/bc' + prompt: Tar or directory containing BC files + type: string + diff --git a/src/swell/suites/convert_ncdiags/suites-convert_ncdiags.yaml b/src/swell/suites/convert_ncdiags/suites-convert_ncdiags.yaml new file mode 100644 index 00000000..b428d7c1 --- /dev/null +++ b/src/swell/suites/convert_ncdiags/suites-convert_ncdiags.yaml @@ -0,0 +1,49 @@ +# Cycle start point +start_cycle_point: + default_value: '2020-12-15T00:00:00Z' + prompt: What is the time of the first cycle (middle of the window)? + type: iso-datetime + +# Cycle final point +final_cycle_point: + default_value: '2020-12-15T06:00:00Z' + prompt: What is the time of the final cycle (middle of the window)? + type: iso-datetime + +# Run ahead limit for workflow +runahead_limit: + default_value: 'P4' + prompt: Since this suite is non-cycling choose how many hours the workflow can run ahead? + type: string + +# Jedi build system +jedi_build_method: + default_value: use_existing + prompt: Create a new JEDI build or use an existing build? + options: ['create', 'use_existing'] + type: string-drop-list + +# Existing JEDI bundle directory +existing_jedi_source_directory: + default_value: {{existing_jedi_source_directory}} + prompt: Provide the path to an existing JEDI bundle directory containing source code repos + type: string + depends: + key: jedi_build_method + value: use_existing + +# Existing JEDI build +existing_jedi_build_directory: + default_value: {{existing_jedi_build_directory}} + prompt: Provide the path to an existing JEDI build directory + type: string + depends: + key: jedi_build_method + value: use_existing + +# Models to use +model_components: + default_value: ['geos_atmosphere'] + prompt: Select models to use (choose at least one) + options: use_method + type: file-check-list diff --git a/src/swell/suites/ufo_testing/flow.cylc b/src/swell/suites/ufo_testing/flow.cylc index 1142db8c..f5953c17 100644 --- a/src/swell/suites/ufo_testing/flow.cylc +++ b/src/swell/suites/ufo_testing/flow.cylc @@ -47,10 +47,13 @@ GetGsiNcdiag GetGsiNcdiag => GsiNcdiagToIoda + GetGeovals + # Run Jedi hofx executable BuildJediByLinking[^]? | BuildJedi[^] => RunJediUfoTestsExecutable GsiNcdiagToIoda => RunJediUfoTestsExecutable GsiBcToIoda => RunJediUfoTestsExecutable + GetGeovals => RunJediTestObsFiltersExecutable # EvaObservations RunJediUfoTestsExecutable => EvaObservations @@ -106,6 +109,9 @@ [[ GsiNcdiagToIoda ]] script = "swell_task GsiNcdiagToIoda $config -d $datetime -m geos_atmosphere" + [[ GetGeovals ]] + script = "swell_task GetGeovals $config -d $datetime -m geos_atmosphere" + [[RunJediUfoTestsExecutable]] script = "swell_task RunJediUfoTestsExecutable $config -d $datetime -m geos_atmosphere" platform = {{platform}} diff --git a/src/swell/suites/ufo_testing/geos_atmosphere/suites-ufo_testing-geos_atmosphere.yaml b/src/swell/suites/ufo_testing/geos_atmosphere/suites-ufo_testing-geos_atmosphere.yaml index 3c93986d..c5c776aa 100644 --- a/src/swell/suites/ufo_testing/geos_atmosphere/suites-ufo_testing-geos_atmosphere.yaml +++ b/src/swell/suites/ufo_testing/geos_atmosphere/suites-ufo_testing-geos_atmosphere.yaml @@ -15,6 +15,9 @@ window_length: observations: default_value: - aircraft + - gmi_gpm + + # - airs_aqua # - amsr2_gcom-w1 # - amsua_aqua @@ -59,7 +62,7 @@ path_to_gsi_nc_diags: # Compute GeoVaLs and observations produce_geovals: - default_value: true + default_value: false prompt: Compute GeoVaLs as well as observations? type: boolean @@ -69,6 +72,13 @@ path_to_gsi_bc_coefficients: prompt: Tar or directory containing BC files type: string +# Observation experiment +geovals_experiment: + default_value: x0048-geovals + prompt: Experiment providing the GeoVaLs? + type: string + + # Fixed options (user not prompted for these) # ------------------------------------------- fixed_options: @@ -103,6 +113,8 @@ fixed_options: background_time_offset: default_value: PT9H prompt: Time before the middle of the window that the background providing forecast began - + geovals_provider: + default_value: ncdiag + prompt: Database providing the geovals. diff --git a/src/swell/tasks/gsi_ncdiag_to_ioda.py b/src/swell/tasks/gsi_ncdiag_to_ioda.py index dd782b8e..b9ea7c4e 100644 --- a/src/swell/tasks/gsi_ncdiag_to_ioda.py +++ b/src/swell/tasks/gsi_ncdiag_to_ioda.py @@ -31,7 +31,7 @@ def execute(self): # Parse configuration # ------------------- observations = self.config.observations() - produce_geovals = self.config.produce_geovals() + produce_geovals = False #self.config.produce_geovals() window_offset = self.config.window_offset() # Get window beginning time @@ -233,6 +233,11 @@ def execute(self): gsi_obs_file = glob.glob(os.path.join(gsi_diag_dir, f'*{observation}*')) + # Skip this observation if not files were found + if len(gsi_obs_file) == 0: + self.logger.info(f'No observation files found for {observation}. Skipping convert') + continue + if observation not in ozone_observations: # Radiances @@ -258,21 +263,36 @@ def execute(self): # ---------------------------------- for observation in observations_orig: + self.logger.info(f'Renaming \'{observation}\' to be swell compliant') + + # Change to gps_bend + search_name = observation + if search_name == 'gps': + search_name = 'gps_bend' + # Input filename - ioda_obs_in_pattern = f'{observation}_obs_*nc*' - ioda_obs_in = glob.glob(os.path.join(self.cycle_dir(), ioda_obs_in_pattern))[0] + ioda_obs_in_pattern = f'{search_name}_obs_*nc*' + + ioda_obs_in_found = glob.glob(os.path.join(self.cycle_dir(), ioda_obs_in_pattern)) + + # If nothing found then skipt this observation + if len(ioda_obs_in_found) == 0: + self.logger.info(f'No observation files found for {observation}. Skipping rename') + continue + + ioda_obs_in = ioda_obs_in_found[0] - ioda_obs_out = f'{observation}.{window_begin}.nc4' + ioda_obs_out = f'{search_name}.{window_begin}.nc4' os.rename(ioda_obs_in, os.path.join(self.cycle_dir(), ioda_obs_out)) # Rename GeoVaLs file if need be if produce_geovals: - ioda_geoval_in_pattern = f'{observation}_geoval_*.nc*' + ioda_geoval_in_pattern = f'{search_name}_geoval_*.nc*' ioda_geoval_in = glob.glob(os.path.join(self.cycle_dir(), ioda_geoval_in_pattern))[0] - ioda_geoval_out = f'{observation}_geovals.{window_begin}.nc4' + ioda_geoval_out = f'{search_name}_geovals.{window_begin}.nc4' os.rename(ioda_geoval_in, os.path.join(self.cycle_dir(), ioda_geoval_out)) From e740eed12521a682d09dbb0f3e9461c09c09ce4d Mon Sep 17 00:00:00 2001 From: danholdaway Date: Fri, 21 Jul 2023 11:03:55 -0400 Subject: [PATCH 106/121] better requirements --- requirements-standalone.txt | 12 ++++++++++++ requirements.txt | 7 +------ src/swell/suites/ufo_testing/flow.cylc | 2 +- 3 files changed, 14 insertions(+), 7 deletions(-) create mode 100644 requirements-standalone.txt diff --git a/requirements-standalone.txt b/requirements-standalone.txt new file mode 100644 index 00000000..bcaaa59a --- /dev/null +++ b/requirements-standalone.txt @@ -0,0 +1,12 @@ +click>=8.1.5 +jinja2>=3.1.2 +pyyaml>=6.0 +pycodestyle>=2.10.0 +pandas>=1.4.0 +isodate>=0.5.4 +f90nml>=1.4.3 +questionary>=1.10.0 +flake8>=6.0.0 +netCDF4 +xarray +matplotlib diff --git a/requirements.txt b/requirements.txt index 615c49e1..ef64ba0a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,8 @@ click>=8.1.5 jinja2>=3.1.2 pyyaml>=6.0 -pycodestyle>=2.10.0 +pycodestyle==2.8.0 pandas>=1.4.0 isodate>=0.5.4 f90nml>=1.4.3 questionary>=1.10.0 -flake8>=6.0.0 -netCDF4 -xarray -matplotlib -cartopy diff --git a/src/swell/suites/ufo_testing/flow.cylc b/src/swell/suites/ufo_testing/flow.cylc index f5953c17..8929cfc6 100644 --- a/src/swell/suites/ufo_testing/flow.cylc +++ b/src/swell/suites/ufo_testing/flow.cylc @@ -53,7 +53,7 @@ BuildJediByLinking[^]? | BuildJedi[^] => RunJediUfoTestsExecutable GsiNcdiagToIoda => RunJediUfoTestsExecutable GsiBcToIoda => RunJediUfoTestsExecutable - GetGeovals => RunJediTestObsFiltersExecutable + GetGeovals => RunJediUfoTestsExecutable # EvaObservations RunJediUfoTestsExecutable => EvaObservations From 8913ab962eea769684385f3ba18dfae0f5dbaefe Mon Sep 17 00:00:00 2001 From: danholdaway Date: Fri, 21 Jul 2023 11:46:19 -0400 Subject: [PATCH 107/121] add da run parameter --- .../convert_ncdiags/suites-convert_ncdiags.yaml | 6 ++++++ .../suites/ufo_testing/suites-ufo_testing.yaml | 16 ++++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/swell/suites/convert_ncdiags/suites-convert_ncdiags.yaml b/src/swell/suites/convert_ncdiags/suites-convert_ncdiags.yaml index b428d7c1..29ff0ec9 100644 --- a/src/swell/suites/convert_ncdiags/suites-convert_ncdiags.yaml +++ b/src/swell/suites/convert_ncdiags/suites-convert_ncdiags.yaml @@ -47,3 +47,9 @@ model_components: prompt: Select models to use (choose at least one) options: use_method type: file-check-list + +# Fixed options +fixed_options: + data_assimilation_run: + default_value: true + prompt: Does this workflow include data assimilation? diff --git a/src/swell/suites/ufo_testing/suites-ufo_testing.yaml b/src/swell/suites/ufo_testing/suites-ufo_testing.yaml index b428d7c1..a145c553 100644 --- a/src/swell/suites/ufo_testing/suites-ufo_testing.yaml +++ b/src/swell/suites/ufo_testing/suites-ufo_testing.yaml @@ -1,12 +1,12 @@ # Cycle start point start_cycle_point: - default_value: '2020-12-15T00:00:00Z' + default_value: '2021-12-12T00:00:00Z' prompt: What is the time of the first cycle (middle of the window)? type: iso-datetime # Cycle final point final_cycle_point: - default_value: '2020-12-15T06:00:00Z' + default_value: '2021-12-12T00:00:00Z' prompt: What is the time of the final cycle (middle of the window)? type: iso-datetime @@ -41,9 +41,21 @@ existing_jedi_build_directory: key: jedi_build_method value: use_existing +# R2D2 Configuration +r2d2_local_path: + default_value: {{r2d2_local_path}} + prompt: Enter the path where R2D2 will store experiment output + type: string + # Models to use model_components: default_value: ['geos_atmosphere'] prompt: Select models to use (choose at least one) options: use_method type: file-check-list + +# Fixed options +fixed_options: + data_assimilation_run: + default_value: true + prompt: Does this workflow include data assimilation? From 73be5cd89c6d304fbfdf2e935550806962af3715 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Fri, 21 Jul 2023 12:35:59 -0400 Subject: [PATCH 108/121] ufo_testing runs in cylc --- src/swell/suites/convert_ncdiags/flow.cylc | 2 ++ src/swell/suites/ufo_testing/flow.cylc | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/swell/suites/convert_ncdiags/flow.cylc b/src/swell/suites/convert_ncdiags/flow.cylc index 81fee698..31e654c3 100644 --- a/src/swell/suites/convert_ncdiags/flow.cylc +++ b/src/swell/suites/convert_ncdiags/flow.cylc @@ -42,10 +42,12 @@ # Convert bias correction to ioda GetGsiBc GetGsiBc => GsiBcToIoda + BuildJediByLinking[^]? | BuildJedi[^] => GsiBcToIoda # Convert ncdiags to ioda GetGsiNcdiag GetGsiNcdiag => GsiNcdiagToIoda + BuildJediByLinking[^]? | BuildJedi[^] => GsiNcdiagToIoda """ {% endfor %} diff --git a/src/swell/suites/ufo_testing/flow.cylc b/src/swell/suites/ufo_testing/flow.cylc index 8929cfc6..c1edb926 100644 --- a/src/swell/suites/ufo_testing/flow.cylc +++ b/src/swell/suites/ufo_testing/flow.cylc @@ -42,15 +42,16 @@ # Convert bias correction to ioda GetGsiBc GetGsiBc => GsiBcToIoda + BuildJediByLinking[^]? | BuildJedi[^] => GsiBcToIoda # Convert ncdiags to ioda GetGsiNcdiag GetGsiNcdiag => GsiNcdiagToIoda + BuildJediByLinking[^]? | BuildJedi[^] => GsiNcdiagToIoda GetGeovals # Run Jedi hofx executable - BuildJediByLinking[^]? | BuildJedi[^] => RunJediUfoTestsExecutable GsiNcdiagToIoda => RunJediUfoTestsExecutable GsiBcToIoda => RunJediUfoTestsExecutable GetGeovals => RunJediUfoTestsExecutable From e54c1e1254d900f1dff8c8947431998740712c94 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Mon, 24 Jul 2023 13:12:56 -0400 Subject: [PATCH 109/121] update hofx to be x0048 --- .../geos_atmosphere/suites-hofx-geos_atmosphere.yaml | 12 ++---------- src/swell/suites/hofx/suites-hofx.yaml | 4 ++-- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/src/swell/suites/hofx/geos_atmosphere/suites-hofx-geos_atmosphere.yaml b/src/swell/suites/hofx/geos_atmosphere/suites-hofx-geos_atmosphere.yaml index 19d2e828..f71aa2ec 100644 --- a/src/swell/suites/hofx/geos_atmosphere/suites-hofx-geos_atmosphere.yaml +++ b/src/swell/suites/hofx/geos_atmosphere/suites-hofx-geos_atmosphere.yaml @@ -37,13 +37,13 @@ npy_proc: # Backgrounds background_experiment: - default_value: x0044 + default_value: x0048 prompt: Experiment providing the background files? type: string # Observation experiment obs_experiment: - default_value: x0044_v3 + default_value: x0048 prompt: Experiment providing the observations? type: string @@ -53,14 +53,6 @@ observations: # Conventional - aircraft # Radiances - - amsua_n19 - - amsua_aqua - - amsua_metop-a - - amsua_metop-b - - amsua_metop-c - - amsua_n15 - - amsua_n18 - - amsua_n19 - gmi_gpm - amsr2_gcom-w1 # Ozone diff --git a/src/swell/suites/hofx/suites-hofx.yaml b/src/swell/suites/hofx/suites-hofx.yaml index 7114e936..3af7c28e 100644 --- a/src/swell/suites/hofx/suites-hofx.yaml +++ b/src/swell/suites/hofx/suites-hofx.yaml @@ -1,12 +1,12 @@ # Cycle start point start_cycle_point: - default_value: '2020-12-15T00:00:00Z' + default_value: '2021-12-12T00:00:00Z' prompt: What is the time of the first cycle (middle of the window)? type: iso-datetime # Cycle final point final_cycle_point: - default_value: '2020-12-15T06:00:00Z' + default_value: '2021-12-12T06:00:00Z' prompt: What is the time of the final cycle (middle of the window)? type: iso-datetime From c2fcc595e0433b854cd86d1e1ee8918991bb7892 Mon Sep 17 00:00:00 2001 From: Akira Sewnath Date: Wed, 26 Jul 2023 14:44:54 -0400 Subject: [PATCH 110/121] updating scatwind yaml (#194) --- .../observations/scatwind.yaml | 170 ++++++++++++------ .../observations/ufo_tests.yaml | 2 +- 2 files changed, 112 insertions(+), 60 deletions(-) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/scatwind.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/scatwind.yaml index fb024aee..9ef016c8 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/scatwind.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/scatwind.yaml @@ -1,8 +1,5 @@ -obs operator: - name: GSISfcModel - observation alias file: '{{experiment_root}}/{{experiment_id}}/configuration/jedi/interfaces/{{model_component}}/observations/obsop_name_map.yaml' obs space: - name: scatwind + name: Scatwind obsdatain: engine: type: H5File @@ -12,107 +9,162 @@ obs space: type: H5File obsfile: '{{cycle_dir}}/{{experiment_id}}.scatwind.{{window_begin}}.nc4' simulated variables: [windEastward, windNorthward] +obs operator: + name: VertInterp + observation alias file: '{{experiment_root}}/{{experiment_id}}/configuration/jedi/interfaces/{{model_component}}/observations/obsop_name_map.yaml' + apply near surface wind scaling: true + obs filters: +# # Reject all obs with PreQC mark already set above 3 -- filter: PreQC - maxvalue: 3 - action: - name: reject +#- filter: PreQC +# maxvalue: 3 +# action: +# name: reject # -# WindSat (289), ASCAT (290), or OSCAT (291) either wind component or velocity greater than 20 m/s, then reject. +# Assign the initial observation error, based on height/pressure - filter: Bounds Check filter variables: - name: windEastward - name: windNorthward - minvalue: -20 - maxvalue: 20 + minvalue: -135 + maxvalue: 135 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000, 105000, 100000, 95000, 90000, 85000, 80000, 75000, 70000, 65000, + 60000, 55000, 50000, 45000, 40000, 35000, 30000, 25000, 20000, 15000, + 10000, 7500, 5000, 4000, 3000, 2000, 1000] #Pressure (Pa) + errors: [2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0] # -- filter: Bounds Check +- filter: Perform Action filter variables: - name: windEastward - - name: windNorthward - test variables: - - name: ObsFunction/Velocity - maxvalue: 20.0 where: - - variable: - name: ObsType/windEastward - is_in: 289, 290, 291 + - variable: PreUseFlag/windEastward + is_in: 100 action: name: reject # -# Similar to satellite winds AMV, reject when obs wind direction is more than 50 degrees different than model. -- filter: Bounds Check +- filter: Perform Action filter variables: - - name: windEastward - name: windNorthward - test variables: - - name: ObsFunction/WindDirAngleDiff - maxvalue: 50.0 + where: + - variable: PreUseFlag/windNorthward + is_in: 100 action: name: reject # -# ASCAT (290), RAPIDSCAT (296) use a LNVD check. -- filter: Bounds Check +#wgu +- filter: BlackList filter variables: - name: windEastward - - name: windNorthward - test variables: - - name: ObsFunction/SatWindsLNVDCheck - maxvalue: 3 where: - - variable: - name: ObsType/windEastward - is_in: 290, 296 + - variable: + name: ObsType/windEastward + is_in: 290 action: - name: reject + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windEastward + inflation factor: 4.0 + adjusted_error_name: GsiAdjustObsError # -# Assign the initial observation error (constant value, 3.5 m/s right now). - filter: BlackList filter variables: - - name: windEastward - name: windNorthward + where: + - variable: + name: ObsType/windNorthward + is_in: 290 action: - name: assign error - error parameter: 3.5 + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4.0 + adjusted_error_name: GsiAdjustObsError # -# For OSCAT, ASCAT, RAPIDSCAT, or WindSat, reject when either wind component differs by at least 6 m/s from model. -- filter: Background Check +- filter: Bounds Check filter variables: - name: windEastward - name: windNorthward - absolute threshold: 6.0 + test variables: + - name: ObsFunction/ScatWindsAmbiguityCheck + maxvalue: 1.e-12 where: - variable: name: ObsType/windEastward - is_in: 289, 290, 291, 296 + is_in: 290 action: name: reject -# + +# Bounds checks. Filter U obs if |v_O-F| > 5, and V obs if |u_O-F| > 5. - filter: Bounds Check filter variables: - name: windEastward - action: - name: reject - maxvalue: 5.0 + - name: windNorthward test variables: - - name: ObsFunction/ObsErrorFactorQuotient + - name: ObsFunction/Arithmetic options: - numerator: - name: ObsErrorData/windEastward # After inflation step - denominator: - name: ObsError/windEastward + variables: + - name: ObsValue/windEastward + - name: HofX/windEastward + coefs: [1.0, -1.0] + minvalue: -5.0 + maxvalue: 5.0 # - filter: Bounds Check filter variables: + - name: windEastward - name: windNorthward + test variables: + - name: ObsFunction/Arithmetic + options: + variables: + - name: ObsValue/windNorthward + - name: HofX/windNorthward + coefs: [1.0, -1.0] + minvalue: -5.0 + maxvalue: 5.0 +# +# Gross Check +- filter: Background Check + filter variables: + - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + wndtype: [ 290 ] + cgross: [ 5.0 ] + error_min: [ 1.4 ] + error_max: [ 6.1 ] + variable: windEastward action: name: reject - maxvalue: 5.0 - test variables: - - name: ObsFunction/ObsErrorFactorQuotient + defer to post: true +# +- filter: Background Check + filter variables: + - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck options: - numerator: - name: ObsErrorData/windNorthward # After inflation step - denominator: - name: ObsError/windNorthward + wndtype: [ 290 ] + cgross: [ 5.0 ] + error_min: [ 1.4 ] + error_max: [ 6.1 ] + variable: windNorthward + action: + name: reject + defer to post: true + diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ufo_tests.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ufo_tests.yaml index 12d8f64b..c8fc4400 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ufo_tests.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ufo_tests.yaml @@ -187,7 +187,7 @@ scatwind: operator_test: rms ref: 1.0e-1 filter_test: - passedBenchmark: 1 + passedBenchmark: 16384 sfcship: operator_test: From be27fa59c63501ede0c7f7a4da8162188a78791b Mon Sep 17 00:00:00 2001 From: Akira Sewnath Date: Wed, 26 Jul 2023 14:45:07 -0400 Subject: [PATCH 111/121] UFO Tests - Satwind (#193) * x48 satwind yaml * clean up and benchmark change * removing @s * yaml fix --- .../geos_atmosphere/observations/satwind.yaml | 437 ++++++++++-------- .../observations/ufo_tests.yaml | 2 +- 2 files changed, 257 insertions(+), 182 deletions(-) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/satwind.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/satwind.yaml index 8a50d727..f875a5d3 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/satwind.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/satwind.yaml @@ -1,6 +1,7 @@ obs operator: name: VertInterp observation alias file: '{{experiment_root}}/{{experiment_id}}/configuration/jedi/interfaces/{{model_component}}/observations/obsop_name_map.yaml' + apply near surface wind scaling: true obs space: name: satwind obsdatain: @@ -19,14 +20,34 @@ obs filters: maxvalue: 3 action: name: reject + +- filter: Perform Action + filter variables: + - name: windEastward + where: + - variable: PreUseFlag/windEastward + is_in: 100 + action: + name: reject + +- filter: Perform Action + filter variables: + - name: windNorthward + where: + - variable: PreUseFlag/windNorthward + is_in: 100 + action: + name: reject + # # Assign the initial observation error, based on height/pressure -- filter: Bounds Check +# add back +- filter: Perform Action filter variables: - name: windEastward - name: windNorthward - minvalue: -135 - maxvalue: 135 +# minvalue: -135 +# maxvalue: 135 action: name: assign error error function: @@ -35,146 +56,225 @@ obs filters: xvar: name: MetaData/pressure xvals: [100000, 95000, 80000, 65000, 60000, 55000, 50000, 45000, 40000, 35000, 30000, 25000, 20000, 15000, 10000] #Pressure (Pa) - errors: [1.4, 1.5, 1.6, 1.8, 1.9, 2.0, 2.1, 2.3, 2.6, 2.8, 3.0, 3.2, 2.7, 2.4, 2.1] + errors: [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2] + +# #Blacklist Obstypes that aren't used in GSI +- filter: BlackList + where: + - variable: + name: ObsType/windEastward + is_in: 240, 241, 248, 249, 251, 255, 256, 260 # -# Observation Range Sanity Check: either wind component or velocity exceeds 135 m/s -- filter: Bounds Check +# Assign the initial observation error, based on height/pressure +# kx = 244 +# reject pressure levels = [110000, 105000, 7500, 5000, 4000, 3000, 2000, 1000, 500, 400, 300, 200, 100, 0] +- filter: Perform Action filter variables: - name: windEastward - name: windNorthward - minvalue: -135 - maxvalue: 135 action: - name: reject -- filter: Bounds Check + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [100000, 95000, 90000, 85000, 80000, 75000, 70000, 65000, 60000, 55000, 50000, 45000, 40000, 35000, 30000, 25000, 20000, 15000, 10000] #Pressure (Pa) + errors: [5.9, 5.9, 5.9, 5.9, 5.45, 5, 5, 5, 5.075, 5.175, 5.3, 5.675, 6.05, 6.25, 6.625, 7, 7, 7, 7] + where: + - variable: + name: ObsType/windEastward + is_in: 244 + + + # Assign the initial observation error, based on height/pressure +- filter: Perform Action filter variables: - name: windEastward - name: windNorthward - test variables: - - name: ObsFunction/Velocity - maxvalue: 135.0 action: - name: reject -# -# All satellite platforms, reject when pressure greater than 950 mb. -- filter: Bounds Check + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000, 105000, 100000, 95000, 90000, 85000, 80000, 75000, 70000, 65000, 60000, 55000, 50000, 45000, 40000, 35000, 30000, 25000, 20000, 15000, 10000, 7500, 5000, 4000, 3000, 2000, 1000, 500, 400, 300, 200, 100, 0] #Pressure (Pa) + errors: [3.8, 3.8, 3.8, 3.8, 3.8, 3.8, 3.8, 3.8, 3.9, 3.9, 4.0, 4.0, 4.1, 5, 6, 6.3, 6.6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7] + where: + - variable: + name: ObsType/windEastward + is_in: 245, 246 + + +# Assign the initial observation error, based on height/pressure +- filter: Perform Action filter variables: - name: windEastward - name: windNorthward - test variables: - - name: MetaData/pressure - maxvalue: 95000 action: - name: reject -# -# Difference check surface_pressure and air_pressure@ObsValue, if less than 100 hPa, reject. -# Starting with 730029 values, 338418 missing (half?), 50883 rejected by difference check, leaving 340728 -- filter: Difference Check + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000, 105000, 100000, 95000, 90000, 85000, 80000, 75000, 70000, 65000, 60000, 55000, 50000, 45000, 40000, 35000, 30000, 25000, 20000, 15000, 10000, 7500, 5000, 4000, 3000, 2000, 1000, 500, 400, 300, 200, 100, 0] #Pressure (Pa) + errors: [3.8, 3.8, 3.8, 3.8, 3.8, 3.8, 3.8, 3.8, 3.9, 3.9, 4.0, 4.0, 4.1, 5, 6, 6.3, 6.6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7] + where: + - variable: + name: ObsType/windEastward + is_in: 242, 243, 247, 252, 253 + +# Assign the initial observation error, based on height/pressure +# kx = 254 +- filter: Perform Action filter variables: - name: windEastward - name: windNorthward - reference: surface_pressure@GeoVaLs - value: pressure@MetaData - maxvalue: -10000 -# -# Multiple satellite platforms, reject when pressure is more than 50 mb above tropopause. -- filter: Difference Check + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000, 105000, 100000, 95000, 90000, 85000, 80000, 75000, 70000, 65000, 60000, 55000, 50000, 45000, 40000, 35000, 30000, 25000, 20000, 15000, 10000, 7500, 5000, 4000, 3000, 2000, 1000, 500, 400, 300, 200, 100, 0] #Pressure (Pa) + errors: [3.8, 3.8, 3.8, 3.8, 3.8, 3.8, 3.8, 3.8, 3.9, 3.9, 4.0, 4.5, 6.1, 6.0, 6.5, 7.3, 7.6, 7, 7.5, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7] + where: + - variable: + name: ObsType/windEastward + is_in: 254 + +# Assign the initial observation error, based on height/pressure +# kx = 250 +- filter: Perform Action filter variables: - name: windEastward - name: windNorthward - reference: TropopauseEstimate@ObsFunction - value: pressure@MetaData - minvalue: -5000 # 50 hPa above tropopause level, negative p-diff action: - name: reject -# -# GOES WV (non-cloudy; itype=247) reject when difference of wind direction is more than 50 degrees. -- filter: Bounds Check + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [110000, 105000, 100000, 95000, 90000, 85000, 80000, 75000, 70000, 65000, 60000, 55000, 50000, 45000, 40000, 35000, 30000, 25000, 20000, 15000, 10000, 7500, 5000, 4000, 3000, 2000, 1000, 500, 400, 300, 200, 100, 0] #Pressure (Pa) + errors: [3.8, 3.8, 3.8, 3.8, 3.8, 3.8, 3.8, 3.8, 3.9, 3.9, 4.0, 4.0, 4.1, 5.0, 7, 7.3, 7.6, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8] + where: + - variable: + name: ObsType/windEastward + is_in: 250 + +# Assign the initial observation error, based on height/pressure +# kx = 257 +# reject [110000, 105000, 7500, 5000, 4000, 3000, 2000, 1000, 500, 400, 300, 200, 100, 0] +- filter: Perform Action filter variables: - name: windEastward - name: windNorthward - test variables: - - name: ObsFunction/WindDirAngleDiff - maxvalue: 50.0 action: - name: reject -# -# GOES IR (245), EUMET IR (253), JMA IR (252) reject when pressure between 400 and 800 mb. -- filter: Bounds Check + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [100000, 95000, 90000, 85000, 80000, 75000, 70000, 65000, 60000, 55000, 50000, 45000, 40000, 35000, 30000, 25000, 20000, 15000, 10000] #Pressure (Pa) + errors: [5.9, 5.9, 5.9, 5.9, 5.9, 5.9, 5.9, 5.9, 5.675, 5.45, 5.525, 5.8, 6.275, 6.575, 6.825, 7.05, 7.05, 7.05, 7.05] + where: + - variable: + name: ObsType/windEastward + is_in: 257 + +# Assign the initial observation error, based on height/pressure +# kx = 258 +# reject = [110000, 105000, 100000, 95000, 90000, 85000, 80000, 75000, 70000, 65000, 7500, 5000, 4000, 3000, 2000, 1000, 500, 400, 300, 200, 100, 0] +- filter: Perform Action filter variables: - name: windEastward - name: windNorthward - test variables: - - name: MetaData/pressure - minvalue: 40000 - maxvalue: 80000 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [60000, 55000, 50000, 45000, 40000, 35000, 30000, 25000, 20000, 15000, 10000] #Pressure (Pa) + errors: [6.05, 5.625, 5.275, 5.375, 5.925, 6.475, 6.775, 7.05, 7.05, 7.05, 7.05] where: - variable: name: ObsType/windEastward - is_in: 245, 252, 253 - action: - name: reject -# -# GOES WV (246, 250, 254), reject when pressure greater than 400 mb. -- filter: Bounds Check + is_in: 258 + +# Assign the initial observation error, based on height/pressure +# kx = 257 +# reject [110000, 105000, 7500, 5000, 4000, 3000, 2000, 1000, 500, 400, 300, 200, 100, 0] +- filter: Perform Action filter variables: - name: windEastward - name: windNorthward - test variables: - - name: MetaData/pressure - maxvalue: 40000 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + xvar: + name: MetaData/pressure + xvals: [100000, 95000, 90000, 85000, 80000, 75000, 70000, 65000, 60000, 55000, 50000, 45000, 40000, 35000, 30000, 25000, 20000, 15000, 10000] #Pressure (Pa) + errors: [6.5, 6.5, 6.5, 6.5, 6.5, 6.5, 6.5, 6.5, 6, 5.5, 5.5, 5.575, 6.025, 6.4, 6.775, 7.15, 7.15, 7.15, 7.15] where: - variable: name: ObsType/windEastward - is_in: 246, 250, 254 - action: - name: reject -# -# EUMET (242) and JMA (243) vis, reject when pressure less than 700 mb. -- filter: Bounds Check + is_in: 259 + +- filter: Difference Check filter variables: - name: windEastward - name: windNorthward - test variables: - - name: MetaData/pressure - minvalue: 70000 + reference: GeoVaLs/surface_pressure + value: MetaData/pressure + maxvalue: -11000 where: - variable: name: ObsType/windEastward - is_in: 242, 243 - action: - name: reject + is_in: 247 + - variable: + name: MetaData/surfaceQualifier + minvalue: 1 # -# MODIS-Aqua/Terra (257) and (259), reject when pressure less than 250 mb. -- filter: Bounds Check +- filter: Difference Check filter variables: - name: windEastward - name: windNorthward - test variables: - - name: MetaData/pressure - minvalue: 25000 + reference: GeoVaLs/surface_pressure + value: MetaData/pressure + maxvalue: -20000 where: - variable: name: ObsType/windEastward - is_in: 257-259 - action: - name: reject -# -# MODIS-Aqua/Terra (258) and (259), reject when pressure greater than 600 mb. + is_in: 244, 257, 258, 259, 260 + - variable: + name: MetaData/surfaceQualifier + minvalue: 1 + +# GOES WV (non-cloudy; itype=247) reject when difference of wind direction is more than 50 degrees. - filter: Bounds Check filter variables: - name: windEastward - name: windNorthward test variables: - - name: MetaData/pressure - maxvalue: 60000 + - name: ObsFunction/WindDirAngleDiff + maxvalue: 50.0 where: - variable: name: ObsType/windEastward - is_in: 258, 259 + is_in: 247 action: name: reject # +# # AVHRR (244), MODIS (257,258,259), VIIRS (260), GOES (247) use a LNVD check. - filter: Bounds Check filter variables: @@ -190,131 +290,106 @@ obs filters: action: name: reject # -# AVHRR and MODIS (ObsType=244,257,258,259) use a SPDB check. -- filter: Bounds Check + +#wgu +- filter: BlackList filter variables: - name: windEastward - - name: windNorthward - test variables: - - name: ObsFunction/SatWindsSPDBCheck - options: - error_min: 1.4 - error_max: 20.0 - maxvalue: 1.75 where: - variable: name: ObsType/windEastward - is_in: 244, 257, 258, 259 + is_in: 240-260 action: - name: reject -# -# GOES (ObsType=245,246,253,254) use a SPDB check only between 300-400 mb. -- filter: Bounds Check + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windEastward + inflation factor: 4 + adjusted_error_name: GsiAdjustObsError + +- filter: BlackList filter variables: - - name: windEastward - name: windNorthward - test variables: - - name: ObsFunction/SatWindsSPDBCheck - options: - error_min: 1.4 - error_max: 20.0 - maxvalue: 1.75 where: - variable: - name: ObsType/windEastward - is_in: 244, 257, 258, 259 - - variable: - name: MetaData/pressure - minvalue: 30000 - maxvalue: 40000 - action: - name: reject -# -- filter: Background Check - filter variables: - - name: windEastward - - name: windNorthward - absolute threshold: 7.5 + name: ObsType/windNorthward + is_in: 240-260 action: name: inflate error - inflation factor: 3.0 - defer to post: true -# -# If the total inflation factor is too big, reject. -- filter: Bounds Check + inflation variable: + name: ObsFunction/ObsErrorFactorPressureCheck + options: + variable: windNorthward + inflation factor: 4 + adjusted_error_name: GsiAdjustObsError + +#wgu +# Gross Check +- filter: Background Check filter variables: - name: windEastward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + error_min: [ 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, + 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] + error_max: [ 6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, + 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1] + cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5, + 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5] + wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, + 250, 251, 252, 253, 254, 256, 257, 258, 259, 260] + variable: windEastward action: name: reject - maxvalue: 2.5 - test variables: - - name: ObsFunction/ObsErrorFactorQuotient - options: - numerator: - name: ObsErrorData/windEastward # After inflation step - denominator: - name: ObsError/windEastward - where: - - variable: - name: ObsType/windEastward - is_in: 240, 241, 242, 244, 247, 248, 249, 250, 252, 255-260 - defer to post: true -# -- filter: Bounds Check + +- filter: Background Check filter variables: - name: windNorthward + function absolute threshold: + - name: ObsFunction/WindsSPDBCheck + options: + cgross: [ 2.5, 2.5, 2.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5, + 2.5, 1.3, 2.5, 1.5, 1.5, 2.5, 2.5, 2.5, 2.5, 2.5] + error_min: [ 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, + 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4, 1.4] + error_max: [ 6.1, 6.1, 15.0, 15.0, 20.0, 20.0, 20.0, 20.0, 20.0, 20.0, + 20.0, 20.0, 20.0, 20.0, 20.0, 20.1, 20.1, 20.1, 20.1, 20.1] + wndtype: [ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, + 250, 251, 252, 253, 254, 256, 257, 258, 259, 260] + variable: windNorthward action: name: reject - maxvalue: 2.5 - test variables: - - name: ObsFunction/ObsErrorFactorQuotient - options: - numerator: - name: ObsErrorData/windNorthward # After inflation step - denominator: - name: ObsError/windNorthward - where: - - variable: - name: ObsType/windNorthward - is_in: 240, 241, 242, 244, 247, 248, 249, 250, 252, 255-260 - defer to post: true -# -# Some satellite platforms have a lower threshold of inflation factor of 1.5 -- filter: Bounds Check + +- filter: BlackList filter variables: - name: windEastward action: - name: reject - maxvalue: 1.5 - test variables: - - name: ObsFunction/ObsErrorFactorQuotient - options: - numerator: - name: ObsErrorData/windEastward # After inflation step - denominator: - name: ObsError/windEastward + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windEastward where: - - variable: - name: ObsType/windEastward - is_in: 243, 245, 246, 251, 253, 254 + - variable: + name: ObsType/windEastward + is_in: 240-260 defer to post: true -# -- filter: Bounds Check + +- filter: BlackList filter variables: - name: windNorthward action: - name: reject - maxvalue: 1.5 - test variables: - - name: ObsFunction/ObsErrorFactorQuotient - options: - numerator: - name: ObsErrorData/windNorthward # After inflation step - denominator: - name: ObsError/windNorthward + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: true + variable: windNorthward where: - - variable: - name: ObsType/windEastward - is_in: 243, 245, 246, 251, 253, 254 + - variable: + name: ObsType/windNorthward + is_in: 240-260 defer to post: true - diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ufo_tests.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ufo_tests.yaml index c8fc4400..ab3b4ab9 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ufo_tests.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ufo_tests.yaml @@ -181,7 +181,7 @@ satwind: operator_test: rms ref: 1.0e-1 filter_test: - passedBenchmark: 1 + passedBenchmark: 231948 scatwind: operator_test: From 63173564c3ac6f17c5e60e06b1dc710ee5b9a7b0 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 26 Jul 2023 14:46:41 -0400 Subject: [PATCH 112/121] add some missing instruments --- ...vhrr3_metop-a.yaml => avhrr3_metop-b.yaml} | 72 +++---- .../observations/avhrr3_n19.yaml | 181 ++++++++++++++++++ .../{iasi_metop-a.yaml => iasi_metop-c.yaml} | 72 +++---- ...uites-convert_ncdiags-geos_atmosphere.yaml | 21 +- .../suites-convert_ncdiags.yaml | 6 + src/swell/tasks/gsi_ncdiag_to_ioda.py | 2 +- 6 files changed, 272 insertions(+), 82 deletions(-) rename src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/{avhrr3_metop-a.yaml => avhrr3_metop-b.yaml} (69%) create mode 100644 src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_n19.yaml rename src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/{iasi_metop-a.yaml => iasi_metop-c.yaml} (94%) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_metop-a.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_metop-b.yaml similarity index 69% rename from src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_metop-a.yaml rename to src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_metop-b.yaml index b8ba9860..f3e2446a 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_metop-a.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_metop-b.yaml @@ -1,32 +1,32 @@ obs space: - name: avhrr3_metop-a + name: avhrr3_metop-b obsdatain: engine: type: H5File - obsfile: '{{cycle_dir}}/avhrr3_metop-a.{{window_begin}}.nc4' + obsfile: '{{cycle_dir}}/avhrr3_metop-b.{{window_begin}}.nc4' obsdataout: engine: type: H5File - obsfile: '{{cycle_dir}}/{{experiment_id}}.avhrr3_metop-a.{{window_begin}}.nc4' + obsfile: '{{cycle_dir}}/{{experiment_id}}.avhrr3_metop-b.{{window_begin}}.nc4' simulated variables: [brightnessTemperature] - channels: &avhrr3_metop-a_channels 3,4,5 + channels: &avhrr3_metop-b_channels 3,4,5 obs operator: name: CRTM Absorbers: [H2O,O3,CO2] obs options: - Sensor_ID: &Sensor_ID avhrr3_metop-a + Sensor_ID: &Sensor_ID avhrr3_metop-b EndianType: little_endian CoefficientPath: '{{crtm_coeff_dir}}' obs bias: - input file: '{{cycle_dir}}/avhrr3_metop-a.{{background_time}}.satbias.nc4' + input file: '{{cycle_dir}}/avhrr3_metop-b.{{background_time}}.satbias.nc4' variational bc: predictors: - name: constant - name: lapse_rate order: 2 - tlapse: &avhrr3_metop-a_tlapse '{{cycle_dir}}/avhrr3_metop-a.{{background_time}}.tlapse.txt' + tlapse: &avhrr3_metop-b_tlapse '{{cycle_dir}}/avhrr3_metop-b.{{background_time}}.tlapse.txt' - name: lapse_rate - tlapse: *avhrr3_metop-a_tlapse + tlapse: *avhrr3_metop-b_tlapse - name: emissivity - name: scan_angle order: 4 @@ -53,32 +53,32 @@ obs filters: - filter: BlackList filter variables: - name: brightnessTemperature - channels: *avhrr3_metop-a_channels + channels: *avhrr3_metop-b_channels action: name: inflate error inflation variable: name: ObsFunction/ObsErrorFactorWavenumIR - channels: *avhrr3_metop-a_channels + channels: *avhrr3_metop-b_channels options: - channels: *avhrr3_metop-a_channels + channels: *avhrr3_metop-b_channels # Topography Check - filter: BlackList filter variables: - name: brightnessTemperature - channels: *avhrr3_metop-a_channels + channels: *avhrr3_metop-b_channels action: name: inflate error inflation variable: name: ObsFunction/ObsErrorFactorTopoRad - channels: *avhrr3_metop-a_channels + channels: *avhrr3_metop-b_channels options: - channels: *avhrr3_metop-a_channels - sensor: avhrr3_metop-a + channels: *avhrr3_metop-b_channels + sensor: avhrr3_metop-b # Observation Range Sanity Check - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *avhrr3_metop-a_channels + channels: *avhrr3_metop-b_channels minvalue: 0.00001 maxvalue: 1000.0 action: @@ -87,24 +87,24 @@ obs filters: - filter: BlackList filter variables: - name: brightnessTemperature - channels: *avhrr3_metop-a_channels + channels: *avhrr3_metop-b_channels action: name: inflate error inflation variable: name: ObsFunction/ObsErrorFactorTransmitTopRad - channels: *avhrr3_metop-a_channels + channels: *avhrr3_metop-b_channels options: - channels: *avhrr3_metop-a_channels + channels: *avhrr3_metop-b_channels # Cloud Detection Check - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *avhrr3_metop-a_channels + channels: *avhrr3_metop-b_channels test variables: - name: ObsFunction/CloudDetectMinResidualAVHRR - channels: *avhrr3_metop-a_channels + channels: *avhrr3_metop-b_channels options: - channels: *avhrr3_metop-a_channels + channels: *avhrr3_metop-b_channels use_flag: [ 1, 1, 1 ] use_flag_clddet: [ 1, 1, 1 ] obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] @@ -115,12 +115,12 @@ obs filters: - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *avhrr3_metop-a_channels + channels: *avhrr3_metop-b_channels test variables: - name: ObsFunction/NearSSTRetCheckIR - channels: *avhrr3_metop-a_channels + channels: *avhrr3_metop-b_channels options: - channels: *avhrr3_metop-a_channels + channels: *avhrr3_metop-b_channels use_flag: [ 1, 1, 1 ] obserr_demisf: [0.01,0.02,0.03,0.02,0.03] obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] @@ -131,14 +131,14 @@ obs filters: - filter: BlackList filter variables: - name: brightnessTemperature - channels: *avhrr3_metop-a_channels + channels: *avhrr3_metop-b_channels action: name: inflate error inflation variable: name: ObsFunction/ObsErrorFactorSurfJacobianRad - channels: *avhrr3_metop-a_channels + channels: *avhrr3_metop-b_channels options: - channels: *avhrr3_metop-a_channels + channels: *avhrr3_metop-b_channels sensor: *Sensor_ID obserr_demisf: [0.01, 0.02, 0.03, 0.02, 0.03] obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] @@ -146,21 +146,21 @@ obs filters: - filter: Background Check filter variables: - name: brightnessTemperature - channels: *avhrr3_metop-a_channels + channels: *avhrr3_metop-b_channels function absolute threshold: - name: ObsFunction/ObsErrorBoundIR - channels: *avhrr3_metop-a_channels + channels: *avhrr3_metop-b_channels options: - channels: *avhrr3_metop-a_channels + channels: *avhrr3_metop-b_channels obserr_bound_latitude: name: ObsFunction/ObsErrorFactorLatRad options: latitude_parameters: [25.0, 0.5, 0.04, 1.0] obserr_bound_transmittop: name: ObsFunction/ObsErrorFactorTransmitTopRad - channels: *avhrr3_metop-a_channels + channels: *avhrr3_metop-b_channels options: - channels: *avhrr3_metop-a_channels + channels: *avhrr3_metop-b_channels obserr_bound_max: [ 6.0, 6.0, 6.0 ] action: name: reject @@ -168,12 +168,12 @@ obs filters: - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *avhrr3_metop-a_channels + channels: *avhrr3_metop-b_channels test variables: - name: ObsFunction/ChannelUseflagCheckRad - channels: *avhrr3_metop-a_channels + channels: *avhrr3_metop-b_channels options: - channels: *avhrr3_metop-a_channels + channels: *avhrr3_metop-b_channels use_flag: [ 1, 1, 1 ] minvalue: 1.0e-12 action: diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_n19.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_n19.yaml new file mode 100644 index 00000000..ae520a1b --- /dev/null +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_n19.yaml @@ -0,0 +1,181 @@ +obs space: + name: avhrr3_n19 + obsdatain: + engine: + type: H5File + obsfile: '{{cycle_dir}}/avhrr3_n19.{{window_begin}}.nc4' + obsdataout: + engine: + type: H5File + obsfile: '{{cycle_dir}}/{{experiment_id}}.avhrr3_n19.{{window_begin}}.nc4' + simulated variables: [brightnessTemperature] + channels: &avhrr3_n19_channels 3,4,5 +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: &Sensor_ID avhrr3_n19 + EndianType: little_endian + CoefficientPath: '{{crtm_coeff_dir}}' +obs bias: + input file: '{{cycle_dir}}/avhrr3_n19.{{background_time}}.satbias.nc4' + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &avhrr3_n19_tlapse '{{cycle_dir}}/avhrr3_n19.{{background_time}}.tlapse.txt' + - name: lapse_rate + tlapse: *avhrr3_n19_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle +obs filters: +# Wavenumber Check +- filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 3 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 88.9999 + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 1.0e-12 + action: + name: reject +- filter: BlackList + filter variables: + - name: brightnessTemperature + channels: *avhrr3_n19_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorWavenumIR + channels: *avhrr3_n19_channels + options: + channels: *avhrr3_n19_channels +# Topography Check +- filter: BlackList + filter variables: + - name: brightnessTemperature + channels: *avhrr3_n19_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: *avhrr3_n19_channels + options: + channels: *avhrr3_n19_channels + sensor: avhrr3_n19 +# Observation Range Sanity Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *avhrr3_n19_channels + minvalue: 0.00001 + maxvalue: 1000.0 + action: + name: reject +# Transmittance Top Check +- filter: BlackList + filter variables: + - name: brightnessTemperature + channels: *avhrr3_n19_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *avhrr3_n19_channels + options: + channels: *avhrr3_n19_channels +# Cloud Detection Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *avhrr3_n19_channels + test variables: + - name: ObsFunction/CloudDetectMinResidualAVHRR + channels: *avhrr3_n19_channels + options: + channels: *avhrr3_n19_channels + use_flag: [ 1, 1, 1 ] + use_flag_clddet: [ 1, 1, 1 ] + obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] + maxvalue: 1.0e-12 + action: + name: reject +# NSST Retrieval Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *avhrr3_n19_channels + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: *avhrr3_n19_channels + options: + channels: *avhrr3_n19_channels + use_flag: [ 1, 1, 1 ] + obserr_demisf: [0.01,0.02,0.03,0.02,0.03] + obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] + maxvalue: 1.0e-12 + action: + name: reject +# Surface Jacobians Check +- filter: BlackList + filter variables: + - name: brightnessTemperature + channels: *avhrr3_n19_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: *avhrr3_n19_channels + options: + channels: *avhrr3_n19_channels + sensor: *Sensor_ID + obserr_demisf: [0.01, 0.02, 0.03, 0.02, 0.03] + obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] +# Gross check +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *avhrr3_n19_channels + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: *avhrr3_n19_channels + options: + channels: *avhrr3_n19_channels + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: [25.0, 0.5, 0.04, 1.0] + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *avhrr3_n19_channels + options: + channels: *avhrr3_n19_channels + obserr_bound_max: [ 6.0, 6.0, 6.0 ] + action: + name: reject +# Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *avhrr3_n19_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *avhrr3_n19_channels + options: + channels: *avhrr3_n19_channels + use_flag: [ 1, 1, 1 ] + minvalue: 1.0e-12 + action: + name: reject + diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-a.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-c.yaml similarity index 94% rename from src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-a.yaml rename to src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-c.yaml index 6515ca2c..8daf2f97 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-a.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-c.yaml @@ -1,15 +1,15 @@ obs space: - name: iasi_metop-a + name: iasi_metop-c obsdatain: engine: type: H5File - obsfile: '{{cycle_dir}}/iasi_metop-a.{{window_begin}}.nc4' + obsfile: '{{cycle_dir}}/iasi_metop-c.{{window_begin}}.nc4' obsdataout: engine: type: H5File - obsfile: '{{cycle_dir}}/{{experiment_id}}.iasi_metop-a.{{window_begin}}.nc4' + obsfile: '{{cycle_dir}}/{{experiment_id}}.iasi_metop-c.{{window_begin}}.nc4' simulated variables: [brightnessTemperature] - channels: &iasi_metop-a_channels 16, 29, 32, 35, 38, 41, 44, 47, 49, 50, 51, 53, + channels: &iasi_metop-c_channels 16, 29, 32, 35, 38, 41, 44, 47, 49, 50, 51, 53, 55, 56, 57, 59, 61, 62, 63, 66, 68, 70, 72, 74, 76, 78, 79, 81, 82, 83, 84, 85, 86, 87, 89, 92, 93, 95, 97, 99, 101, 103, 104, 106, 109, 110, 111, 113, 116, 119, 122, 125, 128, 131, 133, 135, 138, 141, 144, 146, @@ -62,19 +62,19 @@ obs operator: name: CRTM Absorbers: [H2O,O3,CO2] obs options: - Sensor_ID: &Sensor_ID iasi_metop-a + Sensor_ID: &Sensor_ID iasi_metop-c EndianType: little_endian CoefficientPath: '{{crtm_coeff_dir}}' obs bias: - input file: '{{cycle_dir}}/iasi_metop-a.{{background_time}}.satbias.nc4' + input file: '{{cycle_dir}}/iasi_metop-c.{{background_time}}.satbias.nc4' variational bc: predictors: - name: constant - name: lapse_rate order: 2 - tlapse: &iasi_metop-a_tlapse '{{cycle_dir}}/iasi_metop-a.{{background_time}}.tlapse.txt' + tlapse: &iasi_metop-c_tlapse '{{cycle_dir}}/iasi_metop-c.{{background_time}}.tlapse.txt' - name: lapse_rate - tlapse: *iasi_metop-a_tlapse + tlapse: *iasi_metop-c_tlapse - name: emissivity - name: scan_angle order: 4 @@ -106,19 +106,19 @@ obs filters: - filter: BlackList filter variables: - name: brightnessTemperature - channels: *iasi_metop-a_channels + channels: *iasi_metop-c_channels action: name: inflate error inflation variable: name: ObsFunction/ObsErrorFactorWavenumIR - channels: *iasi_metop-a_channels + channels: *iasi_metop-c_channels options: - channels: *iasi_metop-a_channels + channels: *iasi_metop-c_channels # Observation Range Sanity Check - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *iasi_metop-a_channels + channels: *iasi_metop-c_channels minvalue: 50.00001 maxvalue: 449.99999 action: @@ -127,37 +127,37 @@ obs filters: - filter: BlackList filter variables: - name: brightnessTemperature - channels: *iasi_metop-a_channels + channels: *iasi_metop-c_channels action: name: inflate error inflation variable: name: ObsFunction/ObsErrorFactorTopoRad - channels: *iasi_metop-a_channels + channels: *iasi_metop-c_channels options: - channels: *iasi_metop-a_channels - sensor: iasi_metop-a + channels: *iasi_metop-c_channels + sensor: iasi_metop-c # Transmittance Top Check - filter: BlackList filter variables: - name: brightnessTemperature - channels: *iasi_metop-a_channels + channels: *iasi_metop-c_channels action: name: inflate error inflation variable: name: ObsFunction/ObsErrorFactorTransmitTopRad - channels: *iasi_metop-a_channels + channels: *iasi_metop-c_channels options: - channels: *iasi_metop-a_channels + channels: *iasi_metop-c_channels # Cloud Detection Check - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *iasi_metop-a_channels + channels: *iasi_metop-c_channels test variables: - name: ObsFunction/CloudDetectMinResidualIR - channels: *iasi_metop-a_channels + channels: *iasi_metop-c_channels options: - channels: *iasi_metop-a_channels + channels: *iasi_metop-c_channels use_flag: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, @@ -290,12 +290,12 @@ obs filters: - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *iasi_metop-a_channels + channels: *iasi_metop-c_channels test variables: - name: ObsFunction/NearSSTRetCheckIR - channels: *iasi_metop-a_channels + channels: *iasi_metop-c_channels options: - channels: *iasi_metop-a_channels + channels: *iasi_metop-c_channels use_flag: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, @@ -367,14 +367,14 @@ obs filters: - filter: BlackList filter variables: - name: brightnessTemperature - channels: *iasi_metop-a_channels + channels: *iasi_metop-c_channels action: name: inflate error inflation variable: name: ObsFunction/ObsErrorFactorSurfJacobianRad - channels: *iasi_metop-a_channels + channels: *iasi_metop-c_channels options: - channels: *iasi_metop-a_channels + channels: *iasi_metop-c_channels sensor: *Sensor_ID obserr_demisf: [0.01, 0.02, 0.03, 0.02, 0.03] obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] @@ -382,21 +382,21 @@ obs filters: - filter: Background Check filter variables: - name: brightnessTemperature - channels: *iasi_metop-a_channels + channels: *iasi_metop-c_channels function absolute threshold: - name: ObsFunction/ObsErrorBoundIR - channels: *iasi_metop-a_channels + channels: *iasi_metop-c_channels options: - channels: *iasi_metop-a_channels + channels: *iasi_metop-c_channels obserr_bound_latitude: name: ObsFunction/ObsErrorFactorLatRad options: latitude_parameters: [25.0, 0.5, 0.04, 1.0] obserr_bound_transmittop: name: ObsFunction/ObsErrorFactorTransmitTopRad - channels: *iasi_metop-a_channels + channels: *iasi_metop-c_channels options: - channels: *iasi_metop-a_channels + channels: *iasi_metop-c_channels obserr_bound_max: [ 3.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 4.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, @@ -465,12 +465,12 @@ obs filters: - filter: Bounds Check filter variables: - name: brightnessTemperature - channels: *iasi_metop-a_channels + channels: *iasi_metop-c_channels test variables: - name: ObsFunction/ChannelUseflagCheckRad - channels: *iasi_metop-a_channels + channels: *iasi_metop-c_channels options: - channels: *iasi_metop-a_channels + channels: *iasi_metop-c_channels use_flag: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, diff --git a/src/swell/suites/convert_ncdiags/geos_atmosphere/suites-convert_ncdiags-geos_atmosphere.yaml b/src/swell/suites/convert_ncdiags/geos_atmosphere/suites-convert_ncdiags-geos_atmosphere.yaml index f1b22537..b0cdc529 100644 --- a/src/swell/suites/convert_ncdiags/geos_atmosphere/suites-convert_ncdiags-geos_atmosphere.yaml +++ b/src/swell/suites/convert_ncdiags/geos_atmosphere/suites-convert_ncdiags-geos_atmosphere.yaml @@ -27,22 +27,18 @@ observations: - amsua_n19 - atms_n20 - atms_npp - - avhrr_metop-b - - avhrr_n18 - - avhrr_n19 + - avhrr3_metop-b + - avhrr3_n18 + - avhrr3_n19 - cris-fsr_n20 - cris-fsr_npp - gmi_gpm - - hirs4_n18 - - hirs4_n19 - iasi_metop-b - iasi_metop-c - mhs_metop-b - mhs_metop-c - mhs_n19 - - seviri_m08 - ssmis_f17 - - ssmis_f18 # Ozone - mls55_aura - omi_aura @@ -53,7 +49,7 @@ observations: # Path to GSI ncdiags path_to_gsi_nc_diags: - default_value: '/discover/nobackup/drholdaw/SwellTestData/ufo_testing/ncdiag' + default_value: '/discover/nobackup/drholdaw/SwellTestData/ufo_testing/ncdiag/2021121200' prompt: Path to where the ncdiags will be held type: string @@ -65,7 +61,14 @@ produce_geovals: # Location for bias correction coefficients path_to_gsi_bc_coefficients: - default_value: '/discover/nobackup/drholdaw/SwellTestData/ufo_testing/bc' + default_value: '/discover/nobackup/drholdaw/SwellTestData/ufo_testing/bc/2021121200' prompt: Tar or directory containing BC files type: string +fixed_options: + window_offset: + default_value: PT3H + prompt: Time from beginning to middle of the window + background_time_offset: + default_value: PT9H + prompt: Time before the middle of the window that the background providing forecast began diff --git a/src/swell/suites/convert_ncdiags/suites-convert_ncdiags.yaml b/src/swell/suites/convert_ncdiags/suites-convert_ncdiags.yaml index 29ff0ec9..12d7c58d 100644 --- a/src/swell/suites/convert_ncdiags/suites-convert_ncdiags.yaml +++ b/src/swell/suites/convert_ncdiags/suites-convert_ncdiags.yaml @@ -53,3 +53,9 @@ fixed_options: data_assimilation_run: default_value: true prompt: Does this workflow include data assimilation? + +# R2D2 Configuration +r2d2_local_path: + default_value: {{r2d2_local_path}} + prompt: Enter the path where R2D2 will store experiment output + type: string diff --git a/src/swell/tasks/gsi_ncdiag_to_ioda.py b/src/swell/tasks/gsi_ncdiag_to_ioda.py index b9ea7c4e..a033d24f 100644 --- a/src/swell/tasks/gsi_ncdiag_to_ioda.py +++ b/src/swell/tasks/gsi_ncdiag_to_ioda.py @@ -31,7 +31,7 @@ def execute(self): # Parse configuration # ------------------- observations = self.config.observations() - produce_geovals = False #self.config.produce_geovals() + produce_geovals = self.config.produce_geovals() window_offset = self.config.window_offset() # Get window beginning time From 5560e0d2902e4cb71626306ac823bdad39b3367c Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 26 Jul 2023 16:28:38 -0400 Subject: [PATCH 113/121] add gps and avhrr specific cases --- src/swell/tasks/gsi_ncdiag_to_ioda.py | 31 ++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/src/swell/tasks/gsi_ncdiag_to_ioda.py b/src/swell/tasks/gsi_ncdiag_to_ioda.py index a033d24f..c80f3f30 100644 --- a/src/swell/tasks/gsi_ncdiag_to_ioda.py +++ b/src/swell/tasks/gsi_ncdiag_to_ioda.py @@ -213,9 +213,17 @@ def execute(self): elif len(ioda_file_0_) == 3: self.logger.info(f'Skipping combine for {needed_ioda_type}, single file already.') + else: self.logger.abort(f'Combine failed for {needed_ioda_type}, file name issue.') + # Rename gps files from gps_bend if they exist + if 'gps' in observations_orig: + gps_files = glob.glob(os.path.join(self.cycle_dir(), 'gps_bend*')) + for gps_file in gps_files: + gps_file_newname = os.path.basename(gps_file).replace('gps_bend', 'gps') + os.rename(gps_file, os.path.join(self.cycle_dir(), gps_file_newname)) + # Get list of the observations that are ozone observations # -------------------------------------------------------- ozone_sensors = gsid.oz_lay_sensors + gsid.oz_lev_sensors @@ -225,13 +233,19 @@ def execute(self): if ozone_sensor in observation: ozone_observations.append(observation) - # Copy all the files into the cycle directory - # ------------------------------------------- + # Transform radiances and ozone + # ----------------------------- for observation in observations: self.logger.info(f'Converting {observation} to IODA format') - gsi_obs_file = glob.glob(os.path.join(gsi_diag_dir, f'*{observation}*')) + observation_search_name = copy.copy(observation) + + # For avhrr replace the search with just avhrr + if 'avhrr3' in observation_search_name: + observation_search_name = observation_search_name.replace('avhrr3', 'avhrr') + + gsi_obs_file = glob.glob(os.path.join(gsi_diag_dir, f'*{observation_search_name}*')) # Skip this observation if not files were found if len(gsi_obs_file) == 0: @@ -259,6 +273,15 @@ def execute(self): if observation not in ozone_observations: Diag.close() + # Rename avhrr files + # ------------------ + # Rename gps files from gps_bend if they exist + if any('avhrr3' in item for item in observations): + avhrr_files = glob.glob(os.path.join(self.cycle_dir(), 'avhrr*')) + for avhrr_file in avhrr_files: + avhrr_file_newname = os.path.basename(avhrr_file).replace('avhrr', 'avhrr3') + os.rename(avhrr_file, os.path.join(self.cycle_dir(), avhrr_file_newname)) + # Rename files to be swell compliant # ---------------------------------- for observation in observations_orig: @@ -267,8 +290,6 @@ def execute(self): # Change to gps_bend search_name = observation - if search_name == 'gps': - search_name = 'gps_bend' # Input filename ioda_obs_in_pattern = f'{search_name}_obs_*nc*' From b043030576982b2ec3409448ac7298ec365cfaee Mon Sep 17 00:00:00 2001 From: Jianjun Jin <61251050+gmao-jjin3@users.noreply.github.com> Date: Thu, 27 Jul 2023 11:59:10 -0400 Subject: [PATCH 114/121] Feature/jjin3/ufo tests x0048 micorwave (#197) * Update configurations for microwave data in x0048. * Code norm. * Code norm. * Code norm. * Code norm. * Code norm. --- .../observations/amsua_aqua.yaml | 10 +- .../observations/amsua_metop-b.yaml | 2 +- .../observations/amsua_metop-c.yaml | 2 +- .../observations/amsua_n15.yaml | 2 +- .../observations/amsua_n18.yaml | 2 +- .../observations/amsua_n19.yaml | 2 +- .../observations/atms_n20.yaml | 480 ++++++------------ .../observations/atms_npp.yaml | 480 ++++++------------ .../geos_atmosphere/observations/mhs_n19.yaml | 275 +++++++++- .../observations/ufo_tests.yaml | 24 +- 10 files changed, 627 insertions(+), 652 deletions(-) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_aqua.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_aqua.yaml index 8be16c45..2d97b3b6 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_aqua.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_aqua.yaml @@ -3,11 +3,11 @@ obs space: obsdatain: engine: type: H5File - obsfile: '{{cycle_dir}}/amsua_aqua.{{window_begin}}.nc4' + obsfile: '{{cycle_dir}}/amsua_aqua.{{window_begin}}.nc4' obsdataout: engine: type: H5File - obsfile: '{{cycle_dir}}/{{experiment_id}}.amsua_aqua.{{window_begin}}.nc4' + obsfile: '{{cycle_dir}}/{{experiment_id}}.amsua_aqua.{{window_begin}}.nc4' simulated variables: [brightnessTemperature] channels: &amsua_aqua_channels 1-15 obs operator: @@ -160,9 +160,9 @@ obs filters: [2.500, 2.000, 2.000, 0.500, 0.400, 0.400, 0.500, 0.300, 0.350, 0.350, 0.450, 1.000, 1.500, 3.750, 6.300] - obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, + obserr_bound_max: [4.5, 4.5, 4.5, 3.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, - 2.5, 3.5, 4.5, 4.5, 4.5] + 3.0, 3.5, 4.5, 4.5, 4.5] action: name: reject # Inter-channel check @@ -193,7 +193,7 @@ obs filters: channels: 1-15 options: channels: 1-15 - use passive_bc: true +# use passive_bc: true use_flag: [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1] diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-b.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-b.yaml index 8629943a..f86d2ef0 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-b.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-b.yaml @@ -195,7 +195,7 @@ obs filters: channels: 1-15 options: channels: 1-15 - use passive_bc: true +# use passive_bc: true use_flag: [-1, -1, -1, -1, -1, -1, 1, 1, 1, 1, 1, 1, 1, 1, -1] diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-c.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-c.yaml index e8afaca2..3e4e2939 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-c.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-c.yaml @@ -195,7 +195,7 @@ obs filters: channels: 1-15 options: channels: 1-15 - use passive_bc: true +# use passive_bc: true use_flag: [-1, -1, -1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1] diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n15.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n15.yaml index ecc93397..bb2264a0 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n15.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n15.yaml @@ -195,7 +195,7 @@ obs filters: channels: 1-15 options: channels: 1-15 - use passive_bc: true +# use passive_bc: true use_flag: [-1, -1, -1, 1, 1, -1, 1, 1, 1, 1, -1, 1, 1, -1, -1] diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n18.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n18.yaml index 36e147da..4dfdd00b 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n18.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n18.yaml @@ -195,7 +195,7 @@ obs filters: channels: 1-15 options: channels: 1-15 - use passive_bc: true +# use passive_bc: true use_flag: [-1, -1, -1, 1, -1, 1, 1, -1, -1, 1, 1, 1, 1, 1, -1] diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n19.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n19.yaml index 9a69c50a..6230551a 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n19.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n19.yaml @@ -195,7 +195,7 @@ obs filters: channels: 1-15 options: channels: 1-15 - use passive_bc: true +# use passive_bc: true use_flag: [-1, -1, -1, 1, 1, 1, -1, -1, 1, 1, 1, 1, 1, 1, -1] diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/atms_n20.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/atms_n20.yaml index 9140e7d3..4d7ec8b4 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/atms_n20.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/atms_n20.yaml @@ -13,8 +13,8 @@ obs space: obs operator: name: CRTM Absorbers: [H2O,O3,CO2] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 + linear obs operator: + Absorbers: [H2O,O3,CO2] obs options: Sensor_ID: &Sensor_ID atms_n20 EndianType: little_endian @@ -24,6 +24,10 @@ obs bias: variational bc: predictors: - name: constant + - name: cloud_liquid_water + sensor: ATMS + clwdif_ch238: 1 + clwdif_ch314: 2 - name: lapse_rate order: 2 tlapse: &atms_n20_tlapse '{{cycle_dir}}/atms_n20.{{background_time}}.tlapse.txt' @@ -38,323 +42,171 @@ obs bias: order: 2 - name: scan_angle obs filters: -- filter: BlackList - filter variables: - - name: brightnessTemperature - channels: *atms_n20_channels - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelRamp - channels: *atms_n20_channels +# Window and surface-sensitive channels check + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-7,16-22 + test variables: + - name: ObsValue/brightnessTemperature + channels: 1-7,16 + treat missing as out of bounds: true + minvalue: 100.0 + maxvalue: 500.0 + flag all filter variables if any test variable is out of bounds: true +# passedBenchmark: 1500 +# All channels unreasonable values check + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: atms_n20_channels + minvalue: 100.0 + maxvalue: 500.0 +#Hydrometeor Check (cloud/precipitation affected chanels) + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: atms_n20_channels + test variables: + - name: ObsFunction/HydrometeorCheckAMSUAclr + channels: atms_n20_channels options: - channels: *atms_n20_channels - xvar: - name: ObsFunction/CLWRetSymmetricMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue, HofX] - x0: [ 0.030, 0.030, 0.030, 0.020, 0.030, - 0.080, 0.150, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.000, - 0.020, 0.030, 0.030, 0.030, 0.030, - 0.050, 0.100] - x1: [ 0.350, 0.380, 0.400, 0.450, 0.500, - 1.000, 1.000, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.000, - 0.350, 0.500, 0.500, 0.500, 0.500, - 0.500, 0.500] - err0: [ 4.500, 4.500, 4.500, 2.500, 0.550, - 0.300, 0.300, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, - 4.000, 4.000, 3.500, 3.000, 3.000, - 3.000, 3.000] - err1: [20.000, 25.000, 12.000, 7.000, 3.500, - 3.000, 0.800, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, - 19.000, 30.000, 25.000, 16.500, 12.000, - 9.000, 6.500] -# CLW Retrieval Check -- filter: Bounds Check - filter variables: - - name: brightnessTemperature - channels: 1-7, 16-22 - test variables: - - name: ObsFunction/CLWRetMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue] - maxvalue: 999.0 - action: - name: reject -# CLW Retrieval Check -- filter: Bounds Check - filter variables: - - name: brightnessTemperature - channels: 1-7, 16-22 - test variables: - - name: ObsFunction/CLWRetMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [HofX] - maxvalue: 999.0 - action: - name: reject -# Hydrometeor Check (cloud/precipitation affected chanels) -- filter: Bounds Check - filter variables: - - name: brightnessTemperature - channels: *atms_n20_channels - test variables: - - name: ObsFunction/HydrometeorCheckATMS - channels: *atms_n20_channels - options: - channels: *atms_n20_channels - obserr_clearsky: [ 4.500, 4.500, 4.500, 2.500, 0.550, - 0.300, 0.300, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, - 4.000, 4.000, 3.500, 3.000, 3.000, - 3.000, 3.000] - clwret_function: - name: ObsFunction/CLWRetMW + sensor: *Sensor_ID + channels: atms_n20_channels + test_biaspredictor: cloud_liquid_waterPredictor + maxvalue: 0.0 + action: + name: reject +# Assign obs error + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: atms_n20_channels + action: + name: assign error + error parameter vector: + [5.000, 5.000, 5.000, 3.000, 0.550, + 0.300, 0.300, 0.300, 0.300, 0.300, + 0.350, 0.400, 0.550, 0.800, 5.000, + 5.000, 2.500, 2.500, 2.500, 2.500, + 2.500, 2.500] +# Topography check + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: atms_n20_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: atms_n20_channels options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue] - obserr_function: - name: ObsFunction/ObsErrorModelRamp - channels: *atms_n20_channels + sensor: *Sensor_ID + channels: atms_n20_channels +# Transmittance Top Check + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: atms_n20_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: atms_n20_channels options: - channels: *atms_n20_channels - xvar: - name: ObsFunction/CLWRetSymmetricMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue, HofX] - x0: [ 0.030, 0.030, 0.030, 0.020, 0.030, - 0.080, 0.150, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.000, - 0.020, 0.030, 0.030, 0.030, 0.030, - 0.050, 0.100] - x1: [ 0.350, 0.380, 0.400, 0.450, 0.500, - 1.000, 1.000, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.000, - 0.350, 0.500, 0.500, 0.500, 0.500, - 0.500, 0.500] - err0: [ 4.500, 4.500, 4.500, 2.500, 0.550, - 0.300, 0.300, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, - 4.000, 4.000, 3.500, 3.000, 3.000, - 3.000, 3.000] - err1: [20.000, 25.000, 12.000, 7.000, 3.500, - 3.000, 0.800, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, - 19.000, 30.000, 25.000, 16.500, 12.000, - 9.000, 6.500] - maxvalue: 0.0 - action: - name: reject -# Topography check -- filter: BlackList - filter variables: - - name: brightnessTemperature - channels: *atms_n20_channels - action: - name: inflate error - inflation variable: - name: ObsFunction/ObsErrorFactorTopoRad - channels: *atms_n20_channels - options: - sensor: atms_n20 - channels: *atms_n20_channels -# Transmittnace Top Check -- filter: BlackList - filter variables: - - name: brightnessTemperature - channels: *atms_n20_channels - action: - name: inflate error - inflation variable: - name: ObsFunction/ObsErrorFactorTransmitTopRad - channels: *atms_n20_channels - options: - channels: *atms_n20_channels -# Surface Jacobian check -- filter: BlackList - filter variables: - - name: brightnessTemperature - channels: *atms_n20_channels - action: - name: inflate error - inflation variable: - name: ObsFunction/ObsErrorFactorSurfJacobianRad - channels: *atms_n20_channels + channels: atms_n20_channels +# Surface Jacobian check + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: atms_n20_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: atms_n20_channels + options: + channels: atms_n20_channels + use_biasterm: true + test_biasterm: ObsBiasTerm + sensor: *Sensor_ID + obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] + obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] +# Gross check + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: atms_n20_channels + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: atms_n20_channels options: - channels: *atms_n20_channels sensor: *Sensor_ID - obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] - obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] -# Situation dependent Check -- filter: BlackList - filter variables: - - name: brightnessTemperature - channels: *atms_n20_channels - action: - name: inflate error - inflation variable: - name: ObsFunction/ObsErrorFactorSituDependMW - channels: *atms_n20_channels - options: - sensor: atms_n20 - channels: *atms_n20_channels - clwobs_function: - name: ObsFunction/CLWRetMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue] - clwbkg_function: - name: ObsFunction/CLWRetMW + channels: atms_n20_channels + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [HofX] - scatobs_function: - name: ObsFunction/SCATRetMW + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: atms_n20_channels options: - scatret_ch238: 1 - scatret_ch314: 2 - scatret_ch890: 16 - scatret_types: [ObsValue] - clwmatchidx_function: - name: ObsFunction/CLWMatchIndexMW - channels: *atms_n20_channels + channels: atms_n20_channels + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: atms_n20_channels options: - channels: *atms_n20_channels - clwobs_function: - name: ObsFunction/CLWRetMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue] - clwbkg_function: - name: ObsFunction/CLWRetMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [HofX] - clwret_clearsky: [ 0.030, 0.030, 0.030, 0.020, 0.030, - 0.080, 0.150, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.000, - 0.020, 0.030, 0.030, 0.030, 0.030, - 0.050, 0.100] - obserr_clearsky: [ 4.500, 4.500, 4.500, 2.500, 0.550, - 0.300, 0.300, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, - 4.000, 4.000, 3.500, 3.000, 3.000, - 3.000, 3.000] -# Gross check -- filter: Background Check - filter variables: - - name: brightnessTemperature - channels: *atms_n20_channels - function absolute threshold: - - name: ObsFunction/ObsErrorBoundMW - channels: *atms_n20_channels - options: - sensor: atms_n20 - channels: *atms_n20_channels - obserr_bound_latitude: - name: ObsFunction/ObsErrorFactorLatRad - options: - latitude_parameters: [25.0, 0.25, 0.04, 3.0] - obserr_bound_transmittop: - name: ObsFunction/ObsErrorFactorTransmitTopRad - channels: *atms_n20_channels - options: - channels: *atms_n20_channels - obserr_bound_topo: - name: ObsFunction/ObsErrorFactorTopoRad - channels: *atms_n20_channels - options: - channels: *atms_n20_channels - sensor: atms_n20 - obserr_function: - name: ObsFunction/ObsErrorModelRamp - channels: *atms_n20_channels - options: - channels: *atms_n20_channels - xvar: - name: ObsFunction/CLWRetSymmetricMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue, HofX] - x0: [ 0.030, 0.030, 0.030, 0.020, 0.030, - 0.080, 0.150, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.000, - 0.020, 0.030, 0.030, 0.030, 0.030, - 0.050, 0.100] - x1: [ 0.350, 0.380, 0.400, 0.450, 0.500, - 1.000, 1.000, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.000, - 0.350, 0.500, 0.500, 0.500, 0.500, - 0.500, 0.500] - err0: [ 4.500, 4.500, 4.500, 2.500, 0.550, - 0.300, 0.300, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, - 4.000, 4.000, 3.500, 3.000, 3.000, - 3.000, 3.000] - err1: [20.000, 25.000, 12.000, 7.000, 3.500, - 3.000, 0.800, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, - 19.000, 30.000, 25.000, 16.500, 12.000, - 9.000, 6.500] - obserr_bound_max: [4.5, 4.5, 3.0, 3.0, 1.0, - 1.0, 1.0, 1.0, 1.0, 1.0, - 1.0, 1.0, 1.0, 2.0, 4.5, - 4.5, 2.0, 2.0, 2.0, 2.0, - 2.0, 2.0] - action: - name: reject -# Inter-channel check -- filter: Bounds Check - filter variables: - - name: brightnessTemperature - channels: *atms_n20_channels - test variables: - - name: ObsFunction/InterChannelConsistencyCheck - channels: *atms_n20_channels - options: - channels: *atms_n20_channels - sensor: atms_n20 - use_flag: [ 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, - 1, 1, 1, 1, -1, - 1, 1, 1, 1, 1, - 1, 1] - maxvalue: 1.0e-12 - action: - name: reject -# Useflag check -- filter: Bounds Check - filter variables: - - name: brightnessTemperature - channels: *atms_n20_channels - test variables: - - name: ObsFunction/ChannelUseflagCheckRad - channels: *atms_n20_channels - options: - channels: *atms_n20_channels - use_flag: [ 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, - 1, 1, 1, 1, -1, - 1, 1, 1, 1, 1, - 1, 1] - minvalue: 1.0e-12 - action: - name: reject + channels: atms_n20_channels + sensor: *Sensor_ID + error parameter vector: + [5.000, 5.000, 5.000, 3.000, 0.550, + 0.300, 0.300, 0.300, 0.300, 0.300, + 0.350, 0.400, 0.550, 0.800, 5.000, + 5.000, 2.500, 2.500, 2.500, 2.500, + 2.500, 2.500] + obserr_bound_max: [4.5, 4.5, 3.0, 3.0, 1.0, + 1.0, 1.0, 1.0, 1.0, 1.0, + 1.0, 1.0, 1.0, 2.0, 4.5, + 4.5, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0] + action: + name: reject +# Inter-channel check + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: atms_n20_channels + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: atms_n20_channels + options: + channels: atms_n20_channels + use passive_bc: true + sensor: *Sensor_ID + use_flag: [-1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + -1, 1, 1, 1, 1, + 1, 1] + maxvalue: 1.0e-12 + action: + name: reject +#Useflag check + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: atms_n20_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: atms_n20_channels + options: + channels: atms_n20_channels + use passive_bc: true + use_flag: [-1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + -1, 1, 1, 1, 1, + 1, 1] + minvalue: 1.0e-12 + action: + name: reject diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/atms_npp.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/atms_npp.yaml index 1cb1fffe..62be3a1c 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/atms_npp.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/atms_npp.yaml @@ -13,8 +13,8 @@ obs space: obs operator: name: CRTM Absorbers: [H2O,O3,CO2] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 + linear obs operator: + Absorbers: [H2O,O3,CO2] obs options: Sensor_ID: &Sensor_ID atms_npp EndianType: little_endian @@ -24,6 +24,10 @@ obs bias: variational bc: predictors: - name: constant + - name: cloud_liquid_water + sensor: ATMS + clwdif_ch238: 1 + clwdif_ch314: 2 - name: lapse_rate order: 2 tlapse: &atms_npp_tlapse '{{cycle_dir}}/atms_npp.{{background_time}}.tlapse.txt' @@ -38,323 +42,171 @@ obs bias: order: 2 - name: scan_angle obs filters: -- filter: BlackList - filter variables: - - name: brightnessTemperature - channels: *atms_npp_channels - action: - name: assign error - error function: - name: ObsFunction/ObsErrorModelRamp - channels: *atms_npp_channels +# Window and surface-sensitive channels check + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-7,16-22 + test variables: + - name: ObsValue/brightnessTemperature + channels: 1-7,16 + treat missing as out of bounds: true + minvalue: 100.0 + maxvalue: 500.0 + flag all filter variables if any test variable is out of bounds: true +# passedBenchmark: 1500 +# All channels unreasonable values check + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: atms_npp_channels + minvalue: 100.0 + maxvalue: 500.0 +#Hydrometeor Check (cloud/precipitation affected chanels) + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: atms_npp_channels + test variables: + - name: ObsFunction/HydrometeorCheckAMSUAclr + channels: atms_npp_channels options: - channels: *atms_npp_channels - xvar: - name: ObsFunction/CLWRetSymmetricMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue, HofX] - x0: [ 0.030, 0.030, 0.030, 0.020, 0.030, - 0.080, 0.150, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.000, - 0.020, 0.030, 0.030, 0.030, 0.030, - 0.050, 0.100] - x1: [ 0.350, 0.380, 0.400, 0.450, 0.500, - 1.000, 1.000, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.000, - 0.350, 0.500, 0.500, 0.500, 0.500, - 0.500, 0.500] - err0: [ 4.500, 4.500, 4.500, 2.500, 0.550, - 0.300, 0.300, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, - 4.000, 4.000, 3.500, 3.000, 3.000, - 3.000, 3.000] - err1: [20.000, 25.000, 12.000, 7.000, 3.500, - 3.000, 0.800, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, - 19.000, 30.000, 25.000, 16.500, 12.000, - 9.000, 6.500] -# CLW Retrieval Check -- filter: Bounds Check - filter variables: - - name: brightnessTemperature - channels: 1-7, 16-22 - test variables: - - name: ObsFunction/CLWRetMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue] - maxvalue: 999.0 - action: - name: reject -# CLW Retrieval Check -- filter: Bounds Check - filter variables: - - name: brightnessTemperature - channels: 1-7, 16-22 - test variables: - - name: ObsFunction/CLWRetMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [HofX] - maxvalue: 999.0 - action: - name: reject -# Hydrometeor Check (cloud/precipitation affected chanels) -- filter: Bounds Check - filter variables: - - name: brightnessTemperature - channels: *atms_npp_channels - test variables: - - name: ObsFunction/HydrometeorCheckATMS - channels: *atms_npp_channels - options: - channels: *atms_npp_channels - obserr_clearsky: [ 4.500, 4.500, 4.500, 2.500, 0.550, - 0.300, 0.300, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, - 4.000, 4.000, 3.500, 3.000, 3.000, - 3.000, 3.000] - clwret_function: - name: ObsFunction/CLWRetMW + sensor: *Sensor_ID + channels: atms_npp_channels + test_biaspredictor: cloud_liquid_waterPredictor + maxvalue: 0.0 + action: + name: reject +# Assign obs error + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: atms_npp_channels + action: + name: assign error + error parameter vector: + [5.000, 5.000, 5.000, 3.000, 0.550, + 0.300, 0.300, 0.300, 0.300, 0.300, + 0.350, 0.400, 0.550, 0.800, 5.000, + 5.000, 2.500, 2.500, 2.500, 2.500, + 2.500, 2.500] +# Topography check + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: atms_npp_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: atms_npp_channels options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue] - obserr_function: - name: ObsFunction/ObsErrorModelRamp - channels: *atms_npp_channels + sensor: *Sensor_ID + channels: atms_npp_channels +# Transmittance Top Check + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: atms_npp_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: atms_npp_channels options: - channels: *atms_npp_channels - xvar: - name: ObsFunction/CLWRetSymmetricMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue, HofX] - x0: [ 0.030, 0.030, 0.030, 0.020, 0.030, - 0.080, 0.150, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.000, - 0.020, 0.030, 0.030, 0.030, 0.030, - 0.050, 0.100] - x1: [ 0.350, 0.380, 0.400, 0.450, 0.500, - 1.000, 1.000, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.000, - 0.350, 0.500, 0.500, 0.500, 0.500, - 0.500, 0.500] - err0: [ 4.500, 4.500, 4.500, 2.500, 0.550, - 0.300, 0.300, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, - 4.000, 4.000, 3.500, 3.000, 3.000, - 3.000, 3.000] - err1: [20.000, 25.000, 12.000, 7.000, 3.500, - 3.000, 0.800, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, - 19.000, 30.000, 25.000, 16.500, 12.000, - 9.000, 6.500] - maxvalue: 0.0 - action: - name: reject -# Topography check -- filter: BlackList - filter variables: - - name: brightnessTemperature - channels: *atms_npp_channels - action: - name: inflate error - inflation variable: - name: ObsFunction/ObsErrorFactorTopoRad - channels: *atms_npp_channels - options: - sensor: atms_npp - channels: *atms_npp_channels -# Transmittnace Top Check -- filter: BlackList - filter variables: - - name: brightnessTemperature - channels: *atms_npp_channels - action: - name: inflate error - inflation variable: - name: ObsFunction/ObsErrorFactorTransmitTopRad - channels: *atms_npp_channels - options: - channels: *atms_npp_channels -# Surface Jacobian check -- filter: BlackList - filter variables: - - name: brightnessTemperature - channels: *atms_npp_channels - action: - name: inflate error - inflation variable: - name: ObsFunction/ObsErrorFactorSurfJacobianRad - channels: *atms_npp_channels + channels: atms_npp_channels +# Surface Jacobian check + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: atms_npp_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: atms_npp_channels + options: + channels: atms_npp_channels + use_biasterm: true + test_biasterm: ObsBiasTerm + sensor: *Sensor_ID + obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] + obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] +# Gross check + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: atms_npp_channels + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: atms_npp_channels options: - channels: *atms_npp_channels sensor: *Sensor_ID - obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] - obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] -# Situation dependent Check -- filter: BlackList - filter variables: - - name: brightnessTemperature - channels: *atms_npp_channels - action: - name: inflate error - inflation variable: - name: ObsFunction/ObsErrorFactorSituDependMW - channels: *atms_npp_channels - options: - sensor: atms_npp - channels: *atms_npp_channels - clwobs_function: - name: ObsFunction/CLWRetMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue] - clwbkg_function: - name: ObsFunction/CLWRetMW + channels: atms_npp_channels + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [HofX] - scatobs_function: - name: ObsFunction/SCATRetMW + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: atms_npp_channels options: - scatret_ch238: 1 - scatret_ch314: 2 - scatret_ch890: 16 - scatret_types: [ObsValue] - clwmatchidx_function: - name: ObsFunction/CLWMatchIndexMW - channels: *atms_npp_channels + channels: atms_npp_channels + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: atms_npp_channels options: - channels: *atms_npp_channels - clwobs_function: - name: ObsFunction/CLWRetMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue] - clwbkg_function: - name: ObsFunction/CLWRetMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [HofX] - clwret_clearsky: [ 0.030, 0.030, 0.030, 0.020, 0.030, - 0.080, 0.150, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.000, - 0.020, 0.030, 0.030, 0.030, 0.030, - 0.050, 0.100] - obserr_clearsky: [ 4.500, 4.500, 4.500, 2.500, 0.550, - 0.300, 0.300, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, - 4.000, 4.000, 3.500, 3.000, 3.000, - 3.000, 3.000] -# Gross check -- filter: Background Check - filter variables: - - name: brightnessTemperature - channels: *atms_npp_channels - function absolute threshold: - - name: ObsFunction/ObsErrorBoundMW - channels: *atms_npp_channels - options: - sensor: atms_npp - channels: *atms_npp_channels - obserr_bound_latitude: - name: ObsFunction/ObsErrorFactorLatRad - options: - latitude_parameters: [25.0, 0.25, 0.04, 3.0] - obserr_bound_transmittop: - name: ObsFunction/ObsErrorFactorTransmitTopRad - channels: *atms_npp_channels - options: - channels: *atms_npp_channels - obserr_bound_topo: - name: ObsFunction/ObsErrorFactorTopoRad - channels: *atms_npp_channels - options: - channels: *atms_npp_channels - sensor: atms_npp - obserr_function: - name: ObsFunction/ObsErrorModelRamp - channels: *atms_npp_channels - options: - channels: *atms_npp_channels - xvar: - name: ObsFunction/CLWRetSymmetricMW - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue, HofX] - x0: [ 0.030, 0.030, 0.030, 0.020, 0.030, - 0.080, 0.150, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.000, - 0.020, 0.030, 0.030, 0.030, 0.030, - 0.050, 0.100] - x1: [ 0.350, 0.380, 0.400, 0.450, 0.500, - 1.000, 1.000, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.000, - 0.350, 0.500, 0.500, 0.500, 0.500, - 0.500, 0.500] - err0: [ 4.500, 4.500, 4.500, 2.500, 0.550, - 0.300, 0.300, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, - 4.000, 4.000, 3.500, 3.000, 3.000, - 3.000, 3.000] - err1: [20.000, 25.000, 12.000, 7.000, 3.500, - 3.000, 0.800, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 3.000, - 19.000, 30.000, 25.000, 16.500, 12.000, - 9.000, 6.500] - obserr_bound_max: [4.5, 4.5, 3.0, 3.0, 1.0, - 1.0, 1.0, 1.0, 1.0, 1.0, - 1.0, 1.0, 1.0, 2.0, 4.5, - 4.5, 2.0, 2.0, 2.0, 2.0, - 2.0, 2.0] - action: - name: reject -# Inter-channel check -- filter: Bounds Check - filter variables: - - name: brightnessTemperature - channels: *atms_npp_channels - test variables: - - name: ObsFunction/InterChannelConsistencyCheck - channels: *atms_npp_channels - options: - channels: *atms_npp_channels - sensor: atms_npp - use_flag: [ 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, - 1, 1, 1, 1, -1, - 1, 1, 1, 1, 1, - 1, 1] - maxvalue: 1.0e-12 - action: - name: reject -# Useflag check -- filter: Bounds Check - filter variables: - - name: brightnessTemperature - channels: *atms_npp_channels - test variables: - - name: ObsFunction/ChannelUseflagCheckRad - channels: *atms_npp_channels - options: - channels: *atms_npp_channels - use_flag: [ 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, - 1, 1, 1, 1, -1, - 1, 1, 1, 1, 1, - 1, 1] - minvalue: 1.0e-12 - action: - name: reject + channels: atms_npp_channels + sensor: *Sensor_ID + error parameter vector: + [5.000, 5.000, 5.000, 3.000, 0.550, + 0.300, 0.300, 0.300, 0.300, 0.300, + 0.350, 0.400, 0.550, 0.800, 5.000, + 5.000, 2.500, 2.500, 2.500, 2.500, + 2.500, 2.500] + obserr_bound_max: [4.5, 4.5, 3.0, 3.0, 1.0, + 1.0, 1.0, 1.0, 1.0, 1.0, + 1.0, 1.0, 1.0, 2.0, 4.5, + 4.5, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0] + action: + name: reject +# Inter-channel check + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: atms_npp_channels + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: atms_npp_channels + options: + channels: atms_npp_channels + use passive_bc: true + sensor: *Sensor_ID + use_flag: [-1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + -1, 1, 1, 1, 1, + 1, 1] + maxvalue: 1.0e-12 + action: + name: reject +#Useflag check + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: atms_npp_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: atms_npp_channels + options: + channels: atms_npp_channels + use passive_bc: true + use_flag: [-1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + -1, 1, 1, 1, 1, + 1, 1] + minvalue: 1.0e-12 + action: + name: reject diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mhs_n19.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mhs_n19.yaml index c06291d2..21cdf9c4 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mhs_n19.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mhs_n19.yaml @@ -9,12 +9,17 @@ obs space: type: H5File obsfile: '{{cycle_dir}}/{{experiment_id}}.mhs_n19.{{window_begin}}.nc4' simulated variables: [brightnessTemperature] - channels: 1-5 + channels: &mhs_n19_channels 1-5 obs operator: name: CRTM Absorbers: [H2O,O3,CO2] + Clouds: [Water, Ice, Rain, Snow] + Cloud_Fraction: 1.0 + linear obs operator: + Absorbers: [H2O,O3,CO2] + Clouds: [Water] obs options: - Sensor_ID: mhs_n19 + Sensor_ID: &Sensor_ID mhs_n19 EndianType: little_endian CoefficientPath: '{{crtm_coeff_dir}}' obs bias: @@ -35,3 +40,269 @@ obs bias: - name: scan_angle order: 2 - name: scan_angle +obs filters: +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *mhs_n19_channels + minvalue: 50.0 + maxvalue: 550.0 +- filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: mhs_n19_channels + where: + - variable: + name: MetaData/sensorScanPosition + minvalue: 10 + maxvalue: 81 +- filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 1-3 + where: + - variable: + name: MetaData/latitude + minvalue: -25.0 + maxvalue: -10.0 + - variable: + name: MetaData/longitude + minvalue: 260.0 + maxvalue: 300.0 + action: + name: reject +- filter: BlackList + filter variables: + - name: brightnessTemperature + channels: mhs_n19_channels + where: + - variable: + name: GeoVaLs/surface_snow_area_fraction + minvalue: 0.01 +# ice surface +- filter: BlackList + filter variables: + - name: brightnessTemperature + channels: mhs_n19_channels + where: + - variable: + name: GeoVaLs/ice_area_fraction + minvalue: 0.01 +# mixed surface +- filter: BlackList + filter variables: + - name: brightnessTemperature + channels: mhs_n19_channels + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + - variable: + name: GeoVaLs/land_area_fraction + maxvalue: 0.99 +- filter: BlackList + filter variables: + - name: brightnessTemperature + channels: mhs_n19_channels + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + - variable: + name: GeoVaLs/average_surface_temperature_within_field_of_view + maxvalue: 275 +- filter: BlackList + filter variables: + - name: brightnessTemperature + channels: mhs_n19_channels + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelRamp + channels: mhs_n19_channels + options: + channels: mhs_n19_channels + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: [ObsValue, HofX] + bias_application: HofX + test_bias: ObsBiasData + x0: [ 0.050, 0.050, 0.050, + 0.050, 0.050] + x1: [ 25.00, 25.00, 25.00, + 25.00, 25.00] + err0: [300.00, 300.00, 2.500, + 2.000, 2.000] + err1: [700.00, 700.00, 30.00, + 50.00, 60.00] +- filter: BlackList + filter variables: + - name: brightnessTemperature + channels: mhs_n19_channels + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelRamp + channels: mhs_n19_channels + options: + channels: mhs_n19_channels + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: [ObsValue, HofX] + bias_application: HofX + test_bias: ObsBiasData + x0: [ 0.050, 0.050, 0.050, + 0.050, 0.050] + x1: [ 25.00, 25.00, 25.00, + 25.00, 25.00] + err0: [300.00, 300.00, 2.500, + 2.000, 2.000] + err1: [350.00, 350.00, 15.00, + 25.00, 30.00] +#Topography check +- filter: BlackList + filter variables: + - name: brightnessTemperature + channels: *mhs_n19_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: *mhs_n19_channels + options: + sensor: *Sensor_ID + channels: *mhs_n19_channels +#Transmittnace Top Check +- filter: BlackList + filter variables: + - name: brightnessTemperature + channels: *mhs_n19_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *mhs_n19_channels + options: + channels: *mhs_n19_channels +#Gross check +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *mhs_n19_channels + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: *mhs_n19_channels + options: + sensor: *Sensor_ID + channels: *mhs_n19_channels + threshold: 2.0 + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: [0.0, 1.0, 0.0, 1.0] + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *mhs_n19_channels + options: + channels: *mhs_n19_channels + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: *mhs_n19_channels + options: + channels: *mhs_n19_channels + sensor: *Sensor_ID + obserr_function: + name: ObsFunction/ObsErrorModelRamp + channels: *mhs_n19_channels + options: + channels: *mhs_n19_channels + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: [ObsValue, HofX] + bias_application: HofX + x0: [ 0.050, 0.050, 0.050, + 0.050, 0.050] + x1: [ 25.00, 25.00, 25.00, + 25.00, 25.00] + err0: [300.00, 300.00, 2.500, + 2.000, 2.000] + err1: [700.00, 700.00, 30.00, + 50.00, 60.00] + obserr_bound_max: [5.0, 5.0, 10.0, 10.0, 10.0] + action: + name: reject +#Gross check +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *mhs_n19_channels + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: *mhs_n19_channels + options: + sensor: *Sensor_ID + channels: *mhs_n19_channels + threshold: 2.0 + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: [0.0, 1.0, 0.0, 1.0] + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *mhs_n19_channels + options: + channels: *mhs_n19_channels + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: *mhs_n19_channels + options: + channels: *mhs_n19_channels + sensor: *Sensor_ID + obserr_function: + name: ObsFunction/ObsErrorModelRamp + channels: *mhs_n19_channels + options: + channels: *mhs_n19_channels + xvar: + name: ObsFunction/CLWRetSymmetricMW + options: + clwret_ch89v: 1 + clwret_ch166v: 2 + clwret_types: [ObsValue, HofX] + bias_application: HofX + x0: [ 0.050, 0.050, 0.050, + 0.050, 0.050] + x1: [ 25.00, 25.00, 25.00, + 25.00, 25.00] + err0: [300.00, 300.00, 2.500, + 2.000, 2.000] + err1: [350.00, 350.00, 15.00, + 25.00, 30.00] + obserr_bound_max: [5.0, 5.0, 10.0, 10.0, 10.0] + action: + name: reject diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ufo_tests.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ufo_tests.yaml index ab3b4ab9..7792d6e4 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ufo_tests.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ufo_tests.yaml @@ -31,13 +31,13 @@ amsr2_gcom-w1: operator_test: rms ref: 1.0e-1 filter_test: - passedBenchmark: 1 + passedBenchmark: 54558 amsua_aqua: operator_test: rms ref: 1.0e-1 filter_test: - passedBenchmark: 1 + passedBenchmark: 0 amsua_metop-a: operator_test: @@ -49,43 +49,43 @@ amsua_metop-b: operator_test: rms ref: 1.0e-1 filter_test: - passedBenchmark: 1 + passedBenchmark: 71950 amsua_metop-c: operator_test: rms ref: 1.0e-1 filter_test: - passedBenchmark: 1 + passedBenchmark: 91419 amsua_n15: operator_test: rms ref: 1.0e-1 filter_test: - passedBenchmark: 1 + passedBenchmark: 56139 amsua_n18: operator_test: rms ref: 1.0e-1 filter_test: - passedBenchmark: 1 + passedBenchmark: 69693 amsua_n19: operator_test: rms ref: 1.0e-1 filter_test: - passedBenchmark: 1 + passedBenchmark: 73421 atms_n20: operator_test: rms ref: 1.0e-1 filter_test: - passedBenchmark: 1 + passedBenchmark: 159928 atms_npp: operator_test: rms ref: 1.0e-1 filter_test: - passedBenchmark: 1 + passedBenchmark: 158687 avhrr3_metop-a: operator_test: @@ -139,19 +139,19 @@ mhs_metop-b: operator_test: rms ref: 1.0e-1 filter_test: - passedBenchmark: 1 + passedBenchmark: 29630 mhs_metop-c: operator_test: rms ref: 1.0e-1 filter_test: - passedBenchmark: 1 + passedBenchmark: 29815 mhs_n19: operator_test: rms ref: 1.0e-1 filter_test: - passedBenchmark: 1 + passedBenchmark: 27823 mls55_aura: operator_test: From a5bd3374f84ecc7e1798cf41949b5518b72dc18c Mon Sep 17 00:00:00 2001 From: Wei Gu <59488002+gmao-wgu@users.noreply.github.com> Date: Thu, 27 Jul 2023 12:00:58 -0400 Subject: [PATCH 115/121] YAML files for CrIS-FSR(n20,npp), AIRS & SSMIS (#200) Co-authored-by: Dan Holdaway <27729500+danholdaway@users.noreply.github.com> --- .../observations/airs_aqua.yaml | 305 ++++---- .../observations/cris-fsr_n20.yaml | 368 +++++---- .../observations/cris-fsr_npp.yaml | 368 +++++---- .../observations/ssmis_f17.yaml | 714 ++++++++++++++++-- 4 files changed, 1150 insertions(+), 605 deletions(-) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/airs_aqua.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/airs_aqua.yaml index 35ded4fe..e8c03174 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/airs_aqua.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/airs_aqua.yaml @@ -1,5 +1,14 @@ +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + linear obs operator: + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: &Sensor_ID airs_aqua + EndianType: little_endian + CoefficientPath: '{{crtm_coeff_dir}}' obs space: - name: airs_aqua + name: *Sensor_ID obsdatain: engine: type: H5File @@ -31,13 +40,6 @@ obs space: 2122, 2123, 2128, 2134, 2141, 2145, 2149, 2153, 2164, 2189, 2197, 2209, 2226, 2234, 2280, 2318, 2321, 2325, 2328, 2333, 2339, 2348, 2353, 2355, 2357, 2363, 2370, 2371, 2377 -obs operator: - name: CRTM - Absorbers: [H2O,O3,CO2] - obs options: - Sensor_ID: &Sensor_ID airs_aqua - EndianType: little_endian - CoefficientPath: '{{crtm_coeff_dir}}' obs bias: input file: '{{cycle_dir}}/airs_aqua.{{background_time}}.satbias.nc4' variational bc: @@ -57,6 +59,41 @@ obs bias: order: 2 - name: scan_angle obs filters: +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *airs_aqua_channels + action: + name: assign error + error parameter vector: [ 1.2000, 1.2000, 1.3136, 1.4000, 1.4000, 1.2639, 1.4000, 1.4000, 1.1802, 1.2517, + 1.1719, 1.2000, 1.1728, 1.1442, 1.2000, 1.2000, 1.1500, 1.0801, 1.1500, 1.1500, + 1.0396, 1.1500, 1.1500, 1.1500, 1.1500, 1.1500, 1.1500, 1.1500, 0.9946, 1.0500, + 0.9217, 2.0000, 2.0000, 2.0000, 2.0000, 2.0000, 2.0000, 2.0000, 2.0000, 2.0000, + 2.0000, 2.0000, 2.0000, 0.9591, 0.9465, 0.9593, 0.9337, 1.0000, 0.9861, 1.0017, + 1.1000, 1.0083, 1.0024, 1.1000, 0.9967, 1.0094, 0.9412, 1.1000, 0.9980, 0.9807, + 0.8570, 0.8727, 0.8114, 0.8790, 0.8710, 0.8853, 0.7937, 0.8243, 0.8000, 0.8016, + 0.8000, 0.7781, 0.7475, 0.8500, 0.7405, 0.7150, 0.7416, 0.7465, 0.9000, 0.7198, + 0.7157, 0.9000, 0.7270, 0.7246, 0.7040, 0.7039, 0.6600, 0.6694, 0.6669, 0.7031, + 0.6977, 0.6488, 0.6653, 0.9000, 0.6265, 0.6220, 0.6308, 0.6297, 0.6210, 0.6225, + 0.6229, 0.6234, 0.6238, 0.6332, 0.6425, 0.7028, 0.6152, 0.9000, 0.7257, 0.7288, + 1.1500, 0.9000, 0.6673, 0.7473, 0.6767, 0.7056, 0.9000, 0.9500, 0.7271, 0.9500, + 0.7250, 0.7601, 0.6973, 0.7573, 0.6011, 0.6060, 0.9000, 0.6635, 0.5860, 0.5766, + 0.7500, 2.0386, 0.7500, 1.0000, 0.9000, 0.9000, 0.9000, 0.9000, 0.9000, 0.9000, + 1.0000, 1.3386, 1.0000, 1.0000, 0.8500, 0.9500, 1.7386, 0.9500, 0.9000, 0.8000, + 1.7386, 0.7500, 0.7500, 0.7500, 0.8000, 0.7500, 0.8000, 0.9000, 0.7500, 0.8000, + 0.8000, 1.1000, 0.7500, 1.1000, 0.7500, 0.5991, 0.5348, 0.6541, 0.7421, 0.6192, + 0.8186, 1.0616, 0.8848, 1.0240, 2.5000, 1.0249, 1.0795, 1.2199, 2.5000, 2.5000, + 1.3103, 1.3603, 2.5000, 2.5000, 2.5000, 1.3230, 2.5000, 2.5000, 2.5000, 1.4406, + 2.5000, 2.5000, 1.3965, 2.5000, 2.5000, 2.5000, 2.5000, 2.5000, 2.5000, 2.5000, + 2.5000, 1.6997, 2.5000, 2.5000, 2.5000, 2.5000, 2.5000, 1.6264, 2.5000, 2.5000, + 2.5000, 1.3436, 2.5000, 2.5000, 0.5727, 0.6838, 0.5994, 0.5178, 0.5145, 0.5470, + 0.5572, 0.5002, 0.4974, 0.5500, 0.4953, 0.4883, 0.4948, 0.5446, 0.5777, 1.5000, + 1.5000, 3.0000, 3.0000, 2.5000, 2.5000, 2.0000, 1.0000, 1.5000, 1.5000, 1.8000, + 0.6000, 0.7000, 0.6500, 0.6750, 0.7000, 0.7500, 0.7750, 0.8000, 0.8000, 0.8500, + 0.8500, 0.8500, 0.7000, 0.7000, 0.7000, 0.7000, 0.7000, 0.7000, 0.7000, 0.7250, + 0.7500, 0.7750, 0.8000, 0.8250, 0.8000, 0.8000, 0.8000, 0.7500, 0.8000, 0.8000, + 0.8000, 0.8000, 0.8000, 0.8500, 0.8000, 0.8000, 2.5000, 0.7500, 0.7500, 0.7500, + 0.7500 ] # Wavenumber Check - filter: BlackList filter variables: @@ -90,7 +127,7 @@ obs filters: - name: brightnessTemperature channels: *airs_aqua_channels minvalue: 50.00001 - maxvalue: 449.99999 + maxvalue: 549.99999 action: name: reject # Topography Check @@ -128,64 +165,64 @@ obs filters: channels: *airs_aqua_channels options: channels: *airs_aqua_channels - use_flag: [ -1, -1, 1, -1, -1, 1, -1, -1, 1, 1, - 1, -1, 1, 1, -1, -1, -1, 1, -1, -1, - 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, - 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 1, 1, 1, 1, -1, 1, 1, - -1, 1, 1, -1, 1, 1, 1, -1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, - -1, 1, 1, -1, 1, 1, 1, 1, -1, 1, - 1, -1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, -1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, -1, 1, 1, - -1, 1, 1, 1, 1, 1, 1, -1, 1, -1, - 1, 1, 1, 1, 1, 1, -1, 1, 1, 1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, -1, 1, 1, 1, -1, -1, - 1, 1, -1, -1, -1, 1, -1, -1, -1, 1, - -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, - -1, 1, -1, -1, -1, -1, -1, 1, -1, -1, - -1, 1, -1, -1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, -1, 1, 1, 1, 1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1 ] - use_flag_clddet: [ -1, -1, 1, -1, -1, 1, -1, -1, 1, 1, - 1, -1, 1, 1, -1, -1, -1, 1, -1, -1, - 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, - 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 1, 1, 1, 1, -1, 1, 1, - -1, 1, 1, -1, 1, 1, 1, -1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, - -1, 1, 1, -1, 1, 1, 1, 1, -1, 1, - 1, -1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, -1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, -1, 1, 1, - -1, 1, 1, 1, 1, 1, 1, -1, 1, -1, - 1, 1, 1, 1, 1, 1, -1, 1, 1, 1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, -1, 1, 1, 1, -1, -1, - 1, 1, -1, -1, -1, 1, -1, -1, -1, 1, - -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, - -1, 1, -1, -1, -1, -1, -1, 1, -1, -1, - -1, 1, -1, -1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, -1, 1, 1, 1, 1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1 ] + use_flag: &useflag_airs_aqua [ -1, -1, 1, -1, -1, 1, -1, -1, 1, 1, + 1, -1, 1, 1, -1, -1, -1, 1, -1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, -1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, -1, 1, 1, + -1, -1, 1, 1, 1, 1, -1, -1, 1, -1, + 1, 1, 1, 1, 1, 1, -1, 1, 1, 1, + -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 1, -1, -1, -1, -1, 1, -1, -1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, -1, 1, 1, 1, -1, -1, + 1, 1, -1, -1, -1, 1, -1, -1, -1, 1, + -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, + -1, 1, -1, -1, -1, -1, -1, 1, -1, -1, + -1, 1, -1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, 1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1 ] + use_flag_clddet: &clddet_airs_aqua [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1,31, 1,31, 1, 1, 1, 1, 1, 1, + 1,31, 1, 1, 1, 1,31, 1, 1, 1, + 31, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1 ] obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] maxvalue: 1.0e-12 action: @@ -200,35 +237,7 @@ obs filters: channels: *airs_aqua_channels options: channels: *airs_aqua_channels - use_flag: [ -1, -1, 1, -1, -1, 1, -1, -1, 1, 1, - 1, -1, 1, 1, -1, -1, -1, 1, -1, -1, - 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, - 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 1, 1, 1, 1, -1, 1, 1, - -1, 1, 1, -1, 1, 1, 1, -1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, - -1, 1, 1, -1, 1, 1, 1, 1, -1, 1, - 1, -1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, -1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, -1, 1, 1, - -1, 1, 1, 1, 1, 1, 1, -1, 1, -1, - 1, 1, 1, 1, 1, 1, -1, 1, 1, 1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, -1, 1, 1, 1, -1, -1, - 1, 1, -1, -1, -1, 1, -1, -1, -1, 1, - -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, - -1, 1, -1, -1, -1, -1, -1, 1, -1, -1, - -1, 1, -1, -1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, -1, 1, 1, 1, 1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1 ] + use_flag: *useflag_airs_aqua obserr_demisf: [0.01, 0.02, 0.03, 0.02, 0.03] obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] maxvalue: 1.0e-12 @@ -268,35 +277,51 @@ obs filters: channels: *airs_aqua_channels options: channels: *airs_aqua_channels - obserr_bound_max: [ 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, - 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, - 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, - 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, - 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, - 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, - 3.5, 3.5, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, - 3.0, 3.0, 3.0, 3.0, 3.5, 3.5, 3.5, 3.0, 3.0, 3.5, - 3.5, 3.0, 3.0, 3.0, 3.5, 3.0, 3.0, 3.0, 3.0, 3.0, - 3.5, 3.0, 3.5, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, - 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, - 3.5, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, - 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, - 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, - 3.5, 3.5, 3.5, 3.5, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, - 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, - 3.0, 3.5, 3.0, 3.5, 3.0, 3.0, 3.0, 3.0, 3.5, 3.0, - 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, - 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, - 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, - 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, - 4.5, 4.5, 4.5, 4.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5, - 2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 3.5, - 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, - 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, - 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, - 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, - 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, - 3.0 ] + obserr_bound_max: [3.50, 3.50, 3.50, 3.50, 3.50, 3.50, 3.50, 3.50, 3.50, 3.50, + 3.50, 3.50, 3.50, 3.50, 3.50, 3.50, 3.50, 3.50, 3.50, 3.50, + 3.50, 3.50, 3.50, 3.50, 3.50, 3.50, 3.50, 3.50, 3.50, 3.50, + 3.50, 3.50, 3.50, 3.50, 3.50, 3.50, 3.50, 3.50, 3.50, 3.50, + 3.50, 3.50, 3.50, 3.50, 3.50, 3.50, 3.50, 3.50, 3.50, 3.50, + 3.50, 3.50, 3.50, 3.50, 3.50, 3.50, 3.50, 3.50, 3.50, 3.50, + 3.50, 3.50, 3.00, 3.00, 3.00, 3.00, 1.00, 1.00, 3.00, 1.00, + 3.00, 1.00, 1.00, 3.00, 1.00, 1.00, 1.00, 1.00, 3.00, 1.00, + 1.00, 3.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 3.00, 3.00, + 3.50, 3.00, 3.50, 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, + 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, + 3.50, 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, + 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, + 3.00, 1.70, 3.00, 1.00, 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, + 3.50, 1.25, 3.50, 3.50, 3.00, 3.00, 1.50, 3.00, 3.00, 3.00, + 1.50, 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, + 3.00, 3.50, 3.00, 3.50, 3.00, 3.00, 3.00, 3.00, 3.50, 3.00, + 4.50, 4.50, 4.50, 4.50, 4.50, 4.50, 4.50, 4.50, 4.50, 4.50, + 4.50, 4.50, 4.50, 4.50, 4.50, 4.50, 4.50, 4.50, 4.50, 4.50, + 4.50, 4.50, 4.50, 4.50, 4.50, 4.50, 4.50, 4.50, 4.50, 4.50, + 4.50, 4.50, 4.50, 4.50, 4.50, 4.50, 4.50, 4.50, 4.50, 4.50, + 4.50, 4.50, 4.50, 4.50, 2.50, 2.50, 2.50, 2.50, 2.50, 2.50, + 2.50, 2.50, 2.50, 2.50, 2.50, 2.50, 2.50, 2.50, 2.50, 3.50, + 3.50, 3.50, 3.50, 3.50, 3.50, 3.50, 3.50, 3.50, 3.50, 3.50, + 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, + 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, + 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, + 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, 3.00, + 3.00 ] + action: + name: reject +# Surface Type Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *airs_aqua_channels + test variables: + - name: SurfTypeCheckRad@ObsFunction + channels: *airs_aqua_channels + options: + channels: *airs_aqua_channels + use_flag: *useflag_airs_aqua + use_flag_clddet: *clddet_airs_aqua + maxvalue: 1.0e-12 + defer to post: true action: name: reject # Useflag Check @@ -309,35 +334,7 @@ obs filters: channels: *airs_aqua_channels options: channels: *airs_aqua_channels - use_flag: [ -1, -1, 1, -1, -1, 1, -1, -1, 1, 1, - 1, -1, 1, 1, -1, -1, -1, 1, -1, -1, - 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, - 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 1, 1, 1, 1, -1, 1, 1, - -1, 1, 1, -1, 1, 1, 1, -1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, - -1, 1, 1, -1, 1, 1, 1, 1, -1, 1, - 1, -1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, -1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, -1, 1, 1, - -1, 1, 1, 1, 1, 1, 1, -1, 1, -1, - 1, 1, 1, 1, 1, 1, -1, 1, 1, 1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, -1, 1, 1, 1, -1, -1, - 1, 1, -1, -1, -1, 1, -1, -1, -1, 1, - -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, - -1, 1, -1, -1, -1, -1, -1, 1, -1, -1, - -1, 1, -1, -1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, -1, 1, 1, 1, 1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1 ] + use_flag: *useflag_airs_aqua minvalue: 1.0e-12 action: name: reject diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_n20.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_n20.yaml index 9b2c802e..971b3425 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_n20.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_n20.yaml @@ -1,5 +1,14 @@ +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + linear obs operator: + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: &Sensor_ID cris-fsr_n20 + EndianType: little_endian + CoefficientPath: '{{crtm_coeff_dir}}' obs space: - name: cris-fsr_n20 + name: *Sensor_ID obsdatain: engine: type: H5File @@ -41,13 +50,6 @@ obs space: 1969, 1970, 1971, 1972, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1980, 1981, 1982, 1983, 1984, 1985, 1986, 1987, 2119, 2140, 2143, 2147, 2153, 2158, 2161, 2168, 2171, 2175, 2182 -obs operator: - name: CRTM - Absorbers: [H2O,O3,CO2] - obs options: - Sensor_ID: &Sensor_ID cris-fsr_n20 - EndianType: little_endian - CoefficientPath: '{{crtm_coeff_dir}}' obs bias: input file: '{{cycle_dir}}/cris-fsr_n20.{{background_time}}.satbias.nc4' variational bc: @@ -67,6 +69,56 @@ obs bias: order: 2 - name: scan_angle obs filters: +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *cris-fsr_n20_channels + action: + name: assign error + error parameter vector: [ 0.8230, 0.7600, 0.7360, 0.7430, 0.8560, 1.0790, 0.8880, 0.7780, 0.6710, 0.6500, + 0.6430, 0.6290, 0.6290, 0.6180, 0.6380, 0.6190, 0.6100, 0.6270, 0.6010, 0.6170, + 0.6080, 0.4980, 0.5112, 0.4922, 0.4959, 0.4954, 0.4836, 0.5140, 0.5005, 0.4917, + 0.4881, 0.4656, 0.4793, 0.4638, 0.4557, 0.4666, 0.4468, 0.4534, 0.4471, 0.4448, + 0.4469, 0.5360, 0.4426, 0.4388, 0.5340, 0.4368, 0.4380, 0.5310, 0.4379, 0.5350, + 0.4404, 0.4405, 0.4409, 0.4472, 0.4555, 0.4433, 0.4437, 0.4454, 0.4448, 0.4465, + 0.4499, 0.4488, 0.5400, 0.4534, 0.4472, 0.4550, 0.4562, 0.4520, 0.4639, 0.5380, + 0.4573, 0.4604, 0.4533, 0.4692, 0.5660, 0.4457, 0.4457, 0.5154, 0.5084, 0.5280, + 0.5520, 0.5600, 0.5670, 0.5460, 0.4950, 0.4809, 0.4732, 0.4861, 0.4632, 0.5290, + 0.4748, 0.5007, 0.5711, 0.5950, 0.5469, 0.6260, 0.5410, 0.5430, 0.5330, 0.5410, + 0.4695, 0.5300, 0.5390, 0.5290, 0.5420, 0.4681, 0.5360, 0.5420, 0.5350, 0.5630, + 0.4805, 0.6470, 0.6090, 0.5530, 0.5830, 0.5760, 0.6294, 0.5885, 0.5560, 0.5780, + 0.5660, 0.6010, 0.5627, 0.5675, 0.5920, 0.5166, 0.5890, 0.5291, 0.5892, 0.5976, + 0.5834, 0.6512, 0.6748, 0.6615, 0.6003, 0.5669, 0.5587, 0.5507, 0.5871, 0.6160, + 0.6370, 0.6330, 0.6390, 0.6550, 0.6410, 0.6640, 0.6480, 0.6560, 0.6630, 0.6520, + 0.6810, 0.6620, 0.6730, 0.6720, 0.6800, 0.7350, 0.7320, 0.7150, 0.6740, 0.6870, + 0.7020, 0.7050, 0.7150, 0.7250, 0.7070, 0.7400, 0.7400, 0.8740, 0.7370, 0.8190, + 0.7600, 0.8690, 0.9000, 0.6980, 0.8230, 0.6760, 0.6820, 0.7660, 0.6800, 0.6850, + 0.6940, 0.6950, 0.6890, 0.7270, 0.6950, 0.6880, 0.6770, 0.7360, 0.6510, 0.6610, + 0.6199, 0.6223, 0.6036, 0.6003, 0.5991, 0.5980, 0.5910, 0.5764, 0.5770, 0.5593, + 0.5970, 0.5760, 0.5740, 0.5780, 0.5790, 0.5750, 0.5760, 0.5680, 0.5750, 0.5690, + 0.5590, 0.5680, 0.5401, 0.5500, 0.5575, 0.5780, 0.5635, 0.5786, 0.5807, 0.5810, + 0.5730, 0.5690, 0.5670, 0.5520, 0.5500, 0.5580, 0.5520, 0.5620, 0.5740, 0.5750, + 0.6290, 0.6820, 0.7560, 1.0500, 1.1220, 1.1402, 1.1540, 1.1310, 1.1230, 1.1590, + 1.1060, 1.1160, 1.0690, 1.0770, 1.1550, 1.1620, 1.1402, 0.6440, 1.2080, 1.1402, + 1.2950, 1.2580, 1.1402, 0.6060, 0.6030, 0.5630, 0.5920, 0.6070, 0.6110, 0.6120, + 0.6180, 0.6260, 0.6290, 0.5830, 0.8646, 0.6260, 0.6390, 0.5590, 0.8270, 0.6120, + 0.5760, 0.5800, 0.5750, 0.6880, 0.6970, 0.7430, 0.6810, 0.8320, 0.7190, 0.7850, + 0.8780, 0.9402, 1.0054, 1.0730, 1.1290, 1.0350, 1.0270, 0.9703, 1.1950, 0.9153, + 1.2660, 1.1530, 1.3480, 1.1800, 1.2690, 1.3110, 0.9914, 1.3590, 1.1660, 1.1390, + 1.2817, 1.3980, 1.5420, 1.2290, 1.3770, 1.2800, 1.2450, 1.1188, 1.1930, 1.2930, + 1.2750, 1.3310, 1.3400, 1.0990, 1.0480, 1.1240, 1.2250, 1.1830, 1.1960, 1.4000, + 1.3330, 1.4170, 1.3260, 1.3050, 1.0638, 1.2680, 1.2170, 1.2890, 1.3950, 1.2320, + 1.4350, 1.2980, 1.3280, 1.2620, 1.1990, 1.3910, 1.2330, 1.3290, 1.6640, 1.5090, + 1.3490, 1.4810, 1.5950, 1.4850, 1.5320, 1.5040, 1.5840, 1.6090, 1.5160, 1.4890, + 1.5020, 1.5440, 1.6110, 1.5390, 1.2960, 1.2880, 1.2410, 1.3200, 1.3130, 1.3010, + 1.8430, 1.7470, 1.7110, 1.7710, 1.9370, 1.5750, 1.5730, 1.5000, 1.4590, 1.4020, + 1.3630, 2.2010, 2.1270, 2.1770, 2.1570, 2.1920, 2.1460, 2.1510, 2.0710, 1.9860, + 1.8450, 1.6870, 1.5050, 1.3730, 1.2290, 1.1130, 1.0040, 0.9360, 0.8950, 0.8710, + 0.8410, 0.8210, 0.8050, 0.8000, 0.7920, 0.7970, 0.7910, 0.7830, 0.7770, 0.7850, + 0.7870, 0.7890, 0.7930, 0.7940, 0.7450, 0.7500, 0.7480, 0.7490, 0.7440, 0.7330, + 0.7330, 0.7410, 0.7460, 0.7460, 0.7380, 0.7430, 0.7450, 0.7490, 0.7500, 0.7500, + 0.8840, 0.9060, 0.9170, 0.9240, 0.9220, 0.9280, 0.9210, 0.9380, 0.9310, 0.9430, + 0.9460 ] # Wavenumber Check - filter: BlackList filter variables: @@ -100,7 +152,7 @@ obs filters: - name: brightnessTemperature channels: *cris-fsr_n20_channels minvalue: 50.00001 - maxvalue: 449.99999 + maxvalue: 549.99999 action: name: reject # Topography Check @@ -115,7 +167,7 @@ obs filters: channels: *cris-fsr_n20_channels options: channels: *cris-fsr_n20_channels - sensor: cris-fsr_n20 + sensor: *Sensor_ID # Transmittance Top Check - filter: BlackList filter variables: @@ -138,94 +190,94 @@ obs filters: channels: *cris-fsr_n20_channels options: channels: *cris-fsr_n20_channels - use_flag: [ -1, 1, 1, -1, 1, -1, 1, -1, 1, 1, - 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, -1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, -1, -1, 1, -1, -1, - -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, - -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, - -1, 1, -1, -1, 1, -1, -1, -1, 1, -1, - -1, 1, -1, -1, 1, -1, 1, -1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, - 1, 1, -1, -1, -1, -1, 1, 1, -1, -1, - -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, - -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 1, 1, -1, -1, -1, -1, -1, -1, -1, - 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 1, 1, -1, -1, -1, -1, -1, 1, - -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, - 1, -1, -1, -1, -1, -1, -1, 1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1] - use_flag_clddet: [ -1, 1, -1, -1, -1, -1, 1, -1, 1, -1, - 1, -1, 1, -1, -1, 1, -1, -1, -1, 1, - -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, - -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, - -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, - -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, - -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, - -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, - -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, - -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, - -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, - -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, - -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, - -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, - -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, - -1, 1, -1, -1, 1, -1, -1, -1, 1, -1, - -1, 1, -1, -1, 1, -1, 1, -1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, - 1, 1, -1, -1, -1, -1, 1, 1, -1, -1, - -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, - -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 1, 1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1] + use_flag: &useflag_cris-fsr_n20 [ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, -1, 1, 1, -1, 1, 1, -1, 1, -1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, + 1, 1, 1, 1, -1, 1, 1, 1, 1, -1, + -1, -1, -1, -1, 1, 1, 1, 1, 1, -1, + 1, 1, 1, -1, 1, -1, -1, -1, -1, -1, + 1, -1, -1, -1, -1, 1, -1, -1, -1, -1, + 1, -1, -1, -1, -1, -1, 1, 1, -1, -1, + -1, -1, 1, 1, -1, 1, -1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, -1, 1, 1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 1, -1, -1, 1, + -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 1, 1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, + 1, -1, -1, -1, -1, -1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1] + use_flag_clddet: &clddet_cris-fsr_n20 [ 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 30, + 30, 30, 31, 31, 30, 31, 30, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 31, 31, 31, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 31, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 31, 30, 30, 31, + 30, 30, 31, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 31, 30, 30, 30, 30, 30, 30, 31, + 30, 30, 30, 30, 30, 30, 31, 30, 30, 30, + 31, 30, 30, 30, 30, 30, 30, 31, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 31, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30 ] obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] maxvalue: 1.0e-12 action: @@ -253,50 +305,7 @@ obs filters: channels: *cris-fsr_n20_channels options: channels: *cris-fsr_n20_channels - use_flag: [ -1, 1, 1, -1, 1, -1, 1, -1, 1, 1, - 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, -1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, -1, -1, 1, -1, -1, - -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, - -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, - -1, 1, -1, -1, 1, -1, -1, -1, 1, -1, - -1, 1, -1, -1, 1, -1, 1, -1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, - 1, 1, -1, -1, -1, -1, 1, 1, -1, -1, - -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, - -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 1, 1, -1, -1, -1, -1, -1, -1, -1, - 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 1, 1, -1, -1, -1, -1, -1, 1, - -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, - 1, -1, -1, -1, -1, -1, -1, 1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1 ] + use_flag: *useflag_cris-fsr_n20 obserr_demisf: [0.01,0.02,0.03,0.02,0.03] obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] maxvalue: 1.0e-12 @@ -339,7 +348,11 @@ obs filters: obserr_bound_max: [ 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 0.4, 0.4, 0.4, 0.4, 0.4, 0.4, 0.4, + 0.4, 2.0, 0.4, 0.4, 2.0, 0.4, 0.4, 2.0, 0.4, 2.0, + 0.4, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 0.4, 0.4, 0.4, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, @@ -355,13 +368,9 @@ obs filters: 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, - 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, - 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, - 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, - 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, - 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, - 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, - 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 1.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 1.0, 2.0, 2.0, 1.0, + 2.0, 2.0, 1.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, @@ -382,6 +391,22 @@ obs filters: 2.0 ] action: name: reject +# Surface Type Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *cris-fsr_n20_channels + test variables: + - name: SurfTypeCheckRad@ObsFunction + channels: *cris-fsr_n20_channels + options: + channels: *cris-fsr_n20_channels + use_flag: *useflag_cris-fsr_n20 + use_flag_clddet: *clddet_cris-fsr_n20 + maxvalue: 1.0e-12 + defer to post: true + action: + name: reject # Useflag Check - filter: Bounds Check filter variables: @@ -392,50 +417,7 @@ obs filters: channels: *cris-fsr_n20_channels options: channels: *cris-fsr_n20_channels - use_flag: [ -1, 1, 1, -1, 1, -1, 1, -1, 1, 1, - 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, -1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, -1, -1, 1, -1, -1, - -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, - -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, - -1, 1, -1, -1, 1, -1, -1, -1, 1, -1, - -1, 1, -1, -1, 1, -1, 1, -1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, - 1, 1, -1, -1, -1, -1, 1, 1, -1, -1, - -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, - -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 1, 1, -1, -1, -1, -1, -1, -1, -1, - 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 1, 1, -1, -1, -1, -1, -1, 1, - -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, - 1, -1, -1, -1, -1, -1, -1, 1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1 ] + use_flag: *useflag_cris-fsr_n20 minvalue: 1.0e-12 action: name: reject diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_npp.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_npp.yaml index 73d9b764..0e1f04fc 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_npp.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_npp.yaml @@ -1,5 +1,14 @@ +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + linear obs operator: + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: &Sensor_ID cris-fsr_npp + EndianType: little_endian + CoefficientPath: '{{crtm_coeff_dir}}' obs space: - name: cris-fsr_npp + name: *Sensor_ID obsdatain: engine: type: H5File @@ -41,13 +50,6 @@ obs space: 1969, 1970, 1971, 1972, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1980, 1981, 1982, 1983, 1984, 1985, 1986, 1987, 2119, 2140, 2143, 2147, 2153, 2158, 2161, 2168, 2171, 2175, 2182 -obs operator: - name: CRTM - Absorbers: [H2O,O3,CO2] - obs options: - Sensor_ID: &Sensor_ID cris-fsr_npp - EndianType: little_endian - CoefficientPath: '{{crtm_coeff_dir}}' obs bias: input file: '{{cycle_dir}}/cris-fsr_npp.{{background_time}}.satbias.nc4' variational bc: @@ -67,6 +69,56 @@ obs bias: order: 2 - name: scan_angle obs filters: +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *cris-fsr_npp_channels + action: + name: assign error + error parameter vector: [ 0.8230, 0.7600, 0.7360, 0.7430, 0.8560, 1.0790, 0.8880, 0.7780, 0.6710, 0.6500, + 0.6430, 0.6290, 0.6290, 0.6180, 0.6380, 0.6190, 0.6100, 0.6270, 0.6010, 0.6170, + 0.6080, 0.4980, 0.5112, 0.4922, 0.4959, 0.4954, 0.4836, 0.5140, 0.5005, 0.4917, + 0.4881, 0.4656, 0.4793, 0.4638, 0.4557, 0.4666, 0.4468, 0.4534, 0.4471, 0.4448, + 0.4469, 0.5360, 0.4426, 0.4388, 0.5340, 0.4368, 0.4380, 0.5310, 0.4379, 0.5350, + 0.4404, 0.4405, 0.4409, 0.4472, 0.4555, 0.4433, 0.4437, 0.4454, 0.4448, 0.4465, + 0.4499, 0.4488, 0.5400, 0.4534, 0.4472, 0.4550, 0.4562, 0.4520, 0.4639, 0.5380, + 0.4573, 0.4604, 0.4533, 0.4692, 0.5660, 0.4457, 0.4457, 0.5154, 0.5084, 0.5280, + 0.5520, 0.5600, 0.5670, 0.5460, 0.4950, 0.4809, 0.4732, 0.4861, 0.4632, 0.5290, + 0.4748, 0.5007, 0.5711, 0.5950, 0.5469, 0.6260, 0.5410, 0.5430, 0.5330, 0.5410, + 0.4695, 0.5300, 0.5390, 0.5290, 0.5420, 0.4681, 0.5360, 0.5420, 0.5350, 0.5630, + 0.4805, 0.6470, 0.6090, 0.5530, 0.5830, 0.5760, 0.6294, 0.5885, 0.5560, 0.5780, + 0.5660, 0.6010, 0.5627, 0.5675, 0.5920, 0.5166, 0.5890, 0.5291, 0.5892, 0.5976, + 0.5834, 0.6512, 0.6748, 0.6615, 0.6003, 0.5669, 0.5587, 0.5507, 0.5871, 0.6160, + 0.6370, 0.6330, 0.6390, 0.6550, 0.6410, 0.6640, 0.6480, 0.6560, 0.6630, 0.6520, + 0.6810, 0.6620, 0.6730, 0.6720, 0.6800, 0.7350, 0.7320, 0.7150, 0.6740, 0.6870, + 0.7020, 0.7050, 0.7150, 0.7250, 0.7070, 0.7400, 0.7400, 0.8740, 0.7370, 0.8190, + 0.7600, 0.8690, 0.9000, 0.6980, 0.8230, 0.6760, 0.6820, 0.7660, 0.6800, 0.6850, + 0.6940, 0.6950, 0.6890, 0.7270, 0.6950, 0.6880, 0.6770, 0.7360, 0.6510, 0.6610, + 0.6199, 0.6223, 0.6036, 0.6003, 0.5991, 0.5980, 0.5910, 0.5764, 0.5770, 0.5593, + 0.5970, 0.5760, 0.5740, 0.5780, 0.5790, 0.5750, 0.5760, 0.5680, 0.5750, 0.5690, + 0.5590, 0.5680, 0.5401, 0.5500, 0.5575, 0.5780, 0.5635, 0.5786, 0.5807, 0.5810, + 0.5730, 0.5690, 0.5670, 0.5520, 0.5500, 0.5580, 0.5520, 0.5620, 0.5740, 0.5750, + 0.6290, 0.6820, 0.7560, 1.0500, 1.1220, 1.1402, 1.1540, 1.1310, 1.1230, 1.1590, + 1.1060, 1.1160, 1.0690, 1.0770, 1.1550, 1.1620, 1.1402, 0.6440, 1.2080, 1.1402, + 1.2950, 1.2580, 1.1402, 0.6060, 0.6030, 0.5630, 0.5920, 0.6070, 0.6110, 0.6120, + 0.6180, 0.6260, 0.6290, 0.5830, 0.8646, 0.6260, 0.6390, 0.5590, 0.8270, 0.6120, + 0.5760, 0.5800, 0.5750, 0.6880, 0.6970, 0.7430, 0.6810, 0.8320, 0.7190, 0.7850, + 0.8780, 0.9402, 1.0054, 1.0730, 1.1290, 1.0350, 1.0270, 0.9703, 1.1950, 0.9153, + 1.2660, 1.1530, 1.3480, 1.1800, 1.2690, 1.3110, 0.9914, 1.3590, 1.1660, 1.1390, + 1.2817, 1.3980, 1.5420, 1.2290, 1.3770, 1.2800, 1.2450, 1.1188, 1.1930, 1.2930, + 1.2750, 1.3310, 1.3400, 1.0990, 1.0480, 1.1240, 1.2250, 1.1830, 1.1960, 1.4000, + 1.3330, 1.4170, 1.3260, 1.3050, 1.0638, 1.2680, 1.2170, 1.2890, 1.3950, 1.2320, + 1.4350, 1.2980, 1.3280, 1.2620, 1.1990, 1.3910, 1.2330, 1.3290, 1.6640, 1.5090, + 1.3490, 1.4810, 1.5950, 1.4850, 1.5320, 1.5040, 1.5840, 1.6090, 1.5160, 1.4890, + 1.5020, 1.5440, 1.6110, 1.5390, 1.2960, 1.2880, 1.2410, 1.3200, 1.3130, 1.3010, + 1.8430, 1.7470, 1.7110, 1.7710, 1.9370, 1.5750, 1.5730, 1.5000, 1.4590, 1.4020, + 1.3630, 2.2010, 2.1270, 2.1770, 2.1570, 2.1920, 2.1460, 2.1510, 2.0710, 1.9860, + 1.8450, 1.6870, 1.5050, 1.3730, 1.2290, 1.1130, 1.0040, 0.9360, 0.8950, 0.8710, + 0.8410, 0.8210, 0.8050, 0.8000, 0.7920, 0.7970, 0.7910, 0.7830, 0.7770, 0.7850, + 0.7870, 0.7890, 0.7930, 0.7940, 0.7450, 0.7500, 0.7480, 0.7490, 0.7440, 0.7330, + 0.7330, 0.7410, 0.7460, 0.7460, 0.7380, 0.7430, 0.7450, 0.7490, 0.7500, 0.7500, + 0.8840, 0.9060, 0.9170, 0.9240, 0.9220, 0.9280, 0.9210, 0.9380, 0.9310, 0.9430, + 0.9460 ] # Wavenumber Check - filter: BlackList filter variables: @@ -100,7 +152,7 @@ obs filters: - name: brightnessTemperature channels: *cris-fsr_npp_channels minvalue: 50.00001 - maxvalue: 449.99999 + maxvalue: 549.99999 action: name: reject # Topography Check @@ -115,7 +167,7 @@ obs filters: channels: *cris-fsr_npp_channels options: channels: *cris-fsr_npp_channels - sensor: cris-fsr_npp + sensor: *Sensor_ID # Transmittance Top Check - filter: BlackList filter variables: @@ -138,94 +190,94 @@ obs filters: channels: *cris-fsr_npp_channels options: channels: *cris-fsr_npp_channels - use_flag: [ -1, 1, 1, -1, 1, -1, 1, -1, 1, 1, - 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, -1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, -1, -1, 1, -1, -1, - -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, - -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, - -1, 1, -1, -1, 1, -1, -1, -1, 1, -1, - -1, 1, -1, -1, 1, -1, 1, -1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, - 1, 1, -1, -1, -1, -1, 1, 1, -1, -1, - -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, - -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 1, 1, -1, -1, -1, -1, -1, -1, -1, - 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 1, 1, -1, -1, -1, -1, -1, 1, - -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, - 1, -1, -1, -1, -1, -1, -1, 1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1] - use_flag_clddet: [ -1, 1, -1, -1, -1, -1, 1, -1, 1, -1, - 1, -1, 1, -1, -1, 1, -1, -1, -1, 1, - -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, - -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, - -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, - -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, - -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, - -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, - -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, - -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, - -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, - -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, - -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, - -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, - -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, - -1, 1, -1, -1, 1, -1, -1, -1, 1, -1, - -1, 1, -1, -1, 1, -1, 1, -1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, - 1, 1, -1, -1, -1, -1, 1, 1, -1, -1, - -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, - -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 1, 1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1] + use_flag: &useflag_cris-fsr_npp [ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, -1, 1, 1, -1, 1, 1, -1, 1, -1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, + 1, 1, 1, 1, -1, 1, 1, 1, 1, -1, + -1, -1, -1, -1, 1, 1, 1, 1, 1, -1, + 1, 1, 1, -1, 1, -1, -1, -1, -1, -1, + 1, -1, -1, -1, -1, 1, -1, -1, -1, -1, + 1, -1, -1, -1, -1, -1, 1, 1, -1, -1, + -1, -1, 1, 1, -1, 1, -1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, -1, 1, 1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 1, -1, -1, 1, + -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 1, 1, -1, -1, -1, -1, -1, -1, 1, + -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, + 1, -1, -1, -1, -1, -1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1] + use_flag_clddet: &clddet_cris-fsr_npp [ 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 30, + 30, 30, 31, 31, 30, 31, 30, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 31, 31, 31, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 31, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 31, 30, 30, 31, + 30, 30, 31, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 31, 30, 30, 30, 30, 30, 30, 31, + 30, 30, 30, 30, 30, 30, 31, 30, 30, 30, + 31, 30, 30, 30, 30, 30, 30, 31, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 31, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30 ] obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] maxvalue: 1.0e-12 action: @@ -253,50 +305,7 @@ obs filters: channels: *cris-fsr_npp_channels options: channels: *cris-fsr_npp_channels - use_flag: [ -1, 1, 1, -1, 1, -1, 1, -1, 1, 1, - 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, -1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, -1, -1, 1, -1, -1, - -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, - -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, - -1, 1, -1, -1, 1, -1, -1, -1, 1, -1, - -1, 1, -1, -1, 1, -1, 1, -1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, - 1, 1, -1, -1, -1, -1, 1, 1, -1, -1, - -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, - -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 1, 1, -1, -1, -1, -1, -1, -1, -1, - 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 1, 1, -1, -1, -1, -1, -1, 1, - -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, - 1, -1, -1, -1, -1, -1, -1, 1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1 ] + use_flag: *useflag_cris-fsr_npp obserr_demisf: [0.01,0.02,0.03,0.02,0.03] obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] maxvalue: 1.0e-12 @@ -339,7 +348,11 @@ obs filters: obserr_bound_max: [ 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 0.4, 0.4, 0.4, 0.4, 0.4, 0.4, 0.4, + 0.4, 2.0, 0.4, 0.4, 2.0, 0.4, 0.4, 2.0, 0.4, 2.0, + 0.4, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 0.4, 0.4, 0.4, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, @@ -355,13 +368,9 @@ obs filters: 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, - 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, - 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, - 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, - 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, - 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, - 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, - 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 1.0, 2.0, 2.0, 2.0, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 1.0, 2.0, 2.0, 1.0, + 2.0, 2.0, 1.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, @@ -382,6 +391,22 @@ obs filters: 2.0 ] action: name: reject +# Surface Type Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *cris-fsr_npp_channels + test variables: + - name: SurfTypeCheckRad@ObsFunction + channels: *cris-fsr_npp_channels + options: + channels: *cris-fsr_npp_channels + use_flag: *useflag_cris-fsr_npp + use_flag_clddet: *clddet_cris-fsr_npp + maxvalue: 1.0e-12 + defer to post: true + action: + name: reject # Useflag Check - filter: Bounds Check filter variables: @@ -392,50 +417,7 @@ obs filters: channels: *cris-fsr_npp_channels options: channels: *cris-fsr_npp_channels - use_flag: [ -1, 1, 1, -1, 1, -1, 1, -1, 1, 1, - 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, -1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, - -1, 1, -1, 1, -1, -1, -1, 1, -1, -1, - -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, - -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, - -1, 1, -1, -1, 1, -1, -1, -1, 1, -1, - -1, 1, -1, -1, 1, -1, 1, -1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 1, 1, -1, -1, 1, -1, -1, -1, 1, 1, - 1, 1, -1, -1, -1, -1, 1, 1, -1, -1, - -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, - -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 1, 1, -1, -1, -1, -1, -1, -1, -1, - 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 1, 1, -1, -1, -1, -1, -1, 1, - -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, - 1, -1, -1, -1, -1, -1, -1, 1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1 ] + use_flag: *useflag_cris-fsr_npp minvalue: 1.0e-12 action: name: reject diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ssmis_f17.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ssmis_f17.yaml index 5083aa21..7aa7911f 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ssmis_f17.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ssmis_f17.yaml @@ -1,5 +1,14 @@ +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + linear obs operator: + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: &Sensor_ID ssmis_f17 + EndianType: little_endian + CoefficientPath: '{{crtm_coeff_dir}}' obs space: - name: ssmis_f17 + name: *Sensor_ID obsdatain: engine: type: H5File @@ -10,13 +19,6 @@ obs space: obsfile: '{{cycle_dir}}/{{experiment_id}}.ssmis_f17.{{window_begin}}.nc4' simulated variables: [brightnessTemperature] channels: &ssmis_f17_channels 1-24 -obs operator: - name: CRTM - Absorbers: [H2O,O3,CO2] - obs options: - Sensor_ID: &Sensor_ID ssmis_f17 - EndianType: little_endian - CoefficientPath: '{{crtm_coeff_dir}}' obs bias: input file: '{{cycle_dir}}/ssmis_f17.{{background_time}}.satbias.nc4' variational bc: @@ -50,75 +52,49 @@ obs bias: order: 2 - name: scan_angle var_name: sensorScanPosition -obs filters: -#step1: Gross check (setuprad) -- filter: Background Check +obs prior filters: +# Step 1: Initial Observation Error Assignment +- filter: Perform Action filter variables: - name: brightnessTemperature channels: *ssmis_f17_channels - threshold: 1.5 action: - name: reject -#step1: Gross check(qcmod) + name: assign error + error parameter vector: [ 1.50, 0.50, 0.50, 0.50, 0.50, 1.00, 1.00, 3.00, 3.00, 3.00, + 3.00, 2.40, 1.27, 1.44, 3.00, 1.34, 1.74, 3.75, 3.00, 3.00, + 2.00, 6.40, 1.00, 1.00] +obs post filters: +# step 2: Gross check(qcmod) - filter: Background Check filter variables: - name: brightnessTemperature channels: *ssmis_f17_channels absolute threshold: 3.5 remove bias correction: true + where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 action: name: reject -# #step2: clw check -# Keep the CLW check in yaml for further improvement. -# The test case using 2020110112 global SSMIS data shows that CLW check is not activated in GSI. -#- filter: Bounds Check -# filter variables: -# - name: brightnessTemperature -# channels: 1 -# test variables: -# - name: ObsFunction/CLWRetMW_SSMIS -# options: -# satellite: SSMIS -# ch19h: 12 -# ch19v: 13 -# ch22v: 14 -# ch37h: 15 -# ch37v: 16 -# ch91v: 17 -# ch91h: 18 -# varGroup: ObsValue -# minvalue: 0.0 -# maxvalue: 0.1 -# where: -# - variable: -# name: GeoVaLs/water_area_fraction -# minvalue: 0.99 -# action: -# name: reject -#step3: -- filter: Difference Check + +# Step 3: Reject all channels if surface height is greater than 2km +- filter: RejectList filter variables: - name: brightnessTemperature - channels: 1-2,12-16 - reference: brightness_temperature_2@ObsValue - value: brightness_temperature_2@HofX - minvalue: -1.5 - maxvalue: 1.5 + channels: *ssmis_f17_channels where: + - variable: + name: GeoVaLs/surface_geopotential_height + minvalue: 2000.0 + min_exclusive: true - variable: name: GeoVaLs/water_area_fraction maxvalue: 0.99 -#QC_terrain: If ssmis and terrain height > 2km. do not use -- filter: Domain Check - filter variables: - - name: brightnessTemperature - channels: *ssmis_f17_channels - where: - - variable: - name: MetaData/height - maxvalue: 2000.0 -#Do not use over mixed surface -- filter: BlackList + max_exclusive: true + +# Step 4: Reject data over mixed surface type +- filter: RejectList filter variables: - name: brightnessTemperature channels: 1-3,8-18 @@ -126,16 +102,120 @@ obs filters: - variable: name: GeoVaLs/land_area_fraction maxvalue: 0.99 + max_exclusive: true - variable: name: GeoVaLs/water_area_fraction maxvalue: 0.99 + max_exclusive: true - variable: name: GeoVaLs/ice_area_fraction maxvalue: 0.99 + max_exclusive: true - variable: name: GeoVaLs/surface_snow_area_fraction maxvalue: 0.99 -#step4: Generate q.c. bounds and modified variances + max_exclusive: true + +# Step 5: Channel 2 O-F check over non-water dominant area +- filter: Difference Check + filter variables: + - name: brightnessTemperature + channels: 1-2, 12-16 + reference: ObsValue/brightnessTemperature_2 + value: HofX/brightnessTemperature_2 + minvalue: -1.5 + maxvalue: 1.5 + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + max_exclusive: true + +# Step 6: Gross check over non-water dominant area +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *ssmis_f17_channels + absolute threshold: 3.5 + remove bias correction: true + where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + max_exclusive: true + action: + name: reject + +# Step 7: Scattering check for channels 9-11 using channels 8 and 17 +- filter: Difference Check + filter variables: + - name: brightnessTemperature + channels: 9 + reference: ObsValue/brightnessTemperature_9 + value: + name: ObsFunction/Arithmetic + options: + variables: + - name: HofX/brightnessTemperature_17 + - name: ObsBiasData/brightnessTemperature_17 + - name: HofX/brightnessTemperature_8 + - name: ObsBiasData/brightnessTemperature_8 + coefs: [-0.485934, 0.485934, 0.473806, -0.473806] + intercept: 271.252327 + maxvalue: 2 + +- filter: Difference Check + filter variables: + - name: brightnessTemperature + channels: 10 + reference: ObsValue/brightnessTemperature_10 + value: + name: ObsFunction/Arithmetic + options: + variables: + - name: HofX/brightnessTemperature_17 + - name: ObsBiasData/brightnessTemperature_17 + - name: HofX/brightnessTemperature_8 + - name: ObsBiasData/brightnessTemperature_8 + coefs: [-0.413688, 0.413688, 0.361549, -0.361549] + intercept: 272.280341 + maxvalue: 2 + +- filter: Difference Check + filter variables: + - name: brightnessTemperature + channels: 11 + reference: ObsValue/brightnessTemperature_11 + value: + name: ObsFunction/Arithmetic + options: + variables: + - name: HofX/brightnessTemperature_17 + - name: ObsBiasData/brightnessTemperature_17 + - name: HofX/brightnessTemperature_8 + - name: ObsBiasData/brightnessTemperature_8 + coefs: [-0.400882, 0.400882, 0.270510, -0.270510] + intercept: 278.824902 + maxvalue: 2 + +# Step 8: NSST retrieval check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *ssmis_f17_channels + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: *ssmis_f17_channels + options: + channels: *ssmis_f17_channels + use_flag: &useflag_ssmis_f17 [ -1, -1, -1, -1, 1, 1, 1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1] + maxvalue: 1.0e-12 + action: + name: reject + +# Step 9: Error inflation based on surface jacobian - filter: BlackList filter variables: - name: brightnessTemperature @@ -143,7 +223,6 @@ obs filters: action: name: inflate error inflation variable: -#Surface Jacobian check name: ObsFunction/ObsErrorFactorSurfJacobianRad channels: *ssmis_f17_channels options: @@ -151,7 +230,513 @@ obs filters: sensor: *Sensor_ID obserr_demisf: [0.010, 0.010, 0.010, 0.010, 0.010] obserr_dtempf: [0.500, 0.500, 0.500, 0.500, 0.500] -#Useflag Check + +# Step 10: Final gross check +# Channel 1 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor1 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_1 + coefs: [ 2.25 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_1 + threshold: DerivedMetaData/errorInflationFactor1 + absolute threshold: 6.0 + action: + name: reject + +# Channel 2 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor2 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_2 + coefs: [ 0.75 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_2 + threshold: DerivedMetaData/errorInflationFactor2 + absolute threshold: 6.0 + action: + name: reject + +# Channel 3 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor3 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_3 + coefs: [ 0.75 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_3 + threshold: DerivedMetaData/errorInflationFactor3 + absolute threshold: 6.0 + action: + name: reject + +# Channel 4 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor4 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_4 + coefs: [ 0.75 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_4 + threshold: DerivedMetaData/errorInflationFactor4 + absolute threshold: 6.0 + action: + name: reject + +# Channel 5 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor5 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_5 + coefs: [ 0.75 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_5 + threshold: DerivedMetaData/errorInflationFactor5 + absolute threshold: 6.0 + action: + name: reject + +# Channel 6 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor6 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_6 + coefs: [ 1.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_6 + threshold: DerivedMetaData/errorInflationFactor6 + absolute threshold: 6.0 + action: + name: reject + +# Channel 7 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor7 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_7 + coefs: [ 1.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_7 + threshold: DerivedMetaData/errorInflationFactor7 + absolute threshold: 6.0 + action: + name: reject + +# Channel 8 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor8 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_8 + coefs: [ 4.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_8 + threshold: DerivedMetaData/errorInflationFactor8 + absolute threshold: 6.0 + action: + name: reject + +# Channel 9 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor9 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_9 + coefs: [ 4.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_9 + threshold: DerivedMetaData/errorInflationFactor9 + absolute threshold: 6.0 + action: + name: reject + +# Channel 10 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor10 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_10 + coefs: [ 4.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_10 + threshold: DerivedMetaData/errorInflationFactor10 + absolute threshold: 6.0 + action: + name: reject + +# Channel 11 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor11 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_11 + coefs: [ 4.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_11 + threshold: DerivedMetaData/errorInflationFactor11 + absolute threshold: 6.0 + action: + name: reject + +# Channel 12 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor12 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_12 + coefs: [ 3.60 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_12 + threshold: DerivedMetaData/errorInflationFactor12 + absolute threshold: 6.0 + action: + name: reject + +# Channel 13 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor13 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_13 + coefs: [ 1.905 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_13 + threshold: DerivedMetaData/errorInflationFactor13 + absolute threshold: 6.0 + action: + name: reject + +# Channel 14 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor14 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_14 + coefs: [ 2.16 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_14 + threshold: DerivedMetaData/errorInflationFactor14 + absolute threshold: 6.0 + action: + name: reject + +# Channel 15 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor15 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_15 + coefs: [ 4.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_15 + threshold: DerivedMetaData/errorInflationFactor15 + absolute threshold: 6.0 + action: + name: reject + +# Channel 16 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor16 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_16 + coefs: [ 2.01 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_16 + threshold: DerivedMetaData/errorInflationFactor16 + absolute threshold: 6.0 + action: + name: reject + +# Channel 17 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor17 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_17 + coefs: [ 2.61 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_17 + threshold: DerivedMetaData/errorInflationFactor17 + absolute threshold: 6.0 + action: + name: reject + +# Channel 18 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor18 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_18 + coefs: [ 5.625 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_18 + threshold: DerivedMetaData/errorInflationFactor18 + absolute threshold: 6.0 + action: + name: reject + +# Channel 19 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor19 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_19 + coefs: [ 4.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_19 + threshold: DerivedMetaData/errorInflationFactor19 + absolute threshold: 6.0 + action: + name: reject + +# Channel 20 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor20 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_20 + coefs: [ 4.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_20 + threshold: DerivedMetaData/errorInflationFactor20 + absolute threshold: 6.0 + action: + name: reject + +# Channel 21 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor21 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_21 + coefs: [ 3.00 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_21 + threshold: DerivedMetaData/errorInflationFactor21 + absolute threshold: 6.0 + action: + name: reject + +# Channel 22 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor22 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_22 + coefs: [ 9.60 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_22 + threshold: DerivedMetaData/errorInflationFactor22 + absolute threshold: 6.0 + action: + name: reject + +# Channel 23 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor23 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_23 + coefs: [ 1.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_23 + threshold: DerivedMetaData/errorInflationFactor23 + absolute threshold: 6.0 + action: + name: reject + +# Channel 24 +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/errorInflationFactor24 + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsErrorData/brightnessTemperature_24 + coefs: [ 1.50 ] + exponents: [ -1 ] + +- filter: Background Check + filter variables: + - name: brightnessTemperature_24 + threshold: DerivedMetaData/errorInflationFactor24 + absolute threshold: 6.0 + action: + name: reject + +# Useflag Check - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -161,9 +746,8 @@ obs filters: channels: *ssmis_f17_channels options: channels: *ssmis_f17_channels - use_flag: [ 1, -1, -1, -1, 1 , 1, 1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 1] + use_flag: *useflag_ssmis_f17 minvalue: 1.0e-12 action: name: reject + From be042c644f6592718a6c3ee702a13dcfb9aaace1 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Thu, 27 Jul 2023 12:58:28 -0400 Subject: [PATCH 116/121] clean up issues in ncdiag 2 ioda calling task --- .gitignore | 1 + src/swell/tasks/gsi_ncdiag_to_ioda.py | 21 ++++++++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 696f8829..18643848 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # System files *.swp *DS_Store* +._* # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/src/swell/tasks/gsi_ncdiag_to_ioda.py b/src/swell/tasks/gsi_ncdiag_to_ioda.py index c80f3f30..e57b0004 100644 --- a/src/swell/tasks/gsi_ncdiag_to_ioda.py +++ b/src/swell/tasks/gsi_ncdiag_to_ioda.py @@ -151,11 +151,16 @@ def execute(self): self.logger.info('-'*len(log_str)) # Check the number of files that are found - ioda_type_pattern = f'*{needed_ioda_type}*_obs_*' # Pattern, e.g.: *aircraft*_obs_* + ioda_type_pattern = f'*{needed_ioda_type}_*_obs_*' # Pattern, e.g.: *aircraft*_obs_* # List of files for that instrument ioda_path_files = glob.glob(os.path.join(self.cycle_dir(), ioda_type_pattern)) + # Show files that will be combined + self.logger.info(f'Files to combine:') + for ioda_path_file in ioda_path_files: + self.logger.info(f' - {os.path.basename(ioda_path_file)}') + # Check that there are some files to combine self.logger.assert_abort(len(ioda_path_files) > 0, f'In combine of ' + f'{needed_ioda_type} no files where found. Ensure that ' + @@ -317,5 +322,19 @@ def execute(self): os.rename(ioda_geoval_in, os.path.join(self.cycle_dir(), ioda_geoval_out)) + # Remove left over files + # ------------------------------ + + self.logger('Removing residual files...') + + patterns = [ + '*_geoval_*', + ] + + for pattern in patterns: + geoval_files = glob.glob(os.path.join(self.cycle_dir(), pattern)) + for geoval_file in geoval_files: + self.logger.info(f'Removing left over file {os.path.basename(geoval_file)}') + os.remove(geoval_file) # -------------------------------------------------------------------------------------------------- From d68390e9d58d66dc0aad86ad2048b8a5ae6d62f5 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Thu, 27 Jul 2023 14:20:59 -0400 Subject: [PATCH 117/121] fix bug for single file outputs --- src/swell/tasks/gsi_ncdiag_to_ioda.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/swell/tasks/gsi_ncdiag_to_ioda.py b/src/swell/tasks/gsi_ncdiag_to_ioda.py index e57b0004..1476d5a5 100644 --- a/src/swell/tasks/gsi_ncdiag_to_ioda.py +++ b/src/swell/tasks/gsi_ncdiag_to_ioda.py @@ -140,6 +140,13 @@ def execute(self): Diag.close() + # Rename gps files from gps_bend if they exist + if 'gps' in observations_orig: + gps_files = glob.glob(os.path.join(self.cycle_dir(), 'gps_bend*')) + for gps_file in gps_files: + gps_file_newname = os.path.basename(gps_file).replace('gps_bend', 'gps') + os.rename(gps_file, os.path.join(self.cycle_dir(), gps_file_newname)) + # Combine the conventional data # ----------------------------- for needed_ioda_type in needed_ioda_types: @@ -151,7 +158,7 @@ def execute(self): self.logger.info('-'*len(log_str)) # Check the number of files that are found - ioda_type_pattern = f'*{needed_ioda_type}_*_obs_*' # Pattern, e.g.: *aircraft*_obs_* + ioda_type_pattern = f'{needed_ioda_type}*_obs_*' # Pattern, e.g.: *aircraft*_obs_* # List of files for that instrument ioda_path_files = glob.glob(os.path.join(self.cycle_dir(), ioda_type_pattern)) @@ -222,13 +229,6 @@ def execute(self): else: self.logger.abort(f'Combine failed for {needed_ioda_type}, file name issue.') - # Rename gps files from gps_bend if they exist - if 'gps' in observations_orig: - gps_files = glob.glob(os.path.join(self.cycle_dir(), 'gps_bend*')) - for gps_file in gps_files: - gps_file_newname = os.path.basename(gps_file).replace('gps_bend', 'gps') - os.rename(gps_file, os.path.join(self.cycle_dir(), gps_file_newname)) - # Get list of the observations that are ozone observations # -------------------------------------------------------- ozone_sensors = gsid.oz_lay_sensors + gsid.oz_lev_sensors @@ -325,7 +325,7 @@ def execute(self): # Remove left over files # ------------------------------ - self.logger('Removing residual files...') + self.logger.info('Removing residual files...') patterns = [ '*_geoval_*', @@ -334,7 +334,7 @@ def execute(self): for pattern in patterns: geoval_files = glob.glob(os.path.join(self.cycle_dir(), pattern)) for geoval_file in geoval_files: - self.logger.info(f'Removing left over file {os.path.basename(geoval_file)}') + self.logger.info(f' - Removing {os.path.basename(geoval_file)}') os.remove(geoval_file) # -------------------------------------------------------------------------------------------------- From c2add366dfda888c00fa8332a48340d25c744f4c Mon Sep 17 00:00:00 2001 From: Wei Gu <59488002+gmao-wgu@users.noreply.github.com> Date: Fri, 28 Jul 2023 13:52:40 -0400 Subject: [PATCH 118/121] updating YAML resource files for IASI (#202) --- .../observations/airs_aqua.yaml | 1 + .../observations/cris-fsr_n20.yaml | 1 + .../observations/cris-fsr_npp.yaml | 1 + .../observations/iasi_metop-a.yaml | 503 +++++++++++++++ .../observations/iasi_metop-b.yaml | 605 +++++++++--------- .../observations/iasi_metop-c.yaml | 605 +++++++++--------- 6 files changed, 1076 insertions(+), 640 deletions(-) create mode 100644 src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-a.yaml diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/airs_aqua.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/airs_aqua.yaml index e8c03174..b72873e9 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/airs_aqua.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/airs_aqua.yaml @@ -334,6 +334,7 @@ obs filters: channels: *airs_aqua_channels options: channels: *airs_aqua_channels + use passive_bc: true use_flag: *useflag_airs_aqua minvalue: 1.0e-12 action: diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_n20.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_n20.yaml index 971b3425..a9048a7a 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_n20.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_n20.yaml @@ -417,6 +417,7 @@ obs filters: channels: *cris-fsr_n20_channels options: channels: *cris-fsr_n20_channels + use passive_bc: true use_flag: *useflag_cris-fsr_n20 minvalue: 1.0e-12 action: diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_npp.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_npp.yaml index 0e1f04fc..f5f623ff 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_npp.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_npp.yaml @@ -417,6 +417,7 @@ obs filters: channels: *cris-fsr_npp_channels options: channels: *cris-fsr_npp_channels + use passive_bc: true use_flag: *useflag_cris-fsr_npp minvalue: 1.0e-12 action: diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-a.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-a.yaml new file mode 100644 index 00000000..9075bd5a --- /dev/null +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-a.yaml @@ -0,0 +1,503 @@ +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + linear obs operator: + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: &Sensor_ID iasi_metop-a + EndianType: little_endian + CoefficientPath: '{{crtm_coeff_dir}}' +obs space: + name: *Sensor_ID + obsdatain: + engine: + type: H5File + obsfile: '{{cycle_dir}}/iasi_metop-a.{{window_begin}}.nc4' + obsdataout: + engine: + type: H5File + obsfile: '{{cycle_dir}}/{{experiment_id}}.iasi_metop-a.{{window_begin}}.nc4' + simulated variables: [brightnessTemperature] + channels: &iasi_metop-a_channels 16, 29, 32, 35, 38, 41, 44, 47, 49, 50, 51, 53, + 55, 56, 57, 59, 61, 62, 63, 66, 68, 70, 72, 74, 76, 78, 79, 81, 82, 83, + 84, 85, 86, 87, 89, 92, 93, 95, 97, 99, 101, 103, 104, 106, 109, 110, + 111, 113, 116, 119, 122, 125, 128, 131, 133, 135, 138, 141, 144, 146, + 148, 150, 151, 154, 157, 159, 160, 161, 163, 167, 170, 173, 176, 179, + 180, 185, 187, 191, 193, 197, 199, 200, 202, 203, 205, 207, 210, 212, + 213, 214, 217, 218, 219, 222, 224, 225, 226, 228, 230, 231, 232, 236, + 237, 239, 243, 246, 249, 252, 254, 259, 260, 262, 265, 267, 269, 275, + 279, 282, 285, 294, 296, 299, 300, 303, 306, 309, 313, 320, 323, 326, + 327, 329, 332, 335, 345, 347, 350, 354, 356, 360, 363, 366, 371, 372, + 373, 375, 377, 379, 381, 383, 386, 389, 398, 401, 404, 405, 407, 408, + 410, 411, 414, 416, 418, 423, 426, 428, 432, 433, 434, 439, 442, 445, + 450, 457, 459, 472, 477, 483, 509, 515, 546, 552, 559, 566, 571, 573, + 578, 584, 594, 625, 646, 662, 668, 705, 739, 756, 797, 867, 906, 921, + 1027, 1046, 1090, 1098, 1121, 1133, 1173, 1191, 1194, 1222, 1271, 1283, + 1338, 1409, 1414, 1420, 1424, 1427, 1430, 1434, 1440, 1442, 1445, 1450, + 1454, 1460, 1463, 1469, 1474, 1479, 1483, 1487, 1494, 1496, 1502, 1505, + 1509, 1510, 1513, 1518, 1521, 1526, 1529, 1532, 1536, 1537, 1541, 1545, + 1548, 1553, 1560, 1568, 1574, 1579, 1583, 1585, 1587, 1606, 1626, 1639, + 1643, 1652, 1658, 1659, 1666, 1671, 1675, 1681, 1694, 1697, 1710, 1786, + 1791, 1805, 1839, 1884, 1913, 1946, 1947, 1991, 2019, 2094, 2119, 2213, + 2239, 2271, 2289, 2321, 2333, 2346, 2349, 2352, 2359, 2367, 2374, 2398, + 2426, 2562, 2701, 2741, 2745, 2760, 2819, 2889, 2907, 2910, 2919, 2921, + 2939, 2944, 2945, 2948, 2951, 2958, 2971, 2977, 2985, 2988, 2990, 2991, + 2993, 3002, 3008, 3014, 3027, 3029, 3030, 3036, 3047, 3049, 3052, 3053, + 3055, 3058, 3064, 3069, 3087, 3093, 3098, 3105, 3107, 3110, 3116, 3127, + 3129, 3136, 3146, 3151, 3160, 3165, 3168, 3175, 3178, 3189, 3207, 3228, + 3244, 3248, 3252, 3256, 3263, 3281, 3295, 3303, 3309, 3312, 3322, 3326, + 3354, 3366, 3375, 3378, 3411, 3416, 3432, 3438, 3440, 3442, 3444, 3446, + 3448, 3450, 3452, 3454, 3458, 3467, 3476, 3484, 3491, 3497, 3499, 3504, + 3506, 3509, 3518, 3527, 3555, 3575, 3577, 3580, 3582, 3586, 3589, 3599, + 3610, 3626, 3638, 3646, 3653, 3658, 3661, 3673, 3689, 3700, 3710, 3726, + 3763, 3814, 3841, 3888, 4032, 4059, 4068, 4082, 4095, 4160, 4234, 4257, + 4411, 4498, 4520, 4552, 4567, 4608, 4646, 4698, 4808, 4849, 4920, 4939, + 4947, 4967, 4991, 4996, 5015, 5028, 5056, 5128, 5130, 5144, 5170, 5178, + 5183, 5188, 5191, 5368, 5371, 5379, 5381, 5383, 5397, 5399, 5401, 5403, + 5405, 5446, 5455, 5472, 5480, 5483, 5485, 5492, 5497, 5502, 5507, 5509, + 5517, 5528, 5558, 5697, 5714, 5749, 5766, 5785, 5798, 5799, 5801, 5817, + 5833, 5834, 5836, 5849, 5851, 5852, 5865, 5869, 5881, 5884, 5897, 5900, + 5916, 5932, 5948, 5963, 5968, 5978, 5988, 5992, 5994, 5997, 6003, 6008, + 6023, 6026, 6039, 6053, 6056, 6067, 6071, 6082, 6085, 6098, 6112, 6126, + 6135, 6140, 6149, 6154, 6158, 6161, 6168, 6174, 6182, 6187, 6205, 6209, + 6213, 6317, 6339, 6342, 6366, 6381, 6391, 6489, 6962, 6966, 6970, 6975, + 6977, 6982, 6985, 6987, 6989, 6991, 6993, 6995, 6997, 6999, 7000, 7004, + 7008, 7013, 7016, 7021, 7024, 7027, 7029, 7032, 7038, 7043, 7046, 7049, + 7069, 7072, 7076, 7081, 7084, 7089, 7099, 7209, 7222, 7231, 7235, 7247, + 7267, 7269, 7284, 7389, 7419, 7423, 7424, 7426, 7428, 7431, 7436, 7444, + 7475, 7549, 7584, 7665, 7666, 7831, 7836, 7853, 7865, 7885, 7888, 7912, + 7950, 7972, 7980, 7995, 8007, 8015, 8055, 8078 +obs bias: + input file: '{{cycle_dir}}/iasi_metop-a.{{background_time}}.satbias.nc4' + variational bc: + predictors: + - name: constant + - name: lapse_rate + order: 2 + tlapse: &iasi_metop-a_tlapse '{{cycle_dir}}/iasi_metop-a.{{background_time}}.tlapse.txt' + - name: lapse_rate + tlapse: *iasi_metop-a_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle +obs filters: +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + action: + name: assign error + error parameter vector: [ 0.7270, 0.8100, 0.7500, 0.7900, 0.7055, 0.7400, 0.6800, 0.7200, 0.6526, 0.6500, + 0.6650, 0.6900, 0.6394, 0.6400, 0.6528, 0.6065, 0.6246, 0.6100, 0.6423, 0.5995, + 0.5900, 0.6069, 0.6000, 0.5965, 0.6400, 0.6200, 0.5890, 0.5865, 0.6500, 0.5861, + 0.6100, 0.5874, 0.6800, 0.6060, 0.6800, 4.3800, 3.0500, 2.3100, 1.5600, 1.3300, + 1.5800, 0.9300, 0.5832, 0.5587, 0.5867, 0.5800, 0.5655, 0.5522, 0.5864, 0.5475, + 0.5854, 0.5455, 0.5811, 0.5376, 0.5452, 0.5686, 0.5329, 0.5655, 0.5302, 0.5450, + 0.5628, 0.5900, 0.5262, 0.5590, 0.5264, 0.5442, 0.5100, 0.5513, 0.5224, 0.5523, + 0.5188, 0.5487, 0.5245, 0.5800, 0.5437, 0.5343, 0.5364, 0.6400, 0.5338, 0.7200, + 0.5370, 0.7500, 0.5100, 0.6500, 0.5274, 0.5290, 0.5187, 0.5228, 1.1200, 0.5222, + 0.5109, 0.6700, 0.5133, 0.5179, 0.5070, 0.6700, 0.5091, 0.6200, 0.5093, 0.6900, + 0.5048, 0.5024, 0.7800, 0.4970, 0.5337, 0.4865, 0.4915, 0.4835, 0.4869, 0.8700, + 0.4824, 0.4852, 0.8400, 0.8400, 0.8400, 0.5318, 0.8000, 0.4772, 0.9800, 0.4880, + 0.4978, 0.5157, 0.6100, 0.5213, 0.4884, 0.7900, 0.6200, 0.6600, 0.4691, 0.6500, + 0.4809, 0.4680, 0.6200, 0.4679, 0.6913, 0.4705, 0.4785, 0.4700, 0.4773, 0.4703, + 0.9800, 0.4697, 0.4662, 0.6500, 0.4670, 0.4883, 0.4684, 0.4684, 0.4947, 0.5393, + 0.5024, 0.4715, 0.6210, 0.6136, 0.5316, 1.7800, 0.5099, 1.1400, 0.5390, 1.7900, + 0.5080, 0.5723, 1.9400, 2.0100, 0.4900, 0.5647, 0.5022, 1.4700, 0.5815, 0.6782, + 2.1300, 0.5445, 1.5200, 0.5555, 1.9600, 2.3100, 2.3300, 2.3200, 2.3100, 0.6994, + 0.7006, 0.7060, 0.9785, 0.7023, 0.6991, 0.6946, 2.2800, 2.2600, 2.2600, 2.2600, + 0.6608, 0.6835, 0.6822, 2.2400, 2.2600, 0.6735, 2.2800, 0.6670, 0.7732, 0.6642, + 0.6480, 0.6629, 2.2900, 2.2900, 0.6799, 0.6230, 2.3200, 0.6030, 0.6224, 2.3200, + 0.6187, 2.3100, 2.3100, 2.2800, 2.2900, 2.2800, 2.2600, 1.9660, 2.2700, 2.2600, + 2.2500, 2.2700, 2.2400, 2.2100, 2.2400, 2.1700, 2.1800, 2.1700, 2.2100, 1.4000, + 2.1600, 2.2000, 2.1300, 2.1200, 2.1300, 2.1000, 2.1200, 2.1100, 2.0900, 2.0900, + 2.0800, 2.0900, 2.0400, 2.0400, 1.9660, 2.0100, 2.0500, 2.0300, 2.0600, 1.9800, + 1.9500, 1.9400, 1.9100, 1.8570, 1.7600, 1.7480, 1.8300, 2.0400, 1.7480, 1.9900, + 2.0750, 2.0700, 2.0200, 2.0400, 2.1000, 1.9660, 2.1800, 2.2100, 2.2400, 2.2300, + 2.2300, 1.9800, 2.2000, 2.1800, 2.1800, 2.2100, 2.2300, 2.2400, 2.2400, 2.2500, + 1.8000, 2.2400, 1.7300, 1.7300, 2.2700, 1.6700, 2.2100, 1.7200, 2.2300, 2.2300, + 2.2300, 2.2400, 2.2300, 2.1200, 2.1700, 1.7400, 2.0200, 1.8800, 1.6700, 1.7300, + 1.8300, 1.8200, 1.7300, 1.8300, 2.1900, 1.8400, 1.8900, 1.6000, 1.7100, 1.8600, + 1.8500, 1.8400, 1.8700, 1.9100, 1.5200, 1.9500, 1.8700, 1.8900, 1.9100, 1.9100, + 1.9300, 1.9000, 1.9100, 1.9000, 1.8900, 1.8900, 1.9100, 1.9000, 1.9100, 1.9100, + 1.9100, 1.9300, 1.9400, 1.9100, 1.9200, 1.7700, 1.9100, 1.9500, 1.1900, 1.9600, + 1.9800, 1.9400, 1.5500, 1.9100, 1.9200, 1.9200, 1.9700, 1.9300, 1.9900, 1.8600, + 1.1200, 1.9300, 1.9200, 1.9500, 1.8500, 1.8400, 1.9100, 1.1200, 1.8200, 1.8200, + 1.9500, 1.2400, 1.9400, 1.9600, 1.2100, 1.8300, 1.9600, 1.3600, 1.9600, 1.8200, + 1.9200, 1.6800, 1.9300, 1.2300, 1.9600, 1.9300, 1.8600, 1.4100, 1.1600, 1.6000, + 1.2500, 1.2000, 1.6500, 1.6600, 1.8700, 1.9400, 1.9600, 1.9100, 1.2500, 1.9300, + 1.9100, 1.7000, 0.9900, 1.8100, 1.9200, 1.9500, 1.5000, 1.4700, 1.1500, 1.5800, + 1.1800, 1.8200, 1.1300, 1.8300, 1.9100, 1.2600, 1.2700, 1.9100, 1.4500, 1.6000, + 1.2900, 1.9400, 1.9400, 1.2300, 1.9500, 1.2100, 1.9400, 1.8600, 1.9000, 1.3300, + 1.7500, 2.0200, 1.9800, 2.0300, 1.8300, 1.5000, 2.0400, 2.0200, 1.9000, 2.0000, + 2.0200, 1.9500, 1.9300, 1.9500, 1.9500, 1.9900, 2.0000, 1.9400, 1.9600, 1.8600, + 1.9200, 1.8800, 1.8600, 1.8400, 1.8700, 1.7700, 1.8900, 1.8900, 1.8800, 1.9400, + 1.8200, 1.7900, 1.8600, 2.0600, 2.3300, 1.8800, 1.8600, 1.8100, 1.8000, 1.8000, + 1.8600, 1.9000, 2.0000, 2.0600, 2.1000, 2.2000, 2.0000, 2.1600, 1.9800, 1.8000, + 1.8000, 1.8500, 1.7500, 2.0400, 2.1900, 2.1400, 2.1900, 1.8600, 2.1000, 2.1100, + 2.1800, 2.0300, 2.2800, 2.1900, 2.2600, 2.2600, 2.2100, 2.2100, 2.2600, 2.3300, + 2.2700, 2.2100, 2.1200, 2.2300, 2.2600, 2.2500, 1.8800, 2.2600, 2.2400, 2.3600, + 2.2900, 2.3500, 2.3000, 2.2700, 2.0800, 2.0500, 2.2700, 2.2800, 2.2700, 2.2800, + 1.9700, 2.2500, 2.2500, 2.2500, 2.3100, 2.2800, 2.2700, 2.1300, 2.2400, 2.2800, + 2.2800, 2.4100, 2.3400, 9.3200, 2.2800, 2.3800, 2.2700, 2.2700, 2.3900, 2.1100, + 2.0900, 2.1000, 2.0600, 2.1200, 2.0800, 2.0000, 1.9300, 2.0200, 2.5500, 1.5400, + 1.6400, 1.5100, 1.5500, 2.8200, 2.9200, 2.5500, 2.3700, 1.8500, 1.6000, 1.7200, + 1.7400, 1.7900, 1.9000, 1.9400, 2.0000, 2.0400, 2.0800, 2.1200, 2.1300, 2.1600, + 2.1800, 2.1800, 2.2000, 2.2000, 2.4100, 2.3900, 2.3800, 2.4000, 2.4200, 2.4100, + 2.4300, 2.4500, 2.4300, 2.4500, 2.4300, 2.4000, 2.4400, 2.4000, 2.4200, 2.4300, + 2.4500, 2.4500, 2.4500, 2.4600, 2.4500, 2.4500, 2.4300, 2.5100, 2.4800, 2.4800, + 2.5300, 2.4600, 2.4900, 2.5000, 2.5000, 2.5000, 2.5200, 2.5200, 2.5400, 2.5000, + 2.4800, 2.5000, 2.5500, 2.5000, 2.4800, 2.5000, 2.5000, 2.5200, 2.5200, 2.4800, + 2.5000, 2.5000, 2.5200, 2.4600, 2.5300, 9.0000 ] +# Wavenumber Check +- filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 7024, 7027, 7029, 7032, 7038, 7043, 7046, 7049, 7069, 7072, + 7076, 7081, 7084, 7089, 7099, 7209, 7222, 7231, 7235, 7247, + 7267, 7269, 7284, 7389, 7419, 7423, 7424, 7426, 7428, 7431, + 7436, 7444, 7475, 7549, 7584, 7665, 7666, 7831, 7836, 7853, + 7865, 7885, 7888, 7912, 7950, 7972, 7980, 7995, 8007, 8015, + 8055, 8078 + where: + - variable: + name: MetaData/solarZenithAngle + maxvalue: 88.9999 + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 1.0e-12 + action: + name: reject +- filter: BlackList + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorWavenumIR + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels +# Observation Range Sanity Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + minvalue: 50.00001 + maxvalue: 549.99999 + action: + name: reject +# Topography Check +- filter: BlackList + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + sensor: *Sensor_ID +# Transmittance Top Check +- filter: BlackList + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels +# Cloud Detection Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + test variables: + - name: ObsFunction/CloudDetectMinResidualIR + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + use_flag: &useflag_iasi_metop-a [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, + 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, + 1, 1, -1, -1, -1, 1, -1, 1, -1, 1, + 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, + 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, + 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, + -1, -1, -1, 1, -1, 1, -1, -1, 1, -1, + 1, -1, -1, -1, -1, 1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1] + use_flag_clddet: &clddet_iasi_metop-a [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 31, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 31, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 31, 1, 1, 1, 1, 1, + 1, 1, 1, 31, 1, 31, 1, 1, 31, 1, + 31, 1, 1, 1, 1, 31, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1] + obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] + maxvalue: 1.0e-12 + action: + name: reject +# NSST Retrieval Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + use_flag: *useflag_iasi_metop-a + obserr_demisf: [0.01,0.02,0.03,0.02,0.03] + obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] + maxvalue: 1.0e-12 + action: + name: reject +# Surface Jacobians Check +- filter: BlackList + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + sensor: *Sensor_ID + obserr_demisf: [0.01, 0.02, 0.03, 0.02, 0.03] + obserr_dtempf: [0.50, 2.00, 4.00, 2.00, 4.00] +# Gross check +- filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: [25.0, 0.5, 0.04, 1.0] + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + obserr_bound_max: [ 3.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, + 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, + 2.00, 2.00, 4.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, + 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 4.00, 4.00, + 3.50, 2.50, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, + 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, + 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, + 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 0.75, 2.00, + 0.75, 2.00, 2.00, 2.00, 0.75, 0.75, 0.75, 0.75, 2.00, 0.75, + 0.75, 2.00, 0.75, 0.75, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, + 2.00, 2.00, 2.00, 2.00, 2.50, 2.00, 2.50, 2.50, 3.00, 2.50, + 2.50, 2.50, 2.50, 3.50, 2.50, 2.50, 3.00, 3.50, 3.00, 4.00, + 4.00, 0.75, 4.00, 4.00, 4.00, 4.50, 4.50, 4.50, 4.50, 4.50, + 4.00, 4.50, 4.00, 4.00, 4.50, 2.50, 3.00, 2.50, 3.00, 2.50, + 3.00, 2.00, 2.50, 2.50, 3.00, 3.00, 2.50, 3.00, 3.00, 3.00, + 2.50, 2.50, 4.00, 4.50, 4.50, 5.00, 4.00, 4.00, 5.00, 5.00, + 5.00, 5.00, 5.50, 5.50, 4.00, 5.00, 4.00, 4.50, 5.50, 5.50, + 6.00, 4.50, 4.50, 4.00, 5.00, 5.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 1.25, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 1.25, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 1.50, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 1.40, 6.00, 1.40, 6.00, 6.00, 1.40, 6.00, + 1.50, 6.00, 6.00, 6.00, 6.00, 1.50, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 5.50, 4.50, 6.00, + 5.00, 5.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 5.00, 6.00, + 6.00, 6.00, 4.00, 6.00, 6.00, 6.00, 6.00, 4.50, 6.00, 6.00, + 4.50, 6.00, 6.00, 6.00, 6.00, 6.00, 5.00, 6.00, 6.00, 6.00, + 5.00, 6.00, 6.00, 5.00, 6.00, 5.00, 6.00, 6.00, 6.00, 5.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00 ] + action: + name: reject +# Surface Type Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + test variables: + - name: SurfTypeCheckRad@ObsFunction + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + use_flag: *useflag_iasi_metop-a + use_flag_clddet: *clddet_iasi_metop-a + maxvalue: 1.0e-12 + defer to post: true + action: + name: reject +# Useflag Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-a_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *iasi_metop-a_channels + options: + channels: *iasi_metop-a_channels + use passive_bc: true + use_flag: *useflag_iasi_metop-a + minvalue: 1.0e-12 + action: + name: reject diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-b.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-b.yaml index af5f899a..ee513a7f 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-b.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-b.yaml @@ -1,5 +1,14 @@ +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + linear obs operator: + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: &Sensor_ID iasi_metop-b + EndianType: little_endian + CoefficientPath: '{{crtm_coeff_dir}}' obs space: - name: iasi_metop-b + name: *Sensor_ID obsdatain: engine: type: H5File @@ -58,13 +67,6 @@ obs space: 7267, 7269, 7284, 7389, 7419, 7423, 7424, 7426, 7428, 7431, 7436, 7444, 7475, 7549, 7584, 7665, 7666, 7831, 7836, 7853, 7865, 7885, 7888, 7912, 7950, 7972, 7980, 7995, 8007, 8015, 8055, 8078 -obs operator: - name: CRTM - Absorbers: [H2O,O3,CO2] - obs options: - Sensor_ID: &Sensor_ID iasi_metop-b - EndianType: little_endian - CoefficientPath: '{{crtm_coeff_dir}}' obs bias: input file: '{{cycle_dir}}/iasi_metop-b.{{background_time}}.satbias.nc4' variational bc: @@ -84,6 +86,74 @@ obs bias: order: 2 - name: scan_angle obs filters: +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + action: + name: assign error + error parameter vector: [ 0.7270, 0.8100, 0.7500, 0.7900, 0.7055, 0.7400, 0.6800, 0.7200, 0.6526, 0.6500, + 0.6650, 0.6900, 0.6394, 0.6400, 0.6528, 0.6065, 0.6246, 0.6100, 0.6423, 0.5995, + 0.5900, 0.6069, 0.6000, 0.5965, 0.6400, 0.6200, 0.5890, 0.5865, 0.6500, 0.5861, + 0.6100, 0.5874, 0.6800, 0.6060, 0.6800, 4.3800, 3.0500, 2.3100, 1.5600, 1.3300, + 1.5800, 0.9300, 0.5832, 0.5587, 0.5867, 0.5800, 0.5655, 0.5522, 0.5864, 0.5475, + 0.5854, 0.5455, 0.5811, 0.5376, 0.5452, 0.5686, 0.5329, 0.5655, 0.5302, 0.5450, + 0.5628, 0.5900, 0.5262, 0.5590, 0.5264, 0.5442, 0.5100, 0.5513, 0.5224, 0.5523, + 0.5188, 0.5487, 0.5245, 0.5800, 0.5437, 0.5343, 0.5364, 0.6400, 0.5338, 0.7200, + 0.5370, 0.7500, 0.5100, 0.6500, 0.5274, 0.5290, 0.5187, 0.5228, 1.1200, 0.5222, + 0.5109, 0.6700, 0.5133, 0.5179, 0.5070, 0.6700, 0.5091, 0.6200, 0.5093, 0.6900, + 0.5048, 0.5024, 0.7800, 0.4970, 0.5337, 0.4865, 0.4915, 0.4835, 0.4869, 0.8700, + 0.4824, 0.4852, 0.8400, 0.8400, 0.8400, 0.5318, 0.8000, 0.4772, 0.9800, 0.4880, + 0.4978, 0.5157, 0.6100, 0.5213, 0.4884, 0.7900, 0.6200, 0.6600, 0.4691, 0.6500, + 0.4809, 0.4680, 0.6200, 0.4679, 0.6913, 0.4705, 0.4785, 0.4700, 0.4773, 0.4703, + 0.9800, 0.4697, 0.4662, 0.6500, 0.4670, 0.4883, 0.4684, 0.4684, 0.4947, 0.5393, + 0.5024, 0.4715, 0.6210, 0.6136, 0.5316, 1.7800, 0.5099, 1.1400, 0.5390, 1.7900, + 0.5080, 0.5723, 1.9400, 2.0100, 0.4900, 0.5647, 0.5022, 1.4700, 0.5815, 0.6782, + 2.1300, 0.5445, 1.5200, 0.5555, 1.9600, 2.3100, 2.3300, 2.3200, 2.3100, 0.6994, + 0.7006, 0.7060, 0.9785, 0.7023, 0.6991, 0.6946, 2.2800, 2.2600, 2.2600, 2.2600, + 0.6608, 0.6835, 0.6822, 2.2400, 2.2600, 0.6735, 2.2800, 0.6670, 0.7732, 0.6642, + 0.6480, 0.6629, 2.2900, 2.2900, 0.6799, 0.6230, 2.3200, 0.6030, 0.6224, 2.3200, + 0.6187, 2.3100, 2.3100, 2.2800, 2.2900, 2.2800, 2.2600, 1.9660, 2.2700, 2.2600, + 2.2500, 2.2700, 2.2400, 2.2100, 2.2400, 2.1700, 2.1800, 2.1700, 2.2100, 1.4000, + 2.1600, 2.2000, 2.1300, 2.1200, 2.1300, 2.1000, 2.1200, 2.1100, 2.0900, 2.0900, + 2.0800, 2.0900, 2.0400, 2.0400, 1.9660, 2.0100, 2.0500, 2.0300, 2.0600, 1.9800, + 1.9500, 1.9400, 1.9100, 1.8570, 1.7600, 1.7480, 1.8300, 2.0400, 1.7480, 1.9900, + 2.0750, 2.0700, 2.0200, 2.0400, 2.1000, 1.9660, 2.1800, 2.2100, 2.2400, 2.2300, + 2.2300, 1.9800, 2.2000, 2.1800, 2.1800, 2.2100, 2.2300, 2.2400, 2.2400, 2.2500, + 1.8000, 2.2400, 1.7300, 1.7300, 2.2700, 1.6700, 2.2100, 1.7200, 2.2300, 2.2300, + 2.2300, 2.2400, 2.2300, 2.1200, 2.1700, 1.7400, 2.0200, 1.8800, 1.6700, 1.7300, + 1.8300, 1.8200, 1.7300, 1.8300, 2.1900, 1.8400, 1.8900, 1.6000, 1.7100, 1.8600, + 1.8500, 1.8400, 1.8700, 1.9100, 1.5200, 1.9500, 1.8700, 1.8900, 1.9100, 1.9100, + 1.9300, 1.9000, 1.9100, 1.9000, 1.8900, 1.8900, 1.9100, 1.9000, 1.9100, 1.9100, + 1.9100, 1.9300, 1.9400, 1.9100, 1.9200, 1.7700, 1.9100, 1.9500, 1.1900, 1.9600, + 1.9800, 1.9400, 1.5500, 1.9100, 1.9200, 1.9200, 1.9700, 1.9300, 1.9900, 1.8600, + 1.1200, 1.9300, 1.9200, 1.9500, 1.8500, 1.8400, 1.9100, 1.1200, 1.8200, 1.8200, + 1.9500, 1.2400, 1.9400, 1.9600, 1.2100, 1.8300, 1.9600, 1.3600, 1.9600, 1.8200, + 1.9200, 1.6800, 1.9300, 1.2300, 1.9600, 1.9300, 1.8600, 1.4100, 1.1600, 1.6000, + 1.2500, 1.2000, 1.6500, 1.6600, 1.8700, 1.9400, 1.9600, 1.9100, 1.2500, 1.9300, + 1.9100, 1.7000, 0.9900, 1.8100, 1.9200, 1.9500, 1.5000, 1.4700, 1.1500, 1.5800, + 1.1800, 1.8200, 1.1300, 1.8300, 1.9100, 1.2600, 1.2700, 1.9100, 1.4500, 1.6000, + 1.2900, 1.9400, 1.9400, 1.2300, 1.9500, 1.2100, 1.9400, 1.8600, 1.9000, 1.3300, + 1.7500, 2.0200, 1.9800, 2.0300, 1.8300, 1.5000, 2.0400, 2.0200, 1.9000, 2.0000, + 2.0200, 1.9500, 1.9300, 1.9500, 1.9500, 1.9900, 2.0000, 1.9400, 1.9600, 1.8600, + 1.9200, 1.8800, 1.8600, 1.8400, 1.8700, 1.7700, 1.8900, 1.8900, 1.8800, 1.9400, + 1.8200, 1.7900, 1.8600, 2.0600, 2.3300, 1.8800, 1.8600, 1.8100, 1.8000, 1.8000, + 1.8600, 1.9000, 2.0000, 2.0600, 2.1000, 2.2000, 2.0000, 2.1600, 1.9800, 1.8000, + 1.8000, 1.8500, 1.7500, 2.0400, 2.1900, 2.1400, 2.1900, 1.8600, 2.1000, 2.1100, + 2.1800, 2.0300, 2.2800, 2.1900, 2.2600, 2.2600, 2.2100, 2.2100, 2.2600, 2.3300, + 2.2700, 2.2100, 2.1200, 2.2300, 2.2600, 2.2500, 1.8800, 2.2600, 2.2400, 2.3600, + 2.2900, 2.3500, 2.3000, 2.2700, 2.0800, 2.0500, 2.2700, 2.2800, 2.2700, 2.2800, + 1.9700, 2.2500, 2.2500, 2.2500, 2.3100, 2.2800, 2.2700, 2.1300, 2.2400, 2.2800, + 2.2800, 2.4100, 2.3400, 9.3200, 2.2800, 2.3800, 2.2700, 2.2700, 2.3900, 2.1100, + 2.0900, 2.1000, 2.0600, 2.1200, 2.0800, 2.0000, 1.9300, 2.0200, 2.5500, 1.5400, + 1.6400, 1.5100, 1.5500, 2.8200, 2.9200, 2.5500, 2.3700, 1.8500, 1.6000, 1.7200, + 1.7400, 1.7900, 1.9000, 1.9400, 2.0000, 2.0400, 2.0800, 2.1200, 2.1300, 2.1600, + 2.1800, 2.1800, 2.2000, 2.2000, 2.4100, 2.3900, 2.3800, 2.4000, 2.4200, 2.4100, + 2.4300, 2.4500, 2.4300, 2.4500, 2.4300, 2.4000, 2.4400, 2.4000, 2.4200, 2.4300, + 2.4500, 2.4500, 2.4500, 2.4600, 2.4500, 2.4500, 2.4300, 2.5100, 2.4800, 2.4800, + 2.5300, 2.4600, 2.4900, 2.5000, 2.5000, 2.5000, 2.5200, 2.5200, 2.5400, 2.5000, + 2.4800, 2.5000, 2.5500, 2.5000, 2.4800, 2.5000, 2.5000, 2.5200, 2.5200, 2.4800, + 2.5000, 2.5000, 2.5200, 2.4600, 2.5300, 9.0000 ] # Wavenumber Check - filter: BlackList filter variables: @@ -120,7 +190,7 @@ obs filters: - name: brightnessTemperature channels: *iasi_metop-b_channels minvalue: 50.00001 - maxvalue: 449.99999 + maxvalue: 549.99999 action: name: reject # Topography Check @@ -135,7 +205,7 @@ obs filters: channels: *iasi_metop-b_channels options: channels: *iasi_metop-b_channels - sensor: iasi_metop-b + sensor: *Sensor_ID # Transmittance Top Check - filter: BlackList filter variables: @@ -158,130 +228,130 @@ obs filters: channels: *iasi_metop-b_channels options: channels: *iasi_metop-b_channels - use_flag: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, - 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, - -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, - -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, - -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, - 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, - 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, - 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, - 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, - 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, - 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, - 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, - -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, - 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, - -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, - 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, - 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, - 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, - 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, - -1, -1, -1, -1, -1, -1, 1, -1, 1, -1, - 1, -1, -1, -1, 1, -1, -1, -1, -1, -1, - -1, -1, 1, 1, -1, 1, 1, -1, 1, 1, - 1, 1, 1, -1, -1, 1, -1, -1, -1, -1, - -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, - 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, - -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, - -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, - 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, - -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, - -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1] - use_flag_clddet: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, - 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, - -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, - -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, - -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, - 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, - 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, - 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, - 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, - 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, - 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, - 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, - -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, - 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, - -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, - 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, - 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, - 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, - 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, - 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, - -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1] + use_flag: &useflag_iasi_metop-b [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, + 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, + 1, 1, -1, -1, -1, 1, -1, 1, -1, 1, + 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, + 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, + 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, + -1, -1, -1, 1, -1, 1, -1, -1, 1, -1, + 1, -1, -1, -1, -1, 1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1] + use_flag_clddet: &clddet_iasi_metop-b [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 31, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 31, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 31, 1, 1, 1, 1, 1, + 1, 1, 1, 31, 1, 31, 1, 1, 31, 1, + 31, 1, 1, 1, 1, 31, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1] obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] maxvalue: 1.0e-12 action: @@ -296,68 +366,7 @@ obs filters: channels: *iasi_metop-b_channels options: channels: *iasi_metop-b_channels - use_flag: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, - 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, - -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, - -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, - -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, - 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, - 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, - 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, - 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, - 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, - 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, - 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, - -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, - 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, - -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, - 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, - 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, - 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, - 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, - -1, -1, -1, -1, -1, -1, 1, -1, 1, -1, - 1, -1, -1, -1, 1, -1, -1, -1, -1, -1, - -1, -1, 1, 1, -1, 1, 1, -1, 1, 1, - 1, 1, 1, -1, -1, 1, -1, -1, -1, -1, - -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, - 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, - -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, - -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, - 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, - -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, - -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1] + use_flag: *useflag_iasi_metop-b obserr_demisf: [0.01,0.02,0.03,0.02,0.03] obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] maxvalue: 1.0e-12 @@ -397,68 +406,84 @@ obs filters: channels: *iasi_metop-b_channels options: channels: *iasi_metop-b_channels - obserr_bound_max: [ 3.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, - 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, - 2.0, 2.0, 4.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, - 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 4.0, 4.0, - 3.5, 2.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, - 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, - 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, - 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, - 3.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 3.0, 2.0, 2.0, - 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, - 2.0, 2.0, 2.0, 2.0, 2.5, 2.0, 2.5, 2.5, 3.0, 2.5, - 2.5, 2.5, 2.5, 3.5, 2.5, 2.5, 3.0, 3.5, 3.0, 4.0, - 4.0, 4.0, 4.0, 4.0, 4.0, 4.5, 4.5, 4.5, 4.5, 4.5, - 4.0, 4.5, 4.0, 4.0, 4.5, 2.5, 3.0, 2.5, 3.0, 2.5, - 3.0, 2.0, 2.5, 2.5, 3.0, 3.0, 2.5, 3.0, 3.0, 3.0, - 2.5, 2.5, 4.0, 4.5, 4.5, 5.0, 4.0, 4.0, 5.0, 5.0, - 5.0, 5.0, 5.5, 5.5, 4.0, 5.0, 4.0, 4.5, 5.5, 5.5, - 6.0, 4.5, 4.5, 4.0, 5.0, 5.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 5.5, 4.5, 6.0, - 5.0, 5.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 5.0, 6.0, - 6.0, 6.0, 4.0, 6.0, 6.0, 6.0, 6.0, 4.5, 6.0, 6.0, - 4.5, 6.0, 6.0, 6.0, 6.0, 6.0, 5.0, 6.0, 6.0, 6.0, - 5.0, 6.0, 6.0, 5.0, 6.0, 5.0, 6.0, 6.0, 6.0, 5.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0] + obserr_bound_max: [ 3.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, + 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, + 2.00, 2.00, 4.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, + 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 4.00, 4.00, + 3.50, 2.50, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, + 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, + 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, + 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 0.75, 2.00, + 0.75, 2.00, 2.00, 2.00, 0.75, 0.75, 0.75, 0.75, 2.00, 0.75, + 0.75, 2.00, 0.75, 0.75, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, + 2.00, 2.00, 2.00, 2.00, 2.50, 2.00, 2.50, 2.50, 3.00, 2.50, + 2.50, 2.50, 2.50, 3.50, 2.50, 2.50, 3.00, 3.50, 3.00, 4.00, + 4.00, 0.75, 4.00, 4.00, 4.00, 4.50, 4.50, 4.50, 4.50, 4.50, + 4.00, 4.50, 4.00, 4.00, 4.50, 2.50, 3.00, 2.50, 3.00, 2.50, + 3.00, 2.00, 2.50, 2.50, 3.00, 3.00, 2.50, 3.00, 3.00, 3.00, + 2.50, 2.50, 4.00, 4.50, 4.50, 5.00, 4.00, 4.00, 5.00, 5.00, + 5.00, 5.00, 5.50, 5.50, 4.00, 5.00, 4.00, 4.50, 5.50, 5.50, + 6.00, 4.50, 4.50, 4.00, 5.00, 5.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 1.25, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 1.25, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 1.50, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 1.40, 6.00, 1.40, 6.00, 6.00, 1.40, 6.00, + 1.50, 6.00, 6.00, 6.00, 6.00, 1.50, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 5.50, 4.50, 6.00, + 5.00, 5.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 5.00, 6.00, + 6.00, 6.00, 4.00, 6.00, 6.00, 6.00, 6.00, 4.50, 6.00, 6.00, + 4.50, 6.00, 6.00, 6.00, 6.00, 6.00, 5.00, 6.00, 6.00, 6.00, + 5.00, 6.00, 6.00, 5.00, 6.00, 5.00, 6.00, 6.00, 6.00, 5.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00 ] + action: + name: reject +# Surface Type Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-b_channels + test variables: + - name: SurfTypeCheckRad@ObsFunction + channels: *iasi_metop-b_channels + options: + channels: *iasi_metop-b_channels + use_flag: *useflag_iasi_metop-b + use_flag_clddet: *clddet_iasi_metop-b + maxvalue: 1.0e-12 + defer to post: true action: name: reject # Useflag Check @@ -471,68 +496,8 @@ obs filters: channels: *iasi_metop-b_channels options: channels: *iasi_metop-b_channels - use_flag: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, - 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, - -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, - -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, - -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, - 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, - 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, - 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, - 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, - 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, - 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, - 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, - -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, - 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, - -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, - 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, - 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, - 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, - 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, - -1, -1, -1, -1, -1, -1, 1, -1, 1, -1, - 1, -1, -1, -1, 1, -1, -1, -1, -1, -1, - -1, -1, 1, 1, -1, 1, 1, -1, 1, 1, - 1, 1, 1, -1, -1, 1, -1, -1, -1, -1, - -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, - 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, - -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, - -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, - 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, - -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, - -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1] + use passive_bc: true + use_flag: *useflag_iasi_metop-b minvalue: 1.0e-12 action: name: reject diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-c.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-c.yaml index 8daf2f97..97ec2366 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-c.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-c.yaml @@ -1,5 +1,14 @@ +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + linear obs operator: + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: &Sensor_ID iasi_metop-c + EndianType: little_endian + CoefficientPath: '{{crtm_coeff_dir}}' obs space: - name: iasi_metop-c + name: *Sensor_ID obsdatain: engine: type: H5File @@ -58,13 +67,6 @@ obs space: 7267, 7269, 7284, 7389, 7419, 7423, 7424, 7426, 7428, 7431, 7436, 7444, 7475, 7549, 7584, 7665, 7666, 7831, 7836, 7853, 7865, 7885, 7888, 7912, 7950, 7972, 7980, 7995, 8007, 8015, 8055, 8078 -obs operator: - name: CRTM - Absorbers: [H2O,O3,CO2] - obs options: - Sensor_ID: &Sensor_ID iasi_metop-c - EndianType: little_endian - CoefficientPath: '{{crtm_coeff_dir}}' obs bias: input file: '{{cycle_dir}}/iasi_metop-c.{{background_time}}.satbias.nc4' variational bc: @@ -84,6 +86,74 @@ obs bias: order: 2 - name: scan_angle obs filters: +- filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-c_channels + action: + name: assign error + error parameter vector: [ 0.7270, 0.8100, 0.7500, 0.7900, 0.7055, 0.7400, 0.6800, 0.7200, 0.6526, 0.6500, + 0.6650, 0.6900, 0.6394, 0.6400, 0.6528, 0.6065, 0.6246, 0.6100, 0.6423, 0.5995, + 0.5900, 0.6069, 0.6000, 0.5965, 0.6400, 0.6200, 0.5890, 0.5865, 0.6500, 0.5861, + 0.6100, 0.5874, 0.6800, 0.6060, 0.6800, 4.3800, 3.0500, 2.3100, 1.5600, 1.3300, + 1.5800, 0.9300, 0.5832, 0.5587, 0.5867, 0.5800, 0.5655, 0.5522, 0.5864, 0.5475, + 0.5854, 0.5455, 0.5811, 0.5376, 0.5452, 0.5686, 0.5329, 0.5655, 0.5302, 0.5450, + 0.5628, 0.5900, 0.5262, 0.5590, 0.5264, 0.5442, 0.5100, 0.5513, 0.5224, 0.5523, + 0.5188, 0.5487, 0.5245, 0.5800, 0.5437, 0.5343, 0.5364, 0.6400, 0.5338, 0.7200, + 0.5370, 0.7500, 0.5100, 0.6500, 0.5274, 0.5290, 0.5187, 0.5228, 1.1200, 0.5222, + 0.5109, 0.6700, 0.5133, 0.5179, 0.5070, 0.6700, 0.5091, 0.6200, 0.5093, 0.6900, + 0.5048, 0.5024, 0.7800, 0.4970, 0.5337, 0.4865, 0.4915, 0.4835, 0.4869, 0.8700, + 0.4824, 0.4852, 0.8400, 0.8400, 0.8400, 0.5318, 0.8000, 0.4772, 0.9800, 0.4880, + 0.4978, 0.5157, 0.6100, 0.5213, 0.4884, 0.7900, 0.6200, 0.6600, 0.4691, 0.6500, + 0.4809, 0.4680, 0.6200, 0.4679, 0.6913, 0.4705, 0.4785, 0.4700, 0.4773, 0.4703, + 0.9800, 0.4697, 0.4662, 0.6500, 0.4670, 0.4883, 0.4684, 0.4684, 0.4947, 0.5393, + 0.5024, 0.4715, 0.6210, 0.6136, 0.5316, 1.7800, 0.5099, 1.1400, 0.5390, 1.7900, + 0.5080, 0.5723, 1.9400, 2.0100, 0.4900, 0.5647, 0.5022, 1.4700, 0.5815, 0.6782, + 2.1300, 0.5445, 1.5200, 0.5555, 1.9600, 2.3100, 2.3300, 2.3200, 2.3100, 0.6994, + 0.7006, 0.7060, 0.9785, 0.7023, 0.6991, 0.6946, 2.2800, 2.2600, 2.2600, 2.2600, + 0.6608, 0.6835, 0.6822, 2.2400, 2.2600, 0.6735, 2.2800, 0.6670, 0.7732, 0.6642, + 0.6480, 0.6629, 2.2900, 2.2900, 0.6799, 0.6230, 2.3200, 0.6030, 0.6224, 2.3200, + 0.6187, 2.3100, 2.3100, 2.2800, 2.2900, 2.2800, 2.2600, 1.9660, 2.2700, 2.2600, + 2.2500, 2.2700, 2.2400, 2.2100, 2.2400, 2.1700, 2.1800, 2.1700, 2.2100, 1.4000, + 2.1600, 2.2000, 2.1300, 2.1200, 2.1300, 2.1000, 2.1200, 2.1100, 2.0900, 2.0900, + 2.0800, 2.0900, 2.0400, 2.0400, 1.9660, 2.0100, 2.0500, 2.0300, 2.0600, 1.9800, + 1.9500, 1.9400, 1.9100, 1.8570, 1.7600, 1.7480, 1.8300, 2.0400, 1.7480, 1.9900, + 2.0750, 2.0700, 2.0200, 2.0400, 2.1000, 1.9660, 2.1800, 2.2100, 2.2400, 2.2300, + 2.2300, 1.9800, 2.2000, 2.1800, 2.1800, 2.2100, 2.2300, 2.2400, 2.2400, 2.2500, + 1.8000, 2.2400, 1.7300, 1.7300, 2.2700, 1.6700, 2.2100, 1.7200, 2.2300, 2.2300, + 2.2300, 2.2400, 2.2300, 2.1200, 2.1700, 1.7400, 2.0200, 1.8800, 1.6700, 1.7300, + 1.8300, 1.8200, 1.7300, 1.8300, 2.1900, 1.8400, 1.8900, 1.6000, 1.7100, 1.8600, + 1.8500, 1.8400, 1.8700, 1.9100, 1.5200, 1.9500, 1.8700, 1.8900, 1.9100, 1.9100, + 1.9300, 1.9000, 1.9100, 1.9000, 1.8900, 1.8900, 1.9100, 1.9000, 1.9100, 1.9100, + 1.9100, 1.9300, 1.9400, 1.9100, 1.9200, 1.7700, 1.9100, 1.9500, 1.1900, 1.9600, + 1.9800, 1.9400, 1.5500, 1.9100, 1.9200, 1.9200, 1.9700, 1.9300, 1.9900, 1.8600, + 1.1200, 1.9300, 1.9200, 1.9500, 1.8500, 1.8400, 1.9100, 1.1200, 1.8200, 1.8200, + 1.9500, 1.2400, 1.9400, 1.9600, 1.2100, 1.8300, 1.9600, 1.3600, 1.9600, 1.8200, + 1.9200, 1.6800, 1.9300, 1.2300, 1.9600, 1.9300, 1.8600, 1.4100, 1.1600, 1.6000, + 1.2500, 1.2000, 1.6500, 1.6600, 1.8700, 1.9400, 1.9600, 1.9100, 1.2500, 1.9300, + 1.9100, 1.7000, 0.9900, 1.8100, 1.9200, 1.9500, 1.5000, 1.4700, 1.1500, 1.5800, + 1.1800, 1.8200, 1.1300, 1.8300, 1.9100, 1.2600, 1.2700, 1.9100, 1.4500, 1.6000, + 1.2900, 1.9400, 1.9400, 1.2300, 1.9500, 1.2100, 1.9400, 1.8600, 1.9000, 1.3300, + 1.7500, 2.0200, 1.9800, 2.0300, 1.8300, 1.5000, 2.0400, 2.0200, 1.9000, 2.0000, + 2.0200, 1.9500, 1.9300, 1.9500, 1.9500, 1.9900, 2.0000, 1.9400, 1.9600, 1.8600, + 1.9200, 1.8800, 1.8600, 1.8400, 1.8700, 1.7700, 1.8900, 1.8900, 1.8800, 1.9400, + 1.8200, 1.7900, 1.8600, 2.0600, 2.3300, 1.8800, 1.8600, 1.8100, 1.8000, 1.8000, + 1.8600, 1.9000, 2.0000, 2.0600, 2.1000, 2.2000, 2.0000, 2.1600, 1.9800, 1.8000, + 1.8000, 1.8500, 1.7500, 2.0400, 2.1900, 2.1400, 2.1900, 1.8600, 2.1000, 2.1100, + 2.1800, 2.0300, 2.2800, 2.1900, 2.2600, 2.2600, 2.2100, 2.2100, 2.2600, 2.3300, + 2.2700, 2.2100, 2.1200, 2.2300, 2.2600, 2.2500, 1.8800, 2.2600, 2.2400, 2.3600, + 2.2900, 2.3500, 2.3000, 2.2700, 2.0800, 2.0500, 2.2700, 2.2800, 2.2700, 2.2800, + 1.9700, 2.2500, 2.2500, 2.2500, 2.3100, 2.2800, 2.2700, 2.1300, 2.2400, 2.2800, + 2.2800, 2.4100, 2.3400, 9.3200, 2.2800, 2.3800, 2.2700, 2.2700, 2.3900, 2.1100, + 2.0900, 2.1000, 2.0600, 2.1200, 2.0800, 2.0000, 1.9300, 2.0200, 2.5500, 1.5400, + 1.6400, 1.5100, 1.5500, 2.8200, 2.9200, 2.5500, 2.3700, 1.8500, 1.6000, 1.7200, + 1.7400, 1.7900, 1.9000, 1.9400, 2.0000, 2.0400, 2.0800, 2.1200, 2.1300, 2.1600, + 2.1800, 2.1800, 2.2000, 2.2000, 2.4100, 2.3900, 2.3800, 2.4000, 2.4200, 2.4100, + 2.4300, 2.4500, 2.4300, 2.4500, 2.4300, 2.4000, 2.4400, 2.4000, 2.4200, 2.4300, + 2.4500, 2.4500, 2.4500, 2.4600, 2.4500, 2.4500, 2.4300, 2.5100, 2.4800, 2.4800, + 2.5300, 2.4600, 2.4900, 2.5000, 2.5000, 2.5000, 2.5200, 2.5200, 2.5400, 2.5000, + 2.4800, 2.5000, 2.5500, 2.5000, 2.4800, 2.5000, 2.5000, 2.5200, 2.5200, 2.4800, + 2.5000, 2.5000, 2.5200, 2.4600, 2.5300, 9.0000 ] # Wavenumber Check - filter: BlackList filter variables: @@ -120,7 +190,7 @@ obs filters: - name: brightnessTemperature channels: *iasi_metop-c_channels minvalue: 50.00001 - maxvalue: 449.99999 + maxvalue: 549.99999 action: name: reject # Topography Check @@ -135,7 +205,7 @@ obs filters: channels: *iasi_metop-c_channels options: channels: *iasi_metop-c_channels - sensor: iasi_metop-c + sensor: *Sensor_ID # Transmittance Top Check - filter: BlackList filter variables: @@ -158,130 +228,130 @@ obs filters: channels: *iasi_metop-c_channels options: channels: *iasi_metop-c_channels - use_flag: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, - 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, - -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, - -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, - -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, - 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, - 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, - 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, - 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, - 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, - 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, - 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, - -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, - 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, - -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, - 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, - 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, - 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, - 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, - -1, -1, -1, -1, -1, -1, 1, -1, 1, -1, - 1, -1, -1, -1, 1, -1, -1, -1, -1, -1, - -1, -1, 1, 1, -1, 1, 1, -1, 1, 1, - 1, 1, 1, -1, -1, 1, -1, -1, -1, -1, - -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, - 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, - -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, - -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, - 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, - -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, - -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1] - use_flag_clddet: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, - 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, - -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, - -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, - -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, - 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, - 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, - 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, - 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, - 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, - 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, - 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, - -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, - 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, - -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, - 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, - 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, - 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, - 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, - 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, - -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1] + use_flag: &useflag_iasi_metop-c [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, + 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, + -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, + 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, + 1, 1, -1, -1, -1, 1, -1, 1, -1, 1, + 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, + 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, + 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, + -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, + 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, + 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, + 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, 1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, + -1, -1, -1, 1, -1, 1, -1, -1, 1, -1, + 1, -1, -1, -1, -1, 1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1] + use_flag_clddet: &clddet_iasi_metop-c [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 31, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 31, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 31, 1, 1, 1, 1, 1, + 1, 1, 1, 31, 1, 31, 1, 1, 31, 1, + 31, 1, 1, 1, 1, 31, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1] obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] maxvalue: 1.0e-12 action: @@ -296,68 +366,7 @@ obs filters: channels: *iasi_metop-c_channels options: channels: *iasi_metop-c_channels - use_flag: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, - 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, - -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, - -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, - -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, - 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, - 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, - 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, - 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, - 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, - 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, - 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, - -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, - 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, - -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, - 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, - 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, - 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, - 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, - -1, -1, -1, -1, -1, -1, 1, -1, 1, -1, - 1, -1, -1, -1, 1, -1, -1, -1, -1, -1, - -1, -1, 1, 1, -1, 1, 1, -1, 1, 1, - 1, 1, 1, -1, -1, 1, -1, -1, -1, -1, - -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, - 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, - -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, - -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, - 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, - -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, - -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1] + use_flag: *useflag_iasi_metop-c obserr_demisf: [0.01,0.02,0.03,0.02,0.03] obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] maxvalue: 1.0e-12 @@ -397,68 +406,84 @@ obs filters: channels: *iasi_metop-c_channels options: channels: *iasi_metop-c_channels - obserr_bound_max: [ 3.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, - 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, - 2.0, 2.0, 4.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, - 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 4.0, 4.0, - 3.5, 2.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, - 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, - 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, - 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, - 3.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 3.0, 2.0, 2.0, - 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, - 2.0, 2.0, 2.0, 2.0, 2.5, 2.0, 2.5, 2.5, 3.0, 2.5, - 2.5, 2.5, 2.5, 3.5, 2.5, 2.5, 3.0, 3.5, 3.0, 4.0, - 4.0, 4.0, 4.0, 4.0, 4.0, 4.5, 4.5, 4.5, 4.5, 4.5, - 4.0, 4.5, 4.0, 4.0, 4.5, 2.5, 3.0, 2.5, 3.0, 2.5, - 3.0, 2.0, 2.5, 2.5, 3.0, 3.0, 2.5, 3.0, 3.0, 3.0, - 2.5, 2.5, 4.0, 4.5, 4.5, 5.0, 4.0, 4.0, 5.0, 5.0, - 5.0, 5.0, 5.5, 5.5, 4.0, 5.0, 4.0, 4.5, 5.5, 5.5, - 6.0, 4.5, 4.5, 4.0, 5.0, 5.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 5.5, 4.5, 6.0, - 5.0, 5.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 5.0, 6.0, - 6.0, 6.0, 4.0, 6.0, 6.0, 6.0, 6.0, 4.5, 6.0, 6.0, - 4.5, 6.0, 6.0, 6.0, 6.0, 6.0, 5.0, 6.0, 6.0, 6.0, - 5.0, 6.0, 6.0, 5.0, 6.0, 5.0, 6.0, 6.0, 6.0, 5.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, - 6.0, 6.0, 6.0, 6.0, 6.0, 6.0] + obserr_bound_max: [ 3.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, + 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, + 2.00, 2.00, 4.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, + 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 4.00, 4.00, + 3.50, 2.50, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, + 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, + 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, + 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 0.75, 2.00, + 0.75, 2.00, 2.00, 2.00, 0.75, 0.75, 0.75, 0.75, 2.00, 0.75, + 0.75, 2.00, 0.75, 0.75, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, + 2.00, 2.00, 2.00, 2.00, 2.50, 2.00, 2.50, 2.50, 3.00, 2.50, + 2.50, 2.50, 2.50, 3.50, 2.50, 2.50, 3.00, 3.50, 3.00, 4.00, + 4.00, 0.75, 4.00, 4.00, 4.00, 4.50, 4.50, 4.50, 4.50, 4.50, + 4.00, 4.50, 4.00, 4.00, 4.50, 2.50, 3.00, 2.50, 3.00, 2.50, + 3.00, 2.00, 2.50, 2.50, 3.00, 3.00, 2.50, 3.00, 3.00, 3.00, + 2.50, 2.50, 4.00, 4.50, 4.50, 5.00, 4.00, 4.00, 5.00, 5.00, + 5.00, 5.00, 5.50, 5.50, 4.00, 5.00, 4.00, 4.50, 5.50, 5.50, + 6.00, 4.50, 4.50, 4.00, 5.00, 5.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 1.25, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 1.25, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 1.50, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 1.40, 6.00, 1.40, 6.00, 6.00, 1.40, 6.00, + 1.50, 6.00, 6.00, 6.00, 6.00, 1.50, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 5.50, 4.50, 6.00, + 5.00, 5.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 5.00, 6.00, + 6.00, 6.00, 4.00, 6.00, 6.00, 6.00, 6.00, 4.50, 6.00, 6.00, + 4.50, 6.00, 6.00, 6.00, 6.00, 6.00, 5.00, 6.00, 6.00, 6.00, + 5.00, 6.00, 6.00, 5.00, 6.00, 5.00, 6.00, 6.00, 6.00, 5.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, 6.00, + 6.00, 6.00, 6.00, 6.00, 6.00, 6.00 ] + action: + name: reject +# Surface Type Check +- filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *iasi_metop-c_channels + test variables: + - name: SurfTypeCheckRad@ObsFunction + channels: *iasi_metop-c_channels + options: + channels: *iasi_metop-c_channels + use_flag: *useflag_iasi_metop-c + use_flag_clddet: *clddet_iasi_metop-c + maxvalue: 1.0e-12 + defer to post: true action: name: reject # Useflag Check @@ -471,68 +496,8 @@ obs filters: channels: *iasi_metop-c_channels options: channels: *iasi_metop-c_channels - use_flag: [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, - 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, - -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, - -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, - -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, - 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, - 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, - 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, - 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, - 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, - 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, - 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, - -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, - 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, - -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, - 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, - 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, - 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, - 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, - -1, -1, -1, -1, -1, -1, 1, -1, 1, -1, - 1, -1, -1, -1, 1, -1, -1, -1, -1, -1, - -1, -1, 1, 1, -1, 1, 1, -1, 1, 1, - 1, 1, 1, -1, -1, 1, -1, -1, -1, -1, - -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, - 1, 1, 1, 1, 1, 1, -1, 1, -1, -1, - -1, -1, -1, -1, -1, 1, -1, -1, 1, -1, - -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, - 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, - -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 1, -1, - -1, 1, -1, -1, -1, -1, -1, -1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1] + use passive_bc: true + use_flag: *useflag_iasi_metop-c minvalue: 1.0e-12 action: name: reject From f78cfe088838a1561e0b9f3788221c9f2642ea44 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Tue, 8 Aug 2023 08:56:27 -0400 Subject: [PATCH 119/121] ozone working better --- .../geos_atmosphere/model/pseudo-model.yaml | 3 --- .../geos_atmosphere/observations/aircraft.yaml | 2 +- .../geos_atmosphere/observations/mls55_aura.yaml | 6 +++--- .../geos_atmosphere/observations/omi_aura.yaml | 12 ++++++------ .../geos_atmosphere/observations/ompsnm_npp.yaml | 10 +++++----- .../suites-ufo_testing-geos_atmosphere.yaml | 4 ++-- 6 files changed, 17 insertions(+), 20 deletions(-) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/pseudo-model.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/pseudo-model.yaml index 075af1c1..b74aa294 100755 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/pseudo-model.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/pseudo-model.yaml @@ -5,6 +5,3 @@ provider: geos datapath: '' filenames: ['{{cycle_dir}}/bkg.%yyyy%mm%ddT%hh%MM%ssZ.nc4', '{{experiment_root}}/{{experiment_id}}/stage/fv3-jedi/geos_atmosphere/bkg/geos.crtmsrf.{{horizontal_resolution}}.nc4'] -model variables: [u,v,ua,va,t,delp,q,qi,ql,qr,qs,o3ppmv,phis,qls,qcn,cfcn,frocean,frland,varflt,ustar, - bstar,zpbl,cm,ct,cq,kcbl,tsm,khl,khu,frlake,frseaice,vtype,stype,vfrac,sheleg, - ts,soilt,soilm,u10m,v10m] diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/aircraft.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/aircraft.yaml index 26f86c12..fd3721e2 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/aircraft.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/aircraft.yaml @@ -1,7 +1,7 @@ obs operator: name: VertInterp - observation alias file: /discover/nobackup/msienkie/jedi/JediUfoTests/Config/obsop_name_map.yaml + observation alias file: '{{experiment_root}}/{{experiment_id}}/configuration/jedi/interfaces/{{model_component}}/observations/obsop_name_map.yaml' apply near surface wind scaling: true obs space: diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mls55_aura.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mls55_aura.yaml index aa4a5067..7788ce6f 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mls55_aura.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/mls55_aura.yaml @@ -8,7 +8,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/{{experiment_id}}.mls55_aura.{{window_begin}}.nc4' - simulated variables: [ozoneLayer] + simulated variables: [ozoneProfile] obs operator: name: VertInterp vertical coordinate: air_pressure @@ -17,7 +17,7 @@ obs filters: # range sanity check - filter: Bounds Check filter variables: - - name: ozoneLayer + - name: ozoneProfile minvalue: 0 maxvalue: 10000 action: @@ -25,7 +25,7 @@ obs filters: # threshold is really threshold*observation error (threshold= relative threshold) - filter: Background Check filter variables: - - name: ozoneLayer + - name: ozoneProfile threshold: 5.0 action: name: reject diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/omi_aura.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/omi_aura.yaml index ebdc65af..47070cec 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/omi_aura.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/omi_aura.yaml @@ -8,7 +8,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/{{experiment_id}}.omi_aura.{{window_begin}}.nc4' - simulated variables: [ozoneTotal] + simulated variables: [ozoneLayer] obs operator: name: AtmVertInterpLay geovals: [mole_fraction_of_ozone_in_air] @@ -18,14 +18,14 @@ obs operator: obs filters: - filter: Perform Action filter variables: - - name: ozoneTotal + - name: ozoneLayer action: name: assign error error parameter: 6.0 # range sanity check - filter: Bounds Check filter variables: - - name: ozoneTotal + - name: ozoneLayer minvalue: 0 maxvalue: 1000 action: @@ -33,7 +33,7 @@ obs filters: # Reject SZA > 84 - filter: Bounds Check filter variables: - - name: ozoneTotal + - name: ozoneLayer test variables: - name: MetaData/solarZenithAngle maxvalue: 84. @@ -42,7 +42,7 @@ obs filters: # Reject rows 25+ (somewhat stringent but thats what we do in GEOS) - filter: Bounds Check filter variables: - - name: ozoneTotal + - name: ozoneLayer test variables: - name: MetaData/sensorScanPosition minvalue: 3 @@ -53,7 +53,7 @@ obs filters: # Gross check - filter: Background Check filter variables: - - name: ozoneTotal + - name: ozoneLayer threshold: 5.0 action: name: reject diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ompsnm_npp.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ompsnm_npp.yaml index 15fda3af..fb07488a 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ompsnm_npp.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ompsnm_npp.yaml @@ -8,7 +8,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/{{experiment_id}}.ompsnm_npp.{{window_begin}}.nc4' - simulated variables: [ozoneTotal] + simulated variables: [ozoneLayer] obs operator: name: AtmVertInterpLay geovals: [mole_fraction_of_ozone_in_air] @@ -17,14 +17,14 @@ obs operator: obs filters: - filter: Perform Action filter variables: - - name: ozoneTotal + - name: ozoneLayer action: name: assign error error parameter: 6.0 # range sanity check - filter: Bounds Check filter variables: - - name: ozoneTotal + - name: ozoneLayer minvalue: 0 maxvalue: 1000 action: @@ -32,7 +32,7 @@ obs filters: # Reject SZA > 84 - filter: Bounds Check filter variables: - - name: ozoneTotal + - name: ozoneLayer test variables: - name: MetaData/solarZenithAngle maxvalue: 84. @@ -41,7 +41,7 @@ obs filters: # Gross check - filter: Background Check filter variables: - - name: ozoneTotal + - name: ozoneLayer threshold: 5.0 action: name: reject diff --git a/src/swell/suites/ufo_testing/geos_atmosphere/suites-ufo_testing-geos_atmosphere.yaml b/src/swell/suites/ufo_testing/geos_atmosphere/suites-ufo_testing-geos_atmosphere.yaml index c5c776aa..25930f5d 100644 --- a/src/swell/suites/ufo_testing/geos_atmosphere/suites-ufo_testing-geos_atmosphere.yaml +++ b/src/swell/suites/ufo_testing/geos_atmosphere/suites-ufo_testing-geos_atmosphere.yaml @@ -56,7 +56,7 @@ observations: # Path to GSI ncdiags path_to_gsi_nc_diags: - default_value: '/discover/nobackup/drholdaw/SwellTestData/ufo_testing/ncdiag' + default_value: '/discover/nobackup/drholdaw/SwellTestData/ufo_testing/ncdiag/2021121200' prompt: Path to where the ncdiags will be held type: string @@ -68,7 +68,7 @@ produce_geovals: # Location for bias correction coefficients path_to_gsi_bc_coefficients: - default_value: '/discover/nobackup/drholdaw/SwellTestData/ufo_testing/bc' + default_value: '/discover/nobackup/drholdaw/SwellTestData/ufo_testing/bc/2021121200' prompt: Tar or directory containing BC files type: string From c934e62c7125478f71e1be1406effb7ef9a8b33c Mon Sep 17 00:00:00 2001 From: danholdaway Date: Tue, 8 Aug 2023 12:52:51 -0400 Subject: [PATCH 120/121] do not always create the forecast directory --- src/swell/tasks/base/task_base.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/swell/tasks/base/task_base.py b/src/swell/tasks/base/task_base.py index b8a22ae9..e1d1c039 100644 --- a/src/swell/tasks/base/task_base.py +++ b/src/swell/tasks/base/task_base.py @@ -73,11 +73,12 @@ def __init__(self, config_input, datetime_input, model, task_name): # Create cycle and forecast directories # ------------------------------------- cycle_dir = None - forecast_dir = None + self.cycle_forecast_dir = None if datetime_input is not None: - forecast_dir = self.forecast_dir() - os.makedirs(forecast_dir, 0o755, exist_ok=True) + # Name of directory where cycle forecast files will be staged + self.cycle_forecast_dir = os.path.join(self.experiment_path(), 'run', + self.__datetime__.string_directory(), 'forecast') if model is not None: cycle_dir = self.cycle_dir() @@ -90,7 +91,7 @@ def __init__(self, config_input, datetime_input, model, task_name): # Add GEOS utils # -------------- - self.geos = Geos(self.logger, forecast_dir) + self.geos = Geos(self.logger, self.cycle_forecast_dir) # Create some extra helpers available when the datetime is present # ---------------------------------------------------------------- @@ -170,10 +171,13 @@ def cycle_dir(self): def forecast_dir(self, paths=[]): + # Make sure forecast directory exists + # ----------------------------------- + os.makedirs(self.cycle_forecast_dir, 0o755, exist_ok=True) + # Combine datetime string (directory format) with the model # ------------------------------------------------------ - forecast_dir = os.path.join(self.experiment_path(), 'run', - self.__datetime__.string_directory(), 'forecast') + forecast_dir = self.cycle_forecast_dir if len(paths) > 0: # If paths (which should be a list) is not empty, combine with forecast_dir From a5a041bbf6e59bca6fd9c10c1c147e2c883e2a40 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 9 Aug 2023 16:25:41 -0400 Subject: [PATCH 121/121] more updates towards trifecta --- .github/workflows/tier2_application_discover.yml | 6 +++--- .../geos_atmosphere/observations/omi_aura.yaml | 12 ++++++------ .../geos_atmosphere/observations/ompslpnc_npp.yaml | 6 +++--- .../geos_atmosphere/observations/ompsnm_npp.yaml | 10 +++++----- src/swell/suites/convert_ncdiags/flow.cylc | 6 ++++++ .../suites-convert_ncdiags-geos_atmosphere.yaml | 11 +++++++++++ .../convert_ncdiags/suites-convert_ncdiags.yaml | 4 ++-- src/swell/tasks/gsi_ncdiag_to_ioda.py | 12 ++++++++++-- 8 files changed, 46 insertions(+), 21 deletions(-) diff --git a/.github/workflows/tier2_application_discover.yml b/.github/workflows/tier2_application_discover.yml index 80da1b72..979cd877 100644 --- a/.github/workflows/tier2_application_discover.yml +++ b/.github/workflows/tier2_application_discover.yml @@ -3,9 +3,9 @@ name: Tier 2 Applications Tests (Discover) on: workflow_dispatch: -# schedule: -# # * is a special character in YAML so you have to quote this string -# - cron: '0 0 * * *' + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '0 0 * * *' defaults: run: diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/omi_aura.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/omi_aura.yaml index 47070cec..ebdc65af 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/omi_aura.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/omi_aura.yaml @@ -8,7 +8,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/{{experiment_id}}.omi_aura.{{window_begin}}.nc4' - simulated variables: [ozoneLayer] + simulated variables: [ozoneTotal] obs operator: name: AtmVertInterpLay geovals: [mole_fraction_of_ozone_in_air] @@ -18,14 +18,14 @@ obs operator: obs filters: - filter: Perform Action filter variables: - - name: ozoneLayer + - name: ozoneTotal action: name: assign error error parameter: 6.0 # range sanity check - filter: Bounds Check filter variables: - - name: ozoneLayer + - name: ozoneTotal minvalue: 0 maxvalue: 1000 action: @@ -33,7 +33,7 @@ obs filters: # Reject SZA > 84 - filter: Bounds Check filter variables: - - name: ozoneLayer + - name: ozoneTotal test variables: - name: MetaData/solarZenithAngle maxvalue: 84. @@ -42,7 +42,7 @@ obs filters: # Reject rows 25+ (somewhat stringent but thats what we do in GEOS) - filter: Bounds Check filter variables: - - name: ozoneLayer + - name: ozoneTotal test variables: - name: MetaData/sensorScanPosition minvalue: 3 @@ -53,7 +53,7 @@ obs filters: # Gross check - filter: Background Check filter variables: - - name: ozoneLayer + - name: ozoneTotal threshold: 5.0 action: name: reject diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ompslpnc_npp.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ompslpnc_npp.yaml index 48c71dbc..8da30580 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ompslpnc_npp.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ompslpnc_npp.yaml @@ -8,7 +8,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/{{experiment_id}}.ompslpnc_npp.{{window_begin}}.nc4' - simulated variables: [ozoneLayer] + simulated variables: [ozoneTotal] obs operator: name: VertInterp vertical coordinate: ["air_pressure"] @@ -18,7 +18,7 @@ obs filters: # range sanity check - filter: Bounds Check filter variables: - - name: ozoneLayer + - name: ozoneTotal minvalue: 0 maxvalue: 10000 action: @@ -26,7 +26,7 @@ obs filters: # threshold is really threshold*observation error (threshold= relative threshold) - filter: Background Check filter variables: - - name: ozoneLayer + - name: ozoneTotal threshold: 5.0 action: name: reject diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ompsnm_npp.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ompsnm_npp.yaml index fb07488a..15fda3af 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ompsnm_npp.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ompsnm_npp.yaml @@ -8,7 +8,7 @@ obs space: engine: type: H5File obsfile: '{{cycle_dir}}/{{experiment_id}}.ompsnm_npp.{{window_begin}}.nc4' - simulated variables: [ozoneLayer] + simulated variables: [ozoneTotal] obs operator: name: AtmVertInterpLay geovals: [mole_fraction_of_ozone_in_air] @@ -17,14 +17,14 @@ obs operator: obs filters: - filter: Perform Action filter variables: - - name: ozoneLayer + - name: ozoneTotal action: name: assign error error parameter: 6.0 # range sanity check - filter: Bounds Check filter variables: - - name: ozoneLayer + - name: ozoneTotal minvalue: 0 maxvalue: 1000 action: @@ -32,7 +32,7 @@ obs filters: # Reject SZA > 84 - filter: Bounds Check filter variables: - - name: ozoneLayer + - name: ozoneTotal test variables: - name: MetaData/solarZenithAngle maxvalue: 84. @@ -41,7 +41,7 @@ obs filters: # Gross check - filter: Background Check filter variables: - - name: ozoneLayer + - name: ozoneTotal threshold: 5.0 action: name: reject diff --git a/src/swell/suites/convert_ncdiags/flow.cylc b/src/swell/suites/convert_ncdiags/flow.cylc index 31e654c3..4456b23f 100644 --- a/src/swell/suites/convert_ncdiags/flow.cylc +++ b/src/swell/suites/convert_ncdiags/flow.cylc @@ -48,6 +48,9 @@ GetGsiNcdiag GetGsiNcdiag => GsiNcdiagToIoda BuildJediByLinking[^]? | BuildJedi[^] => GsiNcdiagToIoda + + # Clean up + GsiNcdiagToIoda => CleanCycle """ {% endfor %} @@ -96,4 +99,7 @@ [[ GsiNcdiagToIoda ]] script = "swell_task GsiNcdiagToIoda $config -d $datetime -m geos_atmosphere" + [[CleanCycle]] + script = "swell_task CleanCycle $config -d $datetime -m geos_atmosphere" + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/convert_ncdiags/geos_atmosphere/suites-convert_ncdiags-geos_atmosphere.yaml b/src/swell/suites/convert_ncdiags/geos_atmosphere/suites-convert_ncdiags-geos_atmosphere.yaml index b0cdc529..6480ac8b 100644 --- a/src/swell/suites/convert_ncdiags/geos_atmosphere/suites-convert_ncdiags-geos_atmosphere.yaml +++ b/src/swell/suites/convert_ncdiags/geos_atmosphere/suites-convert_ncdiags-geos_atmosphere.yaml @@ -72,3 +72,14 @@ fixed_options: background_time_offset: default_value: PT9H prompt: Time before the middle of the window that the background providing forecast began + clean_patterns: + default_value: + - gsi_bcs/*.nc4 + - gsi_bcs/*.txt + - gsi_bcs/*.yaml + - gsi_bcs + - gsi_ncdiags/*.nc4 + - gsi_ncdiags/aircraft/*.nc4 + - gsi_ncdiags/aircraft + - gsi_ncdiags + prompt: 'Patterns for the files to remove after completing a cycle' diff --git a/src/swell/suites/convert_ncdiags/suites-convert_ncdiags.yaml b/src/swell/suites/convert_ncdiags/suites-convert_ncdiags.yaml index 12d7c58d..c77e8437 100644 --- a/src/swell/suites/convert_ncdiags/suites-convert_ncdiags.yaml +++ b/src/swell/suites/convert_ncdiags/suites-convert_ncdiags.yaml @@ -1,12 +1,12 @@ # Cycle start point start_cycle_point: - default_value: '2020-12-15T00:00:00Z' + default_value: '2021-12-12T00:00:00Z' prompt: What is the time of the first cycle (middle of the window)? type: iso-datetime # Cycle final point final_cycle_point: - default_value: '2020-12-15T06:00:00Z' + default_value: '2021-12-12T00:00:00Z' prompt: What is the time of the final cycle (middle of the window)? type: iso-datetime diff --git a/src/swell/tasks/gsi_ncdiag_to_ioda.py b/src/swell/tasks/gsi_ncdiag_to_ioda.py index 1476d5a5..78ff7180 100644 --- a/src/swell/tasks/gsi_ncdiag_to_ioda.py +++ b/src/swell/tasks/gsi_ncdiag_to_ioda.py @@ -116,9 +116,13 @@ def execute(self): # Get the list of files gsi_conv_file = glob.glob(path_to_search) - # Check that at least one file was found + # Check that some files where found + self.logger.assert_abort(len(gsi_conv_file) != 0, 'The search for GSI ncdiags files ' + + f'returned no files. Search path: \'{path_to_search}\'') + + # Check that only one file was found self.logger.assert_abort(len(gsi_conv_file) == 1, 'The search for GSI ncdiags files ' + - f'returned more than one file. Search: \'{gsi_conv_file}\'') + f'returned more than one file. Files: \'{gsi_conv_file}\'') # Open the file Diag = gsid.Conv(gsi_conv_file[0]) @@ -163,6 +167,10 @@ def execute(self): # List of files for that instrument ioda_path_files = glob.glob(os.path.join(self.cycle_dir(), ioda_type_pattern)) + # For sfc make sure there are no surface ship files + if needed_ioda_type == 'sfc': + ioda_path_files = [x for x in ioda_path_files if 'sfcship' not in x] + # Show files that will be combined self.logger.info(f'Files to combine:') for ioda_path_file in ioda_path_files: