merge with dev

MeteoSwiss · May 16, 2024 · d11f52e · d11f52e
2 parents 228e5d9 + 08115be
commit d11f52e
Show file tree

Hide file tree

Showing 19 changed files with 522 additions and 131 deletions.
diff --git a/.github/workflows/pyrad_tests_base_dev.yml b/.github/workflows/pyrad_tests_base_dev.yml
@@ -18,7 +18,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ "ubuntu-latest" ]
+        os: [ "ubuntu-latest" , "windows-latest"]
         python-version: ["3.9", "3.10", "3.11"]
       max-parallel: 6
 

diff --git a/.github/workflows/pyrad_tests_mch.yml b/.github/workflows/pyrad_tests_mch.yml
@@ -24,7 +24,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ "ubuntu-latest" ]
+        os: [ "ubuntu-latest"]
         python-version: [ "3.9", "3.10", "3.11"]
       max-parallel: 6
 

diff --git a/.github/workflows/pyrad_tests_mch_dev.yml b/.github/workflows/pyrad_tests_mch_dev.yml
@@ -18,7 +18,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ "ubuntu-latest" ]
+        os: [ "ubuntu-latest"]
         python-version: [ "3.9", "3.10", "3.11"]
       max-parallel: 6
 

diff --git a/README.md b/README.md
@@ -19,6 +19,7 @@
 ![version](https://img.shields.io/badge/python-3.8-blue.svg)
 ![version](https://img.shields.io/badge/python-3.9-blue.svg)
 ![version](https://img.shields.io/badge/python-3.10-blue.svg)
+![version](https://img.shields.io/badge/python-3.11-blue.svg)
 
 [![Anaconda-Server Badge](https://anaconda.org/conda-forge/pyrad_mch/badges/version.svg)](https://anaconda.org/conda-forge/pyrad_mch)
 [![Anaconda-Server Badge](https://anaconda.org/conda-forge/pyrad_mch/badges/latest_release_date.svg)](https://anaconda.org/conda-forge/pyrad_mch)

diff --git a/config/pyart/mf_config.py b/config/pyart/mf_config.py
@@ -333,6 +333,7 @@
 # precipitation
 rain_rate = 'rain_rate'
 radar_estimated_rain_rate = 'radar_estimated_rain_rate'
+radar_reflectivity_estimated_rain_rate = 'radar_reflectivity_estimated_rain_rate'
 corrected_radar_estimated_rain_rate = 'corrected_radar_estimated_rain_rate'
 rainfall_accumulation = 'rainfall_accumulation'
 radar_rainrate_relation = 'radar_rainrate_relation'
@@ -734,6 +735,7 @@
     'sun_hit_zdr': sun_hit_zdr,
     'adjustment_factor': adjustment_factor,
     'radar_estimated_rain_rate': radar_estimated_rain_rate,
+    'radar_reflectivity_estimated_rain_rate': radar_reflectivity_estimated_rain_rate,
     'corrected_radar_estimated_rain_rate': corrected_radar_estimated_rain_rate,
     'rainfall_accumulation': rainfall_accumulation,
     'radar_rainrate_relation': radar_rainrate_relation,
@@ -2382,6 +2384,18 @@
                        40., 63., 100., 160., 250., 500.],
         'coordinates': 'elevation azimuth range'},
 
+     radar_reflectivity_estimated_rain_rate: {
+        'units': 'mm/h',
+        'standard_name': 'radar_reflectivity_estimated_rain_rate',
+        'long_name': 'Radar reflectivity estimated rain rate',
+        'labels': ['0.', '0.4', '0.63', '1.', '1.6', '2.5', '4.0', '6.3',
+                   '10.', '16.', '25.', '40.', '63.', '100.', '160.', '250.'],
+        'ticks': [0., 0.4, 0.63, 1., 1.6, 2.5, 4.0, 6.3, 10., 16., 25.,
+                  40., 63., 100., 160., 250.],
+        'boundaries': [0., 0.4, 0.63, 1., 1.6, 2.5, 4.0, 6.3, 10., 16., 25.,
+                       40., 63., 100., 160., 250., 500.],
+        'coordinates': 'elevation azimuth range'},
+
     corrected_radar_estimated_rain_rate: {
         'units': 'mm/h',
         'standard_name': 'corrected_radar_estimated_rain_rate',

diff --git a/doc/source/overview/loc.rst b/doc/source/overview/loc.rst
@@ -32,15 +32,15 @@ ppiImageConfig        STRUCT   Structure defining the PPI image generating (PPI_
 - xmax                FLOAT    Distance of the right image boundary to the radar in km.
 - ymin                FLOAT    Distance of the lower image boundary to the radar in km.
 - ymax                FLOAT    Distance of the upper image boundary to the radar in km.
-- dpi				  INT      Resolution of the image in dots per inch.
+- dpi	              INT      Resolution of the image in dots per inch.
 rhiImageConfig        STRUCT   Structure defining the RHI image generating (RHI_IMAGE or PSEUDORHI_IMAGE products). The following 6 fields are described below:
 - xsize               INT      Number of horizontal pixels of the picture (without frame).
 - ysize               INT      Number of vertical pixels of the picture (without frame).
 - xmin                FLOAT    Distance of the left image boundary to the radar in km.
 - xmax                FLOAT    Distance of the right image boundary to the radar in km.
 - ymin                FLOAT    Distance of the lower image boundary (vertical direction) to the radar in km.
 - ymax                FLOAT    Distance of the upper image boundary (vertical direction) to the radar in km.
-- dpi				  INT      Resolution of the image in dots per inch.
+- dpi		      INT      Resolution of the image in dots per inch.
 ppiMapImageConfig     STRUCT   Structure defining the PPI image overlaid on a map (PPI_MAP product). The following 9 fields are described below:
 - rngRing             FLOAT    Distance between range rings (0 means no range ring) [km].
 - xsize               FLOAT    Image size (inches) [inch].
@@ -53,8 +53,9 @@ ppiMapImageConfig     STRUCT   Structure defining the PPI image overlaid on a ma
 - lonstep             FLOAT    Step in longitude [°] used in the map gridlines.
 - exact_limits	      INT      If set to 1 will use exactly the user-specified latmin/latmax, lonmin/lonmax, if set to 0 will round them to the nearest integer.
 - mapres              STRING   Map resolution. Accepted strings are: “10m”, “50m”, “110m”.
-- maps                STRARR   String array of possible maps to overplot. Accepted entries include: relief, countries, provinces, urban_areas, roads, railroads, coastline, lakes, lakes_europe, rivers, rivers_europe.
-- dpi				  INT      Resolution of the image in dots per inch.
+- maps                STRARR   String array of possible maps to overplot. Accepted entries include: relief, countries, provinces, 
+                                 urban_areas, roads, railroads, coastline, lakes, lakes_europe, rivers, rivers_europe.
+- dpi		      INT      Resolution of the image in dots per inch.
 gridMapImageConfig    STRUCT   Structure defining the display of gridded data overlaid on a map (SURFACE_IMAGE product).
 - xsize               FLOAT    Image size (inches) [inch].
 - ysize               FLOAT    Image size (inches) [inch].
@@ -66,16 +67,17 @@ gridMapImageConfig    STRUCT   Structure defining the display of gridded data ov
 - lonstep             FLOAT    Step in longitude [°] used in the map gridlines.
 - exact_limits	      INT      If set to 1 will use exactly the user-specified latmin/latmax, lonmin/lonmax, if set to 0 will round them to the nearest integer.
 - mapres              STRING   Map resolution. Accepted strings are: “10m”, “50m”, “110m”.
-- maps                STRARR   String array of possible maps to overplot. Accepted entries include: relief, countries, provinces, urban_areas, roads, railroads, coastline, lakes, lakes_europe, rivers, rivers_europe
-- dpi				  INT      Resolution of the image in dots per inch.
+- maps                STRARR   String array of possible maps to overplot. Accepted entries include: relief, countries, provinces, 
+                               urban_areas, roads, railroads, coastline, lakes, lakes_europe, rivers, rivers_europe
+- dpi		      INT      Resolution of the image in dots per inch.
 xsecImageConfig       STRUCT   Structure defining the cross-section images generated from gridded data (CROSS_SECTION, LATITUDE_SLICE and LONGITUDE_SLICE products)
 - xsize               INT      Number of horizontal pixels of the picture (without frame).
 - ysize               INT      Number of vertical pixels of the picture (without frame).
 - xmin                FLOAT    Distance of the left image boundary to the radar in km.
 - xmax                FLOAT    Distance of the right image boundary to the radar in km.
 - ymin                FLOAT    Distance of the lower image boundary (vertical direction) to the radar in km.
 - ymax                FLOAT    Distance of the upper image boundary (vertical direction) to the radar in km.
-- dpi				  INT      Resolution of the image in dots per inch.
+- dpi                 INT      Resolution of the image in dots per inch.
 sunhitsImageConfig    STRUCT   Structure defining the sun hits image. The following 6 fields are described below:
 - xsize               INT      Number of horizontal pixels of the picture (without frame).
 - ysize               INT      Number of vertical pixels of the picture (without frame).

diff --git a/doc/source/overview/main.rst b/doc/source/overview/main.rst
@@ -1,6 +1,5 @@
 Main configuration file
 ==============================
-
 The main configuration file is used to define the global settings, notably the paths to the different sources of data. The parameters of the main configuration file are described in Table 2.
 
 Table 2: Configuration parameters of the main configuration file

diff --git a/src/pyrad_proc/pyrad/flow/flow_aux.py b/src/pyrad_proc/pyrad/flow/flow_aux.py
@@ -56,6 +56,7 @@
 
 from ..io.config import read_config
 from ..io.read_data_radar import get_data
+from ..io.write_data import write_to_s3
 from ..io.io_aux import get_datetime, get_file_list, get_scan_list
 from ..io.io_aux import get_dataset_fields, get_datatype_fields
 from ..io.io_aux import get_new_rainbow_file_name, get_fieldname_pyart
@@ -892,7 +893,14 @@ def _generate_prod(dataset, cfg, prdname, prdfunc, dsname, voltime,
     prdcfg = _create_prdcfg_dict(
         cfg, dsname, prdname, voltime, runinfo=runinfo)
     try:
-        prdfunc(dataset, prdcfg)
+        filenames = prdfunc(dataset, prdcfg)
+        if 's3copypath' in prdcfg: # copy to S3
+            s3AccessPolicy = (prdcfg['s3AccessPolicy'] if 's3AccessPolicy' 
+                                in prdcfg else None)
+            for fname in filenames:
+                if prdcfg['basepath'] in fname: # only products saved to standard basepath
+                    write_to_s3(fname, prdcfg['basepath'], prdcfg['s3copypath'],
+                    s3AccessPolicy)
         return False
     except Exception as inst:
         warn(str(inst))
@@ -1427,6 +1435,12 @@ def _create_prdcfg_dict(cfg, dataset, product, voltime, runinfo=None):
     prdcfg.update({'ScanPeriod': cfg['ScanPeriod']})
     prdcfg.update({'imgformat': cfg['imgformat']})
     prdcfg.update({'RadarName': cfg['RadarName']})
+
+    if 's3copypath' in cfg:
+        prdcfg.update({'s3copypath': cfg['s3copypath']})
+    if 's3AccessPolicy' in cfg:
+        prdcfg.update({'s3AccessPolicy': cfg['s3AccessPolicy']})
+
     if 'RadarBeamwidth' in cfg:
         prdcfg.update({'RadarBeamwidth': cfg['RadarBeamwidth']})
     if 'ppiImageConfig' in cfg:
@@ -1446,6 +1460,7 @@ def _create_prdcfg_dict(cfg, dataset, product, voltime, runinfo=None):
     prdcfg.update({'prdname': product})
     prdcfg.update({'timeinfo': voltime})
     prdcfg.update({'runinfo': runinfo})
+
     if 'dssavename' in cfg[dataset]:
         prdcfg.update({'dssavename': cfg[dataset]['dssavename']})
 

diff --git a/src/pyrad_proc/pyrad/graph/plots_grid.py b/src/pyrad_proc/pyrad/graph/plots_grid.py
@@ -117,7 +117,7 @@ def plot_surface(grid, field_name, level, prdcfg, fname_list, titl=None,
         lon_lines = np.arange(np.floor(min_lon), np.ceil(max_lon) + 1, lonstep)
         lat_lines = np.arange(np.floor(min_lat), np.ceil(max_lat) + 1, latstep)
 
-    if use_basemap:
+    if use_basemap or not _CARTOPY_AVAILABLE:
         resolution = prdcfg['gridMapImageConfig'].get('mapres', 'l')
         if resolution not in ('c', 'l', 'i', 'h', 'f'):
             warn('Unknown map resolution: ' + resolution)
@@ -153,7 +153,7 @@ def plot_surface(grid, field_name, level, prdcfg, fname_list, titl=None,
     if fig is None:
         fig = plt.figure(figsize=[xsize, ysize], dpi=dpi)
 
-        if use_basemap:
+        if use_basemap or not _CARTOPY_AVAILABLE:
             display = pyart.graph.GridMapDisplayBasemap(grid)
             display.plot_basemap(
                 lat_lines=lat_lines, lon_lines=lon_lines,
@@ -179,7 +179,7 @@ def plot_surface(grid, field_name, level, prdcfg, fname_list, titl=None,
             # ax.set_extent([min_lon, max_lon, min_lat, max_lat])
             # display.plot_crosshairs(lon=lon, lat=lat)
     else:
-        if use_basemap:
+        if use_basemap or not _CARTOPY_AVAILABLE:
             display.plot_grid(
                 field_name, level=level, norm=norm, ticks=ticks,
                 lat_lines=lat_lines, lon_lines=lon_lines, title=titl,
@@ -865,7 +865,7 @@ def plot_dda_map(grid, bg_field_name, level, prdcfg, fname_list, titl=None,
     ax.set_xlim([lon_lines[0], lon_lines[-1]])
     ax.set_ylim([lat_lines[0], lat_lines[-1]])
 
-    if embellish:
+    if embellish and _CARTOPY_AVAILABLE:
         for cartomap in maps_list:
             if cartomap == 'countries':
                 # add countries

diff --git a/src/pyrad_proc/pyrad/graph/plots_vol.py b/src/pyrad_proc/pyrad/graph/plots_vol.py
@@ -59,6 +59,15 @@
     warn('shapely not available')
     _SHAPELY_AVAILABLE = False
 
+# Test whether ARM or MCH fork of pyart
+if  hasattr(pyart, '__ismchfordk__'):
+    _PYARTMCH_AVAILABLE = True
+else:
+    warn('MCH version of pyart not installed, some features' + \
+        ' of pyrad are not supported by ARM pyart!')
+    _PYARTMCH_AVAILABLE = False
+
+
 import matplotlib as mpl
 mpl.use('Agg')
 
@@ -263,7 +272,7 @@ def plot_ppi(radar, field_name, ind_el, prdcfg, fname_list, plot_type='PPI',
     return fname_list
 
 
-def plot_ppi_map(radar, field_name, ind_el, prdcfg, fname_list,
+def plot_ppi_map(radar, field_name, ind_el, prdcfg, fname_list, vmin=None, vmax=None,
                  save_fig=True):
     """
     plots a PPI on a geographic map
@@ -321,12 +330,21 @@ def plot_ppi_map(radar, field_name, ind_el, prdcfg, fname_list,
 
     projection = cartopy.crs.PlateCarree()
     display_map = pyart.graph.RadarMapDisplay(radar)
-    display_map.plot_ppi_map(
-        field_name, sweep=ind_el, norm=norm, ticks=ticks, ticklabs=ticklabs,
-        min_lon=min_lon, max_lon=max_lon, min_lat=min_lat, max_lat=max_lat,
-        lat_lines=lat_lines, lon_lines=lon_lines, projection=projection,
-        fig=fig, embellish=False, colorbar_flag=True, alpha=1)
-
+    if _PYARTMCH_AVAILABLE:
+        display_map.plot_ppi_map(
+            field_name, vmin = vmin, vmax = vmax, sweep=ind_el, norm=norm, ticks=ticks, 
+            ticklabs=ticklabs, min_lon=min_lon, max_lon=max_lon, min_lat=min_lat, 
+            max_lat=max_lat, lat_lines=lat_lines, lon_lines=lon_lines, 
+            single_grid_lines_labels = True, projection=projection,
+            fig=fig, embellish=False, colorbar_flag=True, 
+            alpha=1)
+    else:
+        display_map.plot_ppi_map(
+            field_name, vmin = vmin, vmax = vmax, sweep=ind_el, norm=norm, ticks=ticks, 
+            ticklabs=ticklabs, min_lon=min_lon, max_lon=max_lon, min_lat=min_lat, 
+            max_lat=max_lat, lat_lines=lat_lines, lon_lines=lon_lines, 
+            projection=projection, fig=fig, embellish=False, 
+            colorbar_flag=True, alpha=1)        
     ax = display_map.ax
 
     if 'maps' in prdcfg['ppiMapImageConfig']:

diff --git a/src/pyrad_proc/pyrad/io/__init__.py b/src/pyrad_proc/pyrad/io/__init__.py
@@ -167,6 +167,7 @@
     :toctree: generated/
 
     get_rad4alp_prod_fname
+    get_datatype_skyecho
     map_hydro
     map_Doppler
     get_save_dir
@@ -280,6 +281,7 @@
 from .io_aux import find_hzt_file, _get_datetime, get_rad4alp_prod_fname # noqa
 from .io_aux import get_rad4alp_dir, get_rad4alp_grid_dir, find_iso0_file # noqa
 from .io_aux import find_iso0_grib_file, find_date_in_file_name # noqa
+from .io_aux import get_datatype_skyecho # noqa
 
 from .trajectory import Trajectory # noqa
 

diff --git a/src/pyrad_proc/pyrad/io/io_aux.py b/src/pyrad_proc/pyrad/io/io_aux.py
@@ -832,22 +832,41 @@ def get_datatype_skyecho(datatype):
     elif datatype == 'PhiDP':
         datatype_skyecho = 'phidp'
         field_name = 'differential_phase'
+    elif datatype == 'PhiDPc':
+        datatype_skyecho = 'phidp_cor'
+        field_name = 'corrected_differential_phase'
+    elif datatype == 'KDP':
+        datatype_skyecho = 'kdp'
+        field_name = 'specific_differential_phase'
+    elif datatype == 'KDPc':
+        datatype_skyecho = 'kdp_cor'
+        field_name = 'corrected_specific_differential_phase'
     elif datatype == 'ZDR':
         datatype_skyecho = 'ZDR'
         field_name = 'differential_reflectivity'
     elif datatype == 'LDRhv':
         datatype_skyecho = 'LDR_HV'
-        field_name = 'linear_depolariztion_ratio_hv'
+        field_name = 'linear_depolarization_ratio_hv'
     elif datatype == 'LDRvh':
         datatype_skyecho = 'LDR_VH'
-        field_name = 'linear_depolariztion_ratio_vh'
+        field_name = 'linear_depolarization_ratio_vh'
+
+    elif datatype == 'Ah':
+        datatype_skyecho = 'attenuationRain'
+        field_name = 'specific_attenuation'
 
     elif datatype == 'RR':
         datatype_skyecho = 'rainfall_rate'
         field_name = 'radar_estimated_rain_rate'
     elif datatype == 'RR_MP':
         datatype_skyecho = 'rainfall_rate_mp1948'
         field_name = 'Marshall_Palmer_radar_estimated_rain_rate'
+    elif datatype == 'RR_Z':
+        datatype_skyecho = 'rainfall_rate_from_z'
+        field_name = 'radar_reflectivity_estimated_rain_rate'
+    elif datatype == 'RR_KDP':
+        datatype_skyecho = 'rainfall_rate_from_kdp'
+        field_name = 'radar_kdp_estimated_rain_rate'
     elif datatype == 'RR_flag':
         datatype_skyecho = 'rainfall_rate_flag'
         field_name = 'radar_estimated_rain_rate_flag'
@@ -1644,6 +1663,10 @@ def get_fieldname_pyart(datatype):
         field_name = 'radar_estimated_rain_rate'
     elif datatype == 'RR_MP':
         field_name = 'Marshall_Palmer_radar_estimated_rain_rate'
+    elif datatype == 'RR_Z':
+        field_name = 'radar_reflectivity_estimated_rain_rate'
+    elif datatype == 'RR_KDP':
+        field_name = 'radar_kdp_estimated_rain_rate'
     elif datatype == 'RR_flag':
         field_name = 'radar_estimated_rain_rate_flag'
     elif datatype == 'RRc':