Add observations parser

ecgtools/builder.py

@@ -117,7 +117,7 @@ def _filter_files(filelist):
             )
 
         def _glob_dir(directory, extension):
-            return list(directory.rglob(f'*{extension}'))
+            return sorted(list(directory.rglob(f'*{extension}')))
 
         filelist = joblib.Parallel(n_jobs=self.njobs, verbose=5)(
             joblib.delayed(_glob_dir)(directory, self.extension) for directory in self.dirs

ecgtools/parsers/observations.py

@@ -0,0 +1,44 @@
+import pathlib
+import traceback
+from datetime import datetime
+
+import xarray as xr
+
+from ..builder import INVALID_ASSET, TRACEBACK
+
+
+def parse_amwg_obs(file):
+    """Atmospheric observational data stored in"""
+    file = pathlib.Path(file)
+    info = {}
+
+    try:
+        stem = file.stem
+        split = stem.split('_')
+        source = split[0]
+        temporal = split[-2]
+        if len(temporal) == 2:
+            month_number = int(temporal)
+            time_period = 'monthly'
+            temporal = datetime(2020, month_number, 1).strftime('%b').upper()
+
+        elif temporal == 'ANN':
+            time_period = 'annual'
+        else:
+            time_period = 'seasonal'
+
+        with xr.open_dataset(file, chunks={}, decode_times=False) as ds:
+            variable_list = [var for var in ds if 'long_name' in ds[var].attrs]
+
+            info = {
+                'source': source,
+                'temporal': temporal,
+                'time_period': time_period,
+                'variable': variable_list,
+                'path': str(file),
+            }
+
+        return info
+
+    except Exception:
+        return {INVALID_ASSET: file, TRACEBACK: traceback.format_exc()}

tests/test_obs.py

@@ -0,0 +1,37 @@
+import os
+import pathlib
+
+import pandas as pd
+import pytest
+
+from ecgtools import Builder
+from ecgtools.parsers.observations import parse_amwg_obs
+
+sample_data_dir = pathlib.Path(os.path.dirname(__file__)).parent / 'sample_data'
+
+df = pd.DataFrame()
+
+
+@pytest.mark.parametrize(
+    'file_path',
+    [
+        sample_data_dir / 'cesm_obs' / 'AIRS_01_climo.nc',
+        sample_data_dir / 'cesm_obs' / 'MODIS_ANN_climo.nc',
+    ],
+)
+def test_obs_parser(file_path):
+    parse_dict = parse_amwg_obs(file_path)
+    assert isinstance(parse_dict, dict)
+    assert isinstance(df.append(parse_dict, ignore_index=True), pd.DataFrame)
+
+
+@pytest.mark.parametrize(
+    'file_directory',
+    [sample_data_dir / 'cesm_obs'],
+)
+def test_obs_builder(file_directory):
+    b = Builder(file_directory)
+    b.build(parse_amwg_obs)
+    print(file_directory)
+    print(b.df)
+    assert isinstance(b.df, pd.DataFrame)