From c167a093cc5a4cc64dc6319cadac61b4974f5a70 Mon Sep 17 00:00:00 2001 From: Max Grover Date: Thu, 24 Jun 2021 12:37:17 -0600 Subject: [PATCH 01/25] add intial files --- .../cesm2-le-pangeo-forge-execution.ipynb | 177 ++++++++++++++++++ recipes/cesm2_le/cesm_le2_recipe.py | 71 +++++++ 2 files changed, 248 insertions(+) create mode 100644 recipes/cesm2_le/cesm2-le-pangeo-forge-execution.ipynb create mode 100644 recipes/cesm2_le/cesm_le2_recipe.py diff --git a/recipes/cesm2_le/cesm2-le-pangeo-forge-execution.ipynb b/recipes/cesm2_le/cesm2-le-pangeo-forge-execution.ipynb new file mode 100644 index 0000000000..699ce5b4d0 --- /dev/null +++ b/recipes/cesm2_le/cesm2-le-pangeo-forge-execution.ipynb @@ -0,0 +1,177 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "6339b03c-e304-4c3f-96d0-4d9db2a603ad", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/glade/u/home/mgrover/miniconda3/envs/cesm2-marbl/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3361: DtypeWarning: Columns (8,9) have mixed types.Specify dtype option on import or set low_memory=False.\n", + " if (await self.run_code(code, result, async_=asy)):\n" + ] + } + ], + "source": [ + "from cesm_le2_recipe import recipe" + ] + }, + { + "cell_type": "markdown", + "id": "32c613f6-72d3-4e12-bf43-08df5ff4a8cf", + "metadata": {}, + "source": [ + "* Add pr, add directory cesm2-le in staged-recipes/recipes\n", + "## Setup to run the recipe\n", + "\n", + "Set the source and target paths" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "7d5d186b-6e76-48d6-838d-773dd4a7be1b", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "\n", + "# Where the files are stored locally\n", + "direct = '/glade/campaign/cgd/cesm/CESM2-LE/timeseries/atm/proc/tseries/month_1/T'\n", + "\n", + "# Where to write the zarr stores to\n", + "target_dir = '/glade/scratch/mgrover/cesm2-le'" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "1b701c9b-a067-4d8a-9b50-065df33bf18f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "('/glade/campaign/cgd/cesm/CESM2-LE/timeseries/atm/proc/tseries/month_1/T',\n", + " '/glade/scratch/mgrover/cesm2-le',\n", + " '/glade/scratch/mgrover/tmp4kmvzcxz')" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import tempfile\n", + "from fsspec.implementations.local import LocalFileSystem\n", + "from pangeo_forge_recipes.storage import FSSpecTarget, CacheFSSpecTarget\n", + "\n", + "fs_local = LocalFileSystem()\n", + "\n", + "cache_dir = tempfile.TemporaryDirectory()\n", + "cache_target = CacheFSSpecTarget(fs_local, direct)\n", + "\n", + "#target_dir = tempfile.TemporaryDirectory()\n", + "target = FSSpecTarget(fs_local, target_dir)\n", + "\n", + "meta_dir = tempfile.TemporaryDirectory()\n", + "meta_store = FSSpecTarget(fs_local, meta_dir.name)\n", + "\n", + "recipe.input_cache = cache_target\n", + "recipe.target = target\n", + "recipe.metadata_cache = meta_store\n", + "\n", + "cache_target.root_path, target.root_path, meta_store.root_path" + ] + }, + { + "cell_type": "markdown", + "id": "963d8297-d610-440c-8cd9-670276b962f6", + "metadata": {}, + "source": [ + "### Prepare the target\n", + "Basically starts the zarr store - writes the first time step" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "5f6786b1-38d5-4a03-bd04-306591549f28", + "metadata": {}, + "outputs": [ + { + "ename": "FileNotFoundError", + "evalue": "[Errno 2] No such file or directory: '/glade/campaign/cgd/cesm/CESM2-LE/timeseries/atm/proc/tseries/month_1/T/8a8393edfcf123a1af2c1322cc5e77cc-glade_campaign_cgd_cesm_cesm2-le_timeseries_atm_proc_tseries_month_1_t_b.e21.bhistcmip6.f09_g17.le2-1161.009.cam.h0.t.201001-201412.nc'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mGroupNotFoundError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/site-packages/pangeo_forge_recipes/recipes/xarray_zarr.py\u001b[0m in \u001b[0;36mprepare_target\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 212\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 213\u001b[0;31m \u001b[0mds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen_target\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 214\u001b[0m \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Found an existing dataset in target\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/site-packages/pangeo_forge_recipes/recipes/xarray_zarr.py\u001b[0m in \u001b[0;36mopen_target\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 398\u001b[0m \u001b[0mtarget_mapper\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtarget\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_mapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 399\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mxr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen_zarr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtarget_mapper\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 400\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/site-packages/xarray/backends/zarr.py\u001b[0m in \u001b[0;36mopen_zarr\u001b[0;34m(store, group, synchronizer, chunks, decode_cf, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, consolidated, overwrite_encoded_chunks, chunk_store, storage_options, decode_timedelta, use_cftime, **kwargs)\u001b[0m\n\u001b[1;32m 699\u001b[0m \u001b[0mdecode_timedelta\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdecode_timedelta\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 700\u001b[0;31m \u001b[0muse_cftime\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0muse_cftime\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 701\u001b[0m )\n", + "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/site-packages/xarray/backends/api.py\u001b[0m in \u001b[0;36mopen_dataset\u001b[0;34m(filename_or_obj, engine, chunks, cache, decode_cf, mask_and_scale, decode_times, decode_timedelta, use_cftime, concat_characters, decode_coords, drop_variables, backend_kwargs, *args, **kwargs)\u001b[0m\n\u001b[1;32m 499\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mdecoders\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 500\u001b[0;31m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 501\u001b[0m )\n", + "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/site-packages/xarray/backends/zarr.py\u001b[0m in \u001b[0;36mopen_dataset\u001b[0;34m(self, filename_or_obj, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, use_cftime, decode_timedelta, group, mode, synchronizer, consolidated, consolidate_on_close, chunk_store, storage_options, lock)\u001b[0m\n\u001b[1;32m 741\u001b[0m \u001b[0mchunk_store\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mchunk_store\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 742\u001b[0;31m \u001b[0mstorage_options\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstorage_options\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 743\u001b[0m )\n", + "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/site-packages/xarray/backends/zarr.py\u001b[0m in \u001b[0;36mopen_group\u001b[0;34m(cls, store, mode, synchronizer, group, consolidated, consolidate_on_close, chunk_store, storage_options, append_dim, write_region, safe_chunks)\u001b[0m\n\u001b[1;32m 337\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 338\u001b[0;31m \u001b[0mzarr_group\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mzarr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen_group\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstore\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mopen_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 339\u001b[0m return cls(\n", + "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/site-packages/zarr/hierarchy.py\u001b[0m in \u001b[0;36mopen_group\u001b[0;34m(store, mode, cache_attrs, synchronizer, path, chunk_store, storage_options)\u001b[0m\n\u001b[1;32m 1167\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mcontains_group\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstore\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1168\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mGroupNotFoundError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1169\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mGroupNotFoundError\u001b[0m: group not found at path ''", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mrecipe\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprepare_target\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/site-packages/pangeo_forge_recipes/recipes/xarray_zarr.py\u001b[0m in \u001b[0;36mprepare_target\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 225\u001b[0m \u001b[0;31m# need to rewrite this as an append loop\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 226\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mchunk_key\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_init_chunks\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 227\u001b[0;31m \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen_chunk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunk_key\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mds\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 228\u001b[0m \u001b[0;31m# ds is already chunked\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 229\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/contextlib.py\u001b[0m in \u001b[0;36m__enter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 110\u001b[0m \u001b[0;32mdel\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 111\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 112\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgen\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 113\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 114\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"generator didn't yield\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/site-packages/pangeo_forge_recipes/recipes/xarray_zarr.py\u001b[0m in \u001b[0;36mopen_chunk\u001b[0;34m(self, chunk_key)\u001b[0m\n\u001b[1;32m 374\u001b[0m \u001b[0;31m# need to open an unknown number of contexts at the same time\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 375\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mExitStack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mstack\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 376\u001b[0;31m \u001b[0mdsets\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mstack\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menter_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen_input\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 377\u001b[0m \u001b[0;31m# explicitly chunking prevents eager evaluation during concat\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 378\u001b[0m \u001b[0mdsets\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchunk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mds\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdsets\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/site-packages/pangeo_forge_recipes/recipes/xarray_zarr.py\u001b[0m in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 374\u001b[0m \u001b[0;31m# need to open an unknown number of contexts at the same time\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 375\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mExitStack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mstack\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 376\u001b[0;31m \u001b[0mdsets\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mstack\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menter_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen_input\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 377\u001b[0m \u001b[0;31m# explicitly chunking prevents eager evaluation during concat\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 378\u001b[0m \u001b[0mdsets\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchunk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mds\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdsets\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/contextlib.py\u001b[0m in \u001b[0;36menter_context\u001b[0;34m(self, cm)\u001b[0m\n\u001b[1;32m 425\u001b[0m \u001b[0m_cm_type\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcm\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 426\u001b[0m \u001b[0m_exit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_cm_type\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__exit__\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 427\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_cm_type\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__enter__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcm\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 428\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_push_cm_exit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_exit\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 429\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/contextlib.py\u001b[0m in \u001b[0;36m__enter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 110\u001b[0m \u001b[0;32mdel\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 111\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 112\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgen\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 113\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 114\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"generator didn't yield\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/site-packages/pangeo_forge_recipes/recipes/xarray_zarr.py\u001b[0m in \u001b[0;36mopen_input\u001b[0;34m(self, input_key)\u001b[0m\n\u001b[1;32m 345\u001b[0m \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"Opening input with Xarray {input_key}: '{fname}'\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 346\u001b[0m \u001b[0mcache\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minput_cache\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcache_inputs\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 347\u001b[0;31m \u001b[0;32mwith\u001b[0m \u001b[0mfile_opener\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcache\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcache\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy_to_local\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy_input_to_local_file\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 348\u001b[0m \u001b[0mds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mxr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mxarray_open_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 349\u001b[0m \u001b[0mds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfix_scalar_attr_encoding\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/contextlib.py\u001b[0m in \u001b[0;36m__enter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 110\u001b[0m \u001b[0;32mdel\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 111\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 112\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgen\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 113\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 114\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"generator didn't yield\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/site-packages/pangeo_forge_recipes/storage.py\u001b[0m in \u001b[0;36mfile_opener\u001b[0;34m(fname, cache, copy_to_local, **open_kwargs)\u001b[0m\n\u001b[1;32m 198\u001b[0m \u001b[0mntf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# cleans up the temporary file\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 199\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 200\u001b[0;31m \u001b[0;32mwith\u001b[0m \u001b[0mopener\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mfp\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 201\u001b[0m \u001b[0;32myield\u001b[0m \u001b[0mfp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 202\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/contextlib.py\u001b[0m in \u001b[0;36m__enter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 110\u001b[0m \u001b[0;32mdel\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 111\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 112\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgen\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 113\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 114\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"generator didn't yield\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/site-packages/pangeo_forge_recipes/storage.py\u001b[0m in \u001b[0;36mopen\u001b[0;34m(self, path, **kwargs)\u001b[0m\n\u001b[1;32m 106\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mIterator\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 107\u001b[0m \u001b[0;34m\"\"\"Open file with a context manager.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 108\u001b[0;31m \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_full_path\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 109\u001b[0m \u001b[0;32myield\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 110\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/site-packages/fsspec/spec.py\u001b[0m in \u001b[0;36mopen\u001b[0;34m(self, path, mode, block_size, cache_options, **kwargs)\u001b[0m\n\u001b[1;32m 966\u001b[0m \u001b[0mautocommit\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mac\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 967\u001b[0m \u001b[0mcache_options\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcache_options\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 968\u001b[0;31m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 969\u001b[0m )\n\u001b[1;32m 970\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mac\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;34m\"r\"\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/site-packages/fsspec/implementations/local.py\u001b[0m in \u001b[0;36m_open\u001b[0;34m(self, path, mode, block_size, **kwargs)\u001b[0m\n\u001b[1;32m 130\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mauto_mkdir\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;34m\"w\"\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 131\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmakedirs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_parent\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexist_ok\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 132\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mLocalFileOpener\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 133\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 134\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mtouch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/site-packages/fsspec/implementations/local.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, path, mode, autocommit, fs, **kwargs)\u001b[0m\n\u001b[1;32m 218\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautocommit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mautocommit\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 219\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mblocksize\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mio\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDEFAULT_BUFFER_SIZE\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 220\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_open\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 221\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 222\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_open\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/site-packages/fsspec/implementations/local.py\u001b[0m in \u001b[0;36m_open\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 223\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclosed\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 224\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautocommit\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m\"w\"\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 225\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 226\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 227\u001b[0m \u001b[0;31m# TODO: check if path is writable?\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '/glade/campaign/cgd/cesm/CESM2-LE/timeseries/atm/proc/tseries/month_1/T/8a8393edfcf123a1af2c1322cc5e77cc-glade_campaign_cgd_cesm_cesm2-le_timeseries_atm_proc_tseries_month_1_t_b.e21.bhistcmip6.f09_g17.le2-1161.009.cam.h0.t.201001-201412.nc'" + ] + } + ], + "source": [ + "recipe.prepare_target()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "47643312-bf7d-46c6-8c6a-ee44885c9273", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python [conda env:miniconda3-cesm2-marbl]", + "language": "python", + "name": "conda-env-miniconda3-cesm2-marbl-py" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/recipes/cesm2_le/cesm_le2_recipe.py b/recipes/cesm2_le/cesm_le2_recipe.py new file mode 100644 index 0000000000..9cb141f9c4 --- /dev/null +++ b/recipes/cesm2_le/cesm_le2_recipe.py @@ -0,0 +1,71 @@ +import pandas as pd +import xarray as xr +from pangeo_forge_recipes.patterns import FilePattern +from pangeo_forge_recipes.patterns import ConcatDim +from pangeo_forge_recipes.recipes.xarray_zarr import XarrayZarrRecipe + +df = pd.read_csv('/glade/work/mgrover/intake-esm-catalogs/cesm2-le.csv.gz') + +df_sub = df[(df.component == 'atm') & + (df.frequency == 'month_1') & + (df.experiment == 'historical') & + (df.variable == 'T') & + (df.experiment_number == 1161) + ] + +ds = xr.open_dataset(df_sub.path.values[0]) + +def determine_chunk_size(ds): + ntime = len(ds.time) # the number of time slices + chunksize_optimal = 100e6 # desired chunk size in bytes + ncfile_size = ds.nbytes # the netcdf file size + chunksize = max(int(ntime* chunksize_optimal/ ncfile_size),1) + + target_chunks = ds.dims.mapping + target_chunks['time'] = chunksize + + return target_chunks # a dictionary giving the chunk sizes in each dimension + +ntime = len(ds.time) # the number of time slices +chunksize_optimal = 100e6 # desired chunk size in bytes +ncfile_size = ds.nbytes # the netcdf file size +chunksize = max(int(ntime* chunksize_optimal/ ncfile_size),1) + +target_chunks = ds.dims.mapping +target_chunks['time'] = chunksize + +# the netcdf lists some of the coordinate variables as data variables. This is a fix which we want to apply to each chunk. +def set_bnds_as_coords(ds): + new_coords_vars = [var for var in ds.data_vars if 'bnds' in var or 'bounds' in var] + ds = ds.set_coords(new_coords_vars) + return ds + +def make_full_path(time): + ''' + Parameters + ---------- + time : str + + A 13-character string, comprised of two 6-character dates delimited by a dash. + The first four characters of each date are the year, and the final two are the month. + + e.g. The time range from Jan 1850 through Dec 1949 is expressed as '185001-194912'. + + ''' + base_url = '/glade/campaign/cgd/cesm/CESM2-LE/timeseries/atm/proc/tseries/month_1/T/' + return base_url + f'b.e21.BHISTcmip6.f09_g17.LE2-1161.009.cam.h0.T.{time}.nc' + +keys = [] +for path in df_sub.path.values: + keys.append(str(path.split('.')[-2])) + +time_concat_dim = ConcatDim("time", keys=keys) + +pattern = FilePattern(make_full_path, time_concat_dim) + +recipe = XarrayZarrRecipe( + pattern, + target_chunks = target_chunks, + #xarray_open_kwargs = {'decode_coords':False}, + xarray_concat_kwargs = {'coords': 'minimal', 'compat': 'override'}, +) \ No newline at end of file From e377b76f09d079744111e53a7c5d40c358fdbc95 Mon Sep 17 00:00:00 2001 From: cisaacstern <62192187+cisaacstern@users.noreply.github.com> Date: Thu, 24 Jun 2021 12:32:48 -0700 Subject: [PATCH 02/25] input target: CacheFSSpecTarget -> FSSpecTarget --- recipes/cesm2_le/cesm2-le-pangeo-forge-execution.ipynb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/recipes/cesm2_le/cesm2-le-pangeo-forge-execution.ipynb b/recipes/cesm2_le/cesm2-le-pangeo-forge-execution.ipynb index 699ce5b4d0..f0aa1f9200 100644 --- a/recipes/cesm2_le/cesm2-le-pangeo-forge-execution.ipynb +++ b/recipes/cesm2_le/cesm2-le-pangeo-forge-execution.ipynb @@ -68,12 +68,13 @@ "source": [ "import tempfile\n", "from fsspec.implementations.local import LocalFileSystem\n", - "from pangeo_forge_recipes.storage import FSSpecTarget, CacheFSSpecTarget\n", + "from pangeo_forge_recipes.storage import FSSpecTarget #, CacheFSSpecTarget\n", "\n", "fs_local = LocalFileSystem()\n", "\n", "cache_dir = tempfile.TemporaryDirectory()\n", - "cache_target = CacheFSSpecTarget(fs_local, direct)\n", + "# cache_target = CacheFSSpecTarget(fs_local, direct)\n", + "cache_target = FSSpecTarget(fs_local, direct)\n", "\n", "#target_dir = tempfile.TemporaryDirectory()\n", "target = FSSpecTarget(fs_local, target_dir)\n", @@ -174,4 +175,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} +} \ No newline at end of file From 853fde2c26c8d3e315847ff12b7e31a7d4aec9a4 Mon Sep 17 00:00:00 2001 From: Paige Martin Date: Mon, 28 Jun 2021 09:54:23 +1000 Subject: [PATCH 03/25] first commit --- recipes/cesm-pop-lowres-1deg/recipe.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 recipes/cesm-pop-lowres-1deg/recipe.py diff --git a/recipes/cesm-pop-lowres-1deg/recipe.py b/recipes/cesm-pop-lowres-1deg/recipe.py new file mode 100644 index 0000000000..e69de29bb2 From 8d028f70810a5bc79ee299e55c6cde1f4f1332e8 Mon Sep 17 00:00:00 2001 From: cisaacstern <62192187+cisaacstern@users.noreply.github.com> Date: Mon, 28 Jun 2021 12:45:10 -0700 Subject: [PATCH 04/25] recipe outline --- recipes/cesm-pop-lowres-1deg/recipe.py | 31 ++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/recipes/cesm-pop-lowres-1deg/recipe.py b/recipes/cesm-pop-lowres-1deg/recipe.py index e69de29bb2..c70377a3b4 100644 --- a/recipes/cesm-pop-lowres-1deg/recipe.py +++ b/recipes/cesm-pop-lowres-1deg/recipe.py @@ -0,0 +1,31 @@ +from pangeo_forge_recipes.patterns import MergeDim, FilePattern +from pangeo_forge_recipes.recipes import XarrayZarrRecipe + + +def make_full_path(variable): + """Returns a valid path to the source files + + Parameters + ---------- + variable: str + A string representing each variable + """ + return f"fill_in_the_path_here_{variable}.nc" + + +vars = [ + # this should be a list of strings cooresponding to the variable names + # it will eventually be passed into the `make_full_path` function, + # to generate the source file paths +] + +merge_dim = MergeDim("variable", keys=vars) + +pattern = FilePattern(make_full_path, merge_dim) + +chunks = { + # this should be a dictionary mapping the name of the time dimension to + # the desired chunk size, for example: `{"time": 15}` +} + +recipe = XarrayZarrRecipe(pattern, target_chunks=chunks) \ No newline at end of file From e25b91f90eff6c9e5ba9f0247a96dd23651a6d19 Mon Sep 17 00:00:00 2001 From: cisaacstern <62192187+cisaacstern@users.noreply.github.com> Date: Mon, 28 Jun 2021 12:47:58 -0700 Subject: [PATCH 05/25] remove cesm2_le directory --- .../cesm2-le-pangeo-forge-execution.ipynb | 178 ------------------ recipes/cesm2_le/cesm_le2_recipe.py | 71 ------- 2 files changed, 249 deletions(-) delete mode 100644 recipes/cesm2_le/cesm2-le-pangeo-forge-execution.ipynb delete mode 100644 recipes/cesm2_le/cesm_le2_recipe.py diff --git a/recipes/cesm2_le/cesm2-le-pangeo-forge-execution.ipynb b/recipes/cesm2_le/cesm2-le-pangeo-forge-execution.ipynb deleted file mode 100644 index f0aa1f9200..0000000000 --- a/recipes/cesm2_le/cesm2-le-pangeo-forge-execution.ipynb +++ /dev/null @@ -1,178 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "id": "6339b03c-e304-4c3f-96d0-4d9db2a603ad", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/glade/u/home/mgrover/miniconda3/envs/cesm2-marbl/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3361: DtypeWarning: Columns (8,9) have mixed types.Specify dtype option on import or set low_memory=False.\n", - " if (await self.run_code(code, result, async_=asy)):\n" - ] - } - ], - "source": [ - "from cesm_le2_recipe import recipe" - ] - }, - { - "cell_type": "markdown", - "id": "32c613f6-72d3-4e12-bf43-08df5ff4a8cf", - "metadata": {}, - "source": [ - "* Add pr, add directory cesm2-le in staged-recipes/recipes\n", - "## Setup to run the recipe\n", - "\n", - "Set the source and target paths" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "7d5d186b-6e76-48d6-838d-773dd4a7be1b", - "metadata": {}, - "outputs": [], - "source": [ - "from pathlib import Path\n", - "\n", - "# Where the files are stored locally\n", - "direct = '/glade/campaign/cgd/cesm/CESM2-LE/timeseries/atm/proc/tseries/month_1/T'\n", - "\n", - "# Where to write the zarr stores to\n", - "target_dir = '/glade/scratch/mgrover/cesm2-le'" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "1b701c9b-a067-4d8a-9b50-065df33bf18f", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "('/glade/campaign/cgd/cesm/CESM2-LE/timeseries/atm/proc/tseries/month_1/T',\n", - " '/glade/scratch/mgrover/cesm2-le',\n", - " '/glade/scratch/mgrover/tmp4kmvzcxz')" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import tempfile\n", - "from fsspec.implementations.local import LocalFileSystem\n", - "from pangeo_forge_recipes.storage import FSSpecTarget #, CacheFSSpecTarget\n", - "\n", - "fs_local = LocalFileSystem()\n", - "\n", - "cache_dir = tempfile.TemporaryDirectory()\n", - "# cache_target = CacheFSSpecTarget(fs_local, direct)\n", - "cache_target = FSSpecTarget(fs_local, direct)\n", - "\n", - "#target_dir = tempfile.TemporaryDirectory()\n", - "target = FSSpecTarget(fs_local, target_dir)\n", - "\n", - "meta_dir = tempfile.TemporaryDirectory()\n", - "meta_store = FSSpecTarget(fs_local, meta_dir.name)\n", - "\n", - "recipe.input_cache = cache_target\n", - "recipe.target = target\n", - "recipe.metadata_cache = meta_store\n", - "\n", - "cache_target.root_path, target.root_path, meta_store.root_path" - ] - }, - { - "cell_type": "markdown", - "id": "963d8297-d610-440c-8cd9-670276b962f6", - "metadata": {}, - "source": [ - "### Prepare the target\n", - "Basically starts the zarr store - writes the first time step" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "5f6786b1-38d5-4a03-bd04-306591549f28", - "metadata": {}, - "outputs": [ - { - "ename": "FileNotFoundError", - "evalue": "[Errno 2] No such file or directory: '/glade/campaign/cgd/cesm/CESM2-LE/timeseries/atm/proc/tseries/month_1/T/8a8393edfcf123a1af2c1322cc5e77cc-glade_campaign_cgd_cesm_cesm2-le_timeseries_atm_proc_tseries_month_1_t_b.e21.bhistcmip6.f09_g17.le2-1161.009.cam.h0.t.201001-201412.nc'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mGroupNotFoundError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/site-packages/pangeo_forge_recipes/recipes/xarray_zarr.py\u001b[0m in \u001b[0;36mprepare_target\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 212\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 213\u001b[0;31m \u001b[0mds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen_target\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 214\u001b[0m \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Found an existing dataset in target\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/site-packages/pangeo_forge_recipes/recipes/xarray_zarr.py\u001b[0m in \u001b[0;36mopen_target\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 398\u001b[0m \u001b[0mtarget_mapper\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtarget\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_mapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 399\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mxr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen_zarr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtarget_mapper\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 400\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/site-packages/xarray/backends/zarr.py\u001b[0m in \u001b[0;36mopen_zarr\u001b[0;34m(store, group, synchronizer, chunks, decode_cf, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, consolidated, overwrite_encoded_chunks, chunk_store, storage_options, decode_timedelta, use_cftime, **kwargs)\u001b[0m\n\u001b[1;32m 699\u001b[0m \u001b[0mdecode_timedelta\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdecode_timedelta\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 700\u001b[0;31m \u001b[0muse_cftime\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0muse_cftime\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 701\u001b[0m )\n", - "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/site-packages/xarray/backends/api.py\u001b[0m in \u001b[0;36mopen_dataset\u001b[0;34m(filename_or_obj, engine, chunks, cache, decode_cf, mask_and_scale, decode_times, decode_timedelta, use_cftime, concat_characters, decode_coords, drop_variables, backend_kwargs, *args, **kwargs)\u001b[0m\n\u001b[1;32m 499\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mdecoders\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 500\u001b[0;31m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 501\u001b[0m )\n", - "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/site-packages/xarray/backends/zarr.py\u001b[0m in \u001b[0;36mopen_dataset\u001b[0;34m(self, filename_or_obj, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, use_cftime, decode_timedelta, group, mode, synchronizer, consolidated, consolidate_on_close, chunk_store, storage_options, lock)\u001b[0m\n\u001b[1;32m 741\u001b[0m \u001b[0mchunk_store\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mchunk_store\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 742\u001b[0;31m \u001b[0mstorage_options\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstorage_options\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 743\u001b[0m )\n", - "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/site-packages/xarray/backends/zarr.py\u001b[0m in \u001b[0;36mopen_group\u001b[0;34m(cls, store, mode, synchronizer, group, consolidated, consolidate_on_close, chunk_store, storage_options, append_dim, write_region, safe_chunks)\u001b[0m\n\u001b[1;32m 337\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 338\u001b[0;31m \u001b[0mzarr_group\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mzarr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen_group\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstore\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mopen_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 339\u001b[0m return cls(\n", - "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/site-packages/zarr/hierarchy.py\u001b[0m in \u001b[0;36mopen_group\u001b[0;34m(store, mode, cache_attrs, synchronizer, path, chunk_store, storage_options)\u001b[0m\n\u001b[1;32m 1167\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mcontains_group\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstore\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1168\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mGroupNotFoundError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1169\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mGroupNotFoundError\u001b[0m: group not found at path ''", - "\nDuring handling of the above exception, another exception occurred:\n", - "\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mrecipe\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprepare_target\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/site-packages/pangeo_forge_recipes/recipes/xarray_zarr.py\u001b[0m in \u001b[0;36mprepare_target\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 225\u001b[0m \u001b[0;31m# need to rewrite this as an append loop\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 226\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mchunk_key\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_init_chunks\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 227\u001b[0;31m \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen_chunk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunk_key\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mds\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 228\u001b[0m \u001b[0;31m# ds is already chunked\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 229\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/contextlib.py\u001b[0m in \u001b[0;36m__enter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 110\u001b[0m \u001b[0;32mdel\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 111\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 112\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgen\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 113\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 114\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"generator didn't yield\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/site-packages/pangeo_forge_recipes/recipes/xarray_zarr.py\u001b[0m in \u001b[0;36mopen_chunk\u001b[0;34m(self, chunk_key)\u001b[0m\n\u001b[1;32m 374\u001b[0m \u001b[0;31m# need to open an unknown number of contexts at the same time\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 375\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mExitStack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mstack\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 376\u001b[0;31m \u001b[0mdsets\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mstack\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menter_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen_input\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 377\u001b[0m \u001b[0;31m# explicitly chunking prevents eager evaluation during concat\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 378\u001b[0m \u001b[0mdsets\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchunk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mds\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdsets\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/site-packages/pangeo_forge_recipes/recipes/xarray_zarr.py\u001b[0m in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 374\u001b[0m \u001b[0;31m# need to open an unknown number of contexts at the same time\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 375\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mExitStack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mstack\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 376\u001b[0;31m \u001b[0mdsets\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mstack\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menter_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen_input\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 377\u001b[0m \u001b[0;31m# explicitly chunking prevents eager evaluation during concat\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 378\u001b[0m \u001b[0mdsets\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchunk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mds\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdsets\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/contextlib.py\u001b[0m in \u001b[0;36menter_context\u001b[0;34m(self, cm)\u001b[0m\n\u001b[1;32m 425\u001b[0m \u001b[0m_cm_type\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcm\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 426\u001b[0m \u001b[0m_exit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_cm_type\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__exit__\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 427\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_cm_type\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__enter__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcm\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 428\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_push_cm_exit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_exit\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 429\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/contextlib.py\u001b[0m in \u001b[0;36m__enter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 110\u001b[0m \u001b[0;32mdel\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 111\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 112\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgen\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 113\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 114\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"generator didn't yield\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/site-packages/pangeo_forge_recipes/recipes/xarray_zarr.py\u001b[0m in \u001b[0;36mopen_input\u001b[0;34m(self, input_key)\u001b[0m\n\u001b[1;32m 345\u001b[0m \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"Opening input with Xarray {input_key}: '{fname}'\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 346\u001b[0m \u001b[0mcache\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minput_cache\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcache_inputs\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 347\u001b[0;31m \u001b[0;32mwith\u001b[0m \u001b[0mfile_opener\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcache\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcache\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy_to_local\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy_input_to_local_file\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 348\u001b[0m \u001b[0mds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mxr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mxarray_open_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 349\u001b[0m \u001b[0mds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfix_scalar_attr_encoding\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/contextlib.py\u001b[0m in \u001b[0;36m__enter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 110\u001b[0m \u001b[0;32mdel\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 111\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 112\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgen\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 113\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 114\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"generator didn't yield\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/site-packages/pangeo_forge_recipes/storage.py\u001b[0m in \u001b[0;36mfile_opener\u001b[0;34m(fname, cache, copy_to_local, **open_kwargs)\u001b[0m\n\u001b[1;32m 198\u001b[0m \u001b[0mntf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# cleans up the temporary file\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 199\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 200\u001b[0;31m \u001b[0;32mwith\u001b[0m \u001b[0mopener\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mfp\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 201\u001b[0m \u001b[0;32myield\u001b[0m \u001b[0mfp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 202\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/contextlib.py\u001b[0m in \u001b[0;36m__enter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 110\u001b[0m \u001b[0;32mdel\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 111\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 112\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgen\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 113\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 114\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"generator didn't yield\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/site-packages/pangeo_forge_recipes/storage.py\u001b[0m in \u001b[0;36mopen\u001b[0;34m(self, path, **kwargs)\u001b[0m\n\u001b[1;32m 106\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mIterator\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 107\u001b[0m \u001b[0;34m\"\"\"Open file with a context manager.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 108\u001b[0;31m \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_full_path\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 109\u001b[0m \u001b[0;32myield\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 110\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/site-packages/fsspec/spec.py\u001b[0m in \u001b[0;36mopen\u001b[0;34m(self, path, mode, block_size, cache_options, **kwargs)\u001b[0m\n\u001b[1;32m 966\u001b[0m \u001b[0mautocommit\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mac\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 967\u001b[0m \u001b[0mcache_options\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcache_options\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 968\u001b[0;31m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 969\u001b[0m )\n\u001b[1;32m 970\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mac\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;34m\"r\"\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/site-packages/fsspec/implementations/local.py\u001b[0m in \u001b[0;36m_open\u001b[0;34m(self, path, mode, block_size, **kwargs)\u001b[0m\n\u001b[1;32m 130\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mauto_mkdir\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;34m\"w\"\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 131\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmakedirs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_parent\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexist_ok\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 132\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mLocalFileOpener\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 133\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 134\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mtouch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/site-packages/fsspec/implementations/local.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, path, mode, autocommit, fs, **kwargs)\u001b[0m\n\u001b[1;32m 218\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautocommit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mautocommit\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 219\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mblocksize\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mio\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDEFAULT_BUFFER_SIZE\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 220\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_open\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 221\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 222\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_open\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/miniconda3/envs/cesm2-marbl/lib/python3.7/site-packages/fsspec/implementations/local.py\u001b[0m in \u001b[0;36m_open\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 223\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclosed\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 224\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautocommit\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m\"w\"\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 225\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 226\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 227\u001b[0m \u001b[0;31m# TODO: check if path is writable?\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '/glade/campaign/cgd/cesm/CESM2-LE/timeseries/atm/proc/tseries/month_1/T/8a8393edfcf123a1af2c1322cc5e77cc-glade_campaign_cgd_cesm_cesm2-le_timeseries_atm_proc_tseries_month_1_t_b.e21.bhistcmip6.f09_g17.le2-1161.009.cam.h0.t.201001-201412.nc'" - ] - } - ], - "source": [ - "recipe.prepare_target()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "47643312-bf7d-46c6-8c6a-ee44885c9273", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python [conda env:miniconda3-cesm2-marbl]", - "language": "python", - "name": "conda-env-miniconda3-cesm2-marbl-py" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.10" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} \ No newline at end of file diff --git a/recipes/cesm2_le/cesm_le2_recipe.py b/recipes/cesm2_le/cesm_le2_recipe.py deleted file mode 100644 index 9cb141f9c4..0000000000 --- a/recipes/cesm2_le/cesm_le2_recipe.py +++ /dev/null @@ -1,71 +0,0 @@ -import pandas as pd -import xarray as xr -from pangeo_forge_recipes.patterns import FilePattern -from pangeo_forge_recipes.patterns import ConcatDim -from pangeo_forge_recipes.recipes.xarray_zarr import XarrayZarrRecipe - -df = pd.read_csv('/glade/work/mgrover/intake-esm-catalogs/cesm2-le.csv.gz') - -df_sub = df[(df.component == 'atm') & - (df.frequency == 'month_1') & - (df.experiment == 'historical') & - (df.variable == 'T') & - (df.experiment_number == 1161) - ] - -ds = xr.open_dataset(df_sub.path.values[0]) - -def determine_chunk_size(ds): - ntime = len(ds.time) # the number of time slices - chunksize_optimal = 100e6 # desired chunk size in bytes - ncfile_size = ds.nbytes # the netcdf file size - chunksize = max(int(ntime* chunksize_optimal/ ncfile_size),1) - - target_chunks = ds.dims.mapping - target_chunks['time'] = chunksize - - return target_chunks # a dictionary giving the chunk sizes in each dimension - -ntime = len(ds.time) # the number of time slices -chunksize_optimal = 100e6 # desired chunk size in bytes -ncfile_size = ds.nbytes # the netcdf file size -chunksize = max(int(ntime* chunksize_optimal/ ncfile_size),1) - -target_chunks = ds.dims.mapping -target_chunks['time'] = chunksize - -# the netcdf lists some of the coordinate variables as data variables. This is a fix which we want to apply to each chunk. -def set_bnds_as_coords(ds): - new_coords_vars = [var for var in ds.data_vars if 'bnds' in var or 'bounds' in var] - ds = ds.set_coords(new_coords_vars) - return ds - -def make_full_path(time): - ''' - Parameters - ---------- - time : str - - A 13-character string, comprised of two 6-character dates delimited by a dash. - The first four characters of each date are the year, and the final two are the month. - - e.g. The time range from Jan 1850 through Dec 1949 is expressed as '185001-194912'. - - ''' - base_url = '/glade/campaign/cgd/cesm/CESM2-LE/timeseries/atm/proc/tseries/month_1/T/' - return base_url + f'b.e21.BHISTcmip6.f09_g17.LE2-1161.009.cam.h0.T.{time}.nc' - -keys = [] -for path in df_sub.path.values: - keys.append(str(path.split('.')[-2])) - -time_concat_dim = ConcatDim("time", keys=keys) - -pattern = FilePattern(make_full_path, time_concat_dim) - -recipe = XarrayZarrRecipe( - pattern, - target_chunks = target_chunks, - #xarray_open_kwargs = {'decode_coords':False}, - xarray_concat_kwargs = {'coords': 'minimal', 'compat': 'override'}, -) \ No newline at end of file From b59e3efc0dd29c845532f2e0b623f8bdbc5ff28e Mon Sep 17 00:00:00 2001 From: Paige Martin Date: Wed, 30 Jun 2021 10:43:09 +1000 Subject: [PATCH 06/25] add download link, variables, possible chunk size --- recipes/cesm-pop-lowres-1deg/recipe.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/recipes/cesm-pop-lowres-1deg/recipe.py b/recipes/cesm-pop-lowres-1deg/recipe.py index c70377a3b4..3f01229ec1 100644 --- a/recipes/cesm-pop-lowres-1deg/recipe.py +++ b/recipes/cesm-pop-lowres-1deg/recipe.py @@ -10,13 +10,24 @@ def make_full_path(variable): variable: str A string representing each variable """ - return f"fill_in_the_path_here_{variable}.nc" + return f"https://tds.ucar.edu/thredds/fileServer/datazone/campaign/cesm/collections/ASD/v5_rel04_BC5_ne30_g16/ocn/proc/tseries/daily/v5_rel04_BC5_ne30_g16.pop.h.nday1.{variable}.00010101-01661231.nc" vars = [ - # this should be a list of strings cooresponding to the variable names - # it will eventually be passed into the `make_full_path` function, - # to generate the source file paths + 'HMXL_2', + 'SFWF_2', + 'SHF_2', + 'SSH_2', + 'SSS', + 'SST', + 'SST2', + 'TAUX_2', + 'TAUY_2', + 'U1_1', + 'U2_2', + 'V1_1', + 'V2_2', + 'XMXL_2' ] merge_dim = MergeDim("variable", keys=vars) @@ -24,8 +35,7 @@ def make_full_path(variable): pattern = FilePattern(make_full_path, merge_dim) chunks = { - # this should be a dictionary mapping the name of the time dimension to - # the desired chunk size, for example: `{"time": 15}` + 'time:20' } -recipe = XarrayZarrRecipe(pattern, target_chunks=chunks) \ No newline at end of file +recipe = XarrayZarrRecipe(pattern, target_chunks=chunks) From c029fe6ef2f4b53280add761f00cfe49b3aa3417 Mon Sep 17 00:00:00 2001 From: Paige Martin Date: Wed, 30 Jun 2021 11:05:09 +1000 Subject: [PATCH 07/25] updated chunk size --- recipes/cesm-pop-lowres-1deg/recipe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/cesm-pop-lowres-1deg/recipe.py b/recipes/cesm-pop-lowres-1deg/recipe.py index 3f01229ec1..0d699d94a3 100644 --- a/recipes/cesm-pop-lowres-1deg/recipe.py +++ b/recipes/cesm-pop-lowres-1deg/recipe.py @@ -35,7 +35,7 @@ def make_full_path(variable): pattern = FilePattern(make_full_path, merge_dim) chunks = { - 'time:20' + 'time:200' } recipe = XarrayZarrRecipe(pattern, target_chunks=chunks) From ddc88dabd900f50bda2e5e97425d710cfee3100b Mon Sep 17 00:00:00 2001 From: cisaacstern <62192187+cisaacstern@users.noreply.github.com> Date: Fri, 9 Jul 2021 10:31:46 -0700 Subject: [PATCH 08/25] formatting --- recipes/cesm-pop-lowres-1deg/recipe.py | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/recipes/cesm-pop-lowres-1deg/recipe.py b/recipes/cesm-pop-lowres-1deg/recipe.py index 0d699d94a3..26ac40aff0 100644 --- a/recipes/cesm-pop-lowres-1deg/recipe.py +++ b/recipes/cesm-pop-lowres-1deg/recipe.py @@ -10,32 +10,22 @@ def make_full_path(variable): variable: str A string representing each variable """ - return f"https://tds.ucar.edu/thredds/fileServer/datazone/campaign/cesm/collections/ASD/v5_rel04_BC5_ne30_g16/ocn/proc/tseries/daily/v5_rel04_BC5_ne30_g16.pop.h.nday1.{variable}.00010101-01661231.nc" + return ( + f"https://tds.ucar.edu/thredds/fileServer/datazone/campaign/cesm/collections/ASD/" + f"v5_rel04_BC5_ne30_g16/ocn/proc/tseries/daily/v5_rel04_BC5_ne30_g16.pop.h.nday1." + f"{variable}.00010101-01661231.nc" + ) vars = [ - 'HMXL_2', - 'SFWF_2', - 'SHF_2', - 'SSH_2', - 'SSS', - 'SST', - 'SST2', - 'TAUX_2', - 'TAUY_2', - 'U1_1', - 'U2_2', - 'V1_1', - 'V2_2', - 'XMXL_2' + 'HMXL_2', 'SFWF_2', 'SHF_2', 'SSH_2', 'SSS', 'SST', 'SST2', 'TAUX_2', 'TAUY_2', 'U1_1', + 'U2_2', 'V1_1', 'V2_2', 'XMXL_2', ] merge_dim = MergeDim("variable", keys=vars) pattern = FilePattern(make_full_path, merge_dim) -chunks = { - 'time:200' -} +chunks = {'time': 200} recipe = XarrayZarrRecipe(pattern, target_chunks=chunks) From 415e110e9913a9285177a0cac940380dcf750991 Mon Sep 17 00:00:00 2001 From: cisaacstern <62192187+cisaacstern@users.noreply.github.com> Date: Fri, 9 Jul 2021 10:34:56 -0700 Subject: [PATCH 09/25] formatting 2 --- recipes/cesm-pop-lowres-1deg/recipe.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/recipes/cesm-pop-lowres-1deg/recipe.py b/recipes/cesm-pop-lowres-1deg/recipe.py index 26ac40aff0..631bc1446a 100644 --- a/recipes/cesm-pop-lowres-1deg/recipe.py +++ b/recipes/cesm-pop-lowres-1deg/recipe.py @@ -18,14 +18,26 @@ def make_full_path(variable): vars = [ - 'HMXL_2', 'SFWF_2', 'SHF_2', 'SSH_2', 'SSS', 'SST', 'SST2', 'TAUX_2', 'TAUY_2', 'U1_1', - 'U2_2', 'V1_1', 'V2_2', 'XMXL_2', + "HMXL_2", + "SFWF_2", + "SHF_2", + "SSH_2", + "SSS", + "SST", + "SST2", + "TAUX_2", + "TAUY_2", + "U1_1", + "U2_2", + "V1_1", + "V2_2", + "XMXL_2", ] merge_dim = MergeDim("variable", keys=vars) pattern = FilePattern(make_full_path, merge_dim) -chunks = {'time': 200} +chunks = {"time": 200} recipe = XarrayZarrRecipe(pattern, target_chunks=chunks) From 0ad906e685f451f48108951285fa68bc32684354 Mon Sep 17 00:00:00 2001 From: cisaacstern <62192187+cisaacstern@users.noreply.github.com> Date: Fri, 9 Jul 2021 10:36:16 -0700 Subject: [PATCH 10/25] formatting 3 --- recipes/cesm-pop-lowres-1deg/recipe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/cesm-pop-lowres-1deg/recipe.py b/recipes/cesm-pop-lowres-1deg/recipe.py index 631bc1446a..e57fb42e11 100644 --- a/recipes/cesm-pop-lowres-1deg/recipe.py +++ b/recipes/cesm-pop-lowres-1deg/recipe.py @@ -1,4 +1,4 @@ -from pangeo_forge_recipes.patterns import MergeDim, FilePattern +from pangeo_forge_recipes.patterns import FilePattern, MergeDim from pangeo_forge_recipes.recipes import XarrayZarrRecipe From edb6452cf31adb523ca9bd294aa97b3809398e1d Mon Sep 17 00:00:00 2001 From: cisaacstern <62192187+cisaacstern@users.noreply.github.com> Date: Fri, 9 Jul 2021 11:05:34 -0700 Subject: [PATCH 11/25] add ConcatDim --- recipes/cesm-pop-lowres-1deg/recipe.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/recipes/cesm-pop-lowres-1deg/recipe.py b/recipes/cesm-pop-lowres-1deg/recipe.py index e57fb42e11..a1709b571f 100644 --- a/recipes/cesm-pop-lowres-1deg/recipe.py +++ b/recipes/cesm-pop-lowres-1deg/recipe.py @@ -1,10 +1,9 @@ -from pangeo_forge_recipes.patterns import FilePattern, MergeDim +from pangeo_forge_recipes.patterns import ConcatDim, FilePattern, MergeDim from pangeo_forge_recipes.recipes import XarrayZarrRecipe -def make_full_path(variable): +def make_full_path(variable, time): """Returns a valid path to the source files - Parameters ---------- variable: str @@ -13,7 +12,7 @@ def make_full_path(variable): return ( f"https://tds.ucar.edu/thredds/fileServer/datazone/campaign/cesm/collections/ASD/" f"v5_rel04_BC5_ne30_g16/ocn/proc/tseries/daily/v5_rel04_BC5_ne30_g16.pop.h.nday1." - f"{variable}.00010101-01661231.nc" + f"{variable}.{time}.nc" ) @@ -34,9 +33,11 @@ def make_full_path(variable): "XMXL_2", ] +concat_dim = ConcatDim("time", keys=["00010101-01661231",]) + merge_dim = MergeDim("variable", keys=vars) -pattern = FilePattern(make_full_path, merge_dim) +pattern = FilePattern(make_full_path, concat_dim, merge_dim) chunks = {"time": 200} From 86982d66df0690e7d3d0f608f5749fff617d788f Mon Sep 17 00:00:00 2001 From: cisaacstern <62192187+cisaacstern@users.noreply.github.com> Date: Fri, 9 Jul 2021 11:07:22 -0700 Subject: [PATCH 12/25] formatting 4 --- recipes/cesm-pop-lowres-1deg/recipe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/cesm-pop-lowres-1deg/recipe.py b/recipes/cesm-pop-lowres-1deg/recipe.py index a1709b571f..730904d2a6 100644 --- a/recipes/cesm-pop-lowres-1deg/recipe.py +++ b/recipes/cesm-pop-lowres-1deg/recipe.py @@ -33,7 +33,7 @@ def make_full_path(variable, time): "XMXL_2", ] -concat_dim = ConcatDim("time", keys=["00010101-01661231",]) +concat_dim = ConcatDim("time", keys=["00010101-01661231", ]) merge_dim = MergeDim("variable", keys=vars) From 1ebaa695b0ca44dc412e9e4f4ec53c8c28c9ba8e Mon Sep 17 00:00:00 2001 From: cisaacstern <62192187+cisaacstern@users.noreply.github.com> Date: Fri, 9 Jul 2021 11:10:04 -0700 Subject: [PATCH 13/25] formatting 5 --- recipes/cesm-pop-lowres-1deg/recipe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/cesm-pop-lowres-1deg/recipe.py b/recipes/cesm-pop-lowres-1deg/recipe.py index 730904d2a6..b29c077682 100644 --- a/recipes/cesm-pop-lowres-1deg/recipe.py +++ b/recipes/cesm-pop-lowres-1deg/recipe.py @@ -33,7 +33,7 @@ def make_full_path(variable, time): "XMXL_2", ] -concat_dim = ConcatDim("time", keys=["00010101-01661231", ]) +concat_dim = ConcatDim("time", keys=["00010101-01661231"]) merge_dim = MergeDim("variable", keys=vars) From 0548f8bd592a513be9acbfd4dc44f8a477af7af0 Mon Sep 17 00:00:00 2001 From: cisaacstern <62192187+cisaacstern@users.noreply.github.com> Date: Thu, 15 Jul 2021 17:54:52 -0700 Subject: [PATCH 14/25] add instantiate_recipe func --- recipes/cesm-pop-lowres-1deg/recipe.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/recipes/cesm-pop-lowres-1deg/recipe.py b/recipes/cesm-pop-lowres-1deg/recipe.py index b29c077682..548bbb32c6 100644 --- a/recipes/cesm-pop-lowres-1deg/recipe.py +++ b/recipes/cesm-pop-lowres-1deg/recipe.py @@ -2,17 +2,13 @@ from pangeo_forge_recipes.recipes import XarrayZarrRecipe -def make_full_path(variable, time): +def make_full_path(variable, time, api_params=""): """Returns a valid path to the source files - Parameters - ---------- - variable: str - A string representing each variable """ return ( f"https://tds.ucar.edu/thredds/fileServer/datazone/campaign/cesm/collections/ASD/" f"v5_rel04_BC5_ne30_g16/ocn/proc/tseries/daily/v5_rel04_BC5_ne30_g16.pop.h.nday1." - f"{variable}.{time}.nc" + f"{variable}.{time}.nc{api_params}" ) @@ -37,8 +33,12 @@ def make_full_path(variable, time): merge_dim = MergeDim("variable", keys=vars) -pattern = FilePattern(make_full_path, concat_dim, merge_dim) - chunks = {"time": 200} -recipe = XarrayZarrRecipe(pattern, target_chunks=chunks) + +def instantiate_recipe(api_params): + + make_full_path.__defaults__ = (api_params,) + pattern = FilePattern(make_full_path, concat_dim, merge_dim) + + return XarrayZarrRecipe(pattern, target_chunks=chunks) From b3592f0473d02a7d85712fd94219a1cb9e65c510 Mon Sep 17 00:00:00 2001 From: cisaacstern <62192187+cisaacstern@users.noreply.github.com> Date: Thu, 15 Jul 2021 18:02:48 -0700 Subject: [PATCH 15/25] lint --- recipes/cesm-pop-lowres-1deg/recipe.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recipes/cesm-pop-lowres-1deg/recipe.py b/recipes/cesm-pop-lowres-1deg/recipe.py index 548bbb32c6..dfe6ea81dd 100644 --- a/recipes/cesm-pop-lowres-1deg/recipe.py +++ b/recipes/cesm-pop-lowres-1deg/recipe.py @@ -37,8 +37,8 @@ def make_full_path(variable, time, api_params=""): def instantiate_recipe(api_params): - + make_full_path.__defaults__ = (api_params,) pattern = FilePattern(make_full_path, concat_dim, merge_dim) - + return XarrayZarrRecipe(pattern, target_chunks=chunks) From 118372b92dd35e5fc1806f86ef019ba4f092d3ef Mon Sep 17 00:00:00 2001 From: cisaacstern <62192187+cisaacstern@users.noreply.github.com> Date: Fri, 6 Aug 2021 13:47:14 -0700 Subject: [PATCH 16/25] add nitems_per_file based on NCAR netCDF headers --- recipes/cesm-pop-lowres-1deg/recipe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/cesm-pop-lowres-1deg/recipe.py b/recipes/cesm-pop-lowres-1deg/recipe.py index dfe6ea81dd..bc2d78df2f 100644 --- a/recipes/cesm-pop-lowres-1deg/recipe.py +++ b/recipes/cesm-pop-lowres-1deg/recipe.py @@ -29,7 +29,7 @@ def make_full_path(variable, time, api_params=""): "XMXL_2", ] -concat_dim = ConcatDim("time", keys=["00010101-01661231"]) +concat_dim = ConcatDim("time", keys=["00010101-01661231"], nitems_per_file=60590) merge_dim = MergeDim("variable", keys=vars) From 8c58c856910e1f72c986841fa20c5a58a769f218 Mon Sep 17 00:00:00 2001 From: cisaacstern <62192187+cisaacstern@users.noreply.github.com> Date: Fri, 6 Aug 2021 13:52:48 -0700 Subject: [PATCH 17/25] correct for latest query string implementation --- recipes/cesm-pop-lowres-1deg/recipe.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/recipes/cesm-pop-lowres-1deg/recipe.py b/recipes/cesm-pop-lowres-1deg/recipe.py index bc2d78df2f..bc00f24cf3 100644 --- a/recipes/cesm-pop-lowres-1deg/recipe.py +++ b/recipes/cesm-pop-lowres-1deg/recipe.py @@ -2,13 +2,13 @@ from pangeo_forge_recipes.recipes import XarrayZarrRecipe -def make_full_path(variable, time, api_params=""): +def make_full_path(variable, time): """Returns a valid path to the source files """ return ( f"https://tds.ucar.edu/thredds/fileServer/datazone/campaign/cesm/collections/ASD/" f"v5_rel04_BC5_ne30_g16/ocn/proc/tseries/daily/v5_rel04_BC5_ne30_g16.pop.h.nday1." - f"{variable}.{time}.nc{api_params}" + f"{variable}.{time}.nc" ) @@ -35,10 +35,6 @@ def make_full_path(variable, time, api_params=""): chunks = {"time": 200} +pattern = FilePattern(make_full_path, concat_dim, merge_dim) -def instantiate_recipe(api_params): - - make_full_path.__defaults__ = (api_params,) - pattern = FilePattern(make_full_path, concat_dim, merge_dim) - - return XarrayZarrRecipe(pattern, target_chunks=chunks) +recipe = XarrayZarrRecipe(pattern, target_chunks=chunks) From 35f35b6c0a3f106a090abbff9eb5a6fe15e8d5a3 Mon Sep 17 00:00:00 2001 From: cisaacstern <62192187+cisaacstern@users.noreply.github.com> Date: Fri, 6 Aug 2021 18:00:02 -0700 Subject: [PATCH 18/25] increase target_chunks to 300 for ~100 MB/chunk --- recipes/cesm-pop-lowres-1deg/recipe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/cesm-pop-lowres-1deg/recipe.py b/recipes/cesm-pop-lowres-1deg/recipe.py index bc00f24cf3..b89f9918aa 100644 --- a/recipes/cesm-pop-lowres-1deg/recipe.py +++ b/recipes/cesm-pop-lowres-1deg/recipe.py @@ -33,7 +33,7 @@ def make_full_path(variable, time): merge_dim = MergeDim("variable", keys=vars) -chunks = {"time": 200} +chunks = {"time": 300} pattern = FilePattern(make_full_path, concat_dim, merge_dim) From 82ca35863ab5e910fca5ad3dba769b806054a86c Mon Sep 17 00:00:00 2001 From: cisaacstern <62192187+cisaacstern@users.noreply.github.com> Date: Fri, 6 Aug 2021 18:07:55 -0700 Subject: [PATCH 19/25] reorder assignments for clarity --- recipes/cesm-pop-lowres-1deg/recipe.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/recipes/cesm-pop-lowres-1deg/recipe.py b/recipes/cesm-pop-lowres-1deg/recipe.py index b89f9918aa..aa8cc72fc4 100644 --- a/recipes/cesm-pop-lowres-1deg/recipe.py +++ b/recipes/cesm-pop-lowres-1deg/recipe.py @@ -30,11 +30,8 @@ def make_full_path(variable, time): ] concat_dim = ConcatDim("time", keys=["00010101-01661231"], nitems_per_file=60590) - merge_dim = MergeDim("variable", keys=vars) - -chunks = {"time": 300} - pattern = FilePattern(make_full_path, concat_dim, merge_dim) +chunks = {"time": 300} recipe = XarrayZarrRecipe(pattern, target_chunks=chunks) From 942a8a46e6f2330beff95709c4002231d4f8d60b Mon Sep 17 00:00:00 2001 From: cisaacstern <62192187+cisaacstern@users.noreply.github.com> Date: Fri, 6 Aug 2021 18:10:28 -0700 Subject: [PATCH 20/25] add subset_inputs, our new logger.warning did a great job --- recipes/cesm-pop-lowres-1deg/recipe.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/recipes/cesm-pop-lowres-1deg/recipe.py b/recipes/cesm-pop-lowres-1deg/recipe.py index aa8cc72fc4..3a543d2011 100644 --- a/recipes/cesm-pop-lowres-1deg/recipe.py +++ b/recipes/cesm-pop-lowres-1deg/recipe.py @@ -34,4 +34,5 @@ def make_full_path(variable, time): pattern = FilePattern(make_full_path, concat_dim, merge_dim) chunks = {"time": 300} -recipe = XarrayZarrRecipe(pattern, target_chunks=chunks) +subset_inputs = {"time": 60} +recipe = XarrayZarrRecipe(pattern, target_chunks=chunks, subset_inputs=subset_inputs) From 9f5b099b58f7032586d7b20f235d960088db01bd Mon Sep 17 00:00:00 2001 From: cisaacstern <62192187+cisaacstern@users.noreply.github.com> Date: Sun, 15 Aug 2021 12:16:46 -0700 Subject: [PATCH 21/25] subset module and recipe --- .../cesm-pop-lowres-1deg/netcdf_subsets.py | 90 +++++++++++++++++++ recipes/cesm-pop-lowres-1deg/subset_recipe.py | 50 +++++++++++ 2 files changed, 140 insertions(+) create mode 100644 recipes/cesm-pop-lowres-1deg/netcdf_subsets.py create mode 100644 recipes/cesm-pop-lowres-1deg/subset_recipe.py diff --git a/recipes/cesm-pop-lowres-1deg/netcdf_subsets.py b/recipes/cesm-pop-lowres-1deg/netcdf_subsets.py new file mode 100644 index 0000000000..9c9cf0e6c3 --- /dev/null +++ b/recipes/cesm-pop-lowres-1deg/netcdf_subsets.py @@ -0,0 +1,90 @@ +import os +import time + +import numpy as np +import xarray as xr + + +class NetCDFSubsets(): + + def __init__( + self, + cache_fs, + cache_dir, + var_name, + target_bins, + concat_dim_name, + concat_dim_length, + ): + self.cache_fs = cache_fs + self.cache_dir = cache_dir + self.var_name = var_name + self.target_bins = target_bins + self.concat_dim_name = concat_dim_name + self.concat_dim_length = concat_dim_length + + def _fn_from_var(self): + """Assumes one netCDF per variable in cache""" + for filename in self.cache_fs.ls(self.cache_dir): + if self.var_name.lower() in filename: + print(f"Filename for {self.var_name} is {filename}") + return filename + + def _open_dataset(self): + fn = self._fn_from_var() + open_file = self.cache_fs.open(fn) + print(f"Calling `xr.open_dataset` on {open_file}") + start = time.time() + ds = xr.open_dataset(open_file) + print(f"Opened dataset in {time.time()-start:.02f}s") + assert len(ds[self.concat_dim_name]) == self.concat_dim_length + print(f"`len(ds['{self.concat_dim_name}']`) matches expected length") + return ds + + def _assign_time_counter(self): + ds = self._open_dataset() + array = np.arange(1, self.concat_dim_length + 1, 1) + return ds.assign_coords(time_counter=(self.concat_dim_name, array)) + + def _grouby_bins(self): + ds = self._assign_time_counter() + groupby = ds.groupby_bins("time_counter", self.target_bins) + bins, datasets = zip(*groupby) + return bins, datasets + + def _make_target_paths(self, bins): + + def format_filename(interval_object, counter, variable): + out = str(interval_object).replace("(", "") + if "-" in out: # only relevant for the first bin + out = out.replace("-", "") + out = out.replace("]", "") + out = out.replace(", ", "-") + return f"{variable}-{counter}-{out}.nc" + + return [format_filename(b, i, self.var_name) for i, b in enumerate(bins)] + + def subset_netcdf(self): + + bins, datasets = self._grouby_bins() + paths = self._make_target_paths(bins=bins) + + start = time.time() + for i, p in enumerate(paths): + + loop_start = time.time() + + print(f"Writing {p} to local") + datasets[i].to_netcdf(p) + + print(f"Uploading {p} to {self.cache_dir}/subsets/{p}") + self.cache_fs.put(p, f"{self.cache_dir}/subsets/{p}") + + print(f"Removing {p} from local") + os.remove(p) + + print( + f"Total elapsed: {(time.time()-start):.2f}s \n" + f"This iteration: {(time.time()-loop_start):.2f}s" + ) + print("`subset_netcdf` complete") diff --git a/recipes/cesm-pop-lowres-1deg/subset_recipe.py b/recipes/cesm-pop-lowres-1deg/subset_recipe.py new file mode 100644 index 0000000000..e2bf1f19b2 --- /dev/null +++ b/recipes/cesm-pop-lowres-1deg/subset_recipe.py @@ -0,0 +1,50 @@ +import numpy as np +import pandas as pd + +from pangeo_forge_recipes.patterns import ConcatDim, FilePattern, MergeDim +from pangeo_forge_recipes.recipes import XarrayZarrRecipe + + +def format_bins(interval_object): + out = str(interval_object).replace("(", "") + if "-" in out: # only relevant for the first bin + out = out.replace("-", "") + out = out.replace("]", "") + out = out.replace(", ", "-") + return out + + +days = np.arange(1, 60590 + 1, 1) +bins = pd.cut(days, 60) +bins_dict = {i: format_bins(bins.categories[i]) for i in range(len(bins.categories))} + + +def make_full_path(variable, time): + """Returns a valid path to the source files""" + return f"{variable}-{time}-{bins_dict[time]}.nc" + + +variables = [ + "HMXL_2", + "SFWF_2", + "SHF_2", + "SSH_2", + "SSS", + "SST", + "SST2", + "TAUX_2", + "TAUY_2", + "U1_1", + "U2_2", + "V1_1", + "V2_2", + "XMXL_2", +] + +concat_dim = ConcatDim("time", keys=[i for i in range(60)]) +merge_dim = MergeDim("variable", keys=variables) +pattern = FilePattern(make_full_path, concat_dim, merge_dim) + +chunks = {"time": 200} # ~98 MB per chunk, per variable +subset_inputs = {} +recipe = XarrayZarrRecipe(pattern, target_chunks=chunks, subset_inputs=subset_inputs) From 0f3db6448394f143971257ecec3f9036096d0972 Mon Sep 17 00:00:00 2001 From: cisaacstern <62192187+cisaacstern@users.noreply.github.com> Date: Sun, 15 Aug 2021 12:20:42 -0700 Subject: [PATCH 22/25] lint --- recipes/cesm-pop-lowres-1deg/netcdf_subsets.py | 10 +--------- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/recipes/cesm-pop-lowres-1deg/netcdf_subsets.py b/recipes/cesm-pop-lowres-1deg/netcdf_subsets.py index 9c9cf0e6c3..7bb2abbee2 100644 --- a/recipes/cesm-pop-lowres-1deg/netcdf_subsets.py +++ b/recipes/cesm-pop-lowres-1deg/netcdf_subsets.py @@ -6,15 +6,8 @@ class NetCDFSubsets(): - def __init__( - self, - cache_fs, - cache_dir, - var_name, - target_bins, - concat_dim_name, - concat_dim_length, + self, cache_fs, cache_dir, var_name, target_bins, concat_dim_name, concat_dim_length, ): self.cache_fs = cache_fs self.cache_dir = cache_dir @@ -53,7 +46,6 @@ def _grouby_bins(self): return bins, datasets def _make_target_paths(self, bins): - def format_filename(interval_object, counter, variable): out = str(interval_object).replace("(", "") if "-" in out: # only relevant for the first bin diff --git a/setup.cfg b/setup.cfg index 366868d30b..e7597812f7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,7 +3,7 @@ max-line-length = 100 [isort] known_first_party=pangeo_forge_recipes -known_third_party=pandas,pangeo_forge,prefect +known_third_party=numpy,pandas,pangeo_forge,prefect,xarray multi_line_output=3 include_trailing_comma=True force_grid_wrap=0 From cb54d9e386ae0a742d4276c63691a8343c6792ed Mon Sep 17 00:00:00 2001 From: cisaacstern <62192187+cisaacstern@users.noreply.github.com> Date: Sun, 15 Aug 2021 12:22:29 -0700 Subject: [PATCH 23/25] lint 2 --- recipes/cesm-pop-lowres-1deg/netcdf_subsets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/cesm-pop-lowres-1deg/netcdf_subsets.py b/recipes/cesm-pop-lowres-1deg/netcdf_subsets.py index 7bb2abbee2..992be9a00d 100644 --- a/recipes/cesm-pop-lowres-1deg/netcdf_subsets.py +++ b/recipes/cesm-pop-lowres-1deg/netcdf_subsets.py @@ -5,7 +5,7 @@ import xarray as xr -class NetCDFSubsets(): +class NetCDFSubsets: def __init__( self, cache_fs, cache_dir, var_name, target_bins, concat_dim_name, concat_dim_length, ): From f6062f360ce5b96835fc381893326ab68198b0be Mon Sep 17 00:00:00 2001 From: cisaacstern <62192187+cisaacstern@users.noreply.github.com> Date: Tue, 17 Aug 2021 15:24:39 -0700 Subject: [PATCH 24/25] fix SST edge case in _fn_from_var method --- recipes/cesm-pop-lowres-1deg/netcdf_subsets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/cesm-pop-lowres-1deg/netcdf_subsets.py b/recipes/cesm-pop-lowres-1deg/netcdf_subsets.py index 992be9a00d..d10997f968 100644 --- a/recipes/cesm-pop-lowres-1deg/netcdf_subsets.py +++ b/recipes/cesm-pop-lowres-1deg/netcdf_subsets.py @@ -19,7 +19,7 @@ def __init__( def _fn_from_var(self): """Assumes one netCDF per variable in cache""" for filename in self.cache_fs.ls(self.cache_dir): - if self.var_name.lower() in filename: + if f"{self.var_name.lower()}." in filename: print(f"Filename for {self.var_name} is {filename}") return filename From e3c52bc2850a69cdc5359ca4ae2ac1c2263f95ef Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 29 Sep 2022 02:42:28 +0000 Subject: [PATCH 25/25] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../cesm-pop-lowres-1deg/netcdf_subsets.py | 44 ++++++++++-------- recipes/cesm-pop-lowres-1deg/recipe.py | 45 +++++++++--------- recipes/cesm-pop-lowres-1deg/subset_recipe.py | 46 +++++++++---------- 3 files changed, 70 insertions(+), 65 deletions(-) diff --git a/recipes/cesm-pop-lowres-1deg/netcdf_subsets.py b/recipes/cesm-pop-lowres-1deg/netcdf_subsets.py index d10997f968..2054443104 100644 --- a/recipes/cesm-pop-lowres-1deg/netcdf_subsets.py +++ b/recipes/cesm-pop-lowres-1deg/netcdf_subsets.py @@ -7,7 +7,13 @@ class NetCDFSubsets: def __init__( - self, cache_fs, cache_dir, var_name, target_bins, concat_dim_name, concat_dim_length, + self, + cache_fs, + cache_dir, + var_name, + target_bins, + concat_dim_name, + concat_dim_length, ): self.cache_fs = cache_fs self.cache_dir = cache_dir @@ -19,17 +25,17 @@ def __init__( def _fn_from_var(self): """Assumes one netCDF per variable in cache""" for filename in self.cache_fs.ls(self.cache_dir): - if f"{self.var_name.lower()}." in filename: - print(f"Filename for {self.var_name} is {filename}") + if f'{self.var_name.lower()}.' in filename: + print(f'Filename for {self.var_name} is {filename}') return filename def _open_dataset(self): fn = self._fn_from_var() open_file = self.cache_fs.open(fn) - print(f"Calling `xr.open_dataset` on {open_file}") + print(f'Calling `xr.open_dataset` on {open_file}') start = time.time() ds = xr.open_dataset(open_file) - print(f"Opened dataset in {time.time()-start:.02f}s") + print(f'Opened dataset in {time.time()-start:.02f}s') assert len(ds[self.concat_dim_name]) == self.concat_dim_length print(f"`len(ds['{self.concat_dim_name}']`) matches expected length") return ds @@ -41,18 +47,18 @@ def _assign_time_counter(self): def _grouby_bins(self): ds = self._assign_time_counter() - groupby = ds.groupby_bins("time_counter", self.target_bins) + groupby = ds.groupby_bins('time_counter', self.target_bins) bins, datasets = zip(*groupby) return bins, datasets def _make_target_paths(self, bins): def format_filename(interval_object, counter, variable): - out = str(interval_object).replace("(", "") - if "-" in out: # only relevant for the first bin - out = out.replace("-", "") - out = out.replace("]", "") - out = out.replace(", ", "-") - return f"{variable}-{counter}-{out}.nc" + out = str(interval_object).replace('(', '') + if '-' in out: # only relevant for the first bin + out = out.replace('-', '') + out = out.replace(']', '') + out = out.replace(', ', '-') + return f'{variable}-{counter}-{out}.nc' return [format_filename(b, i, self.var_name) for i, b in enumerate(bins)] @@ -66,17 +72,17 @@ def subset_netcdf(self): loop_start = time.time() - print(f"Writing {p} to local") + print(f'Writing {p} to local') datasets[i].to_netcdf(p) - print(f"Uploading {p} to {self.cache_dir}/subsets/{p}") - self.cache_fs.put(p, f"{self.cache_dir}/subsets/{p}") + print(f'Uploading {p} to {self.cache_dir}/subsets/{p}') + self.cache_fs.put(p, f'{self.cache_dir}/subsets/{p}') - print(f"Removing {p} from local") + print(f'Removing {p} from local') os.remove(p) print( - f"Total elapsed: {(time.time()-start):.2f}s \n" - f"This iteration: {(time.time()-loop_start):.2f}s" + f'Total elapsed: {(time.time()-start):.2f}s \n' + f'This iteration: {(time.time()-loop_start):.2f}s' ) - print("`subset_netcdf` complete") + print('`subset_netcdf` complete') diff --git a/recipes/cesm-pop-lowres-1deg/recipe.py b/recipes/cesm-pop-lowres-1deg/recipe.py index 3a543d2011..f770db217b 100644 --- a/recipes/cesm-pop-lowres-1deg/recipe.py +++ b/recipes/cesm-pop-lowres-1deg/recipe.py @@ -3,36 +3,35 @@ def make_full_path(variable, time): - """Returns a valid path to the source files - """ + """Returns a valid path to the source files""" return ( - f"https://tds.ucar.edu/thredds/fileServer/datazone/campaign/cesm/collections/ASD/" - f"v5_rel04_BC5_ne30_g16/ocn/proc/tseries/daily/v5_rel04_BC5_ne30_g16.pop.h.nday1." - f"{variable}.{time}.nc" + f'https://tds.ucar.edu/thredds/fileServer/datazone/campaign/cesm/collections/ASD/' + f'v5_rel04_BC5_ne30_g16/ocn/proc/tseries/daily/v5_rel04_BC5_ne30_g16.pop.h.nday1.' + f'{variable}.{time}.nc' ) vars = [ - "HMXL_2", - "SFWF_2", - "SHF_2", - "SSH_2", - "SSS", - "SST", - "SST2", - "TAUX_2", - "TAUY_2", - "U1_1", - "U2_2", - "V1_1", - "V2_2", - "XMXL_2", + 'HMXL_2', + 'SFWF_2', + 'SHF_2', + 'SSH_2', + 'SSS', + 'SST', + 'SST2', + 'TAUX_2', + 'TAUY_2', + 'U1_1', + 'U2_2', + 'V1_1', + 'V2_2', + 'XMXL_2', ] -concat_dim = ConcatDim("time", keys=["00010101-01661231"], nitems_per_file=60590) -merge_dim = MergeDim("variable", keys=vars) +concat_dim = ConcatDim('time', keys=['00010101-01661231'], nitems_per_file=60590) +merge_dim = MergeDim('variable', keys=vars) pattern = FilePattern(make_full_path, concat_dim, merge_dim) -chunks = {"time": 300} -subset_inputs = {"time": 60} +chunks = {'time': 300} +subset_inputs = {'time': 60} recipe = XarrayZarrRecipe(pattern, target_chunks=chunks, subset_inputs=subset_inputs) diff --git a/recipes/cesm-pop-lowres-1deg/subset_recipe.py b/recipes/cesm-pop-lowres-1deg/subset_recipe.py index e2bf1f19b2..6e52de01f7 100644 --- a/recipes/cesm-pop-lowres-1deg/subset_recipe.py +++ b/recipes/cesm-pop-lowres-1deg/subset_recipe.py @@ -6,11 +6,11 @@ def format_bins(interval_object): - out = str(interval_object).replace("(", "") - if "-" in out: # only relevant for the first bin - out = out.replace("-", "") - out = out.replace("]", "") - out = out.replace(", ", "-") + out = str(interval_object).replace('(', '') + if '-' in out: # only relevant for the first bin + out = out.replace('-', '') + out = out.replace(']', '') + out = out.replace(', ', '-') return out @@ -21,30 +21,30 @@ def format_bins(interval_object): def make_full_path(variable, time): """Returns a valid path to the source files""" - return f"{variable}-{time}-{bins_dict[time]}.nc" + return f'{variable}-{time}-{bins_dict[time]}.nc' variables = [ - "HMXL_2", - "SFWF_2", - "SHF_2", - "SSH_2", - "SSS", - "SST", - "SST2", - "TAUX_2", - "TAUY_2", - "U1_1", - "U2_2", - "V1_1", - "V2_2", - "XMXL_2", + 'HMXL_2', + 'SFWF_2', + 'SHF_2', + 'SSH_2', + 'SSS', + 'SST', + 'SST2', + 'TAUX_2', + 'TAUY_2', + 'U1_1', + 'U2_2', + 'V1_1', + 'V2_2', + 'XMXL_2', ] -concat_dim = ConcatDim("time", keys=[i for i in range(60)]) -merge_dim = MergeDim("variable", keys=variables) +concat_dim = ConcatDim('time', keys=[i for i in range(60)]) +merge_dim = MergeDim('variable', keys=variables) pattern = FilePattern(make_full_path, concat_dim, merge_dim) -chunks = {"time": 200} # ~98 MB per chunk, per variable +chunks = {'time': 200} # ~98 MB per chunk, per variable subset_inputs = {} recipe = XarrayZarrRecipe(pattern, target_chunks=chunks, subset_inputs=subset_inputs)