diff --git a/changelog/504.feature.2.rst b/changelog/504.feature.2.rst new file mode 100644 index 00000000..64d7987d --- /dev/null +++ b/changelog/504.feature.2.rst @@ -0,0 +1,2 @@ +Add `swap_tile_limits` kwarg to `TiledDataset.plot`. +This option allows the user to invert plot limits on either axes to account for WCS values that decrease compared to the pixel axes. diff --git a/changelog/504.feature.rst b/changelog/504.feature.rst new file mode 100644 index 00000000..501016e3 --- /dev/null +++ b/changelog/504.feature.rst @@ -0,0 +1,3 @@ +Update grid orientation of `TiledDataset.plot`. +The grid now has MAXIS1 columns and MAXIS2 rows where MINDEX1 corresponds to column and MINDEX2 corresponds to row. +Additionally, the origin for the grid is now in the lower-left as opposed to the upper-left. diff --git a/dkist/dataset/tests/test_tiled_dataset.py b/dkist/dataset/tests/test_tiled_dataset.py index 30749866..101ce7c2 100644 --- a/dkist/dataset/tests/test_tiled_dataset.py +++ b/dkist/dataset/tests/test_tiled_dataset.py @@ -6,6 +6,7 @@ from dkist import Dataset, TiledDataset, load_dataset from dkist.tests.helpers import figure_test +from dkist.utils.exceptions import DKISTUserWarning def test_tiled_dataset(simple_tiled_dataset, dataset): @@ -79,6 +80,37 @@ def test_tiled_dataset_from_components(dataset): def test_tileddataset_plot(share_zscale): from dkist.data.sample import VBI_AJQWW ori_ds = load_dataset(VBI_AJQWW) + + newtiles = [] + for tile in ori_ds.flat: + newtiles.append(tile.rebin((1, 8, 8), operation=np.sum)) + # ndcube 2.3.0 introduced a deepcopy for rebin, this broke our dataset validation + # https://github.com/sunpy/ndcube/issues/815 + for tile in newtiles: + tile.meta["inventory"] = ori_ds.inventory + ds = TiledDataset(np.array(newtiles).reshape(ori_ds.shape), meta={"inventory": newtiles[0].inventory}) + + fig = plt.figure(figsize=(12, 15)) + with pytest.warns(DKISTUserWarning, + match="The metadata ASDF file that produced this dataset is out of date and will result in " + "incorrect plots. Please re-download the metadata ASDF file."): + #TODO: Once sample data have been updated maybe we should test both paths here (old data and new data) + ds.plot(0, share_zscale=share_zscale, figure=fig) + + return plt.gcf() + +@figure_test +@pytest.mark.parametrize("swap_tile_limits", ["x", "y", "xy", None]) +def test_tileddataset_plot_limit_swapping(swap_tile_limits): + # Also test that row/column sizes are correct + + from dkist.data.sample import VBI_AJQWW + ori_ds = load_dataset(VBI_AJQWW) + + # Swap WCS to make the `swap_tile_limits` option more natural + for tile in ori_ds.flat: + tile.wcs.forward_transform[0].cdelt *= -1 + newtiles = [] for tile in ori_ds.flat: newtiles.append(tile.rebin((1, 8, 8), operation=np.sum)) @@ -87,8 +119,30 @@ def test_tileddataset_plot(share_zscale): for tile in newtiles: tile.meta["inventory"] = ori_ds.inventory ds = TiledDataset(np.array(newtiles).reshape(ori_ds.shape), meta={"inventory": newtiles[0].inventory}) + + non_square_ds = ds[:2, :] + assert non_square_ds.shape[0] != non_square_ds.shape[1] # Just in case the underlying data change for some reason + fig = plt.figure(figsize=(12, 15)) - ds.plot(0, share_zscale=share_zscale, figure=fig) + with pytest.warns(DKISTUserWarning, + match="The metadata ASDF file that produced this dataset is out of date and will result in " + "incorrect plots. Please re-download the metadata ASDF file."): + #TODO: Once sample data have been updated maybe we should test both paths here (old data and new data) + non_square_ds.plot(0, share_zscale=False, swap_tile_limits=swap_tile_limits, figure=fig) + + assert fig.axes[0].get_gridspec().get_geometry() == non_square_ds.shape[::-1] + for ax in fig.axes: + xlims = ax.get_xlim() + ylims = ax.get_ylim() + + if swap_tile_limits in ["x", "xy"]: + assert xlims[0] > xlims[1] + if swap_tile_limits in ["y", "xy"]: + assert ylims[0] > ylims[1] + if swap_tile_limits is None: + assert xlims[0] < xlims[1] + assert ylims[0] < ylims[1] + return plt.gcf() diff --git a/dkist/dataset/tiled_dataset.py b/dkist/dataset/tiled_dataset.py index 0f113d77..0da17fb0 100644 --- a/dkist/dataset/tiled_dataset.py +++ b/dkist/dataset/tiled_dataset.py @@ -7,18 +7,20 @@ """ import types import warnings +from typing import Literal from textwrap import dedent from collections.abc import Collection import matplotlib.pyplot as plt import numpy as np +from matplotlib.gridspec import GridSpec import astropy from astropy.table import vstack from dkist.io.file_manager import FileManager, StripedExternalArray from dkist.io.loaders import AstropyFITSLoader -from dkist.utils.exceptions import DKISTDeprecationWarning +from dkist.utils.exceptions import DKISTDeprecationWarning, DKISTUserWarning from .dataset import Dataset from .utils import dataset_info_str @@ -185,7 +187,7 @@ def _get_axislabels(ax): ylabel = coord.get_axislabel() or coord._get_default_axislabel() return (xlabel, ylabel) - def plot(self, slice_index, share_zscale=False, figure=None, **kwargs): + def plot(self, slice_index, share_zscale=False, figure=None, swap_tile_limits: Literal["x", "y", "xy"] | None = None, **kwargs): """ Plot a slice of each tile in the TiledDataset @@ -202,32 +204,64 @@ def plot(self, slice_index, share_zscale=False, figure=None, **kwargs): figure : `matplotlib.figure.Figure` A figure to use for the plot. If not specified the current pyplot figure will be used, or a new one created. + swap_tile_limits : `"x", "y", "xy"` or `None` (default) + Invert the axis limits of each tile. Either the "x" or "y" axis limits can be inverted separately, or they + can both be inverted with "xy". This option is useful if the orientation of the tile data arrays is flipped + w.r.t. the WCS orientation implied by the mosaic keys. For example, most DL-NIRSP data should be plotted with + `swap_tile_limits="xy"`. """ + if swap_tile_limits not in ["x", "y", "xy", None]: + raise RuntimeError("swap_tile_limits must be one of ['x', 'y', 'xy', None]") + + if len(self.meta.get("history", {}).get("entries", [])) == 0: + warnings.warn("The metadata ASDF file that produced this dataset is out of date and " + "will result in incorrect plots. Please re-download the metadata ASDF file.", + DKISTUserWarning) + if isinstance(slice_index, (int, slice, types.EllipsisType)): slice_index = (slice_index,) + vmin, vmax = np.inf, 0 if figure is None: figure = plt.gcf() - tiles = self.slice_tiles[slice_index].flat - for i, tile in enumerate(tiles): - ax = figure.add_subplot(self.shape[0], self.shape[1], i+1, projection=tile.wcs) - tile.plot(axes=ax, **kwargs) - if i == 0: - xlabel, ylabel = self._get_axislabels(ax) - figure.supxlabel(xlabel, y=0.05) - figure.supylabel(ylabel, x=0.05) - axmin, axmax = ax.get_images()[0].get_clim() - vmin = axmin if axmin < vmin else vmin - vmax = axmax if axmax > vmax else vmax - ax.set_ylabel(" ") - ax.set_xlabel(" ") + sliced_dataset = self.slice_tiles[slice_index] + dataset_ncols, dataset_nrows = sliced_dataset.shape + gridspec = GridSpec(nrows=dataset_nrows, ncols=dataset_ncols, figure=figure) + for col in range(dataset_ncols): + for row in range(dataset_nrows): + tile = sliced_dataset[col, row] + + # Fill up grid from the bottom row + ax_gridspec = gridspec[dataset_nrows - row - 1, col] + ax = figure.add_subplot(ax_gridspec, projection=tile.wcs) + + tile.plot(axes=ax, **kwargs) + + if swap_tile_limits in ["x", "xy"]: + ax.invert_xaxis() + + if swap_tile_limits in ["y", "xy"]: + ax.invert_yaxis() + + ax.set_ylabel(" ") + ax.set_xlabel(" ") + if col == row == 0: + xlabel, ylabel = self._get_axislabels(ax) + figure.supxlabel(xlabel, y=0.05) + figure.supylabel(ylabel, x=0.05) + + axmin, axmax = ax.get_images()[0].get_clim() + vmin = axmin if axmin < vmin else vmin + vmax = axmax if axmax > vmax else vmax + if share_zscale: for ax in figure.get_axes(): ax.get_images()[0].set_clim(vmin, vmax) + title = f"{self.inventory['instrumentName']} Dataset ({self.inventory['datasetId']}) at " - for i, (coord, val) in enumerate(list(tiles[0].global_coords.items())[::-1]): + for i, (coord, val) in enumerate(list(sliced_dataset.flat[0].global_coords.items())[::-1]): if coord == "time": val = val.iso if coord == "stokes": diff --git a/dkist/tests/figure_hashes_mpl_391_ft_261_astropy_611_animators_111_ndcube_222.json b/dkist/tests/figure_hashes_mpl_391_ft_261_astropy_611_animators_111_ndcube_222.json index 2a10e5da..e87ecba7 100644 --- a/dkist/tests/figure_hashes_mpl_391_ft_261_astropy_611_animators_111_ndcube_222.json +++ b/dkist/tests/figure_hashes_mpl_391_ft_261_astropy_611_animators_111_ndcube_222.json @@ -6,6 +6,10 @@ "dkist.dataset.tests.test_plotting.test_2d_plot[aslice1]": "cbb84fbae51d8238803f8f0d6820c575f024fe54b1656f1b181dc4ec645e9ff9", "dkist.dataset.tests.test_plotting.test_2d_plot[aslice2]": "132c5615832daff457dacb4cb770498f1fbb4460a5b90b5d4d01d224c70eeb28", "dkist.dataset.tests.test_plotting.test_2d_plot2": "409b5a10ad8ccf005331261505e63ce8febdc38eb8b5a34f8863e567e3cccb9c", - "dkist.dataset.tests.test_tiled_dataset.test_tileddataset_plot[share_zscale]": "859641d0884ef13a6575ca7125cecea0faaf3722702c22b9a59a728d6c7abe0e", - "dkist.dataset.tests.test_tiled_dataset.test_tileddataset_plot[indpendent_zscale]": "042305abd97f9a59522c5b5e5d7f5389fe010c604b08b9afe00ec7e5a49b7b65" + "dkist.dataset.tests.test_tiled_dataset.test_tileddataset_plot[share_zscale]": "40298abbc680c82de029b02c4e543a60eac1b2d71e06b22c53a1d43194491ac3", + "dkist.dataset.tests.test_tiled_dataset.test_tileddataset_plot[indpendent_zscale]": "b6f2dd9fdeb79bf25ad43a591d8dec242f32e0ba3a521e15791058d51e0ecbaf", + "dkist.dataset.tests.test_tiled_dataset.test_tileddataset_plot_limit_swapping[x]": "0f2fa941c020f9853eff0eaf2f575be193372d7042731349d166a4b3645d78b0", + "dkist.dataset.tests.test_tiled_dataset.test_tileddataset_plot_limit_swapping[y]": "ae3a81c58bf55afed01c90cac9ce6227cddf430c0741d9c2f7b2d4c3ca350a6f", + "dkist.dataset.tests.test_tiled_dataset.test_tileddataset_plot_limit_swapping[xy]": "9098876ebd47e11e2aca7460c29ac1614e383a2386868995ca3b57c61ace0162", + "dkist.dataset.tests.test_tiled_dataset.test_tileddataset_plot_limit_swapping[None]": "0159e3fcd0f7109e216888ea337e8eb9861dbc951ab9cfba5d14cc6c8b501132" } \ No newline at end of file diff --git a/dkist/tests/figure_hashes_mpl_dev_ft_261_astropy_dev_animators_dev_ndcube_dev.json b/dkist/tests/figure_hashes_mpl_dev_ft_261_astropy_dev_animators_dev_ndcube_dev.json index e0dcb458..86c7b2a4 100644 --- a/dkist/tests/figure_hashes_mpl_dev_ft_261_astropy_dev_animators_dev_ndcube_dev.json +++ b/dkist/tests/figure_hashes_mpl_dev_ft_261_astropy_dev_animators_dev_ndcube_dev.json @@ -6,6 +6,10 @@ "dkist.dataset.tests.test_plotting.test_2d_plot[aslice1]": "cbb84fbae51d8238803f8f0d6820c575f024fe54b1656f1b181dc4ec645e9ff9", "dkist.dataset.tests.test_plotting.test_2d_plot[aslice2]": "4b5be9cf1883d0ebd15ff091f52cea2822068e8238a8df7b0f594d69fba27597", "dkist.dataset.tests.test_plotting.test_2d_plot2": "1c10e9db44b0b694a6bb1b493c4c2193278541df7c1302bb11fe3f6372682e35", - "dkist.dataset.tests.test_tiled_dataset.test_tileddataset_plot[share_zscale]": "a5c5e439af14d99110858b552649a334ca2157f146c702cb5ce790fe6ba8ca1a", - "dkist.dataset.tests.test_tiled_dataset.test_tileddataset_plot[indpendent_zscale]": "71022ae3a7bbc2e1250cb5913479d72ad73570eb2e10dd463faf83a1e47865e7" + "dkist.dataset.tests.test_tiled_dataset.test_tileddataset_plot[share_zscale]": "bd0cfadd99f9d3d416f011184f2e9a7971df226879c8786e8ab2349e13909b5c", + "dkist.dataset.tests.test_tiled_dataset.test_tileddataset_plot[indpendent_zscale]": "2d6afac3f582846f4be95b23b524bb670895b0885519d8c13623307d07a3b39e", + "dkist.dataset.tests.test_tiled_dataset.test_tileddataset_plot_limit_swapping[x]": "b35593deb273b02ff1f2384810c4cf825ef5017ecad4d020543c53ad6361cd9e", + "dkist.dataset.tests.test_tiled_dataset.test_tileddataset_plot_limit_swapping[y]": "78de0395df62edd8626014d7b8924b5f3d1d66b27be9c1a328fac7b7639e702b", + "dkist.dataset.tests.test_tiled_dataset.test_tileddataset_plot_limit_swapping[xy]": "05219c0c450825fa7bd555ff27a9a111066082b15e2dde83ac2f3ac43dba5102", + "dkist.dataset.tests.test_tiled_dataset.test_tileddataset_plot_limit_swapping[None]": "64a247b0b54b7de8a8a7c636d60bdceb7d7581a5429c9e8d813b8d81912a2c10" } \ No newline at end of file diff --git a/tools/update_sample_data.py b/tools/update_sample_data.py new file mode 100644 index 00000000..e03414a3 --- /dev/null +++ b/tools/update_sample_data.py @@ -0,0 +1,102 @@ +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "numpy", +# "astropy", +# "sunpy[net]", +# "dkist", +# ] +# /// +""" +This script recreates the sample data files and uploads the recreated versions to Asgard. +""" +import sys +import tarfile +import argparse +from pathlib import Path + +import numpy as np + +from sunpy.net import Fido +from sunpy.net import attrs as a + +import dkist +import dkist.net +from dkist.net.globus import start_transfer_from_file_list, watch_transfer_progress +from dkist.net.globus.endpoints import get_local_endpoint_id, get_transfer_client + +datasets = { + "AJQWW": { + "tiled": True, + "tile_slice": np.s_[0], + "filename": "AJQWW_single_mosaic.tar", + }, + "BKPLX": { + "tiled": False, + "slice": np.s_[0], + "filename": "BKPLX_stokesI.tar", + }, +} + +def main(datasets, working_directory, destination_path="/user_tools_tutorial_data/"): + working_directory = Path(working_directory) + working_directory.mkdir(parents=True, exist_ok=True) + sample_files_for_upload = [] + + for did, props in datasets.items(): + res = Fido.search(a.dkist.Dataset(did)) + asdf_file = Fido.fetch(res, path=working_directory / "{dataset_id}", progress=False, overwrite=False) + + ds = dkist.load_dataset(asdf_file) + if "slice" in props: + ds = ds[props["slice"]] + if "tile_slice" in props: + ds = ds.slice_tiles[props["tile_slice"]] + + if props.get("tiled", False): + for i, sds in enumerate(ds.flat): + sds.files.download(path=working_directory / "{dataset_id}", wait=(i == (len(ds.flat) - 1))) + else: + ds.files.download(path=working_directory / "{dataset_id}", wait=True) + + dataset_path = working_directory / did + # Remove the preview movie and quality report + [f.unlink() for f in dataset_path.glob("*.mp4")] + [f.unlink() for f in dataset_path.glob("*.pdf")] + assert len(list(dataset_path.glob("*.asdf"))) == 1 + + sample_filename = working_directory / props["filename"] + with tarfile.open(sample_filename, mode="w") as tfile: + tfile.add(dataset_path, recursive=True) + + sample_files_for_upload.append(sample_filename) + + + local_endpoint_id = get_local_endpoint_id() + asgard_endpoint_id = "20fa4840-366a-494c-b009-063280ecf70d" + + resp = input(f"About to upload ({', '.join([f.name for f in sample_files_for_upload])}) to {destination_path} on Asgard. Are you sure? [y/N]") + if resp.lower() == "y": + task_id = start_transfer_from_file_list( + local_endpoint_id, + asgard_endpoint_id, + dst_base_path=destination_path, + file_list=sample_files_for_upload, + label="Sample data upload to Asgard", + ) + + watch_transfer_progress(task_id, get_transfer_client(), verbose=True, initial_n=len(sample_files_for_upload)) + +if __name__ == "__main__": + argp = argparse.ArgumentParser(description=__doc__) + argp.add_argument("working_dir", help="local directory to use to build the dataset files.") + argp.add_argument( + "--destination-dir", + default="/user_tools_tutorial_data/test/", + help="path to the destination directory on Asgard (defaults to '/user_tools_tutorial_data/test'" + " so must be explicitly set to override production data)." + ) + + args = argp.parse_args(sys.argv[1:]) + + main(datasets, args.working_dir, args.destination_dir)