Skip to content

Commit

Permalink
Make changes for datasets with only one file (#335)
Browse files Browse the repository at this point in the history
* Test scalar array to filename

* Make changes for datasets with only one file

This is towards fixing VBI single shot mosaics

* Add changelog

---------

Co-authored-by: Drew Leonard <[email protected]>
  • Loading branch information
Cadair and SolarDrew authored Feb 26, 2024
1 parent d172ba6 commit 7063a0e
Show file tree
Hide file tree
Showing 4 changed files with 11 additions and 6 deletions.
1 change: 1 addition & 0 deletions changelog/335.trivial.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Adjust file loading to support single-frame datasets with no time axis.
4 changes: 3 additions & 1 deletion dkist/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,9 @@ def dataset(array, identity_gwcs):
assert ds.data is array
assert ds.wcs is identity_gwcs

ds._file_manager = FileManager.from_parts(['test1.fits'], 0, 'float', array.shape,
# Construct the filename here as a scalar array to make sure that works as
# it's what dkist-inventory does
ds._file_manager = FileManager.from_parts(np.array('test1.fits'), 0, 'float', array.shape,
loader=AstropyFITSLoader)

return ds
Expand Down
7 changes: 5 additions & 2 deletions dkist/io/dask_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ def stack_loader_array(loader_array, chunksize):
# If the chunksize sin't specified then use the whole array shape
chunksize = chunksize or loader_array.flat[0].shape

if loader_array.size == 1:
return tuple(loader_to_dask(loader_array, chunksize))[0]
if len(loader_array.shape) == 1:
return da.stack(loader_to_dask(loader_array, chunksize))
stacks = []
Expand All @@ -38,10 +40,11 @@ def loader_to_dask(loader_array, chunksize):
This is done so that an explicit ``meta=`` argument can be provided to
prevent loading data from disk.
"""

if len(loader_array.shape) != 1:
if loader_array.size != 1 and len(loader_array.shape) != 1:
raise ValueError("Can only be used on one dimensional arrays")

loader_array = np.atleast_1d(loader_array)

# The meta argument to from array is used to determine properties of the
# array, such as dtype. We explicitly specify it here to prevent dask
# trying to auto calculate it by reading from the actual array on disk.
Expand Down
5 changes: 2 additions & 3 deletions dkist/io/file_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ def _output_shape_from_ref_array(shape, loader_array) -> Tuple[int]:
if shape[0] == 1:
shape = shape[1:]

if len(loader_array) == 1:
if loader_array.size == 1:
return shape
else:
return tuple(list(loader_array.shape) + list(shape))
Expand Down Expand Up @@ -99,8 +99,7 @@ def __init__(
self._basepath = None
self.basepath = basepath # Use the setter to convert to a Path
self.chunksize = chunksize

self._fileuri_array = np.array(fileuris)
self._fileuri_array = np.atleast_1d(np.array(fileuris))

loader_array = np.empty_like(self._fileuri_array, dtype=object)
for i, fileuri in enumerate(self._fileuri_array.flat):
Expand Down

0 comments on commit 7063a0e

Please sign in to comment.