Skip to content

Commit

Permalink
Make changes for datasets with only one file
Browse files Browse the repository at this point in the history
This is towards fixing VBI single shot mosaics
  • Loading branch information
Cadair committed Feb 23, 2024
1 parent 7ae7a19 commit 0f23e23
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 5 deletions.
7 changes: 5 additions & 2 deletions dkist/io/dask_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ def stack_loader_array(loader_array, chunksize):
# If the chunksize sin't specified then use the whole array shape
chunksize = chunksize or loader_array.flat[0].shape

if loader_array.size == 1:
return tuple(loader_to_dask(loader_array, chunksize))[0]
if len(loader_array.shape) == 1:
return da.stack(loader_to_dask(loader_array, chunksize))
stacks = []
Expand All @@ -38,10 +40,11 @@ def loader_to_dask(loader_array, chunksize):
This is done so that an explicit ``meta=`` argument can be provided to
prevent loading data from disk.
"""

if len(loader_array.shape) != 1:
if loader_array.size != 1 and len(loader_array.shape) != 1:
raise ValueError("Can only be used on one dimensional arrays")

loader_array = np.atleast_1d(loader_array)

# The meta argument to from array is used to determine properties of the
# array, such as dtype. We explicitly specify it here to prevent dask
# trying to auto calculate it by reading from the actual array on disk.
Expand Down
5 changes: 2 additions & 3 deletions dkist/io/file_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ def _output_shape_from_ref_array(shape, loader_array) -> Tuple[int]:
if shape[0] == 1:
shape = shape[1:]

if len(loader_array) == 1:
if loader_array.size == 1:
return shape
else:
return tuple(list(loader_array.shape) + list(shape))
Expand Down Expand Up @@ -99,8 +99,7 @@ def __init__(
self._basepath = None
self.basepath = basepath # Use the setter to convert to a Path
self.chunksize = chunksize

self._fileuri_array = np.array(fileuris)
self._fileuri_array = np.atleast_1d(np.array(fileuris))

loader_array = np.empty_like(self._fileuri_array, dtype=object)
for i, fileuri in enumerate(self._fileuri_array.flat):
Expand Down

0 comments on commit 0f23e23

Please sign in to comment.