diff --git a/CHANGES.rst b/CHANGES.rst index 811bc400..42ba9cfb 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -24,6 +24,9 @@ Other - Change format of the MirMrsPtCorrModel to use a 1d reference table instead of 2d FITS image extensions [#196] +- Convert ``FITS_rec`` instances to arrays before serializing or + validating with asdf [#205] + 1.8.0 (2023-08-24) ================== diff --git a/src/stdatamodels/fits_support.py b/src/stdatamodels/fits_support.py index 7f94cc3f..d00e91a8 100644 --- a/src/stdatamodels/fits_support.py +++ b/src/stdatamodels/fits_support.py @@ -775,7 +775,8 @@ def callback(node, json_id): data = hdulist[pair].data return data return node - af.tree = treeutil.walk_and_modify(af.tree, callback) + # don't assign to af.tree to avoid an extra validation + af._tree = treeutil.walk_and_modify(af.tree, callback) def from_fits_hdu(hdu, schema, cast_arrays=True): diff --git a/src/stdatamodels/model_base.py b/src/stdatamodels/model_base.py index 96691acb..6079ca56 100644 --- a/src/stdatamodels/model_base.py +++ b/src/stdatamodels/model_base.py @@ -25,7 +25,7 @@ from . import properties from . import schema as mschema from . import validate -from .util import get_envar_as_boolean, remove_none_from_tree +from .util import convert_fitsrec_to_array_in_tree, get_envar_as_boolean, remove_none_from_tree from .history import HistoryList @@ -585,10 +585,15 @@ def to_asdf(self, init, *args, **kwargs): `~asdf.AsdfFile.write_to`. """ self.on_save(init) - self.validate() - asdffile = self.open_asdf(self._instance, **kwargs) + self.validate() # required to trigger ValidationWarning + tree = convert_fitsrec_to_array_in_tree(self._instance) + # don't open_asdf(tree) as this will cause a second validation of the tree + # instead open an empty tree, then assign to the hidden '_tree' + asdffile = self.open_asdf(None, **kwargs) + asdffile._tree = tree asdffile.write_to(init, *args, **kwargs) + @classmethod def from_fits(cls, init, schema=None, **kwargs): """ diff --git a/src/stdatamodels/util.py b/src/stdatamodels/util.py index c5263b6f..3fdcbbc2 100644 --- a/src/stdatamodels/util.py +++ b/src/stdatamodels/util.py @@ -318,6 +318,15 @@ def _remove_none(node): return treeutil.walk_and_modify(tree, _remove_none) +def convert_fitsrec_to_array_in_tree(tree): + def _convert_fitsrec(node): + if isinstance(node, fits.FITS_rec): + return _fits_rec_to_array(node) + else: + return node + return treeutil.walk_and_modify(tree, _convert_fitsrec) + + def rebuild_fits_rec_dtype(fits_rec): dtype = fits_rec.dtype new_dtype = [] @@ -329,3 +338,14 @@ def rebuild_fits_rec_dtype(fits_rec): else: new_dtype.append((field_name, table_dtype)) return np.dtype((np.record, new_dtype)) + + +def _fits_rec_to_array(fits_rec): + bad_columns = [n for n in fits_rec.dtype.fields if np.issubdtype(fits_rec[n].dtype, np.unsignedinteger)] + if not len(bad_columns): + return fits_rec.view(np.ndarray) + new_dtype = rebuild_fits_rec_dtype(fits_rec) + arr = np.asarray(fits_rec, new_dtype).copy() + for name in bad_columns: + arr[name] = fits_rec[name] + return arr diff --git a/src/stdatamodels/validate.py b/src/stdatamodels/validate.py index b657ed1a..3b48d251 100644 --- a/src/stdatamodels/validate.py +++ b/src/stdatamodels/validate.py @@ -11,7 +11,7 @@ from asdf.util import HashableDict import numpy as np -from .util import remove_none_from_tree +from .util import convert_fitsrec_to_array_in_tree, remove_none_from_tree class ValidationWarning(Warning): @@ -144,6 +144,7 @@ def _check_value(value, schema, ctx): # converting to tagged tree, so that we don't have to descend unnecessarily # into nodes for custom types. value = remove_none_from_tree(value) + value = convert_fitsrec_to_array_in_tree(value) value = yamlutil.custom_tree_to_tagged_tree(value, ctx._asdf) if ctx._validate_arrays: