From 5081e209456783c96bdcd8e97d083ed927b3db51 Mon Sep 17 00:00:00 2001
From: Brett <brettgraham@gmail.com>
Date: Tue, 5 Sep 2023 12:03:58 -0400
Subject: [PATCH 1/2] strip fitsrec from tree before serializing with asdf

---
 src/stdatamodels/fits_support.py |  3 ++-
 src/stdatamodels/model_base.py   | 11 ++++++++---
 src/stdatamodels/util.py         | 20 ++++++++++++++++++++
 src/stdatamodels/validate.py     |  3 ++-
 4 files changed, 32 insertions(+), 5 deletions(-)

diff --git a/src/stdatamodels/fits_support.py b/src/stdatamodels/fits_support.py
index 7f94cc3f..d00e91a8 100644
--- a/src/stdatamodels/fits_support.py
+++ b/src/stdatamodels/fits_support.py
@@ -775,7 +775,8 @@ def callback(node, json_id):
             data = hdulist[pair].data
             return data
         return node
-    af.tree = treeutil.walk_and_modify(af.tree, callback)
+    # don't assign to af.tree to avoid an extra validation
+    af._tree = treeutil.walk_and_modify(af.tree, callback)
 
 
 def from_fits_hdu(hdu, schema, cast_arrays=True):
diff --git a/src/stdatamodels/model_base.py b/src/stdatamodels/model_base.py
index 96691acb..6079ca56 100644
--- a/src/stdatamodels/model_base.py
+++ b/src/stdatamodels/model_base.py
@@ -25,7 +25,7 @@
 from . import properties
 from . import schema as mschema
 from . import validate
-from .util import get_envar_as_boolean, remove_none_from_tree
+from .util import convert_fitsrec_to_array_in_tree, get_envar_as_boolean, remove_none_from_tree
 
 from .history import HistoryList
 
@@ -585,10 +585,15 @@ def to_asdf(self, init, *args, **kwargs):
             `~asdf.AsdfFile.write_to`.
         """
         self.on_save(init)
-        self.validate()
-        asdffile = self.open_asdf(self._instance, **kwargs)
+        self.validate()  # required to trigger ValidationWarning
+        tree = convert_fitsrec_to_array_in_tree(self._instance)
+        # don't open_asdf(tree) as this will cause a second validation of the tree
+        # instead open an empty tree, then assign to the hidden '_tree'
+        asdffile = self.open_asdf(None, **kwargs)
+        asdffile._tree = tree
         asdffile.write_to(init, *args, **kwargs)
 
+
     @classmethod
     def from_fits(cls, init, schema=None, **kwargs):
         """
diff --git a/src/stdatamodels/util.py b/src/stdatamodels/util.py
index c5263b6f..3fdcbbc2 100644
--- a/src/stdatamodels/util.py
+++ b/src/stdatamodels/util.py
@@ -318,6 +318,15 @@ def _remove_none(node):
     return treeutil.walk_and_modify(tree, _remove_none)
 
 
+def convert_fitsrec_to_array_in_tree(tree):
+    def _convert_fitsrec(node):
+        if isinstance(node, fits.FITS_rec):
+            return _fits_rec_to_array(node)
+        else:
+            return node
+    return treeutil.walk_and_modify(tree, _convert_fitsrec)
+
+
 def rebuild_fits_rec_dtype(fits_rec):
     dtype = fits_rec.dtype
     new_dtype = []
@@ -329,3 +338,14 @@ def rebuild_fits_rec_dtype(fits_rec):
         else:
             new_dtype.append((field_name, table_dtype))
     return np.dtype((np.record, new_dtype))
+
+
+def _fits_rec_to_array(fits_rec):
+    bad_columns = [n for n in fits_rec.dtype.fields if np.issubdtype(fits_rec[n].dtype, np.unsignedinteger)]
+    if not len(bad_columns):
+        return fits_rec.view(np.ndarray)
+    new_dtype = rebuild_fits_rec_dtype(fits_rec)
+    arr = np.asarray(fits_rec, new_dtype).copy()
+    for name in bad_columns:
+        arr[name] = fits_rec[name]
+    return arr
diff --git a/src/stdatamodels/validate.py b/src/stdatamodels/validate.py
index b657ed1a..3b48d251 100644
--- a/src/stdatamodels/validate.py
+++ b/src/stdatamodels/validate.py
@@ -11,7 +11,7 @@
 from asdf.util import HashableDict
 import numpy as np
 
-from .util import remove_none_from_tree
+from .util import convert_fitsrec_to_array_in_tree, remove_none_from_tree
 
 
 class ValidationWarning(Warning):
@@ -144,6 +144,7 @@ def _check_value(value, schema, ctx):
         # converting to tagged tree, so that we don't have to descend unnecessarily
         # into nodes for custom types.
         value = remove_none_from_tree(value)
+        value = convert_fitsrec_to_array_in_tree(value)
         value = yamlutil.custom_tree_to_tagged_tree(value, ctx._asdf)
 
         if ctx._validate_arrays:

From 751a9f35aca1efda86689c9a71ac6c067ff99506 Mon Sep 17 00:00:00 2001
From: Brett <brettgraham@gmail.com>
Date: Tue, 5 Sep 2023 14:29:24 -0400
Subject: [PATCH 2/2] update changelog

---
 CHANGES.rst | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/CHANGES.rst b/CHANGES.rst
index c1f4a248..9b7b9e7e 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -22,6 +22,9 @@ Other
 - Change format of the MirMrsPtCorrModel to use a 1d reference table
   instead of 2d FITS image extensions [#196]
 
+- Convert ``FITS_rec`` instances to arrays before serializing or
+  validating with asdf [#205]
+
 
 1.8.0 (2023-08-24)
 ==================