RCAL-486: Update Maker Utlity Imports (#654)

* Initial commit.

* Updated changelog.

CHANGES.rst

@@ -3,7 +3,7 @@
 
 general
 -------
--
+- Updated datamodel maker utility imports. [#654]
 
 0.10.0 (2023-02-21)
 ===================

pyproject.toml

@@ -20,8 +20,9 @@ dependencies = [
     'numpy >=1.20',
     'pyparsing >=2.4.7',
     'requests >=2.22',
-    'rad ==0.14.1',
-    'roman_datamodels ==0.14.1',
+    'rad >=0.14.1',
+    #'roman_datamodels >=0.14.1',
+    'roman_datamodels @git+https://github.com/spacetelescope/roman_datamodels.git@main',
     'stcal >=1.3.3',
     'stpipe >=0.4.2, <1.0',
     'tweakwcs >=0.8.0'

romancal/assign_wcs/tests/test_wcs.py

@@ -8,12 +8,12 @@
 
 from romancal.assign_wcs.assign_wcs_step import AssignWcsStep, load_wcs
 from roman_datamodels import datamodels as rdm
-from roman_datamodels.testing import utils as testutil
+from roman_datamodels import maker_utils
 
 from romancal.assign_wcs.utils import wcs_bbox_from_shape
 
 def create_image():
-    l2 = testutil.mk_level2_image()
+    l2 = maker_utils.mk_level2_image()
 
     l2.meta.wcsinfo.v2_ref = -503
     l2.meta.wcsinfo.v3_ref = -318
@@ -27,10 +27,10 @@ def create_image():
 
 
 def create_distortion():
-    distortions = [testutil.mk_distortion()]
+    distortions = [maker_utils.mk_distortion()]
 
     model = create_image()
-    dist = testutil.mk_distortion()
+    dist = maker_utils.mk_distortion()
     dist.coordinate_distortion_transform.bounding_box = wcs_bbox_from_shape(model.data.shape)
     distortions.append(dist)
 

romancal/dark_current/dark_current_step.py

@@ -6,7 +6,7 @@
 from romancal.stpipe import RomanStep
 from roman_datamodels import datamodels as rdd
 from stcal.dark_current import dark_sub
-from roman_datamodels.testing import utils as testutil
+from roman_datamodels import maker_utils
 from roman_datamodels import units as ru
 
 
@@ -92,7 +92,7 @@ def save_dark_data_as_dark_model(dark_data, dark_model):
     """
 
     # Create DarkRef object and copy dark data to it
-    out_dark = testutil.mk_dark(shape=dark_data.data.shape)
+    out_dark = maker_utils.mk_dark(shape=dark_data.data.shape)
     out_dark.data = u.Quantity(dark_data.data, out_dark.data.unit, dtype=out_dark.data.dtype)
     out_dark.dq = dark_data.groupdq
     out_dark.err = u.Quantity(dark_data.err, out_dark.err.unit, dtype=out_dark.err.dtype)

romancal/dark_current/tests/test_dark.py

@@ -11,7 +11,7 @@
 
 from roman_datamodels.datamodels import RampModel, DarkRefModel
 import roman_datamodels as rdm
-from roman_datamodels.testing import utils as testutil
+from roman_datamodels import maker_utils
 from roman_datamodels import units as ru
 
 
@@ -124,7 +124,7 @@ def create_ramp_and_dark(shape, instrument, exptype):
     """Helper function to create test ramp and dark models"""
 
     # Create test ramp model
-    ramp = testutil.mk_ramp(shape)
+    ramp = maker_utils.mk_ramp(shape)
     ramp.meta.instrument.name = instrument
     ramp.meta.instrument.detector = 'WFI01'
     ramp.meta.instrument.optical_element = 'F158'
@@ -133,7 +133,7 @@ def create_ramp_and_dark(shape, instrument, exptype):
     ramp_model = RampModel(ramp)
 
     # Create dark model
-    darkref = testutil.mk_dark(shape)
+    darkref = maker_utils.mk_dark(shape)
     darkref_model = DarkRefModel(darkref)
 
     return ramp_model, darkref_model

romancal/dq_init/dq_init_step.py

@@ -7,7 +7,7 @@
 from romancal.lib import dqflags
 from roman_datamodels.datamodels import RampModel
 import roman_datamodels as rdm
-from roman_datamodels.testing import utils as testutil
+from roman_datamodels import maker_utils
 
 __all__ = ["DQInitStep"]
 
@@ -45,7 +45,7 @@ def process(self, input):
         # Convert to RampModel if needed
         if not isinstance(input_model, RampModel):
             # Create base ramp node with dummy values (for validation)
-            input_ramp = testutil.mk_ramp(input_model.shape)
+            input_ramp = maker_utils.mk_ramp(input_model.shape)
 
             # Copy input_model contents into RampModel
             for key in input_model.keys():

romancal/dq_init/tests/test_dq_init.py

@@ -7,7 +7,7 @@
 
 from roman_datamodels import stnode
 from roman_datamodels.datamodels import MaskRefModel, ScienceRawModel
-from roman_datamodels.testing import utils as testutil
+from roman_datamodels import maker_utils
 from roman_datamodels import units as ru
 
 from romancal.lib import dqflags
@@ -27,7 +27,7 @@ def test_dq_im(xstart, ystart, xsize, ysize, ngroups, instrument, exp_type):
     csize = (ngroups, ysize, xsize)
 
     # create raw input data for step
-    dm_ramp = testutil.mk_ramp(csize)
+    dm_ramp = maker_utils.mk_ramp(csize)
     dm_ramp.meta.instrument.name = instrument
 
     # create a MaskModel elements for the dq input mask
@@ -46,7 +46,7 @@ def test_dq_im(xstart, ystart, xsize, ysize, ngroups, instrument, exp_type):
     dq[400, 200] = 33  # Persistence + do not use
 
     # write mask model
-    ref_data = testutil.mk_mask(csize[1:])
+    ref_data = maker_utils.mk_mask(csize[1:])
 
     # Copy in maskmodel elemnts
     ref_data['dq'] = dq
@@ -81,11 +81,11 @@ def test_groupdq():
     csize = (ngroups, ysize, xsize)
 
     # create raw input data for step
-    dm_ramp = testutil.mk_ramp(csize)
+    dm_ramp = maker_utils.mk_ramp(csize)
     dm_ramp.meta.instrument.name = instrument
 
     # create a MaskModel elements for the dq input mask
-    ref_data = testutil.mk_mask(csize[1:])
+    ref_data = maker_utils.mk_mask(csize[1:])
     ref_data['meta']['instrument']['name'] = instrument
 
     # run the correction step
@@ -109,11 +109,11 @@ def test_err():
     csize = (ngroups, ysize, xsize)
 
     # create raw input data for step
-    dm_ramp = testutil.mk_ramp((ngroups, ysize, xsize))
+    dm_ramp = maker_utils.mk_ramp((ngroups, ysize, xsize))
     dm_ramp.meta.instrument.name = instrument
 
     # create a MaskModel elements for the dq input mask
-    ref_data = testutil.mk_mask(csize[1:])
+    ref_data = maker_utils.mk_mask(csize[1:])
     ref_data['meta']['instrument']['name'] = instrument
 
     # Filter out validation warnings from ref_data
@@ -144,7 +144,7 @@ def test_dq_add1_groupdq():
     csize = (ngroups, ysize, xsize)
 
     # create raw input data for step
-    dm_ramp = testutil.mk_ramp((ngroups, ysize, xsize))
+    dm_ramp = maker_utils.mk_ramp((ngroups, ysize, xsize))
     dm_ramp.meta.instrument.name = instrument
 
     # create a MaskModel elements for the dq input mask
@@ -155,7 +155,7 @@ def test_dq_add1_groupdq():
     dq[400, 500] = 3  # do_not_use and saturated pixel
 
     # write mask model
-    ref_data = testutil.mk_mask(csize[1:])
+    ref_data = maker_utils.mk_mask(csize[1:])
     ref_data['meta']['instrument']['name'] = instrument
 
     # Copy in maskmodel elemnts
@@ -191,7 +191,7 @@ def test_dqinit_step_interface(instrument, exptype):
     shape = (2, 20, 20)
 
     # Create test science raw model
-    wfi_sci_raw = testutil.mk_level1_science_raw(shape)
+    wfi_sci_raw = maker_utils.mk_level1_science_raw(shape)
     wfi_sci_raw.meta.instrument.name = instrument
     wfi_sci_raw.meta.instrument.detector = 'WFI01'
     wfi_sci_raw.meta.instrument.optical_element = 'F158'
@@ -204,7 +204,7 @@ def test_dqinit_step_interface(instrument, exptype):
     # Create mask model
     maskref = stnode.MaskRef()
     meta = {}
-    testutil.add_ref_common(meta)
+    maker_utils.add_ref_common(meta)
     meta['instrument']['optical_element'] = 'F158'
     meta['instrument']['detector'] = 'WFI01'
     meta['reftype'] = 'MASK'
@@ -243,7 +243,7 @@ def test_dqinit_refpix(instrument, exptype):
     shape = (2, 20, 20)
 
     # Create test science raw model
-    wfi_sci_raw = testutil.mk_level1_science_raw(shape)
+    wfi_sci_raw = maker_utils.mk_level1_science_raw(shape)
     wfi_sci_raw.meta.instrument.name = instrument
     wfi_sci_raw.meta.instrument.detector = 'WFI01'
     wfi_sci_raw.meta.instrument.optical_element = 'F158'
@@ -256,7 +256,7 @@ def test_dqinit_refpix(instrument, exptype):
     # Create mask model
     maskref = stnode.MaskRef()
     meta = {}
-    testutil.add_ref_common(meta)
+    maker_utils.add_ref_common(meta)
     meta['instrument']['optical_element'] = 'F158'
     meta['instrument']['detector'] = 'WFI01'
     meta['reftype'] = 'MASK'

romancal/flatfield/tests/test_flatfield.py

@@ -7,7 +7,7 @@
 
 from roman_datamodels import stnode
 from roman_datamodels.datamodels import ImageModel, FlatRefModel
-from roman_datamodels.testing import utils as testutil
+from roman_datamodels import maker_utils
 from roman_datamodels import units as ru
 from romancal.flatfield import FlatFieldStep
 
@@ -28,7 +28,7 @@ def test_flatfield_step_interface(instrument, exptype):
 
     shape = (20, 20)
 
-    wfi_image = testutil.mk_level2_image(shape=shape)
+    wfi_image = maker_utils.mk_level2_image(shape=shape)
     wfi_image.meta.instrument.name = instrument
     wfi_image.meta.instrument.detector = 'WFI01'
     wfi_image.meta.instrument.optical_element = 'F158'
@@ -48,7 +48,7 @@ def test_flatfield_step_interface(instrument, exptype):
     wfi_image_model = ImageModel(wfi_image)
     flatref = stnode.FlatRef()
     meta = {}
-    testutil.add_ref_common(meta)
+    maker_utils.add_ref_common(meta)
     meta['instrument']['optical_element'] = 'F158'
     meta['instrument']['detector'] = 'WFI01'
     meta['reftype'] = 'FLAT'
@@ -79,7 +79,7 @@ def test_flatfield_step_interface(instrument, exptype):
 def test_crds_temporal_match(instrument, exptype):
     """Test that the basic inferface works for data requiring a FLAT reffile"""
 
-    wfi_image = testutil.mk_level2_image()
+    wfi_image = maker_utils.mk_level2_image()
     wfi_image.meta.instrument.name = instrument
     wfi_image.meta.instrument.detector = 'WFI01'
     wfi_image.meta.instrument.optical_element = 'F158'
@@ -113,7 +113,7 @@ def test_crds_temporal_match(instrument, exptype):
 )
 # Test that spectroscopic exposure types will skip flat field step
 def test_spectroscopic_skip(instrument, exptype):
-    wfi_image = testutil.mk_level2_image()
+    wfi_image = maker_utils.mk_level2_image()
     wfi_image.meta.instrument.name = instrument
     wfi_image.meta.instrument.detector = 'WFI01'
     wfi_image.meta.instrument.optical_element = 'F158'

romancal/jump/tests/test_jump_step.py

@@ -13,7 +13,7 @@
 from roman_datamodels import datamodels as rdm
 from roman_datamodels.datamodels import GainRefModel
 from roman_datamodels.datamodels import ReadnoiseRefModel
-from roman_datamodels.testing import utils as testutil
+from roman_datamodels import maker_utils
 from roman_datamodels import units as ru
 
 from romancal.jump import JumpStep
@@ -39,7 +39,7 @@ def generate_wfi_reffiles(tmpdir_factory):
     # Create temporary gain reference file
     gain_ref = rds.GainRef()
     meta = {}
-    testutil.add_ref_common(meta)
+    maker_utils.add_ref_common(meta)
     meta['instrument']['detector'] = 'WFI01'
     meta['instrument']['name'] = 'WFI'
     meta['author'] = 'John Doe'
@@ -59,7 +59,7 @@ def generate_wfi_reffiles(tmpdir_factory):
     # Create temporary readnoise reference file
     rn_ref = rds.ReadnoiseRef()
     meta = {}
-    testutil.add_ref_common(meta)
+    maker_utils.add_ref_common(meta)
     meta['instrument']['detector'] = 'WFI01'
     meta['instrument']['name'] = 'WFI'
     meta['author'] = 'John Doe'
@@ -98,7 +98,7 @@ def _setup(ngroups=10, readnoise=10, nrows=20, ncols=20,
         pixdq = np.zeros(shape=(nrows, ncols), dtype=np.uint32)
 
         csize = (ngroups, nrows, ncols)
-        dm_ramp = rdm.RampModel(testutil.mk_ramp(csize))
+        dm_ramp = rdm.RampModel(maker_utils.mk_ramp(csize))
 
         dm_ramp.meta.instrument.name = 'WFI'
         dm_ramp.meta.instrument.optical_element = 'F158'

romancal/photom/tests/test_photom.py

@@ -6,7 +6,7 @@
 
 from romancal.photom import photom, PhotomStep
 from roman_datamodels.datamodels import ImageModel, WfiImgPhotomRefModel
-from roman_datamodels.testing import utils as testutil
+from roman_datamodels import maker_utils
 
 
 def create_photom_wfi_image(min_r=3.1, delta=0.1):
@@ -60,7 +60,7 @@ def create_photom_wfi_image(min_r=3.1, delta=0.1):
         reftab[element] = key_dict
 
     # Create default datamodel
-    photom_model = testutil.mk_wfi_img_photom()
+    photom_model = maker_utils.mk_wfi_img_photom()
 
     # Copy values above into defautl datamodel
     photom_model.phot_table=reftab
@@ -72,7 +72,7 @@ def test_no_photom_match():
     """Test apply_photom warning for no match"""
 
     # Create sample WFI Level 2 science datamodel
-    input_model = testutil.mk_level2_image()
+    input_model = maker_utils.mk_level2_image()
 
     # Create photom reference datamodel
     photom_model = create_photom_wfi_image(min_r=3.1, delta=0.1)
@@ -108,7 +108,7 @@ def test_apply_photom1():
     """Test apply_photom applies correct metadata"""
 
     # Create sample WFI Level 2 science datamodel
-    input_model = testutil.mk_level2_image()
+    input_model = maker_utils.mk_level2_image()
 
     # Create photom reference datamodel
     photom_model = create_photom_wfi_image(min_r=3.1, delta=0.1)
@@ -162,7 +162,7 @@ def test_apply_photom2():
     """Test apply_photom does not change data values"""
 
     # Create sample WFI Level 2 science datamodel
-    input_model = testutil.mk_level2_image()
+    input_model = maker_utils.mk_level2_image()
 
     # Create photom reference datamodel
     photom_model = create_photom_wfi_image(min_r=3.1, delta=0.1)
@@ -196,11 +196,11 @@ def test_photom_step_interface(instrument, exptype):
     shape = (20, 20)
 
     # Create input model
-    wfi_image = testutil.mk_level2_image(shape=shape)
+    wfi_image = maker_utils.mk_level2_image(shape=shape)
     wfi_image_model = ImageModel(wfi_image)
 
     # Create photom model
-    photom = testutil.mk_wfi_img_photom()
+    photom = maker_utils.mk_wfi_img_photom()
     photom_model = WfiImgPhotomRefModel(photom)
 
     # Run photom correction step
@@ -231,7 +231,7 @@ def test_photom_step_interface_spectroscopic(instrument, exptype):
     shape = (20, 20)
 
     # Create input node
-    wfi_image = testutil.mk_level2_image(shape=shape)
+    wfi_image = maker_utils.mk_level2_image(shape=shape)
 
     # Select exposure type and optical element
     wfi_image.meta.exposure.type = "WFI_PRISM"
@@ -251,7 +251,7 @@ def test_photom_step_interface_spectroscopic(instrument, exptype):
     wfi_image_model = ImageModel(wfi_image)
 
     # Create photom model
-    photom = testutil.mk_wfi_img_photom()
+    photom = maker_utils.mk_wfi_img_photom()
     photom_model = WfiImgPhotomRefModel(photom)
 
     # Run photom correction step

romancal/ramp_fitting/ramp_fit_step.py

@@ -8,7 +8,7 @@
 from romancal.lib import dqflags
 from roman_datamodels import datamodels as rdd
 from roman_datamodels import stnode as rds
-from roman_datamodels.testing import utils as testutil
+from roman_datamodels import maker_utils
 from roman_datamodels import units as ru
 
 from stcal.ramp_fitting import ramp_fit
@@ -92,7 +92,7 @@ def create_image_model(input_model, image_info):
     meta = {}
     meta.update(input_model.meta)
     meta['cal_step']['ramp_fit'] = 'INCOMPLETE'
-    meta['photometry'] = testutil.mk_photometry()
+    meta['photometry'] = maker_utils.mk_photometry()
     inst = {'meta': meta,
             'data': u.Quantity(data, ru.electron / u.s, dtype=data.dtype),
             'dq': dq,