Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

AL-552: Tests and step code needed to change to accommodate for new parameter in ramp_fit. #6072

Merged
merged 8 commits into from
Jul 20, 2021
Merged

AL-552: Tests and step code needed to change to accommodate for new parameter in ramp_fit. #6072

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
2999f13
Adding DQ flags parameter to due to the change in the function signa…
kmacdonald-stsci May 25, 2021
4eb3508
Changing DQ flag parameters based on code review.
kmacdonald-stsci May 26, 2021
e37a9a4
Updated optional results product to get keywords and meta data.
kmacdonald-stsci Jun 15, 2021
148b4bd
Temporarily point to stcal/main in setup.cfg
jdavies-st Jul 19, 2021
9d04156
Fix bug in create_optional_results_model
jdavies-st Jul 19, 2021
e0181d3
Add entry to CHANGES.rst
jdavies-st Jul 19, 2021
da9c879
Require stcal>=0.2.2 for ramp_fit dqflag api change
jdavies-st Jul 19, 2021
3395dce
Merge branch 'master' into al_522_dqflags
jdavies-st Jul 19, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 10 additions & 5 deletions jwst/ramp_fitting/ramp_fit_step.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from .. import datamodels

from stcal.ramp_fitting import ramp_fit
from jwst.datamodels import dqflags

from ..lib import reffile_utils # TODO remove
from ..lib import pipe_utils
Expand Down Expand Up @@ -135,12 +136,14 @@ def create_integration_model(input_model, integ_info):
return int_model


def create_optional_results_model(opt_info):
def create_optional_results_model(input_model, opt_info):
"""
Creates an ImageModel from the computed arrays from ramp_fit.

Parameter
---------
input_model: ~jwst.datamodels.RampModel

opt_info: tuple
The ramp fitting arrays needed for the RampFitOutputModel.

Expand All @@ -162,10 +165,13 @@ def create_optional_results_model(opt_info):
weights=weights,
crmag=crmag)

opt_model.meta.filename = input_model.meta.filename
opt_model.update(input_model) # ... and add all keys from input
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think that calling .update will include the copying of meta.filename from the input to the output model, so line 168 may not even be necessary. Also, do we really want to set the input and output file names to be the same? Shouldn't the opt_model get saved to a file name that at least has a different file type suffix? Or is that handled later?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's done later in the Step code:

jwst/jwst/ramp_fitting/ramp_fit_step.py

Lines 240 to 243 in 8a8f148

# Save the OLS optional fit product, if it exists
if opt_info is not None:
opt_model = create_optional_results_model(opt_info)
self.save_model(opt_model, 'fitopt', output_file=self.opt_name)


return opt_model


class RampFitStep (Step):
class RampFitStep(Step):

"""
This step fits a straight line to the value of counts vs. time to
Expand Down Expand Up @@ -234,12 +240,11 @@ def process(self, input):
image_info, integ_info, opt_info, gls_opt_model = ramp_fit.ramp_fit(
input_model, buffsize,
self.save_opt, readnoise_2d, gain_2d, self.algorithm,
self.weighting, max_cores
)
self.weighting, max_cores, dqflags.pixel)

# Save the OLS optional fit product, if it exists
if opt_info is not None:
opt_model = create_optional_results_model(opt_info)
opt_model = create_optional_results_model(input_model, opt_info)
self.save_model(opt_model, 'fitopt', output_file=self.opt_name)

'''
Expand Down
87 changes: 46 additions & 41 deletions jwst/ramp_fitting/tests/test_ramp_fit.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,11 @@
from jwst.datamodels import dqflags
from jwst.datamodels import RampModel

DO_NOT_USE = dqflags.group['DO_NOT_USE']
JUMP_DET = dqflags.group['JUMP_DET']
SATURATED = dqflags.group['SATURATED']
test_dq_flags = dqflags.pixel

DO_NOT_USE = test_dq_flags["DO_NOT_USE"]
JUMP_DET = test_dq_flags["JUMP_DET"]
SATURATED = test_dq_flags["SATURATED"]

DELIM = "-" * 70

Expand All @@ -24,7 +26,7 @@ def test_one_group_small_buffer_fit_ols():
model1.data[0, 0, 50, 50] = 10.0

slopes, cube, optional, gls_dummy = ramp_fit(
model1, 512, True, rnoise, gain, 'OLS', 'optimal', 'none')
model1, 512, True, rnoise, gain, 'OLS', 'optimal', 'none', test_dq_flags)

data = slopes[0]
np.testing.assert_allclose(data[50, 50], 10.0, 1e-6)
Expand All @@ -36,7 +38,7 @@ def test_drop_frames1_not_set():
model1.meta.exposure.drop_frames1 = None

slopes, cube, optional, gls_dummy = ramp_fit(
model1, 512, True, rnoise, gain, 'OLS', 'optimal', 'none')
model1, 512, True, rnoise, gain, 'OLS', 'optimal', 'none', test_dq_flags)

data = slopes[0]
np.testing.assert_allclose(data[50, 50], 10.0, 1e-6)
Expand Down Expand Up @@ -85,7 +87,7 @@ def test_one_group_small_buffer_fit_gls():
model1.data[0, 0, 50, 50] = 10.0

slopes, cube, optional, gls_dummy = ramp_fit(
model1, 512, True, rnoise, gain, 'GLS', 'optimal', 'none')
model1, 512, True, rnoise, gain, 'GLS', 'optimal', 'none', test_dq_flags)

data = slopes[0]
np.testing.assert_allclose(data[50, 50], 10.0, 1e-6)
Expand All @@ -97,7 +99,7 @@ def test_one_group_two_ints_fit_ols():
model1.data[1, 0, 50, 50] = 12.0

slopes, cube, optional, gls_dummy = ramp_fit(
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none')
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none', test_dq_flags)

data = slopes[0]
np.testing.assert_allclose(data[50, 50], 11.0, 1e-6)
Expand All @@ -114,10 +116,12 @@ def test_gls_vs_ols_two_ints_ols():
model1.data[0, :, 50, 50] = ramp
model1.data[1, :, 50, 50] = ramp * 2

slopes = ramp_fit(model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none')
slopes = ramp_fit(
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none', test_dq_flags)
np.testing.assert_allclose(slopes[0].data[50, 50], 150.0, 1e-6)

slopes_gls = ramp_fit(model1, 1024 * 30000., True, rnoise, gain, 'GLS', 'optimal', 'none')
slopes_gls = ramp_fit(
model1, 1024 * 30000., True, rnoise, gain, 'GLS', 'optimal', 'none', test_dq_flags)
np.testing.assert_allclose(slopes_gls[0].data[50, 50], 150.0, 1e-6)


Expand Down Expand Up @@ -147,10 +151,10 @@ def test_multiprocessing():
algo = "OLS"
# algo = "GLS"
slopes, int_model, opt_model, gls_opt_model = ramp_fit(
model1, 1024 * 30000., False, rnoise, gain, algo, 'optimal', 'none')
model1, 1024 * 30000., False, rnoise, gain, algo, 'optimal', 'none', test_dq_flags)

slopes_multi, int_model_multi, opt_model_multi, gls_opt_model_multi = ramp_fit(
model1, 1024 * 30000., False, rnoise, gain, algo, 'optimal', 'half')
model1, 1024 * 30000., False, rnoise, gain, algo, 'optimal', 'half', test_dq_flags)

np.testing.assert_allclose(slopes.data, slopes_multi.data, rtol=1e-5)

Expand Down Expand Up @@ -178,10 +182,10 @@ def test_multiprocessing2():
algo = "OLS"
# algo = "GLS"
slopes, int_model, opt_model, gls_opt_model = ramp_fit(
model1, 1024 * 30000., True, rnoise, gain, algo, 'optimal', 'none')
model1, 1024 * 30000., True, rnoise, gain, algo, 'optimal', 'none', test_dq_flags)

slopes_multi, int_model_multi, opt_model_multi, gls_opt_model_multi = ramp_fit(
model1, 1024 * 30000., True, rnoise, gain, algo, 'optimal', 'half')
model1, 1024 * 30000., True, rnoise, gain, algo, 'optimal', 'half', test_dq_flags)

np.testing.assert_allclose(slopes.data, slopes_multi.data, rtol=1e-5)

Expand All @@ -192,7 +196,8 @@ def test_one_group_two_ints_fit_gls():
model1.data[0, 0, 50, 50] = 10.0
model1.data[1, 0, 50, 50] = 12.0

slopes = ramp_fit(model1, 1024 * 30000., True, rnoise, gain, 'GLS', 'optimal', 'none')
slopes = ramp_fit(
model1, 1024 * 30000., True, rnoise, gain, 'GLS', 'optimal', 'none', test_dq_flags)

np.testing.assert_allclose(slopes[0].data[50, 50], 11.0, 1e-6)

Expand All @@ -209,7 +214,7 @@ def test_nocrs_noflux(self, method):
model1, gdq, rnoise, pixdq, err, gain = setup_inputs(ngroups=5)

slopes, cube, optional, gls_dummy = ramp_fit(
model1, 60000, False, rnoise, gain, method, 'optimal', 'none')
model1, 60000, False, rnoise, gain, method, 'optimal', 'none', test_dq_flags)

data = slopes[0]
assert(0 == np.max(data))
Expand All @@ -220,7 +225,7 @@ def test_nocrs_noflux_firstrows_are_nan(self, method):
model1.data[0, :, 0:12, :] = np.nan

slopes, cube, optional, gls_dummy = ramp_fit(
model1, 60000, False, rnoise, gain, 'OLS', 'optimal', 'none')
model1, 60000, False, rnoise, gain, 'OLS', 'optimal', 'none', test_dq_flags)

data = slopes[0]
assert(0 == np.max(data))
Expand All @@ -233,7 +238,7 @@ def test_error_when_frame_time_not_set(self, method):
model1.meta.exposure.frame_time = None

slopes, cube, optional, gls_dummy = ramp_fit(
model1, 64000, False, rnoise, gain, 'OLS', 'optimal', 'none')
model1, 64000, False, rnoise, gain, 'OLS', 'optimal', 'none', test_dq_flags)

data = slopes.data
assert(0 == np.max(data))
Expand All @@ -259,7 +264,7 @@ def test_five_groups_two_ints_Poisson_noise_only(self, method):
model1.data[1, 4, 50, 50] = 160.0

slopes, cube, optional, gls_dummy = ramp_fit(
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none')
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none', test_dq_flags)

out_slope = slopes[0][50, 50]
deltaDN1 = 50
Expand All @@ -272,7 +277,7 @@ def test_ngroups_doesnot_match_cube_size(self, method):
model1.meta.exposure.ngroups = 11

slopes, cube, optional, gls_dummy = ramp_fit(
model1, 64000, False, rnoise, gain, 'OLS', 'optimal', 'none')
model1, 64000, False, rnoise, gain, 'OLS', 'optimal', 'none', test_dq_flags)

data = slopes[0]
assert(0 == np.max(data))
Expand All @@ -286,7 +291,7 @@ def test_bad_gain_values(self, method):
gain.data[20, 20] = np.nan

slopes, cube, optional, gls_dummy = ramp_fit(
model1, 64000, False, rnoise, gain, 'OLS', 'optimal', 'none')
model1, 64000, False, rnoise, gain, 'OLS', 'optimal', 'none', test_dq_flags)

data = slopes[0]
dq = slopes[1]
Expand All @@ -310,7 +315,7 @@ def test_simple_ramp(self, method):
model1.data[0, 9, 50, 50] = 190.0

slopes, cube, optional, gls_dummy = ramp_fit(
model1, 64000, True, rnoise, gain, 'OLS', 'optimal', 'none')
model1, 64000, True, rnoise, gain, 'OLS', 'optimal', 'none', test_dq_flags)

# take the ratio of the slopes to get the relative error
data = slopes[0]
Expand All @@ -325,7 +330,7 @@ def test_read_noise_only_fit(self, method):
model1.data[0, 4, 50, 50] = 60.0

slopes, cube, optional, gls_dummy = ramp_fit(
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none')
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none', test_dq_flags)

xvalues = np.arange(5) * 1.0
yvalues = np.array([10, 15, 25, 33, 60])
Expand All @@ -344,7 +349,7 @@ def test_photon_noise_only_fit(self, method):
cds_slope = (model1.data[0, 4, 50, 50] - model1.data[0, 0, 50, 50]) / 4.0

slopes, cube, optional, gls_dummy = ramp_fit(
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none')
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none', test_dq_flags)

data = slopes[0]
np.testing.assert_allclose(data[50, 50], cds_slope, 1e-2)
Expand All @@ -360,7 +365,7 @@ def test_photon_noise_only_bad_last_frame(self, method):
cds_slope = (model1.data[0, 3, 50, 50] - model1.data[0, 0, 50, 50]) / 3.0

slopes, cube, optional, gls_dummy = ramp_fit(
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none')
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none', test_dq_flags)

data = slopes[0]
np.testing.assert_allclose(data[50, 50], cds_slope, 1e-2)
Expand All @@ -375,7 +380,7 @@ def test_photon_noise_only_bad_last_frame_two_groups(self, method):
cds_slope = (model1.data[0, 1, 50, 50] - model1.data[0, 0, 50, 50]) / 1.0

slopes, cube, optional, gls_dummy = ramp_fit(
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none')
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none', test_dq_flags)

data = slopes.data
np.testing.assert_allclose(data[50, 50], cds_slope, 1e-6)
Expand All @@ -389,7 +394,7 @@ def test_photon_noise_with_unweighted_fit(self, method):
model1.data[0, 4, 50, 50] = 60.0

slopes, cube, optional, gls_dummy = ramp_fit(
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'unweighted', 'none')
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'unweighted', 'none', test_dq_flags)

# cds_slope = (model1.data[0,4,500,500] - model1.data[0,0,500,500])/ 4.0
xvalues = np.arange(5) * 1.0
Expand All @@ -416,7 +421,7 @@ def test_two_groups_fit(self, method):
cds_slope = (model1.data[0, 1, 50, 50] - model1.data[0, 0, 50, 50])

slopes, cube, optional, gls_dummy = ramp_fit(
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none')
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none', test_dq_flags)

data = slopes[0]
np.testing.assert_allclose(data[50, 50], cds_slope, 1e-6)
Expand All @@ -437,7 +442,7 @@ def test_four_groups_oneCR_orphangroupatend_fit(self, method):
cds_slope = (model1.data[0, 1, 50, 50] - model1.data[0, 0, 50, 50])

slopes, cube, optional, gls_dummy = ramp_fit(
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none')
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none', test_dq_flags)

data = slopes[0]
np.testing.assert_allclose(data[50, 50], cds_slope, 1e-6)
Expand All @@ -454,7 +459,7 @@ def test_four_groups_two_CRs_at_end(self, method):
cds_slope = (model1.data[0, 1, 50, 50] - model1.data[0, 0, 50, 50])

slopes, cube, optional, gls_dummy = ramp_fit(
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none')
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none', test_dq_flags)

data = slopes[0]
np.testing.assert_allclose(data[50, 50], cds_slope, 1e-6)
Expand All @@ -472,7 +477,7 @@ def test_four_groups_four_CRs(self, method):
model1.groupdq[0, 3, 50, 50] = JUMP_DET

slopes, cube, optional, gls_dummy = ramp_fit(
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none')
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none', test_dq_flags)

data = slopes[0]
np.testing.assert_allclose(data[50, 50], 0, 1e-6)
Expand All @@ -488,7 +493,7 @@ def test_four_groups_three_CRs_at_end(self, method):
model1.groupdq[0, 3, 50, 50] = JUMP_DET

slopes, cube, optional, gls_dummy = ramp_fit(
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none')
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none', test_dq_flags)

expected_slope = 10.0
data = slopes[0]
Expand All @@ -503,7 +508,7 @@ def test_four_groups_CR_causes_orphan_1st_group(self, method):
model1.groupdq[0, 1, 50, 50] = JUMP_DET

slopes, cube, optional, gls_dummy = ramp_fit(
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none')
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none', test_dq_flags)

expected_slope = 20.0
data = slopes[0]
Expand All @@ -514,7 +519,7 @@ def test_one_group_fit(self, method):
model1.data[0, 0, 50, 50] = 10.0

slopes, cube, optional, gls_dummy = ramp_fit(
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none')
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none', test_dq_flags)

data = slopes[0]
np.testing.assert_allclose(data[50, 50], 10.0, 1e-6)
Expand All @@ -532,7 +537,7 @@ def test_two_groups_unc(self, method):
model1.data[0, 1, 50, 50] = 10.0 + deltaDN

slopes, cube, optional, gls_dummy = ramp_fit(
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none')
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none', test_dq_flags)

data, dq, var_poisson, var_rnoise, err = slopes
# delta_electrons = deltaDN * ingain
Expand Down Expand Up @@ -570,7 +575,7 @@ def test_five_groups_unc(self, method):
model1.data[0, 4, 50, 50] = 60.0

slopes, cube, optional, gls_dummy = ramp_fit(
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none')
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none', test_dq_flags)

# out_slope=slopes[0].data[500, 500]
median_slope = np.median(np.diff(model1.data[0, :, 50, 50])) / grouptime
Expand Down Expand Up @@ -618,7 +623,7 @@ def test_oneCR_10_groups_combination(self, method):
model1.groupdq[0, 5, 50, 50] = JUMP_DET

slopes, int_info, opt_info, gls_opt_model = ramp_fit(
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none')
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none', test_dq_flags)

segment_groups = 5
single_sample_readnoise = np.float64(inreadnoise / np.sqrt(2))
Expand Down Expand Up @@ -663,7 +668,7 @@ def test_oneCR_10_groups_combination_noisy2ndSegment(self, method):
model1.groupdq[0, 5, 50, 50] = JUMP_DET

slopes, int_info, opt_info, gls_opt_model = ramp_fit(
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none')
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none', test_dq_flags)

oslope = opt_info[0]
avg_slope = (oslope[0, 0, 50, 50] + oslope[0, 1, 50, 50]) / 2.0
Expand Down Expand Up @@ -699,7 +704,7 @@ def test_twenty_groups_two_segments():
model1.data[0, 15:, 0, 2] = 25000.

new_mod, int_model, opt_info, gls_opt_model = ramp_fit(
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none')
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none', test_dq_flags)

# Check some PRI & OPT output arrays
data = new_mod[0]
Expand Down Expand Up @@ -730,7 +735,7 @@ def test_miri_all_sat():
model1.groupdq[:, :, :, :] = SATURATED

image_info, integ_info, opt_info, gls_opt_model = ramp_fit(
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none')
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none', test_dq_flags)

# Check PRI output arrays
data, dq, var_poisson, var_rnoise, err = image_info
Expand Down Expand Up @@ -795,7 +800,7 @@ def test_miri_first_last():
model1.groupdq[0, 1, 1, 1] = JUMP_DET

image_info, int_model, opt_model, gls_opt_model = ramp_fit(
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none')
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none', test_dq_flags)

data = image_info[0]
np.testing.assert_allclose(data, 10. / 3., rtol=1E-5)
Expand Down Expand Up @@ -823,7 +828,7 @@ def test_miri_no_good_pixel():
model1.groupdq[:, :, :, :] = DO_NOT_USE

image_info, int_model, opt_model, gls_opt_model = ramp_fit(
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none')
model1, 1024 * 30000., True, rnoise, gain, 'OLS', 'optimal', 'none', test_dq_flags)

assert image_info is None

Expand Down
Loading