JP-2297: Properly handle DO_NOT_USE DQ flag for multi-integration processing

CHANGES.rst

@@ -1,3 +1,13 @@
+(unreleased)
+==================
+
+ramp_fitting
+------------
+
+- For slopes with negative median rates, the Poisson variance is zero. [#59]
+- Changed the way the final DQ array gets computed when handling the DO_NOT_USE
+  flag for multi-integration data. [#60]
+
 0.4.1 (2021-10-14)
 ==================
 

src/stcal/ramp_fitting/ols_fit.py

@@ -1168,8 +1168,10 @@ def ramp_fit_compute_variances(ramp_data, gain_2d, readnoise_2d, fit_slopes_ans)
         # Huge variances correspond to non-existing segments, so are reset to 0
         #  to nullify their contribution.
         var_p3[var_p3 > 0.1 * utils.LARGE_VARIANCE] = 0.
+        var_p3[:, med_rates <= 0.] = 0.  # XXX JP-2293
         warnings.resetwarnings()
 
+        var_p4[num_int, :, med_rates <= 0.] = 0.  # XXX JP-2293
         var_both4[num_int, :, :, :] = var_r4[num_int, :, :, :] + var_p4[num_int, :, :, :]
         inv_var_both4[num_int, :, :, :] = 1. / var_both4[num_int, :, :, :]
 
@@ -1420,7 +1422,7 @@ def ramp_fit_overall(
 
     # Compress all integration's dq arrays to create 2D PIXELDDQ array for
     #   primary output
-    final_pixeldq = utils.dq_compress_final(dq_int, n_int)
+    final_pixeldq = utils.dq_compress_final(dq_int, n_int, ramp_data.flags_do_not_use)
 
     if dq_int is not None:
         del dq_int
@@ -1453,6 +1455,7 @@ def ramp_fit_overall(
     # Some contributions to these vars may be NaN as they are from ramps
     # having PIXELDQ=DO_NOT_USE
     var_p2[np.isnan(var_p2)] = 0.
+    var_p2[med_rates <= 0.0] = 0.  # XXX JP-2293
     var_r2[np.isnan(var_r2)] = 0.
 
     # Suppress, then re-enable, harmless arithmetic warning

src/stcal/ramp_fitting/utils.py

@@ -1390,7 +1390,7 @@ def compute_slices(max_cores):
     return number_slices
 
 
-def dq_compress_final(dq_int, n_int):
+def dq_compress_final(dq_int, n_int, dnu_flag):
     """
     Combine the integration-specific dq arrays (which have already been
     compressed and combined with the PIXELDQ array) to create the dq array
@@ -1411,10 +1411,24 @@ def dq_compress_final(dq_int, n_int):
         combination of all integration's pixeldq arrays, 2-D flag
     """
     f_dq = dq_int[0, :, :]
+    nints = dq_int.shape[0]
 
     for jj in range(1, n_int):
         f_dq = np.bitwise_or(f_dq, dq_int[jj, :, :])
 
+    # Sum each pixel over all integrations where DO_NOT_USE is set.  If
+    # the number of integrations with DO_NOT_USE set is less than the
+    # total number of integrations, that means at least one integration
+    # has good data, so the final DQ flag for that pixel should NOT
+    # include the DO_NOT_USE flag.
+    dnu = np.zeros(dq_int.shape, dtype=np.uint32)
+    dnu[np.where(np.bitwise_and(dq_int, dnu_flag))] = 1
+    dnu_sum = dnu.sum(axis=0)
+    not_dnu = np.where(dnu_sum < nints)
+
+    not_dnu_flag = np.uint32(~dnu_flag)
+    f_dq[not_dnu] = np.bitwise_and(f_dq[not_dnu], not_dnu_flag)
+
     return f_dq