Added simple outlier detection step. (#689)

CHANGES.rst

@@ -33,11 +33,16 @@ source_detection
 
 - Small bug fix to ensure that output catalogs are not attached to the file when save_catalogs=False [#684]
 
+outlier_detection
+-----------------
+- Added an empty outlier detection step to the pipeline, as well as a simple test and documentation. [#689]
+
 astrometric_utils
 -----------------
 - Added option to provide epoch so that the coordinates are corrected by proper motion. [#686]
 
 
+
 0.10.0 (2023-02-21)
 ===================
 

docs/roman/error_propagation/main.rst

@@ -18,6 +18,7 @@ Step              Stage Creates arrays          Updates arrays               Ste
 ================= ===== ======================= ============================ =========
 ramp_fitting      ELPP  VAR_POISSON, VAR_RNOISE ERR                          None
 flat_field        ELPP  VAR_FLAT                ERR, VAR_POISSON, VAR_RNOISE None
+outlier_detection HLPP  None                    None                         ERR
 ================= ===== ======================= ============================ =========
 
 ELPP Processing
@@ -48,3 +49,13 @@ reference file ERR array.
 The science data ERR array is then updated to be the square root of the quadratic sum of
 the three variance arrays.
 For more details see :ref:`flat_field <flatfield_step>`.
+
+HLPP Processing
+---------------
+HLPP pipelines perform additional instrument-level and observing-mode corrections and
+calibrations to produce fully calibrated exposures. The various steps that apply corrections and calibrations apply those same corrections/calibrations to all variance arrays and update the total
+ERR array.
+
+outlier_detection
++++++++++++++++++
+The ``outlier_detection`` step is used in all Stage 3 pipelines.

docs/roman/outlier_detection/arguments.rst

@@ -0,0 +1,8 @@
+.. _outlier_detection_step_args:
+
+Step Arguments
+==============
+The `outlier_detection` step has the following optional arguments
+that control the behavior of the processing:
+
+TBD

docs/roman/outlier_detection/description.rst

@@ -0,0 +1,20 @@
+Description
+===========
+
+Algorithm
+---------
+
+Processing multiple datasets together allows for the identification of bad pixels
+or cosmic-rays that remain in each of the input images, many times at levels which
+were not detectable by the :ref:`jump <jump_step>` step. The ``outlier_detection`` step
+implements the following algorithm to identify and flag any remaining cosmic-rays or
+other artifacts left over from previous calibrations:
+
+  - TBD
+
+The outlier detection step serves as a single interface to apply this general
+process to any Roman data, with specific variations of this algorithm for each
+type of data.  Sub-classes of the outlier detection algorithm have been developed
+specifically for
+
+  - TBD

docs/roman/outlier_detection/index.rst

@@ -0,0 +1,13 @@
+.. _outlier_detection_step:
+
+=================
+Outlier Detection
+=================
+
+.. toctree::
+   :maxdepth: 2
+
+   description.rst
+   arguments.rst
+
+.. automodapi:: romancal.outlier_detection

docs/roman/package_index.rst

@@ -15,3 +15,4 @@
    refpix/index.rst
    source_detection/index.rst
    tweakreg/index.rst
+   outlier_detection/index.rst

docs/roman/pipeline/exposure_pipeline.rst

@@ -20,18 +20,19 @@ table below.
 
 .. |check| unicode:: U+2713 .. checkmark
 
-============================================== ========= ========= =========
- Step                                          WFI-Image WFI-Prism WFI-Grism
-============================================== ========= ========= =========
- :ref:`dq_init <dq_init_step>`                  |check|    |check|  |check|
- :ref:`saturation <saturation_step>`            |check|    |check|  |check|
- :ref:`linearity <linearity_step>`              |check|    |check|  |check|
- :ref:`dark_current <dark_current_step>`        |check|    |check|  |check|
- :ref:`jump <jump_step>`                        |check|    |check|  |check|
- :ref:`ramp_fitting <ramp_fitting_step>`        |check|    |check|  |check|
- :ref:`assign_wcs <assign_wcs_step>`            |check|    |check|  |check|
- :ref:`flatfield <flatfield_step>`              |check|
-============================================== ========= ========= =========
+================================================== ========= ========= =========
+ Step                                              WFI-Image WFI-Prism WFI-Grism
+================================================== ========= ========= =========
+ :ref:`dq_init <dq_init_step>`                      |check|    |check|  |check|
+ :ref:`saturation <saturation_step>`                |check|    |check|  |check|
+ :ref:`linearity <linearity_step>`                  |check|    |check|  |check|
+ :ref:`dark_current <dark_current_step>`            |check|    |check|  |check|
+ :ref:`jump <jump_step>`                            |check|    |check|  |check|
+ :ref:`ramp_fitting <ramp_fitting_step>`            |check|    |check|  |check|
+ :ref:`assign_wcs <assign_wcs_step>`                |check|    |check|  |check|
+ :ref:`flatfield <flatfield_step>`                  |check|
+ :ref:`outlier_detection <outlier_detection_step>`  |check|
+================================================== ========= ========= =========
 
 
 Arguments

docs/roman/references_general/references_general.rst

@@ -147,7 +147,7 @@ exposure type    WFI_IMAGE           Allowed values WFI_IMAGE, WFI_GRATING, WFI_
 Tracking Pipeline Progress
 ++++++++++++++++++++++++++
 
-As each pipeline step is applied to a sciece data product, it will record a status
+As each pipeline step is applied to a science data product, it will record a status
 indicator in a cal_step attribute of the science data product. These statuses
 may be included in the primary header of reference files, in order to maintain
 a history of the data that went into creating the reference file.
@@ -174,7 +174,7 @@ Bit  Value         Name              Description
 1    2             SATURATED         Pixel saturated during exposure
 2    4             JUMP_DET          Jump detected during exposure
 3    8             DROPOUT           Data lost in transmission
-4    16            RESERVED_1
+4    16            GW_AFFECTED_DATA  Data affected by the GW read window
 5    32            PERSISTENCE       High persistence (was RESERVED_2)
 6    64            AD_FLOOR          Below A/D floor (0 DN, was RESERVED_3)
 7    128           RESERVED_4
@@ -202,3 +202,45 @@ Bit  Value         Name              Description
 30   1073741824    OTHER_BAD_PIXEL   A catch-all flag
 31   2147483648    REFERENCE_PIXEL   Pixel is a reference pixel
 ===  ==========    ================  ===========================================
+
+Parameter Specification
+=======================
+
+There are a number of steps, such as :ref:`OutlierDetectionStep
+<outlier_detection_step>`, that define
+what data quality flags a pixel is allowed to have to be considered in
+calculations. Such parameters can be set in a number of ways.
+
+First, the flag can be defined as the integer sum of all the DQ bit values from
+the input images DQ arrays that should be considered "good". For example, if
+pixels in the DQ array can have combinations of 1, 2, 4, and 8 and one wants to
+consider DQ flags 2 and 4 as being acceptable for computations, then the
+parameter value should be set to "6" (2+4). In this case a pixel having DQ values
+2, 4, or 6 will be considered a good pixel, while a pixel with a DQ value, e.g.,
+1+2=3, 4+8="12", etc. will be flagged as a "bad" pixel.
+
+Alternatively, one can enter a comma-separated or '+' separated list of integer
+bit flags that should be summed to obtain the final "good" bits. For example,
+both "4,8" and "4+8" are equivalent to a setting of "12".
+
+Finally, instead of integers, the Roman mnemonics, as defined above, may be used.
+For example, all the following specifications are equivalent:
+
+`"12" == "4+8" == "4, 8" == "JUMP_DET, DROPOUT"`
+
+.. note::
+   - The default value (0) will make *all* non-zero
+     pixels in the DQ mask be considered "bad" pixels and the
+     corresponding pixels will not be used in computations.
+
+   - Setting to `None` will turn off the use of the DQ array
+     for computations.
+
+   - In order to reverse the meaning of the flags
+     from indicating values of the "good" DQ flags
+     to indicating the "bad" DQ flags, prepend '~' to the string
+     value. For example, in order to exclude pixels with
+     DQ flags 4 and 8 for computations and to consider
+     as "good" all other pixels (regardless of their DQ flag),
+     use a value of ``~4+8``, or ``~4,8``. A string value of
+     ``~0`` would be equivalent to a setting of ``None``.

romancal/lib/suffix.py

@@ -55,6 +55,7 @@
     "saturation",
     "dark_current",
     "darkcurrent",
+    "outlier_detection",
 ]
 
 # Suffixes that are discovered but should not be considered.
@@ -84,6 +85,7 @@
     "jump",
     "sourcedetectionstep",
     "tweakregstep",
+    "outlierdetectionstep",
 }
 
 

romancal/outlier_detection/__init__.py

@@ -0,0 +1,3 @@
+from .outlier_detection_step import OutlierDetectionStep
+
+__all__ = ["OutlierDetectionStep"]

romancal/outlier_detection/outlier_detection_step.py

@@ -0,0 +1,38 @@
+"""
+Detect and flag outlier in a science image
+"""
+
+import roman_datamodels as rdm
+
+from ..stpipe import RomanStep
+
+__all__ = ["OutlierDetectionStep"]
+
+
+class OutlierDetectionStep(RomanStep):
+    """Detect and flag outliers in a science image."""
+
+    def process(self, step_input):
+
+        input_model = rdm.open(step_input)
+
+        # No reference files
+        # reference_file_model = {}
+        # Do the outlier detection
+        # output_model = outlier_detection.do_correction(
+        #     input_model,
+        #     reference_file_model,
+        # )
+
+        output_model = input_model
+
+        # Close the input and reference files
+        input_model.close()
+
+        if self.save_results:
+            try:
+                self.suffix = "outlier_detection"
+            except AttributeError:
+                self["suffix"] = "outlier_detection"
+
+        return output_model

romancal/outlier_detection/tests/test_outlier_detection.py

@@ -0,0 +1,41 @@
+import os
+
+import pytest
+from astropy import units as u
+from roman_datamodels import maker_utils
+from roman_datamodels.datamodels import ImageModel
+
+from romancal.outlier_detection import OutlierDetectionStep
+
+
+@pytest.mark.skipif(
+    os.environ.get("CI") == "true",
+    reason=(
+        "Roman CRDS servers are not currently available outside the internal network"
+    ),
+)
+@pytest.mark.parametrize(
+    "instrument, exptype",
+    [
+        ("WFI", "WFI_IMAGE"),
+    ],
+)
+def test_outlier_detection_step_interface(instrument, exptype):
+    """Test that the basic inferface works for data requiring outlier detection"""
+
+    shape = (20, 20)
+
+    wfi_image = maker_utils.mk_level2_image(shape=shape)
+    wfi_image.meta.instrument.name = instrument
+    wfi_image.meta.instrument.detector = "WFI01"
+    wfi_image.meta.instrument.optical_element = "F158"
+    wfi_image.meta.exposure.type = exptype
+    wfi_image.data[3:5, 7:10] = 4 * u.electron / u.s
+
+    wfi_image_model = ImageModel(wfi_image)
+
+    result = OutlierDetectionStep.call(wfi_image_model)
+
+    assert (result.data == wfi_image.data).all()
+    # Uncomment after adding outlier_detection added to RAD & RDM
+    # assert result.meta.cal_step.outlier_detection == "COMPLETE"

romancal/step.py

@@ -8,6 +8,7 @@
 from .flatfield.flat_field_step import FlatFieldStep
 from .jump.jump_step import JumpStep
 from .linearity.linearity_step import LinearityStep
+from .outlier_detection.outlier_detection_step import OutlierDetectionStep
 from .photom.photom_step import PhotomStep
 from .ramp_fitting.ramp_fit_step import RampFitStep
 from .saturation.saturation_step import SaturationStep
@@ -24,6 +25,7 @@
     "RampFitStep",
     "SaturationStep",
     "AssignWcsStep",
+    "OutlierDetectionStep",
     "SourceDetectionStep",
     "TweakRegStep",
 ]

romancal/stpipe/integration.py

@@ -31,6 +31,7 @@ def get_steps():
         ("romancal.step.RampFitStep", None, False),
         ("romancal.step.SaturationStep", None, False),
         ("romancal.step.AssignWcsStep", None, False),
+        ("romancal.step.OutlierDetectionStep", None, False),
         ("romancal.step.SourceDetectionStep", None, False),
         ("romancal.step.TweakRegStep", "tweakreg", False),
     ]