From 229934c6dbde98412159b20394449fe13f9c79c3 Mon Sep 17 00:00:00 2001 From: Dave Davis Date: Thu, 27 Jun 2024 12:37:42 -0400 Subject: [PATCH 1/9] Saving the work --- romancal/datamodels/filetype.py | 10 +++++- romancal/pipeline/exposure_pipeline.py | 45 +++++++++++++++----------- 2 files changed, 36 insertions(+), 19 deletions(-) diff --git a/romancal/datamodels/filetype.py b/romancal/datamodels/filetype.py index 9cc8a4f62..8e6150b37 100644 --- a/romancal/datamodels/filetype.py +++ b/romancal/datamodels/filetype.py @@ -2,6 +2,8 @@ import os from pathlib import Path from typing import Union +import roman_datamodels as rdm +from romancal.datamodels import ModelContainer def check(init: Union[os.PathLike, Path, io.FileIO]) -> str: @@ -17,7 +19,7 @@ def check(init: Union[os.PathLike, Path, io.FileIO]) -> str: Returns ------- file_type: str - a string with the file type ("asdf" or "asn") + a string with the file type ("asdf", "asn", "DataModel", or "ModelContainer") """ @@ -41,6 +43,12 @@ def check(init: Union[os.PathLike, Path, io.FileIO]) -> str: return "asn" return ext + elif isinstance(init, rdm.DataModel): + return "DataModel" + + elif isinstance(init, ModelContainer): + return "ModelContainer" + elif hasattr(init, "read") and hasattr(init, "seek"): magic = init.read(5) init.seek(0, 0) diff --git a/romancal/pipeline/exposure_pipeline.py b/romancal/pipeline/exposure_pipeline.py index c6a6ecc84..928b270a6 100644 --- a/romancal/pipeline/exposure_pipeline.py +++ b/romancal/pipeline/exposure_pipeline.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import logging from os.path import basename +import pdb import numpy as np from roman_datamodels import datamodels as rdm @@ -74,12 +75,12 @@ def process(self, input): else: input_filename = None - # open the input file + # determine the input type + pdb.set_trace() file_type = filetype.check(input) if file_type == "asn": asn = ModelContainer.read_asn(input) - - if file_type == "asdf": + elif file_type == "asdf": try: # set the product name based on the input filename asn = asn_from_list([input], product_name=input_filename.split(".")[0]) @@ -92,23 +93,30 @@ def process(self, input): expos_file = [] n_members = 0 # extract the members from the asn to run the files through the steps - for product in asn["products"]: - n_members = len(product["members"]) - for member in product["members"]: - expos_file.append(member["expname"]) - results = ModelContainer() tweakreg_input = ModelContainer() - for in_file in expos_file: - if isinstance(in_file, str): - input_filename = basename(in_file) - log.info(f"Input file name: {input_filename}") - else: - input_filename = None - - # Open the file - input = rdm.open(in_file) - log.info(f"Processing a WFI exposure {in_file}") + if file_type == 'asn': + for product in asn["products"]: + n_members = len(product["members"]) + for member in product["members"]: + expos_file.append(member["expname"]) + + #results = ModelContainer() + #tweakreg_input = ModelContainer() + for in_file in expos_file: + if isinstance(in_file, str): + input_filename = basename(in_file) + log.info(f"Input file name: {input_filename}") + else: + input_filename = None + elif file_type == "DataModel": + in_file = input + + # check to see if in_file is defined, if not assume we have a datamodel + if in_file is not None: + # Open the file + input = rdm.open(in_file) + log.info(f"Processing a WFI exposure {in_file}") self.dq_init.suffix = "dq_init" result = self.dq_init(input) @@ -170,6 +178,7 @@ def process(self, input): self.setup_output(result) self.output_use_model = True + pdb.set_trace() results.append(result) # Now that all the exposures are collated, run tweakreg From 6a5449b917538b28908f8fe3a2a58517b1c4e727 Mon Sep 17 00:00:00 2001 From: Dave Davis Date: Fri, 28 Jun 2024 14:20:33 -0400 Subject: [PATCH 2/9] restore filetype --- romancal/datamodels/filetype.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/romancal/datamodels/filetype.py b/romancal/datamodels/filetype.py index 8e6150b37..9cc8a4f62 100644 --- a/romancal/datamodels/filetype.py +++ b/romancal/datamodels/filetype.py @@ -2,8 +2,6 @@ import os from pathlib import Path from typing import Union -import roman_datamodels as rdm -from romancal.datamodels import ModelContainer def check(init: Union[os.PathLike, Path, io.FileIO]) -> str: @@ -19,7 +17,7 @@ def check(init: Union[os.PathLike, Path, io.FileIO]) -> str: Returns ------- file_type: str - a string with the file type ("asdf", "asn", "DataModel", or "ModelContainer") + a string with the file type ("asdf" or "asn") """ @@ -43,12 +41,6 @@ def check(init: Union[os.PathLike, Path, io.FileIO]) -> str: return "asn" return ext - elif isinstance(init, rdm.DataModel): - return "DataModel" - - elif isinstance(init, ModelContainer): - return "ModelContainer" - elif hasattr(init, "read") and hasattr(init, "seek"): magic = init.read(5) init.seek(0, 0) From 2aad365db06855003ed112ef625f522d062d14ad Mon Sep 17 00:00:00 2001 From: Dave Davis Date: Tue, 2 Jul 2024 13:17:42 -0400 Subject: [PATCH 3/9] rcal-860 Allow roman datamodel as input to the exposure pipeline --- romancal/datamodels/filetype.py | 12 +++++++-- romancal/pipeline/exposure_pipeline.py | 3 --- romancal/regtest/test_wfi_pipeline.py | 34 ++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 5 deletions(-) diff --git a/romancal/datamodels/filetype.py b/romancal/datamodels/filetype.py index 9cc8a4f62..405cca9f0 100644 --- a/romancal/datamodels/filetype.py +++ b/romancal/datamodels/filetype.py @@ -2,6 +2,8 @@ import os from pathlib import Path from typing import Union +import roman_datamodels as rdm +from romancal.datamodels import ModelContainer def check(init: Union[os.PathLike, Path, io.FileIO]) -> str: @@ -17,11 +19,11 @@ def check(init: Union[os.PathLike, Path, io.FileIO]) -> str: Returns ------- file_type: str - a string with the file type ("asdf" or "asn") + a string with the file type ("asdf", "asn", "DataModel", or "ModelContainer") """ - supported = ("asdf", "json") + supported = ("asdf", "json", "DataModel") if isinstance(init, (str, os.PathLike, Path)): path, ext = os.path.splitext(init) @@ -41,6 +43,12 @@ def check(init: Union[os.PathLike, Path, io.FileIO]) -> str: return "asn" return ext + elif isinstance(init, rdm.DataModel): + return "DataModel" + + elif isinstance(init, ModelContainer): + return "ModelContainer" + elif hasattr(init, "read") and hasattr(init, "seek"): magic = init.read(5) init.seek(0, 0) diff --git a/romancal/pipeline/exposure_pipeline.py b/romancal/pipeline/exposure_pipeline.py index 928b270a6..7b9b23a4d 100644 --- a/romancal/pipeline/exposure_pipeline.py +++ b/romancal/pipeline/exposure_pipeline.py @@ -1,7 +1,6 @@ #!/usr/bin/env python import logging from os.path import basename -import pdb import numpy as np from roman_datamodels import datamodels as rdm @@ -76,7 +75,6 @@ def process(self, input): input_filename = None # determine the input type - pdb.set_trace() file_type = filetype.check(input) if file_type == "asn": asn = ModelContainer.read_asn(input) @@ -178,7 +176,6 @@ def process(self, input): self.setup_output(result) self.output_use_model = True - pdb.set_trace() results.append(result) # Now that all the exposures are collated, run tweakreg diff --git a/romancal/regtest/test_wfi_pipeline.py b/romancal/regtest/test_wfi_pipeline.py index 07eb8baaa..7b067647c 100644 --- a/romancal/regtest/test_wfi_pipeline.py +++ b/romancal/regtest/test_wfi_pipeline.py @@ -481,6 +481,40 @@ def test_level2_grism_processing_pipeline(rtdata, ignore_asdf_paths): ) +@pytest.mark.bigdata +def test_elp_input_dm(rtdata, ignore_asdf_paths): + """Test for input roman Datamodel to exposure level pipeline""" + input_data = "r0000101001001001001_01101_0001_WFI01_uncal.asdf" + rtdata.get_data(f"WFI/image/{input_data}") + rtdata.input = rdm.open(input_data) + + # Test Pipeline + output = "r0000101001001001001_01101_0001_WFI01_ALL_SATURATED_cal.asdf" + rtdata.output = output + args = [ + "roman_elp", + rtdata.input, + ] + ExposurePipeline.from_cmdline(args) + rtdata.get_truth(f"truth/WFI/image/{output}") + + diff = compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths) + assert diff.identical, diff.report() + + # Ensure step completion is as expected + model = rdm.open(rtdata.output) + + assert model.meta.cal_step.dq_init == "COMPLETE" + assert model.meta.cal_step.saturation == "COMPLETE" + assert model.meta.cal_step.linearity == "SKIPPED" + assert model.meta.cal_step.dark == "SKIPPED" + assert model.meta.cal_step.jump == "SKIPPED" + assert model.meta.cal_step.ramp_fit == "SKIPPED" + assert model.meta.cal_step.assign_wcs == "SKIPPED" + assert model.meta.cal_step.flat_field == "SKIPPED" + assert model.meta.cal_step.photom == "SKIPPED" + + @pytest.mark.bigdata def test_processing_pipeline_all_saturated(rtdata, ignore_asdf_paths): """Tests for fully saturated data skipping steps in the pipeline""" From 86f3c0c10534a4bd0e6665c7549c6f82fee19342 Mon Sep 17 00:00:00 2001 From: Dave Davis Date: Tue, 2 Jul 2024 14:22:26 -0400 Subject: [PATCH 4/9] rcal-860 Update test input --- romancal/regtest/test_wfi_pipeline.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/romancal/regtest/test_wfi_pipeline.py b/romancal/regtest/test_wfi_pipeline.py index 7b067647c..ea4f730ad 100644 --- a/romancal/regtest/test_wfi_pipeline.py +++ b/romancal/regtest/test_wfi_pipeline.py @@ -486,16 +486,12 @@ def test_elp_input_dm(rtdata, ignore_asdf_paths): """Test for input roman Datamodel to exposure level pipeline""" input_data = "r0000101001001001001_01101_0001_WFI01_uncal.asdf" rtdata.get_data(f"WFI/image/{input_data}") - rtdata.input = rdm.open(input_data) + dm_input = rdm.open(rtdata.input) - # Test Pipeline + # Test Pipeline with input datamodel output = "r0000101001001001001_01101_0001_WFI01_ALL_SATURATED_cal.asdf" rtdata.output = output - args = [ - "roman_elp", - rtdata.input, - ] - ExposurePipeline.from_cmdline(args) + ExposurePipeline.call(dm_input) rtdata.get_truth(f"truth/WFI/image/{output}") diff = compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths) From 79bf81ace1059cc96ac6beb0eac1a4694323f1af Mon Sep 17 00:00:00 2001 From: Dave Davis Date: Tue, 2 Jul 2024 15:53:23 -0400 Subject: [PATCH 5/9] rcal-860 Update output file name --- romancal/regtest/test_wfi_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/romancal/regtest/test_wfi_pipeline.py b/romancal/regtest/test_wfi_pipeline.py index ea4f730ad..ce19e568f 100644 --- a/romancal/regtest/test_wfi_pipeline.py +++ b/romancal/regtest/test_wfi_pipeline.py @@ -489,7 +489,7 @@ def test_elp_input_dm(rtdata, ignore_asdf_paths): dm_input = rdm.open(rtdata.input) # Test Pipeline with input datamodel - output = "r0000101001001001001_01101_0001_WFI01_ALL_SATURATED_cal.asdf" + output = "r0000101001001001001_01101_0001_WFI01_cal.asdf" rtdata.output = output ExposurePipeline.call(dm_input) rtdata.get_truth(f"truth/WFI/image/{output}") From 08fcb7326c41ee7ac0f5cbf60a40c2201ffec800 Mon Sep 17 00:00:00 2001 From: Dave Davis Date: Wed, 3 Jul 2024 08:57:22 -0400 Subject: [PATCH 6/9] rcal-860 Update regression test --- romancal/regtest/test_wfi_pipeline.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/romancal/regtest/test_wfi_pipeline.py b/romancal/regtest/test_wfi_pipeline.py index ce19e568f..ea0b107d2 100644 --- a/romancal/regtest/test_wfi_pipeline.py +++ b/romancal/regtest/test_wfi_pipeline.py @@ -1,6 +1,7 @@ """ Roman tests for flat field correction """ import copy +import os import numpy as np import pytest @@ -491,24 +492,28 @@ def test_elp_input_dm(rtdata, ignore_asdf_paths): # Test Pipeline with input datamodel output = "r0000101001001001001_01101_0001_WFI01_cal.asdf" rtdata.output = output - ExposurePipeline.call(dm_input) + ExposurePipeline.call(dm_input, save_results = True) rtdata.get_truth(f"truth/WFI/image/{output}") - diff = compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths) - assert diff.identical, diff.report() - + # check that the file exists ( don't duplicate checking contents done above) + pipeline = ExposurePipeline() + pipeline.log.info( + "Check that the output file exists " + + passfail(os.path.isfile(rtdata.output)) + ) + # Ensure step completion is as expected model = rdm.open(rtdata.output) assert model.meta.cal_step.dq_init == "COMPLETE" assert model.meta.cal_step.saturation == "COMPLETE" - assert model.meta.cal_step.linearity == "SKIPPED" - assert model.meta.cal_step.dark == "SKIPPED" - assert model.meta.cal_step.jump == "SKIPPED" - assert model.meta.cal_step.ramp_fit == "SKIPPED" - assert model.meta.cal_step.assign_wcs == "SKIPPED" - assert model.meta.cal_step.flat_field == "SKIPPED" - assert model.meta.cal_step.photom == "SKIPPED" + assert model.meta.cal_step.linearity == "COMPLETE" + assert model.meta.cal_step.dark == "COMPLETE" + assert model.meta.cal_step.jump == "COMPLETE" + assert model.meta.cal_step.ramp_fit == "COMPLETE" + assert model.meta.cal_step.assign_wcs == "COMPLETE" + assert model.meta.cal_step.flat_field == "COMPLETE" + assert model.meta.cal_step.photom == "COMPLETE" @pytest.mark.bigdata From 29acc618b2d198bbf9a12d5e4e973c4554cbc7e1 Mon Sep 17 00:00:00 2001 From: Dave Davis Date: Wed, 3 Jul 2024 10:04:47 -0400 Subject: [PATCH 7/9] rcal-860 Add PR number --- CHANGES.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index c20f01241..bf166dc74 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -9,6 +9,8 @@ Documentation general ------- +- Update the exposure pipeline to accept a roman datamodel as input [#1296] + - Update okify script to use GA directory structure [#1282] - pin numpy to <2 [#1275] From 134bfb74b570e41ae0815987838e1e0d3753cd00 Mon Sep 17 00:00:00 2001 From: Dave Davis Date: Wed, 3 Jul 2024 15:52:45 -0400 Subject: [PATCH 8/9] rcal-860 Unit test updates --- romancal/datamodels/tests/test_filetype.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/romancal/datamodels/tests/test_filetype.py b/romancal/datamodels/tests/test_filetype.py index f06085282..dbfa9aee3 100644 --- a/romancal/datamodels/tests/test_filetype.py +++ b/romancal/datamodels/tests/test_filetype.py @@ -2,7 +2,10 @@ import pytest +import roman_datamodels as rdm from romancal.datamodels import filetype +from romancal.datamodels import ModelContainer + DATA_DIRECTORY = Path(__file__).parent / "data" @@ -20,6 +23,11 @@ def test_filetype(): with open(DATA_DIRECTORY / "fake.json") as file_h: file_8 = filetype.check(file_h) file_9 = filetype.check(str(DATA_DIRECTORY / "pluto.asdf")) + model_container = ModelContainer() + file_10 = filetype.check(model_container) + image_node = rdm.maker_utils.mk_level2_image(shape=(20,20)) + im1 = rdm.datamodels.ImageModel(image_node) + file_11 = filetype.check(im1) assert file_1 == "asn" assert file_2 == "asn" @@ -30,6 +38,9 @@ def test_filetype(): assert file_7 == "asdf" assert file_8 == "asn" assert file_9 == "asdf" + assert file_10 == "ModelContainer" + assert file_11 == "DataModel" + with pytest.raises(ValueError): filetype.check(DATA_DIRECTORY / "empty.txt") From b6290fb2140f7fc390aa39c4e7ab6307cf723a78 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 3 Jul 2024 19:53:50 +0000 Subject: [PATCH 9/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- romancal/datamodels/filetype.py | 6 ++++-- romancal/datamodels/tests/test_filetype.py | 7 ++----- romancal/pipeline/exposure_pipeline.py | 8 ++++---- romancal/regtest/test_wfi_pipeline.py | 11 +++++------ 4 files changed, 15 insertions(+), 17 deletions(-) diff --git a/romancal/datamodels/filetype.py b/romancal/datamodels/filetype.py index 405cca9f0..42d578ec3 100644 --- a/romancal/datamodels/filetype.py +++ b/romancal/datamodels/filetype.py @@ -2,7 +2,9 @@ import os from pathlib import Path from typing import Union + import roman_datamodels as rdm + from romancal.datamodels import ModelContainer @@ -45,10 +47,10 @@ def check(init: Union[os.PathLike, Path, io.FileIO]) -> str: return ext elif isinstance(init, rdm.DataModel): return "DataModel" - + elif isinstance(init, ModelContainer): return "ModelContainer" - + elif hasattr(init, "read") and hasattr(init, "seek"): magic = init.read(5) init.seek(0, 0) diff --git a/romancal/datamodels/tests/test_filetype.py b/romancal/datamodels/tests/test_filetype.py index dbfa9aee3..8b938c714 100644 --- a/romancal/datamodels/tests/test_filetype.py +++ b/romancal/datamodels/tests/test_filetype.py @@ -1,11 +1,9 @@ from pathlib import Path import pytest - import roman_datamodels as rdm -from romancal.datamodels import filetype -from romancal.datamodels import ModelContainer +from romancal.datamodels import ModelContainer, filetype DATA_DIRECTORY = Path(__file__).parent / "data" @@ -25,7 +23,7 @@ def test_filetype(): file_9 = filetype.check(str(DATA_DIRECTORY / "pluto.asdf")) model_container = ModelContainer() file_10 = filetype.check(model_container) - image_node = rdm.maker_utils.mk_level2_image(shape=(20,20)) + image_node = rdm.maker_utils.mk_level2_image(shape=(20, 20)) im1 = rdm.datamodels.ImageModel(image_node) file_11 = filetype.check(im1) @@ -40,7 +38,6 @@ def test_filetype(): assert file_9 == "asdf" assert file_10 == "ModelContainer" assert file_11 == "DataModel" - with pytest.raises(ValueError): filetype.check(DATA_DIRECTORY / "empty.txt") diff --git a/romancal/pipeline/exposure_pipeline.py b/romancal/pipeline/exposure_pipeline.py index 7b9b23a4d..56b510a9a 100644 --- a/romancal/pipeline/exposure_pipeline.py +++ b/romancal/pipeline/exposure_pipeline.py @@ -93,14 +93,14 @@ def process(self, input): # extract the members from the asn to run the files through the steps results = ModelContainer() tweakreg_input = ModelContainer() - if file_type == 'asn': + if file_type == "asn": for product in asn["products"]: n_members = len(product["members"]) for member in product["members"]: expos_file.append(member["expname"]) - #results = ModelContainer() - #tweakreg_input = ModelContainer() + # results = ModelContainer() + # tweakreg_input = ModelContainer() for in_file in expos_file: if isinstance(in_file, str): input_filename = basename(in_file) @@ -112,7 +112,7 @@ def process(self, input): # check to see if in_file is defined, if not assume we have a datamodel if in_file is not None: - # Open the file + # Open the file input = rdm.open(in_file) log.info(f"Processing a WFI exposure {in_file}") diff --git a/romancal/regtest/test_wfi_pipeline.py b/romancal/regtest/test_wfi_pipeline.py index ea0b107d2..73911cab7 100644 --- a/romancal/regtest/test_wfi_pipeline.py +++ b/romancal/regtest/test_wfi_pipeline.py @@ -489,19 +489,18 @@ def test_elp_input_dm(rtdata, ignore_asdf_paths): rtdata.get_data(f"WFI/image/{input_data}") dm_input = rdm.open(rtdata.input) - # Test Pipeline with input datamodel + # Test Pipeline with input datamodel output = "r0000101001001001001_01101_0001_WFI01_cal.asdf" rtdata.output = output - ExposurePipeline.call(dm_input, save_results = True) + ExposurePipeline.call(dm_input, save_results=True) rtdata.get_truth(f"truth/WFI/image/{output}") # check that the file exists ( don't duplicate checking contents done above) pipeline = ExposurePipeline() pipeline.log.info( - "Check that the output file exists " - + passfail(os.path.isfile(rtdata.output)) - ) - + "Check that the output file exists " + passfail(os.path.isfile(rtdata.output)) + ) + # Ensure step completion is as expected model = rdm.open(rtdata.output)