From a6d737a11d85437eba05ed258e95fc328125bd47 Mon Sep 17 00:00:00 2001 From: Howard Bushouse Date: Fri, 26 Apr 2024 10:06:42 -0400 Subject: [PATCH 01/15] JP-2922: Use new source_id syntax for level-3 products --- jwst/assign_wcs/nirspec.py | 88 ++++++++++------------ jwst/associations/lib/rules_level3_base.py | 6 +- jwst/pipeline/calwebb_spec3.py | 21 +++++- 3 files changed, 63 insertions(+), 52 deletions(-) diff --git a/jwst/assign_wcs/nirspec.py b/jwst/assign_wcs/nirspec.py index f106dc4c96..0caf55e26e 100644 --- a/jwst/assign_wcs/nirspec.py +++ b/jwst/assign_wcs/nirspec.py @@ -555,8 +555,6 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, For example, something like: (12, 2, 4, 251, 22, 1, 'Y', 'OPEN', nan, nan, 1, 'N'), - column - Parameters ---------- msa_file : str @@ -596,32 +594,14 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, log.error(message) raise MSAFileError(message) - # Get the configuration header from the _msa.fits file. The EXTNAME should be 'SHUTTER_INFO' - msa_conf = msa_file[('SHUTTER_INFO', 1)] - msa_source = msa_file[("SOURCE_INFO", 1)].data + # Get the shutter and source info tables from the _msa.fits file. + msa_conf = msa_file[('SHUTTER_INFO', 1)] # EXTNAME = 'SHUTTER_INFO' + msa_source = msa_file[("SOURCE_INFO", 1)].data # EXTNAME = 'SOURCE_INFO' # First we are going to filter the msa_file data on the msa_metadata_id # and dither_point_index. msa_data = [x for x in msa_conf.data if x['msa_metadata_id'] == msa_metadata_id and x['dither_point_index'] == dither_position] - - # Get all source_ids for slitlets with sources. - # These should not be used when assigning source_id to background slitlets. - source_ids = set([x[5] for x in msa_conf.data if x['msa_metadata_id'] == msa_metadata_id - and x['dither_point_index'] == dither_position]) - # All BKG shutters in the msa metafile have a source_id value of 0. - # Remove it from the list of source ids. - if 0 in source_ids: - source_ids.remove(0) - if source_ids: - max_source_id = max(source_ids) + 1 - else: - max_source_id = 0 - - # define a counter for "all background" slitlets. - # It will be used to assign a "source_id". - bkg_counter = 0 - log.debug(f'msa_data with msa_metadata_id = {msa_metadata_id} {msa_data}') log.info(f'Retrieving open MSA slitlets for msa_metadata_id = {msa_metadata_id} ' f'and dither_index = {dither_position}') @@ -629,25 +609,20 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, # Get the unique slitlet_ids slitlet_ids_unique = list(set([x['slitlet_id'] for x in msa_data])) - # SDP may assign a value of "-1" to ``slitlet_id`` - these need to be ignored. - # JP-436 - if -1 in slitlet_ids_unique: - slitlet_ids_unique.remove(-1) - # add a margin to the slit y limits margin = 0.5 # Now lets look at each unique slitlet id for slitlet_id in slitlet_ids_unique: - # Get the rows for the current slitlet_id + # Get the rows of shutter info for the current slitlet_id slitlets_sid = [x for x in msa_data if x['slitlet_id'] == slitlet_id] open_shutters = [x['shutter_column'] for x in slitlets_sid] # How many shutters in the slitlet are labeled as "main" or "primary"? n_main_shutter = len([s for s in slitlets_sid if s['primary_source'] == 'Y']) - # In the next part we need to calculate, find, determine 5 things: - # quadrant, xcen, ycen, ymin, ymax + # In the next part we need to calculate, find, or determine 5 things for each slit: + # quadrant, xcen, ycen, ymin, ymax # There are no main shutters: all are background if n_main_shutter == 0: @@ -664,11 +639,18 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, xcen = slitlets_sid[0]['shutter_row'] # grab the first as they are all the same source_xpos = 0.0 source_ypos = 0.0 - source_id = _get_bkg_source_id(bkg_counter, max_source_id) + source_id = slitlet_id log.info(f'Slitlet_id {slitlet_id} is background only; assigned source_id = {source_id}') - bkg_counter += 1 - # There is 1 main shutter: phew, that makes it easier. + # Hardwire the source info for background slits, because there's + # no source info for them in the msa_file + source_name = "background_{}".format(slitlet_id) + source_alias = "bkg_{}".format(slitlet_id) + stellarity = 0.0 + source_ra = 0.0 + source_dec = 0.0 + + # There is 1 main shutter: this is a normal slit elif n_main_shutter == 1: xcen, ycen, quadrant, source_xpos, source_ypos = [ (s['shutter_row'], s['shutter_column'], s['shutter_quadrant'], @@ -687,6 +669,30 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, if slitlets_sid[i]['primary_source'] == 'Y': source_id = slitlets_sid[i]['source_id'] + # Normal slits with a source assigned have a source_id > 0 + if source_id > 0: + shutter_id = xcen + (ycen - 1) * 365 # shutter numbers in MSA file are 1-indexed + # Get source info for this normal slitlet + try: + source_name, source_alias, stellarity, source_ra, source_dec = [ + (s['source_name'], s['alias'], s['stellarity'], s['ra'], s['dec']) + for s in msa_source if s['source_id'] == source_id][0] + except IndexError: + log.warning("Could not retrieve source info from MSA file") + + # Slits with source_id < 0 are "virtual" slits, with no source assigned + else: + source_id = slitlet_id + log.info(f'Slitlet_id {slitlet_id} is virtual; assigned source_id = {source_id}') + # Hardwire the source info for this virtual slit, because there's none in the MSA file + source_xpos = 0.0 + source_ypos = 0.0 + source_name = "virtual_{}".format(slitlet_id) + source_alias = "vrt_{}".format(slitlet_id) + stellarity = 0.0 + source_ra = 0.0 + source_dec = 0.0 + # More than 1 main shutter: Not allowed! else: message = ("For slitlet_id = {}, metadata_id = {}, " @@ -697,20 +703,6 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, msa_file.close() raise MSAFileError(message) - # subtract 1 because shutter numbers in the MSA reference file are 1-based. - shutter_id = xcen + (ycen - 1) * 365 - try: - source_name, source_alias, stellarity, source_ra, source_dec = [ - (s['source_name'], s['alias'], s['stellarity'], s['ra'], s['dec']) - for s in msa_source if s['source_id'] == source_id][0] - except IndexError: - # all background shutters - source_name = "background_{}".format(slitlet_id) - source_alias = "bkg_{}".format(slitlet_id) - stellarity = 0.0 - source_ra = 0.0 - source_dec = 0.0 - # Create the output list of tuples that contain the required # data for further computations """ diff --git a/jwst/associations/lib/rules_level3_base.py b/jwst/associations/lib/rules_level3_base.py index f342e080b3..093083b704 100644 --- a/jwst/associations/lib/rules_level3_base.py +++ b/jwst/associations/lib/rules_level3_base.py @@ -583,7 +583,11 @@ def finalize(associations): # Define default product name filling format_product = FormatTemplate( key_formats={ - 'source_id': ['s{:05d}', 's{:s}'], + #'source_id': ['s{:05d}', 's{:s}'], + #'source_id': ['s{:09d}', 's{:s}'], + 'source_id': ['{:s}'], + #'background_id': ['b{:09d}', 'b{:s}'], + #'virtual_id': ['v{:09d}', 'v{:s}'], 'expspcin': ['{:0>2s}'], 'slit_name': ['{:s}'] } diff --git a/jwst/pipeline/calwebb_spec3.py b/jwst/pipeline/calwebb_spec3.py index 71354b2d8c..d85412a722 100644 --- a/jwst/pipeline/calwebb_spec3.py +++ b/jwst/pipeline/calwebb_spec3.py @@ -212,11 +212,26 @@ def process(self, input): source_id, result = source if result[0].meta.exposure.type == "NRS_FIXEDSLIT": slit_name = self._create_nrsfs_slit_name(result) + srcid = f's{source_id.lower()}' self.output_file = format_product( - output_file, source_id=source_id.lower(), slit_name=slit_name) + output_file, source_id=srcid, slit_name=slit_name) + elif result[0].meta.exposure.type == "NRS_MSASPEC": + self.log.debug(f" result[0].name = {result[0].name.lower()}") + source_name = result[0].source_name + self.log.debug(f" result[0].source_name = {source_name}") + if "back" in source_name: + self.log.debug(" background slit") + srcid = f'b{source_id:>09s}' + elif "virt" in source_name: + self.log.debug(" virtual slit") + srcid = f'v{source_id:>09s}' + else: + srcid = f's{source_id:>09s}' + self.output_file = format_product(output_file, source_id=srcid) + self.log.debug(f" output_file = {self.output_file}") else: - self.output_file = format_product( - output_file, source_id=source_id.lower()) + srcid = f's{source_id.lower()}' + self.output_file = format_product(output_file, source_id=srcid) else: result = source From c997cb8379a5653f5c289489ab52a345e043667a Mon Sep 17 00:00:00 2001 From: Howard Bushouse Date: Thu, 9 May 2024 16:12:29 -0400 Subject: [PATCH 02/15] More updates to force unique names for all slits --- jwst/assign_wcs/nirspec.py | 10 ++--- jwst/exp_to_source/exp_to_source.py | 5 ++- jwst/pipeline/calwebb_spec3.py | 62 +++++++++++++++-------------- 3 files changed, 40 insertions(+), 37 deletions(-) diff --git a/jwst/assign_wcs/nirspec.py b/jwst/assign_wcs/nirspec.py index a82857550b..3480203f0b 100644 --- a/jwst/assign_wcs/nirspec.py +++ b/jwst/assign_wcs/nirspec.py @@ -682,13 +682,13 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, # Slits with source_id < 0 are "virtual" slits, with no source assigned else: - source_id = slitlet_id + source_id = abs(source_id) log.info(f'Slitlet_id {slitlet_id} is virtual; assigned source_id = {source_id}') # Hardwire the source info for this virtual slit, because there's none in the MSA file - source_xpos = 0.0 - source_ypos = 0.0 - source_name = "virtual_{}".format(slitlet_id) - source_alias = "vrt_{}".format(slitlet_id) + source_xpos = 0.5 + source_ypos = 0.5 + source_name = "virtual_{}".format(source_id) + source_alias = "vrt_{}".format(source_id) stellarity = 0.0 source_ra = 0.0 source_dec = 0.0 diff --git a/jwst/exp_to_source/exp_to_source.py b/jwst/exp_to_source/exp_to_source.py index b9540b1b16..cf162e1481 100644 --- a/jwst/exp_to_source/exp_to_source.py +++ b/jwst/exp_to_source/exp_to_source.py @@ -37,8 +37,9 @@ def exp_to_source(inputs): log.info(f'Reorganizing data from exposure {exposure.meta.filename}') for slit in exposure.slits: - log.debug(f'Copying source {slit.source_id}') - result_slit = result[str(slit.source_id)] + #log.debug(f'Copying source {slit.source_id}') + log.debug(f'Copying source {slit.source_name}') + result_slit = result[str(slit.source_name)] result_slit.exposures.append(slit) # store values for later use (after merge_tree) # these values are incorrectly getting overwritten by diff --git a/jwst/pipeline/calwebb_spec3.py b/jwst/pipeline/calwebb_spec3.py index d85412a722..ad01720d7f 100644 --- a/jwst/pipeline/calwebb_spec3.py +++ b/jwst/pipeline/calwebb_spec3.py @@ -170,37 +170,37 @@ def process(self, input): ] # Check for negative and large source_id values - if len(sources) > 99999: - self.log.critical("Data contain more than 100,000 sources;" - "filename does not support 6 digit source ids.") - raise Exception - - available_src_ids = set(np.arange(99999) + 1) - used_src_ids = set() - for src in sources: - src_id, model = src - src_id = int(src_id) - used_src_ids.add(src_id) - if 0 < src_id <= 99999: - available_src_ids.remove(src_id) - - hotfixed_sources = [] + #if len(sources) > 99999: + # self.log.critical("Data contain more than 100,000 sources;" + # "filename does not support 6 digit source ids.") + # raise Exception + + #available_src_ids = set(np.arange(99999) + 1) + #used_src_ids = set() + #for src in sources: + # src_id, model = src + # src_id = int(src_id) + # used_src_ids.add(src_id) + # if 0 < src_id <= 99999: + # available_src_ids.remove(src_id) + + #hotfixed_sources = [] # now find and reset bad source_id values - for src in sources: - src_id, model = src - src_id = int(src_id) + #for src in sources: + # src_id, model = src + # src_id = int(src_id) # Replace ids that aren't positive 5-digit integers - if src_id < 0 or src_id > 99999: - src_id_new = available_src_ids.pop() - self.log.info(f"Source ID {src_id} falls outside allowed range.") - self.log.info(f"Reassigning {src_id} to {str(src_id_new).zfill(5)}.") - # Replace source_id for each model in the SourceModelContainers - for contained_model in model: - contained_model.source_id = src_id_new - src_id = src_id_new - hotfixed_sources.append((str(src_id), model)) - - sources = hotfixed_sources + # if src_id < 0 or src_id > 99999: + # src_id_new = available_src_ids.pop() + # self.log.info(f"Source ID {src_id} falls outside allowed range.") + # self.log.info(f"Reassigning {src_id} to {str(src_id_new).zfill(5)}.") + # # Replace source_id for each model in the SourceModelContainers + # for contained_model in model: + # contained_model.source_id = src_id_new + # src_id = src_id_new + # hotfixed_sources.append((str(src_id), model)) + + #sources = hotfixed_sources # Process each source for source in sources: @@ -216,9 +216,11 @@ def process(self, input): self.output_file = format_product( output_file, source_id=srcid, slit_name=slit_name) elif result[0].meta.exposure.type == "NRS_MSASPEC": + self.log.debug(f" source_id = {source_id}") self.log.debug(f" result[0].name = {result[0].name.lower()}") source_name = result[0].source_name - self.log.debug(f" result[0].source_name = {source_name}") + source_id = str(result[0].source_id) + self.log.debug(f" result[0].source_id = {source_id}") if "back" in source_name: self.log.debug(" background slit") srcid = f'b{source_id:>09s}' From 7f3b1ab202f5d14ba12e2f4c43289d2ebf2acc77 Mon Sep 17 00:00:00 2001 From: Howard Bushouse Date: Tue, 14 May 2024 15:29:30 -0400 Subject: [PATCH 03/15] Final (I hope) tweaks to the new code --- jwst/assign_wcs/nirspec.py | 20 ++-- .../outlier_detection_step.py | 1 - jwst/pipeline/calwebb_spec3.py | 95 +++++++++---------- 3 files changed, 55 insertions(+), 61 deletions(-) diff --git a/jwst/assign_wcs/nirspec.py b/jwst/assign_wcs/nirspec.py index 3480203f0b..a556db7f29 100644 --- a/jwst/assign_wcs/nirspec.py +++ b/jwst/assign_wcs/nirspec.py @@ -637,20 +637,23 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, quadrant = slitlets_sid[0]['shutter_quadrant'] ycen = j xcen = slitlets_sid[0]['shutter_row'] # grab the first as they are all the same - source_xpos = 0.5 - source_ypos = 0.5 + + # Background slits all have source_id=0 in the msa_file, + # so assign a unique id based on the slitlet_id source_id = slitlet_id - log.info(f'Slitlet_id {slitlet_id} is background only; assigned source_id = {source_id}') # Hardwire the source info for background slits, because there's # no source info for them in the msa_file + source_xpos = 0.5 + source_ypos = 0.5 source_name = "background_{}".format(slitlet_id) source_alias = "bkg_{}".format(slitlet_id) stellarity = 0.0 source_ra = 0.0 source_dec = 0.0 + log.info(f'Slitlet {slitlet_id} is background only; assigned source_id={source_id}') - # There is 1 main shutter: this is a normal slit + # There is 1 main shutter: this is a slit containing either a real or virtual source elif n_main_shutter == 1: xcen, ycen, quadrant, source_xpos, source_ypos = [ (s['shutter_row'], s['shutter_column'], s['shutter_quadrant'], @@ -669,7 +672,7 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, if slitlets_sid[i]['primary_source'] == 'Y': source_id = slitlets_sid[i]['source_id'] - # Normal slits with a source assigned have a source_id > 0 + # Slits with a real source assigned have a source_id > 0 if source_id > 0: shutter_id = xcen + (ycen - 1) * 365 # shutter numbers in MSA file are 1-indexed # Get source info for this normal slitlet @@ -682,16 +685,15 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, # Slits with source_id < 0 are "virtual" slits, with no source assigned else: - source_id = abs(source_id) - log.info(f'Slitlet_id {slitlet_id} is virtual; assigned source_id = {source_id}') # Hardwire the source info for this virtual slit, because there's none in the MSA file source_xpos = 0.5 source_ypos = 0.5 - source_name = "virtual_{}".format(source_id) - source_alias = "vrt_{}".format(source_id) + source_name = "virtual_{}".format(abs(source_id)) + source_alias = "vrt_{}".format(abs(source_id)) stellarity = 0.0 source_ra = 0.0 source_dec = 0.0 + log.info(f'Slitlet {slitlet_id} is virtual, with source_id={source_id}') # More than 1 main shutter: Not allowed! else: diff --git a/jwst/outlier_detection/outlier_detection_step.py b/jwst/outlier_detection/outlier_detection_step.py index 5eebdce717..11fc831991 100644 --- a/jwst/outlier_detection/outlier_detection_step.py +++ b/jwst/outlier_detection/outlier_detection_step.py @@ -156,7 +156,6 @@ def process(self, input_data): model.meta.cal_step.outlier_detection = "SKIPPED" else: self.input_models.meta.cal_step.outlier_detection = "SKIPPED" - self.skip = True return self.input_models self.log.debug(f"Using {detection_step.__name__} class for outlier_detection") diff --git a/jwst/pipeline/calwebb_spec3.py b/jwst/pipeline/calwebb_spec3.py index ad01720d7f..f4e82357fa 100644 --- a/jwst/pipeline/calwebb_spec3.py +++ b/jwst/pipeline/calwebb_spec3.py @@ -169,69 +169,32 @@ def process(self, input): for name, model in multislit_to_container(source_models).items() ] - # Check for negative and large source_id values - #if len(sources) > 99999: - # self.log.critical("Data contain more than 100,000 sources;" - # "filename does not support 6 digit source ids.") - # raise Exception - - #available_src_ids = set(np.arange(99999) + 1) - #used_src_ids = set() - #for src in sources: - # src_id, model = src - # src_id = int(src_id) - # used_src_ids.add(src_id) - # if 0 < src_id <= 99999: - # available_src_ids.remove(src_id) - - #hotfixed_sources = [] - # now find and reset bad source_id values - #for src in sources: - # src_id, model = src - # src_id = int(src_id) - # Replace ids that aren't positive 5-digit integers - # if src_id < 0 or src_id > 99999: - # src_id_new = available_src_ids.pop() - # self.log.info(f"Source ID {src_id} falls outside allowed range.") - # self.log.info(f"Reassigning {src_id} to {str(src_id_new).zfill(5)}.") - # # Replace source_id for each model in the SourceModelContainers - # for contained_model in model: - # contained_model.source_id = src_id_new - # src_id = src_id_new - # hotfixed_sources.append((str(src_id), model)) - - #sources = hotfixed_sources - # Process each source for source in sources: # If each source is a SourceModelContainer, - # the output name needs to be updated with the source ID, and potentially - # also the slit name (for NIRSpec fixed-slit only). + # the output name needs to be updated based on the source ID, + # and potentially also the slit name (for NIRSpec fixed-slit only). if isinstance(source, tuple): source_id, result = source + + # NIRSpec fixed-slit data if result[0].meta.exposure.type == "NRS_FIXEDSLIT": + # Output file name is constructed using the source_id and the slit name slit_name = self._create_nrsfs_slit_name(result) srcid = f's{source_id.lower()}' - self.output_file = format_product( - output_file, source_id=srcid, slit_name=slit_name) + self.output_file = format_product(output_file, source_id=srcid, slit_name=slit_name) + + # NIRSpec MOS/MSA data elif result[0].meta.exposure.type == "NRS_MSASPEC": - self.log.debug(f" source_id = {source_id}") - self.log.debug(f" result[0].name = {result[0].name.lower()}") - source_name = result[0].source_name - source_id = str(result[0].source_id) - self.log.debug(f" result[0].source_id = {source_id}") - if "back" in source_name: - self.log.debug(" background slit") - srcid = f'b{source_id:>09s}' - elif "virt" in source_name: - self.log.debug(" virtual slit") - srcid = f'v{source_id:>09s}' - else: - srcid = f's{source_id:>09s}' + # Construct the specially formatted source_id to use in the output file + # name that separates source, background, and virtual slits + srcid = self._create_nrsmos_source_id(result) self.output_file = format_product(output_file, source_id=srcid) - self.log.debug(f" output_file = {self.output_file}") + self.log.debug(f"output_file = {self.output_file}") + else: + # All other types just use the source_id directly in the file name srcid = f's{source_id.lower()}' self.output_file = format_product(output_file, source_id=srcid) else: @@ -334,3 +297,33 @@ def _create_nrsfs_slit_name(self, source_models): slit_name = "-".join(slit_names) # append slit names using a dash separator return slit_name + + def _create_nrsmos_source_id(self, source_models): + """Create the complete source_id product field for NIRSpec MOS products. + + The original source_id value has a "s", "b", or "v" character prepended + to uniquely identify source, background, and virtual slits. + """ + + # Get the original source name and ID from the input models + source_name = source_models[0].source_name + source_id = str(source_models[0].source_id) + + # MOS background sources have "background" in the source name + if "back" in source_name: + # prepend "b" to the source_id number and format to 9 chars + srcid = f'b{source_id:>09s}' + self.log.debug(f"Source {source_name} is a MOS background slitlet: ID={srcid}") + + # MOS virtual sources have "virtual" in the source name + elif "virt" in source_name: + # prepend "v" to the source_id number and remove the leading negative sign + srcid = f'v{source_id[1:]:>09s}' + self.log.debug(f"Source {source_name} is a MOS virtual slitlet: ID={srcid}") + + # Regular MOS sources + else: + # prepend "s" to the source_id number and format to 9 chars + srcid = f's{source_id:>09s}' + + return srcid From 58686b3414384cbca5e6b8eb4884bb7198ccd60f Mon Sep 17 00:00:00 2001 From: Howard Bushouse Date: Tue, 14 May 2024 15:31:30 -0400 Subject: [PATCH 04/15] fix style check --- jwst/pipeline/calwebb_spec3.py | 1 - 1 file changed, 1 deletion(-) diff --git a/jwst/pipeline/calwebb_spec3.py b/jwst/pipeline/calwebb_spec3.py index f4e82357fa..290897fe22 100644 --- a/jwst/pipeline/calwebb_spec3.py +++ b/jwst/pipeline/calwebb_spec3.py @@ -1,7 +1,6 @@ #!/usr/bin/env python from collections import defaultdict import os.path as op -import numpy as np from stdatamodels.jwst import datamodels From 18f80385d95b3f806362768b52de6c96d30fe990 Mon Sep 17 00:00:00 2001 From: Howard Bushouse Date: Wed, 15 May 2024 10:35:54 -0400 Subject: [PATCH 05/15] fix bug discovered during testing --- jwst/assign_wcs/nirspec.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jwst/assign_wcs/nirspec.py b/jwst/assign_wcs/nirspec.py index a556db7f29..54d9bf1d8c 100644 --- a/jwst/assign_wcs/nirspec.py +++ b/jwst/assign_wcs/nirspec.py @@ -637,6 +637,7 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, quadrant = slitlets_sid[0]['shutter_quadrant'] ycen = j xcen = slitlets_sid[0]['shutter_row'] # grab the first as they are all the same + shutter_id = xcen + (ycen - 1) * 365 # shutter numbers in MSA file are 1-indexed # Background slits all have source_id=0 in the msa_file, # so assign a unique id based on the slitlet_id @@ -660,6 +661,7 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, s['estimated_source_in_shutter_x'], s['estimated_source_in_shutter_y']) for s in slitlets_sid if s['background'] == 'N'][0] + shutter_id = xcen + (ycen - 1) * 365 # shutter numbers in MSA file are 1-indexed # y-size jmin = min([s['shutter_column'] for s in slitlets_sid]) @@ -667,6 +669,7 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, j = ycen ymax = yhigh + margin + (jmax - j) * 1.15 ymin = -(-ylow + margin) + (jmin - j) * 1.15 + # get the source_id from the primary shutter entry for i in range(len(slitlets_sid)): if slitlets_sid[i]['primary_source'] == 'Y': @@ -674,7 +677,6 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, # Slits with a real source assigned have a source_id > 0 if source_id > 0: - shutter_id = xcen + (ycen - 1) * 365 # shutter numbers in MSA file are 1-indexed # Get source info for this normal slitlet try: source_name, source_alias, stellarity, source_ra, source_dec = [ From 65e14488a719fb4c77d0670c65b68d47e68afb68 Mon Sep 17 00:00:00 2001 From: Howard Bushouse Date: Wed, 15 May 2024 10:47:37 -0400 Subject: [PATCH 06/15] add change log entry --- CHANGES.rst | 23 ++++++++++++++++++++++ jwst/associations/lib/rules_level3_base.py | 4 ---- jwst/exp_to_source/exp_to_source.py | 1 - 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index b9437ab3e5..517cb6493f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -20,6 +20,10 @@ assign_wcs - Move the assigned source position for dedicated NIRSpec MOS background slits from the lower left corner of the slit to the middle of the slit. [#8461] +- Updated the routines that load NIRSpec MOS slit and source data from the MSA meta + data file to properly handle background and virtual slits, and assign appropriate + meta data to them for use downstream. [#8442] + associations ------------ @@ -29,6 +33,9 @@ associations - Match NIRSpec imprint observations to science exposures on mosaic tile location and dither pointing, ``MOSTILNO`` and ``DITHPTIN``. [#8410] +- Updated Level3 rules for new handling of NIRSpec MOS source_id formatting when + constructing output file names. [#8442] + dark_current ------------ @@ -45,6 +52,13 @@ documentation - Added documentation for NIRCam GRISM time series pointing offsets. [#8449] +exp_to_source +------------- + +- Modified slit sorting to use `source_name` as the key, rather than `source_id`, + in order to support changes in `source_id` handling for NIRSpec MOS exposures + that contain background and virtual slits. [#8442] + extract_1d ---------- @@ -99,6 +113,10 @@ outlier_detection finished, unless save_intermediate_results is True. This PR also addressed the _i2d files not being saved in the specified output directory. [#8464] +- Removed the setting of `self.skip = True` when the step gets skipped (due to + inappropriate inputs), so that the step still executes when called again + while processing a list of multiple sources. [#8442] + photom ------ @@ -114,6 +132,11 @@ pipeline - Removed unused ``scale_detection`` argument from ``calwebb_tso3`` pipeline. [#8438] +- Updated the ``calwebb_spec3`` pipeline handling of NIRSpec MOS inputs, to + comply with the new scheme for source ("s"), background ("b"), and + virtual ("v") slits and the construction of output file names for each + type. [#8442] + ramp_fitting ------------ diff --git a/jwst/associations/lib/rules_level3_base.py b/jwst/associations/lib/rules_level3_base.py index 093083b704..ce7c5c631b 100644 --- a/jwst/associations/lib/rules_level3_base.py +++ b/jwst/associations/lib/rules_level3_base.py @@ -583,11 +583,7 @@ def finalize(associations): # Define default product name filling format_product = FormatTemplate( key_formats={ - #'source_id': ['s{:05d}', 's{:s}'], - #'source_id': ['s{:09d}', 's{:s}'], 'source_id': ['{:s}'], - #'background_id': ['b{:09d}', 'b{:s}'], - #'virtual_id': ['v{:09d}', 'v{:s}'], 'expspcin': ['{:0>2s}'], 'slit_name': ['{:s}'] } diff --git a/jwst/exp_to_source/exp_to_source.py b/jwst/exp_to_source/exp_to_source.py index cf162e1481..197bd3aa44 100644 --- a/jwst/exp_to_source/exp_to_source.py +++ b/jwst/exp_to_source/exp_to_source.py @@ -37,7 +37,6 @@ def exp_to_source(inputs): log.info(f'Reorganizing data from exposure {exposure.meta.filename}') for slit in exposure.slits: - #log.debug(f'Copying source {slit.source_id}') log.debug(f'Copying source {slit.source_name}') result_slit = result[str(slit.source_name)] result_slit.exposures.append(slit) From 688ccd12abafbed706b8453446e20015838838a0 Mon Sep 17 00:00:00 2001 From: Howard Bushouse Date: Wed, 15 May 2024 20:10:04 -0400 Subject: [PATCH 07/15] Fix bugs discovered during local regression testing --- jwst/exp_to_source/exp_to_source.py | 11 +++++++++-- jwst/pipeline/calwebb_spec3.py | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/jwst/exp_to_source/exp_to_source.py b/jwst/exp_to_source/exp_to_source.py index 197bd3aa44..baaaa1ce20 100644 --- a/jwst/exp_to_source/exp_to_source.py +++ b/jwst/exp_to_source/exp_to_source.py @@ -37,8 +37,15 @@ def exp_to_source(inputs): log.info(f'Reorganizing data from exposure {exposure.meta.filename}') for slit in exposure.slits: - log.debug(f'Copying source {slit.source_name}') - result_slit = result[str(slit.source_name)] + if slit.source_name is None: + # All MultiSlit data other than NIRSpec MOS get sorted by + # source_id (source_name is not populated) + key = slit.source_id + else: + # NIRSpec MOS slits get sorted by source_name + key = slit.source_name + log.debug(f'Copying source {key}') + result_slit = result[str(key)] result_slit.exposures.append(slit) # store values for later use (after merge_tree) # these values are incorrectly getting overwritten by diff --git a/jwst/pipeline/calwebb_spec3.py b/jwst/pipeline/calwebb_spec3.py index 290897fe22..d8dd0437f6 100644 --- a/jwst/pipeline/calwebb_spec3.py +++ b/jwst/pipeline/calwebb_spec3.py @@ -181,7 +181,7 @@ def process(self, input): if result[0].meta.exposure.type == "NRS_FIXEDSLIT": # Output file name is constructed using the source_id and the slit name slit_name = self._create_nrsfs_slit_name(result) - srcid = f's{source_id.lower()}' + srcid = f's{source_id:>09s}' self.output_file = format_product(output_file, source_id=srcid, slit_name=slit_name) # NIRSpec MOS/MSA data From ca2f008827cddc7ad79b4c92664d9e1181b91607 Mon Sep 17 00:00:00 2001 From: Howard Bushouse Date: Thu, 16 May 2024 09:57:58 -0400 Subject: [PATCH 08/15] fix failing unit test --- jwst/assign_wcs/tests/test_nirspec.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jwst/assign_wcs/tests/test_nirspec.py b/jwst/assign_wcs/tests/test_nirspec.py index a517916835..7b8e16bb39 100644 --- a/jwst/assign_wcs/tests/test_nirspec.py +++ b/jwst/assign_wcs/tests/test_nirspec.py @@ -310,8 +310,8 @@ def test_msa_configuration_all_background(): dither_position = 1 slitlet_info = nirspec.get_open_msa_slits(msaconfl, msa_meta_id, dither_position, slit_y_range=[-.5, .5]) - ref_slit = trmodels.Slit(57, 8281, 1, 251, 23, -2.15, 2.15, 4, 0, '1x1', 'background_57', 'bkg_57', - 0, 0.0, 0.0) + ref_slit = trmodels.Slit(57, 8281, 1, 251, 23, -2.15, 2.15, 4, 57, '1x1', 'background_57', 'bkg_57', + 0.0, 0.0, 0.0) _compare_slits(slitlet_info[0], ref_slit) From 7e3b4ffffaebf427b0989e1b641767d40de561f1 Mon Sep 17 00:00:00 2001 From: Howard Bushouse Date: Thu, 16 May 2024 15:55:34 -0400 Subject: [PATCH 09/15] Updates to spec2 and spec3 regtests --- jwst/regtest/test_nirspec_mos_spec2.py | 10 ++++++---- jwst/regtest/test_nirspec_mos_spec3.py | 10 ++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/jwst/regtest/test_nirspec_mos_spec2.py b/jwst/regtest/test_nirspec_mos_spec2.py index 8bc6f909d6..48e9185fc2 100644 --- a/jwst/regtest/test_nirspec_mos_spec2.py +++ b/jwst/regtest/test_nirspec_mos_spec2.py @@ -12,10 +12,10 @@ def run_pipeline(rtdata_module): rtdata = rtdata_module # Get the MSA metadata file referenced in the input exposure - rtdata.get_data("nirspec/mos/jw01180025001_01_msa.fits") + rtdata.get_data("nirspec/mos/jw01345066001_01_msa.fits") # Get the input ASN file and exposures - rtdata.get_asn("nirspec/mos/jw01180-o025_20221129t204108_spec2_00037_asn.json") + rtdata.get_asn("nirspec/mos/jw01345-o066_20230831t181155_spec2_00010_asn.json") # Run the calwebb_spec2 pipeline; save results from intermediate steps args = ["calwebb_spec2", rtdata.input, @@ -41,11 +41,13 @@ def run_pipeline(rtdata_module): "srctype", "pathloss", "barshadow", "cal", "s2d", "x1d"]) def test_nirspec_mos_spec2(run_pipeline, fitsdiff_default_kwargs, suffix): """Regression test of the calwebb_spec2 pipeline on a - NIRSpec MOS exposure.""" + NIRSpec MOS exposure. Using an exposure that's part of a + 3-shutter nod sequence, so there are nodded exposures available + with which to do background subtraction.""" # Run the pipeline and retrieve outputs rtdata = run_pipeline - output = f"jw01180025001_05101_00001_nrs2_{suffix}.fits" + output = f"jw01345066001_05101_00003_nrs1_{suffix}.fits" rtdata.output = output # Get the truth files diff --git a/jwst/regtest/test_nirspec_mos_spec3.py b/jwst/regtest/test_nirspec_mos_spec3.py index 507eb5c1c5..4e54e30a11 100644 --- a/jwst/regtest/test_nirspec_mos_spec3.py +++ b/jwst/regtest/test_nirspec_mos_spec3.py @@ -11,7 +11,7 @@ def run_pipeline(rtdata_module): """Run calwebb_spec3 on NIRSpec MOS data.""" rtdata = rtdata_module - rtdata.get_asn("nirspec/mos/jw00626-o030_20191210t193826_spec3_001_asn.json") + rtdata.get_asn("nirspec/mos/jw01345-o066_20230831t181155_spec3_00002_asn.json") # Run the calwebb_spec3 pipeline on the association args = ["calwebb_spec3", rtdata.input] @@ -22,13 +22,15 @@ def run_pipeline(rtdata_module): @pytest.mark.bigdata @pytest.mark.parametrize("suffix", ["cal", "crf", "s2d", "x1d"]) -@pytest.mark.parametrize("source_id", ["s00000", "s00227", "s00279", "s00443", - "s00482", "s02315"]) +@pytest.mark.parametrize("source_id", ["b000000030", "b000000031", + "s000004385", "s000007380", + "v000000048", "v000000049", + "v000000053", "v000000056"]) def test_nirspec_mos_spec3(run_pipeline, suffix, source_id, fitsdiff_default_kwargs): """Check results of calwebb_spec3""" rtdata = run_pipeline - output = f"jw00626-o030_{source_id}_nirspec_f170lp-g235m_{suffix}.fits" + output = f"jw01345-o066_{source_id}_nirspec_f170lp-g235m_{suffix}.fits" rtdata.output = output rtdata.get_truth(f"truth/test_nirspec_mos_spec3/{output}") From ad74675c52d79c8baf157298f7bf10289bd75d99 Mon Sep 17 00:00:00 2001 From: Howard Bushouse Date: Fri, 17 May 2024 10:34:42 -0400 Subject: [PATCH 10/15] Update docs and fix regtests --- docs/jwst/data_products/file_naming.rst | 17 ++++++-- docs/jwst/data_products/msa_metadata.rst | 51 ++++++++++++++++++++---- docs/jwst/exp_to_source/main.rst | 4 +- docs/jwst/pipeline/calwebb_spec3.rst | 2 +- jwst/regtest/test_niriss_wfss.py | 2 +- jwst/regtest/test_nirspec_fs_spec3.py | 4 +- 6 files changed, 62 insertions(+), 18 deletions(-) diff --git a/docs/jwst/data_products/file_naming.rst b/docs/jwst/data_products/file_naming.rst index 0f8b6ca30b..c6a48e8a83 100644 --- a/docs/jwst/data_products/file_naming.rst +++ b/docs/jwst/data_products/file_naming.rst @@ -42,14 +42,14 @@ Just as for Stage 2, the suffix distinguishes the different file products of Sta The FITS file naming scheme for Stage 3 "source-based" products is as follows, where items in parentheses are optional: - jw-_[<"t"TargID | "s"SourceID>](-<"epoch"X>)__(-)_(-).fits + jw-_[<"t"TargID | [<"s" | "b" | "v">]](-<"epoch"X>)__(-)_(-).fits where - ppppp: Program ID number - AC_ID: Association candidate ID - TargID: 3-digit Target ID (either TargID or SourceID must be present) - - SourceID: 5-digit Source ID + - SourceID: 9-digit Source ID - epochX: The text "epoch" followed by a single digit epoch number (optional) - instr: Science instrument name (e.g. 'nircam', 'miri') - optElements: A single or hyphen-separated list of optical elements (e.g. filter, grating) @@ -57,10 +57,12 @@ where - prodType: Product type identifier (e.g. 'i2d', 's3d', 'x1d') - ACT_ID: 2-digit activity ID (optional) -An example Stage 3 product FITS file name is: +Example Stage 3 product FITS file names are: jw87600-a3001_t001_niriss_f480m-nrm_amiavg.fits + jw54321-o066_s000123456_nirspec_f170lp_g235m_s2d.fits + Optional Components """"""""""""""""""" @@ -72,7 +74,14 @@ TargID vs SourceID For single-target modes, this is the target identifier as defined in the APT proposal. - For multi-object modes, such as NIRSpec MOS, this will be the slit ID for each object. + For multi-object modes, such as NIRSpec MOS, this will be the source ID for each object. + Note that the SourceID value is preceded by one of three characters "s", "b", or "v". + For most multi-source observation modes, such as Wide Field Slitless Spectroscopy (WFSS) and + NIRSpec Fixed-Slit (FS) spectroscopy, the "s" prefix is used to indicate that the data + correspond to a defined source. In some NIRSpec MOS observations, however, + there can also be "background" and "virtual" sources + (see :ref:`NIRSpec MSA slitlets `). + These cases use the "b" and "v" SourceID prefixes. epochX diff --git a/docs/jwst/data_products/msa_metadata.rst b/docs/jwst/data_products/msa_metadata.rst index edaa703b2f..b251402c80 100644 --- a/docs/jwst/data_products/msa_metadata.rst +++ b/docs/jwst/data_products/msa_metadata.rst @@ -2,7 +2,7 @@ MSA Metadata File: ``msa`` ^^^^^^^^^^^^^^^^^^^^^^^^^^ -While not containing any actual science data, the NIRSpec MSA metadata file is nonetheless +While it doesn't contain any actual science data, the NIRSpec MSA metadata file is nonetheless a crucial component of calibration processing for NIRSpec MOS exposures. It contains all the slitlet, shutter, and source configuration information that's needed by the :ref:`calwebb_spec2 ` pipeline to process a MOS exposure. @@ -128,6 +128,8 @@ where the values of `MSAMETID` and `PATT_NUM` in the science exposure match the values of `msa_metdata_id` and `dither_point_index`, respectively, are loaded. +Slitlets with a catalog source +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ To better understand the ways in which these metadata are used, it's useful to reference a hypothetical example of data within a ``SHUTTER_INFO`` table. The table below shows the first 9 rows of a ``SHUTTER_INFO`` table for a MOS exposure @@ -186,13 +188,46 @@ within each slitlet should be considered the "primary" shutter. This is especial important for slitlets that contain extended sources and hence the `source_id` and `background` entries may indicate that the source is present in multiple shutters. -When a slitlet is found that has no shutters with a primary source (i.e. no shutters -having `primary_source` = "Y"), it is classified as a background slitlet and assigned -a source ID value that's greater than the maximum source ID assigned to other slitlets -(because such slitlets all have a source ID of zero in the MSA metadata coming from -the ground system). -These background slitlets can then be used in :ref:`master background ` -subtraction. +.. _msa_background_and_virtual_slits: + +Slitlets without a catalog source +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +It is possible for users to define slitlets that do not contain a source that's defined +in the MPT catalog when constructing an MSA configuration for an observation. These +kinds of slitlets can be used for two purposes. First, slitlets in which all of the +constituent shutters only contain background can be used to perform "master background" +subtraction during the :ref:`calwebb_spec2 ` pipeline processing +(see :ref:`NIRSpec Master Background ` for more details). +Second, slitlets made up of open shutters that may contain signal from some uncataloged +source in the field can also be created. These are referred to as "virtual" slitlets. + +Background and virtual slitlets have unique metadata in the shutter information table. +The primary defining piece of data is their assigned `source_id` value, because these +slitlets don't have a corresponding source listed in the source information table. +During creation with the MPT, all background slitlets are given a `source_id` of zero. +Virtual slitlets, on the other hand, are assigned *negative* `source_id` values, starting +with -1 and counting downwards from there (i.e. each virtual slit has a unique negative +`source_id` value). + +During the parsing of shutter information described in the previous section, when a +slitlet is found that has no shutters with a primary source (i.e. no shutters +having `primary_source` = "Y"), it is recognized as a background slitlet. In order to +be able to track multiple background slitlets through the remaining processing, they +are reassigned a new `source_id` value equal to their `slitlet_id`. Virtual slitlets, +meanwhile, retain their unique negative `source_id` values throughout processing. + +During Stage 3 processing with the :ref:`calwebb_spec3 ` pipeline, +unique source-based product file names will be created that distinguish data from the +three different kinds of slitlets: source, background, and virtual. As described in +:ref:`source-based file names `, the `SourceID` field of Stage 3 +file names consists of the 9-digit `source_id` number assigned to each MOS slitlet, +preceded by one of the three characters "s", "b", or "v", which identifies whether +the data are from a source, background, or virtual slitlet, respectively. Note that, +as described above, the `source_id` number used here for background slitlets is a +copy of their `slitlet_id` number. For example, a Stage 3 file name for data taken +from a virtual slitlet with `source_id` = -42 will look like: + + jw12345-o066_v000000042_nirspec_f170lp_g235m_x1d.fits The SOURCE_INFO Metadata diff --git a/docs/jwst/exp_to_source/main.rst b/docs/jwst/exp_to_source/main.rst index 14aaa0c8dd..0216274adf 100644 --- a/docs/jwst/exp_to_source/main.rst +++ b/docs/jwst/exp_to_source/main.rst @@ -83,8 +83,8 @@ on the meaning of each field in the file names. The FITS file naming scheme for the source-based "cal" products follows the standard Stage 3 syntax, such as:: - jw10006-o010_s00061_nirspec_f170lp-g235m_cal.fits + jw10006-o010_s000000061_nirspec_f170lp-g235m_cal.fits -where "s00061" in this example is the source ID. +where "s000000061" in this example is the source ID. See :ref:`source-based file names ` for more details on the meaning of each field in this type of file name. diff --git a/docs/jwst/pipeline/calwebb_spec3.rst b/docs/jwst/pipeline/calwebb_spec3.rst index 266852c7dc..a01fdee02e 100644 --- a/docs/jwst/pipeline/calwebb_spec3.rst +++ b/docs/jwst/pipeline/calwebb_spec3.rst @@ -112,7 +112,7 @@ input exposure-based products. The source-based collections of data are saved in intermediate files, one per source/slit. The root names of the source-based files contain the source ID as an identifier and use the same "_cal" suffix as the input calibrated exposure files. An example source-based file name is -"jw00042-o001_s0002_niriss_gr150r_f150w_cal.fits", where "s0002" is the source id. +"jw00042-o001_s00000002_niriss_gr150r_f150w_cal.fits", where "s00000002" is the source id. The reorganized sets of data are sent to subsequent steps to process and combine all the data for one source at a time. diff --git a/jwst/regtest/test_niriss_wfss.py b/jwst/regtest/test_niriss_wfss.py index b723690a2f..d2ab05f4d4 100644 --- a/jwst/regtest/test_niriss_wfss.py +++ b/jwst/regtest/test_niriss_wfss.py @@ -74,7 +74,7 @@ def run_nis_wfss_spec3(run_nis_wfss_spec2, rtdata_module): @pytest.mark.bigdata @pytest.mark.parametrize('suffix', ['cal', 'x1d', 'c1d']) -@pytest.mark.parametrize('source_id', ['s00015', 's00104']) +@pytest.mark.parametrize('source_id', ['s000000015', 's000000104']) def test_nis_wfss_spec3(run_nis_wfss_spec3, rtdata_module, suffix, source_id, fitsdiff_default_kwargs): """Regression test of the calwebb_spec3 pipeline applied to NIRISS WFSS data""" rtdata = rtdata_module diff --git a/jwst/regtest/test_nirspec_fs_spec3.py b/jwst/regtest/test_nirspec_fs_spec3.py index ac15e6a1fd..4dd0d861c4 100644 --- a/jwst/regtest/test_nirspec_fs_spec3.py +++ b/jwst/regtest/test_nirspec_fs_spec3.py @@ -27,8 +27,8 @@ def run_pipeline(rtdata_module): @pytest.mark.bigdata @pytest.mark.parametrize("suffix", ["cal", "crf", "s2d", "x1d"]) -@pytest.mark.parametrize("source_id,slit_name", [("s00001","s200a2"), ("s00021","s200a1"), ("s00023","s400a1"), - ("s00024","s1600a1"), ("s00025","s200b1")]) +@pytest.mark.parametrize("source_id,slit_name", [("s000000001","s200a2"), ("s000000021","s200a1"), + ("s000000023","s400a1"), ("s000000024","s1600a1"), ("s000000025","s200b1")]) def test_nirspec_fs_spec3(run_pipeline, rtdata_module, fitsdiff_default_kwargs, suffix, source_id, slit_name): """Test spec3 pipeline on a set of NIRSpec FS exposures.""" rtdata = rtdata_module From 87ec5de01b00bde7ac2be46d3327de1353c48584 Mon Sep 17 00:00:00 2001 From: Howard Bushouse Date: Fri, 17 May 2024 12:18:55 -0400 Subject: [PATCH 11/15] Yet another bug fix for side effects in WFSS modes --- jwst/pipeline/calwebb_spec3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwst/pipeline/calwebb_spec3.py b/jwst/pipeline/calwebb_spec3.py index d8dd0437f6..a992881439 100644 --- a/jwst/pipeline/calwebb_spec3.py +++ b/jwst/pipeline/calwebb_spec3.py @@ -194,7 +194,7 @@ def process(self, input): else: # All other types just use the source_id directly in the file name - srcid = f's{source_id.lower()}' + srcid = f's{source_id:>09s}' self.output_file = format_product(output_file, source_id=srcid) else: result = source From d3104390e1ca185cdfbd1e95d2e7581b57c8a217 Mon Sep 17 00:00:00 2001 From: Howard Bushouse Date: Tue, 28 May 2024 10:27:05 -0400 Subject: [PATCH 12/15] more virtual slit updates --- jwst/assign_wcs/nirspec.py | 66 +++++++++++----------------------- jwst/pipeline/calwebb_spec3.py | 18 +++++----- 2 files changed, 29 insertions(+), 55 deletions(-) diff --git a/jwst/assign_wcs/nirspec.py b/jwst/assign_wcs/nirspec.py index 54d9bf1d8c..db2c88319a 100644 --- a/jwst/assign_wcs/nirspec.py +++ b/jwst/assign_wcs/nirspec.py @@ -401,6 +401,7 @@ def get_open_slits(input_model, reference_files=None, slit_y_range=[-.55, .55]): """ exp_type = input_model.meta.exposure.type.lower() lamp_mode = input_model.meta.instrument.lamp_mode + prog_id = input_model.meta.observation.program_number.lstrip("0") if isinstance(lamp_mode, str): lamp_mode = lamp_mode.lower() else: @@ -409,7 +410,7 @@ def get_open_slits(input_model, reference_files=None, slit_y_range=[-.55, .55]): (lamp_mode == "msaspec")): msa_metadata_file, msa_metadata_id, dither_point = get_msa_metadata( input_model, reference_files) - slits = get_open_msa_slits(msa_metadata_file, msa_metadata_id, dither_point, slit_y_range) + slits = get_open_msa_slits(prog_id, msa_metadata_file, msa_metadata_id, dither_point, slit_y_range) elif exp_type == "nrs_fixedslit": slits = get_open_fixed_slits(input_model, slit_y_range) elif exp_type == "nrs_brightobj": @@ -510,27 +511,7 @@ def get_msa_metadata(input_model, reference_files): return msa_config, msa_metadata_id, dither_position -def _get_bkg_source_id(bkg_counter, shift_by): - """ - Compute a ``source_id`` for background slitlets. - - All background slitlets are assigned a source_id of 0. - A unique ``source_id`` is necessary to keep them separate in exp_to_source. - A counter is used to assign a unique ``source_id`` that's - greater than the max ID number of all defined sources. - - Parameters - ---------- - bkg_counter : int - The current value of the counter. - shift_by : int - The highest of all source_id values. - """ - - return bkg_counter + shift_by - - -def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, +def get_open_msa_slits(prog_id, msa_file, msa_metadata_id, dither_position, slit_y_range=[-.55, .55]): """ Return the opened MOS slitlets. @@ -557,6 +538,8 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, Parameters ---------- + prog_id : str + The program number msa_file : str MSA meta data file name, FITS keyword ``MSAMETFL``. msa_metadata_id : int @@ -647,8 +630,8 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, # no source info for them in the msa_file source_xpos = 0.5 source_ypos = 0.5 - source_name = "background_{}".format(slitlet_id) - source_alias = "bkg_{}".format(slitlet_id) + source_name = f"{prog_id}_BKG{slitlet_id}" + source_alias = "BKG{}".format(slitlet_id) stellarity = 0.0 source_ra = 0.0 source_dec = 0.0 @@ -670,32 +653,23 @@ def get_open_msa_slits(msa_file, msa_metadata_id, dither_position, ymax = yhigh + margin + (jmax - j) * 1.15 ymin = -(-ylow + margin) + (jmin - j) * 1.15 - # get the source_id from the primary shutter entry + # Get the source_id from the primary shutter entry for i in range(len(slitlets_sid)): if slitlets_sid[i]['primary_source'] == 'Y': source_id = slitlets_sid[i]['source_id'] - # Slits with a real source assigned have a source_id > 0 - if source_id > 0: - # Get source info for this normal slitlet - try: - source_name, source_alias, stellarity, source_ra, source_dec = [ - (s['source_name'], s['alias'], s['stellarity'], s['ra'], s['dec']) - for s in msa_source if s['source_id'] == source_id][0] - except IndexError: - log.warning("Could not retrieve source info from MSA file") - - # Slits with source_id < 0 are "virtual" slits, with no source assigned - else: - # Hardwire the source info for this virtual slit, because there's none in the MSA file - source_xpos = 0.5 - source_ypos = 0.5 - source_name = "virtual_{}".format(abs(source_id)) - source_alias = "vrt_{}".format(abs(source_id)) - stellarity = 0.0 - source_ra = 0.0 - source_dec = 0.0 - log.info(f'Slitlet {slitlet_id} is virtual, with source_id={source_id}') + # Get source info for this slitlet; + # note that slits with a real source assigned have source_id > 0, + # while slits with source_id < 0 contain "virtual" sources + try: + source_name, source_alias, stellarity, source_ra, source_dec = [ + (s['source_name'], s['alias'], s['stellarity'], s['ra'], s['dec']) + for s in msa_source if s['source_id'] == source_id][0] + except IndexError: + log.warning("Could not retrieve source info from MSA file") + + if source_id < 0: + log.info(f'Slitlet {slitlet_id} contains virtual source, with source_id={source_id}') # More than 1 main shutter: Not allowed! else: diff --git a/jwst/pipeline/calwebb_spec3.py b/jwst/pipeline/calwebb_spec3.py index a992881439..7dd0e736dd 100644 --- a/jwst/pipeline/calwebb_spec3.py +++ b/jwst/pipeline/calwebb_spec3.py @@ -300,29 +300,29 @@ def _create_nrsfs_slit_name(self, source_models): def _create_nrsmos_source_id(self, source_models): """Create the complete source_id product field for NIRSpec MOS products. - The original source_id value has a "s", "b", or "v" character prepended + The source_id value gets a "s", "b", or "v" character prepended to uniquely identify source, background, and virtual slits. """ # Get the original source name and ID from the input models source_name = source_models[0].source_name - source_id = str(source_models[0].source_id) + source_id = source_models[0].source_id - # MOS background sources have "background" in the source name - if "back" in source_name: + # MOS background sources have "BKG" in the source name + if "BKG" in source_name: # prepend "b" to the source_id number and format to 9 chars - srcid = f'b{source_id:>09s}' + srcid = f'b{str(source_id):>09s}' self.log.debug(f"Source {source_name} is a MOS background slitlet: ID={srcid}") - # MOS virtual sources have "virtual" in the source name - elif "virt" in source_name: + # MOS virtual sources have a negative source_id value + elif source_id < 0: # prepend "v" to the source_id number and remove the leading negative sign - srcid = f'v{source_id[1:]:>09s}' + srcid = f'v{str(source_id)[1:]:>09s}' self.log.debug(f"Source {source_name} is a MOS virtual slitlet: ID={srcid}") # Regular MOS sources else: # prepend "s" to the source_id number and format to 9 chars - srcid = f's{source_id:>09s}' + srcid = f's{str(source_id):>09s}' return srcid From c85701abaee9f0e03eaf892238158391d70b07a2 Mon Sep 17 00:00:00 2001 From: Howard Bushouse Date: Tue, 28 May 2024 13:44:50 -0400 Subject: [PATCH 13/15] fix failing unit tests --- jwst/assign_wcs/tests/test_nirspec.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/jwst/assign_wcs/tests/test_nirspec.py b/jwst/assign_wcs/tests/test_nirspec.py index 7b8e16bb39..b89ccdcc4b 100644 --- a/jwst/assign_wcs/tests/test_nirspec.py +++ b/jwst/assign_wcs/tests/test_nirspec.py @@ -70,6 +70,7 @@ def create_hdul(detector='NRS1'): phdu.header['detector'] = detector phdu.header['time-obs'] = '8:59:37' phdu.header['date-obs'] = '2016-09-05' + phdu.header['program'] = '1234' scihdu = fits.ImageHDU() scihdu.header['EXTNAME'] = "SCI" @@ -276,10 +277,11 @@ def test_msa_configuration_normal(): """ # Test 1: Reasonably normal as well + prog_id = '1234' msa_meta_id = 12 msaconfl = get_file_path('msa_configuration.fits') dither_position = 1 - slitlet_info = nirspec.get_open_msa_slits(msaconfl, msa_meta_id, dither_position, + slitlet_info = nirspec.get_open_msa_slits(prog_id, msaconfl, msa_meta_id, dither_position, slit_y_range=[-.5, .5]) ref_slit = trmodels.Slit(55, 9376, 1, 251, 26, -5.6, 1.0, 4, 1, '1111x', '95065_1', '2122', 0.13, -0.31716078999999997, -0.18092266) @@ -291,11 +293,12 @@ def test_msa_configuration_no_background(): Test the get_open_msa_slits function. """ # Test 2: Two main shutters, not allowed and should fail + prog_id = '1234' msa_meta_id = 13 msaconfl = get_file_path('msa_configuration.fits') dither_position = 1 with pytest.raises(MSAFileError): - nirspec.get_open_msa_slits(msaconfl, msa_meta_id, dither_position, + nirspec.get_open_msa_slits(prog_id, msaconfl, msa_meta_id, dither_position, slit_y_range=[-.5, .5]) @@ -305,12 +308,13 @@ def test_msa_configuration_all_background(): """ # Test 3: No non-background, not acceptable. + prog_id = '1234' msa_meta_id = 14 msaconfl = get_file_path('msa_configuration.fits') dither_position = 1 - slitlet_info = nirspec.get_open_msa_slits(msaconfl, msa_meta_id, dither_position, + slitlet_info = nirspec.get_open_msa_slits(prog_id, msaconfl, msa_meta_id, dither_position, slit_y_range=[-.5, .5]) - ref_slit = trmodels.Slit(57, 8281, 1, 251, 23, -2.15, 2.15, 4, 57, '1x1', 'background_57', 'bkg_57', + ref_slit = trmodels.Slit(57, 8281, 1, 251, 23, -2.15, 2.15, 4, 57, '1x1', '1234_BKG57', 'BKG57', 0.0, 0.0, 0.0) _compare_slits(slitlet_info[0], ref_slit) @@ -321,10 +325,11 @@ def test_msa_configuration_row_skipped(): """ # Test 4: One row is skipped, should be acceptable. + prog_id = '1234' msa_meta_id = 15 msaconfl = get_file_path('msa_configuration.fits') dither_position = 1 - slitlet_info = nirspec.get_open_msa_slits(msaconfl, msa_meta_id, dither_position, + slitlet_info = nirspec.get_open_msa_slits(prog_id, msaconfl, msa_meta_id, dither_position, slit_y_range=[-.5, .5]) ref_slit = trmodels.Slit(58, 8646, 1, 251, 24, -3.3, 5.6, 4, 1, '11x1011', '95065_1', '2122', 0.130, -0.31716078999999997, -0.18092266) @@ -336,10 +341,11 @@ def test_msa_configuration_multiple_returns(): Test the get_open_msa_slits function. """ # Test 4: One row is skipped, should be acceptable. + prog_id = '1234' msa_meta_id = 16 msaconfl = get_file_path('msa_configuration.fits') dither_position = 1 - slitlet_info = nirspec.get_open_msa_slits(msaconfl, msa_meta_id, dither_position, + slitlet_info = nirspec.get_open_msa_slits(prog_id, msaconfl, msa_meta_id, dither_position, slit_y_range=[-.5, .5]) ref_slit1 = trmodels.Slit(59, 8651, 1, 256, 24, -3.3, 5.6, 4, 1, '11x1011', '95065_1', '2122', 0.13000000000000003, -0.31716078999999997, -0.18092266) From 164afc91429df6f91229b0285fb19692db9d21aa Mon Sep 17 00:00:00 2001 From: Howard Bushouse Date: Tue, 28 May 2024 15:09:56 -0400 Subject: [PATCH 14/15] fix more unit tests --- jwst/msaflagopen/tests/test_msa_open.py | 1 + jwst/resample/tests/test_resample_step.py | 1 + 2 files changed, 2 insertions(+) diff --git a/jwst/msaflagopen/tests/test_msa_open.py b/jwst/msaflagopen/tests/test_msa_open.py index ca8611f248..50e6379984 100644 --- a/jwst/msaflagopen/tests/test_msa_open.py +++ b/jwst/msaflagopen/tests/test_msa_open.py @@ -158,6 +158,7 @@ def test_msaflagopen_step(): 'msa_metadata_id': 12} im.meta.observation = { + 'program_number': '1234', 'date': '2016-09-05', 'time': '8:59:37'} diff --git a/jwst/resample/tests/test_resample_step.py b/jwst/resample/tests/test_resample_step.py index 73f914181a..1f8c010ceb 100644 --- a/jwst/resample/tests/test_resample_step.py +++ b/jwst/resample/tests/test_resample_step.py @@ -70,6 +70,7 @@ def nirspec_rate(): 'ysize': 416, 'ystart': 529} im.meta.observation = { + 'program_number': '1234', 'date': '2016-09-05', 'time': '8:59:37'} im.meta.exposure = { From 3abe70d8713d795308daa035fd71e8140f770e48 Mon Sep 17 00:00:00 2001 From: Howard Bushouse Date: Wed, 29 May 2024 22:39:27 -0400 Subject: [PATCH 15/15] code cleanup --- jwst/assign_wcs/nirspec.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/jwst/assign_wcs/nirspec.py b/jwst/assign_wcs/nirspec.py index db2c88319a..e08834e824 100644 --- a/jwst/assign_wcs/nirspec.py +++ b/jwst/assign_wcs/nirspec.py @@ -401,25 +401,33 @@ def get_open_slits(input_model, reference_files=None, slit_y_range=[-.55, .55]): """ exp_type = input_model.meta.exposure.type.lower() lamp_mode = input_model.meta.instrument.lamp_mode - prog_id = input_model.meta.observation.program_number.lstrip("0") if isinstance(lamp_mode, str): lamp_mode = lamp_mode.lower() else: lamp_mode = 'none' + + # MOS/MSA exposure requiring MSA metadata file if exp_type in ["nrs_msaspec", "nrs_autoflat"] or ((exp_type in ["nrs_lamp", "nrs_autowave"]) and (lamp_mode == "msaspec")): - msa_metadata_file, msa_metadata_id, dither_point = get_msa_metadata( - input_model, reference_files) + prog_id = input_model.meta.observation.program_number.lstrip("0") + msa_metadata_file, msa_metadata_id, dither_point = get_msa_metadata(input_model, reference_files) slits = get_open_msa_slits(prog_id, msa_metadata_file, msa_metadata_id, dither_point, slit_y_range) + + # Fixed slits exposure (non-TSO) elif exp_type == "nrs_fixedslit": slits = get_open_fixed_slits(input_model, slit_y_range) + + # Bright object (TSO) exposure in S1600A1 fixed slit elif exp_type == "nrs_brightobj": slits = [Slit('S1600A1', 3, 0, 0, 0, slit_y_range[0], slit_y_range[1], 5, 1)] + + # Lamp exposure using fixed slits elif exp_type in ["nrs_lamp", "nrs_autowave"]: if lamp_mode in ['fixedslit', 'brightobj']: slits = get_open_fixed_slits(input_model, slit_y_range) else: raise ValueError("EXP_TYPE {0} is not supported".format(exp_type.upper())) + if reference_files is not None and slits: slits = validate_open_slits(input_model, slits, reference_files) log.info("Slits projected on detector {0}: {1}".format(input_model.meta.instrument.detector,