From 95bc9fed1cca5631deb42af04b29028a81ae1c45 Mon Sep 17 00:00:00 2001 From: David Gonzalez Date: Tue, 23 Apr 2024 13:44:19 +0200 Subject: [PATCH 01/11] get_obs_products method supports product_type parameter as string or list --- astroquery/esa/jwst/core.py | 22 +++++++++++++++++++--- docs/esa/jwst/jwst.rst | 2 +- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/astroquery/esa/jwst/core.py b/astroquery/esa/jwst/core.py index 8a1db48bdf..01d7275f08 100644 --- a/astroquery/esa/jwst/core.py +++ b/astroquery/esa/jwst/core.py @@ -83,6 +83,15 @@ def __init__(self, *, tap_plus_handler=None, data_handler=None, show_messages=Tr if show_messages: self.get_status_messages() + def __check_list_strings(self, list): + if list is None: + return False + if list and all(isinstance(elem, str) for elem in list): + return True + else: + raise ValueError("One of the lists is empty or there are " + "elements that are not strings") + def load_tables(self, *, only_names=False, include_shared_tables=False, verbose=False): """Loads all public tables @@ -968,10 +977,10 @@ def get_obs_products(self, *, observation_id=None, cal_level="ALL", composite products based on level 2 products). To request upper levels, please use get_related_observations functions first. Possible values: 'ALL', 3, 2, 1, -1 - product_type : str, optional, default None + product_type : str or list, optional, default None List only products of the given type. If None, all products are \ listed. Possible values: 'thumbnail', 'preview', 'auxiliary', \ - 'science'. + 'science', 'info'. output_file : str, optional Output file. If no value is provided, a temporary one is created. @@ -997,10 +1006,17 @@ def get_obs_products(self, *, observation_id=None, cal_level="ALL", max_cal_level=max_cal_level, is_url=True) params_dict['planeid'] = plane_ids + + if self.__check_list_strings(product_type): + if type(product_type) is list: tap_product_type=",".join(str(elem) for elem in product_type) + else: tap_product_type=product_type + else: + tap_product_type=None + self.__set_additional_parameters(param_dict=params_dict, cal_level=cal_level, max_cal_level=max_cal_level, - product_type=product_type) + product_type=tap_product_type) output_file_full_path, output_dir = self.__set_dirs(output_file=output_file, observation_id=observation_id) # Get file name only diff --git a/docs/esa/jwst/jwst.rst b/docs/esa/jwst/jwst.rst index 8cd59f85f5..47e10009dd 100644 --- a/docs/esa/jwst/jwst.rst +++ b/docs/esa/jwst/jwst.rst @@ -276,7 +276,7 @@ To download a data product >>> output_file = Jwst.get_product(file_name='jw01166091001_02102_00002_nrca3_cal.fits') # doctest: +SKIP To download products by observation identifier, it is possible to use the get_obs_products function, with the same parameters -than get_product_list. +than get_product_list, it also supports product_type parameter as string or list. .. doctest-remote-data:: From e0b66494dbf9b80f4516c3e68ee62ab774169df3 Mon Sep 17 00:00:00 2001 From: David Gonzalez Date: Tue, 23 Apr 2024 14:04:45 +0200 Subject: [PATCH 02/11] align to PEP8 --- astroquery/esa/jwst/core.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/astroquery/esa/jwst/core.py b/astroquery/esa/jwst/core.py index 01d7275f08..ca4ba11e77 100644 --- a/astroquery/esa/jwst/core.py +++ b/astroquery/esa/jwst/core.py @@ -1008,10 +1008,12 @@ def get_obs_products(self, *, observation_id=None, cal_level="ALL", params_dict['planeid'] = plane_ids if self.__check_list_strings(product_type): - if type(product_type) is list: tap_product_type=",".join(str(elem) for elem in product_type) - else: tap_product_type=product_type + if type(product_type) is list: + tap_product_type = ",".join(str(elem) for elem in product_type) + else: + tap_product_type = product_type else: - tap_product_type=None + tap_product_type = None self.__set_additional_parameters(param_dict=params_dict, cal_level=cal_level, From 4d2a20405b80e122000122ffe533e3ae21f30ce9 Mon Sep 17 00:00:00 2001 From: David Gonzalez Date: Fri, 26 Apr 2024 11:20:58 +0200 Subject: [PATCH 03/11] Modify CHANGES.rst file with get_obs_products method changes --- CHANGES.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 9e7b133ee8..2953d907f5 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -30,6 +30,11 @@ gama - Change URL to https and thus making the module functional again. [#3056] +esa.jwst +^^^^^^^^ + +- get_obs_products method supports product_type parameter as string or list [#2995] + mpc ^^^ From b9f52d1d462ab1781fd721e76bb9274441c895c0 Mon Sep 17 00:00:00 2001 From: David Gonzalez Date: Mon, 6 May 2024 13:48:54 +0200 Subject: [PATCH 04/11] Delete check_list_strings method --- astroquery/esa/jwst/core.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/astroquery/esa/jwst/core.py b/astroquery/esa/jwst/core.py index ca4ba11e77..f046e8d043 100644 --- a/astroquery/esa/jwst/core.py +++ b/astroquery/esa/jwst/core.py @@ -83,15 +83,6 @@ def __init__(self, *, tap_plus_handler=None, data_handler=None, show_messages=Tr if show_messages: self.get_status_messages() - def __check_list_strings(self, list): - if list is None: - return False - if list and all(isinstance(elem, str) for elem in list): - return True - else: - raise ValueError("One of the lists is empty or there are " - "elements that are not strings") - def load_tables(self, *, only_names=False, include_shared_tables=False, verbose=False): """Loads all public tables @@ -990,6 +981,11 @@ def get_obs_products(self, *, observation_id=None, cal_level="ALL", Returns the local path where the product(s) are saved. """ + if type(product_type) is list and '' in product_type: + raise ValueError("A list item is empty") + elif not product_type: + raise ValueError("The string is empty") + if observation_id is None: raise ValueError(self.REQUESTED_OBSERVATION_ID) plane_ids, max_cal_level = self._get_plane_id(observation_id=observation_id) @@ -1007,13 +1003,10 @@ def get_obs_products(self, *, observation_id=None, cal_level="ALL", is_url=True) params_dict['planeid'] = plane_ids - if self.__check_list_strings(product_type): - if type(product_type) is list: - tap_product_type = ",".join(str(elem) for elem in product_type) - else: - tap_product_type = product_type + if type(product_type) is list: + tap_product_type = ",".join(str(elem) for elem in product_type) else: - tap_product_type = None + tap_product_type = product_type self.__set_additional_parameters(param_dict=params_dict, cal_level=cal_level, From 21d6c21aaadc11502525bc383a39dd6fbbe96649 Mon Sep 17 00:00:00 2001 From: David Gonzalez Date: Mon, 13 May 2024 10:05:36 +0200 Subject: [PATCH 05/11] cleaner get_obs_products method --- astroquery/esa/jwst/core.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/astroquery/esa/jwst/core.py b/astroquery/esa/jwst/core.py index f046e8d043..84d5eece69 100644 --- a/astroquery/esa/jwst/core.py +++ b/astroquery/esa/jwst/core.py @@ -969,9 +969,10 @@ def get_obs_products(self, *, observation_id=None, cal_level="ALL", levels, please use get_related_observations functions first. Possible values: 'ALL', 3, 2, 1, -1 product_type : str or list, optional, default None - List only products of the given type. If None, all products are \ - listed. Possible values: 'thumbnail', 'preview', 'auxiliary', \ - 'science', 'info'. + List only products of the given type. If None, empty or an element + of the list is empty, all products are listed. + Possible values: 'thumbnail', 'preview', 'auxiliary', + 'science', 'info', ['science', 'preview']... output_file : str, optional Output file. If no value is provided, a temporary one is created. @@ -981,11 +982,9 @@ def get_obs_products(self, *, observation_id=None, cal_level="ALL", Returns the local path where the product(s) are saved. """ - if type(product_type) is list and '' in product_type: - raise ValueError("A list item is empty") - elif not product_type: - raise ValueError("The string is empty") - + if (type(product_type) is list and '' in product_type) or not product_type: + product_type = None + if observation_id is None: raise ValueError(self.REQUESTED_OBSERVATION_ID) plane_ids, max_cal_level = self._get_plane_id(observation_id=observation_id) From a10a87b896f82b9f6e141f74e7b2800d61c80943 Mon Sep 17 00:00:00 2001 From: David Gonzalez Date: Mon, 13 May 2024 10:16:16 +0200 Subject: [PATCH 06/11] clean whitespaces --- astroquery/esa/jwst/core.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/astroquery/esa/jwst/core.py b/astroquery/esa/jwst/core.py index 84d5eece69..f961e9f28e 100644 --- a/astroquery/esa/jwst/core.py +++ b/astroquery/esa/jwst/core.py @@ -984,7 +984,6 @@ def get_obs_products(self, *, observation_id=None, cal_level="ALL", if (type(product_type) is list and '' in product_type) or not product_type: product_type = None - if observation_id is None: raise ValueError(self.REQUESTED_OBSERVATION_ID) plane_ids, max_cal_level = self._get_plane_id(observation_id=observation_id) @@ -1005,7 +1004,7 @@ def get_obs_products(self, *, observation_id=None, cal_level="ALL", if type(product_type) is list: tap_product_type = ",".join(str(elem) for elem in product_type) else: - tap_product_type = product_type + tap_product_type = product_type self.__set_additional_parameters(param_dict=params_dict, cal_level=cal_level, From 89c7d3ba9773f6af8c77f07e4d1bd5de14682b1e Mon Sep 17 00:00:00 2001 From: David Gonzalez Date: Mon, 20 May 2024 11:17:16 +0200 Subject: [PATCH 07/11] remote test for product_type parameter as list --- docs/esa/jwst/jwst.rst | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/esa/jwst/jwst.rst b/docs/esa/jwst/jwst.rst index 47e10009dd..72b7451dc8 100644 --- a/docs/esa/jwst/jwst.rst +++ b/docs/esa/jwst/jwst.rst @@ -275,8 +275,9 @@ To download a data product >>> output_file = Jwst.get_product(artifact_id='6ab73824-6587-4bca-84a8-eb48ac7251be') # doctest: +SKIP >>> output_file = Jwst.get_product(file_name='jw01166091001_02102_00002_nrca3_cal.fits') # doctest: +SKIP + To download products by observation identifier, it is possible to use the get_obs_products function, with the same parameters -than get_product_list, it also supports product_type parameter as string or list. +than get_product_list, it also supports product_type parameter as string or list. product_type as string: .. doctest-remote-data:: @@ -284,6 +285,15 @@ than get_product_list, it also supports product_type parameter as string or list >>> observation_id = 'jw01122001001_0210r_00001_nrs2' >>> results = Jwst.get_obs_products(observation_id=observation_id, cal_level=2, product_type='science') # doctest: +SKIP + +Here product_type as list: + +.. doctest-remote-data:: + + >>> from astroquery.esa.jwst import Jwst + >>> observation_id = 'jw01122001001_0210r_00001_nrs2' + >>> results = Jwst.get_obs_products(observation_id=observation_id, cal_level=2, product_type=['science', 'preview']) # doctest: +SKIP + A temporary directory is created with the files and a list of the them is provided. When more than one product is found, a tar file is retrieved. This method extracts the products. From 667d254ce713adff7f5193bad6e6a73221a501cd Mon Sep 17 00:00:00 2001 From: David Gonzalez Date: Tue, 21 May 2024 10:20:04 +0200 Subject: [PATCH 08/11] delete "# doctest: +SKIP" from get_obs_products tests when they use the product_type attribute --- docs/esa/jwst/jwst.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/esa/jwst/jwst.rst b/docs/esa/jwst/jwst.rst index 72b7451dc8..f0429aa652 100644 --- a/docs/esa/jwst/jwst.rst +++ b/docs/esa/jwst/jwst.rst @@ -283,7 +283,7 @@ than get_product_list, it also supports product_type parameter as string or list >>> from astroquery.esa.jwst import Jwst >>> observation_id = 'jw01122001001_0210r_00001_nrs2' - >>> results = Jwst.get_obs_products(observation_id=observation_id, cal_level=2, product_type='science') # doctest: +SKIP + >>> results = Jwst.get_obs_products(observation_id=observation_id, cal_level=2, product_type='science') Here product_type as list: @@ -292,7 +292,7 @@ Here product_type as list: >>> from astroquery.esa.jwst import Jwst >>> observation_id = 'jw01122001001_0210r_00001_nrs2' - >>> results = Jwst.get_obs_products(observation_id=observation_id, cal_level=2, product_type=['science', 'preview']) # doctest: +SKIP + >>> results = Jwst.get_obs_products(observation_id=observation_id, cal_level=2, product_type=['science', 'preview']) A temporary directory is created with the files and a list of the them is provided. From ec3f48bf7f6497bc8280e866fc5d60dda55fbbf5 Mon Sep 17 00:00:00 2001 From: David Gonzalez Date: Tue, 2 Jul 2024 13:46:50 +0200 Subject: [PATCH 09/11] Replace "type(product_type) is list" by isinstance(product_type, list) in get_obs_products method. Improve the product_type comments in get_obs_products method. --- astroquery/esa/jwst/core.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/astroquery/esa/jwst/core.py b/astroquery/esa/jwst/core.py index f961e9f28e..45293e2995 100644 --- a/astroquery/esa/jwst/core.py +++ b/astroquery/esa/jwst/core.py @@ -969,10 +969,10 @@ def get_obs_products(self, *, observation_id=None, cal_level="ALL", levels, please use get_related_observations functions first. Possible values: 'ALL', 3, 2, 1, -1 product_type : str or list, optional, default None - List only products of the given type. If None, empty or an element - of the list is empty, all products are listed. - Possible values: 'thumbnail', 'preview', 'auxiliary', - 'science', 'info', ['science', 'preview']... + If the string or at least one element of the list is empty, the value is replaced by None. + With None, all products will be downloaded. + Possible string values: 'thumbnail', 'preview', 'auxiliary', 'science' or 'info'. + Posible list values: any combination of string values. output_file : str, optional Output file. If no value is provided, a temporary one is created. @@ -982,7 +982,7 @@ def get_obs_products(self, *, observation_id=None, cal_level="ALL", Returns the local path where the product(s) are saved. """ - if (type(product_type) is list and '' in product_type) or not product_type: + if (isinstance(product_type, list) and '' in product_type) or not product_type: product_type = None if observation_id is None: raise ValueError(self.REQUESTED_OBSERVATION_ID) From c1f0855b0f4254dcc0a88781341fdb1d65c33298 Mon Sep 17 00:00:00 2001 From: David Gonzalez Date: Mon, 8 Jul 2024 11:37:00 +0200 Subject: [PATCH 10/11] Include offline test for product_type as a list --- astroquery/esa/jwst/tests/test_jwsttap.py | 31 +++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/astroquery/esa/jwst/tests/test_jwsttap.py b/astroquery/esa/jwst/tests/test_jwsttap.py index dfb99935a6..a3f359d2e1 100644 --- a/astroquery/esa/jwst/tests/test_jwsttap.py +++ b/astroquery/esa/jwst/tests/test_jwsttap.py @@ -733,6 +733,37 @@ def test_get_obs_products(self): finally: shutil.rmtree(output_file_full_path_dir) + # Test product_type paramater with a list + output_file_full_path_dir = os.getcwd() + os.sep + "temp_test_jwsttap_get_obs_products_1" + try: + os.makedirs(output_file_full_path_dir, exist_ok=True) + except OSError as err: + print(f"Creation of the directory {output_file_full_path_dir} failed: {err.strerror}") + raise err + + file = data_path('single_product_retrieval.tar') + output_file_full_path = output_file_full_path_dir + os.sep + os.path.basename(file) + shutil.copy(file, output_file_full_path) + parameters['output_file'] = output_file_full_path + + expected_files = [] + extracted_file_1 = output_file_full_path_dir + os.sep + 'single_product_retrieval_1.fits' + expected_files.append(extracted_file_1) + product_type_as_list = ['science', 'info'] + try: + files_returned = (jwst.get_obs_products( + observation_id=observation_id, + cal_level='ALL', + product_type=product_type_as_list, + output_file=output_file_full_path)) + parameters['params_dict']['product_type'] = 'science,info' + dummyTapHandler.check_call('load_data', parameters) + self.__check_extracted_files(files_expected=expected_files, + files_returned=files_returned) + finally: + shutil.rmtree(output_file_full_path_dir) + del parameters['params_dict']['product_type'] + # Test single file output_file_full_path_dir = os.getcwd() + os.sep +\ "temp_test_jwsttap_get_obs_products_2" From bb734e0532b6bc2cf492ead72ea13f35fb8f7982 Mon Sep 17 00:00:00 2001 From: Javier Espinosa Date: Mon, 8 Jul 2024 14:31:37 +0200 Subject: [PATCH 11/11] Style fixes --- astroquery/esa/jwst/tests/test_jwsttap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astroquery/esa/jwst/tests/test_jwsttap.py b/astroquery/esa/jwst/tests/test_jwsttap.py index a3f359d2e1..15f8605f8a 100644 --- a/astroquery/esa/jwst/tests/test_jwsttap.py +++ b/astroquery/esa/jwst/tests/test_jwsttap.py @@ -740,7 +740,7 @@ def test_get_obs_products(self): except OSError as err: print(f"Creation of the directory {output_file_full_path_dir} failed: {err.strerror}") raise err - + file = data_path('single_product_retrieval.tar') output_file_full_path = output_file_full_path_dir + os.sep + os.path.basename(file) shutil.copy(file, output_file_full_path)