diff --git a/CHANGES.rst b/CHANGES.rst index ee45ab0369..5ac5e46e76 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -8,6 +8,11 @@ esa.hubble - Added new method ``get_hap_hst_link`` and ``get_member_observations`` to get related observations [#2268] +hsa +^^^ + +- New module to access ESA Herschel mission. [#2122] + Service fixes and enhancements ------------------------------ esa.hubble diff --git a/astroquery/esa/hsa/__init__.py b/astroquery/esa/hsa/__init__.py new file mode 100644 index 0000000000..13394f89ca --- /dev/null +++ b/astroquery/esa/hsa/__init__.py @@ -0,0 +1,29 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +""" +HSA +------- + +The Herschel Science Archive (HSA) is the ESA's archive for the +Herschel mission. +""" +from astropy import config as _config + + +class Conf(_config.ConfigNamespace): + """ + Configuration parameters for `astroquery.esa.hsa`. + """ + DATA_ACTION = _config.ConfigItem("http://archives.esac.esa.int/hsa/whsa-tap-server/data?", + "Main url for retrieving HSA Data Archive files") + + METADATA_ACTION = _config.ConfigItem("http://archives.esac.esa.int/hsa/whsa-tap-server/tap", + "Main url for retrieving HSA Data Archive metadata") + + TIMEOUT = 60 + + +conf = Conf() + +from .core import HSA, HSAClass + +__all__ = ['HSA', 'HSAClass', 'Conf', 'conf'] diff --git a/astroquery/esa/hsa/core.py b/astroquery/esa/hsa/core.py new file mode 100644 index 0000000000..d440bf3f7a --- /dev/null +++ b/astroquery/esa/hsa/core.py @@ -0,0 +1,411 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +import cgi +import os +import re +import shutil +from pathlib import Path + +from astropy import units as u +from astroquery.utils import commons +from astroquery import log +from astroquery.exceptions import LoginError +from astroquery.query import BaseQuery +from astroquery.utils.tap.core import Tap + +from . import conf + +__all__ = ['HSA', 'HSAClass'] + + +class HSAClass(BaseQuery): + + data_url = conf.DATA_ACTION + metadata_url = conf.METADATA_ACTION + timeout = conf.TIMEOUT + + def __init__(self, tap_handler=None): + super().__init__() + if tap_handler is None: + self._tap = Tap(url=self.metadata_url) + else: + self._tap = tap_handler + + def download_data(self, *, retrieval_type="OBSERVATION", observation_id=None, + instrument_name=None, filename=None, observation_oid=None, + instrument_oid=None, product_level=None, verbose=False, + download_dir="", cache=True, **kwargs): + """ + Download data from Herschel + + Parameters + ---------- + observation_id : string, optional + id of the observation to be downloaded + The identifies of the observation we want to retrieve, 10 digits + example: 1342195355 + retrieval_type : string, optional, default 'OBSERVATION' + The type of product that we want to retrieve + values: OBSERVATION, PRODUCT, POSTCARD, POSTCARDFITS, REQUESTFILE_XML, STANDALONE, UPDP, HPDP + instrument_name : string, optional, default 'PACS' + values: PACS, SPIRE, HIFI + The instrument name, by default 'PACS' if the retrieval_type is 'OBSERVATION' + filename : string, optional, default None + If the filename is not set it will use the observation_id as filename + file name to be used to store the file + verbose : bool, optional, default False + flag to display information about the process + observation_oid : string, optional + Observation internal identifies. This is the database identifier + instrument_oid : string, optional + The database identifies of the instrument + values: 1, 2, 3 + product_level : string, optional + level to download + values: ALL, AUXILIARY, CALIBRATION, LEVEL0, LEVEL0_5, LEVEL1, LEVEL2, LEVEL2_5, LEVEL3, ALL-LEVEL3 + download_dir : string, optional + The directory in which the file will be downloaded + + Returns + ------- + File name of downloaded data + """ + if filename is not None: + filename = os.path.splitext(filename)[0] + + params = {'retrieval_type': retrieval_type} + if observation_id is not None: + params['observation_id'] = observation_id + + if retrieval_type == "OBSERVATION" and instrument_name is None: + instrument_name = "PACS" + + if instrument_name is not None: + params['instrument_name'] = instrument_name + + if observation_oid is not None: + params['observation_oid'] = observation_oid + + if instrument_oid is not None: + params['instrument_oid'] = instrument_oid + + if product_level is not None: + params['product_level'] = product_level + + link = self.data_url + "".join(f"&{key}={val}" for key, val in params.items()) + + link += "".join(f"&{key}={val}" for key, val in kwargs.items()) + + if verbose: + log.info(link) + + response = self._request('HEAD', link, save=False, cache=cache) + if response.status_code == 401: + error = "Data protected by proprietary rights. Please check your credentials" + raise LoginError(error) + + response.raise_for_status() + + if filename is None: + if observation_id is not None: + filename = observation_id + else: + error = "Please set either 'obervation_id' or 'filename' for the output" + raise ValueError(error) + + _, res_params = cgi.parse_header(response.headers['Content-Disposition']) + + r_filename = res_params["filename"] + suffixes = Path(r_filename).suffixes + + if len(suffixes) > 1 and suffixes[-1] == ".jpg": + filename += suffixes[-1] + else: + filename += "".join(suffixes) + + filename = os.path.join(download_dir, filename) + + self._download_file(link, filename, head_safe=True, cache=cache) + + if verbose: + log.info(f"Wrote {link} to {filename}") + + return filename + + def get_observation(self, observation_id, instrument_name, *, filename=None, + observation_oid=None, instrument_oid=None, product_level=None, + verbose=False, download_dir="", cache=True, **kwargs): + """ + Download observation from Herschel. + This consists of a .tar file containing: + + - The auxiliary directory: contains all Herschel non-science spacecraft data + - The calibarion directory: contains the uplink and downlink calibration products + - directory: contains the science data distributed in sub-directories called level0/0.5/1/2/2.5/3. + + More information can be found here: + https://www.cosmos.esa.int/web/herschel/data-products-overview + + Parameters + ---------- + observation_id : string + id of the observation to be downloaded + The identifies of the observation we want to retrieve, 10 digits + example: 1342195355 + instrument_name : string + The instrument name + values: PACS, SPIRE, HIFI + filename : string, optional, default None + If the filename is not set it will use the observation_id as filename + file name to be used to store the file + verbose : bool, optional, default 'False' + flag to display information about the process + observation_oid : string, optional + Observation internal identifies. This is the database identifier + istrument_oid : string, optional + The database identifies of the instrument + values: 1, 2, 3 + product_level : string, optional + level to download + values: ALL, AUXILIARY, CALIBRATION, LEVEL0, LEVEL0_5, LEVEL1, LEVEL2, LEVEL2_5, LEVEL3, ALL-LEVEL3 + download_dir : string, optional + The directory in which the file will be downloaded + + Returns + ------- + File name of downloaded data + """ + if filename is not None: + filename = os.path.splitext(filename)[0] + + params = {'retrieval_type': "OBSERVATION", + 'observation_id': observation_id, + 'instrument_name': instrument_name} + + if observation_oid is not None: + params['observation_oid'] = observation_oid + + if instrument_oid is not None: + params['instrument_oid'] = instrument_oid + + if product_level is not None: + params['product_level'] = product_level + + link = self.data_url + "".join(f"&{key}={val}" for key, val in params.items()) + + link += "".join(f"&{key}={val}" for key, val in kwargs.items()) + + if verbose: + log.info(link) + + response = self._request('HEAD', link, save=False, cache=cache) + if response.status_code == 401: + error = "Data protected by proprietary rights. Please check your credentials" + raise LoginError(error) + + response.raise_for_status() + + _, res_params = cgi.parse_header(response.headers['Content-Disposition']) + + r_filename = res_params["filename"] + suffixes = Path(r_filename).suffixes + + if filename is None: + filename = observation_id + + filename += "".join(suffixes) + + filename = os.path.join(download_dir, filename) + + self._download_file(link, filename, head_safe=True, cache=cache) + + if verbose: + log.info(f"Wrote {link} to {filename}") + + return filename + + def get_postcard(self, observation_id, instrument_name, *, filename=None, + verbose=False, download_dir="", cache=True, **kwargs): + """ + Download postcard from Herschel + + Parameters + ---------- + observation_id : string + id of the observation to be downloaded + The identifies of the observation we want to retrieve, 10 digits + example: 1342195355 + instrument_name : string + The instrument name + values: PACS, SPIRE, HIFI + filename : string, optional, default None + If the filename is not set it will use the observation_id as filename + file name to be used to store the file + verbose : bool, optional, default False + flag to display information about the process + observation_oid : string, optional + Observation internal identifies. This is the database identifier + istrument_oid : string, optional + The database identifies of the instrument + values: 1, 2, 3 + product_level : string, optional + level to download + values: ALL, AUXILIARY, CALIBRATION, LEVEL0, LEVEL0_5, LEVEL1, LEVEL2, LEVEL2_5, LEVEL3, ALL-LEVEL3 + postcard_single : string, optional + 'true' to retrieve one single postcard (main one) + values: true, false + download_dir : string, optional + The directory in which the file will be downloaded + + Returns + ------- + File name of downloaded data + """ + if filename is not None: + filename = os.path.splitext(filename)[0] + + params = {'retrieval_type': "POSTCARD", + 'observation_id': observation_id, + 'instrument_name': instrument_name} + + link = self.data_url + "".join(f"&{key}={val}" for key, val in params.items()) + + link += "".join(f"&{key}={val}" for key, val in kwargs.items()) + + if verbose: + log.info(link) + + response = self._request('HEAD', link, save=False, cache=cache) + response.raise_for_status() + local_filepath = self._request('GET', link, cache=True, save=True) + + original_filename = re.findall('filename="(.+)"', + response.headers["Content-Disposition"])[0] + _, ext = os.path.splitext(original_filename) + if filename is None: + filename = observation_id + + filename += ext + + filename = os.path.join(download_dir, filename) + + shutil.move(local_filepath, filename) + + if verbose: + log.info(f"Wrote {link} to {filename}") + + return filename + + def query_hsa_tap(self, query, *, output_file=None, + output_format="votable", verbose=False): + """ + Launches a synchronous job to query HSA Tabular Access Protocol (TAP) Service + + Parameters + ---------- + query : string + query (adql) to be executed + output_file : string, optional, default None + file name where the results are saved if dumpToFile is True. + If this parameter is not provided, the jobid is used instead + output_format : string, optional, default 'votable' + values 'votable' or 'csv' + verbose : bool, optional, default 'False' + flag to display information about the process + + Returns + ------- + A table object + """ + job = self._tap.launch_job(query=query, output_file=output_file, + output_format=output_format, + verbose=verbose, + dump_to_file=output_file is not None) + table = job.get_results() + return table + + def get_tables(self, *, only_names=True, verbose=False): + """ + Get the available table in HSA TAP service + + Parameters + ---------- + only_names : bool, optional, default True + True to load table names only + verbose : bool, optional, default False + flag to display information about the process + + Returns + ------- + A list of tables + """ + tables = self._tap.load_tables(verbose=verbose) + if only_names: + return [t.name for t in tables] + else: + return tables + + def get_columns(self, table_name, *, only_names=True, verbose=False): + """ + Get the available columns for a table in HSA TAP service + + Parameters + ---------- + table_name : string + table name of which, columns will be returned + only_names : bool, optional, default True + True to load column names only + verbose : bool, optional, default False + flag to display information about the process + + Returns + ------- + A list of columns + """ + tables = self._tap.load_tables(verbose=verbose) + + columns = None + for t in tables: + if str(t.name) == str(table_name): + columns = t.columns + break + + if columns is None: + raise ValueError("table name specified was not found in " + "HSA TAP service") + + if only_names: + return [c.name for c in columns] + else: + return columns + + def query_observations(self, coordinate, radius, *, n_obs=10): + """ + Get the observation IDs from a given region + + Parameters + ---------- + coordinate : string / `astropy.coordinates` + the identifier or coordinates around which to query + radius : int / `~astropy.units.Quantity` + the radius of the region + n_obs : int, optional + the number of observations + + Returns + ------- + A table object with the list of observations in the region + """ + r = radius + if not isinstance(radius, u.Quantity): + r = radius*u.deg + coord = commons.parse_coordinates(coordinate).icrs + + query = (f"select top {n_obs} observation_id from hsa.v_active_observation " + f"where contains(" + f"point('ICRS', hsa.v_active_observation.ra, hsa.v_active_observation.dec), " + f"circle('ICRS', {coord.ra.degree},{coord.dec.degree},{r.to(u.deg).value}))=1") + return self.query_hsa_tap(query) + + +HSA = HSAClass() diff --git a/astroquery/esa/hsa/tests/__init__.py b/astroquery/esa/hsa/tests/__init__.py new file mode 100644 index 0000000000..9dce85d06f --- /dev/null +++ b/astroquery/esa/hsa/tests/__init__.py @@ -0,0 +1 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst diff --git a/astroquery/esa/hsa/tests/dummy_handler.py b/astroquery/esa/hsa/tests/dummy_handler.py new file mode 100644 index 0000000000..05bfcc982f --- /dev/null +++ b/astroquery/esa/hsa/tests/dummy_handler.py @@ -0,0 +1,43 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +__all__ = ['DummyHandler'] + + +class DummyHandler(object): + + def __init__(self, method, parameters): + self._invokedMethod = method + self._parameters = parameters + + def reset(self): + self._parameters = {} + self._invokedMethod = None + + def check_call(self, method_name, parameters): + self.check_method(method_name) + self.check_parameters(parameters, method_name) + + def check_method(self, method): + if method == self._invokedMethod: + return + else: + raise ValueError(f"Method '{method}' is not invoked. (Invoked method " + f"is '{self._invokedMethod}'.") + + def check_parameters(self, parameters, method_name): + if parameters is None: + return len(self._parameters) == 0 + if len(parameters) != len(self._parameters): + raise ValueError(f"Wrong number of parameters for method '{method_name}'. " + f"Found: {len(self._parameters)}. Expected {len(parameters)}") + for key in parameters: + if key in self._parameters: + # check value + if self._parameters[key] != parameters[key]: + raise ValueError(f"Wrong '{key}' parameter " + f"value for method '{method_name}'. " + f"Found:'{self._parameters[key]}'. " + f"Expected:'{parameters[key]}'") + else: + raise ValueError(f"Parameter '{key}' not found in method '{method_name}'") + return True diff --git a/astroquery/esa/hsa/tests/dummy_tap_handler.py b/astroquery/esa/hsa/tests/dummy_tap_handler.py new file mode 100644 index 0000000000..e898bf60a5 --- /dev/null +++ b/astroquery/esa/hsa/tests/dummy_tap_handler.py @@ -0,0 +1,75 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +from ....utils.tap.model.taptable import TapTableMeta +from ....utils.tap.model.job import Job + + +class DummyHSATapHandler(object): + + def __init__(self, method, parameters): + self.__invokedMethod = method + self._parameters = parameters + + def reset(self): + self._parameters = {} + self.__invokedMethod = None + + def check_call(self, method_name, parameters): + self.check_method(method_name) + self.check_parameters(parameters, method_name) + + def check_method(self, method): + if method != self.__invokedMethod: + raise Exception(f"Method '{method}' " + f"not invoked. (Invoked method is " + f"'{self.__invokedMethod}')") + + def check_parameters(self, parameters, method_name): + if parameters is None: + return len(self._parameters) == 0 + if len(parameters) != len(self._parameters): + raise Exception(f"Wrong number of parameters for method '{method_name}'. " + f"Found: {len(self._parameters)}. Expected {len(parameters)}") + for key in parameters: + if key in self._parameters: + # check value + if self._parameters[key] != parameters[key]: + raise Exception(f"Wrong '{key}' parameter value for method " + f"'{method_name}'. " + f"Found: '{self._parameters[key]}'. Expected: '{parameters[key]}'") + else: + raise Exception("Parameter '{key}' not found for method 'method_name'") + return False + + def launch_job(self, query, name=None, output_file=None, + output_format="votable", verbose=False, dump_to_file=False, + upload_resource=None, upload_table_name=None): + self.__invokedMethod = 'launch_job' + self._parameters['query'] = query + self._parameters['name'] = name + self._parameters['output_file'] = output_file + self._parameters['output_format'] = output_format + self._parameters['verbose'] = verbose + self._parameters['dump_to_file'] = dump_to_file + self._parameters['upload_resource'] = upload_resource + self._parameters['upload_table_name'] = upload_table_name + return Job(False) + + def get_tables(self, only_names=True, verbose=False): + self.__invokedMethod = 'get_tables' + self._parameters['only_names'] = only_names + self._parameters['verbose'] = verbose + + def get_columns(self, table_name=None, only_names=True, verbose=False): + self.__invokedMethod = 'get_columns' + self._parameters['table_name'] = table_name + self._parameters['only_names'] = only_names + self._parameters['verbose'] = verbose + + def load_tables(self, + only_names=True, + include_shared_tables=False, + verbose=True): + table = TapTableMeta() + table.name = "table" + return [table] diff --git a/astroquery/esa/hsa/tests/test_hsa.py b/astroquery/esa/hsa/tests/test_hsa.py new file mode 100644 index 0000000000..c08dc1078f --- /dev/null +++ b/astroquery/esa/hsa/tests/test_hsa.py @@ -0,0 +1,59 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +import pytest + +from astropy import units as u +from astropy.coordinates import SkyCoord +from ..core import HSAClass +from ..tests.dummy_handler import DummyHandler +from ..tests.dummy_tap_handler import DummyHSATapHandler + + +class TestHSA(): + + def get_dummy_tap_handler(self): + parameterst = {'query': "select top 10 * from hsa.v_active_observation", + 'output_file': "test.vot", + 'output_format': "votable", + 'verbose': False} + dummyTapHandler = DummyHSATapHandler("launch_job", parameterst) + return dummyTapHandler + + def test_query_hsa_tap(self): + parameters = {'query': "select top 10 * from hsa.v_active_observation", + 'output_file': "test.vot", + 'output_format': "votable", + 'verbose': False} + hsa = HSAClass(self.get_dummy_tap_handler()) + hsa.query_hsa_tap(**parameters) + self.get_dummy_tap_handler().check_call("launch_job", parameters) + self.get_dummy_tap_handler().check_parameters(parameters, "launch_job") + self.get_dummy_tap_handler().check_method("launch_job") + self.get_dummy_tap_handler().get_tables() + self.get_dummy_tap_handler().get_columns() + self.get_dummy_tap_handler().load_tables() + + def test_get_tables(self): + parameters = {'only_names': True, + 'verbose': True} + dummyTapHandler = DummyHSATapHandler("get_tables", parameters) + hsa = HSAClass(self.get_dummy_tap_handler()) + hsa.get_tables(**parameters) + dummyTapHandler.check_call("get_tables", parameters) + + def test_get_columns(self): + parameters = {'table_name': "table", + 'only_names': True, + 'verbose': True} + dummyTapHandler = DummyHSATapHandler("get_columns", parameters) + hsa = HSAClass(self.get_dummy_tap_handler()) + hsa.get_columns(**parameters) + dummyTapHandler.check_call("get_columns", parameters) + + def test_query_observations(self): + c = SkyCoord(ra=100.2417*u.degree, dec=9.895*u.degree, frame='icrs') + parameters = {'coordinate': c, + 'radius': 0.5} + dummyTapHandler = DummyHSATapHandler("query_observations", parameters) + hsa = HSAClass(self.get_dummy_tap_handler()) + hsa.query_observations(**parameters) + dummyTapHandler.check_call("query_observations", parameters) diff --git a/astroquery/esa/hsa/tests/test_hsa_remote.py b/astroquery/esa/hsa/tests/test_hsa_remote.py new file mode 100644 index 0000000000..0d8d5efe1a --- /dev/null +++ b/astroquery/esa/hsa/tests/test_hsa_remote.py @@ -0,0 +1,179 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +import tempfile +import pytest + +import os +import tarfile +from requests.exceptions import ChunkedEncodingError + +from ..core import HSAClass + +spire_chksum = [10233, 10762, 9019, 10869, 3944, 11328, 3921, 10999, 10959, 11342, 10974, 3944, 11335, 11323, 11078, 11321, 11089, 11314, 11108, 6281] + +pacs_chksum = [10208, 10755, 8917, 10028, 3924, 3935, 6291] + + +@pytest.mark.remote_data +class TestHSARemote: + tmp_dir = tempfile.TemporaryDirectory() + retries = 2 + + def access_archive_with_retries(self, f, params): + for _ in range(self.retries): + try: + res = f(**params) + return res + except ChunkedEncodingError: + pass + return None + + def test_download_data_observation_pacs(self): + obs_id = "1342191813" + parameters = {'retrieval_type': "OBSERVATION", + 'observation_id': obs_id, + 'instrument_name': "PACS", + 'product_level': 'LEVEL3', + 'cache': False, + 'download_dir': self.tmp_dir.name} + expected_res = os.path.join(self.tmp_dir.name, obs_id + ".tar") + hsa = HSAClass() + res = self.access_archive_with_retries(hsa.download_data, parameters) + if res is None: + pytest.xfail(f"Archive broke the connection {self.retries} times, unable to test") + assert res == expected_res + assert os.path.isfile(res) + tar = tarfile.open(res) + chksum = [m.chksum for m in tar.getmembers()] + assert chksum.sort() == pacs_chksum.sort() + os.remove(res) + + def test_download_data_observation_pacs_filename(self): + obs_id = "1342191813" + fname = "output_file" + parameters = {'retrieval_type': "OBSERVATION", + 'observation_id': obs_id, + 'instrument_name': "PACS", + 'product_level': 'LEVEL3', + 'filename': fname, + 'cache': False, + 'download_dir': self.tmp_dir.name} + expected_res = os.path.join(self.tmp_dir.name, fname + ".tar") + hsa = HSAClass() + res = self.access_archive_with_retries(hsa.download_data, parameters) + if res is None: + pytest.xfail(f"Archive broke the connection {self.retries} times, unable to test") + assert res == expected_res + assert os.path.isfile(res) + tar = tarfile.open(res) + chksum = [m.chksum for m in tar.getmembers()] + assert chksum.sort() == pacs_chksum.sort() + os.remove(res) + + def test_download_data_observation_pacs_compressed(self): + obs_id = "1342191813" + parameters = {'retrieval_type': "OBSERVATION", + 'observation_id': obs_id, + 'instrument_name': "PACS", + 'product_level': 'LEVEL3', + 'compress': 'true', + 'cache': False, + 'download_dir': self.tmp_dir.name} + expected_res = os.path.join(self.tmp_dir.name, obs_id + ".tgz") + hsa = HSAClass() + res = self.access_archive_with_retries(hsa.download_data, parameters) + if res is None: + pytest.xfail(f"Archive broke the connection {self.retries} times, unable to test") + assert res == expected_res + assert os.path.isfile(res) + tar = tarfile.open(res) + chksum = [m.chksum for m in tar.getmembers()] + assert chksum.sort() == pacs_chksum.sort() + os.remove(res) + + def test_download_data_observation_spire(self): + obs_id = "1342191188" + parameters = {'retrieval_type': "OBSERVATION", + 'observation_id': obs_id, + 'instrument_name': "SPIRE", + 'product_level': 'LEVEL2', + 'cache': False, + 'download_dir': self.tmp_dir.name} + expected_res = os.path.join(self.tmp_dir.name, obs_id + ".tar") + hsa = HSAClass() + res = self.access_archive_with_retries(hsa.download_data, parameters) + if res is None: + pytest.xfail(f"Archive broke the connection {self.retries} times, unable to test") + assert res == expected_res + assert os.path.isfile(res) + tar = tarfile.open(res) + chksum = [m.chksum for m in tar.getmembers()] + assert chksum.sort() == spire_chksum.sort() + os.remove(res) + + def test_download_data_postcard_pacs(self): + obs_id = "1342191813" + parameters = {'retrieval_type': "POSTCARD", + 'observation_id': obs_id, + 'instrument_name': "PACS", + 'cache': False, + 'download_dir': self.tmp_dir.name} + expected_res = os.path.join(self.tmp_dir.name, obs_id + ".jpg") + hsa = HSAClass() + res = self.access_archive_with_retries(hsa.download_data, parameters) + if res is None: + pytest.xfail(f"Archive broke the connection {self.retries} times, unable to test") + assert res == expected_res + assert os.path.isfile(res) + os.remove(res) + + def test_download_data_postcard_pacs_filename(self): + obs_id = "1342191813" + fname = "output_file" + parameters = {'retrieval_type': "POSTCARD", + 'observation_id': obs_id, + 'instrument_name': "PACS", + 'filename': fname, + 'cache': False, + 'download_dir': self.tmp_dir.name} + expected_res = os.path.join(self.tmp_dir.name, fname + ".jpg") + hsa = HSAClass() + res = self.access_archive_with_retries(hsa.download_data, parameters) + if res is None: + pytest.xfail(f"Archive broke the connection {self.retries} times, unable to test") + assert res == expected_res + assert os.path.isfile(res) + os.remove(res) + + def test_get_observation(self): + obs_id = "1342191813" + parameters = {'observation_id': obs_id, + 'instrument_name': "PACS", + 'product_level': 'LEVEL3', + 'cache': False, + 'download_dir': self.tmp_dir.name} + expected_res = os.path.join(self.tmp_dir.name, obs_id + ".tar") + hsa = HSAClass() + res = self.access_archive_with_retries(hsa.get_observation, parameters) + if res is None: + pytest.xfail(f"Archive broke the connection {self.retries} times, unable to test") + assert res == expected_res + assert os.path.isfile(res) + tar = tarfile.open(res) + chksum = [m.chksum for m in tar.getmembers()] + assert chksum.sort() == pacs_chksum.sort() + os.remove(res) + + def test_get_postcard(self): + obs_id = "1342191813" + parameters = {'observation_id': obs_id, + 'instrument_name': "PACS", + 'cache': False, + 'download_dir': self.tmp_dir.name} + expected_res = os.path.join(self.tmp_dir.name, obs_id + ".jpg") + hsa = HSAClass() + res = self.access_archive_with_retries(hsa.get_postcard, parameters) + if res is None: + pytest.xfail(f"Archive broke the connection {self.retries} times, unable to test") + assert res == expected_res + assert os.path.isfile(res) + os.remove(res) diff --git a/docs/esa/hsa.rst b/docs/esa/hsa.rst new file mode 100644 index 0000000000..025155828e --- /dev/null +++ b/docs/esa/hsa.rst @@ -0,0 +1,215 @@ +.. _astroquery.esa.hsa: + +*********************************************** +Herschel Science Archive (`astroquery.esa.hsa`) +*********************************************** + +`Herschel `__ was the fourth +cornerstone in ESA's Horizon 2000 science programme, designed to observe the 'cool' universe. +It performed photometry and spectroscopy in the poorly explored 55-670 µm spectral range with a 3.5 m diameter +Cassegrain telescope, providing unique observing capabilities and bridging the gap between earlier infrared +space missions and groundbased facilities. Herschel successfully performed ~37000 science observations and +~6600 science calibration observations which are publicly available to the worldwide astronomical community +through the Herschel Science Archive. + +This package allows the access to the `Herschel Science Archive `__. + +Examples +======== + +1. Getting Herschel data +------------------------ + +.. doctest-remote-data:: + + >>> from astroquery.esa.hsa import HSA + >>> + >>> HSA.download_data(observation_id='1342195355',retrieval_type='OBSERVATION', instrument_name='PACS') # doctest: +IGNORE_OUTPUT + Downloading URL http://archives.esac.esa.int/hsa/whsa-tap-server/data?&retrieval_type=OBSERVATION&observation_id=1342195355&instrument_name=PACS to 1342195355.tar ... [Done] + '1342195355.tar' + +This will download the product of the observation '1342195355' with the instrument 'PACS' and +it will store them in a tar called '1342195355.tar'. The parameters available are detailed in the API. + +For more details of the parameters check the section 6 of the ``Direct Product Access using TAP`` in the +`HSA users guide `_. + +For more details about the products check: + https://www.cosmos.esa.int/web/herschel/data-products-overview + + +2. Getting Observation Products +------------------------------- + +.. doctest-remote-data:: + + >>> from astroquery.esa.hsa import HSA + >>> + >>> HSA.get_observation('1342195355', instrument_name='PACS') # doctest: +IGNORE_OUTPUT + Downloading URL http://archives.esac.esa.int/hsa/whsa-tap-server/data?&retrieval_type=OBSERVATION&observation_id=1342195355&instrument_name=PACS to 1342195355.tar ... [Done] + '1342195355.tar' + +This will download the product of the observation '1342195355' with the instrument 'PACS' and +it will store them in a tar called '1342195355.tar'. The parameters available are detailed in the API. + +.. Note:: There is no difference between the product retrieved with this method and + `~astroquery.esa.hsa.HSAClass.download_data`. `~astroquery.esa.hsa.HSAClass.download_data` + is a more generic interface that allows the user to retrieve any product or metadata and + `~astroquery.esa.hsa.HSAClass.get_observation` allows the user to retrieve only observation products. + +For more information check the section 6.1 of the of the ``Direct Product Access using TAP`` in the +`HSA users guide`_. + +For more details of the parameters check the section 6.2 of the ``Direct Product Access using TAP`` in the +`HSA users guide`_. + + +3. Getting Herschel Postcard +---------------------------- + +.. doctest-remote-data:: + + >>> from astroquery.esa.hsa import HSA + >>> + >>> HSA.get_postcard('1342195355', instrument_name='PACS') # doctest: +IGNORE_OUTPUT + Downloading URL http://archives.esac.esa.int/hsa/whsa-tap-server/data?&retrieval_type=POSTCARD&observation_id=1342195355&instrument_name=PACS to /home/dev/.astropy/cache/astroquery/HSA/data?&retrieval_type=POSTCARD&observation_id=1342195355&instrument_name=PACS ... [Done] + '1342195355.jpg' + +This will download the postcard (static representation in JPG-format of the final product) of the observation +'1342195355' with the instrument 'PACS' and it will store them in a tar called '1342195355.jpg'. +The parameters available are detailed in the API. + +For more details of the parameters check the section 6.2 of the ``Direct Product Access using TAP`` in the +`HSA users guide`_. + + +4. Getting Herschel metadata through TAP +---------------------------------------- + +This function provides access to the Herschel Science Archive database using the Table Access Protocol (TAP) and via the Astronomical Data +Query Language (`ADQL `__). + +.. doctest-remote-data:: + + >>> from astroquery.esa.hsa import HSA + >>> + >>> result = HSA.query_hsa_tap("select top 10 * from hsa.v_active_observation", + ... output_format='csv', output_file='results.csv') + >>> result.pprint(max_width=100) + aor bii ... target_name urn_version + --------------------------------------- ------------------- ... -------------- ----------- + PP2-SWa-NGC3265 28.797292629881316 ... NGC3265 915907 + PRISMAS_W33a_hifi3b_898GHz_A_D2O -17.86672520275456 ... W33A 806737 + GOODS-S_70_d+8+8_forward_r3_shortaxis -27.80919396603746 ... GOODS-S d+8+8 894819 + PSP2_HStars-Set12f - RedRectangle -10.637417697356986 ... Red Rectangle 800938 + SPIRE-A - G126.24-5.52 57.195030974713134 ... G126.24-5.52 810242 + PSP1_PRISMAS_W31C_hifi6a_1477GHz_A_D2H+ -19.93074108498436 ... W31C 920099 + Spire Level-2 GOODS-S 37 - copy -27.81104151290488 ... GOODS-S 898135 + PSP2_HStars-Set13 - 10216 13.27923027337195 ... IRC+10216 801364 + PACS-A - G345.39-3.97 -43.47405026924179 ... G345.39-3.97-1 883176 + PRISMAS_g34_hifi7b_1897GHz_B_C3 1.2495150652937468 ... G34.3+0.1 921086 + +This will execute an ADQL query to download the first 10 observations in the Herschel Science Archive. +The result of the query will be stored in the file ``results.csv``. + + +5. Getting table details of HSA TAP +----------------------------------- + +.. doctest-remote-data:: + + >>> from astroquery.esa.hsa import HSA + >>> + >>> HSA.get_tables() + INFO: Retrieving tables... [astroquery.utils.tap.core] + INFO: Parsing tables... [astroquery.utils.tap.core] + INFO: Done. [astroquery.utils.tap.core] + ['hpdp.latest_observation_hpdp', 'hpdp.vizier_links', 'hpdp.unique_observation_hpdp', 'hpdp.latest_unique_observation_requests', 'hpdp.files', 'hpdp.latest_requests', 'public.dual', 'public.image_formats', 'tap_schema.tables', 'tap_schema.columns', 'tap_schema.keys', 'tap_schema.schemas', 'tap_schema.key_columns', 'hsa.observation_science', 'hsa.proposal_coauthor', 'hsa.proposal_observation', 'hsa.instrument', 'hsa.observing_mode_per_instrument', 'hsa.spire_spectral_feature_finder_catalogue', 'hsa.hifi_spectral_line_smoothed', 'hsa.publication', 'hsa.quality_flag', 'hsa.v_active_observation', 'hsa.proposal_info', 'hsa.pacs_point_source_070', 'hsa.observing_mode', 'hsa.proposal', 'hsa.proposal_pi_user', 'hsa.spire_point_source_350', 'hsa.spire_point_source_250', 'hsa.v_publication', 'hsa.spire_point_source_500', 'hsa.pacs_point_source_100', 'hsa.v_proposal_observation', 'hsa.hifi_spectral_line_native', 'hsa.pacs_point_source_160', 'hsa.ancillary', 'hsa.metadata_expert_panels', 'pubtool.institutions', 'pubtool.v_first_pub_date', 'pubtool.v_first_pub_date_single', 'pubtool.archival_type', 'pubtool.publication', 'pubtool.publication_details', 'pubtool.authors_institutions', 'pubtool.publication_observation', 'pubtool.authors', 'updp2.latest_observation_updp', 'updp2.vizier_links', 'updp2.latest_unique_observation_requests', 'updp2.files', 'updp2.latest_requests', 'updp2.unique_observation_updp'] + +This will show the available tables in HSA TAP service in the Herschel Science Archive. + + +6. Getting columns details of HSA TAP +------------------------------------- + +.. doctest-remote-data:: + + >>> from astroquery.esa.hsa import HSA + >>> + >>> HSA.get_columns('hsa.v_active_observation') + INFO: Retrieving tables... [astroquery.utils.tap.core] + INFO: Parsing tables... [astroquery.utils.tap.core] + INFO: Done. [astroquery.utils.tap.core] + ['aor', 'bii', 'dec', 'duration', 'end_time', 'fov', 'global_science_area', 'icon_image', 'icon_location', 'image_2_5_location', 'image_location', 'ingest_queue_oid', 'instrument_oid', 'is_active_version', 'is_public', 'lii', 'naif_id', 'num_publications', 'observation_id', 'observation_oid', 'observer', 'observing_mode_oid', 'obsstate', 'od_number', 'pa', 'polygon_fov', 'position', 'prop_end', 'proposal_id', 'quality_report_location', 'ra', 'science_area', 'science_category', 'spg_id', 'start_time', 'status', 'target_name', 'urn_version'] + +This will show the column details of the table 'hsa.v_active_observation' in HSA TAP service in the Herschel Science Archive. + + +7. Query Observations +--------------------- + +.. doctest-skip:: + + >>> from astroquery.esa.hsa import HSA + >>> from astropy.coordinates import SkyCoord + >>> from astropy import units as u + >>> + >>> c = SkyCoord(ra=100.2417*u.degree, dec=9.895*u.degree, frame='icrs') + >>> HSA.query_observations(c, 0.5) + + observation_id + object + -------------- + 1342219315 + 1342205057 + 1342205056 + 1342205056 + 1342205057 + +Retrieve a VOTable with the observation IDs of a given region + + +8. Procedure example +-------------------- + +First retrieve the observation IDs based on a position on the sky. To achive this, query the TAP service. + +.. doctest-skip:: + + >>> from astroquery.esa.hsa import HSA + >>> + >>> HSA.query_hsa_tap("select top 10 observation_id from hsa.v_active_observation where contains(point('ICRS', hsa.v_active_observation.ra, hsa.v_active_observation.dec),circle('ICRS', 100.2417,9.895, 1.1))=1", output_format='csv', output_file='results.cvs') +
+ observation_id + int64 + -------------- + 1342228342 + 1342228371 + 1342228372 + 1342219315 + 1342205057 + 1342205056 + 1342205058 + 1342205056 + 1342205057 + +In this example we are doing a circle search of 1.1 degrees in an ICRS (Right ascension [RA], Declination [DEC]) position (100.2417, 9.895). + +For more information on how to use ADQL see: + 'https://www.ivoa.net/documents/latest/ADQL.html' + +After obtaining the desire ID, download the product of the observation '1342205057' with the instrument 'PACS'. + + +.. doctest-skip:: + + >>> HSA.download_data(observation_id='1342205057', retrieval_type='OBSERVATION', instrument_name='PACS') + Downloading URL http://archives.esac.esa.int/hsa/whsa-tap-server/data?&retrieval_type=OBSERVATION&observation_id=1342205057&instrument_name=PACS to 1342205057.tar ... [Done] + '1342205057.tar' + + +Reference/API +============= + +.. automodapi:: astroquery.esa.hsa + :no-inheritance-diagram: diff --git a/docs/index.rst b/docs/index.rst index 4f9be1ecf2..b5019cd8b7 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -187,6 +187,7 @@ The following modules have been completed using a common API: cds/cds.rst linelists/cdms/cdms.rst dace/dace.rst + esa/hsa.rst esa/hubble.rst esa/iso.rst esa/jwst.rst