From 22976b487a54ca0c0002ff4acc8dc1961aef2595 Mon Sep 17 00:00:00 2001 From: lvalerom Date: Sat, 17 Jul 2021 20:37:40 +0200 Subject: [PATCH 01/22] hsa_support init --- astroquery/hsa/__init__.py | 30 ++++++++ astroquery/hsa/core.py | 62 +++++++++++++++++ astroquery/hsa/tests/__init__.py | 1 + astroquery/hsa/tests/dummy_handler.py | 51 ++++++++++++++ astroquery/hsa/tests/dummy_tap_handler.py | 83 +++++++++++++++++++++++ astroquery/hsa/tests/setup_package.py | 17 +++++ astroquery/hsa/tests/test_hsa.py | 47 +++++++++++++ astroquery/hsa/tests/test_hsa_remote.py | 1 + 8 files changed, 292 insertions(+) create mode 100644 astroquery/hsa/__init__.py create mode 100644 astroquery/hsa/core.py create mode 100644 astroquery/hsa/tests/__init__.py create mode 100644 astroquery/hsa/tests/dummy_handler.py create mode 100644 astroquery/hsa/tests/dummy_tap_handler.py create mode 100644 astroquery/hsa/tests/setup_package.py create mode 100644 astroquery/hsa/tests/test_hsa.py create mode 100644 astroquery/hsa/tests/test_hsa_remote.py diff --git a/astroquery/hsa/__init__.py b/astroquery/hsa/__init__.py new file mode 100644 index 0000000000..9714699ec8 --- /dev/null +++ b/astroquery/hsa/__init__.py @@ -0,0 +1,30 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +""" +HSA +------- + +The Herschel Science Archive (HSA) is the ESA's archive for the +Herschel mission. +""" +from astropy import config as _config + + +class Conf(_config.ConfigNamespace): + """ + Configuration parameters for `astroquery.hsa`. + """ + DATA_ACTION = _config.ConfigItem("http://archives.esac.esa.int/hsa/whsa-tap-server/data", + "Main url for retrieving HSA Data Archive files") + + METADATA_ACTION = _config.ConfigItem("http://archives.esac.esa.int/hsa/whsa-tap-server/tap", + "Main url for retrieving HSA Data Archive metadata") + + TIMEOUT = 60 + + +conf = Conf() + +from .core import HSA, HSAClass + +__all__ = ['HSA', 'HSAClass', 'Conf', 'conf'] + diff --git a/astroquery/hsa/core.py b/astroquery/hsa/core.py new file mode 100644 index 0000000000..13f37f85ef --- /dev/null +++ b/astroquery/hsa/core.py @@ -0,0 +1,62 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +from astroquery.query import BaseQuery +from astroquery.utils.tap.core import Tap + +from . import conf + +__all__ = ['HSA', 'HSAClass'] + +class HSAClass(BaseQuery): + + data_url = conf.DATA_ACTION + metadata_url = conf.METADATA_ACTION + timeout = conf.TIMEOUT + + def __init__(self, tap_handler=None): + super(HSAClass, self).__init__() + if tap_handler is None: + self._tap = Tap(url=self.metadata_url) + else: + self._tap = tap_handler + + def query_hsa_tap(self, query, *, output_file=None, + output_format="votable", verbose=False): + """ + """ + job = self._tap.launch_job(query=query, output_file=output_file, + output_format=output_format, + verbose=verbose, + dump_to_file=output_file is not None) + table = job.get_results() + return table + + def get_tables(self, *, only_names=True, verbose=False): + """ + """ + tables = self._tap.load_tables(verbose=verbose) + if only_names: + return [t.name for t in tables] + else: + return tables + + def get_columns(self, table_name, *, only_names=True, verbose=False): + """ + """ + tables = self._tap.load_tables(verbose=verbose) + + columns = None + for t in tables: + if str(t.name) == str(table_name): + columns = t.columns + break + + if columns is None: + raise ValueError("table name specified was not found in " + "HSA TAP service") + + if only_names: + return [c.name for c in columns] + else: + return columns +HSA = HSAClass() diff --git a/astroquery/hsa/tests/__init__.py b/astroquery/hsa/tests/__init__.py new file mode 100644 index 0000000000..9dce85d06f --- /dev/null +++ b/astroquery/hsa/tests/__init__.py @@ -0,0 +1 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst diff --git a/astroquery/hsa/tests/dummy_handler.py b/astroquery/hsa/tests/dummy_handler.py new file mode 100644 index 0000000000..94048b9ba5 --- /dev/null +++ b/astroquery/hsa/tests/dummy_handler.py @@ -0,0 +1,51 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +__all__ = ['DummyHandler'] + + +class DummyHandler(object): + + def __init__(self, method, parameters): + self._invokedMethod = method + self._parameters = parameters + + def reset(self): + self._parameters = {} + self._invokedMethod = None + + def check_call(self, method_name, parameters): + self.check_method(method_name) + self.check_parameters(parameters, method_name) + + def check_method(self, method): + if method == self._invokedMethod: + return + else: + raise ValueError("Method '{}' is not invoked. (Invoked method \ + is '{}'.)").format(method, self_invokedMethod) + + def check_parameters(self, parameters, method_name): + if parameters is None: + return len(self._parameters) == 0 + if len(parameters) != len(self._parameters): + raise ValueError("Wrong number of parameters for method '{}'. \ + Found: {}. Expected {}").format( + method_name, + len(self._parameters), + len(parameters)) + for key in parameters: + if key in self._parameters: + # check value + if self._parameters[key] != parameters[key]: + raise ValueError("Wrong '{}' parameter \ + value for method '{}'. \ + Found:'{}'. Expected:'{}'").format( + method_name, + key, + self._parameters[key], + parameters[key]) + else: + raise ValueError("Parameter '%s' not found in method '%s'", + (str(key), method_name)) + return True + diff --git a/astroquery/hsa/tests/dummy_tap_handler.py b/astroquery/hsa/tests/dummy_tap_handler.py new file mode 100644 index 0000000000..1d4f6e95b5 --- /dev/null +++ b/astroquery/hsa/tests/dummy_tap_handler.py @@ -0,0 +1,83 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +from ...utils.tap.model.taptable import TapTableMeta +from ...utils.tap.model.job import Job + + +class DummyHSATapHandler(object): + + def __init__(self, method, parameters): + self.__invokedMethod = method + self._parameters = parameters + + def reset(self): + self._parameters = {} + self.__invokedMethod = None + + def check_call(self, method_name, parameters): + self.check_method(method_name) + self.check_parameters(parameters, method_name) + + def check_method(self, method): + if method != self.__invokedMethod: + raise Exception("Method '" + str(method) + "" + "' not invoked. (Invoked method is '" + "" + str(self.__invokedMethod)+"')") + + def check_parameters(self, parameters, method_name): + if parameters is None: + return len(self._parameters) == 0 + if len(parameters) != len(self._parameters): + raise Exception("Wrong number of parameters for method '%s'. \ + Found: %d. Expected %d", + (method_name, + len(self._parameters), + len(parameters))) + for key in parameters: + if key in self._parameters: + # check value + if self._parameters[key] != parameters[key]: + raise Exception("Wrong '%s' parameter value for method '%s'. \ + Found: '%s'. Expected: '%s'", ( + method_name, + key, + self._parameters[key], + parameters[key])) + else: + raise Exception("Parameter '%s' not found for method '%s'", + (str(key), method_name)) + return False + + def launch_job(self, query, name=None, output_file=None, + output_format="votable", verbose=False, dump_to_file=False, + upload_resource=None, upload_table_name=None): + self.__invokedMethod = 'launch_job' + self._parameters['query'] = query + self._parameters['name'] = name + self._parameters['output_file'] = output_file + self._parameters['output_format'] = output_format + self._parameters['verbose'] = verbose + self._parameters['dump_to_file'] = dump_to_file + self._parameters['upload_resource'] = upload_resource + self._parameters['upload_table_name'] = upload_table_name + return Job(False) + + def get_tables(self, only_names=True, verbose=False): + self.__invokedMethod = 'get_tables' + self._parameters['only_names'] = only_names + self._parameters['verbose'] = verbose + + def get_columns(self, table_name=None, only_names=True, verbose=False): + self.__invokedMethod = 'get_columns' + self._parameters['table_name'] = table_name + self._parameters['only_names'] = only_names + self._parameters['verbose'] = verbose + + def load_tables(self, + only_names=True, + include_shared_tables=False, + verbose=True): + table = TapTableMeta() + table.name = "table" + return [table] + diff --git a/astroquery/hsa/tests/setup_package.py b/astroquery/hsa/tests/setup_package.py new file mode 100644 index 0000000000..f4dc7f93a3 --- /dev/null +++ b/astroquery/hsa/tests/setup_package.py @@ -0,0 +1,17 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +from __future__ import absolute_import + +import os + + +# setup paths to the test data +# can specify a single file or a list of files +def get_package_data(): + paths = [os.path.join('data', '*.tar'), + os.path.join('data', '*.xml'), + ] # etc, add other extensions + # you can also enlist files individually by names + # finally construct and return a dict for the sub module + return {'astroquery.hsa.tests': paths} + diff --git a/astroquery/hsa/tests/test_hsa.py b/astroquery/hsa/tests/test_hsa.py new file mode 100644 index 0000000000..e115c932f2 --- /dev/null +++ b/astroquery/hsa/tests/test_hsa.py @@ -0,0 +1,47 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +import pytest + +from ..core import HSAClass +from ..tests.dummy_handler import DummyHandler +from ..tests.dummy_tap_handler import DummyHSATapHandler + +class TestHSA(): + + def get_dummy_tap_handler(self): + parameterst = {'query': "select top 10 * from hsa.v_active_observation", + 'output_file': "test.vot", + 'output_format': "votable", + 'verbose': False} + dummyTapHandler = DummyHSATapHandler("launch_job", parameterst) + return dummyTapHandler + + def test_query_hsa_tap(self): + parameters = {'query': "select top 10 * from hsa.v_active_observation", + 'output_file': "test.vot", + 'output_format': "votable", + 'verbose': False} + hsa = HSAClass(self.get_dummy_tap_handler()) + hsa.query_hsa_tap(**parameters) + self.get_dummy_tap_handler().check_call("launch_job", parameters) + self.get_dummy_tap_handler().check_parameters(parameters, "launch_job") + self.get_dummy_tap_handler().check_method("launch_job") + self.get_dummy_tap_handler().get_tables() + self.get_dummy_tap_handler().get_columns() + self.get_dummy_tap_handler().load_tables() + + def test_get_tables(self): + parameters = {'only_names': True, + 'verbose': True} + dummyTapHandler = DummyHSATapHandler("get_tables", parameters) + hsa = HSAClass(self.get_dummy_tap_handler()) + hsa.get_tables(**parameters) + dummyTapHandler.check_call("get_tables", parameters) + + def test_get_columns(self): + parameters = {'table_name': "table", + 'only_names': True, + 'verbose': True} + dummyTapHandler = DummyHSATapHandler("get_columns", parameters) + hsa = HSAClass(self.get_dummy_tap_handler()) + hsa.get_columns(**parameters) + dummyTapHandler.check_call("get_columns", parameters) diff --git a/astroquery/hsa/tests/test_hsa_remote.py b/astroquery/hsa/tests/test_hsa_remote.py new file mode 100644 index 0000000000..9dce85d06f --- /dev/null +++ b/astroquery/hsa/tests/test_hsa_remote.py @@ -0,0 +1 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst From 6089362bb1e60c07fed135f4834900320a2b24f8 Mon Sep 17 00:00:00 2001 From: lvalerom Date: Sun, 18 Jul 2021 16:36:38 +0200 Subject: [PATCH 02/22] hsa support: products and postcards download --- astroquery/hsa/__init__.py | 2 +- astroquery/hsa/core.py | 147 ++++++++++++++++++++++++ astroquery/hsa/tests/test_hsa_remote.py | 45 ++++++++ 3 files changed, 193 insertions(+), 1 deletion(-) diff --git a/astroquery/hsa/__init__.py b/astroquery/hsa/__init__.py index 9714699ec8..150589bc09 100644 --- a/astroquery/hsa/__init__.py +++ b/astroquery/hsa/__init__.py @@ -13,7 +13,7 @@ class Conf(_config.ConfigNamespace): """ Configuration parameters for `astroquery.hsa`. """ - DATA_ACTION = _config.ConfigItem("http://archives.esac.esa.int/hsa/whsa-tap-server/data", + DATA_ACTION = _config.ConfigItem("http://archives.esac.esa.int/hsa/whsa-tap-server/data?", "Main url for retrieving HSA Data Archive files") METADATA_ACTION = _config.ConfigItem("http://archives.esac.esa.int/hsa/whsa-tap-server/tap", diff --git a/astroquery/hsa/core.py b/astroquery/hsa/core.py index 13f37f85ef..8aa6215669 100644 --- a/astroquery/hsa/core.py +++ b/astroquery/hsa/core.py @@ -1,5 +1,13 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst +import cgi +import os +import re +import shutil +from pathlib import Path + +from astroquery import log +from astroquery.exceptions import LoginError from astroquery.query import BaseQuery from astroquery.utils.tap.core import Tap @@ -20,6 +28,145 @@ def __init__(self, tap_handler=None): else: self._tap = tap_handler + def download_data(self, observation_id, *, retrieval_type=None, + instrument_name=None, + filename=None, + verbose=False, + cache=True, + **kwargs): + """ + """ + if filename is not None: + filename = os.path.splitext(filename)[0] + + if retrieval_type is None: + retrieval_type = "OBSERVATION" + + if retrieval_type == "OBSERVATION" and instrument_name is None: + instrument_name = "PACS" + + params = {'retrieval_type': retrieval_type, + 'observation_id': observation_id, + 'instrument_name': instrument_name} + + link = self.data_url + "".join("&{0}={1}".format(key, val) + for key, val in params.items()) + + link += "".join("&{0}={1}".format(key, val) + for key, val in kwargs.items()) + + if verbose: + log.info(link) + + response = self._request('HEAD', link, save=False, cache=cache) + response.raise_for_status() + + if 'Content-Type' in response.headers and 'text' not in response.headers['Content-Type']: + _, params = cgi.parse_header(response.headers['Content-Disposition']) + else: + error = "Data protected by propietary rights. Please check your credentials" + raise LoginError(error) + + r_filename = params["filename"] + suffixes = Path(r_filename).suffixes + + if filename is None: + filename = observation_id + + filename += "".join(suffixes) + + self._download_file(link, filename, head_safe=True, cache=cache) + + if verbose: + log.info("Wrote {0} to {1}".format(link, filename)) + + return filename + + def get_observation(self, observation_id, instrument_name, *, filename=None, + verbose=False, + cache=True, **kwargs): + """ + """ + if filename is not None: + filename = os.path.splitext(filename)[0] + + params = {'retrieval_type': "OBSERVATION", + 'observation_id': observation_id, + 'instrument_name': instrument_name} + + link = self.data_url + "".join("&{0}={1}".format(key, val) + for key, val in params.items()) + + link += "".join("&{0}={1}".format(key, val) + for key, val in kwargs.items()) + + if verbose: + log.info(link) + + response = self._request('HEAD', link, save=False, cache=cache) + response.raise_for_status() + + if 'Content-Type' in response.headers and 'text' not in response.headers['Content-Type']: + _, params = cgi.parse_header(response.headers['Content-Disposition']) + else: + error = "Data protected by propietary rights. Please check your credentials" + raise LoginError(error) + + r_filename = params["filename"] + suffixes = Path(r_filename).suffixes + + if filename is None: + filename = observation_id + + filename += "".join(suffixes) + + self._download_file(link, filename, head_safe=True, cache=cache) + + if verbose: + log.info("Wrote {0} to {1}".format(link, filename)) + + return filename + + def get_postcard(self, observation_id, instrument_name, *, filename=None, + verbose=False, + cache=True, **kwargs): + """ + """ + if filename is not None: + filename = os.path.splitext(filename)[0] + + params = {'retrieval_type': "POSTCARD", + 'observation_id': observation_id, + 'instrument_name': instrument_name} + + link = self.data_url + "".join("&{0}={1}".format(key, val) + for key, val in params.items()) + + link += "".join("&{0}={1}".format(key, val) + for key, val in kwargs.items()) + + if verbose: + log.info(link) + + response = self._request('HEAD', link, save=False, cache=cache) + response.raise_for_status() + local_filepath = self._request('GET', link, cache=True, save=True) + + original_filename = re.findall('filename="(.+)"', + response.headers["Content-Disposition"])[0] + _, ext = os.path.splitext(original_filename) + if filename is None: + filename = observation_id + + filename += ext + + shutil.move(local_filepath, filename) + + if verbose: + log.info("Wrote {0} to {1}".format(link, filename)) + + return filename + def query_hsa_tap(self, query, *, output_file=None, output_format="votable", verbose=False): """ diff --git a/astroquery/hsa/tests/test_hsa_remote.py b/astroquery/hsa/tests/test_hsa_remote.py index 9dce85d06f..4cc4d5b479 100644 --- a/astroquery/hsa/tests/test_hsa_remote.py +++ b/astroquery/hsa/tests/test_hsa_remote.py @@ -1 +1,46 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst +import pytest + +import os + +from ..core import HSAClass +from .test_hsa import TestHSA + +class TestHSARemote(TestHSA): + + @pytest.mark.remote_data + def test_download_data(self): + obs_id = "1342195355" + parameters = {'retrieval_type': "OBSERVATION", + 'observation_id': obs_id, + 'instrument_name': "PACS"} + expected_res = obs_id + ".tar" + hsa = HSAClass(self.get_dummy_tap_handler()) + res = hsa.download_data(**parameters) + assert res == expected_res + assert os.path.isfile(res) + os.remove(res) + + @pytest.mark.remote_data + def test_get_observation(self): + obs_id = "1342195355" + parameters = {'observation_id': obs_id, + 'instrument_name': "PACS"} + expected_res = obs_id + ".tar" + hsa = HSAClass(self.get_dummy_tap_handler()) + res = hsa.get_observation(**parameters) + assert res == expected_res + assert os.path.isfile(res) + os.remove(res) + + @pytest.mark.remote_data + def test_get_postcard(self): + obs_id = "1342195355" + parameters = {'observation_id': obs_id, + 'instrument_name': "PACS"} + expected_res = obs_id + ".jpg" + hsa = HSAClass(self.get_dummy_tap_handler()) + res = hsa.get_postcard(**parameters) + assert res == expected_res + assert os.path.isfile(res) + os.remove(res) From a6769757219cf5df3bfbf2025dd0561de5c5b530 Mon Sep 17 00:00:00 2001 From: lvalerom Date: Sun, 18 Jul 2021 18:11:53 +0200 Subject: [PATCH 03/22] hsa support: documentation --- astroquery/hsa/core.py | 141 +++++++++++++++++++++++++++++++++++++++-- docs/hsa/hsa.rst | 140 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 277 insertions(+), 4 deletions(-) create mode 100644 docs/hsa/hsa.rst diff --git a/astroquery/hsa/core.py b/astroquery/hsa/core.py index 8aa6215669..59d8420736 100644 --- a/astroquery/hsa/core.py +++ b/astroquery/hsa/core.py @@ -35,6 +35,37 @@ def download_data(self, observation_id, *, retrieval_type=None, cache=True, **kwargs): """ + Download data from Herschel + + Parameters + ---------- + observation_id : string, mandatory + id of the observation to be downloaded + The identifies of the observation we want to retrieve, 10 digits + example: 1342195355 + retrieval_type : string, optional, default 'OBSERVATION' + The type of product that we want to retrieve + values: OBSERVATION, PRODUCT, POSTCARD, POSTCARDFITS, REQUESTFILE_XML, STANDALONE, UPDP, HPDP + instrument_name : string, optinal, default 'PACS' + values: PACS, SPIRE, HIFI + The instrument name, by default 'PACS' if the retrieval_type is 'OBSERVATION' + filename : string, optinal, default None + If the filename is not set it will use the observation_id as filename + file name to be used to store the file + verbose : bool, optinal, default False + flag to display information about the process + observation_oid : string, optional + Observation internal identifies. This is the database identifier + istrument_oid : string, optional + The database identifies of the instrument + values: 1, 2, 3 + product_level : string, optional + level to download + values: ALL, AUXILIARY, CALIBRATION, LEVEL0, LEVEL0_5, LEVEL1, LEVEL2, LEVEL2_5, LEVEL3, ALL-LEVEL3 + + Returns + ------- + File name of downloaded data """ if filename is not None: filename = os.path.splitext(filename)[0] @@ -42,12 +73,12 @@ def download_data(self, observation_id, *, retrieval_type=None, if retrieval_type is None: retrieval_type = "OBSERVATION" + params = {'retrieval_type': retrieval_type, + 'observation_id': observation_id} + if retrieval_type == "OBSERVATION" and instrument_name is None: instrument_name = "PACS" - - params = {'retrieval_type': retrieval_type, - 'observation_id': observation_id, - 'instrument_name': instrument_name} + params['instrument_name'] = instrument_name link = self.data_url + "".join("&{0}={1}".format(key, val) for key, val in params.items()) @@ -86,6 +117,34 @@ def get_observation(self, observation_id, instrument_name, *, filename=None, verbose=False, cache=True, **kwargs): """ + Download observation from Herschel + + Parameters + ---------- + observation_id : string, mandatory + id of the observation to be downloaded + The identifies of the observation we want to retrieve, 10 digits + example: 1342195355 + instrument_name : string, mandatory + The instrument name + values: PACS, SPIRE, HIFI + filename : string, optinal, default None + If the filename is not set it will use the observation_id as filename + file name to be used to store the file + verbose : bool, optinal, default 'False' + flag to display information about the process + observation_oid : string, optional + Observation internal identifies. This is the database identifier + istrument_oid : string, optional + The database identifies of the instrument + values: 1, 2, 3 + product_level : string, optional + level to download + values: ALL, AUXILIARY, CALIBRATION, LEVEL0, LEVEL0_5, LEVEL1, LEVEL2, LEVEL2_5, LEVEL3, ALL-LEVEL3 + + Returns + ------- + File name of downloaded data """ if filename is not None: filename = os.path.splitext(filename)[0] @@ -131,6 +190,37 @@ def get_postcard(self, observation_id, instrument_name, *, filename=None, verbose=False, cache=True, **kwargs): """ + Download postcard from Herschel + + Parameters + ---------- + observation_id : string, mandatory + id of the observation to be downloaded + The identifies of the observation we want to retrieve, 10 digits + example: 1342195355 + instrument_name : string, mandatory + The instrument name + values: PACS, SPIRE, HIFI + filename : string, optinal, default None + If the filename is not set it will use the observation_id as filename + file name to be used to store the file + verbose : bool, optinal, default False + flag to display information about the process + observation_oid : string, optional + Observation internal identifies. This is the database identifier + istrument_oid : string, optional + The database identifies of the instrument + values: 1, 2, 3 + product_level : string, optional + level to download + values: ALL, AUXILIARY, CALIBRATION, LEVEL0, LEVEL0_5, LEVEL1, LEVEL2, LEVEL2_5, LEVEL3, ALL-LEVEL3 + postcard_single : string, optional + 'true' to retrieve one single postcard (main one) + values: true, false + + Returns + ------- + File name of downloaded data """ if filename is not None: filename = os.path.splitext(filename)[0] @@ -170,6 +260,23 @@ def get_postcard(self, observation_id, instrument_name, *, filename=None, def query_hsa_tap(self, query, *, output_file=None, output_format="votable", verbose=False): """ + Launches a synchronous job to query HSA Tabular Access Protocol (TAP) Service + + Parameters + ---------- + query : string, mandatory + query (adql) to be executed + output_file : string, optional, default None + file name where the results are saved if dumpToFile is True. + If this parameter is not provided, the jobid is used instead + output_format : string, optional, default 'votable' + values 'votable' or 'csv' + verbose : bool, optional, default 'False' + flag to display information about the process + + Returns + ------- + A table object """ job = self._tap.launch_job(query=query, output_file=output_file, output_format=output_format, @@ -180,6 +287,18 @@ def query_hsa_tap(self, query, *, output_file=None, def get_tables(self, *, only_names=True, verbose=False): """ + Get the available table in HSA TAP service + + Parameters + ---------- + only_names : bool, optional, default True + True to load table names only + verbose : bool, optional, default False + flag to display information about the process + + Returns + ------- + A list of tables """ tables = self._tap.load_tables(verbose=verbose) if only_names: @@ -189,6 +308,20 @@ def get_tables(self, *, only_names=True, verbose=False): def get_columns(self, table_name, *, only_names=True, verbose=False): """ + Get the available columns for a table in HSA TAP service + + Parameters + ---------- + table_name : string, mandatory + table name of which, columns will be returned + only_names : bool, optional, default True + True to load column names only + verbose : bool, optional, default False + + flag to display information about the process + Returns + ------- + A list of columns """ tables = self._tap.load_tables(verbose=verbose) diff --git a/docs/hsa/hsa.rst b/docs/hsa/hsa.rst new file mode 100644 index 0000000000..771ac5f177 --- /dev/null +++ b/docs/hsa/hsa.rst @@ -0,0 +1,140 @@ +.. doctest-skip-all + +.. _astroquery.hsa: + +********************** +hsa (`astroquery.hsa`) +********************** + +Herschel was the fourth cornerstone in ESA's Horizon 2000 science programme, designed to observe the 'cool' universe. +It performed photometry and spectroscopy in the poorly explored 55-670 µm spectral range with a 3.5 m diameter +Cassegrain telescope, providing unique observing capabilities and bridging the gap between earlier infrared space +missions and groundbased facilities. Herschel successfully performed ~37000 science observations and ~6600 science +calibration observations which are publicly available to the worldwide astronomical community through the Herschel Science Archive. + +This package allows the access to the `Herschel Science Archive `__. + +======== +Examples +======== + +------------------------------ +1. Getting Herschel data +------------------------------ + +.. code-block:: python + + >>> from astroquery.hsa import HSA + >>> + >>> HSA.download_data('1342195355',retrieval_type='OBSERVATION', instrument_name='PACS') + Downloading URL http://archives.esac.esa.int/hsa/whsa-tap-server/data?&retrieval_type=OBSERVATION&observation_id=1342195355&instrument_name=PACS to 1342195355.tar ... [Done] + '1342195355.tar' + +This will download the product of the observation '1342195355' with the instrument 'PACS' and +it will store them in a tar called '1342195355.tar'. The parameters available are detailed in the API. + +For more details of the parameters check the section 6 of the 'Direct Product Access using TAP' in the 'HSA users guide' at: + 'http://archives.esac.esa.int/hsa/whsa/' + +------------------------------- +2. Getting Observation Product +------------------------------- + +.. code-block:: python + + >>> from astroquery.hsa import HSA + >>> + >>> HSA.get_observation('1342195355', instrument_name='PACS') + Downloading URL http://archives.esac.esa.int/hsa/whsa-tap-server/data?&retrieval_type=OBSERVATION&observation_id=1342195355&instrument_name=PACS to 1342195355.tar ... [Done] + '1342195355.tar' + +This will download the product of the observation '1342195355' with the instrument 'PACS' and +it will store them in a tar called '1342195355.tar'. The parameters available are detailed in the API. + +For more details of the parameters check the section 6.2 of the 'Direct Product Access using TAP' in the 'HSA users guide' at: + 'http://archives.esac.esa.int/hsa/whsa/' + +------------------------------- +3. Getting Herschel Postcard +------------------------------- + +.. code-block:: python + + >>> from astroquery.hsa import HSA + >>> + >>> HSA.get_postcard('1342195355', instrument_name='PACS') + Downloading URL http://archives.esac.esa.int/hsa/whsa-tap-server/data?&retrieval_type=POSTCARD&observation_id=1342195355&instrument_name=PACS to /home/dev/.astropy/cache/astroquery/HSA/data?&retrieval_type=POSTCARD&observation_id=1342195355&instrument_name=PACS ... [Done] + '1342195355.jpg' + +This will download the postcard of the observation '1342195355' with the instrument 'PACS' and +it will store them in a tar called '1342195355.jpg'. The parameters available are detailed in the API. + +For more details of the parameters check the section 6.2 of the 'Direct Product Access using TAP' in the 'HSA users guide' at: + 'http://archives.esac.esa.int/hsa/whsa/' + +------------------------------------------ +3. Getting Herschel metadata through TAP +------------------------------------------ + +This function provides access to the Herschel Science Archive database using the Table Access Protocol (TAP) and via the Astronomical Data +Query Language (ADQL). + +.. code-block:: python + + >>> from astroquery.hsa import HSA + >>> + >>> result = HSA.query_hsa_tap("select top 10 * from hsa.v_active_observation", output_format='csv', output_file='results.csv') + >>> print(result) + aor bii dec duration ... status target_name urn_version +------------------------------------------------------------ ------------------- ------------------- ---------- ... ------ ---------------------------------- ----------- + PPhot.Cart-perp -33.71633333333334 -33.71633333333334 1759000.0 ... FAILED Cartwheel 925353 + PPhot.Cart -33.71633333333334 -33.71633333333334 1759000.0 ... FAILED Cartwheel 925352 +PPhoto-0005 - cosmos6 - cosmos_328-1 - XMMXCS J2215.9-1738-1 -17.633888888888887 -17.633888888888887 18149000.0 ... FAILED XMMXCS J2215.9-1738-1 cross scan-1 925351 + DRT-B-HD013246 -59.67941666666666 -59.67941666666666 2250000.0 ... FAILED HD013246-1 925350 + DRT-A-HD013246 -59.67941666666666 -59.67941666666666 2250000.0 ... FAILED HD013246-1 925348 + e0102green1_135 - hsc rec -72.03119444444444 -72.03119444444444 4272000.0 ... FAILED 1e0102.2-7219 925346 + e0102green1_45 - hsc rec -72.03119444444444 -72.03119444444444 4272000.0 ... FAILED 1e0102.2-7219 925344 + e0102blue1_135 - hsc rec -72.03119444444444 -72.03119444444444 4272000.0 ... FAILED 1e0102.2-7219 925342 + e0102blue1_45 - hsc rec -72.03119444444444 -72.03119444444444 4272000.0 ... FAILED 1e0102.2-7219 925340 + PPhot.AM06-perp -74.22638888888889 -74.22638888888889 1759000.0 ... FAILED AM0644-741 925338 + +This will execute an ADQL query to download the first 10 observations in the Herschel Science Archive. The result of the query will be +stored in the file 'results.csv'. The result of this query can be printed by doing print(result). + +----------------------------------- +4. Getting table details of HSA TAP +----------------------------------- + +.. code-block:: python + + >>> from astroquery.hsa import HSA + >>> + >>> HSA.get_tables() + INFO: Retrieving tables... [astroquery.utils.tap.core] + INFO: Parsing tables... [astroquery.utils.tap.core] + INFO: Done. [astroquery.utils.tap.core] + ['hpdp.latest_observation_hpdp', 'hpdp.vizier_links', 'hpdp.unique_observation_hpdp', 'hpdp.latest_unique_observation_requests', 'hpdp.files', 'hpdp.latest_requests', 'public.dual', 'public.image_formats', 'tap_schema.tables', 'tap_schema.columns', 'tap_schema.keys', 'tap_schema.schemas', 'tap_schema.key_columns', 'hsa.observation_science', 'hsa.proposal_coauthor', 'hsa.proposal_observation', 'hsa.instrument', 'hsa.observing_mode_per_instrument', 'hsa.spire_spectral_feature_finder_catalogue', 'hsa.hifi_spectral_line_smoothed', 'hsa.publication', 'hsa.quality_flag', 'hsa.v_active_observation', 'hsa.proposal_info', 'hsa.pacs_point_source_070', 'hsa.observing_mode', 'hsa.proposal', 'hsa.proposal_pi_user', 'hsa.spire_point_source_350', 'hsa.spire_point_source_250', 'hsa.v_publication', 'hsa.spire_point_source_500', 'hsa.pacs_point_source_100', 'hsa.v_proposal_observation', 'hsa.hifi_spectral_line_native', 'hsa.pacs_point_source_160', 'hsa.ancillary', 'hsa.metadata_expert_panels', 'pubtool.institutions', 'pubtool.v_first_pub_date', 'pubtool.v_first_pub_date_single', 'pubtool.archival_type', 'pubtool.publication', 'pubtool.publication_details', 'pubtool.authors_institutions', 'pubtool.publication_observation', 'pubtool.authors', 'updp2.latest_observation_updp', 'updp2.vizier_links', 'updp2.latest_unique_observation_requests', 'updp2.files', 'updp2.latest_requests', 'updp2.unique_observation_updp'] + +This will show the available tables in HSA TAP service in the Herschel Science Archive. + +------------------------------------- +5. Getting columns details of HSA TAP +------------------------------------- + +.. code-block:: python + + >>> from astroquery.hsa import HSA + >>> + >>> HSA.get_columns('hsa.v_active_observation') + INFO: Retrieving tables... [astroquery.utils.tap.core] + INFO: Parsing tables... [astroquery.utils.tap.core] + INFO: Done. [astroquery.utils.tap.core] + ['aor', 'bii', 'dec', 'duration', 'end_time', 'fov', 'global_science_area', 'icon_image', 'icon_location', 'image_2_5_location', 'image_location', 'ingest_queue_oid', 'instrument_oid', 'is_active_version', 'is_public', 'lii', 'naif_id', 'num_publications', 'observation_id', 'observation_oid', 'observer', 'observing_mode_oid', 'obsstate', 'od_number', 'pa', 'polygon_fov', 'position', 'prop_end', 'proposal_id', 'quality_report_location', 'ra', 'science_area', 'science_category', 'spg_id', 'start_time', 'status', 'target_name', 'urn_version'] + +This will show the column details of the table 'hsa.v_active_observation' in HSA TAP service in the Herschel Science Archive. + +Reference/API +============= + +.. automodapi:: astroquery.hsa + :no-inheritance-diagram: From 84bd49a7c594682f0d99a9b4ffaece6055722c69 Mon Sep 17 00:00:00 2001 From: lvalerom Date: Sun, 18 Jul 2021 18:14:02 +0200 Subject: [PATCH 04/22] hsa support: fixing tabulation problem in docs --- docs/hsa/hsa.rst | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/hsa/hsa.rst b/docs/hsa/hsa.rst index 771ac5f177..da32473376 100644 --- a/docs/hsa/hsa.rst +++ b/docs/hsa/hsa.rst @@ -85,18 +85,18 @@ Query Language (ADQL). >>> >>> result = HSA.query_hsa_tap("select top 10 * from hsa.v_active_observation", output_format='csv', output_file='results.csv') >>> print(result) - aor bii dec duration ... status target_name urn_version ------------------------------------------------------------- ------------------- ------------------- ---------- ... ------ ---------------------------------- ----------- - PPhot.Cart-perp -33.71633333333334 -33.71633333333334 1759000.0 ... FAILED Cartwheel 925353 - PPhot.Cart -33.71633333333334 -33.71633333333334 1759000.0 ... FAILED Cartwheel 925352 -PPhoto-0005 - cosmos6 - cosmos_328-1 - XMMXCS J2215.9-1738-1 -17.633888888888887 -17.633888888888887 18149000.0 ... FAILED XMMXCS J2215.9-1738-1 cross scan-1 925351 - DRT-B-HD013246 -59.67941666666666 -59.67941666666666 2250000.0 ... FAILED HD013246-1 925350 - DRT-A-HD013246 -59.67941666666666 -59.67941666666666 2250000.0 ... FAILED HD013246-1 925348 - e0102green1_135 - hsc rec -72.03119444444444 -72.03119444444444 4272000.0 ... FAILED 1e0102.2-7219 925346 - e0102green1_45 - hsc rec -72.03119444444444 -72.03119444444444 4272000.0 ... FAILED 1e0102.2-7219 925344 - e0102blue1_135 - hsc rec -72.03119444444444 -72.03119444444444 4272000.0 ... FAILED 1e0102.2-7219 925342 - e0102blue1_45 - hsc rec -72.03119444444444 -72.03119444444444 4272000.0 ... FAILED 1e0102.2-7219 925340 - PPhot.AM06-perp -74.22638888888889 -74.22638888888889 1759000.0 ... FAILED AM0644-741 925338 + aor bii dec duration ... status target_name urn_version + ------------------------------------------------------------ ------------------- ------------------- ---------- ... ------ ---------------------------------- ----------- + PPhot.Cart-perp -33.71633333333334 -33.71633333333334 1759000.0 ... FAILED Cartwheel 925353 + PPhot.Cart -33.71633333333334 -33.71633333333334 1759000.0 ... FAILED Cartwheel 925352 + PPhoto-0005 - cosmos6 - cosmos_328-1 - XMMXCS J2215.9-1738-1 -17.633888888888887 -17.633888888888887 18149000.0 ... FAILED XMMXCS J2215.9-1738-1 cross scan-1 925351 + DRT-B-HD013246 -59.67941666666666 -59.67941666666666 2250000.0 ... FAILED HD013246-1 925350 + DRT-A-HD013246 -59.67941666666666 -59.67941666666666 2250000.0 ... FAILED HD013246-1 925348 + e0102green1_135 - hsc rec -72.03119444444444 -72.03119444444444 4272000.0 ... FAILED 1e0102.2-7219 925346 + e0102green1_45 - hsc rec -72.03119444444444 -72.03119444444444 4272000.0 ... FAILED 1e0102.2-7219 925344 + e0102blue1_135 - hsc rec -72.03119444444444 -72.03119444444444 4272000.0 ... FAILED 1e0102.2-7219 925342 + e0102blue1_45 - hsc rec -72.03119444444444 -72.03119444444444 4272000.0 ... FAILED 1e0102.2-7219 925340 + PPhot.AM06-perp -74.22638888888889 -74.22638888888889 1759000.0 ... FAILED AM0644-741 925338 This will execute an ADQL query to download the first 10 observations in the Herschel Science Archive. The result of the query will be stored in the file 'results.csv'. The result of this query can be printed by doing print(result). From e266db2269fd4610479c458d0a836bd6e70d4df7 Mon Sep 17 00:00:00 2001 From: lvalerom Date: Sun, 18 Jul 2021 22:30:17 +0200 Subject: [PATCH 05/22] hsa_support: added documentation --- CHANGES.rst | 5 ++ astroquery/hsa/__init__.py | 1 - astroquery/hsa/core.py | 46 ++++++++------ astroquery/hsa/tests/dummy_handler.py | 1 - astroquery/hsa/tests/dummy_tap_handler.py | 1 - astroquery/hsa/tests/setup_package.py | 1 - astroquery/hsa/tests/test_hsa.py | 1 + astroquery/hsa/tests/test_hsa_remote.py | 73 ++++++++++++++++++++++- docs/hsa/hsa.rst | 10 ++-- docs/index.rst | 1 + 10 files changed, 113 insertions(+), 27 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index ee45ab0369..b696d39f2d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -268,6 +268,11 @@ heasarc - Add alternative instance of HEASARC Server, maintained by INTEGRAL Science Data Center. [#1988] +hsa +^^^ + +- New module to access ESA Herschel mission. [#] + nasa_exoplanet_archive ^^^^^^^^^^^^^^^^^^^^^^ diff --git a/astroquery/hsa/__init__.py b/astroquery/hsa/__init__.py index 150589bc09..428ef2f4cf 100644 --- a/astroquery/hsa/__init__.py +++ b/astroquery/hsa/__init__.py @@ -27,4 +27,3 @@ class Conf(_config.ConfigNamespace): from .core import HSA, HSAClass __all__ = ['HSA', 'HSAClass', 'Conf', 'conf'] - diff --git a/astroquery/hsa/core.py b/astroquery/hsa/core.py index 59d8420736..52b174cd5a 100644 --- a/astroquery/hsa/core.py +++ b/astroquery/hsa/core.py @@ -7,7 +7,7 @@ from pathlib import Path from astroquery import log -from astroquery.exceptions import LoginError +from astroquery.exceptions import LoginError, InputWarning from astroquery.query import BaseQuery from astroquery.utils.tap.core import Tap @@ -15,6 +15,7 @@ __all__ = ['HSA', 'HSAClass'] + class HSAClass(BaseQuery): data_url = conf.DATA_ACTION @@ -28,7 +29,8 @@ def __init__(self, tap_handler=None): else: self._tap = tap_handler - def download_data(self, observation_id, *, retrieval_type=None, + def download_data(self, *, retrieval_type=None, + observation_id=None, instrument_name=None, filename=None, verbose=False, @@ -39,24 +41,24 @@ def download_data(self, observation_id, *, retrieval_type=None, Parameters ---------- - observation_id : string, mandatory + observation_id : string, optional id of the observation to be downloaded The identifies of the observation we want to retrieve, 10 digits example: 1342195355 retrieval_type : string, optional, default 'OBSERVATION' The type of product that we want to retrieve values: OBSERVATION, PRODUCT, POSTCARD, POSTCARDFITS, REQUESTFILE_XML, STANDALONE, UPDP, HPDP - instrument_name : string, optinal, default 'PACS' + instrument_name : string, optional, default 'PACS' values: PACS, SPIRE, HIFI The instrument name, by default 'PACS' if the retrieval_type is 'OBSERVATION' - filename : string, optinal, default None + filename : string, optional, default None If the filename is not set it will use the observation_id as filename file name to be used to store the file - verbose : bool, optinal, default False + verbose : bool, optional, default False flag to display information about the process observation_oid : string, optional Observation internal identifies. This is the database identifier - istrument_oid : string, optional + instrument_oid : string, optional The database identifies of the instrument values: 1, 2, 3 product_level : string, optional @@ -73,11 +75,14 @@ def download_data(self, observation_id, *, retrieval_type=None, if retrieval_type is None: retrieval_type = "OBSERVATION" - params = {'retrieval_type': retrieval_type, - 'observation_id': observation_id} + params = {'retrieval_type': retrieval_type} + if observation_id is not None: + params['observation_id'] = observation_id if retrieval_type == "OBSERVATION" and instrument_name is None: instrument_name = "PACS" + + if instrument_name is not None: params['instrument_name'] = instrument_name link = self.data_url + "".join("&{0}={1}".format(key, val) @@ -98,13 +103,20 @@ def download_data(self, observation_id, *, retrieval_type=None, error = "Data protected by propietary rights. Please check your credentials" raise LoginError(error) + if filename is None: + if observation_id is not None: + filename = observation_id + else: + error = "Please set a filename for the output" + raise InputWarning(error) + r_filename = params["filename"] suffixes = Path(r_filename).suffixes - if filename is None: - filename = observation_id - - filename += "".join(suffixes) + if len(suffixes) > 1 and suffixes[len(suffixes) - 1] == ".jpg": + filename += suffixes[len(suffixes) - 1] + else: + filename += "".join(suffixes) self._download_file(link, filename, head_safe=True, cache=cache) @@ -128,10 +140,10 @@ def get_observation(self, observation_id, instrument_name, *, filename=None, instrument_name : string, mandatory The instrument name values: PACS, SPIRE, HIFI - filename : string, optinal, default None + filename : string, optional, default None If the filename is not set it will use the observation_id as filename file name to be used to store the file - verbose : bool, optinal, default 'False' + verbose : bool, optional, default 'False' flag to display information about the process observation_oid : string, optional Observation internal identifies. This is the database identifier @@ -201,10 +213,10 @@ def get_postcard(self, observation_id, instrument_name, *, filename=None, instrument_name : string, mandatory The instrument name values: PACS, SPIRE, HIFI - filename : string, optinal, default None + filename : string, optional, default None If the filename is not set it will use the observation_id as filename file name to be used to store the file - verbose : bool, optinal, default False + verbose : bool, optional, default False flag to display information about the process observation_oid : string, optional Observation internal identifies. This is the database identifier diff --git a/astroquery/hsa/tests/dummy_handler.py b/astroquery/hsa/tests/dummy_handler.py index 94048b9ba5..326e12b912 100644 --- a/astroquery/hsa/tests/dummy_handler.py +++ b/astroquery/hsa/tests/dummy_handler.py @@ -48,4 +48,3 @@ def check_parameters(self, parameters, method_name): raise ValueError("Parameter '%s' not found in method '%s'", (str(key), method_name)) return True - diff --git a/astroquery/hsa/tests/dummy_tap_handler.py b/astroquery/hsa/tests/dummy_tap_handler.py index 1d4f6e95b5..ea4ed46632 100644 --- a/astroquery/hsa/tests/dummy_tap_handler.py +++ b/astroquery/hsa/tests/dummy_tap_handler.py @@ -80,4 +80,3 @@ def load_tables(self, table = TapTableMeta() table.name = "table" return [table] - diff --git a/astroquery/hsa/tests/setup_package.py b/astroquery/hsa/tests/setup_package.py index f4dc7f93a3..cb3d19b4bc 100644 --- a/astroquery/hsa/tests/setup_package.py +++ b/astroquery/hsa/tests/setup_package.py @@ -14,4 +14,3 @@ def get_package_data(): # you can also enlist files individually by names # finally construct and return a dict for the sub module return {'astroquery.hsa.tests': paths} - diff --git a/astroquery/hsa/tests/test_hsa.py b/astroquery/hsa/tests/test_hsa.py index e115c932f2..f11ccc4d06 100644 --- a/astroquery/hsa/tests/test_hsa.py +++ b/astroquery/hsa/tests/test_hsa.py @@ -5,6 +5,7 @@ from ..tests.dummy_handler import DummyHandler from ..tests.dummy_tap_handler import DummyHSATapHandler + class TestHSA(): def get_dummy_tap_handler(self): diff --git a/astroquery/hsa/tests/test_hsa_remote.py b/astroquery/hsa/tests/test_hsa_remote.py index 4cc4d5b479..8967b4eb58 100644 --- a/astroquery/hsa/tests/test_hsa_remote.py +++ b/astroquery/hsa/tests/test_hsa_remote.py @@ -6,10 +6,11 @@ from ..core import HSAClass from .test_hsa import TestHSA + class TestHSARemote(TestHSA): @pytest.mark.remote_data - def test_download_data(self): + def test_download_data_observation_pacs(self): obs_id = "1342195355" parameters = {'retrieval_type': "OBSERVATION", 'observation_id': obs_id, @@ -21,6 +22,76 @@ def test_download_data(self): assert os.path.isfile(res) os.remove(res) + @pytest.mark.remote_data + def test_download_data_observation_pacs_filename(self): + obs_id = "1342195355" + fname = "output_file" + parameters = {'retrieval_type': "OBSERVATION", + 'observation_id': obs_id, + 'instrument_name': "PACS", + 'filename': fname} + expected_res = fname + ".tar" + hsa = HSAClass(self.get_dummy_tap_handler()) + res = hsa.download_data(**parameters) + assert res == expected_res + assert os.path.isfile(res) + os.remove(res) + + @pytest.mark.remote_data + def test_download_data_observation_pacs_compressed(self): + obs_id = "1342195355" + parameters = {'retrieval_type': "OBSERVATION", + 'observation_id': obs_id, + 'instrument_name': "PACS", + 'compress': 'true'} + expected_res = obs_id + ".tgz" + hsa = HSAClass(self.get_dummy_tap_handler()) + res = hsa.download_data(**parameters) + assert res == expected_res + assert os.path.isfile(res) + os.remove(res) + + @pytest.mark.remote_data + def test_download_data_observation_spire(self): + obs_id = "1342224960" + parameters = {'retrieval_type': "OBSERVATION", + 'observation_id': obs_id, + 'instrument_name': "SPIRE"} + expected_res = obs_id + ".tar" + hsa = HSAClass(self.get_dummy_tap_handler()) + res = hsa.download_data(**parameters) + assert res == expected_res + assert os.path.isfile(res) + os.remove(res) + + @pytest.mark.remote_data + def test_download_data_postcard_pacs(self): + obs_id = "1342195355" + parameters = {'retrieval_type': "POSTCARD", + 'observation_id': obs_id, + 'instrument_name': "PACS"} + expected_res = obs_id + ".jpg" + hsa = HSAClass(self.get_dummy_tap_handler()) + res = hsa.download_data(**parameters) + assert res == expected_res + assert os.path.isfile(res) + os.remove(res) + + @pytest.mark.remote_data + def test_download_data_postcard_pacs_filename(self): + obs_id = "1342195355" + fname = "output_file" + parameters = {'retrieval_type': "POSTCARD", + 'observation_id': obs_id, + 'instrument_name': "PACS", + 'filename': fname} + expected_res = fname + ".jpg" + hsa = HSAClass(self.get_dummy_tap_handler()) + res = hsa.download_data(**parameters) + assert res == expected_res + assert os.path.isfile(res) + os.remove(res) + @pytest.mark.remote_data def test_get_observation(self): obs_id = "1342195355" diff --git a/docs/hsa/hsa.rst b/docs/hsa/hsa.rst index da32473376..50803d3420 100644 --- a/docs/hsa/hsa.rst +++ b/docs/hsa/hsa.rst @@ -2,9 +2,9 @@ .. _astroquery.hsa: -********************** -hsa (`astroquery.hsa`) -********************** +******************************************* +Herschel Science Archive (`astroquery.hsa`) +******************************************* Herschel was the fourth cornerstone in ESA's Horizon 2000 science programme, designed to observe the 'cool' universe. It performed photometry and spectroscopy in the poorly explored 55-670 µm spectral range with a 3.5 m diameter @@ -12,7 +12,7 @@ Cassegrain telescope, providing unique observing capabilities and bridging the g missions and groundbased facilities. Herschel successfully performed ~37000 science observations and ~6600 science calibration observations which are publicly available to the worldwide astronomical community through the Herschel Science Archive. -This package allows the access to the `Herschel Science Archive `__. +This package allows the access to the `Herschel Science Archive `__. ======== Examples @@ -26,7 +26,7 @@ Examples >>> from astroquery.hsa import HSA >>> - >>> HSA.download_data('1342195355',retrieval_type='OBSERVATION', instrument_name='PACS') + >>> HSA.download_data(observation_id='1342195355',retrieval_type='OBSERVATION', instrument_name='PACS') Downloading URL http://archives.esac.esa.int/hsa/whsa-tap-server/data?&retrieval_type=OBSERVATION&observation_id=1342195355&instrument_name=PACS to 1342195355.tar ... [Done] '1342195355.tar' diff --git a/docs/index.rst b/docs/index.rst index 4f9be1ecf2..9cf9f100f5 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -200,6 +200,7 @@ The following modules have been completed using a common API: heasarc/heasarc.rst hips2fits/hips2fits.rst hitran/hitran.rst + hsa/hsa.rst ipac/irsa/irsa_dust/irsa_dust.rst ipac/irsa/ibe/ibe.rst ipac/irsa/irsa.rst From 47bf1210e956a50b4ec1d98023840b800a8506e7 Mon Sep 17 00:00:00 2001 From: lvalerom Date: Mon, 19 Jul 2021 20:38:18 +0200 Subject: [PATCH 06/22] minor change in CHANGES.rst --- CHANGES.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index b696d39f2d..5ac5e46e76 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -8,6 +8,11 @@ esa.hubble - Added new method ``get_hap_hst_link`` and ``get_member_observations`` to get related observations [#2268] +hsa +^^^ + +- New module to access ESA Herschel mission. [#2122] + Service fixes and enhancements ------------------------------ esa.hubble @@ -268,11 +273,6 @@ heasarc - Add alternative instance of HEASARC Server, maintained by INTEGRAL Science Data Center. [#1988] -hsa -^^^ - -- New module to access ESA Herschel mission. [#] - nasa_exoplanet_archive ^^^^^^^^^^^^^^^^^^^^^^ From b9a0b8dd2ceab50c0115f545d75629be29b923a7 Mon Sep 17 00:00:00 2001 From: lvalerom Date: Mon, 19 Jul 2021 20:50:09 +0200 Subject: [PATCH 07/22] added two line after HSAClass definition to complain with pep8 E305 --- astroquery/hsa/core.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/astroquery/hsa/core.py b/astroquery/hsa/core.py index 52b174cd5a..ca97114edb 100644 --- a/astroquery/hsa/core.py +++ b/astroquery/hsa/core.py @@ -351,4 +351,6 @@ def get_columns(self, table_name, *, only_names=True, verbose=False): return [c.name for c in columns] else: return columns + + HSA = HSAClass() From 4481c553b80a7de2d34f0f11580f14cfc5a2e6d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Valero=20Mart=C3=ADn?= Date: Thu, 6 Jan 2022 14:19:01 +0100 Subject: [PATCH 08/22] Update astroquery/hsa/core.py Co-authored-by: Adam Ginsburg --- astroquery/hsa/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astroquery/hsa/core.py b/astroquery/hsa/core.py index ca97114edb..5ad226303b 100644 --- a/astroquery/hsa/core.py +++ b/astroquery/hsa/core.py @@ -114,7 +114,7 @@ def download_data(self, *, retrieval_type=None, suffixes = Path(r_filename).suffixes if len(suffixes) > 1 and suffixes[len(suffixes) - 1] == ".jpg": - filename += suffixes[len(suffixes) - 1] + filename += suffixes[-1] else: filename += "".join(suffixes) From 23bbf482cac9055cb6aa8727c1e3b9679eae6ac6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Valero=20Mart=C3=ADn?= Date: Thu, 6 Jan 2022 14:19:12 +0100 Subject: [PATCH 09/22] Update astroquery/hsa/core.py Co-authored-by: Adam Ginsburg --- astroquery/hsa/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astroquery/hsa/core.py b/astroquery/hsa/core.py index 5ad226303b..06d607fc52 100644 --- a/astroquery/hsa/core.py +++ b/astroquery/hsa/core.py @@ -113,7 +113,7 @@ def download_data(self, *, retrieval_type=None, r_filename = params["filename"] suffixes = Path(r_filename).suffixes - if len(suffixes) > 1 and suffixes[len(suffixes) - 1] == ".jpg": + if len(suffixes) > 1 and suffixes[-1] == ".jpg": filename += suffixes[-1] else: filename += "".join(suffixes) From b1271bb21650dbf320fcd4018e5af60f24e12c1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Valero=20Mart=C3=ADn?= Date: Thu, 6 Jan 2022 14:19:19 +0100 Subject: [PATCH 10/22] Update astroquery/hsa/core.py Co-authored-by: Adam Ginsburg --- astroquery/hsa/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astroquery/hsa/core.py b/astroquery/hsa/core.py index 06d607fc52..b817bc273f 100644 --- a/astroquery/hsa/core.py +++ b/astroquery/hsa/core.py @@ -100,7 +100,7 @@ def download_data(self, *, retrieval_type=None, if 'Content-Type' in response.headers and 'text' not in response.headers['Content-Type']: _, params = cgi.parse_header(response.headers['Content-Disposition']) else: - error = "Data protected by propietary rights. Please check your credentials" + error = "Data protected by proprietary rights. Please check your credentials" raise LoginError(error) if filename is None: From cc9d950c877ad1a6fdd6e6815ba02f95682440c0 Mon Sep 17 00:00:00 2001 From: lvalerom Date: Thu, 6 Jan 2022 14:51:36 +0100 Subject: [PATCH 11/22] change InputWarning to ValueError --- astroquery/hsa/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astroquery/hsa/core.py b/astroquery/hsa/core.py index b817bc273f..9d02244933 100644 --- a/astroquery/hsa/core.py +++ b/astroquery/hsa/core.py @@ -108,7 +108,7 @@ def download_data(self, *, retrieval_type=None, filename = observation_id else: error = "Please set a filename for the output" - raise InputWarning(error) + raise ValueError(error) r_filename = params["filename"] suffixes = Path(r_filename).suffixes From 0c7bd91016ebca8522b2ab43f0e94b2d4c634b9b Mon Sep 17 00:00:00 2001 From: lvalerom Date: Sat, 15 Jan 2022 16:06:44 +0100 Subject: [PATCH 12/22] procedure example in documentation --- astroquery/hsa/core.py | 9 ++++++++- docs/hsa/hsa.rst | 44 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/astroquery/hsa/core.py b/astroquery/hsa/core.py index 9d02244933..f7ba0774cf 100644 --- a/astroquery/hsa/core.py +++ b/astroquery/hsa/core.py @@ -129,7 +129,14 @@ def get_observation(self, observation_id, instrument_name, *, filename=None, verbose=False, cache=True, **kwargs): """ - Download observation from Herschel + Download observation from Herschel. + This consists of a .tar file containing: + -The auxiliary directory: contains all Herschel non-science spacecraft data + -The calibarion directory: contains the uplink and downlink calibration products + - directory: contains the science data distributed in sub-directories called level0/0.5/1/2/2.5/3. + + More information can be found here: + https://www.cosmos.esa.int/web/herschel/data-products-overview Parameters ---------- diff --git a/docs/hsa/hsa.rst b/docs/hsa/hsa.rst index 50803d3420..4066c5a36c 100644 --- a/docs/hsa/hsa.rst +++ b/docs/hsa/hsa.rst @@ -73,7 +73,7 @@ For more details of the parameters check the section 6.2 of the 'Direct Product 'http://archives.esac.esa.int/hsa/whsa/' ------------------------------------------ -3. Getting Herschel metadata through TAP +4. Getting Herschel metadata through TAP ------------------------------------------ This function provides access to the Herschel Science Archive database using the Table Access Protocol (TAP) and via the Astronomical Data @@ -102,7 +102,7 @@ This will execute an ADQL query to download the first 10 observations in the Her stored in the file 'results.csv'. The result of this query can be printed by doing print(result). ----------------------------------- -4. Getting table details of HSA TAP +5. Getting table details of HSA TAP ----------------------------------- .. code-block:: python @@ -118,7 +118,7 @@ stored in the file 'results.csv'. The result of this query can be printed by doi This will show the available tables in HSA TAP service in the Herschel Science Archive. ------------------------------------- -5. Getting columns details of HSA TAP +6. Getting columns details of HSA TAP ------------------------------------- .. code-block:: python @@ -133,6 +133,44 @@ This will show the available tables in HSA TAP service in the Herschel Science A This will show the column details of the table 'hsa.v_active_observation' in HSA TAP service in the Herschel Science Archive. +------------------------------------- +7. Procedure example +------------------------------------- + +First retrieve the observation IDs based on a position on the sky. To achive this, query the TAP service. + +.. code-block:: python + + >>> from astroquery.hsa import HSA + >>> + >>> HSA.query_hsa_tap("select top 10 observation_id from hsa.v_active_observation where contains(point('ICRS', hsa.v_active_observation.ra, hsa.v_active_observation.dec),circle('ICRS', 100.2417,9.895, 1.1))=1", output_format='csv', output_file='results.cvs') + + observation_id + int64 + -------------- + 1342228342 + 1342228371 + 1342228372 + 1342219315 + 1342205057 + 1342205056 + 1342205058 + 1342205056 + 1342205057 + +In this example we are doing a circle search of 1.1 degrees in an ICRS (Right ascension [RA], Declination [DEC]) position (100.2417, 9.895). + +For more information on how to use ADQL see: + 'https://www.ivoa.net/documents/latest/ADQL.html' + +After obtaining the desire ID, download the product of the observation '1342205057' with the instrument 'PACS'. + +.. code-block:: python + + >>> HSA.download_data(observation_id='1342205057', retrieval_type='OBSERVATION', instrument_name='PACS') + Downloading URL http://archives.esac.esa.int/hsa/whsa-tap-server/data?&retrieval_type=OBSERVATION&observation_id=1342205057&instrument_name=PACS to 1342205057.tar ... [Done] + '1342205057.tar' + Reference/API ============= From 753dbc817a33a3d92adfa516e9a1d74e27497f21 Mon Sep 17 00:00:00 2001 From: lvalerom Date: Sat, 5 Feb 2022 23:42:05 +0100 Subject: [PATCH 13/22] addressing review --- astroquery/hsa/core.py | 7 ++++--- docs/hsa/hsa.rst | 39 ++++++++++++++++++++++++--------------- 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/astroquery/hsa/core.py b/astroquery/hsa/core.py index f7ba0774cf..323c895c84 100644 --- a/astroquery/hsa/core.py +++ b/astroquery/hsa/core.py @@ -131,9 +131,10 @@ def get_observation(self, observation_id, instrument_name, *, filename=None, """ Download observation from Herschel. This consists of a .tar file containing: - -The auxiliary directory: contains all Herschel non-science spacecraft data - -The calibarion directory: contains the uplink and downlink calibration products - - directory: contains the science data distributed in sub-directories called level0/0.5/1/2/2.5/3. + + - The auxiliary directory: contains all Herschel non-science spacecraft data + - The calibarion directory: contains the uplink and downlink calibration products + - directory: contains the science data distributed in sub-directories called level0/0.5/1/2/2.5/3. More information can be found here: https://www.cosmos.esa.int/web/herschel/data-products-overview diff --git a/docs/hsa/hsa.rst b/docs/hsa/hsa.rst index 4066c5a36c..4181adf576 100644 --- a/docs/hsa/hsa.rst +++ b/docs/hsa/hsa.rst @@ -6,10 +6,10 @@ Herschel Science Archive (`astroquery.hsa`) ******************************************* -Herschel was the fourth cornerstone in ESA's Horizon 2000 science programme, designed to observe the 'cool' universe. -It performed photometry and spectroscopy in the poorly explored 55-670 µm spectral range with a 3.5 m diameter -Cassegrain telescope, providing unique observing capabilities and bridging the gap between earlier infrared space -missions and groundbased facilities. Herschel successfully performed ~37000 science observations and ~6600 science +`Herschel `__ was the fourth cornerstone in ESA's Horizon 2000 science programme, designed to observe the 'cool' universe. +It performed photometry and spectroscopy in the poorly explored 55-670 µm spectral range with a 3.5 m diameter +Cassegrain telescope, providing unique observing capabilities and bridging the gap between earlier infrared space +missions and groundbased facilities. Herschel successfully performed ~37000 science observations and ~6600 science calibration observations which are publicly available to the worldwide astronomical community through the Herschel Science Archive. This package allows the access to the `Herschel Science Archive `__. @@ -24,33 +24,42 @@ Examples .. code-block:: python - >>> from astroquery.hsa import HSA + >>> from astroquery.hsa import HSA >>> >>> HSA.download_data(observation_id='1342195355',retrieval_type='OBSERVATION', instrument_name='PACS') Downloading URL http://archives.esac.esa.int/hsa/whsa-tap-server/data?&retrieval_type=OBSERVATION&observation_id=1342195355&instrument_name=PACS to 1342195355.tar ... [Done] '1342195355.tar' -This will download the product of the observation '1342195355' with the instrument 'PACS' and +This will download the product of the observation '1342195355' with the instrument 'PACS' and it will store them in a tar called '1342195355.tar'. The parameters available are detailed in the API. For more details of the parameters check the section 6 of the 'Direct Product Access using TAP' in the 'HSA users guide' at: 'http://archives.esac.esa.int/hsa/whsa/' +For more details about the products check: + 'https://www.cosmos.esa.int/web/herschel/data-products-overview' + ------------------------------- -2. Getting Observation Product +2. Getting Observation Products ------------------------------- .. code-block:: python - >>> from astroquery.hsa import HSA + >>> from astroquery.hsa import HSA >>> >>> HSA.get_observation('1342195355', instrument_name='PACS') Downloading URL http://archives.esac.esa.int/hsa/whsa-tap-server/data?&retrieval_type=OBSERVATION&observation_id=1342195355&instrument_name=PACS to 1342195355.tar ... [Done] '1342195355.tar' -This will download the product of the observation '1342195355' with the instrument 'PACS' and +This will download the product of the observation '1342195355' with the instrument 'PACS' and it will store them in a tar called '1342195355.tar'. The parameters available are detailed in the API. +`Notice`: There is no difference between the product retrieved with this method and `download_data`. `download_data` is a more generic +interface that allows the user to retrieve any product or metadata and `get_observation` allows the user to retrieve only observation products. + +For more information check the section 6.1 of the of the 'direct Product Access using TAP' in the 'HSA users guide' at: + 'http://archives.esac.esa.int/hsa/whsa/' + For more details of the parameters check the section 6.2 of the 'Direct Product Access using TAP' in the 'HSA users guide' at: 'http://archives.esac.esa.int/hsa/whsa/' @@ -60,13 +69,13 @@ For more details of the parameters check the section 6.2 of the 'Direct Product .. code-block:: python - >>> from astroquery.hsa import HSA + >>> from astroquery.hsa import HSA >>> >>> HSA.get_postcard('1342195355', instrument_name='PACS') Downloading URL http://archives.esac.esa.int/hsa/whsa-tap-server/data?&retrieval_type=POSTCARD&observation_id=1342195355&instrument_name=PACS to /home/dev/.astropy/cache/astroquery/HSA/data?&retrieval_type=POSTCARD&observation_id=1342195355&instrument_name=PACS ... [Done] '1342195355.jpg' -This will download the postcard of the observation '1342195355' with the instrument 'PACS' and +This will download the postcard (static representation in JPG-format of the final product) of the observation '1342195355' with the instrument 'PACS' and it will store them in a tar called '1342195355.jpg'. The parameters available are detailed in the API. For more details of the parameters check the section 6.2 of the 'Direct Product Access using TAP' in the 'HSA users guide' at: @@ -77,7 +86,7 @@ For more details of the parameters check the section 6.2 of the 'Direct Product ------------------------------------------ This function provides access to the Herschel Science Archive database using the Table Access Protocol (TAP) and via the Astronomical Data -Query Language (ADQL). +Query Language (`ADQL `__). .. code-block:: python @@ -99,7 +108,7 @@ Query Language (ADQL). PPhot.AM06-perp -74.22638888888889 -74.22638888888889 1759000.0 ... FAILED AM0644-741 925338 This will execute an ADQL query to download the first 10 observations in the Herschel Science Archive. The result of the query will be -stored in the file 'results.csv'. The result of this query can be printed by doing print(result). +stored in the file 'results.csv'. The result of this query can be printed by doing `print(result)`. ----------------------------------- 5. Getting table details of HSA TAP @@ -146,7 +155,7 @@ First retrieve the observation IDs based on a position on the sky. To achive thi >>> HSA.query_hsa_tap("select top 10 observation_id from hsa.v_active_observation where contains(point('ICRS', hsa.v_active_observation.ra, hsa.v_active_observation.dec),circle('ICRS', 100.2417,9.895, 1.1))=1", output_format='csv', output_file='results.cvs')
observation_id - int64 + int64 -------------- 1342228342 1342228371 @@ -162,7 +171,7 @@ In this example we are doing a circle search of 1.1 degrees in an ICRS (Right as For more information on how to use ADQL see: 'https://www.ivoa.net/documents/latest/ADQL.html' - + After obtaining the desire ID, download the product of the observation '1342205057' with the instrument 'PACS'. .. code-block:: python From b3c3b034f9a6e24de3937aa9f1840852133a6289 Mon Sep 17 00:00:00 2001 From: lvalerom Date: Sat, 2 Apr 2022 21:46:52 +0200 Subject: [PATCH 14/22] addressing review --- astroquery/{ => esa}/hsa/__init__.py | 0 astroquery/{ => esa}/hsa/core.py | 95 +++++++---- astroquery/{ => esa}/hsa/tests/__init__.py | 0 .../{ => esa}/hsa/tests/dummy_handler.py | 0 .../{ => esa}/hsa/tests/dummy_tap_handler.py | 4 +- .../{ => esa}/hsa/tests/setup_package.py | 2 +- astroquery/{ => esa}/hsa/tests/test_hsa.py | 11 ++ astroquery/esa/hsa/tests/test_hsa_remote.py | 149 ++++++++++++++++++ astroquery/hsa/tests/test_hsa_remote.py | 117 -------------- docs/{hsa => esa}/hsa.rst | 62 +++++--- docs/index.rst | 2 +- 11 files changed, 267 insertions(+), 175 deletions(-) rename astroquery/{ => esa}/hsa/__init__.py (100%) rename astroquery/{ => esa}/hsa/core.py (81%) rename astroquery/{ => esa}/hsa/tests/__init__.py (100%) rename astroquery/{ => esa}/hsa/tests/dummy_handler.py (100%) rename astroquery/{ => esa}/hsa/tests/dummy_tap_handler.py (97%) rename astroquery/{ => esa}/hsa/tests/setup_package.py (92%) rename astroquery/{ => esa}/hsa/tests/test_hsa.py (79%) create mode 100644 astroquery/esa/hsa/tests/test_hsa_remote.py delete mode 100644 astroquery/hsa/tests/test_hsa_remote.py rename docs/{hsa => esa}/hsa.rst (90%) diff --git a/astroquery/hsa/__init__.py b/astroquery/esa/hsa/__init__.py similarity index 100% rename from astroquery/hsa/__init__.py rename to astroquery/esa/hsa/__init__.py diff --git a/astroquery/hsa/core.py b/astroquery/esa/hsa/core.py similarity index 81% rename from astroquery/hsa/core.py rename to astroquery/esa/hsa/core.py index 323c895c84..f383405855 100644 --- a/astroquery/hsa/core.py +++ b/astroquery/esa/hsa/core.py @@ -6,6 +6,8 @@ import shutil from pathlib import Path +from astropy import units as u +from ...utils import commons from astroquery import log from astroquery.exceptions import LoginError, InputWarning from astroquery.query import BaseQuery @@ -29,13 +31,9 @@ def __init__(self, tap_handler=None): else: self._tap = tap_handler - def download_data(self, *, retrieval_type=None, - observation_id=None, - instrument_name=None, - filename=None, - verbose=False, - cache=True, - **kwargs): + def download_data(self, *, retrieval_type="OBSERVATION", observation_id=None, + instrument_name=None, filename=None, verbose=False, + download_dir="", cache=True, **kwargs): """ Download data from Herschel @@ -64,6 +62,8 @@ def download_data(self, *, retrieval_type=None, product_level : string, optional level to download values: ALL, AUXILIARY, CALIBRATION, LEVEL0, LEVEL0_5, LEVEL1, LEVEL2, LEVEL2_5, LEVEL3, ALL-LEVEL3 + download_dir : string, optional + The directory in which the file will be downloaded Returns ------- @@ -72,9 +72,6 @@ def download_data(self, *, retrieval_type=None, if filename is not None: filename = os.path.splitext(filename)[0] - if retrieval_type is None: - retrieval_type = "OBSERVATION" - params = {'retrieval_type': retrieval_type} if observation_id is not None: params['observation_id'] = observation_id @@ -85,11 +82,9 @@ def download_data(self, *, retrieval_type=None, if instrument_name is not None: params['instrument_name'] = instrument_name - link = self.data_url + "".join("&{0}={1}".format(key, val) - for key, val in params.items()) + link = self.data_url + "".join("&{0}={1}".format(key, val) for key, val in params.items()) - link += "".join("&{0}={1}".format(key, val) - for key, val in kwargs.items()) + link += "".join("&{0}={1}".format(key, val) for key, val in kwargs.items()) if verbose: log.info(link) @@ -107,7 +102,7 @@ def download_data(self, *, retrieval_type=None, if observation_id is not None: filename = observation_id else: - error = "Please set a filename for the output" + error = "Please set either 'obervation_id' or 'filename' for the output" raise ValueError(error) r_filename = params["filename"] @@ -118,6 +113,8 @@ def download_data(self, *, retrieval_type=None, else: filename += "".join(suffixes) + filename = os.path.join(download_dir, filename) + self._download_file(link, filename, head_safe=True, cache=cache) if verbose: @@ -126,8 +123,7 @@ def download_data(self, *, retrieval_type=None, return filename def get_observation(self, observation_id, instrument_name, *, filename=None, - verbose=False, - cache=True, **kwargs): + verbose=False, download_dir="", cache=True, **kwargs): """ Download observation from Herschel. This consists of a .tar file containing: @@ -141,11 +137,11 @@ def get_observation(self, observation_id, instrument_name, *, filename=None, Parameters ---------- - observation_id : string, mandatory + observation_id : string id of the observation to be downloaded The identifies of the observation we want to retrieve, 10 digits example: 1342195355 - instrument_name : string, mandatory + instrument_name : string The instrument name values: PACS, SPIRE, HIFI filename : string, optional, default None @@ -161,6 +157,8 @@ def get_observation(self, observation_id, instrument_name, *, filename=None, product_level : string, optional level to download values: ALL, AUXILIARY, CALIBRATION, LEVEL0, LEVEL0_5, LEVEL1, LEVEL2, LEVEL2_5, LEVEL3, ALL-LEVEL3 + download_dir : string, optional + The directory in which the file will be downloaded Returns ------- @@ -173,11 +171,9 @@ def get_observation(self, observation_id, instrument_name, *, filename=None, 'observation_id': observation_id, 'instrument_name': instrument_name} - link = self.data_url + "".join("&{0}={1}".format(key, val) - for key, val in params.items()) + link = self.data_url + "".join("&{0}={1}".format(key, val) for key, val in params.items()) - link += "".join("&{0}={1}".format(key, val) - for key, val in kwargs.items()) + link += "".join("&{0}={1}".format(key, val) for key, val in kwargs.items()) if verbose: log.info(link) @@ -199,6 +195,8 @@ def get_observation(self, observation_id, instrument_name, *, filename=None, filename += "".join(suffixes) + filename = os.path.join(download_dir, filename) + self._download_file(link, filename, head_safe=True, cache=cache) if verbose: @@ -207,18 +205,17 @@ def get_observation(self, observation_id, instrument_name, *, filename=None, return filename def get_postcard(self, observation_id, instrument_name, *, filename=None, - verbose=False, - cache=True, **kwargs): + verbose=False, download_dir="", cache=True, **kwargs): """ Download postcard from Herschel Parameters ---------- - observation_id : string, mandatory + observation_id : string id of the observation to be downloaded The identifies of the observation we want to retrieve, 10 digits example: 1342195355 - instrument_name : string, mandatory + instrument_name : string The instrument name values: PACS, SPIRE, HIFI filename : string, optional, default None @@ -237,6 +234,8 @@ def get_postcard(self, observation_id, instrument_name, *, filename=None, postcard_single : string, optional 'true' to retrieve one single postcard (main one) values: true, false + download_dir : string, optional + The directory in which the file will be downloaded Returns ------- @@ -249,11 +248,9 @@ def get_postcard(self, observation_id, instrument_name, *, filename=None, 'observation_id': observation_id, 'instrument_name': instrument_name} - link = self.data_url + "".join("&{0}={1}".format(key, val) - for key, val in params.items()) + link = self.data_url + "".join("&{0}={1}".format(key, val) for key, val in params.items()) - link += "".join("&{0}={1}".format(key, val) - for key, val in kwargs.items()) + link += "".join("&{0}={1}".format(key, val) for key, val in kwargs.items()) if verbose: log.info(link) @@ -270,6 +267,8 @@ def get_postcard(self, observation_id, instrument_name, *, filename=None, filename += ext + filename = os.path.join(download_dir, filename) + shutil.move(local_filepath, filename) if verbose: @@ -284,7 +283,7 @@ def query_hsa_tap(self, query, *, output_file=None, Parameters ---------- - query : string, mandatory + query : string query (adql) to be executed output_file : string, optional, default None file name where the results are saved if dumpToFile is True. @@ -332,13 +331,13 @@ def get_columns(self, table_name, *, only_names=True, verbose=False): Parameters ---------- - table_name : string, mandatory + table_name : string table name of which, columns will be returned only_names : bool, optional, default True True to load column names only verbose : bool, optional, default False - flag to display information about the process + Returns ------- A list of columns @@ -360,5 +359,33 @@ def get_columns(self, table_name, *, only_names=True, verbose=False): else: return columns + def query_observations(self, coordinate, radius, n_obs=10): + """ + Get the observation IDs from a given region + + Parameters + ---------- + coordinate : string / `astropy.coordinates` + the identifier or coordinates around which to query + radius : int / `~astropy.units.Quantity` + the radius of the region + n_obs : int, optional + the number of observations + + Returns + ------- + A table object with the list of observations in the region + """ + r = radius + if not isinstance(radius, u.Quantity): + r = radius*u.deg + coord = commons.parse_coordinates(coordinate).icrs + + query = ("select top {} observation_id from hsa.v_active_observation " + "where contains(point('ICRS', hsa.v_active_observation.ra, " + "hsa.v_active_observation.dec), circle('ICRS', {},{},{}))=1")\ + .format(n_obs, coord.ra.degree, coord.dec.degree, r.to(u.deg).value) + return self.query_hsa_tap(query) + HSA = HSAClass() diff --git a/astroquery/hsa/tests/__init__.py b/astroquery/esa/hsa/tests/__init__.py similarity index 100% rename from astroquery/hsa/tests/__init__.py rename to astroquery/esa/hsa/tests/__init__.py diff --git a/astroquery/hsa/tests/dummy_handler.py b/astroquery/esa/hsa/tests/dummy_handler.py similarity index 100% rename from astroquery/hsa/tests/dummy_handler.py rename to astroquery/esa/hsa/tests/dummy_handler.py diff --git a/astroquery/hsa/tests/dummy_tap_handler.py b/astroquery/esa/hsa/tests/dummy_tap_handler.py similarity index 97% rename from astroquery/hsa/tests/dummy_tap_handler.py rename to astroquery/esa/hsa/tests/dummy_tap_handler.py index ea4ed46632..20c7d53d25 100644 --- a/astroquery/hsa/tests/dummy_tap_handler.py +++ b/astroquery/esa/hsa/tests/dummy_tap_handler.py @@ -1,7 +1,7 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst -from ...utils.tap.model.taptable import TapTableMeta -from ...utils.tap.model.job import Job +from ....utils.tap.model.taptable import TapTableMeta +from ....utils.tap.model.job import Job class DummyHSATapHandler(object): diff --git a/astroquery/hsa/tests/setup_package.py b/astroquery/esa/hsa/tests/setup_package.py similarity index 92% rename from astroquery/hsa/tests/setup_package.py rename to astroquery/esa/hsa/tests/setup_package.py index cb3d19b4bc..0cb4513e22 100644 --- a/astroquery/hsa/tests/setup_package.py +++ b/astroquery/esa/hsa/tests/setup_package.py @@ -1,6 +1,6 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst -from __future__ import absolute_import +#from __future__ import absolute_import import os diff --git a/astroquery/hsa/tests/test_hsa.py b/astroquery/esa/hsa/tests/test_hsa.py similarity index 79% rename from astroquery/hsa/tests/test_hsa.py rename to astroquery/esa/hsa/tests/test_hsa.py index f11ccc4d06..81a4f7facf 100644 --- a/astroquery/hsa/tests/test_hsa.py +++ b/astroquery/esa/hsa/tests/test_hsa.py @@ -1,6 +1,8 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst import pytest +from astropy import units as u +from astropy.coordinates import SkyCoord from ..core import HSAClass from ..tests.dummy_handler import DummyHandler from ..tests.dummy_tap_handler import DummyHSATapHandler @@ -46,3 +48,12 @@ def test_get_columns(self): hsa = HSAClass(self.get_dummy_tap_handler()) hsa.get_columns(**parameters) dummyTapHandler.check_call("get_columns", parameters) + + def test_query_observations(self): + c = SkyCoord(ra=100.2417*u.degree, dec=9.895*u.degree, frame='icrs') + parameters = {'coordinate': c, + 'radius': 0.5} + dummyTapHandler = DummyHSATapHandler("query_observations",parameters) + hsa = HSAClass(self.get_dummy_tap_handler()) + hsa.query_observations(**parameters) + dummyTapHandler.check_call("query_observations", parameters) diff --git a/astroquery/esa/hsa/tests/test_hsa_remote.py b/astroquery/esa/hsa/tests/test_hsa_remote.py new file mode 100644 index 0000000000..8a91b847ed --- /dev/null +++ b/astroquery/esa/hsa/tests/test_hsa_remote.py @@ -0,0 +1,149 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +import tempfile +import pytest + +import os,tarfile + +from ..core import HSAClass + +spire_chksum = [10243, 10772, 9029, 11095, 3875, 3893, 3903, 3868, 3888, 3951, 3905, 3875, 3887, 3908, 3871, 3892, 3894, 3900, 3870, 3892, 3938, 3904, 3893, 3866, 3893, 3930, 3901, 3867, 3893, 3927, 3903, 3868, 3892, 3926, 3904, 3869, 3889, 3937, 3899, 3868, 3887, 3928, 3912, 3865, 3887, 3937, 3899, 3865, 3891, 3925, 3911, 3868, 3883, 3937, 3902, 3870, 3893, 3922, 3914, 3869, 3886, 3924, 3903, 3870, 3889, 3922, 3903, 3867, 3889, 3932, 3902, 3869, 3892, 3930, 3900, 3868, 3892, 3916, 3910, 3868, 3889, 3932, 3907, 3869, 3892, 3914, 3900, 3871, 3884, 3918, 3903, 3894, 3874, 3892, 3906, 3872, 3889, 3895, 9724, 10495, 9293, 11299, 12132, 10602, 11283, 9582, 10568, 11354, 11228, 11812, 11006, 11224, 11382, 10232, 11049, 11011, 3862, 9671, 10844, 10899, 3946, 3941, 3953, 3954, 3948, 3940, 3954, 3948, 3945, 9286, 11197, 3902, 10137, 11664, 12038, 3963, 3948, 11970, 3867, 11452, 3908, 3908, 3894, 3908, 3908, 3894, 3885, 3914, 3876, 11820, 3958, 3905, 3963, 3925, 3974, 3923, 12241, 3897, 3900, 12328, 11911, 3893, 3885, 3895, 3895, 3891, 11970, 12252, 3899, 12503, 12135, 11826, 3899, 3899, 3912, 3912, 11530, 11625, 11850, 10981, 3872, 3890, 3896, 3895, 3867, 3891, 10561, 10991, 11209, 3853, 3871, 3869, 11372, 10826, 10995, 3887, 3882, 3883, 3879, 3884, 3883, 3889, 3883, 3887, 3876, 3885, 3883, 3882, 3888, 3886, 3882, 3882, 3882, 3887, 3885, 3885, 3884, 3880, 3881, 3878, 3882, 3884, 3879, 3887, 3882, 3884, 3877, 3883, 3882, 3881, 3882, 3876, 3879, 3881, 3881, 3884, 3882, 3882, 3882, 3888, 3883, 3880, 3880, 3875, 3880, 3884, 3883, 3887, 3880, 3879, 3874, 3882, 3879, 3887, 3876, 3888, 3888, 3885, 3885, 3876, 3880, 3887, 3877, 3884, 3886, 3879, 3879, 3877, 3880, 3876, 3889, 3885, 3886, 3885, 3879, 3879, 12424, 10112, 10885, 3895, 3922, 3888, 3900, 3898, 3919, 3925, 3930, 3923, 3932, 3931, 3917, 3922, 3917, 3919, 3935, 3929, 3923, 3929, 3924, 3906, 3918, 3895, 3892, 11322, 10879, 3944, 11338, 3921, 11009, 10969, 11352, 10984, 3944, 11345, 11333, 11088, 11331, 11099, 11324, 11118, 10591, 11502, 6291] + +pacs_chksum = [10205, 10752, 8914, 3869, 9736, 3925, 3942, 11201, 10454, 11393, 10520, 11080, 3979, 11121, 3961, 9745, 10492, 9317, 11304, 12025, 10584, 11295, 9564, 10585, 11367, 11229, 11808, 11039, 11225, 11411, 10246, 11066, 11013, 10158, 3939, 3927, 3941, 3936, 3942, 3925, 9691, 10381, 10944, 3990, 10970, 3969, 9453, 10802, 3881, 11926, 3896, 12528, 3908, 11898, 3883, 3894, 3878, 11231, 3910, 3885, 3876, 3913, 3967, 3970, 3908, 3903, 3901, 3958, 3898, 3870, 3915, 3878, 3895, 3901, 3888, 3880, 3922, 3881, 3926, 3883, 12229, 3958, 3949, 3936, 3962, 3949, 3947, 3868, 13210, 3885, 3901, 11463, 3939, 3967, 3878, 3900, 3911, 3947, 3917, 3940, 3953, 3904, 3869, 3874, 3880, 3893, 3964, 3980, 3889, 3871, 3908, 3882, 3888, 3895, 3905, 3889, 13163, 3879, 11036, 12469, 3886, 3887, 3863, 3921, 3881, 3911, 3878, 3921, 11149, 13032, 3880, 3881, 3883, 3883, 3895, 3899, 3919, 12498, 3896, 3942, 3894, 11407, 3866, 11716, 3889, 3890, 3888, 3873, 9870, 10738, 3978, 10759, 3949, 10720, 3921, 10390, 3938, 10365, 11182, 3905, 10724, 3943, 10742, 3942, 10025, 3924, 3935, 10388, 11185, 11206, 6288] + +@pytest.mark.remote_data +class TestHSARemote: + tmp_dir = tempfile.TemporaryDirectory() + + def test_download_data_observation_pacs(self): + #obs_id = "1342195355" + obs_id = "1342191813" + parameters = {'retrieval_type': "OBSERVATION", + 'observation_id': obs_id, + 'instrument_name': "PACS", + 'download_dir': self.tmp_dir.name} + expected_res = os.path.join(self.tmp_dir.name, obs_id + ".tar") + hsa = HSAClass() + res = hsa.download_data(**parameters) + assert res == expected_res + assert os.path.isfile(res) + chksum = [] + tar = tarfile.open(res) + for m in tar.getmembers(): + chksum.append(m.chksum) + assert chksum.sort() == pacs_chksum.sort() + os.remove(res) + + def test_download_data_observation_pacs_filename(self): + obs_id = "1342191813" + fname = "output_file" + parameters = {'retrieval_type': "OBSERVATION", + 'observation_id': obs_id, + 'instrument_name': "PACS", + 'filename': fname, + 'download_dir': self.tmp_dir.name} + expected_res = os.path.join(self.tmp_dir.name, fname + ".tar") + hsa = HSAClass() + res = hsa.download_data(**parameters) + assert res == expected_res + assert os.path.isfile(res) + chksum = [] + tar = tarfile.open(res) + for m in tar.getmembers(): + chksum.append(m.chksum) + assert chksum.sort() == pacs_chksum.sort() + os.remove(res) + + def test_download_data_observation_pacs_compressed(self): + obs_id = "1342191813" + parameters = {'retrieval_type': "OBSERVATION", + 'observation_id': obs_id, + 'instrument_name': "PACS", + 'compress': 'true', + 'download_dir': self.tmp_dir.name} + expected_res = os.path.join(self.tmp_dir.name, obs_id + ".tgz") + hsa = HSAClass() + res = hsa.download_data(**parameters) + assert res == expected_res + assert os.path.isfile(res) + chksum = [] + tar = tarfile.open(res) + for m in tar.getmembers(): + chksum.append(m.chksum) + assert chksum.sort() == pacs_chksum.sort() + os.remove(res) + + def test_download_data_observation_spire(self): + #obs_id = "1342224960" + obs_id = "1342191188" + parameters = {'retrieval_type': "OBSERVATION", + 'observation_id': obs_id, + 'instrument_name': "SPIRE", + 'download_dir': self.tmp_dir.name} + expected_res = os.path.join(self.tmp_dir.name, obs_id + ".tar") + hsa = HSAClass() + res = hsa.download_data(**parameters) + assert res == expected_res + assert os.path.isfile(res) + chksum = [] + tar = tarfile.open(res) + for m in tar.getmembers(): + chksum.append(m.chksum) + assert chksum.sort() == spire_chksum.sort() + os.remove(res) + + def test_download_data_postcard_pacs(self): + obs_id = "1342191813" + parameters = {'retrieval_type': "POSTCARD", + 'observation_id': obs_id, + 'instrument_name': "PACS", + 'download_dir': self.tmp_dir.name} + expected_res = os.path.join(self.tmp_dir.name, obs_id + ".jpg") + hsa = HSAClass() + res = hsa.download_data(**parameters) + assert res == expected_res + assert os.path.isfile(res) + os.remove(res) + + def test_download_data_postcard_pacs_filename(self): + obs_id = "1342191813" + fname = "output_file" + parameters = {'retrieval_type': "POSTCARD", + 'observation_id': obs_id, + 'instrument_name': "PACS", + 'filename': fname, + 'download_dir': self.tmp_dir.name} + expected_res = os.path.join(self.tmp_dir.name, fname + ".jpg") + hsa = HSAClass() + res = hsa.download_data(**parameters) + assert res == expected_res + assert os.path.isfile(res) + os.remove(res) + + def test_get_observation(self): + obs_id = "1342191813" + parameters = {'observation_id': obs_id, + 'instrument_name': "PACS", + 'download_dir': self.tmp_dir.name} + expected_res = os.path.join(self.tmp_dir.name, obs_id + ".tar") + hsa = HSAClass() + res = hsa.get_observation(**parameters) + assert res == expected_res + assert os.path.isfile(res) + chksum = [] + tar = tarfile.open(res) + for m in tar.getmembers(): + chksum.append(m.chksum) + assert chksum.sort() == pacs_chksum.sort() + os.remove(res) + + def test_get_postcard(self): + obs_id = "1342191813" + parameters = {'observation_id': obs_id, + 'instrument_name': "PACS", + 'download_dir': self.tmp_dir.name} + expected_res = os.path.join(self.tmp_dir.name, obs_id + ".jpg") + hsa = HSAClass() + res = hsa.get_postcard(**parameters) + assert res == expected_res + assert os.path.isfile(res) + os.remove(res) diff --git a/astroquery/hsa/tests/test_hsa_remote.py b/astroquery/hsa/tests/test_hsa_remote.py deleted file mode 100644 index 8967b4eb58..0000000000 --- a/astroquery/hsa/tests/test_hsa_remote.py +++ /dev/null @@ -1,117 +0,0 @@ -# Licensed under a 3-clause BSD style license - see LICENSE.rst -import pytest - -import os - -from ..core import HSAClass -from .test_hsa import TestHSA - - -class TestHSARemote(TestHSA): - - @pytest.mark.remote_data - def test_download_data_observation_pacs(self): - obs_id = "1342195355" - parameters = {'retrieval_type': "OBSERVATION", - 'observation_id': obs_id, - 'instrument_name': "PACS"} - expected_res = obs_id + ".tar" - hsa = HSAClass(self.get_dummy_tap_handler()) - res = hsa.download_data(**parameters) - assert res == expected_res - assert os.path.isfile(res) - os.remove(res) - - @pytest.mark.remote_data - def test_download_data_observation_pacs_filename(self): - obs_id = "1342195355" - fname = "output_file" - parameters = {'retrieval_type': "OBSERVATION", - 'observation_id': obs_id, - 'instrument_name': "PACS", - 'filename': fname} - expected_res = fname + ".tar" - hsa = HSAClass(self.get_dummy_tap_handler()) - res = hsa.download_data(**parameters) - assert res == expected_res - assert os.path.isfile(res) - os.remove(res) - - @pytest.mark.remote_data - def test_download_data_observation_pacs_compressed(self): - obs_id = "1342195355" - parameters = {'retrieval_type': "OBSERVATION", - 'observation_id': obs_id, - 'instrument_name': "PACS", - 'compress': 'true'} - expected_res = obs_id + ".tgz" - hsa = HSAClass(self.get_dummy_tap_handler()) - res = hsa.download_data(**parameters) - assert res == expected_res - assert os.path.isfile(res) - os.remove(res) - - @pytest.mark.remote_data - def test_download_data_observation_spire(self): - obs_id = "1342224960" - parameters = {'retrieval_type': "OBSERVATION", - 'observation_id': obs_id, - 'instrument_name': "SPIRE"} - expected_res = obs_id + ".tar" - hsa = HSAClass(self.get_dummy_tap_handler()) - res = hsa.download_data(**parameters) - assert res == expected_res - assert os.path.isfile(res) - os.remove(res) - - @pytest.mark.remote_data - def test_download_data_postcard_pacs(self): - obs_id = "1342195355" - parameters = {'retrieval_type': "POSTCARD", - 'observation_id': obs_id, - 'instrument_name': "PACS"} - expected_res = obs_id + ".jpg" - hsa = HSAClass(self.get_dummy_tap_handler()) - res = hsa.download_data(**parameters) - assert res == expected_res - assert os.path.isfile(res) - os.remove(res) - - @pytest.mark.remote_data - def test_download_data_postcard_pacs_filename(self): - obs_id = "1342195355" - fname = "output_file" - parameters = {'retrieval_type': "POSTCARD", - 'observation_id': obs_id, - 'instrument_name': "PACS", - 'filename': fname} - expected_res = fname + ".jpg" - hsa = HSAClass(self.get_dummy_tap_handler()) - res = hsa.download_data(**parameters) - assert res == expected_res - assert os.path.isfile(res) - os.remove(res) - - @pytest.mark.remote_data - def test_get_observation(self): - obs_id = "1342195355" - parameters = {'observation_id': obs_id, - 'instrument_name': "PACS"} - expected_res = obs_id + ".tar" - hsa = HSAClass(self.get_dummy_tap_handler()) - res = hsa.get_observation(**parameters) - assert res == expected_res - assert os.path.isfile(res) - os.remove(res) - - @pytest.mark.remote_data - def test_get_postcard(self): - obs_id = "1342195355" - parameters = {'observation_id': obs_id, - 'instrument_name': "PACS"} - expected_res = obs_id + ".jpg" - hsa = HSAClass(self.get_dummy_tap_handler()) - res = hsa.get_postcard(**parameters) - assert res == expected_res - assert os.path.isfile(res) - os.remove(res) diff --git a/docs/hsa/hsa.rst b/docs/esa/hsa.rst similarity index 90% rename from docs/hsa/hsa.rst rename to docs/esa/hsa.rst index 4181adf576..05613abf61 100644 --- a/docs/hsa/hsa.rst +++ b/docs/esa/hsa.rst @@ -1,10 +1,8 @@ -.. doctest-skip-all +.. _astroquery.esa.hsa: -.. _astroquery.hsa: - -******************************************* -Herschel Science Archive (`astroquery.hsa`) -******************************************* +*********************************************** +Herschel Science Archive (`astroquery.esa.hsa`) +*********************************************** `Herschel `__ was the fourth cornerstone in ESA's Horizon 2000 science programme, designed to observe the 'cool' universe. It performed photometry and spectroscopy in the poorly explored 55-670 µm spectral range with a 3.5 m diameter @@ -22,9 +20,9 @@ Examples 1. Getting Herschel data ------------------------------ -.. code-block:: python +.. doctest-remote-data:: - >>> from astroquery.hsa import HSA + >>> from astroquery.esa.hsa import HSA >>> >>> HSA.download_data(observation_id='1342195355',retrieval_type='OBSERVATION', instrument_name='PACS') Downloading URL http://archives.esac.esa.int/hsa/whsa-tap-server/data?&retrieval_type=OBSERVATION&observation_id=1342195355&instrument_name=PACS to 1342195355.tar ... [Done] @@ -43,9 +41,9 @@ For more details about the products check: 2. Getting Observation Products ------------------------------- -.. code-block:: python +.. doctest-remote-data:: - >>> from astroquery.hsa import HSA + >>> from astroquery.esa.hsa import HSA >>> >>> HSA.get_observation('1342195355', instrument_name='PACS') Downloading URL http://archives.esac.esa.int/hsa/whsa-tap-server/data?&retrieval_type=OBSERVATION&observation_id=1342195355&instrument_name=PACS to 1342195355.tar ... [Done] @@ -67,9 +65,9 @@ For more details of the parameters check the section 6.2 of the 'Direct Product 3. Getting Herschel Postcard ------------------------------- -.. code-block:: python +.. doctest-remote-data:: - >>> from astroquery.hsa import HSA + >>> from astroquery.esa.hsa import HSA >>> >>> HSA.get_postcard('1342195355', instrument_name='PACS') Downloading URL http://archives.esac.esa.int/hsa/whsa-tap-server/data?&retrieval_type=POSTCARD&observation_id=1342195355&instrument_name=PACS to /home/dev/.astropy/cache/astroquery/HSA/data?&retrieval_type=POSTCARD&observation_id=1342195355&instrument_name=PACS ... [Done] @@ -88,9 +86,9 @@ For more details of the parameters check the section 6.2 of the 'Direct Product This function provides access to the Herschel Science Archive database using the Table Access Protocol (TAP) and via the Astronomical Data Query Language (`ADQL `__). -.. code-block:: python +.. doctest-remote-data:: - >>> from astroquery.hsa import HSA + >>> from astroquery.esa.hsa import HSA >>> >>> result = HSA.query_hsa_tap("select top 10 * from hsa.v_active_observation", output_format='csv', output_file='results.csv') >>> print(result) @@ -114,9 +112,9 @@ stored in the file 'results.csv'. The result of this query can be printed by doi 5. Getting table details of HSA TAP ----------------------------------- -.. code-block:: python +.. doctest-remote-data:: - >>> from astroquery.hsa import HSA + >>> from astroquery.esa.hsa import HSA >>> >>> HSA.get_tables() INFO: Retrieving tables... [astroquery.utils.tap.core] @@ -130,9 +128,9 @@ This will show the available tables in HSA TAP service in the Herschel Science A 6. Getting columns details of HSA TAP ------------------------------------- -.. code-block:: python +.. doctest-remote-data:: - >>> from astroquery.hsa import HSA + >>> from astroquery.esa.hsa import HSA >>> >>> HSA.get_columns('hsa.v_active_observation') INFO: Retrieving tables... [astroquery.utils.tap.core] @@ -143,14 +141,38 @@ This will show the available tables in HSA TAP service in the Herschel Science A This will show the column details of the table 'hsa.v_active_observation' in HSA TAP service in the Herschel Science Archive. ------------------------------------- -7. Procedure example +7. Query Observations +------------------------------------- + +.. code-block:: python + + >>> from astroquery.esa.hsa import HSA + >>> from astropy.coordinates import SkyCoord + >>> from astropy import units as u + >>> + >>> c = SkyCoord(ra=100.2417*u.degree, dec=9.895*u.degree, frame='icrs') + >>> HSA.query_observations(c, 0.5) +
+ observation_id + object + -------------- + 1342219315 + 1342205057 + 1342205056 + 1342205056 + 1342205057 + +Retrieve a VOTable with the observation IDs of a given region + +------------------------------------- +8. Procedure example ------------------------------------- First retrieve the observation IDs based on a position on the sky. To achive this, query the TAP service. .. code-block:: python - >>> from astroquery.hsa import HSA + >>> from astroquery.esa.hsa import HSA >>> >>> HSA.query_hsa_tap("select top 10 observation_id from hsa.v_active_observation where contains(point('ICRS', hsa.v_active_observation.ra, hsa.v_active_observation.dec),circle('ICRS', 100.2417,9.895, 1.1))=1", output_format='csv', output_file='results.cvs')
diff --git a/docs/index.rst b/docs/index.rst index 9cf9f100f5..b5019cd8b7 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -187,6 +187,7 @@ The following modules have been completed using a common API: cds/cds.rst linelists/cdms/cdms.rst dace/dace.rst + esa/hsa.rst esa/hubble.rst esa/iso.rst esa/jwst.rst @@ -200,7 +201,6 @@ The following modules have been completed using a common API: heasarc/heasarc.rst hips2fits/hips2fits.rst hitran/hitran.rst - hsa/hsa.rst ipac/irsa/irsa_dust/irsa_dust.rst ipac/irsa/ibe/ibe.rst ipac/irsa/irsa.rst From 0382c1212f6479236b4cfba3ba0d4ef3e24eddfd Mon Sep 17 00:00:00 2001 From: lvalerom Date: Sat, 2 Apr 2022 23:06:31 +0200 Subject: [PATCH 15/22] style --- astroquery/esa/hsa/tests/setup_package.py | 2 -- astroquery/esa/hsa/tests/test_hsa.py | 2 +- astroquery/esa/hsa/tests/test_hsa_remote.py | 6 +++--- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/astroquery/esa/hsa/tests/setup_package.py b/astroquery/esa/hsa/tests/setup_package.py index 0cb4513e22..5d47518d13 100644 --- a/astroquery/esa/hsa/tests/setup_package.py +++ b/astroquery/esa/hsa/tests/setup_package.py @@ -1,7 +1,5 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst -#from __future__ import absolute_import - import os diff --git a/astroquery/esa/hsa/tests/test_hsa.py b/astroquery/esa/hsa/tests/test_hsa.py index 81a4f7facf..c08dc1078f 100644 --- a/astroquery/esa/hsa/tests/test_hsa.py +++ b/astroquery/esa/hsa/tests/test_hsa.py @@ -53,7 +53,7 @@ def test_query_observations(self): c = SkyCoord(ra=100.2417*u.degree, dec=9.895*u.degree, frame='icrs') parameters = {'coordinate': c, 'radius': 0.5} - dummyTapHandler = DummyHSATapHandler("query_observations",parameters) + dummyTapHandler = DummyHSATapHandler("query_observations", parameters) hsa = HSAClass(self.get_dummy_tap_handler()) hsa.query_observations(**parameters) dummyTapHandler.check_call("query_observations", parameters) diff --git a/astroquery/esa/hsa/tests/test_hsa_remote.py b/astroquery/esa/hsa/tests/test_hsa_remote.py index 8a91b847ed..9befcffcf6 100644 --- a/astroquery/esa/hsa/tests/test_hsa_remote.py +++ b/astroquery/esa/hsa/tests/test_hsa_remote.py @@ -2,7 +2,8 @@ import tempfile import pytest -import os,tarfile +import os +import tarfile from ..core import HSAClass @@ -10,12 +11,12 @@ pacs_chksum = [10205, 10752, 8914, 3869, 9736, 3925, 3942, 11201, 10454, 11393, 10520, 11080, 3979, 11121, 3961, 9745, 10492, 9317, 11304, 12025, 10584, 11295, 9564, 10585, 11367, 11229, 11808, 11039, 11225, 11411, 10246, 11066, 11013, 10158, 3939, 3927, 3941, 3936, 3942, 3925, 9691, 10381, 10944, 3990, 10970, 3969, 9453, 10802, 3881, 11926, 3896, 12528, 3908, 11898, 3883, 3894, 3878, 11231, 3910, 3885, 3876, 3913, 3967, 3970, 3908, 3903, 3901, 3958, 3898, 3870, 3915, 3878, 3895, 3901, 3888, 3880, 3922, 3881, 3926, 3883, 12229, 3958, 3949, 3936, 3962, 3949, 3947, 3868, 13210, 3885, 3901, 11463, 3939, 3967, 3878, 3900, 3911, 3947, 3917, 3940, 3953, 3904, 3869, 3874, 3880, 3893, 3964, 3980, 3889, 3871, 3908, 3882, 3888, 3895, 3905, 3889, 13163, 3879, 11036, 12469, 3886, 3887, 3863, 3921, 3881, 3911, 3878, 3921, 11149, 13032, 3880, 3881, 3883, 3883, 3895, 3899, 3919, 12498, 3896, 3942, 3894, 11407, 3866, 11716, 3889, 3890, 3888, 3873, 9870, 10738, 3978, 10759, 3949, 10720, 3921, 10390, 3938, 10365, 11182, 3905, 10724, 3943, 10742, 3942, 10025, 3924, 3935, 10388, 11185, 11206, 6288] + @pytest.mark.remote_data class TestHSARemote: tmp_dir = tempfile.TemporaryDirectory() def test_download_data_observation_pacs(self): - #obs_id = "1342195355" obs_id = "1342191813" parameters = {'retrieval_type': "OBSERVATION", 'observation_id': obs_id, @@ -73,7 +74,6 @@ def test_download_data_observation_pacs_compressed(self): os.remove(res) def test_download_data_observation_spire(self): - #obs_id = "1342224960" obs_id = "1342191188" parameters = {'retrieval_type': "OBSERVATION", 'observation_id': obs_id, From c30e59ef7148e6c4cc43811c1957fe44c68e8392 Mon Sep 17 00:00:00 2001 From: lvalerom Date: Sat, 2 Apr 2022 23:27:24 +0200 Subject: [PATCH 16/22] fixing docs --- astroquery/esa/hsa/__init__.py | 2 +- astroquery/esa/hsa/tests/setup_package.py | 2 +- docs/esa/hsa.rst | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/astroquery/esa/hsa/__init__.py b/astroquery/esa/hsa/__init__.py index 428ef2f4cf..13394f89ca 100644 --- a/astroquery/esa/hsa/__init__.py +++ b/astroquery/esa/hsa/__init__.py @@ -11,7 +11,7 @@ class Conf(_config.ConfigNamespace): """ - Configuration parameters for `astroquery.hsa`. + Configuration parameters for `astroquery.esa.hsa`. """ DATA_ACTION = _config.ConfigItem("http://archives.esac.esa.int/hsa/whsa-tap-server/data?", "Main url for retrieving HSA Data Archive files") diff --git a/astroquery/esa/hsa/tests/setup_package.py b/astroquery/esa/hsa/tests/setup_package.py index 5d47518d13..fbfb8d1e98 100644 --- a/astroquery/esa/hsa/tests/setup_package.py +++ b/astroquery/esa/hsa/tests/setup_package.py @@ -11,4 +11,4 @@ def get_package_data(): ] # etc, add other extensions # you can also enlist files individually by names # finally construct and return a dict for the sub module - return {'astroquery.hsa.tests': paths} + return {'astroquery.esa.hsa.tests': paths} diff --git a/docs/esa/hsa.rst b/docs/esa/hsa.rst index 05613abf61..b22e982bb3 100644 --- a/docs/esa/hsa.rst +++ b/docs/esa/hsa.rst @@ -144,7 +144,7 @@ This will show the column details of the table 'hsa.v_active_observation' in HSA 7. Query Observations ------------------------------------- -.. code-block:: python +.. doctest-skip:: >>> from astroquery.esa.hsa import HSA >>> from astropy.coordinates import SkyCoord @@ -154,7 +154,7 @@ This will show the column details of the table 'hsa.v_active_observation' in HSA >>> HSA.query_observations(c, 0.5)
observation_id - object + object -------------- 1342219315 1342205057 @@ -170,7 +170,7 @@ Retrieve a VOTable with the observation IDs of a given region First retrieve the observation IDs based on a position on the sky. To achive this, query the TAP service. -.. code-block:: python +.. doctest-skip:: >>> from astroquery.esa.hsa import HSA >>> @@ -196,7 +196,7 @@ For more information on how to use ADQL see: After obtaining the desire ID, download the product of the observation '1342205057' with the instrument 'PACS'. -.. code-block:: python +.. doctest-skip:: >>> HSA.download_data(observation_id='1342205057', retrieval_type='OBSERVATION', instrument_name='PACS') Downloading URL http://archives.esac.esa.int/hsa/whsa-tap-server/data?&retrieval_type=OBSERVATION&observation_id=1342205057&instrument_name=PACS to 1342205057.tar ... [Done] @@ -205,5 +205,5 @@ After obtaining the desire ID, download the product of the observation '13422050 Reference/API ============= -.. automodapi:: astroquery.hsa +.. automodapi:: astroquery.esa.hsa :no-inheritance-diagram: From 842f2da92ccc456f0b6c32b960af79faf0c7b91c Mon Sep 17 00:00:00 2001 From: lvalerom Date: Sat, 9 Apr 2022 17:22:50 +0200 Subject: [PATCH 17/22] review comments --- astroquery/esa/hsa/core.py | 51 ++++++++++----- astroquery/esa/hsa/tests/test_hsa_remote.py | 69 ++++++++++++++++++--- 2 files changed, 95 insertions(+), 25 deletions(-) diff --git a/astroquery/esa/hsa/core.py b/astroquery/esa/hsa/core.py index f383405855..9a3fdf1626 100644 --- a/astroquery/esa/hsa/core.py +++ b/astroquery/esa/hsa/core.py @@ -9,7 +9,7 @@ from astropy import units as u from ...utils import commons from astroquery import log -from astroquery.exceptions import LoginError, InputWarning +from astroquery.exceptions import LoginError from astroquery.query import BaseQuery from astroquery.utils.tap.core import Tap @@ -32,7 +32,8 @@ def __init__(self, tap_handler=None): self._tap = tap_handler def download_data(self, *, retrieval_type="OBSERVATION", observation_id=None, - instrument_name=None, filename=None, verbose=False, + instrument_name=None, filename=None, observation_oid=None, + instrument_oid=None, product_level=None, verbose=False, download_dir="", cache=True, **kwargs): """ Download data from Herschel @@ -82,6 +83,15 @@ def download_data(self, *, retrieval_type="OBSERVATION", observation_id=None, if instrument_name is not None: params['instrument_name'] = instrument_name + if observation_oid is not None: + params['observation_oid'] = observation_oid + + if instrument_oid is not None: + params['instrument_oid'] = instrument_oid + + if product_level is not None: + params['product_level'] = product_level + link = self.data_url + "".join("&{0}={1}".format(key, val) for key, val in params.items()) link += "".join("&{0}={1}".format(key, val) for key, val in kwargs.items()) @@ -90,14 +100,12 @@ def download_data(self, *, retrieval_type="OBSERVATION", observation_id=None, log.info(link) response = self._request('HEAD', link, save=False, cache=cache) - response.raise_for_status() - - if 'Content-Type' in response.headers and 'text' not in response.headers['Content-Type']: - _, params = cgi.parse_header(response.headers['Content-Disposition']) - else: + if response.status_code == 401: error = "Data protected by proprietary rights. Please check your credentials" raise LoginError(error) + response.raise_for_status() + if filename is None: if observation_id is not None: filename = observation_id @@ -105,7 +113,10 @@ def download_data(self, *, retrieval_type="OBSERVATION", observation_id=None, error = "Please set either 'obervation_id' or 'filename' for the output" raise ValueError(error) - r_filename = params["filename"] + _, res_params = cgi.parse_header(response.headers['Content-Disposition']) + + + r_filename = res_params["filename"] suffixes = Path(r_filename).suffixes if len(suffixes) > 1 and suffixes[-1] == ".jpg": @@ -123,6 +134,7 @@ def download_data(self, *, retrieval_type="OBSERVATION", observation_id=None, return filename def get_observation(self, observation_id, instrument_name, *, filename=None, + observation_oid=None, instrument_oid=None, product_level=None, verbose=False, download_dir="", cache=True, **kwargs): """ Download observation from Herschel. @@ -171,6 +183,15 @@ def get_observation(self, observation_id, instrument_name, *, filename=None, 'observation_id': observation_id, 'instrument_name': instrument_name} + if observation_oid is not None: + params['observation_oid'] = observation_oid + + if instrument_oid is not None: + params['instrument_oid'] = instrument_oid + + if product_level is not None: + params['product_level'] = product_level + link = self.data_url + "".join("&{0}={1}".format(key, val) for key, val in params.items()) link += "".join("&{0}={1}".format(key, val) for key, val in kwargs.items()) @@ -179,15 +200,15 @@ def get_observation(self, observation_id, instrument_name, *, filename=None, log.info(link) response = self._request('HEAD', link, save=False, cache=cache) + if response.status_code == 401: + error = "Data protected by proprietary rights. Please check your credentials" + raise LoginError(error) + response.raise_for_status() - if 'Content-Type' in response.headers and 'text' not in response.headers['Content-Type']: - _, params = cgi.parse_header(response.headers['Content-Disposition']) - else: - error = "Data protected by propietary rights. Please check your credentials" - raise LoginError(error) + _, res_params = cgi.parse_header(response.headers['Content-Disposition']) - r_filename = params["filename"] + r_filename = res_params["filename"] suffixes = Path(r_filename).suffixes if filename is None: @@ -359,7 +380,7 @@ def get_columns(self, table_name, *, only_names=True, verbose=False): else: return columns - def query_observations(self, coordinate, radius, n_obs=10): + def query_observations(self, coordinate, radius, *, n_obs=10): """ Get the observation IDs from a given region diff --git a/astroquery/esa/hsa/tests/test_hsa_remote.py b/astroquery/esa/hsa/tests/test_hsa_remote.py index 9befcffcf6..90c4cc4d15 100644 --- a/astroquery/esa/hsa/tests/test_hsa_remote.py +++ b/astroquery/esa/hsa/tests/test_hsa_remote.py @@ -4,27 +4,44 @@ import os import tarfile +from requests.exceptions import ChunkedEncodingError from ..core import HSAClass -spire_chksum = [10243, 10772, 9029, 11095, 3875, 3893, 3903, 3868, 3888, 3951, 3905, 3875, 3887, 3908, 3871, 3892, 3894, 3900, 3870, 3892, 3938, 3904, 3893, 3866, 3893, 3930, 3901, 3867, 3893, 3927, 3903, 3868, 3892, 3926, 3904, 3869, 3889, 3937, 3899, 3868, 3887, 3928, 3912, 3865, 3887, 3937, 3899, 3865, 3891, 3925, 3911, 3868, 3883, 3937, 3902, 3870, 3893, 3922, 3914, 3869, 3886, 3924, 3903, 3870, 3889, 3922, 3903, 3867, 3889, 3932, 3902, 3869, 3892, 3930, 3900, 3868, 3892, 3916, 3910, 3868, 3889, 3932, 3907, 3869, 3892, 3914, 3900, 3871, 3884, 3918, 3903, 3894, 3874, 3892, 3906, 3872, 3889, 3895, 9724, 10495, 9293, 11299, 12132, 10602, 11283, 9582, 10568, 11354, 11228, 11812, 11006, 11224, 11382, 10232, 11049, 11011, 3862, 9671, 10844, 10899, 3946, 3941, 3953, 3954, 3948, 3940, 3954, 3948, 3945, 9286, 11197, 3902, 10137, 11664, 12038, 3963, 3948, 11970, 3867, 11452, 3908, 3908, 3894, 3908, 3908, 3894, 3885, 3914, 3876, 11820, 3958, 3905, 3963, 3925, 3974, 3923, 12241, 3897, 3900, 12328, 11911, 3893, 3885, 3895, 3895, 3891, 11970, 12252, 3899, 12503, 12135, 11826, 3899, 3899, 3912, 3912, 11530, 11625, 11850, 10981, 3872, 3890, 3896, 3895, 3867, 3891, 10561, 10991, 11209, 3853, 3871, 3869, 11372, 10826, 10995, 3887, 3882, 3883, 3879, 3884, 3883, 3889, 3883, 3887, 3876, 3885, 3883, 3882, 3888, 3886, 3882, 3882, 3882, 3887, 3885, 3885, 3884, 3880, 3881, 3878, 3882, 3884, 3879, 3887, 3882, 3884, 3877, 3883, 3882, 3881, 3882, 3876, 3879, 3881, 3881, 3884, 3882, 3882, 3882, 3888, 3883, 3880, 3880, 3875, 3880, 3884, 3883, 3887, 3880, 3879, 3874, 3882, 3879, 3887, 3876, 3888, 3888, 3885, 3885, 3876, 3880, 3887, 3877, 3884, 3886, 3879, 3879, 3877, 3880, 3876, 3889, 3885, 3886, 3885, 3879, 3879, 12424, 10112, 10885, 3895, 3922, 3888, 3900, 3898, 3919, 3925, 3930, 3923, 3932, 3931, 3917, 3922, 3917, 3919, 3935, 3929, 3923, 3929, 3924, 3906, 3918, 3895, 3892, 11322, 10879, 3944, 11338, 3921, 11009, 10969, 11352, 10984, 3944, 11345, 11333, 11088, 11331, 11099, 11324, 11118, 10591, 11502, 6291] +spire_chksum = [10233, 10762, 9019, 10869, 3944, 11328, 3921, 10999, 10959, 11342, 10974, 3944, 11335, 11323, 11078, 11321, 11089, 11314, 11108, 6281] -pacs_chksum = [10205, 10752, 8914, 3869, 9736, 3925, 3942, 11201, 10454, 11393, 10520, 11080, 3979, 11121, 3961, 9745, 10492, 9317, 11304, 12025, 10584, 11295, 9564, 10585, 11367, 11229, 11808, 11039, 11225, 11411, 10246, 11066, 11013, 10158, 3939, 3927, 3941, 3936, 3942, 3925, 9691, 10381, 10944, 3990, 10970, 3969, 9453, 10802, 3881, 11926, 3896, 12528, 3908, 11898, 3883, 3894, 3878, 11231, 3910, 3885, 3876, 3913, 3967, 3970, 3908, 3903, 3901, 3958, 3898, 3870, 3915, 3878, 3895, 3901, 3888, 3880, 3922, 3881, 3926, 3883, 12229, 3958, 3949, 3936, 3962, 3949, 3947, 3868, 13210, 3885, 3901, 11463, 3939, 3967, 3878, 3900, 3911, 3947, 3917, 3940, 3953, 3904, 3869, 3874, 3880, 3893, 3964, 3980, 3889, 3871, 3908, 3882, 3888, 3895, 3905, 3889, 13163, 3879, 11036, 12469, 3886, 3887, 3863, 3921, 3881, 3911, 3878, 3921, 11149, 13032, 3880, 3881, 3883, 3883, 3895, 3899, 3919, 12498, 3896, 3942, 3894, 11407, 3866, 11716, 3889, 3890, 3888, 3873, 9870, 10738, 3978, 10759, 3949, 10720, 3921, 10390, 3938, 10365, 11182, 3905, 10724, 3943, 10742, 3942, 10025, 3924, 3935, 10388, 11185, 11206, 6288] +pacs_chksum = [10208, 10755, 8917, 10028, 3924, 3935, 6291] @pytest.mark.remote_data class TestHSARemote: tmp_dir = tempfile.TemporaryDirectory() + retries = 2 + + def access_archive_with_retries(self, f, params): + for _ in range(self.retries): + try: + res = f(**params) + return res + except ChunkedEncodingError: + pass + return None + def test_download_data_observation_pacs(self): obs_id = "1342191813" parameters = {'retrieval_type': "OBSERVATION", 'observation_id': obs_id, 'instrument_name': "PACS", + 'product_level': 'LEVEL3', + 'cache': False, 'download_dir': self.tmp_dir.name} expected_res = os.path.join(self.tmp_dir.name, obs_id + ".tar") hsa = HSAClass() - res = hsa.download_data(**parameters) + #res = hsa.download_data(**parameters) + res = self.access_archive_with_retries(hsa.download_data, parameters) + if res == None: + pytest.skip("Archive broke the connection {} times, unable to test".format(self.retries)) assert res == expected_res assert os.path.isfile(res) chksum = [] @@ -40,11 +57,16 @@ def test_download_data_observation_pacs_filename(self): parameters = {'retrieval_type': "OBSERVATION", 'observation_id': obs_id, 'instrument_name': "PACS", + 'product_level': 'LEVEL3', 'filename': fname, + 'cache': False, 'download_dir': self.tmp_dir.name} expected_res = os.path.join(self.tmp_dir.name, fname + ".tar") hsa = HSAClass() - res = hsa.download_data(**parameters) + #res = hsa.download_data(**parameters) + res = self.access_archive_with_retries(hsa.download_data, parameters) + if res == None: + pytest.skip("Archive broke the connection {} times, unable to test".format(self.retries)) assert res == expected_res assert os.path.isfile(res) chksum = [] @@ -59,11 +81,16 @@ def test_download_data_observation_pacs_compressed(self): parameters = {'retrieval_type': "OBSERVATION", 'observation_id': obs_id, 'instrument_name': "PACS", + 'product_level': 'LEVEL3', 'compress': 'true', + 'cache': False, 'download_dir': self.tmp_dir.name} expected_res = os.path.join(self.tmp_dir.name, obs_id + ".tgz") hsa = HSAClass() - res = hsa.download_data(**parameters) + #res = hsa.download_data(**parameters) + res = self.access_archive_with_retries(hsa.download_data, parameters) + if res == None: + pytest.skip("Archive broke the connection {} times, unable to test".format(self.retries)) assert res == expected_res assert os.path.isfile(res) chksum = [] @@ -78,10 +105,15 @@ def test_download_data_observation_spire(self): parameters = {'retrieval_type': "OBSERVATION", 'observation_id': obs_id, 'instrument_name': "SPIRE", + 'product_level': 'LEVEL2', + 'cache': False, 'download_dir': self.tmp_dir.name} expected_res = os.path.join(self.tmp_dir.name, obs_id + ".tar") hsa = HSAClass() - res = hsa.download_data(**parameters) + #res = hsa.download_data(**parameters) + res = self.access_archive_with_retries(hsa.download_data, parameters) + if res == None: + pytest.skip("Archive broke the connection {} times, unable to test".format(self.retries)) assert res == expected_res assert os.path.isfile(res) chksum = [] @@ -96,10 +128,14 @@ def test_download_data_postcard_pacs(self): parameters = {'retrieval_type': "POSTCARD", 'observation_id': obs_id, 'instrument_name': "PACS", + 'cache': False, 'download_dir': self.tmp_dir.name} expected_res = os.path.join(self.tmp_dir.name, obs_id + ".jpg") hsa = HSAClass() - res = hsa.download_data(**parameters) + #res = hsa.download_data(**parameters) + res = self.access_archive_with_retries(hsa.download_data, parameters) + if res == None: + pytest.skip("Archive broke the connection {} times, unable to test".format(self.retries)) assert res == expected_res assert os.path.isfile(res) os.remove(res) @@ -111,10 +147,14 @@ def test_download_data_postcard_pacs_filename(self): 'observation_id': obs_id, 'instrument_name': "PACS", 'filename': fname, + 'cache': False, 'download_dir': self.tmp_dir.name} expected_res = os.path.join(self.tmp_dir.name, fname + ".jpg") hsa = HSAClass() - res = hsa.download_data(**parameters) + #res = hsa.download_data(**parameters) + res = self.access_archive_with_retries(hsa.download_data, parameters) + if res == None: + pytest.skip("Archive broke the connection {} times, unable to test".format(self.retries)) assert res == expected_res assert os.path.isfile(res) os.remove(res) @@ -123,10 +163,15 @@ def test_get_observation(self): obs_id = "1342191813" parameters = {'observation_id': obs_id, 'instrument_name': "PACS", + 'product_level': 'LEVEL3', + 'cache': False, 'download_dir': self.tmp_dir.name} expected_res = os.path.join(self.tmp_dir.name, obs_id + ".tar") hsa = HSAClass() - res = hsa.get_observation(**parameters) + #res = hsa.get_observation(**parameters) + res = self.access_archive_with_retries(hsa.get_observation, parameters) + if res == None: + pytest.skip("Archive broke the connection {} times, unable to test".format(self.retries)) assert res == expected_res assert os.path.isfile(res) chksum = [] @@ -140,10 +185,14 @@ def test_get_postcard(self): obs_id = "1342191813" parameters = {'observation_id': obs_id, 'instrument_name': "PACS", + 'cache': False, 'download_dir': self.tmp_dir.name} expected_res = os.path.join(self.tmp_dir.name, obs_id + ".jpg") hsa = HSAClass() - res = hsa.get_postcard(**parameters) + #res = hsa.get_postcard(**parameters) + res = self.access_archive_with_retries(hsa.get_postcard, parameters) + if res == None: + pytest.skip("Archive broke the connection {} times, unable to test".format(self.retries)) assert res == expected_res assert os.path.isfile(res) os.remove(res) From 7419d9af103341f0d5ddff2ed7a5cc5db3c76f86 Mon Sep 17 00:00:00 2001 From: lvalerom Date: Sat, 9 Apr 2022 17:29:48 +0200 Subject: [PATCH 18/22] style --- astroquery/esa/hsa/core.py | 1 - astroquery/esa/hsa/tests/test_hsa_remote.py | 45 ++++++--------------- 2 files changed, 13 insertions(+), 33 deletions(-) diff --git a/astroquery/esa/hsa/core.py b/astroquery/esa/hsa/core.py index 9a3fdf1626..689b7acce1 100644 --- a/astroquery/esa/hsa/core.py +++ b/astroquery/esa/hsa/core.py @@ -115,7 +115,6 @@ def download_data(self, *, retrieval_type="OBSERVATION", observation_id=None, _, res_params = cgi.parse_header(response.headers['Content-Disposition']) - r_filename = res_params["filename"] suffixes = Path(r_filename).suffixes diff --git a/astroquery/esa/hsa/tests/test_hsa_remote.py b/astroquery/esa/hsa/tests/test_hsa_remote.py index 90c4cc4d15..b73d62de5f 100644 --- a/astroquery/esa/hsa/tests/test_hsa_remote.py +++ b/astroquery/esa/hsa/tests/test_hsa_remote.py @@ -27,7 +27,6 @@ def access_archive_with_retries(self, f, params): pass return None - def test_download_data_observation_pacs(self): obs_id = "1342191813" parameters = {'retrieval_type': "OBSERVATION", @@ -38,16 +37,13 @@ def test_download_data_observation_pacs(self): 'download_dir': self.tmp_dir.name} expected_res = os.path.join(self.tmp_dir.name, obs_id + ".tar") hsa = HSAClass() - #res = hsa.download_data(**parameters) res = self.access_archive_with_retries(hsa.download_data, parameters) - if res == None: + if res is None: pytest.skip("Archive broke the connection {} times, unable to test".format(self.retries)) assert res == expected_res assert os.path.isfile(res) - chksum = [] tar = tarfile.open(res) - for m in tar.getmembers(): - chksum.append(m.chksum) + chksum = [m.chksum for m in tar.getmembers()] assert chksum.sort() == pacs_chksum.sort() os.remove(res) @@ -63,16 +59,13 @@ def test_download_data_observation_pacs_filename(self): 'download_dir': self.tmp_dir.name} expected_res = os.path.join(self.tmp_dir.name, fname + ".tar") hsa = HSAClass() - #res = hsa.download_data(**parameters) res = self.access_archive_with_retries(hsa.download_data, parameters) - if res == None: + if res is None: pytest.skip("Archive broke the connection {} times, unable to test".format(self.retries)) assert res == expected_res assert os.path.isfile(res) - chksum = [] tar = tarfile.open(res) - for m in tar.getmembers(): - chksum.append(m.chksum) + chksum = [m.chksum for m in tar.getmembers()] assert chksum.sort() == pacs_chksum.sort() os.remove(res) @@ -87,16 +80,13 @@ def test_download_data_observation_pacs_compressed(self): 'download_dir': self.tmp_dir.name} expected_res = os.path.join(self.tmp_dir.name, obs_id + ".tgz") hsa = HSAClass() - #res = hsa.download_data(**parameters) res = self.access_archive_with_retries(hsa.download_data, parameters) - if res == None: + if res is None: pytest.skip("Archive broke the connection {} times, unable to test".format(self.retries)) assert res == expected_res assert os.path.isfile(res) - chksum = [] tar = tarfile.open(res) - for m in tar.getmembers(): - chksum.append(m.chksum) + chksum = [m.chksum for m in tar.getmembers()] assert chksum.sort() == pacs_chksum.sort() os.remove(res) @@ -110,16 +100,13 @@ def test_download_data_observation_spire(self): 'download_dir': self.tmp_dir.name} expected_res = os.path.join(self.tmp_dir.name, obs_id + ".tar") hsa = HSAClass() - #res = hsa.download_data(**parameters) res = self.access_archive_with_retries(hsa.download_data, parameters) - if res == None: + if res is None: pytest.skip("Archive broke the connection {} times, unable to test".format(self.retries)) assert res == expected_res assert os.path.isfile(res) - chksum = [] tar = tarfile.open(res) - for m in tar.getmembers(): - chksum.append(m.chksum) + chksum = [m.chksum for m in tar.getmembers()] assert chksum.sort() == spire_chksum.sort() os.remove(res) @@ -132,9 +119,8 @@ def test_download_data_postcard_pacs(self): 'download_dir': self.tmp_dir.name} expected_res = os.path.join(self.tmp_dir.name, obs_id + ".jpg") hsa = HSAClass() - #res = hsa.download_data(**parameters) res = self.access_archive_with_retries(hsa.download_data, parameters) - if res == None: + if res is None: pytest.skip("Archive broke the connection {} times, unable to test".format(self.retries)) assert res == expected_res assert os.path.isfile(res) @@ -151,9 +137,8 @@ def test_download_data_postcard_pacs_filename(self): 'download_dir': self.tmp_dir.name} expected_res = os.path.join(self.tmp_dir.name, fname + ".jpg") hsa = HSAClass() - #res = hsa.download_data(**parameters) res = self.access_archive_with_retries(hsa.download_data, parameters) - if res == None: + if res is None: pytest.skip("Archive broke the connection {} times, unable to test".format(self.retries)) assert res == expected_res assert os.path.isfile(res) @@ -168,16 +153,13 @@ def test_get_observation(self): 'download_dir': self.tmp_dir.name} expected_res = os.path.join(self.tmp_dir.name, obs_id + ".tar") hsa = HSAClass() - #res = hsa.get_observation(**parameters) res = self.access_archive_with_retries(hsa.get_observation, parameters) - if res == None: + if res is None: pytest.skip("Archive broke the connection {} times, unable to test".format(self.retries)) assert res == expected_res assert os.path.isfile(res) - chksum = [] tar = tarfile.open(res) - for m in tar.getmembers(): - chksum.append(m.chksum) + chksum = [m.chksum for m in tar.getmembers()] assert chksum.sort() == pacs_chksum.sort() os.remove(res) @@ -189,9 +171,8 @@ def test_get_postcard(self): 'download_dir': self.tmp_dir.name} expected_res = os.path.join(self.tmp_dir.name, obs_id + ".jpg") hsa = HSAClass() - #res = hsa.get_postcard(**parameters) res = self.access_archive_with_retries(hsa.get_postcard, parameters) - if res == None: + if res is None: pytest.skip("Archive broke the connection {} times, unable to test".format(self.retries)) assert res == expected_res assert os.path.isfile(res) From 4d6e353361c0eb1d1d60f3df7e7283c9cae9d9f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Wed, 13 Apr 2022 11:46:45 -0700 Subject: [PATCH 19/22] Fix doctests --- docs/esa/hsa.rst | 38 ++++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/docs/esa/hsa.rst b/docs/esa/hsa.rst index b22e982bb3..5749075a1e 100644 --- a/docs/esa/hsa.rst +++ b/docs/esa/hsa.rst @@ -24,7 +24,7 @@ Examples >>> from astroquery.esa.hsa import HSA >>> - >>> HSA.download_data(observation_id='1342195355',retrieval_type='OBSERVATION', instrument_name='PACS') + >>> HSA.download_data(observation_id='1342195355',retrieval_type='OBSERVATION', instrument_name='PACS') # doctest: +IGNORE_OUTPUT Downloading URL http://archives.esac.esa.int/hsa/whsa-tap-server/data?&retrieval_type=OBSERVATION&observation_id=1342195355&instrument_name=PACS to 1342195355.tar ... [Done] '1342195355.tar' @@ -45,7 +45,7 @@ For more details about the products check: >>> from astroquery.esa.hsa import HSA >>> - >>> HSA.get_observation('1342195355', instrument_name='PACS') + >>> HSA.get_observation('1342195355', instrument_name='PACS') # doctest: +IGNORE_OUTPUT Downloading URL http://archives.esac.esa.int/hsa/whsa-tap-server/data?&retrieval_type=OBSERVATION&observation_id=1342195355&instrument_name=PACS to 1342195355.tar ... [Done] '1342195355.tar' @@ -69,7 +69,7 @@ For more details of the parameters check the section 6.2 of the 'Direct Product >>> from astroquery.esa.hsa import HSA >>> - >>> HSA.get_postcard('1342195355', instrument_name='PACS') + >>> HSA.get_postcard('1342195355', instrument_name='PACS') # doctest: +IGNORE_OUTPUT Downloading URL http://archives.esac.esa.int/hsa/whsa-tap-server/data?&retrieval_type=POSTCARD&observation_id=1342195355&instrument_name=PACS to /home/dev/.astropy/cache/astroquery/HSA/data?&retrieval_type=POSTCARD&observation_id=1342195355&instrument_name=PACS ... [Done] '1342195355.jpg' @@ -90,23 +90,21 @@ Query Language (`ADQL >> from astroquery.esa.hsa import HSA >>> - >>> result = HSA.query_hsa_tap("select top 10 * from hsa.v_active_observation", output_format='csv', output_file='results.csv') - >>> print(result) - aor bii dec duration ... status target_name urn_version - ------------------------------------------------------------ ------------------- ------------------- ---------- ... ------ ---------------------------------- ----------- - PPhot.Cart-perp -33.71633333333334 -33.71633333333334 1759000.0 ... FAILED Cartwheel 925353 - PPhot.Cart -33.71633333333334 -33.71633333333334 1759000.0 ... FAILED Cartwheel 925352 - PPhoto-0005 - cosmos6 - cosmos_328-1 - XMMXCS J2215.9-1738-1 -17.633888888888887 -17.633888888888887 18149000.0 ... FAILED XMMXCS J2215.9-1738-1 cross scan-1 925351 - DRT-B-HD013246 -59.67941666666666 -59.67941666666666 2250000.0 ... FAILED HD013246-1 925350 - DRT-A-HD013246 -59.67941666666666 -59.67941666666666 2250000.0 ... FAILED HD013246-1 925348 - e0102green1_135 - hsc rec -72.03119444444444 -72.03119444444444 4272000.0 ... FAILED 1e0102.2-7219 925346 - e0102green1_45 - hsc rec -72.03119444444444 -72.03119444444444 4272000.0 ... FAILED 1e0102.2-7219 925344 - e0102blue1_135 - hsc rec -72.03119444444444 -72.03119444444444 4272000.0 ... FAILED 1e0102.2-7219 925342 - e0102blue1_45 - hsc rec -72.03119444444444 -72.03119444444444 4272000.0 ... FAILED 1e0102.2-7219 925340 - PPhot.AM06-perp -74.22638888888889 -74.22638888888889 1759000.0 ... FAILED AM0644-741 925338 - -This will execute an ADQL query to download the first 10 observations in the Herschel Science Archive. The result of the query will be -stored in the file 'results.csv'. The result of this query can be printed by doing `print(result)`. + >>> result = HSA.query_hsa_tap("select top 10 * from hsa.v_active_observation", + ... output_format='csv', output_file='results.csv') + >>> result.pprint(max_width=100) + aor bii ... target_name urn_version + --------------------------------------- ------------------- ... -------------- ----------- + PP2-SWa-NGC3265 28.797292629881316 ... NGC3265 915907 + PRISMAS_W33a_hifi3b_898GHz_A_D2O -17.86672520275456 ... W33A 806737 + GOODS-S_70_d+8+8_forward_r3_shortaxis -27.80919396603746 ... GOODS-S d+8+8 894819 + PSP2_HStars-Set12f - RedRectangle -10.637417697356986 ... Red Rectangle 800938 + SPIRE-A - G126.24-5.52 57.195030974713134 ... G126.24-5.52 810242 + PSP1_PRISMAS_W31C_hifi6a_1477GHz_A_D2H+ -19.93074108498436 ... W31C 920099 + Spire Level-2 GOODS-S 37 - copy -27.81104151290488 ... GOODS-S 898135 + PSP2_HStars-Set13 - 10216 13.27923027337195 ... IRC+10216 801364 + PACS-A - G345.39-3.97 -43.47405026924179 ... G345.39-3.97-1 883176 + PRISMAS_g34_hifi7b_1897GHz_B_C3 1.2495150652937468 ... G34.3+0.1 921086 ----------------------------------- 5. Getting table details of HSA TAP From b4daf03add84f10cc6cc750e76c1b0a14224ea4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Wed, 13 Apr 2022 11:47:11 -0700 Subject: [PATCH 20/22] Fix sphinx links and headings --- docs/esa/hsa.rst | 70 +++++++++++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 31 deletions(-) diff --git a/docs/esa/hsa.rst b/docs/esa/hsa.rst index 5749075a1e..025155828e 100644 --- a/docs/esa/hsa.rst +++ b/docs/esa/hsa.rst @@ -4,21 +4,21 @@ Herschel Science Archive (`astroquery.esa.hsa`) *********************************************** -`Herschel `__ was the fourth cornerstone in ESA's Horizon 2000 science programme, designed to observe the 'cool' universe. +`Herschel `__ was the fourth +cornerstone in ESA's Horizon 2000 science programme, designed to observe the 'cool' universe. It performed photometry and spectroscopy in the poorly explored 55-670 µm spectral range with a 3.5 m diameter -Cassegrain telescope, providing unique observing capabilities and bridging the gap between earlier infrared space -missions and groundbased facilities. Herschel successfully performed ~37000 science observations and ~6600 science -calibration observations which are publicly available to the worldwide astronomical community through the Herschel Science Archive. +Cassegrain telescope, providing unique observing capabilities and bridging the gap between earlier infrared +space missions and groundbased facilities. Herschel successfully performed ~37000 science observations and +~6600 science calibration observations which are publicly available to the worldwide astronomical community +through the Herschel Science Archive. This package allows the access to the `Herschel Science Archive `__. -======== Examples ======== ------------------------------- 1. Getting Herschel data ------------------------------- +------------------------ .. doctest-remote-data:: @@ -31,13 +31,13 @@ Examples This will download the product of the observation '1342195355' with the instrument 'PACS' and it will store them in a tar called '1342195355.tar'. The parameters available are detailed in the API. -For more details of the parameters check the section 6 of the 'Direct Product Access using TAP' in the 'HSA users guide' at: - 'http://archives.esac.esa.int/hsa/whsa/' +For more details of the parameters check the section 6 of the ``Direct Product Access using TAP`` in the +`HSA users guide `_. For more details about the products check: - 'https://www.cosmos.esa.int/web/herschel/data-products-overview' + https://www.cosmos.esa.int/web/herschel/data-products-overview + -------------------------------- 2. Getting Observation Products ------------------------------- @@ -52,18 +52,20 @@ For more details about the products check: This will download the product of the observation '1342195355' with the instrument 'PACS' and it will store them in a tar called '1342195355.tar'. The parameters available are detailed in the API. -`Notice`: There is no difference between the product retrieved with this method and `download_data`. `download_data` is a more generic -interface that allows the user to retrieve any product or metadata and `get_observation` allows the user to retrieve only observation products. +.. Note:: There is no difference between the product retrieved with this method and + `~astroquery.esa.hsa.HSAClass.download_data`. `~astroquery.esa.hsa.HSAClass.download_data` + is a more generic interface that allows the user to retrieve any product or metadata and + `~astroquery.esa.hsa.HSAClass.get_observation` allows the user to retrieve only observation products. -For more information check the section 6.1 of the of the 'direct Product Access using TAP' in the 'HSA users guide' at: - 'http://archives.esac.esa.int/hsa/whsa/' +For more information check the section 6.1 of the of the ``Direct Product Access using TAP`` in the +`HSA users guide`_. + +For more details of the parameters check the section 6.2 of the ``Direct Product Access using TAP`` in the +`HSA users guide`_. -For more details of the parameters check the section 6.2 of the 'Direct Product Access using TAP' in the 'HSA users guide' at: - 'http://archives.esac.esa.int/hsa/whsa/' -------------------------------- 3. Getting Herschel Postcard -------------------------------- +---------------------------- .. doctest-remote-data:: @@ -73,15 +75,16 @@ For more details of the parameters check the section 6.2 of the 'Direct Product Downloading URL http://archives.esac.esa.int/hsa/whsa-tap-server/data?&retrieval_type=POSTCARD&observation_id=1342195355&instrument_name=PACS to /home/dev/.astropy/cache/astroquery/HSA/data?&retrieval_type=POSTCARD&observation_id=1342195355&instrument_name=PACS ... [Done] '1342195355.jpg' -This will download the postcard (static representation in JPG-format of the final product) of the observation '1342195355' with the instrument 'PACS' and -it will store them in a tar called '1342195355.jpg'. The parameters available are detailed in the API. +This will download the postcard (static representation in JPG-format of the final product) of the observation +'1342195355' with the instrument 'PACS' and it will store them in a tar called '1342195355.jpg'. +The parameters available are detailed in the API. + +For more details of the parameters check the section 6.2 of the ``Direct Product Access using TAP`` in the +`HSA users guide`_. -For more details of the parameters check the section 6.2 of the 'Direct Product Access using TAP' in the 'HSA users guide' at: - 'http://archives.esac.esa.int/hsa/whsa/' ------------------------------------------- 4. Getting Herschel metadata through TAP ------------------------------------------- +---------------------------------------- This function provides access to the Herschel Science Archive database using the Table Access Protocol (TAP) and via the Astronomical Data Query Language (`ADQL `__). @@ -106,7 +109,10 @@ Query Language (`ADQL >> HSA.download_data(observation_id='1342205057', retrieval_type='OBSERVATION', instrument_name='PACS') Downloading URL http://archives.esac.esa.int/hsa/whsa-tap-server/data?&retrieval_type=OBSERVATION&observation_id=1342205057&instrument_name=PACS to 1342205057.tar ... [Done] '1342205057.tar' + Reference/API ============= From aa4078c172489c747809f80b2b46effc193c7920 Mon Sep 17 00:00:00 2001 From: lvalerom Date: Thu, 14 Apr 2022 00:16:26 +0200 Subject: [PATCH 21/22] review comments --- astroquery/esa/hsa/core.py | 4 ++-- astroquery/esa/hsa/tests/dummy_handler.py | 22 +++++++++++----------- astroquery/esa/hsa/tests/setup_package.py | 14 -------------- 3 files changed, 13 insertions(+), 27 deletions(-) delete mode 100644 astroquery/esa/hsa/tests/setup_package.py diff --git a/astroquery/esa/hsa/core.py b/astroquery/esa/hsa/core.py index 689b7acce1..7a68e4e1bc 100644 --- a/astroquery/esa/hsa/core.py +++ b/astroquery/esa/hsa/core.py @@ -7,7 +7,7 @@ from pathlib import Path from astropy import units as u -from ...utils import commons +from astroquery.utils import commons from astroquery import log from astroquery.exceptions import LoginError from astroquery.query import BaseQuery @@ -25,7 +25,7 @@ class HSAClass(BaseQuery): timeout = conf.TIMEOUT def __init__(self, tap_handler=None): - super(HSAClass, self).__init__() + super().__init__() if tap_handler is None: self._tap = Tap(url=self.metadata_url) else: diff --git a/astroquery/esa/hsa/tests/dummy_handler.py b/astroquery/esa/hsa/tests/dummy_handler.py index 326e12b912..45fc35d0b9 100644 --- a/astroquery/esa/hsa/tests/dummy_handler.py +++ b/astroquery/esa/hsa/tests/dummy_handler.py @@ -21,30 +21,30 @@ def check_method(self, method): if method == self._invokedMethod: return else: - raise ValueError("Method '{}' is not invoked. (Invoked method \ - is '{}'.)").format(method, self_invokedMethod) + raise ValueError("Method '{}' is not invoked. (Invoked method " + "is '{}'.)".format(method, self._invokedMethod)) def check_parameters(self, parameters, method_name): if parameters is None: return len(self._parameters) == 0 if len(parameters) != len(self._parameters): - raise ValueError("Wrong number of parameters for method '{}'. \ - Found: {}. Expected {}").format( + raise ValueError("Wrong number of parameters for method '{}'. " + "Found: {}. Expected {}".format( method_name, len(self._parameters), - len(parameters)) + len(parameters))) for key in parameters: if key in self._parameters: # check value if self._parameters[key] != parameters[key]: - raise ValueError("Wrong '{}' parameter \ - value for method '{}'. \ - Found:'{}'. Expected:'{}'").format( + raise ValueError("Wrong '{}' parameter " + "value for method '{}'. " + "Found:'{}'. Expected:'{}'".format( method_name, key, self._parameters[key], - parameters[key]) + parameters[key])) else: - raise ValueError("Parameter '%s' not found in method '%s'", - (str(key), method_name)) + raise ValueError("Parameter '{}' not found in method '{}'" + .format(str(key), method_name)) return True diff --git a/astroquery/esa/hsa/tests/setup_package.py b/astroquery/esa/hsa/tests/setup_package.py deleted file mode 100644 index fbfb8d1e98..0000000000 --- a/astroquery/esa/hsa/tests/setup_package.py +++ /dev/null @@ -1,14 +0,0 @@ -# Licensed under a 3-clause BSD style license - see LICENSE.rst - -import os - - -# setup paths to the test data -# can specify a single file or a list of files -def get_package_data(): - paths = [os.path.join('data', '*.tar'), - os.path.join('data', '*.xml'), - ] # etc, add other extensions - # you can also enlist files individually by names - # finally construct and return a dict for the sub module - return {'astroquery.esa.hsa.tests': paths} From 914345308e00796040a90c97452a4c4578ee4d2f Mon Sep 17 00:00:00 2001 From: lvalerom Date: Thu, 14 Apr 2022 15:39:36 +0200 Subject: [PATCH 22/22] fix string formatting, and change skip to xfail --- astroquery/esa/hsa/core.py | 26 +++++++++---------- astroquery/esa/hsa/tests/dummy_handler.py | 25 +++++++----------- astroquery/esa/hsa/tests/dummy_tap_handler.py | 25 +++++++----------- astroquery/esa/hsa/tests/test_hsa_remote.py | 16 ++++++------ 4 files changed, 39 insertions(+), 53 deletions(-) diff --git a/astroquery/esa/hsa/core.py b/astroquery/esa/hsa/core.py index 7a68e4e1bc..d440bf3f7a 100644 --- a/astroquery/esa/hsa/core.py +++ b/astroquery/esa/hsa/core.py @@ -92,9 +92,9 @@ def download_data(self, *, retrieval_type="OBSERVATION", observation_id=None, if product_level is not None: params['product_level'] = product_level - link = self.data_url + "".join("&{0}={1}".format(key, val) for key, val in params.items()) + link = self.data_url + "".join(f"&{key}={val}" for key, val in params.items()) - link += "".join("&{0}={1}".format(key, val) for key, val in kwargs.items()) + link += "".join(f"&{key}={val}" for key, val in kwargs.items()) if verbose: log.info(link) @@ -128,7 +128,7 @@ def download_data(self, *, retrieval_type="OBSERVATION", observation_id=None, self._download_file(link, filename, head_safe=True, cache=cache) if verbose: - log.info("Wrote {0} to {1}".format(link, filename)) + log.info(f"Wrote {link} to {filename}") return filename @@ -191,9 +191,9 @@ def get_observation(self, observation_id, instrument_name, *, filename=None, if product_level is not None: params['product_level'] = product_level - link = self.data_url + "".join("&{0}={1}".format(key, val) for key, val in params.items()) + link = self.data_url + "".join(f"&{key}={val}" for key, val in params.items()) - link += "".join("&{0}={1}".format(key, val) for key, val in kwargs.items()) + link += "".join(f"&{key}={val}" for key, val in kwargs.items()) if verbose: log.info(link) @@ -220,7 +220,7 @@ def get_observation(self, observation_id, instrument_name, *, filename=None, self._download_file(link, filename, head_safe=True, cache=cache) if verbose: - log.info("Wrote {0} to {1}".format(link, filename)) + log.info(f"Wrote {link} to {filename}") return filename @@ -268,9 +268,9 @@ def get_postcard(self, observation_id, instrument_name, *, filename=None, 'observation_id': observation_id, 'instrument_name': instrument_name} - link = self.data_url + "".join("&{0}={1}".format(key, val) for key, val in params.items()) + link = self.data_url + "".join(f"&{key}={val}" for key, val in params.items()) - link += "".join("&{0}={1}".format(key, val) for key, val in kwargs.items()) + link += "".join(f"&{key}={val}" for key, val in kwargs.items()) if verbose: log.info(link) @@ -292,7 +292,7 @@ def get_postcard(self, observation_id, instrument_name, *, filename=None, shutil.move(local_filepath, filename) if verbose: - log.info("Wrote {0} to {1}".format(link, filename)) + log.info(f"Wrote {link} to {filename}") return filename @@ -401,10 +401,10 @@ def query_observations(self, coordinate, radius, *, n_obs=10): r = radius*u.deg coord = commons.parse_coordinates(coordinate).icrs - query = ("select top {} observation_id from hsa.v_active_observation " - "where contains(point('ICRS', hsa.v_active_observation.ra, " - "hsa.v_active_observation.dec), circle('ICRS', {},{},{}))=1")\ - .format(n_obs, coord.ra.degree, coord.dec.degree, r.to(u.deg).value) + query = (f"select top {n_obs} observation_id from hsa.v_active_observation " + f"where contains(" + f"point('ICRS', hsa.v_active_observation.ra, hsa.v_active_observation.dec), " + f"circle('ICRS', {coord.ra.degree},{coord.dec.degree},{r.to(u.deg).value}))=1") return self.query_hsa_tap(query) diff --git a/astroquery/esa/hsa/tests/dummy_handler.py b/astroquery/esa/hsa/tests/dummy_handler.py index 45fc35d0b9..05bfcc982f 100644 --- a/astroquery/esa/hsa/tests/dummy_handler.py +++ b/astroquery/esa/hsa/tests/dummy_handler.py @@ -21,30 +21,23 @@ def check_method(self, method): if method == self._invokedMethod: return else: - raise ValueError("Method '{}' is not invoked. (Invoked method " - "is '{}'.)".format(method, self._invokedMethod)) + raise ValueError(f"Method '{method}' is not invoked. (Invoked method " + f"is '{self._invokedMethod}'.") def check_parameters(self, parameters, method_name): if parameters is None: return len(self._parameters) == 0 if len(parameters) != len(self._parameters): - raise ValueError("Wrong number of parameters for method '{}'. " - "Found: {}. Expected {}".format( - method_name, - len(self._parameters), - len(parameters))) + raise ValueError(f"Wrong number of parameters for method '{method_name}'. " + f"Found: {len(self._parameters)}. Expected {len(parameters)}") for key in parameters: if key in self._parameters: # check value if self._parameters[key] != parameters[key]: - raise ValueError("Wrong '{}' parameter " - "value for method '{}'. " - "Found:'{}'. Expected:'{}'".format( - method_name, - key, - self._parameters[key], - parameters[key])) + raise ValueError(f"Wrong '{key}' parameter " + f"value for method '{method_name}'. " + f"Found:'{self._parameters[key]}'. " + f"Expected:'{parameters[key]}'") else: - raise ValueError("Parameter '{}' not found in method '{}'" - .format(str(key), method_name)) + raise ValueError(f"Parameter '{key}' not found in method '{method_name}'") return True diff --git a/astroquery/esa/hsa/tests/dummy_tap_handler.py b/astroquery/esa/hsa/tests/dummy_tap_handler.py index 20c7d53d25..e898bf60a5 100644 --- a/astroquery/esa/hsa/tests/dummy_tap_handler.py +++ b/astroquery/esa/hsa/tests/dummy_tap_handler.py @@ -20,32 +20,25 @@ def check_call(self, method_name, parameters): def check_method(self, method): if method != self.__invokedMethod: - raise Exception("Method '" + str(method) + "" - "' not invoked. (Invoked method is '" - "" + str(self.__invokedMethod)+"')") + raise Exception(f"Method '{method}' " + f"not invoked. (Invoked method is " + f"'{self.__invokedMethod}')") def check_parameters(self, parameters, method_name): if parameters is None: return len(self._parameters) == 0 if len(parameters) != len(self._parameters): - raise Exception("Wrong number of parameters for method '%s'. \ - Found: %d. Expected %d", - (method_name, - len(self._parameters), - len(parameters))) + raise Exception(f"Wrong number of parameters for method '{method_name}'. " + f"Found: {len(self._parameters)}. Expected {len(parameters)}") for key in parameters: if key in self._parameters: # check value if self._parameters[key] != parameters[key]: - raise Exception("Wrong '%s' parameter value for method '%s'. \ - Found: '%s'. Expected: '%s'", ( - method_name, - key, - self._parameters[key], - parameters[key])) + raise Exception(f"Wrong '{key}' parameter value for method " + f"'{method_name}'. " + f"Found: '{self._parameters[key]}'. Expected: '{parameters[key]}'") else: - raise Exception("Parameter '%s' not found for method '%s'", - (str(key), method_name)) + raise Exception("Parameter '{key}' not found for method 'method_name'") return False def launch_job(self, query, name=None, output_file=None, diff --git a/astroquery/esa/hsa/tests/test_hsa_remote.py b/astroquery/esa/hsa/tests/test_hsa_remote.py index b73d62de5f..0d8d5efe1a 100644 --- a/astroquery/esa/hsa/tests/test_hsa_remote.py +++ b/astroquery/esa/hsa/tests/test_hsa_remote.py @@ -39,7 +39,7 @@ def test_download_data_observation_pacs(self): hsa = HSAClass() res = self.access_archive_with_retries(hsa.download_data, parameters) if res is None: - pytest.skip("Archive broke the connection {} times, unable to test".format(self.retries)) + pytest.xfail(f"Archive broke the connection {self.retries} times, unable to test") assert res == expected_res assert os.path.isfile(res) tar = tarfile.open(res) @@ -61,7 +61,7 @@ def test_download_data_observation_pacs_filename(self): hsa = HSAClass() res = self.access_archive_with_retries(hsa.download_data, parameters) if res is None: - pytest.skip("Archive broke the connection {} times, unable to test".format(self.retries)) + pytest.xfail(f"Archive broke the connection {self.retries} times, unable to test") assert res == expected_res assert os.path.isfile(res) tar = tarfile.open(res) @@ -82,7 +82,7 @@ def test_download_data_observation_pacs_compressed(self): hsa = HSAClass() res = self.access_archive_with_retries(hsa.download_data, parameters) if res is None: - pytest.skip("Archive broke the connection {} times, unable to test".format(self.retries)) + pytest.xfail(f"Archive broke the connection {self.retries} times, unable to test") assert res == expected_res assert os.path.isfile(res) tar = tarfile.open(res) @@ -102,7 +102,7 @@ def test_download_data_observation_spire(self): hsa = HSAClass() res = self.access_archive_with_retries(hsa.download_data, parameters) if res is None: - pytest.skip("Archive broke the connection {} times, unable to test".format(self.retries)) + pytest.xfail(f"Archive broke the connection {self.retries} times, unable to test") assert res == expected_res assert os.path.isfile(res) tar = tarfile.open(res) @@ -121,7 +121,7 @@ def test_download_data_postcard_pacs(self): hsa = HSAClass() res = self.access_archive_with_retries(hsa.download_data, parameters) if res is None: - pytest.skip("Archive broke the connection {} times, unable to test".format(self.retries)) + pytest.xfail(f"Archive broke the connection {self.retries} times, unable to test") assert res == expected_res assert os.path.isfile(res) os.remove(res) @@ -139,7 +139,7 @@ def test_download_data_postcard_pacs_filename(self): hsa = HSAClass() res = self.access_archive_with_retries(hsa.download_data, parameters) if res is None: - pytest.skip("Archive broke the connection {} times, unable to test".format(self.retries)) + pytest.xfail(f"Archive broke the connection {self.retries} times, unable to test") assert res == expected_res assert os.path.isfile(res) os.remove(res) @@ -155,7 +155,7 @@ def test_get_observation(self): hsa = HSAClass() res = self.access_archive_with_retries(hsa.get_observation, parameters) if res is None: - pytest.skip("Archive broke the connection {} times, unable to test".format(self.retries)) + pytest.xfail(f"Archive broke the connection {self.retries} times, unable to test") assert res == expected_res assert os.path.isfile(res) tar = tarfile.open(res) @@ -173,7 +173,7 @@ def test_get_postcard(self): hsa = HSAClass() res = self.access_archive_with_retries(hsa.get_postcard, parameters) if res is None: - pytest.skip("Archive broke the connection {} times, unable to test".format(self.retries)) + pytest.xfail(f"Archive broke the connection {self.retries} times, unable to test") assert res == expected_res assert os.path.isfile(res) os.remove(res)