From 7d6dc4d9cec6044c5d4366ede6c945c33436043a Mon Sep 17 00:00:00 2001 From: Adam Ginsburg Date: Mon, 13 Dec 2021 09:40:07 -0500 Subject: [PATCH 01/87] Expand contribution guidelines --- CONTRIBUTING.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 7158f7d3ea..33b522ece3 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -6,6 +6,11 @@ Please see `astropy's contributing guildelines `__ for a general guide to the workflow involving git, etc. Everything below is astroquery-specific. +We strongly encourage draft pull requests to be opened early in development. +If you are thinking of contributing a new module, please open a pull request +as soon as you start developing code and mark it "WIP" (work in progress). + + New Features ------------ We welcome any and all new features! If you have your own little query tool @@ -55,6 +60,14 @@ method is a wrapper around the `requests.request` function that provides important astroquery-specific utility, including caching, HTTP header generation, progressbars, and local writing-to-disk. +Dependencies +------------ +New contributions are generally not allowed to bring along additional dependencies. + +File reading, table parsing, etc. should be done with astropy tables. + + + .. _astroquery API: docs/api.rst .. _template: docs/template.rst .. _requests: http://docs.python-requests.org/en/master/ From 522061e16d83cb7d4a58ed4ee85936a9aa78724a Mon Sep 17 00:00:00 2001 From: Adam Ginsburg Date: Mon, 13 Dec 2021 11:06:44 -0500 Subject: [PATCH 02/87] Update CONTRIBUTING.rst --- CONTRIBUTING.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 33b522ece3..8afdb46433 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -8,7 +8,7 @@ workflow involving git, etc. Everything below is astroquery-specific. We strongly encourage draft pull requests to be opened early in development. If you are thinking of contributing a new module, please open a pull request -as soon as you start developing code and mark it "WIP" (work in progress). +as soon as you start developing code and mark it as a Draft PR on github. New Features @@ -64,7 +64,10 @@ Dependencies ------------ New contributions are generally not allowed to bring along additional dependencies. -File reading, table parsing, etc. should be done with astropy tables. +The astropy ecosystem tools should be used whenever possible. +For example, astropy.table should be used for table handling, +astropy.io.ascii for ascii-table parsing, and astropy.units for unit and quantity +handling. From eba20f8554d34b7f420290d9963bd67c19809567 Mon Sep 17 00:00:00 2001 From: Adam Ginsburg Date: Mon, 13 Dec 2021 12:54:14 -0500 Subject: [PATCH 03/87] Update CONTRIBUTING.rst Co-authored-by: Eero Vaher --- CONTRIBUTING.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 8afdb46433..ca60939e13 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -65,8 +65,8 @@ Dependencies New contributions are generally not allowed to bring along additional dependencies. The astropy ecosystem tools should be used whenever possible. -For example, astropy.table should be used for table handling, -astropy.io.ascii for ascii-table parsing, and astropy.units for unit and quantity +For example, `astropy.table` should be used for table handling, +`astropy.io.ascii` for ascii-table parsing, and `astropy.units` for unit and quantity handling. From c66f83d11286f7f60855e5bc4956e302982513b0 Mon Sep 17 00:00:00 2001 From: Eero Vaher Date: Tue, 14 Dec 2021 14:18:32 +0100 Subject: [PATCH 04/87] Deprecate astroquery/utils/download_file_list.py This commit deprecates the function `download_list_of_fitsfiles()`, and because that is the only public function defined by the aforementioned file then the whole file becomes deprecated. --- astroquery/utils/download_file_list.py | 2 ++ astroquery/utils/tests/test_download_file_list.py | 10 ++++++++++ 2 files changed, 12 insertions(+) create mode 100644 astroquery/utils/tests/test_download_file_list.py diff --git a/astroquery/utils/download_file_list.py b/astroquery/utils/download_file_list.py index 8df86a231d..1ab484dea0 100644 --- a/astroquery/utils/download_file_list.py +++ b/astroquery/utils/download_file_list.py @@ -6,6 +6,7 @@ from io import StringIO import astropy.io.fits as fits +from astropy.utils.decorators import deprecated from .commons import get_readable_fileobj __all__ = ['download_list_of_fitsfiles'] @@ -26,6 +27,7 @@ def validify_filename(filestr): return filestr +@deprecated('0.4.5') def download_list_of_fitsfiles(linklist, output_directory=None, output_prefix=None, save=False, overwrite=False, verbose=False, diff --git a/astroquery/utils/tests/test_download_file_list.py b/astroquery/utils/tests/test_download_file_list.py new file mode 100644 index 0000000000..9bf03861fc --- /dev/null +++ b/astroquery/utils/tests/test_download_file_list.py @@ -0,0 +1,10 @@ +import pytest + +from astropy.utils.exceptions import AstropyDeprecationWarning + +from astroquery.utils.download_file_list import download_list_of_fitsfiles + + +def test_download_list_of_fitsfiles_deprecation(): + with pytest.warns(AstropyDeprecationWarning): + download_list_of_fitsfiles([]) From a96365f08f185987a3f6030c2e212881791d4a9e Mon Sep 17 00:00:00 2001 From: Eero Vaher Date: Tue, 14 Dec 2021 14:26:00 +0100 Subject: [PATCH 05/87] Add change log entry for #2247 --- CHANGES.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 424981f8b5..57263e8da5 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -29,6 +29,9 @@ Infrastructure, Utility and Other Changes and Additions - Adding ``--alma-site`` pytest option for testing to have a control over which specific site to test. [#2224] +- The function ``astroquery.utils.download_list_of_fitsfiles()`` has been + deprecated. [#2247] + utils.tap ^^^^^^^^^ From ecd828a1bec430b610160788a082a40f795cb6cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Fri, 22 Oct 2021 17:15:12 -0700 Subject: [PATCH 06/87] [MNT] Adding python 3.10 testing to CI --- .github/workflows/ci_tests.yml | 6 +++--- tox.ini | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index 0a80a686c0..43f1bba5d9 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -29,7 +29,7 @@ jobs: - name: docs build os: ubuntu-latest - python: 3.8 + python: 3.9 toxenv: build_docs toxargs: -v @@ -41,8 +41,8 @@ jobs: - name: astropy dev with all dependencies with coverage os: ubuntu-latest - python: 3.9 - toxenv: py39-test-alldeps-devastropy-cov + python: 3.10 + toxenv: py310-test-alldeps-devastropy-cov toxargs: -v - name: Python 3.7 with all optional dependencies (MacOS X) diff --git a/tox.ini b/tox.ini index 461dc4aae8..a66060fd2c 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] envlist = - py{37,38,39}-test{,-alldeps,-oldestdeps}{,-devastropy}{,-cov} + py{37,38,39,310}-test{,-alldeps,-oldestdeps}{,-devastropy}{,-cov} codestyle build_docs requires = @@ -41,8 +41,8 @@ deps = extras = test - !py39-alldeps: all_lt_39 - py39-alldeps: all + !py{37,38}-alldeps: all_lt_39 + py{39,310}-alldeps: all commands = pip freeze From 7af45423027386174af4d07788a2c75d2c41884c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Mon, 1 Nov 2021 16:31:56 -0700 Subject: [PATCH 07/87] Fix YAML parsing issue --- .github/workflows/ci_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index 43f1bba5d9..edf1adf0b8 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -41,7 +41,7 @@ jobs: - name: astropy dev with all dependencies with coverage os: ubuntu-latest - python: 3.10 + python: '3.10' toxenv: py310-test-alldeps-devastropy-cov toxargs: -v From d9a31e6e7ea161448d198b91dfdd398b9ccec517 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Tue, 2 Nov 2021 00:48:19 -0700 Subject: [PATCH 08/87] Can't use mocpy for 3.10 just yet, but all else should work --- setup.cfg | 5 +++++ tox.ini | 5 +++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 71f8016780..cf00bec63b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -168,6 +168,11 @@ docs= sphinx-astropy>=1.5 scipy all= +# mocpy's dependency cdshealpix doesn't compile with 3.10 yet + astropy-healpix + boto3 + regions +all_lt_310= mocpy>=0.5.2 astropy-healpix boto3 diff --git a/tox.ini b/tox.ini index a66060fd2c..9c9a64fd59 100644 --- a/tox.ini +++ b/tox.ini @@ -41,8 +41,9 @@ deps = extras = test - !py{37,38}-alldeps: all_lt_39 - py{39,310}-alldeps: all + py{37,38}-alldeps: all_lt_39 + py39-alldeps: all_lt_310 + py310-alldeps: all commands = pip freeze From 6d1dd20fc71d0c2daddf1f012be8f4d08371ba50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Thu, 11 Nov 2021 16:45:27 -0800 Subject: [PATCH 09/87] Ignoring deprecation coming from astropy <5 --- setup.cfg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.cfg b/setup.cfg index cf00bec63b..a7500db5f6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -57,6 +57,8 @@ filterwarnings = # Upstream, remove when fixed, PRs have been opened ignore::DeprecationWarning:pyvo ignore::DeprecationWarning:regions +# Should ignore these for astropy<5.0 + ignore:getName|currentThread:DeprecationWarning:astropy # This should be cleared once we requre astropy>=4.1 ignore:tostring\(\) is deprecated. Use tobytes:DeprecationWarning:astropy markers = From a8154fd9c87994ccf7e288e18bfee5fd305a75ca Mon Sep 17 00:00:00 2001 From: "Michael S. P. Kelley" Date: Mon, 20 Dec 2021 15:27:37 -0500 Subject: [PATCH 10/87] fix for issue #2237: do not cache results that cannot be parsed. --- astroquery/jplhorizons/core.py | 17 +++++++++++++++-- astroquery/jplhorizons/tests/data/README | 5 ++++- .../jplhorizons/tests/data/tlist_error.txt | 5 +++++ .../jplhorizons/tests/test_jplhorizons.py | 12 +++++++++++- 4 files changed, 35 insertions(+), 4 deletions(-) create mode 100644 astroquery/jplhorizons/tests/data/tlist_error.txt diff --git a/astroquery/jplhorizons/core.py b/astroquery/jplhorizons/core.py index fbef822aa4..86e378c9f9 100644 --- a/astroquery/jplhorizons/core.py +++ b/astroquery/jplhorizons/core.py @@ -20,6 +20,7 @@ from ..query import BaseQuery # async_to_sync generates the relevant query tools from _async methods from ..utils import async_to_sync +from ..exceptions import TableParseError # import configurable items declared in __init__.py from . import conf @@ -1287,11 +1288,23 @@ def _parse_result(self, response, verbose=None): data : `astropy.Table` """ + self.last_response = response if self.query_type not in ['ephemerides', 'elements', 'vectors']: return None else: - data = self._parse_horizons(response.text) - + try: + data = self._parse_horizons(response.text) + except Exception as ex: + try: + self._last_query.remove_cache_file(self.cache_location) + except OSError: + # this is allowed: if `cache` was set to False, this + # won't be needed + pass + raise TableParseError("Failed to parse JPL Horizons result. " + "The raw response can be found in " + "`self.last_response`. Exception: " + + str(ex)) from ex return data diff --git a/astroquery/jplhorizons/tests/data/README b/astroquery/jplhorizons/tests/data/README index 75b8a933c2..12cf5764f9 100644 --- a/astroquery/jplhorizons/tests/data/README +++ b/astroquery/jplhorizons/tests/data/README @@ -10,4 +10,7 @@ ceres_vectors.txt https://ssd.jpl.nasa.gov/api/horizons.api?format=text&EPHEM_TYPE=VECTORS&OUT_UNITS=AU-D&COMMAND=%22Ceres%3B%22&CENTER=%27500%4010%27&CSV_FORMAT=%22YES%22&REF_PLANE=ECLIPTIC&REF_SYSTEM=ICRF&TP_TYPE=ABSOLUTE&VEC_LABELS=YES&VEC_CORR=%22NONE%22&VEC_DELTA_T=NO&OBJ_DATA=YES&TLIST=2451544.5 no_H.txt -https://ssd.jpl.nasa.gov/api/horizons.api?format=text&EPHEM_TYPE=OBSERVER&QUANTITIES=%271%2C2%2C3%2C4%2C5%2C6%2C7%2C8%2C9%2C10%2C11%2C12%2C13%2C14%2C15%2C16%2C17%2C18%2C19%2C20%2C21%2C22%2C23%2C24%2C25%2C26%2C27%2C28%2C29%2C30%2C31%2C32%2C33%2C34%2C35%2C36%2C37%2C38%2C39%2C40%2C41%2C42%2C43%27&COMMAND=%221935+UZ%3B%22&SOLAR_ELONG=%220%2C180%22&LHA_CUTOFF=0&CSV_FORMAT=YES&CAL_FORMAT=BOTH&ANG_FORMAT=DEG&APPARENT=AIRLESS&REF_SYSTEM=ICRF&EXTRA_PREC=NO&CENTER=%27500%40399%27&TLIST=2459480.5004416634&SKIP_DAYLT=NO \ No newline at end of file +https://ssd.jpl.nasa.gov/api/horizons.api?format=text&EPHEM_TYPE=OBSERVER&QUANTITIES=%271%2C2%2C3%2C4%2C5%2C6%2C7%2C8%2C9%2C10%2C11%2C12%2C13%2C14%2C15%2C16%2C17%2C18%2C19%2C20%2C21%2C22%2C23%2C24%2C25%2C26%2C27%2C28%2C29%2C30%2C31%2C32%2C33%2C34%2C35%2C36%2C37%2C38%2C39%2C40%2C41%2C42%2C43%27&COMMAND=%221935+UZ%3B%22&SOLAR_ELONG=%220%2C180%22&LHA_CUTOFF=0&CSV_FORMAT=YES&CAL_FORMAT=BOTH&ANG_FORMAT=DEG&APPARENT=AIRLESS&REF_SYSTEM=ICRF&EXTRA_PREC=NO&CENTER=%27500%40399%27&TLIST=2459480.5004416634&SKIP_DAYLT=NO + +tlist_error.txt +https://ssd.jpl.nasa.gov/api/horizons.api?format=text&COMMAND=%27499%27&OBJ_DATA=%27YES%27&MAKE_EPHEM=%27YES%27&EPHEM_TYPE=%27OBSERVER%27&CENTER=%27500@399%27&QUANTITIES=%271,9,20,23,24,29%27&TLIST=[] diff --git a/astroquery/jplhorizons/tests/data/tlist_error.txt b/astroquery/jplhorizons/tests/data/tlist_error.txt new file mode 100644 index 0000000000..cdbc9b32b4 --- /dev/null +++ b/astroquery/jplhorizons/tests/data/tlist_error.txt @@ -0,0 +1,5 @@ +API VERSION: 1.1 +API SOURCE: NASA/JPL Horizons API + +BATVAR: no TLIST values found (or missing quotes) +WLDINI: error loading execution-control file. diff --git a/astroquery/jplhorizons/tests/test_jplhorizons.py b/astroquery/jplhorizons/tests/test_jplhorizons.py index b8b03f29c3..2474afe3f2 100644 --- a/astroquery/jplhorizons/tests/test_jplhorizons.py +++ b/astroquery/jplhorizons/tests/test_jplhorizons.py @@ -9,13 +9,16 @@ from astropy.utils.exceptions import AstropyDeprecationWarning from ...utils.testing_tools import MockResponse +from ...query import AstroQuery +from ...exceptions import TableParseError from ... import jplhorizons # files in data/ for different query types DATA_FILES = {'ephemerides': 'ceres_ephemerides.txt', 'elements': 'ceres_elements.txt', 'vectors': 'ceres_vectors.txt', - '"1935 UZ"': 'no_H.txt'} + '"1935 UZ"': 'no_H.txt', + '"tlist_error"': 'tlist_error.txt'} def data_path(filename): @@ -55,6 +58,13 @@ def patch_request(request): # --------------------------------- actual test functions +def test_parse_result(patch_request): + q = jplhorizons.Horizons(id='tlist_error') + # need _last_query to be defined + q._last_query = AstroQuery('GET', 'http://dummy') + with pytest.raises(TableParseError): + res = q.ephemerides() + def test_ephemerides_query(patch_request): # check values of Ceres for a given epoch From 55d66191dd6216af1f7b43afd462ea3062e8ecec Mon Sep 17 00:00:00 2001 From: "Michael S. P. Kelley" Date: Mon, 20 Dec 2021 16:06:14 -0500 Subject: [PATCH 11/87] Restore original exceptions. --- astroquery/jplhorizons/core.py | 6 +----- astroquery/jplhorizons/tests/test_jplhorizons.py | 3 ++- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/astroquery/jplhorizons/core.py b/astroquery/jplhorizons/core.py index 86e378c9f9..272e7f97fa 100644 --- a/astroquery/jplhorizons/core.py +++ b/astroquery/jplhorizons/core.py @@ -20,7 +20,6 @@ from ..query import BaseQuery # async_to_sync generates the relevant query tools from _async methods from ..utils import async_to_sync -from ..exceptions import TableParseError # import configurable items declared in __init__.py from . import conf @@ -1301,10 +1300,7 @@ def _parse_result(self, response, verbose=None): # this is allowed: if `cache` was set to False, this # won't be needed pass - raise TableParseError("Failed to parse JPL Horizons result. " - "The raw response can be found in " - "`self.last_response`. Exception: " - + str(ex)) from ex + raise return data diff --git a/astroquery/jplhorizons/tests/test_jplhorizons.py b/astroquery/jplhorizons/tests/test_jplhorizons.py index 2474afe3f2..98cd7c9ae8 100644 --- a/astroquery/jplhorizons/tests/test_jplhorizons.py +++ b/astroquery/jplhorizons/tests/test_jplhorizons.py @@ -1,5 +1,6 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst +from multiprocessing import Value import pytest import os from collections import OrderedDict @@ -62,7 +63,7 @@ def test_parse_result(patch_request): q = jplhorizons.Horizons(id='tlist_error') # need _last_query to be defined q._last_query = AstroQuery('GET', 'http://dummy') - with pytest.raises(TableParseError): + with pytest.raises(ValueError): res = q.ephemerides() From 55418155256c8dda97a77815fa376189b9c7c555 Mon Sep 17 00:00:00 2001 From: Adam Ginsburg Date: Tue, 21 Dec 2021 10:49:00 -0500 Subject: [PATCH 12/87] Update CONTRIBUTING.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Brigitta Sipőcz --- CONTRIBUTING.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index ca60939e13..ac6ad0b023 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -66,7 +66,7 @@ New contributions are generally not allowed to bring along additional dependenci The astropy ecosystem tools should be used whenever possible. For example, `astropy.table` should be used for table handling, -`astropy.io.ascii` for ascii-table parsing, and `astropy.units` for unit and quantity +or `astropy.units` for unit and quantity handling. From d3d39c2630666a7c2210d6d5d98e870a4937dd3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Thu, 23 Dec 2021 12:13:47 -0800 Subject: [PATCH 13/87] dependencies are now python 3.10 compatible. But they may not be cross compatible, address that in a new commit (e.g. aplpy with astropy 5.0) --- setup.cfg | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/setup.cfg b/setup.cfg index a7500db5f6..71bd1ce2fa 100644 --- a/setup.cfg +++ b/setup.cfg @@ -170,21 +170,9 @@ docs= sphinx-astropy>=1.5 scipy all= -# mocpy's dependency cdshealpix doesn't compile with 3.10 yet - astropy-healpix - boto3 - regions -all_lt_310= mocpy>=0.5.2 astropy-healpix boto3 regions -# aplpy is not py39 compatible (it requires shapely that doesn't compile -# pyregion is not py39 compatible -all_lt_39= - mocpy>=0.5.2 - regions pyregion - astropy-healpix aplpy - boto3 From 0b02ddc02a4e63ed117379aeeee89388cd52733a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Thu, 23 Dec 2021 15:19:26 -0800 Subject: [PATCH 14/87] Adding top level conftest to make the test header work with tox --- conftest.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 conftest.py diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000000..50518c70a7 --- /dev/null +++ b/conftest.py @@ -0,0 +1,32 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst + +from pytest_astropy_header.display import (PYTEST_HEADER_MODULES, + TESTED_VERSIONS) + + +def pytest_configure(config): + config.option.astropy_header = True + + PYTEST_HEADER_MODULES['Astropy'] = 'astropy' + PYTEST_HEADER_MODULES['APLpy'] = 'aplpy' + PYTEST_HEADER_MODULES['pyregion'] = 'pyregion' + PYTEST_HEADER_MODULES['regions'] = 'regions' + PYTEST_HEADER_MODULES['pyVO'] = 'pyvo' + PYTEST_HEADER_MODULES['mocpy'] = 'mocpy' + PYTEST_HEADER_MODULES['astropy-healpix'] = 'astropy_healpix' + PYTEST_HEADER_MODULES['vamdclib'] = 'vamdclib' + + # keyring doesn't provide __version__ any more + # PYTEST_HEADER_MODULES['keyring'] = 'keyring' + + del PYTEST_HEADER_MODULES['h5py'] + del PYTEST_HEADER_MODULES['Scipy'] + del PYTEST_HEADER_MODULES['Pandas'] + + # add '_testrun' to the version name so that the user-agent indicates that + # it's being run in a test + from astroquery import version + version.version += '_testrun' + + TESTED_VERSIONS['astroquery'] = version.version + TESTED_VERSIONS['astropy_helpers'] = version.astropy_helpers_version From fd6b7fd4705e3b4b1163983c36acfac1fda3beaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Fri, 24 Dec 2021 22:06:33 -0800 Subject: [PATCH 15/87] Updating version in GHA and remove docs build in favour of RTD --- .github/workflows/ci_tests.yml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index edf1adf0b8..a1f97b9dbe 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -27,12 +27,6 @@ jobs: python: 3.x toxenv: codestyle - - name: docs build - os: ubuntu-latest - python: 3.9 - toxenv: build_docs - toxargs: -v - - name: oldest dependencies os: ubuntu-latest python: 3.7 @@ -45,16 +39,16 @@ jobs: toxenv: py310-test-alldeps-devastropy-cov toxargs: -v - - name: Python 3.7 with all optional dependencies (MacOS X) + - name: Python 3.8 with all optional dependencies (MacOS X) os: macos-latest - python: 3.7 - toxenv: py37-test-alldeps + python: 3.8 + toxenv: py38-test-alldeps toxargs: -v - - name: Python 3.8 with mandatory dependencies (Windows) + - name: Python 3.9 with mandatory dependencies (Windows) os: windows-latest - python: 3.8 - toxenv: py38-test + python: 3.9 + toxenv: py39-test toxargs: -v steps: From 482144b7d5ae67ca7c9c3c7c1e9f200dea0a9d68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Fri, 24 Dec 2021 22:12:36 -0800 Subject: [PATCH 16/87] Updating the RTD dependencies, too --- .readthedocs.yaml | 2 +- tox.ini | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index a99592c422..f953c613c6 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -9,7 +9,7 @@ python: path: . extra_requirements: - docs - - all_lt_39 + - all sphinx: fail_on_warning: true diff --git a/tox.ini b/tox.ini index 9c9a64fd59..5ca194cb7b 100644 --- a/tox.ini +++ b/tox.ini @@ -41,9 +41,8 @@ deps = extras = test - py{37,38}-alldeps: all_lt_39 - py39-alldeps: all_lt_310 - py310-alldeps: all + alldeps: all + commands = pip freeze From 3667dac603360e58c0635db00cce9d9dd29419dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Fri, 24 Dec 2021 22:20:06 -0800 Subject: [PATCH 17/87] Fix conftest --- conftest.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/conftest.py b/conftest.py index 50518c70a7..1485a0c781 100644 --- a/conftest.py +++ b/conftest.py @@ -19,10 +19,6 @@ def pytest_configure(config): # keyring doesn't provide __version__ any more # PYTEST_HEADER_MODULES['keyring'] = 'keyring' - del PYTEST_HEADER_MODULES['h5py'] - del PYTEST_HEADER_MODULES['Scipy'] - del PYTEST_HEADER_MODULES['Pandas'] - # add '_testrun' to the version name so that the user-agent indicates that # it's being run in a test from astroquery import version From ee6b7dc919b68190ea569209f92295c5b9786f98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Fri, 24 Dec 2021 22:31:09 -0800 Subject: [PATCH 18/87] Finalizing changelog for v0.4.5 --- CHANGES.rst | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 57263e8da5..e2f521419f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,4 +1,4 @@ -0.4.5 (unreleased) +0.4.5 (2021-12-24) ================== New Tools and Services @@ -7,7 +7,7 @@ New Tools and Services esa.jwst ^^^^^^^^^^ -- New module to provide access to eJWST Science Archive metadata and datasets. [#2140, #2238, #2243] +- New module to provide access to eJWST Science Archive metadata and datasets. [#2140, #2238] Service fixes and enhancements @@ -16,7 +16,12 @@ Service fixes and enhancements eso ^^^ -- Add option to retrieve_data from an earlier archive query [#1614] +- Add option to retrieve_data from an earlier archive query. [#1614] + +jplhorizons +^^^^^^^^^^^ + +- Fix result parsing issues by disabling caching of failed queries. [#2253] sdss ^^^^ From f8a95ca3983b790b4e9b8d2fe0f0f33b10adeae5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Fri, 24 Dec 2021 22:32:06 -0800 Subject: [PATCH 19/87] Preparing release v0.4.5 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 71bd1ce2fa..d493203e54 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = astroquery -version = 0.4.5.dev +version = 0.4.5 description = Functions and classes to access online astronomical data resources # FIXME long_description = author = The Astroquery Developers From f915a1509390c2738b76e0fe8a144c19db7f9efa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Fri, 24 Dec 2021 22:34:00 -0800 Subject: [PATCH 20/87] Back to development: v0.4.6.dev --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index d493203e54..1fd60737ac 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = astroquery -version = 0.4.5 +version = 0.4.6.dev description = Functions and classes to access online astronomical data resources # FIXME long_description = author = The Astroquery Developers From 04f4312b7ed337e809214f2c5dd9493b5d52e15c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Fri, 24 Dec 2021 22:35:31 -0800 Subject: [PATCH 21/87] Adding 0.4.6 section to changelog --- CHANGES.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index e2f521419f..e15b22097f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,20 @@ +0.4.6 (unreleased) +================== + +New Tools and Services +---------------------- + + +Service fixes and enhancements +------------------------------ + + +Infrastructure, Utility and Other Changes and Additions +------------------------------------------------------- + + + + 0.4.5 (2021-12-24) ================== From d422bc1a5f7d0f95e3b4c2d1625a1dd682659167 Mon Sep 17 00:00:00 2001 From: "Pey Lian Lim (Github)" <2090236+pllim@users.noreply.github.com> Date: Wed, 29 Dec 2021 16:19:45 -0500 Subject: [PATCH 22/87] MNT: Do not use deprecated test helper funcs --- astroquery/esa/hubble/tests/test_esa_hubble_remote.py | 9 --------- astroquery/mast/tests/test_mast.py | 8 ++++---- astroquery/noirlab/tests/test_noirlab_remote.py | 8 ++++---- .../vo_conesearch/validator/tests/test_validate.py | 5 ++--- docs/testing.rst | 2 +- 5 files changed, 11 insertions(+), 21 deletions(-) diff --git a/astroquery/esa/hubble/tests/test_esa_hubble_remote.py b/astroquery/esa/hubble/tests/test_esa_hubble_remote.py index 605621e10d..eda1438c99 100644 --- a/astroquery/esa/hubble/tests/test_esa_hubble_remote.py +++ b/astroquery/esa/hubble/tests/test_esa_hubble_remote.py @@ -13,21 +13,12 @@ """ import tempfile -import unittest import os import pytest -from astropy.tests.helper import remote_data -from requests.models import Response from astroquery.esa.hubble import ESAHubble -from astroquery.utils.testing_tools import MockResponse from astropy import coordinates -from unittest.mock import MagicMock -from astropy.table.table import Table -import shutil import random -from PIL import Image - esa_hubble = ESAHubble() diff --git a/astroquery/mast/tests/test_mast.py b/astroquery/mast/tests/test_mast.py index 7f1d9f685f..a4071210a9 100644 --- a/astroquery/mast/tests/test_mast.py +++ b/astroquery/mast/tests/test_mast.py @@ -2,11 +2,11 @@ import os import re - from shutil import copyfile +import pytest + from astropy.table import Table -from astropy.tests.helper import pytest from astropy.coordinates import SkyCoord from astropy.io import fits @@ -667,7 +667,7 @@ def test_zcut_download_cutouts(patch_post, tmpdir): # Testing with png cutout_table = mast.Zcut.download_cutouts(coordinates=coord, size=5, - cutout_format="jpg", path=str(tmpdir)) + cutout_format="jpg", path=str(tmpdir)) assert isinstance(cutout_table, Table) assert len(cutout_table) == 3 assert cutout_table["Local Path"][0][-4:] == ".jpg" @@ -675,7 +675,7 @@ def test_zcut_download_cutouts(patch_post, tmpdir): # Testing with img_param cutout_table = mast.Zcut.download_cutouts(coordinates=coord, size=5, - cutout_format="jpg", path=str(tmpdir), invert=True) + cutout_format="jpg", path=str(tmpdir), invert=True) assert isinstance(cutout_table, Table) assert len(cutout_table) == 3 assert cutout_table["Local Path"][0][-4:] == ".jpg" diff --git a/astroquery/noirlab/tests/test_noirlab_remote.py b/astroquery/noirlab/tests/test_noirlab_remote.py index 71669a865d..5a44e54a22 100644 --- a/astroquery/noirlab/tests/test_noirlab_remote.py +++ b/astroquery/noirlab/tests/test_noirlab_remote.py @@ -2,21 +2,21 @@ # Python library # External packages +import pytest from astropy import units as u from astropy.coordinates import SkyCoord -from astropy.tests.helper import remote_data + # Local packages from .. import Noirlab from . import expected as expsia -# #!import pytest # performs similar tests as test_module.py, but performs # the actual HTTP request rather than monkeypatching them. # should be disabled or enabled at will - use the -# remote_data decorator from astropy: +# remote_data decorator from pytest-remotedata: -@remote_data +@pytest.mark.remote_data class TestNoirlabClass: def test_query_region_1(self): diff --git a/astroquery/vo_conesearch/validator/tests/test_validate.py b/astroquery/vo_conesearch/validator/tests/test_validate.py index 5262d19cf1..f30354e51e 100644 --- a/astroquery/vo_conesearch/validator/tests/test_validate.py +++ b/astroquery/vo_conesearch/validator/tests/test_validate.py @@ -18,8 +18,8 @@ from numpy.testing import assert_allclose # ASTROPY -from astropy.tests.helper import catch_warnings from astropy.utils.data import get_pkg_data_filename +from astropy.utils.exceptions import AstropyUserWarning # LOCAL from .. import conf, validate, tstquery @@ -83,9 +83,8 @@ def teardown_class(self): @pytest.mark.remote_data def test_tstquery(): - with catch_warnings() as w: + with pytest.warns(AstropyUserWarning, match='too large') as w: d = tstquery.parse_cs('ivo://cds.vizier/i/252', cap_index=4) assert len(w) == 1 - assert 'too large' in str(w[0].message) assert_allclose([d['RA'], d['DEC'], d['SR']], [45, 0.07460390065517808, 0.1]) diff --git a/docs/testing.rst b/docs/testing.rst index 13d83b3525..f273544556 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -85,7 +85,7 @@ the ``test_module.py`` file. ------------------------- The remote tests are much easier. Just decorate the test class or test -functions with ``astropy.tests.helper.remote_data``. +functions with ``@pytest.mark.remote_data``. ``setup_package.py`` -------------------- From 9c127f81b1d78a7056dd363f1eb3e2efd8fdb3a4 Mon Sep 17 00:00:00 2001 From: "Pey Lian Lim (Github)" <2090236+pllim@users.noreply.github.com> Date: Wed, 29 Dec 2021 16:25:17 -0500 Subject: [PATCH 23/87] Ignore distutils deprecation --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 1fd60737ac..9714405121 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,6 +37,7 @@ filterwarnings = error ignore: Experimental:UserWarning: # This is a temporary measure, all of these should be fixed: + ignore:distutils Version classes are deprecated:DeprecationWarning ignore::pytest.PytestUnraisableExceptionWarning ignore::numpy.VisibleDeprecationWarning ignore: unclosed file:ResourceWarning From 9bea166ec0fac891ae165d513cf3c5f821d23ace Mon Sep 17 00:00:00 2001 From: jespinosaar Date: Mon, 10 Jan 2022 10:33:24 +0100 Subject: [PATCH 24/87] login_gui method removed from docs --- docs/esa/jwst.rst | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/docs/esa/jwst.rst b/docs/esa/jwst.rst index de072c462e..f871790e6a 100644 --- a/docs/esa/jwst.rst +++ b/docs/esa/jwst.rst @@ -651,7 +651,7 @@ To remove asynchronous ----------------------- Authenticated users are able to access to TAP+ capabilities (shared tables, persistent jobs, etc.) -In order to authenticate a user, ``login``, ``login_gui`` or ``login_token_gui`` methods must be called. After a successful +In order to authenticate a user, ``login`` method must be called. After a successful authentication, the user will be authenticated until ``logout`` method is called. All previous methods (``query_object``, ``cone_search``, ``load_table``, ``load_tables``, ``launch_job``) explained for @@ -667,18 +667,6 @@ The main differences are: 2.1. Login/Logout ~~~~~~~~~~~~~~~~~ -Using the graphic interface: - - -*Note: Tkinter module is required to use login_gui method.* - -.. code-block:: python - - >>> from astroquery.esa.jwst import Jwst - >>> from astroquery.esa.jwst import Jwst - >>> Jwst.login_gui() - - Using the command line: .. code-block:: python From 6788573c854476e7d52548b4b052ee19a2795863 Mon Sep 17 00:00:00 2001 From: jespinosaar Date: Mon, 10 Jan 2022 11:22:52 +0100 Subject: [PATCH 25/87] login_gui removed from JWST --- astroquery/esa/jwst/core.py | 11 ----------- astroquery/esa/jwst/data_access.py | 3 ++- astroquery/esa/jwst/tests/test_jwsttap.py | 8 -------- 3 files changed, 2 insertions(+), 20 deletions(-) diff --git a/astroquery/esa/jwst/core.py b/astroquery/esa/jwst/core.py index d67a51cc50..a9a4bd59c5 100644 --- a/astroquery/esa/jwst/core.py +++ b/astroquery/esa/jwst/core.py @@ -647,17 +647,6 @@ def login(self, *, user=None, password=None, credentials_file=None, if token: self.set_token(token=token) - def login_gui(self, *, verbose=False): - """Performs a login using a GUI dialog - TAP+ only - - Parameters - ---------- - verbose : bool, optional, default 'False' - flag to display information about the process - """ - return self.__jwsttap.login_gui(verbose) - def logout(self, *, verbose=False): """Performs a logout TAP+ only diff --git a/astroquery/esa/jwst/data_access.py b/astroquery/esa/jwst/data_access.py index 310a7133d9..f39bd35d01 100644 --- a/astroquery/esa/jwst/data_access.py +++ b/astroquery/esa/jwst/data_access.py @@ -10,6 +10,7 @@ """ from astropy.utils import data +from . import conf __all__ = ['JwstDataHandler'] @@ -17,7 +18,7 @@ class JwstDataHandler: def __init__(self, base_url=None): if base_url is None: - self.base_url = "http://jwstdummydata.com" + self.base_url = conf.JWST_DATA_SERVER else: self.base_url = base_url diff --git a/astroquery/esa/jwst/tests/test_jwsttap.py b/astroquery/esa/jwst/tests/test_jwsttap.py index 523fd2f5c6..54b127d16e 100644 --- a/astroquery/esa/jwst/tests/test_jwsttap.py +++ b/astroquery/esa/jwst/tests/test_jwsttap.py @@ -1002,14 +1002,6 @@ def test_login(self): tap.login(user='test_user', password='test_password') dummyTapHandler.check_call('login', parameters) - def test_login_gui(self): - dummyTapHandler = DummyTapHandler() - tap = JwstClass(tap_plus_handler=dummyTapHandler, show_messages=False) - parameters = {} - parameters['verbose'] = False - tap.login_gui() - dummyTapHandler.check_call('login_gui', parameters) - def test_logout(self): dummyTapHandler = DummyTapHandler() tap = JwstClass(tap_plus_handler=dummyTapHandler, show_messages=False) From 713107171f476b8eebcc98b23bf7aaab6aecdfe9 Mon Sep 17 00:00:00 2001 From: jespinosaar Date: Mon, 10 Jan 2022 11:29:22 +0100 Subject: [PATCH 26/87] changes.rst updated --- CHANGES.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index e15b22097f..5d3da70957 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -8,6 +8,10 @@ New Tools and Services Service fixes and enhancements ------------------------------ +esa.jwst +^^^^^^^^^^ + +- Minor fixes, documentation updated. [#2257] Infrastructure, Utility and Other Changes and Additions ------------------------------------------------------- From a93ea7c76967995e2159c384d0f428f2aca97622 Mon Sep 17 00:00:00 2001 From: Eero Vaher Date: Wed, 12 Jan 2022 00:57:48 +0100 Subject: [PATCH 27/87] Test Gaia.query_object with `columns` and `radius` Currently the `astroquery.gaia.core.GaiaClass.query_object()` method ignores the `column` argument if `radius` is specified. This commit adds a remote test that reveals the problem. --- astroquery/gaia/tests/test_gaia_remote.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/astroquery/gaia/tests/test_gaia_remote.py b/astroquery/gaia/tests/test_gaia_remote.py index 98f34f100f..dcebe328b9 100644 --- a/astroquery/gaia/tests/test_gaia_remote.py +++ b/astroquery/gaia/tests/test_gaia_remote.py @@ -5,6 +5,15 @@ from .. import GaiaClass +@pytest.mark.remote_data +def test_query_object_columns_with_radius(): + # Regression test: `columns` were ignored if `radius` was provided [#2025] + Gaia = GaiaClass() + sc = SkyCoord(ra=0*u.deg, dec=0*u.deg) + table = Gaia.query_object_async(sc, radius=10*u.arcsec, columns=['ra']) + assert table.colnames == ['ra', 'dist'] + + @pytest.mark.remote_data def test_query_object_row_limit(): Gaia = GaiaClass() From a46e7d58d9d2787a0a26be020450e9d6754e4272 Mon Sep 17 00:00:00 2001 From: Eero Vaher Date: Wed, 12 Jan 2022 00:57:55 +0100 Subject: [PATCH 28/87] Fix Gaia.query_object with `columns` and `radius` The method `astroquery.gaia.core.GaiaClass.__query_object()` no longer ignores its `column` argument when `radius` is specified. --- astroquery/gaia/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/astroquery/gaia/core.py b/astroquery/gaia/core.py index 7170f72214..68c4a5cf12 100644 --- a/astroquery/gaia/core.py +++ b/astroquery/gaia/core.py @@ -386,8 +386,8 @@ def __query_object(self, coordinate, radius=None, width=None, height=None, coord = self.__getCoordInput(coordinate, "coordinate") job = None if radius is not None: - job = self.__cone_search(coord, radius, - async_job=async_job, verbose=verbose) + job = self.__cone_search(coord, radius, async_job=async_job, + verbose=verbose, columns=columns) else: raHours, dec = commons.coord_to_radec(coord) ra = raHours * 15.0 # Converts to degrees From 252b1162931450d9d8b0690e127978a62660ad7f Mon Sep 17 00:00:00 2001 From: Eero Vaher Date: Mon, 27 Dec 2021 17:02:35 +0100 Subject: [PATCH 29/87] Add change log entry for #2249 --- CHANGES.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 5d3da70957..b7c524e53a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -13,6 +13,13 @@ esa.jwst - Minor fixes, documentation updated. [#2257] +gaia +^^^^ + +- The ``query_object()`` and ``query_object_async()`` methods of + ``astroquery.gaia.Gaia`` no longer ignore their ``columns`` argument when + ``radius`` is specified. [#2249] + Infrastructure, Utility and Other Changes and Additions ------------------------------------------------------- From bcef71672fdb6d47a535479f7252af71cef34c63 Mon Sep 17 00:00:00 2001 From: Natanael Date: Sun, 16 Jan 2022 10:20:40 -0300 Subject: [PATCH 30/87] Fix documentation typo about default radius value --- astroquery/sdss/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astroquery/sdss/core.py b/astroquery/sdss/core.py index 9e636a8648..13009dd7ab 100644 --- a/astroquery/sdss/core.py +++ b/astroquery/sdss/core.py @@ -81,7 +81,7 @@ def query_crossid_async(self, coordinates, obj_names=None, radius : str or `~astropy.units.Quantity` object, optional The string must be parsable by `~astropy.coordinates.Angle`. The appropriate `~astropy.units.Quantity` object from - `astropy.units` may also be used. Defaults to 2 arcsec. + `astropy.units` may also be used. Defaults to 5 arcsec. timeout : float, optional Time limit (in seconds) for establishing successful connection with remote server. Defaults to `SDSSClass.TIMEOUT`. From 777f80b4db026360cd461d93498d7da8e86c8f14 Mon Sep 17 00:00:00 2001 From: Natanael Date: Sun, 16 Jan 2022 10:21:08 -0300 Subject: [PATCH 31/87] Fix spec query not working for DR >= 12 --- astroquery/sdss/core.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/astroquery/sdss/core.py b/astroquery/sdss/core.py index 13009dd7ab..bb22be4270 100644 --- a/astroquery/sdss/core.py +++ b/astroquery/sdss/core.py @@ -144,7 +144,10 @@ def query_crossid_async(self, coordinates, obj_names=None, sql_query += ',dbo.fPhotoTypeN(p.type) as type \ FROM #upload u JOIN #x x ON x.up_id = u.up_id \ - JOIN PhotoObjAll p ON p.objID = x.objID ORDER BY x.up_id' + JOIN PhotoObjAll p ON p.objID = x.objID ' + if specobj_fields: + sql_query += 'JOIN SpecObjAll s ON p.objID = s.bestObjID ' + sql_query += 'ORDER BY x.up_id' data = "obj_id ra dec \n" data += " \n ".join(['{0} {1} {2}'.format(obj_names[i], From 4fb7217d125d42ea52a2b5d47f2cfc82050228cb Mon Sep 17 00:00:00 2001 From: Natanael Date: Sun, 16 Jan 2022 10:21:25 -0300 Subject: [PATCH 32/87] Fix photo and spec query not working for DR 17 --- astroquery/sdss/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astroquery/sdss/core.py b/astroquery/sdss/core.py index bb22be4270..3c61800ffe 100644 --- a/astroquery/sdss/core.py +++ b/astroquery/sdss/core.py @@ -168,7 +168,7 @@ def query_crossid_async(self, coordinates, obj_names=None, if get_query_payload: return request_payload url = self._get_crossid_url(data_release) - response = self._request("POST", url, params=request_payload, + response = self._request("POST", url, data=request_payload, timeout=timeout, cache=cache) return response From 46739aa29eacc5fbe9f89a8eda3943c45218af98 Mon Sep 17 00:00:00 2001 From: javier-ballester Date: Fri, 3 Dec 2021 10:26:27 +0100 Subject: [PATCH 33/87] NXSAPCR-1093: modification of urls to https --- astroquery/esa/xmm_newton/__init__.py | 6 +++--- astroquery/esa/xmm_newton/core.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/astroquery/esa/xmm_newton/__init__.py b/astroquery/esa/xmm_newton/__init__.py index 0dbaceb6fa..5f5c44d99a 100644 --- a/astroquery/esa/xmm_newton/__init__.py +++ b/astroquery/esa/xmm_newton/__init__.py @@ -17,13 +17,13 @@ class Conf(_config.ConfigNamespace): """ Configuration parameters for `astroquery.esa.xmm_newton`. """ - DATA_ACTION = _config.ConfigItem("http://nxsa.esac.esa.int/" + DATA_ACTION = _config.ConfigItem("https://nxsa.esac.esa.int/" "nxsa-sl/servlet/data-action?", "Main url for retriving XSA files") - DATA_ACTION_AIO = _config.ConfigItem("http://nxsa.esac.esa.int/" + DATA_ACTION_AIO = _config.ConfigItem("https://nxsa.esac.esa.int/" "nxsa-sl/servlet/data-action-aio?", "Main url for retriving XSA files") - METADATA_ACTION = _config.ConfigItem("http://nxsa.esac.esa.int/" + METADATA_ACTION = _config.ConfigItem("https://nxsa.esac.esa.int/" "nxsa-sl/servlet/" "metadata-action?", "Main url for retriving XSA metadata") diff --git a/astroquery/esa/xmm_newton/core.py b/astroquery/esa/xmm_newton/core.py index 23cb392ef4..66ec1f6d0a 100644 --- a/astroquery/esa/xmm_newton/core.py +++ b/astroquery/esa/xmm_newton/core.py @@ -41,8 +41,8 @@ def __init__(self, tap_handler=None): super(XMMNewtonClass, self).__init__() if tap_handler is None: - self._tap = TapPlus(url="http://nxsa.esac.esa.int" - "/tap-server/tap/") + self._tap = TapPlus(url="https://nxsa.esac.esa.int" + "/tap-server/tap") else: self._tap = tap_handler self._rmf_ftp = str("http://sasdev-xmm.esac.esa.int/pub/ccf/constituents/extras/responses/") From 045469fee1028d3f77e1ce844b56f37562c2de76 Mon Sep 17 00:00:00 2001 From: javier-ballester Date: Fri, 3 Dec 2021 10:59:51 +0100 Subject: [PATCH 34/87] NXSAPCR-1091: Added functionality to download proprietary data - added corresponding tests --- astroquery/esa/xmm_newton/core.py | 143 +++++++++------- astroquery/esa/xmm_newton/tests/my_config.ini | 3 + .../esa/xmm_newton/tests/setup_package.py | 1 + .../esa/xmm_newton/tests/test_xmm_newton.py | 86 ++++++++-- .../tests/test_xmm_newton_remote.py | 153 +++++++++++++++--- 5 files changed, 294 insertions(+), 92 deletions(-) create mode 100644 astroquery/esa/xmm_newton/tests/my_config.ini diff --git a/astroquery/esa/xmm_newton/core.py b/astroquery/esa/xmm_newton/core.py index 66ec1f6d0a..836b0ae738 100644 --- a/astroquery/esa/xmm_newton/core.py +++ b/astroquery/esa/xmm_newton/core.py @@ -12,6 +12,7 @@ """ import re +from getpass import getpass from ...utils.tap.core import TapPlus from ...query import BaseQuery import shutil @@ -21,7 +22,7 @@ import os from astropy.io import fits -from . import conf +from . import conf, config from astroquery import log from astropy.coordinates import SkyCoord from ...exceptions import LoginError @@ -41,14 +42,13 @@ def __init__(self, tap_handler=None): super(XMMNewtonClass, self).__init__() if tap_handler is None: - self._tap = TapPlus(url="https://nxsa.esac.esa.int" - "/tap-server/tap") + self._tap = TapPlus(url="https://nxsa.esac.esa.int/tap-server/tap") else: self._tap = tap_handler self._rmf_ftp = str("http://sasdev-xmm.esac.esa.int/pub/ccf/constituents/extras/responses/") def download_data(self, observation_id, *, filename=None, verbose=False, - cache=True, **kwargs): + cache=True, prop=False, username=None, password=None, **kwargs): """ Download data from XMM-Newton @@ -100,42 +100,38 @@ def download_data(self, observation_id, *, filename=None, verbose=False, None if not verbose. It downloads the observation indicated If verbose returns the filename """ - if filename is not None: - filename = os.path.splitext(filename)[0] + """ + Here we change the log level so that it is above 20, this is to stop a log.debug in query.py. this debug + reveals the url being sent which in turn reveals the users username and password + """ + previouslevel = log.getEffectiveLevel() + log.setLevel(50) - link = self.data_aio_url + "obsno=" + observation_id + # create url to access the aio + link = self._create_link(observation_id, **kwargs) - link = link + "".join("&{0}={1}".format(key, val) - for key, val in kwargs.items()) + # If the user wants to access proprietary data, ask them for there credentials + if prop: + username, password = self._get_username_and_password(credentials_file) + link = f"{link}&AIOUSER={username}&AIOPWD={password}" if verbose: log.info(link) - # we can cache this HEAD request - the _download_file one will check - # the file size and will never cache - response = self._request('HEAD', link, save=False, cache=cache) - - # Get original extension - if 'Content-Type' in response.headers and 'text' not in response.headers['Content-Type']: - _, params = cgi.parse_header(response.headers['Content-Disposition']) - else: - if response.status_code == 401: - error = "Data protected by proprietary rights. Please check your credentials" - raise LoginError(error) - response.raise_for_status() - + # get response of created url + params = self._request_link(link, cache) r_filename = params["filename"] suffixes = Path(r_filename).suffixes + print(suffixes) - if filename is None: - filename = observation_id - - filename += "".join(suffixes) + # get desired filename + filename = self._create_filename(filename, observation_id, suffixes) self._download_file(link, filename, head_safe=True, cache=cache) if verbose: - log.info("Wrote {0} to {1}".format(link, filename)) + log.info(f"Wrote {link} to {filename}") + log.setLevel(previouslevel) def get_postcard(self, observation_id, *, image_type="OBS_EPIC", filename=None, verbose=False): @@ -186,12 +182,12 @@ def get_postcard(self, observation_id, *, image_type="OBS_EPIC", else: filename = observation_id + ".png" - log.info("Copying file to {0}...".format(filename)) + log.info(f"Copying file to {filename}...") shutil.move(local_filepath, filename) if verbose: - log.info("Wrote {0} to {1}".format(link, filename)) + log.info(f"Wrote {link} to {filename}") return filename @@ -273,14 +269,53 @@ def get_columns(self, table_name, *, only_names=True, verbose=False): break if columns is None: - raise ValueError("table name specified is not found in " - "XSA TAP service") + raise ValueError("table name specified is not found in XSA TAP service") if only_names: return [c.name for c in columns] else: return columns + def _create_link(self, observation_id, **kwargs): + link = f"{self.data_aio_url}obsno={observation_id}" + link = link + "".join("&{0}={1}".format(key, val) + for key, val in kwargs.items()) + return link + + def _request_link(self, link, cache): + # we can cache this HEAD request - the _download_file one will check + # the file size and will never cache + response = self._request('HEAD', link, save=False, cache=cache) + # Get original extension + if 'Content-Type' in response.headers and 'text' not in response.headers['Content-Type']: + _, params = cgi.parse_header(response.headers['Content-Disposition']) + elif response.status_code == 401: + error = "Data protected by proprietary rights. Please check your credentials" + raise LoginError(error) + elif 'Content-Type' not in response.headers: + error = "Incorrect credentials" + raise LoginError(error) + response.raise_for_status() + return params + + def _get_username_and_password(self, credentials_file): + if credentials_file != None: + self.configuration.read(credentials_file) + username = self.configuration.get('user', 'username') + password = self.configuration.get('user', 'password') + else: + username = input("Username: ") + password = getpass("Password: ") + return username, password + + def _create_filename(self, filename, observation_id, suffixes): + if filename is not None: + filename = os.path.splitext(filename)[0] + else: + filename = observation_id + filename += "".join(suffixes) + return filename + def _parse_filename(self, filename): """Parses the file's name of a product @@ -572,9 +607,9 @@ def get_epic_metadata(self, *, target_name=None, Tables containing the metadata of the target """ if not target_name and not coordinates: - raise Exception("Input parameters needed, " - "please provide the name " - "or the coordinates of the target") + raise Exception("Input parameters needed, " + "please provide the name " + "or the coordinates of the target") epic_source = {"table": "xsa.v_epic_source", "column": "epic_source_equatorial_spoint"} @@ -600,29 +635,29 @@ def get_epic_metadata(self, *, target_name=None, query_fmt = ("select {} from {} " "where 1=contains({}, circle('ICRS', {}, {}, {}));") epic_source_table = self.query_xsa_tap(query_fmt.format(cols, - epic_source["table"], - epic_source["column"], - c.ra.degree, - c.dec.degree, - radius)) + epic_source["table"], + epic_source["column"], + c.ra.degree, + c.dec.degree, + radius)) cat_4xmm_table = self.query_xsa_tap(query_fmt.format(cols, - cat_4xmm["table"], - cat_4xmm["column"], - c.ra.degree, - c.dec.degree, - radius)) + cat_4xmm["table"], + cat_4xmm["column"], + c.ra.degree, + c.dec.degree, + radius)) stack_4xmm_table = self.query_xsa_tap(query_fmt.format(cols, - stack_4xmm["table"], - stack_4xmm["column"], - c.ra.degree, - c.dec.degree, - radius)) + stack_4xmm["table"], + stack_4xmm["column"], + c.ra.degree, + c.dec.degree, + radius)) slew_source_table = self.query_xsa_tap(query_fmt.format(cols, - slew_source["table"], - slew_source["column"], - c.ra.degree, - c.dec.degree, - radius)) + slew_source["table"], + slew_source["column"], + c.ra.degree, + c.dec.degree, + radius)) return epic_source_table, cat_4xmm_table, stack_4xmm_table, slew_source_table def get_epic_lightcurve(self, filename, source_number, *, diff --git a/astroquery/esa/xmm_newton/tests/my_config.ini b/astroquery/esa/xmm_newton/tests/my_config.ini new file mode 100644 index 0000000000..97c415a7ed --- /dev/null +++ b/astroquery/esa/xmm_newton/tests/my_config.ini @@ -0,0 +1,3 @@ +[user] +username=test +password=test \ No newline at end of file diff --git a/astroquery/esa/xmm_newton/tests/setup_package.py b/astroquery/esa/xmm_newton/tests/setup_package.py index ae93358d3e..19bc88a7de 100644 --- a/astroquery/esa/xmm_newton/tests/setup_package.py +++ b/astroquery/esa/xmm_newton/tests/setup_package.py @@ -19,6 +19,7 @@ def get_package_data(): paths = [os.path.join('data', '*.tar'), os.path.join('data', '*.xml'), + os.path.join('my_config.ini') ] # etc, add other extensions # you can also enlist files individually by names # finally construct and return a dict for the sub module diff --git a/astroquery/esa/xmm_newton/tests/test_xmm_newton.py b/astroquery/esa/xmm_newton/tests/test_xmm_newton.py index d487ae6a68..fa3329cd30 100644 --- a/astroquery/esa/xmm_newton/tests/test_xmm_newton.py +++ b/astroquery/esa/xmm_newton/tests/test_xmm_newton.py @@ -9,33 +9,40 @@ Created on 4 Sept. 2019 """ +from unittest.mock import patch import pytest - -import sys import tarfile import os import errno import shutil -from astropy.coordinates import SkyCoord -from astropy.utils.diff import report_diff_values -from astroquery.utils.tap.core import TapPlus from ..core import XMMNewtonClass from ..tests.dummy_tap_handler import DummyXMMNewtonTapHandler from ..tests.dummy_handler import DummyHandler -from fileinput import filename -from tarfile import is_tarfile +from astroquery.exceptions import LoginError -class TestXMMNewton(): +class mockResponse(): + headers = {'Date': 'Wed, 24 Nov 2021 13:43:50 GMT', + 'Server': 'Apache/2.4.6 (Red Hat Enterprise Linux) OpenSSL/1.0.2k-fips', + 'Content-Disposition': 'inline; filename="0560181401.tar.gz"', + 'Content-Type': 'application/x-gzip', + 'Content-Length': '6590874', 'Connection': 'close'} + status_code = 400 + + @staticmethod + def raise_for_status(): + pass + +class TestXMMNewton(): def get_dummy_tap_handler(self): - parameterst = {'query': "select top 10 * from v_public_observations", - 'output_file': "test2.vot", - 'output_format': "votable", - 'verbose': False} - dummyTapHandler = DummyXMMNewtonTapHandler("launch_job", parameterst) + parameters = {'query': "select top 10 * from v_public_observations", + 'output_file': "test2.vot", + 'output_format': "votable", + 'verbose': False} + dummyTapHandler = DummyXMMNewtonTapHandler("launch_job", parameters) return dummyTapHandler def test_query_xsa_tap(self): @@ -235,7 +242,7 @@ def _create_tar(self, tarname, files): os.makedirs(os.path.join(ob_name, ftype)) except OSError as exc: if exc.errno == errno.EEXIST and \ - os.path.isdir(os.path.join(ob_name, ftype)): + os.path.isdir(os.path.join(ob_name, ftype)): pass else: raise @@ -255,7 +262,7 @@ def _create_tar_lightcurves(self, tarname, files): os.makedirs(os.path.join(ob_name, ftype)) except OSError as exc: if exc.errno == errno.EEXIST and \ - os.path.isdir(os.path.join(ob_name, ftype)): + os.path.isdir(os.path.join(ob_name, ftype)): pass else: raise @@ -377,7 +384,7 @@ def test_get_epic_images(self): xsa = XMMNewtonClass(self.get_dummy_tap_handler()) res = xsa.get_epic_images(_tarname, band=[], instrument=[], get_detmask=True, get_exposure_map=True) - assert len(res) == 6 # Number of different bands + assert len(res) == 6 # Number of different bands assert len(res[1]) == 9 # Number of different inst within band 1 assert len(res[2]) == 9 # Number of different inst within band 2 assert len(res[3]) == 9 # Number of different inst within band 3 @@ -510,3 +517,50 @@ def test_get_epic_lightcurve_invalid_source_number(self, capsys): % (_tarname, _invalid_source_number, _default_instrument)) os.remove(_tarname) + + def test_create_link(self): + xsa = XMMNewtonClass(self.get_dummy_tap_handler()) + link = xsa._create_link("0560181401") + assert link == "https://nxsa.esac.esa.int/nxsa-sl/servlet/data-action-aio?obsno=0560181401" + + @patch('astroquery.query.BaseQuery._request') + def test_request_link(self, mock_request): + xsa = XMMNewtonClass(self.get_dummy_tap_handler()) + mock_request.return_value = mockResponse + params = xsa._request_link("https://nxsa.esac.esa.int/nxsa-sl/servlet/data-action-aio?obsno=0560181401", None) + assert params == {'filename': '0560181401.tar.gz'} + + @pytest.mark.xfail(raises=LoginError) + @patch('astroquery.query.BaseQuery._request') + def test_request_link_protected(self, mock_request): + xsa = XMMNewtonClass(self.get_dummy_tap_handler()) + dummyclass = mockResponse + dummyclass.headers = {} + mock_request.return_value = dummyclass + xsa._request_link("https://nxsa.esac.esa.int/nxsa-sl/servlet/data-action-aio?obsno=0560181401", None) + + @pytest.mark.xfail(raises=LoginError) + @patch('astroquery.query.BaseQuery._request') + def test_request_link_incorrect_credentials(self, mock_request): + xsa = XMMNewtonClass(self.get_dummy_tap_handler()) + dummyclass = mockResponse + dummyclass.headers = {} + dummyclass.status_code = 10 + mock_request.return_value = dummyclass + xsa._request_link("https://nxsa.esac.esa.int/nxsa-sl/servlet/data-action-aio?obsno=0560181401", None) + + def test_get_username_and_password(self): + xsa = XMMNewtonClass(self.get_dummy_tap_handler()) + username, password = xsa._get_username_and_password("astroquery/esa/xmm_newton/tests/my_config.ini") + assert username == "test" + assert password == "test" + + def test_create_filename_None(self): + xsa = XMMNewtonClass(self.get_dummy_tap_handler()) + filename = xsa._create_filename(None, "0560181401", ['.tar', '.gz']) + assert filename == "0560181401.tar.gz" + + def test_create_filename_Not_None(self): + xsa = XMMNewtonClass(self.get_dummy_tap_handler()) + filename = xsa._create_filename("Test", "0560181401", ['.tar', '.gz']) + assert filename == "Test.tar.gz" diff --git a/astroquery/esa/xmm_newton/tests/test_xmm_newton_remote.py b/astroquery/esa/xmm_newton/tests/test_xmm_newton_remote.py index c0464a87b2..72a08f06df 100644 --- a/astroquery/esa/xmm_newton/tests/test_xmm_newton_remote.py +++ b/astroquery/esa/xmm_newton/tests/test_xmm_newton_remote.py @@ -1,18 +1,15 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst """ - @author: Elena Colomo @contact: ecolomo@esa.int - European Space Astronomy Centre (ESAC) European Space Agency (ESA) - Created on 4 Sept. 2019 """ import pytest +from astroquery.exceptions import LoginError -import sys import tarfile import os import errno @@ -22,9 +19,96 @@ from astroquery.utils.tap.core import TapPlus from ..core import XMMNewtonClass - +from ..tests.dummy_tap_handler import DummyXMMNewtonTapHandler class TestXMMNewtonRemote(): + _files = { + "0405320501": { + "pps": [ + "P0405320501M1S002EXPMAP1000.FTZ", + "P0405320501M1S002IMAGE_4000.FTZ", + "P0405320501M2S003EXPMAP2000.FTZ", + "P0405320501M2S003IMAGE_5000.FTZ", + "P0405320501PNS001EXPMAP3000.FTZ", + "P0405320501PNS001IMAGE_8000.FTZ", + "P0405320501M1S002EXPMAP2000.FTZ", + "P0405320501M1S002IMAGE_5000.FTZ", + "P0405320501M2S003EXPMAP3000.FTZ", + "P0405320501M2S003IMAGE_8000.FTZ", + "P0405320501PNS001EXPMAP4000.FTZ", + "P0405320501PNX000DETMSK1000.FTZ", + "P0405320501M1S002EXPMAP3000.FTZ", + "P0405320501M1S002IMAGE_8000.FTZ", + "P0405320501M2S003EXPMAP4000.FTZ", + "P0405320501M2X000DETMSK1000.FTZ", + "P0405320501PNS001EXPMAP5000.FTZ", + "P0405320501PNX000DETMSK2000.FTZ", + "P0405320501M1S002EXPMAP4000.FTZ", + "P0405320501M1X000DETMSK1000.FTZ", + "P0405320501M2S003EXPMAP5000.FTZ", + "P0405320501M2X000DETMSK2000.FTZ", + "P0405320501PNS001EXPMAP8000.FTZ", + "P0405320501PNX000DETMSK3000.FTZ", + "P0405320501M1S002EXPMAP5000.FTZ", + "P0405320501M1X000DETMSK2000.FTZ", + "P0405320501M2S003EXPMAP8000.FTZ", + "P0405320501M2X000DETMSK3000.FTZ", + "P0405320501PNS001IMAGE_1000.FTZ", + "P0405320501PNX000DETMSK4000.FTZ", + "P0405320501M1S002EXPMAP8000.FTZ", + "P0405320501M1X000DETMSK3000.FTZ", + "P0405320501M2S003IMAGE_1000.FTZ", + "P0405320501M2X000DETMSK4000.FTZ", + "P0405320501PNS001IMAGE_2000.FTZ", + "P0405320501PNX000DETMSK5000.FTZ", + "P0405320501M1S002IMAGE_1000.FTZ", + "P0405320501M1X000DETMSK4000.FTZ", + "P0405320501M2S003IMAGE_2000.FTZ", + "P0405320501M2X000DETMSK5000.FTZ", + "P0405320501PNS001IMAGE_3000.FTZ", + "P0405320501M1S002IMAGE_2000.FTZ", + "P0405320501M1X000DETMSK5000.FTZ", + "P0405320501M2S003IMAGE_3000.FTZ", + "P0405320501PNS001EXPMAP1000.FTZ", + "P0405320501PNS001IMAGE_4000.FTZ", + "P0405320501M1S002IMAGE_3000.FTZ", + "P0405320501M2S003EXPMAP1000.FTZ", + "P0405320501M2S003IMAGE_4000.FTZ", + "P0405320501PNS001EXPMAP2000.FTZ", + "P0405320501PNS001IMAGE_5000.FTZ", + "P0405320501PNU001IMAGE_5000.FTZ", + "P0405320501PNX001IMAGE_5000.FTZ" + ] + } + } + + def get_dummy_tap_handler(self): + parameters = {'query': "select top 10 * from v_public_observations", + 'output_file': "test2.vot", + 'output_format': "votable", + 'verbose': False} + dummyTapHandler = DummyXMMNewtonTapHandler("launch_job", parameters) + return dummyTapHandler + + def _create_tar(self, tarname, files): + with tarfile.open(tarname, "w") as tar: + for ob_name, ob in self._files.items(): + for ftype, ftype_val in ob.items(): + for f in ftype_val: + try: + os.makedirs(os.path.join(ob_name, ftype)) + except OSError as exc: + if exc.errno == errno.EEXIST and \ + os.path.isdir(os.path.join(ob_name, ftype)): + pass + else: + raise + _file = open(os.path.join(ob_name, ftype, f), "w") + _file.close() + tar.add(os.path.join(ob_name, ftype, f)) + os.remove(os.path.join(ob_name, ftype, f)) + shutil.rmtree(os.path.join(ob_name, ftype)) + shutil.rmtree(ob_name) @pytest.mark.remote_data def test_download_data(self): @@ -65,23 +149,6 @@ def test_get_postcard_filename(self): xsa = XMMNewtonClass(self.get_dummy_tap_handler()) xsa.get_postcard(**parameters) - @pytest.mark.remote_data - def test_get_epic_spectra(self): - _tarname = "tarfile.tar" - _source_number = 83 - _instruments = ["M1", "M1_arf", "M1_bkg", "M1_rmf", - "M2", "M2_arf", "M2_bkg", "M2_rmf", - "PN", "PN_arf", "PN_bkg", "PN_rmf"] - self._create_tar(_tarname, self._files) - xsa = XMMNewtonClass(self.get_dummy_tap_handler()) - res = xsa.get_epic_spectra(_tarname, _source_number, - instrument=[]) - assert len(res) == 8 - # Removing files created in this test - for ob_name in self._files: - shutil.rmtree(ob_name) - os.remove(_tarname) - @pytest.mark.remote_data def test_get_epic_metadata(self): tap_url = "http://nxsadev.esac.esa.int/tap-server/tap/" @@ -125,3 +192,45 @@ def test_get_epic_metadata(self): c.dec.degree, radius)) assert report_diff_values(slew_source, table) + + @pytest.mark.remote_data + @pytest.mark.xfail(raises=LoginError) + def test_download_proprietary_data_incorrect_credentials(self): + parameters = {'observation_id': "0762470101", + 'prop': 'True', + 'credentials_file': "astroquery/esa/xmm_newton/tests/my_config.ini", + 'level': "PPS", + 'name': 'OBSMLI', + 'filename': 'single', + 'instname': 'OM', + 'extension': 'FTZ', + 'verbose': False} + xsa = XMMNewtonClass(self.get_dummy_tap_handler()) + xsa.download_data(**parameters) + + @pytest.mark.remote_data + @pytest.mark.xfail(raises=LoginError) + def test_download_proprietary_data_without_credentials(self): + parameters = {'observation_id': "0883780101", + 'level': "PPS", + 'name': 'OBSMLI', + 'filename': 'single', + 'instname': 'OM', + 'extension': 'FTZ', + 'verbose': False} + xsa = XMMNewtonClass(self.get_dummy_tap_handler()) + xsa.download_data(**parameters) + + + @pytest.mark.remote_data + def test_get_epic_spectra(self): + _tarname = "tarfile.tar" + _source_number = 83 + _instruments = ["M1", "M1_arf", "M1_bkg", "M1_rmf", + "M2", "M2_arf", "M2_bkg", "M2_rmf", + "PN", "PN_arf", "PN_bkg", "PN_rmf"] + self._create_tar(_tarname, self._files) + xsa = XMMNewtonClass(self.get_dummy_tap_handler()) + res = xsa.get_epic_spectra(_tarname, _source_number, + instrument=[]) + assert len(res) == 0 \ No newline at end of file From 3323d7df76bc33c4c19a8fccfdbf466051ac3fa2 Mon Sep 17 00:00:00 2001 From: javier-ballester Date: Fri, 3 Dec 2021 11:01:20 +0100 Subject: [PATCH 35/87] NXSAPCR-1091: Added documentation for downloading proprietary data. --- docs/esa/xmm_newton.rst | 41 +++++++++++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/docs/esa/xmm_newton.rst b/docs/esa/xmm_newton.rst index 643297fbd0..8bbbc53737 100644 --- a/docs/esa/xmm_newton.rst +++ b/docs/esa/xmm_newton.rst @@ -36,8 +36,37 @@ it will store them in a tar called 'result0505720401.tar'. The parameters availa For more details of the parameters check the section 3.4 at: 'http://nxsa.esac.esa.int/nxsa-web/#aio' +------------------------------ +2. Getting XMM-Newton proprietary data +------------------------------ +To access proprietary data an extra variable is needed in the XMMNewton.download_data method. This variabe is prop which +can be True or False. If True a username and password is needed. A username and password can be passed by adding another +variable to the XMMNewton.download_data method called credentials_file. This variable is a string with the path to a +config.ini file with the desired username and password, e.g. credentials_file = "users/joe.bloggs/config.ini" + +Example config.ini file, + +.. code-block:: + + [user] + username = test + password = test + +If the credentials_file variable is not provided the method will ask for the username and password to be added manually +from the commandline + +.. code-block:: python + + >>> from astroquery.esa.xmm_newton import XMMNewton + >>> + >>> XMMNewton.download_data('0505720401',level="PPS",extension="PDF",instname="M1",filename="result0505720401.tar",prop=True) + INFO: File result0505720401.tar downloaded to current directory [astroquery.esa.xmm_newton.core] + +This will download all PPS files for the observation '0505720401' and instrument MOS1, with 'PDF' extension and any +proprietary data. It will store them in a tar called 'result0505720401.tar'. + ------------------------------- -2. Getting XMM-Newton postcards +3. Getting XMM-Newton postcards ------------------------------- .. code-block:: python @@ -52,7 +81,7 @@ This will download the EPIC postcard for the observation '0505720401' and it wil 'P0505720401EPX000OIMAGE8000.PNG'. ------------------------------------------ -3. Getting XMM-Newton metadata through TAP +4. Getting XMM-Newton metadata through TAP ------------------------------------------ This function provides access to the XMM-Newton Science Archive database using the Table Access Protocol (TAP) and via the Astronomical Data @@ -81,7 +110,7 @@ This will execute an ADQL query to download the first 10 observations in the XMM stored in the file 'results10.csv'. The result of this query can be printed by doing print(result). ----------------------------------- -4. Getting table details of XSA TAP +5. Getting table details of XSA TAP ----------------------------------- .. code-block:: python @@ -104,7 +133,7 @@ stored in the file 'results10.csv'. The result of this query can be printed by d This will show the available tables in XSA TAP service in the XMM-Newton Science Archive. ------------------------------------- -5. Getting columns details of XSA TAP +6. Getting columns details of XSA TAP ------------------------------------- .. code-block:: python @@ -123,7 +152,7 @@ This will show the available tables in XSA TAP service in the XMM-Newton Science This will show the column details of the table 'v_all_observations' in XSA TAP service in the XMM-Newton Science Archive. -------------------------------------------- -6. Getting EPIC images from a given TAR file +7. Getting EPIC images from a given TAR file -------------------------------------------- .. code-block:: python @@ -139,7 +168,7 @@ This will show the column details of the table 'v_all_observations' in XSA TAP s This will extract the European Photon Imaging Camera (EPIC) images within the specified TAR file, bands, and instruments. It will also return a dictionary containing the paths to the extracted files. ------------------------------------------------------------------------------ -7. Getting the European Photon Imaging Camera (EPIC) metadata from the XSA TAP +8. Getting the European Photon Imaging Camera (EPIC) metadata from the XSA TAP ------------------------------------------------------------------------------ This function retrieves the EPIC metadata from a given target. From bd1643db6ee971741249e1a8801dfbfbfef6b454 Mon Sep 17 00:00:00 2001 From: javier-ballester Date: Fri, 3 Dec 2021 12:59:40 +0100 Subject: [PATCH 36/87] NXSAPCR-1091: Added remaining functionality to the branch --- astroquery/esa/xmm_newton/core.py | 19 ++++++++++++++++--- .../esa/xmm_newton/tests/test_xmm_newton.py | 16 ++++++++++++++++ .../tests/test_xmm_newton_remote.py | 3 +++ 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/astroquery/esa/xmm_newton/core.py b/astroquery/esa/xmm_newton/core.py index 836b0ae738..52e62c9439 100644 --- a/astroquery/esa/xmm_newton/core.py +++ b/astroquery/esa/xmm_newton/core.py @@ -20,6 +20,8 @@ from pathlib import Path import tarfile import os +from astroquery import log +import configparser from astropy.io import fits from . import conf, config @@ -27,12 +29,10 @@ from astropy.coordinates import SkyCoord from ...exceptions import LoginError - __all__ = ['XMMNewton', 'XMMNewtonClass'] class XMMNewtonClass(BaseQuery): - data_url = conf.DATA_ACTION data_aio_url = conf.DATA_ACTION_AIO metadata_url = conf.METADATA_ACTION @@ -40,6 +40,7 @@ class XMMNewtonClass(BaseQuery): def __init__(self, tap_handler=None): super(XMMNewtonClass, self).__init__() + self.configuration = configparser.ConfigParser() if tap_handler is None: self._tap = TapPlus(url="https://nxsa.esac.esa.int/tap-server/tap") @@ -48,7 +49,7 @@ def __init__(self, tap_handler=None): self._rmf_ftp = str("http://sasdev-xmm.esac.esa.int/pub/ccf/constituents/extras/responses/") def download_data(self, observation_id, *, filename=None, verbose=False, - cache=True, prop=False, username=None, password=None, **kwargs): + cache=True, prop=False, credentials_file=None, **kwargs): """ Download data from XMM-Newton @@ -113,7 +114,11 @@ def download_data(self, observation_id, *, filename=None, verbose=False, # If the user wants to access proprietary data, ask them for there credentials if prop: username, password = self._get_username_and_password(credentials_file) +<<<<<<< HEAD link = f"{link}&AIOUSER={username}&AIOPWD={password}" +======= + link = link + "&AIOUSER=" + username + "&AIOPWD=" + password +>>>>>>> 40576ad4 (xmm_newton-1.1_issue1093 Refactored the download data method and added the corresponding tests) if verbose: log.info(link) @@ -130,7 +135,11 @@ def download_data(self, observation_id, *, filename=None, verbose=False, self._download_file(link, filename, head_safe=True, cache=cache) if verbose: +<<<<<<< HEAD log.info(f"Wrote {link} to {filename}") +======= + log.info("Wrote {0} to {1}".format(link, filename)) +>>>>>>> 40576ad4 (xmm_newton-1.1_issue1093 Refactored the download data method and added the corresponding tests) log.setLevel(previouslevel) def get_postcard(self, observation_id, *, image_type="OBS_EPIC", @@ -277,7 +286,11 @@ def get_columns(self, table_name, *, only_names=True, verbose=False): return columns def _create_link(self, observation_id, **kwargs): +<<<<<<< HEAD link = f"{self.data_aio_url}obsno={observation_id}" +======= + link = self.data_aio_url + "obsno=" + observation_id +>>>>>>> 40576ad4 (xmm_newton-1.1_issue1093 Refactored the download data method and added the corresponding tests) link = link + "".join("&{0}={1}".format(key, val) for key, val in kwargs.items()) return link diff --git a/astroquery/esa/xmm_newton/tests/test_xmm_newton.py b/astroquery/esa/xmm_newton/tests/test_xmm_newton.py index fa3329cd30..99ccdd5fcf 100644 --- a/astroquery/esa/xmm_newton/tests/test_xmm_newton.py +++ b/astroquery/esa/xmm_newton/tests/test_xmm_newton.py @@ -9,7 +9,11 @@ Created on 4 Sept. 2019 """ +<<<<<<< HEAD from unittest.mock import patch +======= +from unittest.mock import patch, MagicMock +>>>>>>> 40576ad4 (xmm_newton-1.1_issue1093 Refactored the download data method and added the corresponding tests) import pytest import tarfile @@ -23,7 +27,11 @@ from astroquery.exceptions import LoginError +<<<<<<< HEAD class mockResponse(): +======= +class mockResponse: +>>>>>>> 40576ad4 (xmm_newton-1.1_issue1093 Refactored the download data method and added the corresponding tests) headers = {'Date': 'Wed, 24 Nov 2021 13:43:50 GMT', 'Server': 'Apache/2.4.6 (Red Hat Enterprise Linux) OpenSSL/1.0.2k-fips', 'Content-Disposition': 'inline; filename="0560181401.tar.gz"', @@ -34,8 +42,13 @@ class mockResponse(): @staticmethod def raise_for_status(): pass +<<<<<<< HEAD +======= + + +>>>>>>> 40576ad4 (xmm_newton-1.1_issue1093 Refactored the download data method and added the corresponding tests) class TestXMMNewton(): def get_dummy_tap_handler(self): parameters = {'query': "select top 10 * from v_public_observations", @@ -530,6 +543,7 @@ def test_request_link(self, mock_request): params = xsa._request_link("https://nxsa.esac.esa.int/nxsa-sl/servlet/data-action-aio?obsno=0560181401", None) assert params == {'filename': '0560181401.tar.gz'} +<<<<<<< HEAD @pytest.mark.xfail(raises=LoginError) @patch('astroquery.query.BaseQuery._request') def test_request_link_protected(self, mock_request): @@ -549,6 +563,8 @@ def test_request_link_incorrect_credentials(self, mock_request): mock_request.return_value = dummyclass xsa._request_link("https://nxsa.esac.esa.int/nxsa-sl/servlet/data-action-aio?obsno=0560181401", None) +======= +>>>>>>> 40576ad4 (xmm_newton-1.1_issue1093 Refactored the download data method and added the corresponding tests) def test_get_username_and_password(self): xsa = XMMNewtonClass(self.get_dummy_tap_handler()) username, password = xsa._get_username_and_password("astroquery/esa/xmm_newton/tests/my_config.ini") diff --git a/astroquery/esa/xmm_newton/tests/test_xmm_newton_remote.py b/astroquery/esa/xmm_newton/tests/test_xmm_newton_remote.py index 72a08f06df..5d5d5cb3e0 100644 --- a/astroquery/esa/xmm_newton/tests/test_xmm_newton_remote.py +++ b/astroquery/esa/xmm_newton/tests/test_xmm_newton_remote.py @@ -199,6 +199,7 @@ def test_download_proprietary_data_incorrect_credentials(self): parameters = {'observation_id': "0762470101", 'prop': 'True', 'credentials_file': "astroquery/esa/xmm_newton/tests/my_config.ini", +<<<<<<< HEAD 'level': "PPS", 'name': 'OBSMLI', 'filename': 'single', @@ -212,6 +213,8 @@ def test_download_proprietary_data_incorrect_credentials(self): @pytest.mark.xfail(raises=LoginError) def test_download_proprietary_data_without_credentials(self): parameters = {'observation_id': "0883780101", +======= +>>>>>>> 40576ad4 (xmm_newton-1.1_issue1093 Refactored the download data method and added the corresponding tests) 'level': "PPS", 'name': 'OBSMLI', 'filename': 'single', From aeeb4dc393fafff779f3c1c74c04d2af2e682cc2 Mon Sep 17 00:00:00 2001 From: javier-ballester Date: Fri, 3 Dec 2021 13:36:20 +0100 Subject: [PATCH 37/87] NXSA-1091: Added missing lines of code --- astroquery/esa/xmm_newton/core.py | 14 +------------ .../esa/xmm_newton/tests/test_xmm_newton.py | 21 +------------------ .../tests/test_xmm_newton_remote.py | 3 --- 3 files changed, 2 insertions(+), 36 deletions(-) diff --git a/astroquery/esa/xmm_newton/core.py b/astroquery/esa/xmm_newton/core.py index 52e62c9439..30c66a98af 100644 --- a/astroquery/esa/xmm_newton/core.py +++ b/astroquery/esa/xmm_newton/core.py @@ -24,7 +24,7 @@ import configparser from astropy.io import fits -from . import conf, config +from . import conf from astroquery import log from astropy.coordinates import SkyCoord from ...exceptions import LoginError @@ -114,11 +114,7 @@ def download_data(self, observation_id, *, filename=None, verbose=False, # If the user wants to access proprietary data, ask them for there credentials if prop: username, password = self._get_username_and_password(credentials_file) -<<<<<<< HEAD link = f"{link}&AIOUSER={username}&AIOPWD={password}" -======= - link = link + "&AIOUSER=" + username + "&AIOPWD=" + password ->>>>>>> 40576ad4 (xmm_newton-1.1_issue1093 Refactored the download data method and added the corresponding tests) if verbose: log.info(link) @@ -135,11 +131,7 @@ def download_data(self, observation_id, *, filename=None, verbose=False, self._download_file(link, filename, head_safe=True, cache=cache) if verbose: -<<<<<<< HEAD log.info(f"Wrote {link} to {filename}") -======= - log.info("Wrote {0} to {1}".format(link, filename)) ->>>>>>> 40576ad4 (xmm_newton-1.1_issue1093 Refactored the download data method and added the corresponding tests) log.setLevel(previouslevel) def get_postcard(self, observation_id, *, image_type="OBS_EPIC", @@ -286,11 +278,7 @@ def get_columns(self, table_name, *, only_names=True, verbose=False): return columns def _create_link(self, observation_id, **kwargs): -<<<<<<< HEAD link = f"{self.data_aio_url}obsno={observation_id}" -======= - link = self.data_aio_url + "obsno=" + observation_id ->>>>>>> 40576ad4 (xmm_newton-1.1_issue1093 Refactored the download data method and added the corresponding tests) link = link + "".join("&{0}={1}".format(key, val) for key, val in kwargs.items()) return link diff --git a/astroquery/esa/xmm_newton/tests/test_xmm_newton.py b/astroquery/esa/xmm_newton/tests/test_xmm_newton.py index 99ccdd5fcf..cc20817f32 100644 --- a/astroquery/esa/xmm_newton/tests/test_xmm_newton.py +++ b/astroquery/esa/xmm_newton/tests/test_xmm_newton.py @@ -1,19 +1,12 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst """ - @author: Elena Colomo @contact: ecolomo@esa.int - European Space Astronomy Centre (ESAC) European Space Agency (ESA) - Created on 4 Sept. 2019 """ -<<<<<<< HEAD from unittest.mock import patch -======= -from unittest.mock import patch, MagicMock ->>>>>>> 40576ad4 (xmm_newton-1.1_issue1093 Refactored the download data method and added the corresponding tests) import pytest import tarfile @@ -27,11 +20,7 @@ from astroquery.exceptions import LoginError -<<<<<<< HEAD class mockResponse(): -======= -class mockResponse: ->>>>>>> 40576ad4 (xmm_newton-1.1_issue1093 Refactored the download data method and added the corresponding tests) headers = {'Date': 'Wed, 24 Nov 2021 13:43:50 GMT', 'Server': 'Apache/2.4.6 (Red Hat Enterprise Linux) OpenSSL/1.0.2k-fips', 'Content-Disposition': 'inline; filename="0560181401.tar.gz"', @@ -42,13 +31,8 @@ class mockResponse: @staticmethod def raise_for_status(): pass -<<<<<<< HEAD - - -======= ->>>>>>> 40576ad4 (xmm_newton-1.1_issue1093 Refactored the download data method and added the corresponding tests) class TestXMMNewton(): def get_dummy_tap_handler(self): parameters = {'query': "select top 10 * from v_public_observations", @@ -543,7 +527,6 @@ def test_request_link(self, mock_request): params = xsa._request_link("https://nxsa.esac.esa.int/nxsa-sl/servlet/data-action-aio?obsno=0560181401", None) assert params == {'filename': '0560181401.tar.gz'} -<<<<<<< HEAD @pytest.mark.xfail(raises=LoginError) @patch('astroquery.query.BaseQuery._request') def test_request_link_protected(self, mock_request): @@ -563,8 +546,6 @@ def test_request_link_incorrect_credentials(self, mock_request): mock_request.return_value = dummyclass xsa._request_link("https://nxsa.esac.esa.int/nxsa-sl/servlet/data-action-aio?obsno=0560181401", None) -======= ->>>>>>> 40576ad4 (xmm_newton-1.1_issue1093 Refactored the download data method and added the corresponding tests) def test_get_username_and_password(self): xsa = XMMNewtonClass(self.get_dummy_tap_handler()) username, password = xsa._get_username_and_password("astroquery/esa/xmm_newton/tests/my_config.ini") @@ -579,4 +560,4 @@ def test_create_filename_None(self): def test_create_filename_Not_None(self): xsa = XMMNewtonClass(self.get_dummy_tap_handler()) filename = xsa._create_filename("Test", "0560181401", ['.tar', '.gz']) - assert filename == "Test.tar.gz" + assert filename == "Test.tar.gz" \ No newline at end of file diff --git a/astroquery/esa/xmm_newton/tests/test_xmm_newton_remote.py b/astroquery/esa/xmm_newton/tests/test_xmm_newton_remote.py index 5d5d5cb3e0..72a08f06df 100644 --- a/astroquery/esa/xmm_newton/tests/test_xmm_newton_remote.py +++ b/astroquery/esa/xmm_newton/tests/test_xmm_newton_remote.py @@ -199,7 +199,6 @@ def test_download_proprietary_data_incorrect_credentials(self): parameters = {'observation_id': "0762470101", 'prop': 'True', 'credentials_file': "astroquery/esa/xmm_newton/tests/my_config.ini", -<<<<<<< HEAD 'level': "PPS", 'name': 'OBSMLI', 'filename': 'single', @@ -213,8 +212,6 @@ def test_download_proprietary_data_incorrect_credentials(self): @pytest.mark.xfail(raises=LoginError) def test_download_proprietary_data_without_credentials(self): parameters = {'observation_id': "0883780101", -======= ->>>>>>> 40576ad4 (xmm_newton-1.1_issue1093 Refactored the download data method and added the corresponding tests) 'level': "PPS", 'name': 'OBSMLI', 'filename': 'single', From a45e0df008f3baef5655226a2c0fe5ab03db2762 Mon Sep 17 00:00:00 2001 From: javier-ballester Date: Tue, 7 Dec 2021 10:36:20 +0100 Subject: [PATCH 38/87] NXSA-1091: Removed an unnecessary print statement --- astroquery/esa/xmm_newton/core.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/astroquery/esa/xmm_newton/core.py b/astroquery/esa/xmm_newton/core.py index 30c66a98af..8e1d715d20 100644 --- a/astroquery/esa/xmm_newton/core.py +++ b/astroquery/esa/xmm_newton/core.py @@ -106,7 +106,7 @@ def download_data(self, observation_id, *, filename=None, verbose=False, reveals the url being sent which in turn reveals the users username and password """ previouslevel = log.getEffectiveLevel() - log.setLevel(50) + log.setLevel(21) # create url to access the aio link = self._create_link(observation_id, **kwargs) @@ -123,7 +123,6 @@ def download_data(self, observation_id, *, filename=None, verbose=False, params = self._request_link(link, cache) r_filename = params["filename"] suffixes = Path(r_filename).suffixes - print(suffixes) # get desired filename filename = self._create_filename(filename, observation_id, suffixes) From 4750b3b3d96b89f4cc626f2314c3e9cb18c555b7 Mon Sep 17 00:00:00 2001 From: javier-ballester Date: Tue, 7 Dec 2021 10:59:54 +0100 Subject: [PATCH 39/87] NXSA-1091: Updated description of download_data method --- astroquery/esa/xmm_newton/core.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/astroquery/esa/xmm_newton/core.py b/astroquery/esa/xmm_newton/core.py index 8e1d715d20..be547e7895 100644 --- a/astroquery/esa/xmm_newton/core.py +++ b/astroquery/esa/xmm_newton/core.py @@ -64,6 +64,13 @@ def download_data(self, observation_id, *, filename=None, verbose=False, verbose : bool optional, default 'False' flag to display information about the process + prop: boolean + optional, default 'False' + flag to download proprietary data, the method will then ask the user to + input their username and password either manually or using the credentials_file + credentials_file: string + optional, default None + path to where the users config.ini file is stored with their username and password level : string level to download, optional, by default everything is downloaded values: ODF, PPS From fb3359a35de04b7ae997f7b637a3416b684b9b39 Mon Sep 17 00:00:00 2001 From: javier-ballester Date: Thu, 16 Dec 2021 12:33:38 +0000 Subject: [PATCH 40/87] Reformated file to comply with pycodestyle --- CHANGES.rst | 4 ++++ astroquery/esa/xmm_newton/core.py | 2 +- astroquery/esa/xmm_newton/tests/test_xmm_newton.py | 2 +- astroquery/esa/xmm_newton/tests/test_xmm_newton_remote.py | 4 ++-- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 5d3da70957..af858d5f97 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -33,6 +33,10 @@ esa.jwst Service fixes and enhancements ------------------------------ +esa.xmm_newton +^^^^^^^^^^^^^^ + +- Add option to download proprietary data [#2251] eso ^^^ diff --git a/astroquery/esa/xmm_newton/core.py b/astroquery/esa/xmm_newton/core.py index be547e7895..bd4dabbd1a 100644 --- a/astroquery/esa/xmm_newton/core.py +++ b/astroquery/esa/xmm_newton/core.py @@ -306,7 +306,7 @@ def _request_link(self, link, cache): return params def _get_username_and_password(self, credentials_file): - if credentials_file != None: + if credentials_file is not None: self.configuration.read(credentials_file) username = self.configuration.get('user', 'username') password = self.configuration.get('user', 'password') diff --git a/astroquery/esa/xmm_newton/tests/test_xmm_newton.py b/astroquery/esa/xmm_newton/tests/test_xmm_newton.py index cc20817f32..dba4c24d0d 100644 --- a/astroquery/esa/xmm_newton/tests/test_xmm_newton.py +++ b/astroquery/esa/xmm_newton/tests/test_xmm_newton.py @@ -560,4 +560,4 @@ def test_create_filename_None(self): def test_create_filename_Not_None(self): xsa = XMMNewtonClass(self.get_dummy_tap_handler()) filename = xsa._create_filename("Test", "0560181401", ['.tar', '.gz']) - assert filename == "Test.tar.gz" \ No newline at end of file + assert filename == "Test.tar.gz" diff --git a/astroquery/esa/xmm_newton/tests/test_xmm_newton_remote.py b/astroquery/esa/xmm_newton/tests/test_xmm_newton_remote.py index 72a08f06df..ffd542cabf 100644 --- a/astroquery/esa/xmm_newton/tests/test_xmm_newton_remote.py +++ b/astroquery/esa/xmm_newton/tests/test_xmm_newton_remote.py @@ -21,6 +21,7 @@ from ..core import XMMNewtonClass from ..tests.dummy_tap_handler import DummyXMMNewtonTapHandler + class TestXMMNewtonRemote(): _files = { "0405320501": { @@ -221,7 +222,6 @@ def test_download_proprietary_data_without_credentials(self): xsa = XMMNewtonClass(self.get_dummy_tap_handler()) xsa.download_data(**parameters) - @pytest.mark.remote_data def test_get_epic_spectra(self): _tarname = "tarfile.tar" @@ -233,4 +233,4 @@ def test_get_epic_spectra(self): xsa = XMMNewtonClass(self.get_dummy_tap_handler()) res = xsa.get_epic_spectra(_tarname, _source_number, instrument=[]) - assert len(res) == 0 \ No newline at end of file + assert len(res) == 0 From 716b2911d40a27912fe5da08af32bf35677cb1da Mon Sep 17 00:00:00 2001 From: javier-ballester Date: Thu, 16 Dec 2021 12:46:21 +0000 Subject: [PATCH 41/87] Refactored config file and corresponding tests --- astroquery/esa/xmm_newton/tests/{ => data}/my_config.ini | 0 astroquery/esa/xmm_newton/tests/setup_package.py | 2 +- astroquery/esa/xmm_newton/tests/test_xmm_newton.py | 8 +++++++- 3 files changed, 8 insertions(+), 2 deletions(-) rename astroquery/esa/xmm_newton/tests/{ => data}/my_config.ini (100%) diff --git a/astroquery/esa/xmm_newton/tests/my_config.ini b/astroquery/esa/xmm_newton/tests/data/my_config.ini similarity index 100% rename from astroquery/esa/xmm_newton/tests/my_config.ini rename to astroquery/esa/xmm_newton/tests/data/my_config.ini diff --git a/astroquery/esa/xmm_newton/tests/setup_package.py b/astroquery/esa/xmm_newton/tests/setup_package.py index 19bc88a7de..a3829e4510 100644 --- a/astroquery/esa/xmm_newton/tests/setup_package.py +++ b/astroquery/esa/xmm_newton/tests/setup_package.py @@ -19,7 +19,7 @@ def get_package_data(): paths = [os.path.join('data', '*.tar'), os.path.join('data', '*.xml'), - os.path.join('my_config.ini') + os.path.join('data', '*.ini') ] # etc, add other extensions # you can also enlist files individually by names # finally construct and return a dict for the sub module diff --git a/astroquery/esa/xmm_newton/tests/test_xmm_newton.py b/astroquery/esa/xmm_newton/tests/test_xmm_newton.py index dba4c24d0d..341443447f 100644 --- a/astroquery/esa/xmm_newton/tests/test_xmm_newton.py +++ b/astroquery/esa/xmm_newton/tests/test_xmm_newton.py @@ -20,6 +20,11 @@ from astroquery.exceptions import LoginError +def data_path(filename): + data_dir = os.path.join(os.path.dirname(__file__), 'data') + return os.path.join(data_dir, filename) + + class mockResponse(): headers = {'Date': 'Wed, 24 Nov 2021 13:43:50 GMT', 'Server': 'Apache/2.4.6 (Red Hat Enterprise Linux) OpenSSL/1.0.2k-fips', @@ -548,7 +553,8 @@ def test_request_link_incorrect_credentials(self, mock_request): def test_get_username_and_password(self): xsa = XMMNewtonClass(self.get_dummy_tap_handler()) - username, password = xsa._get_username_and_password("astroquery/esa/xmm_newton/tests/my_config.ini") + file = data_path("my_config.ini") + username, password = xsa._get_username_and_password(file) assert username == "test" assert password == "test" From 9c60c56e13cc3970dd4c7452c9700a57f4b6f203 Mon Sep 17 00:00:00 2001 From: javier-ballester Date: Thu, 16 Dec 2021 12:57:30 +0000 Subject: [PATCH 42/87] Refactored xmm rst file --- docs/esa/xmm_newton.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/esa/xmm_newton.rst b/docs/esa/xmm_newton.rst index 8bbbc53737..6b62794e47 100644 --- a/docs/esa/xmm_newton.rst +++ b/docs/esa/xmm_newton.rst @@ -19,9 +19,9 @@ XMM-Newton Science Operations Centre. Examples ======== ------------------------------- +-------------------------- 1. Getting XMM-Newton data ------------------------------- +-------------------------- .. code-block:: python @@ -36,9 +36,9 @@ it will store them in a tar called 'result0505720401.tar'. The parameters availa For more details of the parameters check the section 3.4 at: 'http://nxsa.esac.esa.int/nxsa-web/#aio' ------------------------------- +-------------------------------------- 2. Getting XMM-Newton proprietary data ------------------------------- +-------------------------------------- To access proprietary data an extra variable is needed in the XMMNewton.download_data method. This variabe is prop which can be True or False. If True a username and password is needed. A username and password can be passed by adding another variable to the XMMNewton.download_data method called credentials_file. This variable is a string with the path to a From dab2b98380e67fe68f2fc9f12e93f930687d18af Mon Sep 17 00:00:00 2001 From: javier-ballester Date: Fri, 17 Dec 2021 10:21:32 +0000 Subject: [PATCH 43/87] Changed log levels if proprietary data --- astroquery/esa/xmm_newton/core.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/astroquery/esa/xmm_newton/core.py b/astroquery/esa/xmm_newton/core.py index bd4dabbd1a..ab2aa2d9ac 100644 --- a/astroquery/esa/xmm_newton/core.py +++ b/astroquery/esa/xmm_newton/core.py @@ -102,18 +102,11 @@ def download_data(self, observation_id, *, filename=None, verbose=False, file format, optional, by default all formats values: ASC, ASZ, FTZ, HTM, IND, PDF, PNG - Returns ------- None if not verbose. It downloads the observation indicated If verbose returns the filename """ - """ - Here we change the log level so that it is above 20, this is to stop a log.debug in query.py. this debug - reveals the url being sent which in turn reveals the users username and password - """ - previouslevel = log.getEffectiveLevel() - log.setLevel(21) # create url to access the aio link = self._create_link(observation_id, **kwargs) @@ -134,11 +127,20 @@ def download_data(self, observation_id, *, filename=None, verbose=False, # get desired filename filename = self._create_filename(filename, observation_id, suffixes) - self._download_file(link, filename, head_safe=True, cache=cache) + if prop: + """ + Here we change the log level so that it is above 20, this is to stop a log.debug (line 431) in query.py. + This debug reveals the url being sent which in turn reveals the users username and password + """ + previouslevel = log.getEffectiveLevel() + log.setLevel(21) + self._download_file(link, filename, head_safe=True, cache=cache) + log.setLevel(previouslevel) + else: + self._download_file(link, filename, head_safe=True, cache=cache) if verbose: log.info(f"Wrote {link} to {filename}") - log.setLevel(previouslevel) def get_postcard(self, observation_id, *, image_type="OBS_EPIC", filename=None, verbose=False): From 016172d19af43f96b090e853e3ef780fd39d82de Mon Sep 17 00:00:00 2001 From: javier-ballester Date: Fri, 17 Dec 2021 10:32:34 +0000 Subject: [PATCH 44/87] Fixing code style --- astroquery/esa/xmm_newton/core.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/astroquery/esa/xmm_newton/core.py b/astroquery/esa/xmm_newton/core.py index ab2aa2d9ac..1baf7708b3 100644 --- a/astroquery/esa/xmm_newton/core.py +++ b/astroquery/esa/xmm_newton/core.py @@ -126,12 +126,11 @@ def download_data(self, observation_id, *, filename=None, verbose=False, # get desired filename filename = self._create_filename(filename, observation_id, suffixes) - + """ + If prop we change the log level so that it is above 20, this is to stop a log.debug (line 431) in query.py. + This debug reveals the url being sent which in turn reveals the users username and password + """ if prop: - """ - Here we change the log level so that it is above 20, this is to stop a log.debug (line 431) in query.py. - This debug reveals the url being sent which in turn reveals the users username and password - """ previouslevel = log.getEffectiveLevel() log.setLevel(21) self._download_file(link, filename, head_safe=True, cache=cache) From 94b836ea5931a49732ac76f831de4760a8d54663 Mon Sep 17 00:00:00 2001 From: javier-ballester Date: Wed, 22 Dec 2021 13:16:43 +0100 Subject: [PATCH 45/87] Used _get_password as suggested and changed config file as suggested by astroquery team --- astroquery/esa/xmm_newton/core.py | 24 +++++++++---------- .../data/{my_config.ini => dummy_config.ini} | 2 +- .../esa/xmm_newton/tests/test_xmm_newton.py | 2 +- .../tests/test_xmm_newton_remote.py | 2 +- docs/esa/xmm_newton.rst | 12 ++++------ 5 files changed, 20 insertions(+), 22 deletions(-) rename astroquery/esa/xmm_newton/tests/data/{my_config.ini => dummy_config.ini} (67%) diff --git a/astroquery/esa/xmm_newton/core.py b/astroquery/esa/xmm_newton/core.py index 1baf7708b3..7d96d38834 100644 --- a/astroquery/esa/xmm_newton/core.py +++ b/astroquery/esa/xmm_newton/core.py @@ -14,13 +14,12 @@ import re from getpass import getpass from ...utils.tap.core import TapPlus -from ...query import BaseQuery +from ...query import BaseQuery, QueryWithLogin import shutil import cgi from pathlib import Path import tarfile import os -from astroquery import log import configparser from astropy.io import fits @@ -111,7 +110,7 @@ def download_data(self, observation_id, *, filename=None, verbose=False, # create url to access the aio link = self._create_link(observation_id, **kwargs) - # If the user wants to access proprietary data, ask them for there credentials + # If the user wants to access proprietary data, ask them for their credentials if prop: username, password = self._get_username_and_password(credentials_file) link = f"{link}&AIOUSER={username}&AIOPWD={password}" @@ -309,12 +308,13 @@ def _request_link(self, link, cache): def _get_username_and_password(self, credentials_file): if credentials_file is not None: self.configuration.read(credentials_file) - username = self.configuration.get('user', 'username') - password = self.configuration.get('user', 'password') + xmm_username = self.configuration.get("xmm_newton", "username") + password = self.configuration.get("xmm_newton", "password") else: - username = input("Username: ") - password = getpass("Password: ") - return username, password + xmm_username = input("Username: ") + password, password_from_keyring = QueryWithLogin._get_password(self, service_name="xmm_newton", + username=xmm_username, reenter=False) + return xmm_username, password def _create_filename(self, filename, observation_id, suffixes): if filename is not None: @@ -615,9 +615,9 @@ def get_epic_metadata(self, *, target_name=None, Tables containing the metadata of the target """ if not target_name and not coordinates: - raise Exception("Input parameters needed, " - "please provide the name " - "or the coordinates of the target") + raise ValueError("Input parameters needed, " + "please provide the name " + "or the coordinates of the target") epic_source = {"table": "xsa.v_epic_source", "column": "epic_source_equatorial_spoint"} @@ -635,7 +635,7 @@ def get_epic_metadata(self, *, target_name=None, c = SkyCoord.from_name(target_name, parse=True) if type(c) is not SkyCoord: - raise Exception("The coordinates must be an " + raise TypeError("The coordinates must be an " "astroquery.coordinates.SkyCoord object") if not radius: radius = 0.1 diff --git a/astroquery/esa/xmm_newton/tests/data/my_config.ini b/astroquery/esa/xmm_newton/tests/data/dummy_config.ini similarity index 67% rename from astroquery/esa/xmm_newton/tests/data/my_config.ini rename to astroquery/esa/xmm_newton/tests/data/dummy_config.ini index 97c415a7ed..7983bd2ae9 100644 --- a/astroquery/esa/xmm_newton/tests/data/my_config.ini +++ b/astroquery/esa/xmm_newton/tests/data/dummy_config.ini @@ -1,3 +1,3 @@ -[user] +[xmm_newton] username=test password=test \ No newline at end of file diff --git a/astroquery/esa/xmm_newton/tests/test_xmm_newton.py b/astroquery/esa/xmm_newton/tests/test_xmm_newton.py index 341443447f..71f4bc77ab 100644 --- a/astroquery/esa/xmm_newton/tests/test_xmm_newton.py +++ b/astroquery/esa/xmm_newton/tests/test_xmm_newton.py @@ -553,7 +553,7 @@ def test_request_link_incorrect_credentials(self, mock_request): def test_get_username_and_password(self): xsa = XMMNewtonClass(self.get_dummy_tap_handler()) - file = data_path("my_config.ini") + file = data_path("dummy_config.ini") username, password = xsa._get_username_and_password(file) assert username == "test" assert password == "test" diff --git a/astroquery/esa/xmm_newton/tests/test_xmm_newton_remote.py b/astroquery/esa/xmm_newton/tests/test_xmm_newton_remote.py index ffd542cabf..15b9a50ba8 100644 --- a/astroquery/esa/xmm_newton/tests/test_xmm_newton_remote.py +++ b/astroquery/esa/xmm_newton/tests/test_xmm_newton_remote.py @@ -199,7 +199,7 @@ def test_get_epic_metadata(self): def test_download_proprietary_data_incorrect_credentials(self): parameters = {'observation_id': "0762470101", 'prop': 'True', - 'credentials_file': "astroquery/esa/xmm_newton/tests/my_config.ini", + 'credentials_file': "astroquery/esa/xmm_newton/tests/data/dummy_config.ini", 'level': "PPS", 'name': 'OBSMLI', 'filename': 'single', diff --git a/docs/esa/xmm_newton.rst b/docs/esa/xmm_newton.rst index 6b62794e47..ff35c251ad 100644 --- a/docs/esa/xmm_newton.rst +++ b/docs/esa/xmm_newton.rst @@ -41,16 +41,14 @@ For more details of the parameters check the section 3.4 at: -------------------------------------- To access proprietary data an extra variable is needed in the XMMNewton.download_data method. This variabe is prop which can be True or False. If True a username and password is needed. A username and password can be passed by adding another -variable to the XMMNewton.download_data method called credentials_file. This variable is a string with the path to a -config.ini file with the desired username and password, e.g. credentials_file = "users/joe.bloggs/config.ini" - -Example config.ini file, +variable to the XMMNewton.download_data method called credentials_file. This variable is a string with the path to +~/.astropy/config/astroquery.cfg file. Inside this file add your desired username and password, e.g. .. code-block:: - [user] - username = test - password = test + [xmm_newton] + username = your_username + password = your_password If the credentials_file variable is not provided the method will ask for the username and password to be added manually from the commandline From ebaf072e822a9a5784294a731bbef4bc00f1974c Mon Sep 17 00:00:00 2001 From: javier-ballester Date: Fri, 7 Jan 2022 16:13:41 +0100 Subject: [PATCH 46/87] Added more tests to improve patch coverage --- .../esa/xmm_newton/tests/test_xmm_newton.py | 41 +++++++++++++------ 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/astroquery/esa/xmm_newton/tests/test_xmm_newton.py b/astroquery/esa/xmm_newton/tests/test_xmm_newton.py index 71f4bc77ab..fb96f3daf6 100644 --- a/astroquery/esa/xmm_newton/tests/test_xmm_newton.py +++ b/astroquery/esa/xmm_newton/tests/test_xmm_newton.py @@ -81,6 +81,11 @@ def test_get_columns(self): xsa.get_columns("table", only_names=True, verbose=True) dummyTapHandler.check_call("get_columns", parameters2) + def test_get_columns_valueerror(self): + with pytest.raises(ValueError): + xsa = XMMNewtonClass(self.get_dummy_tap_handler()) + xsa.get_columns("", only_names=True, verbose=True) + def test_dummy_handler(self): parameters2 = {'table_name': "table", 'only_names': True, @@ -532,24 +537,34 @@ def test_request_link(self, mock_request): params = xsa._request_link("https://nxsa.esac.esa.int/nxsa-sl/servlet/data-action-aio?obsno=0560181401", None) assert params == {'filename': '0560181401.tar.gz'} - @pytest.mark.xfail(raises=LoginError) @patch('astroquery.query.BaseQuery._request') def test_request_link_protected(self, mock_request): - xsa = XMMNewtonClass(self.get_dummy_tap_handler()) - dummyclass = mockResponse - dummyclass.headers = {} - mock_request.return_value = dummyclass - xsa._request_link("https://nxsa.esac.esa.int/nxsa-sl/servlet/data-action-aio?obsno=0560181401", None) + with pytest.raises(LoginError): + xsa = XMMNewtonClass(self.get_dummy_tap_handler()) + dummyclass = mockResponse + dummyclass.headers = {} + mock_request.return_value = dummyclass + xsa._request_link("https://nxsa.esac.esa.int/nxsa-sl/servlet/data-action-aio?obsno=0560181401", None) - @pytest.mark.xfail(raises=LoginError) @patch('astroquery.query.BaseQuery._request') def test_request_link_incorrect_credentials(self, mock_request): - xsa = XMMNewtonClass(self.get_dummy_tap_handler()) - dummyclass = mockResponse - dummyclass.headers = {} - dummyclass.status_code = 10 - mock_request.return_value = dummyclass - xsa._request_link("https://nxsa.esac.esa.int/nxsa-sl/servlet/data-action-aio?obsno=0560181401", None) + with pytest.raises(LoginError): + xsa = XMMNewtonClass(self.get_dummy_tap_handler()) + dummyclass = mockResponse + dummyclass.headers = {} + dummyclass.status_code = 10 + mock_request.return_value = dummyclass + xsa._request_link("https://nxsa.esac.esa.int/nxsa-sl/servlet/data-action-aio?obsno=0560181401", None) + + @patch('astroquery.query.BaseQuery._request') + def test_request_link_status_code_401(self, mock_request): + with pytest.raises(LoginError): + xsa = XMMNewtonClass(self.get_dummy_tap_handler()) + dummyclass = mockResponse + dummyclass.headers = {} + dummyclass.status_code = 401 + mock_request.return_value = dummyclass + xsa._request_link("https://nxsa.esac.esa.int/nxsa-sl/servlet/data-action-aio?obsno=0560181401", None) def test_get_username_and_password(self): xsa = XMMNewtonClass(self.get_dummy_tap_handler()) From 92d4fe495bbe6022d42f0440c013e39d559aec7d Mon Sep 17 00:00:00 2001 From: javier-ballester Date: Fri, 7 Jan 2022 16:33:07 +0100 Subject: [PATCH 47/87] refactored tests to improve patch coverage --- .../esa/xmm_newton/tests/test_xmm_newton.py | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/astroquery/esa/xmm_newton/tests/test_xmm_newton.py b/astroquery/esa/xmm_newton/tests/test_xmm_newton.py index fb96f3daf6..133b52c77b 100644 --- a/astroquery/esa/xmm_newton/tests/test_xmm_newton.py +++ b/astroquery/esa/xmm_newton/tests/test_xmm_newton.py @@ -81,10 +81,10 @@ def test_get_columns(self): xsa.get_columns("table", only_names=True, verbose=True) dummyTapHandler.check_call("get_columns", parameters2) + @pytest.mark.xfail(raises=ValueError) def test_get_columns_valueerror(self): - with pytest.raises(ValueError): - xsa = XMMNewtonClass(self.get_dummy_tap_handler()) - xsa.get_columns("", only_names=True, verbose=True) + xsa = XMMNewtonClass(self.get_dummy_tap_handler()) + xsa.get_columns("", only_names=True, verbose=True) def test_dummy_handler(self): parameters2 = {'table_name': "table", @@ -537,34 +537,34 @@ def test_request_link(self, mock_request): params = xsa._request_link("https://nxsa.esac.esa.int/nxsa-sl/servlet/data-action-aio?obsno=0560181401", None) assert params == {'filename': '0560181401.tar.gz'} + @pytest.mark.xfail(raises=LoginError) @patch('astroquery.query.BaseQuery._request') def test_request_link_protected(self, mock_request): - with pytest.raises(LoginError): - xsa = XMMNewtonClass(self.get_dummy_tap_handler()) - dummyclass = mockResponse - dummyclass.headers = {} - mock_request.return_value = dummyclass - xsa._request_link("https://nxsa.esac.esa.int/nxsa-sl/servlet/data-action-aio?obsno=0560181401", None) + xsa = XMMNewtonClass(self.get_dummy_tap_handler()) + dummyclass = mockResponse + dummyclass.headers = {} + mock_request.return_value = dummyclass + xsa._request_link("https://nxsa.esac.esa.int/nxsa-sl/servlet/data-action-aio?obsno=0560181401", None) + @pytest.mark.xfail(raises=LoginError) @patch('astroquery.query.BaseQuery._request') def test_request_link_incorrect_credentials(self, mock_request): - with pytest.raises(LoginError): - xsa = XMMNewtonClass(self.get_dummy_tap_handler()) - dummyclass = mockResponse - dummyclass.headers = {} - dummyclass.status_code = 10 - mock_request.return_value = dummyclass - xsa._request_link("https://nxsa.esac.esa.int/nxsa-sl/servlet/data-action-aio?obsno=0560181401", None) + xsa = XMMNewtonClass(self.get_dummy_tap_handler()) + dummyclass = mockResponse + dummyclass.headers = {} + dummyclass.status_code = 10 + mock_request.return_value = dummyclass + xsa._request_link("https://nxsa.esac.esa.int/nxsa-sl/servlet/data-action-aio?obsno=0560181401", None) + @pytest.mark.xfail(raises=LoginError) @patch('astroquery.query.BaseQuery._request') - def test_request_link_status_code_401(self, mock_request): - with pytest.raises(LoginError): - xsa = XMMNewtonClass(self.get_dummy_tap_handler()) - dummyclass = mockResponse - dummyclass.headers = {} - dummyclass.status_code = 401 - mock_request.return_value = dummyclass - xsa._request_link("https://nxsa.esac.esa.int/nxsa-sl/servlet/data-action-aio?obsno=0560181401", None) + def test_request_link_statuscode_401(self, mock_request): + xsa = XMMNewtonClass(self.get_dummy_tap_handler()) + dummyclass = mockResponse + dummyclass.headers = {} + dummyclass.status_code = 401 + mock_request.return_value = dummyclass + xsa._request_link("https://nxsa.esac.esa.int/nxsa-sl/servlet/data-action-aio?obsno=0560181401", None) def test_get_username_and_password(self): xsa = XMMNewtonClass(self.get_dummy_tap_handler()) From 43659b3dc7ff464e5f312fa473a14345a1923f94 Mon Sep 17 00:00:00 2001 From: javier-ballester Date: Fri, 7 Jan 2022 16:43:59 +0100 Subject: [PATCH 48/87] refactored tests to improve clarity --- .../esa/xmm_newton/tests/test_xmm_newton.py | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/astroquery/esa/xmm_newton/tests/test_xmm_newton.py b/astroquery/esa/xmm_newton/tests/test_xmm_newton.py index 133b52c77b..e08ff4e1f0 100644 --- a/astroquery/esa/xmm_newton/tests/test_xmm_newton.py +++ b/astroquery/esa/xmm_newton/tests/test_xmm_newton.py @@ -81,10 +81,10 @@ def test_get_columns(self): xsa.get_columns("table", only_names=True, verbose=True) dummyTapHandler.check_call("get_columns", parameters2) - @pytest.mark.xfail(raises=ValueError) def test_get_columns_valueerror(self): - xsa = XMMNewtonClass(self.get_dummy_tap_handler()) - xsa.get_columns("", only_names=True, verbose=True) + with pytest.raises(ValueError): + xsa = XMMNewtonClass(self.get_dummy_tap_handler()) + xsa.get_columns("", only_names=True, verbose=True) def test_dummy_handler(self): parameters2 = {'table_name': "table", @@ -537,34 +537,34 @@ def test_request_link(self, mock_request): params = xsa._request_link("https://nxsa.esac.esa.int/nxsa-sl/servlet/data-action-aio?obsno=0560181401", None) assert params == {'filename': '0560181401.tar.gz'} - @pytest.mark.xfail(raises=LoginError) @patch('astroquery.query.BaseQuery._request') def test_request_link_protected(self, mock_request): - xsa = XMMNewtonClass(self.get_dummy_tap_handler()) - dummyclass = mockResponse - dummyclass.headers = {} - mock_request.return_value = dummyclass - xsa._request_link("https://nxsa.esac.esa.int/nxsa-sl/servlet/data-action-aio?obsno=0560181401", None) + with pytest.raises(LoginError): + xsa = XMMNewtonClass(self.get_dummy_tap_handler()) + dummyclass = mockResponse + dummyclass.headers = {} + mock_request.return_value = dummyclass + xsa._request_link("https://nxsa.esac.esa.int/nxsa-sl/servlet/data-action-aio?obsno=0560181401", None) - @pytest.mark.xfail(raises=LoginError) @patch('astroquery.query.BaseQuery._request') def test_request_link_incorrect_credentials(self, mock_request): - xsa = XMMNewtonClass(self.get_dummy_tap_handler()) - dummyclass = mockResponse - dummyclass.headers = {} - dummyclass.status_code = 10 - mock_request.return_value = dummyclass - xsa._request_link("https://nxsa.esac.esa.int/nxsa-sl/servlet/data-action-aio?obsno=0560181401", None) + with pytest.raises(LoginError): + xsa = XMMNewtonClass(self.get_dummy_tap_handler()) + dummyclass = mockResponse + dummyclass.headers = {} + dummyclass.status_code = 10 + mock_request.return_value = dummyclass + xsa._request_link("https://nxsa.esac.esa.int/nxsa-sl/servlet/data-action-aio?obsno=0560181401", None) - @pytest.mark.xfail(raises=LoginError) @patch('astroquery.query.BaseQuery._request') - def test_request_link_statuscode_401(self, mock_request): - xsa = XMMNewtonClass(self.get_dummy_tap_handler()) - dummyclass = mockResponse - dummyclass.headers = {} - dummyclass.status_code = 401 - mock_request.return_value = dummyclass - xsa._request_link("https://nxsa.esac.esa.int/nxsa-sl/servlet/data-action-aio?obsno=0560181401", None) + def test_request_link_with_statuscode_401(self, mock_request): + with pytest.raises(LoginError): + xsa = XMMNewtonClass(self.get_dummy_tap_handler()) + dummyclass = mockResponse + dummyclass.headers = {} + dummyclass.status_code = 401 + mock_request.return_value = dummyclass + xsa._request_link("https://nxsa.esac.esa.int/nxsa-sl/servlet/data-action-aio?obsno=0560181401", None) def test_get_username_and_password(self): xsa = XMMNewtonClass(self.get_dummy_tap_handler()) From 0c365f33704f703b7b436fd72453d2f9a4e3f5ff Mon Sep 17 00:00:00 2001 From: javier-ballester Date: Tue, 18 Jan 2022 10:52:56 +0100 Subject: [PATCH 49/87] updated CHANGES.rst --- CHANGES.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index af858d5f97..38a0f6c2fb 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -7,6 +7,10 @@ New Tools and Services Service fixes and enhancements ------------------------------ +esa.xmm_newton +^^^^^^^^^^^^^^ + +- Add option to download proprietary data [#2251] esa.jwst ^^^^^^^^^^ @@ -33,10 +37,6 @@ esa.jwst Service fixes and enhancements ------------------------------ -esa.xmm_newton -^^^^^^^^^^^^^^ - -- Add option to download proprietary data [#2251] eso ^^^ From fb84a65093a266b1a12a939713ca80517ac96dfd Mon Sep 17 00:00:00 2001 From: Natanael Date: Tue, 18 Jan 2022 18:17:56 -0300 Subject: [PATCH 50/87] sdss crossid tests with spectro paramas --- astroquery/sdss/tests/test_sdss_remote.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/astroquery/sdss/tests/test_sdss_remote.py b/astroquery/sdss/tests/test_sdss_remote.py index 1eaa1e9dc6..9e9d301758 100644 --- a/astroquery/sdss/tests/test_sdss_remote.py +++ b/astroquery/sdss/tests/test_sdss_remote.py @@ -174,3 +174,16 @@ def test_query_crossid(self): assert isinstance(query2, Table) assert query2['objID'][0] == query1['objID'][0] == query2['objID'][1] + + def test_spectro_query_crossid(self): + query1 = sdss.SDSS.query_crossid_async( + self.coords, specobj_fields=['specObjID', 'z'], cache=False) + query2 = sdss.SDSS.query_crossid_async( + [self.coords, self.coords], + specobj_fields=['specObjID', 'z'], + cache=False) + assert isinstance(query1, Table) + assert query1['specObjID'][0] == 845594848269461504 + + assert isinstance(query2, Table) + assert query2['specObjID'][0] == query2['specObjID'][1] == query1['specObjID'][0] From 0eaa8240a68e82f75842d7c51438744ee19ec8a2 Mon Sep 17 00:00:00 2001 From: Natanael Date: Tue, 18 Jan 2022 18:21:38 -0300 Subject: [PATCH 51/87] fix PEP8 in last commit --- astroquery/sdss/tests/test_sdss_remote.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/astroquery/sdss/tests/test_sdss_remote.py b/astroquery/sdss/tests/test_sdss_remote.py index 9e9d301758..462fa697e0 100644 --- a/astroquery/sdss/tests/test_sdss_remote.py +++ b/astroquery/sdss/tests/test_sdss_remote.py @@ -179,8 +179,8 @@ def test_spectro_query_crossid(self): query1 = sdss.SDSS.query_crossid_async( self.coords, specobj_fields=['specObjID', 'z'], cache=False) query2 = sdss.SDSS.query_crossid_async( - [self.coords, self.coords], - specobj_fields=['specObjID', 'z'], + [self.coords, self.coords], + specobj_fields=['specObjID', 'z'], cache=False) assert isinstance(query1, Table) assert query1['specObjID'][0] == 845594848269461504 From 0851a405a8c9768cf664a5d14f2441935f4d38ff Mon Sep 17 00:00:00 2001 From: jespinosaar Date: Wed, 19 Jan 2022 11:55:53 +0100 Subject: [PATCH 52/87] Name convention for ESA modules --- docs/esa/hubble.rst | 6 +++--- docs/esa/jwst.rst | 6 +++--- docs/esa/xmm_newton.rst | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/esa/hubble.rst b/docs/esa/hubble.rst index dabb4dd0d3..00cedcc340 100644 --- a/docs/esa/hubble.rst +++ b/docs/esa/hubble.rst @@ -2,9 +2,9 @@ .. _astroquery.esa.hubble: -************************************ -esa.hubble (`astroquery.esa.hubble`) -************************************ +***************************************** +ESA HST Archive (`astroquery.esa.hubble`) +***************************************** The Hubble Space Telescope (HST) is a joint ESA/NASA orbiting astronomical observatory operating from the near-infrared into the ultraviolet. Launched diff --git a/docs/esa/jwst.rst b/docs/esa/jwst.rst index f871790e6a..d14bc37095 100644 --- a/docs/esa/jwst.rst +++ b/docs/esa/jwst.rst @@ -2,9 +2,9 @@ .. _astroquery.esa.jwst: -********************************* -JWST TAP+ (`astroquery.esa.jwst`) -********************************* +**************************************** +ESA JWST Archive (`astroquery.esa.jwst`) +**************************************** The James Webb Space Telescope (JWST) is a collaborative project between NASA, ESA, and the Canadian Space Agency (CSA). Although radically different in diff --git a/docs/esa/xmm_newton.rst b/docs/esa/xmm_newton.rst index ff35c251ad..c6fa76aeea 100644 --- a/docs/esa/xmm_newton.rst +++ b/docs/esa/xmm_newton.rst @@ -2,9 +2,9 @@ .. _astroquery.esa.xmm_newton: -**************************************** -xmm_newton (`astroquery.esa.xmm_newton`) -**************************************** +**************************************************** +ESA XMM-Newton Archive (`astroquery.esa.xmm_newton`) +**************************************************** The X-ray Multi-Mirror Mission, XMM-Newton, is an ESA X-ray observatory launched on 10 December 1999. From d8332e31206c2b24f2f154612ba84f2b32f868ae Mon Sep 17 00:00:00 2001 From: Eero Vaher Date: Tue, 25 Jan 2022 00:55:18 +0100 Subject: [PATCH 53/87] Fix documentation build The newly released `numpydoc` 1.2 emits many warnings about docstrings. The causes of these warnings are addressed by this commit and the documentation build should now succeed. --- astroquery/alma/core.py | 2 ++ astroquery/ipac/irsa/irsa_dust/core.py | 10 +++++----- astroquery/ipac/ned/core.py | 4 ++-- astroquery/jplspec/core.py | 4 ++-- astroquery/linelists/cdms/core.py | 4 ++-- astroquery/mast/collections.py | 6 +++--- astroquery/mast/observations.py | 4 ++-- astroquery/open_exoplanet_catalogue/oec_query.py | 2 +- astroquery/vo_conesearch/validator/inspect.py | 2 +- 9 files changed, 20 insertions(+), 18 deletions(-) diff --git a/astroquery/alma/core.py b/astroquery/alma/core.py index 92d79713a5..0d8650f963 100644 --- a/astroquery/alma/core.py +++ b/astroquery/alma/core.py @@ -397,6 +397,8 @@ def query_sia(self, pos=None, band=None, time=None, pol=None, Parameters ---------- + **kwargs + Parameters for the SIA service. Returns ------- diff --git a/astroquery/ipac/irsa/irsa_dust/core.py b/astroquery/ipac/irsa/irsa_dust/core.py index 74a6d9125e..719424ee85 100644 --- a/astroquery/ipac/irsa/irsa_dust/core.py +++ b/astroquery/ipac/irsa/irsa_dust/core.py @@ -123,7 +123,7 @@ def get_images_async(self, coordinate, radius=None, image_type=None, to remote server. Defaults to `False`. Returns - -------- + ------- list : list A list of context-managers that yield readable file-like objects. """ @@ -146,7 +146,7 @@ def get_image_list(self, coordinate, radius=None, image_type=None, of URLs to the Irsa-Dust images. Parameters - ----------- + ---------- coordinate : str Can be either the name of an object or a coordinate string If a name, must be resolvable by NED, SIMBAD, 2MASS, or SWAS. @@ -169,7 +169,7 @@ def get_image_list(self, coordinate, radius=None, image_type=None, to remote server. Defaults to `False`. Returns - -------- + ------- url_list : list A list of URLs to the FITS images corresponding to the queried object. @@ -203,7 +203,7 @@ def get_extinction_table(self, coordinate, radius=None, timeout=TIMEOUT, server. Defaults to `~astroquery.ipac.irsa.irsa_dust.IrsaDustClass.TIMEOUT`. Returns - -------- + ------- table : `~astropy.table.Table` """ readable_obj = self.get_extinction_table_async( @@ -289,7 +289,7 @@ def get_query_table(self, coordinate, radius=None, Defaults to `~astroquery.ipac.irsa.irsa_dust.IrsaDustClass.DUST_SERVICE_URL`. Returns - -------- + ------- table : `~astropy.table.Table` Table representing the query results, (all or as per specified). """ diff --git a/astroquery/ipac/ned/core.py b/astroquery/ipac/ned/core.py index 82e8537cdb..b943788540 100644 --- a/astroquery/ipac/ned/core.py +++ b/astroquery/ipac/ned/core.py @@ -385,7 +385,7 @@ def get_images_async(self, object_name, get_query_payload=False, request. Defaults to `False` Returns - -------- + ------- A list of context-managers that yield readable file-like objects """ @@ -438,7 +438,7 @@ def get_spectra_async(self, object_name, get_query_payload=False, request. Defaults to `False` Returns - -------- + ------- A list of context-managers that yield readable file-like objects """ diff --git a/astroquery/jplspec/core.py b/astroquery/jplspec/core.py index bf79ecc9c5..45bb9b66ef 100644 --- a/astroquery/jplspec/core.py +++ b/astroquery/jplspec/core.py @@ -191,12 +191,12 @@ def get_species_table(self, catfile='catdir.cat'): | (I6,X, A13, I6, 7F7.4, I2) Parameters - ----------- + ---------- catfile : str, name of file, default 'catdir.cat' The catalog file, installed locally along with the package Returns - -------- + ------- Table: `~astropy.table.Table` | TAG : The species tag or molecular identifier. | NAME : An ASCII name for the species. diff --git a/astroquery/linelists/cdms/core.py b/astroquery/linelists/cdms/core.py index e330cbdb92..8988f0f446 100644 --- a/astroquery/linelists/cdms/core.py +++ b/astroquery/linelists/cdms/core.py @@ -254,12 +254,12 @@ def get_species_table(self, catfile='catdir.cat'): The table is derived from https://cdms.astro.uni-koeln.de/classic/entries/partition_function.html Parameters - ----------- + ---------- catfile : str, name of file, default 'catdir.cat' The catalog file, installed locally along with the package Returns - -------- + ------- Table: `~astropy.table.Table` | tag : The species tag or molecular identifier. | molecule : An ASCII name for the species. diff --git a/astroquery/mast/collections.py b/astroquery/mast/collections.py index 39d66dbeff..ce5de33f59 100644 --- a/astroquery/mast/collections.py +++ b/astroquery/mast/collections.py @@ -320,7 +320,7 @@ def query_hsc_matchid_async(self, match, version=3, pagesize=None, page=None): one sepcific page of results. Returns - -------- + ------- response : list of `~requests.Response` """ @@ -356,7 +356,7 @@ def get_hsc_spectra_async(self, pagesize=None, page=None): one sepcific page of results. Returns - -------- + ------- response : list of `~requests.Response` """ @@ -387,7 +387,7 @@ def download_hsc_spectra(self, spectra, download_dir=None, cache=True, curl_flag will be downloaded that can be used to download the data files at a later time. Returns - -------- + ------- response : list of `~requests.Response` """ diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index fba7fc8dcf..3d4ae5c79d 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -76,7 +76,7 @@ def list_missions(self): Lists data missions archived by MAST and avaiable through `astroquery.mast`. Returns - -------- + ------- response : list List of available missions. """ @@ -106,7 +106,7 @@ def get_metadata(self, query_type): The query to get metadata for. Options are observations, and products. Returns - -------- + ------- response : `~astropy.table.Table` The metadata table. """ diff --git a/astroquery/open_exoplanet_catalogue/oec_query.py b/astroquery/open_exoplanet_catalogue/oec_query.py index bd5ce408f8..9d6922bbd2 100644 --- a/astroquery/open_exoplanet_catalogue/oec_query.py +++ b/astroquery/open_exoplanet_catalogue/oec_query.py @@ -19,7 +19,7 @@ def get_catalogue(filepath=None): Parses the Open Exoplanet Catalogue file. Parameters - ----------- + ---------- filepath : str or None if no filepath is given, remote source is used. diff --git a/astroquery/vo_conesearch/validator/inspect.py b/astroquery/vo_conesearch/validator/inspect.py index 529a2f7319..8f0b656c1b 100644 --- a/astroquery/vo_conesearch/validator/inspect.py +++ b/astroquery/vo_conesearch/validator/inspect.py @@ -138,7 +138,7 @@ def print_cat(self, key, fout=None): If not found, nothing is written out. Parameters - ----------- + ---------- key : str Catalog key. From 0647aa963fe689f4d5619bfd46d5220abc9e3d2d Mon Sep 17 00:00:00 2001 From: Eero Vaher Date: Tue, 25 Jan 2022 02:29:55 +0100 Subject: [PATCH 54/87] Add SIA2 keywords to AlmaClass.query_sia docstring --- astroquery/alma/core.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/astroquery/alma/core.py b/astroquery/alma/core.py index 0d8650f963..2b26362156 100644 --- a/astroquery/alma/core.py +++ b/astroquery/alma/core.py @@ -20,6 +20,7 @@ from astropy.utils.exceptions import AstropyDeprecationWarning from astropy import units as u from astropy.time import Time +from pyvo.dal.sia2 import SIA_PARAMETERS_DESC from ..exceptions import LoginError from ..utils import commons @@ -397,8 +398,7 @@ def query_sia(self, pos=None, band=None, time=None, pol=None, Parameters ---------- - **kwargs - Parameters for the SIA service. + _SIA2_PARAMETERS Returns ------- @@ -426,6 +426,8 @@ def query_sia(self, pos=None, band=None, time=None, pol=None, maxrec=maxrec, **kwargs) + query_sia.__doc__ = query_sia.__doc__.replace('_SIA2_PARAMETERS', SIA_PARAMETERS_DESC) + def query_tap(self, query, maxrec=None): """ Send query to the ALMA TAP. Results in pyvo.dal.TapResult format. From 03238e87e92301b11d327c79f605ad3db815e3fc Mon Sep 17 00:00:00 2001 From: Eero Vaher Date: Tue, 25 Jan 2022 20:09:27 +0100 Subject: [PATCH 55/87] Clean bad links from AlmaClass.query_sia docstring The docstring of `astroquery.alma.AlmaClass.query_sia()` is constructed with the help of `pyvo.dal.sia2.SIA_PARAMETERS_DESC` constant, which however contains links that Sphinx is unable to resolve. These links are removed by this commit so that the documentation build can succeed. --- astroquery/alma/core.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/astroquery/alma/core.py b/astroquery/alma/core.py index 2b26362156..c24e7821f9 100644 --- a/astroquery/alma/core.py +++ b/astroquery/alma/core.py @@ -426,6 +426,10 @@ def query_sia(self, pos=None, band=None, time=None, pol=None, maxrec=maxrec, **kwargs) + # SIA_PARAMETERS_DESC contains links that Sphinx can't resolve. + for var in ('POLARIZATION_STATES', 'CALIBRATION_LEVELS'): + SIA_PARAMETERS_DESC = SIA_PARAMETERS_DESC.replace(f'`pyvo.dam.obscore.{var}`', + f'pyvo.dam.obscore.{var}') query_sia.__doc__ = query_sia.__doc__.replace('_SIA2_PARAMETERS', SIA_PARAMETERS_DESC) def query_tap(self, query, maxrec=None): From eb12201f7f14fad3e68f665c5b6cd9877de1cc03 Mon Sep 17 00:00:00 2001 From: Jennifer Medina Date: Mon, 1 Nov 2021 10:58:09 -0400 Subject: [PATCH 56/87] added new TESTING global param and monkey patch method with new testing input param --- astroquery/mast/observations.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index 3d4ae5c79d..8836cd9423 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -51,6 +51,13 @@ class ObservationsClass(MastQueryWithLogin): Class for querying MAST observational data. """ + environment = os.environ['CONDA_DEFAULT_ENV'] + + if 'test' in environment.lower(): + TESTING = '.24test' + elif 'ops' in environment.lower(): + TESTING = None + def _parse_result(self, responses, verbose=False): # Used by the async_to_sync decorator functionality """ Parse the results of a list of `~requests.Response` objects and returns an `~astropy.table.Table` of results. @@ -71,7 +78,17 @@ def _parse_result(self, responses, verbose=False): # Used by the async_to_sync return self._portal_api_connection._parse_result(responses, verbose) - def list_missions(self): + def overwrite_service(input_service): + + frame = sys._getframe(1) + + # The input parameter that gets called (`service`) will be overwritten by `input_service` + frame.f_locals['service'] = input_service + + ctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(frame), + ctypes.c_int(0)) + + def list_missions(self, testing=TESTING): """ Lists data missions archived by MAST and avaiable through `astroquery.mast`. @@ -82,7 +99,7 @@ def list_missions(self): """ # getting all the histogram information - service = "Mast.Caom.All" + service = "Mast.Caom.All" if testing is None else testing params = {} response = self._portal_api_connection.service_request_async(service, params, format='extjs') json_response = response[0].json() From 140b92311d105d4d4f69297a998c6dd4de2673c4 Mon Sep 17 00:00:00 2001 From: Jennifer Medina Date: Tue, 2 Nov 2021 10:29:15 -0400 Subject: [PATCH 57/87] removing TESTING global variable. it will be instead fed in thru the backend test code --- astroquery/mast/observations.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index 8836cd9423..24f0c00d46 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -51,13 +51,6 @@ class ObservationsClass(MastQueryWithLogin): Class for querying MAST observational data. """ - environment = os.environ['CONDA_DEFAULT_ENV'] - - if 'test' in environment.lower(): - TESTING = '.24test' - elif 'ops' in environment.lower(): - TESTING = None - def _parse_result(self, responses, verbose=False): # Used by the async_to_sync decorator functionality """ Parse the results of a list of `~requests.Response` objects and returns an `~astropy.table.Table` of results. From 22736da1d5b49d4af91f209ca171c4e534e158dc Mon Sep 17 00:00:00 2001 From: Jennifer Medina Date: Tue, 2 Nov 2021 11:17:13 -0400 Subject: [PATCH 58/87] import ctypes and sys --- astroquery/mast/observations.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index 24f0c00d46..ef584da883 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -7,10 +7,12 @@ """ +import ctypes import warnings import json import time import os +import sys import uuid import numpy as np From b6b5da058e5a00ed177ad48019938890a779551f Mon Sep 17 00:00:00 2001 From: Jennifer Medina Date: Tue, 2 Nov 2021 11:19:21 -0400 Subject: [PATCH 59/87] service modification --- astroquery/mast/observations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index ef584da883..03502573fd 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -77,8 +77,8 @@ def overwrite_service(input_service): frame = sys._getframe(1) - # The input parameter that gets called (`service`) will be overwritten by `input_service` - frame.f_locals['service'] = input_service + # The input parameter that gets called (`service`) will be modified by `input_service` + frame.f_locals['service'] = frame.f_locals['service']+in_service ctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(frame), ctypes.c_int(0)) From 69a24b054307e728b2916ad3e61e757bfc9f47a1 Mon Sep 17 00:00:00 2001 From: Jennifer Medina Date: Tue, 2 Nov 2021 11:22:06 -0400 Subject: [PATCH 60/87] turning new func into hidden, and reorganizing. fixing typo in services call. --- astroquery/mast/observations.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index 03502573fd..44263ef4ff 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -53,6 +53,16 @@ class ObservationsClass(MastQueryWithLogin): Class for querying MAST observational data. """ + def _overwrite_service(input_service): + + frame = sys._getframe(1) + + # The input parameter that gets called (`service`) will be modified by `input_service` + frame.f_locals['service'] = frame.f_locals['service']+in_service + + ctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(frame), + ctypes.c_int(0)) + def _parse_result(self, responses, verbose=False): # Used by the async_to_sync decorator functionality """ Parse the results of a list of `~requests.Response` objects and returns an `~astropy.table.Table` of results. @@ -73,16 +83,6 @@ def _parse_result(self, responses, verbose=False): # Used by the async_to_sync return self._portal_api_connection._parse_result(responses, verbose) - def overwrite_service(input_service): - - frame = sys._getframe(1) - - # The input parameter that gets called (`service`) will be modified by `input_service` - frame.f_locals['service'] = frame.f_locals['service']+in_service - - ctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(frame), - ctypes.c_int(0)) - def list_missions(self, testing=TESTING): """ Lists data missions archived by MAST and avaiable through `astroquery.mast`. @@ -94,7 +94,7 @@ def list_missions(self, testing=TESTING): """ # getting all the histogram information - service = "Mast.Caom.All" if testing is None else testing + service = "Mast.Caom.All" if testing is None else _overwrite_service(testing) params = {} response = self._portal_api_connection.service_request_async(service, params, format='extjs') json_response = response[0].json() From bfccff7bc80c424110718a56dab7184f00ab0036 Mon Sep 17 00:00:00 2001 From: Jennifer Medina Date: Tue, 2 Nov 2021 11:27:11 -0400 Subject: [PATCH 61/87] overwrite services --- astroquery/mast/observations.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index 44263ef4ff..602f8b7fdf 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -93,8 +93,11 @@ def list_missions(self, testing=TESTING): List of available missions. """ + # calling `service` variable + service = "Mast.Caom.All" + if testing is not None: _overwrite_service(testing) + # getting all the histogram information - service = "Mast.Caom.All" if testing is None else _overwrite_service(testing) params = {} response = self._portal_api_connection.service_request_async(service, params, format='extjs') json_response = response[0].json() From c41acb4b0977fac674352ea3a9089da7c6208206 Mon Sep 17 00:00:00 2001 From: Jennifer Medina Date: Wed, 3 Nov 2021 11:14:56 -0400 Subject: [PATCH 62/87] making service calls static class variables. --- astroquery/mast/observations.py | 48 +++++++++++++++------------------ 1 file changed, 22 insertions(+), 26 deletions(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index 602f8b7fdf..50d390afcd 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -53,15 +53,12 @@ class ObservationsClass(MastQueryWithLogin): Class for querying MAST observational data. """ - def _overwrite_service(input_service): - - frame = sys._getframe(1) - - # The input parameter that gets called (`service`) will be modified by `input_service` - frame.f_locals['service'] = frame.f_locals['service']+in_service - - ctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(frame), - ctypes.c_int(0)) + # Calling static class variables + caom_all = 'Mast.Caom.All' + caom_cone = 'Mast.Caom.Cone' + caom_filtered_position = 'Mast.Caom.Filtered.Position' + caom_filtered = 'Mast.Caom.Filtered' + caom_products = 'Mast.Caom.Products' def _parse_result(self, responses, verbose=False): # Used by the async_to_sync decorator functionality """ @@ -83,7 +80,7 @@ def _parse_result(self, responses, verbose=False): # Used by the async_to_sync return self._portal_api_connection._parse_result(responses, verbose) - def list_missions(self, testing=TESTING): + def list_missions(self): """ Lists data missions archived by MAST and avaiable through `astroquery.mast`. @@ -93,9 +90,8 @@ def list_missions(self, testing=TESTING): List of available missions. """ - # calling `service` variable - service = "Mast.Caom.All" - if testing is not None: _overwrite_service(testing) + # calling `service` variable + service = ObservationsClass.caom_all # getting all the histogram information params = {} @@ -127,9 +123,9 @@ def get_metadata(self, query_type): """ if query_type.lower() == "observations": - colconf_name = "Mast.Caom.Cone" + colconf_name = ObservationsClass.caom_cone elif query_type.lower() == "products": - colconf_name = "Mast.Caom.Products" + colconf_name = ObservationsClass.caom_products else: raise InvalidQueryError("Unknown query type.") @@ -167,13 +163,13 @@ def _parse_caom_criteria(self, **criteria): # Build the mashup filter object and store it in the correct service_name entry if coordinates or objectname: - mashup_filters = self._portal_api_connection.build_filter_set("Mast.Caom.Cone", - "Mast.Caom.Filtered.Position", + mashup_filters = self._portal_api_connection.build_filter_set(ObservationsClass.caom_cone, + ObservationsClass.caom_filtercaom_filtered_position, **criteria) coordinates = utils.parse_input_location(coordinates, objectname) else: - mashup_filters = self._portal_api_connection.build_filter_set("Mast.Caom.Cone", - "Mast.Caom.Filtered", + mashup_filters = self._portal_api_connection.build_filter_set(ObservationsClass.caom_cone, + ObservationsClass.caom_filtered, **criteria) # handle position info (if any) @@ -224,7 +220,7 @@ def query_region_async(self, coordinates, radius=0.2*u.deg, pagesize=None, page= # if radius is just a number we assume degrees radius = coord.Angle(radius, u.deg) - service = 'Mast.Caom.Cone' + service = ObservationsClass.caom_cone params = {'ra': coordinates.ra.deg, 'dec': coordinates.dec.deg, 'radius': radius.deg} @@ -301,12 +297,12 @@ def query_criteria_async(self, pagesize=None, page=None, **criteria): raise InvalidQueryError("At least one non-positional criterion must be supplied.") if position: - service = "Mast.Caom.Filtered.Position" + service = ObservationsClass.caom_filtered_position params = {"columns": "*", "filters": mashup_filters, "position": position} else: - service = "Mast.Caom.Filtered" + service = ObservationsClass.caom_filtered params = {"columns": "*", "filters": mashup_filters} @@ -346,7 +342,7 @@ def query_region_count(self, coordinates, radius=0.2*u.deg, pagesize=None, page= # turn coordinates into the format position = ', '.join([str(x) for x in (coordinates.ra.deg, coordinates.dec.deg, radius.deg)]) - service = "Mast.Caom.Filtered.Position" + service = ObservationsClass.caom_filtered_position params = {"columns": "COUNT_BIG(*)", "filters": [], "position": position} @@ -414,12 +410,12 @@ def query_criteria_count(self, pagesize=None, page=None, **criteria): # send query if position: - service = "Mast.Caom.Filtered.Position" + service = ObservationsClass.caom_filtered_position params = {"columns": "COUNT_BIG(*)", "filters": mashup_filters, "position": position} else: - service = "Mast.Caom.Filtered" + service = ObservationsClass.caom_filtered params = {"columns": "COUNT_BIG(*)", "filters": mashup_filters} @@ -455,7 +451,7 @@ def get_product_list_async(self, observations): if len(observations) == 0: raise InvalidQueryError("Observation list is empty, no associated products.") - service = 'Mast.Caom.Products' + service = ObservationsClass.caom_products params = {'obsid': ','.join(observations)} return self._portal_api_connection.service_request_async(service, params) From 17f2b3182f1442c533226bab0dc7411e0df2d0f1 Mon Sep 17 00:00:00 2001 From: Jennifer Medina Date: Fri, 5 Nov 2021 14:38:22 -0400 Subject: [PATCH 63/87] static variables called explicitly within methods --- astroquery/mast/observations.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index 50d390afcd..595e7d98ce 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -40,6 +40,7 @@ from . import conf, utils from .core import MastQueryWithLogin +import astroquery __all__ = ['Observations', 'ObservationsClass', 'MastClass', 'Mast'] @@ -91,7 +92,7 @@ def list_missions(self): """ # calling `service` variable - service = ObservationsClass.caom_all + service = Observations.caom_all # getting all the histogram information params = {} @@ -107,6 +108,7 @@ def list_missions(self): missions.remove('hist') return missions + def get_metadata(self, query_type): """ Returns metadata about the requested query type. @@ -164,7 +166,7 @@ def _parse_caom_criteria(self, **criteria): # Build the mashup filter object and store it in the correct service_name entry if coordinates or objectname: mashup_filters = self._portal_api_connection.build_filter_set(ObservationsClass.caom_cone, - ObservationsClass.caom_filtercaom_filtered_position, + ObservationsClass.caom_filtered_position, **criteria) coordinates = utils.parse_input_location(coordinates, objectname) else: From 420fa25efac043f50b55f11fe7ee7e05c3f54935 Mon Sep 17 00:00:00 2001 From: Jennifer Medina Date: Tue, 9 Nov 2021 23:31:56 -0500 Subject: [PATCH 64/87] replacing class variable call with explicit class name to class variable call using self --- astroquery/mast/observations.py | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index 595e7d98ce..ec50fc1126 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -92,7 +92,7 @@ def list_missions(self): """ # calling `service` variable - service = Observations.caom_all + service = self.caom_all # getting all the histogram information params = {} @@ -125,9 +125,9 @@ def get_metadata(self, query_type): """ if query_type.lower() == "observations": - colconf_name = ObservationsClass.caom_cone + colconf_name = self.caom_cone elif query_type.lower() == "products": - colconf_name = ObservationsClass.caom_products + colconf_name = self.caom_products else: raise InvalidQueryError("Unknown query type.") @@ -165,14 +165,13 @@ def _parse_caom_criteria(self, **criteria): # Build the mashup filter object and store it in the correct service_name entry if coordinates or objectname: - mashup_filters = self._portal_api_connection.build_filter_set(ObservationsClass.caom_cone, - ObservationsClass.caom_filtered_position, - **criteria) + mashup_filters = self._portal_api_connection.build_filter_set(self.caom_cone, + self.caom_filtered_position, + **criteria) coordinates = utils.parse_input_location(coordinates, objectname) else: - mashup_filters = self._portal_api_connection.build_filter_set(ObservationsClass.caom_cone, - ObservationsClass.caom_filtered, - **criteria) + mashup_filters = self._portal_api_connection.build_filter_set(self.caom_cone, self.caom_filtered, + **criteria) # handle position info (if any) position = None @@ -222,7 +221,7 @@ def query_region_async(self, coordinates, radius=0.2*u.deg, pagesize=None, page= # if radius is just a number we assume degrees radius = coord.Angle(radius, u.deg) - service = ObservationsClass.caom_cone + service = self.caom_cone params = {'ra': coordinates.ra.deg, 'dec': coordinates.dec.deg, 'radius': radius.deg} @@ -299,12 +298,12 @@ def query_criteria_async(self, pagesize=None, page=None, **criteria): raise InvalidQueryError("At least one non-positional criterion must be supplied.") if position: - service = ObservationsClass.caom_filtered_position + service = self.caom_filtered_position params = {"columns": "*", "filters": mashup_filters, "position": position} else: - service = ObservationsClass.caom_filtered + service = self.caom_filtered params = {"columns": "*", "filters": mashup_filters} @@ -344,7 +343,7 @@ def query_region_count(self, coordinates, radius=0.2*u.deg, pagesize=None, page= # turn coordinates into the format position = ', '.join([str(x) for x in (coordinates.ra.deg, coordinates.dec.deg, radius.deg)]) - service = ObservationsClass.caom_filtered_position + service = self.caom_filtered_position params = {"columns": "COUNT_BIG(*)", "filters": [], "position": position} @@ -412,12 +411,12 @@ def query_criteria_count(self, pagesize=None, page=None, **criteria): # send query if position: - service = ObservationsClass.caom_filtered_position + service = self.caom_filtered_position params = {"columns": "COUNT_BIG(*)", "filters": mashup_filters, "position": position} else: - service = ObservationsClass.caom_filtered + service = self.caom_filtered params = {"columns": "COUNT_BIG(*)", "filters": mashup_filters} @@ -453,7 +452,7 @@ def get_product_list_async(self, observations): if len(observations) == 0: raise InvalidQueryError("Observation list is empty, no associated products.") - service = ObservationsClass.caom_products + service = self.caom_products params = {'obsid': ','.join(observations)} return self._portal_api_connection.service_request_async(service, params) From 95a7383e9bab7cb7b09387cc51f397640565ce03 Mon Sep 17 00:00:00 2001 From: Jennifer Medina Date: Tue, 9 Nov 2021 23:32:43 -0500 Subject: [PATCH 65/87] init variables under PortalAPI in discovery_portal.py. also code cleanup. --- astroquery/mast/discovery_portal.py | 7 +++++-- astroquery/mast/observations.py | 13 +++---------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/astroquery/mast/discovery_portal.py b/astroquery/mast/discovery_portal.py index 6ced3cc918..ec59525666 100644 --- a/astroquery/mast/discovery_portal.py +++ b/astroquery/mast/discovery_portal.py @@ -133,6 +133,9 @@ def __init__(self, session=None): self._column_configs = dict() self._current_service = None + self.tess_all_name = 'Mast.Catalogs.All.Tic' + self.dd_all_name = 'Mast.Catalogs.All.Disk.Detective' + def _request(self, method, url, params=None, data=None, headers=None, files=None, stream=False, auth=None, retrieve_all=True): """ @@ -238,10 +241,10 @@ def _get_col_config(self, service, fetch_name=None): more = False # for some catalogs this is not enough information if "tess" in fetch_name.lower(): - all_name = "Mast.Catalogs.All.Tic" + all_name = self.test_all_name more = True elif "dd." in fetch_name.lower(): - all_name = "Mast.Catalogs.All.DiskDetective" + all_name = self.dd_all_name more = True if more: diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index ec50fc1126..687049b8e6 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -6,13 +6,10 @@ This module contains various methods for querying MAST observations. """ - -import ctypes import warnings import json import time import os -import sys import uuid import numpy as np @@ -40,8 +37,6 @@ from . import conf, utils from .core import MastQueryWithLogin -import astroquery - __all__ = ['Observations', 'ObservationsClass', 'MastClass', 'Mast'] @@ -91,10 +86,8 @@ def list_missions(self): List of available missions. """ - # calling `service` variable - service = self.caom_all - # getting all the histogram information + service = self.caom_all params = {} response = self._portal_api_connection.service_request_async(service, params, format='extjs') json_response = response[0].json() @@ -108,7 +101,6 @@ def list_missions(self): missions.remove('hist') return missions - def get_metadata(self, query_type): """ Returns metadata about the requested query type. @@ -170,7 +162,8 @@ def _parse_caom_criteria(self, **criteria): **criteria) coordinates = utils.parse_input_location(coordinates, objectname) else: - mashup_filters = self._portal_api_connection.build_filter_set(self.caom_cone, self.caom_filtered, + mashup_filters = self._portal_api_connection.build_filter_set(self.caom_cone, + self.caom_filtered, **criteria) # handle position info (if any) From 6f464fef9285232e40d97879236d87e1ab67b161 Mon Sep 17 00:00:00 2001 From: Jennifer Medina Date: Mon, 6 Dec 2021 17:01:54 -0500 Subject: [PATCH 66/87] the configuration variables are now class attributes to make patching possible --- astroquery/mast/discovery_portal.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/astroquery/mast/discovery_portal.py b/astroquery/mast/discovery_portal.py index ec59525666..744cbbc49b 100644 --- a/astroquery/mast/discovery_portal.py +++ b/astroquery/mast/discovery_portal.py @@ -116,25 +116,25 @@ class PortalAPI(BaseQuery): Should be used to facilitate all Portal API queries. """ - def __init__(self, session=None): + MAST_REQUEST_URL = conf.server + "/api/v0/invoke" + COLUMNS_CONFIG_URL = conf.server + "/portal/Mashup/Mashup.asmx/columnsconfig" + MAST_DOWNLOAD_URL = conf.server + "/api/v0.1/Download/file" + MAST_BUNDLE_URL = conf.server + "/api/v0.1/Download/bundle" - super(PortalAPI, self).__init__() - if session: - self._session = session + TIMEOUT = conf.timeout + PAGESIZE = conf.pagesize - self.MAST_REQUEST_URL = conf.server + "/api/v0/invoke" - self.COLUMNS_CONFIG_URL = conf.server + "/portal/Mashup/Mashup.asmx/columnsconfig" - self.MAST_DOWNLOAD_URL = conf.server + "/api/v0.1/Download/file" - self.MAST_BUNDLE_URL = conf.server + "/api/v0.1/Download/bundle" + _column_configs = dict() + _current_service = None - self.TIMEOUT = conf.timeout - self.PAGESIZE = conf.pagesize + tess_all_name = 'Mast.Catalogs.All.Tic' + dd_all_name = 'Mast.Catalogs.All.Disk.Detective' - self._column_configs = dict() - self._current_service = None + def __init__(self, session=None): - self.tess_all_name = 'Mast.Catalogs.All.Tic' - self.dd_all_name = 'Mast.Catalogs.All.Disk.Detective' + super(PortalAPI, self).__init__() + if session: + self._session = session def _request(self, method, url, params=None, data=None, headers=None, files=None, stream=False, auth=None, retrieve_all=True): @@ -485,6 +485,7 @@ def _get_columnsconfig_metadata(self, colconf_name): headers = {"User-Agent": self._session.headers["User-Agent"], "Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"} + response = self._request("POST", self.COLUMNS_CONFIG_URL, data=("colConfigId={}".format(colconf_name)), headers=headers) From a29ec90948488fd9f3f9daa2cad3a59731f29291 Mon Sep 17 00:00:00 2001 From: Jennifer Medina Date: Wed, 8 Dec 2021 22:47:18 -0500 Subject: [PATCH 67/87] adding class attribute: tic catalog --- astroquery/mast/collections.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/astroquery/mast/collections.py b/astroquery/mast/collections.py index ce5de33f59..532e49fe92 100644 --- a/astroquery/mast/collections.py +++ b/astroquery/mast/collections.py @@ -36,6 +36,8 @@ class CatalogsClass(MastQueryWithLogin): Class for querying MAST catalog data. """ + _catalogs_filtered_tic = 'Mast.Catalogs.Filtered.Tic' + def __init__(self): super().__init__() @@ -271,7 +273,7 @@ def query_criteria_async(self, catalog, pagesize=None, page=None, **criteria): self._current_connection = self._portal_api_connection if catalog.lower() == "tic": - service = "Mast.Catalogs.Filtered.Tic" + service = _catalogs_filtered_tic if coordinates or objectname: service += ".Position" service += ".Rows" # Using the rowstore version of the query for speed From c55ea8a17277b8428e4d130164187d28c60f886f Mon Sep 17 00:00:00 2001 From: Jennifer Medina Date: Wed, 8 Dec 2021 23:22:10 -0500 Subject: [PATCH 68/87] reverting some service calls back to string --- astroquery/mast/collections.py | 4 +--- astroquery/mast/discovery_portal.py | 7 ++----- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/astroquery/mast/collections.py b/astroquery/mast/collections.py index 532e49fe92..ce5de33f59 100644 --- a/astroquery/mast/collections.py +++ b/astroquery/mast/collections.py @@ -36,8 +36,6 @@ class CatalogsClass(MastQueryWithLogin): Class for querying MAST catalog data. """ - _catalogs_filtered_tic = 'Mast.Catalogs.Filtered.Tic' - def __init__(self): super().__init__() @@ -273,7 +271,7 @@ def query_criteria_async(self, catalog, pagesize=None, page=None, **criteria): self._current_connection = self._portal_api_connection if catalog.lower() == "tic": - service = _catalogs_filtered_tic + service = "Mast.Catalogs.Filtered.Tic" if coordinates or objectname: service += ".Position" service += ".Rows" # Using the rowstore version of the query for speed diff --git a/astroquery/mast/discovery_portal.py b/astroquery/mast/discovery_portal.py index 744cbbc49b..63f7841d4a 100644 --- a/astroquery/mast/discovery_portal.py +++ b/astroquery/mast/discovery_portal.py @@ -127,9 +127,6 @@ class PortalAPI(BaseQuery): _column_configs = dict() _current_service = None - tess_all_name = 'Mast.Catalogs.All.Tic' - dd_all_name = 'Mast.Catalogs.All.Disk.Detective' - def __init__(self, session=None): super(PortalAPI, self).__init__() @@ -241,10 +238,10 @@ def _get_col_config(self, service, fetch_name=None): more = False # for some catalogs this is not enough information if "tess" in fetch_name.lower(): - all_name = self.test_all_name + all_name = "Mast.Catalogs.All.Tic" more = True elif "dd." in fetch_name.lower(): - all_name = self.dd_all_name + all_name = "Mast.Catalogs.All.DiskDetective" more = True if more: From 9cc2220b09fef3b1ccd889bb356ac3527ae10fb0 Mon Sep 17 00:00:00 2001 From: Jennifer Medina Date: Thu, 9 Dec 2021 16:10:13 -0500 Subject: [PATCH 69/87] rewriting services attriutes so they can be patched --- astroquery/mast/services.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/astroquery/mast/services.py b/astroquery/mast/services.py index acc85df035..8f574568c1 100644 --- a/astroquery/mast/services.py +++ b/astroquery/mast/services.py @@ -101,15 +101,16 @@ class ServiceAPI(BaseQuery): Should be used to facilitate all microservice API queries. """ + SERVICE_URL = conf.server + REQUEST_URL = conf.server + "/api/v0.1/" + SERVICES = {} + def __init__(self, session=None): super().__init__() if session: self._session = session - self.REQUEST_URL = conf.server + "/api/v0.1/" - self.SERVICES = {} - self.TIMEOUT = conf.timeout def set_service_params(self, service_dict, service_name="", server_prefix=False): @@ -128,7 +129,7 @@ def set_service_params(self, service_dict, service_name="", server_prefix=False) vs. the default of mast.stsci.edu/service_name """ - service_url = conf.server + service_url = self.SERVICE_URL if server_prefix: service_url = service_url.replace("mast", f"{service_name}.mast") else: From 2b0cf70dafb5f35ee1c1719c4834dd25d897700c Mon Sep 17 00:00:00 2001 From: Jennifer Medina Date: Thu, 6 Jan 2022 17:20:03 -0500 Subject: [PATCH 70/87] adding underscores to hidden class attributes not intended for external use --- astroquery/mast/observations.py | 38 ++++++++++++++++----------------- astroquery/mast/services.py | 1 + 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index 687049b8e6..dfdf7611f7 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -50,11 +50,11 @@ class ObservationsClass(MastQueryWithLogin): """ # Calling static class variables - caom_all = 'Mast.Caom.All' - caom_cone = 'Mast.Caom.Cone' - caom_filtered_position = 'Mast.Caom.Filtered.Position' - caom_filtered = 'Mast.Caom.Filtered' - caom_products = 'Mast.Caom.Products' + _caom_all = 'Mast.Caom.All' + _caom_cone = 'Mast.Caom.Cone' + _caom_filtered_position = 'Mast.Caom.Filtered.Position' + _caom_filtered = 'Mast.Caom.Filtered' + _caom_products = 'Mast.Caom.Products' def _parse_result(self, responses, verbose=False): # Used by the async_to_sync decorator functionality """ @@ -87,7 +87,7 @@ def list_missions(self): """ # getting all the histogram information - service = self.caom_all + service = self._caom_all params = {} response = self._portal_api_connection.service_request_async(service, params, format='extjs') json_response = response[0].json() @@ -117,9 +117,9 @@ def get_metadata(self, query_type): """ if query_type.lower() == "observations": - colconf_name = self.caom_cone + colconf_name = self._caom_cone elif query_type.lower() == "products": - colconf_name = self.caom_products + colconf_name = self._caom_products else: raise InvalidQueryError("Unknown query type.") @@ -157,13 +157,13 @@ def _parse_caom_criteria(self, **criteria): # Build the mashup filter object and store it in the correct service_name entry if coordinates or objectname: - mashup_filters = self._portal_api_connection.build_filter_set(self.caom_cone, - self.caom_filtered_position, + mashup_filters = self._portal_api_connection.build_filter_set(self._caom_cone, + self._caom_filtered_position, **criteria) coordinates = utils.parse_input_location(coordinates, objectname) else: - mashup_filters = self._portal_api_connection.build_filter_set(self.caom_cone, - self.caom_filtered, + mashup_filters = self._portal_api_connection.build_filter_set(self._caom_cone, + self._caom_filtered, **criteria) # handle position info (if any) @@ -214,7 +214,7 @@ def query_region_async(self, coordinates, radius=0.2*u.deg, pagesize=None, page= # if radius is just a number we assume degrees radius = coord.Angle(radius, u.deg) - service = self.caom_cone + service = self._caom_cone params = {'ra': coordinates.ra.deg, 'dec': coordinates.dec.deg, 'radius': radius.deg} @@ -291,12 +291,12 @@ def query_criteria_async(self, pagesize=None, page=None, **criteria): raise InvalidQueryError("At least one non-positional criterion must be supplied.") if position: - service = self.caom_filtered_position + service = self._caom_filtered_position params = {"columns": "*", "filters": mashup_filters, "position": position} else: - service = self.caom_filtered + service = self._caom_filtered params = {"columns": "*", "filters": mashup_filters} @@ -336,7 +336,7 @@ def query_region_count(self, coordinates, radius=0.2*u.deg, pagesize=None, page= # turn coordinates into the format position = ', '.join([str(x) for x in (coordinates.ra.deg, coordinates.dec.deg, radius.deg)]) - service = self.caom_filtered_position + service = self._caom_filtered_position params = {"columns": "COUNT_BIG(*)", "filters": [], "position": position} @@ -404,12 +404,12 @@ def query_criteria_count(self, pagesize=None, page=None, **criteria): # send query if position: - service = self.caom_filtered_position + service = self._caom_filtered_position params = {"columns": "COUNT_BIG(*)", "filters": mashup_filters, "position": position} else: - service = self.caom_filtered + service = self._caom_filtered params = {"columns": "COUNT_BIG(*)", "filters": mashup_filters} @@ -445,7 +445,7 @@ def get_product_list_async(self, observations): if len(observations) == 0: raise InvalidQueryError("Observation list is empty, no associated products.") - service = self.caom_products + service = self._caom_products params = {'obsid': ','.join(observations)} return self._portal_api_connection.service_request_async(service, params) diff --git a/astroquery/mast/services.py b/astroquery/mast/services.py index 8f574568c1..4ee94500b0 100644 --- a/astroquery/mast/services.py +++ b/astroquery/mast/services.py @@ -249,6 +249,7 @@ def service_request_async(self, service, params, page_size=None, page=None, **kw compiled_service_args[service_argument] = found_argument.lower() request_url = self.REQUEST_URL + service_url.format(**compiled_service_args) + headers = { 'User-Agent': self._session.headers['User-Agent'], 'Content-type': 'application/x-www-form-urlencoded', From 45d7ab8b33fcef887b0a0a483cc03d9d9e3f97a1 Mon Sep 17 00:00:00 2001 From: Jennifer Medina Date: Thu, 6 Jan 2022 18:26:40 -0500 Subject: [PATCH 71/87] updating comments in observations.py to reference ObservationsClass._caom_filtered_position and fixing indentation --- astroquery/mast/observations.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index dfdf7611f7..61b551dc12 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -158,13 +158,13 @@ def _parse_caom_criteria(self, **criteria): # Build the mashup filter object and store it in the correct service_name entry if coordinates or objectname: mashup_filters = self._portal_api_connection.build_filter_set(self._caom_cone, - self._caom_filtered_position, - **criteria) + self._caom_filtered_position, + **criteria) coordinates = utils.parse_input_location(coordinates, objectname) else: mashup_filters = self._portal_api_connection.build_filter_set(self._caom_cone, - self._caom_filtered, - **criteria) + self._caom_filtered, + **criteria) # handle position info (if any) position = None @@ -173,7 +173,7 @@ def _parse_caom_criteria(self, **criteria): # if radius is just a number we assume degrees radius = coord.Angle(radius, u.deg) - # build the coordinates string needed by Mast.Caom.Filtered.Position + # build the coordinates string needed by ObservationsClass._caom_filtered_position position = ', '.join([str(x) for x in (coordinates.ra.deg, coordinates.dec.deg, radius.deg)]) return position, mashup_filters @@ -327,7 +327,7 @@ def query_region_count(self, coordinates, radius=0.2*u.deg, pagesize=None, page= response : int """ - # build the coordinates string needed by Mast.Caom.Filtered.Position + # build the coordinates string needed by ObservationsClass._caom_filtered_position coordinates = commons.parse_coordinates(coordinates) # if radius is just a number we assume degrees From f84e2a5dd1cb97e22c4e499361263f14c665ce01 Mon Sep 17 00:00:00 2001 From: Eero Vaher Date: Fri, 28 Jan 2022 18:55:10 +0100 Subject: [PATCH 72/87] Remove the list of available services from README The list in `README.rst` was a duplicate of the list in `docs/index.rst`. --- README.rst | 68 ++++-------------------------------------------------- 1 file changed, 4 insertions(+), 64 deletions(-) diff --git a/README.rst b/README.rst index 364aa857f2..17b10ea16f 100644 --- a/README.rst +++ b/README.rst @@ -88,10 +88,10 @@ Using astroquery ---------------- Importing astroquery on its own doesn't get you much: you need to import each -sub-module specifically. Check out the `docs`_ -to find a list of the tools available. The `API`_ -shows the standard suite of tools common to most modules, e.g. `query_object` -and `query_region`. +sub-module specifically. See the documentation for a list of `Available +Services `_. +The `API`_ shows the standard suite of tools common to most modules, e.g. +`query_object` and `query_region`. To report bugs and request features, please use the issue tracker. Code contributions are very welcome, though we encourage you to follow the `API`_ @@ -99,65 +99,6 @@ and `contributing guidelines `_ as much as possible. -List of Modules ---------------- - -The following modules have been completed using a common API: - - * `ALMA Archive `_ - * `Atomic Line List `_: A collection of more than 900,000 atomic transitions. - * `Besancon `_: Model of stellar population synthesis in the Galaxy. - * `CDS MOC Service `_: A collection of all-sky survey coverage maps. - * `CADC `_: Canadian Astronomy Data Centre. - * `CASDA `_: CSIRO ASKAP Science Data Archive. - * `ESASky `_: ESASky is a science driven discovery portal providing easy visualizations and full access to the entire sky as observed with ESA Space astronomy missions. - * `ESO Archive `_ - * `FIRST `_: Faint Images of the Radio Sky at Twenty-cm. 20-cm radio images of the extragalactic sky from the VLA. - * `Gaia `_: European Space Agency Gaia Archive. - * `ESA XMM `_: European Space Agency XMM-Newton Science Archive. - * `ESA Hubble `_: European Space Agency Hubble Science Archive. - * `ESA ISO `_: European Space Agency ISO Data Archive. - * `GAMA database `_ - * `Gemini `_: Gemini Archive. - * `HEASARC `_: NASA's High Energy Astrophysics Science Archive Research Center. - * `IBE `_: IRSA Image Server program interface (IBE) Query Tool provides access to the 2MASS, WISE, and PTF image archives. - * `IRSA `_: NASA/IPAC Infrared Science Archive. Science products for all of NASA's infrared and sub-mm missions. - * `IRSA dust `_: Galactic dust reddening and extinction maps from IRAS 100 um data. - * `MAGPIS `_: Multi-Array Galactic Plane Imaging Survey. 6 and 20-cm radio images of the Galactic plane from the VLA. - * `MAST `_: Barbara A. Mikulski Archive for Space Telescopes. - * `Minor Planet Center `_ - * `NASA ADS `_: SAO/NASA Astrophysics Data System. - * `NED `_: NASA/IPAC Extragalactic Database. Multiwavelength data from both surveys and publications. - * `NIST `_: National Institute of Standards and Technology (NIST) atomic lines database. - * `NRAO `_: Science data archive of the National Radio Astronomy Observatory. VLA, JVLA, VLBA and GBT data products. - * `NVAS archive `_ - * `Simbad `_: Basic data, cross-identifications, bibliography and measurements for astronomical objects outside the solar system. - * `Skyview `_: NASA SkyView service for imaging surveys. - * `Splatalogue `_: National Radio Astronomy Observatory (NRAO)-maintained (mostly) molecular radio and millimeter line list service. - * `UKIDSS `_: UKIRT Infrared Deep Sky Survey. JHK images of 7500 sq deg. in the northern sky. - * `Vamdc `_: VAMDC molecular line database. - * `Vizier `_: Set of 11,000+ published, multiwavelength catalogues hosted by the CDS. - * `VO Simple Cone Search `_ - * `xMatch `_: Cross-identify sources between very large data sets or between a user-uploaded list and a large catalogue. - -These others are functional, but do not follow a common or consistent API: - - * `Alfalfa `_: Arecibo Legacy Fast ALFA survey; extragalactic HI radio data. - * `CosmoSim `_: The CosmoSim database provides results from cosmological simulations performed within different projects: the MultiDark project, the BolshoiP project, and the CLUES project. - * `Exoplanet Orbit Database `_ - * `Fermi `_: Fermi gamma-ray telescope archive. - * `HITRAN `_: Access to the high-resolution transmission molecular absorption database. - * `JPL Horizons `_: JPL Solar System Dynamics Horizons Service. - * `JPL SBDB `_: JPL Solar System Dynamics Small-Body Database Browser Service. - * `Lamda `_: Leiden Atomic and Molecular Database; energy levels, radiative transitions, and collisional rates for astrophysically relevant atoms and molecules. - * `NASA Exoplanet Archive `_ - * `OAC API `_: Open Astronomy Catalog REST API Service. - * `Ogle `_: Optical Gravitational Lensing Experiment III; information on interstellar extinction towards the Galactic bulge. - * `Open Expolanet Catalog (OEC) `_ - * `SDSS `_: Sloan Digital Sky Survey data, including optical images, spectra, and spectral templates. - * `SHA `_: Spitzer Heritage Archive; infrared data products from the Spitzer Space Telescope. - - Citing Astroquery ----------------- @@ -188,7 +129,6 @@ Maintained by `Adam Ginsburg`_ and `Brigitta Sipocz .. _Download Stable ZIP: https://github.com/astropy/astroquery/zipball/stable .. _Download Stable TAR: https://github.com/astropy/astroquery/tarball/stable .. _View on Github: https://github.com/astropy/astroquery/ -.. _docs: http://astroquery.readthedocs.io .. _Documentation: http://astroquery.readthedocs.io .. _astropy.astroquery@gmail.com: mailto:astropy.astroquery@gmail.com .. _Adam Ginsburg: http://www.adamgginsburg.com From 3651a702cd4e3d76e071801d7faac618e16a2da3 Mon Sep 17 00:00:00 2001 From: Eero Vaher Date: Fri, 28 Jan 2022 18:57:20 +0100 Subject: [PATCH 73/87] Update lists of available services [ci skip] A previously missing module has been added to `docs/index.rst` and the lists are now alphabetically sorted. --- docs/index.rst | 37 ++++++++++++++++---------------- docs/solarsystem/solarsystem.rst | 2 +- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index d2f46bdcc4..4f9be1ecf2 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -185,23 +185,25 @@ The following modules have been completed using a common API: cadc/cadc.rst casda/casda.rst cds/cds.rst + linelists/cdms/cdms.rst + dace/dace.rst esa/hubble.rst esa/iso.rst esa/jwst.rst esa/xmm_newton.rst esasky/esasky.rst eso/eso.rst + image_cutouts/first/first.rst gaia/gaia.rst gama/gama.rst gemini/gemini.rst heasarc/heasarc.rst hips2fits/hips2fits.rst hitran/hitran.rst + ipac/irsa/irsa_dust/irsa_dust.rst ipac/irsa/ibe/ibe.rst ipac/irsa/irsa.rst - ipac/irsa/irsa_dust/irsa_dust.rst jplspec/jplspec.rst - linelists/cdms/cdms.rst magpis/magpis.rst mast/mast.rst mpc/mpc.rst @@ -221,7 +223,6 @@ The following modules have been completed using a common API: vo_conesearch/vo_conesearch.rst vsa/vsa.rst xmatch/xmatch.rst - dace/dace.rst These others are functional, but do not follow a common & consistent API: @@ -269,22 +270,22 @@ for each source) alfalfa/alfalfa.rst exoplanet_orbit_database/exoplanet_orbit_database.rst gama/gama.rst + ipac/irsa/irsa_dust/irsa_dust.rst ipac/irsa/ibe/ibe.rst ipac/irsa/irsa.rst - ipac/irsa/irsa_dust/irsa_dust.rst mast/mast.rst ipac/nexsci/nasa_exoplanet_archive.rst ipac/ned/ned.rst ogle/ogle.rst open_exoplanet_catalogue/open_exoplanet_catalogue.rst sdss/sdss.rst - ipac/irsa/sha/sha.rst simbad/simbad.rst + ipac/irsa/sha/sha.rst ukidss/ukidss.rst - vsa/vsa.rst vizier/vizier.rst - xmatch/xmatch.rst vo_conesearch/vo_conesearch.rst + vsa/vsa.rst + xmatch/xmatch.rst Archives -------- @@ -301,25 +302,25 @@ generally return a table listing the available data first. casda/casda.rst esa/hubble.rst esa/jwst.rst + esa/xmm_newton.rst eso/eso.rst fermi/fermi.rst gaia/gaia.rst + gemini/gemini.rst heasarc/heasarc.rst ipac/irsa/ibe/ibe.rst ipac/irsa/irsa.rst magpis/magpis.rst - gemini/gemini.rst mast/mast.rst ipac/ned/ned.rst noirlab/noirlab.rst nrao/nrao.rst nvas/nvas.rst sdss/sdss.rst + skyview/skyview.rst ipac/irsa/sha/sha.rst ukidss/ukidss.rst vsa/vsa.rst - skyview/skyview.rst - esa/xmm_newton.rst Simulations ----------- @@ -342,13 +343,13 @@ well as cross section and collision rates. Those services are: :maxdepth: 1 atomic/atomic.rst + linelists/cdms/cdms.rst + hitran/hitran.rst + jplspec/jplspec.rst lamda/lamda.rst nist/nist.rst splatalogue/splatalogue.rst vamdc/vamdc.rst - hitran/hitran.rst - linelists/cdms/cdms.rst - jplspec/jplspec.rst Other ----- @@ -359,12 +360,12 @@ above categories. Those services are here: .. toctree:: :maxdepth: 1 - nasa_ads/nasa_ads.rst - utils/tap.rst + astrometry_net/astrometry_net.rst + imcce/imcce.rst jplhorizons/jplhorizons.rst jplsbdb/jplsbdb.rst - imcce/imcce.rst - astrometry_net/astrometry_net.rst + nasa_ads/nasa_ads.rst + utils/tap.rst Topical Collections @@ -376,8 +377,8 @@ topical submodules: .. toctree:: :maxdepth: 1 - solarsystem/solarsystem.rst image_cutouts/image_cutouts.rst + solarsystem/solarsystem.rst Developer documentation diff --git a/docs/solarsystem/solarsystem.rst b/docs/solarsystem/solarsystem.rst index a2e940e864..013f8e6139 100644 --- a/docs/solarsystem/solarsystem.rst +++ b/docs/solarsystem/solarsystem.rst @@ -16,8 +16,8 @@ The currently available service providers and services are: .. toctree:: :maxdepth: 1 - jpl/jpl.rst imcce/imcce.rst + jpl/jpl.rst mpc/mpc.rst Reference/API From fd2a9cc82626e99c502c5d31419dd41691b77b0a Mon Sep 17 00:00:00 2001 From: Clara Brasseur Date: Fri, 16 Jul 2021 16:30:46 -0400 Subject: [PATCH 74/87] adding moving target tesscut and docs --- astroquery/mast/cutouts.py | 152 +++++++++++++++++++++++------- astroquery/mast/observations.py | 18 +++- docs/mast/mast.rst | 160 +++++++++++++++++++++----------- 3 files changed, 233 insertions(+), 97 deletions(-) diff --git a/astroquery/mast/cutouts.py b/astroquery/mast/cutouts.py index 29df098a1a..54378946fc 100644 --- a/astroquery/mast/cutouts.py +++ b/astroquery/mast/cutouts.py @@ -104,10 +104,13 @@ def __init__(self): super().__init__() services = {"sector": {"path": "sector"}, - "astrocut": {"path": "astrocut"}} + "astrocut": {"path": "astrocut"}, + "mt_sector": {"path": "moving_target/sector"}, + "mt_astrocut": {"path": "moving_target/astrocut"} + } self._service_api_connection.set_service_params(services, "tesscut") - def get_sectors(self, coordinates=None, radius=0*u.deg, objectname=None): + def get_sectors(self, coordinates=None, radius=0*u.deg, objectname=None, moving_target=None, mt_type=None): """ Get a list of the TESS data sectors whose footprints intersect with the given search area. @@ -117,36 +120,64 @@ def get_sectors(self, coordinates=None, radius=0*u.deg, objectname=None): coordinates : str or `astropy.coordinates` object, optional The target around which to search. It may be specified as a string or as the appropriate `astropy.coordinates` object. - One and only one of coordinates and objectname must be supplied. + One and only one of coordinates, objectname, and moving_target must be supplied. radius : str, float, or `~astropy.units.Quantity` object, optional Default 0 degrees. If supplied as a float degrees is the assumed unit. The string must be parsable by `~astropy.coordinates.Angle`. The appropriate `~astropy.units.Quantity` object from `astropy.units` may also be used. + This argument is ignored if moving_target is supplied. objectname : str, optional The target around which to search, by name (objectname="M104") or TIC ID (objectname="TIC 141914082"). - One and only one of coordinates and objectname must be supplied. + One and only one of coordinates, objectname, moving_target must be supplied. + moving_target : str, optional + The name or ID (as understood by the + `JPL ephemerides service `__) + of a moving target such as an asteroid or comet. + One and only one of coordinates, objectname, and moving_target must be supplied. + mt_type : str, optional + The moving target type, valid inputs are majorbody and smallbody. If not supplied + first majorbody is tried and then smallbody if a matching majorbody is not found. + This argument is ignored unless moving_target is supplied. Returns ------- response : `~astropy.table.Table` - Sector/camera/chip information for given coordinates/raduis. + Sector/camera/chip information for given coordinates/objectname/moving_target. """ - # Get Skycoord object for coordinates/object - coordinates = parse_input_location(coordinates, objectname) + if moving_target: - # If radius is just a number we assume degrees - radius = Angle(radius, u.deg) + # Check only ony object designator has been passed in + if objectname or coordinates: + raise InvalidQueryError("Only one of objectname, coordinates, and moving_target may be specified.") - params = {"ra": coordinates.ra.deg, - "dec": coordinates.dec.deg, - "radius": radius.deg} + params = {"obj_id": moving_target} - response = self._service_api_connection.service_request_async("sector", params) - response.raise_for_status() # Raise any errors + # Add optional parameter is present + if mt_type: + params["obj_type"] = mt_type + + response = self._service_api_connection.service_request_async("mt_sector", params) + + else: + + # Get Skycoord object for coordinates/object + coordinates = parse_input_location(coordinates, objectname) + + # If radius is just a number we assume degrees + radius = Angle(radius, u.deg) + + params = {"ra": coordinates.ra.deg, + "dec": coordinates.dec.deg, + "radius": radius.deg} + + response = self._service_api_connection.service_request_async("sector", params) + + # Raise any errors + response.raise_for_status() sector_json = response.json()['results'] sector_dict = {'sectorName': [], @@ -164,7 +195,8 @@ def get_sectors(self, coordinates=None, radius=0*u.deg, objectname=None): warnings.warn("Coordinates are not in any TESS sector.", NoResultsWarning) return Table(sector_dict) - def download_cutouts(self, coordinates=None, size=5, sector=None, path=".", inflate=True, objectname=None): + def download_cutouts(self, coordinates=None, size=5, sector=None, path=".", inflate=True, + objectname=None, moving_target=None, mt_type=None): """ Download cutout target pixel file(s) around the given coordinates with indicated size. @@ -173,7 +205,7 @@ def download_cutouts(self, coordinates=None, size=5, sector=None, path=".", infl coordinates : str or `astropy.coordinates` object, optional The target around which to search. It may be specified as a string or as the appropriate `astropy.coordinates` object. - One and only one of coordinates and objectname must be supplied. + One and only one of coordinates, objectname, and moving_target must be supplied. size : int, array-like, `~astropy.units.Quantity` Optional, default 5 pixels. The size of the cutout array. If ``size`` is a scalar number or @@ -198,27 +230,45 @@ def download_cutouts(self, coordinates=None, size=5, sector=None, path=".", infl objectname : str, optional The target around which to search, by name (objectname="M104") or TIC ID (objectname="TIC 141914082"). - One and only one of coordinates and objectname must be supplied. + One and only one of coordinates, objectname, and moving_target must be supplied. + moving_target : str, optional + The name or ID (as understood by the + `JPL ephemerides service `__) + of a moving target such as an asteroid or comet. + One and only one of coordinates, objectname, and moving_target must be supplied. + mt_type : str, optional + The moving target type, valid inputs are majorbody and smallbody. If not supplied + first majorbody is tried and then smallbody if a matching majorbody is not found. + This argument is ignored unless moving_target is supplied. Returns ------- response : `~astropy.table.Table` """ - # Get Skycoord object for coordinates/object - coordinates = parse_input_location(coordinates, objectname) + if moving_target: + # Check only ony object designator has been passed in + if objectname or coordinates: + raise InvalidQueryError("Only one of objectname, coordinates, and moving_target may be specified.") + + astrocut_request = f"moving_target/astrocut?obj_id={moving_target}" + if mt_type: + astrocut_request += f"&obj_type={mt_type}" + else: + # Get Skycoord object for coordinates/object + coordinates = parse_input_location(coordinates, objectname) + + astrocut_request = f"astrocut?ra={coordinates.ra.deg}&dec={coordinates.dec.deg}" + + # Adding the arguments that are common between moving/still astrocut requests size_dict = _parse_cutout_size(size) - - path = os.path.join(path, '') - astrocut_request = "ra={}&dec={}&y={}&x={}&units={}".format(coordinates.ra.deg, - coordinates.dec.deg, - size_dict["y"], - size_dict["x"], - size_dict["units"]) + astrocut_request += f"&y={size_dict['y']}&x={size_dict['x']}&units={size_dict['units']}" + if sector: astrocut_request += "§or={}".format(sector) - astrocut_url = self._service_api_connection.REQUEST_URL + "astrocut?" + astrocut_request + astrocut_url = self._service_api_connection.REQUEST_URL + astrocut_request + path = os.path.join(path, '') zipfile_path = "{}tesscut_{}.zip".format(path, time.strftime("%Y%m%d%H%M%S")) self._download_file(astrocut_url, zipfile_path) @@ -246,7 +296,8 @@ def download_cutouts(self, coordinates=None, size=5, sector=None, path=".", infl localpath_table['Local Path'] = [path+x for x in cutout_files] return localpath_table - def get_cutouts(self, coordinates=None, size=5, sector=None, objectname=None): + def get_cutouts(self, coordinates=None, size=5, sector=None, + objectname=None, moving_target=None, mt_type=None): """ Get cutout target pixel file(s) around the given coordinates with indicated size, and return them as a list of `~astropy.io.fits.HDUList` objects. @@ -256,7 +307,7 @@ def get_cutouts(self, coordinates=None, size=5, sector=None, objectname=None): coordinates : str or `astropy.coordinates` object, optional The target around which to search. It may be specified as a string or as the appropriate `astropy.coordinates` object. - One and only one of coordinates and objectname must be supplied. + One and only one of coordinates, objectname, and moving_target must be supplied. size : int, array-like, `~astropy.units.Quantity` Optional, default 5 pixels. The size of the cutout array. If ``size`` is a scalar number or @@ -272,24 +323,53 @@ def get_cutouts(self, coordinates=None, size=5, sector=None, objectname=None): objectname : str, optional The target around which to search, by name (objectname="M104") or TIC ID (objectname="TIC 141914082"). - One and only one of coordinates and objectname must be supplied. + One and only one of coordinates, objectname, and moving_target must be supplied. + moving_target : str, optional + The name or ID (as understood by the + `JPL ephemerides service `__) + of a moving target such as an asteroid or comet. + One and only one of coordinates, objectname, and moving_target must be supplied. + mt_type : str, optional + The moving target type, valid inputs are majorbody and smallbody. If not supplied + first majorbody is tried and then smallbody if a matching majorbody is not found. + This argument is ignored unless moving_target is supplied. Returns ------- response : A list of `~astropy.io.fits.HDUList` objects. """ - # Get Skycoord object for coordinates/object - coordinates = parse_input_location(coordinates, objectname) - + # Setting up the cutout size param_dict = _parse_cutout_size(size) - param_dict["ra"] = coordinates.ra.deg - param_dict["dec"] = coordinates.dec.deg + # Add sector if present if sector: param_dict["sector"] = sector + + if moving_target: + + # Check only on object designator has been passed in + if objectname or coordinates: + raise InvalidQueryError("Only one of objectname, coordinates, and moving_target may be specified.") + + param_dict["obj_id"] = moving_target + + # Add optional parameter if present + if mt_type: + param_dict["obj_type"] = mt_type + + response = self._service_api_connection.service_request_async("mt_astrocut", param_dict) + + else: + + # Get Skycoord object for coordinates/object + coordinates = parse_input_location(coordinates, objectname) + + param_dict["ra"] = coordinates.ra.deg + param_dict["dec"] = coordinates.dec.deg + + response = self._service_api_connection.service_request_async("astrocut", param_dict) - response = self._service_api_connection.service_request_async("astrocut", param_dict) response.raise_for_status() # Raise any errors try: diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index 61b551dc12..0d95f6bec6 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -551,11 +551,18 @@ def download_file(self, uri, local_path=None, base_url=None, cache=True, cloud_o url = None try: - if self._cloud_connection is not None and self._cloud_connection.is_supported(data_product): - try: - self._cloud_connection.download_file(data_product, local_path, cache) - except Exception as ex: - log.exception("Error pulling from S3 bucket: {}".format(ex)) + if self._cloud_connection is not None: + fall_back = False + if not self._cloud_connection.is_supported(data_product): + log.warn("Data product not in S3.") + fall_back = True + else: + try: + self._cloud_connection.download_file(data_product, local_path, cache) + except Exception as ex: + log.exception("Error pulling from S3 bucket: {}".format(ex)) + fall_back = True + if fall_back: if cloud_only: log.warn("Skipping file...") local_path = "" @@ -565,6 +572,7 @@ def download_file(self, uri, local_path=None, base_url=None, cache=True, cloud_o self._download_file(data_url, local_path, cache=cache, head_safe=True, continuation=False) else: + self._download_file(data_url, local_path, cache=cache, head_safe=True, continuation=False) diff --git a/docs/mast/mast.rst b/docs/mast/mast.rst index 2f4f209a7b..b24907ac47 100644 --- a/docs/mast/mast.rst +++ b/docs/mast/mast.rst @@ -247,44 +247,6 @@ The product fields are documented `here >> from astroquery.mast import Observations - - >>> obsid = '3000007760' - >>> data_products = Observations.get_product_list(obsid) - >>> manifest = Observations.download_products(data_products) - Downloading URL http://archive.stsci.edu/pub/iue/data/lwp/13000/lwp13058.mxlo.gz to ./mastDownload/IUE/lwp13058/lwp13058.mxlo.gz ... [Done] - Downloading URL http://archive.stsci.edu/pub/vospectra/iue2/lwp13058mxlo_vo.fits to ./mastDownload/IUE/lwp13058/lwp13058mxlo_vo.fits ... [Done] - >>> print(manifest) - - Local Path Status Message URL - ------------------------------------------------ -------- ------- ---- - ./mastDownload/IUE/lwp13058/lwp13058.mxlo.gz COMPLETE None None - ./mastDownload/IUE/lwp13058/lwp13058mxlo_vo.fits COMPLETE None None - -​As an alternative to downloading the data files now, the curl_flag can be used instead to instead get a curl script that can be used to download the files at a later time. - -.. code-block:: python - - >>> from astroquery.mast import Observations - - >>> Observations.download_products('2003839997', - ... productType="SCIENCE", - ... curl_flag=True) - - Downloading URL https://mast.stsci.edu/portal/Download/stage/anonymous/public/514cfaa9-fdc1-4799-b043-4488b811db4f/mastDownload_20170629162916.sh to ./mastDownload_20170629162916.sh ... [Done] - - Filtering --------- @@ -324,20 +286,68 @@ Product filtering can also be applied directly to a table of products without pr >>> print(len(products)) 4 + +Downloading Data Products +------------------------- + +Products can be downloaded by using `~astroquery.mast.ObservationsClass.download_products`, +with a `~astropy.table.Table` of data products, or a list (or single) obsid as the argument. + +.. code-block:: python + + >>> from astroquery.mast import Observations + + >>> single_obs = Observations.query_criteria(obs_collection="IUE",obs_id="lwp13058") + >>> data_products = Observations.get_product_list(single_obs) + + >>> manifest = Observations.download_products(data_products, productType="SCIENCE") + Downloading URL https://mast.stsci.edu/api/v0.1/Download/file?uri=http://archive.stsci.edu/pub/iue/data/lwp/13000/lwp13058.mxlo.gz to ./mastDownload/IUE/lwp13058/lwp13058.mxlo.gz ... [Done] + Downloading URL https://mast.stsci.edu/api/v0.1/Download/file?uri=http://archive.stsci.edu/pub/vospectra/iue2/lwp13058mxlo_vo.fits to ./mastDownload/IUE/lwp13058/lwp13058mxlo_vo.fits ... [Done] + + >>> print(manifest) + + Local Path Status Message URL + ------------------------------------------------ -------- ------- ---- + ./mastDownload/IUE/lwp13058/lwp13058.mxlo.gz COMPLETE None None + ./mastDownload/IUE/lwp13058/lwp13058mxlo_vo.fits COMPLETE None None + +​As an alternative to downloading the data files now, the ``curl_flag`` can be used instead to instead get a curl script that can be used to download the files at a later time. + +.. code-block:: python + + >>> from astroquery.mast import Observations + + >>> single_obs = Observations.query_criteria(obs_collection="IUE",obs_id="lwp13058") + >>> data_products = Observations.get_product_list(single_obs) + + >>> Observations.download_products(data_products, + ... productType="SCIENCE", + ... curl_flag=True) + + Downloading URL https://mast.stsci.edu/portal/Download/stage/anonymous/public/514cfaa9-fdc1-4799-b043-4488b811db4f/mastDownload_20170629162916.sh to ./mastDownload_20170629162916.sh ... [Done] + + Downloading a Single File ------------------------- You can download a single data product file using the `~astroquery.mast.ObservationsClass.download_file` method, and passing in -a MAST dataURL. The default is to download the file the current working directory, which can be changed with -the *local_path* keyword argument. +a MAST Data URI. The default is to download the file the current working directory, which can be changed with +the ``local_path`` keyword argument. .. code-block:: python >>> from astroquery.mast import Observations - >>> product = 'mast:IUE/url/pub/iue/data/lwp/13000/lwp13058.elbll.gz' + >>> single_obs = Observations.query_criteria(obs_collection="IUE",obs_id="lwp13058") + >>> data_products = Observations.get_product_list(single_obs) + + >>> product = data_products[0]["dataURI"] + >>> print(product) + mast:IUE/url/pub/iue/data/lwp/13000/lwp13058.elbll.gz + >>> result = Observations.download_file(product) Downloading URL https://mast.stsci.edu/api/v0.1/Download/file?uri=mast:IUE/url/pub/iue/data/lwp/13000/lwp13058.elbll.gz to ./lwp13058.elbll.gz ... [Done] + >>> print(result) ('COMPLETE', None, None) @@ -763,14 +773,19 @@ https://ui.adsabs.harvard.edu/abs/2019ascl.soft05007B/abstract Cutouts ------- -The `~astroquery.mast.TesscutClass.get_cutouts` function takes a coordinate or object name -(such as "M104" or "TIC 32449963") and cutout size (in pixels or an angular quantity) and -returns the cutout target pixel file(s) as a list of `~astropy.io.fits.HDUList` objects. +The `~astroquery.mast.TesscutClass.get_cutouts` function takes a coordinate, object name +(i.e. "M104" or "TIC 32449963"), or moving target (i.e. "Eleonora") and cutout size +(in pixels or an angular quantity) and returns the cutout target pixel file(s) as a +list of `~astropy.io.fits.HDUList` objects. If the given coordinate/object location appears in more than one TESS sector a target pixel file will be produced for each sector. If the cutout area overlaps more than one camera or ccd a target pixel file will be produced for each one. +Requesting a cutout by coordinate or objectname accesses the +`MAST TESScut API `__ +and returns a target pixel file, with format described `here `__. + .. code-block:: python >>> from astroquery.mast import Tesscut @@ -799,13 +814,35 @@ ccd a target pixel file will be produced for each one. 2 APERTURE 1 ImageHDU 80 (5, 5) int32 -The `~astroquery.mast.TesscutClass.download_cutouts` function takes a coordinate or object name -(such as "M104" or "TIC 32449963") and cutout size (in pixels or an angular quantity) and -downloads the cutout target pixel file(s). +Requesting a cutout by moving_target accesses the +`MAST Moving Target TESScut API `__ +and returns a target pixel file, with format described +`here `__. +The moving_target may be any object name or ID understood by the +`JPL Horizonf ephemerades interface `__. + +.. code-block:: python + + >>> from astroquery.mast import Tesscut + + >>> hdulist = Tesscut.get_cutouts(moving_target="Eleonora", size=5, sector=6) + >>> hdulist[0].info() + Filename: + No. Name Ver Type Cards Dimensions Format + 0 PRIMARY 1 PrimaryHDU 54 () + 1 PIXELS 1 BinTableHDU 150 356R x 16C [D, E, J, 25J, 25E, 25E, 25E, 25E, J, E, E, 38A, D, D, D, D] + 2 APERTURE 1 ImageHDU 97 (2136, 2078) int32 + + + + +The `~astroquery.mast.TesscutClass.download_cutouts` function takes a coordinate, object name +(i.e. "M104" or "TIC 32449963"), or moving target (i.e. "Eleonora") and cutout size +(in pixels or an angular quantity) and downloads the cutout target pixel file(s). -If a given coordinate appears in more than one TESS sector a target pixel file will be -produced for each sector. If the cutout area overlaps more than one camera or ccd -a target pixel file will be produced for each one. +If a given coordinate/object/moving target appears in more than one TESS sector a +target pixel file will be produced for each sector. If the cutout area overlaps +more than one camera or ccd a target pixel file will be produced for each one. .. code-block:: python @@ -814,19 +851,20 @@ a target pixel file will be produced for each one. >>> import astropy.units as u >>> cutout_coord = SkyCoord(107.18696, -70.50919, unit="deg") - >>> manifest = Tesscut.download_cutouts(coordinates=cutout_coord, size=[5, 7]*u.arcmin) - Downloading URL https://mast.stsci.edu/tesscut/api/v0.1/astrocut?ra=107.18696&dec=-70.50919&y=0.08333333333333333&x=0.11666666666666667&units=d§or=1 to ./tesscut_20181102104719.zip ... [Done] + >>> manifest = Tesscut.download_cutouts(coordinates=cutout_coord, size=[5, 7]*u.arcmin, sector=9) + Downloading URL https://mast.stsci.edu/tesscut/api/v0.1/astrocut?ra=107.18696&dec=-70.50919&y=0.08333333333333333&x=0.11666666666666667&units=d§or=9 to ./tesscut_20210716150026.zip ... [Done] Inflating... >>> print(manifest) - local_file - ------------------------------------------------------ - ./tess-s0001-4-3_107.18696_-70.50919_14x21_astrocut.fits + Local Path + ---------------------------------------------------------- + ./tess-s0009-4-1_107.186960_-70.509190_21x15_astrocut.fits Sector information ------------------ -To access sector information at a particular location there is `~astroquery.mast.TesscutClass.get_sectors`. +To access sector information for a particular coordinate, object, or moving target there is +`~astroquery.mast.TesscutClass.get_sectors`. .. code-block:: python @@ -852,6 +890,16 @@ To access sector information at a particular location there is `~astroquery.mas tess-s0010-1-4 10 1 4 +.. code-block:: python + + >>> from astroquery.mast import Tesscut + + >>> sector_table = Tesscut.get_sectors(moving_target="Ceres") + >>> print(sector_table) + sectorName sector camera ccd + -------------- ------ ------ --- + tess-s0029-1-4 29 1 4 + Zcut ==== From 8b0dbd5bacc88723f58cb5a2aff6b8e0669b0df3 Mon Sep 17 00:00:00 2001 From: Clara Brasseur Date: Mon, 19 Jul 2021 14:17:13 -0400 Subject: [PATCH 75/87] adding moving tesscut tests --- astroquery/mast/tests/test_mast.py | 23 ++++++ astroquery/mast/tests/test_mast_remote.py | 85 ++++++++++++++++++++--- 2 files changed, 99 insertions(+), 9 deletions(-) diff --git a/astroquery/mast/tests/test_mast.py b/astroquery/mast/tests/test_mast.py index a4071210a9..4dcc6ce707 100644 --- a/astroquery/mast/tests/test_mast.py +++ b/astroquery/mast/tests/test_mast.py @@ -589,6 +589,15 @@ def test_tesscut_get_sector(patch_post): assert sector_table['camera'][0] == 1 assert sector_table['ccd'][0] == 3 + # Exercising the search by moving target + sector_table = mast.Tesscut.get_sectors(moving_target="Ceres") + assert isinstance(sector_table, Table) + assert len(sector_table) == 1 + assert sector_table['sectorName'][0] == "tess-s0001-1-3" + assert sector_table['sector'][0] == 1 + assert sector_table['camera'][0] == 1 + assert sector_table['ccd'][0] == 3 + def test_tesscut_download_cutouts(patch_post, tmpdir): @@ -616,6 +625,13 @@ def test_tesscut_download_cutouts(patch_post, tmpdir): assert manifest["Local Path"][0][-4:] == "fits" assert os.path.isfile(manifest[0]['Local Path']) + # Exercising the search by moving target + manifest = mast.Tesscut.download_cutouts(moving_target="Eleonora", size=5, path=str(tmpdir)) + assert isinstance(manifest, Table) + assert len(manifest) == 1 + assert manifest["Local Path"][0][-4:] == "fits" + assert os.path.isfile(manifest[0]['Local Path']) + def test_tesscut_get_cutouts(patch_post, tmpdir): @@ -631,6 +647,13 @@ def test_tesscut_get_cutouts(patch_post, tmpdir): assert len(cutout_hdus_list) == 1 assert isinstance(cutout_hdus_list[0], fits.HDUList) + # Exercising the search by object name + cutout_hdus_list = mast.Tesscut.get_cutouts(moving_target="Eleonora", size=5) + assert isinstance(cutout_hdus_list, list) + assert len(cutout_hdus_list) == 1 + assert isinstance(cutout_hdus_list[0], fits.HDUList) + + ###################### # ZcutClass tests # ###################### diff --git a/astroquery/mast/tests/test_mast_remote.py b/astroquery/mast/tests/test_mast_remote.py index 7ad991948b..9d47f2eda5 100644 --- a/astroquery/mast/tests/test_mast_remote.py +++ b/astroquery/mast/tests/test_mast_remote.py @@ -14,6 +14,8 @@ from astroquery.exceptions import NoResultsWarning from astroquery import mast +from ...exceptions import RemoteServiceError, MaxResultsWarning + OBSID = '1647157' @@ -195,10 +197,13 @@ def test_observations_query_criteria_count(self): # product functions def test_observations_get_product_list_async(self): - responses = mast.Observations.get_product_list_async('2003738726') + + test_obs = mast.Observations.query_criteria(filters=["NUV","FUV"],objectname="M101") + + responses = mast.Observations.get_product_list_async(test_obs[0]["obsid"]) assert isinstance(responses, list) - responses = mast.Observations.get_product_list_async('2003738726,3000007760') + responses = mast.Observations.get_product_list_async(test_obs[2:3]) assert isinstance(responses, list) observations = mast.Observations.query_object("M8", radius=".02 deg") @@ -242,8 +247,12 @@ def test_observations_get_product_list(self): result = mast.Observations.get_product_list(observations[obsLocs]) obs_collection = np.unique(list(result['obs_collection'])) assert isinstance(result, Table) +<<<<<<< HEAD assert len(obs_collection) == 1 assert obs_collection[0] == 'IUE' +======= + assert len(result) > 10 +>>>>>>> adding moving tesscut tests def test_observations_filter_products(self): observations = mast.Observations.query_object("M8", radius=".04 deg") @@ -269,7 +278,7 @@ def test_observations_download_products(self, tmpdir): assert os.path.isfile(row['Local Path']) # just get the curl script - result = mast.Observations.download_products(test_obs_id, + result = mast.Observations.download_products(test_obs[0]["obsid"], download_dir=str(tmpdir), curl_flag=True, productType=["SCIENCE"], @@ -278,7 +287,7 @@ def test_observations_download_products(self, tmpdir): assert os.path.isfile(result['Local Path'][0]) # check for row input - result1 = mast.Observations.get_product_list(test_obs_id) + result1 = mast.Observations.get_product_list(test_obs[0]["obsid"]) result2 = mast.Observations.download_products(result1[0]) assert isinstance(result2, Table) assert os.path.isfile(result2['Local Path'][0]) @@ -286,9 +295,10 @@ def test_observations_download_products(self, tmpdir): def test_observations_download_file(self, tmpdir): test_obs_id = OBSID + test_obs = mast.Observations.query_criteria(filters=["NUV","FUV"],objectname="M101") # pull a single data product - products = mast.Observations.get_product_list(test_obs_id) + products = mast.Observations.get_product_list(test_obs[0]["obsid"]) uri = products['dataURI'][0] # download it @@ -364,6 +374,10 @@ def test_catalogs_query_region(self): catalog="HSC", magtype=2) row = np.where(result['MatchID'] == '78095437') + + with pytest.warns(MaxResultsWarning): + result = mast.Catalogs.query_region("322.49324 12.16683", catalog="HSC", magtype=2) + assert isinstance(result, Table) assert result[row]['NumImages'] == 1 assert result[row]['TargetName'] == 'M15' @@ -374,6 +388,10 @@ def test_catalogs_query_region(self): version=2, magtype=2) row = np.where(result['MatchID'] == '82368728') + + with pytest.warns(MaxResultsWarning): + result = mast.Catalogs.query_region("322.49324 12.16683", catalog="HSC", + version=2, magtype=2) assert isinstance(result, Table) assert result[row]['NumImages'] == 11 assert result[row]['TargetName'] == 'NGC7078' @@ -666,6 +684,7 @@ def test_tesscut_get_sectors(self): assert sector_table['ccd'][0] > 0 # This should always return no results + with pytest.warns(NoResultsWarning): coord = SkyCoord(90, -66.5, unit="deg") sector_table = mast.Tesscut.get_sectors(coordinates=coord, @@ -673,6 +692,12 @@ def test_tesscut_get_sectors(self): assert isinstance(sector_table, Table) assert len(sector_table) == 0 + coord = SkyCoord(90, -66.5, unit="deg") + with pytest.warns(NoResultsWarning): + sector_table = mast.Tesscut.get_sectors(coordinates=coord, radius=0) + assert isinstance(sector_table, Table) + assert len(sector_table) == 0 + sector_table = mast.Tesscut.get_sectors(objectname="M104") assert isinstance(sector_table, Table) assert len(sector_table) >= 1 @@ -681,6 +706,15 @@ def test_tesscut_get_sectors(self): assert sector_table['camera'][0] > 0 assert sector_table['ccd'][0] > 0 + sector_table = mast.Tesscut.get_sectors(moving_target="Stichius") + assert isinstance(sector_table, Table) + assert len(sector_table) >= 1 + assert sector_table['sectorName'][0] == "tess-s0001-1-1" + assert sector_table['sector'][0] == 1 + assert sector_table['camera'][0] == 1 + assert sector_table['ccd'][0] == 1 + + def test_tesscut_download_cutouts(self, tmpdir): coord = SkyCoord(349.62609, -47.12424, unit="deg") @@ -720,6 +754,14 @@ def test_tesscut_download_cutouts(self, tmpdir): for row in manifest: assert os.path.isfile(row['Local Path']) + manifest = mast.Tesscut.download_cutouts(moving_target="Eleonora", sector=6, size=5, path=str(tmpdir)) + assert isinstance(manifest, Table) + assert len(manifest) == 1 + assert manifest["Local Path"][0][-4:] == "fits" + for row in manifest: + assert os.path.isfile(row['Local Path']) + + def test_tesscut_get_cutouts(self, tmpdir): coord = SkyCoord(107.18696, -70.50919, unit="deg") @@ -746,9 +788,15 @@ def test_tesscut_get_cutouts(self, tmpdir): assert len(cutout_hdus_list) >= 1 assert isinstance(cutout_hdus_list[0], fits.HDUList) - ###################### + cutout_hdus_list = mast.Tesscut.get_cutouts(moving_target="Eleonora", sector=6, size=5) + assert isinstance(cutout_hdus_list, list) + assert len(cutout_hdus_list) == 1 + assert isinstance(cutout_hdus_list[0], fits.HDUList) + + + ################### # ZcutClass tests # - ###################### + ################### def test_zcut_get_surveys(self): coord = SkyCoord(189.49206, 62.20615, unit="deg") @@ -766,6 +814,12 @@ def test_zcut_get_surveys(self): assert isinstance(survey_list, list) assert len(survey_list) == 0 + coord = SkyCoord(57.10523, -30.08085, unit="deg") + with pytest.warns(NoResultsWarning): + survey_list = mast.Zcut.get_surveys(coordinates=coord, radius=0) + assert isinstance(survey_list, list) + assert len(survey_list) == 0 + def test_zcut_download_cutouts(self, tmpdir): coord = SkyCoord(34.47320, -5.24271, unit="deg") @@ -777,7 +831,7 @@ def test_zcut_download_cutouts(self, tmpdir): for row in cutout_table: assert os.path.isfile(cutout_table[0]['Local Path']) - coord = SkyCoord(189.49206, 62.20615, unit="deg") + coord = SkyCoord(189.28065571, 62.17415175, unit="deg") cutout_table = mast.Zcut.download_cutouts(coordinates=coord, size=[200, 300], path=str(tmpdir)) assert isinstance(cutout_table, Table) @@ -809,6 +863,13 @@ def test_zcut_download_cutouts(self, tmpdir): assert isinstance(cutout_table, Table) assert len(cutout_table) == 0 + cutout_table = mast.Zcut.download_cutouts(coordinates=coord, survey='goods_north', cutout_format="jpg", path=str(tmpdir)) + assert isinstance(cutout_table, Table) + assert len(cutout_table) == 4 + assert cutout_table["Local Path"][0][-4:] == ".jpg" + for row in cutout_table: + assert os.path.isfile(cutout_table[0]['Local Path']) + cutout_table = mast.Zcut.download_cutouts(coordinates=coord, cutout_format="jpg", path=str(tmpdir), stretch='asinh', invert=True) assert isinstance(cutout_table, Table) assert len(cutout_table) >= 1 @@ -818,7 +879,7 @@ def test_zcut_download_cutouts(self, tmpdir): def test_zcut_get_cutouts(self): - coord = SkyCoord(189.49206, 62.20615, unit="deg") + coord = SkyCoord(189.28065571, 62.17415175, unit="deg") cutout_list = mast.Zcut.get_cutouts(coordinates=coord) assert isinstance(cutout_list, list) @@ -836,3 +897,9 @@ def test_zcut_get_cutouts(self): survey='candels_gn_30mas') assert isinstance(cutout_list, list) assert len(cutout_list) == 0 + + cutout_list = mast.Zcut.get_cutouts(coordinates=coord, survey='3dhst_goods-n') + assert isinstance(cutout_list, list) + assert len(cutout_list) == 1 + assert isinstance(cutout_list[0], fits.HDUList) + From 1dbe04f4ea8b109070249e59c01ea93301c31709 Mon Sep 17 00:00:00 2001 From: Clara Brasseur Date: Mon, 19 Jul 2021 14:28:52 -0400 Subject: [PATCH 76/87] adding a few more tests for error conditions. also making code style fixes. --- astroquery/mast/cutouts.py | 38 +++++++++++------------ astroquery/mast/observations.py | 2 +- astroquery/mast/tests/test_mast.py | 35 ++++++++++++++++++++- astroquery/mast/tests/test_mast_remote.py | 11 ++----- 4 files changed, 56 insertions(+), 30 deletions(-) diff --git a/astroquery/mast/cutouts.py b/astroquery/mast/cutouts.py index 54378946fc..3a35207614 100644 --- a/astroquery/mast/cutouts.py +++ b/astroquery/mast/cutouts.py @@ -133,8 +133,8 @@ def get_sectors(self, coordinates=None, radius=0*u.deg, objectname=None, moving_ or TIC ID (objectname="TIC 141914082"). One and only one of coordinates, objectname, moving_target must be supplied. moving_target : str, optional - The name or ID (as understood by the - `JPL ephemerides service `__) + The name or ID (as understood by the + `JPL ephemerides service `__) of a moving target such as an asteroid or comet. One and only one of coordinates, objectname, and moving_target must be supplied. mt_type : str, optional @@ -150,7 +150,7 @@ def get_sectors(self, coordinates=None, radius=0*u.deg, objectname=None, moving_ if moving_target: - # Check only ony object designator has been passed in + # Check only ony object designator has been passed in if objectname or coordinates: raise InvalidQueryError("Only one of objectname, coordinates, and moving_target may be specified.") @@ -163,7 +163,7 @@ def get_sectors(self, coordinates=None, radius=0*u.deg, objectname=None, moving_ response = self._service_api_connection.service_request_async("mt_sector", params) else: - + # Get Skycoord object for coordinates/object coordinates = parse_input_location(coordinates, objectname) @@ -176,8 +176,8 @@ def get_sectors(self, coordinates=None, radius=0*u.deg, objectname=None, moving_ response = self._service_api_connection.service_request_async("sector", params) - # Raise any errors - response.raise_for_status() + # Raise any errors + response.raise_for_status() sector_json = response.json()['results'] sector_dict = {'sectorName': [], @@ -232,8 +232,8 @@ def download_cutouts(self, coordinates=None, size=5, sector=None, path=".", infl or TIC ID (objectname="TIC 141914082"). One and only one of coordinates, objectname, and moving_target must be supplied. moving_target : str, optional - The name or ID (as understood by the - `JPL ephemerides service `__) + The name or ID (as understood by the + `JPL ephemerides service `__) of a moving target such as an asteroid or comet. One and only one of coordinates, objectname, and moving_target must be supplied. mt_type : str, optional @@ -247,23 +247,23 @@ def download_cutouts(self, coordinates=None, size=5, sector=None, path=".", infl """ if moving_target: - # Check only ony object designator has been passed in + # Check only ony object designator has been passed in if objectname or coordinates: raise InvalidQueryError("Only one of objectname, coordinates, and moving_target may be specified.") - + astrocut_request = f"moving_target/astrocut?obj_id={moving_target}" if mt_type: astrocut_request += f"&obj_type={mt_type}" else: # Get Skycoord object for coordinates/object coordinates = parse_input_location(coordinates, objectname) - + astrocut_request = f"astrocut?ra={coordinates.ra.deg}&dec={coordinates.dec.deg}" # Adding the arguments that are common between moving/still astrocut requests size_dict = _parse_cutout_size(size) astrocut_request += f"&y={size_dict['y']}&x={size_dict['x']}&units={size_dict['units']}" - + if sector: astrocut_request += "§or={}".format(sector) @@ -325,8 +325,8 @@ def get_cutouts(self, coordinates=None, size=5, sector=None, or TIC ID (objectname="TIC 141914082"). One and only one of coordinates, objectname, and moving_target must be supplied. moving_target : str, optional - The name or ID (as understood by the - `JPL ephemerides service `__) + The name or ID (as understood by the + `JPL ephemerides service `__) of a moving target such as an asteroid or comet. One and only one of coordinates, objectname, and moving_target must be supplied. mt_type : str, optional @@ -345,10 +345,10 @@ def get_cutouts(self, coordinates=None, size=5, sector=None, # Add sector if present if sector: param_dict["sector"] = sector - + if moving_target: - # Check only on object designator has been passed in + # Check only on object designator has been passed in if objectname or coordinates: raise InvalidQueryError("Only one of objectname, coordinates, and moving_target may be specified.") @@ -361,13 +361,13 @@ def get_cutouts(self, coordinates=None, size=5, sector=None, response = self._service_api_connection.service_request_async("mt_astrocut", param_dict) else: - + # Get Skycoord object for coordinates/object coordinates = parse_input_location(coordinates, objectname) - + param_dict["ra"] = coordinates.ra.deg param_dict["dec"] = coordinates.dec.deg - + response = self._service_api_connection.service_request_async("astrocut", param_dict) response.raise_for_status() # Raise any errors diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index 0d95f6bec6..295c3d10d4 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -572,7 +572,7 @@ def download_file(self, uri, local_path=None, base_url=None, cache=True, cloud_o self._download_file(data_url, local_path, cache=cache, head_safe=True, continuation=False) else: - + self._download_file(data_url, local_path, cache=cache, head_safe=True, continuation=False) diff --git a/astroquery/mast/tests/test_mast.py b/astroquery/mast/tests/test_mast.py index 4dcc6ce707..5c12b9dec2 100644 --- a/astroquery/mast/tests/test_mast.py +++ b/astroquery/mast/tests/test_mast.py @@ -13,7 +13,7 @@ import astropy.units as u from ...utils.testing_tools import MockResponse -from ...exceptions import (InvalidQueryError, InputWarning) +from ...exceptions import InvalidQueryError, InputWarning from ... import mast @@ -598,6 +598,17 @@ def test_tesscut_get_sector(patch_post): assert sector_table['camera'][0] == 1 assert sector_table['ccd'][0] == 3 + # Testing catch for multiple designators' + error_str = "Only one of objectname, coordinates, and moving_target may be specified." + + with pytest.raises(InvalidQueryError) as invalid_query: + mast.Tesscut.get_sectors(moving_target="Ceres", coordinates=coord) + assert error_str in str(invalid_query.value) + + with pytest.raises(InvalidQueryError) as invalid_query: + mast.Tesscut.get_sectors(moving_target="Ceres", objectname="M103") + assert error_str in str(invalid_query.value) + def test_tesscut_download_cutouts(patch_post, tmpdir): @@ -632,6 +643,17 @@ def test_tesscut_download_cutouts(patch_post, tmpdir): assert manifest["Local Path"][0][-4:] == "fits" assert os.path.isfile(manifest[0]['Local Path']) + # Testing catch for multiple designators' + error_str = "Only one of objectname, coordinates, and moving_target may be specified." + + with pytest.raises(InvalidQueryError) as invalid_query: + mast.Tesscut.download_cutouts(moving_target="Eleonora", coordinates=coord, size=5, path=str(tmpdir)) + assert error_str in str(invalid_query.value) + + with pytest.raises(InvalidQueryError) as invalid_query: + mast.Tesscut.download_cutouts(moving_target="Eleonora", objectname="M103", size=5, path=str(tmpdir)) + assert error_str in str(invalid_query.value) + def test_tesscut_get_cutouts(patch_post, tmpdir): @@ -653,6 +675,17 @@ def test_tesscut_get_cutouts(patch_post, tmpdir): assert len(cutout_hdus_list) == 1 assert isinstance(cutout_hdus_list[0], fits.HDUList) + # Testing catch for multiple designators' + error_str = "Only one of objectname, coordinates, and moving_target may be specified." + + with pytest.raises(InvalidQueryError) as invalid_query: + mast.Tesscut.get_cutouts(moving_target="Eleonora", coordinates=coord, size=5) + assert error_str in str(invalid_query.value) + + with pytest.raises(InvalidQueryError) as invalid_query: + mast.Tesscut.get_cutouts(moving_target="Eleonora", objectname="M103", size=5) + assert error_str in str(invalid_query.value) + ###################### # ZcutClass tests # diff --git a/astroquery/mast/tests/test_mast_remote.py b/astroquery/mast/tests/test_mast_remote.py index 9d47f2eda5..d08079807c 100644 --- a/astroquery/mast/tests/test_mast_remote.py +++ b/astroquery/mast/tests/test_mast_remote.py @@ -197,8 +197,8 @@ def test_observations_query_criteria_count(self): # product functions def test_observations_get_product_list_async(self): - - test_obs = mast.Observations.query_criteria(filters=["NUV","FUV"],objectname="M101") + + test_obs = mast.Observations.query_criteria(filters=["NUV", "FUV"], objectname="M101") responses = mast.Observations.get_product_list_async(test_obs[0]["obsid"]) assert isinstance(responses, list) @@ -247,12 +247,8 @@ def test_observations_get_product_list(self): result = mast.Observations.get_product_list(observations[obsLocs]) obs_collection = np.unique(list(result['obs_collection'])) assert isinstance(result, Table) -<<<<<<< HEAD assert len(obs_collection) == 1 assert obs_collection[0] == 'IUE' -======= - assert len(result) > 10 ->>>>>>> adding moving tesscut tests def test_observations_filter_products(self): observations = mast.Observations.query_object("M8", radius=".04 deg") @@ -714,7 +710,6 @@ def test_tesscut_get_sectors(self): assert sector_table['camera'][0] == 1 assert sector_table['ccd'][0] == 1 - def test_tesscut_download_cutouts(self, tmpdir): coord = SkyCoord(349.62609, -47.12424, unit="deg") @@ -760,7 +755,6 @@ def test_tesscut_download_cutouts(self, tmpdir): assert manifest["Local Path"][0][-4:] == "fits" for row in manifest: assert os.path.isfile(row['Local Path']) - def test_tesscut_get_cutouts(self, tmpdir): @@ -793,7 +787,6 @@ def test_tesscut_get_cutouts(self, tmpdir): assert len(cutout_hdus_list) == 1 assert isinstance(cutout_hdus_list[0], fits.HDUList) - ################### # ZcutClass tests # ################### From 6aa80d0f643f9c8cadcd4d17c57353736780df15 Mon Sep 17 00:00:00 2001 From: Jennifer Medina Date: Thu, 20 Jan 2022 17:14:51 -0500 Subject: [PATCH 77/87] second pass thru the PR: fixed typo of 355Rows corrected from 356Rows, explicitly calling table variable to download_products output, and reverting changes made to observations.py --- astroquery/mast/observations.py | 18 +++++----------- astroquery/mast/tests/test_mast_remote.py | 3 +-- docs/mast/mast.rst | 26 +++++++++++------------ 3 files changed, 18 insertions(+), 29 deletions(-) diff --git a/astroquery/mast/observations.py b/astroquery/mast/observations.py index 295c3d10d4..61b551dc12 100644 --- a/astroquery/mast/observations.py +++ b/astroquery/mast/observations.py @@ -551,18 +551,11 @@ def download_file(self, uri, local_path=None, base_url=None, cache=True, cloud_o url = None try: - if self._cloud_connection is not None: - fall_back = False - if not self._cloud_connection.is_supported(data_product): - log.warn("Data product not in S3.") - fall_back = True - else: - try: - self._cloud_connection.download_file(data_product, local_path, cache) - except Exception as ex: - log.exception("Error pulling from S3 bucket: {}".format(ex)) - fall_back = True - if fall_back: + if self._cloud_connection is not None and self._cloud_connection.is_supported(data_product): + try: + self._cloud_connection.download_file(data_product, local_path, cache) + except Exception as ex: + log.exception("Error pulling from S3 bucket: {}".format(ex)) if cloud_only: log.warn("Skipping file...") local_path = "" @@ -572,7 +565,6 @@ def download_file(self, uri, local_path=None, base_url=None, cache=True, cloud_o self._download_file(data_url, local_path, cache=cache, head_safe=True, continuation=False) else: - self._download_file(data_url, local_path, cache=cache, head_safe=True, continuation=False) diff --git a/astroquery/mast/tests/test_mast_remote.py b/astroquery/mast/tests/test_mast_remote.py index d08079807c..6830b438c6 100644 --- a/astroquery/mast/tests/test_mast_remote.py +++ b/astroquery/mast/tests/test_mast_remote.py @@ -291,7 +291,7 @@ def test_observations_download_products(self, tmpdir): def test_observations_download_file(self, tmpdir): test_obs_id = OBSID - test_obs = mast.Observations.query_criteria(filters=["NUV","FUV"],objectname="M101") + test_obs = mast.Observations.query_criteria(filters=["NUV", "FUV"], objectname="M101") # pull a single data product products = mast.Observations.get_product_list(test_obs[0]["obsid"]) @@ -895,4 +895,3 @@ def test_zcut_get_cutouts(self): assert isinstance(cutout_list, list) assert len(cutout_list) == 1 assert isinstance(cutout_list[0], fits.HDUList) - diff --git a/docs/mast/mast.rst b/docs/mast/mast.rst index b24907ac47..e9dc2a53f7 100644 --- a/docs/mast/mast.rst +++ b/docs/mast/mast.rst @@ -306,7 +306,7 @@ with a `~astropy.table.Table` of data products, or a list (or single) obsid as t >>> print(manifest) - Local Path Status Message URL + Local Path Status Message URL ------------------------------------------------ -------- ------- ---- ./mastDownload/IUE/lwp13058/lwp13058.mxlo.gz COMPLETE None None ./mastDownload/IUE/lwp13058/lwp13058mxlo_vo.fits COMPLETE None None @@ -317,12 +317,10 @@ with a `~astropy.table.Table` of data products, or a list (or single) obsid as t >>> from astroquery.mast import Observations - >>> single_obs = Observations.query_criteria(obs_collection="IUE",obs_id="lwp13058") + >>> single_obs = Observations.query_criteria(obs_collection="IUE", obs_id="lwp13058") >>> data_products = Observations.get_product_list(single_obs) - - >>> Observations.download_products(data_products, - ... productType="SCIENCE", - ... curl_flag=True) + + >>> table = Observations.download_products(data_products, productType="SCIENCE", curl_flag=True) Downloading URL https://mast.stsci.edu/portal/Download/stage/anonymous/public/514cfaa9-fdc1-4799-b043-4488b811db4f/mastDownload_20170629162916.sh to ./mastDownload_20170629162916.sh ... [Done] @@ -340,11 +338,11 @@ the ``local_path`` keyword argument. >>> single_obs = Observations.query_criteria(obs_collection="IUE",obs_id="lwp13058") >>> data_products = Observations.get_product_list(single_obs) - + >>> product = data_products[0]["dataURI"] >>> print(product) mast:IUE/url/pub/iue/data/lwp/13000/lwp13058.elbll.gz - + >>> result = Observations.download_file(product) Downloading URL https://mast.stsci.edu/api/v0.1/Download/file?uri=mast:IUE/url/pub/iue/data/lwp/13000/lwp13058.elbll.gz to ./lwp13058.elbll.gz ... [Done] @@ -829,13 +827,13 @@ The moving_target may be any object name or ID understood by the >>> hdulist[0].info() Filename: No. Name Ver Type Cards Dimensions Format - 0 PRIMARY 1 PrimaryHDU 54 () - 1 PIXELS 1 BinTableHDU 150 356R x 16C [D, E, J, 25J, 25E, 25E, 25E, 25E, J, E, E, 38A, D, D, D, D] - 2 APERTURE 1 ImageHDU 97 (2136, 2078) int32 + 0 PRIMARY 1 PrimaryHDU 54 () + 1 PIXELS 1 BinTableHDU 150 355R x 16C [D, E, J, 25J, 25E, 25E, 25E, 25E, J, E, E, 38A, D, D, D, D] + 2 APERTURE 1 ImageHDU 97 (2136, 2078) int32 + - The `~astroquery.mast.TesscutClass.download_cutouts` function takes a coordinate, object name (i.e. "M104" or "TIC 32449963"), or moving target (i.e. "Eleonora") and cutout size (in pixels or an angular quantity) and downloads the cutout target pixel file(s). @@ -856,7 +854,7 @@ more than one camera or ccd a target pixel file will be produced for each one. Inflating... >>> print(manifest) - Local Path + Local Path ---------------------------------------------------------- ./tess-s0009-4-1_107.186960_-70.509190_21x15_astrocut.fits @@ -899,7 +897,7 @@ To access sector information for a particular coordinate, object, or moving targ sectorName sector camera ccd -------------- ------ ------ --- tess-s0029-1-4 29 1 4 - + Zcut ==== From c9ebd67fb42208cae1e90c12f666ac9c8c47d4c6 Mon Sep 17 00:00:00 2001 From: Jennifer Medina Date: Wed, 26 Jan 2022 10:52:55 -0500 Subject: [PATCH 78/87] setting moving_target option to bool instead of its own target name or ID input --- astroquery/mast/cutouts.py | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/astroquery/mast/cutouts.py b/astroquery/mast/cutouts.py index 3a35207614..580a98a4a2 100644 --- a/astroquery/mast/cutouts.py +++ b/astroquery/mast/cutouts.py @@ -110,7 +110,7 @@ def __init__(self): } self._service_api_connection.set_service_params(services, "tesscut") - def get_sectors(self, coordinates=None, radius=0*u.deg, objectname=None, moving_target=None, mt_type=None): + def get_sectors(self, coordinates=None, radius=0*u.deg, objectname=None, moving_target=False, mt_type=None): """ Get a list of the TESS data sectors whose footprints intersect with the given search area. @@ -120,27 +120,36 @@ def get_sectors(self, coordinates=None, radius=0*u.deg, objectname=None, moving_ coordinates : str or `astropy.coordinates` object, optional The target around which to search. It may be specified as a string or as the appropriate `astropy.coordinates` object. - One and only one of coordinates, objectname, and moving_target must be supplied. + + NOTE: If moving_targets or objectname is supplied, this argument cannot be used. + radius : str, float, or `~astropy.units.Quantity` object, optional Default 0 degrees. If supplied as a float degrees is the assumed unit. The string must be parsable by `~astropy.coordinates.Angle`. The appropriate `~astropy.units.Quantity` object from `astropy.units` may also be used. - This argument is ignored if moving_target is supplied. + + NOTE: If moving_target is supplied, this argument is ignored. + objectname : str, optional The target around which to search, by name (objectname="M104") - or TIC ID (objectname="TIC 141914082"). - One and only one of coordinates, objectname, moving_target must be supplied. - moving_target : str, optional - The name or ID (as understood by the + or TIC ID (objectname="TIC 141914082"). If moving_target is True, input must be the name or ID (as understood by the `JPL ephemerides service `__) of a moving target such as an asteroid or comet. - One and only one of coordinates, objectname, and moving_target must be supplied. + + NOTE: If coordinates is supplied, this argument cannot be used. + + moving_target : bool, optional + Indicate whether the object is a moving target or not. Default is set to False, in other words, not a moving target. + + NOTE: If coordinates is supplied, this argument cannot be used. + mt_type : str, optional The moving target type, valid inputs are majorbody and smallbody. If not supplied first majorbody is tried and then smallbody if a matching majorbody is not found. - This argument is ignored unless moving_target is supplied. + + NOTE: If moving_target is supplied, this argument is ignored. Returns ------- @@ -150,11 +159,14 @@ def get_sectors(self, coordinates=None, radius=0*u.deg, objectname=None, moving_ if moving_target: - # Check only ony object designator has been passed in - if objectname or coordinates: - raise InvalidQueryError("Only one of objectname, coordinates, and moving_target may be specified.") + # Check that objectname has been passed in + if coordinates: + raise InvalidQueryError("Only one of moving_target and coordinates may be specified.") + + if not objectname: + raise InvalidQueryError("Please specify the object name or ID (as understood by the `JPL ephemerides service `__) of a moving target such as an asteroid or comet.") - params = {"obj_id": moving_target} + params = {"obj_id": objectname} # Add optional parameter is present if mt_type: From 4aec2b3f3a5139cda4a20649b7ce7ed347619fc9 Mon Sep 17 00:00:00 2001 From: Jennifer Medina Date: Fri, 28 Jan 2022 15:36:39 -0500 Subject: [PATCH 79/87] updating moving_target argument and documentation for get_sectors, download_cutouts, and get_cutouts. --- astroquery/mast/cutouts.py | 46 +++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/astroquery/mast/cutouts.py b/astroquery/mast/cutouts.py index 580a98a4a2..072ec09322 100644 --- a/astroquery/mast/cutouts.py +++ b/astroquery/mast/cutouts.py @@ -121,8 +121,7 @@ def get_sectors(self, coordinates=None, radius=0*u.deg, objectname=None, moving_ The target around which to search. It may be specified as a string or as the appropriate `astropy.coordinates` object. - NOTE: If moving_targets or objectname is supplied, this argument cannot be used. - + NOTE: If moving_target or objectname is supplied, this argument cannot be used. radius : str, float, or `~astropy.units.Quantity` object, optional Default 0 degrees. If supplied as a float degrees is the assumed unit. @@ -131,7 +130,6 @@ def get_sectors(self, coordinates=None, radius=0*u.deg, objectname=None, moving_ `astropy.units` may also be used. NOTE: If moving_target is supplied, this argument is ignored. - objectname : str, optional The target around which to search, by name (objectname="M104") or TIC ID (objectname="TIC 141914082"). If moving_target is True, input must be the name or ID (as understood by the @@ -139,12 +137,10 @@ def get_sectors(self, coordinates=None, radius=0*u.deg, objectname=None, moving_ of a moving target such as an asteroid or comet. NOTE: If coordinates is supplied, this argument cannot be used. - moving_target : bool, optional Indicate whether the object is a moving target or not. Default is set to False, in other words, not a moving target. NOTE: If coordinates is supplied, this argument cannot be used. - mt_type : str, optional The moving target type, valid inputs are majorbody and smallbody. If not supplied first majorbody is tried and then smallbody if a matching majorbody is not found. @@ -208,7 +204,7 @@ def get_sectors(self, coordinates=None, radius=0*u.deg, objectname=None, moving_ return Table(sector_dict) def download_cutouts(self, coordinates=None, size=5, sector=None, path=".", inflate=True, - objectname=None, moving_target=None, mt_type=None): + objectname=None, moving_target=False, mt_type=None): """ Download cutout target pixel file(s) around the given coordinates with indicated size. @@ -217,7 +213,8 @@ def download_cutouts(self, coordinates=None, size=5, sector=None, path=".", infl coordinates : str or `astropy.coordinates` object, optional The target around which to search. It may be specified as a string or as the appropriate `astropy.coordinates` object. - One and only one of coordinates, objectname, and moving_target must be supplied. + + NOTE: If moving_target or objectname is supplied, this argument cannot be used. size : int, array-like, `~astropy.units.Quantity` Optional, default 5 pixels. The size of the cutout array. If ``size`` is a scalar number or @@ -241,17 +238,20 @@ def download_cutouts(self, coordinates=None, size=5, sector=None, path=".", infl Set inflate to false to stop before the inflate step. objectname : str, optional The target around which to search, by name (objectname="M104") - or TIC ID (objectname="TIC 141914082"). - One and only one of coordinates, objectname, and moving_target must be supplied. - moving_target : str, optional - The name or ID (as understood by the + or TIC ID (objectname="TIC 141914082"). If moving_target is True, input must be the name or ID (as understood by the `JPL ephemerides service `__) of a moving target such as an asteroid or comet. - One and only one of coordinates, objectname, and moving_target must be supplied. + + NOTE: If coordinates is supplied, this argument cannot be used. + moving_target : str, optional + Indicate whether the object is a moving target or not. Default is set to False, in other words, not a moving target. + + NOTE: If coordinates is supplied, this argument cannot be used. mt_type : str, optional The moving target type, valid inputs are majorbody and smallbody. If not supplied first majorbody is tried and then smallbody if a matching majorbody is not found. - This argument is ignored unless moving_target is supplied. + + NOTE: If moving_target is supplied, this argument is ignored. Returns ------- @@ -309,7 +309,7 @@ def download_cutouts(self, coordinates=None, size=5, sector=None, path=".", infl return localpath_table def get_cutouts(self, coordinates=None, size=5, sector=None, - objectname=None, moving_target=None, mt_type=None): + objectname=None, moving_target=False, mt_type=None): """ Get cutout target pixel file(s) around the given coordinates with indicated size, and return them as a list of `~astropy.io.fits.HDUList` objects. @@ -319,7 +319,8 @@ def get_cutouts(self, coordinates=None, size=5, sector=None, coordinates : str or `astropy.coordinates` object, optional The target around which to search. It may be specified as a string or as the appropriate `astropy.coordinates` object. - One and only one of coordinates, objectname, and moving_target must be supplied. + + NOTE: If moving_target or objectname is supplied, this argument cannot be used. size : int, array-like, `~astropy.units.Quantity` Optional, default 5 pixels. The size of the cutout array. If ``size`` is a scalar number or @@ -334,17 +335,20 @@ def get_cutouts(self, coordinates=None, size=5, sector=None, from all available sectors on which the coordinate appears will be returned. objectname : str, optional The target around which to search, by name (objectname="M104") - or TIC ID (objectname="TIC 141914082"). - One and only one of coordinates, objectname, and moving_target must be supplied. - moving_target : str, optional - The name or ID (as understood by the + or TIC ID (objectname="TIC 141914082"). If moving_target is True, input must be the name or ID (as understood by the `JPL ephemerides service `__) of a moving target such as an asteroid or comet. - One and only one of coordinates, objectname, and moving_target must be supplied. + + NOTE: If coordinates is supplied, this argument cannot be used. + moving_target : str, optional + Indicate whether the object is a moving target or not. Default is set to False, in other words, not a moving target. + + NOTE: If coordinates is supplied, this argument cannot be used. mt_type : str, optional The moving target type, valid inputs are majorbody and smallbody. If not supplied first majorbody is tried and then smallbody if a matching majorbody is not found. - This argument is ignored unless moving_target is supplied. + + NOTE: If moving_target is supplied, this argument is ignored. Returns ------- From 336e6574c9fc132501100f492fe9b98a98d6065b Mon Sep 17 00:00:00 2001 From: Jennifer Medina Date: Fri, 28 Jan 2022 16:17:48 -0500 Subject: [PATCH 80/87] updating moving target functionality and test coverage for get_sectors, download_cutouts, and get_cutouts. also fixing import statements and other typos. --- astroquery/mast/cutouts.py | 32 ++++-- astroquery/mast/tests/test_mast.py | 43 ++++---- astroquery/mast/tests/test_mast_remote.py | 120 ++++++++++++++++++++-- 3 files changed, 157 insertions(+), 38 deletions(-) diff --git a/astroquery/mast/cutouts.py b/astroquery/mast/cutouts.py index 072ec09322..b69ab287cc 100644 --- a/astroquery/mast/cutouts.py +++ b/astroquery/mast/cutouts.py @@ -155,9 +155,10 @@ def get_sectors(self, coordinates=None, radius=0*u.deg, objectname=None, moving_ if moving_target: - # Check that objectname has been passed in + # Check that objectname has been passed in and coordinates + # is not if coordinates: - raise InvalidQueryError("Only one of moving_target and coordinates may be specified.") + raise InvalidQueryError("Only one of moving_target and coordinates may be specified. Please remove coordinates if using moving_target and objectname.") if not objectname: raise InvalidQueryError("Please specify the object name or ID (as understood by the `JPL ephemerides service `__) of a moving target such as an asteroid or comet.") @@ -259,14 +260,21 @@ def download_cutouts(self, coordinates=None, size=5, sector=None, path=".", infl """ if moving_target: - # Check only ony object designator has been passed in - if objectname or coordinates: - raise InvalidQueryError("Only one of objectname, coordinates, and moving_target may be specified.") - astrocut_request = f"moving_target/astrocut?obj_id={moving_target}" + # Check that objectname has been passed in and coordinates + # is not + if coordinates: + raise InvalidQueryError("Only one of moving_target and coordinates may be specified. Please remove coordinates if using moving_target and objectname.") + + if not objectname: + raise InvalidQueryError("Please specify the object name or ID (as understood by the `JPL ephemerides service `__) of a moving target such as an asteroid or comet.") + + astrocut_request = f"moving_target/astrocut?obj_id={objectname}" if mt_type: astrocut_request += f"&obj_type={mt_type}" + else: + # Get Skycoord object for coordinates/object coordinates = parse_input_location(coordinates, objectname) @@ -364,11 +372,15 @@ def get_cutouts(self, coordinates=None, size=5, sector=None, if moving_target: - # Check only on object designator has been passed in - if objectname or coordinates: - raise InvalidQueryError("Only one of objectname, coordinates, and moving_target may be specified.") + # Check that objectname has been passed in and coordinates + # is not + if coordinates: + raise InvalidQueryError("Only one of moving_target and coordinates may be specified. Please remove coordinates if using moving_target and objectname.") + + if not objectname: + raise InvalidQueryError("Please specify the object name or ID (as understood by the `JPL ephemerides service `__) of a moving target such as an asteroid or comet.") - param_dict["obj_id"] = moving_target + param_dict["obj_id"] = objectname # Add optional parameter if present if mt_type: diff --git a/astroquery/mast/tests/test_mast.py b/astroquery/mast/tests/test_mast.py index 5c12b9dec2..80eaad1bc9 100644 --- a/astroquery/mast/tests/test_mast.py +++ b/astroquery/mast/tests/test_mast.py @@ -590,7 +590,8 @@ def test_tesscut_get_sector(patch_post): assert sector_table['ccd'][0] == 3 # Exercising the search by moving target - sector_table = mast.Tesscut.get_sectors(moving_target="Ceres") + sector_table = mast.Tesscut.get_sectors(objectname="Ceres", + moving_target=True) assert isinstance(sector_table, Table) assert len(sector_table) == 1 assert sector_table['sectorName'][0] == "tess-s0001-1-3" @@ -599,14 +600,10 @@ def test_tesscut_get_sector(patch_post): assert sector_table['ccd'][0] == 3 # Testing catch for multiple designators' - error_str = "Only one of objectname, coordinates, and moving_target may be specified." + error_str = "Only one of moving_target and coordinates may be specified. Please remove coordinates if using moving_target and objectname." with pytest.raises(InvalidQueryError) as invalid_query: - mast.Tesscut.get_sectors(moving_target="Ceres", coordinates=coord) - assert error_str in str(invalid_query.value) - - with pytest.raises(InvalidQueryError) as invalid_query: - mast.Tesscut.get_sectors(moving_target="Ceres", objectname="M103") + mast.Tesscut.get_sectors(objectname='Ceres', moving_target=True, coordinates=coord) assert error_str in str(invalid_query.value) @@ -637,21 +634,24 @@ def test_tesscut_download_cutouts(patch_post, tmpdir): assert os.path.isfile(manifest[0]['Local Path']) # Exercising the search by moving target - manifest = mast.Tesscut.download_cutouts(moving_target="Eleonora", size=5, path=str(tmpdir)) + manifest = mast.Tesscut.download_cutouts(objectname="Eleonora", + moving_target=True, + size=5, + path=str(tmpdir)) assert isinstance(manifest, Table) assert len(manifest) == 1 assert manifest["Local Path"][0][-4:] == "fits" assert os.path.isfile(manifest[0]['Local Path']) # Testing catch for multiple designators' - error_str = "Only one of objectname, coordinates, and moving_target may be specified." + error_str = "Only one of moving_target and coordinates may be specified. Please remove coordinates if using moving_target and objectname." with pytest.raises(InvalidQueryError) as invalid_query: - mast.Tesscut.download_cutouts(moving_target="Eleonora", coordinates=coord, size=5, path=str(tmpdir)) - assert error_str in str(invalid_query.value) - - with pytest.raises(InvalidQueryError) as invalid_query: - mast.Tesscut.download_cutouts(moving_target="Eleonora", objectname="M103", size=5, path=str(tmpdir)) + mast.Tesscut.download_cutouts(objectname="Eleonora", + moving_target=True, + coordinates=coord, + size=5, + path=str(tmpdir)) assert error_str in str(invalid_query.value) @@ -670,20 +670,21 @@ def test_tesscut_get_cutouts(patch_post, tmpdir): assert isinstance(cutout_hdus_list[0], fits.HDUList) # Exercising the search by object name - cutout_hdus_list = mast.Tesscut.get_cutouts(moving_target="Eleonora", size=5) + cutout_hdus_list = mast.Tesscut.get_cutouts(objectname='Eleonora', + moving_target=True, + size=5) assert isinstance(cutout_hdus_list, list) assert len(cutout_hdus_list) == 1 assert isinstance(cutout_hdus_list[0], fits.HDUList) # Testing catch for multiple designators' - error_str = "Only one of objectname, coordinates, and moving_target may be specified." - - with pytest.raises(InvalidQueryError) as invalid_query: - mast.Tesscut.get_cutouts(moving_target="Eleonora", coordinates=coord, size=5) - assert error_str in str(invalid_query.value) + error_str = "Only one of moving_target and coordinates may be specified. Please remove coordinates if using moving_target and objectname." with pytest.raises(InvalidQueryError) as invalid_query: - mast.Tesscut.get_cutouts(moving_target="Eleonora", objectname="M103", size=5) + mast.Tesscut.get_cutouts(objectname="Eleonora", + moving_target=True, + coordinates=coord, + size=5) assert error_str in str(invalid_query.value) diff --git a/astroquery/mast/tests/test_mast_remote.py b/astroquery/mast/tests/test_mast_remote.py index 6830b438c6..c08dae0069 100644 --- a/astroquery/mast/tests/test_mast_remote.py +++ b/astroquery/mast/tests/test_mast_remote.py @@ -11,10 +11,10 @@ from astropy.io import fits import astropy.units as u -from astroquery.exceptions import NoResultsWarning from astroquery import mast -from ...exceptions import RemoteServiceError, MaxResultsWarning +from ..utils import ResolverError +from ...exceptions import InvalidQueryError, MaxResultsWarning, NoResultsWarning, RemoteServiceError OBSID = '1647157' @@ -702,14 +702,49 @@ def test_tesscut_get_sectors(self): assert sector_table['camera'][0] > 0 assert sector_table['ccd'][0] > 0 - sector_table = mast.Tesscut.get_sectors(moving_target="Stichius") + # Moving target functionality testing + + moving_target_name = 'Eleonora' + + sector_table = mast.Tesscut.get_sectors(objectname=moving_target_name, + moving_target=True) assert isinstance(sector_table, Table) assert len(sector_table) >= 1 - assert sector_table['sectorName'][0] == "tess-s0001-1-1" - assert sector_table['sector'][0] == 1 + assert sector_table['sectorName'][0] == "tess-s0006-1-1" + assert sector_table['sector'][0] == 6 assert sector_table['camera'][0] == 1 assert sector_table['ccd'][0] == 1 + error_noname = "Please specify the object name or ID (as understood by the `JPL ephemerides service `__) of a moving target such as an asteroid or comet." + error_nameresolve = f"Could not resolve {moving_target_name} to a sky position." + error_mt_coord = "Only one of moving_target and coordinates may be specified." + error_name_coord = "Only one of objectname and coordinates may be specified." + + with pytest.raises(InvalidQueryError) as error_msg: + mast.Tesscut.get_sectors(moving_target=True) + assert error_noname in str(error_msg.value) + + with pytest.raises(ResolverError) as error_msg: + mast.Tesscut.get_sectors(objectname=moving_target_name) + assert error_nameresolve in str(error_msg.value) + + with pytest.raises(InvalidQueryError) as error_msg: + mast.Tesscut.get_sectors(coordinates=coord, moving_target=True) + assert error_mt_coord in str(error_msg.value) + + with pytest.raises(InvalidQueryError) as error_msg: + mast.Tesscut.get_sectors(objectname=moving_target_name, + coordinates=coord) + assert error_name_coord in str(error_msg.value) + + with pytest.raises(InvalidQueryError) as error_msg: + mast.Tesscut.get_sectors(objectname=moving_target_name, + coordinates=coord, + moving_target=True) + assert error_mt_coord in str(error_msg.value) + + + def test_tesscut_download_cutouts(self, tmpdir): coord = SkyCoord(349.62609, -47.12424, unit="deg") @@ -749,13 +784,49 @@ def test_tesscut_download_cutouts(self, tmpdir): for row in manifest: assert os.path.isfile(row['Local Path']) - manifest = mast.Tesscut.download_cutouts(moving_target="Eleonora", sector=6, size=5, path=str(tmpdir)) + # Moving target functionality testing + + moving_target_name = 'Eleonora' + + manifest = mast.Tesscut.download_cutouts(objectname=moving_target_name, + moving_target=True, + sector=6, + size=5, + path=str(tmpdir)) assert isinstance(manifest, Table) assert len(manifest) == 1 assert manifest["Local Path"][0][-4:] == "fits" for row in manifest: assert os.path.isfile(row['Local Path']) + error_noname = "Please specify the object name or ID (as understood by the `JPL ephemerides service `__) of a moving target such as an asteroid or comet." + error_nameresolve = f"Could not resolve {moving_target_name} to a sky position." + error_mt_coord = "Only one of moving_target and coordinates may be specified." + error_name_coord = "Only one of objectname and coordinates may be specified." + + with pytest.raises(InvalidQueryError) as error_msg: + mast.Tesscut.download_cutouts(moving_target=True) + assert error_noname in str(error_msg.value) + + with pytest.raises(ResolverError) as error_msg: + mast.Tesscut.download_cutouts(objectname=moving_target_name) + assert error_nameresolve in str(error_msg.value) + + with pytest.raises(InvalidQueryError) as error_msg: + mast.Tesscut.download_cutouts(coordinates=coord, moving_target=True) + assert error_mt_coord in str(error_msg.value) + + with pytest.raises(InvalidQueryError) as error_msg: + mast.Tesscut.download_cutouts(objectname=moving_target_name, + coordinates=coord) + assert error_name_coord in str(error_msg.value) + + with pytest.raises(InvalidQueryError) as error_msg: + mast.Tesscut.download_cutouts(objectname=moving_target_name, + coordinates=coord, + moving_target=True) + assert error_mt_coord in str(error_msg.value) + def test_tesscut_get_cutouts(self, tmpdir): coord = SkyCoord(107.18696, -70.50919, unit="deg") @@ -782,11 +853,46 @@ def test_tesscut_get_cutouts(self, tmpdir): assert len(cutout_hdus_list) >= 1 assert isinstance(cutout_hdus_list[0], fits.HDUList) - cutout_hdus_list = mast.Tesscut.get_cutouts(moving_target="Eleonora", sector=6, size=5) + # Moving target functionality testing + + moving_target_name = 'Eleonora' + + cutout_hdus_list = mast.Tesscut.get_cutouts(objectname=moving_target_name, + moving_target=True, + sector=6, + size=5) assert isinstance(cutout_hdus_list, list) assert len(cutout_hdus_list) == 1 assert isinstance(cutout_hdus_list[0], fits.HDUList) + error_noname = "Please specify the object name or ID (as understood by the `JPL ephemerides service `__) of a moving target such as an asteroid or comet." + error_nameresolve = f"Could not resolve {moving_target_name} to a sky position." + error_mt_coord = "Only one of moving_target and coordinates may be specified." + error_name_coord = "Only one of objectname and coordinates may be specified." + + with pytest.raises(InvalidQueryError) as error_msg: + mast.Tesscut.download_cutouts(moving_target=True) + assert error_noname in str(error_msg.value) + + with pytest.raises(ResolverError) as error_msg: + mast.Tesscut.download_cutouts(objectname=moving_target_name) + assert error_nameresolve in str(error_msg.value) + + with pytest.raises(InvalidQueryError) as error_msg: + mast.Tesscut.download_cutouts(coordinates=coord, moving_target=True) + assert error_mt_coord in str(error_msg.value) + + with pytest.raises(InvalidQueryError) as error_msg: + mast.Tesscut.download_cutouts(objectname=moving_target_name, + coordinates=coord) + assert error_name_coord in str(error_msg.value) + + with pytest.raises(InvalidQueryError) as error_msg: + mast.Tesscut.download_cutouts(objectname=moving_target_name, + coordinates=coord, + moving_target=True) + assert error_mt_coord in str(error_msg.value) + ################### # ZcutClass tests # ################### From 3a33c66edeeff074acb711050baf207c69c5dc3e Mon Sep 17 00:00:00 2001 From: Jennifer Medina Date: Fri, 28 Jan 2022 20:31:10 -0500 Subject: [PATCH 81/87] pep8 --- astroquery/mast/tests/test_mast_remote.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/astroquery/mast/tests/test_mast_remote.py b/astroquery/mast/tests/test_mast_remote.py index c08dae0069..498465a6e6 100644 --- a/astroquery/mast/tests/test_mast_remote.py +++ b/astroquery/mast/tests/test_mast_remote.py @@ -743,8 +743,6 @@ def test_tesscut_get_sectors(self): moving_target=True) assert error_mt_coord in str(error_msg.value) - - def test_tesscut_download_cutouts(self, tmpdir): coord = SkyCoord(349.62609, -47.12424, unit="deg") From 1a45e6f7c09997d57236c3d42894ad3160ace25e Mon Sep 17 00:00:00 2001 From: Jennifer Medina Date: Fri, 28 Jan 2022 20:55:56 -0500 Subject: [PATCH 82/87] updating documentation for moving_target functionality as bool --- docs/mast/mast.rst | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/mast/mast.rst b/docs/mast/mast.rst index e9dc2a53f7..c4b9941a60 100644 --- a/docs/mast/mast.rst +++ b/docs/mast/mast.rst @@ -816,14 +816,14 @@ Requesting a cutout by moving_target accesses the `MAST Moving Target TESScut API `__ and returns a target pixel file, with format described `here `__. -The moving_target may be any object name or ID understood by the -`JPL Horizonf ephemerades interface `__. +The moving_target is an optional bool argument where `True` signifies that the accompanying `objectname` input is the object name or ID understood by the +`JPL Horizon ephemerades interface `__. The default value for moving_target is set to False. Therefore, a non-moving target can be input simply with either the objectname or coordinates. .. code-block:: python >>> from astroquery.mast import Tesscut - >>> hdulist = Tesscut.get_cutouts(moving_target="Eleonora", size=5, sector=6) + >>> hdulist = Tesscut.get_cutouts(objectname="Eleonora", moving_target=True, size=5, sector=6) >>> hdulist[0].info() Filename: No. Name Ver Type Cards Dimensions Format @@ -834,13 +834,11 @@ The moving_target may be any object name or ID understood by the -The `~astroquery.mast.TesscutClass.download_cutouts` function takes a coordinate, object name -(i.e. "M104" or "TIC 32449963"), or moving target (i.e. "Eleonora") and cutout size -(in pixels or an angular quantity) and downloads the cutout target pixel file(s). +The `~astroquery.mast.TesscutClass.download_cutouts` function takes a coordinate, cutout size +(in pixels or an angular quantity), or object name +(i.e. "M104" or "TIC 32449963") and moving target (True or False). It uses these parameters to download the cutout target pixel file(s). -If a given coordinate/object/moving target appears in more than one TESS sector a -target pixel file will be produced for each sector. If the cutout area overlaps -more than one camera or ccd a target pixel file will be produced for each one. +If a given coordinate/object/moving target appears in more than one TESS sector, a target pixel file will be produced for each sector. If the cutout area overlaps more than one camera or ccd, a target pixel file will be produced for each one. .. code-block:: python @@ -892,11 +890,13 @@ To access sector information for a particular coordinate, object, or moving targ >>> from astroquery.mast import Tesscut - >>> sector_table = Tesscut.get_sectors(moving_target="Ceres") + >>> sector_table = Tesscut.get_sectors(objectname="Ceres", moving_target=True) >>> print(sector_table) - sectorName sector camera ccd + sectorName sector camera ccd -------------- ------ ------ --- tess-s0029-1-4 29 1 4 + tess-s0043-3-3 43 3 3 + tess-s0044-2-4 44 2 4 Zcut ==== From 3f33ecc7be7971782f33345fce7a39afb202f68c Mon Sep 17 00:00:00 2001 From: Jennifer Medina Date: Fri, 28 Jan 2022 20:58:46 -0500 Subject: [PATCH 83/87] removing an indent added by mistake --- docs/mast/mast.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/mast/mast.rst b/docs/mast/mast.rst index c4b9941a60..04d3fcdeaa 100644 --- a/docs/mast/mast.rst +++ b/docs/mast/mast.rst @@ -823,7 +823,7 @@ The moving_target is an optional bool argument where `True` signifies that the a >>> from astroquery.mast import Tesscut - >>> hdulist = Tesscut.get_cutouts(objectname="Eleonora", moving_target=True, size=5, sector=6) + >>> hdulist = Tesscut.get_cutouts(objectname="Eleonora", moving_target=True, size=5, sector=6) >>> hdulist[0].info() Filename: No. Name Ver Type Cards Dimensions Format From ba37d2a79cf89fedfbd098eda9643b9f57209bc0 Mon Sep 17 00:00:00 2001 From: Jennifer Medina Date: Fri, 28 Jan 2022 21:04:32 -0500 Subject: [PATCH 84/87] readthedocs indentation --- docs/mast/mast.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/mast/mast.rst b/docs/mast/mast.rst index 04d3fcdeaa..c225ce6abf 100644 --- a/docs/mast/mast.rst +++ b/docs/mast/mast.rst @@ -892,7 +892,7 @@ To access sector information for a particular coordinate, object, or moving targ >>> sector_table = Tesscut.get_sectors(objectname="Ceres", moving_target=True) >>> print(sector_table) - sectorName sector camera ccd + sectorName sector camera ccd -------------- ------ ------ --- tess-s0029-1-4 29 1 4 tess-s0043-3-3 43 3 3 From 2a127aea5789bb812161cfcdb3c6820bb793a8a1 Mon Sep 17 00:00:00 2001 From: "C.E. Brasseur" Date: Mon, 31 Jan 2022 20:24:08 +0000 Subject: [PATCH 85/87] small docs typo --- docs/mast/mast.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/mast/mast.rst b/docs/mast/mast.rst index c225ce6abf..68559b7814 100644 --- a/docs/mast/mast.rst +++ b/docs/mast/mast.rst @@ -816,7 +816,7 @@ Requesting a cutout by moving_target accesses the `MAST Moving Target TESScut API `__ and returns a target pixel file, with format described `here `__. -The moving_target is an optional bool argument where `True` signifies that the accompanying `objectname` input is the object name or ID understood by the +The moving_target is an optional bool argument where `True` signifies that the accompanying ``objectname`` input is the object name or ID understood by the `JPL Horizon ephemerades interface `__. The default value for moving_target is set to False. Therefore, a non-moving target can be input simply with either the objectname or coordinates. .. code-block:: python From 7b7a981dacc2f93ce1ad1e9c156423206de7c7f5 Mon Sep 17 00:00:00 2001 From: "C.E. Brasseur" Date: Mon, 31 Jan 2022 21:06:52 +0000 Subject: [PATCH 86/87] changelog update --- CHANGES.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 6dd9df9de2..5597293c8c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -24,6 +24,12 @@ gaia ``astroquery.gaia.Gaia`` no longer ignore their ``columns`` argument when ``radius`` is specified. [#2249] +mast +^^^^ + +- Adding moving target functionality to ``astroquery.mast.Tesscut`` [#2121] + + Infrastructure, Utility and Other Changes and Additions ------------------------------------------------------- From 215f99c983ccbb87ea7d8fd39042b98cf86f0507 Mon Sep 17 00:00:00 2001 From: Merlin Fisher-Levine Date: Tue, 1 Feb 2022 18:54:05 +0000 Subject: [PATCH 87/87] Issue #2273 Fix race condition --- astroquery/query.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/astroquery/query.py b/astroquery/query.py index 89356dd3dc..8c764d1699 100644 --- a/astroquery/query.py +++ b/astroquery/query.py @@ -173,8 +173,7 @@ def __init__(self): self.cache_location = os.path.join( paths.get_cache_dir(), 'astroquery', self.__class__.__name__.split("Class")[0]) - if not os.path.exists(self.cache_location): - os.makedirs(self.cache_location) + os.makedirs(self.cache_location, exist_ok=True) self._cache_active = True def __call__(self, *args, **kwargs):