Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Removed temp files once they are no longer needed from hubble and xmm #2465

Merged
merged 3 commits into from
Jul 20, 2022
Merged

Removed temp files once they are no longer needed from hubble and xmm #2465

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
03a871c
Removed temp files once they are no longer needed from hubble and xmm…
javier-ballester Jul 13, 2022
61226bd
Added temp directory to handle temp files for hubble, xmm_newton and …
javier-ballester Jul 19, 2022
ae9de56
Using os.path for more rouboust filepath generation
bsipocz Jul 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 16 additions & 12 deletions astroquery/esa/hubble/tests/test_esa_hubble.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,19 +11,20 @@


"""
import numpy as np
import pytest
import os
import shutil
import tempfile
from unittest.mock import MagicMock
from unittest.mock import patch

import numpy as np
import pytest
from astropy import coordinates
from astropy.table.table import Table
from requests.models import Response

from astroquery.esa.hubble import ESAHubbleClass
from astroquery.esa.hubble.tests.dummy_tap_handler import DummyHubbleTapHandler
from astroquery.utils.mocks import MockResponse
from astropy import coordinates
from unittest.mock import MagicMock
from astropy.table.table import Table
import shutil


def data_path(filename):
Expand Down Expand Up @@ -93,9 +94,10 @@ def test_download_product_errors(self):
assert "This product_type is not allowed" in err.value.args[0]

def test_download_product_by_calibration(self):
tempdir = tempfile.mkdtemp("temp")
parameters = {'observation_id': "J6FL25S4Q",
'cal_level': "RAW",
'filename': "J6FL25S4Q.vot.test",
'filename': os.path.join(tempdir, "J6FL25S4Q.vot.test"),
'verbose': True}
ehst = ESAHubbleClass(self.get_dummy_tap_handler())
ehst.download_product(observation_id=parameters['observation_id'],
Expand All @@ -104,9 +106,10 @@ def test_download_product_by_calibration(self):
verbose=parameters['verbose'])

def test_download_product_by_product_type(self):
tempdir = tempfile.mkdtemp("temp")
parameters = {'observation_id': "J6FL25S4Q",
'product_type': "SCIENCE_PRODUCT",
'filename': "J6FL25S4Q.vot.test",
'filename': os.path.join(tempdir, "J6FL25S4Q.vot.test"),
'verbose': True}
ehst = ESAHubbleClass(self.get_dummy_tap_handler())
ehst.download_product(observation_id=parameters['observation_id'],
Expand All @@ -127,11 +130,11 @@ def test_download_product_by_product_type(self):
verbose=parameters['verbose'])

def test_get_postcard(self):
tempdir = tempfile.mkdtemp("temp")
ehst = ESAHubbleClass(self.get_dummy_tap_handler())
ehst.get_postcard(observation_id="X0MC5101T",
filename="X0MC5101T.vot",
filename=os.path.join(tempdir, "X0MC5101T.vot"),
verbose=True)
os.remove("X0MC5101T.vot")

@patch.object(ESAHubbleClass, 'cone_search')
@patch.object(ESAHubbleClass, '_query_tap_target')
Expand Down Expand Up @@ -225,8 +228,9 @@ def test_get_tables(self):
ehst.get_tables(True, True)

def test_get_artifact(self):
tempdir = tempfile.mkdtemp("temp")
ehst = ESAHubbleClass(self.get_dummy_tap_handler())
ehst.get_artifact("w0ji0v01t_c2f.fits.gz")
ehst.get_artifact(os.path.join(tempdir, "w0ji0v01t_c2f.fits.gz"))

def test_get_columns(self):
parameters = {'query': "select top 10 * from hsc_v2.hubble_sc2",
Expand Down
94 changes: 49 additions & 45 deletions astroquery/esa/xmm_newton/tests/test_xmm_newton.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,19 @@
European Space Agency (ESA)
Created on 4 Sept. 2019
"""
import errno
import os
import shutil
import tarfile
import tempfile
from unittest.mock import patch

import pytest
import tarfile
import os
import errno
import shutil

from astroquery.exceptions import LoginError
from ..core import XMMNewtonClass
from ..tests.dummy_tap_handler import DummyXMMNewtonTapHandler
from ..tests.dummy_handler import DummyHandler
from astroquery.exceptions import LoginError
from ..tests.dummy_tap_handler import DummyXMMNewtonTapHandler


def data_path(filename):
Expand Down Expand Up @@ -98,9 +99,11 @@ def test_dummy_handler(self):
dummyHandler.reset()

def test_parse_filename(self):
self._create_tar("filename.tar", self._files)
tempdir = tempfile.mkdtemp("temp")
filename = os.path.join(tempdir, "filename.tar")
self._create_tar(filename, self._files)
xsa = XMMNewtonClass(self.get_dummy_tap_handler())
with tarfile.open("filename.tar", "r") as tar:
with tarfile.open(filename, "r") as tar:
for i in tar.getmembers():
paths = os.path.split(i.name)
fname = paths[1]
Expand All @@ -109,7 +112,6 @@ def test_parse_filename(self):
continue
fname_info = xsa._parse_filename(fname)
assert fname_info["X"] == "P"
os.remove("filename.tar")

_files = {
"0405320501": {
Expand Down Expand Up @@ -294,9 +296,10 @@ def _create_tar_lightcurves(self, tarname, files):
shutil.rmtree(ob_name)

def test_create_tar_lightcurves(self):
_tarname = "tarfile_lightcurves.tar"
self._create_tar_lightcurves(_tarname, self._files_lightcurves)
assert os.path.isfile(_tarname)
tempdir = tempfile.mkdtemp("temp")
_tarpath = os.path.join(tempdir, "tarfile_lightcurves.tar")
self._create_tar_lightcurves(_tarpath, self._files_lightcurves)
assert os.path.isfile(_tarpath)

def test_get_epic_spectra_non_existing_file(self, capsys):
_tarname = "nonexistingfile.tar"
Expand All @@ -310,27 +313,28 @@ def test_get_epic_spectra_non_existing_file(self, capsys):
"[astroquery.esa.xmm_newton.core]\n" % _tarname)

def test_get_epic_spectra_invalid_instrumnet(self, capsys):
_tarname = "tarfile.tar"
tempdir = tempfile.mkdtemp("temp")
_tarpath = os.path.join(tempdir, "tarfile.tar")
_invalid_instrument = "II"
_source_number = 83
self._create_tar(_tarname, self._files)
self._create_tar(_tarpath, self._files)
xsa = XMMNewtonClass(self.get_dummy_tap_handler())
res = xsa.get_epic_spectra(_tarname, _source_number,
res = xsa.get_epic_spectra(_tarpath, _source_number,
instrument=[_invalid_instrument])
assert res == {}
out, err = capsys.readouterr()
assert err == ("WARNING: Invalid instrument %s "
"[astroquery.esa.xmm_newton.core]\n"
% _invalid_instrument)
os.remove(_tarname)

def test_get_epic_spectra_invalid_source_number(self, capsys):
_tarname = "tarfile.tar"
tempdir = tempfile.mkdtemp("temp")
_tarpath = os.path.join(tempdir, "tarfile.tar")
_invalid_source_number = 833
_default_instrument = ['M1', 'M2', 'PN', 'EP']
self._create_tar(_tarname, self._files)
self._create_tar(_tarpath, self._files)
xsa = XMMNewtonClass(self.get_dummy_tap_handler())
res = xsa.get_epic_spectra(_tarname, _invalid_source_number,
res = xsa.get_epic_spectra(_tarpath, _invalid_source_number,
instrument=[])
assert res == {}
out, err = capsys.readouterr()
Expand All @@ -339,9 +343,8 @@ def test_get_epic_spectra_invalid_source_number(self, capsys):
" Source Number: %u\n"
" Instrument: %s\n"
" [astroquery.esa.xmm_newton.core]\n"
% (_tarname, _invalid_source_number,
% (_tarpath, _invalid_source_number,
_default_instrument))
os.remove(_tarname)

def test_get_epic_images_non_existing_file(self, capsys):
_tarname = "nonexistingfile.tar"
Expand All @@ -354,43 +357,44 @@ def test_get_epic_images_non_existing_file(self, capsys):
"[astroquery.esa.xmm_newton.core]\n" % _tarname)

def test_get_epic_images_invalid_instrument(self, capsys):
_tarname = "tarfile.tar"
tempdir = tempfile.mkdtemp("temp")
_tarpath = os.path.join(tempdir, "tarfile.tar")
_invalid_instrument = "II"
self._create_tar(_tarname, self._files)
self._create_tar(_tarpath, self._files)
xsa = XMMNewtonClass(self.get_dummy_tap_handler())
res = xsa.get_epic_images(_tarname,
res = xsa.get_epic_images(_tarpath,
band=[], instrument=[_invalid_instrument],
get_detmask=True, get_exposure_map=True)
assert res == {}
out, err = capsys.readouterr()
assert err == ("WARNING: Invalid instrument %s "
"[astroquery.esa.xmm_newton.core]\n"
% _invalid_instrument)
os.remove(_tarname)

def test_get_epic_images_invalid_band(self, capsys):
_tarname = "tarfile.tar"
tempdir = tempfile.mkdtemp("temp")
_tarpath = os.path.join(tempdir, "tarfile.tar")
_invalid_band = 10
self._create_tar(_tarname, self._files)
self._create_tar(_tarpath, self._files)
xsa = XMMNewtonClass(self.get_dummy_tap_handler())
res = xsa.get_epic_images(_tarname,
res = xsa.get_epic_images(_tarpath,
band=[_invalid_band], instrument=[],
get_detmask=True, get_exposure_map=True)
assert res == {}
out, err = capsys.readouterr()
assert err == ("WARNING: Invalid band %u "
"[astroquery.esa.xmm_newton.core]\n" % _invalid_band)
os.remove(_tarname)

def test_get_epic_images(self):
_tarname = "tarfile.tar"
tempdir = tempfile.mkdtemp("temp")
_tarpath = os.path.join(tempdir, "tarfile.tar")
_instruments = ["M1", "M1_expo", "M1_det",
"M2", "M2_expo", "M2_det",
"PN", "PN_expo", "PN_det",
"EP", "EP_expo", "EP_det"]
self._create_tar(_tarname, self._files)
self._create_tar(_tarpath, self._files)
xsa = XMMNewtonClass(self.get_dummy_tap_handler())
res = xsa.get_epic_images(_tarname, band=[], instrument=[],
res = xsa.get_epic_images(_tarpath, band=[], instrument=[],
get_detmask=True, get_exposure_map=True)
assert len(res) == 6 # Number of different bands
assert len(res[1]) == 9 # Number of different inst within band 1
Expand Down Expand Up @@ -471,14 +475,14 @@ def test_get_epic_images(self):
# Removing files created in this test
for ob_name in self._files:
shutil.rmtree(ob_name)
os.remove(_tarname)

def test_get_epic_lightcurve(self):
_tarname = "tarfile.tar"
self._create_tar(_tarname, self._files)
tempdir = tempfile.mkdtemp("temp")
_tarpath = os.path.join(tempdir, "tarfile.tar")
self._create_tar(_tarpath, self._files)
_source_number = 1
xsa = XMMNewtonClass(self.get_dummy_tap_handler())
res = xsa.get_epic_lightcurve(_tarname, _source_number,
res = xsa.get_epic_lightcurve(_tarpath, _source_number,
instrument=['M1', 'M2', 'PN'])
assert res == {}

Expand All @@ -494,26 +498,27 @@ def test_get_epic_lightcurve_non_existing_file(self, capsys):
"[astroquery.esa.xmm_newton.core]\n" % _tarname)

def test_get_epic_lightcurve_invalid_instrument(self, capsys):
_tarname = "tarfile.tar"
tempdir = tempfile.mkdtemp("temp")
_tarpath = os.path.join(tempdir, "tarfile.tar")
_invalid_instrument = "II"
self._create_tar(_tarname, self._files)
self._create_tar(_tarpath, self._files)
xsa = XMMNewtonClass(self.get_dummy_tap_handler())
res = xsa.get_epic_images(_tarname, [], [_invalid_instrument],
res = xsa.get_epic_images(_tarpath, [], [_invalid_instrument],
get_detmask=True, get_exposure_map=True)
assert res == {}
out, err = capsys.readouterr()
assert err == ("WARNING: Invalid instrument %s "
"[astroquery.esa.xmm_newton.core]\n"
% _invalid_instrument)
os.remove(_tarname)

def test_get_epic_lightcurve_invalid_source_number(self, capsys):
_tarname = "tarfile.tar"
tempdir = tempfile.mkdtemp("temp")
_tarpath = os.path.join(tempdir, "tarfile.tar")
_invalid_source_number = 833
_default_instrument = ['M1', 'M2', 'PN', 'EP']
self._create_tar(_tarname, self._files)
self._create_tar(_tarpath, self._files)
xsa = XMMNewtonClass(self.get_dummy_tap_handler())
res = xsa.get_epic_lightcurve(_tarname, _invalid_source_number,
res = xsa.get_epic_lightcurve(_tarpath, _invalid_source_number,
instrument=[])
assert res == {}
out, err = capsys.readouterr()
Expand All @@ -522,9 +527,8 @@ def test_get_epic_lightcurve_invalid_source_number(self, capsys):
" Source Number: %u\n"
" Instrument: %s\n"
" [astroquery.esa.xmm_newton.core]\n"
% (_tarname, _invalid_source_number,
% (_tarpath, _invalid_source_number,
_default_instrument))
os.remove(_tarname)

def test_create_link(self):
xsa = XMMNewtonClass(self.get_dummy_tap_handler())
Expand Down
36 changes: 16 additions & 20 deletions astroquery/esasky/tests/test_esasky_remote.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,15 @@

import os
import shutil
import tempfile

import pytest
from astroquery import log
from astroquery.utils.tap.model.taptable import TapTableMeta
from astroquery.utils.tap.model.tapcolumn import TapColumn
from astroquery.utils.commons import TableList
from astropy.io.fits.hdu.hdulist import HDUList

from astroquery import log
from astroquery.utils.commons import TableList
from astroquery.utils.tap.model.tapcolumn import TapColumn
from astroquery.utils.tap.model.taptable import TapTableMeta
from ... import esasky

ESASkyClass = esasky.core.ESASkyClass()
Expand Down Expand Up @@ -87,11 +89,11 @@ def test_esasky_get_images_obs_id(self):
assert os.path.exists(file_path)
log.info("Checking {} data.".format(mission))
if mission.upper() == "HERSCHEL":
assert(isinstance(result[mission.upper()][0]["250"], HDUList))
assert(isinstance(result[mission.upper()][0]["350"], HDUList))
assert(isinstance(result[mission.upper()][0]["500"], HDUList))
assert (isinstance(result[mission.upper()][0]["250"], HDUList))
assert (isinstance(result[mission.upper()][0]["350"], HDUList))
assert (isinstance(result[mission.upper()][0]["500"], HDUList))
else:
assert(isinstance(result[mission.upper()][0], HDUList))
assert (isinstance(result[mission.upper()][0], HDUList))

result = None

Expand All @@ -113,10 +115,10 @@ def test_esasky_get_spectra_obs_id(self):
assert os.path.exists(file_path)
log.info("Checking {} data.".format(mission))
if mission.upper() == "HERSCHEL":
assert(isinstance(result[mission.upper()]["1342253595"]["WBS"]["WBS-V_USB_4b"], HDUList))
assert(isinstance(result[mission.upper()]["1342253595"]["HRS"]["HRS-H_LSB_4b"], HDUList))
assert (isinstance(result[mission.upper()]["1342253595"]["WBS"]["WBS-V_USB_4b"], HDUList))
assert (isinstance(result[mission.upper()]["1342253595"]["HRS"]["HRS-H_LSB_4b"], HDUList))
else:
assert(isinstance(result[mission.upper()][0], HDUList))
assert (isinstance(result[mission.upper()][0], HDUList))

result = None

Expand All @@ -132,23 +134,17 @@ def test_esasky_query_object_maps(self):

@pytest.mark.bigdata
def test_esasky_get_images(self):
download_directory = "ESASkyRemoteTest"
if not os.path.exists(download_directory):
os.makedirs(download_directory)
tempdir = tempfile.mkdtemp("ESASkyRemoteTest")

missions = ESASkyClass.list_maps()
# Remove very large map missions & missions with many results
# & missions without proper download url (INTEGRAL, SUZAKU, ALMA, AKARI)
missions = [mission for mission in missions if mission not in
("HST-OPTICAL", "HST-IR", "HST-UV", "XMM-OM-UV", "INTEGRAL", "SUZAKU", "ALMA", "AKARI")]

ESASkyClass.get_images(position="M51", missions=missions, download_dir=download_directory)

for mission in missions:
file_path = os.path.join(download_directory, mission)
assert os.path.exists(file_path)
ESASkyClass.get_images(position="M51", missions=missions, download_dir=tempdir)

shutil.rmtree(download_directory)
assert (os.path.getsize(tempdir) > 0)

def test_esasky_get_images_small(self):
download_directory = "ESASkyRemoteTest"
Expand Down