Merge pull request #1031 from TOMToolkit/1023-experience-upgrading-th…

…e-single-target-processors-for-aws-compatibility

generalize single target data service data structure for AWS, etc.

tom_dataproducts/processors/atlas_processor.py

@@ -1,13 +1,17 @@
+import logging
 import mimetypes
 
 from astropy import units
 import astropy.io.ascii
 from astropy.time import Time, TimezoneInfo
 import numpy as np
+from django.core.files.storage import default_storage
 
 from tom_dataproducts.data_processor import DataProcessor
 from tom_dataproducts.exceptions import InvalidFileFormatException
 
+logger = logging.getLogger(__name__)
+
 
 class AtlasProcessor(DataProcessor):
 
@@ -16,7 +20,7 @@ def data_type_override(self):
 
     def process_data(self, data_product):
         """
-        Routes a atlas processing call to a method specific to a file-format.
+        Routes an atlas processing call to a method specific to a file-format.
 
         :param data_product: Photometric DataProduct which will be processed into the specified format for database
         ingestion
@@ -26,7 +30,12 @@ def process_data(self, data_product):
         :rtype: list
         """
 
-        mimetype = mimetypes.guess_type(data_product.data.path)[0]
+        try:
+            mimetype = mimetypes.guess_type(data_product.data.path)[0]
+        except NotImplementedError:
+            mimetype = 'text/plain'
+        logger.debug(f'Processing Atlas data with mimetype {mimetype}')
+
         if mimetype in self.PLAINTEXT_MIMETYPES:
             photometry = self._process_photometry_from_plaintext(data_product)
             return [(datum.pop('timestamp'), datum, datum.pop('source', 'ATLAS')) for datum in photometry]
@@ -52,7 +61,8 @@ def _process_photometry_from_plaintext(self, data_product):
         photometry = []
         signal_to_noise_cutoff = 3.0  # cutoff to turn magnitudes into non-detection limits
 
-        data = astropy.io.ascii.read(data_product.data.path)
+        data_file = default_storage.open(data_product.data.name, 'r')
+        data = astropy.io.ascii.read(data_file.read())
         if len(data) < 1:
             raise InvalidFileFormatException('Empty table or invalid file type')
 

tom_dataproducts/processors/panstarrs_processor.py

@@ -4,6 +4,7 @@
 from astropy import units
 import astropy.io.ascii
 from astropy.time import Time, TimezoneInfo
+from django.core.files.storage import default_storage
 
 from tom_dataproducts.data_processor import DataProcessor
 from tom_dataproducts.exceptions import InvalidFileFormatException
@@ -32,7 +33,10 @@ def process_data(self, data_product):
         :rtype: list
         """
 
-        mimetype = mimetypes.guess_type(data_product.data.path)[0]
+        try:
+            mimetype = mimetypes.guess_type(data_product.data.path)[0]
+        except NotImplementedError:
+            mimetype = 'text/plain'
         logger.debug(f'Processing PanSTARRS data with mimetype {mimetype}')
 
         if mimetype in self.PLAINTEXT_MIMETYPES:
@@ -58,7 +62,9 @@ def _process_photometry_from_plaintext(self, data_product):
         """
         photometry = []
 
-        data = astropy.io.ascii.read(data_product.data.path)
+        data_file = default_storage.open(data_product.data.name, 'r')
+        data = astropy.io.ascii.read(data_file.read())
+
         if len(data) < 1:
             raise InvalidFileFormatException('Empty table or invalid file type')
 

tom_dataproducts/single_target_data_service/panstarrs_service/panstarrs.py

@@ -57,7 +57,7 @@ def __init__(self, *args, **kwargs):
         # initialize query time range to reasonable values
         now = datetime.now()
         now_mjd = Time((now - timedelta(minutes=1))).mjd
-        past_mjd = Time((now - timedelta(days=20))).mjd
+        past_mjd = Time(datetime(2008, 12, 1)).mjd  # Pan-STARRS first went online in December 2008
         self.fields['max_date_mjd'].initial = now_mjd
         self.fields['min_date_mjd'].initial = past_mjd