From 18546a1592803eaed7fd6f5e91e3cbfab74abbbc Mon Sep 17 00:00:00 2001
From: mairan <mteodoro@stsci.edu>
Date: Thu, 13 Apr 2023 12:15:17 -0400
Subject: [PATCH] TweakReg: update to use VO instead of astroquery. (#678)

* Update method to fetch catalog data.

* Unit test for get_catalog and some refactoring.

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Add astroquery to dependencies.

* add code owners file (#670)

* add code owners file

* fix usernames

* use maintainer team

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

* Bug fix.

* Add astrometric_utils test suite and Improve error message.

* Add pandas to dependencies.

* Bump numpy version.

* Update get_catalog to use VO service instead of astroquery.

* Remove astroquery dependencies.

* Update default abs refcat.

* Style fixes.

* Remove pandas formats.

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Zach Burnett <zburnett@stsci.edu>
Co-authored-by: Nadia Dencheva <nadia.dencheva@gmail.com>
---
 pyproject.toml                                |  4 +-
 romancal/tweakreg/astrometric_utils.py        | 44 ++++++++++++-------
 .../tweakreg/tests/test_astrometric_utils.py  | 13 +++++-
 3 files changed, 41 insertions(+), 20 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 8dee21d9d..e41ffdc22 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -18,7 +18,7 @@ dependencies = [
     'crds >=11.16.16',
     'gwcs >=0.18.1',
     'jsonschema >=4.0.1',
-    'numpy >=1.20.3',
+    'numpy >=1.20',
     'photutils >=1.6.0',
     'pyparsing >=2.4.7',
     'requests >=2.22',
@@ -29,8 +29,6 @@ dependencies = [
     'stcal >=1.3.3',
     'stpipe >=0.4.2',
     'tweakwcs >=0.8.0',
-    'astroquery >= 0.4.6',
-    'pandas >= 1.5.3'
 ]
 dynamic = ['version']
 
diff --git a/romancal/tweakreg/astrometric_utils.py b/romancal/tweakreg/astrometric_utils.py
index bfae2f485..ab746cb85 100644
--- a/romancal/tweakreg/astrometric_utils.py
+++ b/romancal/tweakreg/astrometric_utils.py
@@ -1,14 +1,20 @@
-import traceback
+import os
 
+import requests
 from astropy import table
 from astropy import units as u
 from astropy.coordinates import SkyCoord
-from astroquery.mast import Catalogs
 
 from ..assign_wcs import utils as wcsutil
 from ..resample import resample_utils
 
-DEF_CAT = "GAIADR3"
+ASTROMETRIC_CAT_ENVVAR = "ASTROMETRIC_CATALOG_URL"
+DEF_CAT_URL = "http://gsss.stsci.edu/webservices"
+
+if ASTROMETRIC_CAT_ENVVAR in os.environ:
+    SERVICELOCATION = os.environ[ASTROMETRIC_CAT_ENVVAR]
+else:
+    SERVICELOCATION = DEF_CAT_URL
 
 """
 
@@ -19,7 +25,7 @@
 
 def create_astrometric_catalog(
     input_models,
-    catalog=DEF_CAT,
+    catalog="GAIADR3",
     output="ref_cat.ecsv",
     gaia_only=False,
     table_format="ascii.ecsv",
@@ -148,7 +154,7 @@ def compute_radius(wcs):
     return radius, fiducial
 
 
-def get_catalog(ra, dec, sr=0.1, catalog=DEF_CAT):
+def get_catalog(ra, dec, sr=0.1, catalog="GAIADR3"):
     """Extract catalog from VO web service.
 
     Parameters
@@ -168,18 +174,24 @@ def get_catalog(ra, dec, sr=0.1, catalog=DEF_CAT):
 
     Returns
     -------
-    csv : CSV object
-        CSV object of returned sources with all columns as provided by catalog
+        A Table object of returned sources with all columns as provided by catalog.
 
     """
-    try:
-        catalog_data_csv = Catalogs.query_object(
-            f"{ra} {dec}", radius=sr, catalog=catalog, format="csv"
-        )
-        catalog_data_csv.rename_column("source_id", "objID")
-        catalog_data_csv.rename_column("phot_g_mean_mag", "mag")
-    except Exception:
-        traceback.format_exc()
+    service_type = "vo/CatalogSearch.aspx"
+    spec_str = "RA={}&DEC={}&SR={}&FORMAT={}&CAT={}&MINDET=5"
+    headers = {"Content-Type": "text/csv"}
+    fmt = "CSV"
+
+    spec = spec_str.format(ra, dec, sr, fmt, catalog)
+    service_url = f"{SERVICELOCATION}/{service_type}?{spec}"
+    rawcat = requests.get(service_url, headers=headers)
+    r_contents = rawcat.content.decode()  # convert from bytes to a String
+    rstr = r_contents.split("\r\n")
+    # remove initial line describing the number of sources returned
+    # CRITICAL to proper interpretation of CSV data
+    del rstr[0]
+    if len(rstr) == 0:
+        print(Exception("VO catalog service returned no results."))
         raise
 
-    return catalog_data_csv
+    return table.Table.read(rstr, format="csv")
diff --git a/romancal/tweakreg/tests/test_astrometric_utils.py b/romancal/tweakreg/tests/test_astrometric_utils.py
index cd4fe0730..8a5c4f226 100644
--- a/romancal/tweakreg/tests/test_astrometric_utils.py
+++ b/romancal/tweakreg/tests/test_astrometric_utils.py
@@ -290,7 +290,18 @@ def test_create_astrometric_catalog_write_results_to_disk(tmp_path, base_image):
         if x.strip().split()[1].lower() == "yes"
     ]
     # exclude data formats
-    [list_of_supported_formats.remove(x) for x in ["asdf", "fits", "hdf5", "parquet"]]
+    [
+        list_of_supported_formats.remove(x)
+        for x in [
+            "pandas.csv",
+            "pandas.html",
+            "pandas.json",
+            "asdf",
+            "fits",
+            "hdf5",
+            "parquet",
+        ]
+    ]
 
     for table_format in list_of_supported_formats:
         res = create_astrometric_catalog(