From 4f8ee16f14f8afdba2c9ed0f47fa3b57507545d9 Mon Sep 17 00:00:00 2001 From: Nate Tellis Date: Wed, 24 Jan 2024 16:17:31 -0500 Subject: [PATCH] Remove Pandera as dependency --- cutouts/filter.py | 11 +-- cutouts/io/nsc.py | 9 +-- cutouts/io/skymapper.py | 7 +- cutouts/io/tests/test_nsc_dr2.py | 24 ++++++ cutouts/io/types.py | 42 ---------- cutouts/io/ztf.py | 7 +- cutouts/main.py | 12 +-- requirements.txt | 134 +++++++++++++++++++++++++++---- setup.cfg | 1 - 9 files changed, 158 insertions(+), 89 deletions(-) diff --git a/cutouts/filter.py b/cutouts/filter.py index 8daee7d..74fe8d0 100644 --- a/cutouts/filter.py +++ b/cutouts/filter.py @@ -1,17 +1,15 @@ import logging import numpy as np -import pandera as pa -from pandera.typing import DataFrame +import pandas as pd -from .io.types import CutoutRequest, CutoutResult, CutoutsResultSchema +from .io.types import CutoutRequest, CutoutResult logger = logging.getLogger(__file__) -@pa.check_types def select_cutout( - results_df: DataFrame[CutoutsResultSchema], cutout_request: CutoutRequest + results_df: pd.DataFrame, cutout_request: CutoutRequest ) -> CutoutResult: """ Select the cutout closest to the requested exposure start time +- delta_time. @@ -91,9 +89,8 @@ def select_cutout( return result -@pa.check_types def select_comparison_cutout( - results_df: DataFrame[CutoutsResultSchema], + results_df: pd.DataFrame, cutout_result: CutoutResult, cutout_request: CutoutRequest, min_time_separation: float = 1 / 24, diff --git a/cutouts/io/nsc.py b/cutouts/io/nsc.py index bdbe360..4ad5cd0 100644 --- a/cutouts/io/nsc.py +++ b/cutouts/io/nsc.py @@ -1,11 +1,9 @@ import logging -import pandera as pa -from pandera.typing import DataFrame +import pandas as pd from pyvo.dal.sia import SIAResults from .sia import SIAHandler -from .types import CutoutRequest, CutoutsResultSchema from .util import exposure_id_from_url logger = logging.getLogger(__name__) @@ -16,10 +14,9 @@ def _get_generic_image_url_from_cutout_url(cutout_url: str): return cutout_url.split("&POS=")[0] -@pa.check_types def find_cutouts_nsc_dr2( - cutout_request: CutoutRequest, -) -> DataFrame[CutoutsResultSchema]: + cutout_request: pd.DataFrame, +) -> pd.DataFrame: """ Search the NOIRLab Archive for cutouts and images at a given RA, Dec. diff --git a/cutouts/io/skymapper.py b/cutouts/io/skymapper.py index 68dd8ad..9b2db7e 100644 --- a/cutouts/io/skymapper.py +++ b/cutouts/io/skymapper.py @@ -1,12 +1,10 @@ import logging import pandas as pd -import pandera as pa -from pandera.typing import DataFrame from pyvo.dal.sia import SIAResults from .sia import SIAHandler -from .types import CutoutRequest, CutoutsResultSchema +from .types import CutoutRequest logger = logging.getLogger(__name__) @@ -22,10 +20,9 @@ def _get_generic_image_url_from_cutout_url(cutout_url: str): return url_string -@pa.check_types def find_cutouts_skymapper_dr2( cutout_request: CutoutRequest, -) -> DataFrame[CutoutsResultSchema]: +) -> pd.DataFrame: """ Search the Skymapper SIA service for cutouts and images at a given RA, Dec. diff --git a/cutouts/io/tests/test_nsc_dr2.py b/cutouts/io/tests/test_nsc_dr2.py index 2de7a9c..5bee073 100644 --- a/cutouts/io/tests/test_nsc_dr2.py +++ b/cutouts/io/tests/test_nsc_dr2.py @@ -1,11 +1,15 @@ import os from contextlib import contextmanager from unittest.mock import patch +import pathlib +import pickle +import pandas as pd from astropy.io.votable import parse from pyvo.dal.sia import SIAResults from ..nsc import NSC_DR2_SIA, find_cutouts_nsc_dr2 +from ...main import get_cutouts from ..types import CutoutRequest # 2014 HE199 (2014-04-28T08:07:52.435) @@ -67,3 +71,23 @@ def test_sia_nsc_dr2_query(): assert col in results.columns assert "VR" in results["filter"].values + + +def test_sia_nsc_dr2_query_(): + + with mock_sia_nsc_dr2_query( + "nsc_dr2_227.5251615214173_-27.026013823449265_56775.33880132809.xml" + ) as mock: + results, comparison_results = get_cutouts( + pd.DataFrame(cutout_request1), + out_dir=pathlib.Path(os.path.dirname(os.path.abspath(__file__))) + ) + print(results) + print(comparison_results) + + with open('cutout_results.pickle', 'wb') as f: + pickle.dump(results, f) + with open('cutout_comparison_results.pickle', 'wb') as f: + pickle.dump(comparison_results, f) + + assert False \ No newline at end of file diff --git a/cutouts/io/types.py b/cutouts/io/types.py index e1206d3..83c8305 100644 --- a/cutouts/io/types.py +++ b/cutouts/io/types.py @@ -1,50 +1,8 @@ from typing import Optional -import pandera as pa -from pandera.typing import Series from pydantic import BaseModel -class CutoutRequestSchema(pa.SchemaModel): - """ - Dataframe validation for multiple cutout requests - """ - - request_id: Optional[Series[str]] = pa.Field(nullable=True) - observatory_code: Series[str] = pa.Field(coerce=True) - exposure_start_mjd: Series[float] = pa.Field(nullable=False, coerce=True) - ra_deg: Series[float] = pa.Field(ge=0, le=360, coerce=True) - dec_deg: Series[float] = pa.Field(ge=-90, le=90, coerce=True) - filter: Optional[ - Series[str] - ] = pa.Field() # TODO: validate against real list of filters? - exposure_id: Optional[Series[str]] = pa.Field(nullable=True) - exposure_duration: Optional[Series[float]] = pa.Field( - ge=0, le=2000, coerce=True, nullable=True - ) - height_arcsec: Series[float] = pa.Field(ge=0, le=200, coerce=True, nullable=True) - width_arcsec: Series[float] = pa.Field(ge=0, le=200, coerce=True, nullable=True) - delta_time: Series[float] = pa.Field(ge=0, le=100, coerce=True, nullable=True) - - -class CutoutsResultSchema(pa.SchemaModel): - # TODO: ra, dec here are the ra, dec returned by the query and - # these should be equal to the queried ra and dec. - # However, this may not always be true and we may want to consider - # adding additional fields to allow backends to return things such as the - # the center of the image/cutout. - ra_deg: Series[float] = pa.Field(ge=0, le=360, coerce=True) - dec_deg: Series[float] = pa.Field(ge=-90, le=90, coerce=True) - filter: Series[str] = pa.Field() - exposure_id: Series[str] = pa.Field() - exposure_start_mjd: Series[float] = pa.Field(nullable=False, coerce=True) - exposure_duration: Series[float] = pa.Field(ge=0, le=2000, coerce=True) - cutout_url: Series[str] = pa.Field(coerce=True) - image_url: Series[str] = pa.Field(coerce=True) - height_arcsec: Series[float] = pa.Field(ge=0, le=200, coerce=True) - width_arcsec: Series[float] = pa.Field(ge=0, le=200, coerce=True) - - class CutoutRequest(BaseModel): """ A single cutout request diff --git a/cutouts/io/ztf.py b/cutouts/io/ztf.py index f8f1290..030fd37 100644 --- a/cutouts/io/ztf.py +++ b/cutouts/io/ztf.py @@ -5,11 +5,9 @@ import backoff import numpy as np import pandas as pd -import pandera as pa import requests -from pandera.typing import DataFrame -from .types import CutoutRequest, CutoutsResultSchema +from .types import CutoutRequest logger = logging.getLogger(__file__) @@ -83,8 +81,7 @@ def perform_request(search_url): return response.text -@pa.check_types() -def find_cutouts_ztf(cutout_request: CutoutRequest) -> DataFrame[CutoutsResultSchema]: +def find_cutouts_ztf(cutout_request: CutoutRequest) -> pd.DataFrame: """ Search the ZTF service for cutouts and images at a given RA, Dec. diff --git a/cutouts/main.py b/cutouts/main.py index 0bc19a8..c9be92b 100644 --- a/cutouts/main.py +++ b/cutouts/main.py @@ -2,17 +2,15 @@ import logging import pathlib import sys -from typing import Any, Dict, Iterable, Optional, Tuple, cast +from typing import Any, Dict, Iterable, Optional, Tuple import numpy as np import pandas as pd -import pandera as pa from astropy.time import Time -from pandera.typing import DataFrame from .filter import select_comparison_cutout, select_cutout from .io import download_cutout, find_cutouts -from .io.types import CutoutRequest, CutoutRequestSchema +from .io.types import CutoutRequest from .plot import generate_gif, plot_comparison_cutouts, plot_cutouts logger = logging.getLogger("cutouts") @@ -27,9 +25,8 @@ } -@pa.check_types def get_cutouts( - cutout_requests: DataFrame[CutoutRequestSchema], + cutout_requests: pd.DataFrame, out_dir: pathlib.Path, timeout: Optional[int] = 180, full_image_timeout: Optional[int] = 600, @@ -226,7 +223,6 @@ def run_cutouts_from_precovery( compare: bool = False, compare_kwargs: Optional[dict] = None, ): - # This seems unecessary but linting fails without it out_dir_path = pathlib.Path(str(out_dir)) out_file_path = pathlib.Path(str(out_file)) @@ -268,7 +264,7 @@ def run_cutouts_from_precovery( cutout_requests["delta_time"].fillna(1e-8, inplace=True) cutout_results, comparison_results = get_cutouts( - cast(DataFrame[CutoutRequestSchema], cutout_requests), + cutout_requests, out_dir=out_dir_path, download_full_image=download_full_image, timeout=timeout, diff --git a/requirements.txt b/requirements.txt index d142f81..ab2ff65 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,15 +1,119 @@ -astropy -backoff -numpy -pandas -pandera -pyvo -matplotlib -multimethod>=1.9.1 -imageio -pytest -pytest-cov -pre-commit -setuptools>=45 -wheel -setuptools_scm>=6.0 +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --all-extras +# +annotated-types==0.6.0 + # via pydantic +astropy==6.0.0 + # via + # cutouts (pyproject.toml) + # pyvo +astropy-iers-data==0.2024.1.22.0.30.30 + # via astropy +backoff==2.2.1 + # via cutouts (pyproject.toml) +certifi==2023.11.17 + # via requests +cfgv==3.4.0 + # via pre-commit +charset-normalizer==3.3.2 + # via requests +contourpy==1.2.0 + # via matplotlib +coverage[toml]==7.4.0 + # via + # coverage + # pytest-cov +cycler==0.12.1 + # via matplotlib +distlib==0.3.8 + # via virtualenv +filelock==3.13.1 + # via virtualenv +fonttools==4.47.2 + # via matplotlib +identify==2.5.33 + # via pre-commit +idna==3.6 + # via requests +imageio==2.33.1 + # via cutouts (pyproject.toml) +iniconfig==2.0.0 + # via pytest +kiwisolver==1.4.5 + # via matplotlib +matplotlib==3.8.2 + # via cutouts (pyproject.toml) +nodeenv==1.8.0 + # via pre-commit +numpy==1.26.3 + # via + # astropy + # contourpy + # cutouts (pyproject.toml) + # imageio + # matplotlib + # pandas + # pyerfa +packaging==23.2 + # via + # astropy + # matplotlib + # pytest +pandas==2.2.0 + # via cutouts (pyproject.toml) +pillow==10.2.0 + # via + # imageio + # matplotlib +platformdirs==4.1.0 + # via virtualenv +pluggy==1.4.0 + # via pytest +pre-commit==3.6.0 + # via cutouts (pyproject.toml) +pydantic==2.5.3 + # via cutouts (pyproject.toml) +pydantic-core==2.14.6 + # via pydantic +pyerfa==2.0.1.1 + # via astropy +pyparsing==3.1.1 + # via matplotlib +pytest==7.4.4 + # via + # cutouts (pyproject.toml) + # pytest-cov +pytest-cov==4.1.0 + # via cutouts (pyproject.toml) +python-dateutil==2.8.2 + # via + # matplotlib + # pandas +pytz==2023.3.post1 + # via pandas +pyvo==1.5 + # via cutouts (pyproject.toml) +pyyaml==6.0.1 + # via + # astropy + # pre-commit +requests==2.31.0 + # via pyvo +six==1.16.0 + # via python-dateutil +typing-extensions==4.9.0 + # via + # pydantic + # pydantic-core +tzdata==2023.4 + # via pandas +urllib3==2.1.0 + # via requests +virtualenv==20.25.0 + # via pre-commit + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/setup.cfg b/setup.cfg index 0554220..0a520bc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -39,7 +39,6 @@ install_requires = pyvo matplotlib imageio - pandera pydantic [options.extras_require]