Pydantic debug

app/api/metadata.py

@@ -18,13 +18,7 @@
 from fastapi import APIRouter, Depends, HTTPException
 from fastapi.responses import JSONResponse
 
-from core.config import (
-    BASE_URL,
-    CURRENT_API_VERSION,
-    M2M_URL,
-    METADATA_SOURCE,
-    settings,
-)
+from core.config import settings
 from cache.redis import redis_dependency, ConnectionError
 from store import META
 from utils.conn import send_request, retrieve_deployments
@@ -98,7 +92,7 @@ async def _set_cache(
 
 
 async def _check_version(version: float = 2.0):
-    if version == CURRENT_API_VERSION:
+    if version == settings.CURRENT_API_VERSION:
         return True
     else:
         raise HTTPException(status_code=404, detail="API version not found.")
@@ -149,7 +143,7 @@ async def _fetch_table(
         }
 
         tabledf = dataframe.read_parquet(
-            os.path.join(METADATA_SOURCE, table_name),
+            os.path.join(settings.METADATA_SOURCE, table_name),
             engine="pyarrow-dataset",
             filters=filters,
             index=False,
@@ -172,7 +166,7 @@ async def _fetch_table(
 async def _get_annotations(
     reference_designator, stream_method, stream_rd, begin_date, end_date
 ):
-    url = f"{BASE_URL}/{M2M_URL}/12580/anno/find"
+    url = f"{settings.BASE_URL}/{settings.M2M_URL}/12580/anno/find"
     params = {
         "beginDT": begin_date,
         "endDT": end_date,

app/api/models.py

@@ -6,11 +6,11 @@
 
 
 class InstrumentRequest(BaseModel):
-    site: Optional[str]
-    group: Optional[str]
-    infrastructure: Optional[str]
-    area: Optional[str]
-    refdes: Optional[str]
+    site: Optional[str] = None
+    group: Optional[str] = None
+    infrastructure: Optional[str] = None
+    area: Optional[str] = None
+    refdes: Optional[str] = None
     include_params: bool = False
 
     _key: str = PrivateAttr()

app/core/config.py

@@ -1,19 +1,21 @@
 import os
 
-from pydantic import BaseSettings, RedisDsn
+from typing import List, Dict, Optional, ClassVar
+from pydantic import RedisDsn
+from pydantic_settings import BaseSettings
 
 
 class Settings(BaseSettings):
     """Setting for the whole application"""
 
     # TODO: Switch over fully to this settings
-    SERVICE_NAME = "Metadata Service"
-    SERVICE_ID = "metadata"
-    OPENAPI_URL = f"/{SERVICE_ID}/openapi.json"
-    DOCS_URL = f"/{SERVICE_ID}/"
-    SERVICE_DESCRIPTION = """Metadata service for Interactive Oceans."""
+    SERVICE_NAME: ClassVar[str] = "Metadata Service"
+    SERVICE_ID: ClassVar[str] = "metadata"
+    OPENAPI_URL: ClassVar[str] = f"/{SERVICE_ID}/openapi.json"
+    DOCS_URL: ClassVar[str] = f"/{SERVICE_ID}/"
+    SERVICE_DESCRIPTION: ClassVar[str] = """Metadata service for Interactive Oceans."""
 
-    CORS_ORIGINS = [
+    CORS_ORIGINS: List[str] = [
         "http://localhost",
         "http://localhost:8000",
         "http://localhost:5000",
@@ -25,28 +27,28 @@ class Settings(BaseSettings):
         "https://api.interactiveoceans.washington.edu",
     ]
 
-    BASE_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    BASE_PATH: str = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
     # API VERSION
-    CURRENT_API_VERSION = 2.0
+    CURRENT_API_VERSION: str = '2.0'
 
     # Cloud Credentials
-    AWS_KEY = os.environ.get("AWS_ACCESS_KEY_ID", None)
-    AWS_SECRET = os.environ.get("AWS_SECRET_ACCESS_KEY", None)
+    AWS_KEY: Optional[str] = os.environ.get("AWS_ACCESS_KEY_ID", None)
+    AWS_SECRET: Optional[str] = os.environ.get("AWS_SECRET_ACCESS_KEY", None)
 
     # Redis configurations
     REDIS_URI: RedisDsn = os.environ.get(
         "REDIS_URI", "redis://localhost:6379/0"
     )
 
     # OOI Configurations
-    BASE_URL = "https://ooinet.oceanobservatories.org"
-    M2M_URL = "api/m2m"
-    USERNAME = os.environ.get("OOI_USERNAME", "")
-    TOKEN = os.environ.get("OOI_TOKEN", "")
+    BASE_URL: str = "https://ooinet.oceanobservatories.org"
+    M2M_URL: str = "api/m2m"
+    USERNAME: str = os.environ.get("OOI_USERNAME", "")
+    TOKEN: str = os.environ.get("OOI_TOKEN", "")
 
     # File Systems Configurations
-    FILE_SYSTEMS = {
+    FILE_SYSTEMS: Dict = {
         "minio_s3": dict(
             protocol="s3", client_kwargs={"endpoint_url": "http://minio:9000"}
         ),
@@ -57,76 +59,15 @@ class Settings(BaseSettings):
             config_kwargs={"max_pool_connections": 1000},
         ),
     }
-    GOOGLE_SERVICE_JSON = os.environ.get(
+    GOOGLE_SERVICE_JSON: str = os.environ.get(
         "GOOGLE_SERVICE_JSON",
         "",
     )
-    DATA_BUCKET = 'ooi-data-prod'
+    DATA_BUCKET: str = 'ooi-data-prod'
 
     # Data sources
-    METADATA_SOURCE = "s3://ooi-metadata-prod"
-    METADATA_BUCKET = "ooi-metadata-prod"
+    METADATA_SOURCE: str = "s3://ooi-metadata-prod"
+    METADATA_BUCKET: str = "ooi-metadata-prod"
 
 
 settings = Settings()
-
-# API SETTINGS
-SERVICE_NAME = "Metadata Service"
-SERVICE_ID = "metadata"
-OPENAPI_URL = f"/{SERVICE_ID}/openapi.json"
-DOCS_URL = f"/{SERVICE_ID}/"
-SERVICE_DESCRIPTION = """Metadata service for Interactive Oceans."""
-
-CORS_ORIGINS = [
-    "http://localhost",
-    "http://localhost:8000",
-    "http://localhost:5000",
-    "http://localhost:4000",
-    "https://appdev.ooica.net",
-    "https://app-dev.ooica.net",
-    "https://app.interactiveoceans.washington.edu",
-    "https://api-dev.ooica.net",
-    "https://api.interactiveoceans.washington.edu",
-    "https://api-development.ooica.net",
-    "https://cava-portal.netlify.app",
-]
-
-BASE_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-
-# API VERSION
-CURRENT_API_VERSION = 2.0
-
-# Cloud Credentials
-AWS_KEY = os.environ.get("AWS_ACCESS_KEY_ID", None)
-AWS_SECRET = os.environ.get("AWS_SECRET_ACCESS_KEY", None)
-
-# Redis configurations
-REDIS_HOST = os.environ.get("REDIS_HOST", "localhost")
-REDIS_PORT = os.environ.get("REDIS_PORT", 6379)
-
-# OOI Configurations
-BASE_URL = "https://ooinet.oceanobservatories.org"
-M2M_URL = "api/m2m"
-USERNAME = os.environ.get("OOI_USERNAME", None)
-TOKEN = os.environ.get("OOI_TOKEN", None)
-
-# File Systems Configurations
-FILE_SYSTEMS = {
-    "minio_s3": dict(
-        protocol="s3", client_kwargs={"endpoint_url": "http://minio:9000"}
-    ),
-    "aws_s3": dict(
-        protocol="s3",
-        skip_instance_cache=True,
-        use_listings_cache=False,
-        config_kwargs={"max_pool_connections": 1000},
-    ),
-}
-GOOGLE_SERVICE_JSON = os.environ.get(
-    "GOOGLE_SERVICE_JSON",
-    "",
-)
-DATA_BUCKET = 'ooi-data-prod'
-
-# Data sources
-METADATA_SOURCE = "s3://ooi-metadata-prod"

app/main.py

@@ -7,31 +7,24 @@
 from prometheus_fastapi_instrumentator import Instrumentator
 
 from api import metadata
-from core.config import (
-    CORS_ORIGINS,
-    CURRENT_API_VERSION,
-    DOCS_URL,
-    OPENAPI_URL,
-    SERVICE_DESCRIPTION,
-    SERVICE_ID,
-    SERVICE_NAME,
-)
+from core.config import settings
+
 from scripts import LoadMeta, load_instrument_catalog
 
-logger = logging.getLogger(f"{SERVICE_ID}-app")
+logger = logging.getLogger(f"{settings.SERVICE_ID}-app")
 
 app = FastAPI(
-    title=SERVICE_NAME,
-    openapi_url=OPENAPI_URL,
-    docs_url=DOCS_URL,
+    title=settings.SERVICE_NAME,
+    openapi_url=settings.OPENAPI_URL,
+    docs_url=settings.DOCS_URL,
     redoc_url=None,
-    version=CURRENT_API_VERSION,
-    description=SERVICE_DESCRIPTION,
+    version=settings.CURRENT_API_VERSION,
+    description=settings.SERVICE_DESCRIPTION,
 )
 
 app.add_middleware(
     CORSMiddleware,
-    allow_origins=CORS_ORIGINS,
+    allow_origins=settings.CORS_ORIGINS,
     # Regex for dev in netlify
     allow_origin_regex='https://.*cava-portal\.netlify\.app',
     allow_credentials=True,
@@ -42,7 +35,7 @@
 
 @app.get("/", include_in_schema=False)
 def home():
-    return RedirectResponse(url=f"/{SERVICE_ID}")
+    return RedirectResponse(url=f"/{settings.SERVICE_ID}")
 
 
 @app.on_event("startup")
@@ -52,7 +45,7 @@ def startup_event():
 
 
 app.include_router(
-    metadata.router, prefix=f"/{SERVICE_ID}", tags=[f"{SERVICE_ID}"]
+    metadata.router, prefix=f"/{settings.SERVICE_ID}", tags=[f"{settings.SERVICE_ID}"]
 )
 
 # Prometheus instrumentation

app/scripts/baseloader.py

@@ -2,15 +2,15 @@
 import threading
 import fsspec
 
-from core.config import FILE_SYSTEMS
+from core.config import settings
 
 
 class Loader:
     def __init__(self):
         self._in_progress = True
         self._name = "Loader"
         self._logger = logger
-        self._fs_kwargs = FILE_SYSTEMS["aws_s3"]
+        self._fs_kwargs = settings.FILE_SYSTEMS["aws_s3"]
         self._daemon = True
         self._fs = fsspec.filesystem(**self._fs_kwargs)
 

app/scripts/instrument_catalog.py

@@ -3,12 +3,12 @@
 import fsspec
 
 from store import META
-from core.config import METADATA_SOURCE
+from core.config import settings
 
 
 def load_instrument_catalog():
     fs = fsspec.filesystem("s3")
     with fs.open(
-        os.path.join(METADATA_SOURCE, "instruments_catalog.json")
+        os.path.join(settings.METADATA_SOURCE, "instruments_catalog.json")
     ) as f:
         META.update({"instruments_catalog": json.load(f)})

app/scripts/metaloader.py

@@ -8,20 +8,15 @@
 import gspread
 import pandas as pd
 
-from core.config import (
-    BASE_PATH,
-    BASE_URL,
-    GOOGLE_SERVICE_JSON,
-    M2M_URL,
-    DATA_BUCKET,
-)
+from core.config import settings
+
 from store import META
 from utils.conn import map_concurrency, send_request
 from scripts.baseloader import Loader
 
 
 def get_stream(stream):
-    url = f"{BASE_URL}/{M2M_URL}/12575/stream/byname/{stream}"
+    url = f"{settings.BASE_URL}/{settings.M2M_URL}/12575/stream/byname/{stream}"
     stream_dict = send_request(url)
     return {
         "stream_id": stream_dict["id"],
@@ -78,7 +73,7 @@ def parse_global_range_dataframe(global_ranges):
 def retrieve_deployments(refdes):
     dep_port = 12587
     reflist = list(split_refdes(refdes))
-    base_url_list = [BASE_URL, M2M_URL, str(dep_port), "events/deployment/inv"]
+    base_url_list = [settings.BASE_URL, settings.M2M_URL, str(dep_port), "events/deployment/inv"]
     dep_list = send_request("/".join(base_url_list + reflist))
     deployments = []
     if isinstance(dep_list, list):
@@ -118,12 +113,12 @@ def read_cava_assets(asset_xfile="CAVA_Assets.xlsx"):
 
 
 def get_toc():
-    url = f"{BASE_URL}/{M2M_URL}/12576/sensor/inv/toc"
+    url = f"{settings.BASE_URL}/{settings.M2M_URL}/12576/sensor/inv/toc"
     return send_request(url)
 
 
 def get_vocab():
-    url = f"{BASE_URL}/{M2M_URL}/12586/vocab"
+    url = f"{settings.BASE_URL}/{settings.M2M_URL}/12586/vocab"
     return send_request(url)
 
 
@@ -369,7 +364,7 @@ def _perform_refresh(self, metadata_cache):
         self._logger.info("Done pickling metadata .")
 
     def initialize_metadata(self):
-        meta_path = os.path.join(BASE_PATH, "core/meta")
+        meta_path = os.path.join(settings.BASE_PATH, "core/meta")
         metadata_cache = os.path.join(meta_path, "metadata.pkl")
         if os.path.exists(metadata_cache):
             with open(metadata_cache, "rb") as f:
@@ -410,6 +405,6 @@ def read_cava_assets(self):
 
     def fetch_creds(self):
         self._fs.get(
-            GOOGLE_SERVICE_JSON,
+            settings.GOOGLE_SERVICE_JSON,
             os.path.join(self._gspread_dir, "service_account.json"),
         )

app/utils/conn.py

@@ -3,7 +3,7 @@
 
 import requests
 
-from core.config import TOKEN, USERNAME, BASE_URL, M2M_URL
+from core.config import settings
 
 logger = logging.getLogger(__name__)
 
@@ -54,7 +54,7 @@ def send_request(url, params=None):
     """Send request to OOI. Username and Token already included."""
     try:
         prepped_request = requests.Request(
-            "GET", url, params=params, auth=(USERNAME, TOKEN)
+            "GET", url, params=params, auth=(settings.USERNAME, settings.TOKEN)
         ).prepare()
         r = fetch_url(prepped_request, session=SESSION)
         if isinstance(r, requests.Response):
@@ -73,7 +73,7 @@ def split_refdes(refdes):
 async def retrieve_deployments(refdes):
     dep_port = 12587
     reflist = list(split_refdes(refdes))
-    base_url_list = [BASE_URL, M2M_URL, str(dep_port), "events/deployment/inv"]
+    base_url_list = [settings.BASE_URL, settings.M2M_URL, str(dep_port), "events/deployment/inv"]
     dep_list = send_request("/".join(base_url_list + reflist))
     deployments = []
     if isinstance(dep_list, list):

environment.yaml

@@ -25,3 +25,5 @@ dependencies:
   - loguru
   - zarr
   - prometheus-fastapi-instrumentator
+  - pydantic > 2
+  - pydantic-settings