From f6810ddbf5b6a68c4f61c2ea4558e9b938174625 Mon Sep 17 00:00:00 2001 From: nsheff Date: Tue, 17 Oct 2023 17:13:13 -0400 Subject: [PATCH] clean up config handling --- bedhost/cli.py | 11 +---- bedhost/const.py | 2 +- bedhost/data_models.py | 14 ++++-- bedhost/dependencies.py | 14 ------ bedhost/helpers.py | 55 ++++++++++++++++++--- bedhost/main.py | 93 ++++++++++------------------------- bedhost/routers/base.py | 15 +++--- bedhost/routers/bed_api.py | 27 +++++----- bedhost/routers/bedset_api.py | 12 ++--- bedhost/routers/search_api.py | 7 +-- 10 files changed, 114 insertions(+), 136 deletions(-) diff --git a/bedhost/cli.py b/bedhost/cli.py index 1500ae25..50192fbf 100644 --- a/bedhost/cli.py +++ b/bedhost/cli.py @@ -1,5 +1,5 @@ -from bedhost import PKG_NAME -from bedhost._version import __version__ +from . import PKG_NAME +from ._version import __version__ from ubiquerg import VersionInHelpParser from yacman import select_config @@ -47,11 +47,4 @@ def add_subparser(cmd, description): f"the first available environment variable among: {', '.join(CFG_ENV_VARS)} will be used if set." f" Currently: {env_var_val}", ) - sps[cmd].add_argument( - "-d", - "--dbg", - action="store_true", - dest="debug", - help="Set logger verbosity to debug", - ) return parser diff --git a/bedhost/const.py b/bedhost/const.py index 0977e684..bb061416 100644 --- a/bedhost/const.py +++ b/bedhost/const.py @@ -12,7 +12,7 @@ CFG_SERVER_PORT_KEY, ) -from bedhost._version import __version__ as SERVER_VERSION +from ._version import __version__ as SERVER_VERSION PKG_NAME = "bedhost" LOG_FORMAT = "%(levelname)s in %(funcName)s: %(message)s" diff --git a/bedhost/data_models.py b/bedhost/data_models.py index 8630ead0..8d51d3bf 100644 --- a/bedhost/data_models.py +++ b/bedhost/data_models.py @@ -3,13 +3,14 @@ from pydantic import BaseModel from enum import Enum -from bedhost.const import CFG_REMOTE_KEY +from .const import CFG_REMOTE_KEY # from bedhost.main import bbc -from bedhost.dependencies import get_bbconf +# from bedhost.dependencies import get_bbconf -bbc = get_bbconf() +# bbc = get_bbconf() +# from .main import bbc class DBResponse(BaseModel): """ @@ -20,9 +21,14 @@ class DBResponse(BaseModel): data: Union[List[List], List[Dict], Tuple, Dict] +# RemoteClassEnum = Enum( +# "RemoteClassEnum", +# {r: r for r in bbc.config[CFG_REMOTE_KEY]} if bbc.is_remote else {"http": "http"}, +# ) + RemoteClassEnum = Enum( "RemoteClassEnum", - {r: r for r in bbc.config[CFG_REMOTE_KEY]} if bbc.is_remote else {"http": "http"}, + {"http": "http"}, ) BedsetDigest = Path( diff --git a/bedhost/dependencies.py b/bedhost/dependencies.py index 4db5f071..e69de29b 100644 --- a/bedhost/dependencies.py +++ b/bedhost/dependencies.py @@ -1,14 +0,0 @@ -from bedhost.helpers import BedHostConf -import os -from bedhost.exceptions import BedHostException - -bb_conf_file_path = os.environ.get("BEDBASE_CONFIG") or None - -try: - bedbase_conf = BedHostConf(bb_conf_file_path) -except Exception as e: - raise BedHostException(f"Bedbase config was not provided or is incorrect: {e}") - - -def get_bbconf() -> BedHostConf: - return bedbase_conf diff --git a/bedhost/helpers.py b/bedhost/helpers.py index a6f17f09..80e47f40 100644 --- a/bedhost/helpers.py +++ b/bedhost/helpers.py @@ -1,19 +1,20 @@ import os -from urllib import parse -from typing import List, Union from bbconf import BedBaseConf from starlette.responses import FileResponse, RedirectResponse +from typing import List, Union +from urllib import parse -from bedhost import _LOGGER - -from bedhost.const import ( +from . import _LOGGER +from .const import ( + CFG_PATH_KEY, + CFG_PATH_PIPELINE_OUTPUT_KEY, + CFG_REMOTE_KEY, TYPES_MAPPING, VALIDATIONS_MAPPING, OPERATORS_MAPPING, ) -from bedhost.exceptions import IncorrectSchemaException - +from .exceptions import BedHostException class BedHostConf(BedBaseConf): """ @@ -194,6 +195,46 @@ def get_openapi_version(app): return "3.0.2" +def attach_routers(app): + _LOGGER.info("Mounting routers...") + from .routers import bed_api, bedset_api, base, search_api + app.include_router(base.router) + app.include_router(bed_api.router) + app.include_router(bedset_api.router) + app.include_router(search_api.search_router) + return app + + +def configure(bbconf_file_path): + try: + # bbconf_file_path = os.environ.get("BEDBASE_CONFIG") or None + _LOGGER.info(f"Loading config...{bbconf_file_path}") + bbc = BedHostConf(bbconf_file_path) + except Exception as e: + raise BedHostException(f"Bedbase config was not provided or is incorrect: {e}") + + if not CFG_REMOTE_KEY in bbc.config: + _LOGGER.debug( + f"Using local files for serving: " + f"{bbc.config[CFG_PATH_KEY][CFG_PATH_PIPELINE_OUTPUT_KEY]}" + ) + app.mount( + bbc.get_bedstat_output_path(), + StaticFiles(directory=bbc.get_bedstat_output_path()), + name="bedfile", + ) + app.mount( + bbc.get_bedbuncher_output_path(), + StaticFiles(directory=bbc.get_bedbuncher_output_path()), + name="bedset", + ) + else: + _LOGGER.debug( + f"Using remote files for serving. Prefix: {bbc.config[CFG_REMOTE_KEY]['http']['prefix']}" + ) + return bbc + + # def get_id_map(bbc, table_name, file_type): # """ # Get a dict for avalible file/figure ids diff --git a/bedhost/main.py b/bedhost/main.py index f816787b..3e81a96f 100644 --- a/bedhost/main.py +++ b/bedhost/main.py @@ -1,16 +1,15 @@ +import coloredlogs import logging -import sys import os - -import coloredlogs +import sys import uvicorn -from fastapi import FastAPI -from fastapi.middleware.cors import CORSMiddleware -from fastapi.staticfiles import StaticFiles -from bedhost import _LOGGER -from bedhost.cli import build_parser -from bedhost.const import ( +from . import _LOGGER +import bedhost.dependencies as dependencies +from .helpers import FileResponse, configure, attach_routers + +from .cli import build_parser +from .const import ( CFG_PATH_KEY, CFG_PATH_PIPELINE_OUTPUT_KEY, CFG_REMOTE_KEY, @@ -21,30 +20,9 @@ STATIC_PATH, SERVER_VERSION, ) - -from bedhost.helpers import FileResponse - -from bedhost.dependencies import get_bbconf -from bedhost.routers import bed_api, bedset_api, base, search_api - - -_LOGGER_UVICORN = logging.getLogger("uvicorn.access") -coloredlogs.install( - logger=_LOGGER_UVICORN, - level=logging.INFO, - datefmt="%b %d %Y %H:%M:%S", - fmt="[%(levelname)s] [%(asctime)s] [BEDHOST] %(message)s", -) - - -_LOGGER_BEDHOST = logging.getLogger("bedhost") -coloredlogs.install( - logger=_LOGGER_BEDHOST, - level=logging.INFO, - datefmt="%b %d %Y %H:%M:%S", - fmt="[%(levelname)s] [%(asctime)s] [BEDHOST] %(message)s", -) - +from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware +from fastapi.staticfiles import StaticFiles app = FastAPI( title=PKG_NAME, @@ -58,10 +36,9 @@ "http://localhost:8000", "http://localhost:5173", "https://bedbase.org", - "*", + "*", # allow cross origin resource sharing, since this is a public API ] -# uncomment below for development, to allow cross origin resource sharing app.add_middleware( CORSMiddleware, allow_origins=origins, @@ -78,37 +55,12 @@ async def index(): """ return FileResponse(os.path.join(STATIC_PATH, "index.html")) - -def attach_routers(app): - _LOGGER.debug("Mounting routers") - - app.include_router(base.router) - app.include_router(bed_api.router) - app.include_router(bedset_api.router) - app.include_router(search_api.search_router) - - bbc = get_bbconf() - - if not CFG_REMOTE_KEY in bbc.config: - _LOGGER.debug( - f"Using local files for serving: " - f"{bbc.config[CFG_PATH_KEY][CFG_PATH_PIPELINE_OUTPUT_KEY]}" - ) - app.mount( - bbc.get_bedstat_output_path(), - StaticFiles(directory=bbc.get_bedstat_output_path()), - name="bedfile", - ) - app.mount( - bbc.get_bedbuncher_output_path(), - StaticFiles(directory=bbc.get_bedbuncher_output_path()), - name="bedset", - ) - else: - _LOGGER.debug( - f"Using remote files for serving. Prefix: {bbc.config[CFG_REMOTE_KEY]['http']['prefix']}" - ) - +@app.get("/test", response_model=int) +async def get_bedfile_count(): + """ + Returns the number of bedfiles available in the database + """ + return int(bbc['conf'].bed.record_count) def main(): parser = build_parser() @@ -119,9 +71,11 @@ def main(): sys.exit(1) if args.command == "serve": - attach_routers(app) _LOGGER.info(f"Running {PKG_NAME} app...") - bbc = get_bbconf() + bbconf_file_path = args.config or os.environ.get("BEDBASE_CONFIG") or None + global bbc + bbc = configure(bbconf_file_path) + attach_routers(app) uvicorn.run( app, host=bbc.config[CFG_SERVER_KEY][CFG_SERVER_HOST_KEY], @@ -131,6 +85,9 @@ def main(): if __name__ != "__main__": if os.environ.get("BEDBASE_CONFIG"): + bbconf_file_path = os.environ.get("BEDBASE_CONFIG") or None + global bbc + bbc = configure(bbconf_file_path) attach_routers(app) else: raise EnvironmentError( diff --git a/bedhost/routers/base.py b/bedhost/routers/base.py index 97eab688..1c8374c4 100644 --- a/bedhost/routers/base.py +++ b/bedhost/routers/base.py @@ -1,18 +1,17 @@ import os +from fastapi import APIRouter +from fastapi.responses import FileResponse from typing import Dict -from bedhost.const import ( + +from ..const import ( STATIC_PATH, ALL_VERSIONS, ) -from fastapi import APIRouter -from fastapi.responses import FileResponse -from bedhost.main import _LOGGER -from bedhost.helpers import get_openapi_version -from bedhost.dependencies import get_bbconf - -bbc = get_bbconf() +from ..helpers import get_openapi_version +from .. import _LOGGER +from ..main import bbc router = APIRouter(prefix="/api", tags=["base"]) diff --git a/bedhost/routers/bed_api.py b/bedhost/routers/bed_api.py index d176b476..a75f73f4 100644 --- a/bedhost/routers/bed_api.py +++ b/bedhost/routers/bed_api.py @@ -1,31 +1,34 @@ import subprocess import os -from typing import Annotated, Dict, Optional, List +try: + from typing import Annotated, Dict, Optional, List +except: + from typing_extensions import Annotated + from typing import Dict, Optional, List + import tempfile from fastapi import APIRouter, HTTPException, Query, Response, Path, Depends from fastapi.responses import PlainTextResponse, StreamingResponse from pipestat.exceptions import RecordNotFoundError -from bedhost.main import _LOGGER - -from bedhost.data_models import ( - DBResponse, - RemoteClassEnum, - BedDigest, - chromosome_number, -) -from bedhost.const import ( +from .. import _LOGGER +from ..main import bbc +from ..const import ( CFG_PATH_PIPELINE_OUTPUT_KEY, CFG_REMOTE_KEY, CFG_PATH_KEY, FIG_FORMAT, ) -from bedhost.dependencies import get_bbconf +from ..data_models import ( + DBResponse, + RemoteClassEnum, + BedDigest, + chromosome_number, +) router = APIRouter(prefix="/api/bed", tags=["bed"]) -bbc = get_bbconf() @router.get("/genomes") diff --git a/bedhost/routers/bedset_api.py b/bedhost/routers/bedset_api.py index 37a4005b..d32d4457 100644 --- a/bedhost/routers/bedset_api.py +++ b/bedhost/routers/bedset_api.py @@ -5,19 +5,15 @@ import os -from bedhost import _LOGGER -from bedhost.const import ( +from .. import _LOGGER +from ..const import ( CFG_REMOTE_KEY, CFG_PATH_KEY, CFG_PATH_PIPELINE_OUTPUT_KEY, FIG_FORMAT, ) - -# from bedhost.helpers import -from bedhost.data_models import DBResponse, RemoteClassEnum, BedsetDigest, BedList -from bedhost.dependencies import get_bbconf - -bbc = get_bbconf() +from ..data_models import DBResponse, RemoteClassEnum, BedsetDigest, BedList +from ..main import bbc router = APIRouter(prefix="/api/bedset", tags=["bedset"]) diff --git a/bedhost/routers/search_api.py b/bedhost/routers/search_api.py index 2b240c56..87c1c723 100644 --- a/bedhost/routers/search_api.py +++ b/bedhost/routers/search_api.py @@ -1,13 +1,10 @@ from fastapi import APIRouter -from bedhost.dependencies import get_bbconf -from bedhost import _LOGGER - -bbc = get_bbconf() +from .. import _LOGGER +from ..main import bbc search_router = APIRouter(prefix="/api/search", tags=["search"]) - @search_router.get("/bed/{query}") async def text_to_bed_search(query): _LOGGER.info(f"Searching for: {query}")