From 968d067959a4c433b41e617bc2516f31166f436d Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Fri, 4 Mar 2022 06:18:40 -0500 Subject: [PATCH 1/7] add mosaic search --- .github/workflows/tests/test_raster.py | 115 ++++++++++++-- Dockerfile.raster | 6 +- Dockerfile.stac | 7 - README.md | 8 +- deployment/cdk/config.py | 13 +- deployment/dockerfiles/Dockerfile.features | 2 +- deployment/dockerfiles/Dockerfile.raster | 2 +- deployment/dockerfiles/Dockerfile.stac | 2 +- deployment/dockerfiles/Dockerfile.vector | 2 +- deployment/handlers/db_handler.py | 15 +- docker-compose.yml | 4 +- src/eoapi/raster/eoapi/raster/app.py | 11 +- src/eoapi/raster/eoapi/raster/config.py | 3 + src/eoapi/raster/eoapi/raster/factory.py | 168 ++++++++++++++++++++- src/eoapi/raster/setup.py | 3 +- 15 files changed, 314 insertions(+), 47 deletions(-) diff --git a/.github/workflows/tests/test_raster.py b/.github/workflows/tests/test_raster.py index 5beba0d..8c90129 100644 --- a/.github/workflows/tests/test_raster.py +++ b/.github/workflows/tests/test_raster.py @@ -26,7 +26,7 @@ def test_mosaic_api(): assert resp.headers["content-type"] == "application/json" assert resp.status_code == 200 assert resp.json()["searchid"] - assert resp.json()["metadata"] + assert resp.json()["links"] searchid = resp.json()["searchid"] @@ -42,15 +42,112 @@ def test_mosaic_api(): assert list(resp.json()[0]) == ["id", "bbox", "assets"] assert resp.json()[0]["id"] == "20200307aC0853900w361030" - z, x, y = 15, 8589, 12849 - resp = httpx.get( - f"{raster_endpoint}/mosaic/tiles/{searchid}/{z}/{x}/{y}", - params={"assets": "cog"}, - headers={"Accept-Encoding": "br, gzip"}, - ) + # z, x, y = 15, 8589, 12849 + # resp = httpx.get( + # f"{raster_endpoint}/mosaic/tiles/{searchid}/{z}/{x}/{y}", + # params={"assets": "cog"}, + # headers={"Accept-Encoding": "br, gzip"}, + # ) + # assert resp.status_code == 200 + # assert resp.headers["content-type"] == "image/jpeg" + # assert "content-encoding" not in resp.headers + + +def test_mosaic_search(): + """test mosaic.""" + # register some fake mosaic + searches = [ + { + "filter": {"op": "=", "args": [{"property": "collection"}, "collection1"]}, + "metadata": {"owner": "vincent"}, + }, + { + "filter": {"op": "=", "args": [{"property": "collection"}, "collection2"]}, + "metadata": {"owner": "vincent"}, + }, + { + "filter": {"op": "=", "args": [{"property": "collection"}, "collection3"]}, + "metadata": {"owner": "vincent"}, + }, + { + "filter": {"op": "=", "args": [{"property": "collection"}, "collection4"]}, + "metadata": {"owner": "vincent"}, + }, + { + "filter": {"op": "=", "args": [{"property": "collection"}, "collection5"]}, + "metadata": {"owner": "vincent"}, + }, + { + "filter": {"op": "=", "args": [{"property": "collection"}, "collection6"]}, + "metadata": {"owner": "vincent"}, + }, + { + "filter": {"op": "=", "args": [{"property": "collection"}, "collection7"]}, + "metadata": {"owner": "vincent"}, + }, + { + "filter": {"op": "=", "args": [{"property": "collection"}, "collection8"]}, + "metadata": {"owner": "sean"}, + }, + { + "filter": {"op": "=", "args": [{"property": "collection"}, "collection9"]}, + "metadata": {"owner": "sean"}, + }, + { + "filter": {"op": "=", "args": [{"property": "collection"}, "collection10"]}, + "metadata": {"owner": "drew"}, + }, + { + "filter": {"op": "=", "args": [{"property": "collection"}, "collection11"]}, + "metadata": {"owner": "drew"}, + }, + { + "filter": {"op": "=", "args": [{"property": "collection"}, "collection12"]}, + "metadata": {"owner": "drew"}, + }, + ] + for search in searches: + resp = httpx.post(f"{raster_endpoint}/mosaic/register", json=search) + assert resp.status_code == 200 + assert resp.json()["searchid"] + + resp = httpx.get(f"{raster_endpoint}/mosaic/list") + assert resp.headers["content-type"] == "application/json" + assert resp.status_code == 200 + assert ( + resp.json()["numberMatched"] > 10 + ) # there should be at least 12 mosaic registered + assert resp.json()["numberReturned"] == 10 # default limit is 10 + + # Make sure all mosaics returned have + for mosaic in resp.json()["searches"]: + assert mosaic["search"]["metadata"]["type"] == "mosaic" + + links = resp.json()["links"] + assert len(links) == 2 + assert links[0]["rel"] == "self" + assert links[1]["rel"] == "next" + assert links[1]["href"] == f"{raster_endpoint}/mosaic/list?limit=10&offset=10" + + resp = httpx.get(f"{raster_endpoint}/mosaic/list", params={"limit": 1, "offset": 1}) + assert resp.status_code == 200 + assert resp.json()["numberMatched"] > 10 + assert resp.json()["numberReturned"] == 1 + + links = resp.json()["links"] + assert len(links) == 3 + assert links[0]["rel"] == "self" + assert links[0]["href"] == f"{raster_endpoint}/mosaic/list?limit=1&offset=1" + assert links[1]["rel"] == "next" + assert links[1]["href"] == f"{raster_endpoint}/mosaic/list?limit=1&offset=2" + assert links[2]["rel"] == "prev" + assert links[2]["href"] == f"{raster_endpoint}/mosaic/list?limit=1&offset=0" + + # Filter on mosaic metadata + resp = httpx.get(f"{raster_endpoint}/mosaic/list", params={"owner": "vincent"}) assert resp.status_code == 200 - assert resp.headers["content-type"] == "image/jpeg" - assert "content-encoding" not in resp.headers + assert resp.json()["numberMatched"] == 7 + assert resp.json()["numberReturned"] == 7 def test_stac_api(): diff --git a/Dockerfile.raster b/Dockerfile.raster index d727789..b58b4ce 100644 --- a/Dockerfile.raster +++ b/Dockerfile.raster @@ -4,11 +4,7 @@ FROM ghcr.io/vincentsarago/uvicorn-gunicorn:${PYTHON_VERSION} ENV CURL_CA_BUNDLE /etc/ssl/certs/ca-certificates.crt -# Speed up dev cycle by pre-installing titiler -RUN pip install \ - titiler.core==0.4.* \ - titiler.mosaic==0.4.* \ - psycopg[binary,pool] +RUN pip install psycopg[binary,pool] COPY src/eoapi/raster /tmp/raster RUN pip install /tmp/raster diff --git a/Dockerfile.stac b/Dockerfile.stac index 5a40ee8..aa73347 100644 --- a/Dockerfile.stac +++ b/Dockerfile.stac @@ -4,13 +4,6 @@ FROM ghcr.io/vincentsarago/uvicorn-gunicorn:${PYTHON_VERSION} ENV CURL_CA_BUNDLE /etc/ssl/certs/ca-certificates.crt -# Speed up dev cycle by pre-installing stac-fastapi -RUN pip install \ - stac-fastapi.api==2.3.* \ - stac-fastapi.types==2.3.* \ - stac-fastapi.extensions==2.3.* \ - stac-fastapi.pgstac==2.3.* - COPY src/eoapi/stac /tmp/stac RUN pip install /tmp/stac RUN rm -rf /tmp/stac diff --git a/README.md b/README.md index 8abd8c3..2411b19 100644 --- a/README.md +++ b/README.md @@ -136,19 +136,19 @@ The stack is deployed by the [AWS CDK](https://aws.amazon.com/cdk/) utility. Und 3. Update settings - Set environment variable or hard code in `deployment/.env` file (e.g `EOAPI_DB_PGSTAC_VERSION=0.4.3`). + Set environment variable or hard code in `deployment/.env` file (e.g `CDK_EOAPI_DB_PGSTAC_VERSION=0.4.3`). **Important**: - `EOAPI_DB_PGSTAC_VERSION` is a required env - - You can choose which functions to deploy by setting `EOAPI_FUNCTIONS` env (e.g `EOAPI_FUNCTIONS='["stac","raster","features"]'`) + - You can choose which functions to deploy by setting `CDK_EOAPI_FUNCTIONS` env (e.g `CDK_EOAPI_FUNCTIONS='["stac","raster","features"]'`) 4. Deploy ```bash - $ EOAPI_STAGE=staging EOAPI_DB_PGSTAC_VERSION=0.4.3 npm run cdk deploy eoapi-staging --profile {my-aws-profile} + $ EOAPI_STAGE=staging CDK_EOAPI_DB_PGSTAC_VERSION=0.4.3 npm run cdk deploy eoapi-staging --profile {my-aws-profile} # Deploy in specific region - $ AWS_DEFAULT_REGION=eu-central-1 AWS_REGION=eu-central-1 EOAPI_DB_PGSTAC_VERSION=0.4.3 npm run cdk deploy eoapi-production --profile {my-aws-profile} + $ AWS_DEFAULT_REGION=eu-central-1 AWS_REGION=eu-central-1 CDK_EOAPI_DB_PGSTAC_VERSION=0.4.3 npm run cdk deploy eoapi-production --profile {my-aws-profile} ``` diff --git a/deployment/cdk/config.py b/deployment/cdk/config.py index f6d26a4..f190f6f 100644 --- a/deployment/cdk/config.py +++ b/deployment/cdk/config.py @@ -28,7 +28,7 @@ class Config: """model config""" env_file = "deployment/.env" - env_prefix = "EOAPI_" + env_prefix = "CDK_EOAPI_" use_enum_values = True @@ -45,7 +45,7 @@ class Config: """model config""" env_file = "deployment/.env" - env_prefix = "EOAPI_DB_" + env_prefix = "CDK_EOAPI_DB_" class eoSTACSettings(pydantic.BaseSettings): @@ -60,7 +60,7 @@ class Config: """model config""" env_file = "deployment/.env" - env_prefix = "EOAPI_STAC_" + env_prefix = "CDK_EOAPI_STAC_" class eoRasterSettings(pydantic.BaseSettings): @@ -73,6 +73,7 @@ class eoRasterSettings(pydantic.BaseSettings): "CPL_VSIL_CURL_ALLOWED_EXTENSIONS": ".tif,.TIF,.tiff", "GDAL_CACHEMAX": "200", # 200 mb "GDAL_DISABLE_READDIR_ON_OPEN": "EMPTY_DIR", + "GDAL_INGESTED_BYTES_AT_OPEN": "32768", "GDAL_HTTP_MERGE_CONSECUTIVE_RANGES": "YES", "GDAL_HTTP_MULTIPLEX": "YES", "GDAL_HTTP_VERSION": "2", @@ -99,7 +100,7 @@ class Config: """model config""" env_file = "deployment/.env" - env_prefix = "EOAPI_RASTER_" + env_prefix = "CDK_EOAPI_RASTER_" class eoVectorSettings(pydantic.BaseSettings): @@ -114,7 +115,7 @@ class Config: """model config""" env_file = "deployment/.env" - env_prefix = "EOAPI_VECTOR_" + env_prefix = "CDK_EOAPI_VECTOR_" class eoFeaturesSettings(pydantic.BaseSettings): @@ -129,4 +130,4 @@ class Config: """model config""" env_file = "deployment/.env" - env_prefix = "EOAPI_FEATURES_" + env_prefix = "CDK_EOAPI_FEATURES_" diff --git a/deployment/dockerfiles/Dockerfile.features b/deployment/dockerfiles/Dockerfile.features index b9e850a..dc7371d 100644 --- a/deployment/dockerfiles/Dockerfile.features +++ b/deployment/dockerfiles/Dockerfile.features @@ -3,7 +3,7 @@ FROM lambci/lambda:build-python3.8 WORKDIR /tmp COPY src/eoapi/features /tmp/features -RUN pip install mangum /tmp/features -t /asset --no-binary pydantic +RUN pip install mangum>=0.14,<0.15 /tmp/features -t /asset --no-binary pydantic RUN rm -rf /tmp/features # Reduce package size and remove useless files diff --git a/deployment/dockerfiles/Dockerfile.raster b/deployment/dockerfiles/Dockerfile.raster index 7a05458..1f248a2 100644 --- a/deployment/dockerfiles/Dockerfile.raster +++ b/deployment/dockerfiles/Dockerfile.raster @@ -3,7 +3,7 @@ FROM lambci/lambda:build-python3.8 WORKDIR /tmp COPY src/eoapi/raster /tmp/raster -RUN pip install mangum /tmp/raster["psycopg-binary"] -t /asset --no-binary pydantic +RUN pip install mangum>=0.14,<0.15 /tmp/raster["psycopg-binary"] -t /asset --no-binary pydantic RUN rm -rf /tmp/raster # Reduce package size and remove useless files diff --git a/deployment/dockerfiles/Dockerfile.stac b/deployment/dockerfiles/Dockerfile.stac index d237af2..752a8d9 100644 --- a/deployment/dockerfiles/Dockerfile.stac +++ b/deployment/dockerfiles/Dockerfile.stac @@ -3,7 +3,7 @@ FROM lambci/lambda:build-python3.8 WORKDIR /tmp COPY src/eoapi/stac /tmp/stac -RUN pip install mangum /tmp/stac -t /asset --no-binary pydantic +RUN pip install mangum>=0.14,<0.15 /tmp/stac -t /asset --no-binary pydantic RUN rm -rf /tmp/stac # Reduce package size and remove useless files diff --git a/deployment/dockerfiles/Dockerfile.vector b/deployment/dockerfiles/Dockerfile.vector index 04560fc..1ba1e51 100644 --- a/deployment/dockerfiles/Dockerfile.vector +++ b/deployment/dockerfiles/Dockerfile.vector @@ -3,7 +3,7 @@ FROM lambci/lambda:build-python3.8 WORKDIR /tmp COPY src/eoapi/vector /tmp/vector -RUN pip install mangum /tmp/vector -t /asset --no-binary pydantic +RUN pip install mangum>=0.14,<0.15 /tmp/vector -t /asset --no-binary pydantic RUN rm -rf /tmp/vector # Reduce package size and remove useless files diff --git a/deployment/handlers/db_handler.py b/deployment/handlers/db_handler.py index 74411a3..7e69739 100644 --- a/deployment/handlers/db_handler.py +++ b/deployment/handlers/db_handler.py @@ -190,15 +190,28 @@ def handler(event, context): register_extensions(cursor=cur) dsn = "postgresql://{user}:{password}@{host}:{port}/{dbname}".format( - dbname=user_params.get("dbname", "postgres"), + dbname=user_params["dbname"], user=user_params["username"], password=user_params["password"], host=connection_params["host"], port=connection_params["port"], ) + print("Running to PgSTAC migration...") asyncio.run(run_migration(dsn)) + print("Adding mosaic index...") + with psycopg.connect( + dsn, + autocommit=True, + options="-c search_path=pgstac,public -c application_name=pgstac", + ) as conn: + conn.execute( + sql.SQL( + "CREATE INDEX IF NOT EXISTS searches_mosaic ON searches ((true)) WHERE metadata->>'type'='mosaic';" + ) + ) + except Exception as e: print(e) return send(event, context, "FAILED", {"message": str(e)}) diff --git a/docker-compose.yml b/docker-compose.yml index 8429967..7c670d2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -85,6 +85,8 @@ services: # AWS S3 endpoint config - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + # API Config + - EOAPI_RASTER_ENABLE_MOSAIC_SEARCH=TRUE depends_on: - database command: @@ -155,7 +157,7 @@ services: database: container_name: eoapi.db platform: linux/amd64 - image: ghcr.io/stac-utils/pgstac:v0.4.3 + image: ghcr.io/stac-utils/pgstac:v0.4.5 environment: - POSTGRES_USER=username - POSTGRES_PASSWORD=password diff --git a/src/eoapi/raster/eoapi/raster/app.py b/src/eoapi/raster/eoapi/raster/app.py index ec82347..407ea1e 100644 --- a/src/eoapi/raster/eoapi/raster/app.py +++ b/src/eoapi/raster/eoapi/raster/app.py @@ -10,11 +10,10 @@ from titiler.core.resources.enums import OptionalHeader from titiler.mosaic.errors import MOSAIC_STATUS_CODES from titiler.pgstac.db import close_db_connection, connect_to_db -from titiler.pgstac.factory import MosaicTilerFactory from eoapi.raster.config import ApiSettings from eoapi.raster.dependencies import DatasetPathParams -from eoapi.raster.factory import MultiBaseTilerFactory +from eoapi.raster.factory import MosaicTilerFactory, MultiBaseTilerFactory from eoapi.raster.reader import STACReader from eoapi.raster.version import __version__ as eoapi_raster_version @@ -33,8 +32,12 @@ add_exception_handlers(app, DEFAULT_STATUS_CODES) add_exception_handlers(app, MOSAIC_STATUS_CODES) -# PgSTAC mosaic tiler -mosaic = MosaicTilerFactory(router_prefix="mosaic", optional_headers=optional_headers) +# Custom PgSTAC mosaic tiler +mosaic = MosaicTilerFactory( + router_prefix="mosaic", + enable_mosaic_search=settings.enable_mosaic_search, + optional_headers=optional_headers, +) app.include_router(mosaic.router, prefix="/mosaic", tags=["PgSTAC Mosaic"]) # Custom STAC titiler endpoint (not added to the openapi docs) diff --git a/src/eoapi/raster/eoapi/raster/config.py b/src/eoapi/raster/eoapi/raster/config.py index 9eec63b..3672c0a 100644 --- a/src/eoapi/raster/eoapi/raster/config.py +++ b/src/eoapi/raster/eoapi/raster/config.py @@ -13,6 +13,9 @@ class _ApiSettings(pydantic.BaseSettings): cachecontrol: str = "public, max-age=3600" debug: bool = False + # MosaicTiler settings + enable_mosaic_search: bool = False + @pydantic.validator("cors_origins") def parse_cors_origin(cls, v): """Parse CORS origins.""" diff --git a/src/eoapi/raster/eoapi/raster/factory.py b/src/eoapi/raster/eoapi/raster/factory.py index 7107531..5d34e2f 100644 --- a/src/eoapi/raster/eoapi/raster/factory.py +++ b/src/eoapi/raster/eoapi/raster/factory.py @@ -1,13 +1,21 @@ """Custom MultiBaseTilerFactory.""" +import os from dataclasses import dataclass -from typing import Any +from typing import Any, List, Optional -from fastapi import Depends +from psycopg import sql +from psycopg.rows import class_row +from pydantic import BaseModel + +from fastapi import Depends, Query +from starlette.datastructures import QueryParams from starlette.requests import Request from starlette.responses import HTMLResponse from starlette.templating import Jinja2Templates -from titiler.core import factory +from titiler.core import factory as TitilerFactory +from titiler.pgstac import factory as TitilerPgSTACFactory +from titiler.pgstac import model try: from importlib.resources import files as resources_files # type: ignore @@ -21,7 +29,7 @@ @dataclass -class MultiBaseTilerFactory(factory.MultiBaseTilerFactory): +class MultiBaseTilerFactory(TitilerFactory.MultiBaseTilerFactory): """Custom endpoints factory.""" def register_routes(self) -> None: @@ -46,3 +54,155 @@ def stac_demo( }, media_type="text/html", ) + + +class Infos(BaseModel): + """Response model for /list endpoint.""" + + searches: List[model.Info] + links: Optional[List[model.Link]] + numberMatched: Optional[int] + numberReturned: Optional[int] + + +@dataclass +class MosaicTilerFactory(TitilerPgSTACFactory.MosaicTilerFactory): + """Custom endpoints factory.""" + + enable_mosaic_search: bool = False + + def register_routes(self) -> None: + """This Method register routes to the router.""" + super().register_routes() + if self.enable_mosaic_search: + self._mosaic_search() + + def _mosaic_search(self) -> None: + """register mosaic search route.""" + + @self.router.get( + "/list", + responses={200: {"description": "List Mosaics in PgSTAC."}}, + response_model=Infos, + response_model_exclude_none=True, + ) + def list_mosaic( + request: Request, + limit: int = Query( + 10, + ge=1, + le=int(os.environ.get("EOAPI_RASTER_MAX_MOSAIC", "10000")), + description="Page size limit", + ), + offset: int = Query( + 0, + ge=0, + description="Page offset", + ), + ): + """List a Search query.""" + offset_and_limit = [ + sql.SQL("LIMIT {number}").format(number=sql.Literal(limit)), + sql.SQL("OFFSET {start}").format(start=sql.Literal(offset)), + ] + + # filter to only return `metadata->type == 'mosaic'` + mosaic_filter = sql.SQL("metadata::json->>{key} = {value}").format( + key=sql.Literal("type"), value=sql.Literal("mosaic") + ) + + # additional metadata property filter + # =val - filter for a metadata property. Multiple property filters are ANDed together. + qs_key_to_remove = ["limit", "offset", "properties", "sortby"] + additional_filter = [ + sql.SQL("metadata::json->>{key} = {value}").format( + key=sql.Literal(key), value=sql.Literal(value) + ) + for (key, value) in request.query_params.items() + if key.lower() not in qs_key_to_remove + ] + filters = [ + sql.SQL("WHERE"), + sql.SQL("AND ").join([mosaic_filter, *additional_filter]), + ] + + # TODO: enable SortBy + with request.app.state.dbpool.connection() as conn: + with conn.cursor() as cursor: + # Get Total Number of searches rows + query = [ + sql.SQL("SELECT count(*) FROM searches"), + *filters, + ] + cursor.execute(sql.SQL(" ").join(query)) + nb_items = cursor.fetchone()[0] + + # Get rows + cursor.row_factory = class_row(model.Search) + query = [ + sql.SQL("SELECT * FROM searches"), + *filters, + *offset_and_limit, + ] + + cursor.execute(sql.SQL(" ").join(query)) + + searches_info = cursor.fetchall() + + qs = QueryParams({**request.query_params, "limit": limit, "offset": offset}) + links = [ + model.Link( + rel="self", + href=self.url_for(request, "list_mosaic") + f"?{qs}", + ), + ] + + if len(searches_info) < int(nb_items): + next_token = offset + len(searches_info) + qs = QueryParams( + {**request.query_params, "limit": limit, "offset": next_token} + ) + links.append( + model.Link( + rel="next", + href=self.url_for(request, "list_mosaic") + f"?{qs}", + ), + ) + + if offset > 0: + prev_token = offset - limit if (offset - limit) > 0 else 0 + qs = QueryParams( + {**request.query_params, "limit": limit, "offset": prev_token} + ) + links.append( + model.Link( + rel="prev", + href=self.url_for(request, "list_mosaic") + f"?{qs}", + ), + ) + + return Infos( + searches=[ + model.Info( + search=search, + links=[ + model.Link( + rel="metadata", + href=self.url_for( + request, "info_search", searchid=search.id + ), + ), + model.Link( + rel="tilejson", + href=self.url_for( + request, "tilejson", searchid=search.id + ), + ), + ], + ) + for search in searches_info + ], + links=links, + numberMatched=int(nb_items), + numberReturned=len(searches_info), + ) diff --git a/src/eoapi/raster/setup.py b/src/eoapi/raster/setup.py index 1a9d8f8..a242e9f 100644 --- a/src/eoapi/raster/setup.py +++ b/src/eoapi/raster/setup.py @@ -6,8 +6,7 @@ long_description = f.read() inst_reqs = [ - "titiler.pgstac==0.1.0a3", - "jinja2>=3.0,<4.0", + "titiler.pgstac==0.1.0.a5", "starlette-cramjam>=0.1.0,<0.2", "importlib_resources>=1.1.0;python_version<'3.9'", ] From 558724cbe528a4fc7a4c14eed0385df222ab0307 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Mon, 7 Mar 2022 12:05:11 +0100 Subject: [PATCH 2/7] do not cast to json --- src/eoapi/raster/eoapi/raster/factory.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/eoapi/raster/eoapi/raster/factory.py b/src/eoapi/raster/eoapi/raster/factory.py index 5d34e2f..0338c2b 100644 --- a/src/eoapi/raster/eoapi/raster/factory.py +++ b/src/eoapi/raster/eoapi/raster/factory.py @@ -107,7 +107,7 @@ def list_mosaic( ] # filter to only return `metadata->type == 'mosaic'` - mosaic_filter = sql.SQL("metadata::json->>{key} = {value}").format( + mosaic_filter = sql.SQL("metadata->>{key} = {value}").format( key=sql.Literal("type"), value=sql.Literal("mosaic") ) @@ -115,7 +115,7 @@ def list_mosaic( # =val - filter for a metadata property. Multiple property filters are ANDed together. qs_key_to_remove = ["limit", "offset", "properties", "sortby"] additional_filter = [ - sql.SQL("metadata::json->>{key} = {value}").format( + sql.SQL("metadata->>{key} = {value}").format( key=sql.Literal(key), value=sql.Literal(value) ) for (key, value) in request.query_params.items() From a88a488199a1a382d5d3805a77ca856a1fb503dc Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Mon, 7 Mar 2022 12:06:28 +0100 Subject: [PATCH 3/7] uncomment --- .github/workflows/tests/test_raster.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/tests/test_raster.py b/.github/workflows/tests/test_raster.py index 8c90129..54d489c 100644 --- a/.github/workflows/tests/test_raster.py +++ b/.github/workflows/tests/test_raster.py @@ -42,15 +42,15 @@ def test_mosaic_api(): assert list(resp.json()[0]) == ["id", "bbox", "assets"] assert resp.json()[0]["id"] == "20200307aC0853900w361030" - # z, x, y = 15, 8589, 12849 - # resp = httpx.get( - # f"{raster_endpoint}/mosaic/tiles/{searchid}/{z}/{x}/{y}", - # params={"assets": "cog"}, - # headers={"Accept-Encoding": "br, gzip"}, - # ) - # assert resp.status_code == 200 - # assert resp.headers["content-type"] == "image/jpeg" - # assert "content-encoding" not in resp.headers + z, x, y = 15, 8589, 12849 + resp = httpx.get( + f"{raster_endpoint}/mosaic/tiles/{searchid}/{z}/{x}/{y}", + params={"assets": "cog"}, + headers={"Accept-Encoding": "br, gzip"}, + ) + assert resp.status_code == 200 + assert resp.headers["content-type"] == "image/jpeg" + assert "content-encoding" not in resp.headers def test_mosaic_search(): From 2fa23a386d510a4929acbc384d423c584a06ae42 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Thu, 10 Mar 2022 13:41:08 +0100 Subject: [PATCH 4/7] handle sortBy --- .github/workflows/tests/test_raster.py | 17 ++++++++++ src/eoapi/raster/eoapi/raster/factory.py | 42 ++++++++++++++++++++++-- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests/test_raster.py b/.github/workflows/tests/test_raster.py index 54d489c..9c43f38 100644 --- a/.github/workflows/tests/test_raster.py +++ b/.github/workflows/tests/test_raster.py @@ -149,6 +149,23 @@ def test_mosaic_search(): assert resp.json()["numberMatched"] == 7 assert resp.json()["numberReturned"] == 7 + # sortBy + resp = httpx.get(f"{raster_endpoint}/mosaic/list", params={"sortby": "lastused"}) + assert resp.status_code == 200 + + resp = httpx.get(f"{raster_endpoint}/mosaic/list", params={"sortby": "usecount"}) + assert resp.status_code == 200 + + resp = httpx.get(f"{raster_endpoint}/mosaic/list", params={"sortby": "-owner"}) + assert resp.status_code == 200 + assert ( + "owner" not in resp.json()["searches"][0]["search"]["metadata"] + ) # some mosaic don't have owners + + resp = httpx.get(f"{raster_endpoint}/mosaic/list", params={"sortby": "owner"}) + assert resp.status_code == 200 + assert "owner" not in resp.json()["searches"][0]["search"]["metadata"] + def test_stac_api(): """test stac proxy.""" diff --git a/src/eoapi/raster/eoapi/raster/factory.py b/src/eoapi/raster/eoapi/raster/factory.py index 0338c2b..2874194 100644 --- a/src/eoapi/raster/eoapi/raster/factory.py +++ b/src/eoapi/raster/eoapi/raster/factory.py @@ -1,8 +1,9 @@ """Custom MultiBaseTilerFactory.""" import os +import re from dataclasses import dataclass -from typing import Any, List, Optional +from typing import Any, Generator, List, Optional from psycopg import sql from psycopg.rows import class_row @@ -99,6 +100,10 @@ def list_mosaic( ge=0, description="Page offset", ), + sortby: Optional[str] = Query( + None, + description="Sort the response items by a property (ascending (default) or descending).", + ), ): """List a Search query.""" offset_and_limit = [ @@ -126,7 +131,39 @@ def list_mosaic( sql.SQL("AND ").join([mosaic_filter, *additional_filter]), ] - # TODO: enable SortBy + def parse_sort_by(sortby: str) -> Generator[sql.Composable, None, None]: + """Parse SortBy expression.""" + for s in sortby.split(","): + parts = re.match( + "^(?P[+-]?)(?P.*)$", s + ).groupdict() # type:ignore + prop = parts["prop"] + if parts["prop"] in ["lastused", "usecount"]: + prop = sql.Identifier(parts["prop"]) + else: + prop = sql.SQL("metadata->>{}").format( + sql.Literal(parts["prop"]) + ) + + if parts["dir"] == "-": + order = sql.SQL("{} DESC").format(prop) + else: + order = sql.SQL("{} ASC").format(prop) + + yield order + + # sortby=[+|-]PROP - sort the response items by a property (ascending (default) or descending). + order_by = [] + if sortby: + sort_expr = list(parse_sort_by(sortby)) + + print(sort_expr) + if sort_expr: + order_by = [ + sql.SQL("ORDER BY"), + sql.SQL(", ").join(sort_expr), + ] + with request.app.state.dbpool.connection() as conn: with conn.cursor() as cursor: # Get Total Number of searches rows @@ -142,6 +179,7 @@ def list_mosaic( query = [ sql.SQL("SELECT * FROM searches"), *filters, + *order_by, *offset_and_limit, ] From 5d5b96f0a3d1884acbe97fc42fcd4a4af8d0c194 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Thu, 10 Mar 2022 13:46:15 +0100 Subject: [PATCH 5/7] fix test --- .github/workflows/tests/test_raster.py | 2 +- README.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests/test_raster.py b/.github/workflows/tests/test_raster.py index 9c43f38..268f0b9 100644 --- a/.github/workflows/tests/test_raster.py +++ b/.github/workflows/tests/test_raster.py @@ -164,7 +164,7 @@ def test_mosaic_search(): resp = httpx.get(f"{raster_endpoint}/mosaic/list", params={"sortby": "owner"}) assert resp.status_code == 200 - assert "owner" not in resp.json()["searches"][0]["search"]["metadata"] + assert "owner" in resp.json()["searches"][0]["search"]["metadata"] def test_stac_api(): diff --git a/README.md b/README.md index 2411b19..5f3ae55 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,9 @@

- + Downloads From 3795019026d7fb92683cb06b62e5173d56c7f4fe Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Thu, 10 Mar 2022 18:59:02 +0100 Subject: [PATCH 6/7] remove literal for limit/offset --- .github/workflows/tests/test_raster.py | 12 +++--- src/eoapi/raster/eoapi/raster/factory.py | 55 +++++++++++++++--------- 2 files changed, 41 insertions(+), 26 deletions(-) diff --git a/.github/workflows/tests/test_raster.py b/.github/workflows/tests/test_raster.py index 268f0b9..3ae0470 100644 --- a/.github/workflows/tests/test_raster.py +++ b/.github/workflows/tests/test_raster.py @@ -117,7 +117,7 @@ def test_mosaic_search(): assert ( resp.json()["numberMatched"] > 10 ) # there should be at least 12 mosaic registered - assert resp.json()["numberReturned"] == 10 # default limit is 10 + assert resp.json()["context"]["returned"] == 10 # default limit is 10 # Make sure all mosaics returned have for mosaic in resp.json()["searches"]: @@ -131,8 +131,9 @@ def test_mosaic_search(): resp = httpx.get(f"{raster_endpoint}/mosaic/list", params={"limit": 1, "offset": 1}) assert resp.status_code == 200 - assert resp.json()["numberMatched"] > 10 - assert resp.json()["numberReturned"] == 1 + assert resp.json()["context"]["matched"] > 10 + assert resp.json()["context"]["limit"] == 1 + assert resp.json()["context"]["returned"] == 1 links = resp.json()["links"] assert len(links) == 3 @@ -146,8 +147,9 @@ def test_mosaic_search(): # Filter on mosaic metadata resp = httpx.get(f"{raster_endpoint}/mosaic/list", params={"owner": "vincent"}) assert resp.status_code == 200 - assert resp.json()["numberMatched"] == 7 - assert resp.json()["numberReturned"] == 7 + assert resp.json()["context"]["matched"] == 7 + assert resp.json()["context"]["limit"] == 10 + assert resp.json()["context"]["returned"] == 7 # sortBy resp = httpx.get(f"{raster_endpoint}/mosaic/list", params={"sortby": "lastused"}) diff --git a/src/eoapi/raster/eoapi/raster/factory.py b/src/eoapi/raster/eoapi/raster/factory.py index 2874194..1621652 100644 --- a/src/eoapi/raster/eoapi/raster/factory.py +++ b/src/eoapi/raster/eoapi/raster/factory.py @@ -7,7 +7,7 @@ from psycopg import sql from psycopg.rows import class_row -from pydantic import BaseModel +from pydantic import BaseModel, validator from fastapi import Depends, Query from starlette.datastructures import QueryParams @@ -57,13 +57,29 @@ def stac_demo( ) +class Context(BaseModel): + """Context Model.""" + + returned: int + limit: Optional[int] + matched: Optional[int] + + @validator("limit") + def validate_limit(cls, v, values): + """validate limit.""" + if values["returned"] > v: + raise ValueError( + "Number of returned items must be less than or equal to the limit" + ) + return v + + class Infos(BaseModel): """Response model for /list endpoint.""" searches: List[model.Info] links: Optional[List[model.Link]] - numberMatched: Optional[int] - numberReturned: Optional[int] + context: Context @dataclass @@ -106,19 +122,12 @@ def list_mosaic( ), ): """List a Search query.""" - offset_and_limit = [ - sql.SQL("LIMIT {number}").format(number=sql.Literal(limit)), - sql.SQL("OFFSET {start}").format(start=sql.Literal(offset)), - ] + # Default filter to only return `metadata->type == 'mosaic'` + mosaic_filter = sql.SQL("metadata->>'type' = 'mosaic'") - # filter to only return `metadata->type == 'mosaic'` - mosaic_filter = sql.SQL("metadata->>{key} = {value}").format( - key=sql.Literal("type"), value=sql.Literal("mosaic") - ) - - # additional metadata property filter + # additional metadata property filter passed in query-parameters # =val - filter for a metadata property. Multiple property filters are ANDed together. - qs_key_to_remove = ["limit", "offset", "properties", "sortby"] + qs_key_to_remove = ["limit", "offset", "sortby"] additional_filter = [ sql.SQL("metadata->>{key} = {value}").format( key=sql.Literal(key), value=sql.Literal(value) @@ -172,7 +181,7 @@ def parse_sort_by(sortby: str) -> Generator[sql.Composable, None, None]: *filters, ] cursor.execute(sql.SQL(" ").join(query)) - nb_items = cursor.fetchone()[0] + nb_items = int(cursor.fetchone()[0]) # Get rows cursor.row_factory = class_row(model.Search) @@ -180,10 +189,11 @@ def parse_sort_by(sortby: str) -> Generator[sql.Composable, None, None]: sql.SQL("SELECT * FROM searches"), *filters, *order_by, - *offset_and_limit, + sql.SQL("LIMIT %(limit)s OFFSET %(offset)s"), ] - - cursor.execute(sql.SQL(" ").join(query)) + cursor.execute( + sql.SQL(" ").join(query), {"limit": limit, "offset": offset} + ) searches_info = cursor.fetchall() @@ -195,7 +205,7 @@ def parse_sort_by(sortby: str) -> Generator[sql.Composable, None, None]: ), ] - if len(searches_info) < int(nb_items): + if len(searches_info) < nb_items: next_token = offset + len(searches_info) qs = QueryParams( {**request.query_params, "limit": limit, "offset": next_token} @@ -241,6 +251,9 @@ def parse_sort_by(sortby: str) -> Generator[sql.Composable, None, None]: for search in searches_info ], links=links, - numberMatched=int(nb_items), - numberReturned=len(searches_info), + context=Context( + returned=len(searches_info), + matched=nb_items, + limit=limit, + ), ) From 07dd7c7404f9e51da982a0a4c949e8d98b41f97f Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Thu, 10 Mar 2022 19:11:09 +0100 Subject: [PATCH 7/7] fix test --- .github/workflows/tests/test_raster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests/test_raster.py b/.github/workflows/tests/test_raster.py index 3ae0470..80dc8d2 100644 --- a/.github/workflows/tests/test_raster.py +++ b/.github/workflows/tests/test_raster.py @@ -115,7 +115,7 @@ def test_mosaic_search(): assert resp.headers["content-type"] == "application/json" assert resp.status_code == 200 assert ( - resp.json()["numberMatched"] > 10 + resp.json()["context"]["matched"] > 10 ) # there should be at least 12 mosaic registered assert resp.json()["context"]["returned"] == 10 # default limit is 10