Merge pull request #36 from developmentseed/MosaicSearch

add mosaic search + other updates

.github/workflows/tests/test_raster.py

@@ -26,7 +26,7 @@ def test_mosaic_api():
     assert resp.headers["content-type"] == "application/json"
     assert resp.status_code == 200
     assert resp.json()["searchid"]
-    assert resp.json()["metadata"]
+    assert resp.json()["links"]
 
     searchid = resp.json()["searchid"]
 
@@ -53,6 +53,122 @@ def test_mosaic_api():
     assert "content-encoding" not in resp.headers
 
 
+def test_mosaic_search():
+    """test mosaic."""
+    # register some fake mosaic
+    searches = [
+        {
+            "filter": {"op": "=", "args": [{"property": "collection"}, "collection1"]},
+            "metadata": {"owner": "vincent"},
+        },
+        {
+            "filter": {"op": "=", "args": [{"property": "collection"}, "collection2"]},
+            "metadata": {"owner": "vincent"},
+        },
+        {
+            "filter": {"op": "=", "args": [{"property": "collection"}, "collection3"]},
+            "metadata": {"owner": "vincent"},
+        },
+        {
+            "filter": {"op": "=", "args": [{"property": "collection"}, "collection4"]},
+            "metadata": {"owner": "vincent"},
+        },
+        {
+            "filter": {"op": "=", "args": [{"property": "collection"}, "collection5"]},
+            "metadata": {"owner": "vincent"},
+        },
+        {
+            "filter": {"op": "=", "args": [{"property": "collection"}, "collection6"]},
+            "metadata": {"owner": "vincent"},
+        },
+        {
+            "filter": {"op": "=", "args": [{"property": "collection"}, "collection7"]},
+            "metadata": {"owner": "vincent"},
+        },
+        {
+            "filter": {"op": "=", "args": [{"property": "collection"}, "collection8"]},
+            "metadata": {"owner": "sean"},
+        },
+        {
+            "filter": {"op": "=", "args": [{"property": "collection"}, "collection9"]},
+            "metadata": {"owner": "sean"},
+        },
+        {
+            "filter": {"op": "=", "args": [{"property": "collection"}, "collection10"]},
+            "metadata": {"owner": "drew"},
+        },
+        {
+            "filter": {"op": "=", "args": [{"property": "collection"}, "collection11"]},
+            "metadata": {"owner": "drew"},
+        },
+        {
+            "filter": {"op": "=", "args": [{"property": "collection"}, "collection12"]},
+            "metadata": {"owner": "drew"},
+        },
+    ]
+    for search in searches:
+        resp = httpx.post(f"{raster_endpoint}/mosaic/register", json=search)
+        assert resp.status_code == 200
+        assert resp.json()["searchid"]
+
+    resp = httpx.get(f"{raster_endpoint}/mosaic/list")
+    assert resp.headers["content-type"] == "application/json"
+    assert resp.status_code == 200
+    assert (
+        resp.json()["context"]["matched"] > 10
+    )  # there should be at least 12 mosaic registered
+    assert resp.json()["context"]["returned"] == 10  # default limit is 10
+
+    # Make sure all mosaics returned have
+    for mosaic in resp.json()["searches"]:
+        assert mosaic["search"]["metadata"]["type"] == "mosaic"
+
+    links = resp.json()["links"]
+    assert len(links) == 2
+    assert links[0]["rel"] == "self"
+    assert links[1]["rel"] == "next"
+    assert links[1]["href"] == f"{raster_endpoint}/mosaic/list?limit=10&offset=10"
+
+    resp = httpx.get(f"{raster_endpoint}/mosaic/list", params={"limit": 1, "offset": 1})
+    assert resp.status_code == 200
+    assert resp.json()["context"]["matched"] > 10
+    assert resp.json()["context"]["limit"] == 1
+    assert resp.json()["context"]["returned"] == 1
+
+    links = resp.json()["links"]
+    assert len(links) == 3
+    assert links[0]["rel"] == "self"
+    assert links[0]["href"] == f"{raster_endpoint}/mosaic/list?limit=1&offset=1"
+    assert links[1]["rel"] == "next"
+    assert links[1]["href"] == f"{raster_endpoint}/mosaic/list?limit=1&offset=2"
+    assert links[2]["rel"] == "prev"
+    assert links[2]["href"] == f"{raster_endpoint}/mosaic/list?limit=1&offset=0"
+
+    # Filter on mosaic metadata
+    resp = httpx.get(f"{raster_endpoint}/mosaic/list", params={"owner": "vincent"})
+    assert resp.status_code == 200
+    assert resp.json()["context"]["matched"] == 7
+    assert resp.json()["context"]["limit"] == 10
+    assert resp.json()["context"]["returned"] == 7
+
+    # sortBy
+    resp = httpx.get(f"{raster_endpoint}/mosaic/list", params={"sortby": "lastused"})
+    assert resp.status_code == 200
+
+    resp = httpx.get(f"{raster_endpoint}/mosaic/list", params={"sortby": "usecount"})
+    assert resp.status_code == 200
+
+    resp = httpx.get(f"{raster_endpoint}/mosaic/list", params={"sortby": "-owner"})
+    assert resp.status_code == 200
+    assert (
+        "owner" not in resp.json()["searches"][0]["search"]["metadata"]
+    )  # some mosaic don't have owners
+
+    resp = httpx.get(f"{raster_endpoint}/mosaic/list", params={"sortby": "owner"})
+    assert resp.status_code == 200
+    assert "owner" in resp.json()["searches"][0]["search"]["metadata"]
+
+
 def test_stac_api():
     """test stac proxy."""
     item = {

Dockerfile.raster

@@ -4,11 +4,7 @@ FROM ghcr.io/vincentsarago/uvicorn-gunicorn:${PYTHON_VERSION}
 
 ENV CURL_CA_BUNDLE /etc/ssl/certs/ca-certificates.crt
 
-# Speed up dev cycle by pre-installing titiler
-RUN pip install \
-    titiler.core==0.4.* \
-    titiler.mosaic==0.4.* \
-    psycopg[binary,pool]
+RUN pip install psycopg[binary,pool]
 
 COPY src/eoapi/raster /tmp/raster
 RUN pip install /tmp/raster

Dockerfile.stac

@@ -4,13 +4,6 @@ FROM ghcr.io/vincentsarago/uvicorn-gunicorn:${PYTHON_VERSION}
 
 ENV CURL_CA_BUNDLE /etc/ssl/certs/ca-certificates.crt
 
-# Speed up dev cycle by pre-installing stac-fastapi
-RUN pip install \
-    stac-fastapi.api==2.3.* \
-    stac-fastapi.types==2.3.* \
-    stac-fastapi.extensions==2.3.* \
-    stac-fastapi.pgstac==2.3.*
-
 COPY src/eoapi/stac /tmp/stac
 RUN pip install /tmp/stac
 RUN rm -rf /tmp/stac

README.md

@@ -6,9 +6,9 @@
 </p>
 
 <p align="center">
-  <!-- <a href="https://github.com/developmentseed/eoAPI/actions?query=workflow%3ACI" target="_blank">
+  <a href="https://github.com/developmentseed/eoAPI/actions?query=workflow%3ACI" target="_blank">
       <img src="https://github.com/developmentseed/eoAPI/workflows/CI/badge.svg" alt="Test">
-  </a> -->
+  </a>
   <a href="https://github.com/developmentseed/eoAPI/blob/master/LICENSE" target="_blank">
       <img src="https://img.shields.io/github/license/developmentseed/titiler.svg" alt="Downloads">
   </a>
@@ -136,19 +136,19 @@ The stack is deployed by the [AWS CDK](https://aws.amazon.com/cdk/) utility. Und
 
 3. Update settings
 
-    Set environment variable or hard code in `deployment/.env` file (e.g `EOAPI_DB_PGSTAC_VERSION=0.4.3`).
+    Set environment variable or hard code in `deployment/.env` file (e.g `CDK_EOAPI_DB_PGSTAC_VERSION=0.4.3`).
 
     **Important**:
       - `EOAPI_DB_PGSTAC_VERSION` is a required env
-      - You can choose which functions to deploy by setting `EOAPI_FUNCTIONS` env (e.g `EOAPI_FUNCTIONS='["stac","raster","features"]'`)
+      - You can choose which functions to deploy by setting `CDK_EOAPI_FUNCTIONS` env (e.g `CDK_EOAPI_FUNCTIONS='["stac","raster","features"]'`)
 
 4. Deploy
 
     ```bash
-    $ EOAPI_STAGE=staging EOAPI_DB_PGSTAC_VERSION=0.4.3 npm run cdk deploy eoapi-staging --profile {my-aws-profile}
+    $ EOAPI_STAGE=staging CDK_EOAPI_DB_PGSTAC_VERSION=0.4.3 npm run cdk deploy eoapi-staging --profile {my-aws-profile}
 
     # Deploy in specific region
-    $ AWS_DEFAULT_REGION=eu-central-1 AWS_REGION=eu-central-1 EOAPI_DB_PGSTAC_VERSION=0.4.3 npm run cdk deploy eoapi-production --profile {my-aws-profile}
+    $ AWS_DEFAULT_REGION=eu-central-1 AWS_REGION=eu-central-1 CDK_EOAPI_DB_PGSTAC_VERSION=0.4.3 npm run cdk deploy eoapi-production --profile {my-aws-profile}
     ```
 
 

deployment/cdk/config.py

@@ -28,7 +28,7 @@ class Config:
         """model config"""
 
         env_file = "deployment/.env"
-        env_prefix = "EOAPI_"
+        env_prefix = "CDK_EOAPI_"
         use_enum_values = True
 
 
@@ -45,7 +45,7 @@ class Config:
         """model config"""
 
         env_file = "deployment/.env"
-        env_prefix = "EOAPI_DB_"
+        env_prefix = "CDK_EOAPI_DB_"
 
 
 class eoSTACSettings(pydantic.BaseSettings):
@@ -60,7 +60,7 @@ class Config:
         """model config"""
 
         env_file = "deployment/.env"
-        env_prefix = "EOAPI_STAC_"
+        env_prefix = "CDK_EOAPI_STAC_"
 
 
 class eoRasterSettings(pydantic.BaseSettings):
@@ -73,6 +73,7 @@ class eoRasterSettings(pydantic.BaseSettings):
         "CPL_VSIL_CURL_ALLOWED_EXTENSIONS": ".tif,.TIF,.tiff",
         "GDAL_CACHEMAX": "200",  # 200 mb
         "GDAL_DISABLE_READDIR_ON_OPEN": "EMPTY_DIR",
+        "GDAL_INGESTED_BYTES_AT_OPEN": "32768",
         "GDAL_HTTP_MERGE_CONSECUTIVE_RANGES": "YES",
         "GDAL_HTTP_MULTIPLEX": "YES",
         "GDAL_HTTP_VERSION": "2",
@@ -99,7 +100,7 @@ class Config:
         """model config"""
 
         env_file = "deployment/.env"
-        env_prefix = "EOAPI_RASTER_"
+        env_prefix = "CDK_EOAPI_RASTER_"
 
 
 class eoVectorSettings(pydantic.BaseSettings):
@@ -114,7 +115,7 @@ class Config:
         """model config"""
 
         env_file = "deployment/.env"
-        env_prefix = "EOAPI_VECTOR_"
+        env_prefix = "CDK_EOAPI_VECTOR_"
 
 
 class eoFeaturesSettings(pydantic.BaseSettings):
@@ -129,4 +130,4 @@ class Config:
         """model config"""
 
         env_file = "deployment/.env"
-        env_prefix = "EOAPI_FEATURES_"
+        env_prefix = "CDK_EOAPI_FEATURES_"

deployment/dockerfiles/Dockerfile.features

@@ -3,7 +3,7 @@ FROM lambci/lambda:build-python3.8
 WORKDIR /tmp
 
 COPY src/eoapi/features /tmp/features
-RUN pip install mangum /tmp/features -t /asset --no-binary pydantic
+RUN pip install mangum>=0.14,<0.15 /tmp/features -t /asset --no-binary pydantic
 RUN rm -rf /tmp/features
 
 # Reduce package size and remove useless files

deployment/dockerfiles/Dockerfile.raster

@@ -3,7 +3,7 @@ FROM lambci/lambda:build-python3.8
 WORKDIR /tmp
 
 COPY src/eoapi/raster /tmp/raster
-RUN pip install mangum /tmp/raster["psycopg-binary"] -t /asset --no-binary pydantic
+RUN pip install mangum>=0.14,<0.15 /tmp/raster["psycopg-binary"] -t /asset --no-binary pydantic
 RUN rm -rf /tmp/raster
 
 # Reduce package size and remove useless files

deployment/dockerfiles/Dockerfile.stac

@@ -3,7 +3,7 @@ FROM lambci/lambda:build-python3.8
 WORKDIR /tmp
 
 COPY src/eoapi/stac /tmp/stac
-RUN pip install mangum /tmp/stac -t /asset --no-binary pydantic
+RUN pip install mangum>=0.14,<0.15 /tmp/stac -t /asset --no-binary pydantic
 RUN rm -rf /tmp/stac
 
 # Reduce package size and remove useless files

deployment/dockerfiles/Dockerfile.vector

@@ -3,7 +3,7 @@ FROM lambci/lambda:build-python3.8
 WORKDIR /tmp
 
 COPY src/eoapi/vector /tmp/vector
-RUN pip install mangum /tmp/vector -t /asset --no-binary pydantic
+RUN pip install mangum>=0.14,<0.15 /tmp/vector -t /asset --no-binary pydantic
 RUN rm -rf /tmp/vector
 
 # Reduce package size and remove useless files

deployment/handlers/db_handler.py

@@ -190,15 +190,28 @@ def handler(event, context):
                 register_extensions(cursor=cur)
 
         dsn = "postgresql://{user}:{password}@{host}:{port}/{dbname}".format(
-            dbname=user_params.get("dbname", "postgres"),
+            dbname=user_params["dbname"],
             user=user_params["username"],
             password=user_params["password"],
             host=connection_params["host"],
             port=connection_params["port"],
         )
 
+        print("Running to PgSTAC migration...")
         asyncio.run(run_migration(dsn))
 
+        print("Adding mosaic index...")
+        with psycopg.connect(
+            dsn,
+            autocommit=True,
+            options="-c search_path=pgstac,public -c application_name=pgstac",
+        ) as conn:
+            conn.execute(
+                sql.SQL(
+                    "CREATE INDEX IF NOT EXISTS searches_mosaic ON searches ((true)) WHERE metadata->>'type'='mosaic';"
+                )
+            )
+
     except Exception as e:
         print(e)
         return send(event, context, "FAILED", {"message": str(e)})

docker-compose.yml

@@ -85,6 +85,8 @@ services:
       # AWS S3 endpoint config
       - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
       - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+      # API Config
+      - EOAPI_RASTER_ENABLE_MOSAIC_SEARCH=TRUE
     depends_on:
       - database
     command:
@@ -155,7 +157,7 @@ services:
   database:
     container_name: eoapi.db
     platform: linux/amd64
-    image: ghcr.io/stac-utils/pgstac:v0.4.3
+    image: ghcr.io/stac-utils/pgstac:v0.4.5
     environment:
       - POSTGRES_USER=username
       - POSTGRES_PASSWORD=password

src/eoapi/raster/eoapi/raster/app.py

@@ -10,11 +10,10 @@
 from titiler.core.resources.enums import OptionalHeader
 from titiler.mosaic.errors import MOSAIC_STATUS_CODES
 from titiler.pgstac.db import close_db_connection, connect_to_db
-from titiler.pgstac.factory import MosaicTilerFactory
 
 from eoapi.raster.config import ApiSettings
 from eoapi.raster.dependencies import DatasetPathParams
-from eoapi.raster.factory import MultiBaseTilerFactory
+from eoapi.raster.factory import MosaicTilerFactory, MultiBaseTilerFactory
 from eoapi.raster.reader import STACReader
 from eoapi.raster.version import __version__ as eoapi_raster_version
 
@@ -33,8 +32,12 @@
 add_exception_handlers(app, DEFAULT_STATUS_CODES)
 add_exception_handlers(app, MOSAIC_STATUS_CODES)
 
-# PgSTAC mosaic tiler
-mosaic = MosaicTilerFactory(router_prefix="mosaic", optional_headers=optional_headers)
+# Custom PgSTAC mosaic tiler
+mosaic = MosaicTilerFactory(
+    router_prefix="mosaic",
+    enable_mosaic_search=settings.enable_mosaic_search,
+    optional_headers=optional_headers,
+)
 app.include_router(mosaic.router, prefix="/mosaic", tags=["PgSTAC Mosaic"])
 
 # Custom STAC titiler endpoint (not added to the openapi docs)

src/eoapi/raster/eoapi/raster/config.py

@@ -13,6 +13,9 @@ class _ApiSettings(pydantic.BaseSettings):
     cachecontrol: str = "public, max-age=3600"
     debug: bool = False
 
+    # MosaicTiler settings
+    enable_mosaic_search: bool = False
+
     @pydantic.validator("cors_origins")
     def parse_cors_origin(cls, v):
         """Parse CORS origins."""