diff --git a/.idea/tileserver.iml b/.idea/tileserver.iml index d108e33..60f20ee 100644 --- a/.idea/tileserver.iml +++ b/.idea/tileserver.iml @@ -5,6 +5,7 @@ + diff --git a/Dockerfile b/Dockerfile index 3447e6e..8f2c1b6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ ENV PIP_DISABLE_PIP_VERSION_CHECK=1 POETRY_VIRTUALENVS_CREATE=false RUN python3 -m venv /poetry-env RUN /poetry-env/bin/pip install -U pip setuptools -RUN /poetry-env/bin/pip install poetry +RUN /poetry-env/bin/pip install poetry==1.8.4 WORKDIR /app/ diff --git a/macrostrat_tileserver/main.py b/macrostrat_tileserver/main.py index ebda3b3..7773d98 100644 --- a/macrostrat_tileserver/main.py +++ b/macrostrat_tileserver/main.py @@ -14,7 +14,6 @@ connect_to_db, register_table_catalog, ) -from timvt.settings import PostgresSettings from timvt.layer import FunctionRegistry from titiler.core.errors import DEFAULT_STATUS_CODES, add_exception_handlers from titiler.core.factory import TilerFactory @@ -30,6 +29,7 @@ from pathlib import Path from time import time +from .map_ingestion import register_map_ingestion_routes from typing import Any, Optional from buildpg import asyncpg @@ -100,9 +100,8 @@ async def startup_event(): # Apply fixtures # apply_fixtures(db_settings.database_url) - await register_table_catalog(app, schemas=["sources"]) + # await register_table_catalog(app, schemas=["sources"]) prepare_image_tile_subsystem() - print("Application started.") @app.on_event("startup") @@ -138,6 +137,10 @@ async def shutdown_event(): app.add_middleware(CompressionMiddleware, minimum_size=0) +# Map ingestion +register_map_ingestion_routes(app) + + MapnikLayerFactory(app) cog = TilerFactory() @@ -148,8 +151,8 @@ async def shutdown_event(): # Register endpoints. mvt_tiler = CachedVectorTilerFactory( - with_tables_metadata=True, - with_functions_metadata=True, # add Functions metadata endpoints (/functions.json, /{function_name}.json) + with_tables_metadata=False, + with_functions_metadata=False, # add Functions metadata endpoints (/functions.json, /{function_name}.json) with_viewer=False, ) diff --git a/macrostrat_tileserver/map_bounds/__init__.py b/macrostrat_tileserver/map_bounds/__init__.py index 5bcd922..352a692 100644 --- a/macrostrat_tileserver/map_bounds/__init__.py +++ b/macrostrat_tileserver/map_bounds/__init__.py @@ -17,23 +17,33 @@ async def rgeom( y: int, ): """Get a tile from the tileserver.""" - pool = request.app.state.pool + return await get_rgeom(request.app.state.pool, z=z, x=x, y=y) + +@router.get("/bounds/{slug}/{z}/{x}/{y}") +async def rgeom_slug( + request: Request, + slug: str, + z: int, + x: int, + y: int, +): + """Get a tile from the tileserver.""" + return await get_rgeom( + request.app.state.pool, where="slug = :slug", z=z, x=x, y=y, slug=slug + ) + + +async def get_rgeom(pool, *, where="is_finalized = true", **params): async with pool.acquire() as con: - data = await run_layer_query( - con, - "bounds", - z=z, - x=x, - y=y, - ) + data = await run_layer_query(con, "bounds", where=where, **params) kwargs = {} kwargs.setdefault("media_type", MimeTypes.pbf.value) return Response(data, **kwargs) -async def run_layer_query(con, layer_name, **params): - query = get_layer_sql(layer_name) +async def run_layer_query(con, layer_name, *, where="true", **params): + query = get_layer_sql(layer_name, where=where) q, p = render(query, layer_name=layer_name, **params) # Overcomes a shortcoming in buildpg that deems casting to an array as unsafe @@ -43,7 +53,7 @@ async def run_layer_query(con, layer_name, **params): return await con.fetchval(q, *p) -def get_layer_sql(layer: str): +def get_layer_sql(layer: str, *, where="true"): query = __here__ / "queries" / (layer + ".sql") q = query.read_text() q = q.strip() @@ -52,6 +62,7 @@ def get_layer_sql(layer: str): # Replace the envelope with the function call. Kind of awkward. q = q.replace(":envelope", "tile_utils.envelope(:x, :y, :z)") + q = q.replace(":where", where) # Wrap with MVT creation return f"WITH feature_query AS ({q}) SELECT ST_AsMVT(feature_query, :layer_name) FROM feature_query" diff --git a/macrostrat_tileserver/map_bounds/queries/bounds.sql b/macrostrat_tileserver/map_bounds/queries/bounds.sql index e839d3d..598ccc5 100644 --- a/macrostrat_tileserver/map_bounds/queries/bounds.sql +++ b/macrostrat_tileserver/map_bounds/queries/bounds.sql @@ -5,6 +5,7 @@ WITH tile AS ( ), sources AS ( SELECT source_id, + is_finalized, name, slug, scale, @@ -15,7 +16,7 @@ WITH tile AS ( FROM maps.sources, tile WHERE rgeom is NOT NULL - AND status_code = 'active' + AND :where AND ST_Intersects(rgeom, envelope_4326) ) SELECT * FROM sources z diff --git a/macrostrat_tileserver/map_ingestion/__init__.py b/macrostrat_tileserver/map_ingestion/__init__.py new file mode 100644 index 0000000..600add3 --- /dev/null +++ b/macrostrat_tileserver/map_ingestion/__init__.py @@ -0,0 +1,249 @@ +from pathlib import Path + +from buildpg import render, Renderer +from fastapi import APIRouter, Request, Response +from timvt.resources.enums import MimeTypes +from titiler.core.models.mapbox import TileJSON +from asyncpg import UndefinedTableError +from enum import Enum +from macrostrat.utils import get_logger +from macrostrat.database.utils import format as format_sql + +print_sql_statements = False + +router = APIRouter() +log = get_logger("uvicorn.error") + +__here__ = Path(__file__).parent + + +class FeatureType(str, Enum): + """Feature types.""" + + polygons = "polygons" + lines = "lines" + points = "points" + + +@router.get( + "/{slug}/tilejson.json", + response_model=TileJSON, + responses={200: {"description": "Return a tilejson"}}, + response_model_exclude_none=True, +) +async def tilejson( + request: Request, + slug: str, +): + """Return TileJSON document.""" + url_path = request.url_for( + "tile", **{"slug": slug, "z": "{z}", "x": "{x}", "y": "{y}"} + ) + + tile_endpoint = str(url_path) + + bounds_query = f""" + SELECT geom FROM sources.{slug}_polygons + UNION + SELECT geom FROM sources.{slug}_lines + UNION + SELECT geom FROM sources.{slug}_points + """ + + sql = get_bounds(bounds_query, geometry_column="geom") + pool = request.app.state.pool + async with pool.acquire() as con: + bounds = await con.fetchval(sql) + + return { + "minzoom": 0, + "maxzoom": 18, + "name": slug, + "bounds": bounds, + "tiles": [tile_endpoint], + } + + +@router.get("/{slug}/{z}/{x}/{y}") +async def tile( + request: Request, + slug: str, + z: int, + x: int, + y: int, +): + + # if feature_type != FeatureType.polygons: + # return Response(status_code=404, content="Only polygons are supported for now") + + """Get a tile from the tileserver.""" + pool = request.app.state.pool + + data = b"" + success = False + for layer in FeatureType: + try: + data += await get_layer(pool, slug, layer, z=z, x=x, y=y) + success = True + except UndefinedTableError: + pass + if not success: + return Response(status_code=404, content=f"No tables found for {slug}") + + kwargs = {} + kwargs.setdefault("media_type", MimeTypes.pbf.value) + return Response(data, **kwargs) + + +async def get_layer(pool, slug, layer: FeatureType, **params): + async with pool.acquire() as con: + table_name = f"{slug}_{layer}" + alias = "s" + column_dict = await get_table_columns(con, table_name, schema="sources") + log.debug("Columns: %s", column_dict) + columns = [ + format_column(k, v, cast_empty_strings=True, table_alias=alias) + for k, v in column_dict.items() + if k != "geom" + ] + columns.append("tile_layers.tile_geom(s.geom, :envelope) AS geometry") + + joins = None + if layer == FeatureType.polygons: + joins = [ + "LEFT JOIN macrostrat.intervals i0 ON s.b_interval = i0.id", + "LEFT JOIN macrostrat.intervals i1 ON s.t_interval = i1.id", + ] + + b_age = "i0.age_bottom" + t_age = "i1.age_top" + # Eventually we will allow b_age and t_age to be set directly + # b_age = "coalesce(s.b_age, i0.age_bottom)" + # t_age = "coalesce(s.t_age, i1.age_top)" + columns += [ + b_age + "::float AS b_age", + t_age + "::float AS t_age", + _color_subquery(b_age, t_age, "color"), + ] + + return await run_layer_query( + con, + f"sources.{table_name}", + columns, + joins=joins, + table_alias=alias, + layer_name=f"{layer}", + **params, + ) + + +string_data_types = [ + "character varying", + "text", +] + + +def format_column( + col, data_type, table_alias=None, cast_empty_strings=False, name=None +): + val = _wrap_with_quotes(col) + if name is None: + name = val + if table_alias is not None: + val = f"{table_alias}.{val}" + if cast_empty_strings and data_type in string_data_types: + val = f"NULLIF({val}, '')::text" + return f"{val} AS {name}" + + +def _color_subquery(b_age, t_age, alias): + return f"""( + SELECT interval_color + FROM macrostrat.intervals + WHERE age_top <= {t_age} AND age_bottom >= {b_age} + ORDER BY age_bottom - age_top + LIMIT 1 + ) AS {alias}""" + + +async def run_layer_query( + con, + table_name, + columns, + *, + joins=None, + layer_name="default", + table_alias=None, + **params, +): + _cols = ", ".join(columns) + query = f"SELECT {_cols} FROM {table_name}" + if table_alias: + query += f" AS {table_alias}" + + if joins: + query += "\n" + "\n".join(joins) + + query = extend_sql(query) + params = dict(layer_name=layer_name, **params) + + if print_sql_statements: + log.debug( + "Running query:\n%s\nParameters: %s", + format_sql(query, reindent=True), + params, + ) + + q, p = render(query, **params) + + return await con.fetchval(q, *p) + + +def _wrap_with_quotes(col): + if col[0] == '"' and col[-1] == '"': + col = col[1:-1] + if '"' in col: + col = col.replace('"', '""') + return '"' + col + '"' + + +def extend_sql(sql): + q = sql.strip() + if q.endswith(";"): + q = q[:-1] + + # Replace the envelope with the function call. Kind of awkward. + q = q.replace(":envelope", "tile_utils.envelope(:x, :y, :z)") + + # Wrap with MVT creation + return f"WITH feature_query AS ({q}) SELECT ST_AsMVT(feature_query, :layer_name, 4096, 'geometry') FROM feature_query" + + +def get_bounds(base_query, geometry_column="geometry"): + return f"""WITH b AS ( + SELECT ST_Union(a.{geometry_column}::box2d)::box2d env + FROM ({base_query}) a + ) + SELECT ARRAY[ST_XMin(env), ST_YMin(env), ST_XMax(env), ST_YMax(env)] + FROM b; + """ + + +async def get_table_columns(con, table, schema="sources"): + base_sql = f""" + SELECT column_name, data_type + FROM information_schema.columns + WHERE table_name = :table + AND table_schema = :schema; + """ + + q, p = render(base_sql, table=table, schema=schema) + res = await con.fetch(q, *p) + if len(res) == 0: + raise UndefinedTableError(f"Table {schema}.{table} not found") + + return {i[0]: i[1] for i in res} + + +def register_map_ingestion_routes(app): + app.include_router(router, tags=["Map ingestion"], prefix="/ingestion")