Merge pull request #48 from UW-Macrostrat/modernize-build

Streamline Docker build

.github/workflows/build-image.yaml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: "recursive"
       - name: Docker meta
@@ -28,6 +28,7 @@ jobs:
             type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-')}}
             type=raw,value=latest-itb-{{date 'YYYYMMDDHHmmss'}}
             type=ref,event=pr,suffix=-{{date 'YYYYMMDDHHmmss'}}
+            type=ref,event=branch
             type=ref,event=branch,suffix=-{{date 'YYYYMMDDHHmmss'}}
             type=ref,event=tag,suffix=-{{date 'YYYYMMDDHHmmss'}}
             type=semver,pattern={{version}}

.mise.toml

@@ -0,0 +1,3 @@
+[tools]
+python = "3.10"
+node = "20"

Dockerfile

@@ -1,51 +1,20 @@
 # From here:
 # https://stackoverflow.com/questions/53835198/integrating-python-poetry-with-docker
 # NOTE: we might want to make things a bit nicer here
-FROM python:3.9@sha256:c0dcc146710fed0a6d62cb55b92f00bfbfc3b931fff6218f4958bab58333c37b
+FROM python:3.10
 
-# MAPNIK
-# Install mapnik for compiling legacy image tiles
 RUN apt-get update -y && \
   apt-get install -y --no-install-recommends \
-  build-essential software-properties-common curl \
-  libboost-dev libboost-filesystem-dev libboost-program-options-dev libboost-python-dev \
-  libboost-regex-dev libboost-system-dev libboost-thread-dev libicu-dev libtiff5-dev \
-  libfreetype-dev libpng-dev libxml2-dev libproj-dev libcairo-dev \
-  postgresql-contrib libharfbuzz-dev python-dev && \
+  libgdal-dev libproj-dev libgeos-dev && \
   rm -rf /var/lib/apt/lists/*
 
-# Mapnik
-ARG MAPNIK_VERSION=3.1.0
-RUN curl -L -s https://github.com/mapnik/mapnik/releases/download/v${MAPNIK_VERSION}/mapnik-v${MAPNIK_VERSION}.tar.bz2 | tar -xj -C /tmp/
-RUN cd /tmp/mapnik-v${MAPNIK_VERSION} && python scons/scons.py configure
-RUN cd /tmp/mapnik-v${MAPNIK_VERSION} && make JOBS=4 && make install JOBS=4
-
-ENV BOOST_PYTHON_LIB=boost_python39
-# Python bindings to mapnik
-ARG PYTHON_MAPNIK_COMMIT=7da019cf9eb12af8f8aa88b7d75789dfcd1e901b
-RUN mkdir -p /opt/python-mapnik && curl -L https://github.com/mapnik/python-mapnik/archive/${PYTHON_MAPNIK_COMMIT}.tar.gz | tar xz -C /opt/python-mapnik --strip-components=1
-RUN cd /opt/python-mapnik && python3 setup.py install && rm -r /opt/python-mapnik/build
-
-# Remove build dependencies
-RUN apt-get remove -y \
-  build-essential software-properties-common \
-  libboost-dev libboost-filesystem-dev libboost-program-options-dev libboost-python-dev \
-  libboost-regex-dev libboost-system-dev libboost-thread-dev libicu-dev libtiff5-dev \
-  libfreetype-dev libpng-dev libxml2-dev libproj-dev libcairo-dev libharfbuzz-dev python-dev
-
-# CartoCSS stylesheet generation
-# Install nodejs
-RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - && \
-  apt-get install -y nodejs && \
-  rm -rf /var/lib/apt/lists/*
-
-# Install carto
-RUN npm install -g carto
-
 # The rest of this (for vector tile generation and the server itself) should be easier.
 ENV PIP_DISABLE_PIP_VERSION_CHECK=1 POETRY_VIRTUALENVS_CREATE=false
 
-RUN pip install "pip==23.2.1" && pip install "setuptools==68.2.2" && pip install "poetry==1.6.1"
+RUN python3 -m venv /poetry-env
+RUN /poetry-env/bin/pip install -U pip setuptools
+RUN /poetry-env/bin/pip install poetry
+
 
 WORKDIR /app/
 
@@ -59,14 +28,17 @@ COPY ./pyproject.toml ./poetry.lock /app/
 RUN python3 -m venv /venv
 ENV PATH="/venv/bin:$PATH"
 
-RUN poetry install --no-interaction --no-ansi --no-root --no-dev
+# Command that always fails
+#RUN false
+
+RUN /poetry-env/bin/poetry install --no-interaction --no-ansi --no-root --no-dev
 
 EXPOSE 8000
 
 # Creating folders, and files for a project:
 COPY ./ /app/
 
 # Install the root package
-RUN poetry install --no-interaction --no-ansi --no-dev
+RUN /poetry-env/bin/poetry install --no-interaction --no-ansi --no-dev
 
 CMD uvicorn --host 0.0.0.0 --port 8000 macrostrat_tileserver.main:app

Dockerfile.mapnik

@@ -0,0 +1,83 @@
+# From here:
+# https://stackoverflow.com/questions/53835198/integrating-python-poetry-with-docker
+# NOTE: we might want to make things a bit nicer here
+FROM python:3.10
+
+# MAPNIK
+# Install mapnik for compiling legacy image tiles
+RUN apt-get update -y && \
+    apt-get install -y --no-install-recommends \
+        build-essential software-properties-common curl \
+        libboost-dev libboost-filesystem-dev libboost-program-options-dev libboost-python-dev \
+        libboost-regex-dev libboost-system-dev libboost-thread-dev libicu-dev libtiff5-dev \
+        libfreetype-dev libpng-dev libxml2-dev libgdal-dev libgeos-dev libproj-dev libcairo-dev \
+        libharfbuzz-dev postgresql-contrib && \
+    rm -rf /var/lib/apt/lists/*
+
+# Mapnik
+ARG MAPNIK_VERSION=4.0.3
+
+WORKDIR /tmp/
+
+RUN git clone --depth 1 --branch v${MAPNIK_VERSION} https://github.com/mapnik/mapnik.git && cd mapnik && git submodule update --init deps
+# Install mapnik
+WORKDIR /tmp/mapnik
+RUN ./configure && make JOBS=4 && make install
+
+ENV BOOST_PYTHON_LIB=boost_python310
+
+# The rest of this (for vector tile generation and the server itself) should be easier.
+ENV PIP_DISABLE_PIP_VERSION_CHECK=1 POETRY_VIRTUALENVS_CREATE=false
+
+RUN pip install "pip==24.3.1" "setuptools>=75.6.0" "poetry==1.8.4"
+
+# Create and activate our own virtual envrionment so that we can keep
+# our application dependencies separate from Poetry's
+RUN python3 -m venv /venv
+ENV PATH="/venv/bin:$PATH"
+
+WORKDIR /tmp/
+# Clone the mapnik python bindings and install them
+RUN git clone https://github.com/mapnik/python-mapnik.git && \
+    cd python-mapnik && \
+    git checkout 10315a6d898ed341f5df5975395f3dc67814ebf6
+WORKDIR /tmp/python-mapnik
+RUN pip install "pybind11==2.13.6" && pip install .
+
+RUN rm -rf /tmp/*
+
+# Remove build dependencies
+RUN apt-get remove -y \
+  build-essential software-properties-common \
+  libboost-dev libboost-filesystem-dev libboost-program-options-dev libboost-python-dev \
+  libboost-regex-dev libboost-system-dev libboost-thread-dev libicu-dev libtiff5-dev \
+  libfreetype-dev libpng-dev libxml2-dev libproj-dev libgdal-dev libgeos-dev libcairo-dev libharfbuzz-dev
+
+# Software needed to actually run the tileserver
+
+# CartoCSS stylesheet generation
+# TODO: we could make this run as a separate step, potentially
+# Install nodejs version 20
+RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - && apt-get install -y nodejs && npm install -g carto
+
+# Install carto
+RUN npm install -g carto
+
+WORKDIR /app/
+
+# Copy only requirements to cache them in docker layer
+# Right now, Poetry lock file must exist to avoid hanging on dependency resolution
+COPY ./deps/timvt/ /app/deps/timvt/
+COPY ./pyproject.toml ./poetry.lock /app/
+
+RUN poetry install --no-interaction --no-ansi --no-root --no-dev
+
+EXPOSE 8000
+
+# Creating folders, and files for a project:
+COPY ./ /app/
+
+# Install the root package
+RUN poetry install --no-interaction --no-ansi --no-dev
+
+CMD uvicorn --host 0.0.0.0 --port 8000 macrostrat_tileserver.main:app

Makefile

@@ -4,5 +4,15 @@ serve:
 fast:
 	poetry run uvicorn macrostrat_tileserver.main:app --log-level debug --port 8000 --workers 8
 
+test-dev:
+	poetry run pytest -s -x --no-drop --skip-legacy-raster
+
 test:
-	strat compose run tileserver pytest
+	poetry run pytest -s -x
+
+docker-build:
+	docker build -t macrostrat-tileserver .
+
+styles:
+	npm install -g carto
+	poetry run tileserver create-mapnik-xml mapnik-xml-test

README.md

@@ -37,7 +37,7 @@ There is a bundled `tileserver` CLI that will create the layers. In Docker:
 
 > docker run macrostrat/tileserver \
 >   -e POSTGRES_DB=postgresql://user:password@db.server:5432 \
->   tileserver create-fixtures  
+>   tileserver create-fixtures
 
 Or in the running docker container:
 
@@ -66,3 +66,14 @@ Macrostrat core layers:
 - New layers can be defined using SQL or PL/PGSQL functions.
 - Currently, layers must be initialized by editing the `macrostrat_tileserver/main.py` file to
   add the appropriate initialization function. This will be improved in the future.
+
+## Testing
+
+Testing is done with `pytest`. To run the tests, use:
+
+```make test```.
+
+To omit legacy raster tests, use
+
+  ```make test-dev```.
+

conftest.py

@@ -0,0 +1,40 @@
+import pytest
+from dotenv import load_dotenv
+
+load_dotenv()
+
+from macrostrat_tileserver.tests.fixtures import (
+    app,
+    client,
+    db,
+    test_database_url,
+)  # noqa
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--skip-legacy-raster", action="store_true", help="Skip legacy raster tests"
+    )
+    parser.addoption(
+        "--no-drop",
+        action="store_true",
+        default=False,
+        help="Keep the database after tests",
+    )
+
+
+def pytest_configure(config):
+    # register an additional marker
+    # This is kind of an annoying way to specify this
+    config.addinivalue_line(
+        "markers", "legacy_raster: mark test as legacy raster test."
+    )
+
+
+def pytest_collection_modifyitems(config, items):
+    if config.getoption("--skip-legacy-raster"):
+        # --runslow given in cli: do not skip slow tests
+        skip_slow = pytest.mark.skip(reason="--skip-legacy-raster option provided")
+        for item in items:
+            if "legacy_raster" in item.keywords:
+                item.add_marker(skip_slow)

macrostrat_tileserver/cli.py

@@ -3,6 +3,7 @@
 from macrostrat.utils import relative_path
 from typer import Typer
 from dotenv import load_dotenv
+from pathlib import Path
 
 load_dotenv()
 
@@ -39,3 +40,20 @@ def list_layers():
         print(k)
         print(v)
         print()
+
+
+@_cli.command(name="create-mapnik-xml")
+def create_mapnik_xml(outdir: Path):
+    """Create image styles"""
+    # Note: Carto NodeJS module must be installed globally for this to work
+    from .main import app
+
+    from .image_tiles.mapnik_styles import make_mapnik_xml
+
+    outdir.mkdir(parents=True, exist_ok=True)
+    # This makes files without database connection information,
+    # for testing purposes essentially
+    for scale in ("large", "medium", "small", "tiny"):
+        xml = make_mapnik_xml(scale)
+        with (outdir / f"{scale}.xml").open("w") as f:
+            f.write(xml)

macrostrat_tileserver/image_tiles/core.py

@@ -14,6 +14,8 @@
 
 tile_settings = TileSettings()
 
+db = Database(environ.get("DATABASE_URL"))
+
 
 class ImageTileSubsystem:
     """Macrostrat's image tile subsystem allows image tiles to be generated using Mapnik.
@@ -32,7 +34,7 @@ def build_layer_cache(self):
         for scale in scales:
             # Log timings
             t = time.time()
-            self.layer_cache[scale] = make_mapnik_xml(scale)
+            self.layer_cache[scale] = make_mapnik_xml(scale, db.engine.url)
             print(
                 f"Generated mapnik XML for scale {scale} in {time.time() - t} seconds"
             )
@@ -67,7 +69,7 @@ async def handle_tile_request(
         request: Request,
         background_tasks: BackgroundTasks,
         tile: Tile = Depends(TileParams),
-        cache: CacheMode = CacheMode.prefer
+        cache: CacheMode = CacheMode.prefer,
     ):
         """Return vector tile."""
         pool = request.app.state.pool
@@ -102,4 +104,6 @@ async def handle_tile_request(
                 set_cached_tile, pool, "carto-image", tile, content
             )
 
-        return TileResponse(content, timer, cache_status=CacheStatus.miss, media_type="image/png")
+        return TileResponse(
+            content, timer, cache_status=CacheStatus.miss, media_type="image/png"
+        )