From 047e2fa08830e71b265b6f1ac35f3d14fb509363 Mon Sep 17 00:00:00 2001 From: havok2063 Date: Fri, 3 May 2024 10:56:10 -0400 Subject: [PATCH 1/4] adding new dockerfile setup --- .dockerignore | 14 ++++++++++++++ Dockerfile | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ Dockerfile.dev | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 110 insertions(+) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 Dockerfile.dev diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..ebea18b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,14 @@ +# Ignore .git dirs +.git +.github + +# Ignore tests directory +**/tests/* +**/test_*.py + +# Ignore docs directory +**/docs/* + +# Ignore Python bytecode +**/__pycache__/ +**/*.pyc \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..d9f593c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,48 @@ +# Stage 1: Development stage for Python dependencies +FROM python:3.10-slim as dep-stage + +# Set up app dir +WORKDIR /tmp + +# Copy project files over +COPY ./pyproject.toml ./poetry.lock ./ + +# Install build-essential package +RUN apt-get update && \ + apt-get install -y \ + build-essential \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Install poetry and project dependencies +RUN pip install poetry && \ + poetry config virtualenvs.create false && \ + poetry install --no-root && \ + rm -rf ~/.cache + +# Stage 2: Development stage for the project +FROM dep-stage as dev-stage + +# Copy the main project files over and install +COPY ./ ./ +RUN poetry install --only main + +# Create dir for socket and logs +RUN mkdir -p /tmp/webapp + +# Setting environment variables +# these can be manually overridden +ENV MODULE_NAME="valis.wsgi" +ENV VALIS_SOCKET_DIR='/tmp/webapp' +ENV VALIS_LOGS_DIR='/tmp/webapp' +ENV VALIS_ALLOW_ORIGIN="http://localhost:8080" +ENV VALIS_DB_REMOTE=True + +# Stage 3: Build stage (inherits from dev-stage) +FROM dev-stage as build-stage + +# Expose the port +EXPOSE 8000 + +# Start the FastAPI app for production +CMD ["poetry", "run", "gunicorn", "-c", "python/valis/wsgi_conf.py", "valis.wsgi:app"] diff --git a/Dockerfile.dev b/Dockerfile.dev new file mode 100644 index 0000000..fe68148 --- /dev/null +++ b/Dockerfile.dev @@ -0,0 +1,48 @@ +# Stage 1: Development stage for Python dependencies +FROM python:3.10-slim as dep-stage + +# Set up app dir +WORKDIR /tmp + +# Copy project files over +COPY ./pyproject.toml ./poetry.lock ./ + +# Install build-essential package +RUN apt-get update && \ + apt-get install -y \ + build-essential \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Install poetry and project dependencies +RUN pip install poetry && \ + poetry config virtualenvs.create false && \ + poetry install --no-root && \ + rm -rf ~/.cache + +# Stage 2: Development stage for the project +FROM dep-stage as dev-stage + +# Copy the main project files over and install +COPY ./ ./ +RUN poetry install --only main + +# setting environment variables +# these can be manually overridden +ENV MODULE_NAME="valis.wsgi" +ENV VALIS_SOCKET_DIR='/tmp/valis' +ENV VALIS_LOGS_DIR='/tmp' +ENV VALIS_ALLOW_ORIGIN="http://localhost:3000" +ENV VALIS_DB_REMOTE=True + +# Stage 3: Build stage (inherits from dev-stage) +FROM dev-stage as build-stage + +# Set a label +LABEL org.opencontainers.image.source https://github.com/sdss/valis + +# Expose port 8000 +EXPOSE 8000 + +# Start the FastAPI app with hot-reloading for development +CMD ["uvicorn", "valis.wsgi:app", "--host", "0.0.0.0", "--port", "8000", "--reload"] From b1582561552cd0ca161f2ffe51714c1028142811 Mon Sep 17 00:00:00 2001 From: havok2063 Date: Fri, 3 May 2024 10:56:53 -0400 Subject: [PATCH 2/4] removing old docker files --- docker/docker-compose.yml | 28 ------------------------- docker/nginx/Dockerfile | 3 --- docker/nginx/nginx.conf | 36 --------------------------------- docker/valis/Dockerfile | 34 ------------------------------- docker/valis/docker_prestart.sh | 6 ------ 5 files changed, 107 deletions(-) delete mode 100644 docker/docker-compose.yml delete mode 100644 docker/nginx/Dockerfile delete mode 100644 docker/nginx/nginx.conf delete mode 100644 docker/valis/Dockerfile delete mode 100644 docker/valis/docker_prestart.sh diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml deleted file mode 100644 index ab0714b..0000000 --- a/docker/docker-compose.yml +++ /dev/null @@ -1,28 +0,0 @@ -version: '3.9' -services: - nginx: - image: nginx-valis:latest - container_name: nginx-valis - build: - context: nginx - volumes: - - ${VALIS_SOCKET_DIR:-/tmp/valis}:/tmp/valis # (host_socket_dir:upstream_socket_location) - ports: - - 5000:8080 - networks: - - valisnet - - valis: - build: - context: valis - image: valis:0.1.0 - container_name: valis - volumes: - - ${VALIS_SOCKET_DIR:-/tmp/valis}:/tmp/valis # (host_socket_dir:container_socket_dir) - networks: - - valisnet - -networks: - valisnet: - - diff --git a/docker/nginx/Dockerfile b/docker/nginx/Dockerfile deleted file mode 100644 index a779348..0000000 --- a/docker/nginx/Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM nginx:1.21 - -COPY nginx.conf /etc/nginx/nginx.conf \ No newline at end of file diff --git a/docker/nginx/nginx.conf b/docker/nginx/nginx.conf deleted file mode 100644 index f301a22..0000000 --- a/docker/nginx/nginx.conf +++ /dev/null @@ -1,36 +0,0 @@ -events { - -} - -http { - error_log /etc/nginx/nginx-error.log warn; - access_log /etc/nginx/nginx-access.log; - - client_max_body_size 20m; - - proxy_cache_path /etc/nginx/cache keys_zone=one:500m max_size=1000m; - - upstream valis { - server unix:/tmp/valis/valis.sock; - } - - server { - listen 8080 default_server; - server_name localhost; - - location /valis { - proxy_set_header Host $http_host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_redirect off; - proxy_buffering off; - proxy_pass http://valis; - proxy_cache one; - proxy_http_version 1.1; - - error_log /etc/nginx/valis_error.log error; - access_log /etc/nginx/valis_access.log; - - } - } -} \ No newline at end of file diff --git a/docker/valis/Dockerfile b/docker/valis/Dockerfile deleted file mode 100644 index 9ae3a03..0000000 --- a/docker/valis/Dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -FROM tiangolo/uvicorn-gunicorn-fastapi:python3.7 - -# Git checkout of valis -WORKDIR /tmp -RUN git clone --depth 1 https://github.com/sdss/valis project - -# Install Poetry -RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | POETRY_HOME=/opt/poetry python && \ - cd /usr/local/bin && \ - ln -s /opt/poetry/bin/poetry && \ - poetry config virtualenvs.create false - -# Install the project and its dependencies -WORKDIR /tmp/project -RUN poetry install --no-dev - -# copy over application -RUN cp -r python/valis /app - -WORKDIR /app - -# copying over a pre-start script -COPY ./docker_prestart.sh /app/prestart.sh - -# see https://github.com/tiangolo/uvicorn-gunicorn-fastapi-docker -# for docs on setup and configuration options - -# setting environment variables -ENV MODULE_NAME="valis.wsgi" - -# set container socket directory to /tmp/valis ; push that to gunicorn conf -ENV VALIS_SOCKET_DIR='/tmp/valis' -ENV GUNICORN_CONF="/app/app/wsgi_conf.py" - diff --git a/docker/valis/docker_prestart.sh b/docker/valis/docker_prestart.sh deleted file mode 100644 index 4632465..0000000 --- a/docker/valis/docker_prestart.sh +++ /dev/null @@ -1,6 +0,0 @@ -#! /usr/bin/env bash - -# create a symlink from /app/app to /app/valis so container -# understands that valis is an importable python package -# -ln -sf valis app From 49a1be9b3226497af7151b80eb3a789bfb797a2b Mon Sep 17 00:00:00 2001 From: havok2063 Date: Fri, 3 May 2024 12:34:03 -0400 Subject: [PATCH 3/4] updating readme with new docker instructions --- README.md | 62 +++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 49 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index d3258e9..e5e0b27 100644 --- a/README.md +++ b/README.md @@ -81,23 +81,59 @@ db_user: {unid} ## Deployment -### Deploying at Utah in api.sdss.org docker -- Login to `lore` or `manga` machines -- Login to the api.sdss.org docker with `ssh -p 2209 [unid]@api.sdss.org` -- If needed, load valis module with `ml load valis` -- cd to `$VALIS_DIR` -- run `run_valis` alias or `poetry run gunicorn -c python/valis/wsgi_conf.py valis.wsgi:app` + +This section describes a variety of deployment methods. Valis uses gunicorn as its +wsgi http server. It binds the app both to port 8000, and a unix socket. The defaut mode +is to start valis with an awsgi uvicorn server, with 4 workers. + +### Deploying Zora + Valis together +See the SDSS [Zora+Valis Docker](https://github.com/sdss/zora_valis_dockers) repo page. + +### Deploying at Utah in dataviz-dm +TBD ### Running manually via gunicorn + nginx - Setup a local nginx server with a /valis location - export VALIS_SOCKET_DIR=/tmp/valis - run `gunicorn -c python/valis/wsgi_conf.py valis.wsgi:app` -### Running Docker Compose -This builds and sets up the valis docker running with nginx, mapped to a unix socket at `unix:/tmp/valis/valis.sock`. It binds the internal nginx port 8080 to localhost port 5000. -- Navigate to `docker` folder -- Run `docker-compose -f docker-compose.yml build` to build the docker images -- Run `docker-compose -f docker-compose.yml up -d` to start the containers in detached mode -- Navigate to `http://localhost:5000/valis` -- To stop the service and remove containers, run `docker-compose -f docker-compose.yml down` +This also exposes valis to port 8000, and should be available at `http://localhost:8000`. + +### Running the Docker + +There are two dockerfiles, one for running in development mode and one for production. To connect valis to the `sdss5db` database, you'll need to set several **VALIS_DB_XXX** environment variables during the `docker run` command. + +- VALIS_DB_REMOTE: Set this to True +- VALIS_DB_HOST: the database host machine +- VALIS_DB_USER: the database user +- VALIS_DB_PASS: the database password +- VALIS_DB_PORT: the database port + +The following examples show how to connect the valis docker to a database running on the same machine, following the database setup instructions above. + +**Development** + +To build the docker image, run + +`docker build -t valis-dev -f Dockerfile.dev .` + +To start a container, run +```bash +docker run -p 8000:8000 -e VALIS_DB_REMOTE=True -e VALIS_DB_HOST=host.docker.internal -e VALIS_DB_USER=[user] -e VALIS_DB_PASS=[password] -e VALIS_DB_PORT=6000 valis-dev +``` + +**Production** + +To build the docker image, run + +`docker build -t valis -f Dockerfile .` + +To start a container, run +```bash +docker run -p 8000:8000 -e VALIS_DB_REMOTE=True -e VALIS_DB_HOST=host.docker.internal -e VALIS_DB_USER=[user] -e VALIS_DB_PASS=[password] -e VALIS_DB_PORT=6000 valis +``` +Note: If your docker vm has only a small resource allocation, the production container may crash on start, due to the number of workers allocated. You can adjust the number of workers with the `VALIS_WORKERS` envvar. For example, add `-e VALIS_WORKERS=2` to scale the number of workers down to 2. + +### Podman +All dockerfiles work with `podman`, and the syntax is the same as above. Simply replace `docker` with `podman`. \ No newline at end of file From d9eba9ea18f2a9fe681fdfeed89dab18d7c49e33 Mon Sep 17 00:00:00 2001 From: havok2063 Date: Fri, 3 May 2024 12:42:34 -0400 Subject: [PATCH 4/4] updating labels in valis docker --- Dockerfile | 4 ++++ Dockerfile.dev | 1 + 2 files changed, 5 insertions(+) diff --git a/Dockerfile b/Dockerfile index d9f593c..5fd5adb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,6 +41,10 @@ ENV VALIS_DB_REMOTE=True # Stage 3: Build stage (inherits from dev-stage) FROM dev-stage as build-stage +# Set a label +LABEL org.opencontainers.image.source https://github.com/sdss/valis +LABEL org.opencontainers.image.description "valis production image" + # Expose the port EXPOSE 8000 diff --git a/Dockerfile.dev b/Dockerfile.dev index fe68148..f04f9ad 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -40,6 +40,7 @@ FROM dev-stage as build-stage # Set a label LABEL org.opencontainers.image.source https://github.com/sdss/valis +LABEL org.opencontainers.image.description "valis development image" # Expose port 8000 EXPOSE 8000