diff --git a/.github/workflows/cypress-tests.yml b/.github/workflows/cypress-tests.yml index 6c90209c2..03df4e9b1 100644 --- a/.github/workflows/cypress-tests.yml +++ b/.github/workflows/cypress-tests.yml @@ -27,7 +27,7 @@ jobs: - name: Build and start Docker containers run: | - docker compose -f docker-compose.yml -f docker-compose.test.yml --env-file test.env up -d --build + docker compose -f docker-compose.yml -f docker-compose.test.yml --env-file test.env --profile postgres up -d --build - name: Wait for application to be ready run: | @@ -43,7 +43,7 @@ jobs: elapsed=$((current_time - start_time)) if [ $elapsed -ge $timeout ]; then echo "Timeout waiting for application to be ready. Dumping logs:" - docker compose -f docker-compose.yml -f docker-compose.test.yml --env-file test.env logs + docker compose -f docker-compose.yml -f docker-compose.test.yml --env-file test.env --profile postgres logs exit 1 fi sleep 5 @@ -72,4 +72,4 @@ jobs: - name: Stop Docker containers if: always() - run: docker compose -f docker-compose.yml -f docker-compose.test.yml --env-file test.env down + run: docker compose -f docker-compose.yml -f docker-compose.test.yml --env-file test.env --profile postgres down diff --git a/.github/workflows/release-docker-images.yml b/.github/workflows/release-docker-images.yml index 6d22a8be8..c1f444f64 100644 --- a/.github/workflows/release-docker-images.yml +++ b/.github/workflows/release-docker-images.yml @@ -36,7 +36,7 @@ jobs: password: ${{ secrets.DOCKER_TOKEN }} - name: Build full project via docker-compose - run: docker compose build --parallel --build-arg GIT_HASH=${GITHUB_SHA:0:6} + run: docker compose --profile postgres build --parallel --build-arg GIT_HASH=${GITHUB_SHA:0:6} - name: Push images to Docker Hub - run: docker compose push --ignore-push-failures + run: docker compose --profile postgres push --ignore-push-failures diff --git a/Makefile b/Makefile index f2709054a..70f041970 100644 --- a/Makefile +++ b/Makefile @@ -8,23 +8,44 @@ SHELL=/bin/bash E2E_RUN = cd e2e; +define parse_env_value + $(shell grep -e ^$(1) ${ENV_FILE} | awk -F'[=]' '{gsub(/ /,"");print $$2}') +endef + +define parse_env_bool + $(if $(filter false,$(1)),false,true) +endef + +# Default environment and settings (dev) export ENV_FILE = .env -export TAG = $(shell grep -e ^TAG ${ENV_FILE} | awk -F'[=]' '{gsub(/ /,""); print $$2}') +export TAG = $(call parse_env_value,TAG) export GIT_HASH = $(shell git rev-parse --short HEAD) +POSTGRES_DOCKER_RAW = $(shell echo $(call parse_env_value,POSTGRES_DOCKER) | tr '[:upper:]' '[:lower:]') +export POSTGRES_DOCKER = $(call parse_env_bool,$(POSTGRES_DOCKER_RAW)) + +# Default compose file args export COMPOSE_FILE_ARGS = -f docker-compose.yml -f docker-compose.dev.yml +COMPOSE_FILE_ARGS += $(if $(POSTGRES_DOCKER),--profile postgres,) + +# Set up environment-specific values +define setup_env + $(eval ENV_FILE = $(1)) + $(eval TAG = $(call parse_env_value,TAG)) + $(eval POSTGRES_DOCKER_RAW = $(shell echo $(call parse_env_value,POSTGRES_DOCKER) | tr '[:upper:]' '[:lower:]')) + $(eval POSTGRES_DOCKER = $(call parse_env_bool,$(POSTGRES_DOCKER_RAW))) + $(eval COMPOSE_FILE_ARGS = $(2)) + $(eval COMPOSE_FILE_ARGS += $(if $(POSTGRES_DOCKER),--profile postgres,)) +endef -PROD: ## Run in prod mode (e.g. `make PROD start`, etc.) using config in `prod.env` - $(eval ENV_FILE = prod.env) - $(eval TAG = $(shell grep -e ^TAG ${ENV_FILE} | awk -F'[=]' '{gsub(/ /,"");print $$2}')) - $(eval COMPOSE_FILE_ARGS = -f docker-compose.yml) +PROD: + $(call setup_env,prod.env,-f docker-compose.yml) -TEST: ## Run in test mode (e.g. `make TEST e2e-run`, etc.) using config in `test.env` - $(eval ENV_FILE = test.env) - $(eval TAG = $(shell grep -e ^TAG ${ENV_FILE} | awk -F'[=]' '{gsub(/ /,"");print $$2}')) - $(eval COMPOSE_FILE_ARGS = -f docker-compose.yml -f docker-compose.test.yml) +TEST: + $(call setup_env,test.env,-f docker-compose.yml -f docker-compose.test.yml) echo_vars: @echo ENV_FILE=${ENV_FILE} + @echo POSTGRES_DOCKER=${POSTGRES_DOCKER} @echo TAG=${TAG} pull: echo_vars ## Pull most recent Docker container builds (nightlies) diff --git a/README.md b/README.md index 50974216d..4d15603f2 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,7 @@ cp example.env .env ```sh -docker compose up --build +docker compose --profile postgres up --build ``` If you get a permission error, try running this command with `sudo`. @@ -87,7 +87,7 @@ To avoid having to use `sudo` in the future (on a Linux or Windows machine with Once you've built the docker images, you can run without `--build`, which may be faster. Run ```sh -docker compose up +docker compose --profile postgres up ``` or simply @@ -103,16 +103,22 @@ If you have only changed configuration values in .env, you can recreate your con fully rebuilding them with `--force-recreate`. For example: ```sh -docker compose down -docker compose up --force-recreate +docker compose --profile postgres down +docker compose --profile postgres up --force-recreate ``` To see what the environment of your containers is going to look like, run: ```sh -docker compose convert +docker compose --profile postgres convert ``` +#### Using a local or remote (non-docker) database + +Omit the `--profile postgres` flag to use a local or remote database. You will need to set the `DATABASE_URL` environment variable in your `.env` file to point to your database. + +When using `make` commands, setting POSTGRES_DOCKER to `true` or `false` will determine whether to automatically include `--profile postgres` when it calls out to `docker compose`. + #### Production Mode Shortcuts The commands in the Makefile can be prefaced with PROD. If so, the "dev overlay" configuration in `docker-compose.dev.yml` will be ignored. @@ -138,8 +144,8 @@ You can now test your setup by visiting `http://localhost:80/home`. Once the index page loads, you can create an account using the `/createuser` path. You'll be logged in right away; email validation is not required. -When you're done working, you can end the process using `Ctrl+C`, or typing `docker compose down` -if you are running in "detched mode". +When you're done working, you can end the process using `Ctrl+C`, or typing `docker compose --profile postgres down` +if you are running in "detached mode". ### Updating the system @@ -174,7 +180,7 @@ However, if you are deploying in a high impact context and need help, please [re Once you've gotten [Polis running (as described above)](#-running-polis), you can enable developer conveniences by running ```bash -docker compose -f docker-compose.yml -f docker-compose.dev.yml up +docker compose -f docker-compose.yml -f docker-compose.dev.yml --profile postgres up ``` (run with `--build` if this is your first time running, or if you need to rebuild containers) @@ -188,7 +194,7 @@ This enables: - etc. This command takes advantage of the `docker-compose.dev.yml` _overlay_ file, which layers the developer conveniences describe above into the base system, as described in the `docker-compose.yml` file. -You can specify these `-f docker-compose.yml -f docker-compose.dev.yml` arguments for any `docker` command which you need to take advantage of these features (not just `docker compose up`). +You can specify these `-f docker-compose.yml -f docker-compose.dev.yml` arguments for any `docker` command which you need to take advantage of these features (not just `docker compose --profile postgres up`). You can create your own `docker-compose.x.yml` file as an overlay and add or modify any values you need to differ from the defaults found in the `docker-compose.yml` file and pass it as the second argument to the `docker compose -f` command above. @@ -219,7 +225,7 @@ git config --local include.path ../.gitconfig #### Running as a background process -If you would like to run docker compose as a background process, run the `up` commands with the `--detach` flag, and use `docker compose down` to stop. +If you would like to run docker compose as a background process, run the `up` commands with the `--detach` flag, and use `docker compose --profile postgres down` to stop. #### Using Docker Machine as your development environment diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 2c2d290a2..0aa471343 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -3,7 +3,7 @@ # server, and exposing most ports. DO NOT USE THIS IN PRODUCTION. # This configuration does not work on its own and must be run together with docker-compose.yml, -# e.g. `docker compose -f docker-compose.yml -f docker-compose.dev.yml up`. +# e.g. `docker compose --profile postgres -f docker-compose.yml -f docker-compose.dev.yml up`. # (Note the usage in Makefile.) # For more information see https://docs.docker.com/compose/extends/ @@ -32,10 +32,10 @@ services: environment: CHOKIDAR_USEPOLLING: "true" - # postgres: - # restart: no - # ports: - # - "${POSTGRES_PORT:-5432}:5432" + postgres: + restart: no + ports: + - "${POSTGRES_PORT:-5432}:5432" file-server: build: diff --git a/docker-compose.test.yml b/docker-compose.test.yml index aa4e4d0fb..17d0261cd 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -3,7 +3,7 @@ # server. DO NOT USE THIS IN PRODUCTION. # This configuration does not work on its own and must be run together with docker-compose.yml, -# e.g. `docker compose -f docker-compose.yml -f docker-compose.test.yml --env-file test.env up`. +# e.g. `docker compose --profile postgres -f docker-compose.yml -f docker-compose.test.yml --env-file test.env up`. # For more information see https://docs.docker.com/compose/extends/ diff --git a/docker-compose.yml b/docker-compose.yml index 4d6751128..e1417d280 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,24 +1,28 @@ -# Before running docker compose up for the first time, +# Before running docker compose --profile postgres up for the first time, # first copy the env.example file to .env and fill in the values. # `cp env.example .env` # (edit values as needed) # # Then, build and run with: -# `docker compose build` -# `docker compose up` +# `docker compose --profile postgres build` +# `docker compose --profile postgres up` # # Subsequently you should only need to run: -# `docker compose up` +# `docker compose --profile postgres up` # # Force a full re-build with no cache from previous builds: -# `docker compose build --no-cache` -# `docker compose up` +# `docker compose --profile postgres build --no-cache` +# `docker compose --profile postgres up` # # If you only changed the .env file, you can do a quick rebuild: -# `docker compose up --force-recreate` +# `docker compose --profile postgres up --force-recreate` # # To stop, press CTRL+C (if not running in --detach mode), or run: -# `docker compose down` +# `docker compose --profile postgres down` +# +# omit the --profile postgres flag to run without the postgres service, +# such as when using postgres on your host machine, or a remote server. +# (by default, the Makefile will include --profile postgres, unless POSTGRES_DOCKER=false in your env) services: server: @@ -34,7 +38,6 @@ services: labels: polis_tag: ${TAG:-dev} depends_on: - # - "postgres" - "file-server" networks: - "polis-net" @@ -46,8 +49,6 @@ services: math: image: docker.io/compdem/polis-math:${TAG:-dev} - # depends_on: - # - "postgres" build: context: ./math labels: @@ -60,23 +61,25 @@ services: networks: - "polis-net" - # postgres: - # image: docker.io/compdem/polis-postgres:${TAG:-dev} - # restart: always - # build: - # context: ./server - # dockerfile: Dockerfile-db - # labels: - # polis_tag: ${TAG:-dev} - # environment: - # - POSTGRES_DB=${POSTGRES_DB:?error} - # - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:?error} - # - POSTGRES_USER=${POSTGRES_USER:?error} - # networks: - # - "polis-net" - # volumes: - # - "backups:/backups" - # - "postgres:/var/lib/postgresql/data" + postgres: + image: docker.io/compdem/polis-postgres:${TAG:-dev} + restart: always + build: + context: ./server + dockerfile: Dockerfile-db + labels: + polis_tag: ${TAG:-dev} + environment: + - POSTGRES_DB=${POSTGRES_DB:?error} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:?error} + - POSTGRES_USER=${POSTGRES_USER:?error} + networks: + - "polis-net" + volumes: + - "backups:/backups" + - "postgres:/var/lib/postgresql/data" + profiles: + - postgres nginx-proxy: image: docker.io/compdem/polis-nginx-proxy:${TAG:-dev} @@ -120,9 +123,9 @@ volumes: backups: labels: polis_tag: ${TAG:-dev} - # postgres: - # labels: - # polis_tag: ${TAG:-dev} + postgres: + labels: + polis_tag: ${TAG:-dev} server-logs: labels: polis_tag: ${TAG:-dev} diff --git a/docs/migrations.md b/docs/migrations.md index 766eba4f0..307609c77 100644 --- a/docs/migrations.md +++ b/docs/migrations.md @@ -12,7 +12,7 @@ But if we update the database schema after your initial provisioning of your ser - Please submit an issue if you'd like to work on enabling backups through Docker Compose. - Your database data is stored on a docker volume, which means that it will persist even when you destroy all your docker containers. Be mindful of this. - - You can remove ALL volumes defined within a `docker-compose` file via: `docker compose down --volumes` + - You can remove ALL volumes defined within a `docker-compose` file via: `docker compose --profile postgres down --volumes` - You can remove ONE volume via `docker volume ls` and `docker volume rm ` - SQL migrations can be found in [`server/postgres/migrations/`][] of this repo. @@ -24,7 +24,7 @@ For example, if we add the migration file host system: ```sh -docker compose exec postgres psql --username postgres --dbname polis-dev --file=/docker-entrypoint-initdb.d/000001_update_pwreset_table.sql +docker compose --profile postgres exec postgres psql --username postgres --dbname polis-dev --file=/docker-entrypoint-initdb.d/000001_update_pwreset_table.sql ``` You can also run a local .sql file on a postgres container instance with this syntax: diff --git a/example.env b/example.env index 009be62b4..df5bd4cb9 100644 --- a/example.env +++ b/example.env @@ -20,12 +20,16 @@ SERVER_LOG_LEVEL= ###### DATABASE ###### # Optional DB replica for reads: READ_ONLY_DATABASE_URL= +# Required for using dockerized postgres (ie --profile postgres). POSTGRES_DB=polis-dev POSTGRES_PORT=5432 POSTGRES_HOST=postgres:${POSTGRES_PORT} POSTGRES_USER=postgres POSTGRES_PASSWORD=oiPorg3Nrz0yqDLE +# Always required. Replace with your own database URL if not using dockerized postgres. DATABASE_URL=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB} +# Makefile will read this to determine if the database is running in a docker container. +POSTGRES_DOCKER=true ###### DOCKER CONCERNS ###### diff --git a/math/README.md b/math/README.md index 7951bf88f..01d4e9b80 100644 --- a/math/README.md +++ b/math/README.md @@ -15,7 +15,7 @@ This allows you to connect to and evaluate code from within the running math wor To use this infrastructure, run the following from the root of this repository: ```sh -docker compose -f docker-compose.yml -f docker-compose.dev.yml up +docker compose -f docker-compose.yml -f docker-compose.dev.yml --profile postgres up ``` The first time you run this (or if you've edited any of the docker files, or certain other parts of the system), you may need to run this command with the `--build` flag. @@ -59,7 +59,7 @@ There are also a number of commands which can be run, either locally or with `do * `clojure -M:run full` - run a full system (poller plus auxiliary task processing) * etc. -If you are exporting data, you will need to run `docker compose` here with the same `-f docker-compose.yml -f docker-compose.dev.yml` options as described above, so that the directory (volume) mounting in the dev configuration file will mirror the generated files over onto your local filesystem. +If you are exporting data, you will need to run `docker compose` here with the same `-f docker-compose.yml -f docker-compose.dev.yml --profile postgres` options as described above, so that the directory (volume) mounting in the dev configuration file will mirror the generated files over onto your local filesystem. ## Tests diff --git a/server/README.md b/server/README.md index adb814bb4..3a24eabd5 100644 --- a/server/README.md +++ b/server/README.md @@ -49,9 +49,9 @@ flavors of node. Another popular option is to run the database in docker, and perhaps other services as well, while running this API server locally (not docker). Ensure that the postgres port is published from your docker container. This will be - the default behavior if you run `docker compose -f docker-compose.yml -f docker-compose.dev.yml up postgres` from the + the default behavior if you run `docker compose -f docker-compose.yml -f docker-compose.dev.yml --profile postgres up postgres` from the root folder of the polis project. To run everything but the API server in this fashion you can use - `docker compose -f docker-compose.yml -f docker-compose.dev.yml up math postgres file-server maildev`. In this case + `docker compose -f docker-compose.yml -f docker-compose.dev.yml --profile postgres up math postgres file-server maildev`. In this case the polis-dev database should be accessible at the default DATABASE_URL seen in server/example.env. 3\. Connect to the new database then run the migrations in its shell. You can skip this step if you built the diff --git a/test.env b/test.env index bce989237..c921cc70d 100644 --- a/test.env +++ b/test.env @@ -11,6 +11,7 @@ DOMAIN_OVERRIDE=localhost EMBED_SERVICE_HOSTNAME=localhost STATIC_FILES_HOST=file-server +POSTGRES_DOCKER=true POSTGRES_DB=polis-test POSTGRES_HOST=postgres:5432 POSTGRES_PASSWORD=PdwPNS2mDN73Vfbc