working traefik configuration for Docker Swarm (with ARM images)

[francescor]

Hi,
we've invested a lot in reaching this working setup. We'll like to share (and collect feedback).

Lsp is working, too.

version: "3.9"

# Official traefik configuration
# https://www.windmill.dev/docs/advanced/self_host#traefik-configuration

x-image: &common-image
  # https://github.com/windmill-labs/windmill/blob/main/CHANGELOG.md
  image: ghcr.io/windmill-labs/windmill:1.432.0  # ok, image is multiarch

x-deploy: &deploy-snippet
  replicas: 1
  placement:
    constraints:
      - node.role == worker
      - node.platform.arch == aarch64
      - engine.labels.jobs-type == stateless
  update_config:
    parallelism: 1
    order: stop-first
    delay: 90s
    failure_action: rollback
  rollback_config:
    parallelism: 1
    delay: 10s
    failure_action: pause         # default: pause or continue
  restart_policy:
    condition: on-failure
    delay: 90s                    # default: 5s, between attempts
    max_attempts: 50
  resources:
    reservations:
      cpus: '0.2'
      memory: 64M
    limits:
      cpus: '2'
      memory: 4G


networks:
  traefik-mix:
    external: true
    name: traefik-mix
  backplane:
    external: true
    name: backplane
  internal:
  swarm-backplane:
    external: true
    name: swarm-backplane

volumes:
  worker_dependency_cache: null
  pgdata16:

services:
  server:
    <<: *common-image
    networks:
      internal:
        aliases:
          - windmill-qa_server
      traefik-mix:
    environment:
      - DATABASE_URL=postgres://windmill:XXXXXXXXXXXXXX@db-windmill:5432/windmill?sslmode=disable
      - BASE_URL=https://windmill-qa.example.com
      - RUST_LOG=info
      # You can set the number of workers to > 0 and not need any separate worker service
      - NUM_WORKERS=0
      # - DISABLE_SERVER=false
      - MODE=server
      - METRICS_ADDR=false
      - SUPERADMIN_SECRET=XXXXXXXXXXXXXXXXXXXXXXXX
    volumes:
      - ./config/windmill-qa/oauth.json:/usr/src/app/oauth.json
      - ./config/windmill-qa/pull_hub_items.sh:/usr/bin/pull_hub_items
    deploy:
      <<: *deploy-snippet
      endpoint_mode: dnsrr
      replicas: 1
      update_config:
        order: stop-first
        parallelism: 1
        delay: 20s
      restart_policy:
        condition: any
        max_attempts: 10
        window: 120s
      labels:
        diun.enable: 'true'
        diun.watch_repo: 'true'
        traefik.enable: 'true'
        traefik.docker.network: traefik-mix
        traefik.http.services.windmill-qa-server.loadbalancer.server.port: 8000
        traefik.http.routers.windmill-qa-server.rule: Host(`windmill-qa.example.com`)
        traefik.http.routers.windmill-qa-server.tls: 'true'
        traefik.http.routers.windmill-qa-server.tls.certresolver: myhttpchallenge
    healthcheck:
      test: [ "CMD", "wget", "--spider", "http://localhost:8000" ]
      interval: 120s
      timeout: 2s


  worker:
    <<: *common-image
    hostname: "{{.Node.Hostname}}"
    networks:
      - internal
      - swarm-backplane
    deploy:
      <<: *deploy-snippet
      replicas: 1
      placement:
        max_replicas_per_node: 1
        constraints:
          - node.role == worker
          - node.platform.arch == aarch64
          - engine.labels.jobs-type == stateless
    environment:
      - DATABASE_URL=postgres://windmill:XXXXXXXXXXXXXX@db-windmill:5432/windmill?sslmode=disable
      - BASE_URL=https://windmill-qa.example.com
      - BASE_INTERNAL_URL=http://windmill-qa_server:8000
      # - RUST_LOG=debug
      - RUST_LOG=info
      # WARN windmill: We STRONGLY recommend using at most 1 worker per container, use at your own risks
      - NUM_WORKERS=1
      # - DISABLE_SERVER=true
      - MODE=worker
      - KEEP_JOB_DIR=false
      - DENO_PATH=/usr/bin/deno
      - PYTHON_PATH=/usr/local/bin/python3
      - METRICS_ADDR=false
      - DOCKER_HOST=http://dockersocket:2375
      - WORKER_GROUP=default
    # to mount the worker folder to debug,, KEEP_JOB_DIR=true and mount /tmp/windmill
    volumes:
      - worker_dependency_cache:/tmp/windmill/cache
      # mount the docker socket to allow to run docker containers from within the workers
      - /var/run/docker.sock:/var/run/docker.sock

  # Websocket for linters (eg. shellcheck) 
  lsp:
    image: ghcr.io/windmill-labs/windmill-lsp:1.432.0 
    networks:
      - traefik-mix
    environment:
      - LOGLEVEL=DEBUG
    deploy:
      <<: *deploy-snippet
      replicas: 1
      update_config:
        order: start-first
        parallelism: 1
        delay: 20s
      restart_policy:
        condition: any
        max_attempts: 1
        window: 120s
      labels:
        traefik.enable: 'true'
        traefik.docker.network: traefik-mix
        traefik.http.services.windmill-qa-lsp.loadbalancer.server.port: 3001
        traefik.http.services.windmill-qa-lsp.loadbalancer.healthcheck.path: /health
        # esempio chiamate: vedi https://github.com/windmill-labs/windmill/blob/main/lsp/pyls_launcher.py
        # attenzione che ci si connette solo come websocket e va bene che ci sia il /ws
        traefik.http.routers.windmill-qa-lsp.rule: 'Host(`windmill-qa.example.com`) && PathPrefix(`/ws`)'
        traefik.http.routers.windmill-qa-lsp.tls: 'true'
        traefik.http.routers.windmill-qa-lsp.tls.certresolver: myhttpchallenge

  db-windmill:
    image: wodby/postgres:16
    stop_grace_period: 30s
    environment:
      POSTGRES_PASSWORD: XXXXXXXXXXXXXX
      POSTGRES_USER: windmill
      POSTGRES_DB: windmill
      PGDATA: /var/lib/postgresql/data/pgdata
    volumes:
      #- ./compose_conf/postgres:/docker-entrypoint-initdb.d
      - pgdata16:/var/lib/postgresql/data/pgdata
      - /srv/data/db_dumps/windmill-qa:/db_dumps
    healthcheck:
      test: [ "CMD-SHELL", "pg_isready --user windmill --dbname windmill" ]
      interval: 10s
      timeout: 5s
      retries: 5
    networks:
      internal:
      backplane:
        aliases:
          - windmill-qa_postgres
    ports:
      - target: 54320
        published: 54320
        protocol: tcp
        #mode: vip
        mode: host
    deploy:
      replicas: 1
      update_config:
        parallelism: 1
        delay: 10s
      restart_policy:
        condition: on-failure
        max_attempts: 3
        window: 120s
      resources:
        limits:
          cpus: '0.5'
          memory: 1G
        reservations:
          memory: 128M
      placement:
        constraints: [node.hostname == boat-tank-f8ba]
      labels:
        traefik.enable: 'false'