feat(repo): Move to use src-layout

Containerfile

@@ -1,34 +1,24 @@
-FROM python:3.12-slim
-
-# Checkout and install dagster libraries needed to run the gRPC server by exposing
-# your code location to dagster-webserver and dagster-daemon, and loading the
-# DagsterInstance.
-
-RUN pip install \
-    dagster \
-    dagster-postgres \
-    dagster-docker
-
-# Set $DAGSTER_HOME and copy dagster instance there
-ENV DAGSTER_HOME=/opt/dagster/dagster_home
-
-RUN mkdir -p $DAGSTER_HOME
-COPY dagster.yaml $DAGSTER_HOME
+FROM python:3.12-slim-bookworm
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
 
+ENV DAGSTER_HOME=/opt/dagster/home
 
 # Add repository code
 WORKDIR /opt/dagster/app
-COPY workspace.yaml /opt/dagster/app
-COPY local_archives /opt/dagster/app
-COPY cloud_archives	/opt/dagster/app
+COPY src /opt/dagster/app
+COPY pyproject.toml /opt/dagster/app
 
-# Run dagster gRPC server on port 4000
-EXPOSE 4000
+# Checkout and install dagster libraries needed to run the gRPC server by exposing
+# your code location to dagster-webserver and dagster-daemon, and loading the
+# DagsterInstance.
+RUN uv sync
 
 # Set the code location module to be loaded by the gRPC server
 ENV MODULE_NAME=local_archives
 
-# Using CMD rather than ENTRYPOINT allows the command to be overridden in
-# run launchers or executors to run other commands using this image
-CMD ["dagster", "api", "grpc", "-h", "0.0.0.0", "-p", "4000", "-m", ${MODULE_NAME:-local_archives}]
+# Using CMD rather than RUN allows the command to be overridden in
+# run launchers or executors to run other commands using this image.
+# This is important as runs are executed inside this container.
+ENTRYPOINT ["uv", "run"]
+CMD ["dagster", "api", "grpc", "-h", "0.0.0.0", "-p", "4000", "-m", "local_archives"]
 

infrastructure/docker-compose.yaml

@@ -0,0 +1,156 @@
+name: dagster
+
+x-postgres-variables: &postgres-variables
+  POSTGRES_USER: ${POSTGRES_USER:-dagster_user}
+  POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-dagster_password}
+  POSTGRES_DB: ${POSTGRES_DB:-dagster_db}
+  POSTGRES_PORT: ${POSTGRES_PORT:-5438}
+  POSTGRES_HOST: "dagster-postgres"
+
+x-dagster-configs: &dagster-configs
+  - source: dagster.yaml
+    target: /opt/dagster/dagster.yaml
+  - source: workspace.yaml
+    target: /opt/dagster/home/workspace.yaml
+
+
+services:
+  # This service runs the postgres DB used by dagster for run storage, schedule storage,
+  # and event log storage. Depending on the hardware you run this Compose on, you may be able
+  # to reduce the interval and timeout in the healthcheck to speed up your `docker-compose up` times.
+  dagster-postgres:
+    image: postgres:16
+    container_name: dagster-postgres
+    environment:
+      <<: *postgres-variables
+      PGDATA: "/var/lib/postgresql/data"
+    volumes:
+      - dagster-pgdata-vol:/var/lib/postgresql/data
+    expose:
+      - ${POSTGRES_PORT:-5438} # Publishes on network but not to host
+    networks: ["dagster-network"]
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_PASSWORD}"]
+      interval: 10s
+      timeout: 8s
+      retries: 5
+
+  # This service runs the gRPC server that loads user code, used by both dagster-webserver
+  # and dagster-daemon. By setting DAGSTER_CURRENT_IMAGE to its own image, we tell the
+  # run launcher to use this same image when launching runs in a new container as well.
+  dagster-codeserver_local-archives:
+    container_name: dagster-codeserver_local-archives
+    image: ghcr.io/openclimatefix/dagster-dags:devsjc-code-container
+    restart: always
+    environment:
+      <<: *postgres-variables
+      DAGSTER_CURRENT_IMAGE: "ghcr.io/openclimatefix/dagster-dags"
+      DAGSTER_HOME: "/opt/dagster/home"
+    expose:
+      - "4000"
+    configs: *dagster-configs
+    networks: ["dagster-network"]
+
+  # This service runs dagster-webserver, which loads your user code from the user code container.
+  # Since our instance uses the QueuedRunCoordinator, any runs submitted from the webserver will be put on
+  # a queue and later dequeued and launched by dagster-daemon.
+  dagster-webserver:
+    container_name: dagster-webserver
+    image: dagster/dagster-k8s:latest
+    command: ["dagster-webserver", "-h", "0.0.0.0", "-p", "3008", "-w", "${DAGSTER_HOME}/workspace.yaml"]
+    ports:
+      - "3008:3008"
+    environment:
+      <<: *postgres-variables
+      DAGSTER_HOME: "/opt/dagster/home"
+    configs: *dagster-configs
+    volumes:
+      # Enable termination of runs from the webserver
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /tmp/io_manager_storage:/tmp/io_manager_storage
+    networks: ["dagster-network"]
+    depends_on:
+      dagster-postgres:
+        condition: service_healthy
+      dagster-codeserver_local-archives:
+        condition: service_started
+
+  # This service runs the dagster-daemon process, which is responsible for taking runs
+  # off of the queue and launching them, as well as creating runs from schedules or sensors.
+  dagster-daemon:
+    container_name: dagster-daemon
+    image: dagster/dagster-k8s:latest
+    command: ["dagster-daemon", "run"]
+    restart: on-failure
+    environment:
+      <<: *postgres-variables
+      DAGSTER_HOME: "/opt/dagster/home"
+    configs: *dagster-configs
+    volumes:
+      # Enable kicking off of runs from the daemon
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /tmp/io_manager_storage:/tmp/io_manager_storage
+    networks: ["dagster-network"]
+    depends_on:
+      dagster-postgres:
+        condition: service_healthy
+      dagster-codeserver_local-archives:
+        condition: service_started
+
+networks:
+  dagster-network:
+    driver: bridge
+    name: dagster-network
+
+volumes:
+  # Volume for the postgres data directory
+  dagster-pgdata-vol:
+    name: dagster-pgdata-vol
+
+configs:
+  workspace.yaml:
+    content: |
+      load_from:
+        - grpc_server:
+            host: "dagster-codeserver_local-archives"
+            port: 4000
+            location_name: "local_archives"
+
+  dagster.yaml:
+    content: |
+      storage:
+        postgres:
+          username:
+            env: POSTGRES_USER
+          password:
+            env: POSTGRES_PASSWORD
+          hostname:
+            env: POSTGRES_HOST
+          db_name:
+            env: POSTGRES_DB
+          port:
+            env: POSTGRES_PORT
+      local_artifact_storage:
+        module: dagster.core.storage.root
+        class: LocalArtifactStorage
+        config:
+          base_dir: "/opt/dagster/local/"
+      run_coordinator:
+        module: dagster.core.run_coordinator
+        class: QueuedRunCoordinator
+        config:
+          max_concurrent_runs: 30
+          tag_concurrency_limits:
+            - key: "dagster/backfill"
+              limit: 15
+            - key: "nwp-consumer"
+              limit: 1
+      retention:
+        schedule:
+          purge_after_days: 90
+        sensor:
+          purge_after_days:
+            skipped: 7
+            failure: 30
+            success: -1
+