Merge branch 'master' of https://github.com/scaleoutsystems/fedn into…

… feature/SK-1229

.ci/tests/examples/run.sh

@@ -38,7 +38,7 @@ python ../../.ci/tests/examples/wait_for.py reducer
 python ../../.ci/tests/examples/wait_for.py combiners
 
 >&2 echo "Upload compute package"
-python ../../.ci/tests/examples/api_test.py set_package --path package.tgz --helper "$helper"
+python ../../.ci/tests/examples/api_test.py set_package --path package.tgz --helper "$helper" --name test
 
 >&2 echo "Wait for clients to connect"
 python ../../.ci/tests/examples/wait_for.py clients

.github/workflows/build-containers.yaml

@@ -53,7 +53,7 @@ jobs:
           tags: ${{ steps.meta1.outputs.tags }}
           labels: ${{ steps.meta1.outputs.labels }}
           file: Dockerfile
-      
+
       # if push to master of release, run trivy scan on the image
       - name: Trivy scan
         if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}

.trivyignore

@@ -0,0 +1,9 @@
+# zlib version 1:1.3.dfsg+really1.3.1-1+b1 is installed from Debian Testing (Trixie) repository,
+# but Trivy assumes an older version of zlib because base image uses Debian Bookworm and
+# therefore raises the vulnerability alert CVE-2023-45853.
+# 
+# See this discussion about a similar issue: https://github.com/aquasecurity/trivy/discussions/6059
+# 
+# Ignoring this vulnerability since it is fixed in this PR: https://github.com/scaleoutsystems/fedn/pull/787
+# 
+CVE-2023-45853

Dockerfile

@@ -1,12 +1,21 @@
 # Stage 1: Builder
 ARG BASE_IMG=python:3.12-slim
-FROM $BASE_IMG as builder
+FROM $BASE_IMG AS builder
 
 ARG GRPC_HEALTH_PROBE_VERSION=""
 ARG REQUIREMENTS=""
 
 WORKDIR /build
 
+# Temporarily add the Debian Testing repository to install zlib1g 1:1.3.dfsg+really1.3.1-1+b1 (fixed CVE-2023-45853)
+# Both zlib1g and zlib1g-dev are installed in the builder stage.
+RUN echo "deb http://deb.debian.org/debian testing main" > /etc/apt/sources.list.d/testing.list \
+  && apt-get update \
+  && apt-get install -y --no-install-recommends -t testing zlib1g=1:1.3.dfsg+really1.3.1-1+b1 zlib1g-dev=1:1.3.dfsg+really1.3.1-1+b1 \
+  && rm -rf /etc/apt/sources.list.d/testing.list \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
+
 # Install build dependencies
 RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends python3-dev gcc wget \
   && rm -rf /var/lib/apt/lists/*
@@ -49,12 +58,19 @@ RUN set -ex \
   # Creare application specific tmp directory, set ENV TMPDIR to /app/tmp
   && mkdir -p /app/tmp \
   && chown -R appuser:appgroup /venv /app \
-  # Upgrade the package index and install security upgrades
+  # Temporarily add the Debian Testing repository to install zlib1g 1:1.3.dfsg+really1.3.1-1+b1 (fixed CVE-2023-45853)
+  && echo "deb http://deb.debian.org/debian testing main" > /etc/apt/sources.list.d/testing.list \
+  && apt-get update \
+  && apt-get install -y --no-install-recommends -t testing zlib1g=1:1.3.dfsg+really1.3.1-1+b1 \
+  && rm -rf /etc/apt/sources.list.d/testing.list \
+  # Update package index and upgrade all installed packages
   && apt-get update \
   && apt-get upgrade -y \
+  # Clean up
   && apt-get autoremove -y \
   && apt-get clean -y \
   && rm -rf /var/lib/apt/lists/*
+
 USER appuser
 
 ENTRYPOINT [ "/venv/bin/fedn" ]

Dockerfile.dev

@@ -0,0 +1,54 @@
+# Base image
+ARG BASE_IMG=python:3.10-slim
+FROM $BASE_IMG
+
+ARG GRPC_HEALTH_PROBE_VERSION=""
+
+# Requirements (use MNIST Keras as default)
+ARG REQUIREMENTS=""
+
+# Add FEDn and default configs
+COPY . /app
+COPY config/settings-client.yaml.template /app/config/settings-client.yaml
+COPY config/settings-combiner.yaml.template /app/config/settings-combiner.yaml
+COPY config/settings-hooks.yaml.template /app/config/settings-hooks.yaml
+COPY config/settings-reducer.yaml.template /app/config/settings-reducer.yaml
+COPY $REQUIREMENTS /app/config/requirements.txt
+
+# Install developer tools (needed for psutil)
+RUN apt-get update && apt-get install -y python3-dev gcc
+
+# Install grpc health probe checker
+RUN if [ ! -z "$GRPC_HEALTH_PROBE_VERSION" ]; then \
+  apt-get install -y wget && \
+  wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \
+  chmod +x /bin/grpc_health_probe && \
+  apt-get remove -y wget && apt autoremove -y; \
+  else \
+  echo "No grpc_health_probe version specified, skipping installation"; \
+  fi
+
+# Setup working directory
+WORKDIR /app
+
+# Create FEDn app directory
+SHELL ["/bin/bash", "-c"]
+RUN mkdir -p /app \
+  && mkdir -p /app/client \
+  && mkdir -p /app/certs \
+  && mkdir -p /app/client/package \
+  && mkdir -p /app/certs \
+  #
+  # Install FEDn and requirements
+  && python -m venv /venv \
+  && /venv/bin/pip install --upgrade pip \
+  && /venv/bin/pip install --no-cache-dir 'setuptools>=65' \
+  && /venv/bin/pip install --no-cache-dir -e . \
+  && if [[ ! -z "$REQUIREMENTS" ]]; then \
+  /venv/bin/pip install --no-cache-dir -r /app/config/requirements.txt; \
+  fi \
+  #
+  # Clean up
+  && rm -r /app/config/requirements.txt
+
+ENTRYPOINT [ "/venv/bin/fedn" ]

config/settings-combiner.yaml.local.template

@@ -10,6 +10,7 @@ cert_path: tmp/server.crt
 key_path: tmp/server.key
 
 statestore:
+  # Available DB types are MongoDB, PostgreSQL, SQLite
   type: MongoDB
   mongo_config:
     username: fedn_admin

config/settings-combiner.yaml.template

@@ -8,13 +8,19 @@ port: 12080
 max_clients: 30
 
 statestore:
+  # Available DB types are MongoDB, PostgreSQL, SQLite
   type: MongoDB
   mongo_config:
     username: fedn_admin
     password: password
     host: mongo
     port: 6534
-
+  postgres_config:
+    username: fedn_admin
+    password: password
+    host: fedn_postgres
+    port: 5432
+
 storage:
   storage_type: S3
   storage_config:

config/settings-controller.yaml.local.template

@@ -5,6 +5,7 @@ controller:
   debug: True
 
 statestore:
+  # Available DB types are MongoDB, PostgreSQL, SQLite
   type: MongoDB
   mongo_config:
     username: fedn_admin

config/settings-reducer.yaml.template

@@ -5,12 +5,18 @@ controller:
   debug: True
 
 statestore:
+  # Available DB types are MongoDB, PostgreSQL, SQLite
   type: MongoDB
   mongo_config:
     username: fedn_admin
     password: password
     host: mongo
     port: 6534
+  postgres_config:
+    username: fedn_admin
+    password: password
+    host: fedn_postgres
+    port: 5432
 
 storage:
   storage_type: S3

docker-compose.dev.yaml

@@ -0,0 +1,173 @@
+# Compose schema version
+version: '3.4'
+
+# Setup network
+networks:
+  default:
+    name: fedn_default
+
+services:
+  # Base services
+  minio:
+    image: minio/minio:14128-5ee91dc
+    hostname: minio
+    environment:
+      - GET_HOSTS_FROM=dns
+      - MINIO_HOST=minio
+      - MINIO_PORT=9000
+      - MINIO_ROOT_USER=fedn_admin
+      - MINIO_ROOT_PASSWORD=password
+    command: server /data --console-address minio:9001
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://minio:9000/minio/health/live" ]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+    ports:
+      - 9000:9000
+      - 9001:9001
+
+  mongo:
+    image: mongo:7.0
+    restart: always
+    environment:
+      - MONGO_INITDB_ROOT_USERNAME=fedn_admin
+      - MONGO_INITDB_ROOT_PASSWORD=password
+    ports:
+      - 6534:6534
+    command: mongod --port 6534
+
+  mongo-express:
+    image: mongo-express:latest
+    restart: always
+    depends_on:
+      - "mongo"
+    environment:
+      - ME_CONFIG_MONGODB_SERVER=mongo
+      - ME_CONFIG_MONGODB_PORT=6534
+      - ME_CONFIG_MONGODB_ADMINUSERNAME=fedn_admin
+      - ME_CONFIG_MONGODB_ADMINPASSWORD=password
+      - ME_CONFIG_BASICAUTH_USERNAME=fedn_admin
+      - ME_CONFIG_BASICAUTH_PASSWORD=password
+    ports:
+      - 8081:8081
+
+  fedn_postgres:
+    image: postgres:15
+    environment:
+      POSTGRES_USER: fedn_admin
+      POSTGRES_PASSWORD: password
+      POSTGRES_DB: fedn_db
+    ports:
+      - "5432:5432"
+
+  api-server:
+    environment:
+      - GET_HOSTS_FROM=dns
+      - USER=test
+      - PROJECT=project
+      - FLASK_DEBUG=1
+      - STATESTORE_CONFIG=/app/config/settings-reducer.yaml.template
+      - MODELSTORAGE_CONFIG=/app/config/settings-reducer.yaml.template
+      - FEDN_COMPUTE_PACKAGE_DIR=/app
+      - TMPDIR=/app/tmp
+    build:
+      context: .
+      dockerfile: Dockerfile.dev
+      args:
+        BASE_IMG: ${BASE_IMG:-python:3.12-slim}
+    working_dir: /app
+    volumes:
+      - ${HOST_REPO_DIR:-.}/fedn:/app/fedn
+    depends_on:
+      - minio
+      - mongo
+      - fedn_postgres
+    command:
+      - controller
+      - start
+    ports:
+      - 8092:8092
+
+  # Combiner
+  combiner:
+    environment:
+      - PYTHONUNBUFFERED=0
+      - GET_HOSTS_FROM=dns
+      - STATESTORE_CONFIG=/app/config/settings-combiner.yaml.template
+      - MODELSTORAGE_CONFIG=/app/config/settings-combiner.yaml.template
+      - HOOK_SERVICE_HOST=hook:12081
+      - TMPDIR=/app/tmp
+    build:
+      context: .
+      dockerfile: Dockerfile.dev
+      args:
+        BASE_IMG: ${BASE_IMG:-python:3.12-slim}
+        GRPC_HEALTH_PROBE_VERSION: v0.4.35
+    working_dir: /app
+    volumes:
+      - ${HOST_REPO_DIR:-.}/fedn:/app/fedn
+    command:
+      - combiner
+      - start
+      - --init
+      - config/settings-combiner.yaml.template
+    ports:
+      - 12080:12080
+    healthcheck:
+      test: [ "CMD", "/app/grpc_health_probe", "-addr=localhost:12080" ]
+      interval: 20s
+      timeout: 10s
+      retries: 5
+    depends_on:
+      - api-server
+      - hooks
+  # Hooks
+  hooks:
+    container_name: hook
+    environment:
+      - GET_HOSTS_FROM=dns
+      - TMPDIR=/app/tmp
+    build:
+      context: .
+      dockerfile: Dockerfile.dev
+      args:
+        BASE_IMG: ${BASE_IMG:-python:3.12-slim}
+        GRPC_HEALTH_PROBE_VERSION: v0.4.35
+    working_dir: /app
+    volumes:
+      - ${HOST_REPO_DIR:-.}/fedn:/app/fedn
+    entrypoint: [ "sh", "-c" ]
+    command:
+      - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/fedn hooks start"
+    ports:
+      - 12081:12081
+    healthcheck:
+      test: [ "CMD", "/bin/grpc_health_probe", "-addr=localhost:12081" ]
+      interval: 20s
+      timeout: 10s
+      retries: 5
+
+  # Client
+  client:
+    environment:
+      - GET_HOSTS_FROM=dns
+      - FEDN_PACKAGE_EXTRACT_DIR=package
+    build:
+      context: .
+      dockerfile: Dockerfile.dev
+      args:
+        BASE_IMG: ${BASE_IMG:-python:3.10-slim}
+    working_dir: /app
+    volumes:
+      - ${HOST_REPO_DIR:-.}/fedn:/app/fedn
+    command:
+      - client
+      - start
+      - --api-url
+      - http://api-server:8092
+    deploy:
+      replicas: 0
+    depends_on:
+      combiner:
+        condition: service_healthy

docker-compose.yaml

@@ -51,6 +51,15 @@ services:
       - ME_CONFIG_BASICAUTH_PASSWORD=password
     ports:
       - 8081:8081
+
+  fedn_postgres:
+    image: postgres:15
+    environment:
+      POSTGRES_USER: fedn_admin
+      POSTGRES_PASSWORD: password
+      POSTGRES_DB: fedn_db
+    ports:
+      - "5432:5432"
 
   api-server:
     environment:
@@ -72,6 +81,7 @@ services:
     depends_on:
       - minio
       - mongo
+      - fedn_postgres
     command:
       - controller
       - start