Merge branch 'main' into dependabot/pip/rope-eq-1.13.0

.devcontainer/Dockerfile

@@ -29,8 +29,8 @@ RUN apt-get install -y curl wget gnupg python3 python-is-python3 python3-pip git
     build-essential tmux vim
 
 RUN python -m pip install \
-    pip==23.3.1 \
-    setuptools==68.2.2 \
+    pip==24.1.2 \
+    setuptools==70.3.0 \
     poetry==1.7.1
 
 USER $USERNAME

.editorconfig

@@ -16,6 +16,14 @@ profile = black
 indent_style = space
 indent_size = 2
 
+[*.md]
+indent_style = space
+indent_size = 2
+
 [*.yml]
 indent_style = space
 indent_size = 2
+
+[*.toml]
+indent_style = space
+indent_size = 4

.github/CODEOWNERS

@@ -1,23 +1,38 @@
 # Details: https://help.github.com/en/articles/about-code-owners
 
 # Default code owners
-* @danieljanes @tanertopal
+* @tanertopal @danieljanes
 
 # README.md
 README.md @jafermarq @tanertopal @danieljanes
 
 # Flower Baselines
 /baselines @jafermarq @tanertopal @danieljanes
 
+# Flower Benchmarks
+/benchmarks @jafermarq @tanertopal @danieljanes
+
 # Flower Datasets
 /datasets @jafermarq @tanertopal @danieljanes
 
 # Flower Examples
 /examples @jafermarq @tanertopal @danieljanes
 
+# Flower Templates
+/src/py/flwr/cli/new/templates @jafermarq @tanertopal @danieljanes
+
 # Changelog
-/doc/source/ref-changelog.md @jafermarq @tanertopal @danieljanes
+/framework/docs/source/ref-changelog.md @jafermarq @tanertopal @danieljanes
+
+# Translations
+/framework/docs/locales @charlesbvll @tanertopal @danieljanes
 
 # GitHub Actions and Workflows
 /.github/workflows @Robert-Steiner @tanertopal @danieljanes
 /.github/actions @Robert-Steiner @tanertopal @danieljanes
+
+# Docker-related files
+/.devcontainer @Robert-Steiner @Moep90 @tanertopal @danieljanes
+**/Dockerfile @Robert-Steiner @Moep90 @tanertopal @danieljanes
+**/*.Dockerfile @Robert-Steiner @Moep90 @tanertopal @danieljanes
+/src/docker @Robert-Steiner @Moep90 @tanertopal @danieljanes

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,6 +1,7 @@
 <!--
 Thank you for opening a pull request (PR)!
 
+Please rename your PRs following this [format](https://flower.ai/docs/framework/contributor-tutorial-contribute-on-github.html#pr-title-format).
 Contribution guidelines: https://github.com/adap/flower/blob/main/CONTRIBUTING.md
 -->
 
@@ -37,29 +38,9 @@ Example: The variable `rnd` was renamed to `server_round` to improve readability
 - [ ] Implement proposed change
 - [ ] Write tests
 - [ ] Update [documentation](https://flower.ai/docs/writing-documentation.html)
-- [ ] Update the changelog entry below
 - [ ] Make CI checks pass
 - [ ] Ping maintainers on [Slack](https://flower.ai/join-slack/) (channel `#contributions`)
 
-<!--
-Inside the following 'Changelog entry' section, you should put the description of your changes that will be added to the changelog alongside your PR title.
-
-If the section is completely empty (without any token) or non-existant, the changelog will just contain the title of the PR for the changelog entry, without any description. If the section contains some text other than tokens, it will use it to add a description to the change. If the section contains one of the following tokens it will ignore any other text and put the PR under the corresponding section of the changelog:
-
-<general> is for classifying a PR as a general improvement.
-<skip> is to not add the PR to the changelog
-<baselines> is to add a general baselines change to the PR
-<examples> is to add a general examples change to the PR
-<sdk> is to add a general sdk change to the PR
-<simulations> is to add a general simulations change to the PR
-
-Note that only one token should be used.
--->
-
-### Changelog entry
-
-
-
 ### Any other comments?
 
 <!--

.github/actions/bootstrap/action.yml

@@ -3,16 +3,20 @@ description: "Bootstrap Python environment (install and configure Python version
 inputs:
   python-version:
     description: "Version range or exact version of Python or PyPy to use, using SemVer's version range syntax."
-    default: 3.8
+    default: 3.9
   pip-version:
     description: "Version of pip to be installed using pip"
-    default: 23.3.1
+    default: 24.1.2
   setuptools-version:
     description: "Version of setuptools to be installed using pip"
-    default: 68.2.2
+    default: 70.3.0
   poetry-version:
     description: "Version of poetry to be installed using pip"
     default: 1.7.1
+  poetry-skip:
+    description: "Option to skip the poetry installation"
+    required: false
+    default: "false"
 outputs:
   python-version:
     description: "Version range or exact version of Python or PyPy"
@@ -35,10 +39,12 @@ runs:
         python-version: ${{ inputs.python-version }}
     - name: Install build tools
       run: |
-        python -m pip install -U pip==${{ inputs.pip-version }}
-        python -m pip install -U setuptools==${{ inputs.setuptools-version }}
-        python -m pip install -U poetry==${{ inputs.poetry-version }}
-        python -m poetry config virtualenvs.create false
+        if ${{ inputs.poetry-skip == 'false' }}; then
+          python -m pip install -U pip==${{ inputs.pip-version }}
+          python -m pip install -U setuptools==${{ inputs.setuptools-version }}
+          python -m pip install -U poetry==${{ inputs.poetry-version }}
+          python -m poetry config virtualenvs.create false
+        fi
       shell: bash
     - id: versions
       shell: bash

.github/workflows/_docker-build.yml

@@ -36,18 +36,16 @@ permissions:
 jobs:
   build:
     name: Build image
-    runs-on: ubuntu-22.04
-    timeout-minutes: 60
+    runs-on: ${{ matrix.platform.runner-os }}
+    timeout-minutes: 180
     outputs:
       build-id: ${{ steps.build-id.outputs.id }}
     strategy:
       fail-fast: true
       matrix:
         platform: [
-            # build-push action and qemu use different platform names
-            # therefore we create a map
-            { name: "amd64", qemu: "", docker: "linux/amd64" },
-            { name: "arm64", qemu: "arm64", docker: "linux/arm64" },
+            { name: "amd64", docker: "linux/amd64", runner-os: "ubuntu-22.04" },
+            { name: "arm64", docker: "linux/arm64", runner-os: "ubuntu-4-core-arm64" },
           ]
     steps:
       - name: Create build id
@@ -60,47 +58,64 @@ jobs:
           hash = hashlib.sha256('''${{ inputs.namespace-repository }}
           ${{ inputs.file-dir }}
           ${{ inputs.build-args }}'''.encode())
+          # Adds two spaces to the line breaks to ensure proper indentation
+          # when passing the multi-line string to the wretry.action.
+          # Without it, the multi-line string is passed like this:
+          #
+          # build-args: |
+          #   ARG1=
+          # ARG2=
+          # ARG3=
+          #
+          # This causes the Docker action to interpret ARG2 and ARG3 as keys instead
+          # of values ​​of the multi-line string.
+          build_args = '''${{ inputs.build-args }}'''.replace("\n", "\n  ")
+
           with open(os.environ['GITHUB_OUTPUT'], 'a') as fh:
               print(f"id={hash.hexdigest()}", file=fh)
-
-      - name: Set up QEMU
-        if: matrix.platform.qemu != ''
-        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
-        with:
-          platforms: ${{ matrix.platform.qemu }}
+              print("build-args<<EOF", file=fh)
+              print(build_args, file=fh)
+              print("EOF", file=fh)
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
+        uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1
         with:
           images: ${{ inputs.namespace-repository }}
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0
+        uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5 # v3.8.0
 
       - name: Login to Docker Hub
-        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
         with:
           username: ${{ secrets.dockerhub-user }}
           password: ${{ secrets.dockerhub-token }}
 
       - name: Build and push
+        uses: Wandalen/wretry.action@ffdd254f4eaf1562b8a2c66aeaa37f1ff2231179 # v3.7.3
         id: build
-        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
         with:
-          platforms: ${{ matrix.platform.docker }}
-          context: "{{defaultContext}}:${{ inputs.file-dir }}"
-          build-args: ${{ inputs.build-args }}
-          outputs: type=image,name=${{ inputs.namespace-repository }},push-by-digest=true,name-canonical=true,push=true
+          action: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75 # v6.9.0
+          attempt_limit: 60 # 60 attempts * (9 secs delay + 1 sec retry) = ~10 mins
+          attempt_delay: 9000 # 9 secs
+          with: |
+            pull: true
+            sbom: true
+            platforms: ${{ matrix.platform.docker }}
+            context: "{{defaultContext}}:${{ inputs.file-dir }}"
+            outputs: type=image,name=${{ inputs.namespace-repository }},push-by-digest=true,name-canonical=true,push=true
+            build-args: |
+              ${{ steps.build-id.outputs.build-args }}
 
       - name: Export digest
         run: |
           mkdir -p /tmp/digests
-          digest="${{ steps.build.outputs.digest }}"
+          digest="${{ fromJSON(steps.build.outputs.outputs).digest }}"
           touch "/tmp/digests/${digest#sha256:}"
 
       - name: Upload digest
-        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
         with:
           name: digests-${{ steps.build-id.outputs.id }}-${{ matrix.platform.name }}
           path: /tmp/digests/*
@@ -116,24 +131,24 @@ jobs:
       metadata: ${{ steps.meta.outputs.json }}
     steps:
       - name: Download digests
-        uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
         with:
           pattern: digests-${{ needs.build.outputs.build-id }}-*
           path: /tmp/digests
           merge-multiple: true
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
+        uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1
         with:
           images: ${{ inputs.namespace-repository }}
           tags: ${{ inputs.tags }}
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0
+        uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5 # v3.8.0
 
       - name: Login to Docker Hub
-        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
         with:
           username: ${{ secrets.dockerhub-user }}
           password: ${{ secrets.dockerhub-token }}

.github/workflows/baselines.yml

@@ -36,7 +36,7 @@ jobs:
                 FILTER+=$(echo "$DIR: ${BASELINES_PATH}/**\n")
             done < <(find baselines -maxdepth 1 \
                 -name ".*" -prune -o \
-                -path "baselines/doc" -prune -o \
+                -path "baselines/docs" -prune -o \
                 -path "baselines/dev" -prune -o \
                 -path "baselines/baseline_template" -prune -o \
                 -path "baselines/flwr_baselines" -prune -o \

.github/workflows/cache-cleanup.yml

@@ -0,0 +1,63 @@
+name: Cache cleanup
+
+on:
+  workflow_dispatch
+
+jobs:
+  e2e-cleanup:
+    runs-on: ubuntu-22.04
+    timeout-minutes: 10
+    # Using approach described here:
+    # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs
+    strategy:
+      matrix:
+        include:
+          - directory: bare
+          - directory: bare-https
+          - directory: bare-client-auth
+          - directory: jax
+          - directory: pytorch
+          - directory: tensorflow
+          - directory: tabnet
+          - directory: opacus
+          - directory: pytorch-lightning
+          - directory: scikit-learn
+          - directory: fastai
+          - directory: pandas
+
+    name: Framework / ${{ matrix.directory }}
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.9
+
+      - name: Cleanup caches by directories
+        # Only keep caches that match the latest keys for each directory
+        run: |
+          gh extension install actions/gh-actions-cache
+          
+          REPO=${{ github.repository }}
+          LATEST_KEY=pythonloc-${{ matrix.directory }}-${{ env.pythonLocation }}-${{ hashFiles(format('./e2e/{0}/pyproject.toml', matrix.directory)) }}
+
+          echo "Fetching list of cache keys"
+          cacheKeys=$(gh actions-cache list -R $REPO | grep "${{ matrix.directory }}" | cut -f 1 )
+
+          ## Setting this to not fail the workflow while deleting cache keys. 
+          set +e
+          echo "Deleting caches..."
+          for cacheKey in $cacheKeys
+          do
+            if [ "$cacheKey" != "$LATEST_KEY" ]; then
+              echo -e "\tOld key found -> $cacheKey"
+              gh actions-cache delete "$cacheKey" --confirm
+            fi
+          done
+
+          echo "Done"
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/datasets-e2e.yml

@@ -45,7 +45,7 @@ jobs:
       - name: Bootstrap
         uses: ./.github/actions/bootstrap
         with:
-          python-version: 3.8
+          python-version: 3.9
       - name: Install dependencies
         run: python -m poetry install
       - name: Run tests

.github/workflows/datasets.yml

@@ -6,16 +6,23 @@ on:
       - main
     paths:
       - "datasets/**"
+      - ".github/workflows/datasets.yml"
+      - ".github/workflows/datasets-e2e.yml"
   pull_request:
     branches:
       - main
     paths:
       - "datasets/**"
+      - ".github/workflows/datasets.yml"
+      - ".github/workflows/datasets-e2e.yml"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
+env:
+  FLWR_TELEMETRY_ENABLED: 0
+
 defaults:
   run:
     working-directory: datasets
@@ -28,9 +35,9 @@ jobs:
         # Latest version which comes cached in the host image can be found here:
         # https://github.com/actions/runner-images/blob/main/images/linux/Ubuntu2204-Readme.md#python
         # In case of a mismatch, the job has to download Python to install it.
-        # Note: Due to a bug in actions/setup-python we have to put 3.10 in
-        # qoutes as it will otherwise will assume 3.1
-        python: [3.8, 3.9, '3.10']
+        # Note: Due to a bug in actions/setup-python, we have to put "3.10" in
+        # quotes as it will otherwise assume "3.1"
+        python: ['3.9', '3.10', '3.11']
 
     name: Python ${{ matrix.python }}