Skip to content

Commit

Permalink
Merge branch 'main' into semantic-partitioner
Browse files Browse the repository at this point in the history
  • Loading branch information
KarhouTam authored Oct 23, 2024
2 parents 7cc8acd + 76e1c28 commit e187c37
Show file tree
Hide file tree
Showing 723 changed files with 125,020 additions and 91,002 deletions.
8 changes: 8 additions & 0 deletions .editorconfig
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,14 @@ profile = black
indent_style = space
indent_size = 2

[*.md]
indent_style = space
indent_size = 2

[*.yml]
indent_style = space
indent_size = 2

[*.toml]
indent_style = space
indent_size = 4
11 changes: 10 additions & 1 deletion .github/CODEOWNERS
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,10 @@
README.md @jafermarq @tanertopal @danieljanes

# Flower Baselines
/baselines @jafermarq @tanertopal @danieljanes
/baselines @jafermarq @danieljanes

# Flower Benchmarks
/benchmarks @jafermarq @danieljanes

# Flower Datasets
/datasets @jafermarq @tanertopal @danieljanes
Expand All @@ -27,3 +30,9 @@ README.md @jafermarq @tanertopal @danieljanes
# GitHub Actions and Workflows
/.github/workflows @Robert-Steiner @tanertopal @danieljanes
/.github/actions @Robert-Steiner @tanertopal @danieljanes

# Docker-related files
/.devcontainer @Robert-Steiner @Moep90 @tanertopal @danieljanes
**/Dockerfile @Robert-Steiner @Moep90 @tanertopal @danieljanes
**/*.Dockerfile @Robert-Steiner @Moep90 @tanertopal @danieljanes
/src/docker @Robert-Steiner @Moep90 @tanertopal @danieljanes
2 changes: 1 addition & 1 deletion .github/actions/bootstrap/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ description: "Bootstrap Python environment (install and configure Python version
inputs:
python-version:
description: "Version range or exact version of Python or PyPy to use, using SemVer's version range syntax."
default: 3.8
default: 3.9
pip-version:
description: "Version of pip to be installed using pip"
default: 24.1.2
Expand Down
23 changes: 8 additions & 15 deletions .github/workflows/_docker-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,18 +36,16 @@ permissions:
jobs:
build:
name: Build image
runs-on: ubuntu-22.04
runs-on: ${{ matrix.platform.runner-os }}
timeout-minutes: 180
outputs:
build-id: ${{ steps.build-id.outputs.id }}
strategy:
fail-fast: true
matrix:
platform: [
# build-push action and qemu use different platform names
# therefore we create a map
{ name: "amd64", qemu: "", docker: "linux/amd64" },
{ name: "arm64", qemu: "arm64", docker: "linux/arm64" },
{ name: "amd64", docker: "linux/amd64", runner-os: "ubuntu-22.04" },
{ name: "arm64", docker: "linux/arm64", runner-os: "ubuntu-4-core-arm64" },
]
steps:
- name: Create build id
Expand Down Expand Up @@ -79,20 +77,14 @@ jobs:
print(build_args, file=fh)
print("EOF", file=fh)
- name: Set up QEMU
if: matrix.platform.qemu != ''
uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0
with:
platforms: ${{ matrix.platform.qemu }}

- name: Extract metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
with:
images: ${{ inputs.namespace-repository }}

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3.6.1
uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1

- name: Login to Docker Hub
uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
Expand All @@ -104,11 +96,12 @@ jobs:
uses: Wandalen/wretry.action@6feedb7dedadeb826de0f45ff482b53b379a7844 # v3.5.0
id: build
with:
action: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
action: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0
attempt_limit: 60 # 60 attempts * (9 secs delay + 1 sec retry) = ~10 mins
attempt_delay: 9000 # 9 secs
with: |
pull: true
sbom: true
platforms: ${{ matrix.platform.docker }}
context: "{{defaultContext}}:${{ inputs.file-dir }}"
outputs: type=image,name=${{ inputs.namespace-repository }},push-by-digest=true,name-canonical=true,push=true
Expand All @@ -122,7 +115,7 @@ jobs:
touch "/tmp/digests/${digest#sha256:}"
- name: Upload digest
uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
with:
name: digests-${{ steps.build-id.outputs.id }}-${{ matrix.platform.name }}
path: /tmp/digests/*
Expand Down Expand Up @@ -152,7 +145,7 @@ jobs:
tags: ${{ inputs.tags }}

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3.6.1
uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1

- name: Login to Docker Hub
uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/cache-cleanup.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ jobs:
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: 3.8
python-version: 3.9

- name: Cleanup caches by directories
# Only keep caches that match the latest keys for each directory
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/datasets-e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ jobs:
- name: Bootstrap
uses: ./.github/actions/bootstrap
with:
python-version: 3.8
python-version: 3.9
- name: Install dependencies
run: python -m poetry install
- name: Run tests
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/datasets.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ jobs:
# In case of a mismatch, the job has to download Python to install it.
# Note: Due to a bug in actions/setup-python, we have to put "3.10" in
# quotes as it will otherwise assume "3.1"
python: [3.8, 3.9, '3.10', '3.11']
python: ['3.9', '3.10', '3.11']

name: Python ${{ matrix.python }}

Expand Down
69 changes: 69 additions & 0 deletions .github/workflows/docker-build-main.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
name: Build Docker Images Main Branch

on:
push:
branches:
- 'main'

jobs:
parameters:
if: github.repository == 'adap/flower'
name: Collect docker build parameters
runs-on: ubuntu-22.04
timeout-minutes: 10
outputs:
pip-version: ${{ steps.versions.outputs.pip-version }}
setuptools-version: ${{ steps.versions.outputs.setuptools-version }}
flwr-version-ref: ${{ steps.versions.outputs.flwr-version-ref }}
steps:
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

- uses: ./.github/actions/bootstrap
id: bootstrap

- id: versions
run: |
echo "pip-version=${{ steps.bootstrap.outputs.pip-version }}" >> "$GITHUB_OUTPUT"
echo "setuptools-version=${{ steps.bootstrap.outputs.setuptools-version }}" >> "$GITHUB_OUTPUT"
echo "flwr-version-ref=git+${{ github.server_url }}/${{ github.repository }}.git@${{ github.sha }}" >> "$GITHUB_OUTPUT"
build-docker-base-images:
name: Build base images
if: github.repository == 'adap/flower'
uses: ./.github/workflows/_docker-build.yml
needs: parameters
with:
namespace-repository: flwr/base
file-dir: src/docker/base/ubuntu
build-args: |
PIP_VERSION=${{ needs.parameters.outputs.pip-version }}
SETUPTOOLS_VERSION=${{ needs.parameters.outputs.setuptools-version }}
FLWR_VERSION_REF=${{ needs.parameters.outputs.flwr-version-ref }}
tags: unstable
secrets:
dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }}
dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}

build-docker-binary-images:
name: Build binary images
if: github.repository == 'adap/flower'
uses: ./.github/workflows/_docker-build.yml
needs: build-docker-base-images
strategy:
fail-fast: false
matrix:
images: [
{ repository: "flwr/superlink", file_dir: "src/docker/superlink" },
{ repository: "flwr/supernode", file_dir: "src/docker/supernode" },
{ repository: "flwr/serverapp", file_dir: "src/docker/serverapp" },
{ repository: "flwr/superexec", file_dir: "src/docker/superexec" },
{ repository: "flwr/clientapp", file_dir: "src/docker/clientapp" }
]
with:
namespace-repository: ${{ matrix.images.repository }}
file-dir: ${{ matrix.images.file_dir }}
build-args: BASE_IMAGE=unstable
tags: unstable
secrets:
dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }}
dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
51 changes: 51 additions & 0 deletions .github/workflows/docker-readme.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
name: Update Docker READMEs

on:
push:
branches:
- 'main'
paths:
- 'src/docker/**/README.md'

jobs:
collect:
if: ${{ github.repository == 'adap/flower' }}
name: Collect Docker READMEs
runs-on: ubuntu-22.04
timeout-minutes: 10
outputs:
readme_files: ${{ steps.filter.outputs.readme_files }}
steps:
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

- uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
id: filter
with:
list-files: "json"
filters: |
readme:
- 'src/docker/**/README.md'
update:
if: ${{ needs.collect.outputs.readme_files != '' && toJson(fromJson(needs.collect.outputs.readme_files)) != '[]' }}
name: Update Docker READMEs
runs-on: ubuntu-22.04
timeout-minutes: 10
needs: collect
strategy:
matrix:
readme_path: ${{ fromJSON(needs.collect.outputs.readme_files) }}

steps:
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

- id: repository
run: echo "name=$(basename $(dirname ${{ matrix.readme_path }}))" >> "$GITHUB_OUTPUT"

- name: Docker Hub Description
uses: peter-evans/dockerhub-description@e98e4d1628a5f3be2be7c231e50981aee98723ae # v4.0.0
with:
repository: flwr/${{ steps.repository.outputs.name }}
readme-filepath: ${{ matrix.readme_path }}
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
Loading

0 comments on commit e187c37

Please sign in to comment.