Skip to content

Commit

Permalink
Merge pull request #181 from OmegaSquad82/rechunk
Browse files Browse the repository at this point in the history 
fix: aligned executed times to execute sequentially after build
  • Loading branch information
@OmegaSquad82
OmegaSquad82 authored Feb 6, 2025
2 parents 75765b2 + e6b6127 commit 847afb4
Show file tree
Hide file tree
Showing 4 changed files with 78 additions and 252 deletions.
129 changes: 12 additions & 117 deletions .github/workflows/build-image.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,37 +3,30 @@ on:
workflow_dispatch: # allow manually triggering builds
push:
paths-ignore:
- '**/release-please.yml'
- '**/recompress-latest.yml'
- '**/vulnerability-scan.yml'
- '**/composition-analysis.yml'
- "**/release-please.yml"
- "**/recompress-latest.yml"
- "**/vulnerability-scan.yml"
- "**/composition-analysis.yml"
- "**.md"
- "**/.gitkeep"
- "build-isos"
- "renovate.json"
schedule:
- cron: "10 06 * * 1,3,5" # build at 06:10 UTC on Mon, Wed, Fr
- cron:
"10 06 * * 1,3,5" # build at 06:10 UTC on Mon, Wed, Fr
# (30 minutes after last ublue images start building)

concurrency:
group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
cancel-in-progress: true

env:
IMAGE_REGISTRY: ghcr.io/${{ github.repository_owner }}
IMAGE_TAGS: latest
REGISTRY_USER: ${{ github.actor }}
REGISTRY_PASSWORD: ${{ github.token }}
ZSTD_CHUNKED: true

jobs:
imagebuild:
name: Build Custom Image
runs-on: ubuntu-24.04
permissions:
contents: read
packages: write
security-events: write
strategy:
fail-fast: true
matrix:
Expand All @@ -43,7 +36,6 @@ jobs:
- carbonatcyanotrichit
- flaviramea
steps:

# Gimme more room to unfold
- name: Maximize build space
uses: easimon/maximize-build-space@master
Expand All @@ -70,20 +62,16 @@ jobs:
# https://github.com/radxa/armbian-compile-action/commit/82fe40188f2f3b4dbd4ae8fb9c64f49f117085c7
sudo mkfs.btrfs -L buildlv -M -m single --force "${LV_NAME}"
sudo mount -o defaults,noatime,compress=zstd:3,discard=sync "${LV_NAME}" "${GITHUB_WORKSPACE}"
sudo mount -o defaults,noatime,compress=zstd:1,discard=sync "${LV_NAME}" "${GITHUB_WORKSPACE}"
sudo chown -R "$WORKSPACE_OWNER" "${GITHUB_WORKSPACE}"
# Create bootc image
- name: Checkout codeZ
uses: actions/checkout@v4

- name: Get default branch name
run: echo "DEFAULT_BRANCH=${{ github.event.repository.default_branch }}" >$GITHUB_ENV


- name: Check if on default (main) or test (rechunk) branch to rechunk it
run: |
if [[ '${{ github.ref_name }}' = "$DEFAULT_BRANCH" || '${{ github.ref_name }}' = 'rechunk' ]]; then
if [[ '${{ github.ref_name }}' = "${{ github.event.repository.default_branch }}" || '${{ github.ref_name }}' = 'rechunk' ]]; then
OUT='IS_DEFAULT=true'
else
OUT='IS_DEFAULT=false'
Expand All @@ -92,108 +80,15 @@ jobs:
- name: Build Custom Image
uses: blue-build/[email protected]
id: build
with:
cosign_private_key: ${{ secrets.SIGNING_SECRET }}
maximize_build_space: ${{ env.IS_DEFAULT }}
pr_event_number: ${{ github.event.number }}
rechunk: ${{ env.IS_DEFAULT }}
recipe: recipe-${{ matrix.package }}.yml
registry_token: ${{ github.token }}
build_opts: |
--compression-format zstd
# Workaround bug where capital letters in your GitHub username make it impossible to push to GHCR.
# https://github.com/macbre/push-to-ghcr/issues/12
- name: Lowercase Registry
uses: ASzc/change-string-case-action@v6
if: env.IS_DEFAULT == 'true'
id: registry_case
with:
string: ${{ env.IMAGE_REGISTRY }}

- name: Composite image name
if: env.IS_DEFAULT == 'true'
run: echo "IMAGE_NAME=${{ steps.registry_case.outputs.lowercase }}/${{ matrix.package }}" >$GITHUB_ENV

# This is something the community works for quite a long time
# but it likely will improve the efficiency of transfers;
# this was made to workaround the fact that blue-build
# does not yet support zstd:chunked and it depends
# on bootc/rpm-ostree having implemented this.
# No OS image must be pushed in this form,
# except to non-productive image tags.
#
# Changetracker
# https://fedoraproject.org/wiki/Changes/zstd:chunked
# https://github.com/containers/storage/pull/775
# Documentation
# https://universal-blue.discourse.group/t/zst-chunked-faq/3462
# Projectstuffz
# https://github.com/blue-build/cli/issues/112
# https://github.com/containers/bootc/issues/20
# https://github.com/containers/bootc/issues/963
- name: bootc does not support zstd:chunked atm
run: |
if [[ '${{ env.IS_DEFAULT }}' = 'true' && '${{ env.ZSTD_CHUNKED }}' = 'true' ]]; then
OUT='DO_ZSTD_CHUNKED=true'
else
OUT='DO_ZSTD_CHUNKED=false'
fi
echo "$OUT" >$GITHUB_ENV
# Compression overdrive
- name: Tag latest as latest-zstchd
if: env.DO_ZSTD_CHUNKED == 'true'
id: pull
run: |
set -eux pipefail
IMAGES=()
podman login \
--username ${{ env.REGISTRY_USER }} \
--password ${{ env.REGISTRY_PASSWORD }} \
${{ steps.registry_case.outputs.lowercase }}
for tag in ${{ env.IMAGE_TAGS }}; do
podman pull "${{ env.IMAGE_NAME }}:$tag"
podman tag "${{ env.IMAGE_NAME }}:$tag" "${{ env.IMAGE_NAME }}:${tag}-zstchd"
IMAGES+=("${{ env.IMAGE_NAME }}:${tag}-zstchd") # zstd:chunked
done
echo "ALL_TAGS=${IMAGES[@]}" >$GITHUB_ENV
- name: Push To GHCR with Podman in zstd:chunked
uses: redhat-actions/push-to-registry@v2
if: env.DO_ZSTD_CHUNKED == 'true'
id: push
with:
tags: ${{ env.ALL_TAGS }}
username: ${{ env.REGISTRY_USER }}
password: ${{ env.REGISTRY_PASSWORD }}
extra-args: |
--compression-format=zstd:chunked
# Sign container
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
if: env.DO_ZSTD_CHUNKED == 'true'
with:
registry: ghcr.io
username: ${{ env.REGISTRY_USER }}
password: ${{ env.REGISTRY_PASSWORD }}

- uses: sigstore/[email protected]
if: env.DO_ZSTD_CHUNKED == 'true'

- name: Sign container image
if: env.DO_ZSTD_CHUNKED == 'true'
run: |
echo "${{ env.COSIGN_PRIVATE_KEY }}" >cosign.key
wc -c cosign.key
cosign sign -y --key cosign.key ${{ env.IMAGE_NAME }}@${TAGS}
env:
COSIGN_EXPERIMENTAL: false
COSIGN_PRIVATE_KEY: ${{ secrets.SIGNING_SECRET }}
TAGS: ${{ steps.push.outputs.digest }}

- name: Echo outputs
if: env.DO_ZSTD_CHUNKED == 'true'
- name: Print outputs
run: |
echo "${{ toJSON(steps.push.outputs) }}"
echo '${{ toJSON(steps.build.outputs) }}' | jq .
92 changes: 18 additions & 74 deletions .github/workflows/composition-analysis.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,31 +3,31 @@ on:
workflow_dispatch: # allow manually triggering builds
push:
paths-ignore:
- '**/build-image.yml'
- '**/release-please.yml'
- '**/recompress-latest.yml'
- '**/vulnerability-scan.yml'
- "**/build-image.yml"
- "**/release-please.yml"
- "**/recompress-latest.yml"
- "**/vulnerability-scan.yml"
- "**.md"
- "**/.gitkeep"
- "build-isos"
- "renovate.json"
schedule:
- cron: "10 07 * * 1,3,5" # scan at 06:10 UTC on Mon, Wed, Fr
- cron:
"00 07 * * 1,3,5" # scan at 07:00 UTC on Mon, Wed, Fr
# (1 hour after bluebuild images started building)

concurrency:
group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
cancel-in-progress: true

env:
IMAGE_REGISTRY: ghcr.io/${{ github.repository_owner }}
IMAGE_TAGS: latest
REGISTRY_USER: ${{ github.actor }}
REGISTRY_USERNAME: ${{ github.actor }}
REGISTRY_PASSWORD: ${{ github.token }}
REGISTRY_IMG_PATH: ghcr.io/${{ github.repository_owner }}

jobs:
analysis:
name: Scan Custom Image
name: Analyze Custom Image
runs-on: ubuntu-24.04
permissions:
contents: read
Expand All @@ -42,86 +42,30 @@ jobs:
- carbonatcyanotrichit
- flaviramea
steps:

# Gimme more room to unfold
- name: Maximize build space
uses: easimon/maximize-build-space@master
with:
overprovision-lvm: true
remove-android: true
remove-codeql: true
remove-docker-images: true
remove-dotnet: true
remove-haskell: true
- name: Convert to compressed filesystem
shell: bash
run: |
set -eux pipefail
# https://github.com/easimon/maximize-build-space/issues/23
# https://github.com/easimon/maximize-build-space/pull/37
WORKSPACE_OWNER="$(stat -c '%U:%G' "${GITHUB_WORKSPACE}")"
VG_NAME=buildvg
LV_NAME="/dev/mapper/${VG_NAME}-buildlv"
cd ~
sudo umount "${LV_NAME}"
# https://github.com/radxa/armbian-compile-action/commit/82fe40188f2f3b4dbd4ae8fb9c64f49f117085c7
sudo mkfs.btrfs -L buildlv -M -m single --force "${LV_NAME}"
sudo mount -o defaults,noatime,compress=zstd:3,discard=sync "${LV_NAME}" "${GITHUB_WORKSPACE}"
sudo chown -R "$WORKSPACE_OWNER" "${GITHUB_WORKSPACE}"
# Prepare analysis
- name: Checkout codeZ
uses: actions/checkout@v4

- name: Get default branch name
run: echo "DEFAULT_BRANCH=${{ github.event.repository.default_branch }}" >$GITHUB_ENV

- name: Check if on default (main) or test (rechunk) branch to rechunk it
run: |
if [[ '${{ github.ref_name }}' = "$DEFAULT_BRANCH" || '${{ github.ref_name }}' = 'analyze' ]]; then
OUT='IS_DEFAULT=true'
else
OUT='IS_DEFAULT=false'
fi
echo "$OUT" >$GITHUB_ENV
# Workaround bug where capital letters in your GitHub username make it impossible to push to GHCR.
# https://github.com/macbre/push-to-ghcr/issues/12
- name: Lowercase Registry
uses: ASzc/change-string-case-action@v6
id: case
with:
string: ${{ env.IMAGE_REGISTRY }}

- name: Composite image name
run: echo "IMAGE_NAME=${{ steps.case.outputs.lowercase }}/${{ matrix.package }}:$IMAGE_TAGS" >$GITHUB_ENV
- name: Construct lowercase image reference
run: echo "IMAGE_REF=${REGISTRY_IMG_PATH@L}/${{ matrix.package }}:latest" >> ${GITHUB_ENV}

# SCA, Vuln and License scanning
- name: Run GitHub Action for ORT
uses: oss-review-toolkit/ort-ci-github-action@v1
id: ort
with:
sw-name: ${{ matrix.package }}

- name: Echo outputs
run: |
echo "${{ toJSON(steps.ort.outputs) }}"
- uses: philips-labs/[email protected]
- name: Run GitHub Action for Tern
uses: philips-labs/[email protected]
id: tern
with:
image: ${{ env.IMAGE_NAME }}
image: ${{ env.IMAGE_REF }}

- uses: actions/upload-artifact@v4
with:
name: ${{ matrix.package }}
name: ${{ matrix.package }}
path: ${{ steps.tern.outputs.file }}

- name: Echo outputs
- name: Print outputs
run: |
echo "${{ toJSON(steps.ort.outputs) }}"
echo "${{ toJSON(steps.tern.outputs) }}"
echo '${{ toJSON(steps.ort.outputs) }}' | jq .
echo '${{ toJSON(steps.tern.outputs) }}' | jq .
Loading

0 comments on commit 847afb4

Please sign in to comment.