From 199197deeda9ee70437f084c19027a242081d3a5 Mon Sep 17 00:00:00 2001 From: OmegaSquad82 <34405234+OmegaSquad82@users.noreply.github.com> Date: Wed, 5 Feb 2025 22:18:24 +0100 Subject: [PATCH 01/11] feat: add philips-labs/tern-action@v1.1.0 Signed-off-by: OmegaSquad82 <34405234+OmegaSquad82@users.noreply.github.com> --- .github/workflows/build-image.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 60ac318..781412d 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -201,3 +201,11 @@ jobs: uses: oss-review-toolkit/ort-ci-github-action@v1 with: sw-name: ${{ matrix.package }} + - uses: philips-labs/tern-action@v1.1.0 + id: tern + with: + image: ${{ env.ALL_TAGS }} + - uses: actions/upload-artifact@v2 + with: + name: ${{ matrix.package }} + path: ${{ steps.tern.outputs.file }} From 0a5acdb8b183c04d58a98e8cea2908211eea95eb Mon Sep 17 00:00:00 2001 From: OmegaSquad82 <34405234+OmegaSquad82@users.noreply.github.com> Date: Wed, 5 Feb 2025 23:11:57 +0100 Subject: [PATCH 02/11] fix: GitHub blocks actions/upload-artifact@v2 so use v4 Signed-off-by: OmegaSquad82 <34405234+OmegaSquad82@users.noreply.github.com> --- .github/workflows/build-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 781412d..2f25532 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -205,7 +205,7 @@ jobs: id: tern with: image: ${{ env.ALL_TAGS }} - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 with: name: ${{ matrix.package }} path: ${{ steps.tern.outputs.file }} From 1b4ec5ea31bb81eb128e294bf17064b485fb3548 Mon Sep 17 00:00:00 2001 From: OmegaSquad82 <34405234+OmegaSquad82@users.noreply.github.com> Date: Wed, 5 Feb 2025 23:25:57 +0100 Subject: [PATCH 03/11] fix: hand over full image name feat: only run software composition analysis in default branch Signed-off-by: OmegaSquad82 <34405234+OmegaSquad82@users.noreply.github.com> --- .github/workflows/build-image.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 2f25532..f61dc43 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -196,16 +196,21 @@ jobs: run: | echo "${{ toJSON(steps.push.outputs) }}" - # Free and Open Source Software license scanning + # SCA, Vuln and License scanning - name: Run GitHub Action for ORT + if: env.IS_DEFAULT == 'true' uses: oss-review-toolkit/ort-ci-github-action@v1 with: sw-name: ${{ matrix.package }} + - uses: philips-labs/tern-action@v1.1.0 + if: env.IS_DEFAULT == 'true' id: tern with: - image: ${{ env.ALL_TAGS }} + image: ${{ env.IMAGE_NAME }} + - uses: actions/upload-artifact@v4 + if: env.IS_DEFAULT == 'true' with: name: ${{ matrix.package }} path: ${{ steps.tern.outputs.file }} From b29154119ed8cb3eec5392da7009b4eb0b247bd7 Mon Sep 17 00:00:00 2001 From: OmegaSquad82 <34405234+OmegaSquad82@users.noreply.github.com> Date: Thu, 6 Feb 2025 00:03:12 +0100 Subject: [PATCH 04/11] feat: separate composition-analysis workflow Signed-off-by: OmegaSquad82 <34405234+OmegaSquad82@users.noreply.github.com> --- .github/workflows/composition-analysis | 126 +++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 .github/workflows/composition-analysis diff --git a/.github/workflows/composition-analysis b/.github/workflows/composition-analysis new file mode 100644 index 0000000..c744b29 --- /dev/null +++ b/.github/workflows/composition-analysis @@ -0,0 +1,126 @@ +name: composition-analysis +on: + workflow_dispatch: # allow manually triggering builds + push: + paths-ignore: + - '**/build-image.yml' + - '**/release-please.yml' + - '**/vulnerability-scan.yml' + - "**.md" + - "**/.gitkeep" + - "build-isos" + - "renovate.json" + schedule: + - cron: "10 07 * * 1,3,5" # scan at 06:10 UTC on Mon, Wed, Fr + # (1 hour after bluebuild images started building) + +concurrency: + group: ${{ github.workflow }}-${{ github.ref || github.run_id }} + cancel-in-progress: true + +env: + IMAGE_REGISTRY: ghcr.io/${{ github.repository_owner }} + IMAGE_TAGS: latest + REGISTRY_USER: ${{ github.actor }} + REGISTRY_PASSWORD: ${{ github.token }} + +jobs: + analysis: + name: Scan Custom Image + runs-on: ubuntu-24.04 + permissions: + contents: read + packages: write + security-events: write + strategy: + fail-fast: true + matrix: + package: + - aubertit + - buttgenbachit + - carbonatcyanotrichit + - flaviramea + steps: + + # Gimme more room to unfold + - name: Maximize build space + uses: easimon/maximize-build-space@master + with: + overprovision-lvm: true + remove-android: true + remove-codeql: true + remove-docker-images: true + remove-dotnet: true + remove-haskell: true + - name: Convert to compressed filesystem + shell: bash + run: | + set -eux pipefail + + # https://github.com/easimon/maximize-build-space/issues/23 + # https://github.com/easimon/maximize-build-space/pull/37 + WORKSPACE_OWNER="$(stat -c '%U:%G' "${GITHUB_WORKSPACE}")" + VG_NAME=buildvg + LV_NAME="/dev/mapper/${VG_NAME}-buildlv" + + cd ~ + sudo umount "${LV_NAME}" + + # https://github.com/radxa/armbian-compile-action/commit/82fe40188f2f3b4dbd4ae8fb9c64f49f117085c7 + sudo mkfs.btrfs -L buildlv -M -m single --force "${LV_NAME}" + sudo mount -o defaults,noatime,compress=zstd:3,discard=sync "${LV_NAME}" "${GITHUB_WORKSPACE}" + + sudo chown -R "$WORKSPACE_OWNER" "${GITHUB_WORKSPACE}" + + # Prepare analysis + - name: Checkout codeZ + uses: actions/checkout@v4 + + - name: Get default branch name + run: echo "DEFAULT_BRANCH=${{ github.event.repository.default_branch }}" >$GITHUB_ENV + + - name: Check if on default (main) or test (rechunk) branch to rechunk it + run: | + if [[ '${{ github.ref_name }}' = "$DEFAULT_BRANCH" || '${{ github.ref_name }}' = 'analyze' ]]; then + OUT='IS_DEFAULT=true' + else + OUT='IS_DEFAULT=false' + fi + echo "$OUT" >$GITHUB_ENV + + # Workaround bug where capital letters in your GitHub username make it impossible to push to GHCR. + # https://github.com/macbre/push-to-ghcr/issues/12 + - name: Lowercase Registry + uses: ASzc/change-string-case-action@v6 + id: registry_case + with: + string: ${{ env.IMAGE_REGISTRY }} + + - name: Composite image name + run: echo "IMAGE_NAME=${{ steps.registry_case.outputs.lowercase }}/${{ matrix.package }}:$IMAGE_TAGS" >$GITHUB_ENV + + # SCA, Vuln and License scanning + - name: Run GitHub Action for ORT + uses: oss-review-toolkit/ort-ci-github-action@v1 + id: ort + with: + sw-name: ${{ matrix.package }} + + - name: Echo outputs + run: | + echo "${{ toJSON(steps.ort.outputs) }}" + + - uses: philips-labs/tern-action@v1.1.0 + id: tern + with: + image: ${{ env.ALL_TAGS }} + image: ${{ env.IMAGE_NAME }} + + - uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.package }} + path: ${{ steps.tern.outputs.file }} + + - name: Echo outputs + run: | + echo "${{ toJSON(steps.tern.outputs) }}" From 4049b74a9934b07b33bad7d0bcc1c797cfcbccf8 Mon Sep 17 00:00:00 2001 From: OmegaSquad82 <34405234+OmegaSquad82@users.noreply.github.com> Date: Thu, 6 Feb 2025 00:08:58 +0100 Subject: [PATCH 05/11] feat: remove SCA from building workflow Signed-off-by: OmegaSquad82 <34405234+OmegaSquad82@users.noreply.github.com> --- .github/workflows/build-image.yml | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index f61dc43..0321ae3 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -5,6 +5,7 @@ on: paths-ignore: - '**/release-please.yml' - '**/vulnerability-scan.yml' + - '**/composition-analysis.yml' - "**.md" - "**/.gitkeep" - "build-isos" @@ -25,7 +26,7 @@ env: ZSTD_CHUNKED: true jobs: - bluebuild: + imagebuild: name: Build Custom Image runs-on: ubuntu-24.04 permissions: @@ -195,22 +196,3 @@ jobs: if: env.DO_ZSTD_CHUNKED == 'true' run: | echo "${{ toJSON(steps.push.outputs) }}" - - # SCA, Vuln and License scanning - - name: Run GitHub Action for ORT - if: env.IS_DEFAULT == 'true' - uses: oss-review-toolkit/ort-ci-github-action@v1 - with: - sw-name: ${{ matrix.package }} - - - uses: philips-labs/tern-action@v1.1.0 - if: env.IS_DEFAULT == 'true' - id: tern - with: - image: ${{ env.IMAGE_NAME }} - - - uses: actions/upload-artifact@v4 - if: env.IS_DEFAULT == 'true' - with: - name: ${{ matrix.package }} - path: ${{ steps.tern.outputs.file }} From 65742bafbaf74466ff112ae65e01d75cbb7b9370 Mon Sep 17 00:00:00 2001 From: OmegaSquad82 <34405234+OmegaSquad82@users.noreply.github.com> Date: Thu, 6 Feb 2025 00:09:22 +0100 Subject: [PATCH 06/11] fix: Rename composition-analysis to composition-analysis.yml Signed-off-by: OmegaSquad82 <34405234+OmegaSquad82@users.noreply.github.com> --- .../workflows/{composition-analysis => composition-analysis.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{composition-analysis => composition-analysis.yml} (100%) diff --git a/.github/workflows/composition-analysis b/.github/workflows/composition-analysis.yml similarity index 100% rename from .github/workflows/composition-analysis rename to .github/workflows/composition-analysis.yml From cc73ad713b749596b803ac7d8fea0e9f08e9eb79 Mon Sep 17 00:00:00 2001 From: OmegaSquad82 <34405234+OmegaSquad82@users.noreply.github.com> Date: Thu, 6 Feb 2025 00:10:39 +0100 Subject: [PATCH 07/11] fix: no execution after change in composition-analysis.yml Signed-off-by: OmegaSquad82 <34405234+OmegaSquad82@users.noreply.github.com> --- .github/workflows/vulnerability-scan.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/vulnerability-scan.yml b/.github/workflows/vulnerability-scan.yml index b8ccd5c..527ed3f 100644 --- a/.github/workflows/vulnerability-scan.yml +++ b/.github/workflows/vulnerability-scan.yml @@ -8,6 +8,7 @@ on: paths-ignore: - '**/build-image.yml' - '**/release-please.yml' + - '**/composition-analysis.yml' - "**.md" - "**/.gitkeep" - "build-isos" From de8182f2121e7fc686896a621b087f4f57521dea Mon Sep 17 00:00:00 2001 From: OmegaSquad82 <34405234+OmegaSquad82@users.noreply.github.com> Date: Thu, 6 Feb 2025 00:25:43 +0100 Subject: [PATCH 08/11] feat: Create recompress-latest.yml Signed-off-by: OmegaSquad82 <34405234+OmegaSquad82@users.noreply.github.com> --- .github/workflows/recompress-latest.yml | 161 ++++++++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 .github/workflows/recompress-latest.yml diff --git a/.github/workflows/recompress-latest.yml b/.github/workflows/recompress-latest.yml new file mode 100644 index 0000000..195b4d0 --- /dev/null +++ b/.github/workflows/recompress-latest.yml @@ -0,0 +1,161 @@ +name: recompress-latest +on: + workflow_dispatch: # allow manually triggering builds + push: + paths-ignore: + - '**/build-image.yml' + - '**/release-please.yml' + - '**/vulnerability-scan.yml' + - '**/composition-analysis.yml' + - "**.md" + - "**/.gitkeep" + - "build-isos" + - "renovate.json" + schedule: + - cron: "10 07 * * 1,3,5" # build at 07:10 UTC on Mon, Wed, Fr + # (60 minutes after bluebuild images start building) + +concurrency: + group: ${{ github.workflow }}-${{ github.ref || github.run_id }} + cancel-in-progress: true + +env: + IMAGE_REGISTRY: ghcr.io/${{ github.repository_owner }} + IMAGE_TAGS: latest + REGISTRY_USER: ${{ github.actor }} + REGISTRY_PASSWORD: ${{ github.token }} + +jobs: + recompress: + name: Build Custom Image + runs-on: ubuntu-24.04 + permissions: + contents: read + packages: write + strategy: + fail-fast: true + matrix: + package: + - aubertit + - buttgenbachit + - carbonatcyanotrichit + - flaviramea + steps: + + # Gimme more room to unfold + - name: Maximize build space + uses: easimon/maximize-build-space@master + with: + overprovision-lvm: true + remove-android: true + remove-codeql: true + remove-docker-images: true + remove-dotnet: true + remove-haskell: true + - name: Convert to compressed filesystem + shell: bash + run: | + set -eux pipefail + + # https://github.com/easimon/maximize-build-space/issues/23 + # https://github.com/easimon/maximize-build-space/pull/37 + WORKSPACE_OWNER="$(stat -c '%U:%G' "${GITHUB_WORKSPACE}")" + VG_NAME=buildvg + LV_NAME="/dev/mapper/${VG_NAME}-buildlv" + + cd ~ + sudo umount "${LV_NAME}" + + # https://github.com/radxa/armbian-compile-action/commit/82fe40188f2f3b4dbd4ae8fb9c64f49f117085c7 + sudo mkfs.btrfs -L buildlv -M -m single --force "${LV_NAME}" + sudo mount -o defaults,noatime,compress=zstd:3,discard=sync "${LV_NAME}" "${GITHUB_WORKSPACE}" + + sudo chown -R "$WORKSPACE_OWNER" "${GITHUB_WORKSPACE}" + + # Create bootc image + - name: Checkout codeZ + uses: actions/checkout@v4 + + - name: Get default branch name + run: echo "DEFAULT_BRANCH=${{ github.event.repository.default_branch }}" >$GITHUB_ENV + + # Workaround bug where capital letters in your GitHub username make it impossible to push to GHCR. + # https://github.com/macbre/push-to-ghcr/issues/12 + - name: Lowercase Registry + uses: ASzc/change-string-case-action@v6 + id: registry_case + with: + string: ${{ env.IMAGE_REGISTRY }} + + - name: Composite image name + run: echo "IMAGE_NAME=${{ steps.registry_case.outputs.lowercase }}/${{ matrix.package }}" >$GITHUB_ENV + + # This is something the community works for quite a long time + # but it likely will improve the efficiency of transfers; + # this was made to workaround the fact that blue-build + # does not yet support zstd:chunked and it depends + # on bootc/rpm-ostree having implemented this. + # No OS image must be pushed in this form, + # except to non-productive image tags. + # + # Changetracker + # https://fedoraproject.org/wiki/Changes/zstd:chunked + # https://github.com/containers/storage/pull/775 + # Documentation + # https://universal-blue.discourse.group/t/zst-chunked-faq/3462 + # Projectstuffz + # https://github.com/blue-build/cli/issues/112 + # https://github.com/containers/bootc/issues/20 + # https://github.com/containers/bootc/issues/963 + - name: Tag latest as latest-zstchd + id: pull + run: | + set -eux pipefail + IMAGES=() + podman login \ + --username ${{ env.REGISTRY_USER }} \ + --password ${{ env.REGISTRY_PASSWORD }} \ + ${{ steps.registry_case.outputs.lowercase }} + for tag in ${{ env.IMAGE_TAGS }}; do + podman pull "${{ env.IMAGE_NAME }}:$tag" + podman tag "${{ env.IMAGE_NAME }}:$tag" "${{ env.IMAGE_NAME }}:${tag}-zstchd" + IMAGES+=("${{ env.IMAGE_NAME }}:${tag}-zstchd") # zstd:chunked + done + echo "ALL_TAGS=${IMAGES[@]}" >$GITHUB_ENV + + - name: Push To GHCR with Podman in zstd:chunked + uses: redhat-actions/push-to-registry@v2 + id: push + with: + tags: ${{ env.ALL_TAGS }} + username: ${{ env.REGISTRY_USER }} + password: ${{ env.REGISTRY_PASSWORD }} + extra-args: | + --compression-format=zstd:chunked + + # Sign recompressed container image + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ env.REGISTRY_USER }} + password: ${{ env.REGISTRY_PASSWORD }} + + - uses: sigstore/cosign-installer@v3.8.0 + + - name: Sign container image + id: sign + run: | + echo "${{ env.COSIGN_PRIVATE_KEY }}" >cosign.key + wc -c cosign.key + cosign sign -y --key cosign.key ${{ env.IMAGE_NAME }}@${TAGS} + env: + COSIGN_EXPERIMENTAL: false + COSIGN_PRIVATE_KEY: ${{ secrets.SIGNING_SECRET }} + TAGS: ${{ steps.push.outputs.digest }} + + - name: Echo outputs + run: | + echo "${{ toJSON(steps.pull.outputs) }}" + echo "${{ toJSON(steps.push.outputs) }}" + echo "${{ toJSON(steps.sign.outputs) }}" From 7ab5213e1e2ef0e28f0906b8852ae5b07a53f7ea Mon Sep 17 00:00:00 2001 From: OmegaSquad82 <34405234+OmegaSquad82@users.noreply.github.com> Date: Thu, 6 Feb 2025 00:26:37 +0100 Subject: [PATCH 09/11] fix: Update build-image.yml to exclude recompress-latest.yml Signed-off-by: OmegaSquad82 <34405234+OmegaSquad82@users.noreply.github.com> --- .github/workflows/build-image.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 0321ae3..13c0e19 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -4,6 +4,7 @@ on: push: paths-ignore: - '**/release-please.yml' + - '**/recompress-latest.yml' - '**/vulnerability-scan.yml' - '**/composition-analysis.yml' - "**.md" From c9c2bea02fae8fe1264ad5cf35b3244e8e8bbd57 Mon Sep 17 00:00:00 2001 From: OmegaSquad82 <34405234+OmegaSquad82@users.noreply.github.com> Date: Thu, 6 Feb 2025 00:28:49 +0100 Subject: [PATCH 10/11] fix: Update composition-analysis.yml to exclude recompress-latest.yml Signed-off-by: OmegaSquad82 <34405234+OmegaSquad82@users.noreply.github.com> --- .github/workflows/composition-analysis.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/composition-analysis.yml b/.github/workflows/composition-analysis.yml index c744b29..d23a416 100644 --- a/.github/workflows/composition-analysis.yml +++ b/.github/workflows/composition-analysis.yml @@ -5,6 +5,7 @@ on: paths-ignore: - '**/build-image.yml' - '**/release-please.yml' + - '**/recompress-latest.yml' - '**/vulnerability-scan.yml' - "**.md" - "**/.gitkeep" @@ -30,7 +31,7 @@ jobs: runs-on: ubuntu-24.04 permissions: contents: read - packages: write + packages: read security-events: write strategy: fail-fast: true @@ -92,12 +93,12 @@ jobs: # https://github.com/macbre/push-to-ghcr/issues/12 - name: Lowercase Registry uses: ASzc/change-string-case-action@v6 - id: registry_case + id: case with: string: ${{ env.IMAGE_REGISTRY }} - name: Composite image name - run: echo "IMAGE_NAME=${{ steps.registry_case.outputs.lowercase }}/${{ matrix.package }}:$IMAGE_TAGS" >$GITHUB_ENV + run: echo "IMAGE_NAME=${{ steps.case.outputs.lowercase }}/${{ matrix.package }}:$IMAGE_TAGS" >$GITHUB_ENV # SCA, Vuln and License scanning - name: Run GitHub Action for ORT @@ -113,7 +114,6 @@ jobs: - uses: philips-labs/tern-action@v1.1.0 id: tern with: - image: ${{ env.ALL_TAGS }} image: ${{ env.IMAGE_NAME }} - uses: actions/upload-artifact@v4 @@ -123,4 +123,5 @@ jobs: - name: Echo outputs run: | + echo "${{ toJSON(steps.ort.outputs) }}" echo "${{ toJSON(steps.tern.outputs) }}" From 1f4aff10f05cfc3259c52fb0d4da7324e7825f58 Mon Sep 17 00:00:00 2001 From: OmegaSquad82 <34405234+OmegaSquad82@users.noreply.github.com> Date: Thu, 6 Feb 2025 00:32:18 +0100 Subject: [PATCH 11/11] fix: Update vulnerability-scan.yml to exclude recompress-latest.yml feat: run vulnerability-scan workflow once a day after build Signed-off-by: OmegaSquad82 <34405234+OmegaSquad82@users.noreply.github.com> --- .github/workflows/vulnerability-scan.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/vulnerability-scan.yml b/.github/workflows/vulnerability-scan.yml index 527ed3f..ad90f72 100644 --- a/.github/workflows/vulnerability-scan.yml +++ b/.github/workflows/vulnerability-scan.yml @@ -2,12 +2,13 @@ name: vulnerability-scan on: schedule: - cron: - "50 0,6,12,18 * * *" # build at {0,6,12,18}:55 UTC every day - # (50 minutes after custom image build was triggered, then every 6h) + "55 06 * * *" # build at 06:55 am UTC every day + # (50 minutes after custom image build was triggered) push: paths-ignore: - '**/build-image.yml' - '**/release-please.yml' + - '**/recompress-latest.yml' - '**/composition-analysis.yml' - "**.md" - "**/.gitkeep" @@ -23,9 +24,9 @@ jobs: name: Scan Custom Image runs-on: ubuntu-24.04 permissions: - security-events: write contents: read packages: read + security-events: write strategy: fail-fast: true matrix: