[OPS 293] multiversion maven repo

.github/workflows/build-container.yml

@@ -3,16 +3,106 @@ name: Build CI Container
 on:
   repository_dispatch:
     types: [build-container]
+    inputs:
+      push_image:
+        description: If the image should be pushed to the registry
+        type: boolean
+        default: true
   workflow_dispatch:
-  push:
-    branches:
-      - "*"
-      - "!main"
+    inputs:
+      commit:
+        description: 'Initial commit to layer the resulting image'
+        required: true
+        type: string
+      push_image:
+        description: If the image should be pushed to the registry
+        type: boolean
+        default: true
 
+  # Build container on PR open and switch to ready
+  pull_request:
+    types: [opened, ready_for_review]
+
 jobs:
-  build-container:
+  parse-inputs:
     runs-on: ubuntu-latest
-    container: hashicorp/packer:1.11
+    outputs:
+      # A list of commits to include in the build. Expected to be the head commit when incrementing, or the full range when rebuilding.
+      commits: ${{ steps.check.outputs.commits }}
+      # The image to use for the build. Expected to be the previous image when incrementing, or the java base image when rebuilding.
+      action_build_container: ${{ steps.check.outputs.action_build_container }}
+      # The first commit SHA to be included in the image (not just this HEAD run)
+      base_commit: ${{ steps.check.outputs.base_commit }}
+      # The last commit SHA to be included in the image for this run
+      last_commit: ${{ steps.check.outputs.last_commit }}
+      # The version label for the image if provided
+      version: ${{ steps.check.outputs.version }}
+      # The commit SHA used to build the pipeline-java base image used to build this image.
+      pipeline_java_commit: ${{ steps.check.outputs.pipeline_java_commit }}
+      # The version of the pipeline-java base image used to build this image.
+      pipeline_java_version: ${{ steps.check.outputs.pipeline_java_version }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Check and parse inputs
+        id: check
+        run: |
+          # Check whether the commit was provided
+          if [ "z${{ inputs.commit }}" != "z" ]; then
+
+            # We're on some commit, so rebuild the whole image
+            ## find all commits to layer in the image
+
+            ## This is a special format of putting a multiline
+            ## string into a value inside $GITHUB_OUTPUT
+            ## It's using the heredoc format, so
+            ## that's why it's not in a standard Github's "var=value" syntax
+            echo "commits<<EOF" >> $GITHUB_OUTPUT
+            echo "$(git rev-list --ancestry-path ${{ inputs.commit }}~1..HEAD)" >> $GITHUB_OUTPUT
+            echo "EOF" >> $GITHUB_OUTPUT
+
+            ## detect the container to build the repository with
+            echo "action_build_container=zepben/pipeline-java" >> $GITHUB_OUTPUT
+
+            ## find the first commit to start the build with
+            echo "base_commmit=${{ inputs.commit }}" >> $GITHUB_OUTPUT
+          else
+            # We're on HEAD, so update the existing image
+            echo "commits=HEAD" >> $GITHUB_OUTPUT
+            echo "action_build_container=zepben/pipeline-java-ewb" >> $GITHUB_OUTPUT
+
+            # We don't have any base commit, so
+            # fetch it from the latest image
+            pipeline_java_ewb_labels=$(skopeo inspect docker://zepben/pipeline-java-ewb | jq .Labels)
+            echo "base_commit=$(echo $pipeline_java_ewb_labels | jq .base_commit)" >> $GITHUB_OUTPUT
+
+            # Figure out a version
+            if [ "z${{ github.event.client_payload.version }}" != "z" ]; then
+              # If our automated dispatch provided a version, use it
+              echo "version=${{ github.event.client_payload.version }}" >> $GITHUB_OUTPUT
+            else
+              # Otherwise, it means we're just rebasing on a new pipeline, 
+              # so fetch it from the latest self image
+              echo "version=$(echo $pipeline_java_ewb_labels | jq .version)" >> $GITHUB_OUTPUT
+            fi 
+          fi
+
+          # basic pipeline-java labels to include in the our final container
+          pipeline_java_labels=$(skopeo inspect docker://zepben/pipeline-java | jq .Labels)
+          echo "pipeline_java_commit=$(echo $pipeline-java-labels | jq .commit)" >> $GITHUB_OUTPUT
+          echo "pipeline_java_version=$(echo $pipeline-java-labels | jq .version)" >> $GITHUB_OUTPUT
+
+          # set the last commit to whatever caused this flow to run
+          # this would be HEAD off the branch where launched
+          #   or main, if built from remote_dispatch
+          echo "last_commit=$GITHUB_SHA" >> $GITHUB_OUTPUT
+
+  build-repository:
+    runs-on: ubuntu-latest
+    needs: [parse-inputs]
+    container: ${{ needs.parse-inputs.outputs.action_build_container }} 
     env:
       ZEPBEN_GPG_KEY: ${{ secrets.ZEPBEN_GPG_KEY }}
       MAVEN_SETTINGS: ${{ secrets.MAVEN_SETTINGS }}
@@ -22,63 +112,126 @@ jobs:
       GPG_KEY_PASSWORD: ${{ secrets.GPG_KEY_PASSWORD }}
       DOCKER_HUB_ACCESS_TOKEN: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
       DOCKER_HUB_USER: ${{ secrets.DOCKER_HUB_USER }}
+      LOCAL_REPO: /maven
     steps:
       - uses: actions/checkout@v4
 
-      - name: Install deps
+      - name: Work around git permission issue
         run: |
-          apk add libxml2-utils docker
+          dname=$(echo ${{github.repository}} | cut -d'/' -f2)
+          git config --global --add safe.directory /__w/$dname/$dname
+        shell: sh
 
-      - name: Packer Version
+      - name: Build repository
+        id: build
         run: |
-          packer version
+          # parse commits as a list
+          commits=( ${{ needs.parse-inputs.outputs.commits }} )
+          for commit in "${commits[@]}"; do
+            git checkout $commit
+            ./maven-build.sh 
+          done 
+        shell: bash
+
+      - name: Store repository for wrapping in container  
+        uses: actions/upload-artifact@v4
+        id: upload
+        if: steps.build.outcome == 'success'
+        with:
+          name: repo
+          path: /maven
+          if-no-files-found: error
+        continue-on-error: false
 
-      - name: Create docker container
+  build-container:
+    runs-on: ubuntu-latest
+    needs: [parse-inputs, build-repository]
+    env:
+      DOCKER_HUB_ACCESS_TOKEN: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+      DOCKER_HUB_USER: ${{ secrets.DOCKER_HUB_USER }}
+      LOCAL_REPO: local_repo
+      BASE_IMAGE: zepben/pipeline-java
+    steps:
+
+      - uses: actions/checkout@v4
+
+      - name: Work around git permission issue
         run: |
-          # Scripts to copy dependencies from dependencyManagement
-          cp for-ci-build-image/deps-pom-template.xml deps-pom.xml
-          version=$(cat pom.xml | sed 's/xmlns.*=".*"//g' | xmllint --xpath '/project/version/text()' -)
-          sed -i "s/{{version}}/$version/g" deps-pom.xml
-          deps=$(cat pom.xml | sed 's/xmlns.*=".*"//g' | xmllint --xpath '/project/dependencyManagement/dependencies/dependency' -)
-          deps=$(echo $deps | sed 's/\//\\\//g; s/ /\\n/g')
-          ln=$(cat deps-pom.xml | grep -n "<dependencies>" | cut -d':' -f1)
-          ln=$((ln + 1))
-          sed -i "$ln i $deps" deps-pom.xml
-          # Scripts to copy plugins from pluginManagement
-          plugins=$(cat pom.xml | sed 's/xmlns.*=".*"//g' | xmllint --xpath '/project/build/pluginManagement/plugins/plugin' -)
-          plugins=$(echo $plugins | sed 's/\//\\\//g; s/ /\\n/g')
-          ln=$(cat deps-pom.xml | grep -n "<plugins>" | cut -d':' -f1)
-          ln=$((ln + 1))
-          sed -i "$ln i $plugins" deps-pom.xml
-          xmllint -format deps-pom.xml > deps-pom-new.xml
-          mv -f deps-pom-new.xml deps-pom.xml
-          ADDITIONAL_ARGS=""
-          if [[ $GITHUB_REF_NAME != "main" ]]; then
-            ADDITIONAL_ARGS="-except docker.push,docker.tag"
-          fi
+          dname=$(echo ${{github.repository}} | cut -d'/' -f2)
+          git config --global --add safe.directory /__w/$dname/$dname
+        shell: sh
 
-          # Handle the base image if provided
-          if [ "z${{ github.event.client_payload.base_image }}" != "z" ]; then
-            # Some label was given via remote call
-            PIPELINE_JAVA_IMAGE="-var base_image=${{ github.event.client_payload.base_image }}"
-          fi
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Github Container Registry
+        uses: docker/login-action@v3
+        with:
+          username: ${{ env.DOCKER_HUB_USER }}
+          password: ${{ env.DOCKER_HUB_ACCESS_TOKEN }}
+          registry: docker.io
 
-          # Handle the resulting container labels if provided
-          if [ "z${{ github.event.client_payload.labels }}" != "z" ]; then
-            # the labels come in as "label1 label2" but we need to provide them as
-            # ["label1", "label2"]
-            # So we replace the original values with quoted ones and then wrap in braces
-
-            # Split the passed values into an array
-            array=(${{ github.event.client_payload.labels }})
-            # Print every array entry encased by double quotes
-            tags=$(printf "\"%s\"," "${array[@]}")
-            # Construct the list parameter for the container tags
-            IMAGE_TAGS="-var tags=[$tags]"
+      - name: Download repository
+        uses: actions/download-artifact@v4
+        with:
+          name: repo
+          path: ${{ env.LOCAL_REPO }}
+        continue-on-error: false
+
+      - name: Figure out tags and labels
+        id: labels
+        run: |
+          if [ "z${{ github.event.client_payload.base_image }}" != "z" ]; then
+            # Some base_image with label was given via a remote call, so override the env
+            echo "BASE_IMAGE=${{ github.event.client_payload.base_image }}" >> $GITHUB_ENV
           fi
 
+          # Create the Dockerfile
+          # Replace BASE_IMAGE and LOCAL_REPO in Dockerfile template
+          # BASE_IMAGE is set as env on the whole job and updated above if needed
+          # LOCAL_REPO is set as env on the whole job and used to download the artefact
           cd for-ci-build-image
-          packer init build-container.pkr.hcl
-          packer validate build-container.pkr.hcl
-          packer build -on-error=abort -var dockerhub_user=$DOCKER_HUB_USER -var dockerhub_pw=$DOCKER_HUB_ACCESS_TOKEN $PIPELINE_JAVA_IMAGE $IMAGE_TAGS $ADDITIONAL_ARGS build-container.pkr.hcl
-        shell: bash
+          cat Dockerfile.in | envsubst > Dockerfile
+
+          # Check Dockerfile
+          cat Dockerfile
+
+      - name: Get SHAs
+        id: sha
+        run: |
+          SHORT_REF=$(git rev-parse --short HEAD)
+          LONG_REF=$(git rev-parse HEAD)
+          echo "short_ref=$SHORT_REF" >> $GITHUB_OUTPUT
+          echo "long_ref=$LONG_REF" >> $GITHUB_OUTPUT
+
+      # Priority sorting determines the tag used in the OCI label
+      # The current order preferences the version, then commit, then any special tags
+      # We always push a commit based tag
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: zepben/pipeline-java-ewb
+          labels: |
+            base_commit=${{ needs.parse-inputs.outputs.base_commit }}
+            last_commit=${{ needs.parse-inputs.outputs.last_commit }}
+            pipeline_java_commit=${{ needs.parse-inputs.outputs.pipeline_java_commit }}
+            pipeline_java_version=${{ needs.parse-inputs.outputs.pipeline_java_version }}
+            version=${{ needs.parse-inputs.outputs.version }}
+            org.opencontainers.image.vendor=Zepben
+          tags: |
+            type=raw,value=${{ github.ref_name }}-${{ steps.sha.outputs.short_ref }},enable=${{ github.ref_name != 'main' }},priority=200
+            type=raw,value=${{ steps.sha.outputs.short_ref }},enable=${{ github.ref_name == 'main' }},priority=200
+            type=raw,value=${{ github.ref_name }},enable=${{ github.ref_name != 'main' }},priority=110
+            type=raw,value=${{ needs.parse-inputs.outputs.version }},enable=${{ needs.parse-inputs.outputs.version  != '' }},priority=105
+            type=raw,value=latest,enable=${{ github.ref_name == 'main' }},priority=100
+
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          push: ${{ inputs.push_image }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/check-and-tag.yml

@@ -8,7 +8,7 @@ on:
 jobs:
   check-and-tag:
     runs-on: ubuntu-latest
-    container: zepben/pipeline-basic:5.0.0
+    container: zepben/pipeline-basic
     env:
       NEXUS_MAVEN_REPO: ${{ secrets.NEXUS_MAVEN_REPO }}
       NEXUS_USERNAME: ${{ secrets.NEXUS_USERNAME }}
@@ -18,17 +18,21 @@ jobs:
       SLACK_NOTIFICATION: YES
       SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Check and tag version
+        id: version
         run: |
           /scripts/release-checks.sh --java --maven pom.xml
           source /scripts/common.sh
           version=$(xmlstarlet pyx pom.xml | grep -v ^A | xmlstarlet p2x | xmlstarlet sel -t -v "/project/version")
           tag_finalize_version $version
+          echo "version=$version" >> $GITHUB_OUTPUT
         shell: bash
 
       - name: Trigger release
         uses: peter-evans/repository-dispatch@v1
         with:
           token: ${{ secrets.CI_GITHUB_TOKEN }}
-          event-type: trigger-release
+          event-type: trigger-release
+          client-payload: '{"version": "${{ steps.version.outputs.version }}"}'
+

.github/workflows/release.yml

@@ -12,12 +12,12 @@ on:
 jobs:
   deploy-central:
     runs-on: ubuntu-latest
-    container: zepben/pipeline-java:4.5.1
+    container: zepben/pipeline-ewb-java
     env:
       DOCKER_HUB_USER: ${{ secrets.DOCKER_HUB_USER }}
       DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Maven deploy to Central
         id: build
@@ -42,4 +42,4 @@ jobs:
         with:
           token: ${{ secrets.CI_GITHUB_TOKEN }}
           event-type: build-container
-
+          client-payload: '{"version": "${{ github.event.client_payload.version }}"}'

README.md

@@ -8,3 +8,18 @@ While the major version is <1.0.0, changes should bump the minor (y in x.y.z) ve
 (CI does not do this automatically like other Zepben repos).
 If you have fixed a bug in the pom you can probably just increment the z number, but while
 we are less than 1.0.0 it probably doesn't make much difference...
+
+## Building pipeline-java-ewb container
+
+We build the `pipeline-java-ewb` container after releasing a new superpom, or on demand. The reason for this container is to
+include a Maven repository with as many dependencies as possible, so that our actual Maven builds are quicker.
+
+There are 3 ways that this container is built:
+
+ 1. Automatically for superpom releases via a series of flow dispatches. In this case, we want to only update the current repository with new packages/versions, preserving all the packages that are already there. This allows us to cache packages for many superpom releases in the same repository.
+
+ 2. Automatically for rebasing on a newer `pipeline-java` container to include fixes and updates to ci-utils/tools. In this case we only want to rebase the current container image on top of a new one.
+
+ 3. Manually, by providing an initial commit; then a new container is built via a series of steps by checking out all the commits starting with the provided SHA and ending on HEAD. Can be used in cases where the container gets too big by carrying old and unnecessary dependencies and we want to start afresh.
+
+ The container might also be built for the PR checks, but only to test the build process, i.e without pushing it to a repository.

for-ci-build-image/Dockerfile.in

@@ -0,0 +1,5 @@
+FROM ${BASE_IMAGE}
+
+COPY maven-settings.xml /usr/share/maven/conf/settings.xml
+COPY ${LOCAL_REPO} /root/local_repo
+