diff --git a/.github/workflows/build-publish-containers.yml b/.github/workflows/build-publish-containers.yml index ec2bcd9f..a7c9b607 100644 --- a/.github/workflows/build-publish-containers.yml +++ b/.github/workflows/build-publish-containers.yml @@ -12,7 +12,7 @@ on: - main paths: - containers/Dockerfile.EESSI-* - - containers/build-or-download-cvmfs-rpm.sh + - containers/build-or-download-cvmfs-*.sh # also rebuild the containers for new releases of filesystem-layer: # this often means that we have a new CVMFS configuration package @@ -32,8 +32,14 @@ jobs: contents: read strategy: matrix: - tag: ['client:centos7', 'build-node:debian11'] - platform: [amd64, arm64, ppc64le] + tag: ['client:centos7', 'build-node:debian11', 'build-node:debian-sid'] + platform: [amd64, arm64, riscv64] + exclude: + # exclude images that don't support RISC-V + - tag: client:centos7 + platform: riscv64 + - tag: build-node:debian11 + platform: riscv64 steps: - name: Check out the repo @@ -91,7 +97,7 @@ jobs: contents: read strategy: matrix: - tag: ['client:centos7', 'build-node:debian11'] + tag: ['client:centos7', 'build-node:debian11', 'build-node:debian-sid'] steps: - name: Login to GitHub Container Registry if: github.event_name != 'pull_request' @@ -106,12 +112,19 @@ jobs: echo REPOSITORY_OWNER=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]') >> $GITHUB_ENV - name: Create the final image using "docker buildx imagetools create" - if: github.event_name != 'pull_request' + if: github.event_name != 'pull_request' && matrix.tag == 'build-node:debian-sid' run: | docker buildx imagetools create -t ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }} \ ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}-amd64 \ ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}-arm64 \ - ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}-ppc64le + ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}-riscv64 + + - name: Create the final image using "docker buildx imagetools create" + if: github.event_name != 'pull_request' && matrix.tag != 'build-node:debian-sid' + run: | + docker buildx imagetools create -t ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }} \ + ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}-amd64 \ + ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}-arm64 - name: Inspect the final image if: github.event_name != 'pull_request' diff --git a/containers/Dockerfile.EESSI-build-node-debian-sid b/containers/Dockerfile.EESSI-build-node-debian-sid new file mode 100644 index 00000000..864919e5 --- /dev/null +++ b/containers/Dockerfile.EESSI-build-node-debian-sid @@ -0,0 +1,44 @@ +ARG cvmfsversion=2.11.2 +ARG archspecversion=0.2.2 +ARG awscliversion=1.32.22 +ARG fuseoverlayfsversion=1.10 + +FROM debian:sid-20240330-slim AS prepare-deb +ARG cvmfsversion +COPY ./containers/build-or-download-cvmfs-debs.sh /build-or-download-cvmfs-debs.sh +RUN sh /build-or-download-cvmfs-debs.sh ${cvmfsversion} + + +FROM debian:sid-20240330-slim +ARG cvmfsversion +ARG archspecversion +ARG awscliversion +ARG fuseoverlayfsversion + +COPY --from=prepare-deb /root/deb /root/deb + +RUN apt-get update +RUN apt-get install -y sudo vim openssh-client gawk autofs curl attr uuid fuse3 libfuse2 psmisc gdb uuid-dev lsof strace +# python3 and jq are required for eessi-upload-to-staging script (next to awscli) +RUN apt-get install -y python3-pip python3-venv jq +RUN dpkg -i /root/deb/cvmfs_${cvmfsversion}~1+debian13_$(dpkg --print-architecture).deb \ + /root/deb/cvmfs-fuse3_${cvmfsversion}~1+debian13_$(dpkg --print-architecture).deb \ + /root/deb/cvmfs-libs_${cvmfsversion}~1+debian13_$(dpkg --print-architecture).deb \ + /root/deb/cvmfs-config-default_latest_all.deb \ + /root/deb/cvmfs-config-eessi_latest_all.deb + +# download binary for specific version of fuse-overlayfs +RUN curl -L -o /usr/local/bin/fuse-overlayfs https://github.com/containers/fuse-overlayfs/releases/download/v${fuseoverlayfsversion}/fuse-overlayfs-$(uname -m) \ + && chmod +x /usr/local/bin/fuse-overlayfs + +RUN echo 'CVMFS_QUOTA_LIMIT=10000' > /etc/cvmfs/default.local \ + && echo 'CVMFS_CLIENT_PROFILE="single"' >> /etc/cvmfs/default.local \ + && echo 'CVMFS_HIDE_MAGIC_XATTRS=yes' >> /etc/cvmfs/default.local + +RUN mkdir -p /cvmfs/software.eessi.io + +RUN useradd -ms /bin/bash eessi + +RUN python3 -m venv /opt/archspec && . /opt/archspec/bin/activate && pip3 install archspec==${archspecversion} && deactivate && ln -s /opt/archspec/bin/archspec /usr/local/bin/archspec +# stick to awscli v1.x, 2.x is not available through PyPI (see https://github.com/aws/aws-cli/issues/4947) +RUN python3 -m venv /opt/awscli && . /opt/awscli/bin/activate && pip3 install awscli==${awscliversion} && deactivate && ln -s /opt/awscli/bin/aws /usr/local/bin/aws diff --git a/containers/Dockerfile.EESSI-build-node-debian11 b/containers/Dockerfile.EESSI-build-node-debian11 index 4fe34ae3..7b20a8e8 100644 --- a/containers/Dockerfile.EESSI-build-node-debian11 +++ b/containers/Dockerfile.EESSI-build-node-debian11 @@ -39,7 +39,3 @@ RUN useradd -ms /bin/bash eessi # stick to awscli v1.x, 2.x is not available through PyPI (see https://github.com/aws/aws-cli/issues/4947) RUN pip3 install archspec awscli==${awscliversion} - -RUN curl -OL https://raw.githubusercontent.com/EESSI/eessi-bot-software-layer/develop/scripts/eessi-upload-to-staging \ - && mv eessi-upload-to-staging /usr/bin \ - && chmod a+x /usr/bin/eessi-upload-to-staging diff --git a/containers/build-or-download-cvmfs-debs.sh b/containers/build-or-download-cvmfs-debs.sh index 071fa867..cd81bee9 100644 --- a/containers/build-or-download-cvmfs-debs.sh +++ b/containers/build-or-download-cvmfs-debs.sh @@ -1,22 +1,52 @@ #/bin/bash cvmfsversion=$1 arch=$(dpkg --print-architecture) -os=debian11 apt-get update -apt-get install -y wget -if [ "$arch" = "ppc64el" ] || [ "$arch" = "arm64" ] +apt-get install -y wget lsb-release + +distro=$(lsb_release -si | tr [:upper:] [:lower:]) +release=$(lsb_release -sr) + +# lsb_release -sr prints n/a for debian sid, replace it by 13 +if [ "${distro}" = "debian" ] && [ "${release}" = "n/a" ] +then + release=13 +fi + +os="${distro}${release}" + +if [ "$arch" = "arm64" ] || [ "$arch" = "riscv64" ] || [ "${os}" = "debian13" ] then - apt-get install -y devscripts libfuse3-dev cmake cpio libcap-dev libssl-dev libfuse-dev pkg-config libattr1-dev python-dev python-setuptools python3-dev python3-setuptools uuid-dev valgrind libz-dev lsb-release - # Set Python 2 as default Python - update-alternatives --install /usr/bin/python python /usr/bin/python2 1 - update-alternatives --install /usr/bin/python python /usr/bin/python3 2 + apt-get install -y devscripts libfuse3-dev cmake cpio libcap-dev libssl-dev libfuse-dev pkg-config libattr1-dev python3-dev python3-setuptools python3-dev python3-setuptools uuid-dev libz-dev lsb-release cd /tmp wget https://github.com/cvmfs/cvmfs/archive/refs/tags/cvmfs-${cvmfsversion}.tar.gz tar xzf cvmfs-${cvmfsversion}.tar.gz - cd cvmfs-cvmfs-${cvmfsversion}/ci/cvmfs + cd cvmfs-cvmfs-${cvmfsversion} mkdir /root/deb - sed -i 's/Architecture: i386 amd64 armhf arm64/Architecture: i386 amd64 armhf arm64 ppc64el/' ../../packaging/debian/cvmfs/control.in + sed -i 's/Architecture: i386 amd64 armhf arm64/Architecture: i386 amd64 armhf arm64 riscv64/' packaging/debian/cvmfs/control.in + sed -i 's/python-dev/python3-dev/' packaging/debian/cvmfs/control.in + sed -i 's/python-setuptools/python3-setuptools/' packaging/debian/cvmfs/control.in + if [ "$arch" = "riscv64" ] + then + # valgrind is not available (yet) for RISC-V + sed -i 's/, valgrind//' packaging/debian/cvmfs/control.in + # for RISC-V we need to run autoreconf, see: + # https://github.com/cvmfs/cvmfs/pull/3446 + wget https://github.com/cvmfs/cvmfs/pull/3446.patch + patch -p 1 -i ./3446.patch + rm 3446.patch + # QEMU shows the host CPU in /proc/cpuinfo, so we need to tweak the CPU detection for some packages and use uname -m instead + sed -i "s/^ISA=.*/ISA=\$(uname -m)/" externals/libcrypto/src/configureHook.sh + sed -i "s/rv64/riscv64/" externals/libcrypto/src/configureHook.sh + sed -i "s/^ISA=.*/ISA=\$(uname -m)/" externals/protobuf/src/configureHook.sh + sed -i "s/rv64/riscv64/" externals/protobuf/src/configureHook.sh + else + apt-get install -y valgrind + fi + cd ci/cvmfs + # make sure the cvmfs package also uses debian 13 for debian sid + [ $release = "13" ] && sed -i "s@\$(lsb_release -sr)@13@" ./deb.sh && sed -i "s/focal/trixie/" ./deb.sh ./deb.sh /tmp/cvmfs-cvmfs-${cvmfsversion} /root/deb else mkdir -p /root/deb diff --git a/containers/build-or-download-cvmfs-rpms.sh b/containers/build-or-download-cvmfs-rpms.sh index 2e955768..a4cbcbf3 100644 --- a/containers/build-or-download-cvmfs-rpms.sh +++ b/containers/build-or-download-cvmfs-rpms.sh @@ -2,7 +2,7 @@ cvmfsversion=$1 arch=$(uname -m) yum install -y wget -if [ "$arch" = "ppc64le" ] +if [ "$arch" = "riscv64" ] then yum install -y epel-release yum install -y rpm-build checkpolicy cmake fuse-devel fuse3-devel gcc gcc-c++ golang libattr-devel libcap-devel libuuid-devel openssl-devel python-devel python-setuptools python3-devel python3-setuptools selinux-policy-devel valgrind-devel hardlink selinux-policy-targeted