From 3ded669d9a51d82e304a4555c75c71a2c6c1a5bf Mon Sep 17 00:00:00 2001 From: Ben Cressey Date: Fri, 3 May 2024 04:54:28 +0000 Subject: [PATCH] twoliter: update Dockerfile for per-package rpm directories Update the buildsys Dockerfile so that it works with RPM packages being outputted to sub-directories, i.e. build/package-name/ Co-authored-by: Matt Briggs Co-authored-by: Ben Cressey Signed-off-by: Ben Cressey --- twoliter/embedded/Dockerfile | 102 +++++++++++++++++----- twoliter/embedded/Dockerfile.dockerignore | 7 +- 2 files changed, 82 insertions(+), 27 deletions(-) diff --git a/twoliter/embedded/Dockerfile b/twoliter/embedded/Dockerfile index 3c619f797..d01a71ed6 100644 --- a/twoliter/embedded/Dockerfile +++ b/twoliter/embedded/Dockerfile @@ -1,7 +1,8 @@ # syntax=docker/dockerfile:1.4.3 -# This Dockerfile has two sections which are used to build rpm.spec packages and to create -# Bottlerocket images, respectively. They are marked as Section 1 and Section 2. buildsys -# uses Section 1 during build-package calls and Section 2 during build-variant calls. +# This Dockerfile has three sections which are used to build rpm.spec packages, to create +# kits, and to create Bottlerocket images, respectively. They are marked as Sections 1-3. +# buildsys uses Section 1 during build-package calls, Section 2 during build-kit calls, +# and Section 3 during build-variant calls. # # Several commands start with RUN --mount=target=/host, which mounts the docker build # context (which in practice is the root of the Bottlerocket repository) as a read-only @@ -94,6 +95,8 @@ RUN \ # Builds an RPM package from a spec file. FROM sdk AS rpmbuild ARG PACKAGE +ARG PACKAGE_DEPENDENCIES +ARG KIT_DEPENDENCIES ARG ARCH ARG NOCACHE ARG VARIANT @@ -129,18 +132,26 @@ RUN \ USER root RUN --mount=target=/host \ - ln -s /host/build/rpms/*.rpm ./rpmbuild/RPMS \ - && createrepo_c \ + for pkg in ${PACKAGE_DEPENDENCIES} ; do \ + ln -s /host/build/rpms/${pkg}/*.rpm ./rpmbuild/RPMS ; \ + done && \ + createrepo_c \ -o ./rpmbuild/RPMS \ -x '*-debuginfo-*.rpm' \ -x '*-debugsource-*.rpm' \ --no-database \ - /host/build/rpms \ - && cp .rpmmacros /etc/rpm/macros \ - && dnf -y \ + ./rpmbuild/RPMS && \ + cp .rpmmacros /etc/rpm/macros && \ + declare -a KIT_REPOS && \ + for kit in ${KIT_DEPENDENCIES} ; do \ + KIT_REPOS+=("--repofrompath=${kit},/host/build/kits/${kit}/${ARCH}" --enablerepo "${kit}") ; \ + done && \ + echo "${KIT_REPOS[@]}" && \ + dnf -y \ --disablerepo '*' \ --repofrompath repo,./rpmbuild/RPMS \ --enablerepo 'repo' \ + "${KIT_REPOS[@]}" \ --nogpgcheck \ --forcearch "${ARCH}" \ builddep rpmbuild/SPECS/${PACKAGE}.spec @@ -167,13 +178,47 @@ FROM scratch AS package COPY --from=rpmbuild /home/builder/rpmbuild/RPMS/*/*.rpm /output/ ############################################################################################ -# Section 2: The following build stages are used to create a Bottlerocket image once all of -# the rpm files have been created by repeatedly using Section 1. +# Section 2: The following build stages are used to create a Bottlerocket kit once all of +# the rpm files have been created by repeatedly using Section 1. This process can occur more +# than once because packages can depend on kits and those kits depend on packages that must +# be built first. # =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= -# Creates an RPM repository from packages created in Section 1. +# Builds a kit from RPM packages. +FROM sdk AS kitbuild +ARG KIT +ARG PACKAGE_DEPENDENCIES +ARG ARCH +ARG NOCACHE + +WORKDIR /home/builder +USER builder + +RUN --mount=target=/host \ + /host/build/tools/rpm2kit \ + --packages-dir=/host/build/rpms \ + --arch="${ARCH}" \ + "${PACKAGE_DEPENDENCIES[@]/#/--package=}" \ + --output-dir=/home/builder/output \ + && echo ${NOCACHE} + +# Copies kit artifacts from the previous stage to their expected location so that buildsys +# can find them and copy them out. +FROM scratch AS kit +COPY --from=kitbuild /home/builder/output/. /output/ + +############################################################################################ +# Section 3: The following build stages are used to create a Bottlerocket image once all of +# the rpm files have been created by repeatedly using Sections 1 and 2. + +# =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= +# Creates an RPM repository from packages created in Section 1 and kits from Section 2. FROM sdk AS repobuild +# The list of packages from the variant Cargo.toml package.metadata.build-variant.packages section. ARG PACKAGES +# The complete list of non-kit packages required by way of pure package-to-package dependencies. +ARG PACKAGE_DEPENDENCIES +ARG KIT_DEPENDENCIES ARG ARCH ARG NOCACHE @@ -196,28 +241,36 @@ RUN --mount=target=/host \ WORKDIR /root USER root RUN --mount=target=/host \ - mkdir -p /local/rpms ./rpmbuild/RPMS \ - && ln -s /host/build/rpms/*.rpm ./rpmbuild/RPMS \ - && ln -s /home/builder/rpmbuild/RPMS/*/*.rpm ./rpmbuild/RPMS \ - && createrepo_c \ + mkdir -p ./rpmbuild/RPMS && \ + for pkg in ${PACKAGE_DEPENDENCIES} ; do \ + ln -s /host/build/rpms/${pkg}/*.rpm ./rpmbuild/RPMS ; \ + done && \ + ln -s /home/builder/rpmbuild/RPMS/*/*.rpm ./rpmbuild/RPMS && \ + createrepo_c \ -o ./rpmbuild/RPMS \ -x '*-debuginfo-*.rpm' \ -x '*-debugsource-*.rpm' \ --no-database \ - ./rpmbuild/RPMS \ - && echo '%_dbpath %{_sharedstatedir}/rpm' >> /etc/rpm/macros \ - && dnf -y \ + ./rpmbuild/RPMS && \ + echo '%_dbpath %{_sharedstatedir}/rpm' >> /etc/rpm/macros && \ + declare -a KIT_REPOS && \ + for kit in ${KIT_DEPENDENCIES} ; do \ + KIT_REPOS+=("--repofrompath=${kit},/host/build/kits/${kit}/${ARCH}" --enablerepo "${kit}") ; \ + done && \ + dnf -y \ --disablerepo '*' \ --repofrompath repo,./rpmbuild/RPMS \ --enablerepo 'repo' \ + "${KIT_REPOS[@]}" \ --nogpgcheck \ --downloadonly \ --downloaddir . \ --forcearch "${ARCH}" \ - install $(printf "bottlerocket-%s\n" metadata ${PACKAGES}) \ - && mv *.rpm /local/rpms \ - && createrepo_c /local/rpms \ - && echo ${NOCACHE} + install $(printf "bottlerocket-%s\n" metadata ${PACKAGES}) && \ + mkdir -p /local/rpms && \ + mv *.rpm /local/rpms && \ + createrepo_c /local/rpms && \ + echo ${NOCACHE} # =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= # Builds a Bottlerocket image. @@ -288,7 +341,7 @@ WORKDIR /root USER root RUN --mount=target=/host \ mkdir -p /local/migrations \ - && find /host/build/rpms/ -maxdepth 1 -type f \ + && find /host/build/rpms/os/ -maxdepth 1 -type f \ -name "bottlerocket-migrations-*.rpm" \ -not -iname '*debuginfo*' \ -exec cp '{}' '/local/migrations/' ';' \ @@ -300,6 +353,7 @@ RUN --mount=target=/host \ # =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= # Creates an archive of kernel development sources and toolchain. FROM repobuild as kmodkitbuild +# The list of packages from the variant Cargo.toml package.metadata.build-variant.packages section. ARG PACKAGES ARG ARCH ARG VERSION_ID @@ -314,7 +368,7 @@ WORKDIR /tmp RUN --mount=target=/host \ mkdir -p /local/archives \ && KERNEL="$(printf "%s\n" ${PACKAGES} | awk '/^kernel-/{print $1}')" \ - && find /host/build/rpms/ -maxdepth 1 -type f \ + && find /host/build/rpms/${KERNEL}/ -maxdepth 1 -type f \ -name "bottlerocket-${KERNEL}-archive-*.rpm" \ -exec cp '{}' '/local/archives/' ';' \ && /host/build/tools/rpm2kmodkit \ diff --git a/twoliter/embedded/Dockerfile.dockerignore b/twoliter/embedded/Dockerfile.dockerignore index c2bf40d23..2f9c7757e 100644 --- a/twoliter/embedded/Dockerfile.dockerignore +++ b/twoliter/embedded/Dockerfile.dockerignore @@ -3,9 +3,10 @@ /build/* !/build/rpms/ /build/rpms/* -!/build/rpms/*.rpm -/build/rpms/*-debuginfo-*.rpm -/build/rpms/*-debugsource-*.rpm +!/build/rpms/*/*.rpm +/build/rpms/*/*-debuginfo-*.rpm +/build/rpms/*/*-debugsource-*.rpm +!/build/kits/ !/build/tools/* **/target/* /sbkeys