Merge pull request #67 from fayaaz/bookworm-update

Update to bookworm

Dockerfile

@@ -8,7 +8,7 @@ RUN apt-get -y update && \
         git vim parted \
         quilt coreutils qemu-user-static debootstrap zerofree zip dosfstools \
         libarchive-tools libcap2-bin rsync grep udev xz-utils curl xxd file kmod bc\
-        binfmt-support ca-certificates qemu-utils kpartx \
+        binfmt-support ca-certificates qemu-utils kpartx fdisk gpg pigz\
     && rm -rf /var/lib/apt/lists/*
 
 COPY . /pi-gen/

build-docker.sh

@@ -1,13 +1,20 @@
-#!/bin/bash -eu
+#!/usr/bin/env bash
+# Note: Avoid usage of arrays as MacOS users have an older version of bash (v3.x) which does not supports arrays
+set -eu
 
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+DIR="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)"
 
 BUILD_OPTS="$*"
 
-DOCKER="docker"
+# Allow user to override docker command
+DOCKER=${DOCKER:-docker}
 
-if ! ${DOCKER} ps >/dev/null 2>&1; then
-	DOCKER="sudo docker"
+# Ensure that default docker command is not set up in rootless mode
+if \
+  ! ${DOCKER} ps    >/dev/null 2>&1 || \
+    ${DOCKER} info 2>/dev/null | grep -q rootless \
+; then
+	DOCKER="sudo ${DOCKER}"
 fi
 if ! ${DOCKER} ps >/dev/null; then
 	echo "error connecting to docker:"
@@ -48,7 +55,7 @@ fi
 CONTAINER_NAME=${CONTAINER_NAME:-pigen_work}
 CONTINUE=${CONTINUE:-0}
 PRESERVE_CONTAINER=${PRESERVE_CONTAINER:-0}
-PIGEN_DOCKER_OPTS=${PIGEN_DOCKER_OPTS:-""}  
+PIGEN_DOCKER_OPTS=${PIGEN_DOCKER_OPTS:-""}
 
 if [ -z "${IMG_NAME}" ]; then
 	echo "IMG_NAME not set in 'config'" 1>&2
@@ -75,50 +82,83 @@ fi
 # Modify original build-options to allow config file to be mounted in the docker container
 BUILD_OPTS="$(echo "${BUILD_OPTS:-}" | sed -E 's@\-c\s?([^ ]+)@-c /config@')"
 
-# Check the arch of the machine we're running on. If it's 64-bit, use a 32-bit base image instead
-case "$(uname -m)" in
-  x86_64|aarch64)
-    BASE_IMAGE=i386/debian:bullseye
+${DOCKER} build --build-arg BASE_IMAGE=debian:bullseye -t pi-gen "${DIR}"
+
+if [ "${CONTAINER_EXISTS}" != "" ]; then
+  DOCKER_CMDLINE_NAME="${CONTAINER_NAME}_cont"
+  DOCKER_CMDLINE_PRE="--rm"
+  DOCKER_CMDLINE_POST="--volumes-from=${CONTAINER_NAME}"
+else
+  DOCKER_CMDLINE_NAME="${CONTAINER_NAME}"
+  DOCKER_CMDLINE_PRE=""
+  DOCKER_CMDLINE_POST=""
+fi
+
+# Check if binfmt_misc is required
+binfmt_misc_required=1
+case $(uname -m) in
+  aarch64)
+    binfmt_misc_required=0
     ;;
-  *)
-    BASE_IMAGE=debian:bullseye
+  arm*)
+    binfmt_misc_required=0
     ;;
 esac
-${DOCKER} build --build-arg BASE_IMAGE=${BASE_IMAGE} -t pi-gen "${DIR}"
 
-if [ "${CONTAINER_EXISTS}" != "" ]; then
-	trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${CONTAINER_NAME}_cont' SIGINT SIGTERM
-	time ${DOCKER} run --rm --privileged \
-		--cap-add=ALL \
-		-v /dev:/dev \
-		-v /lib/modules:/lib/modules \
-		${PIGEN_DOCKER_OPTS} \
-		--volume "${CONFIG_FILE}":/config:ro \
-		-e "GIT_HASH=${GIT_HASH}" \
-		--volumes-from="${CONTAINER_NAME}" --name "${CONTAINER_NAME}_cont" \
-		pi-gen \
-		bash -e -o pipefail -c "dpkg-reconfigure qemu-user-static &&
-	cd /pi-gen; ./build.sh ${BUILD_OPTS} &&
-	rsync -av work/*/build.log deploy/" &
-	wait "$!"
-else
-	trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${CONTAINER_NAME}' SIGINT SIGTERM
-	time ${DOCKER} run --name "${CONTAINER_NAME}" --privileged \
-		--cap-add=ALL \
-		-v /dev:/dev \
-		-v /lib/modules:/lib/modules \
-		${PIGEN_DOCKER_OPTS} \
-		--volume "${CONFIG_FILE}":/config:ro \
-		-e "GIT_HASH=${GIT_HASH}" \
-		pi-gen \
-		bash -e -o pipefail -c "dpkg-reconfigure qemu-user-static &&
-	cd /pi-gen; ./build.sh ${BUILD_OPTS} &&
-	rsync -av work/*/build.log deploy/" &
-	wait "$!"
+# Check if qemu-aarch64-static and /proc/sys/fs/binfmt_misc are present
+if [[ "${binfmt_misc_required}" == "1" ]]; then
+  if ! qemu_arm=$(which qemu-aarch64-static) ; then
+    echo "qemu-aarch64-static not found (please install qemu-user-static)"
+    exit 1
+  fi
+  if [ ! -f /proc/sys/fs/binfmt_misc/register ]; then
+    echo "binfmt_misc required but not mounted, trying to mount it..."
+    if ! mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc ; then
+        echo "mounting binfmt_misc failed"
+        exit 1
+    fi
+    echo "binfmt_misc mounted"
+  fi
+  if ! grep -q "^interpreter ${qemu_arm}" /proc/sys/fs/binfmt_misc/qemu-aarch64* ; then
+    # Register qemu-aarch64 for binfmt_misc
+    reg="echo ':qemu-aarch64-rpi:M::"\
+"\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xb7\x00:"\
+"\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:"\
+"${qemu_arm}:F' > /proc/sys/fs/binfmt_misc/register"
+    echo "Registering qemu-aarch64 for binfmt_misc..."
+    sudo bash -c "${reg}" 2>/dev/null || true
+  fi
 fi
 
+trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${DOCKER_CMDLINE_NAME}' SIGINT SIGTERM
+time ${DOCKER} run \
+  $DOCKER_CMDLINE_PRE \
+  --name "${DOCKER_CMDLINE_NAME}" \
+  --privileged \
+  --cap-add=ALL \
+  -v /dev:/dev \
+  -v /lib/modules:/lib/modules \
+  ${PIGEN_DOCKER_OPTS} \
+  --volume "${CONFIG_FILE}":/config:ro \
+  -e "GIT_HASH=${GIT_HASH}" \
+  $DOCKER_CMDLINE_POST \
+  pi-gen \
+  bash -e -o pipefail -c "
+    dpkg-reconfigure qemu-user-static &&
+    # binfmt_misc is sometimes not mounted with debian bullseye image
+    (mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc || true) &&
+    cd /pi-gen; ./build.sh ${BUILD_OPTS} &&
+    rsync -av work/*/build.log deploy/
+  " &
+  wait "$!"
+
+# Ensure that deploy/ is always owned by calling user
 echo "copying results from deploy/"
-${DOCKER} cp "${CONTAINER_NAME}":/pi-gen/deploy .
+${DOCKER} cp "${CONTAINER_NAME}":/pi-gen/deploy - | tar -xf -
+
+echo "copying log from container ${CONTAINER_NAME} to depoy/"
+${DOCKER} logs --timestamps "${CONTAINER_NAME}" &>deploy/build-docker.log
+
 ls -lah deploy
 
 # cleanup

build.sh

@@ -21,7 +21,7 @@ EOF
 			PACKAGES="$(sed -f "${SCRIPT_DIR}/remove-comments.sed" < "${i}-packages-nr")"
 			if [ -n "$PACKAGES" ]; then
 				on_chroot << EOF
-apt-get -o APT::Acquire::Retries=3 install --no-install-recommends -y $PACKAGES
+apt-get -o Acquire::Retries=3 install --no-install-recommends -y $PACKAGES
 EOF
 				if [ "${USE_QCOW2}" = "1" ]; then
 					on_chroot << EOF
@@ -36,7 +36,7 @@ EOF
 			PACKAGES="$(sed -f "${SCRIPT_DIR}/remove-comments.sed" < "${i}-packages")"
 			if [ -n "$PACKAGES" ]; then
 				on_chroot << EOF
-apt-get -o APT::Acquire::Retries=3 install -y $PACKAGES
+apt-get -o Acquire::Retries=3 install -y $PACKAGES
 EOF
 				if [ "${USE_QCOW2}" = "1" ]; then
 					on_chroot << EOF
@@ -133,7 +133,7 @@ run_stage(){
 		done
 	fi
 
-	if [ "${USE_QCOW2}" = "1" ]; then 
+	if [ "${USE_QCOW2}" = "1" ]; then
 		unload_qimage
 	else
 		# make sure we are not umounting during export-image stage
@@ -155,6 +155,14 @@ if [ "$(id -u)" != "0" ]; then
 fi
 
 BASE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+if [[ $BASE_DIR = *" "* ]]; then
+	echo "There is a space in the base path of pi-gen"
+	echo "This is not a valid setup supported by debootstrap."
+	echo "Please remove the spaces, or move pi-gen directory to a base path without spaces" 1>&2
+	exit 1
+fi
+
 export BASE_DIR
 
 if [ -f config ]; then
@@ -195,19 +203,29 @@ fi
 export USE_QEMU="${USE_QEMU:-0}"
 export IMG_DATE="${IMG_DATE:-"$(date +%Y-%m-%d)"}"
 export IMG_FILENAME="${IMG_FILENAME:-"${IMG_DATE}-${IMG_NAME}"}"
-export ZIP_FILENAME="${ZIP_FILENAME:-"image_${IMG_DATE}-${IMG_NAME}"}"
+export ARCHIVE_FILENAME="${ARCHIVE_FILENAME:-"image_${IMG_DATE}-${IMG_NAME}"}"
 
 export SCRIPT_DIR="${BASE_DIR}/scripts"
 export WORK_DIR="${WORK_DIR:-"${BASE_DIR}/work/${IMG_NAME}"}"
 export DEPLOY_DIR=${DEPLOY_DIR:-"${BASE_DIR}/deploy"}
-export DEPLOY_ZIP="${DEPLOY_ZIP:-1}"
+
+# DEPLOY_ZIP was deprecated in favor of DEPLOY_COMPRESSION
+# This preserve the old behavior with DEPLOY_ZIP=0 where no archive was created
+if [ -z "${DEPLOY_COMPRESSION}" ] && [ "${DEPLOY_ZIP:-1}" = "0" ]; then
+	echo "DEPLOY_ZIP has been deprecated in favor of DEPLOY_COMPRESSION"
+	echo "Similar behavior to DEPLOY_ZIP=0 can be obtained with DEPLOY_COMPRESSION=none"
+	echo "Please update your config file"
+	DEPLOY_COMPRESSION=none
+fi
+export DEPLOY_COMPRESSION=${DEPLOY_COMPRESSION:-zip}
+export COMPRESSION_LEVEL=${COMPRESSION_LEVEL:-6}
 export LOG_FILE="${WORK_DIR}/build.log"
 
 export TARGET_HOSTNAME=${TARGET_HOSTNAME:-mixxxpi}
 
 export FIRST_USER_NAME=${FIRST_USER_NAME:-pi}
 export FIRST_USER_PASS=${FIRST_USER_PASS:-mixxx}
-export RELEASE=${RELEASE:-bullseye}
+export RELEASE=${RELEASE:-bookworm}
 export WPA_ESSID
 export WPA_PASSWORD
 export WPA_COUNTRY
@@ -265,6 +283,10 @@ fi
 
 export NO_PRERUN_QCOW2="${NO_PRERUN_QCOW2:-1}"
 
+if [ "$SETFCAP" != "1" ]; then
+	export CAPSH_ARG="--drop=cap_setfcap"
+fi
+
 dependencies_check "${BASE_DIR}/depends"
 
 #check username is valid
@@ -273,6 +295,17 @@ if [[ ! "$FIRST_USER_NAME" =~ ^[a-z][-a-z0-9_]*$ ]]; then
 	exit 1
 fi
 
+if [[ "$DISABLE_FIRST_BOOT_USER_RENAME" == "1" ]] && [ -z "${FIRST_USER_PASS}" ]; then
+	echo "To disable user rename on first boot, FIRST_USER_PASS needs to be set"
+	echo "Not setting FIRST_USER_PASS makes your system vulnerable and open to cyberattacks"
+	exit 1
+fi
+
+if [[ "$DISABLE_FIRST_BOOT_USER_RENAME" == "1" ]]; then
+	echo "User rename on the first boot is disabled"
+	echo "Be advised of the security risks linked to shipping a device with default username/password set."
+fi
+
 if [[ -n "${APT_PROXY}" ]] && ! curl --silent "${APT_PROXY}" >/dev/null ; then
 	echo "Could not reach APT_PROXY server: ${APT_PROXY}"
 	exit 1
@@ -370,7 +403,7 @@ for EXPORT_DIR in ${EXPORT_DIRS}; do
 
 	else
 		run_stage
-	fi 
+	fi
 	if [ "${USE_QEMU}" != "1" ]; then
 		if [ -e "${EXPORT_DIR}/EXPORT_NOOBS" ]; then
 			# shellcheck source=/dev/null

depends

@@ -19,3 +19,5 @@ lsmod:kmod
 bc
 qemu-nbd:qemu-utils
 kpartx
+gpg
+pigz

export-image/01-user-rename/00-packages

@@ -0,0 +1 @@
+userconf-pi

export-image/01-user-rename/01-run.sh

@@ -0,0 +1,9 @@
+#!/bin/bash -e
+
+if [[ "${DISABLE_FIRST_BOOT_USER_RENAME}" == "0" ]]; then
+	on_chroot <<- EOF
+		SUDO_USER="${FIRST_USER_NAME}" rename-user -f -s
+	EOF
+else
+	rm -f "${ROOTFS_DIR}/etc/xdg/autostart/piwiz.desktop"
+fi

export-image/01-set-sources/01-run.sh → export-image/02-set-sources/01-run.sh

@@ -4,6 +4,6 @@ rm -f "${ROOTFS_DIR}/etc/apt/apt.conf.d/51cache"
 find "${ROOTFS_DIR}/var/lib/apt/lists/" -type f -delete
 on_chroot << EOF
 apt-get update
-apt-get -y dist-upgrade
+apt-get -y dist-upgrade --auto-remove --purge
 apt-get clean
 EOF

export-image/03-set-partuuid/00-run.sh → export-image/04-set-partuuid/00-run.sh

@@ -12,7 +12,6 @@ if [ "${NO_PRERUN_QCOW2}" = "0" ]; then
 	sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"
 	sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"
 
-	sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/boot/cmdline.txt"
-
+	sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/boot/firmware/cmdline.txt"
 fi
 

export-image/04-finalise/01-run.sh → export-image/05-finalise/01-run.sh

@@ -4,7 +4,10 @@ IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"
 INFO_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.info"
 MIXXX_VERSION_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}-mixxx.version"
 
+sed -i 's/^update_initramfs=.*/update_initramfs=all/' "${ROOTFS_DIR}/etc/initramfs-tools/update-initramfs.conf"
+
 on_chroot << EOF
+update-initramfs -u
 if [ -x /etc/init.d/fake-hwclock ]; then
 	/etc/init.d/fake-hwclock stop
 fi
@@ -54,7 +57,7 @@ rm -f "${ROOTFS_DIR}/root/.vnc/private.key"
 rm -f "${ROOTFS_DIR}/etc/vnc/updateid"
 
 update_issue "$(basename "${EXPORT_DIR}")"
-install -m 644 "${ROOTFS_DIR}/etc/rpi-issue" "${ROOTFS_DIR}/boot/issue.txt"
+install -m 644 "${ROOTFS_DIR}/etc/rpi-issue" "${ROOTFS_DIR}/boot/firmware/issue.txt"
 
 cp "$ROOTFS_DIR/etc/rpi-issue" "$INFO_FILE"
 cp "$ROOTFS_DIR/opt/mixxx.version" "$MIXXX_VERSION_FILE"
@@ -78,9 +81,10 @@ cp "$ROOTFS_DIR/opt/mixxx.version" "$MIXXX_VERSION_FILE"
 } >> "$INFO_FILE"
 
 mkdir -p "${DEPLOY_DIR}"
-rm -f "${DEPLOY_DIR}/${ZIP_FILENAME}${IMG_SUFFIX}.zip"
+rm -f "${DEPLOY_DIR}/${ARCHIVE_FILENAME}${IMG_SUFFIX}.*"
 rm -f "${DEPLOY_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"
 
+
 mv "$INFO_FILE" "$DEPLOY_DIR/"
 
 if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then
@@ -93,15 +97,27 @@ if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then
 else
 	unload_qimage
 	make_bootable_image "${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.qcow2" "$IMG_FILE"
+	cp "$IMG_FILE" "$DEPLOY_DIR"
 fi
 
-if [ "${DEPLOY_ZIP}" == "1" ]; then
+case "${DEPLOY_COMPRESSION}" in
+zip)
 	pushd "${STAGE_WORK_DIR}" > /dev/null
-	zip "${DEPLOY_DIR}/${ZIP_FILENAME}${IMG_SUFFIX}.zip" \
-		"$(basename "${IMG_FILE}")"
+	zip -"${COMPRESSION_LEVEL}" \
+	"${DEPLOY_DIR}/${ARCHIVE_FILENAME}${IMG_SUFFIX}.zip" "$(basename "${IMG_FILE}")"
 	popd > /dev/null
-else
-	mv "$IMG_FILE" "$DEPLOY_DIR/"
-fi
+	;;
+gz)
+	pigz --force -"${COMPRESSION_LEVEL}" "$IMG_FILE" --stdout > \
+	"${DEPLOY_DIR}/${ARCHIVE_FILENAME}${IMG_SUFFIX}.img.gz"
+	;;
+xz)
+	xz --compress --force --threads 0 --memlimit-compress=50% -"${COMPRESSION_LEVEL}" \
+	--stdout "$IMG_FILE" > "${DEPLOY_DIR}/${ARCHIVE_FILENAME}${IMG_SUFFIX}.img.xz"
+	;;
+none | *)
+	cp "$IMG_FILE" "$DEPLOY_DIR/"
+;;
+esac
 
 mv "$MIXXX_VERSION_FILE" "$DEPLOY_DIR"