Fix support for Fedora probe builder

Fedora 31 introduced zstd-compressed RPMs, which older rpm2cpio
can't read. Work around it by using a helper Alpine-based Docker
image that includes a new enough version.

Add a Fedora 31 gcc 9.2 builder too for future-proofing.

Also fix the crawler to include Fedora 30/31 kernels, as f29
and older are now archived.

While we're removing dependencies, move the kpartx device mapper
manipulation to the toolkit image as well

probe-builder/Dockerfile.centos-gcc9.2

@@ -0,0 +1,20 @@
+FROM fedora:31
+
+RUN yum -y install \
+	wget \
+	git \
+	gcc \
+	gcc-c++ \
+	autoconf \
+	bison \
+	flex \
+	make \
+	cmake \
+	elfutils-devel \
+	findutils \
+	kmod \
+	python-lxml && yum clean all
+
+ADD builder-entrypoint.sh /
+WORKDIR /build/probe
+ENTRYPOINT [ "/builder-entrypoint.sh" ]

probe-builder/Dockerfile.toolkit

@@ -0,0 +1,5 @@
+FROM alpine
+
+RUN apk add rpm2cpio multipath-tools
+ADD toolkit-entrypoint.sh /toolkit-entrypoint.sh
+ENTRYPOINT ["/toolkit-entrypoint.sh"]

probe-builder/build-probe-binaries

@@ -227,6 +227,15 @@ function checkout_sysdig {
 	HAVE_SYSDIG_CHECKOUT=1
 }
 
+function build_toolkit {
+	if [ -z "${HAVE_BUILDER_TOOLKIT:-}" ]
+	then
+		IMAGE_NAME="${BUILDER_IMAGE_PREFIX:-}sysdig-probe-builder:toolkit"
+		docker build -t "$IMAGE_NAME" -f $BUILDER_SOURCE/Dockerfile.toolkit --pull $BUILDER_SOURCE
+		HAVE_BUILDER_TOOLKIT=1
+	fi
+}
+
 declare -A builders
 function build_probe {
 	# Skip Kernel 4.15.0-29 because probe does not build against it
@@ -345,31 +354,12 @@ function coreos_build {
 		VERSION_DIR=build/$DISTRO/$VERSION
 		if [ ! -e $VERSION_DIR/config_orig ]
 		then
+			mkdir -p $VERSION_DIR
 			echo "Unpacking $(basename $IMG)"
-			mkdir -p $VERSION_DIR/container
-			if [ ! -e $VERSION_DIR/container.img ]
-			then
-				bzcat $IMG > $VERSION_DIR/container.img
-			fi
-
-			# mount developer container is a very stateful part of this script
-			# the section between mount/unmounting should be kept very small
-			# otherwise if something fails there are many inconsistencies that can happen
-			LOOPDEV=$(sudo kpartx -asv $VERSION_DIR/container.img | cut -d\  -f 3)
-			sudo mount /dev/mapper/$LOOPDEV $VERSION_DIR/container
-			# Copy kernel headers
-			cp -r $VERSION_DIR/container/lib/modules $VERSION_DIR
-
-			# Copy kernel config
-			rm -f $VERSION_DIR/config-*
-			cp $VERSION_DIR/container/usr/boot/config-* $VERSION_DIR/
-			cp $VERSION_DIR/config-* $VERSION_DIR/config_orig
-
-			# umount and remove the developer container
-			sudo umount $VERSION_DIR/container
-			sudo kpartx -dv $VERSION_DIR/container.img
-			rmdir $VERSION_DIR/container
-			rm $VERSION_DIR/container.img
+			FULL_IMG=$(readlink -f "$IMG")
+			FULL_VERSION_DIR=$(readlink -f "$VERSION_DIR")
+			build_toolkit
+			docker run --rm --privileged -v "$FULL_IMG:$FULL_IMG:ro" -v "$FULL_VERSION_DIR:$FULL_VERSION_DIR" "${BUILDER_IMAGE_PREFIX:-}sysdig-probe-builder:toolkit" coreos "$FULL_IMG" "$FULL_VERSION_DIR"
 
 			KERNEL_RELEASE=$(cd $VERSION_DIR && ls config-* | sed s/config-//)
 			export COREOS_BUILD=${VERSION%%.*}
@@ -484,7 +474,10 @@ function rhel_build {
 		else
 			echo "Unpacking $RPM to $TARGET"
 			mkdir -p $TARGET
-			rpm2cpio $RPM | (cd $TARGET ; cpio -idm)
+			FULL_RPM=$(readlink -f "$RPM")
+			FULL_TARGET=$(readlink -f "$TARGET")
+			build_toolkit
+			docker run --rm --privileged -v "$FULL_RPM:$FULL_RPM:ro" -v "$FULL_TARGET:$FULL_TARGET" "${BUILDER_IMAGE_PREFIX:-}sysdig-probe-builder:toolkit" rpm "$FULL_RPM" "$FULL_TARGET"
 			touch $MARKER
 		fi
 	done

probe-builder/kernel-crawler.py

@@ -155,7 +155,7 @@
     "Fedora" : [
         {
             "root" : "https://mirrors.kernel.org/fedora/releases/",
-            "discovery_pattern": "/html/body//a[regex:test(@href, '^2[2-9]/$')]/@href",
+            "discovery_pattern": "/html/body//a[regex:test(@href, '^[3-9][0-9]/$')]/@href",
             "subdirs" : [
                 "Everything/x86_64/os/Packages/k/"
             ],
@@ -164,7 +164,7 @@
 
         {
             "root" : "https://mirrors.kernel.org/fedora/updates/",
-            "discovery_pattern": "/html/body//a[regex:test(@href, '^2[2-9]/$')]/@href",
+            "discovery_pattern": "/html/body//a[regex:test(@href, '^[3-9][0-9]/$')]/@href",
             "subdirs" : [
                 "x86_64/Packages/k/"
             ],
@@ -173,7 +173,7 @@
 
         {
             "root" : "https://mirrors.kernel.org/fedora/updates/",
-            "discovery_pattern": "/html/body//a[regex:test(@href, '^2[2-9]/$')]/@href",
+            "discovery_pattern": "/html/body//a[regex:test(@href, '^[3-9][0-9]/$')]/@href",
             "subdirs" : [
                 "Everything/x86_64/Packages/k/"
             ],

probe-builder/toolkit-entrypoint.sh

@@ -0,0 +1,56 @@
+#!/bin/sh
+
+set -euo pipefail
+
+usage() {
+	cat >&2 <<EOF
+Usage:
+	$0 DISTRIBUTION KERNEL_PACKAGE OUTPUT_DIR
+EOF
+	exit 1
+}
+
+unpack_coreos_kernel()
+{
+	KERNEL_PACKAGE="$1"
+	OUTPUT_DIR="$2"
+
+	bzcat "$KERNEL_PACKAGE" > /tmp/container.img
+
+	# mount developer container is a very stateful part of this script
+	# the section between mount/unmounting should be kept very small
+	# otherwise if something fails there are many inconsistencies that can happen
+	LOOPDEV=$(kpartx -asv /tmp/container.img | cut -d\  -f 3)
+	mount /dev/mapper/$LOOPDEV /mnt
+	# Copy kernel headers
+	cp -r /mnt/lib/modules "$OUTPUT_DIR"
+
+	# Copy kernel config
+	rm -f $OUTPUT_DIR/config-*
+	cp /mnt/usr/boot/config-* $OUTPUT_DIR/
+	cp $OUTPUT_DIR/config-* $OUTPUT_DIR/config_orig
+	# umount and remove the developer container
+	umount /mnt
+	kpartx -dv /tmp/container.img
+}
+
+unpack_rpm()
+{
+	KERNEL_PACKAGE="$1"
+	OUTPUT_DIR="$2"
+
+	rpm2cpio "$KERNEL_PACKAGE" | (cd "$OUTPUT_DIR" && cpio -idm)
+}
+
+case "$1" in
+	coreos)
+		unpack_coreos_kernel "$2" "$3"
+		;;
+	rpm)
+		unpack_rpm "$2" "$3"
+		;;
+	*)
+		echo "Unsupported distribution $1"
+		exit 1
+		;;
+esac