piraeus-server: deal with large backups

There is a limit of around 1MB per resource in kubernetes. Big clusters might
need to make DB backups that are larger than that. We improve our current
solution in two ways:

* Provide a meaningful output should the run-migration not succeed. Instead
  of crashing by default, print what needs to be done to stderr and wait.
  Waiting can be skipped with an environment variable.
* Try to split the backup into smaller chunks, that can be reassembled on
  restore.

Signed-off-by: Moritz Wanzenböck <[email protected]>

dockerfiles/piraeus-server/entry.sh

@@ -2,6 +2,29 @@
 
 set -e
 
+LB_WAIT_FOR_BACKUP=${LB_WAIT_FOR_BACKUP:-"true"}
+
+create_backup_secret() {
+	BACKUP_NAME="$1"
+	BACKUP_FILE="$2"
+	VERSION="$3"
+
+	if ! kubectl create secret generic "${BACKUP_NAME}" --type piraeus.io/linstor-backup --from-file=backup.tar.gz="${BACKUP_FILE}"; then
+		echo "Backup ${BACKUP_FILE} may be too large to fit into a single secret, trying to split into chunks" >&2
+		split -d -b 512k "${BACKUP_FILE}" "${BACKUP_FILE}." || return 1
+		for SPLIT_FILE in "${BACKUP_FILE}".* ; do
+			kubectl create secret generic "${BACKUP_NAME}-${SPLIT_FILE##"${BACKUP_FILE}".}" --type piraeus.io/linstor-backup-part --from-file=backup.tar.gz="${SPLIT_FILE}" || return 1
+			kubectl label secret "${BACKUP_NAME}-${SPLIT_FILE##"${BACKUP_FILE}".}" piraeus.io/backup="${BACKUP_NAME}" || return 1
+		done
+
+		kubectl create secret generic "${BACKUP_NAME}" --type piraeus.io/linstor-backup || return 1
+	else
+		kubectl label secret "${BACKUP_NAME}" piraeus.io/backup="${BACKUP_NAME}" || return 1
+	fi
+
+	kubectl annotate secrets "${BACKUP_NAME}" "piraeus.io/linstor-version=${VERSION}" "piraeus.io/backup-version=2"
+}
+
 run_migration() {
 	if /usr/share/linstor-server/bin/linstor-config all-migrations-applied --logs=/var/log/linstor-controller --config-directory=/etc/linstor "$@" ; then
 		return 0
@@ -19,8 +42,28 @@ run_migration() {
 			kubectl get "${CRD}" -oyaml > "${CRD}.yaml"
 		done
 		tar -czvf backup.tar.gz -- *.yaml
-		kubectl create secret generic "${BACKUP_NAME}" --type piraeus.io/linstor-backup --from-file=backup.tar.gz
-		kubectl annotate secrets "${BACKUP_NAME}" "piraeus.io/linstor-version=${VERSION}"
+
+		if ! create_backup_secret "${BACKUP_NAME}" backup.tar.gz "${VERSION}"; then
+			echo "===============================================================================" >&2
+			echo "Backup backup.tar.gz too large, even after chunking, to fit into secrets." >&2
+			echo "Please manually copy it to a safe location, by running:" >&2
+			echo "" >&2
+			echo "  kubectl cp -c run-migration $(hostname):/run/migration/backup.tar.gz ." >&2
+			echo "" >&2
+			echo "This container will wait until downloading of the backup has been confirmed." >&2
+			echo "To confirm the backup has been stored locally, create a secret using:" >&2
+			echo "" >&2
+			echo "  kubectl create secret generic ${BACKUP_NAME} --type piraeus.io/linstor-backup" >&2
+			echo "" >&2
+
+			if [ "${LB_WAIT_FOR_BACKUP}" != "true" ]; then
+				return 1
+			fi
+
+			while ! kubectl get secrets "${BACKUP_NAME}" 2>/dev/null ; do
+				sleep 10
+			done
+		fi
 	fi
 
 	/usr/share/linstor-server/bin/linstor-config run-migration --config-directory=/etc/linstor "$@"