[#43] Switch to new MariaDB image.

t2
drevops · Dec 2, 2024 · bc02d7c · bc02d7c
1 parent 17394f2
commit bc02d7c
Show file tree

Hide file tree

Showing 10 changed files with 290 additions and 113 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -6,8 +6,8 @@ jobs:
       - image: drevops/ci-runner:24.11.0
         environment:
           BUILDX_VERSION: v0.10.4
-          BUILDX_PLATFORMS: linux/amd64,linux/arm64,linux/arm/v7
     working_directory: /root/project
+
     steps:
       - checkout
       - run:
@@ -39,6 +39,8 @@ jobs:
           command: codecov --fail-on-error -t $CODECOV_TOKEN -s coverage
       - run:
           name: Deploy image
+          environment:
+            BUILDX_PLATFORMS: linux/amd64,linux/arm64
           command: |
             if [ -n "${CIRCLE_TAG}" ]; then
               export TAG="${CIRCLE_TAG}"

diff --git a/9999-mariadb-init.bash b/9999-mariadb-init.bash
@@ -52,6 +52,9 @@ if [ "$1" = 'mysqld' -a -z "$wantHelp" ]; then
     chown -R mysql:mysql /run/mysqld
   fi
 
+  MARIADB_INIT_WAIT_SECONDS=${MARIADB_INIT_WAIT_SECONDS:-30}
+  MARIADB_INIT_PERIOD_SECONDS=${MARIADB_INIT_PERIOD_SECONDS:-1}
+
   # @note: If data dir exists and is not empty - most likely the DB has
   # already been initialised.
   if [ -d ${MARIADB_DATA_DIR:-/var/lib/mysql} ] && [ "$(ls -A "${MARIADB_DATA_DIR:-/var/lib/mysql}")" ]; then
@@ -70,18 +73,21 @@ if [ "$1" = 'mysqld' -a -z "$wantHelp" ]; then
     pid="$!"
     echo "pid is $pid"
 
-    for i in {30..0}; do
+    for i in $(seq 0 $MARIADB_INIT_WAIT_SECONDS); do
       if echo 'SELECT 1' | mysql -u root; then
         break
       fi
       echo 'MySQL init process in progress...'
-      sleep 1
+      sleep $MARIADB_INIT_PERIOD_SECONDS
     done
 
     # @note: Added a flag to force upgrade.
     if [ "${FORCE_MYSQL_UPGRADE:-}" = "1" ]; then
       echo "starting mysql upgrade"
-      mysql_upgrade --force
+      # @note: mariadb-upgrade may fail on the first run due to the unresolved
+      # permissions, but will succeed on the second run.
+      # @see https://mariadb.com/kb/en/mariadb-upgrade/#
+      mariadb-upgrade --force || mariadb-upgrade --force
     fi
 
     if ! kill -s TERM "$pid" || ! wait "$pid"; then
@@ -98,12 +104,12 @@ if [ "$1" = 'mysqld' -a -z "$wantHelp" ]; then
     pid="$!"
     echo "pid is $pid"
 
-    for i in {30..0}; do
+    for i in $(seq 0 $MARIADB_INIT_WAIT_SECONDS); do
       if echo 'SELECT 1' | mysql -u root; then
         break
       fi
       echo 'MySQL init process in progress...'
-      sleep 1
+      sleep $MARIADB_INIT_PERIOD_SECONDS
     done
 
     if [ "$MARIADB_ROOT_PASSWORD" = "" ]; then
@@ -124,6 +130,8 @@ if [ "$1" = 'mysqld' -a -z "$wantHelp" ]; then
 DROP DATABASE IF EXISTS test;
 USE mysql;
 ALTER USER root@localhost IDENTIFIED VIA mysql_native_password USING PASSWORD("$MARIADB_ROOT_PASSWORD");
+DELETE FROM global_priv WHERE User='root' AND Host NOT IN ('localhost', '127.0.0.1', '::1');
+DELETE FROM proxies_priv WHERE User='root' AND Host NOT IN ('localhost', '127.0.0.1', '::1');
 FLUSH PRIVILEGES;
 
 EOF
@@ -148,13 +156,14 @@ EOF
     echo "[mysql]" >> ${MARIADB_DATA_DIR:-/var/lib/mysql}/.my.cnf
     echo "database=${MARIADB_DATABASE}" >> ${MARIADB_DATA_DIR:-/var/lib/mysql}/.my.cnf
 
-    for f in `ls /docker-entrypoint-initdb.d/*`; do
-      case "$f" in
-        *.sh)     echo "$0: running $f"; . "$f" ;;
-        *.sql)    echo "$0: running $f"; cat $f| envsubst | tee | mysql -u root -p${MARIADB_ROOT_PASSWORD}; echo ;;
-        *)        echo "$0: ignoring $f" ;;
-      esac
-    echo
+    for f in /docker-entrypoint-initdb.d/*; do
+      if [ -e "$f" ]; then
+        case "$f" in
+          *.sh)     echo "$0: running $f"; . "$f" ;;
+          *.sql)    echo "$0: running $f"; cat $f| envsubst | tee | mysql -u root -p${MARIADB_ROOT_PASSWORD}; echo ;;
+          *)        echo "$0: ignoring $f" ;;
+        esac
+      fi
     done
 
     if ! kill -s TERM "$pid" || ! wait "$pid"; then
@@ -165,6 +174,7 @@ EOF
   fi
 
   echo "done, now starting daemon"
+  touch /tmp/mariadb-init-complete
 
 fi
 # LCOV_EXCL_END
diff --git a/Dockerfile b/Dockerfile
@@ -7,22 +7,21 @@
 # support setting data directory as an environment variable) to support new
 # location and overriding default CMD to include our custom data directory.
 #
-FROM uselagoon/mariadb-10.11-drupal:24.8.0
+FROM uselagoon/mariadb-10.11-drupal:24.11.0
 
 # Set the data directory to a different location that a mounted volume.
-ENV MARIADB_DATA_DIR=/var/lib/db-data
+ENV MARIADB_DATA_DIR=/home/db-data
 
 # Add customised entrypoint script.
 COPY 9999-mariadb-init.bash /lagoon/entrypoints/
 
+# Create the custom data directory and set permissions.
 USER root
-
-RUN mkdir -p /var/lib/db-data \
-    && chown -R mysql /var/lib/db-data \
-    && chgrp -R mysql /var/lib/db-data \
-    && /bin/fix-permissions /var/lib/db-data
-
+RUN mkdir -p /home/db-data \
+    && chown -R mysql:mysql /home/db-data \
+    && /bin/fix-permissions /home/db-data \
+    && chmod -R 775 /home/db-data
 USER mysql
 
 # @todo Try removing the CMD override.
-CMD ["mysqld", "--datadir=/var/lib/db-data"]
+CMD ["mysqld", "--datadir=/home/db-data"]
diff --git a/Dockerfile.seed b/Dockerfile.seed
@@ -1,16 +1,12 @@
-ARG SEED_IMAGE=drevops/mariadb-drupal-data:latest
+ARG BASE_IMAGE=drevops/mariadb-drupal-data:latest
 
-FROM ${SEED_IMAGE}
-
-# Data directory to copy from.
-ARG SRC_DATADIR=.data
-
-COPY ${SRC_DATADIR} /var/lib/db-data/
+FROM ${BASE_IMAGE}
 
 USER root
 
-RUN chown -R mysql /var/lib/db-data \
-    && chgrp -R mysql /var/lib/db-data \
-    && /bin/fix-permissions /var/lib/db-data
+COPY .data /home/db-data/
 
+RUN chown -R mysql:mysql /home/db-data \
+    && /bin/fix-permissions /home/db-data \
+    && chmod -R 775 /home/db-data
 USER mysql
diff --git a/README.md b/README.md
@@ -33,7 +33,7 @@ captured as a Docker layer and stored as an image to docker registry.
 Image consumers download the image and start containers with instantaneously
 available data (no time-consuming database imports required).
 
-Technically, majority of the functionality is relying on upstream [`uselagoon/mariadb-drupal`](https://github.com/uselagoon/lagoon-images/blob/main/images/mariadb-drupal/10.6.Dockerfile) Docker image.
+Technically, majority of the functionality is relying on upstream [`uselagoon/mariadb-drupal`](https://github.com/uselagoon/lagoon-images/blob/main/images/mariadb-drupal/10.11.Dockerfile) Docker image.
 [Entrypoint script](entrypoint.bash) had to be copied from [upstream script](https://github.com/uselagoon/lagoon-images/blob/main/images/mariadb/entrypoints/9999-mariadb-init.bash) and adjusted to support custom data directory.
 
 ## Use case
@@ -80,4 +80,4 @@ This image is built and pushed automatically to DockerHub:
 Versions are following versions of the [upstream image](https://hub.docker.com/r/uselagoon/mariadb-drupal/tags) to ease maintenance.
 
 ---
-Repository created using https://getscaffold.dev/ project scaffold template
+_This repository was created using the [Scaffold](https://getscaffold.dev/) project template_
diff --git a/seed-db.sh b/seed-db.sh
@@ -2,8 +2,8 @@
 ##
 # Seed image with a database from file.
 #
-# The seeding process has 3-phases build:
-# 1. Create extracted DB files by starting a temporary container and importing database.
+# The seeding process has 3-phases:
+# 1. Create extracted DB files by starting a temporary container and importing the database.
 # 2. Build a new image from the base image and extracted DB files.
 # 3. Start a container from the new image and verify that the database was imported.
 #
@@ -15,7 +15,7 @@
 # shellcheck disable=SC2002,SC2015
 
 set -eu
-[ -n "${DREVOPS_DEBUG:-}" ] && set -x
+[ -n "${DEBUG:-}" ] && set -x
 
 # Database dump file as a first argument to the script.
 DB_FILE="${DB_FILE:-$1}"
@@ -42,11 +42,12 @@ LOG_DIR="${LOG_DIR:-.logs}"
 # Temporary data directory on host.
 TMP_DATA_DIR="${TMP_DATA_DIR:-.data}"
 
+# Show verbose output.
+LOG_IS_VERBOSE="${LOG_IS_VERBOSE:-}"
+
 # ------------------------------------------------------------------------------
 
 # @formatter:off
-#info() { printf "%s\n" "$1"; echo;}
-
 info() { [ -z "${TERM_NO_COLOR:-}" ] && tput colors >/dev/null 2>&1 && printf "\n[\033[36mINFO\033[0m] %s\n\n" "$1" || printf "\n[INFO] %s\n" "$1"; }
 task() { [ -z "${TERM_NO_COLOR:-}" ] && tput colors >/dev/null 2>&1 && printf "[\033[34mTASK\033[0m] %s\n" "$1" || printf "[TASK] %s\n" "$1"; }
 pass() { [ -z "${TERM_NO_COLOR:-}" ] && tput colors >/dev/null 2>&1 && printf "[ \033[32mOK\033[0m ] %s\n" "$1" || printf "[ OK ] %s\n" "$1"; }
@@ -60,20 +61,65 @@ note() { printf "       %s\n" "$1"; }
 [ "${BASE_IMAGE##*/}" = "$BASE_IMAGE" ] && fail "${BASE_IMAGE} should be in a format myorg/myimage." && exit 1
 [ "${DST_IMAGE##*/}" = "$DST_IMAGE" ] && fail "${DST_IMAGE} should be in a format myorg/myimage." && exit 1
 
+# Function to collect logs and display them on script exit.
+cleanup() {
+  if [ $? -ne 0 ]; then
+    fail "Collecting logs after failure."
+    if [ -d "${LOG_DIR}" ] && [ -z "${LOG_IS_VERBOSE}" ]; then
+      for log_file in "${LOG_DIR}"/*.log; do
+        echo
+        note "--- Displaying ${log_file} ---"
+        echo
+        cat "${log_file}"
+        echo
+      done
+    else
+      note "No logs available to display."
+    fi
+  fi
+}
+
+trap cleanup EXIT
+
 log_container() {
+  name="${1?Missing log name}"
+  prefix=${2:-}
+
   mkdir -p "${LOG_DIR}" >/dev/null
-  docker logs "${1}" >>"${LOG_DIR}/${2:-}${1}.log" 2>&1
+  log_file="${LOG_DIR}/${prefix}${name}.log"
+
+  if [ -n "${LOG_IS_VERBOSE}" ]; then
+    docker logs "${1}" | tee -a "${log_file}"
+  else
+    docker logs "${1}" &>>"${log_file}"
+  fi
 }
 
 wait_for_db_service() {
+  cid="${1}"
+
+  user=()
+  if [ -n "${2-}" ]; then
+    user=("--user=${2}")
+  fi
+
   echo -n "       Waiting for the service to become ready."
-  docker exec --user 1000 -i "${1}" sh -c "until nc -z localhost 3306; do sleep 1; echo -n .; done; echo"
+  if ! docker exec "${user[@]}" -i "${cid}" sh -c "until nc -z localhost 3306; do sleep 1; echo -n .; done; echo"; then
+    fail "MYSQL service did not start successfully."
+    log_container "${cid}"
+    return 1
+  fi
   log_container "${cid}"
   pass "MYSQL is running."
 }
 
 assert_db_system_tables_present() {
-  if docker exec --user 1000 "${1}" /usr/bin/mysql -e "show tables from information_schema;" | grep -q user_variables; then
+  user=()
+  if [ -n "${2-}" ]; then
+    user=("--user=${2}")
+  fi
+
+  if docker exec "${user[@]}" "${1}" /usr/bin/mysql -e "show tables from information_schema;" | grep -q user_variables; then
     pass "Database system tables present."
   else
     pass "Database system tables are not present in container ${1}"
@@ -82,7 +128,12 @@ assert_db_system_tables_present() {
 }
 
 assert_db_was_imported() {
-  if docker exec --user 1000 "${1}" /usr/bin/mysql -e "show tables;" | grep -q users; then
+  user=()
+  if [ -n "${2-}" ]; then
+    user=("--user=${2}")
+  fi
+
+  if docker exec "${user[@]}" "${1}" /usr/bin/mysql -e "show tables;" | grep -q users; then
     pass "Imported database exists."
   else
     fail "Imported database does not exist in container ${1}"
@@ -92,19 +143,27 @@ assert_db_was_imported() {
 
 start_container() {
   task "Start container from the image ${1}"
-  cid=$(docker run -d --rm "${1}" 2>"$LOG_DIR"/container-start.log)
+
+  user=()
+  if [ -n "${2-}" ]; then
+    user=("--user=${2}")
+  fi
+
+  cid=$(docker run "${user[@]}" -d "${1}" 2>"$LOG_DIR"/container-start.log)
   cat "${LOG_DIR}"/container-start.log >>"$LOG_DIR/${cid}.log" && rm "${LOG_DIR}"/container-start.log || true
+
+  wait_for_db_service "${cid}" "${2-}"
+  assert_db_system_tables_present "${cid}" "${2-}"
+
   pass "Started container ${cid}"
-  wait_for_db_service "${cid}"
-  assert_db_system_tables_present "${cid}"
 }
 
 get_started_container_id() {
   docker ps -q --filter ancestor="${1}" --filter status=running | head -n 1
 }
 
 stop_container() {
-  task "Stop and removing container ${1}"
+  task "Stop and remove container ${1}"
   # Log container output before stopping it into a separate log file for debugging.
   log_container "${1}" "stopped-"
   docker stop "${1}" >/dev/null
@@ -134,7 +193,7 @@ fi
 note "Destination image: ${DST_IMAGE}"
 note "Destination platform(s): ${DESTINATION_PLATFORMS}"
 
-info "Stage 1: Produce database structure files from dump"
+info "Stage 1: Produce database structure files from dump file"
 
 start_container "${BASE_IMAGE}"
 cid="$(get_started_container_id "${BASE_IMAGE}")"
@@ -143,29 +202,42 @@ task "Import database from the ${DB_FILE} file."
 cat "${DB_FILE}" | docker exec -i "${cid}" /usr/bin/mysql
 assert_db_was_imported "${cid}"
 
+#task "Release locks."
+#docker exec "${cid}" /usr/bin/mysql -e "FLUSH TABLES WITH READ LOCK;"
+#docker exec "${cid}" /usr/bin/mysql -e "UNLOCK TABLES;"
+#docker exec "${cid}" sh -c "mysql_upgrade --force"
+#docker exec "${cid}" /usr/bin/mysql -e "FLUSH TABLES WITH READ LOCK;"
+#assert_db_was_imported "${cid}"
+#pass "Released locks."
+
 task "Update permissions on the seeded database files."
-docker exec "${cid}" bash -c "chown -R mysql /var/lib/db-data && /bin/fix-permissions /var/lib/db-data" || true
+docker exec "${cid}" bash -c "chown -R mysql /home/db-data && /bin/fix-permissions /home/db-data" || true
 pass "Updated permissions on the seeded database files."
 
 task "Copy expanded database files to host"
 mkdir -p "${TMP_DATA_DIR}"
-docker cp "${cid}":/var/lib/db-data/. "${TMP_DATA_DIR}/"
+docker cp "${cid}":/home/db-data/. "${TMP_DATA_DIR}/" >/dev/null
 [ ! -d "${TMP_DATA_DIR}/mysql" ] && fail "Unable to copy expanded database files to host " && ls -al "${TMP_DATA_DIR}" && exit 1
 pass "Copied expanded database files to host"
 
+task "Remove logs and temporary database files"
+#rm -f "${TMP_DATA_DIR}"/ib_logfile* || true
+rm -f "${TMP_DATA_DIR}"/aria_log* || true
+pass "Removed logs and temporary database files"
+
 stop_container "${cid}"
 
 info "Stage 2: Build image"
 
-task "Build image ${DST_IMAGE} for ${DESTINATION_PLATFORMS} platform(s)."
-docker buildx build --no-cache --platform "${DESTINATION_PLATFORMS}" --tag "${DST_IMAGE}" --push -f Dockerfile.seed .
-pass "Built image ${DST_IMAGE} for ${DESTINATION_PLATFORMS} platform(s)."
+task "Build image ${DST_IMAGE} for ${DESTINATION_PLATFORMS} platform(s) from ${BASE_IMAGE}."
+docker buildx build -D --no-cache --build-arg="BASE_IMAGE=${BASE_IMAGE}" --platform "${DESTINATION_PLATFORMS}" --tag "${DST_IMAGE}" --push -f Dockerfile.seed .
+pass "Built image ${DST_IMAGE} for ${DESTINATION_PLATFORMS} platform(s) from ${BASE_IMAGE}."
 
 info "Stage 3: Test image"
 
-start_container "${DST_IMAGE}"
+start_container "${DST_IMAGE}" 1000
 cid="$(get_started_container_id "${DST_IMAGE}")"
-assert_db_was_imported "${cid}"
+assert_db_was_imported "${cid}" 1000
 stop_container "${cid}"
 
 info "Finished database seeding."