extra log message while extracting

Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.15.0
+FROM alpine:3.19.1
 LABEL maintainer "Arulraj V <[email protected]>"
 
 RUN apk add --no-cache --update \
@@ -10,13 +10,13 @@ RUN apk add --no-cache --update \
       curl \
       python3 \
       py3-pip \
-      gettext && \
-      pip3 install --no-cache-dir \
-      awscli && \
-      # mariadb-connector-c && \
+      gettext \
+      aws-cli \
+      pv \
+      mariadb-connector-c-dev && \
       rm -rf /var/cache/apk/*
 
-RUN curl -L --insecure https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-alpine-linux-amd64-v0.6.1.tar.gz | tar -xz -C /usr/local/bin/
+RUN curl -L --insecure https://github.com/jwilder/dockerize/releases/download/v0.7.0/dockerize-alpine-linux-amd64-v0.7.0.tar.gz | tar -xz -C /usr/local/bin/
 RUN chmod +x /usr/local/bin/dockerize
 
 ARG GIT_COMMIT_ID=unspecified

README.md

@@ -14,7 +14,8 @@ This docker image will backup the mediawiki database (MySQL based) and config fi
 To start the backup container
 
 ```bash
-docker-compose up -d wiki-backup
+docker-compose build backup
+docker-compose up -d backup
 ```
 
 The **required environmental** variables are
@@ -65,15 +66,15 @@ This will backup MySQL database and everything in `/mediawiki` mounted folder.
 
 To restore the from s3 backup
 
-```
-docker-compose run wiki-backup restore
+```bash
+docker-compose run backup restore
 ```
 
 Then you will entered into a shell. By default it will display the latest 10 backup files like below
 
 ```bash
-docker-compose run --rm wiki-backup restore
-Creating mediawiki-backup-restore_wiki-backup_run ... done
+docker-compose run --rm backup restore
+Creating mediawiki-backup-restore_backup_run ... done
 2022/03/06 15:39:13 Waiting for: tcp://db:3306
 2022/03/06 15:39:13 Connected to tcp://db:3306
 2022-03-06 07:50:04  440 Bytes 2022-03-06T075000Z.dump.sql.gz
@@ -91,7 +92,7 @@ RESTORE_DATABASE
 
 The available commands are
 
-```
+```bash
 list_s3_top_ten
 list_s3
 restore <fileName>
@@ -121,17 +122,17 @@ restore daily/2022-09-27T154250Z.daily
 
 Set your S3 credentials in .env file. Then
 
-```
+```bash
 docker-compose up -d db
-docker-compose run --rm -e "RESTORE_DATABASE=my_wiki" wiki-backup restore
+docker-compose run --rm -e "RESTORE_DATABASE=my_wiki" backup restore
 ```
 
 Then exec into the restore container
 
 ### To override anything on restore
 
 ```bash
-docker-compose run --rm -e "RESTORE_DATABASE=new_my_wiki" -v "/var/www/html:/mediawiki" wiki-backup restore
+docker-compose run --rm -e "RESTORE_DATABASE=new_my_wiki" -v "/var/www/html:/mediawiki" backup restore
 ```
 
 Refer
@@ -144,18 +145,17 @@ While retention policy on s3 is supposed to keep the folders tidy, these command
 
 `$AWS_ARGS` is loaded within the backup container. Run the following manually to populate credentials.
 
-```
+```bash
 export AWS_ACCESS_KEY_ID=$S3_ACCESS_KEY_ID
 export AWS_SECRET_ACCESS_KEY=$S3_SECRET_ACCESS_KEY
 ```
 
 You can run the following to list and delete files.
-```
+
+```bash
 source restore.sh
 list_s3 hourly
 aws $AWS_ARGS s3 rm s3://$S3_BUCKET/wiki/testing/hourly/ --dryrun --recursive --exclude "*" --include "*.gz"
 ```
 
-The `--dryrun` flag does not delete files, instead shows what would be deleted. When you are confident about deleting the files listed, you can run the command without the `--dryrun` flag. 
-
-
+The `--dryrun` flag does not delete files, instead shows what would be deleted. When you are confident about deleting the files listed, you can run the command without the `--dryrun` flag.

backup.sh

@@ -1,6 +1,10 @@
 #! /usr/bin/env bash
 # set -e
 
+logger() {
+  echo $(date +"%Y-%m-%dT%H%M%SZ") "$@"
+}
+
 copy_s3 () {
   SRC_FILE=$1
   DEST_FILE=$2
@@ -13,7 +17,7 @@ copy_s3 () {
     AWS_ARGS="--endpoint-url $S3_ENDPOINT"
   fi
 
-  echo "Uploading ${DEST_FILE} on S3..."
+  logger "Uploading ${DEST_FILE} on S3..."
   if [[ -z "$S3_PREFIX" ]]; then
     # backup without prefix
     aws $AWS_ARGS s3 cp $SRC_FILE s3://$S3_BUCKET/$FREQUENCY/$DEST_FILE && \
@@ -26,47 +30,47 @@ copy_s3 () {
     fi
   fi
   if [ "$?" == "0" ]; then
-    echo "Successfully uploading ${FREQUENCY} backup file ${DEST_FILE} on S3"
+    logger "Successfully uploading ${FREQUENCY} backup file ${DEST_FILE} on S3"
   else
-    echo "Error uploading ${FREQUENCY} backup file ${DEST_FILE} on S3"
+    logger "Error uploading ${FREQUENCY} backup file ${DEST_FILE} on S3"
   fi
   rm -f $SRC_FILE
 }
-FREQUENCY=$1
+FREQUENCY=${1:-hourly}
 MYSQL_HOST_OPTS="-h $MYSQL_HOST -P $MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD"
 DUMP_START_TIME=$(date +"%Y-%m-%dT%H%M%SZ")
 BACKUP_DIR="/backup"
 
-echo "Backup frequency ${FREQUENCY}"
-echo "Backup is started at ${DUMP_START_TIME}"
-echo "Creating dump for ${MYSQLDUMP_DATABASE} from ${MYSQL_HOST}..."
+logger "Backup frequency ${FREQUENCY}"
+logger "Backup is started at ${DUMP_START_TIME}"
+logger "Creating dump for ${MYSQLDUMP_DATABASE} from ${MYSQL_HOST}..."
 DUMP_SQL_FILE="$BACKUP_DIR/$DUMP_START_TIME.dump.sql"
 DUMP_FILE="$DUMP_SQL_FILE.gz"
 mysqldump --single-transaction $MYSQLDUMP_OPTIONS $MYSQL_HOST_OPTS $MYSQL_SSL_OPTS $MYSQLDUMP_DATABASE > $DUMP_SQL_FILE
-echo "Compressing $DUMP_SQL_FILE"
+logger "Compressing $DUMP_SQL_FILE"
 gzip -f "$DUMP_SQL_FILE"
 
 if [ "$?" == "0" ]; then
   S3_FILE="$DUMP_START_TIME.$FREQUENCY.dump.sql.gz"
   copy_s3 $DUMP_FILE $S3_FILE $FREQUENCY
 else
-  echo "Error creating mysqldump"
+  logger "Error creating mysqldump"
 fi
 
 MEDIAWIKI_DIR="/mediawiki"
 DUMP_FILE="$BACKUP_DIR/$DUMP_START_TIME.mediawiki.tar.gz"
 # backup mediawiki folder
 if [ -d $MEDIAWIKI_DIR ]; then
-  echo "Creating $DUMP_FILE from $MEDIAWIKI_DIR"
+  logger "Creating $DUMP_FILE from $MEDIAWIKI_DIR"
   # Gzip mediawiki folder
   tar -czf $DUMP_FILE -C $(dirname $MEDIAWIKI_DIR) $(basename $MEDIAWIKI_DIR)
   if [ "$?" == "0" ]; then
     S3_FILE="$DUMP_START_TIME.$FREQUENCY.mediawiki.tar.gz"
     copy_s3 $DUMP_FILE $S3_FILE $FREQUENCY
   else
-    echo "Error creating mediawiki"
+    logger "Error creating mediawiki"
   fi
 fi
 
 DUMP_END_TIME=$(date +"%Y-%m-%dT%H%M%SZ")
-echo "Backup is ends at ${DUMP_END_TIME}"
+logger "Backup is ends at ${DUMP_END_TIME}"

docker-compose.yml

@@ -34,15 +34,15 @@ services:
       - mysqldb-data:/var/lib/mysql
     restart: on-failure
 
-  wiki-backup:
+  backup:
     <<: *default
     build:
       context: .
       args:
         # GIT_COMMIT load from .envrc via direnv
         GIT_COMMIT_ID: ${GIT_COMMIT:-unspecified}
     image: tamilwiki/mediawiki_backup_restore:dev
-    environment:            
+    environment:
       INIT_BACKUP: 1
       MYSQL_HOST: ${MYSQL_HOST}
       MYSQL_PASSWORD: ${MYSQL_PASSWORD}

restore.sh

@@ -1,16 +1,22 @@
 #! /usr/bin/env bash
 # set -x
 # set -e
+set -o pipefail
 
 RESTORE_DIR="/restore"
 MEDIAWIKI_DIR="/mediawiki"
+DONT_CHANGE_DUMP_FILE=${DONT_CHANGE_DUMP_FILE:-true}
 
 if [[ -z "$S3_ENDPOINT" ]]; then
   AWS_ARGS=""
 else
   AWS_ARGS="--endpoint-url $S3_ENDPOINT"
 fi
 
+logger() {
+  echo $(date +"%Y-%m-%dT%H%M%SZ") "$@"
+}
+
 # Validate the given file exists in S3
 _s3_key_exists() {
   if [[ -z "$S3_PREFIX" ]]; then
@@ -36,7 +42,7 @@ list_s3_top_ten() {
     S3_PATH="$S3_PATH/$1"
   fi
 
-  echo "Listing top 10 files from $S3_PATH"
+  logger "Listing top 10 files from $S3_PATH"
   aws $AWS_ARGS s3 ls $S3_PATH/$(date +"%Y-%m-%d") --human-readable | sort -r | head -n 10
 }
 
@@ -51,13 +57,13 @@ list_s3() {
     S3_PATH="$S3_PATH/$1"
   fi
 
-  echo "Listing all files from $S3_PATH"
+  logger "Listing all files from $S3_PATH"
   aws $AWS_ARGS s3 ls $S3_PATH/ --human-readable
 }
 
 restore_db() {
   mkdir -p $RESTORE_DIR
-  echo "Restoring DB $RESTORE_DATABASE ..."
+  logger "Restoring DB $RESTORE_DATABASE ..."
   success="1"
   MYSQL_HOST_OPTS="-h $MYSQL_HOST -P $MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD"
   # Drop all tables if exists
@@ -75,58 +81,69 @@ restore_db() {
   fi
   RESTORE_FILE=$(basename $1)
   if [[ -f $RESTORE_DIR/$RESTORE_FILE ]]; then
-    echo "${MYSQL_RESTORE_OPTIONS}" > $RESTORE_DIR/options.sql
-    gzip -dkc $RESTORE_DIR/$RESTORE_FILE > $RESTORE_DIR/content.sql
-    cat $RESTORE_DIR/options.sql $RESTORE_DIR/content.sql > $RESTORE_DIR/dump.sql
-
-    defaultCollationName=$(mysql -s -N $MYSQL_HOST_OPTS $RESTORE_DATABASE -e "SELECT @@collation_database;")
-    defaultCharset=$(mysql -s -N $MYSQL_HOST_OPTS $RESTORE_DATABASE -e "SELECT @@character_set_database;")
-
-    # Replace default collation if utf8mb4_0900_ai_ci is not supported.
-    collation0900aiciName=$(mysql -s -N $MYSQL_HOST_OPTS $RESTORE_DATABASE -e "SELECT collation_name FROM information_schema.COLLATIONS WHERE collation_name='utf8mb4_0900_ai_ci';")
-    if [[ -z $collation0900aiciName ]]; then
-      sed -i "s/utf8mb4_0900_ai_ci/$defaultCollationName/g" $RESTORE_DIR/dump.sql
-    fi
-
-    # Replace default charset if utf8mb4 is not supported.
-    charsetutf8mb4Name=$(mysql -s -N $MYSQL_HOST_OPTS $RESTORE_DATABASE -e "SELECT character_set_name FROM information_schema.CHARACTER_SETS WHERE character_set_name='utf8mb4';")
-    if [[ -z $charsetutf8mb4Name ]]; then
-      sed -i "s/CHARSET=utf8mb4/CHARSET=$defaultCharset/g" $RESTORE_DIR/dump.sql
+    logger "Extracting..."
+    gzip -dvkc $RESTORE_DIR/$RESTORE_FILE > $RESTORE_DIR/dump.sql
+    logger "The size of the restore file is $(du -hs $RESTORE_DIR/dump.sql | awk '{print $1}')."
+
+    if [[ "$DONT_CHANGE_DUMP_FILE" == "false" ]]; then
+      if [[ ! -z "$MYSQL_RESTORE_OPTIONS" ]]; then
+        logger "Adding restore options..."
+        sed -i "1i${MYSQL_RESTORE_OPTIONS}" $RESTORE_DIR/dump.sql
+      fi
+
+      defaultCollationName=$(mysql -s -N $MYSQL_HOST_OPTS $RESTORE_DATABASE -e "SELECT @@collation_database;")
+      defaultCharset=$(mysql -s -N $MYSQL_HOST_OPTS $RESTORE_DATABASE -e "SELECT @@character_set_database;")
+
+      # Replace default collation if utf8mb4_0900_ai_ci is not supported.
+      collation0900aiciName=$(mysql -s -N $MYSQL_HOST_OPTS $RESTORE_DATABASE -e "SELECT collation_name FROM information_schema.COLLATIONS WHERE collation_name='utf8mb4_0900_ai_ci';")
+      if [[ -z $collation0900aiciName ]]; then
+        logger "Replacing default collation..."
+        sed -i "s/utf8mb4_0900_ai_ci/$defaultCollationName/g" $RESTORE_DIR/dump.sql
+      fi
+
+      # Replace default charset if utf8mb4 is not supported.
+      charsetutf8mb4Name=$(mysql -s -N $MYSQL_HOST_OPTS $RESTORE_DATABASE -e "SELECT character_set_name FROM information_schema.CHARACTER_SETS WHERE character_set_name='utf8mb4';")
+      if [[ -z $charsetutf8mb4Name ]]; then
+        logger "Replacing default charset..."
+        sed -i "s/CHARSET=utf8mb4/CHARSET=$defaultCharset/g" $RESTORE_DIR/dump.sql
+      fi
     fi
 
+    logger "Restoring..."
     mysql $MYSQL_HOST_OPTS $RESTORE_DATABASE < $RESTORE_DIR/dump.sql
     if [ "$?" == "0" ]; then
       success="0"
     fi
-    rm -rf $RESTORE_DIR/$RESTORE_FILE $RESTORE_DIR/dump.sql $RESTORE_DIR/content.sql $RESTORE_DIR/options.sql
+    rm -rf $RESTORE_DIR/$RESTORE_FILE $RESTORE_DIR/dump.sql
   else
-    echo "File $1 not exits."
+    logger "File $1 not exits."
   fi
 
   if [ "$success" == "0" ]; then
-    echo "Restoring DB $RESTORE_DATABASE success!"
+    logger "Restoring DB $RESTORE_DATABASE success!"
   else
-    echo "Restoring DB $RESTORE_DATABASE failed"
+    logger "Restoring DB $RESTORE_DATABASE failed"
   fi
 
 }
 
 restore_mediawiki() {
   mkdir -p $RESTORE_DIR
-  echo "Restoring Mediawiki ..."
+  logger "Restoring Mediawiki ..."
   RESTORE_FILE=$RESTORE_DIR/$1
   if [[ -z "$S3_PREFIX" ]]; then
     aws $AWS_ARGS s3 cp s3://$S3_BUCKET/$1 $RESTORE_FILE
   else
     aws $AWS_ARGS s3 cp s3://$S3_BUCKET/$S3_PREFIX/$1 $RESTORE_FILE
   fi
 
-  tar -xzvf $RESTORE_FILE -C $(dirname $MEDIAWIKI_DIR)
+  logger "Extracting..."
+  pv $RESTORE_FILE | tar -xzf - -C $(dirname $MEDIAWIKI_DIR)
 
   if [ "$?" == "0" ]; then
-    echo "Restoring Mediawiki $1 success!"
+    logger "Restoring Mediawiki $1 success!"
   else
-    echo "Restoring Mediawiki $1 failed"
+    logger "Restoring Mediawiki $1 failed"
   fi
   rm -rf $RESTORE_FILE
 }
@@ -139,12 +156,12 @@ restore() {
   # The fileName will be without extensions like 2022-03-06T075000Z or latest
   fileName=$1
   RESTORE_START_TIME=$(date +"%Y-%m-%dT%H%M%SZ")
-  echo "Restoring Started at $RESTORE_START_TIME"
+  logger "Restoring Started at $RESTORE_START_TIME"
 
   # Restoring DB
   SQL_DUMP_FILE="$fileName.dump.sql.gz"
   if [[ "$(_s3_key_exists $SQL_DUMP_FILE)" != "0" ]]; then
-    echo "The given dump ${SQL_DUMP_FILE} file is does not exists."
+    logger "The given dump ${SQL_DUMP_FILE} file is does not exists."
     return 1
   fi
   restore_db $SQL_DUMP_FILE
@@ -153,12 +170,12 @@ restore() {
   if [ -d $MEDIAWIKI_DIR ]; then
     WIKI_DUMP_FILE="$fileName.mediawiki.tar.gz"
     if [[ "$(_s3_key_exists $WIKI_DUMP_FILE)" != "0" ]]; then
-      echo "The given mediawiki ${WIKI_DUMP_FILE} file is does not exists."
+      logger "The given mediawiki ${WIKI_DUMP_FILE} file is does not exists."
       return 1
     fi
     restore_mediawiki $WIKI_DUMP_FILE
   fi
 
   RESTORE_END_TIME=$(date +"%Y-%m-%dT%H%M%SZ")
-  echo "Restoring Ends at $RESTORE_END_TIME"
+  logger "Restoring Ends at $RESTORE_END_TIME"
 }