diff --git a/Dockerfile b/Dockerfile index 5283984..ae72cac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.15.0 +FROM alpine:3.19.1 LABEL maintainer "Arulraj V " RUN apk add --no-cache --update \ @@ -10,13 +10,13 @@ RUN apk add --no-cache --update \ curl \ python3 \ py3-pip \ - gettext && \ - pip3 install --no-cache-dir \ - awscli && \ - # mariadb-connector-c && \ + gettext \ + aws-cli \ + pv \ + mariadb-connector-c-dev && \ rm -rf /var/cache/apk/* -RUN curl -L --insecure https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-alpine-linux-amd64-v0.6.1.tar.gz | tar -xz -C /usr/local/bin/ +RUN curl -L --insecure https://github.com/jwilder/dockerize/releases/download/v0.7.0/dockerize-alpine-linux-amd64-v0.7.0.tar.gz | tar -xz -C /usr/local/bin/ RUN chmod +x /usr/local/bin/dockerize ARG GIT_COMMIT_ID=unspecified diff --git a/README.md b/README.md index 14c34c7..e58cf5e 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,8 @@ This docker image will backup the mediawiki database (MySQL based) and config fi To start the backup container ```bash -docker-compose up -d wiki-backup +docker-compose build backup +docker-compose up -d backup ``` The **required environmental** variables are @@ -65,15 +66,15 @@ This will backup MySQL database and everything in `/mediawiki` mounted folder. To restore the from s3 backup -``` -docker-compose run wiki-backup restore +```bash +docker-compose run backup restore ``` Then you will entered into a shell. By default it will display the latest 10 backup files like below ```bash -docker-compose run --rm wiki-backup restore -Creating mediawiki-backup-restore_wiki-backup_run ... done +docker-compose run --rm backup restore +Creating mediawiki-backup-restore_backup_run ... done 2022/03/06 15:39:13 Waiting for: tcp://db:3306 2022/03/06 15:39:13 Connected to tcp://db:3306 2022-03-06 07:50:04 440 Bytes 2022-03-06T075000Z.dump.sql.gz @@ -91,7 +92,7 @@ RESTORE_DATABASE The available commands are -``` +```bash list_s3_top_ten list_s3 restore @@ -121,9 +122,9 @@ restore daily/2022-09-27T154250Z.daily Set your S3 credentials in .env file. Then -``` +```bash docker-compose up -d db -docker-compose run --rm -e "RESTORE_DATABASE=my_wiki" wiki-backup restore +docker-compose run --rm -e "RESTORE_DATABASE=my_wiki" backup restore ``` Then exec into the restore container @@ -131,7 +132,7 @@ Then exec into the restore container ### To override anything on restore ```bash -docker-compose run --rm -e "RESTORE_DATABASE=new_my_wiki" -v "/var/www/html:/mediawiki" wiki-backup restore +docker-compose run --rm -e "RESTORE_DATABASE=new_my_wiki" -v "/var/www/html:/mediawiki" backup restore ``` Refer @@ -144,18 +145,17 @@ While retention policy on s3 is supposed to keep the folders tidy, these command `$AWS_ARGS` is loaded within the backup container. Run the following manually to populate credentials. -``` +```bash export AWS_ACCESS_KEY_ID=$S3_ACCESS_KEY_ID export AWS_SECRET_ACCESS_KEY=$S3_SECRET_ACCESS_KEY ``` You can run the following to list and delete files. -``` + +```bash source restore.sh list_s3 hourly aws $AWS_ARGS s3 rm s3://$S3_BUCKET/wiki/testing/hourly/ --dryrun --recursive --exclude "*" --include "*.gz" ``` -The `--dryrun` flag does not delete files, instead shows what would be deleted. When you are confident about deleting the files listed, you can run the command without the `--dryrun` flag. - - +The `--dryrun` flag does not delete files, instead shows what would be deleted. When you are confident about deleting the files listed, you can run the command without the `--dryrun` flag. diff --git a/backup.sh b/backup.sh index b2facb8..8c613b8 100755 --- a/backup.sh +++ b/backup.sh @@ -1,6 +1,10 @@ #! /usr/bin/env bash # set -e +logger() { + echo $(date +"%Y-%m-%dT%H%M%SZ") "$@" +} + copy_s3 () { SRC_FILE=$1 DEST_FILE=$2 @@ -13,7 +17,7 @@ copy_s3 () { AWS_ARGS="--endpoint-url $S3_ENDPOINT" fi - echo "Uploading ${DEST_FILE} on S3..." + logger "Uploading ${DEST_FILE} on S3..." if [[ -z "$S3_PREFIX" ]]; then # backup without prefix aws $AWS_ARGS s3 cp $SRC_FILE s3://$S3_BUCKET/$FREQUENCY/$DEST_FILE && \ @@ -26,47 +30,47 @@ copy_s3 () { fi fi if [ "$?" == "0" ]; then - echo "Successfully uploading ${FREQUENCY} backup file ${DEST_FILE} on S3" + logger "Successfully uploading ${FREQUENCY} backup file ${DEST_FILE} on S3" else - echo "Error uploading ${FREQUENCY} backup file ${DEST_FILE} on S3" + logger "Error uploading ${FREQUENCY} backup file ${DEST_FILE} on S3" fi rm -f $SRC_FILE } -FREQUENCY=$1 +FREQUENCY=${1:-hourly} MYSQL_HOST_OPTS="-h $MYSQL_HOST -P $MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD" DUMP_START_TIME=$(date +"%Y-%m-%dT%H%M%SZ") BACKUP_DIR="/backup" -echo "Backup frequency ${FREQUENCY}" -echo "Backup is started at ${DUMP_START_TIME}" -echo "Creating dump for ${MYSQLDUMP_DATABASE} from ${MYSQL_HOST}..." +logger "Backup frequency ${FREQUENCY}" +logger "Backup is started at ${DUMP_START_TIME}" +logger "Creating dump for ${MYSQLDUMP_DATABASE} from ${MYSQL_HOST}..." DUMP_SQL_FILE="$BACKUP_DIR/$DUMP_START_TIME.dump.sql" DUMP_FILE="$DUMP_SQL_FILE.gz" mysqldump --single-transaction $MYSQLDUMP_OPTIONS $MYSQL_HOST_OPTS $MYSQL_SSL_OPTS $MYSQLDUMP_DATABASE > $DUMP_SQL_FILE -echo "Compressing $DUMP_SQL_FILE" +logger "Compressing $DUMP_SQL_FILE" gzip -f "$DUMP_SQL_FILE" if [ "$?" == "0" ]; then S3_FILE="$DUMP_START_TIME.$FREQUENCY.dump.sql.gz" copy_s3 $DUMP_FILE $S3_FILE $FREQUENCY else - echo "Error creating mysqldump" + logger "Error creating mysqldump" fi MEDIAWIKI_DIR="/mediawiki" DUMP_FILE="$BACKUP_DIR/$DUMP_START_TIME.mediawiki.tar.gz" # backup mediawiki folder if [ -d $MEDIAWIKI_DIR ]; then - echo "Creating $DUMP_FILE from $MEDIAWIKI_DIR" + logger "Creating $DUMP_FILE from $MEDIAWIKI_DIR" # Gzip mediawiki folder tar -czf $DUMP_FILE -C $(dirname $MEDIAWIKI_DIR) $(basename $MEDIAWIKI_DIR) if [ "$?" == "0" ]; then S3_FILE="$DUMP_START_TIME.$FREQUENCY.mediawiki.tar.gz" copy_s3 $DUMP_FILE $S3_FILE $FREQUENCY else - echo "Error creating mediawiki" + logger "Error creating mediawiki" fi fi DUMP_END_TIME=$(date +"%Y-%m-%dT%H%M%SZ") -echo "Backup is ends at ${DUMP_END_TIME}" +logger "Backup is ends at ${DUMP_END_TIME}" diff --git a/docker-compose.yml b/docker-compose.yml index 482d3d9..7058139 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,7 +34,7 @@ services: - mysqldb-data:/var/lib/mysql restart: on-failure - wiki-backup: + backup: <<: *default build: context: . @@ -42,7 +42,7 @@ services: # GIT_COMMIT load from .envrc via direnv GIT_COMMIT_ID: ${GIT_COMMIT:-unspecified} image: tamilwiki/mediawiki_backup_restore:dev - environment: + environment: INIT_BACKUP: 1 MYSQL_HOST: ${MYSQL_HOST} MYSQL_PASSWORD: ${MYSQL_PASSWORD} diff --git a/restore.sh b/restore.sh index 637610c..31da2da 100755 --- a/restore.sh +++ b/restore.sh @@ -1,9 +1,11 @@ #! /usr/bin/env bash # set -x # set -e +set -o pipefail RESTORE_DIR="/restore" MEDIAWIKI_DIR="/mediawiki" +DONT_CHANGE_DUMP_FILE=${DONT_CHANGE_DUMP_FILE:-true} if [[ -z "$S3_ENDPOINT" ]]; then AWS_ARGS="" @@ -11,6 +13,10 @@ else AWS_ARGS="--endpoint-url $S3_ENDPOINT" fi +logger() { + echo $(date +"%Y-%m-%dT%H%M%SZ") "$@" +} + # Validate the given file exists in S3 _s3_key_exists() { if [[ -z "$S3_PREFIX" ]]; then @@ -36,7 +42,7 @@ list_s3_top_ten() { S3_PATH="$S3_PATH/$1" fi - echo "Listing top 10 files from $S3_PATH" + logger "Listing top 10 files from $S3_PATH" aws $AWS_ARGS s3 ls $S3_PATH/$(date +"%Y-%m-%d") --human-readable | sort -r | head -n 10 } @@ -51,13 +57,13 @@ list_s3() { S3_PATH="$S3_PATH/$1" fi - echo "Listing all files from $S3_PATH" + logger "Listing all files from $S3_PATH" aws $AWS_ARGS s3 ls $S3_PATH/ --human-readable } restore_db() { mkdir -p $RESTORE_DIR - echo "Restoring DB $RESTORE_DATABASE ..." + logger "Restoring DB $RESTORE_DATABASE ..." success="1" MYSQL_HOST_OPTS="-h $MYSQL_HOST -P $MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD" # Drop all tables if exists @@ -75,45 +81,55 @@ restore_db() { fi RESTORE_FILE=$(basename $1) if [[ -f $RESTORE_DIR/$RESTORE_FILE ]]; then - echo "${MYSQL_RESTORE_OPTIONS}" > $RESTORE_DIR/options.sql - gzip -dkc $RESTORE_DIR/$RESTORE_FILE > $RESTORE_DIR/content.sql - cat $RESTORE_DIR/options.sql $RESTORE_DIR/content.sql > $RESTORE_DIR/dump.sql - - defaultCollationName=$(mysql -s -N $MYSQL_HOST_OPTS $RESTORE_DATABASE -e "SELECT @@collation_database;") - defaultCharset=$(mysql -s -N $MYSQL_HOST_OPTS $RESTORE_DATABASE -e "SELECT @@character_set_database;") - - # Replace default collation if utf8mb4_0900_ai_ci is not supported. - collation0900aiciName=$(mysql -s -N $MYSQL_HOST_OPTS $RESTORE_DATABASE -e "SELECT collation_name FROM information_schema.COLLATIONS WHERE collation_name='utf8mb4_0900_ai_ci';") - if [[ -z $collation0900aiciName ]]; then - sed -i "s/utf8mb4_0900_ai_ci/$defaultCollationName/g" $RESTORE_DIR/dump.sql - fi - - # Replace default charset if utf8mb4 is not supported. - charsetutf8mb4Name=$(mysql -s -N $MYSQL_HOST_OPTS $RESTORE_DATABASE -e "SELECT character_set_name FROM information_schema.CHARACTER_SETS WHERE character_set_name='utf8mb4';") - if [[ -z $charsetutf8mb4Name ]]; then - sed -i "s/CHARSET=utf8mb4/CHARSET=$defaultCharset/g" $RESTORE_DIR/dump.sql + logger "Extracting..." + gzip -dvkc $RESTORE_DIR/$RESTORE_FILE > $RESTORE_DIR/dump.sql + logger "The size of the restore file is $(du -hs $RESTORE_DIR/dump.sql | awk '{print $1}')." + + if [[ "$DONT_CHANGE_DUMP_FILE" == "false" ]]; then + if [[ ! -z "$MYSQL_RESTORE_OPTIONS" ]]; then + logger "Adding restore options..." + sed -i "1i${MYSQL_RESTORE_OPTIONS}" $RESTORE_DIR/dump.sql + fi + + defaultCollationName=$(mysql -s -N $MYSQL_HOST_OPTS $RESTORE_DATABASE -e "SELECT @@collation_database;") + defaultCharset=$(mysql -s -N $MYSQL_HOST_OPTS $RESTORE_DATABASE -e "SELECT @@character_set_database;") + + # Replace default collation if utf8mb4_0900_ai_ci is not supported. + collation0900aiciName=$(mysql -s -N $MYSQL_HOST_OPTS $RESTORE_DATABASE -e "SELECT collation_name FROM information_schema.COLLATIONS WHERE collation_name='utf8mb4_0900_ai_ci';") + if [[ -z $collation0900aiciName ]]; then + logger "Replacing default collation..." + sed -i "s/utf8mb4_0900_ai_ci/$defaultCollationName/g" $RESTORE_DIR/dump.sql + fi + + # Replace default charset if utf8mb4 is not supported. + charsetutf8mb4Name=$(mysql -s -N $MYSQL_HOST_OPTS $RESTORE_DATABASE -e "SELECT character_set_name FROM information_schema.CHARACTER_SETS WHERE character_set_name='utf8mb4';") + if [[ -z $charsetutf8mb4Name ]]; then + logger "Replacing default charset..." + sed -i "s/CHARSET=utf8mb4/CHARSET=$defaultCharset/g" $RESTORE_DIR/dump.sql + fi fi + logger "Restoring..." mysql $MYSQL_HOST_OPTS $RESTORE_DATABASE < $RESTORE_DIR/dump.sql if [ "$?" == "0" ]; then success="0" fi - rm -rf $RESTORE_DIR/$RESTORE_FILE $RESTORE_DIR/dump.sql $RESTORE_DIR/content.sql $RESTORE_DIR/options.sql + rm -rf $RESTORE_DIR/$RESTORE_FILE $RESTORE_DIR/dump.sql else - echo "File $1 not exits." + logger "File $1 not exits." fi if [ "$success" == "0" ]; then - echo "Restoring DB $RESTORE_DATABASE success!" + logger "Restoring DB $RESTORE_DATABASE success!" else - echo "Restoring DB $RESTORE_DATABASE failed" + logger "Restoring DB $RESTORE_DATABASE failed" fi } restore_mediawiki() { mkdir -p $RESTORE_DIR - echo "Restoring Mediawiki ..." + logger "Restoring Mediawiki ..." RESTORE_FILE=$RESTORE_DIR/$1 if [[ -z "$S3_PREFIX" ]]; then aws $AWS_ARGS s3 cp s3://$S3_BUCKET/$1 $RESTORE_FILE @@ -121,12 +137,13 @@ restore_mediawiki() { aws $AWS_ARGS s3 cp s3://$S3_BUCKET/$S3_PREFIX/$1 $RESTORE_FILE fi - tar -xzvf $RESTORE_FILE -C $(dirname $MEDIAWIKI_DIR) + logger "Extracting..." + pv $RESTORE_FILE | tar -xzf - -C $(dirname $MEDIAWIKI_DIR) if [ "$?" == "0" ]; then - echo "Restoring Mediawiki $1 success!" + logger "Restoring Mediawiki $1 success!" else - echo "Restoring Mediawiki $1 failed" + logger "Restoring Mediawiki $1 failed" fi rm -rf $RESTORE_FILE } @@ -139,12 +156,12 @@ restore() { # The fileName will be without extensions like 2022-03-06T075000Z or latest fileName=$1 RESTORE_START_TIME=$(date +"%Y-%m-%dT%H%M%SZ") - echo "Restoring Started at $RESTORE_START_TIME" + logger "Restoring Started at $RESTORE_START_TIME" # Restoring DB SQL_DUMP_FILE="$fileName.dump.sql.gz" if [[ "$(_s3_key_exists $SQL_DUMP_FILE)" != "0" ]]; then - echo "The given dump ${SQL_DUMP_FILE} file is does not exists." + logger "The given dump ${SQL_DUMP_FILE} file is does not exists." return 1 fi restore_db $SQL_DUMP_FILE @@ -153,12 +170,12 @@ restore() { if [ -d $MEDIAWIKI_DIR ]; then WIKI_DUMP_FILE="$fileName.mediawiki.tar.gz" if [[ "$(_s3_key_exists $WIKI_DUMP_FILE)" != "0" ]]; then - echo "The given mediawiki ${WIKI_DUMP_FILE} file is does not exists." + logger "The given mediawiki ${WIKI_DUMP_FILE} file is does not exists." return 1 fi restore_mediawiki $WIKI_DUMP_FILE fi RESTORE_END_TIME=$(date +"%Y-%m-%dT%H%M%SZ") - echo "Restoring Ends at $RESTORE_END_TIME" + logger "Restoring Ends at $RESTORE_END_TIME" }