From 3b2b4c1dbeea9b0179c09d86b6424125aa45ceac Mon Sep 17 00:00:00 2001 From: marbonilla <miguel.ramirez@edunext.co> Date: Fri, 17 Nov 2023 19:02:16 -0500 Subject: [PATCH 01/10] fix: update variables names and jumplines --- drydock_backups/README.rst | 2 +- .../patches/drydock-multipurpose-jobs | 38 ++++++++++--------- drydock_backups/plugin.py | 2 +- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/drydock_backups/README.rst b/drydock_backups/README.rst index 8c22438b..87494edf 100644 --- a/drydock_backups/README.rst +++ b/drydock_backups/README.rst @@ -1,4 +1,4 @@ -Drydock-backuyps +Drydock-backups ================ This is a tutor plugin used to easily store backups of mysql and mongodb databases through k8s cronjobs. It backups the databases and stores them in a bucket. diff --git a/drydock_backups/patches/drydock-multipurpose-jobs b/drydock_backups/patches/drydock-multipurpose-jobs index 62c980f3..f60a1428 100644 --- a/drydock_backups/patches/drydock-multipurpose-jobs +++ b/drydock_backups/patches/drydock-multipurpose-jobs @@ -33,7 +33,9 @@ spec: value: '{{ BACKUP_MYSQL_USERNAME }}' - name: MYSQL_ROOT_PASSWORD value: '{{ BACKUP_MYSQL_PASSWORD }}' - {% if BACKUP_STORAGE_SERVICE == "aws-s3" %} + - name: BACKUP_STORAGE_SERVICE + value: '{{ BACKUP_STORAGE_SERVICE }}' + {%- if BACKUP_STORAGE_SERVICE == "aws-s3" %} - name: AWS_ACCESS_KEY_ID value: '{{ BACKUP_AWS_ACCESS_KEY }}' - name: AWS_SECRET_ACCESS_KEY @@ -42,6 +44,8 @@ spec: value: '{{ BACKUP_BUCKET_NAME }}' - name: BUCKET_PATH value: '{{ BACKUP_BUCKET_PATH }}' + - name: BACKUP_STORAGE_SERVICE + value: '{{ BACKUP_STORAGE_SERVICE }}' {% elif BACKUP_STORAGE_SERVICE == "azure-blob" %} - name: AZURE_CONTAINER_NAME value: '{{ BACKUP_AZURE_CONTAINER_NAME }}' @@ -49,19 +53,18 @@ spec: value: '{{ BACKUP_AZURE_CONTAINER_SAS_TOKEN }}' - name: AZURE_ACCOUNT_NAME value: '{{ BACKUP_AZURE_ACCOUNT_NAME }}' - {% endif %} - {% if BACKUP_CUSTOM_STORAGE_ENDPOINT %} + {% endif -%} + {%- if BACKUP_CUSTOM_STORAGE_ENDPOINT %} - name: BACKUP_CUSTOM_STORAGE_ENDPOINT value: '{{ BACKUP_CUSTOM_STORAGE_ENDPOINT }}' - {% endif %} - {% if BACKUP_K8S_USE_EPHEMERAL_VOLUMES %} + {% endif -%} + {%- if BACKUP_K8S_USE_EPHEMERAL_VOLUMES %} volumeMounts: - mountPath: /data/ name: backup-volume {% endif %} - args: - - mysql - {% if BACKUP_K8S_USE_EPHEMERAL_VOLUMES %} + command: ['mysql'] + {%- if BACKUP_K8S_USE_EPHEMERAL_VOLUMES %} volumes: - name: backup-volume ephemeral: @@ -109,8 +112,10 @@ spec: - name: MONGODB_PASSWORD value: '{{ BACKUP_MONGO_PASSWORD }}' - name: MONGODB_DATABASES - value: '{{ BACKUP_MONGODB_DATABASE }}' - {% if BACKUP_STORAGE_SERVICE == "aws-s3" %} + value: '{{ MONGODB_DATABASE }}' + - name: BACKUP_STORAGE_SERVICE + value: '{{ BACKUP_STORAGE_SERVICE }}' + {%- if BACKUP_STORAGE_SERVICE == "aws-s3" %} - name: AWS_ACCESS_KEY_ID value: '{{ BACKUP_AWS_ACCESS_KEY }}' - name: AWS_SECRET_ACCESS_KEY @@ -119,10 +124,10 @@ spec: value: '{{ BACKUP_BUCKET_NAME }}' - name: BUCKET_PATH value: '{{ BACKUP_BUCKET_PATH }}' - {% if BACKUP_CUSTOM_STORAGE_ENDPOINT %} + {%- if BACKUP_CUSTOM_STORAGE_ENDPOINT %} - name: BACKUP_CUSTOM_STORAGE_ENDPOINT value: '{{ BACKUP_CUSTOM_STORAGE_ENDPOINT }}' - {% endif %} + {% endif -%} {% elif BACKUP_STORAGE_SERVICE == "azure-blob" %} - name: AZURE_CONTAINER_NAME value: '{{ BACKUP_AZURE_CONTAINER_NAME }}' @@ -130,15 +135,14 @@ spec: value: '{{ BACKUP_AZURE_CONTAINER_SAS_TOKEN }}' - name: AZURE_ACCOUNT_NAME value: '{{ BACKUP_AZURE_ACCOUNT_NAME }}' - {% endif %} - {% if BACKUP_K8S_USE_EPHEMERAL_VOLUMES %} + {% endif -%} + {%- if BACKUP_K8S_USE_EPHEMERAL_VOLUMES %} volumeMounts: - mountPath: /data/ name: backup-volume {% endif %} - args: - - mongo - {% if BACKUP_K8S_USE_EPHEMERAL_VOLUMES %} + command: ['mongo'] + {%- if BACKUP_K8S_USE_EPHEMERAL_VOLUMES %} volumes: - name: backup-volume ephemeral: diff --git a/drydock_backups/plugin.py b/drydock_backups/plugin.py index 8c932a68..75208ebf 100644 --- a/drydock_backups/plugin.py +++ b/drydock_backups/plugin.py @@ -15,7 +15,7 @@ "VERSION": __version__, "IMAGE": "ednxops/shipyard-utils:v1.1.0", "CRON_SCHEDULE": '0 2 * * *', - "BACKUP_STORAGE_SERVICE": "aws-s3", + "STORAGE_SERVICE": "aws-s3", "AWS_ACCESS_KEY": "", "AWS_SECRET_KEY": "", "BUCKET_NAME": "", From 7758e715ecc9b8c94ced1d2e7360d93b572b3e7a Mon Sep 17 00:00:00 2001 From: marbonilla <miguel.ramirez@edunext.co> Date: Mon, 20 Nov 2023 11:10:21 -0500 Subject: [PATCH 02/10] fix: duplicate key --- drydock_backups/patches/drydock-multipurpose-jobs | 2 -- 1 file changed, 2 deletions(-) diff --git a/drydock_backups/patches/drydock-multipurpose-jobs b/drydock_backups/patches/drydock-multipurpose-jobs index f60a1428..d4af57c9 100644 --- a/drydock_backups/patches/drydock-multipurpose-jobs +++ b/drydock_backups/patches/drydock-multipurpose-jobs @@ -44,8 +44,6 @@ spec: value: '{{ BACKUP_BUCKET_NAME }}' - name: BUCKET_PATH value: '{{ BACKUP_BUCKET_PATH }}' - - name: BACKUP_STORAGE_SERVICE - value: '{{ BACKUP_STORAGE_SERVICE }}' {% elif BACKUP_STORAGE_SERVICE == "azure-blob" %} - name: AZURE_CONTAINER_NAME value: '{{ BACKUP_AZURE_CONTAINER_NAME }}' From 480418cea7c7d5917a064f146d7a8774a7565c50 Mon Sep 17 00:00:00 2001 From: marbonilla <miguel.ramirez@edunext.co> Date: Mon, 20 Nov 2023 11:14:01 -0500 Subject: [PATCH 03/10] fix: include custom_storage_endpoint in aws block --- drydock_backups/patches/drydock-multipurpose-jobs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drydock_backups/patches/drydock-multipurpose-jobs b/drydock_backups/patches/drydock-multipurpose-jobs index d4af57c9..b44825ba 100644 --- a/drydock_backups/patches/drydock-multipurpose-jobs +++ b/drydock_backups/patches/drydock-multipurpose-jobs @@ -44,6 +44,10 @@ spec: value: '{{ BACKUP_BUCKET_NAME }}' - name: BUCKET_PATH value: '{{ BACKUP_BUCKET_PATH }}' + {%- if BACKUP_CUSTOM_STORAGE_ENDPOINT %} + - name: BACKUP_CUSTOM_STORAGE_ENDPOINT + value: '{{ BACKUP_CUSTOM_STORAGE_ENDPOINT }}' + {% endif -%} {% elif BACKUP_STORAGE_SERVICE == "azure-blob" %} - name: AZURE_CONTAINER_NAME value: '{{ BACKUP_AZURE_CONTAINER_NAME }}' From 5a02fa50718640d904733dcb75cfc439d9f035c1 Mon Sep 17 00:00:00 2001 From: marbonilla <miguel.ramirez@edunext.co> Date: Mon, 20 Nov 2023 14:33:14 -0500 Subject: [PATCH 04/10] fix: newlines control --- .../patches/drydock-multipurpose-jobs | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/drydock_backups/patches/drydock-multipurpose-jobs b/drydock_backups/patches/drydock-multipurpose-jobs index b44825ba..ce7f6297 100644 --- a/drydock_backups/patches/drydock-multipurpose-jobs +++ b/drydock_backups/patches/drydock-multipurpose-jobs @@ -47,24 +47,20 @@ spec: {%- if BACKUP_CUSTOM_STORAGE_ENDPOINT %} - name: BACKUP_CUSTOM_STORAGE_ENDPOINT value: '{{ BACKUP_CUSTOM_STORAGE_ENDPOINT }}' - {% endif -%} - {% elif BACKUP_STORAGE_SERVICE == "azure-blob" %} + {%- endif %} + {%- elif BACKUP_STORAGE_SERVICE == "azure-blob" %} - name: AZURE_CONTAINER_NAME value: '{{ BACKUP_AZURE_CONTAINER_NAME }}' - name: AZURE_CONTAINER_SAS_TOKEN value: '{{ BACKUP_AZURE_CONTAINER_SAS_TOKEN }}' - name: AZURE_ACCOUNT_NAME value: '{{ BACKUP_AZURE_ACCOUNT_NAME }}' - {% endif -%} - {%- if BACKUP_CUSTOM_STORAGE_ENDPOINT %} - - name: BACKUP_CUSTOM_STORAGE_ENDPOINT - value: '{{ BACKUP_CUSTOM_STORAGE_ENDPOINT }}' - {% endif -%} + {%- endif %} {%- if BACKUP_K8S_USE_EPHEMERAL_VOLUMES %} volumeMounts: - mountPath: /data/ name: backup-volume - {% endif %} + {%- endif %} command: ['mysql'] {%- if BACKUP_K8S_USE_EPHEMERAL_VOLUMES %} volumes: @@ -77,7 +73,7 @@ spec: resources: requests: storage: {{ BACKUP_K8S_EPHEMERAL_VOLUME_SIZE }} - {% endif %} + {%- endif %} --- apiVersion: batch/v1 kind: CronJob @@ -129,20 +125,20 @@ spec: {%- if BACKUP_CUSTOM_STORAGE_ENDPOINT %} - name: BACKUP_CUSTOM_STORAGE_ENDPOINT value: '{{ BACKUP_CUSTOM_STORAGE_ENDPOINT }}' - {% endif -%} - {% elif BACKUP_STORAGE_SERVICE == "azure-blob" %} + {%- endif %} + {%- elif BACKUP_STORAGE_SERVICE == "azure-blob" %} - name: AZURE_CONTAINER_NAME value: '{{ BACKUP_AZURE_CONTAINER_NAME }}' - name: AZURE_CONTAINER_SAS_TOKEN value: '{{ BACKUP_AZURE_CONTAINER_SAS_TOKEN }}' - name: AZURE_ACCOUNT_NAME value: '{{ BACKUP_AZURE_ACCOUNT_NAME }}' - {% endif -%} + {%- endif %} {%- if BACKUP_K8S_USE_EPHEMERAL_VOLUMES %} volumeMounts: - mountPath: /data/ name: backup-volume - {% endif %} + {%- endif %} command: ['mongo'] {%- if BACKUP_K8S_USE_EPHEMERAL_VOLUMES %} volumes: @@ -155,4 +151,4 @@ spec: resources: requests: storage: {{ BACKUP_K8S_EPHEMERAL_VOLUME_SIZE }} - {% endif %} + {%- endif %} From 4c514f279d291a483e8bb347f766e61bd80bb4f4 Mon Sep 17 00:00:00 2001 From: marbonilla <miguel.ramirez@edunext.co> Date: Mon, 20 Nov 2023 15:05:35 -0500 Subject: [PATCH 05/10] fix: args for command --- drydock_backups/patches/drydock-multipurpose-jobs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drydock_backups/patches/drydock-multipurpose-jobs b/drydock_backups/patches/drydock-multipurpose-jobs index ce7f6297..34f2de66 100644 --- a/drydock_backups/patches/drydock-multipurpose-jobs +++ b/drydock_backups/patches/drydock-multipurpose-jobs @@ -61,7 +61,8 @@ spec: - mountPath: /data/ name: backup-volume {%- endif %} - command: ['mysql'] + args: + - mysql {%- if BACKUP_K8S_USE_EPHEMERAL_VOLUMES %} volumes: - name: backup-volume @@ -139,7 +140,8 @@ spec: - mountPath: /data/ name: backup-volume {%- endif %} - command: ['mongo'] + args: + - mongo {%- if BACKUP_K8S_USE_EPHEMERAL_VOLUMES %} volumes: - name: backup-volume From fa4ed56cb1081ce91b3e2f426596cced0bacd21e Mon Sep 17 00:00:00 2001 From: marbonilla <miguel.ramirez@edunext.co> Date: Mon, 20 Nov 2023 15:47:26 -0500 Subject: [PATCH 06/10] fix: environment azure variables --- drydock_backups/docker-entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drydock_backups/docker-entrypoint.sh b/drydock_backups/docker-entrypoint.sh index 3518f306..c090e8a8 100755 --- a/drydock_backups/docker-entrypoint.sh +++ b/drydock_backups/docker-entrypoint.sh @@ -32,7 +32,7 @@ if [ "$BACKUP_STORAGE_SERVICE" = 'aws-s3' ]; then aws --endpoint-url $BACKUP_CUSTOM_STORAGE_ENDPOINT s3 mv $FILENAME s3://$S3_BUCKET_NAME/$BUCKET_PATH/$1/ fi elif [ "$BACKUP_STORAGE_SERVICE" = 'azure-blob' ]; then - azcopy cp $FILENAME "https://${AZURE_STORAGE_ACCOUNT}.blob.core.windows.net/${AZURE_STORAGE_CONTAINER_NAME}/$BUCKET_PATH/${1}/$FILENAME?${AZURE_STORAGE_SAS_TOKEN}" + azcopy cp $FILENAME https://$AZURE_ACCOUNT_NAME.blob.core.windows.net/$AZURE_CONTAINER_NAME/$BUCKET_PATH/${1}/$FILENAME?$AZURE_CONTAINER_SAS_TOKEN else echo "Unknown storage service" exit 1 From feb00c9ce8030c85ef0aed838b7a86757d986993 Mon Sep 17 00:00:00 2001 From: marbonilla <miguel.ramirez@edunext.co> Date: Tue, 21 Nov 2023 12:56:51 -0500 Subject: [PATCH 07/10] fix: include bucket path in all options --- drydock_backups/patches/drydock-multipurpose-jobs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drydock_backups/patches/drydock-multipurpose-jobs b/drydock_backups/patches/drydock-multipurpose-jobs index 34f2de66..543af0a8 100644 --- a/drydock_backups/patches/drydock-multipurpose-jobs +++ b/drydock_backups/patches/drydock-multipurpose-jobs @@ -35,6 +35,8 @@ spec: value: '{{ BACKUP_MYSQL_PASSWORD }}' - name: BACKUP_STORAGE_SERVICE value: '{{ BACKUP_STORAGE_SERVICE }}' + - name: BUCKET_PATH + value: '{{ BACKUP_BUCKET_PATH }}' {%- if BACKUP_STORAGE_SERVICE == "aws-s3" %} - name: AWS_ACCESS_KEY_ID value: '{{ BACKUP_AWS_ACCESS_KEY }}' @@ -42,8 +44,6 @@ spec: value: '{{ BACKUP_AWS_SECRET_KEY }}' - name: S3_BUCKET_NAME value: '{{ BACKUP_BUCKET_NAME }}' - - name: BUCKET_PATH - value: '{{ BACKUP_BUCKET_PATH }}' {%- if BACKUP_CUSTOM_STORAGE_ENDPOINT %} - name: BACKUP_CUSTOM_STORAGE_ENDPOINT value: '{{ BACKUP_CUSTOM_STORAGE_ENDPOINT }}' @@ -114,6 +114,8 @@ spec: value: '{{ MONGODB_DATABASE }}' - name: BACKUP_STORAGE_SERVICE value: '{{ BACKUP_STORAGE_SERVICE }}' + - name: BUCKET_PATH + value: '{{ BACKUP_BUCKET_PATH }}' {%- if BACKUP_STORAGE_SERVICE == "aws-s3" %} - name: AWS_ACCESS_KEY_ID value: '{{ BACKUP_AWS_ACCESS_KEY }}' @@ -121,8 +123,6 @@ spec: value: '{{ BACKUP_AWS_SECRET_KEY }}' - name: S3_BUCKET_NAME value: '{{ BACKUP_BUCKET_NAME }}' - - name: BUCKET_PATH - value: '{{ BACKUP_BUCKET_PATH }}' {%- if BACKUP_CUSTOM_STORAGE_ENDPOINT %} - name: BACKUP_CUSTOM_STORAGE_ENDPOINT value: '{{ BACKUP_CUSTOM_STORAGE_ENDPOINT }}' From e3003d9fbab0328c5cd30c4ee7d96bfee35036df Mon Sep 17 00:00:00 2001 From: marbonilla <miguel.ramirez@edunext.co> Date: Tue, 21 Nov 2023 13:35:06 -0500 Subject: [PATCH 08/10] fix: delete databases variable --- drydock_backups/patches/drydock-multipurpose-jobs | 2 -- drydock_backups/plugin.py | 1 - 2 files changed, 3 deletions(-) diff --git a/drydock_backups/patches/drydock-multipurpose-jobs b/drydock_backups/patches/drydock-multipurpose-jobs index 543af0a8..02074efb 100644 --- a/drydock_backups/patches/drydock-multipurpose-jobs +++ b/drydock_backups/patches/drydock-multipurpose-jobs @@ -110,8 +110,6 @@ spec: value: '{{ BACKUP_MONGO_USERNAME }}' - name: MONGODB_PASSWORD value: '{{ BACKUP_MONGO_PASSWORD }}' - - name: MONGODB_DATABASES - value: '{{ MONGODB_DATABASE }}' - name: BACKUP_STORAGE_SERVICE value: '{{ BACKUP_STORAGE_SERVICE }}' - name: BUCKET_PATH diff --git a/drydock_backups/plugin.py b/drydock_backups/plugin.py index 75208ebf..4696a0be 100644 --- a/drydock_backups/plugin.py +++ b/drydock_backups/plugin.py @@ -30,7 +30,6 @@ "MYSQL_PASSWORD": '{{ MYSQL_ROOT_PASSWORD }}', "MONGO_PASSWORD": '{{ MONGODB_PASSWORD }}', "MONGO_USERNAME": '{{ MONGODB_USERNAME }}', - "MONGODB_DATABASE": "" }, } From efcd1b7aabcf43e8f3d75d71bd821df10ec266b5 Mon Sep 17 00:00:00 2001 From: marbonilla <miguel.ramirez@edunext.co> Date: Wed, 22 Nov 2023 18:10:50 -0500 Subject: [PATCH 09/10] fix: update .sh --- drydock_backups/Dockerfile | 4 ++-- drydock_backups/docker-entrypoint.sh | 7 +++++-- drydock_backups/patches/k8s-jobs | 1 - 3 files changed, 7 insertions(+), 5 deletions(-) delete mode 100644 drydock_backups/patches/k8s-jobs diff --git a/drydock_backups/Dockerfile b/drydock_backups/Dockerfile index f471f9af..1d1845a9 100644 --- a/drydock_backups/Dockerfile +++ b/drydock_backups/Dockerfile @@ -2,9 +2,9 @@ FROM mysql:5.7-debian as base RUN apt-get update && apt-get install -y wget gnupg2 -RUN wget -qO - https://www.mongodb.org/static/pgp/server-6.0.asc | apt-key add - +RUN wget -qO - https://www.mongodb.org/static/pgp/server-4.4.asc | apt-key add - -RUN echo "deb http://repo.mongodb.org/apt/debian buster/mongodb-org/6.0 main" | tee /etc/apt/sources.list.d/mongodb-org-6.0.list +RUN echo "deb http://repo.mongodb.org/apt/debian buster/mongodb-org/4.4 main" | tee /etc/apt/sources.list.d/mongodb-org-4.4.list RUN apt-get update && apt-get install -y \ mongodb-org \ diff --git a/drydock_backups/docker-entrypoint.sh b/drydock_backups/docker-entrypoint.sh index c090e8a8..9eba32ee 100755 --- a/drydock_backups/docker-entrypoint.sh +++ b/drydock_backups/docker-entrypoint.sh @@ -8,12 +8,15 @@ MONGODB_URI="mongodb://" MONGODB_PORT=${MONGODB_PORT:-27017} MONGODB_HOST=${MONGODB_HOST:-mongodb} MONGODB_AUTHDB=${MONGODB_AUTHDB:-admin} +MONGODB_AUTHDB_STRING="authSource=${MONGODB_AUTHDB}" -if [ -n "${MONGODB_USERNAME}" ] && [ -n "${MONGODB_PASSWORD}" ]; then +if [ -z ${MONGODB_USERNAME} ] && [ -z ${MONGODB_PASSWORD} ]; then + MONGODB_AUTHDB_STRING="" +else MONGODB_URI+="${MONGODB_USERNAME}:${MONGODB_PASSWORD}@" fi -MONGODB_URI+="${MONGODB_HOST}:${MONGODB_PORT}/${MONGODB_DATABASES}?authSource=${MONGODB_AUTHDB}" +MONGODB_URI+="${MONGODB_HOST}:${MONGODB_PORT}/?${MONGODB_AUTHDB_STRING}" if [ "$1" = 'mysql' ]; then mysqldump -u $MYSQL_ROOT_USERNAME -h $MYSQL_HOST -P $MYSQL_PORT --password=$MYSQL_ROOT_PASSWORD \ diff --git a/drydock_backups/patches/k8s-jobs b/drydock_backups/patches/k8s-jobs deleted file mode 100644 index cb6f96b6..00000000 --- a/drydock_backups/patches/k8s-jobs +++ /dev/null @@ -1 +0,0 @@ -{{ patch("drydock-multipurpose-jobs") }} From f25b729955f934ba2cfb37b7ce164fa84d9ea9b2 Mon Sep 17 00:00:00 2001 From: marbonilla <miguel.ramirez@edunext.co> Date: Thu, 23 Nov 2023 11:18:44 -0500 Subject: [PATCH 10/10] fix: update default shipyard-utils image --- drydock_backups/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drydock_backups/plugin.py b/drydock_backups/plugin.py index 4696a0be..70ab570b 100644 --- a/drydock_backups/plugin.py +++ b/drydock_backups/plugin.py @@ -13,7 +13,7 @@ config = { "defaults": { "VERSION": __version__, - "IMAGE": "ednxops/shipyard-utils:v1.1.0", + "IMAGE": "ednxops/shipyard-utils:v15.4.0", "CRON_SCHEDULE": '0 2 * * *', "STORAGE_SERVICE": "aws-s3", "AWS_ACCESS_KEY": "",