From bd93b2b2ee05f3df0607ad4ded84e4e1874c7357 Mon Sep 17 00:00:00 2001 From: Tesshu Flower Date: Mon, 25 Mar 2024 13:40:14 -0400 Subject: [PATCH 1/3] Restic allow restore from non-initialized path Fixes: https://github.com/backube/volsync/issues/1172 Signed-off-by: Tesshu Flower --- CHANGELOG.md | 7 ++++ controllers/mover/restic/logfilter.go | 4 +++ controllers/mover/restic/logfilter_test.go | 41 +++++++++++++++++++++- mover-restic/entry.sh | 7 +++- 4 files changed, 57 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3d8e3181..221320f1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [unreleased] + +### Fixed + +- Allow restic restore from empty or non-initialized path + ## [0.9.0] ### Changed @@ -282,6 +288,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Support for rsync & rclone replication - Helm chart to deploy operator +[Unreleased]: https://github.com/backube/volsync/compare/release-0.9...HEAD [0.9.0]: https://github.com/backube/volsync/compare/release-0.8...v0.9.0 [0.8.1]: https://github.com/backube/volsync/compare/release-0.8.0...v0.8.1 [0.8.0]: https://github.com/backube/volsync/compare/release-0.7...v0.8.0 diff --git a/controllers/mover/restic/logfilter.go b/controllers/mover/restic/logfilter.go index ee3581430..674592635 100644 --- a/controllers/mover/restic/logfilter.go +++ b/controllers/mover/restic/logfilter.go @@ -24,6 +24,10 @@ import ( var resticRegex = regexp.MustCompile( `^\s*([pP]rocessed)\s.+([fF]iles)|` + `^\s*([sS]napshot)\s.+([sS]aved)|` + + `^\s*(No eligible)|` + + `(No data)|` + + `(Directory is empty)|` + + `^\s*([cC]reated)|` + `^\s*([rR]epository)\s.+([oO]pened)|` + `^\s*([rR]estoring)|` + `^\s*([nN]o parent snapshot)|` + diff --git a/controllers/mover/restic/logfilter_test.go b/controllers/mover/restic/logfilter_test.go index de5790dce..7c2daa875 100644 --- a/controllers/mover/restic/logfilter_test.go +++ b/controllers/mover/restic/logfilter_test.go @@ -66,7 +66,9 @@ Restic completed in 18s === Done ===` // nolint:lll - expectedFilteredResticSourceLogSuccessful := `repository f5bccd54 opened (repository version 2) successfully, password is correct + expectedFilteredResticSourceLogSuccessful := `created restic repository f5bccd54c8 at s3:http://minio-api-minio.apps.app-aws-411ga-sno-net2-zp5jq.dev06.red-chesterfield.com/ttest-restic-new +repository f5bccd54 opened (repository version 2) successfully, password is correct +created new cache in /home/testuser/DEVFEDORA/volsync/RESTICTESTS/CACHE no parent snapshot found, will read all files Added to the repository: 12.941 MiB (12.529 MiB stored) processed 25 files, 36.658 MiB in 0:12 @@ -199,4 +201,41 @@ Restic completed in 9s` Expect(filteredLines).To(Equal(expectedFilteredResticDestlogSuccessful)) }) }) + + Context("Restic dest mover logs - empty repo/path (not initialized previously)", func() { + // Sample restore log for restic mover + // nolint:lll + resticDestlogSuccessful := `Starting container +VolSync restic container version: v0.9.0+169bc5b-dirty +restore +restic 0.16.4 compiled with go1.21.8 on linux/amd64 +Testing mandatory env variables +=== Check for dir initialized === +=== Initialize Dir === +created restic repository 374c6313cb at s3:http://minio.minio.svc.cluster.local:9000/resticbucket1b + +Please note that knowledge of your password is required to access +the repository. Losing your password means that your data is +irrecoverably lost. +=== Starting restore === +No eligible snapshots found +=== No data will be restored === +Restic completed in 4s +=== Done ===` + + // nolint:lll + expectedFilteredResticDestlogSuccessful := `created restic repository 374c6313cb at s3:http://minio.minio.svc.cluster.local:9000/resticbucket1b +No eligible snapshots found +=== No data will be restored === +Restic completed in 4s` + + It("Should filter the logs from a successful replication dest (restic restore)", func() { + reader := strings.NewReader(resticDestlogSuccessful) + filteredLines, err := utils.FilterLogs(reader, restic.LogLineFilterSuccess) + Expect(err).NotTo(HaveOccurred()) + + logger.Info("Logs after filter", "filteredLines", filteredLines) + Expect(filteredLines).To(Equal(expectedFilteredResticDestlogSuccessful)) + }) + }) }) diff --git a/mover-restic/entry.sh b/mover-restic/entry.sh index f5eab8cc2..889eb03d1 100755 --- a/mover-restic/entry.sh +++ b/mover-restic/entry.sh @@ -49,13 +49,16 @@ function check_contents { # Ensure the repo has been initialized function ensure_initialized { - echo "== Initialize Dir =======" + echo "=== Check for dir initialized ===" # Try a restic command and capture the rc & output outfile=$(mktemp -q) if ! "${RESTIC[@]}" snapshots 2>"$outfile"; then output=$(<"$outfile") # Match against error string for uninitialized repo + # This string also appears when credentials are incorrect (in which case + # the following cmd `restic init` will also fail) if [[ $output =~ .*(Is there a repository at the following location).* ]]; then + echo "=== Initialize Dir ===" "${RESTIC[@]}" init else cat "$outfile" @@ -252,6 +255,7 @@ function do_restore { snapshot_id=$(select_restic_snapshot_to_restore) if [[ -z ${snapshot_id} ]]; then echo "No eligible snapshots found" + echo "=== No data will be restored ===" else pushd "${DATA_DIR}" echo "Selected restic snapshot with id: ${snapshot_id}" @@ -286,6 +290,7 @@ for op in "$@"; do do_prune ;; "restore") + ensure_initialized do_restore sync ;; From 3993ac61c59d960a4069252de7d30d48a6bdaeb1 Mon Sep 17 00:00:00 2001 From: Tesshu Flower Date: Mon, 25 Mar 2024 20:14:09 -0400 Subject: [PATCH 2/3] restic backup ignore lost+found in empty dir check Fixes: https://github.com/backube/volsync/issues/1181 Signed-off-by: Tesshu Flower --- CHANGELOG.md | 1 + controllers/mover/restic/logfilter_test.go | 22 ++++++++++++++++++++++ mover-restic/entry.sh | 2 +- 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 221320f1d..e0709229e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Allow restic restore from empty or non-initialized path +- Ignore lost+found on restic backup when checking for empty source volume ## [0.9.0] diff --git a/controllers/mover/restic/logfilter_test.go b/controllers/mover/restic/logfilter_test.go index 7c2daa875..8b6571c8b 100644 --- a/controllers/mover/restic/logfilter_test.go +++ b/controllers/mover/restic/logfilter_test.go @@ -83,6 +83,28 @@ Restic completed in 18s` Expect(filteredLines).To(Equal(expectedFilteredResticSourceLogSuccessful)) }) + It("Should filter the logs from a successful replication source of an empty dir (skip restic backup)", func() { + // Sample backup log for restic mover where data is empty + // nolint:lll + resticSourceLogSuccessful := `Starting container +VolSync restic container version: v0.9.1+aa9ab46-dirty +backup +restic 0.16.4 compiled with go1.21.8 on linux/amd64 +Testing mandatory env variables +== Checking directory for content === +== Directory is empty skipping backup ===` + + // nolint:lll + expectedFilteredResticSourceLogSuccessful := `== Directory is empty skipping backup ===` + + reader := strings.NewReader(resticSourceLogSuccessful) + filteredLines, err := utils.FilterLogs(reader, restic.LogLineFilterSuccess) + Expect(err).NotTo(HaveOccurred()) + + logger.Info("Logs after filter", "filteredLines", filteredLines) + Expect(filteredLines).To(Equal(expectedFilteredResticSourceLogSuccessful)) + }) + It("Should filter the logs from a replication source (restic backup) that performed an unlock", func() { // Sample backup log for restic mover resticSourceLog := `Starting container diff --git a/mover-restic/entry.sh b/mover-restic/entry.sh index 889eb03d1..7b8bfa735 100755 --- a/mover-restic/entry.sh +++ b/mover-restic/entry.sh @@ -40,7 +40,7 @@ function check_var_defined { function check_contents { echo "== Checking directory for content ===" - DIR_CONTENTS="$(ls -A "${DATA_DIR}")" + DIR_CONTENTS="$(ls -A "${DATA_DIR}" | grep -v "lost+found" || true)" if [ -z "${DIR_CONTENTS}" ]; then echo "== Directory is empty skipping backup ===" exit 0 From f0c8f61d54eab92fa5663444f06c0cf1fe575ed1 Mon Sep 17 00:00:00 2001 From: Tesshu Flower Date: Mon, 25 Mar 2024 22:01:36 -0400 Subject: [PATCH 3/3] Linting fix - use ls --ignore instead of grep -v Signed-off-by: Tesshu Flower --- mover-restic/entry.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mover-restic/entry.sh b/mover-restic/entry.sh index 7b8bfa735..d71febf38 100755 --- a/mover-restic/entry.sh +++ b/mover-restic/entry.sh @@ -40,7 +40,7 @@ function check_var_defined { function check_contents { echo "== Checking directory for content ===" - DIR_CONTENTS="$(ls -A "${DATA_DIR}" | grep -v "lost+found" || true)" + DIR_CONTENTS="$(ls -A "${DATA_DIR}" --ignore="lost+found")" if [ -z "${DIR_CONTENTS}" ]; then echo "== Directory is empty skipping backup ===" exit 0