Fix: Removed dir on loadmeta failure

If loadmeta fails during SyncBlocks, the block directory is left behind, resulting in subsequent failures to read the meta file. FailureToLoadMeta unit test Signed-off-by: Thor <[email protected]>
thanos-io · Nov 27, 2019 · f4930f3 · f4930f3
1 parent 56c1ae6
commit f4930f3
Show file tree

Hide file tree

Showing 2 changed files with 42 additions and 1 deletion.
diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go
@@ -343,7 +343,7 @@ func (s *BucketStore) SyncBlocks(ctx context.Context) error {
 		bdir := path.Join(s.dir, id.String())
 		meta, err := loadMeta(ctx, s.logger, s.bucket, bdir, id)
 		if err != nil {
-			return errors.Wrap(err, "load meta")
+			return errors.Wrap(os.RemoveAll(bdir), "load meta")
 		}
 
 		inRange, err := s.isBlockInMinMaxRange(ctx, meta)

diff --git a/pkg/store/bucket_test.go b/pkg/store/bucket_test.go
@@ -805,3 +805,44 @@ func expectedTouchedBlockOps(all []ulid.ULID, expected []ulid.ULID, cached []uli
 	sort.Strings(ops)
 	return ops
 }
+
+func TestFailureToLoadMeta(t *testing.T) {
+
+	ctx := context.Background()
+	logger := log.NewNopLogger()
+
+	dir, err := ioutil.TempDir("", "test-failure-to-load-meta")
+	testutil.Ok(t, err)
+	defer func() { testutil.Ok(t, os.RemoveAll(dir)) }()
+
+	syncdir, err := ioutil.TempDir("", "test-failure-to-load-meta")
+	testutil.Ok(t, err)
+	defer func() { testutil.Ok(t, os.RemoveAll(dir)) }()
+
+	bkt := inmem.NewBucket()
+	series := []labels.Labels{labels.FromStrings("a", "1", "b", "1")}
+
+	id1, err := testutil.CreateBlock(ctx, dir, series, 10, 0, 1000, labels.Labels{{Name: "cluster", Value: "a"}, {Name: "region", Value: "r1"}}, 0)
+	testutil.Ok(t, err)
+
+	testutil.Ok(t, block.Upload(ctx, logger, bkt, filepath.Join(dir, id1.String())))
+
+	ok, err := bkt.Exists(ctx, filepath.Join(id1.String(), "meta.json"))
+	testutil.Ok(t, err)
+	testutil.Equals(t, true, ok)
+
+	// Delete the meta.json file
+	testutil.Ok(t, bkt.Delete(ctx, filepath.Join(id1.String(), "meta.json")))
+
+	rec := &recorder{Bucket: bkt}
+	var relabelConf []*relabel.Config
+	bucketStore, err := NewBucketStore(logger, nil, rec, syncdir, noopCache{}, 0, 0, 99, false, 20,
+		filterConf, relabelConf, true)
+	testutil.Ok(t, err)
+
+	bucketStore.SyncBlocks(context.Background())
+
+	// Ensure the directory does not exist (cleaned up on load meta error)
+	_, err = os.Stat(filepath.Join(syncdir, id1.String()))
+	testutil.Equals(t, true, os.IsNotExist(err))
+}