Merge pull request #2500 from thanos-io/release-0.12

Merge v0.12.1 into master
thanos-io · Apr 22, 2020 · 45a6bc4 · 45a6bc4
2 parents 822bc7c + 60a2c9e
commit 45a6bc4
Show file tree

Hide file tree

Showing 14 changed files with 224 additions and 81 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -9,7 +9,14 @@ NOTE: As semantic versioning states all 0.y.z releases can contain breaking chan
 
 We use *breaking* word for marking changes that are not backward compatible (relates only to v0.y.z releases.)
 
-## Unreleased
+## [v0.12.1](https://github.com/thanos-io/thanos/releases/tag/v0.12.1) - 2020.04.20
+
+### Fixed
+
+- [#2411](https://github.com/thanos-io/thanos/pull/2411) Query: fix a bug where queries might not time out sometimes due to issues with one or more StoreAPIs.
+- [#2474](https://github.com/thanos-io/thanos/pull/2474) Store: fix a panic caused by concurrent memory access during block filtering.
+- [#2472](https://github.com/thanos-io/thanos/pull/2472) Compact: fix a bug where partial blocks were never deleted, causing spam of warnings.
+- [#2484](https://github.com/thanos-io/thanos/pull/2484) Query/Ruler: fix issue #2483, when web.route-prefix is set, it is added twice in HTTP router prefix.
 
 ### Fixed
 

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-0.12.0
+0.12.1
diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go
@@ -420,7 +420,7 @@ func runCompact(
 		}
 
 		// No need to resync before partial uploads and delete marked blocks. Last sync should be valid.
-		compact.BestEffortCleanAbortedPartialUploads(ctx, logger, sy.Partial(), bkt, partialUploadDeleteAttempts, blocksMarkedForDeletion)
+		compact.BestEffortCleanAbortedPartialUploads(ctx, logger, sy.Partial(), bkt, partialUploadDeleteAttempts, blocksCleaned, blockCleanupFailures)
 		if err := blocksCleaner.DeleteMarkedBlocks(ctx); err != nil {
 			return errors.Wrap(err, "error cleaning blocks")
 		}

diff --git a/cmd/thanos/query.go b/cmd/thanos/query.go
@@ -8,7 +8,6 @@ import (
 	"fmt"
 	"math"
 	"net/http"
-	"path"
 	"strings"
 	"time"
 
@@ -357,7 +356,7 @@ func runQuery(
 
 		api := v1.NewAPI(logger, reg, engine, queryableCreator, enableAutodownsampling, enablePartialResponse, replicaLabels, instantDefaultMaxSourceResolution)
 
-		api.Register(router.WithPrefix(path.Join(webRoutePrefix, "/api/v1")), tracer, logger, ins)
+		api.Register(router.WithPrefix("/api/v1"), tracer, logger, ins)
 
 		srv := httpserver.New(logger, reg, comp, httpProbe,
 			httpserver.WithListen(httpBindAddr),

diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go
@@ -8,7 +8,6 @@ import (
 	"math/rand"
 	"net/http"
 	"net/url"
-	"path"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -576,7 +575,7 @@ func runRule(
 		ui.NewRuleUI(logger, reg, ruleMgr, alertQueryURL.String(), webExternalPrefix, webPrefixHeaderName).Register(router, ins)
 
 		api := v1.NewAPI(logger, reg, ruleMgr)
-		api.Register(router.WithPrefix(path.Join(webRoutePrefix, "/api/v1")), tracer, logger, ins)
+		api.Register(router.WithPrefix("/api/v1"), tracer, logger, ins)
 
 		srv := httpserver.New(logger, reg, comp, httpProbe,
 			httpserver.WithListen(httpBindAddr),

diff --git a/docs/release-process.md b/docs/release-process.md
@@ -33,7 +33,7 @@ Release shepherd responsibilities:
 | Release   | Time of first RC         | Shepherd (GitHub handle) |
 |-----------|--------------------------|--------------------------|
 | v0.13.0   | (planned) 2020.05.13     | `TBD`                    |
-| v0.12.0   | (planned) 2020.04.01     | `@squat`                 |
+| v0.12.0   | 2020.04.15               | `@squat`                 |
 | v0.11.0   | 2020.02.19               | `@metalmatze`            |
 | v0.10.0   | 2020.01.08               | `@GiedriusS`             |
 | v0.9.0    | 2019.11.26               | `@bwplotka`              |

diff --git a/pkg/block/fetcher.go b/pkg/block/fetcher.go
@@ -550,6 +550,7 @@ var _ MetadataFilter = &DeduplicateFilter{}
 // Not go-routine safe.
 type DeduplicateFilter struct {
 	duplicateIDs []ulid.ULID
+	mu           sync.Mutex
 }
 
 // NewDeduplicateFilter creates DeduplicateFilter.
@@ -603,11 +604,13 @@ func (f *DeduplicateFilter) filterForResolution(root *Node, metaSlice []*metadat
 
 	duplicateULIDs := getNonRootIDs(root)
 	for _, id := range duplicateULIDs {
+		f.mu.Lock()
 		if metas[id] != nil {
 			f.duplicateIDs = append(f.duplicateIDs, id)
 		}
 		synced.WithLabelValues(duplicateMeta).Inc()
 		delete(metas, id)
+		f.mu.Unlock()
 	}
 }
 

diff --git a/pkg/compact/clean.go b/pkg/compact/clean.go
@@ -27,7 +27,8 @@ func BestEffortCleanAbortedPartialUploads(
 	partial map[ulid.ULID]error,
 	bkt objstore.Bucket,
 	deleteAttempts prometheus.Counter,
-	blocksMarkedForDeletion prometheus.Counter,
+	blockCleanups prometheus.Counter,
+	blockCleanupFailures prometheus.Counter,
 ) {
 	level.Info(logger).Log("msg", "started cleaning of aborted partial uploads")
 
@@ -45,10 +46,15 @@ func BestEffortCleanAbortedPartialUploads(
 
 		deleteAttempts.Inc()
 		level.Info(logger).Log("msg", "found partially uploaded block; marking for deletion", "block", id)
-		if err := block.MarkForDeletion(ctx, logger, bkt, id, blocksMarkedForDeletion); err != nil {
-			level.Warn(logger).Log("msg", "failed to delete aborted partial upload; skipping", "block", id, "thresholdAge", PartialUploadThresholdAge, "err", err)
-			return
+		// We don't gather any information about deletion marks for partial blocks, so let's simply remove it. We waited
+		// long PartialUploadThresholdAge already.
+		// TODO(bwplotka): Fix some edge cases: https://github.com/thanos-io/thanos/issues/2470 .
+		if err := block.Delete(ctx, logger, bkt, id); err != nil {
+			blockCleanupFailures.Inc()
+			level.Warn(logger).Log("msg", "failed to delete aborted partial upload; will retry in next iteration", "block", id, "thresholdAge", PartialUploadThresholdAge, "err", err)
+			continue
 		}
+		blockCleanups.Inc()
 		level.Info(logger).Log("msg", "deleted aborted partial upload", "block", id, "thresholdAge", PartialUploadThresholdAge)
 	}
 	level.Info(logger).Log("msg", "cleaning of aborted partial uploads done")

diff --git a/pkg/compact/clean_test.go b/pkg/compact/clean_test.go
@@ -59,22 +59,19 @@ func TestBestEffortCleanAbortedPartialUploads(t *testing.T) {
 	testutil.Ok(t, bkt.Upload(ctx, path.Join(shouldIgnoreID2.String(), "chunks", "000001"), &fakeChunk))
 
 	deleteAttempts := promauto.With(nil).NewCounter(prometheus.CounterOpts{})
-	blocksMarkedForDeletion := promauto.With(nil).NewCounter(prometheus.CounterOpts{})
-
+	blockCleanups := promauto.With(nil).NewCounter(prometheus.CounterOpts{})
+	blockCleanupFailures := promauto.With(nil).NewCounter(prometheus.CounterOpts{})
 	_, partial, err := metaFetcher.Fetch(ctx)
 	testutil.Ok(t, err)
 
-	BestEffortCleanAbortedPartialUploads(ctx, logger, partial, bkt, deleteAttempts, blocksMarkedForDeletion)
+	BestEffortCleanAbortedPartialUploads(ctx, logger, partial, bkt, deleteAttempts, blockCleanups, blockCleanupFailures)
 	testutil.Equals(t, 1.0, promtest.ToFloat64(deleteAttempts))
+	testutil.Equals(t, 1.0, promtest.ToFloat64(blockCleanups))
+	testutil.Equals(t, 0.0, promtest.ToFloat64(blockCleanupFailures))
 
 	exists, err := bkt.Exists(ctx, path.Join(shouldDeleteID.String(), "chunks", "000001"))
 	testutil.Ok(t, err)
-	testutil.Equals(t, true, exists)
-
-	exists, err = bkt.Exists(ctx, path.Join(shouldDeleteID.String(), metadata.DeletionMarkFilename))
-	testutil.Ok(t, err)
-	testutil.Equals(t, true, exists)
-	testutil.Equals(t, 1.0, promtest.ToFloat64(blocksMarkedForDeletion))
+	testutil.Equals(t, false, exists)
 
 	exists, err = bkt.Exists(ctx, path.Join(shouldIgnoreID1.String(), "chunks", "000001"))
 	testutil.Ok(t, err)