Merge branch 'main' of https://github.com/metonymic-smokey/thanos int…

…o receive
thanos-io · Feb 12, 2022 · f489478 · f489478
2 parents 69c7ed6 + 71ad51d
commit f489478
Show file tree

Hide file tree

Showing 102 changed files with 3,784 additions and 1,042 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -12,6 +12,9 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 
 ### Added
 
+- [#5110](https://github.com/thanos-io/thanos/pull/5110) Block: Do not upload DebugMeta files to obj store.
+- [#4963](https://github.com/thanos-io/thanos/pull/4963) Compactor, Store, Tools: Loading block metadata now only filters out duplicates within a source (or compaction group if replica labels are configured), and does so in parallel over sources.
+- [#5089](https://github.com/thanos-io/thanos/pull/5089) S3: Create an empty map in the case SSE-KMS is used and no KMSEncryptionContext is passed.
 - [#4970](https://github.com/thanos-io/thanos/pull/4970) Added a new flag `exclude-delete` to `tools bucket ls`, which excludes blocks marked for deletion.
 - [#4903](https://github.com/thanos-io/thanos/pull/4903) Compactor: Added tracing support for compaction.
 - [#4909](https://github.com/thanos-io/thanos/pull/4909) Compactor: Add flag --max-time / --min-time to filter blocks that are ready to be compacted.
@@ -22,6 +25,13 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 - [#4974](https://github.com/thanos-io/thanos/pull/4974) Store: Support tls_config configuration for connecting with Azure storage.
 - [#4999](https://github.com/thanos-io/thanos/pull/4999) COS: Support `endpoint` configuration for vpc internal endpoint.
 - [#5059](https://github.com/thanos-io/thanos/pull/5059) Compactor: Adding minimum retention flag validation for downsampling retention.
+- [#4667](https://github.com/thanos-io/thanos/pull/4667) Add a pure aws-sdk auth for s3 storage.
+- [#5111](https://github.com/thanos-io/thanos/pull/5111) Add matcher support to Query Rules endpoint.
+- [#5117](https://github.com/thanos-io/thanos/pull/5117) Bucket replicate: Added flag `--ignore-marked-for-deletion` to avoid replication of blocks with the deletion mark.
+
+## Changed
+- [#5144](https://github.com/thanos-io/thanos/pull/5144) UI: Improve graph color
+- [#5119](https://github.com/thanos-io/thanos/pull/5119) UI: Optimize Target, Alert and Service Discovery page and on each of them add a search bar.
 
 ### Fixed
 
@@ -30,6 +40,8 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 - [#4879](https://github.com/thanos-io/thanos/pull/4879) Bucket verify: Fixed bug causing wrong number of blocks to be checked.
 - [#4908](https://github.com/thanos-io/thanos/pull/4908) UI: Show 'minus' icon and add tooltip when store min / max time is not available.
 - [#4883](https://github.com/thanos-io/thanos/pull/4883) Mixin: adhere to RFC 1123 compatible component naming.
+- [#5114](https://github.com/thanos-io/thanos/pull/5114) Tools `thanos bucket inspect` fix time formatting.
+- [#5139](https://github.com/thanos-io/thanos/pull/5139) COS: Support multi-part upload, fix upload issue when index size more than 5GB.
 
 ## [v0.24.0](https://github.com/thanos-io/thanos/tree/release-0.24) - 2021.12.22
 

diff --git a/MAINTAINERS.md b/MAINTAINERS.md
@@ -1,15 +1,16 @@
 # Core Maintainers of this repository
 
-| Name                  | Email                  | Slack                    | GitHub                                       | Company       |
-|-----------------------|------------------------|--------------------------|----------------------------------------------|---------------|
-| Bartłomiej Płotka     | [email protected]     | `@bwplotka`              | [@bwplotka](https://github.com/bwplotka)     | Red Hat       |
-| Frederic Branczyk     | [email protected]    | `@brancz`                | [@brancz](https://github.com/brancz)         | Polar Signals |
-| Giedrius Statkevičius | [email protected] | `@Giedrius Statkevičius` | [@GiedriusS](https://github.com/GiedriusS)   | Vinted        |
-| Kemal Akkoyun         | [email protected]     | `@kakkoyun`              | [@kakkoyun](https://github.com/kakkoyun)     | Polar Signals |
-| Lucas Servén Marín    | [email protected]      | `@squat`                 | [@squat](https://github.com/squat)           | Red Hat       |
-| Prem Saraswat         | [email protected]     | `@Prem Saraswat`         | [@onprem](https://github.com/onprem)         | Red Hat       |
-| Matthias Loibl        | [email protected] | `@metalmatze`            | [@metalmatze](https://github.com/metalmatze) | Polar Signals |
-| Ben Ye                | [email protected]  | `@yeya24`                | [@yeya24](https://github.com/yeya24)         | ByteDance     |
+| Name                  | Email                  | Slack                    | GitHub                                         | Company       |
+|-----------------------|------------------------|--------------------------|------------------------------------------------|---------------|
+| Bartłomiej Płotka     | [email protected]     | `@bwplotka`              | [@bwplotka](https://github.com/bwplotka)       | Red Hat       |
+| Frederic Branczyk     | [email protected]    | `@brancz`                | [@brancz](https://github.com/brancz)           | Polar Signals |
+| Giedrius Statkevičius | [email protected] | `@Giedrius Statkevičius` | [@GiedriusS](https://github.com/GiedriusS)     | Vinted        |
+| Kemal Akkoyun         | [email protected]     | `@kakkoyun`              | [@kakkoyun](https://github.com/kakkoyun)       | Polar Signals |
+| Lucas Servén Marín    | [email protected]      | `@squat`                 | [@squat](https://github.com/squat)             | Red Hat       |
+| Prem Saraswat         | [email protected]     | `@Prem Saraswat`         | [@onprem](https://github.com/onprem)           | Red Hat       |
+| Matthias Loibl        | [email protected] | `@metalmatze`            | [@metalmatze](https://github.com/metalmatze)   | Polar Signals |
+| Ben Ye                | [email protected]  | `@yeya24`                | [@yeya24](https://github.com/yeya24)           | ByteDance     |
+| Wiard van Rij         | [email protected]      | `@wiard van Rij`         | [@wiardvanrij](https://github.com/wiardvanrij) | Roku          |
 
 We are bunch of people from different companies with various interests and skills. We are from different parts of the world: Germany, Italy, Lithuania, Poland, UK, India and China. We have something in common though: We all share the love for OpenSource, Go, Prometheus, :coffee: and Observability topics.
 
@@ -27,15 +28,15 @@ We also have some nice souls that help triaging issues and PRs. See [here](https
 
 Full list of triage persons is displayed below:
 
-| Name          | Slack            | GitHub                                         | Company |
-|---------------|------------------|------------------------------------------------|---------|
-| Adrien Fillon | `@Adrien F`      | [@adrien-f](https://github.com/adrien-f)       |         |
-| Ian Billett   | `@billett`       | [@ianbillett](https://github.com/ianbillett)   | Red Hat |
-| Martin Chodur | `@FUSAKLA`       | [@fusakla](https://github.com/fusakla)         |         |
-| Michael Dai   | `@jojohappy`     | [@jojohappy](https://github.com/jojohappy)     |         |
-| Xiang Dai     | `@daixiang0`     | [@daixiang0](https://github.com/daixiang0)     |         |
-| Wiard van Rij | `@wiard van Rij` | [@wiardvanrij](https://github.com/wiardvanrij) | Roku    |
-| Jimmie Han    | `@hanjm`         | [@hanjm](https://github.com/hanjm)             | Tencent |
+| Name          | Slack         | GitHub                                       | Company |
+|---------------|---------------|----------------------------------------------|---------|
+| Adrien Fillon | `@Adrien F`   | [@adrien-f](https://github.com/adrien-f)     |         |
+| Ian Billett   | `@billett`    | [@ianbillett](https://github.com/ianbillett) | Red Hat |
+| Martin Chodur | `@FUSAKLA`    | [@fusakla](https://github.com/fusakla)       |         |
+| Michael Dai   | `@jojohappy`  | [@jojohappy](https://github.com/jojohappy)   |         |
+| Xiang Dai     | `@daixiang0`  | [@daixiang0](https://github.com/daixiang0)   |         |
+| Jimmie Han    | `@hanjm`      | [@hanjm](https://github.com/hanjm)           | Tencent |
+| Matej Gera    | `@Matej Gera` | [@matej-g](https://github.com/matej-g)       | Red Hat |
 
 Please reach any of the maintainer on slack or email if you want to help as well.
 

diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go
@@ -228,7 +228,7 @@ func runCompact(
 	// The delay of deleteDelay/2 is added to ensure we fetch blocks that are meant to be deleted but do not have a replacement yet.
 	// This is to make sure compactor will not accidentally perform compactions with gap instead.
 	ignoreDeletionMarkFilter := block.NewIgnoreDeletionMarkFilter(logger, bkt, deleteDelay/2, conf.blockMetaFetchConcurrency)
-	duplicateBlocksFilter := block.NewDeduplicateFilter()
+	duplicateBlocksFilter := block.NewDeduplicateFilter(conf.blockMetaFetchConcurrency)
 	noCompactMarkerFilter := compact.NewGatherNoCompactionMarkFilter(logger, bkt, conf.blockMetaFetchConcurrency)
 	labelShardedMetaFilter := block.NewLabelShardedMetaFilter(relabelConfig)
 	consistencyDelayMetaFilter := block.NewConsistencyDelayMetaFilter(logger, conf.consistencyDelay, extprom.WrapRegistererWithPrefix("thanos_", reg))
@@ -271,9 +271,10 @@ func runCompact(
 				labelShardedMetaFilter,
 				consistencyDelayMetaFilter,
 				ignoreDeletionMarkFilter,
+				block.NewReplicaLabelRemover(logger, conf.dedupReplicaLabels),
 				duplicateBlocksFilter,
 				noCompactMarkerFilter,
-			}, []block.MetadataModifier{block.NewReplicaLabelRemover(logger, conf.dedupReplicaLabels)},
+			},
 		)
 		cf.UpdateOnChange(func(blocks []metadata.Meta, err error) {
 			compactorView.Set(blocks, err)
@@ -440,7 +441,7 @@ func runCompact(
 			}
 
 			for _, meta := range sy.Metas() {
-				groupKey := compact.DefaultGroupKey(meta.Thanos)
+				groupKey := meta.Thanos.GroupKey()
 				downsampleMetrics.downsamples.WithLabelValues(groupKey)
 				downsampleMetrics.downsampleFailures.WithLabelValues(groupKey)
 			}
@@ -532,7 +533,7 @@ func runCompact(
 
 		// Separate fetcher for global view.
 		// TODO(bwplotka): Allow Bucket UI to visualize the state of the block as well.
-		f := baseMetaFetcher.NewMetaFetcher(extprom.WrapRegistererWithPrefix("thanos_bucket_ui", reg), nil, nil, "component", "globalBucketUI")
+		f := baseMetaFetcher.NewMetaFetcher(extprom.WrapRegistererWithPrefix("thanos_bucket_ui", reg), nil, "component", "globalBucketUI")
 		f.UpdateOnChange(func(blocks []metadata.Meta, err error) {
 			global.Set(blocks, err)
 			api.SetGlobal(blocks, err)

diff --git a/cmd/thanos/downsample.go b/cmd/thanos/downsample.go
@@ -24,7 +24,6 @@ import (
 
 	"github.com/thanos-io/thanos/pkg/block"
 	"github.com/thanos-io/thanos/pkg/block/metadata"
-	"github.com/thanos-io/thanos/pkg/compact"
 	"github.com/thanos-io/thanos/pkg/compact/downsample"
 	"github.com/thanos-io/thanos/pkg/component"
 	"github.com/thanos-io/thanos/pkg/errutil"
@@ -86,8 +85,8 @@ func RunDownsample(
 	}
 
 	metaFetcher, err := block.NewMetaFetcher(logger, block.FetcherConcurrency, bkt, "", extprom.WrapRegistererWithPrefix("thanos_", reg), []block.MetadataFilter{
-		block.NewDeduplicateFilter(),
-	}, nil)
+		block.NewDeduplicateFilter(block.FetcherConcurrency),
+	})
 	if err != nil {
 		return errors.Wrap(err, "create meta fetcher")
 	}
@@ -121,7 +120,7 @@ func RunDownsample(
 			}
 
 			for _, meta := range metas {
-				groupKey := compact.DefaultGroupKey(meta.Thanos)
+				groupKey := meta.Thanos.GroupKey()
 				metrics.downsamples.WithLabelValues(groupKey)
 				metrics.downsampleFailures.WithLabelValues(groupKey)
 			}
@@ -252,10 +251,11 @@ func downsampleBucket(
 					errMsg = "downsampling to 60 min"
 				}
 				if err := processDownsampling(workerCtx, logger, bkt, m, dir, resolution, hashFunc, metrics); err != nil {
-					metrics.downsampleFailures.WithLabelValues(compact.DefaultGroupKey(m.Thanos)).Inc()
+					metrics.downsampleFailures.WithLabelValues(m.Thanos.GroupKey()).Inc()
 					errCh <- errors.Wrap(err, errMsg)
+
 				}
-				metrics.downsamples.WithLabelValues(compact.DefaultGroupKey(m.Thanos)).Inc()
+				metrics.downsamples.WithLabelValues(m.Thanos.GroupKey()).Inc()
 			}
 		}()
 	}
@@ -377,7 +377,7 @@ func processDownsampling(
 	downsampleDuration := time.Since(begin)
 	level.Info(logger).Log("msg", "downsampled block",
 		"from", m.ULID, "to", id, "duration", downsampleDuration, "duration_ms", downsampleDuration.Milliseconds())
-	metrics.downsampleDuration.WithLabelValues(compact.DefaultGroupKey(m.Thanos)).Observe(downsampleDuration.Seconds())
+	metrics.downsampleDuration.WithLabelValues(m.Thanos.GroupKey()).Observe(downsampleDuration.Seconds())
 
 	if err := block.VerifyIndex(logger, filepath.Join(resdir, block.IndexFilename), m.MinTime, m.MaxTime); err != nil {
 		return errors.Wrap(err, "output block index not valid")

diff --git a/cmd/thanos/main_test.go b/cmd/thanos/main_test.go
@@ -22,7 +22,6 @@ import (
 
 	"github.com/thanos-io/thanos/pkg/block"
 	"github.com/thanos-io/thanos/pkg/block/metadata"
-	"github.com/thanos-io/thanos/pkg/compact"
 	"github.com/thanos-io/thanos/pkg/compact/downsample"
 	"github.com/thanos-io/thanos/pkg/objstore"
 	"github.com/thanos-io/thanos/pkg/testutil"
@@ -154,8 +153,8 @@ func TestRegression4960_Deadlock(t *testing.T) {
 	testutil.Ok(t, err)
 
 	metrics := newDownsampleMetrics(prometheus.NewRegistry())
-	testutil.Equals(t, 0.0, promtest.ToFloat64(metrics.downsamples.WithLabelValues(compact.DefaultGroupKey(meta.Thanos))))
-	metaFetcher, err := block.NewMetaFetcher(nil, block.FetcherConcurrency, bkt, "", nil, nil, nil)
+	testutil.Equals(t, 0.0, promtest.ToFloat64(metrics.downsamples.WithLabelValues(meta.Thanos.GroupKey())))
+	metaFetcher, err := block.NewMetaFetcher(nil, block.FetcherConcurrency, bkt, "", nil, nil)
 	testutil.Ok(t, err)
 
 	metas, _, err := metaFetcher.Fetch(ctx)
@@ -194,14 +193,14 @@ func TestCleanupDownsampleCacheFolder(t *testing.T) {
 	testutil.Ok(t, err)
 
 	metrics := newDownsampleMetrics(prometheus.NewRegistry())
-	testutil.Equals(t, 0.0, promtest.ToFloat64(metrics.downsamples.WithLabelValues(compact.DefaultGroupKey(meta.Thanos))))
-	metaFetcher, err := block.NewMetaFetcher(nil, block.FetcherConcurrency, bkt, "", nil, nil, nil)
+	testutil.Equals(t, 0.0, promtest.ToFloat64(metrics.downsamples.WithLabelValues(meta.Thanos.GroupKey())))
+	metaFetcher, err := block.NewMetaFetcher(nil, block.FetcherConcurrency, bkt, "", nil, nil)
 	testutil.Ok(t, err)
 
 	metas, _, err := metaFetcher.Fetch(ctx)
 	testutil.Ok(t, err)
 	testutil.Ok(t, downsampleBucket(ctx, logger, metrics, bkt, metas, dir, 1, metadata.NoneFunc))
-	testutil.Equals(t, 1.0, promtest.ToFloat64(metrics.downsamples.WithLabelValues(compact.DefaultGroupKey(meta.Thanos))))
+	testutil.Equals(t, 1.0, promtest.ToFloat64(metrics.downsamples.WithLabelValues(meta.Thanos.GroupKey())))
 
 	_, err = os.Stat(dir)
 	testutil.Assert(t, os.IsNotExist(err), "index cache dir should not exist at the end of execution")

diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go
@@ -304,8 +304,8 @@ func runStore(
 			block.NewLabelShardedMetaFilter(relabelConfig),
 			block.NewConsistencyDelayMetaFilter(logger, time.Duration(conf.consistencyDelay), extprom.WrapRegistererWithPrefix("thanos_", reg)),
 			ignoreDeletionMarkFilter,
-			block.NewDeduplicateFilter(),
-		}, nil)
+			block.NewDeduplicateFilter(conf.blockMetaFetchConcurrency),
+		})
 	if err != nil {
 		return errors.Wrap(err, "meta fetcher")
 	}