From fc8b489d7cc849b09877548e1afad0ac85050f2b Mon Sep 17 00:00:00 2001 From: Philip Gough Date: Mon, 31 Jul 2023 16:04:38 +0100 Subject: [PATCH 1/2] tools:replicate: Allow configuring min and max compaction range via flags Signed-off-by: Philip Gough --- cmd/thanos/tools_bucket.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/cmd/thanos/tools_bucket.go b/cmd/thanos/tools_bucket.go index 508522a406..9a31444f0e 100644 --- a/cmd/thanos/tools_bucket.go +++ b/cmd/thanos/tools_bucket.go @@ -123,6 +123,8 @@ type bucketWebConfig struct { type bucketReplicateConfig struct { resolutions []time.Duration + compactMin int + compactMax int compactions []int matcherStrs string singleRun bool @@ -207,7 +209,11 @@ func (tbc *bucketWebConfig) registerBucketWebFlag(cmd extkingpin.FlagClause) *bu func (tbc *bucketReplicateConfig) registerBucketReplicateFlag(cmd extkingpin.FlagClause) *bucketReplicateConfig { cmd.Flag("resolution", "Only blocks with these resolutions will be replicated. Repeated flag.").Default("0s", "5m", "1h").HintAction(listResLevel).DurationListVar(&tbc.resolutions) - cmd.Flag("compaction", "Only blocks with these compaction levels will be replicated. Repeated flag.").Default("1", "2", "3", "4").IntsVar(&tbc.compactions) + cmd.Flag("compaction-min", "Only blocks with at least this compaction level will be replicated.").Default("1").IntVar(&tbc.compactMin) + + cmd.Flag("compaction-max", "Only blocks up to a maximum of this compaction level will be replicated.").Default("4").IntVar(&tbc.compactMax) + + cmd.Flag("compaction", "Only blocks with these compaction levels will be replicated. Repeated flag. Overrides compaction-min and compaction-max if set.").Default().IntsVar(&tbc.compactions) cmd.Flag("matcher", "blocks whose external labels match this matcher will be replicated. All Prometheus matchers are supported, including =, !=, =~ and !~.").StringVar(&tbc.matcherStrs) @@ -721,6 +727,16 @@ func registerBucketReplicate(app extkingpin.AppClause, objStoreConfig *extflag.P resolutionLevels = append(resolutionLevels, compact.ResolutionLevel(lvl.Milliseconds())) } + if len(tbc.compactions) == 0 { + if tbc.compactMin > tbc.compactMax { + return errors.New("compaction-min must be less than or equal to compaction-max") + } + tbc.compactions = []int{} + for compactionLevel := tbc.compactMin; compactionLevel <= tbc.compactMax; compactionLevel++ { + tbc.compactions = append(tbc.compactions, compactionLevel) + } + } + blockIDs := make([]ulid.ULID, 0, len(*ids)) for _, id := range *ids { bid, err := ulid.Parse(id) From c042a8aff3cdbc71c37cf91c3afe345519be8fde Mon Sep 17 00:00:00 2001 From: Philip Gough Date: Mon, 31 Jul 2023 16:13:42 +0100 Subject: [PATCH 2/2] docs: Generate docs and add changelog Signed-off-by: Philip Gough --- CHANGELOG.md | 3 ++- docs/components/tools.md | 10 ++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d5910deb72..dfdc1dd8c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#6369](https://github.com/thanos-io/thanos/pull/6369) Receive: add az-aware replication support for Ketama algorithm - [#6185](https://github.com/thanos-io/thanos/pull/6185) Tracing: tracing in OTLP support configuring service_name. - [#6192](https://github.com/thanos-io/thanos/pull/6192) Store: add flag `bucket-web-label` to select the label to use as timeline title in web UI -- [#6167](https://github.com/thanos-io/thanos/pull/6195) Receive: add flag `tsdb.too-far-in-future.time-window` to prevent clock skewed samples to pollute TSDB head and block all valid incoming samples. +- [#6195](https://github.com/thanos-io/thanos/pull/6195) Receive: add flag `tsdb.too-far-in-future.time-window` to prevent clock skewed samples to pollute TSDB head and block all valid incoming samples. - [#6273](https://github.com/thanos-io/thanos/pull/6273) Mixin: Allow specifying an instance name filter in dashboards - [#6163](https://github.com/thanos-io/thanos/pull/6163) Receiver: Add hidden flag `--receive-forward-max-backoff` to configure the max backoff for forwarding requests. - [#5777](https://github.com/thanos-io/thanos/pull/5777) Receive: Allow specifying tenant-specific external labels in Router Ingestor. @@ -27,6 +27,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#6467](https://github.com/thanos-io/thanos/pull/6467) Mixin (Receive): add alert for tenant reaching head series limit. - [#6528](https://github.com/thanos-io/thanos/pull/6528) Index Cache: Add histogram metric `thanos_store_index_cache_stored_data_size_bytes` for item size. - [#6560](https://github.com/thanos-io/thanos/pull/6560) Thanos ruler: add flag to optionally disable adding Thanos params when querying metrics +- [#6574](https://github.com/thanos-io/thanos/pull/6574) Tools: Add min and max compactions range flags to `bucket replicate` command. ### Fixed - [#6503](https://github.com/thanos-io/thanos/pull/6503) *: Change the engine behind `ContentPathReloader` to be completely independent of any filesystem concept. This effectively fixes this configuration reload when used with Kubernetes ConfigMaps, Secrets, or other volume mounts. diff --git a/docs/components/tools.md b/docs/components/tools.md index e394dc0148..3c552bf3a9 100644 --- a/docs/components/tools.md +++ b/docs/components/tools.md @@ -507,8 +507,14 @@ Replicate data from one object storage to another. NOTE: Currently it works only with Thanos blocks (meta.json has to have Thanos metadata). Flags: - --compaction=1... ... Only blocks with these compaction levels will be - replicated. Repeated flag. + --compaction=COMPACTION ... + Only blocks with these compaction levels + will be replicated. Repeated flag. Overrides + compaction-min and compaction-max if set. + --compaction-max=4 Only blocks up to a maximum of this compaction + level will be replicated. + --compaction-min=1 Only blocks with at least this compaction level + will be replicated. -h, --help Show context-sensitive help (also try --help-long and --help-man). --http-address="0.0.0.0:10902"