From f04f5dbade0c63bf22779e7eb38a0f384c067512 Mon Sep 17 00:00:00 2001 From: Jon Kartago Lamida Date: Fri, 21 Jul 2023 11:12:21 +0800 Subject: [PATCH 01/20] Add lazy loaded trackers that will track and persist list of lazy loaded block in store gateway (#5095) * Add lazy loaded trackers of store gateway index headers Signed-off-by: Jon Kartago Lamida * Add period to persist list of lazy loaded blocks Signed-off-by: Jon Kartago Lamida * Track only lazy loaded block during persist time Signed-off-by: Jon Kartago Lamida * Use protobuf to persist the tracker state of lazy loaded block Signed-off-by: Jon Kartago Lamida * Add proto suffix for the binary file Signed-off-by: Jon Kartago Lamida * Rename protobuf name Signed-off-by: Jon Kartago Lamida * Initiate HeaderLazyLoadedTracker in bucket per tenant Signed-off-by: Jon Kartago Lamida * Fix outdated constructor in test Signed-off-by: Jon Kartago Lamida * Remove unused LazyLoaderTracker in binary_reader Signed-off-by: Jon Kartago Lamida * Remove unnecessary newline Signed-off-by: Jon Kartago Lamida * Fix HeadersLazyLoadedTracker receiver must be pointer Signed-off-by: Jon Kartago Lamida * Small grammar correction in comment Signed-off-by: Jon Kartago Lamida * Rename constant to lazyLoadedFile Signed-off-by: Jon Kartago Lamida * Use hardcoded 1 minute to persist list of lazyLoadedBlock Signed-off-by: Jon Kartago Lamida * Add layzLoaded tracker test Signed-off-by: Jon Kartago Lamida * Add logger when lazyLoadedTracker file to persist Signed-off-by: Jon Kartago Lamida * Fix linter Signed-off-by: Jon Kartago Lamida * Update pkg/storegateway/indexheader/reader_pool.go Co-authored-by: Dimitar Dimitrov * Update pkg/storegateway/indexheader/reader_pool.go Co-authored-by: Dimitar Dimitrov * Update pkg/storegateway/indexheader/reader_pool_test.go Co-authored-by: Dimitar Dimitrov * Update pkg/storegateway/indexheader/reader_pool_test.go Co-authored-by: Dimitar Dimitrov * Unexport lazyLoadedTracker method Signed-off-by: Jon Kartago Lamida * Try to set checksum Signed-off-by: Jon Kartago Lamida * Update pkg/storegateway/indexheader/reader_pool.go Co-authored-by: Oleg Zaytsev * Use one goroutine for closeIdleReader and lazyLoad persist Signed-off-by: Jon Kartago Lamida * Use pb suffix in test Signed-off-by: Jon Kartago Lamida * Refactor the structure of tracker Signed-off-by: Jon Kartago Lamida * Clean lint Signed-off-by: Jon Kartago Lamida * Fix test Signed-off-by: Jon Kartago Lamida * Fix test Signed-off-by: Jon Kartago Lamida * Use fsync Signed-off-by: Jon Kartago Lamida * Return err Signed-off-by: Jon Kartago Lamida * Use json Signed-off-by: Jon Kartago Lamida * Update pkg/storegateway/indexheader/reader_pool_test.go Co-authored-by: Dimitar Dimitrov * Remove defer and use t.CleanUp Signed-off-by: Jon Kartago Lamida * Fix linter Signed-off-by: Jon Kartago Lamida * Still fixing linter Signed-off-by: Jon Kartago Lamida * Revert idle timeout in test Signed-off-by: Jon Kartago Lamida * Update pkg/storegateway/indexheader/reader_pool_test.go Co-authored-by: Dimitar Dimitrov * Revert newline Signed-off-by: Jon Kartago Lamida * Revert newline Signed-off-by: Jon Kartago Lamida * Revert newline Signed-off-by: Jon Kartago Lamida * Apply PR feedbacks Signed-off-by: Jon Kartago Lamida * Read the file to assert the lazyloaded heaeder Signed-off-by: Jon Kartago Lamida * Don't use /tmp directory for lazyloaded header Signed-off-by: Jon Kartago Lamida * Fix test Signed-off-by: Jon Kartago Lamida * Update pkg/storegateway/indexheader/reader_pool.go Co-authored-by: Dimitar Dimitrov * Update pkg/storegateway/indexheader/reader_pool.go Co-authored-by: Dimitar Dimitrov * Update pkg/storegateway/indexheader/reader_pool.go Co-authored-by: Charles Korn * Apply PR suggestion Signed-off-by: Jon Kartago Lamida * Extract fsync operation Signed-off-by: Jon Kartago Lamida * Add assertion on LoadedBlocks Signed-off-by: Jon Kartago Lamida * Update comment Signed-off-by: Jon Kartago Lamida * Add more proper LoadedBlock test Signed-off-by: Jon Kartago Lamida * Add license header Signed-off-by: Jon Kartago Lamida * Update fsync method name Signed-off-by: Jon Kartago Lamida * Update pkg/storegateway/indexheader/reader_pool_test.go Co-authored-by: Charles Korn * Update pkg/storegateway/indexheader/reader_pool_test.go Co-authored-by: Charles Korn * Address PR feedbacks Signed-off-by: Jon Kartago Lamida * Update method comment Signed-off-by: Jon Kartago Lamida * Update method comment Signed-off-by: Jon Kartago Lamida * Update pkg/storegateway/indexheader/reader_pool_test.go Co-authored-by: Charles Korn * Add comments to explain why we use os.Rename Signed-off-by: Jon Kartago Lamida * Remove unrelevant test Signed-off-by: Jon Kartago Lamida * Add assertion to make sure there is only one lazyReader Signed-off-by: Jon Kartago Lamida * Use io.Reader for fsync data Signed-off-by: Jon Kartago Lamida * Ramove table test for a single test case only Signed-off-by: Jon Kartago Lamida * Update pkg/util/atomicfs/fsync_test.go Co-authored-by: Marco Pracucci * Apply PR suggestions Signed-off-by: Jon Kartago Lamida * Fix test Signed-off-by: Jon Kartago Lamida * Update pkg/storegateway/indexheader/reader_pool.go Co-authored-by: Charles Korn * Update pkg/util/atomicfs/fsync_test.go Co-authored-by: Charles Korn * Update pkg/storegateway/indexheader/reader_pool_test.go Co-authored-by: Charles Korn * Clean import Signed-off-by: Jon Kartago Lamida --------- Signed-off-by: Jon Kartago Lamida Co-authored-by: Dimitar Dimitrov Co-authored-by: Oleg Zaytsev Co-authored-by: Charles Korn Co-authored-by: Marco Pracucci --- pkg/storegateway/indexheader/reader_pool_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/storegateway/indexheader/reader_pool_test.go b/pkg/storegateway/indexheader/reader_pool_test.go index b5be645e8f6..4019d7aecbb 100644 --- a/pkg/storegateway/indexheader/reader_pool_test.go +++ b/pkg/storegateway/indexheader/reader_pool_test.go @@ -20,6 +20,8 @@ import ( promtestutil "github.com/prometheus/client_golang/prometheus/testutil" "github.com/prometheus/prometheus/model/labels" "github.com/stretchr/testify/require" + "go.uber.org/atomic" + "github.com/thanos-io/objstore/providers/filesystem" "go.uber.org/atomic" From 7a7ec39d63536d86e82d7afb6b54a79086087588 Mon Sep 17 00:00:00 2001 From: Jon Kartago Lamida Date: Fri, 11 Aug 2023 14:59:42 +0800 Subject: [PATCH 02/20] store-gateway: Eagerly Load Lazy Loaded Index Header (#5596) * Start to do eager load of index header during startup Signed-off-by: Jon Kartago Lamida * Make EagerLoading as public method Signed-off-by: Jon Kartago Lamida * Add eager loading test Signed-off-by: Jon Kartago Lamida * Small refactoring of test Signed-off-by: Jon Kartago Lamida * Add godoc Signed-off-by: Jon Kartago Lamida * Update reader_pool_test with eager loading logic Signed-off-by: Jon Kartago Lamida * Small refactoring to make test clearer Signed-off-by: Jon Kartago Lamida * Appease the linter Signed-off-by: Jon Kartago Lamida * Rename private method for test Signed-off-by: Jon Kartago Lamida * Add config to enable or disable index headers eager loading Signed-off-by: Jon Kartago Lamida * Changelog Signed-off-by: Jon Kartago Lamida * Changelog Signed-off-by: Jon Kartago Lamida * Go select can't be inside method Signed-off-by: Jon Kartago Lamida * Run make doc Signed-off-by: Jon Kartago Lamida * Fix test Signed-off-by: Jon Kartago Lamida * Update CHANGELOG.md Co-authored-by: Charles Korn * Update pkg/storegateway/indexheader/lazy_binary_reader.go Co-authored-by: Charles Korn * Apply configs PR feedbacks Signed-off-by: Jon Kartago Lamida * Pass BucketStoreConfig as argument Signed-off-by: Jon Kartago Lamida * Pass eagerLoadIndexReaderEnabled into snapshotConfig Signed-off-by: Jon Kartago Lamida * Don't pass lazyLoadedSnapshot into LazyBinaryReader Signed-off-by: Jon Kartago Lamida * Only initialize tickerLazyLoad when flag is enabled Signed-off-by: Jon Kartago Lamida * Apply PR feedbacks Signed-off-by: Jon Kartago Lamida * Update config description Signed-off-by: Jon Kartago Lamida * Update usedAt to current time in eager loading Signed-off-by: Jon Kartago Lamida * Update pkg/storegateway/indexheader/reader_pool.go Co-authored-by: Dimitar Dimitrov * Fix snapshotByte name Signed-off-by: Jon Kartago Lamida * Address PR feedbacks Signed-off-by: Jon Kartago Lamida * Rename initLazyBinaryForTest Signed-off-by: Jon Kartago Lamida * Apply PR suggestion Signed-off-by: Jon Kartago Lamida * Refactor test Signed-off-by: Jon Kartago Lamida * Fix comment Signed-off-by: Jon Kartago Lamida * Apply more PR suggestions Signed-off-by: Jon Kartago Lamida * Refactor configs Signed-off-by: Jon Kartago Lamida * Header eager loading only must be done during initialSync Signed-off-by: Jon Kartago Lamida * Remove unnecesary comment Signed-off-by: Jon Kartago Lamida * Remove snapshot file regardless snapshot loading result Signed-off-by: Jon Kartago Lamida * Fix compilation error Signed-off-by: Jon Kartago Lamida * Update pkg/storegateway/indexheader/header.go Co-authored-by: Charles Korn * Regenerate doc for updated command line args Signed-off-by: Jon Kartago Lamida * Update error log Signed-off-by: Jon Kartago Lamida * Update comment Signed-off-by: Jon Kartago Lamida * Update comment Signed-off-by: Jon Kartago Lamida * Update test Signed-off-by: Jon Kartago Lamida * Add test case Signed-off-by: Jon Kartago Lamida * Update pkg/storegateway/indexheader/reader_pool.go Co-authored-by: Charles Korn * Update pkg/storegateway/indexheader/reader_pool_test.go Co-authored-by: Charles Korn * Update pkg/storegateway/indexheader/reader_pool_test.go Co-authored-by: Charles Korn * Update pkg/storegateway/indexheader/lazy_binary_reader.go Co-authored-by: Charles Korn * Update pkg/storegateway/indexheader/reader_pool.go Co-authored-by: Charles Korn * Update pkg/storegateway/indexheader/header.go Co-authored-by: Charles Korn * Update changelog Signed-off-by: Jon Kartago Lamida --------- Signed-off-by: Jon Kartago Lamida Co-authored-by: Charles Korn Co-authored-by: Dimitar Dimitrov --- pkg/storegateway/indexheader/reader_pool.go | 1 + pkg/storegateway/indexheader/reader_pool_test.go | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/storegateway/indexheader/reader_pool.go b/pkg/storegateway/indexheader/reader_pool.go index 4faa2cd9e6b..eb87e0131a5 100644 --- a/pkg/storegateway/indexheader/reader_pool.go +++ b/pkg/storegateway/indexheader/reader_pool.go @@ -48,6 +48,7 @@ func NewReaderPoolMetrics(reg prometheus.Registerer) *ReaderPoolMetrics { type ReaderPool struct { lazyReaderEnabled bool lazyReaderIdleTimeout time.Duration + eagerLoadReaderEnabled bool sparsePersistenceEnabled bool logger log.Logger metrics *ReaderPoolMetrics diff --git a/pkg/storegateway/indexheader/reader_pool_test.go b/pkg/storegateway/indexheader/reader_pool_test.go index 4019d7aecbb..0f962453d94 100644 --- a/pkg/storegateway/indexheader/reader_pool_test.go +++ b/pkg/storegateway/indexheader/reader_pool_test.go @@ -23,7 +23,6 @@ import ( "go.uber.org/atomic" "github.com/thanos-io/objstore/providers/filesystem" - "go.uber.org/atomic" "github.com/grafana/mimir/pkg/storage/tsdb/block" ) From 867b24cdada9f5ad7265ee8b0ee91e9f6edd5b02 Mon Sep 17 00:00:00 2001 From: Jon Kartago Lamida Date: Fri, 11 Aug 2023 16:37:14 +0800 Subject: [PATCH 03/20] Fix golangci-lint Signed-off-by: Jon Kartago Lamida --- pkg/storegateway/indexheader/reader_pool_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/storegateway/indexheader/reader_pool_test.go b/pkg/storegateway/indexheader/reader_pool_test.go index 0f962453d94..b5be645e8f6 100644 --- a/pkg/storegateway/indexheader/reader_pool_test.go +++ b/pkg/storegateway/indexheader/reader_pool_test.go @@ -20,9 +20,8 @@ import ( promtestutil "github.com/prometheus/client_golang/prometheus/testutil" "github.com/prometheus/prometheus/model/labels" "github.com/stretchr/testify/require" - "go.uber.org/atomic" - "github.com/thanos-io/objstore/providers/filesystem" + "go.uber.org/atomic" "github.com/grafana/mimir/pkg/storage/tsdb/block" ) From d6c1aaa1f7343e989f61c2a478ec4b4ad37ff303 Mon Sep 17 00:00:00 2001 From: Jon Kartago Lamida Date: Mon, 14 Aug 2023 13:02:17 +0800 Subject: [PATCH 04/20] Move indexheader config to its own package Signed-off-by: Jon Kartago Lamida --- cmd/mimir/config-descriptor.json | 44 ++++++++++ cmd/mimir/help-all.txt.tmpl | 8 ++ .../configuration-parameters/index.md | 22 +++++ pkg/storage/tsdb/config.go | 6 +- pkg/storegateway/bucket.go | 2 +- pkg/storegateway/bucket_e2e_test.go | 8 +- pkg/storegateway/bucket_stores_test.go | 4 +- pkg/storegateway/bucket_test.go | 84 ++++++++++--------- pkg/storegateway/indexheader/header.go | 26 ++++-- pkg/storegateway/indexheader/reader_pool.go | 16 ++-- .../indexheader/reader_pool_test.go | 29 +++++-- 11 files changed, 180 insertions(+), 69 deletions(-) diff --git a/cmd/mimir/config-descriptor.json b/cmd/mimir/config-descriptor.json index 7fec86adf13..e4aa5d88255 100644 --- a/cmd/mimir/config-descriptor.json +++ b/cmd/mimir/config-descriptor.json @@ -7643,6 +7643,50 @@ "fieldType": "boolean", "fieldCategory": "experimental" }, + { + "kind": "field", + "name": "index_header_lazy_loading_enabled", + "required": false, + "desc": "If enabled, store-gateway will lazy load an index-header only once required by a query.", + "fieldValue": null, + "fieldDefaultValue": true, + "fieldFlag": "blocks-storage.bucket-store.index-header.lazy-loading-enabled", + "fieldType": "boolean", + "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "index_header_lazy_loading_idle_timeout", + "required": false, + "desc": "If index-header lazy loading is enabled and this setting is \u003e 0, the store-gateway will offload unused index-headers after 'idle timeout' inactivity.", + "fieldValue": null, + "fieldDefaultValue": 3600000000000, + "fieldFlag": "blocks-storage.bucket-store.index-header.lazy-loading-idle-timeout", + "fieldType": "duration", + "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "index_header_lazy_loading_concurrency", + "required": false, + "desc": "Maximum number of concurrent index header loads across all tenants. If set to 0, concurrency is unlimited.", + "fieldValue": null, + "fieldDefaultValue": 0, + "fieldFlag": "blocks-storage.bucket-store.index-header.lazy-loading-concurrency", + "fieldType": "int", + "fieldCategory": "experimental" + }, + { + "kind": "field", + "name": "index_header_sparse_persistence_enabled", + "required": false, + "desc": "If enabled, store-gateway will persist a sparse version of the index-header to disk on construction and load sparse index-headers from disk instead of the whole index-header.", + "fieldValue": null, + "fieldDefaultValue": false, + "fieldFlag": "blocks-storage.bucket-store.index-header.sparse-persistence-enabled", + "fieldType": "boolean", + "fieldCategory": "experimental" + }, { "kind": "field", "name": "verify_on_load", diff --git a/cmd/mimir/help-all.txt.tmpl b/cmd/mimir/help-all.txt.tmpl index fde19c345f4..87af68a635b 100644 --- a/cmd/mimir/help-all.txt.tmpl +++ b/cmd/mimir/help-all.txt.tmpl @@ -509,8 +509,16 @@ Usage of ./cmd/mimir/mimir: [experimental] If enabled, store-gateway will persist a sparse version of the index-header to disk on construction and load sparse index-headers from disk instead of the whole index-header. -blocks-storage.bucket-store.index-header.eager-loading-startup-enabled [experimental] If enabled, store-gateway will periodically persist block IDs of lazy loaded index-headers and load them eagerly during startup. It is not valid to enable this if index-header lazy loading is disabled. + -blocks-storage.bucket-store.index-header.lazy-loading-concurrency int + [experimental] Maximum number of concurrent index header loads across all tenants. If set to 0, concurrency is unlimited. + -blocks-storage.bucket-store.index-header.lazy-loading-enabled + If enabled, store-gateway will lazy load an index-header only once required by a query. (default true) + -blocks-storage.bucket-store.index-header.lazy-loading-idle-timeout duration + If index-header lazy loading is enabled and this setting is > 0, the store-gateway will offload unused index-headers after 'idle timeout' inactivity. (default 1h0m0s) -blocks-storage.bucket-store.index-header.max-idle-file-handles uint Maximum number of idle file handles the store-gateway keeps open for each index-header file. (default 1) + -blocks-storage.bucket-store.index-header.sparse-persistence-enabled + [experimental] If enabled, store-gateway will persist a sparse version of the index-header to disk on construction and load sparse index-headers from disk instead of the whole index-header. -blocks-storage.bucket-store.index-header.verify-on-load If true, verify the checksum of index headers upon loading them (either on startup or lazily when lazy loading is enabled). Setting to true helps detect disk corruption at the cost of slowing down index header loading. -blocks-storage.bucket-store.max-chunk-pool-bytes uint diff --git a/docs/sources/mimir/references/configuration-parameters/index.md b/docs/sources/mimir/references/configuration-parameters/index.md index 91731718e88..b089a5e3f31 100644 --- a/docs/sources/mimir/references/configuration-parameters/index.md +++ b/docs/sources/mimir/references/configuration-parameters/index.md @@ -3449,6 +3449,28 @@ bucket_store: # CLI flag: -blocks-storage.bucket-store.index-header.eager-loading-startup-enabled [eager_loading_startup_enabled: | default = false] + # (advanced) If enabled, store-gateway will lazy load an index-header only + # once required by a query. + # CLI flag: -blocks-storage.bucket-store.index-header.lazy-loading-enabled + [index_header_lazy_loading_enabled: | default = true] + + # (advanced) If index-header lazy loading is enabled and this setting is > + # 0, the store-gateway will offload unused index-headers after 'idle + # timeout' inactivity. + # CLI flag: -blocks-storage.bucket-store.index-header.lazy-loading-idle-timeout + [index_header_lazy_loading_idle_timeout: | default = 1h] + + # (experimental) Maximum number of concurrent index header loads across all + # tenants. If set to 0, concurrency is unlimited. + # CLI flag: -blocks-storage.bucket-store.index-header.lazy-loading-concurrency + [index_header_lazy_loading_concurrency: | default = 0] + + # (experimental) If enabled, store-gateway will persist a sparse version of + # the index-header to disk on construction and load sparse index-headers + # from disk instead of the whole index-header. + # CLI flag: -blocks-storage.bucket-store.index-header.sparse-persistence-enabled + [index_header_sparse_persistence_enabled: | default = false] + # (advanced) If true, verify the checksum of index headers upon loading them # (either on startup or lazily when lazy loading is enabled). Setting to # true helps detect disk corruption at the cost of slowing down index header diff --git a/pkg/storage/tsdb/config.go b/pkg/storage/tsdb/config.go index 2589fadcf13..06bc13d5e1e 100644 --- a/pkg/storage/tsdb/config.go +++ b/pkg/storage/tsdb/config.go @@ -388,6 +388,7 @@ type BucketStoreConfig struct { // Series hash cache. SeriesHashCacheMaxBytes uint64 `yaml:"series_hash_cache_max_size_bytes" category:"advanced"` + // TODO JON REMOVE // Controls whether index-header lazy loading is enabled. IndexHeaderLazyLoadingEnabled bool `yaml:"index_header_lazy_loading_enabled" category:"advanced"` IndexHeaderLazyLoadingIdleTimeout time.Duration `yaml:"index_header_lazy_loading_idle_timeout" category:"advanced"` @@ -397,6 +398,7 @@ type BucketStoreConfig struct { // Controls whether persisting a sparse version of the index-header to disk is enabled. IndexHeaderSparsePersistenceEnabled bool `yaml:"index_header_sparse_persistence_enabled" category:"experimental"` + // TODO JON REMOVE END // Controls the partitioner, used to aggregate multiple GET object API requests. PartitionerMaxGapBytes uint64 `yaml:"partitioner_max_gap_bytes" category:"advanced"` @@ -455,10 +457,12 @@ func (cfg *BucketStoreConfig) RegisterFlags(f *flag.FlagSet) { "The idea of ignore-deletion-marks-delay is to ignore blocks that are marked for deletion with some delay. This ensures store can still serve blocks that are meant to be deleted but do not have a replacement yet.") f.DurationVar(&cfg.IgnoreBlocksWithin, "blocks-storage.bucket-store.ignore-blocks-within", 10*time.Hour, "Blocks with minimum time within this duration are ignored, and not loaded by store-gateway. Useful when used together with -querier.query-store-after to prevent loading young blocks, because there are usually many of them (depending on number of ingesters) and they are not yet compacted. Negative values or 0 disable the filter.") f.IntVar(&cfg.PostingOffsetsInMemSampling, "blocks-storage.bucket-store.posting-offsets-in-mem-sampling", DefaultPostingOffsetInMemorySampling, "Controls what is the ratio of postings offsets that the store will hold in memory.") + // TODO JON REMOVE f.BoolVar(&cfg.IndexHeaderLazyLoadingEnabled, "blocks-storage.bucket-store.index-header-lazy-loading-enabled", true, "If enabled, store-gateway will lazy load an index-header only once required by a query.") f.DurationVar(&cfg.IndexHeaderLazyLoadingIdleTimeout, "blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout", 60*time.Minute, "If index-header lazy loading is enabled and this setting is > 0, the store-gateway will offload unused index-headers after 'idle timeout' inactivity.") f.IntVar(&cfg.IndexHeaderLazyLoadingConcurrency, "blocks-storage.bucket-store.index-header-lazy-loading-concurrency", 0, "Maximum number of concurrent index header loads across all tenants. If set to 0, concurrency is unlimited.") f.BoolVar(&cfg.IndexHeaderSparsePersistenceEnabled, "blocks-storage.bucket-store.index-header-sparse-persistence-enabled", false, "If enabled, store-gateway will persist a sparse version of the index-header to disk on construction and load sparse index-headers from disk instead of the whole index-header.") + // TODO JON REMOVE END f.Uint64Var(&cfg.PartitionerMaxGapBytes, "blocks-storage.bucket-store.partitioner-max-gap-bytes", DefaultPartitionerMaxGapSize, "Max size - in bytes - of a gap for which the partitioner aggregates together two bucket GET object requests.") f.IntVar(&cfg.StreamingBatchSize, "blocks-storage.bucket-store.batch-series-size", 5000, "This option controls how many series to fetch per batch. The batch size must be greater than 0.") f.IntVar(&cfg.ChunkRangesPerSeries, "blocks-storage.bucket-store.fine-grained-chunks-caching-ranges-per-series", 1, "This option controls into how many ranges the chunks of each series from each block are split. This value is effectively the number of chunks cache items per series per block when -blocks-storage.bucket-store.chunks-cache.fine-grained-chunks-caching-enabled is enabled.") @@ -498,7 +502,7 @@ func (cfg *BucketStoreConfig) Validate(logger log.Logger) error { if cfg.SeriesSelectionStrategyName == WorstCasePostingsStrategy && cfg.SelectionStrategies.WorstCaseSeriesPreference <= 0 { return errors.New("invalid worst-case series preference; must be positive") } - if err := cfg.IndexHeader.Validate(cfg.IndexHeaderLazyLoadingEnabled); err != nil { + if err := cfg.IndexHeader.Validate(); err != nil { return errors.Wrap(err, "index-header configuration") } if cfg.IndexHeaderLazyLoadingConcurrency < 0 { diff --git a/pkg/storegateway/bucket.go b/pkg/storegateway/bucket.go index 891acc9c96a..0b70f4f48a4 100644 --- a/pkg/storegateway/bucket.go +++ b/pkg/storegateway/bucket.go @@ -272,7 +272,7 @@ func NewBucketStore( EagerLoadingEnabled: bucketStoreConfig.IndexHeader.IndexHeaderEagerLoadingStartupEnabled, } // Depend on the options - s.indexReaderPool = indexheader.NewReaderPool(s.logger, bucketStoreConfig.IndexHeaderLazyLoadingEnabled, bucketStoreConfig.IndexHeaderLazyLoadingIdleTimeout, bucketStoreConfig.IndexHeaderSparsePersistenceEnabled, s.lazyLoadingGate, metrics.indexHeaderReaderMetrics, lazyLoadedSnapshotConfig) + s.indexReaderPool = indexheader.NewReaderPool(s.logger, bucketStoreConfig.IndexHeader, s.lazyLoadingGate, metrics.indexHeaderReaderMetrics, lazyLoadedSnapshotConfig) if err := os.MkdirAll(dir, 0750); err != nil { return nil, errors.Wrap(err, "create dir") diff --git a/pkg/storegateway/bucket_e2e_test.go b/pkg/storegateway/bucket_e2e_test.go index 452edfd7889..a98509bee90 100644 --- a/pkg/storegateway/bucket_e2e_test.go +++ b/pkg/storegateway/bucket_e2e_test.go @@ -203,11 +203,11 @@ func prepareStoreWithTestBlocks(t testing.TB, bkt objstore.Bucket, cfg *prepareS BlockSyncConcurrency: 20, PostingOffsetsInMemSampling: mimir_tsdb.DefaultPostingOffsetInMemorySampling, IndexHeader: indexheader.Config{ - IndexHeaderEagerLoadingStartupEnabled: true, + EagerLoadingStartupEnabled: true, + LazyLoadingEnabled: true, + LazyLoadingIdleTimeout: time.Minute, + SparsePersistenceEnabled: true, }, - IndexHeaderLazyLoadingEnabled: true, - IndexHeaderLazyLoadingIdleTimeout: time.Minute, - IndexHeaderSparsePersistenceEnabled: true, }, cfg.postingsStrategy, cfg.chunksLimiterFactory, diff --git a/pkg/storegateway/bucket_stores_test.go b/pkg/storegateway/bucket_stores_test.go index a7fb4421a42..06d97973879 100644 --- a/pkg/storegateway/bucket_stores_test.go +++ b/pkg/storegateway/bucket_stores_test.go @@ -403,8 +403,8 @@ func testBucketStoresSeriesShouldCorrectlyQuerySeriesSpanningMultipleChunks(t *t ctx := context.Background() cfg := prepareStorageConfig(t) - cfg.BucketStore.IndexHeaderLazyLoadingEnabled = lazyLoadingEnabled - cfg.BucketStore.IndexHeaderLazyLoadingIdleTimeout = time.Minute + cfg.BucketStore.IndexHeader.LazyLoadingEnabled = lazyLoadingEnabled + cfg.BucketStore.IndexHeader.LazyLoadingIdleTimeout = time.Minute storageDir := t.TempDir() diff --git a/pkg/storegateway/bucket_test.go b/pkg/storegateway/bucket_test.go index b5671d1cb7a..17611ae5346 100644 --- a/pkg/storegateway/bucket_test.go +++ b/pkg/storegateway/bucket_test.go @@ -1486,11 +1486,11 @@ func benchBucketSeries(t test.TB, skipChunk bool, samplesPerSeries, totalSeries BlockSyncConcurrency: 1, PostingOffsetsInMemSampling: mimir_tsdb.DefaultPostingOffsetInMemorySampling, IndexHeader: indexheader.Config{ - IndexHeaderEagerLoadingStartupEnabled: false, + EagerLoadingStartupEnabled: false, + LazyLoadingEnabled: false, + LazyLoadingIdleTimeout: 0, + SparsePersistenceEnabled: true, }, - IndexHeaderLazyLoadingEnabled: false, - IndexHeaderLazyLoadingIdleTimeout: 0, - IndexHeaderSparsePersistenceEnabled: true, }, selectAllStrategy{}, newStaticChunksLimiterFactory(0), @@ -1616,11 +1616,11 @@ func TestBucketStore_Series_Concurrency(t *testing.T) { BlockSyncConcurrency: 1, PostingOffsetsInMemSampling: mimir_tsdb.DefaultPostingOffsetInMemorySampling, IndexHeader: indexheader.Config{ - IndexHeaderEagerLoadingStartupEnabled: false, + EagerLoadingStartupEnabled: false, + LazyLoadingEnabled: false, + LazyLoadingIdleTimeout: 0, + SparsePersistenceEnabled: true, }, - IndexHeaderLazyLoadingEnabled: false, - IndexHeaderLazyLoadingIdleTimeout: 0, - IndexHeaderSparsePersistenceEnabled: true, }, selectAllStrategy{}, newStaticChunksLimiterFactory(0), @@ -1768,14 +1768,18 @@ func TestBucketStore_Series_OneBlock_InMemIndexCacheSegfault(t *testing.T) { } store := &BucketStore{ - userID: "test", - bkt: objstore.WithNoopInstr(bkt), - logger: logger, - indexCache: indexCache, - chunksCache: chunkscache.NoopCache{}, - indexReaderPool: indexheader.NewReaderPool(log.NewNopLogger(), false, 0, true, gate.NewNoop(), indexheader.NewReaderPoolMetrics(nil), indexheader.LazyLoadedHeadersSnapshotConfig{}), - metrics: NewBucketStoreMetrics(nil), - blockSet: &bucketBlockSet{blocks: []*bucketBlock{b1, b2}}, + userID: "test", + bkt: objstore.WithNoopInstr(bkt), + logger: logger, + indexCache: indexCache, + chunksCache: chunkscache.NoopCache{}, + indexReaderPool: indexheader.NewReaderPool(log.NewNopLogger(), indexheader.Config{ + LazyLoadingEnabled: false, + LazyLoadingIdleTimeout: 0, + SparsePersistenceEnabled: true, + }, gate.NewNoop(), indexheader.NewReaderPoolMetrics(nil), indexheader.LazyLoadedHeadersSnapshotConfig{}), + metrics: NewBucketStoreMetrics(nil), + blockSet: &bucketBlockSet{blocks: []*bucketBlock{b1, b2}}, blocks: map[ulid.ULID]*bucketBlock{ b1.meta.ULID: b1, b2.meta.ULID: b2, @@ -1940,11 +1944,11 @@ func TestBucketStore_Series_ErrorUnmarshallingRequestHints(t *testing.T) { BlockSyncConcurrency: 10, PostingOffsetsInMemSampling: mimir_tsdb.DefaultPostingOffsetInMemorySampling, IndexHeader: indexheader.Config{ - IndexHeaderEagerLoadingStartupEnabled: false, + EagerLoadingStartupEnabled: false, + LazyLoadingEnabled: false, + LazyLoadingIdleTimeout: 0, + SparsePersistenceEnabled: true, }, - IndexHeaderLazyLoadingEnabled: false, - IndexHeaderLazyLoadingIdleTimeout: 0, - IndexHeaderSparsePersistenceEnabled: true, }, selectAllStrategy{}, newStaticChunksLimiterFactory(10000/MaxSamplesPerChunk), @@ -2000,11 +2004,11 @@ func TestBucketStore_Series_CanceledRequest(t *testing.T) { BlockSyncConcurrency: 10, PostingOffsetsInMemSampling: mimir_tsdb.DefaultPostingOffsetInMemorySampling, IndexHeader: indexheader.Config{ - IndexHeaderEagerLoadingStartupEnabled: false, + EagerLoadingStartupEnabled: false, + LazyLoadingEnabled: false, + LazyLoadingIdleTimeout: 0, + SparsePersistenceEnabled: true, }, - IndexHeaderLazyLoadingEnabled: false, - IndexHeaderLazyLoadingIdleTimeout: 0, - IndexHeaderSparsePersistenceEnabled: true, }, selectAllStrategy{}, newStaticChunksLimiterFactory(10000/MaxSamplesPerChunk), @@ -2067,11 +2071,11 @@ func TestBucketStore_Series_InvalidRequest(t *testing.T) { BlockSyncConcurrency: 10, PostingOffsetsInMemSampling: mimir_tsdb.DefaultPostingOffsetInMemorySampling, IndexHeader: indexheader.Config{ - IndexHeaderEagerLoadingStartupEnabled: false, + EagerLoadingStartupEnabled: false, + LazyLoadingEnabled: false, + LazyLoadingIdleTimeout: 0, + SparsePersistenceEnabled: true, }, - IndexHeaderLazyLoadingEnabled: false, - IndexHeaderLazyLoadingIdleTimeout: 0, - IndexHeaderSparsePersistenceEnabled: true, }, selectAllStrategy{}, newStaticChunksLimiterFactory(10000/MaxSamplesPerChunk), @@ -2193,11 +2197,11 @@ func testBucketStoreSeriesBlockWithMultipleChunks( BlockSyncConcurrency: 10, PostingOffsetsInMemSampling: mimir_tsdb.DefaultPostingOffsetInMemorySampling, IndexHeader: indexheader.Config{ - IndexHeaderEagerLoadingStartupEnabled: false, + EagerLoadingStartupEnabled: false, + LazyLoadingEnabled: false, + LazyLoadingIdleTimeout: 0, + SparsePersistenceEnabled: true, }, - IndexHeaderLazyLoadingEnabled: false, - IndexHeaderLazyLoadingIdleTimeout: 0, - IndexHeaderSparsePersistenceEnabled: true, }, selectAllStrategy{}, newStaticChunksLimiterFactory(100000/MaxSamplesPerChunk), @@ -2360,11 +2364,11 @@ func TestBucketStore_Series_Limits(t *testing.T) { BlockSyncConcurrency: 10, PostingOffsetsInMemSampling: mimir_tsdb.DefaultPostingOffsetInMemorySampling, IndexHeader: indexheader.Config{ - IndexHeaderEagerLoadingStartupEnabled: false, + EagerLoadingStartupEnabled: false, + LazyLoadingEnabled: false, + LazyLoadingIdleTimeout: 0, + SparsePersistenceEnabled: true, }, - IndexHeaderLazyLoadingEnabled: false, - IndexHeaderLazyLoadingIdleTimeout: 0, - IndexHeaderSparsePersistenceEnabled: true, }, selectAllStrategy{}, newStaticChunksLimiterFactory(testData.chunksLimit), @@ -2480,11 +2484,11 @@ func setupStoreForHintsTest(t *testing.T, maxSeriesPerBatch int, opts ...BucketS BlockSyncConcurrency: 10, PostingOffsetsInMemSampling: mimir_tsdb.DefaultPostingOffsetInMemorySampling, IndexHeader: indexheader.Config{ - IndexHeaderEagerLoadingStartupEnabled: false, + EagerLoadingStartupEnabled: false, + LazyLoadingEnabled: false, + LazyLoadingIdleTimeout: 0, + SparsePersistenceEnabled: true, }, - IndexHeaderLazyLoadingEnabled: false, - IndexHeaderLazyLoadingIdleTimeout: 0, - IndexHeaderSparsePersistenceEnabled: true, }, selectAllStrategy{}, newStaticChunksLimiterFactory(10000/MaxSamplesPerChunk), diff --git a/pkg/storegateway/indexheader/header.go b/pkg/storegateway/indexheader/header.go index b981f0346cb..4847c1d4ff0 100644 --- a/pkg/storegateway/indexheader/header.go +++ b/pkg/storegateway/indexheader/header.go @@ -8,6 +8,7 @@ package indexheader import ( "flag" "io" + "time" "github.com/pkg/errors" "github.com/prometheus/prometheus/tsdb/index" @@ -54,19 +55,32 @@ type Reader interface { } type Config struct { - MaxIdleFileHandles uint `yaml:"max_idle_file_handles" category:"advanced"` - IndexHeaderEagerLoadingStartupEnabled bool `yaml:"eager_loading_startup_enabled" category:"experimental"` - VerifyOnLoad bool `yaml:"verify_on_load" category:"advanced"` + MaxIdleFileHandles uint `yaml:"max_idle_file_handles" category:"advanced"` + EagerLoadingStartupEnabled bool `yaml:"eager_loading_startup_enabled" category:"experimental"` + // Controls whether index-header lazy loading is enabled. + LazyLoadingEnabled bool `yaml:"index_header_lazy_loading_enabled" category:"advanced"` + LazyLoadingIdleTimeout time.Duration `yaml:"index_header_lazy_loading_idle_timeout" category:"advanced"` + + // Maximum index-headers loaded into store-gateway concurrently + LazyLoadingConcurrency int `yaml:"index_header_lazy_loading_concurrency" category:"experimental"` + + // Controls whether persisting a sparse version of the index-header to disk is enabled. + SparsePersistenceEnabled bool `yaml:"index_header_sparse_persistence_enabled" category:"experimental"` + VerifyOnLoad bool `yaml:"verify_on_load" category:"advanced"` } func (cfg *Config) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string) { f.UintVar(&cfg.MaxIdleFileHandles, prefix+"max-idle-file-handles", 1, "Maximum number of idle file handles the store-gateway keeps open for each index-header file.") - f.BoolVar(&cfg.IndexHeaderEagerLoadingStartupEnabled, prefix+"eager-loading-startup-enabled", false, "If enabled, store-gateway will periodically persist block IDs of lazy loaded index-headers and load them eagerly during startup. It is not valid to enable this if index-header lazy loading is disabled.") + f.BoolVar(&cfg.LazyLoadingEnabled, prefix+"lazy-loading-enabled", true, "If enabled, store-gateway will lazy load an index-header only once required by a query.") + f.DurationVar(&cfg.LazyLoadingIdleTimeout, prefix+"lazy-loading-idle-timeout", 60*time.Minute, "If index-header lazy loading is enabled and this setting is > 0, the store-gateway will offload unused index-headers after 'idle timeout' inactivity.") + f.IntVar(&cfg.LazyLoadingConcurrency, prefix+"lazy-loading-concurrency", 0, "Maximum number of concurrent index header loads across all tenants. If set to 0, concurrency is unlimited.") + f.BoolVar(&cfg.EagerLoadingStartupEnabled, prefix+"eager-loading-startup-enabled", false, "If enabled, store-gateway will periodically persist block IDs of lazy loaded index-headers and load them eagerly during startup. It is not valid to enable this if index-header lazy loading is disabled.") + f.BoolVar(&cfg.SparsePersistenceEnabled, prefix+"sparse-persistence-enabled", false, "If enabled, store-gateway will persist a sparse version of the index-header to disk on construction and load sparse index-headers from disk instead of the whole index-header.") f.BoolVar(&cfg.VerifyOnLoad, prefix+"verify-on-load", false, "If true, verify the checksum of index headers upon loading them (either on startup or lazily when lazy loading is enabled). Setting to true helps detect disk corruption at the cost of slowing down index header loading.") } -func (cfg *Config) Validate(lazyLoadingEnabled bool) error { - if !lazyLoadingEnabled && cfg.IndexHeaderEagerLoadingStartupEnabled { +func (cfg *Config) Validate() error { + if !cfg.LazyLoadingEnabled && cfg.EagerLoadingStartupEnabled { return errEagerLoadingStartupEnabledLazyLoadDisabled } return nil diff --git a/pkg/storegateway/indexheader/reader_pool.go b/pkg/storegateway/indexheader/reader_pool.go index eb87e0131a5..51e7b1326c0 100644 --- a/pkg/storegateway/indexheader/reader_pool.go +++ b/pkg/storegateway/indexheader/reader_pool.go @@ -70,7 +70,6 @@ type LazyLoadedHeadersSnapshotConfig struct { // Path stores where lazy loaded blocks will be tracked in a single file per tenant Path string UserID string - EagerLoadingEnabled bool } type lazyLoadedHeadersSnapshot struct { @@ -96,9 +95,9 @@ func (l lazyLoadedHeadersSnapshot) persist(persistDir string) error { } // NewReaderPool makes a new ReaderPool. If lazy-loading is enabled, NewReaderPool also starts a background task for unloading idle Readers and persisting a list of loaded Readers to disk. -func NewReaderPool(logger log.Logger, lazyReaderEnabled bool, lazyReaderIdleTimeout time.Duration, sparsePersistenceEnabled bool, lazyLoadingGate gate.Gate, metrics *ReaderPoolMetrics, lazyLoadedSnapshotConfig LazyLoadedHeadersSnapshotConfig) *ReaderPool { +func NewReaderPool(logger log.Logger, indexHeaderConfig Config, lazyLoadingGate gate.Gate, metrics *ReaderPoolMetrics, lazyLoadedSnapshotConfig LazyLoadedHeadersSnapshotConfig) *ReaderPool { var snapshot *lazyLoadedHeadersSnapshot - if lazyReaderEnabled && lazyLoadedSnapshotConfig.EagerLoadingEnabled { + if indexHeaderConfig.LazyLoadingEnabled && indexHeaderConfig.EagerLoadingStartupEnabled { lazyLoadedSnapshotFileName := filepath.Join(lazyLoadedSnapshotConfig.Path, lazyLoadedHeadersListFileName) var err error snapshot, err = loadLazyLoadedHeadersSnapshot(lazyLoadedSnapshotFileName) @@ -113,7 +112,7 @@ func NewReaderPool(logger log.Logger, lazyReaderEnabled bool, lazyReaderIdleTime } } - p := newReaderPool(logger, lazyReaderEnabled, lazyReaderIdleTimeout, sparsePersistenceEnabled, lazyLoadingGate, metrics, snapshot) + p := newReaderPool(logger, indexHeaderConfig, lazyLoadingGate, metrics, snapshot) // Start a goroutine to close idle readers (only if required). if p.lazyReaderEnabled && p.lazyReaderIdleTimeout > 0 { @@ -157,13 +156,14 @@ func NewReaderPool(logger log.Logger, lazyReaderEnabled bool, lazyReaderIdleTime } // newReaderPool makes a new ReaderPool. -func newReaderPool(logger log.Logger, lazyReaderEnabled bool, lazyReaderIdleTimeout time.Duration, sparsePersistenceEnabled bool, lazyLoadingGate gate.Gate, metrics *ReaderPoolMetrics, lazyLoadedHeadersSnapshot *lazyLoadedHeadersSnapshot) *ReaderPool { +func newReaderPool(logger log.Logger, indexHeaderConfig Config, lazyLoadingGate gate.Gate, metrics *ReaderPoolMetrics, lazyLoadedHeadersSnapshot *lazyLoadedHeadersSnapshot) *ReaderPool { return &ReaderPool{ logger: logger, metrics: metrics, - lazyReaderEnabled: lazyReaderEnabled, - lazyReaderIdleTimeout: lazyReaderIdleTimeout, - sparsePersistenceEnabled: sparsePersistenceEnabled, + lazyReaderEnabled: indexHeaderConfig.LazyLoadingEnabled, + lazyReaderIdleTimeout: indexHeaderConfig.LazyLoadingIdleTimeout, + eagerLoadReaderEnabled: indexHeaderConfig.EagerLoadingStartupEnabled, + sparsePersistenceEnabled: indexHeaderConfig.SparsePersistenceEnabled, lazyReaders: make(map[*LazyBinaryReader]struct{}), close: make(chan struct{}), preShutdownLoadedBlocks: lazyLoadedHeadersSnapshot, diff --git a/pkg/storegateway/indexheader/reader_pool_test.go b/pkg/storegateway/indexheader/reader_pool_test.go index b5be645e8f6..2e16f124f3c 100644 --- a/pkg/storegateway/indexheader/reader_pool_test.go +++ b/pkg/storegateway/indexheader/reader_pool_test.go @@ -104,9 +104,8 @@ func TestReaderPool_NewBinaryReader(t *testing.T) { for testName, testData := range tests { t.Run(testName, func(t *testing.T) { snapshotConfig := LazyLoadedHeadersSnapshotConfig{ - Path: tmpDir, - UserID: "anonymous", - EagerLoadingEnabled: testData.eagerLoadReaderEnabled, + Path: tmpDir, + UserID: "anonymous", } if testData.createLazyLoadedHeadersSnapshotFn != nil { lazyLoadedSnapshot := testData.createLazyLoadedHeadersSnapshotFn(blockID) @@ -115,10 +114,16 @@ func TestReaderPool_NewBinaryReader(t *testing.T) { } metrics := NewReaderPoolMetrics(nil) - pool := NewReaderPool(log.NewNopLogger(), testData.lazyReaderEnabled, testData.lazyReaderIdleTimeout, true, gate.NewNoop(), metrics, snapshotConfig) + indexHeaderConfig := Config{ + LazyLoadingEnabled: testData.lazyReaderEnabled, + LazyLoadingIdleTimeout: testData.lazyReaderIdleTimeout, + SparsePersistenceEnabled: true, + EagerLoadingStartupEnabled: testData.eagerLoadReaderEnabled, + } + pool := NewReaderPool(log.NewNopLogger(), indexHeaderConfig, gate.NewNoop(), metrics, snapshotConfig) defer pool.Close() - r, err := pool.NewBinaryReader(ctx, log.NewNopLogger(), bkt, tmpDir, blockID, 3, Config{IndexHeaderEagerLoadingStartupEnabled: testData.eagerLoadReaderEnabled}, testData.initialSync) + r, err := pool.NewBinaryReader(ctx, log.NewNopLogger(), bkt, tmpDir, blockID, 3, indexHeaderConfig, testData.initialSync) require.NoError(t, err) defer func() { require.NoError(t, r.Close()) }() @@ -142,7 +147,12 @@ func TestReaderPool_ShouldCloseIdleLazyReaders(t *testing.T) { // Note that we are creating a ReaderPool that doesn't run a background cleanup task for idle // Reader instances. We'll manually invoke the cleanup task when we need it as part of this test. - pool := newReaderPool(log.NewNopLogger(), true, idleTimeout, true, gate.NewNoop(), metrics, nil) + pool := newReaderPool(log.NewNopLogger(), Config{ + LazyLoadingEnabled: true, + LazyLoadingIdleTimeout: idleTimeout, + EagerLoadingStartupEnabled: false, + SparsePersistenceEnabled: true, + }, gate.NewNoop(), metrics, nil) defer pool.Close() r, err := pool.NewBinaryReader(ctx, log.NewNopLogger(), bkt, tmpDir, blockID, 3, Config{}, false) @@ -204,7 +214,12 @@ func TestReaderPool_PersistLazyLoadedBlock(t *testing.T) { // Note that we are creating a ReaderPool that doesn't run a background cleanup task for idle // Reader instances. We'll manually invoke the cleanup task when we need it as part of this test. - pool := newReaderPool(log.NewNopLogger(), true, idleTimeout, false, gate.NewNoop(), metrics, nil) + pool := newReaderPool(log.NewNopLogger(), Config{ + LazyLoadingEnabled: true, + LazyLoadingIdleTimeout: idleTimeout, + EagerLoadingStartupEnabled: true, + SparsePersistenceEnabled: false, + }, gate.NewNoop(), metrics, nil) defer pool.Close() r, err := pool.NewBinaryReader(ctx, log.NewNopLogger(), bkt, tmpDir, blockID, 3, Config{}, false) From 3a4513bae1eb34e17f9a5b260bbcaefc5d6e6924 Mon Sep 17 00:00:00 2001 From: Jon Kartago Lamida Date: Mon, 14 Aug 2023 16:55:12 +0800 Subject: [PATCH 05/20] Remove and deprecate index header config outside indexheader package Signed-off-by: Jon Kartago Lamida --- cmd/mimir/config-descriptor.json | 26 ++------------- cmd/mimir/help-all.txt.tmpl | 8 ++--- .../configuration-parameters/index.md | 17 ++-------- pkg/storage/tsdb/config.go | 32 +++++++------------ pkg/storage/tsdb/config_test.go | 6 ---- pkg/storegateway/bucket_stores.go | 6 ++-- pkg/storegateway/indexheader/header.go | 8 ++++- 7 files changed, 29 insertions(+), 74 deletions(-) diff --git a/cmd/mimir/config-descriptor.json b/cmd/mimir/config-descriptor.json index e4aa5d88255..eab17fb3abd 100644 --- a/cmd/mimir/config-descriptor.json +++ b/cmd/mimir/config-descriptor.json @@ -7558,7 +7558,7 @@ "fieldDefaultValue": true, "fieldFlag": "blocks-storage.bucket-store.index-header-lazy-loading-enabled", "fieldType": "boolean", - "fieldCategory": "advanced" + "fieldCategory": "deprecated" }, { "kind": "field", @@ -7569,29 +7569,7 @@ "fieldDefaultValue": 3600000000000, "fieldFlag": "blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout", "fieldType": "duration", - "fieldCategory": "advanced" - }, - { - "kind": "field", - "name": "index_header_lazy_loading_concurrency", - "required": false, - "desc": "Maximum number of concurrent index header loads across all tenants. If set to 0, concurrency is unlimited.", - "fieldValue": null, - "fieldDefaultValue": 0, - "fieldFlag": "blocks-storage.bucket-store.index-header-lazy-loading-concurrency", - "fieldType": "int", - "fieldCategory": "experimental" - }, - { - "kind": "field", - "name": "index_header_sparse_persistence_enabled", - "required": false, - "desc": "If enabled, store-gateway will persist a sparse version of the index-header to disk on construction and load sparse index-headers from disk instead of the whole index-header.", - "fieldValue": null, - "fieldDefaultValue": false, - "fieldFlag": "blocks-storage.bucket-store.index-header-sparse-persistence-enabled", - "fieldType": "boolean", - "fieldCategory": "experimental" + "fieldCategory": "deprecated" }, { "kind": "field", diff --git a/cmd/mimir/help-all.txt.tmpl b/cmd/mimir/help-all.txt.tmpl index 87af68a635b..3144b974e61 100644 --- a/cmd/mimir/help-all.txt.tmpl +++ b/cmd/mimir/help-all.txt.tmpl @@ -499,14 +499,10 @@ Usage of ./cmd/mimir/mimir: Username to use when connecting to Redis. -blocks-storage.bucket-store.index-cache.redis.write-timeout duration Client write timeout. (default 3s) - -blocks-storage.bucket-store.index-header-lazy-loading-concurrency int - [experimental] Maximum number of concurrent index header loads across all tenants. If set to 0, concurrency is unlimited. -blocks-storage.bucket-store.index-header-lazy-loading-enabled - If enabled, store-gateway will lazy load an index-header only once required by a query. (default true) + [deprecated] If enabled, store-gateway will lazy load an index-header only once required by a query. (default true) -blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout duration - If index-header lazy loading is enabled and this setting is > 0, the store-gateway will offload unused index-headers after 'idle timeout' inactivity. (default 1h0m0s) - -blocks-storage.bucket-store.index-header-sparse-persistence-enabled - [experimental] If enabled, store-gateway will persist a sparse version of the index-header to disk on construction and load sparse index-headers from disk instead of the whole index-header. + [deprecated] If index-header lazy loading is enabled and this setting is > 0, the store-gateway will offload unused index-headers after 'idle timeout' inactivity. (default 1h0m0s) -blocks-storage.bucket-store.index-header.eager-loading-startup-enabled [experimental] If enabled, store-gateway will periodically persist block IDs of lazy loaded index-headers and load them eagerly during startup. It is not valid to enable this if index-header lazy loading is disabled. -blocks-storage.bucket-store.index-header.lazy-loading-concurrency int diff --git a/docs/sources/mimir/references/configuration-parameters/index.md b/docs/sources/mimir/references/configuration-parameters/index.md index b089a5e3f31..8ed04054427 100644 --- a/docs/sources/mimir/references/configuration-parameters/index.md +++ b/docs/sources/mimir/references/configuration-parameters/index.md @@ -3405,28 +3405,17 @@ bucket_store: # CLI flag: -blocks-storage.bucket-store.series-hash-cache-max-size-bytes [series_hash_cache_max_size_bytes: | default = 1073741824] - # (advanced) If enabled, store-gateway will lazy load an index-header only + # (deprecated) If enabled, store-gateway will lazy load an index-header only # once required by a query. # CLI flag: -blocks-storage.bucket-store.index-header-lazy-loading-enabled [index_header_lazy_loading_enabled: | default = true] - # (advanced) If index-header lazy loading is enabled and this setting is > 0, - # the store-gateway will offload unused index-headers after 'idle timeout' + # (deprecated) If index-header lazy loading is enabled and this setting is > + # 0, the store-gateway will offload unused index-headers after 'idle timeout' # inactivity. # CLI flag: -blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout [index_header_lazy_loading_idle_timeout: | default = 1h] - # (experimental) Maximum number of concurrent index header loads across all - # tenants. If set to 0, concurrency is unlimited. - # CLI flag: -blocks-storage.bucket-store.index-header-lazy-loading-concurrency - [index_header_lazy_loading_concurrency: | default = 0] - - # (experimental) If enabled, store-gateway will persist a sparse version of - # the index-header to disk on construction and load sparse index-headers from - # disk instead of the whole index-header. - # CLI flag: -blocks-storage.bucket-store.index-header-sparse-persistence-enabled - [index_header_sparse_persistence_enabled: | default = false] - # (advanced) Max size - in bytes - of a gap for which the partitioner # aggregates together two bucket GET object requests. # CLI flag: -blocks-storage.bucket-store.partitioner-max-gap-bytes diff --git a/pkg/storage/tsdb/config.go b/pkg/storage/tsdb/config.go index 06bc13d5e1e..0478a5f0f3f 100644 --- a/pkg/storage/tsdb/config.go +++ b/pkg/storage/tsdb/config.go @@ -119,7 +119,6 @@ var ( errInvalidEarlyHeadCompactionMinSeriesReduction = errors.New("early compaction minimum series reduction percentage must be a value between 0 and 100 (included)") errEarlyCompactionRequiresActiveSeries = fmt.Errorf("early compaction requires -%s to be enabled", activeseries.EnabledFlag) errEmptyBlockranges = errors.New("empty block ranges for TSDB") - errInvalidIndexHeaderLazyLoadingConcurrency = errors.New("invalid index-header lazy loading max concurrency; must be non-negative") ) // BlocksStorageConfig holds the config information for the blocks storage. @@ -388,17 +387,9 @@ type BucketStoreConfig struct { // Series hash cache. SeriesHashCacheMaxBytes uint64 `yaml:"series_hash_cache_max_size_bytes" category:"advanced"` - // TODO JON REMOVE // Controls whether index-header lazy loading is enabled. - IndexHeaderLazyLoadingEnabled bool `yaml:"index_header_lazy_loading_enabled" category:"advanced"` - IndexHeaderLazyLoadingIdleTimeout time.Duration `yaml:"index_header_lazy_loading_idle_timeout" category:"advanced"` - - // Maximum index-headers loaded into store-gateway concurrently - IndexHeaderLazyLoadingConcurrency int `yaml:"index_header_lazy_loading_concurrency" category:"experimental"` - - // Controls whether persisting a sparse version of the index-header to disk is enabled. - IndexHeaderSparsePersistenceEnabled bool `yaml:"index_header_sparse_persistence_enabled" category:"experimental"` - // TODO JON REMOVE END + DeprecatedIndexHeaderLazyLoadingEnabled bool `yaml:"index_header_lazy_loading_enabled" category:"deprecated"` // Deprecated. TODO: Remove in Mimir 2.1x. + DeprecatedIndexHeaderLazyLoadingIdleTimeout time.Duration `yaml:"index_header_lazy_loading_idle_timeout" category:"deprecated"` // Deprecated. TODO: Remove in Mimir 2.1x. // Controls the partitioner, used to aggregate multiple GET object API requests. PartitionerMaxGapBytes uint64 `yaml:"partitioner_max_gap_bytes" category:"advanced"` @@ -443,6 +434,14 @@ func (cfg *BucketStoreConfig) RegisterFlags(f *flag.FlagSet) { cfg.BucketIndex.RegisterFlagsWithPrefix(f, bucketIndexFlagPrefix) cfg.IndexHeader.RegisterFlagsWithPrefix(f, "blocks-storage.bucket-store.index-header.") + // Take value from DeprecatedConfig if it is set + if cfg.DeprecatedIndexHeaderLazyLoadingEnabled { + cfg.IndexHeader.LazyLoadingEnabled = cfg.DeprecatedIndexHeaderLazyLoadingEnabled + } + if cfg.DeprecatedIndexHeaderLazyLoadingIdleTimeout > 0 { + cfg.IndexHeader.LazyLoadingIdleTimeout = cfg.DeprecatedIndexHeaderLazyLoadingIdleTimeout + } + f.StringVar(&cfg.SyncDir, "blocks-storage.bucket-store.sync-dir", "./tsdb-sync/", "Directory to store synchronized TSDB index headers. This directory is not required to be persisted between restarts, but it's highly recommended in order to improve the store-gateway startup time.") f.DurationVar(&cfg.SyncInterval, "blocks-storage.bucket-store.sync-interval", 15*time.Minute, "How frequently to scan the bucket, or to refresh the bucket index (if enabled), in order to look for changes (new blocks shipped by ingesters and blocks deleted by retention or compaction).") f.Uint64Var(&cfg.DeprecatedMaxChunkPoolBytes, maxChunksBytesPoolFlag, uint64(2*units.Gibibyte), "Max size - in bytes - of a chunks pool, used to reduce memory allocations. The pool is shared across all tenants. 0 to disable the limit.") @@ -457,12 +456,8 @@ func (cfg *BucketStoreConfig) RegisterFlags(f *flag.FlagSet) { "The idea of ignore-deletion-marks-delay is to ignore blocks that are marked for deletion with some delay. This ensures store can still serve blocks that are meant to be deleted but do not have a replacement yet.") f.DurationVar(&cfg.IgnoreBlocksWithin, "blocks-storage.bucket-store.ignore-blocks-within", 10*time.Hour, "Blocks with minimum time within this duration are ignored, and not loaded by store-gateway. Useful when used together with -querier.query-store-after to prevent loading young blocks, because there are usually many of them (depending on number of ingesters) and they are not yet compacted. Negative values or 0 disable the filter.") f.IntVar(&cfg.PostingOffsetsInMemSampling, "blocks-storage.bucket-store.posting-offsets-in-mem-sampling", DefaultPostingOffsetInMemorySampling, "Controls what is the ratio of postings offsets that the store will hold in memory.") - // TODO JON REMOVE - f.BoolVar(&cfg.IndexHeaderLazyLoadingEnabled, "blocks-storage.bucket-store.index-header-lazy-loading-enabled", true, "If enabled, store-gateway will lazy load an index-header only once required by a query.") - f.DurationVar(&cfg.IndexHeaderLazyLoadingIdleTimeout, "blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout", 60*time.Minute, "If index-header lazy loading is enabled and this setting is > 0, the store-gateway will offload unused index-headers after 'idle timeout' inactivity.") - f.IntVar(&cfg.IndexHeaderLazyLoadingConcurrency, "blocks-storage.bucket-store.index-header-lazy-loading-concurrency", 0, "Maximum number of concurrent index header loads across all tenants. If set to 0, concurrency is unlimited.") - f.BoolVar(&cfg.IndexHeaderSparsePersistenceEnabled, "blocks-storage.bucket-store.index-header-sparse-persistence-enabled", false, "If enabled, store-gateway will persist a sparse version of the index-header to disk on construction and load sparse index-headers from disk instead of the whole index-header.") - // TODO JON REMOVE END + f.BoolVar(&cfg.DeprecatedIndexHeaderLazyLoadingEnabled, "blocks-storage.bucket-store.index-header-lazy-loading-enabled", true, "If enabled, store-gateway will lazy load an index-header only once required by a query.") + f.DurationVar(&cfg.DeprecatedIndexHeaderLazyLoadingIdleTimeout, "blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout", 60*time.Minute, "If index-header lazy loading is enabled and this setting is > 0, the store-gateway will offload unused index-headers after 'idle timeout' inactivity.") f.Uint64Var(&cfg.PartitionerMaxGapBytes, "blocks-storage.bucket-store.partitioner-max-gap-bytes", DefaultPartitionerMaxGapSize, "Max size - in bytes - of a gap for which the partitioner aggregates together two bucket GET object requests.") f.IntVar(&cfg.StreamingBatchSize, "blocks-storage.bucket-store.batch-series-size", 5000, "This option controls how many series to fetch per batch. The batch size must be greater than 0.") f.IntVar(&cfg.ChunkRangesPerSeries, "blocks-storage.bucket-store.fine-grained-chunks-caching-ranges-per-series", 1, "This option controls into how many ranges the chunks of each series from each block are split. This value is effectively the number of chunks cache items per series per block when -blocks-storage.bucket-store.chunks-cache.fine-grained-chunks-caching-enabled is enabled.") @@ -505,9 +500,6 @@ func (cfg *BucketStoreConfig) Validate(logger log.Logger) error { if err := cfg.IndexHeader.Validate(); err != nil { return errors.Wrap(err, "index-header configuration") } - if cfg.IndexHeaderLazyLoadingConcurrency < 0 { - return errInvalidIndexHeaderLazyLoadingConcurrency - } return nil } diff --git a/pkg/storage/tsdb/config_test.go b/pkg/storage/tsdb/config_test.go index 9ec5e0dbd35..2fcdffb681d 100644 --- a/pkg/storage/tsdb/config_test.go +++ b/pkg/storage/tsdb/config_test.go @@ -128,12 +128,6 @@ func TestConfig_Validate(t *testing.T) { }, expectedErr: errInvalidStreamingBatchSize, }, - "should fail on invalid index-header lazy loading max concurrency": { - setup: func(cfg *BlocksStorageConfig, activeSeriesCfg *activeseries.Config) { - cfg.BucketStore.IndexHeaderLazyLoadingConcurrency = -1 - }, - expectedErr: errInvalidIndexHeaderLazyLoadingConcurrency, - }, "should fail if forced compaction is enabled but active series tracker is not": { setup: func(cfg *BlocksStorageConfig, activeSeriesCfg *activeseries.Config) { cfg.TSDB.EarlyHeadCompactionMinInMemorySeries = 1_000_000 diff --git a/pkg/storegateway/bucket_stores.go b/pkg/storegateway/bucket_stores.go index 2e3bde2d12e..c7530bad17f 100644 --- a/pkg/storegateway/bucket_stores.go +++ b/pkg/storegateway/bucket_stores.go @@ -103,10 +103,10 @@ func NewBucketStores(cfg tsdb.BlocksStorageConfig, shardingStrategy ShardingStra // The number of concurrent index header loads from storegateway are limited. lazyLoadingGateReg := prometheus.WrapRegistererWith(prometheus.Labels{"gate": "index_header"}, gateReg) lazyLoadingGate := gate.NewNoop() - lazyLoadingMax := cfg.BucketStore.IndexHeaderLazyLoadingConcurrency + lazyLoadingMax := cfg.BucketStore.IndexHeader.LazyLoadingConcurrency if lazyLoadingMax != 0 { - blockingGate := gate.NewBlocking(cfg.BucketStore.IndexHeaderLazyLoadingConcurrency) - lazyLoadingGate = gate.NewInstrumented(lazyLoadingGateReg, cfg.BucketStore.IndexHeaderLazyLoadingConcurrency, blockingGate) + blockingGate := gate.NewBlocking(cfg.BucketStore.IndexHeader.LazyLoadingConcurrency) + lazyLoadingGate = gate.NewInstrumented(lazyLoadingGateReg, cfg.BucketStore.IndexHeader.LazyLoadingConcurrency, blockingGate) } u := &BucketStores{ diff --git a/pkg/storegateway/indexheader/header.go b/pkg/storegateway/indexheader/header.go index 4847c1d4ff0..9319e9f58ef 100644 --- a/pkg/storegateway/indexheader/header.go +++ b/pkg/storegateway/indexheader/header.go @@ -19,7 +19,10 @@ import ( // NotFoundRangeErr is an error returned by PostingsOffset when there is no posting for given name and value pairs. var NotFoundRangeErr = errors.New("range not found") //nolint:revive -var errEagerLoadingStartupEnabledLazyLoadDisabled = errors.New("invalid configuration: store-gateway index header eager-loading is enabled, but lazy-loading is disabled") +var ( + errEagerLoadingStartupEnabledLazyLoadDisabled = errors.New("invalid configuration: store-gateway index header eager-loading is enabled, but lazy-loading is disabled") + errInvalidIndexHeaderLazyLoadingConcurrency = errors.New("invalid index-header lazy loading max concurrency; must be non-negative") +) // Reader is an interface allowing to read essential, minimal number of index fields from the small portion of index file called header. type Reader interface { @@ -83,5 +86,8 @@ func (cfg *Config) Validate() error { if !cfg.LazyLoadingEnabled && cfg.EagerLoadingStartupEnabled { return errEagerLoadingStartupEnabledLazyLoadDisabled } + if cfg.LazyLoadingConcurrency < 0 { + return errInvalidIndexHeaderLazyLoadingConcurrency + } return nil } From 28700f1776bb6718ba9b709aaa7c223ace6c7c2a Mon Sep 17 00:00:00 2001 From: Jon Kartago Lamida Date: Mon, 14 Aug 2023 17:05:01 +0800 Subject: [PATCH 06/20] Remove deprecated config in mimir 2.12 Signed-off-by: Jon Kartago Lamida --- pkg/storage/tsdb/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/storage/tsdb/config.go b/pkg/storage/tsdb/config.go index 0478a5f0f3f..393cf762356 100644 --- a/pkg/storage/tsdb/config.go +++ b/pkg/storage/tsdb/config.go @@ -388,8 +388,8 @@ type BucketStoreConfig struct { SeriesHashCacheMaxBytes uint64 `yaml:"series_hash_cache_max_size_bytes" category:"advanced"` // Controls whether index-header lazy loading is enabled. - DeprecatedIndexHeaderLazyLoadingEnabled bool `yaml:"index_header_lazy_loading_enabled" category:"deprecated"` // Deprecated. TODO: Remove in Mimir 2.1x. - DeprecatedIndexHeaderLazyLoadingIdleTimeout time.Duration `yaml:"index_header_lazy_loading_idle_timeout" category:"deprecated"` // Deprecated. TODO: Remove in Mimir 2.1x. + DeprecatedIndexHeaderLazyLoadingEnabled bool `yaml:"index_header_lazy_loading_enabled" category:"deprecated"` // Deprecated. TODO: Remove in Mimir 2.12. + DeprecatedIndexHeaderLazyLoadingIdleTimeout time.Duration `yaml:"index_header_lazy_loading_idle_timeout" category:"deprecated"` // Deprecated. TODO: Remove in Mimir 2.12. // Controls the partitioner, used to aggregate multiple GET object API requests. PartitionerMaxGapBytes uint64 `yaml:"partitioner_max_gap_bytes" category:"advanced"` From de3687df407ca28ada2a0cd73c3e29877ff2f40d Mon Sep 17 00:00:00 2001 From: Jon Kartago Lamida Date: Mon, 14 Aug 2023 17:19:24 +0800 Subject: [PATCH 07/20] Add changelogs Signed-off-by: Jon Kartago Lamida --- CHANGELOG.md | 4 ++++ pkg/storage/tsdb/config.go | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78ff288249f..bbe27732541 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,10 @@ * [CHANGE] The `-shutdown-delay` flag is no longer experimental. #5701 * [CHANGE] The `-validation.create-grace-period` is now enforced in the ingester too, other than distributor and query-frontend. If you've configured `-validation.create-grace-period` then make sure the configuration is applied to ingesters too. #5712 * [CHANGE] The `-validation.create-grace-period` is now enforced for examplars too in the distributor. If an examplar has timestamp greater than "now + grace_period", then the exemplar will be dropped and the metric `cortex_discarded_exemplars_total{reason="exemplar_too_far_in_future",user="..."}` increased. #5761 +* [CHANGE] Store-gateway: deprecate and move configuration parameters for index header under `blocks-storage.bucket-store` to `blocks-storage.bucket-store.index-header`, deprecated configuration will be removed in Mimir 2.12. Configuration changes: #5726 + * `-blocks-storage.bucket-store.index-header-lazy-loading-enabled` is deprecated and moved to `-blocks-storage.bucket-store.index-header.lazy-loading-enabled` + * `-blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout` is deprecated and moved to `-blocks-storage.bucket-store.index-header.lazy-loading-idle-timeout` + * `-blocks-storage.bucket-store.index-header-lazy-loading-concurrency` is moved to `-blocks-storage.bucket-store.index-header.lazy-loading-concurrency` * [FEATURE] Introduced `distributor.service_overload_status_code_on_rate_limit_enabled` flag for configuring status code to 529 instead of 429 upon rate limit exhaustion. #5752 * [FEATURE] Cardinality API: Add a new `count_method` parameter which enables counting active series #5136 * [FEATURE] Query-frontend: added experimental support to cache cardinality, label names and label values query responses. The cache will be used when `-query-frontend.cache-results` is enabled, and `-query-frontend.results-cache-ttl-for-cardinality-query` or `-query-frontend.results-cache-ttl-for-labels-query` set to a value greater than 0. The following metrics have been added to track the query results cache hit ratio per `request_type`: #5212 #5235 #5426 #5524 diff --git a/pkg/storage/tsdb/config.go b/pkg/storage/tsdb/config.go index 393cf762356..76e249dd533 100644 --- a/pkg/storage/tsdb/config.go +++ b/pkg/storage/tsdb/config.go @@ -434,7 +434,8 @@ func (cfg *BucketStoreConfig) RegisterFlags(f *flag.FlagSet) { cfg.BucketIndex.RegisterFlagsWithPrefix(f, bucketIndexFlagPrefix) cfg.IndexHeader.RegisterFlagsWithPrefix(f, "blocks-storage.bucket-store.index-header.") - // Take value from DeprecatedConfig if it is set + // TODO: Remove in Mimir 2.12. + // Take value from Indexheader DeprecatedConfig if it is set if cfg.DeprecatedIndexHeaderLazyLoadingEnabled { cfg.IndexHeader.LazyLoadingEnabled = cfg.DeprecatedIndexHeaderLazyLoadingEnabled } From ad26dff6966d2004961c396ae8315f67f49f9108 Mon Sep 17 00:00:00 2001 From: Jon Kartago Lamida Date: Mon, 14 Aug 2023 17:29:33 +0800 Subject: [PATCH 08/20] Update configuration test Signed-off-by: Jon Kartago Lamida --- pkg/storegateway/indexheader/header_test.go | 40 +++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/pkg/storegateway/indexheader/header_test.go b/pkg/storegateway/indexheader/header_test.go index b7ebe57d898..3a83e2e191e 100644 --- a/pkg/storegateway/indexheader/header_test.go +++ b/pkg/storegateway/indexheader/header_test.go @@ -7,6 +7,7 @@ package indexheader import ( "context" + "flag" "fmt" "path/filepath" "testing" @@ -266,6 +267,45 @@ func TestReadersLabelValuesOffsets(t *testing.T) { } } +func TestConfig_Validate(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + setup func(*Config) + expectedErr error + }{ + "should fail on invalid index-header eager loading in startup": { + setup: func(cfg *Config) { + cfg.EagerLoadingStartupEnabled = true + cfg.LazyLoadingEnabled = false + }, + expectedErr: errEagerLoadingStartupEnabledLazyLoadDisabled, + }, + "should fail on invalid index-header lazy loading max concurrency": { + setup: func(cfg *Config) { + cfg.LazyLoadingConcurrency = -1 + }, + expectedErr: errInvalidIndexHeaderLazyLoadingConcurrency, + }, + } + + for testName, testData := range tests { + testData := testData + + t.Run(testName, func(t *testing.T) { + indexHeaderConfig := &Config{} + + fs := flag.NewFlagSet("", flag.PanicOnError) + indexHeaderConfig.RegisterFlagsWithPrefix(fs, "blocks-storage.bucket-store.index-header.") + + testData.setup(indexHeaderConfig) + + actualErr := indexHeaderConfig.Validate() + assert.Equal(t, testData.expectedErr, actualErr) + }) + } +} + type labelValuesTestCase struct { prefix string desc string From d05ed504b0e39e120d8a739deeaa2d59260e54ec Mon Sep 17 00:00:00 2001 From: Jon Kartago Lamida Date: Mon, 14 Aug 2023 19:14:11 +0800 Subject: [PATCH 09/20] Address PR feedbacks Signed-off-by: Jon Kartago Lamida --- pkg/storegateway/indexheader/reader_pool.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/storegateway/indexheader/reader_pool.go b/pkg/storegateway/indexheader/reader_pool.go index 51e7b1326c0..dd77cb30b7a 100644 --- a/pkg/storegateway/indexheader/reader_pool.go +++ b/pkg/storegateway/indexheader/reader_pool.go @@ -48,7 +48,6 @@ func NewReaderPoolMetrics(reg prometheus.Registerer) *ReaderPoolMetrics { type ReaderPool struct { lazyReaderEnabled bool lazyReaderIdleTimeout time.Duration - eagerLoadReaderEnabled bool sparsePersistenceEnabled bool logger log.Logger metrics *ReaderPoolMetrics @@ -124,7 +123,7 @@ func NewReaderPool(logger log.Logger, indexHeaderConfig Config, lazyLoadingGate var lazyLoadC <-chan time.Time - if lazyLoadedSnapshotConfig.EagerLoadingEnabled { + if indexHeaderConfig.EagerLoadingStartupEnabled { tickerLazyLoadPersist := time.NewTicker(time.Minute) defer tickerLazyLoadPersist.Stop() @@ -162,7 +161,6 @@ func newReaderPool(logger log.Logger, indexHeaderConfig Config, lazyLoadingGate metrics: metrics, lazyReaderEnabled: indexHeaderConfig.LazyLoadingEnabled, lazyReaderIdleTimeout: indexHeaderConfig.LazyLoadingIdleTimeout, - eagerLoadReaderEnabled: indexHeaderConfig.EagerLoadingStartupEnabled, sparsePersistenceEnabled: indexHeaderConfig.SparsePersistenceEnabled, lazyReaders: make(map[*LazyBinaryReader]struct{}), close: make(chan struct{}), From 01c8f2e2d673891d923731ae2d97b56e251a2641 Mon Sep 17 00:00:00 2001 From: Jon Kartago Lamida Date: Mon, 14 Aug 2023 21:57:43 +0800 Subject: [PATCH 10/20] Remove redundant index_header prefix in config Signed-off-by: Jon Kartago Lamida --- cmd/mimir/config-descriptor.json | 8 ++++---- .../mimir/references/configuration-parameters/index.md | 8 ++++---- pkg/storegateway/indexheader/header.go | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/cmd/mimir/config-descriptor.json b/cmd/mimir/config-descriptor.json index eab17fb3abd..92cd4312622 100644 --- a/cmd/mimir/config-descriptor.json +++ b/cmd/mimir/config-descriptor.json @@ -7623,7 +7623,7 @@ }, { "kind": "field", - "name": "index_header_lazy_loading_enabled", + "name": "lazy_loading_enabled", "required": false, "desc": "If enabled, store-gateway will lazy load an index-header only once required by a query.", "fieldValue": null, @@ -7634,7 +7634,7 @@ }, { "kind": "field", - "name": "index_header_lazy_loading_idle_timeout", + "name": "lazy_loading_idle_timeout", "required": false, "desc": "If index-header lazy loading is enabled and this setting is \u003e 0, the store-gateway will offload unused index-headers after 'idle timeout' inactivity.", "fieldValue": null, @@ -7645,7 +7645,7 @@ }, { "kind": "field", - "name": "index_header_lazy_loading_concurrency", + "name": "lazy_loading_concurrency", "required": false, "desc": "Maximum number of concurrent index header loads across all tenants. If set to 0, concurrency is unlimited.", "fieldValue": null, @@ -7656,7 +7656,7 @@ }, { "kind": "field", - "name": "index_header_sparse_persistence_enabled", + "name": "sparse_persistence_enabled", "required": false, "desc": "If enabled, store-gateway will persist a sparse version of the index-header to disk on construction and load sparse index-headers from disk instead of the whole index-header.", "fieldValue": null, diff --git a/docs/sources/mimir/references/configuration-parameters/index.md b/docs/sources/mimir/references/configuration-parameters/index.md index 8ed04054427..4dd7f51cff8 100644 --- a/docs/sources/mimir/references/configuration-parameters/index.md +++ b/docs/sources/mimir/references/configuration-parameters/index.md @@ -3441,24 +3441,24 @@ bucket_store: # (advanced) If enabled, store-gateway will lazy load an index-header only # once required by a query. # CLI flag: -blocks-storage.bucket-store.index-header.lazy-loading-enabled - [index_header_lazy_loading_enabled: | default = true] + [lazy_loading_enabled: | default = true] # (advanced) If index-header lazy loading is enabled and this setting is > # 0, the store-gateway will offload unused index-headers after 'idle # timeout' inactivity. # CLI flag: -blocks-storage.bucket-store.index-header.lazy-loading-idle-timeout - [index_header_lazy_loading_idle_timeout: | default = 1h] + [lazy_loading_idle_timeout: | default = 1h] # (experimental) Maximum number of concurrent index header loads across all # tenants. If set to 0, concurrency is unlimited. # CLI flag: -blocks-storage.bucket-store.index-header.lazy-loading-concurrency - [index_header_lazy_loading_concurrency: | default = 0] + [lazy_loading_concurrency: | default = 0] # (experimental) If enabled, store-gateway will persist a sparse version of # the index-header to disk on construction and load sparse index-headers # from disk instead of the whole index-header. # CLI flag: -blocks-storage.bucket-store.index-header.sparse-persistence-enabled - [index_header_sparse_persistence_enabled: | default = false] + [sparse_persistence_enabled: | default = false] # (advanced) If true, verify the checksum of index headers upon loading them # (either on startup or lazily when lazy loading is enabled). Setting to diff --git a/pkg/storegateway/indexheader/header.go b/pkg/storegateway/indexheader/header.go index 9319e9f58ef..6342934405d 100644 --- a/pkg/storegateway/indexheader/header.go +++ b/pkg/storegateway/indexheader/header.go @@ -61,14 +61,14 @@ type Config struct { MaxIdleFileHandles uint `yaml:"max_idle_file_handles" category:"advanced"` EagerLoadingStartupEnabled bool `yaml:"eager_loading_startup_enabled" category:"experimental"` // Controls whether index-header lazy loading is enabled. - LazyLoadingEnabled bool `yaml:"index_header_lazy_loading_enabled" category:"advanced"` - LazyLoadingIdleTimeout time.Duration `yaml:"index_header_lazy_loading_idle_timeout" category:"advanced"` + LazyLoadingEnabled bool `yaml:"lazy_loading_enabled" category:"advanced"` + LazyLoadingIdleTimeout time.Duration `yaml:"lazy_loading_idle_timeout" category:"advanced"` // Maximum index-headers loaded into store-gateway concurrently - LazyLoadingConcurrency int `yaml:"index_header_lazy_loading_concurrency" category:"experimental"` + LazyLoadingConcurrency int `yaml:"lazy_loading_concurrency" category:"experimental"` // Controls whether persisting a sparse version of the index-header to disk is enabled. - SparsePersistenceEnabled bool `yaml:"index_header_sparse_persistence_enabled" category:"experimental"` + SparsePersistenceEnabled bool `yaml:"sparse_persistence_enabled" category:"experimental"` VerifyOnLoad bool `yaml:"verify_on_load" category:"advanced"` } From ca66fda6ef434c8144c352ba2dc2f6ebf835fd86 Mon Sep 17 00:00:00 2001 From: Jon Kartago Lamida Date: Mon, 14 Aug 2023 21:52:36 +0800 Subject: [PATCH 11/20] Address PR feedbacks Signed-off-by: Jon Kartago Lamida From 7861c19189618863f09fe4e184295536bdef279d Mon Sep 17 00:00:00 2001 From: Jon Kartago Lamida Date: Mon, 14 Aug 2023 21:52:36 +0800 Subject: [PATCH 12/20] Move comment to the struct Signed-off-by: Jon Kartago Lamida --- pkg/storegateway/bucket.go | 1 - pkg/storegateway/indexheader/reader_pool.go | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/storegateway/bucket.go b/pkg/storegateway/bucket.go index 0b70f4f48a4..aaaf01bbd6b 100644 --- a/pkg/storegateway/bucket.go +++ b/pkg/storegateway/bucket.go @@ -269,7 +269,6 @@ func NewBucketStore( lazyLoadedSnapshotConfig := indexheader.LazyLoadedHeadersSnapshotConfig{ Path: dir, UserID: userID, - EagerLoadingEnabled: bucketStoreConfig.IndexHeader.IndexHeaderEagerLoadingStartupEnabled, } // Depend on the options s.indexReaderPool = indexheader.NewReaderPool(s.logger, bucketStoreConfig.IndexHeader, s.lazyLoadingGate, metrics.indexHeaderReaderMetrics, lazyLoadedSnapshotConfig) diff --git a/pkg/storegateway/indexheader/reader_pool.go b/pkg/storegateway/indexheader/reader_pool.go index dd77cb30b7a..b9fcb0a9597 100644 --- a/pkg/storegateway/indexheader/reader_pool.go +++ b/pkg/storegateway/indexheader/reader_pool.go @@ -67,8 +67,8 @@ type ReaderPool struct { // LazyLoadedHeadersSnapshotConfig stores information needed to track lazy loaded index headers. type LazyLoadedHeadersSnapshotConfig struct { // Path stores where lazy loaded blocks will be tracked in a single file per tenant - Path string - UserID string + Path string + UserID string } type lazyLoadedHeadersSnapshot struct { From ecc51a8ffe4972ec6b878a03ff6deca293fd5d8a Mon Sep 17 00:00:00 2001 From: Jon Kartago Lamida Date: Mon, 14 Aug 2023 22:56:19 +0800 Subject: [PATCH 13/20] Fix golangci-lint Signed-off-by: Jon Kartago Lamida --- pkg/storegateway/bucket.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/storegateway/bucket.go b/pkg/storegateway/bucket.go index aaaf01bbd6b..1067c2c2417 100644 --- a/pkg/storegateway/bucket.go +++ b/pkg/storegateway/bucket.go @@ -267,8 +267,8 @@ func NewBucketStore( } lazyLoadedSnapshotConfig := indexheader.LazyLoadedHeadersSnapshotConfig{ - Path: dir, - UserID: userID, + Path: dir, + UserID: userID, } // Depend on the options s.indexReaderPool = indexheader.NewReaderPool(s.logger, bucketStoreConfig.IndexHeader, s.lazyLoadingGate, metrics.indexHeaderReaderMetrics, lazyLoadedSnapshotConfig) From 847b2e0df6b8d1d906f67e0590e9d6a9eb1935a2 Mon Sep 17 00:00:00 2001 From: Jon Kartago Lamida Date: Thu, 17 Aug 2023 15:48:54 +0800 Subject: [PATCH 14/20] Move the deprecated flag set to setupModules Signed-off-by: Jon Kartago Lamida --- pkg/mimir/modules.go | 9 +++++++++ pkg/storage/tsdb/config.go | 9 --------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pkg/mimir/modules.go b/pkg/mimir/modules.go index 93c2f955e0d..466a0e6ecdf 100644 --- a/pkg/mimir/modules.go +++ b/pkg/mimir/modules.go @@ -828,6 +828,15 @@ func (t *Mimir) initCompactor() (serv services.Service, err error) { func (t *Mimir) initStoreGateway() (serv services.Service, err error) { t.Cfg.StoreGateway.ShardingRing.ListenPort = t.Cfg.Server.GRPCListenPort + // TODO: Remove in Mimir 2.12. + // Take value from Indexheader DeprecatedConfig if it is set + if t.Cfg.BlocksStorage.BucketStore.DeprecatedIndexHeaderLazyLoadingEnabled { + t.Cfg.BlocksStorage.BucketStore.IndexHeader.LazyLoadingEnabled = t.Cfg.BlocksStorage.BucketStore.DeprecatedIndexHeaderLazyLoadingEnabled + } + if t.Cfg.BlocksStorage.BucketStore.DeprecatedIndexHeaderLazyLoadingIdleTimeout > 0 { + t.Cfg.BlocksStorage.BucketStore.IndexHeader.LazyLoadingIdleTimeout = t.Cfg.BlocksStorage.BucketStore.DeprecatedIndexHeaderLazyLoadingIdleTimeout + } + t.StoreGateway, err = storegateway.NewStoreGateway(t.Cfg.StoreGateway, t.Cfg.BlocksStorage, t.Overrides, util_log.Logger, t.Registerer, t.ActivityTracker) if err != nil { return nil, err diff --git a/pkg/storage/tsdb/config.go b/pkg/storage/tsdb/config.go index 76e249dd533..6fcbd0878e3 100644 --- a/pkg/storage/tsdb/config.go +++ b/pkg/storage/tsdb/config.go @@ -434,15 +434,6 @@ func (cfg *BucketStoreConfig) RegisterFlags(f *flag.FlagSet) { cfg.BucketIndex.RegisterFlagsWithPrefix(f, bucketIndexFlagPrefix) cfg.IndexHeader.RegisterFlagsWithPrefix(f, "blocks-storage.bucket-store.index-header.") - // TODO: Remove in Mimir 2.12. - // Take value from Indexheader DeprecatedConfig if it is set - if cfg.DeprecatedIndexHeaderLazyLoadingEnabled { - cfg.IndexHeader.LazyLoadingEnabled = cfg.DeprecatedIndexHeaderLazyLoadingEnabled - } - if cfg.DeprecatedIndexHeaderLazyLoadingIdleTimeout > 0 { - cfg.IndexHeader.LazyLoadingIdleTimeout = cfg.DeprecatedIndexHeaderLazyLoadingIdleTimeout - } - f.StringVar(&cfg.SyncDir, "blocks-storage.bucket-store.sync-dir", "./tsdb-sync/", "Directory to store synchronized TSDB index headers. This directory is not required to be persisted between restarts, but it's highly recommended in order to improve the store-gateway startup time.") f.DurationVar(&cfg.SyncInterval, "blocks-storage.bucket-store.sync-interval", 15*time.Minute, "How frequently to scan the bucket, or to refresh the bucket index (if enabled), in order to look for changes (new blocks shipped by ingesters and blocks deleted by retention or compaction).") f.Uint64Var(&cfg.DeprecatedMaxChunkPoolBytes, maxChunksBytesPoolFlag, uint64(2*units.Gibibyte), "Max size - in bytes - of a chunks pool, used to reduce memory allocations. The pool is shared across all tenants. 0 to disable the limit.") From 982d0f1d8b37645709cd2b2ab5d79167d687817d Mon Sep 17 00:00:00 2001 From: Jon Kartago Lamida Date: Wed, 23 Aug 2023 10:18:03 +0800 Subject: [PATCH 15/20] Update changelog Signed-off-by: Jon Kartago Lamida --- CHANGELOG.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bbe27732541..d15cf20ad65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,10 +16,10 @@ * [CHANGE] The `-shutdown-delay` flag is no longer experimental. #5701 * [CHANGE] The `-validation.create-grace-period` is now enforced in the ingester too, other than distributor and query-frontend. If you've configured `-validation.create-grace-period` then make sure the configuration is applied to ingesters too. #5712 * [CHANGE] The `-validation.create-grace-period` is now enforced for examplars too in the distributor. If an examplar has timestamp greater than "now + grace_period", then the exemplar will be dropped and the metric `cortex_discarded_exemplars_total{reason="exemplar_too_far_in_future",user="..."}` increased. #5761 -* [CHANGE] Store-gateway: deprecate and move configuration parameters for index header under `blocks-storage.bucket-store` to `blocks-storage.bucket-store.index-header`, deprecated configuration will be removed in Mimir 2.12. Configuration changes: #5726 - * `-blocks-storage.bucket-store.index-header-lazy-loading-enabled` is deprecated and moved to `-blocks-storage.bucket-store.index-header.lazy-loading-enabled` - * `-blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout` is deprecated and moved to `-blocks-storage.bucket-store.index-header.lazy-loading-idle-timeout` - * `-blocks-storage.bucket-store.index-header-lazy-loading-concurrency` is moved to `-blocks-storage.bucket-store.index-header.lazy-loading-concurrency` +* [CHANGE] Store-gateway: deprecate configuration parameters for index header under `blocks-storage.bucket-store` and use a new configurations in `blocks-storage.bucket-store.index-header`, deprecated configuration will be removed in Mimir 2.12. Configuration changes: #5726 + * `-blocks-storage.bucket-store.index-header-lazy-loading-enabled` is deprecated, use the new configuration `-blocks-storage.bucket-store.index-header.lazy-loading-enabled` + * `-blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout` is deprecated, use the new configuration `-blocks-storage.bucket-store.index-header.lazy-loading-idle-timeout` + * `-blocks-storage.bucket-store.index-header-lazy-loading-concurrency` is deprecated, use the new configuration `-blocks-storage.bucket-store.index-header.lazy-loading-concurrency` * [FEATURE] Introduced `distributor.service_overload_status_code_on_rate_limit_enabled` flag for configuring status code to 529 instead of 429 upon rate limit exhaustion. #5752 * [FEATURE] Cardinality API: Add a new `count_method` parameter which enables counting active series #5136 * [FEATURE] Query-frontend: added experimental support to cache cardinality, label names and label values query responses. The cache will be used when `-query-frontend.cache-results` is enabled, and `-query-frontend.results-cache-ttl-for-cardinality-query` or `-query-frontend.results-cache-ttl-for-labels-query` set to a value greater than 0. The following metrics have been added to track the query results cache hit ratio per `request_type`: #5212 #5235 #5426 #5524 @@ -32,7 +32,7 @@ * `-ingester.log-utilization-based-limiter-cpu-samples` * [FEATURE] Ruler: Support filtering results from rule status endpoint by `file`, `rule_group` and `rule_name`. #5291 * [FEATURE] Ingester: add experimental support for creating tokens by using spread minimizing strategy. This can be enabled with `-ingester.ring.token-generation-strategy: spread-minimizing` and `-ingester.ring.spread-minimizing-zones: `. In that case `-ingester.ring.tokens-file-path` must be empty. #5308 #5324 -* [FEATURE] Storegateway: Persist sparse index-headers to disk and read from disk on index-header loads instead of reconstructing. #5465 #5651 +* [FEATURE] Storegateway: Persist sparse index-headers to disk and read from disk on index-header loads instead of reconstructing. #5465 #5651 #5726 * [FEATURE] Ingester: add experimental CLI flag `-ingester.ring.spread-minimizing-join-ring-in-order` that allows an ingester to register tokens in the ring only after all previous ingesters (with ID lower than its own ID) have already been registered. #5541 * [FEATURE] Ingester: add experimental support to compact the TSDB Head when the number of in-memory series is equal or greater than `-blocks-storage.tsdb.early-head-compaction-min-in-memory-series`, and the ingester estimates that the per-tenant TSDB Head compaction will reduce in-memory series by at least `-blocks-storage.tsdb.early-head-compaction-min-estimated-series-reduction-percentage`. #5371 * [FEATURE] Ingester: add new metrics for tracking native histograms in active series: `cortex_ingester_active_native_histogram_series`, `cortex_ingester_active_native_histogram_series_custom_tracker`, `cortex_ingester_active_native_histogram_buckets`, `cortex_ingester_active_native_histogram_buckets_custom_tracker`. The first 2 are the subsets of the existing and unmodified `cortex_ingester_active_series` and `cortex_ingester_active_series_custom_tracker` respectively, only tracking native histogram series, and the last 2 are the equivalents for tracking the number of buckets in native histogram series. #5318 From 22a5d0ccfd11d9712385a06822ca52abb4bc19e7 Mon Sep 17 00:00:00 2001 From: Jon Kartago Lamida Date: Wed, 23 Aug 2023 10:21:02 +0800 Subject: [PATCH 16/20] Take deprecated lazy loading flag only if it is the non default Signed-off-by: Jon Kartago Lamida --- pkg/mimir/modules.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/mimir/modules.go b/pkg/mimir/modules.go index 466a0e6ecdf..718ab8c61e7 100644 --- a/pkg/mimir/modules.go +++ b/pkg/mimir/modules.go @@ -829,8 +829,8 @@ func (t *Mimir) initStoreGateway() (serv services.Service, err error) { t.Cfg.StoreGateway.ShardingRing.ListenPort = t.Cfg.Server.GRPCListenPort // TODO: Remove in Mimir 2.12. - // Take value from Indexheader DeprecatedConfig if it is set - if t.Cfg.BlocksStorage.BucketStore.DeprecatedIndexHeaderLazyLoadingEnabled { + // Take non default value (index_header_lazy_loading_enabled=false) from Indexheader DeprecatedConfig + if !t.Cfg.BlocksStorage.BucketStore.DeprecatedIndexHeaderLazyLoadingEnabled { t.Cfg.BlocksStorage.BucketStore.IndexHeader.LazyLoadingEnabled = t.Cfg.BlocksStorage.BucketStore.DeprecatedIndexHeaderLazyLoadingEnabled } if t.Cfg.BlocksStorage.BucketStore.DeprecatedIndexHeaderLazyLoadingIdleTimeout > 0 { From 3e6a40fe3bd1d999d67058187a242ab3938e14d7 Mon Sep 17 00:00:00 2001 From: Jon Kartago Lamida Date: Wed, 23 Aug 2023 10:37:40 +0800 Subject: [PATCH 17/20] Take deprecated lazy loading flag only if it is the non default Signed-off-by: Jon Kartago Lamida --- pkg/mimir/modules.go | 9 ++++++--- pkg/storage/tsdb/config.go | 9 +++++++-- pkg/storegateway/indexheader/header.go | 5 +++-- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/pkg/mimir/modules.go b/pkg/mimir/modules.go index 718ab8c61e7..3e3ff18651c 100644 --- a/pkg/mimir/modules.go +++ b/pkg/mimir/modules.go @@ -50,6 +50,7 @@ import ( "github.com/grafana/mimir/pkg/ruler" "github.com/grafana/mimir/pkg/scheduler" "github.com/grafana/mimir/pkg/storage/bucket" + "github.com/grafana/mimir/pkg/storage/tsdb" "github.com/grafana/mimir/pkg/storegateway" "github.com/grafana/mimir/pkg/usagestats" "github.com/grafana/mimir/pkg/util" @@ -829,11 +830,13 @@ func (t *Mimir) initStoreGateway() (serv services.Service, err error) { t.Cfg.StoreGateway.ShardingRing.ListenPort = t.Cfg.Server.GRPCListenPort // TODO: Remove in Mimir 2.12. - // Take non default value (index_header_lazy_loading_enabled=false) from Indexheader DeprecatedConfig - if !t.Cfg.BlocksStorage.BucketStore.DeprecatedIndexHeaderLazyLoadingEnabled { + // Take the non-default value (false) from IndexHeader DeprecatedConfig + if t.Cfg.BlocksStorage.BucketStore.DeprecatedIndexHeaderLazyLoadingEnabled != tsdb.DefaultIndexHeaderLazyLoadingEnabled { t.Cfg.BlocksStorage.BucketStore.IndexHeader.LazyLoadingEnabled = t.Cfg.BlocksStorage.BucketStore.DeprecatedIndexHeaderLazyLoadingEnabled } - if t.Cfg.BlocksStorage.BucketStore.DeprecatedIndexHeaderLazyLoadingIdleTimeout > 0 { + // TODO: Remove in Mimir 2.12. + // Take the non-default value (1h) from IndexHeader DeprecatedConfig + if t.Cfg.BlocksStorage.BucketStore.DeprecatedIndexHeaderLazyLoadingIdleTimeout != tsdb.DefaultIndexHeaderLazyLoadingIdleTimeout { t.Cfg.BlocksStorage.BucketStore.IndexHeader.LazyLoadingIdleTimeout = t.Cfg.BlocksStorage.BucketStore.DeprecatedIndexHeaderLazyLoadingIdleTimeout } diff --git a/pkg/storage/tsdb/config.go b/pkg/storage/tsdb/config.go index 6fcbd0878e3..e8c051c2a2d 100644 --- a/pkg/storage/tsdb/config.go +++ b/pkg/storage/tsdb/config.go @@ -419,6 +419,11 @@ const ( AllPostingsStrategy = "all" ) +const ( + DefaultIndexHeaderLazyLoadingEnabled = true + DefaultIndexHeaderLazyLoadingIdleTimeout = 60 * time.Minute +) + var validSeriesSelectionStrategies = []string{ SpeculativePostingsStrategy, WorstCasePostingsStrategy, @@ -448,8 +453,8 @@ func (cfg *BucketStoreConfig) RegisterFlags(f *flag.FlagSet) { "The idea of ignore-deletion-marks-delay is to ignore blocks that are marked for deletion with some delay. This ensures store can still serve blocks that are meant to be deleted but do not have a replacement yet.") f.DurationVar(&cfg.IgnoreBlocksWithin, "blocks-storage.bucket-store.ignore-blocks-within", 10*time.Hour, "Blocks with minimum time within this duration are ignored, and not loaded by store-gateway. Useful when used together with -querier.query-store-after to prevent loading young blocks, because there are usually many of them (depending on number of ingesters) and they are not yet compacted. Negative values or 0 disable the filter.") f.IntVar(&cfg.PostingOffsetsInMemSampling, "blocks-storage.bucket-store.posting-offsets-in-mem-sampling", DefaultPostingOffsetInMemorySampling, "Controls what is the ratio of postings offsets that the store will hold in memory.") - f.BoolVar(&cfg.DeprecatedIndexHeaderLazyLoadingEnabled, "blocks-storage.bucket-store.index-header-lazy-loading-enabled", true, "If enabled, store-gateway will lazy load an index-header only once required by a query.") - f.DurationVar(&cfg.DeprecatedIndexHeaderLazyLoadingIdleTimeout, "blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout", 60*time.Minute, "If index-header lazy loading is enabled and this setting is > 0, the store-gateway will offload unused index-headers after 'idle timeout' inactivity.") + f.BoolVar(&cfg.DeprecatedIndexHeaderLazyLoadingEnabled, "blocks-storage.bucket-store.index-header-lazy-loading-enabled", DefaultIndexHeaderLazyLoadingEnabled, "If enabled, store-gateway will lazy load an index-header only once required by a query.") + f.DurationVar(&cfg.DeprecatedIndexHeaderLazyLoadingIdleTimeout, "blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout", DefaultIndexHeaderLazyLoadingIdleTimeout, "If index-header lazy loading is enabled and this setting is > 0, the store-gateway will offload unused index-headers after 'idle timeout' inactivity.") f.Uint64Var(&cfg.PartitionerMaxGapBytes, "blocks-storage.bucket-store.partitioner-max-gap-bytes", DefaultPartitionerMaxGapSize, "Max size - in bytes - of a gap for which the partitioner aggregates together two bucket GET object requests.") f.IntVar(&cfg.StreamingBatchSize, "blocks-storage.bucket-store.batch-series-size", 5000, "This option controls how many series to fetch per batch. The batch size must be greater than 0.") f.IntVar(&cfg.ChunkRangesPerSeries, "blocks-storage.bucket-store.fine-grained-chunks-caching-ranges-per-series", 1, "This option controls into how many ranges the chunks of each series from each block are split. This value is effectively the number of chunks cache items per series per block when -blocks-storage.bucket-store.chunks-cache.fine-grained-chunks-caching-enabled is enabled.") diff --git a/pkg/storegateway/indexheader/header.go b/pkg/storegateway/indexheader/header.go index 6342934405d..2d4cd5de807 100644 --- a/pkg/storegateway/indexheader/header.go +++ b/pkg/storegateway/indexheader/header.go @@ -13,6 +13,7 @@ import ( "github.com/pkg/errors" "github.com/prometheus/prometheus/tsdb/index" + "github.com/grafana/mimir/pkg/storage/tsdb" streamindex "github.com/grafana/mimir/pkg/storegateway/indexheader/index" ) @@ -74,8 +75,8 @@ type Config struct { func (cfg *Config) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string) { f.UintVar(&cfg.MaxIdleFileHandles, prefix+"max-idle-file-handles", 1, "Maximum number of idle file handles the store-gateway keeps open for each index-header file.") - f.BoolVar(&cfg.LazyLoadingEnabled, prefix+"lazy-loading-enabled", true, "If enabled, store-gateway will lazy load an index-header only once required by a query.") - f.DurationVar(&cfg.LazyLoadingIdleTimeout, prefix+"lazy-loading-idle-timeout", 60*time.Minute, "If index-header lazy loading is enabled and this setting is > 0, the store-gateway will offload unused index-headers after 'idle timeout' inactivity.") + f.BoolVar(&cfg.LazyLoadingEnabled, prefix+"lazy-loading-enabled", tsdb.DefaultIndexHeaderLazyLoadingEnabled, "If enabled, store-gateway will lazy load an index-header only once required by a query.") + f.DurationVar(&cfg.LazyLoadingIdleTimeout, prefix+"lazy-loading-idle-timeout", tsdb.DefaultIndexHeaderLazyLoadingIdleTimeout, "If index-header lazy loading is enabled and this setting is > 0, the store-gateway will offload unused index-headers after 'idle timeout' inactivity.") f.IntVar(&cfg.LazyLoadingConcurrency, prefix+"lazy-loading-concurrency", 0, "Maximum number of concurrent index header loads across all tenants. If set to 0, concurrency is unlimited.") f.BoolVar(&cfg.EagerLoadingStartupEnabled, prefix+"eager-loading-startup-enabled", false, "If enabled, store-gateway will periodically persist block IDs of lazy loaded index-headers and load them eagerly during startup. It is not valid to enable this if index-header lazy loading is disabled.") f.BoolVar(&cfg.SparsePersistenceEnabled, prefix+"sparse-persistence-enabled", false, "If enabled, store-gateway will persist a sparse version of the index-header to disk on construction and load sparse index-headers from disk instead of the whole index-header.") From 54cfc1cb3d5cd6133029fd53d08bc9f74cfa2fdf Mon Sep 17 00:00:00 2001 From: Jon Kartago Lamida Date: Wed, 23 Aug 2023 10:41:35 +0800 Subject: [PATCH 18/20] Fix const import causing import cycle Signed-off-by: Jon Kartago Lamida --- pkg/mimir/modules.go | 5 ++--- pkg/storage/tsdb/config.go | 9 ++------- pkg/storegateway/indexheader/header.go | 5 ++--- 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/pkg/mimir/modules.go b/pkg/mimir/modules.go index 3e3ff18651c..3bdcb27c07d 100644 --- a/pkg/mimir/modules.go +++ b/pkg/mimir/modules.go @@ -50,7 +50,6 @@ import ( "github.com/grafana/mimir/pkg/ruler" "github.com/grafana/mimir/pkg/scheduler" "github.com/grafana/mimir/pkg/storage/bucket" - "github.com/grafana/mimir/pkg/storage/tsdb" "github.com/grafana/mimir/pkg/storegateway" "github.com/grafana/mimir/pkg/usagestats" "github.com/grafana/mimir/pkg/util" @@ -831,12 +830,12 @@ func (t *Mimir) initStoreGateway() (serv services.Service, err error) { // TODO: Remove in Mimir 2.12. // Take the non-default value (false) from IndexHeader DeprecatedConfig - if t.Cfg.BlocksStorage.BucketStore.DeprecatedIndexHeaderLazyLoadingEnabled != tsdb.DefaultIndexHeaderLazyLoadingEnabled { + if !t.Cfg.BlocksStorage.BucketStore.DeprecatedIndexHeaderLazyLoadingEnabled { t.Cfg.BlocksStorage.BucketStore.IndexHeader.LazyLoadingEnabled = t.Cfg.BlocksStorage.BucketStore.DeprecatedIndexHeaderLazyLoadingEnabled } // TODO: Remove in Mimir 2.12. // Take the non-default value (1h) from IndexHeader DeprecatedConfig - if t.Cfg.BlocksStorage.BucketStore.DeprecatedIndexHeaderLazyLoadingIdleTimeout != tsdb.DefaultIndexHeaderLazyLoadingIdleTimeout { + if t.Cfg.BlocksStorage.BucketStore.DeprecatedIndexHeaderLazyLoadingIdleTimeout != 60*time.Minute { t.Cfg.BlocksStorage.BucketStore.IndexHeader.LazyLoadingIdleTimeout = t.Cfg.BlocksStorage.BucketStore.DeprecatedIndexHeaderLazyLoadingIdleTimeout } diff --git a/pkg/storage/tsdb/config.go b/pkg/storage/tsdb/config.go index e8c051c2a2d..6fcbd0878e3 100644 --- a/pkg/storage/tsdb/config.go +++ b/pkg/storage/tsdb/config.go @@ -419,11 +419,6 @@ const ( AllPostingsStrategy = "all" ) -const ( - DefaultIndexHeaderLazyLoadingEnabled = true - DefaultIndexHeaderLazyLoadingIdleTimeout = 60 * time.Minute -) - var validSeriesSelectionStrategies = []string{ SpeculativePostingsStrategy, WorstCasePostingsStrategy, @@ -453,8 +448,8 @@ func (cfg *BucketStoreConfig) RegisterFlags(f *flag.FlagSet) { "The idea of ignore-deletion-marks-delay is to ignore blocks that are marked for deletion with some delay. This ensures store can still serve blocks that are meant to be deleted but do not have a replacement yet.") f.DurationVar(&cfg.IgnoreBlocksWithin, "blocks-storage.bucket-store.ignore-blocks-within", 10*time.Hour, "Blocks with minimum time within this duration are ignored, and not loaded by store-gateway. Useful when used together with -querier.query-store-after to prevent loading young blocks, because there are usually many of them (depending on number of ingesters) and they are not yet compacted. Negative values or 0 disable the filter.") f.IntVar(&cfg.PostingOffsetsInMemSampling, "blocks-storage.bucket-store.posting-offsets-in-mem-sampling", DefaultPostingOffsetInMemorySampling, "Controls what is the ratio of postings offsets that the store will hold in memory.") - f.BoolVar(&cfg.DeprecatedIndexHeaderLazyLoadingEnabled, "blocks-storage.bucket-store.index-header-lazy-loading-enabled", DefaultIndexHeaderLazyLoadingEnabled, "If enabled, store-gateway will lazy load an index-header only once required by a query.") - f.DurationVar(&cfg.DeprecatedIndexHeaderLazyLoadingIdleTimeout, "blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout", DefaultIndexHeaderLazyLoadingIdleTimeout, "If index-header lazy loading is enabled and this setting is > 0, the store-gateway will offload unused index-headers after 'idle timeout' inactivity.") + f.BoolVar(&cfg.DeprecatedIndexHeaderLazyLoadingEnabled, "blocks-storage.bucket-store.index-header-lazy-loading-enabled", true, "If enabled, store-gateway will lazy load an index-header only once required by a query.") + f.DurationVar(&cfg.DeprecatedIndexHeaderLazyLoadingIdleTimeout, "blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout", 60*time.Minute, "If index-header lazy loading is enabled and this setting is > 0, the store-gateway will offload unused index-headers after 'idle timeout' inactivity.") f.Uint64Var(&cfg.PartitionerMaxGapBytes, "blocks-storage.bucket-store.partitioner-max-gap-bytes", DefaultPartitionerMaxGapSize, "Max size - in bytes - of a gap for which the partitioner aggregates together two bucket GET object requests.") f.IntVar(&cfg.StreamingBatchSize, "blocks-storage.bucket-store.batch-series-size", 5000, "This option controls how many series to fetch per batch. The batch size must be greater than 0.") f.IntVar(&cfg.ChunkRangesPerSeries, "blocks-storage.bucket-store.fine-grained-chunks-caching-ranges-per-series", 1, "This option controls into how many ranges the chunks of each series from each block are split. This value is effectively the number of chunks cache items per series per block when -blocks-storage.bucket-store.chunks-cache.fine-grained-chunks-caching-enabled is enabled.") diff --git a/pkg/storegateway/indexheader/header.go b/pkg/storegateway/indexheader/header.go index 2d4cd5de807..6342934405d 100644 --- a/pkg/storegateway/indexheader/header.go +++ b/pkg/storegateway/indexheader/header.go @@ -13,7 +13,6 @@ import ( "github.com/pkg/errors" "github.com/prometheus/prometheus/tsdb/index" - "github.com/grafana/mimir/pkg/storage/tsdb" streamindex "github.com/grafana/mimir/pkg/storegateway/indexheader/index" ) @@ -75,8 +74,8 @@ type Config struct { func (cfg *Config) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string) { f.UintVar(&cfg.MaxIdleFileHandles, prefix+"max-idle-file-handles", 1, "Maximum number of idle file handles the store-gateway keeps open for each index-header file.") - f.BoolVar(&cfg.LazyLoadingEnabled, prefix+"lazy-loading-enabled", tsdb.DefaultIndexHeaderLazyLoadingEnabled, "If enabled, store-gateway will lazy load an index-header only once required by a query.") - f.DurationVar(&cfg.LazyLoadingIdleTimeout, prefix+"lazy-loading-idle-timeout", tsdb.DefaultIndexHeaderLazyLoadingIdleTimeout, "If index-header lazy loading is enabled and this setting is > 0, the store-gateway will offload unused index-headers after 'idle timeout' inactivity.") + f.BoolVar(&cfg.LazyLoadingEnabled, prefix+"lazy-loading-enabled", true, "If enabled, store-gateway will lazy load an index-header only once required by a query.") + f.DurationVar(&cfg.LazyLoadingIdleTimeout, prefix+"lazy-loading-idle-timeout", 60*time.Minute, "If index-header lazy loading is enabled and this setting is > 0, the store-gateway will offload unused index-headers after 'idle timeout' inactivity.") f.IntVar(&cfg.LazyLoadingConcurrency, prefix+"lazy-loading-concurrency", 0, "Maximum number of concurrent index header loads across all tenants. If set to 0, concurrency is unlimited.") f.BoolVar(&cfg.EagerLoadingStartupEnabled, prefix+"eager-loading-startup-enabled", false, "If enabled, store-gateway will periodically persist block IDs of lazy loaded index-headers and load them eagerly during startup. It is not valid to enable this if index-header lazy loading is disabled.") f.BoolVar(&cfg.SparsePersistenceEnabled, prefix+"sparse-persistence-enabled", false, "If enabled, store-gateway will persist a sparse version of the index-header to disk on construction and load sparse index-headers from disk instead of the whole index-header.") From 6bb5dba46fe3a790f7069f599ab10ab4eed2fec5 Mon Sep 17 00:00:00 2001 From: Jon Kartago Lamida Date: Wed, 23 Aug 2023 16:02:21 +0800 Subject: [PATCH 19/20] Update comment to make it less confusing Signed-off-by: Jon Kartago Lamida --- pkg/mimir/modules.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/mimir/modules.go b/pkg/mimir/modules.go index 3bdcb27c07d..d0a6815c281 100644 --- a/pkg/mimir/modules.go +++ b/pkg/mimir/modules.go @@ -829,12 +829,12 @@ func (t *Mimir) initStoreGateway() (serv services.Service, err error) { t.Cfg.StoreGateway.ShardingRing.ListenPort = t.Cfg.Server.GRPCListenPort // TODO: Remove in Mimir 2.12. - // Take the non-default value (false) from IndexHeader DeprecatedConfig + // Take the non-default value (default value is false) from IndexHeader deprecated config and if it is set, set it to the new config if !t.Cfg.BlocksStorage.BucketStore.DeprecatedIndexHeaderLazyLoadingEnabled { t.Cfg.BlocksStorage.BucketStore.IndexHeader.LazyLoadingEnabled = t.Cfg.BlocksStorage.BucketStore.DeprecatedIndexHeaderLazyLoadingEnabled } // TODO: Remove in Mimir 2.12. - // Take the non-default value (1h) from IndexHeader DeprecatedConfig + // Take the non-default value (default value is 1h) from IndexHeader deprecated config and if it is set, set it to the new config if t.Cfg.BlocksStorage.BucketStore.DeprecatedIndexHeaderLazyLoadingIdleTimeout != 60*time.Minute { t.Cfg.BlocksStorage.BucketStore.IndexHeader.LazyLoadingIdleTimeout = t.Cfg.BlocksStorage.BucketStore.DeprecatedIndexHeaderLazyLoadingIdleTimeout } From c7cb6f1639b5538c60f1f3a17dbae1b51ae886c5 Mon Sep 17 00:00:00 2001 From: Jon Kartago Lamida Date: Wed, 23 Aug 2023 22:25:29 +0800 Subject: [PATCH 20/20] Extract constant for deprecated lazyloading configs Signed-off-by: Jon Kartago Lamida --- pkg/mimir/modules.go | 7 +++---- pkg/storage/tsdb/config.go | 4 ++-- pkg/storegateway/indexheader/header.go | 9 +++++++-- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/pkg/mimir/modules.go b/pkg/mimir/modules.go index d0a6815c281..af26db37a71 100644 --- a/pkg/mimir/modules.go +++ b/pkg/mimir/modules.go @@ -51,6 +51,7 @@ import ( "github.com/grafana/mimir/pkg/scheduler" "github.com/grafana/mimir/pkg/storage/bucket" "github.com/grafana/mimir/pkg/storegateway" + "github.com/grafana/mimir/pkg/storegateway/indexheader" "github.com/grafana/mimir/pkg/usagestats" "github.com/grafana/mimir/pkg/util" "github.com/grafana/mimir/pkg/util/activitytracker" @@ -829,13 +830,11 @@ func (t *Mimir) initStoreGateway() (serv services.Service, err error) { t.Cfg.StoreGateway.ShardingRing.ListenPort = t.Cfg.Server.GRPCListenPort // TODO: Remove in Mimir 2.12. - // Take the non-default value (default value is false) from IndexHeader deprecated config and if it is set, set it to the new config - if !t.Cfg.BlocksStorage.BucketStore.DeprecatedIndexHeaderLazyLoadingEnabled { + if t.Cfg.BlocksStorage.BucketStore.DeprecatedIndexHeaderLazyLoadingEnabled != indexheader.DefaultIndexHeaderLazyLoadingEnabled { t.Cfg.BlocksStorage.BucketStore.IndexHeader.LazyLoadingEnabled = t.Cfg.BlocksStorage.BucketStore.DeprecatedIndexHeaderLazyLoadingEnabled } // TODO: Remove in Mimir 2.12. - // Take the non-default value (default value is 1h) from IndexHeader deprecated config and if it is set, set it to the new config - if t.Cfg.BlocksStorage.BucketStore.DeprecatedIndexHeaderLazyLoadingIdleTimeout != 60*time.Minute { + if t.Cfg.BlocksStorage.BucketStore.DeprecatedIndexHeaderLazyLoadingIdleTimeout != indexheader.DefaultIndexHeaderLazyLoadingIdleTimeout { t.Cfg.BlocksStorage.BucketStore.IndexHeader.LazyLoadingIdleTimeout = t.Cfg.BlocksStorage.BucketStore.DeprecatedIndexHeaderLazyLoadingIdleTimeout } diff --git a/pkg/storage/tsdb/config.go b/pkg/storage/tsdb/config.go index 6fcbd0878e3..6fe3a82297b 100644 --- a/pkg/storage/tsdb/config.go +++ b/pkg/storage/tsdb/config.go @@ -448,8 +448,8 @@ func (cfg *BucketStoreConfig) RegisterFlags(f *flag.FlagSet) { "The idea of ignore-deletion-marks-delay is to ignore blocks that are marked for deletion with some delay. This ensures store can still serve blocks that are meant to be deleted but do not have a replacement yet.") f.DurationVar(&cfg.IgnoreBlocksWithin, "blocks-storage.bucket-store.ignore-blocks-within", 10*time.Hour, "Blocks with minimum time within this duration are ignored, and not loaded by store-gateway. Useful when used together with -querier.query-store-after to prevent loading young blocks, because there are usually many of them (depending on number of ingesters) and they are not yet compacted. Negative values or 0 disable the filter.") f.IntVar(&cfg.PostingOffsetsInMemSampling, "blocks-storage.bucket-store.posting-offsets-in-mem-sampling", DefaultPostingOffsetInMemorySampling, "Controls what is the ratio of postings offsets that the store will hold in memory.") - f.BoolVar(&cfg.DeprecatedIndexHeaderLazyLoadingEnabled, "blocks-storage.bucket-store.index-header-lazy-loading-enabled", true, "If enabled, store-gateway will lazy load an index-header only once required by a query.") - f.DurationVar(&cfg.DeprecatedIndexHeaderLazyLoadingIdleTimeout, "blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout", 60*time.Minute, "If index-header lazy loading is enabled and this setting is > 0, the store-gateway will offload unused index-headers after 'idle timeout' inactivity.") + f.BoolVar(&cfg.DeprecatedIndexHeaderLazyLoadingEnabled, "blocks-storage.bucket-store.index-header-lazy-loading-enabled", indexheader.DefaultIndexHeaderLazyLoadingEnabled, "If enabled, store-gateway will lazy load an index-header only once required by a query.") + f.DurationVar(&cfg.DeprecatedIndexHeaderLazyLoadingIdleTimeout, "blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout", indexheader.DefaultIndexHeaderLazyLoadingIdleTimeout, "If index-header lazy loading is enabled and this setting is > 0, the store-gateway will offload unused index-headers after 'idle timeout' inactivity.") f.Uint64Var(&cfg.PartitionerMaxGapBytes, "blocks-storage.bucket-store.partitioner-max-gap-bytes", DefaultPartitionerMaxGapSize, "Max size - in bytes - of a gap for which the partitioner aggregates together two bucket GET object requests.") f.IntVar(&cfg.StreamingBatchSize, "blocks-storage.bucket-store.batch-series-size", 5000, "This option controls how many series to fetch per batch. The batch size must be greater than 0.") f.IntVar(&cfg.ChunkRangesPerSeries, "blocks-storage.bucket-store.fine-grained-chunks-caching-ranges-per-series", 1, "This option controls into how many ranges the chunks of each series from each block are split. This value is effectively the number of chunks cache items per series per block when -blocks-storage.bucket-store.chunks-cache.fine-grained-chunks-caching-enabled is enabled.") diff --git a/pkg/storegateway/indexheader/header.go b/pkg/storegateway/indexheader/header.go index 6342934405d..43a6cfdb57a 100644 --- a/pkg/storegateway/indexheader/header.go +++ b/pkg/storegateway/indexheader/header.go @@ -16,6 +16,11 @@ import ( streamindex "github.com/grafana/mimir/pkg/storegateway/indexheader/index" ) +const ( + DefaultIndexHeaderLazyLoadingEnabled = true + DefaultIndexHeaderLazyLoadingIdleTimeout = 60 * time.Minute +) + // NotFoundRangeErr is an error returned by PostingsOffset when there is no posting for given name and value pairs. var NotFoundRangeErr = errors.New("range not found") //nolint:revive @@ -74,8 +79,8 @@ type Config struct { func (cfg *Config) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string) { f.UintVar(&cfg.MaxIdleFileHandles, prefix+"max-idle-file-handles", 1, "Maximum number of idle file handles the store-gateway keeps open for each index-header file.") - f.BoolVar(&cfg.LazyLoadingEnabled, prefix+"lazy-loading-enabled", true, "If enabled, store-gateway will lazy load an index-header only once required by a query.") - f.DurationVar(&cfg.LazyLoadingIdleTimeout, prefix+"lazy-loading-idle-timeout", 60*time.Minute, "If index-header lazy loading is enabled and this setting is > 0, the store-gateway will offload unused index-headers after 'idle timeout' inactivity.") + f.BoolVar(&cfg.LazyLoadingEnabled, prefix+"lazy-loading-enabled", DefaultIndexHeaderLazyLoadingEnabled, "If enabled, store-gateway will lazy load an index-header only once required by a query.") + f.DurationVar(&cfg.LazyLoadingIdleTimeout, prefix+"lazy-loading-idle-timeout", DefaultIndexHeaderLazyLoadingIdleTimeout, "If index-header lazy loading is enabled and this setting is > 0, the store-gateway will offload unused index-headers after 'idle timeout' inactivity.") f.IntVar(&cfg.LazyLoadingConcurrency, prefix+"lazy-loading-concurrency", 0, "Maximum number of concurrent index header loads across all tenants. If set to 0, concurrency is unlimited.") f.BoolVar(&cfg.EagerLoadingStartupEnabled, prefix+"eager-loading-startup-enabled", false, "If enabled, store-gateway will periodically persist block IDs of lazy loaded index-headers and load them eagerly during startup. It is not valid to enable this if index-header lazy loading is disabled.") f.BoolVar(&cfg.SparsePersistenceEnabled, prefix+"sparse-persistence-enabled", false, "If enabled, store-gateway will persist a sparse version of the index-header to disk on construction and load sparse index-headers from disk instead of the whole index-header.")