store-gateway: Eagerly Load Lazy Loaded Index Header (#5596)

* Start to do eager load of index header during startup Signed-off-by: Jon Kartago Lamida <[email protected]> * Make EagerLoading as public method Signed-off-by: Jon Kartago Lamida <[email protected]> * Add eager loading test Signed-off-by: Jon Kartago Lamida <[email protected]> * Small refactoring of test Signed-off-by: Jon Kartago Lamida <[email protected]> * Add godoc Signed-off-by: Jon Kartago Lamida <[email protected]> * Update reader_pool_test with eager loading logic Signed-off-by: Jon Kartago Lamida <[email protected]> * Small refactoring to make test clearer Signed-off-by: Jon Kartago Lamida <[email protected]> * Appease the linter Signed-off-by: Jon Kartago Lamida <[email protected]> * Rename private method for test Signed-off-by: Jon Kartago Lamida <[email protected]> * Add config to enable or disable index headers eager loading Signed-off-by: Jon Kartago Lamida <[email protected]> * Changelog Signed-off-by: Jon Kartago Lamida <[email protected]> * Changelog Signed-off-by: Jon Kartago Lamida <[email protected]> * Go select can't be inside method Signed-off-by: Jon Kartago Lamida <[email protected]> * Run make doc Signed-off-by: Jon Kartago Lamida <[email protected]> * Fix test Signed-off-by: Jon Kartago Lamida <[email protected]> * Update CHANGELOG.md Co-authored-by: Charles Korn <[email protected]> * Update pkg/storegateway/indexheader/lazy_binary_reader.go Co-authored-by: Charles Korn <[email protected]> * Apply configs PR feedbacks Signed-off-by: Jon Kartago Lamida <[email protected]> * Pass BucketStoreConfig as argument Signed-off-by: Jon Kartago Lamida <[email protected]> * Pass eagerLoadIndexReaderEnabled into snapshotConfig Signed-off-by: Jon Kartago Lamida <[email protected]> * Don't pass lazyLoadedSnapshot into LazyBinaryReader Signed-off-by: Jon Kartago Lamida <[email protected]> * Only initialize tickerLazyLoad when flag is enabled Signed-off-by: Jon Kartago Lamida <[email protected]> * Apply PR feedbacks Signed-off-by: Jon Kartago Lamida <[email protected]> * Update config description Signed-off-by: Jon Kartago Lamida <[email protected]> * Update usedAt to current time in eager loading Signed-off-by: Jon Kartago Lamida <[email protected]> * Update pkg/storegateway/indexheader/reader_pool.go Co-authored-by: Dimitar Dimitrov <[email protected]> * Fix snapshotByte name Signed-off-by: Jon Kartago Lamida <[email protected]> * Address PR feedbacks Signed-off-by: Jon Kartago Lamida <[email protected]> * Rename initLazyBinaryForTest Signed-off-by: Jon Kartago Lamida <[email protected]> * Apply PR suggestion Signed-off-by: Jon Kartago Lamida <[email protected]> * Refactor test Signed-off-by: Jon Kartago Lamida <[email protected]> * Fix comment Signed-off-by: Jon Kartago Lamida <[email protected]> * Apply more PR suggestions Signed-off-by: Jon Kartago Lamida <[email protected]> * Refactor configs Signed-off-by: Jon Kartago Lamida <[email protected]> * Header eager loading only must be done during initialSync Signed-off-by: Jon Kartago Lamida <[email protected]> * Remove unnecesary comment Signed-off-by: Jon Kartago Lamida <[email protected]> * Remove snapshot file regardless snapshot loading result Signed-off-by: Jon Kartago Lamida <[email protected]> * Fix compilation error Signed-off-by: Jon Kartago Lamida <[email protected]> * Update pkg/storegateway/indexheader/header.go Co-authored-by: Charles Korn <[email protected]> * Regenerate doc for updated command line args Signed-off-by: Jon Kartago Lamida <[email protected]> * Update error log Signed-off-by: Jon Kartago Lamida <[email protected]> * Update comment Signed-off-by: Jon Kartago Lamida <[email protected]> * Update comment Signed-off-by: Jon Kartago Lamida <[email protected]> * Update test Signed-off-by: Jon Kartago Lamida <[email protected]> * Add test case Signed-off-by: Jon Kartago Lamida <[email protected]> * Update pkg/storegateway/indexheader/reader_pool.go Co-authored-by: Charles Korn <[email protected]> * Update pkg/storegateway/indexheader/reader_pool_test.go Co-authored-by: Charles Korn <[email protected]> * Update pkg/storegateway/indexheader/reader_pool_test.go Co-authored-by: Charles Korn <[email protected]> * Update pkg/storegateway/indexheader/lazy_binary_reader.go Co-authored-by: Charles Korn <[email protected]> * Update pkg/storegateway/indexheader/reader_pool.go Co-authored-by: Charles Korn <[email protected]> * Update pkg/storegateway/indexheader/header.go Co-authored-by: Charles Korn <[email protected]> * Update changelog Signed-off-by: Jon Kartago Lamida <[email protected]> --------- Signed-off-by: Jon Kartago Lamida <[email protected]> Co-authored-by: Charles Korn <[email protected]> Co-authored-by: Dimitar Dimitrov <[email protected]>
grafana · Aug 11, 2023 · 2109b06 · 2109b06
1 parent a03dcb1
commit 2109b06
Show file tree

Hide file tree

Showing 14 changed files with 408 additions and 190 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -181,6 +181,7 @@
 * [CHANGE] Compactor: change default of `-compactor.first-level-compaction-wait-period` to 25m. #5128
 * [CHANGE] Ruler: changed default of `-ruler.poll-interval` from `1m` to `10m`. Starting from this release, the configured rule groups will also be re-synced each time they're modified calling the ruler configuration API. #5170
 * [FEATURE] Query-frontend: add `-query-frontend.log-query-request-headers` to enable logging of request headers in query logs. #5030
+* [FEATURE] Store-gateway: add experimental feature to retain lazy-loaded index headers between restarts by eagerly loading them during startup. This is disabled by default and can only be enabled if lazy loading is enabled. #5606
 * [ENHANCEMENT] Add per-tenant limit `-validation.max-native-histogram-buckets` to be able to ignore native histogram samples that have too many buckets. #4765
 * [ENHANCEMENT] Store-gateway: reduce memory usage in some LabelValues calls. #4789
 * [ENHANCEMENT] Store-gateway: add a `stage` label to the metric `cortex_bucket_store_series_data_touched`. This label now applies to `data_type="chunks"` and `data_type="series"`. The `stage` label has 2 values: `processed` - the number of series that parsed - and `returned` - the number of series selected from the processed bytes to satisfy the query. #4797 #4830

diff --git a/cmd/mimir/config-descriptor.json b/cmd/mimir/config-descriptor.json
@@ -7516,13 +7516,24 @@
                   "kind": "field",
                   "name": "max_idle_file_handles",
                   "required": false,
-                  "desc": "Maximum number of idle file handles the store-gateway keeps open for each index header file.",
+                  "desc": "Maximum number of idle file handles the store-gateway keeps open for each index-header file.",
                   "fieldValue": null,
                   "fieldDefaultValue": 1,
                   "fieldFlag": "blocks-storage.bucket-store.index-header.max-idle-file-handles",
                   "fieldType": "int",
                   "fieldCategory": "advanced"
                 },
+                {
+                  "kind": "field",
+                  "name": "eager_loading_startup_enabled",
+                  "required": false,
+                  "desc": "If enabled, store-gateway will periodically persist block IDs of lazy loaded index-headers and load them eagerly during startup. It is not valid to enable this if index-header lazy loading is disabled.",
+                  "fieldValue": null,
+                  "fieldDefaultValue": false,
+                  "fieldFlag": "blocks-storage.bucket-store.index-header.eager-loading-startup-enabled",
+                  "fieldType": "boolean",
+                  "fieldCategory": "experimental"
+                },
                 {
                   "kind": "field",
                   "name": "verify_on_load",

diff --git a/cmd/mimir/help-all.txt.tmpl b/cmd/mimir/help-all.txt.tmpl
@@ -503,8 +503,10 @@ Usage of ./cmd/mimir/mimir:
     	If index-header lazy loading is enabled and this setting is > 0, the store-gateway will offload unused index-headers after 'idle timeout' inactivity. (default 1h0m0s)
   -blocks-storage.bucket-store.index-header-sparse-persistence-enabled
     	[experimental] If enabled, store-gateway will persist a sparse version of the index-header to disk on construction and load sparse index-headers from disk instead of the whole index-header.
+  -blocks-storage.bucket-store.index-header.eager-loading-startup-enabled
+    	[experimental] If enabled, store-gateway will periodically persist block IDs of lazy loaded index-headers and load them eagerly during startup. It is not valid to enable this if index-header lazy loading is disabled.
   -blocks-storage.bucket-store.index-header.max-idle-file-handles uint
-    	Maximum number of idle file handles the store-gateway keeps open for each index header file. (default 1)
+    	Maximum number of idle file handles the store-gateway keeps open for each index-header file. (default 1)
   -blocks-storage.bucket-store.index-header.verify-on-load
     	If true, verify the checksum of index headers upon loading them (either on startup or lazily when lazy loading is enabled). Setting to true helps detect disk corruption at the cost of slowing down index header loading.
   -blocks-storage.bucket-store.max-chunk-pool-bytes uint

diff --git a/docs/sources/mimir/references/configuration-parameters/index.md b/docs/sources/mimir/references/configuration-parameters/index.md
@@ -3412,10 +3412,16 @@ bucket_store:
 
   index_header:
     # (advanced) Maximum number of idle file handles the store-gateway keeps
-    # open for each index header file.
+    # open for each index-header file.
     # CLI flag: -blocks-storage.bucket-store.index-header.max-idle-file-handles
     [max_idle_file_handles: <int> | default = 1]
 
+    # (experimental) If enabled, store-gateway will periodically persist block
+    # IDs of lazy loaded index-headers and load them eagerly during startup. It
+    # is not valid to enable this if index-header lazy loading is disabled.
+    # CLI flag: -blocks-storage.bucket-store.index-header.eager-loading-startup-enabled
+    [eager_loading_startup_enabled: <boolean> | default = false]
+
     # (advanced) If true, verify the checksum of index headers upon loading them
     # (either on startup or lazily when lazy loading is enabled). Setting to
     # true helps detect disk corruption at the cost of slowing down index header

diff --git a/pkg/storage/tsdb/config.go b/pkg/storage/tsdb/config.go
@@ -498,6 +498,9 @@ func (cfg *BucketStoreConfig) Validate(logger log.Logger) error {
 	if cfg.SeriesSelectionStrategyName == WorstCasePostingsStrategy && cfg.SelectionStrategies.WorstCaseSeriesPreference <= 0 {
 		return errors.New("invalid worst-case series preference; must be positive")
 	}
+	if err := cfg.IndexHeader.Validate(cfg.IndexHeaderLazyLoadingEnabled); err != nil {
+		return errors.Wrap(err, "index-header configuration")
+	}
 	if cfg.IndexHeaderLazyLoadingConcurrency < 0 {
 		return errInvalidIndexHeaderLazyLoadingConcurrency
 	}

diff --git a/pkg/storegateway/bucket.go b/pkg/storegateway/bucket.go
@@ -42,6 +42,7 @@ import (
 
 	"github.com/grafana/mimir/pkg/mimirpb"
 	"github.com/grafana/mimir/pkg/storage/sharding"
+	"github.com/grafana/mimir/pkg/storage/tsdb"
 	"github.com/grafana/mimir/pkg/storage/tsdb/block"
 	"github.com/grafana/mimir/pkg/storage/tsdb/bucketcache"
 	"github.com/grafana/mimir/pkg/storegateway/chunkscache"
@@ -227,18 +228,11 @@ func NewBucketStore(
 	bkt objstore.InstrumentedBucketReader,
 	fetcher block.MetadataFetcher,
 	dir string,
-	maxSeriesPerBatch int,
-	numChunksRangesPerSeries int,
+	bucketStoreConfig tsdb.BucketStoreConfig,
 	postingsStrategy postingsSelectionStrategy,
 	chunksLimiterFactory ChunksLimiterFactory,
 	seriesLimiterFactory SeriesLimiterFactory,
 	partitioners blockPartitioners,
-	blockSyncConcurrency int,
-	postingOffsetsInMemSampling int,
-	indexHeaderCfg indexheader.Config,
-	lazyIndexReaderEnabled bool,
-	lazyIndexReaderIdleTimeout time.Duration,
-	sparsePersistenceEnabled bool,
 	seriesHashCache *hashcache.SeriesHashCache,
 	metrics *BucketStoreMetrics,
 	options ...BucketStoreOption,
@@ -252,19 +246,19 @@ func NewBucketStore(
 		chunksCache:                 chunkscache.NoopCache{},
 		blocks:                      map[ulid.ULID]*bucketBlock{},
 		blockSet:                    newBucketBlockSet(),
-		blockSyncConcurrency:        blockSyncConcurrency,
+		blockSyncConcurrency:        bucketStoreConfig.BlockSyncConcurrency,
 		queryGate:                   gate.NewNoop(),
 		lazyLoadingGate:             gate.NewNoop(),
 		chunksLimiterFactory:        chunksLimiterFactory,
 		seriesLimiterFactory:        seriesLimiterFactory,
 		partitioners:                partitioners,
-		postingOffsetsInMemSampling: postingOffsetsInMemSampling,
-		indexHeaderCfg:              indexHeaderCfg,
+		postingOffsetsInMemSampling: bucketStoreConfig.PostingOffsetsInMemSampling,
+		indexHeaderCfg:              bucketStoreConfig.IndexHeader,
 		seriesHashCache:             seriesHashCache,
 		metrics:                     metrics,
 		userID:                      userID,
-		maxSeriesPerBatch:           maxSeriesPerBatch,
-		numChunksRangesPerSeries:    numChunksRangesPerSeries,
+		maxSeriesPerBatch:           bucketStoreConfig.StreamingBatchSize,
+		numChunksRangesPerSeries:    bucketStoreConfig.ChunkRangesPerSeries,
 		postingsStrategy:            postingsStrategy,
 	}
 
@@ -274,11 +268,12 @@ func NewBucketStore(
 
 	lazyLoadedSnapshotConfig := indexheader.LazyLoadedHeadersSnapshotConfig{
 		// Path stores where lazy loaded blocks will be tracked in a single file per tenant
-		Path:   dir,
-		UserID: userID,
+		Path:                dir,
+		UserID:              userID,
+		EagerLoadingEnabled: bucketStoreConfig.IndexHeader.IndexHeaderEagerLoadingStartupEnabled,
 	}
 	// Depend on the options
-	s.indexReaderPool = indexheader.NewReaderPool(s.logger, lazyIndexReaderEnabled, lazyIndexReaderIdleTimeout, sparsePersistenceEnabled, s.lazyLoadingGate, metrics.indexHeaderReaderMetrics, lazyLoadedSnapshotConfig)
+	s.indexReaderPool = indexheader.NewReaderPool(s.logger, bucketStoreConfig.IndexHeaderLazyLoadingEnabled, bucketStoreConfig.IndexHeaderLazyLoadingIdleTimeout, bucketStoreConfig.IndexHeaderSparsePersistenceEnabled, s.lazyLoadingGate, metrics.indexHeaderReaderMetrics, lazyLoadedSnapshotConfig)
 
 	if err := os.MkdirAll(dir, 0750); err != nil {
 		return nil, errors.Wrap(err, "create dir")
@@ -311,6 +306,10 @@ func (s *BucketStore) Stats() BucketStoreStats {
 // SyncBlocks synchronizes the stores state with the Bucket bucket.
 // It will reuse disk space as persistent cache based on s.dir param.
 func (s *BucketStore) SyncBlocks(ctx context.Context) error {
+	return s.syncBlocks(ctx, false)
+}
+
+func (s *BucketStore) syncBlocks(ctx context.Context, initialSync bool) error {
 	metas, _, metaFetchErr := s.fetcher.Fetch(ctx)
 	// For partial view allow adding new blocks at least.
 	if metaFetchErr != nil && metas == nil {
@@ -324,7 +323,7 @@ func (s *BucketStore) SyncBlocks(ctx context.Context) error {
 		wg.Add(1)
 		go func() {
 			for meta := range blockc {
-				if err := s.addBlock(ctx, meta); err != nil {
+				if err := s.addBlock(ctx, meta, initialSync); err != nil {
 					continue
 				}
 			}
@@ -366,7 +365,7 @@ func (s *BucketStore) SyncBlocks(ctx context.Context) error {
 // InitialSync perform blocking sync with extra step at the end to delete locally saved blocks that are no longer
 // present in the bucket. The mismatch of these can only happen between restarts, so we can do that only once per startup.
 func (s *BucketStore) InitialSync(ctx context.Context) error {
-	if err := s.SyncBlocks(ctx); err != nil {
+	if err := s.syncBlocks(ctx, true); err != nil {
 		return errors.Wrap(err, "sync block")
 	}
 
@@ -402,7 +401,7 @@ func (s *BucketStore) getBlock(id ulid.ULID) *bucketBlock {
 	return s.blocks[id]
 }
 
-func (s *BucketStore) addBlock(ctx context.Context, meta *block.Meta) (err error) {
+func (s *BucketStore) addBlock(ctx context.Context, meta *block.Meta, initialSync bool) (err error) {
 	dir := filepath.Join(s.dir, meta.ULID.String())
 	start := time.Now()
 
@@ -428,6 +427,7 @@ func (s *BucketStore) addBlock(ctx context.Context, meta *block.Meta) (err error
 		meta.ULID,
 		s.postingOffsetsInMemSampling,
 		s.indexHeaderCfg,
+		initialSync,
 	)
 	if err != nil {
 		return errors.Wrap(err, "create index header reader")

diff --git a/pkg/storegateway/bucket_e2e_test.go b/pkg/storegateway/bucket_e2e_test.go
@@ -197,18 +197,22 @@ func prepareStoreWithTestBlocks(t testing.TB, bkt objstore.Bucket, cfg *prepareS
 		objstore.WithNoopInstr(bkt),
 		metaFetcher,
 		cfg.tempDir,
-		cfg.maxSeriesPerBatch,
-		1,
+		mimir_tsdb.BucketStoreConfig{
+			StreamingBatchSize:          cfg.maxSeriesPerBatch,
+			ChunkRangesPerSeries:        1,
+			BlockSyncConcurrency:        20,
+			PostingOffsetsInMemSampling: mimir_tsdb.DefaultPostingOffsetInMemorySampling,
+			IndexHeader: indexheader.Config{
+				IndexHeaderEagerLoadingStartupEnabled: true,
+			},
+			IndexHeaderLazyLoadingEnabled:       true,
+			IndexHeaderLazyLoadingIdleTimeout:   time.Minute,
+			IndexHeaderSparsePersistenceEnabled: true,
+		},
 		cfg.postingsStrategy,
 		cfg.chunksLimiterFactory,
 		cfg.seriesLimiterFactory,
 		newGapBasedPartitioners(mimir_tsdb.DefaultPartitionerMaxGapSize, nil),
-		20,
-		mimir_tsdb.DefaultPostingOffsetInMemorySampling,
-		indexheader.Config{},
-		true,
-		time.Minute,
-		true,
 		hashcache.NewSeriesHashCache(1024*1024),
 		NewBucketStoreMetrics(s.metricsRegistry),
 		storeOpts...,

diff --git a/pkg/storegateway/bucket_stores.go b/pkg/storegateway/bucket_stores.go
@@ -482,8 +482,7 @@ func (u *BucketStores) getOrCreateStore(userID string) (*BucketStore, error) {
 		userBkt,
 		fetcher,
 		u.syncDirForUser(userID),
-		u.cfg.BucketStore.StreamingBatchSize,
-		u.cfg.BucketStore.ChunkRangesPerSeries,
+		u.cfg.BucketStore,
 		selectPostingsStrategy(u.logger, u.cfg.BucketStore.SeriesSelectionStrategyName, u.cfg.BucketStore.SelectionStrategies.WorstCaseSeriesPreference),
 		NewChunksLimiterFactory(func() uint64 {
 			return uint64(u.limits.MaxChunksPerQuery(userID))
@@ -492,12 +491,6 @@ func (u *BucketStores) getOrCreateStore(userID string) (*BucketStore, error) {
 			return uint64(u.limits.MaxFetchedSeriesPerQuery(userID))
 		}),
 		u.partitioners,
-		u.cfg.BucketStore.BlockSyncConcurrency,
-		u.cfg.BucketStore.PostingOffsetsInMemSampling,
-		u.cfg.BucketStore.IndexHeader,
-		u.cfg.BucketStore.IndexHeaderLazyLoadingEnabled,
-		u.cfg.BucketStore.IndexHeaderLazyLoadingIdleTimeout,
-		u.cfg.BucketStore.IndexHeaderSparsePersistenceEnabled,
 		u.seriesHashCache,
 		u.bucketStoreMetrics,
 		bucketStoreOpts...,