cache: Enable embedded cache if no other cache is configured. (#10620)

**What this PR does / why we need it**:

FIFO cache is replaced by embedded-cache in
#6821
This PR enables embedded cache for query range results, chunks cache,
index_stats_results and volume_results if no other cache is explicitly
configured (redis, memcache).

**Special notes for your reviewer**:
Also removes a deprecated flag `cache-split-interval`

**Checklist**
- [x] Reviewed the
[`CONTRIBUTING.md`](https://github.com/grafana/loki/blob/main/CONTRIBUTING.md)
guide (**required**)
- [ ] Documentation added
- [x] Tests updated
- [x] `CHANGELOG.md` updated
- [ ] If the change is worth mentioning in the release notes, add
`add-to-release-notes` label
- [x] Changes that require user attention or interaction to upgrade are
documented in `docs/sources/setup/upgrade/_index.md`
- [ ] For Helm chart changes bump the Helm chart version in
`production/helm/loki/Chart.yaml` and update
`production/helm/loki/CHANGELOG.md` and
`production/helm/loki/README.md`. [Example
PR](d10549e)

---------

Co-authored-by: Christian Haudum <[email protected]>

CHANGELOG.md

@@ -17,6 +17,7 @@
 * [10395](https://github.com/grafana/loki/pull/10395/) **shantanualshi** Remove deprecated `split_queries_by_interval` and `forward_headers_list` configuration options in the `query_range` section
 * [10456](https://github.com/grafana/loki/pull/10456) **dannykopping** Add `loki_distributor_ingester_append_timeouts_total` metric, remove `loki_distributor_ingester_append_failures_total` metric
 * [10534](https://github.com/grafana/loki/pull/10534) **chaudum** Remove configuration `use_boltdb_shipper_as_backup`
+* [10620](https://github.com/grafana/loki/pull/10620) **ashwanthgoli** Enable embedded cache if no other cache is explicitly enabled.
 * [10655](https://github.com/grafana/loki/pull/10655) **chaudum** Remove legacy ingester shutdown handler `/ingester/flush_shutdown`.
 
 ##### Fixes

cmd/migrate/main.go

@@ -76,17 +76,15 @@ func main() {
 
 	// This is a little brittle, if we add a new cache it may easily get missed here but it's important to disable
 	// any of the chunk caches to save on memory because we write chunks to the cache when we call Put operations on the store.
-	sourceConfig.ChunkStoreConfig.ChunkCacheConfig.EnableFifoCache = false
+	sourceConfig.ChunkStoreConfig.ChunkCacheConfig.EmbeddedCache.Enabled = false
 	sourceConfig.ChunkStoreConfig.ChunkCacheConfig.MemcacheClient = defaultsConfig.ChunkStoreConfig.ChunkCacheConfig.MemcacheClient
 	sourceConfig.ChunkStoreConfig.ChunkCacheConfig.Redis = defaultsConfig.ChunkStoreConfig.ChunkCacheConfig.Redis
-	sourceConfig.ChunkStoreConfig.WriteDedupeCacheConfig.EnableFifoCache = false
 	sourceConfig.ChunkStoreConfig.WriteDedupeCacheConfig.MemcacheClient = defaultsConfig.ChunkStoreConfig.WriteDedupeCacheConfig.MemcacheClient
 	sourceConfig.ChunkStoreConfig.WriteDedupeCacheConfig.Redis = defaultsConfig.ChunkStoreConfig.WriteDedupeCacheConfig.Redis
 
-	destConfig.ChunkStoreConfig.ChunkCacheConfig.EnableFifoCache = false
+	destConfig.ChunkStoreConfig.ChunkCacheConfig.EmbeddedCache.Enabled = false
 	destConfig.ChunkStoreConfig.ChunkCacheConfig.MemcacheClient = defaultsConfig.ChunkStoreConfig.ChunkCacheConfig.MemcacheClient
 	destConfig.ChunkStoreConfig.ChunkCacheConfig.Redis = defaultsConfig.ChunkStoreConfig.ChunkCacheConfig.Redis
-	destConfig.ChunkStoreConfig.WriteDedupeCacheConfig.EnableFifoCache = false
 	destConfig.ChunkStoreConfig.WriteDedupeCacheConfig.MemcacheClient = defaultsConfig.ChunkStoreConfig.WriteDedupeCacheConfig.MemcacheClient
 	destConfig.ChunkStoreConfig.WriteDedupeCacheConfig.Redis = defaultsConfig.ChunkStoreConfig.WriteDedupeCacheConfig.Redis
 

docs/sources/configure/_index.md

@@ -3929,11 +3929,6 @@ The cache block configures the cache backend. The supported CLI flags `<prefix>`
 &nbsp;
 
 ```yaml
-# (deprecated: use embedded-cache instead) Enable in-memory cache (auto-enabled
-# for the chunks & query results cache if no other cache is configured).
-# CLI flag: -<prefix>.cache.enable-fifocache
-[enable_fifocache: <boolean> | default = false]
-
 # The default validity of entries for caches unless overridden.
 # CLI flag: -<prefix>.default-validity
 [default_validity: <duration> | default = 1h]
@@ -4084,35 +4079,14 @@ embedded_cache:
   # CLI flag: -<prefix>.embedded-cache.max-size-mb
   [max_size_mb: <int> | default = 100]
 
-  # The time to live for items in the cache before they get purged.
-  # CLI flag: -<prefix>.embedded-cache.ttl
-  [ttl: <duration> | default = 1h]
-
-fifocache:
-  # Maximum memory size of the cache in bytes. A unit suffix (KB, MB, GB) may be
-  # applied.
-  # CLI flag: -<prefix>.fifocache.max-size-bytes
-  [max_size_bytes: <string> | default = "1GB"]
-
-  # deprecated: Maximum number of entries in the cache.
-  # CLI flag: -<prefix>.fifocache.max-size-items
+  # Maximum number of entries in the cache.
+  # CLI flag: -<prefix>.embedded-cache.max-size-items
   [max_size_items: <int> | default = 0]
 
   # The time to live for items in the cache before they get purged.
-  # CLI flag: -<prefix>.fifocache.ttl
+  # CLI flag: -<prefix>.embedded-cache.ttl
   [ttl: <duration> | default = 1h]
 
-  # Deprecated (use ttl instead): The expiry duration for the cache.
-  # CLI flag: -<prefix>.fifocache.duration
-  [validity: <duration> | default = 0s]
-
-  # Deprecated (use max-size-items or max-size-bytes instead): The number of
-  # entries to cache.
-  # CLI flag: -<prefix>.fifocache.size
-  [size: <int> | default = 0]
-
-  [purgeinterval: <duration>]
-
 # The maximum number of concurrent asynchronous writeback cache can occur.
 # CLI flag: -<prefix>.max-async-cache-write-back-concurrency
 [async_cache_write_back_concurrency: <int> | default = 16]

docs/sources/configure/query-frontend.md

@@ -53,11 +53,11 @@ data:
 
       results_cache:
         cache:
-          # We're going to use the in-process "FIFO" cache
-          enable_fifocache: true
-          fifocache:
-            size: 1024
-            validity: 24h
+          # We're going to use the embedded cache
+          embedded_cache:
+            enabled: true
+            max_size_mb: 100
+            ttl: 24h
 
     limits_config:
       max_cache_freshness_per_query: '10m'

docs/sources/setup/upgrade/_index.md

@@ -54,6 +54,7 @@ The previous default value `false` is applied.
 5. `experimental.ruler.enable-api` is removed. Use `ruler.enable-api` instead.
 6. `split_queries_by_interval` is removed from `query_range` YAML section. You can instead configure it in [Limits Config](/docs/loki/latest/configuration/#limits_config).
 7. `frontend.forward-headers-list` CLI flag and its corresponding YAML setting are removed.
+8. `frontend.cache-split-interval` CLI flag is removed. Results caching interval is now determined by `querier.split-queries-by-interval`.
 
 #### Legacy ingester shutdown handler is removed
 
@@ -64,6 +65,9 @@ The already deprecated handler `/ingester/flush_shutdown` is removed in favor of
 The `loki_distributor_ingester_append_failures_total` metric has been removed in favour of `loki_distributor_ingester_append_timeouts_total`.
 This new metric will provide a more clear signal that there is an issue with ingesters, and this metric can be used for high-signal alerting.
 
+#### Changes to default configuration values
+
+1. `frontend.embedded-cache.max-size-mb` Embedded results cache size now defaults to 100MB.
 
 ## 2.9.0
 

integration/loki_micro_services_test.go

@@ -588,7 +588,7 @@ func TestQueryTSDB_WithCachedPostings(t *testing.T) {
 			"index-gateway",
 			"-target=index-gateway",
 			"-tsdb.enable-postings-cache=true",
-			"-store.index-cache-read.cache.enable-fifocache=true",
+			"-store.index-cache-read.embedded-cache.enabled=true",
 		)
 	)
 	require.NoError(t, clu.Run())

pkg/loki/config_wrapper.go

@@ -120,7 +120,7 @@ func (c *ConfigWrapper) ApplyDynamicConfig() cfg.Source {
 			betterTSDBShipperDefaults(r, &defaults, r.SchemaConfig.Configs[i])
 		}
 
-		applyFIFOCacheConfig(r)
+		applyEmbeddedCacheConfig(r)
 		applyIngesterFinalSleep(r)
 		applyIngesterReplicationFactor(r)
 		applyChunkRetain(r, &defaults)
@@ -557,23 +557,18 @@ func betterTSDBShipperDefaults(cfg, defaults *ConfigWrapper, period config.Perio
 	}
 }
 
-// applyFIFOCacheConfig turns on FIFO cache for the chunk store, for the query range results,
-// and for the index stats results, but only if no other cache storage is configured (redis or memcache).
-// This behavior is only applied for the chunk store cache, for the query range results cache, and for
-// the index stats results (i.e: not applicable for the index queries cache or for the write dedupe cache).
-func applyFIFOCacheConfig(r *ConfigWrapper) {
+// applyEmbeddedCacheConfig turns on Embedded cache for the chunk store, query range results,
+// index stats and volume results only if no other cache storage is configured (redis or memcache).
+// Not applicable for the index queries cache or for the write dedupe cache.
+func applyEmbeddedCacheConfig(r *ConfigWrapper) {
 	chunkCacheConfig := r.ChunkStoreConfig.ChunkCacheConfig
 	if !cache.IsCacheConfigured(chunkCacheConfig) {
-		r.ChunkStoreConfig.ChunkCacheConfig.EnableFifoCache = true
+		r.ChunkStoreConfig.ChunkCacheConfig.EmbeddedCache.Enabled = true
 	}
 
 	resultsCacheConfig := r.QueryRange.ResultsCacheConfig.CacheConfig
 	if !cache.IsCacheConfigured(resultsCacheConfig) {
-		r.QueryRange.ResultsCacheConfig.CacheConfig.EnableFifoCache = true
-		// The query results fifocache is still in Cortex so we couldn't change the flag defaults
-		// so instead we will override them here.
-		r.QueryRange.ResultsCacheConfig.CacheConfig.Fifocache.MaxSizeBytes = "1GB"
-		r.QueryRange.ResultsCacheConfig.CacheConfig.Fifocache.TTL = 1 * time.Hour
+		r.QueryRange.ResultsCacheConfig.CacheConfig.EmbeddedCache.Enabled = true
 	}
 
 	indexStatsCacheConfig := r.QueryRange.StatsCacheConfig.CacheConfig