Update config defaults (#913)

* Updated defaults Signed-off-by: Joe Elliott <[email protected]> * updated manifest Signed-off-by: Joe Elliott <[email protected]> * Updated docs Signed-off-by: Joe Elliott <[email protected]> * changelog Signed-off-by: Joe Elliott <[email protected]>
grafana · Aug 30, 2021 · 457a234 · 457a234
1 parent 6f99d44
commit 457a234
Show file tree

Hide file tree

Showing 13 changed files with 67 additions and 37 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,28 @@
 * [BUGFIX] Update port spec for GCS docker-compose example [#869](https://github.com/grafana/tempo/pull/869) (@zalegrala)
 * [BUGFIX] Cortex upgrade to fix an issue where unhealthy compactors can't be forgotten [#878](https://github.com/grafana/tempo/pull/878) (@joe-elliott)
 * [ENHANCEMENT] Added "query blocks" cli option. [#876](https://github.com/grafana/tempo/pull/876) (@joe-elliott)
+* [ENHANCEMENT] Added traceid to `trace too large message`. [#888](https://github.com/grafana/tempo/pull/888) (@mritunjaysharma394)
+* [ENHANCEMENT] Add support to tempo workloads to `overrides` from single configmap in microservice mode. [#896](https://github.com/grafana/tempo/pull/896) (@kavirajk)
+* [ENHANCEMENT] Make `overrides_config` block name consistent with Loki and Cortex in microservice mode. [#906](https://github.com/grafana/tempo/pull/906) (@kavirajk)
+* [ENHANCEMENT] Updated config defaults to reflect better capture operational knowledge. [#913](https://github.com/grafana/tempo/pull/913) (@joe-elliott)
+  ```
+  ingester:
+    trace_idle_period: 30s => 10s  # reduce ingester memory requirements with little impact on querying
+    flush_check_period: 30s => 10s
+  query_frontend:
+    query_shards: 2 => 20          # will massively improve performance on large installs
+  storage:
+    trace:
+      wal:
+        encoding: none => snappy   # snappy has been tested thoroughly and ready for production use
+      block:
+        bloom_filter_false_positive: .05 => .01          # will increase total bloom filter size but improve query performance
+        bloom_filter_shard_size_bytes: 256KiB => 100 KiB # will improve query performance
+  compactor:
+    compaction:
+      chunk_size_bytes: 10 MiB => 5 MiB  # will reduce compactor memory needs
+      compaction_window: 4h => 1h        # will allow more compactors to participate in compaction without substantially increasing blocks
+  ```
 * [ENHANCEMENT] Make s3 backend readError logic more robust [#905](https://github.com/grafana/tempo/pull/905) (@wei840222)
 * [ENHANCEMENT] Include additional detail when searching for traces [#916](https://github.com/grafana/tempo/pull/916) (@zalegrala)
 * [ENHANCEMENT] Add `gen index` and `gen bloom` commands to tempo-cli. [#903](https://github.com/grafana/tempo/pull/903) (@annanay25)

diff --git a/docs/tempo/website/configuration/_index.md b/docs/tempo/website/configuration/_index.md
@@ -142,10 +142,14 @@ ingester:
             # number of replicas of each span to make while pushing to the backend
             replication_factor: 3
 
-    # amount of time before considering a trace complete and flushing it to a block
-    # (default: 30s)
+    # amount of time a trace must be idle before flushing it to the wal.
+    # (default: 10s)
     [trace_idle_period: <duration>]
 
+    # how often to sweep all tenants and move traces from live -> wal -> completed blocks.
+    # (default: 10s)
+    [flush_check_period: <duration>]
+
     # maximum size of a block before cutting it
     # (default: 1073741824 = 1GB)
     [max_block_bytes: <int>]
@@ -169,7 +173,7 @@ query_frontend:
     [max_retries: <int>]
 
     # number of shards to split the query into
-    # (default: 2)
+    # (default: 20)
     [query_shards: <int>]
 ```
 
@@ -218,10 +222,10 @@ compactor:
         # Optional. Duration to keep blocks that have been compacted elsewhere. Default is 1h.
         [compacted_block_retention: <duration>]
 
-        # Optional. Blocks in this time window will be compacted together. Default is 4h.
+        # Optional. Blocks in this time window will be compacted together. Default is 1h.
         [compaction_window: <duration>]
 
-        # Optional. Amount of data to buffer from input blocks. Default is 10 MB.
+        # Optional. Amount of data to buffer from input blocks. Default is 5 MiB.
         [chunk_size_bytes: <int>]
 
         # Optional. Flush data to backend when buffer is this large. Default is 30 MB.
@@ -518,19 +522,24 @@ storage:
             # Example: "wal: /var/tempo/wal"
             [path: <string>] 
 
-            # (experimental) wal encoding/compression.
+            # wal encoding/compression.
             # options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd
+            # (default: snappy)
             [encoding: <string>]
 
         # block configuration
         block:
 
             # bloom filter false positive rate.  lower values create larger filters but fewer false positives
-            # (default: .05)
+            # (default: .01)
             [bloom_filter_false_positive: <float>]
 
+            # maximum size of each bloom filter shard
+            # (default: 100 KiB)
+            [bloom_filter_shard_size_bytes: <int>]
+
             # number of bytes per index record
-            # (defaultL 1MB)
+            # (default: 1MiB)
             [index_downsample_bytes: <uint64>]
 
             # block encoding/compression.  options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd

diff --git a/docs/tempo/website/configuration/compression.md b/docs/tempo/website/configuration/compression.md
@@ -34,16 +34,15 @@ It is important to note that although all of these compression formats are suppo
 we use zstd and it's possible/probable that the other compression algorithms may have issue at scale.  Please 
 file an issue if you stumble upon any problems!
 
-## WAL (Experimental)
+## WAL
 
-The WAL also supports compression. By default this is turned off because it comes with a small performance penalty.
-However, it does reduce disk i/o and adds checksums to the WAL which are valuable in higher volume installations.
+The WAL also supports compression. By default this is configured to use snappy. This comes with a small performance
+penalty but reduces disk I/O and and adds checksums to the WAL. All of the above configuration options are supported
+but only snappy has been tested at scale.
 
 ```
 storage:
   trace:
     wal:
-      encoding: none
-```
-
-If WAL compression is turned on it is recommend to use snappy. All of the above options are supported.
+      encoding: snappy
+```
diff --git a/docs/tempo/website/configuration/manifest.md b/docs/tempo/website/configuration/manifest.md
@@ -16,7 +16,7 @@ go run ./cmd/tempo --storage.trace.backend=local --storage.trace.local.path=/tmp
 
 ## Complete Configuration
 
-> **Note**: This manifest was generated on 7th of June 2021.
+> **Note**: This manifest was generated on 2021-08-25.
 
 ```yaml
 target: all
@@ -181,7 +181,7 @@ query_frontend:
   port: 0
   downstream_url: ""
   max_retries: 2
-  query_shards: 2
+  query_shards: 20
 compactor:
   ring:
     kvstore:
@@ -223,9 +223,9 @@ compactor:
     instance_addr: ""
     wait_active_instance_timeout: 10m0s
   compaction:
-    chunk_size_bytes: 10485760
+    chunk_size_bytes: 5242880
     flush_size_bytes: 31457280
-    compaction_window: 4h0m0s
+    compaction_window: 1h0m0s
     max_compaction_objects: 6000000
     max_block_bytes: 107374182400
     block_retention: 336h0m0s
@@ -282,9 +282,9 @@ ingester:
     port: 0
     id: hostname
   concurrent_flushes: 16
-  flush_check_period: 30s
+  flush_check_period: 10s
   flush_op_timeout: 5m0s
-  trace_idle_period: 30s
+  trace_idle_period: 10s
   max_block_duration: 1h0m0s
   max_block_bytes: 1073741824
   complete_block_timeout: 15m0s
@@ -298,12 +298,12 @@ storage:
       path: /tmp/tempo/wal
       completedfilepath: /tmp/tempo/wal/completed
       blocksfilepath: /tmp/tempo/wal/blocks
-      encoding: none
+      encoding: snappy
     block:
       index_downsample_bytes: 1048576
       index_page_size_bytes: 256000
-      bloom_filter_false_positive: 0.05
-      bloom_filter_shard_size_bytes: 256000
+      bloom_filter_false_positive: 0.01
+      bloom_filter_shard_size_bytes: 102400
       encoding: zstd
     blocklist_poll: 5m0s
     blocklist_poll_concurrency: 50

diff --git a/example/docker-compose/azure/tempo-azure.yaml b/example/docker-compose/azure/tempo-azure.yaml
@@ -37,7 +37,7 @@ storage:
       encoding: zstd                   # block encoding/compression.  options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd
     wal:
       path: /tmp/tempo/wal             # where to store the the wal locally
-      encoding: none                   # wal encoding/compression.  options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd
+      encoding: snappy                 # wal encoding/compression.  options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd
     azure:
       container-name: tempo                    # how to store data in azure
       endpoint-suffix: azurite:10000

diff --git a/example/docker-compose/distributed/tempo-distributed.yaml b/example/docker-compose/distributed/tempo-distributed.yaml
@@ -55,7 +55,7 @@ storage:
       insecure: true
     wal:
       path: /tmp/tempo/wal             # where to store the the wal locally
-      encoding: none                   # wal encoding/compression.  options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd
+      encoding: snappy                 # wal encoding/compression.  options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd
     local:
       path: /tmp/tempo/blocks
     pool:

diff --git a/example/docker-compose/gcs/tempo-gcs.yaml b/example/docker-compose/gcs/tempo-gcs.yaml
@@ -38,7 +38,7 @@ storage:
       encoding: zstd                   # block encoding/compression.  options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd
     wal:
       path: /tmp/tempo/wal             # where to store the the wal locally
-      encoding: none                   # wal encoding/compression.  options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd
+      encoding: snappy                 # wal encoding/compression.  options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd
     gcs:
       bucket_name: tempo
       endpoint: https://gcs:4443/storage/v1/

diff --git a/example/docker-compose/local/tempo-local.yaml b/example/docker-compose/local/tempo-local.yaml
@@ -37,7 +37,7 @@ storage:
       encoding: zstd                   # block encoding/compression.  options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd
     wal:
       path: /tmp/tempo/wal             # where to store the the wal locally
-      encoding: none                   # wal encoding/compression.  options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd
+      encoding: snappy                 # wal encoding/compression.  options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd
     local:
       path: /tmp/tempo/blocks
     pool:

diff --git a/example/docker-compose/s3/tempo-s3.yaml b/example/docker-compose/s3/tempo-s3.yaml
@@ -38,7 +38,7 @@ storage:
       encoding: zstd                   # block encoding/compression.  options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd
     wal:
       path: /tmp/tempo/wal             # where to store the the wal locally
-      encoding: none                   # wal encoding/compression.  options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd
+      encoding: snappy                 # wal encoding/compression.  options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd
     s3:
       bucket: tempo                    # how to store data in s3
       endpoint: minio:9000

diff --git a/modules/compactor/config.go b/modules/compactor/config.go
@@ -20,7 +20,7 @@ type Config struct {
 // RegisterFlagsAndApplyDefaults registers the flags.
 func (cfg *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet) {
 	cfg.Compactor = tempodb.CompactorConfig{
-		ChunkSizeBytes:          10 * 1024 * 1024, // 10 MiB
+		ChunkSizeBytes:          5 * 1024 * 1024, // 5 MiB
 		FlushSizeBytes:          tempodb.DefaultFlushSizeBytes,
 		CompactedBlockRetention: time.Hour,
 		RetentionConcurrency:    tempodb.DefaultRetentionConcurrency,
@@ -33,6 +33,6 @@ func (cfg *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet)
 	f.DurationVar(&cfg.Compactor.BlockRetention, util.PrefixConfig(prefix, "compaction.block-retention"), 14*24*time.Hour, "Duration to keep blocks/traces.")
 	f.IntVar(&cfg.Compactor.MaxCompactionObjects, util.PrefixConfig(prefix, "compaction.max-objects-per-block"), 6000000, "Maximum number of traces in a compacted block.")
 	f.Uint64Var(&cfg.Compactor.MaxBlockBytes, util.PrefixConfig(prefix, "compaction.max-block-bytes"), 100*1024*1024*1024 /* 100GB */, "Maximum size of a compacted block.")
-	f.DurationVar(&cfg.Compactor.MaxCompactionRange, util.PrefixConfig(prefix, "compaction.compaction-window"), 4*time.Hour, "Maximum time window across which to compact blocks.")
+	f.DurationVar(&cfg.Compactor.MaxCompactionRange, util.PrefixConfig(prefix, "compaction.compaction-window"), time.Hour, "Maximum time window across which to compact blocks.")
 	cfg.OverrideRingKey = ring.CompactorRingKey
 }
diff --git a/modules/frontend/config.go b/modules/frontend/config.go
@@ -18,7 +18,7 @@ func (cfg *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet)
 	cfg.Config.Handler.LogQueriesLongerThan = 0
 	cfg.Config.FrontendV1.MaxOutstandingPerTenant = 100
 	cfg.MaxRetries = 2
-	cfg.QueryShards = 2
+	cfg.QueryShards = 20
 }
 
 type CortexNoQuerierLimits struct{}

diff --git a/modules/ingester/config.go b/modules/ingester/config.go
@@ -35,10 +35,10 @@ func (cfg *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet)
 	cfg.LifecyclerConfig.RingConfig.HeartbeatTimeout = 5 * time.Minute
 
 	cfg.ConcurrentFlushes = 16
-	cfg.FlushCheckPeriod = 30 * time.Second
+	cfg.FlushCheckPeriod = 10 * time.Second
 	cfg.FlushOpTimeout = 5 * time.Minute
 
-	f.DurationVar(&cfg.MaxTraceIdle, prefix+".trace-idle-period", 30*time.Second, "Duration after which to consider a trace complete if no spans have been received")
+	f.DurationVar(&cfg.MaxTraceIdle, prefix+".trace-idle-period", 10*time.Second, "Duration after which to consider a trace complete if no spans have been received")
 	f.DurationVar(&cfg.MaxBlockDuration, prefix+".max-block-duration", time.Hour, "Maximum duration which the head block can be appended to before cutting it.")
 	f.Uint64Var(&cfg.MaxBlockBytes, prefix+".max-block-bytes", 1024*1024*1024, "Maximum size of the head block before cutting it.")
 	f.DurationVar(&cfg.CompleteBlockTimeout, prefix+".complete-block-timeout", 3*tempodb.DefaultBlocklistPoll, "Duration to keep head blocks in the ingester after they have been cut.")

diff --git a/modules/storage/config.go b/modules/storage/config.go
@@ -34,11 +34,11 @@ func (cfg *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet)
 
 	cfg.Trace.WAL = &wal.Config{}
 	f.StringVar(&cfg.Trace.WAL.Filepath, util.PrefixConfig(prefix, "trace.wal.path"), "/var/tempo/wal", "Path at which store WAL blocks.")
-	cfg.Trace.WAL.Encoding = backend.EncNone
+	cfg.Trace.WAL.Encoding = backend.EncSnappy
 
 	cfg.Trace.Block = &encoding.BlockConfig{}
-	f.Float64Var(&cfg.Trace.Block.BloomFP, util.PrefixConfig(prefix, "trace.block.bloom-filter-false-positive"), .05, "Bloom Filter False Positive.")
-	f.IntVar(&cfg.Trace.Block.BloomShardSizeBytes, util.PrefixConfig(prefix, "trace.block.bloom-filter-shard-size-bytes"), 250*1024, "Bloom Filter Shard Size in bytes.")
+	f.Float64Var(&cfg.Trace.Block.BloomFP, util.PrefixConfig(prefix, "trace.block.bloom-filter-false-positive"), .01, "Bloom Filter False Positive.")
+	f.IntVar(&cfg.Trace.Block.BloomShardSizeBytes, util.PrefixConfig(prefix, "trace.block.bloom-filter-shard-size-bytes"), 100*1024, "Bloom Filter Shard Size in bytes.")
 	f.IntVar(&cfg.Trace.Block.IndexDownsampleBytes, util.PrefixConfig(prefix, "trace.block.index-downsample-bytes"), 1024*1024, "Number of bytes (before compression) per index record.")
 	f.IntVar(&cfg.Trace.Block.IndexPageSizeBytes, util.PrefixConfig(prefix, "trace.block.index-page-size-bytes"), 250*1024, "Number of bytes per index page.")
 	cfg.Trace.Block.Encoding = backend.EncZstd