Revisit Memberlist defaults (#889)

* Added memberlist defaults Signed-off-by: Joe Elliott <[email protected]> * regenn'ed manifest Signed-off-by: Joe Elliott <[email protected]> * Cleaned up memberlist settings Signed-off-by: Joe Elliott <[email protected]> * disable compression Signed-off-by: Joe Elliott <[email protected]>
grafana · Aug 18, 2021 · 1c7af6b · 1c7af6b
1 parent ca82cb8
commit 1c7af6b
Show file tree

Hide file tree

Showing 3 changed files with 43 additions and 26 deletions.
diff --git a/cmd/tempo/app/app.go b/cmd/tempo/app/app.go
@@ -85,9 +85,16 @@ func (c *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet) {
 	f.IntVar(&c.Server.GRPCListenPort, "server.grpc-listen-port", 9095, "gRPC server listen port.")
 
 	// Memberlist settings
-	fs := flag.NewFlagSet("", flag.PanicOnError)
+	fs := flag.NewFlagSet("", flag.PanicOnError) // create a new flag set b/c we don't want all of the memberlist settings in our flags. we're just doing this to get defaults
 	c.MemberlistKV.RegisterFlags(fs)
 	_ = fs.Parse([]string{})
+	// these defaults were chosen to balance resource usage vs. ring propagation speed. they are a "toned down" version of
+	// the memberlist defaults
+	c.MemberlistKV.RetransmitMult = 2
+	c.MemberlistKV.GossipInterval = time.Second
+	c.MemberlistKV.GossipNodes = 2
+	c.MemberlistKV.EnableCompression = false
+
 	f.Var(&c.MemberlistKV.JoinMembers, "memberlist.host-port", "Host port to connect to memberlist cluster.")
 	f.IntVar(&c.MemberlistKV.TCPTransport.BindPort, "memberlist.bind-port", 7946, "Port for memberlist to communicate on")
 

diff --git a/docs/tempo/website/configuration/_index.md b/docs/tempo/website/configuration/_index.md
@@ -549,24 +549,24 @@ memberlist:
     [randomize_node_name: <boolean> | default = true]
     
     # The timeout for establishing a connection with a remote node, and for
-    # read/write operations. Uses memberlist LAN defaults if 0.
-    [stream_timeout: <duration> | default = 0s]
+    # read/write operations.
+    [stream_timeout: <duration> | default = 10s]
     
     # Multiplication factor used when sending out messages (factor * log(N+1)).
-    [retransmit_factor: <int> | default = 0]
+    [retransmit_factor: <int> | default = 2]
     
-    # How often to use pull/push sync. Uses memberlist LAN defaults if 0.
-    [pull_push_interval: <duration> | default = 0s]
+    # How often to use pull/push sync.
+    [pull_push_interval: <duration> | default = 30s]
     
-    # How often to gossip. Uses memberlist LAN defaults if 0.
-    [gossip_interval: <duration> | default = 0s]
+    # How often to gossip.
+    [gossip_interval: <duration> | default = 1s]
     
-    # How many nodes to gossip to. Uses memberlist LAN defaults if 0.
-    [gossip_nodes: <int> | default = 0]
+    # How many nodes to gossip to.
+    [gossip_nodes: <int> | default = 2]
     
     # How long to keep gossiping to dead nodes, to give them chance to refute their
-    # death. Uses memberlist LAN defaults if 0.
-    [gossip_to_dead_nodes_time: <duration> | default = 0s]
+    # death.
+    [gossip_to_dead_nodes_time: <duration> | default = 30s]
     
     # How soon can dead node's name be reclaimed with new address. Defaults to 0,
     # which is disabled.

diff --git a/docs/tempo/website/configuration/manifest.md b/docs/tempo/website/configuration/manifest.md
@@ -81,14 +81,16 @@ distributor:
         tls_ca_path: ""
         tls_server_name: ""
         tls_insecure_skip_verify: false
+        username: ""
+        password: ""
       multi:
         primary: ""
         secondary: ""
         mirror_enabled: false
         mirror_timeout: 2s
     heartbeat_period: 5s
     heartbeat_timeout: 5m0s
-    instance_id: Annanays-Mac.local
+    instance_id: hostname
     instance_interface_names:
       - eth0
       - en0
@@ -177,8 +179,8 @@ query_frontend:
   instance_interface_names: []
   address: ""
   port: 0
-  compress_responses: true
   downstream_url: ""
+  max_retries: 2
   query_shards: 2
 compactor:
   ring:
@@ -202,6 +204,8 @@ compactor:
         tls_ca_path: ""
         tls_server_name: ""
         tls_insecure_skip_verify: false
+        username: ""
+        password: ""
       multi:
         primary: ""
         secondary: ""
@@ -211,12 +215,13 @@ compactor:
     heartbeat_timeout: 1m0s
     wait_stability_min_duration: 1m0s
     wait_stability_max_duration: 5m0s
-    instance_id: Annanays-Mac.local
+    instance_id: hostname
     instance_interface_names:
       - eth0
       - en0
     instance_port: 0
     instance_addr: ""
+    wait_active_instance_timeout: 10m0s
   compaction:
     chunk_size_bytes: 10485760
     flush_size_bytes: 31457280
@@ -251,6 +256,8 @@ ingester:
           tls_ca_path: ""
           tls_server_name: ""
           tls_insecure_skip_verify: false
+          username: ""
+          password: ""
         multi:
           primary: ""
           secondary: ""
@@ -273,14 +280,14 @@ ingester:
     unregister_on_shutdown: true
     address: 127.0.0.1
     port: 0
-    id: Annanays-Mac.local
+    id: hostname
   concurrent_flushes: 16
   flush_check_period: 30s
   flush_op_timeout: 5m0s
   trace_idle_period: 30s
   max_block_duration: 1h0m0s
   max_block_bytes: 1073741824
-  complete_block_timeout: 6m0s
+  complete_block_timeout: 15m0s
   override_ring_key: ring
 storage:
   trace:
@@ -310,7 +317,7 @@ storage:
       chunk_buffer_size: 10485760
       endpoint: ""
       insecure: false
-      hedge_requests_at: 0
+      hedge_requests_at: 0s
     s3:
       bucket: ""
       endpoint: ""
@@ -319,18 +326,20 @@ storage:
       secret_key: ""
       insecure: false
       part_size: 0
+      hedge_requests_at: 0s
       signature_v2: false
       forcepathstyle: false
-      hedge_requests_at: 0
     azure:
       storage-account-name: ""
       storage-account-key: ""
       container-name: ""
       endpoint-suffix: blob.core.windows.net
       max-buffers: 4
       buffer-size: 3145728
-      hedge-requests-at: 0
+      hedge-requests-at: 0s
     cache: ""
+    cache_min_compaction_level: 0
+    cache_max_block_age: 0s
     background_cache:
       writeback_goroutines: 10
       writeback_buffer: 10000
@@ -349,13 +358,14 @@ overrides:
 memberlist:
   node_name: ""
   randomize_node_name: true
-  stream_timeout: 0s
-  retransmit_factor: 0
-  pull_push_interval: 0s
-  gossip_interval: 0s
-  gossip_nodes: 0
-  gossip_to_dead_nodes_time: 0s
+  stream_timeout: 10s
+  retransmit_factor: 2
+  pull_push_interval: 30s
+  gossip_interval: 1s
+  gossip_nodes: 2
+  gossip_to_dead_nodes_time: 30s
   dead_node_reclaim_time: 0s
+  compression_enabled: false
   join_members: []
   min_join_backoff: 1s
   max_join_backoff: 1m0s