Persisting failed transactions on disk when in-mem buffer is full (Da…

…taDog#6921) persisting failed transactions on disk when in-mem buffer is full
deliveroo · Mar 24, 2021 · 77f3687 · 77f3687
1 parent c568988
commit 77f3687
Show file tree

Hide file tree

Showing 17 changed files with 2,812 additions and 63 deletions.
diff --git a/pkg/config/config.go b/pkg/config/config.go
@@ -13,6 +13,7 @@ import (
 	"net"
 	"net/url"
 	"os"
+	"path"
 	"path/filepath"
 	"regexp"
 	"strings"
@@ -321,6 +322,13 @@ func InitConfig(config Config) {
 	config.BindEnvAndSetDefault("forwarder_recovery_interval", DefaultForwarderRecoveryInterval)
 	config.BindEnvAndSetDefault("forwarder_recovery_reset", false)
 
+	// Forwarder storage on disk
+	defaultForwarderStoragePath := path.Join(config.GetString("run_path"), "transactions_to_retry")
+	config.BindEnvAndSetDefault("forwarder_storage_path", defaultForwarderStoragePath)
+	config.BindEnvAndSetDefault("forwarder_outdated_file_in_days", 10)
+	config.BindEnvAndSetDefault("forwarder_flush_to_disk_mem_ratio", 0.5)
+	config.BindEnvAndSetDefault("forwarder_storage_max_size_in_bytes", 0) // 0 means disabled. This is a BETA feature.
+
 	// Dogstatsd
 	config.BindEnvAndSetDefault("use_dogstatsd", true)
 	config.BindEnvAndSetDefault("dogstatsd_port", 8125)    // Notice: 0 means UDP port closed