Port: Telemetry For Lease Expiration Times

command/server/config_test.go

@@ -30,6 +30,10 @@ func TestLoadConfigFileIntegerAndBooleanValuesJson(t *testing.T) {
 	testLoadConfigFileIntegerAndBooleanValuesJson(t)
 }
 
+func TestLoadConfigFileWithLeaseMetricTelemetry(t *testing.T) {
+	testLoadConfigFileLeaseMetrics(t)
+}
+
 func TestLoadConfigDir(t *testing.T) {
 	testLoadConfigDir(t)
 }

command/server/config_test_helpers.go

@@ -62,14 +62,17 @@ func testLoadConfigFile_topLevel(t *testing.T, entropy *configutil.Entropy) {
 			},
 
 			Telemetry: &configutil.Telemetry{
-				StatsdAddr:              "bar",
-				StatsiteAddr:            "foo",
-				DisableHostname:         false,
-				DogStatsDAddr:           "127.0.0.1:7254",
-				DogStatsDTags:           []string{"tag_1:val_1", "tag_2:val_2"},
-				PrometheusRetentionTime: 30 * time.Second,
-				UsageGaugePeriod:        5 * time.Minute,
-				MaximumGaugeCardinality: 125,
+				StatsdAddr:                  "bar",
+				StatsiteAddr:                "foo",
+				DisableHostname:             false,
+				DogStatsDAddr:               "127.0.0.1:7254",
+				DogStatsDTags:               []string{"tag_1:val_1", "tag_2:val_2"},
+				PrometheusRetentionTime:     30 * time.Second,
+				UsageGaugePeriod:            5 * time.Minute,
+				MaximumGaugeCardinality:     125,
+				LeaseMetricsEpsilon:         time.Hour,
+				NumLeaseMetricsTimeBuckets:  168,
+				LeaseMetricsNameSpaceLabels: false,
 			},
 
 			DisableMlock: true,
@@ -192,6 +195,9 @@ func testLoadConfigFile_json2(t *testing.T, entropy *configutil.Entropy) {
 				CirconusBrokerID:                   "0",
 				CirconusBrokerSelectTag:            "dc:sfo",
 				PrometheusRetentionTime:            30 * time.Second,
+				LeaseMetricsEpsilon:                time.Hour,
+				NumLeaseMetricsTimeBuckets:         168,
+				LeaseMetricsNameSpaceLabels:        false,
 			},
 		},
 
@@ -371,15 +377,18 @@ func testLoadConfigFile(t *testing.T) {
 			},
 
 			Telemetry: &configutil.Telemetry{
-				StatsdAddr:              "bar",
-				StatsiteAddr:            "foo",
-				DisableHostname:         false,
-				UsageGaugePeriod:        5 * time.Minute,
-				MaximumGaugeCardinality: 100,
-				DogStatsDAddr:           "127.0.0.1:7254",
-				DogStatsDTags:           []string{"tag_1:val_1", "tag_2:val_2"},
-				PrometheusRetentionTime: configutil.PrometheusDefaultRetentionTime,
-				MetricsPrefix:           "myprefix",
+				StatsdAddr:                  "bar",
+				StatsiteAddr:                "foo",
+				DisableHostname:             false,
+				UsageGaugePeriod:            5 * time.Minute,
+				MaximumGaugeCardinality:     100,
+				DogStatsDAddr:               "127.0.0.1:7254",
+				DogStatsDTags:               []string{"tag_1:val_1", "tag_2:val_2"},
+				PrometheusRetentionTime:     configutil.PrometheusDefaultRetentionTime,
+				MetricsPrefix:               "myprefix",
+				LeaseMetricsEpsilon:         time.Hour,
+				NumLeaseMetricsTimeBuckets:  168,
+				LeaseMetricsNameSpaceLabels: false,
 			},
 
 			DisableMlock: true,
@@ -477,6 +486,9 @@ func testLoadConfigFile_json(t *testing.T) {
 				CirconusBrokerID:                   "",
 				CirconusBrokerSelectTag:            "",
 				PrometheusRetentionTime:            configutil.PrometheusDefaultRetentionTime,
+				LeaseMetricsEpsilon:                time.Hour,
+				NumLeaseMetricsTimeBuckets:         168,
+				LeaseMetricsNameSpaceLabels:        false,
 			},
 
 			PidFile:     "./pidfile",
@@ -540,12 +552,15 @@ func testLoadConfigDir(t *testing.T) {
 			},
 
 			Telemetry: &configutil.Telemetry{
-				StatsiteAddr:            "qux",
-				StatsdAddr:              "baz",
-				DisableHostname:         true,
-				UsageGaugePeriod:        5 * time.Minute,
-				MaximumGaugeCardinality: 100,
-				PrometheusRetentionTime: configutil.PrometheusDefaultRetentionTime,
+				StatsiteAddr:                "qux",
+				StatsdAddr:                  "baz",
+				DisableHostname:             true,
+				UsageGaugePeriod:            5 * time.Minute,
+				MaximumGaugeCardinality:     100,
+				PrometheusRetentionTime:     configutil.PrometheusDefaultRetentionTime,
+				LeaseMetricsEpsilon:         time.Hour,
+				NumLeaseMetricsTimeBuckets:  168,
+				LeaseMetricsNameSpaceLabels: false,
 			},
 			ClusterName: "testcluster",
 		},
@@ -668,6 +683,9 @@ func testConfig_Sanitized(t *testing.T) {
 			"stackdriver_debug_logs":                 false,
 			"statsd_address":                         "bar",
 			"statsite_address":                       "",
+			"lease_metrics_epsilon":                  time.Hour,
+			"num_lease_metrics_buckets":              168,
+			"add_lease_metrics_namespace_labels":     false,
 		},
 	}
 
@@ -787,3 +805,93 @@ func testParseSeals(t *testing.T) {
 	}
 	require.Equal(t, config, expected)
 }
+
+func testLoadConfigFileLeaseMetrics(t *testing.T) {
+	config, err := LoadConfigFile("./test-fixtures/config5.hcl")
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	expected := &Config{
+		SharedConfig: &configutil.SharedConfig{
+			Listeners: []*configutil.Listener{
+				{
+					Type:    "tcp",
+					Address: "127.0.0.1:443",
+				},
+			},
+
+			Telemetry: &configutil.Telemetry{
+				StatsdAddr:                  "bar",
+				StatsiteAddr:                "foo",
+				DisableHostname:             false,
+				UsageGaugePeriod:            5 * time.Minute,
+				MaximumGaugeCardinality:     100,
+				DogStatsDAddr:               "127.0.0.1:7254",
+				DogStatsDTags:               []string{"tag_1:val_1", "tag_2:val_2"},
+				PrometheusRetentionTime:     configutil.PrometheusDefaultRetentionTime,
+				MetricsPrefix:               "myprefix",
+				LeaseMetricsEpsilon:         time.Hour,
+				NumLeaseMetricsTimeBuckets:  2,
+				LeaseMetricsNameSpaceLabels: true,
+			},
+
+			DisableMlock: true,
+
+			Entropy: nil,
+
+			PidFile: "./pidfile",
+
+			ClusterName: "testcluster",
+		},
+
+		Storage: &Storage{
+			Type:         "consul",
+			RedirectAddr: "foo",
+			Config: map[string]string{
+				"foo": "bar",
+			},
+		},
+
+		HAStorage: &Storage{
+			Type:         "consul",
+			RedirectAddr: "snafu",
+			Config: map[string]string{
+				"bar": "baz",
+			},
+			DisableClustering: true,
+		},
+
+		ServiceRegistration: &ServiceRegistration{
+			Type: "consul",
+			Config: map[string]string{
+				"foo": "bar",
+			},
+		},
+
+		DisableCache:             true,
+		DisableCacheRaw:          true,
+		DisablePrintableCheckRaw: true,
+		DisablePrintableCheck:    true,
+		EnableUI:                 true,
+		EnableUIRaw:              true,
+
+		EnableRawEndpoint:    true,
+		EnableRawEndpointRaw: true,
+
+		DisableSealWrap:    true,
+		DisableSealWrapRaw: true,
+
+		MaxLeaseTTL:        10 * time.Hour,
+		MaxLeaseTTLRaw:     "10h",
+		DefaultLeaseTTL:    10 * time.Hour,
+		DefaultLeaseTTLRaw: "10h",
+	}
+
+	addExpectedEntConfig(expected, []string{})
+
+	config.Listeners[0].RawConfig = nil
+	if diff := deep.Equal(config, expected); diff != nil {
+		t.Fatal(diff)
+	}
+}

command/server/test-fixtures/config5.hcl

@@ -0,0 +1,51 @@
+disable_cache = true
+ disable_mlock = true
+
+ ui = true
+
+ listener "tcp" {
+     address = "127.0.0.1:443"
+ 	allow_stuff = true
+ }
+
+ backend "consul" {
+     foo = "bar"
+     advertise_addr = "foo"
+ }
+
+ ha_backend "consul" {
+     bar = "baz"
+     advertise_addr = "snafu"
+     disable_clustering = "true"
+ }
+
+ service_registration "consul" {
+     foo = "bar"
+ }
+
+ telemetry {
+     statsd_address = "bar"
+     usage_gauge_period = "5m"
+     maximum_gauge_cardinality = 100
+
+     statsite_address = "foo"
+     dogstatsd_addr = "127.0.0.1:7254"
+     dogstatsd_tags = ["tag_1:val_1", "tag_2:val_2"]
+     metrics_prefix = "myprefix"
+
+     lease_metrics_epsilon = "1h"
+     num_lease_metrics_buckets = 2
+     add_lease_metrics_namespace_labels = true 
+ }
+
+ sentinel {
+     additional_enabled_modules = []
+ }
+
+ max_lease_ttl = "10h"
+ default_lease_ttl = "10h"
+ cluster_name = "testcluster"
+ pid_file = "./pidfile"
+ raw_storage_endpoint = true
+ disable_sealwrap = true
+ disable_printable_check = true

helper/metricsutil/bucket.go

@@ -37,3 +37,22 @@ func TTLBucket(ttl time.Duration) string {
 	}
 
 }
+
+func ExpiryBucket(expiryTime time.Time, leaseEpsilon time.Duration, rollingWindow time.Time, labelNS string, useNS bool) *LeaseExpiryLabel {
+	if !useNS {
+		labelNS = ""
+	}
+	leaseExpiryLabel := LeaseExpiryLabel{LabelNS: labelNS}
+
+	// calculate rolling window
+	if expiryTime.Before(rollingWindow) {
+		leaseExpiryLabel.LabelName = expiryTime.Round(leaseEpsilon).String()
+		return &leaseExpiryLabel
+	}
+	return nil
+}
+
+type LeaseExpiryLabel = struct {
+	LabelName string
+	LabelNS   string
+}

helper/metricsutil/wrapped_metrics.go

@@ -28,6 +28,15 @@ type ClusterMetricSink struct {
 
 	// Sink is the go-metrics instance to send to.
 	Sink metrics.MetricSink
+
+	// Constants that are helpful for metrics within the metrics sink
+	TelemetryConsts TelemetryConstConfig
+}
+
+type TelemetryConstConfig struct {
+	LeaseMetricsEpsilon         time.Duration
+	NumLeaseMetricsTimeBuckets  int
+	LeaseMetricsNameSpaceLabels bool
 }
 
 type Metrics interface {
@@ -83,8 +92,9 @@ func BlackholeSink() *ClusterMetricSink {
 
 func NewClusterMetricSink(clusterName string, sink metrics.MetricSink) *ClusterMetricSink {
 	cms := &ClusterMetricSink{
-		ClusterName: atomic.Value{},
-		Sink:        sink,
+		ClusterName:     atomic.Value{},
+		Sink:            sink,
+		TelemetryConsts: TelemetryConstConfig{},
 	}
 	cms.ClusterName.Store(clusterName)
 	return cms

internalshared/configutil/config.go

@@ -214,6 +214,9 @@ func (c *SharedConfig) Sanitized() map[string]interface{} {
 			"stackdriver_location":                   c.Telemetry.StackdriverLocation,
 			"stackdriver_namespace":                  c.Telemetry.StackdriverNamespace,
 			"stackdriver_debug_logs":                 c.Telemetry.StackdriverDebugLogs,
+			"lease_metrics_epsilon":                  c.Telemetry.LeaseMetricsEpsilon,
+			"num_lease_metrics_buckets":              c.Telemetry.NumLeaseMetricsTimeBuckets,
+			"add_lease_metrics_namespace_labels":     c.Telemetry.LeaseMetricsNameSpaceLabels,
 		}
 		result["telemetry"] = sanitizedTelemetry
 	}