From 643573bb78462d475be4f81c78641dc58215c96e Mon Sep 17 00:00:00 2001 From: Christopher Snell Date: Fri, 8 Jun 2018 16:40:53 -0500 Subject: [PATCH 1/5] Add support for tags --- config.go | 11 ++++++----- jobs.go | 32 +++++++++++++++++++++++++++----- selenium.go | 19 +++++-------------- simple.go | 20 ++++++++++---------- storage.go | 18 ++++++++++++++---- storage_dogstatsd.go | 26 ++++++++++++++++---------- storage_riemann.go | 8 ++++++++ 7 files changed, 86 insertions(+), 48 deletions(-) diff --git a/config.go b/config.go index a90f0ce..3731a1e 100644 --- a/config.go +++ b/config.go @@ -9,11 +9,12 @@ import ( // Config is the base of our configuration data structure type Config struct { - Jobs []Job `yaml:"jobs"` - Selenium SeleniumConfig `yaml:"selenium"` - Storage StorageConfig `yaml:"storage,omitempty"` - ReportInternalMetrics bool `yaml:"report-internal-metrics,omitempty"` - InternalMetricsInterval uint `yaml:"internal-metrics-gathering-interval,omitempty"` + Jobs []Job `yaml:"jobs"` + Selenium SeleniumConfig `yaml:"selenium"` + Storage StorageConfig `yaml:"storage,omitempty"` + ReportInternalMetrics bool `yaml:"report-internal-metrics,omitempty"` + InternalMetricsInterval uint `yaml:"internal-metrics-gathering-interval,omitempty"` + ServerTags map[string]string `yaml:"server-tags,omitempty"` } // SeleniumConfig holds the configuration for our Selenium service diff --git a/jobs.go b/jobs.go index ffc12fd..5997b76 100644 --- a/jobs.go +++ b/jobs.go @@ -6,17 +6,19 @@ import ( "log" "math/rand" "net/http" + "net/url" "sync" "time" ) // Job holds a single Selenium type Job struct { - Name string `yaml:"name"` - URL string `yaml:"url"` - Type string `yaml:"type"` - Interval uint16 `yaml:"interval"` - Cookies []Cookie `yaml:"cookies,omitempty"` + Name string `yaml:"name"` + URL string `yaml:"url"` + Type string `yaml:"type"` + Tags map[string]string `yaml:"tags,omitempty"` + Interval uint16 `yaml:"interval"` + Cookies []Cookie `yaml:"cookies,omitempty"` } // JobRunner holds channels and state related to running Jobs @@ -72,6 +74,8 @@ func StartJobs(ctx context.Context, wg *sync.WaitGroup, c *Config, storage *Stor for _, j := range jobs { + j.populateAutomaticTags(c) + // If we've been provided with an offset for staggering jobs, sleep for a random // time interval (where: 0 < sleepDur < offset) before starting that job's timer if c.Selenium.JobStaggerOffset > 0 { @@ -85,3 +89,21 @@ func StartJobs(ctx context.Context, wg *sync.WaitGroup, c *Config, storage *Stor } } + +func (j *Job) populateAutomaticTags(c *Config) { + u, err := url.Parse(j.URL) + if err != nil { + return + } + + // Add all of our server tags + for k, v := range c.ServerTags { + j.Tags[k] = v + } + + // Add the hostname from our job + j.Tags["crabby.job_hostname"] = u.Hostname() + + // Add the type of check (simple, selenium) + j.Tags["crabby.job_type"] = j.Type +} diff --git a/selenium.go b/selenium.go index d13d8c4..22b0642 100644 --- a/selenium.go +++ b/selenium.go @@ -120,20 +120,11 @@ func RunSeleniumTest(j Job, seleniumServer string, storage *Storage) { return } - fmt.Println(j.Name, "DNS time:", wr.ri.dnsDuration) - storage.MetricDistributor <- makeMetric(j.Name, "dns_duration", wr.ri.dnsDuration) - - fmt.Println(j.Name, "Connection establishment time", wr.ri.serverConnectionDuration) - storage.MetricDistributor <- makeMetric(j.Name, "server_connection_duration", wr.ri.serverConnectionDuration) - - fmt.Println(j.Name, "Response time:", wr.ri.serverResponseDuration) - storage.MetricDistributor <- makeMetric(j.Name, "server_response_duration", wr.ri.serverResponseDuration) - - fmt.Println(j.Name, "Server processing time:", wr.ri.serverProcessingDuration) - storage.MetricDistributor <- makeMetric(j.Name, "server_processing_duration", wr.ri.serverProcessingDuration) - - fmt.Println(j.Name, "DOM rendering time:", wr.ri.domRenderingDuration) - storage.MetricDistributor <- makeMetric(j.Name, "dom_rendering_duration", wr.ri.domRenderingDuration) + storage.MetricDistributor <- makeMetric(j, "dns_duration", wr.ri.dnsDuration) + storage.MetricDistributor <- makeMetric(j, "server_connection_duration", wr.ri.serverConnectionDuration) + storage.MetricDistributor <- makeMetric(j, "server_response_duration", wr.ri.serverResponseDuration) + storage.MetricDistributor <- makeMetric(j, "server_processing_duration", wr.ri.serverProcessingDuration) + storage.MetricDistributor <- makeMetric(j, "dom_rendering_duration", wr.ri.domRenderingDuration) err = wr.wd.Close() if err != nil { diff --git a/simple.go b/simple.go index 17eb923..7fb917f 100644 --- a/simple.go +++ b/simple.go @@ -88,7 +88,7 @@ func RunSimpleTest(ctx context.Context, j Job, storage *Storage, client *http.Cl } // Send our server response code as an event - storage.EventDistributor <- makeEvent(j.Name, resp.StatusCode) + storage.EventDistributor <- makeEvent(j, resp.StatusCode) // Even though we never read the response body, if we don't close it, // the http.Transport goroutines will terminate and the app will eventually @@ -109,16 +109,16 @@ func RunSimpleTest(ctx context.Context, j Job, storage *Storage, client *http.Cl switch url.Scheme { case "https": - storage.MetricDistributor <- makeMetric(j.Name, "dns_duration", t1.Sub(t0).Seconds()*1000) - storage.MetricDistributor <- makeMetric(j.Name, "server_connection_duration", t2.Sub(t1).Seconds()*1000) - storage.MetricDistributor <- makeMetric(j.Name, "tls_handshake_duration", t3.Sub(t2).Seconds()*1000) - storage.MetricDistributor <- makeMetric(j.Name, "server_processing_duration", t4.Sub(t3).Seconds()*1000) - storage.MetricDistributor <- makeMetric(j.Name, "server_response_duration", t5.Sub(t4).Seconds()*1000) + storage.MetricDistributor <- makeMetric(j, "dns_duration", t1.Sub(t0).Seconds()*1000) + storage.MetricDistributor <- makeMetric(j, "server_connection_duration", t2.Sub(t1).Seconds()*1000) + storage.MetricDistributor <- makeMetric(j, "tls_handshake_duration", t3.Sub(t2).Seconds()*1000) + storage.MetricDistributor <- makeMetric(j, "server_processing_duration", t4.Sub(t3).Seconds()*1000) + storage.MetricDistributor <- makeMetric(j, "server_response_duration", t5.Sub(t4).Seconds()*1000) case "http": - storage.MetricDistributor <- makeMetric(j.Name, "dns_duration", t1.Sub(t0).Seconds()*1000) - storage.MetricDistributor <- makeMetric(j.Name, "server_connection_duration", t3.Sub(t1).Seconds()*1000) - storage.MetricDistributor <- makeMetric(j.Name, "server_processing_duration", t4.Sub(t3).Seconds()*1000) - storage.MetricDistributor <- makeMetric(j.Name, "server_response_duration", t5.Sub(t4).Seconds()*1000) + storage.MetricDistributor <- makeMetric(j, "dns_duration", t1.Sub(t0).Seconds()*1000) + storage.MetricDistributor <- makeMetric(j, "server_connection_duration", t3.Sub(t1).Seconds()*1000) + storage.MetricDistributor <- makeMetric(j, "server_processing_duration", t4.Sub(t3).Seconds()*1000) + storage.MetricDistributor <- makeMetric(j, "server_response_duration", t5.Sub(t4).Seconds()*1000) } } diff --git a/storage.go b/storage.go index aa683cd..e367872 100644 --- a/storage.go +++ b/storage.go @@ -12,6 +12,7 @@ import ( type Metric struct { Name string Value float64 + Tags map[string]string Timestamp time.Time } @@ -19,6 +20,7 @@ type Metric struct { type Event struct { Name string ServerStatus int + Tags map[string]string Timestamp time.Time } @@ -168,25 +170,33 @@ func (s *Storage) storageDistributor(ctx context.Context, wg *sync.WaitGroup) er } // makeMetric creates a Metric from raw values and metric names -func makeMetric(name string, timing string, value float64) Metric { +func makeMetric(j Job, timing string, value float64) Metric { m := Metric{ - Name: fmt.Sprintf("%v.%v", name, timing), + Name: fmt.Sprintf("%v.%v", j.Name, timing), Value: value, Timestamp: time.Now(), } + for k, v := range j.Tags { + m.Tags[k] = v + } + return m } // makeEvent creates an Event from raw values and event names -func makeEvent(name string, status int) Event { +func makeEvent(j Job, status int) Event { e := Event{ - Name: name, + Name: j.Name, ServerStatus: status, Timestamp: time.Now(), } + for k, v := range j.Tags { + e.Tags[k] = v + } + return e } diff --git a/storage_dogstatsd.go b/storage_dogstatsd.go index c45c7fb..f667835 100644 --- a/storage_dogstatsd.go +++ b/storage_dogstatsd.go @@ -12,10 +12,10 @@ import ( // DogstatsdConfig describes the YAML-provided configuration for a Datadog // DogstatsD storage backend type DogstatsdConfig struct { - Host string `yaml:"host"` - Port int `yaml:"port"` - Namespace string `yaml:"metric-namespace"` - Tags []string `yaml:"tags,omitempty"` + Host string `yaml:"host"` + Port int `yaml:"port"` + Namespace string `yaml:"metric-namespace"` + Tags map[string]string `yaml:"tags,omitempty"` } // DogstatsdStorage holds the configuration for a DogstatsD storage backend @@ -69,6 +69,11 @@ func (d DogstatsdStorage) sendMetric(m Metric) error { metricName = fmt.Sprintf("%v.%v", d.Namespace, m.Name) } + // Add all of our metric tags to the metric payload + for k, v := range m.Tags { + d.DogstatsdConn.Tags = append(d.DogstatsdConn.Tags, k+":"+v) + } + err := d.DogstatsdConn.TimeInMilliseconds(metricName, m.Value, nil, 1) if err != nil { log.Printf("Could not send metric %v: %v\n", m.Name, err) @@ -100,16 +105,17 @@ func (d DogstatsdStorage) sendEvent(e Event) error { Message: fmt.Sprintf("%v is returning a HTTP status code of %v", e.Name, e.ServerStatus), } + // Add all of our metric tags to the metric payload + for k, v := range e.Tags { + d.DogstatsdConn.Tags = append(d.DogstatsdConn.Tags, k+":"+v) + } + if (e.ServerStatus < 400) && (e.ServerStatus > 0) { sc.Status = statsd.Ok } else { sc.Status = statsd.Critical } - for _, t := range d.DogstatsdConn.Tags { - sc.Tags = append(sc.Tags, t) - } - err := d.DogstatsdConn.ServiceCheck(sc) return err @@ -127,8 +133,8 @@ func NewDogstatsdStorage(c *Config) DogstatsdStorage { log.Println("Warning: could not create dogstatsd connection", err) } - for _, t := range c.Storage.Dogstatsd.Tags { - d.DogstatsdConn.Tags = append(d.DogstatsdConn.Tags, t) + for k, v := range c.Storage.Dogstatsd.Tags { + d.DogstatsdConn.Tags = append(d.DogstatsdConn.Tags, k+":"+v) } return d } diff --git a/storage_riemann.go b/storage_riemann.go index c163e82..34c501b 100644 --- a/storage_riemann.go +++ b/storage_riemann.go @@ -93,6 +93,10 @@ func (r RiemannStorage) sendMetric(m Metric) error { Tags: r.Tags, } + for k, v := range m.Tags { + ev.Tags = append(ev.Tags, k+":"+v) + } + err := r.Client.SendEvent(ev) if err != nil { return err @@ -124,6 +128,10 @@ func (r RiemannStorage) sendEvent(e Event) error { Tags: r.Tags, } + for k, v := range e.Tags { + ev.Tags = append(ev.Tags, k+":"+v) + } + err := r.Client.SendEvent(ev) if err != nil { return err From bc74022b1b1d12751327e35edb3d85665a8f37a3 Mon Sep 17 00:00:00 2001 From: Christopher Snell Date: Fri, 8 Jun 2018 16:51:46 -0500 Subject: [PATCH 2/5] Change tag format for Riemann config --- storage_riemann.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/storage_riemann.go b/storage_riemann.go index 34c501b..4debc0d 100644 --- a/storage_riemann.go +++ b/storage_riemann.go @@ -12,10 +12,10 @@ import ( // RiemannConfig describes the YAML-provided configuration for a Riemann // storage backend type RiemannConfig struct { - Host string `yaml:"host"` - Port int `yaml:"port"` - Namespace string `yaml:"metric-namespace,omitempty"` - Tags []string `yaml:"tags,omitempty"` + Host string `yaml:"host"` + Port int `yaml:"port"` + Namespace string `yaml:"metric-namespace,omitempty"` + Tags map[string]string `yaml:"tags,omitempty"` } // RiemannStorage holds the configuration for a Riemann storage backend @@ -30,7 +30,10 @@ func NewRiemannStorage(c *Config) (RiemannStorage, error) { r := RiemannStorage{} r.Namespace = c.Storage.Riemann.Namespace - r.Tags = c.Storage.Riemann.Tags + + for k, v := range c.Storage.Riemann.Tags { + r.Tags = append(r.Tags, k+":"+v) + } r.Client = goryman.NewGorymanClient(fmt.Sprint(c.Storage.Riemann.Host, ":", c.Storage.Riemann.Port)) err := r.Client.Connect() From 23bc463f1e4e2d73340ff3402b5c2cf38d37341f Mon Sep 17 00:00:00 2001 From: Christopher Snell Date: Fri, 8 Jun 2018 16:53:30 -0500 Subject: [PATCH 3/5] Add server tags and update new tag format elsewhere --- example/config.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/example/config.yaml b/example/config.yaml index 3dff246..b1434fa 100644 --- a/example/config.yaml +++ b/example/config.yaml @@ -32,7 +32,7 @@ storage: port: 8125 metric-namespace: crabby.crabby-sfo-1 tags: - - crabby-sfo-1 + - instance: crabby-sfo-1 prometheus: host: prometheus.mysite.org port: 9091 @@ -42,6 +42,8 @@ storage: port: 5555 metric-namespace: crabby.crabby-sfo-1 tags: - - crabby-sfo-1 + - instance: crabby-sfo-1 report-internal-metrics: true internal-metrics-gathering-interval: 15 +server-tags: + - region: us-east-1 \ No newline at end of file From 7f3691fd7c85dc0d5ee6403fe094a1d31e1f3b87 Mon Sep 17 00:00:00 2001 From: Chris Snell Date: Sat, 9 Jun 2018 17:10:05 -0500 Subject: [PATCH 4/5] More work on Prometheus support --- storage.go | 4 ++- storage_dogstatsd.go | 6 ++-- storage_graphite.go | 6 ++-- storage_prometheus.go | 72 ++++++++++++++++++++++++++++++++++++++----- storage_riemann.go | 4 +-- 5 files changed, 76 insertions(+), 16 deletions(-) diff --git a/storage.go b/storage.go index e367872..4cf9cb0 100644 --- a/storage.go +++ b/storage.go @@ -11,6 +11,7 @@ import ( // Metric holds one metric data point type Metric struct { Name string + Timing string Value float64 Tags map[string]string Timestamp time.Time @@ -173,7 +174,8 @@ func (s *Storage) storageDistributor(ctx context.Context, wg *sync.WaitGroup) er func makeMetric(j Job, timing string, value float64) Metric { m := Metric{ - Name: fmt.Sprintf("%v.%v", j.Name, timing), + Name: j.Name, + Timing: timing, Value: value, Timestamp: time.Now(), } diff --git a/storage_dogstatsd.go b/storage_dogstatsd.go index f667835..0678e85 100644 --- a/storage_dogstatsd.go +++ b/storage_dogstatsd.go @@ -64,9 +64,9 @@ func (d DogstatsdStorage) sendMetric(m Metric) error { var metricName string if d.Namespace == "" { - metricName = fmt.Sprintf("crabby.%v", m.Name) + metricName = fmt.Sprintf("crabby.%v.%v", m.Name, m.Timing) } else { - metricName = fmt.Sprintf("%v.%v", d.Namespace, m.Name) + metricName = fmt.Sprintf("%v.%v.%v", d.Namespace, m.Name, m.Timing) } // Add all of our metric tags to the metric payload @@ -76,7 +76,7 @@ func (d DogstatsdStorage) sendMetric(m Metric) error { err := d.DogstatsdConn.TimeInMilliseconds(metricName, m.Value, nil, 1) if err != nil { - log.Printf("Could not send metric %v: %v\n", m.Name, err) + log.Printf("Could not send metric %v.%v: %v\n", m.Name, m.Timing, err) return err } diff --git a/storage_graphite.go b/storage_graphite.go index ea5acc0..d65d468 100644 --- a/storage_graphite.go +++ b/storage_graphite.go @@ -67,9 +67,9 @@ func (g GraphiteStorage) sendMetric(m Metric) error { valStr := strconv.FormatFloat(m.Value, 'f', 3, 64) if g.Namespace == "" { - metricName = fmt.Sprintf("crabby.%v", m.Name) + metricName = fmt.Sprintf("crabby.%v.%v", m.Name, m.Timing) } else { - metricName = fmt.Sprintf("%v.%v", g.Namespace, m.Name) + metricName = fmt.Sprintf("%v.%v.%v", g.Namespace, m.Name, m.Timing) } if m.Timestamp.IsZero() { @@ -89,7 +89,7 @@ func (g GraphiteStorage) sendMetric(m Metric) error { err := g.GraphiteConn.SendMetric(gm) if err != nil { - log.Printf("Could not send metric %v: %v\n", m.Name, err) + log.Printf("Could not send metric %v.%v: %v\n", m.Name, m.Timing, err) return err } diff --git a/storage_prometheus.go b/storage_prometheus.go index 5946988..f1ddaaa 100644 --- a/storage_prometheus.go +++ b/storage_prometheus.go @@ -18,11 +18,28 @@ type PrometheusConfig struct { Namespace string `yaml:"metric-namespace,omitempty"` } +// CurrentMetrics is a map of Metrics + a mutex that maintains the most recent metrics +// that we are collecting. These are maintained so that a Prometheus client can connect +// to the built-in Prometheus endpoint and fetch the very latest result for this +// metric. +type CurrentMetrics struct { + metrics map[string]MetricStore + sync.RWMutex +} + +// MetricStore holds a Prometheus GaugeVec for a given metric +type MetricStore struct { + name string + timing string + gaugeVec prometheus.GaugeVec +} + // PrometheusStorage holds the configuration for a Graphite storage backend type PrometheusStorage struct { - Namespace string - Registry *prometheus.Registry - url string + Namespace string + Registry *prometheus.Registry + url string + currentMetrics *CurrentMetrics } // NewPrometheusStorage sets up a new Prometheus storage backend @@ -61,10 +78,7 @@ func (p PrometheusStorage) processMetrics(ctx context.Context, wg *sync.WaitGrou for { select { case m := <-mchan: - err := p.sendMetric(m) - if err != nil { - log.Println(err) - } + p.storeCurrentMetric(m) case <-ctx.Done(): log.Println("Cancellation request recieved. Cancelling metrics processor.") return @@ -105,3 +119,47 @@ func (p PrometheusStorage) sendEvent(e Event) error { var err error return err } + +func (p PrometheusStorage) storeCurrentMetric(m Metric) { + p.currentMetrics.Lock() + defer p.currentMetrics.Unlock() + + // See if this metric name exists in the current metrics map + cm, exists := p.currentMetrics.metrics[m.Timing] + + // If the metric doesn't exist, create a new MetricsStore and Gauge for it + if !exists { + var glabel []string + + for k := range m.Tags { + glabel = append(glabel, k) + } + + gauge := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: m.Timing, + }, glabel, + ) + + // Create a MetricsStore/Gauge + ms := MetricStore{ + name: m.Name, + gaugeVec: *gauge, + } + + // ...and add that to our map + p.currentMetrics.metrics[m.Name] = ms + + // Register the Gauge with Prometheus + prometheus.MustRegister(p.currentMetrics.metrics[m.Timing].gaugeVec) + + } else { + labels := prometheus.Labels{} + for k, v := range m.Tags { + labels[k] = v + } + labels["name"] = m.Name + cm.gaugeVec.With(labels).Set(m.Value) + } + +} diff --git a/storage_riemann.go b/storage_riemann.go index 4debc0d..9b6dcb3 100644 --- a/storage_riemann.go +++ b/storage_riemann.go @@ -85,9 +85,9 @@ func (r RiemannStorage) sendMetric(m Metric) error { var metricName string if r.Namespace == "" { - metricName = fmt.Sprintf("crabby.%v", m.Name) + metricName = fmt.Sprintf("crabby.%v.%v", m.Name, m.Timing) } else { - metricName = fmt.Sprintf("%v.%v", r.Namespace, m.Name) + metricName = fmt.Sprintf("%v.%v.%v", r.Namespace, m.Name, m.Timing) } ev := &goryman.Event{ From 7cab2a764627a5d10f30ec3fbf60e06641fc49a9 Mon Sep 17 00:00:00 2001 From: Chris Snell Date: Sat, 9 Jun 2018 17:58:26 -0500 Subject: [PATCH 5/5] Remove storage backend tags...unneeded --- storage_dogstatsd.go | 10 +++------- storage_riemann.go | 11 +++-------- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/storage_dogstatsd.go b/storage_dogstatsd.go index 0678e85..94c19d1 100644 --- a/storage_dogstatsd.go +++ b/storage_dogstatsd.go @@ -12,10 +12,9 @@ import ( // DogstatsdConfig describes the YAML-provided configuration for a Datadog // DogstatsD storage backend type DogstatsdConfig struct { - Host string `yaml:"host"` - Port int `yaml:"port"` - Namespace string `yaml:"metric-namespace"` - Tags map[string]string `yaml:"tags,omitempty"` + Host string `yaml:"host"` + Port int `yaml:"port"` + Namespace string `yaml:"metric-namespace"` } // DogstatsdStorage holds the configuration for a DogstatsD storage backend @@ -133,8 +132,5 @@ func NewDogstatsdStorage(c *Config) DogstatsdStorage { log.Println("Warning: could not create dogstatsd connection", err) } - for k, v := range c.Storage.Dogstatsd.Tags { - d.DogstatsdConn.Tags = append(d.DogstatsdConn.Tags, k+":"+v) - } return d } diff --git a/storage_riemann.go b/storage_riemann.go index 9b6dcb3..659d434 100644 --- a/storage_riemann.go +++ b/storage_riemann.go @@ -12,10 +12,9 @@ import ( // RiemannConfig describes the YAML-provided configuration for a Riemann // storage backend type RiemannConfig struct { - Host string `yaml:"host"` - Port int `yaml:"port"` - Namespace string `yaml:"metric-namespace,omitempty"` - Tags map[string]string `yaml:"tags,omitempty"` + Host string `yaml:"host"` + Port int `yaml:"port"` + Namespace string `yaml:"metric-namespace,omitempty"` } // RiemannStorage holds the configuration for a Riemann storage backend @@ -31,10 +30,6 @@ func NewRiemannStorage(c *Config) (RiemannStorage, error) { r.Namespace = c.Storage.Riemann.Namespace - for k, v := range c.Storage.Riemann.Tags { - r.Tags = append(r.Tags, k+":"+v) - } - r.Client = goryman.NewGorymanClient(fmt.Sprint(c.Storage.Riemann.Host, ":", c.Storage.Riemann.Port)) err := r.Client.Connect() if err != nil {