refactor: implement scraping through prometheus collector

clambin · Aug 3, 2021 · 3abe570 · 3abe570
1 parent 99d4c86
commit 3abe570
Show file tree

Hide file tree

Showing 11 changed files with 309 additions and 161 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -5,6 +5,7 @@ on:
     branches:
     - master
     - refactor
+    - metrics
 
 jobs:
   test:

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -5,6 +5,7 @@ on:
     branches-ignore:
       - master
       - refactor
+      - metrics
   pull_request_target:
 
 jobs:

diff --git a/assets/grafana/dashboards/pinger.json b/assets/grafana/dashboards/pinger.json
@@ -15,19 +15,18 @@
   "editable": true,
   "gnetId": null,
   "graphTooltip": 0,
-  "id": 156,
+  "id": 450,
   "links": [],
   "panels": [
     {
       "aliasColors": {},
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": null,
+      "datasource": "Prometheus",
       "description": "",
       "fieldConfig": {
         "defaults": {
-          "custom": {},
           "links": [],
           "unit": "s"
         },
@@ -61,7 +60,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "7.3.4",
+      "pluginVersion": "8.0.6",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -71,6 +70,7 @@
       "steppedLine": false,
       "targets": [
         {
+          "exemplar": true,
           "expr": "sum by (host) (pinger_latency_seconds / pinger_packet_count)",
           "interval": "",
           "legendFormat": "{{host}}",
@@ -97,6 +97,7 @@
       },
       "yaxes": [
         {
+          "$$hashKey": "object:83",
           "format": "s",
           "label": null,
           "logBase": 1,
@@ -105,6 +106,7 @@
           "show": true
         },
         {
+          "$$hashKey": "object:84",
           "format": "short",
           "label": null,
           "logBase": 1,
@@ -119,108 +121,101 @@
       }
     },
     {
-      "aliasColors": {},
-      "bars": false,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": null,
-      "decimals": 0,
+      "datasource": "Prometheus",
       "description": "",
       "fieldConfig": {
         "defaults": {
-          "custom": {},
-          "links": []
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "stepAfter",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "links": [],
+          "mappings": [],
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "short"
         },
         "overrides": []
       },
-      "fill": 1,
-      "fillGradient": 0,
       "gridPos": {
         "h": 11,
         "w": 12,
         "x": 12,
         "y": 0
       },
-      "hiddenSeries": false,
       "id": 5,
-      "legend": {
-        "alignAsTable": true,
-        "avg": true,
-        "current": true,
-        "max": true,
-        "min": true,
-        "rightSide": false,
-        "show": true,
-        "total": false,
-        "values": true
-      },
-      "lines": true,
-      "linewidth": 1,
-      "nullPointMode": "null",
       "options": {
-        "alertThreshold": true
+        "legend": {
+          "calcs": [
+            "mean",
+            "lastNotNull",
+            "max",
+            "min"
+          ],
+          "displayMode": "table",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single"
+        }
       },
-      "percentage": false,
-      "pluginVersion": "7.3.4",
-      "pointradius": 2,
-      "points": false,
-      "renderer": "flot",
-      "seriesOverrides": [],
-      "spaceLength": 10,
-      "stack": false,
-      "steppedLine": false,
+      "pluginVersion": "8.0.6",
       "targets": [
         {
-          "expr": "sum by (host) (increase(pinger_packet_loss_count[5m]))",
+          "exemplar": true,
+          "expr": "sum by (host) (increase(pinger_packet_loss_count[2m]))",
           "interval": "",
           "legendFormat": "{{host}}",
           "refId": "B"
         }
       ],
-      "thresholds": [],
       "timeFrom": null,
-      "timeRegions": [],
       "timeShift": null,
       "title": "Packet Loss",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
-      },
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
-      },
-      "yaxes": [
-        {
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": "0",
-          "show": true
-        },
-        {
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": "0",
-          "show": true
-        }
-      ],
-      "yaxis": {
-        "align": false,
-        "alignLevel": null
-      }
+      "type": "timeseries"
     }
   ],
   "refresh": "10s",
-  "schemaVersion": 26,
+  "schemaVersion": 30,
   "style": "dark",
   "tags": [],
   "templating": {
@@ -247,5 +242,5 @@
   "timezone": "",
   "title": "Pinger",
   "uid": "WHj4zUggz",
-  "version": 2
+  "version": 5
 }
diff --git a/go.mod b/go.mod
@@ -4,8 +4,8 @@ go 1.15
 
 require (
 	github.com/alecthomas/units v0.0.0-20201120081800-1786d5ef83d4 // indirect
-	github.com/clambin/gotools v0.3.90
 	github.com/prometheus/client_golang v1.11.0
+	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/sirupsen/logrus v1.8.1
 	github.com/stretchr/testify v1.7.0
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6

diff --git a/pinger.go b/pinger.go
@@ -1,9 +1,11 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"github.com/clambin/pinger/pinger"
 	"github.com/clambin/pinger/version"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	log "github.com/sirupsen/logrus"
 	"gopkg.in/alecthomas/kingpin.v2"
@@ -29,7 +31,7 @@ func main() {
 	a.Flag("port", "Metrics listener port").Default("8080").IntVar(&cfg.port)
 	a.Flag("endpoint", "Metrics listener endpoint").Default("/metrics").StringVar(&cfg.endpoint)
 	a.Flag("debug", "Log debug messages").BoolVar(&cfg.debug)
-	a.Flag("interval", "Measurement interval").Default("5s").DurationVar(&cfg.interval)
+	// a.Flag("interval", "Measurement interval").Default("5s").DurationVar(&cfg.interval)
 	hosts := a.Arg("hosts", "hosts to ping").Strings()
 
 	_, err := a.Parse(os.Args[1:])
@@ -47,12 +49,17 @@ func main() {
 		hosts = &values
 	}
 
-	log.Infof("pinger %s - hosts: %s", version.BuildVersion, *hosts)
+	log.WithField("hosts", *hosts).Infof("pinger %s", version.BuildVersion)
 
-	go pinger.Run(*hosts, cfg.interval)
+	p := pinger.New(*hosts)
+	prometheus.MustRegister(p)
 
-	// Run initialized & runs the metrics
+	go p.Run(context.Background())
+
+	// Run the metrics server
 	listenAddress := fmt.Sprintf(":%d", cfg.port)
 	http.Handle(cfg.endpoint, promhttp.Handler())
-	_ = http.ListenAndServe(listenAddress, nil)
+	err = http.ListenAndServe(listenAddress, nil)
+
+	log.WithError(err).Error("failed to start http server")
 }
diff --git a/pinger/collect.go b/pinger/collect.go
@@ -0,0 +1,26 @@
+package pinger
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	log "github.com/sirupsen/logrus"
+)
+
+// Describe interface for Prometheus collector
+func (pinger *Pinger) Describe(ch chan<- *prometheus.Desc) {
+	ch <- pinger.packetsMetric
+	ch <- pinger.lossMetric
+	ch <- pinger.latencyMetric
+}
+
+// Collect interface for Prometheus collector
+func (pinger *Pinger) Collect(ch chan<- prometheus.Metric) {
+	for host, tracker := range pinger.Trackers {
+		count, loss, latency := tracker.Calculate()
+
+		log.WithFields(log.Fields{"host": host, "count": count, "loss": loss, "latency": latency}).Debug()
+
+		ch <- prometheus.MustNewConstMetric(pinger.packetsMetric, prometheus.GaugeValue, float64(count), host)
+		ch <- prometheus.MustNewConstMetric(pinger.lossMetric, prometheus.GaugeValue, float64(loss), host)
+		ch <- prometheus.MustNewConstMetric(pinger.latencyMetric, prometheus.GaugeValue, latency.Seconds(), host)
+	}
+}
-Original file line number
+Diff line change
@@ Expand Up / @@ -5,6 +5,7 @@ on: @@
         branches:
         - master
         - refactor
+        - metrics
     jobs:
       test:
@@ Expand Down @@