Skip to content

Commit

Permalink
feat: add prometheus metrics & endpoint
Browse files Browse the repository at this point in the history
  • Loading branch information
maigl committed Jan 19, 2023
1 parent c92dfb5 commit d654738
Show file tree
Hide file tree
Showing 308 changed files with 36,108 additions and 1,658 deletions.
44 changes: 44 additions & 0 deletions apiserver/controllers/controllers.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ import (
"io/ioutil"
"log"
"net/http"
"os"
"strings"

"garm/apiserver/params"
"garm/auth"
Expand All @@ -29,6 +31,7 @@ import (

"github.com/gorilla/websocket"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
)

func NewAPIController(r *runner.Runner, auth *auth.Authenticator, hub *wsWriter.Hub) (*APIController, error) {
Expand Down Expand Up @@ -82,6 +85,26 @@ func handleError(w http.ResponseWriter, err error) {
json.NewEncoder(w).Encode(apiErr)
}

// GetControllerInfo returns means to identify this very garm instance.
// This is very useful for debugging and monitoring purposes.
func (a *APIController) GetControllerInfo() (hostname, controllerId string) {

var err error
hostname, err = os.Hostname()
if err != nil {
log.Printf("error getting hostname: %q", err)
return "", ""
}

info, err := a.r.GetControllerID()
if err != nil {
log.Printf("error getting controller info: %q", err)
return hostname, ""
}

return hostname, info.String()
}

func (a *APIController) authenticateHook(body []byte, headers http.Header) error {
// signature := headers.Get("X-Hub-Signature-256")
hookType := headers.Get("X-Github-Hook-Installation-Target-Type")
Expand All @@ -99,6 +122,18 @@ func (a *APIController) authenticateHook(body []byte, headers http.Header) error
return nil
}

// metric to count total webhooks received
// at this point the webhook is not yet authenticated and
// we don't know if it's meant for us or not
var webhooksReceived = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "garm_webhooks_received",
Help: "The total number of webhooks received",
}, []string{"valid", "reason", "hostname", "controller_id"})

func init() {
prometheus.MustRegister(webhooksReceived)
}

func (a *APIController) handleWorkflowJobEvent(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
body, err := ioutil.ReadAll(r.Body)
Expand All @@ -110,14 +145,23 @@ func (a *APIController) handleWorkflowJobEvent(w http.ResponseWriter, r *http.Re
signature := r.Header.Get("X-Hub-Signature-256")
hookType := r.Header.Get("X-Github-Hook-Installation-Target-Type")

hostname, controllerId := a.GetControllerInfo()

if err := a.r.DispatchWorkflowJob(hookType, signature, body); err != nil {
if errors.Is(err, gErrors.ErrNotFound) {
webhooksReceived.WithLabelValues("false", "owner_unknown", hostname, controllerId).Inc()
log.Printf("got not found error from DispatchWorkflowJob. webhook not meant for us?: %q", err)
return
} else if strings.Contains(err.Error(), "signature") {
webhooksReceived.WithLabelValues("false", "signature_invalid", hostname, controllerId).Inc()
} else {
webhooksReceived.WithLabelValues("false", "unknown", hostname, controllerId).Inc()
}

handleError(w, err)
return
}
webhooksReceived.WithLabelValues("true", "", hostname, controllerId).Inc()
}

func (a *APIController) CatchAll(w http.ResponseWriter, r *http.Request) {
Expand Down
93 changes: 93 additions & 0 deletions apiserver/controllers/metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
package controllers

import (
"log"

"garm/auth"
"github.com/prometheus/client_golang/prometheus"
)

type GarmCollector struct {
instanceMetric *prometheus.Desc
apiController *APIController
}

func NewGarmCollector(a *APIController) *GarmCollector {
return &GarmCollector{
apiController: a,
instanceMetric: prometheus.NewDesc(
"garm_runner_status",
"Status of the runner",
[]string{"name", "status", "runner_status", "pool", "pool_type", "hostname", "controller_id"}, nil,
)}
}

func (c *GarmCollector) Describe(ch chan<- *prometheus.Desc) {
ch <- c.instanceMetric
}

func (c *GarmCollector) Collect(ch chan<- prometheus.Metric) {
c.CollectInstanceMetric(ch)
}

// CollectInstanceMetric collects the metrics for the runner instances
// reflecting the statuses and the pool they belong to.
func (c *GarmCollector) CollectInstanceMetric(ch chan<- prometheus.Metric) {

ctx := auth.GetAdminContext()

instances, err := c.apiController.r.ListAllInstances(ctx)
if err != nil {
log.Printf("cannot collect metrics, listing instances: %s", err)
return
}

pools, err := c.apiController.r.ListAllPools(ctx)
if err != nil {
log.Printf("listing pools: %s", err)
// continue anyway
}

type poolInfo struct {
Name string
Type string
}

poolNames := make(map[string]poolInfo)
for _, pool := range pools {
if pool.EnterpriseName != "" {
poolNames[pool.ID] = poolInfo{
Name: pool.EnterpriseName,
Type: "enterprise",
}
} else if pool.OrgName != "" {
poolNames[pool.ID] = poolInfo{
Name: pool.OrgName,
Type: "organization",
}
} else {
poolNames[pool.ID] = poolInfo{
Name: pool.RepoName,
Type: "repository",
}
}
}

hostname, controllerID := c.apiController.GetControllerInfo()

for _, instance := range instances {

ch <- prometheus.MustNewConstMetric(
c.instanceMetric,
prometheus.GaugeValue,
1,
instance.Name,
string(instance.Status),
string(instance.RunnerStatus),
poolNames[instance.PoolID].Name,
poolNames[instance.PoolID].Type,
hostname,
controllerID,
)
}
}
7 changes: 7 additions & 0 deletions apiserver/routers/routers.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,20 @@ import (
"garm/apiserver/controllers"
"garm/auth"
"garm/util"

"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
)

func NewAPIRouter(han *controllers.APIController, logWriter io.Writer, authMiddleware, initMiddleware, instanceMiddleware auth.Middleware) *mux.Router {
router := mux.NewRouter()
logMiddleware := util.NewLoggingMiddleware(logWriter)
router.Use(logMiddleware)

// Handles github webhooks
router.Handle("/metrics", promhttp.Handler())
prometheus.MustRegister(controllers.NewGarmCollector(han))

// Handles github webhooks
webhookRouter := router.PathPrefix("/webhooks").Subrouter()
webhookRouter.PathPrefix("/").Handler(http.HandlerFunc(han.CatchAll))
Expand Down
13 changes: 10 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ require (
github.com/spf13/cobra v1.4.1-0.20220504202302-9e88759b19cd
github.com/stretchr/testify v1.8.0
golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8
golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f
golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a
gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0
gopkg.in/natefinch/lumberjack.v2 v2.0.0
gopkg.in/yaml.v3 v3.0.1
Expand All @@ -33,6 +33,8 @@ require (
)

require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/felixge/httpsnoop v1.0.1 // indirect
Expand All @@ -52,9 +54,14 @@ require (
github.com/kr/pretty v0.3.0 // indirect
github.com/mattn/go-runewidth v0.0.13 // indirect
github.com/mattn/go-sqlite3 v1.14.12 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/pborman/uuid v1.2.1 // indirect
github.com/pkg/sftp v1.13.4 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.14.0 // indirect
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.37.0 // indirect
github.com/prometheus/procfs v0.8.0 // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/robfig/cron/v3 v3.0.1 // indirect
github.com/rogpeppe/fastuuid v1.2.0 // indirect
Expand All @@ -66,7 +73,7 @@ require (
golang.org/x/net v0.0.0-20220325170049-de3da57026de // indirect
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.28.0 // indirect
google.golang.org/protobuf v1.28.1 // indirect
gopkg.in/errgo.v1 v1.0.1 // indirect
gopkg.in/httprequest.v1 v1.2.1 // indirect
gopkg.in/macaroon-bakery.v2 v2.3.0 // indirect
Expand Down
Loading

0 comments on commit d654738

Please sign in to comment.