Skip to content

Commit

Permalink
Parameterize pagespeed categories (#45)
Browse files Browse the repository at this point in the history 
This creates an environment variable and CLI flag to set which pagespeed
categories to fetch. By default, it fetches all five, preserving this package's
default functionality.

If categories are specified in a target as JSON, it will supercede the
categories specified via the envvar or flag.
  • Loading branch information
@penmanglewood
penmanglewood authored Jun 28, 2022
1 parent 78700a4 commit 8442695
Show file tree
Hide file tree
Showing 9 changed files with 218 additions and 70 deletions.
26 changes: 17 additions & 9 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ The provided dashboard (Pagespeed) will be loaded with data after the first scra

The dashboard can be found at [grafana](https://grafana.com/dashboards/9510)

Note: The example dashboard assumes you're fetching all pagespeed categories.

## Understanding Metrics

* https://github.com/GoogleChrome/lighthouse/blob/master/docs/understanding-results.md
Expand Down Expand Up @@ -62,12 +64,16 @@ Or via JSON which adds additional parameters
// URL can't be invalid
// Strategy can only be mobile/desktop
// If strategy is not specified, both desktop & mobile will be used
// Categories can be any of accessibility/best-practices/performance/pwa/seo
// If categories are not specified, all categories will be used
// Parameters are passed down to google pagespeed api
{"url":"https://github.com/foomo/pagespeed_exporter","campaign":"test","locale":"en","source":"source"}
{"url":"https://mysite.com/test?test=true","strategy":"mobile"}
{"url":"https://mysite.com/test?test=true","categories": ["best-practices"]}
```

Configuration specification in JSON and plain is supported both in command line & prometheus configuration
Expand All @@ -76,15 +82,16 @@ Configuration specification in JSON and plain is supported both in command line

Configuration of targets can be done via docker and via prometheus

| Flag | Variable | Description | Default | Required |
|------------------|--------------------|-----------------------------------------------|--------------------|----------|
| -api-key | PAGESPEED_API_KEY | sets the google API key used for pagespeed | | False |
| -targets | PAGESPEED_TARGETS | comma separated list of targets to measure | | False |
| -t | NONE | multi-value target array (check docker comp) | | False |
| -listener | PAGESPEED_LISTENER | sets the listener address for the exporters | :9271 | False |
| -parallel | PAGESPEED_PARALLEL | sets the execution of targets to be parallel | false | False |
| -pushGatewayUrl | PUSHGATEWAY_URL | sets the pushgateway url to send the metrics | | False |
| -pushGatewayJob | PUSHGATEWAY_JOB | sets the pushgateway job name | pagespeed_exporter | False |
| Flag | Variable | Description | Default | Required |
|------------------|----------------------|-----------------------------------------------|--------------------------------------------------|----------|
| -api-key | PAGESPEED_API_KEY | sets the google API key used for pagespeed | | False |
| -targets | PAGESPEED_TARGETS | comma separated list of targets to measure | | False |
| -categories | PAGESPEED_CATEGORIES | comma separated list of categories to check | accessibility,best-practices,performance,pwa,seo | False |
| -t | NONE | multi-value target array (check docker comp) | | False |
| -listener | PAGESPEED_LISTENER | sets the listener address for the exporters | :9271 | False |
| -parallel | PAGESPEED_PARALLEL | sets the execution of targets to be parallel | false | False |
| -pushGatewayUrl | PUSHGATEWAY_URL | sets the pushgateway url to send the metrics | | False |
| -pushGatewayJob | PUSHGATEWAY_JOB | sets the pushgateway job name | pagespeed_exporter | False |

Note: google api key is required only if scraping more than 2 targets/second

Expand Down Expand Up @@ -137,6 +144,7 @@ or
$ docker run -p "9271:9271" --rm \
--env PAGESPEED_API_KEY={KEY} \
--env PAGESPEED_TARGETS=https://google.com,https://prometheus.io \
--env PAGESPEED_CATEGORIES=accessibility,pwa \
foomo/pagespeed_exporter
```

Expand Down
33 changes: 18 additions & 15 deletions collector/collector.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,11 @@ var (
_ Factory = factory{}
)

var (
timeValueRe = regexp.MustCompile(`(\d*[.]?\d+(ms|s))|0`)
timeUnitRe = regexp.MustCompile(`(ms|s)`)
)

type Factory interface {
Create(config Config) (prometheus.Collector, error)
}
Expand Down Expand Up @@ -124,7 +129,7 @@ func collect(scrape *ScrapeResult, ch chan<- prometheus.Metric) error {
}

if r.LighthouseResult != nil {
collectLighthouseResults("lighthouse", r.LighthouseResult, constLabels, ch)
collectLighthouseResults("lighthouse", scrape.Request.Categories, r.LighthouseResult, constLabels, ch)
}
return nil
}
Expand Down Expand Up @@ -162,46 +167,44 @@ func collectLoadingExperience(prefix string, lexp *pagespeedonline.PagespeedApiL

}

func collectLighthouseResults(prefix string, lhr *pagespeedonline.LighthouseResultV5, constLabels prometheus.Labels, ch chan<- prometheus.Metric) {
func collectLighthouseResults(prefix string, cats []string, lhr *pagespeedonline.LighthouseResultV5, constLabels prometheus.Labels, ch chan<- prometheus.Metric) {

ch <- prometheus.MustNewConstMetric(
prometheus.NewDesc(fqname(prefix, "total_duration_seconds"), "The total time spent in seconds loading the page and evaluating audits.", nil, constLabels),
prometheus.GaugeValue,
lhr.Timing.Total/1000) //ms -> seconds

categories := map[string]*pagespeedonline.LighthouseCategoryV5{
"performance": lhr.Categories.Performance,
"accessibility": lhr.Categories.Accessibility,
"pwa": lhr.Categories.Pwa,
"best-practices": lhr.Categories.BestPractices,
"seo": lhr.Categories.Seo,
CategoryPerformance: lhr.Categories.Performance,
CategoryAccessibility: lhr.Categories.Accessibility,
CategoryPWA: lhr.Categories.Pwa,
CategoryBestPractices: lhr.Categories.BestPractices,
CategorySEO: lhr.Categories.Seo,
}

for k, v := range categories {
score, err := strconv.ParseFloat(fmt.Sprint(v.Score), 64)
for _, c := range cats {
score, err := strconv.ParseFloat(fmt.Sprint(categories[c].Score), 64)
if err != nil {
logrus.WithError(err).Warn("could not parse category score")
continue
}

ch <- prometheus.MustNewConstMetric(
prometheus.NewDesc(fqname(prefix, "category_score"), "Lighthouse score for the specified category", []string{"category"}, constLabels),
prometheus.GaugeValue,
score,
k)
c)
}

for k, v := range lhr.Audits {
re := regexp.MustCompile(`(\d*[.]?\d+(ms|s))|0`)
timeRe := regexp.MustCompile(`(ms|s)`)

if timeAuditMetrics[k] {
displayValue := strings.Replace(v.DisplayValue, "\u00a0", "", -1)
displayValue = strings.Replace(displayValue, ",", "", -1)
if !timeRe.MatchString(displayValue) {
if !timeUnitRe.MatchString(displayValue) {
displayValue = displayValue + "s"
}

if duration, errDuration := time.ParseDuration(re.FindString(displayValue)); errDuration == nil {
if duration, errDuration := time.ParseDuration(timeValueRe.FindString(displayValue)); errDuration == nil {
ch <- prometheus.MustNewConstMetric(
prometheus.NewDesc(fqname(prefix, k, "duration_seconds"), v.Description, nil, constLabels),
prometheus.GaugeValue,
Expand Down
63 changes: 53 additions & 10 deletions collector/model.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,12 @@ const (
StrategyMobile = Strategy("mobile")
StrategyDesktop = Strategy("desktop")

CategoryAccessibility = "accessibility"
CategoryBestPractices = "best-practices"
CategorySEO = "seo"
CategoryPWA = "pwa"
CategoryPerformance = "performance"

Namespace = "pagespeed"
)

Expand All @@ -22,17 +28,26 @@ var availableStrategies = map[Strategy]bool{
StrategyDesktop: true,
}

var availableCategories = map[string]bool{
CategoryAccessibility: true,
CategoryBestPractices: true,
CategorySEO: true,
CategoryPWA: true,
CategoryPerformance: true,
}

type ScrapeResult struct {
Request ScrapeRequest
Result *pagespeedonline.PagespeedApiPagespeedResponseV5
}

type ScrapeRequest struct {
Url string `json:"url"`
Strategy Strategy `json:"strategy"`
Campaign string `json:"campaign"`
Source string `json:"source"`
Locale string `json:"locale"`
Url string `json:"url"`
Strategy Strategy `json:"strategy"`
Campaign string `json:"campaign"`
Source string `json:"source"`
Locale string `json:"locale"`
Categories []string `json:"categories"`
}

func (sr ScrapeRequest) IsValid() bool {
Expand All @@ -43,6 +58,13 @@ func (sr ScrapeRequest) IsValid() bool {
if !availableStrategies[sr.Strategy] {
return false
}

for _, c := range sr.Categories {
if !availableCategories[c] {
return false
}
}

if _, err := url.ParseRequestURI(sr.Url); err != nil {
return false
}
Expand All @@ -58,7 +80,7 @@ type Config struct {
ScrapeTimeout time.Duration
}

func CalculateScrapeRequests(targets ...string) []ScrapeRequest {
func CalculateScrapeRequests(targets, categories []string) []ScrapeRequest {
if len(targets) == 0 {
return nil
}
Expand All @@ -67,6 +89,7 @@ func CalculateScrapeRequests(targets ...string) []ScrapeRequest {
for _, t := range targets {
var request ScrapeRequest
if err := json.Unmarshal([]byte(t), &request); err == nil {
populateCategories(&request, categories)
if request.Strategy != "" {
requests = append(requests, request)
} else {
Expand All @@ -77,10 +100,11 @@ func CalculateScrapeRequests(targets ...string) []ScrapeRequest {
requests = append(requests, desktop, mobile)
}
} else {
requests = append(requests,
ScrapeRequest{Url: t, Strategy: StrategyDesktop},
ScrapeRequest{Url: t, Strategy: StrategyMobile},
)
desktop := ScrapeRequest{Url: t, Strategy: StrategyDesktop}
mobile := ScrapeRequest{Url: t, Strategy: StrategyMobile}
populateCategories(&desktop, categories)
populateCategories(&mobile, categories)
requests = append(requests, desktop, mobile)
}
}

Expand All @@ -94,3 +118,22 @@ func CalculateScrapeRequests(targets ...string) []ScrapeRequest {

return filtered
}

// populateCategories sets categories in the scrape request if not already set
func populateCategories(r *ScrapeRequest, cats []string) {
if r.Categories != nil && len(r.Categories) != 0 {
return
}

if cats == nil {
cats = make([]string, 0, len(availableCategories))
}

if len(cats) == 0 {
for c := range availableCategories {
cats = append(cats, c)
}
}

r.Categories = cats
}
Loading

0 comments on commit 8442695

Please sign in to comment.