Skip to content

Commit

Permalink
feat(metrics): add basic metrics type support (#789)
Browse files Browse the repository at this point in the history
Co-authored-by: Michi Hoffmann <[email protected]>
viglia and cleptric authored Apr 30, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent 84427d8 commit 9a9f285
Showing 6 changed files with 726 additions and 0 deletions.
4 changes: 4 additions & 0 deletions interfaces.go
Original file line number Diff line number Diff line change
@@ -25,6 +25,9 @@ const profileType = "profile"
// checkInType is the type of a check in event.
const checkInType = "check_in"

// metricType is the type of a metric event.
const metricType = "statsd"

// Level marks the severity of the event.
type Level string

@@ -320,6 +323,7 @@ type Event struct {
Exception []Exception `json:"exception,omitempty"`
DebugMeta *DebugMeta `json:"debug_meta,omitempty"`
Attachments []*Attachment `json:"-"`
Metrics []Metric `json:"-"`

// The fields below are only relevant for transactions.

36 changes: 36 additions & 0 deletions interfaces_test.go
Original file line number Diff line number Diff line change
@@ -513,3 +513,39 @@ func TestStructSnapshots(t *testing.T) {
})
}
}

func TestMarshalMetrics(t *testing.T) {
tests := []struct {
name string
metrics []Metric
want string
}{
{
name: "allowed characters",
metrics: []Metric{
NewCounterMetric("counter", Second(), map[string]string{"foo": "bar", "route": "GET /foo"}, 1597790835, 1.0),
NewDistributionMetric("distribution", Second(), map[string]string{"$foo$": "%bar%"}, 1597790835, 1.0),
NewGaugeMetric("gauge", Second(), map[string]string{"föö": "bär"}, 1597790835, 1.0),
NewSetMetric[int]("set", Second(), map[string]string{"%{key}": "$value$"}, 1597790835, 1),
NewCounterMetric("no_tags", Second(), nil, 1597790835, 1.0),
},

want: strings.Join([]string{
"counter@second:1|c|#foo:bar,route:GET /foo|T1597790835",
"distribution@second:1|d|#_foo_:bar|T1597790835",
"gauge@second:1:1:1:1:1|g|#f_:br|T1597790835",
"set@second:1|s|#_key_:$value$|T1597790835",
"no_tags@second:1|c|T1597790835",
}, "\n"),
},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
serializedMetric := marshalMetrics(test.metrics)
if diff := cmp.Diff(string(serializedMetric), test.want); diff != "" {
t.Errorf("Context mismatch (-want +got):\n%s", diff)
}
})
}
}
427 changes: 427 additions & 0 deletions metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,427 @@
package sentry

import (
"fmt"
"hash/crc32"
"math"
"regexp"
"sort"
"strings"
)

type (
NumberOrString interface {
int | string
}

void struct{}
)

var (
member void
keyRegex = regexp.MustCompile(`[^a-zA-Z0-9_/.-]+`)
valueRegex = regexp.MustCompile(`[^\w\d\s_:/@\.{}\[\]$-]+`)
unitRegex = regexp.MustCompile(`[^a-z]+`)
)

type MetricUnit struct {
unit string
}

func (m MetricUnit) toString() string {
return m.unit
}

func NanoSecond() MetricUnit {
return MetricUnit{
"nanosecond",
}
}

func MicroSecond() MetricUnit {
return MetricUnit{
"microsecond",
}
}

func MilliSecond() MetricUnit {
return MetricUnit{
"millisecond",
}
}

func Second() MetricUnit {
return MetricUnit{
"second",
}
}

func Minute() MetricUnit {
return MetricUnit{
"minute",
}
}

func Hour() MetricUnit {
return MetricUnit{
"hour",
}
}

func Day() MetricUnit {
return MetricUnit{
"day",
}
}

func Week() MetricUnit {
return MetricUnit{
"week",
}
}

func Bit() MetricUnit {
return MetricUnit{
"bit",
}
}

func Byte() MetricUnit {
return MetricUnit{
"byte",
}
}

func KiloByte() MetricUnit {
return MetricUnit{
"kilobyte",
}
}

func KibiByte() MetricUnit {
return MetricUnit{
"kibibyte",
}
}

func MegaByte() MetricUnit {
return MetricUnit{
"megabyte",
}
}

func MebiByte() MetricUnit {
return MetricUnit{
"mebibyte",
}
}

func GigaByte() MetricUnit {
return MetricUnit{
"gigabyte",
}
}

func GibiByte() MetricUnit {
return MetricUnit{
"gibibyte",
}
}

func TeraByte() MetricUnit {
return MetricUnit{
"terabyte",
}
}

func TebiByte() MetricUnit {
return MetricUnit{
"tebibyte",
}
}

func PetaByte() MetricUnit {
return MetricUnit{
"petabyte",
}
}

func PebiByte() MetricUnit {
return MetricUnit{
"pebibyte",
}
}

func ExaByte() MetricUnit {
return MetricUnit{
"exabyte",
}
}

func ExbiByte() MetricUnit {
return MetricUnit{
"exbibyte",
}
}

func Ratio() MetricUnit {
return MetricUnit{
"ratio",
}
}

func Percent() MetricUnit {
return MetricUnit{
"percent",
}
}

func CustomUnit(unit string) MetricUnit {
return MetricUnit{
unitRegex.ReplaceAllString(unit, ""),
}
}

type Metric interface {
GetType() string
GetTags() map[string]string
GetKey() string
GetUnit() string
GetTimestamp() int64
SerializeValue() string
SerializeTags() string
}

type abstractMetric struct {
key string
unit MetricUnit
tags map[string]string
// A unix timestamp (full seconds elapsed since 1970-01-01 00:00 UTC).
timestamp int64
}

func (am abstractMetric) GetTags() map[string]string {
return am.tags
}

func (am abstractMetric) GetKey() string {
return am.key
}

func (am abstractMetric) GetUnit() string {
return am.unit.toString()
}

func (am abstractMetric) GetTimestamp() int64 {
return am.timestamp
}

func (am abstractMetric) SerializeTags() string {
var sb strings.Builder

values := make([]string, 0, len(am.tags))
for k := range am.tags {
values = append(values, k)
}
sortSlice(values)

for _, key := range values {
val := sanitizeValue(am.tags[key])
key = sanitizeKey(key)
sb.WriteString(fmt.Sprintf("%s:%s,", key, val))
}
s := sb.String()
if len(s) > 0 {
s = s[:len(s)-1]
}
return s
}

// Counter Metric.
type CounterMetric struct {
value float64
abstractMetric
}

func (c *CounterMetric) Add(value float64) {
c.value += value
}

func (c CounterMetric) GetType() string {
return "c"
}

func (c CounterMetric) SerializeValue() string {
return fmt.Sprintf(":%v", c.value)
}

// timestamp: A unix timestamp (full seconds elapsed since 1970-01-01 00:00 UTC).
func NewCounterMetric(key string, unit MetricUnit, tags map[string]string, timestamp int64, value float64) CounterMetric {
am := abstractMetric{
key,
unit,
tags,
timestamp,
}

return CounterMetric{
value,
am,
}
}

// Distribution Metric.
type DistributionMetric struct {
values []float64
abstractMetric
}

func (d *DistributionMetric) Add(value float64) {
d.values = append(d.values, value)
}

func (d DistributionMetric) GetType() string {
return "d"
}

func (d DistributionMetric) SerializeValue() string {
var sb strings.Builder
for _, el := range d.values {
sb.WriteString(fmt.Sprintf(":%v", el))
}
return sb.String()
}

// timestamp: A unix timestamp (full seconds elapsed since 1970-01-01 00:00 UTC).
func NewDistributionMetric(key string, unit MetricUnit, tags map[string]string, timestamp int64, value float64) DistributionMetric {
am := abstractMetric{
key,
unit,
tags,
timestamp,
}

return DistributionMetric{
[]float64{value},
am,
}
}

// Gauge Metric.
type GaugeMetric struct {
last float64
min float64
max float64
sum float64
count float64
abstractMetric
}

func (g *GaugeMetric) Add(value float64) {
g.last = value
g.min = math.Min(g.min, value)
g.max = math.Max(g.max, value)
g.sum += value
g.count++
}

func (g GaugeMetric) GetType() string {
return "g"
}

func (g GaugeMetric) SerializeValue() string {
return fmt.Sprintf(":%v:%v:%v:%v:%v", g.last, g.min, g.max, g.sum, g.count)
}

// timestamp: A unix timestamp (full seconds elapsed since 1970-01-01 00:00 UTC).
func NewGaugeMetric(key string, unit MetricUnit, tags map[string]string, timestamp int64, value float64) GaugeMetric {
am := abstractMetric{
key,
unit,
tags,
timestamp,
}

return GaugeMetric{
value, // last
value, // min
value, // max
value, // sum
value, // count
am,
}
}

// Set Metric.
type SetMetric[T NumberOrString] struct {
values map[T]void
abstractMetric
}

func (s *SetMetric[T]) Add(value T) {
s.values[value] = member
}

func (s SetMetric[T]) GetType() string {
return "s"
}

func (s SetMetric[T]) SerializeValue() string {
_hash := func(s string) uint32 {
return crc32.ChecksumIEEE([]byte(s))
}

values := make([]T, 0, len(s.values))
for k := range s.values {
values = append(values, k)
}
sortSlice(values)

var sb strings.Builder
for _, el := range values {
switch any(el).(type) {
case int:
sb.WriteString(fmt.Sprintf(":%v", el))
case string:
s := fmt.Sprintf("%v", el)
sb.WriteString(fmt.Sprintf(":%d", _hash(s)))
}
}

return sb.String()
}

// timestamp: A unix timestamp (full seconds elapsed since 1970-01-01 00:00 UTC).
func NewSetMetric[T NumberOrString](key string, unit MetricUnit, tags map[string]string, timestamp int64, value T) SetMetric[T] {
am := abstractMetric{
key,
unit,
tags,
timestamp,
}

return SetMetric[T]{
map[T]void{
value: member,
},
am,
}
}

func sanitizeKey(s string) string {
return keyRegex.ReplaceAllString(s, "_")
}

func sanitizeValue(s string) string {
return valueRegex.ReplaceAllString(s, "")
}

type Ordered interface {
~int | ~int8 | ~int16 | ~int32 | ~int64 | ~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64 | ~uintptr | ~float32 | ~float64 | ~string
}

func sortSlice[T Ordered](s []T) {
sort.Slice(s, func(i, j int) bool {
return s[i] < s[j]
})
}
160 changes: 160 additions & 0 deletions metrics_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,160 @@
package sentry

import (
"testing"

"github.com/google/go-cmp/cmp"
)

func TestSanitizeKey(t *testing.T) {
tests := []struct {
name string
key string
want string
}{
{
name: "allowed characters",
key: "test.metric-1",
want: "test.metric-1",
},
{
name: "forbidden characters",
key: "@test.me^tri'@c-1{}[]",
want: "_test.me_tri_c-1_",
},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if diff := cmp.Diff(sanitizeKey(test.key), test.want); diff != "" {
t.Errorf("Context mismatch (-want +got):\n%s", diff)
}
})
}
}

func TestSanitizeValue(t *testing.T) {
tests := []struct {
name string
value string
want string
}{
{
name: "allowed characters",
value: "test.metric-1",
want: "test.metric-1",
},
{
name: "forbidden characters",
value: "@test.me^tri'+@c-1{}[]",
want: "@test.metri@c-1{}[]",
},
{
name: "allow empty character",
value: "@route /test",
want: "@route /test",
},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if diff := cmp.Diff(sanitizeValue(test.value), test.want); diff != "" {
t.Errorf("Context mismatch (-want +got):\n%s", diff)
}
})
}
}

func TestSerializeTags(t *testing.T) {
tests := []struct {
name string
metric abstractMetric
want string
}{
{
name: "normal tags",
metric: abstractMetric{
tags: map[string]string{"tag1": "val1", "tag2": "val2"},
},
want: "tag1:val1,tag2:val2",
},
{
name: "empty tags",
metric: abstractMetric{
tags: map[string]string{},
},
want: "",
},
{
name: "un-sanitized tags",
metric: abstractMetric{
tags: map[string]string{"@env": "pro+d", "vers^^ion": `\release@`},
},
want: "_env:prod,vers_ion:release@",
},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if diff := cmp.Diff(test.metric.SerializeTags(), test.want); diff != "" {
t.Errorf("Context mismatch (-want +got):\n%s", diff)
}
})
}
}

func TestSerializeValue(t *testing.T) {
tests := []struct {
name string
metric Metric
want string
}{
{
name: "distribution metric",
metric: DistributionMetric{
values: []float64{2.0, 4.0, 3.0, 6.0},
},
want: ":2:4:3:6",
},
{
name: "gauge metric",
metric: GaugeMetric{
last: 1.0,
min: 1.0,
max: 1.0,
sum: 1.0,
count: 1.0,
},
want: ":1:1:1:1:1",
},
{
name: "set metric with strings",
metric: SetMetric[string]{
values: map[string]void{"Hello": member, "World": member},
},
want: ":4157704578:4223024711",
},
{
name: "set metric with integers",
metric: SetMetric[int]{
values: map[int]void{1: member, 2: member},
},
want: ":1:2",
},
{
name: "counter metric",
metric: CounterMetric{
value: 2.0,
},
want: ":2",
},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if diff := cmp.Diff(test.metric.SerializeValue(), test.want); diff != "" {
t.Errorf("Context mismatch (-want +got):\n%s", diff)
}
})
}
}
68 changes: 68 additions & 0 deletions transport.go
Original file line number Diff line number Diff line change
@@ -94,6 +94,55 @@
return nil
}

func marshalMetrics(metrics []Metric) []byte {
var b bytes.Buffer
for i, metric := range metrics {
b.WriteString(metric.GetKey())
if unit := metric.GetUnit(); unit != "" {
b.WriteString(fmt.Sprintf("@%s", unit))
}
b.WriteString(fmt.Sprintf("%s|%s", metric.SerializeValue(), metric.GetType()))
if serializedTags := metric.SerializeTags(); serializedTags != "" {
b.WriteString(fmt.Sprintf("|#%s", serializedTags))
}
b.WriteString(fmt.Sprintf("|T%d", metric.GetTimestamp()))

if i < len(metrics)-1 {
b.WriteString("\n")
}
}
return b.Bytes()
}

func encodeMetric(enc *json.Encoder, b io.Writer, metrics []Metric) error {
body := marshalMetrics(metrics)
// Item header
err := enc.Encode(struct {
Type string `json:"type"`
Length int `json:"length"`
}{
Type: metricType,
Length: len(body),
})
if err != nil {
return err
}

// metric payload
if _, err = b.Write(body); err != nil {
return err
}

// "Envelopes should be terminated with a trailing newline."
//
// [1]: https://develop.sentry.dev/sdk/envelopes/#envelopes
if _, err := b.Write([]byte("\n")); err != nil {
return err
}

return err
}

func encodeAttachment(enc *json.Encoder, b io.Writer, attachment *Attachment) error {
// Attachment header
err := enc.Encode(struct {
@@ -175,8 +224,10 @@
return nil, err
}

if event.Type == transactionType || event.Type == checkInType {

Check failure on line 227 in transport.go

GitHub Actions / Lint

ifElseChain: rewrite if-else to switch statement (gocritic)
err = encodeEnvelopeItem(enc, event.Type, body)
} else if event.Type == metricType {
err = encodeMetric(enc, &b, event.Metrics)
} else {
err = encodeEnvelopeItem(enc, eventType, body)
}
@@ -478,6 +529,13 @@
Logger.Printf("There was an issue with sending an event: %v", err)
continue
}
if response.StatusCode >= 400 && response.StatusCode <= 599 {
b, err := io.ReadAll(response.Body)
if err != nil {
Logger.Printf("Error while reading response code: %v", err)
}
Logger.Printf("Sending %s failed with the following error: %s", eventType, string(b))
}
t.mu.Lock()
t.limits.Merge(ratelimit.FromResponse(response))
t.mu.Unlock()
@@ -583,8 +641,10 @@
}

var eventType string
if event.Type == transactionType {

Check failure on line 644 in transport.go

GitHub Actions / Lint

ifElseChain: rewrite if-else to switch statement (gocritic)
eventType = "transaction"
} else if event.Type == metricType {
eventType = metricType
} else {
eventType = fmt.Sprintf("%s event", event.Level)
}
@@ -601,6 +661,14 @@
Logger.Printf("There was an issue with sending an event: %v", err)
return
}
if response.StatusCode >= 400 && response.StatusCode <= 599 {
b, err := io.ReadAll(response.Body)
if err != nil {
Logger.Printf("Error while reading response code: %v", err)
}
Logger.Printf("Sending %s failed with the following error: %s", eventType, string(b))
}

t.mu.Lock()
t.limits.Merge(ratelimit.FromResponse(response))
t.mu.Unlock()
31 changes: 31 additions & 0 deletions transport_test.go
Original file line number Diff line number Diff line change
@@ -680,3 +680,34 @@ func testRateLimiting(t *testing.T, tr Transport) {
t.Errorf("got transactionEvent = %d, want %d", n, 1)
}
}

func TestEnvelopeFromMetricBody(t *testing.T) {
event := newTestEvent(metricType)
event.Metrics = append(event.Metrics,
NewCounterMetric("counter", Second(), map[string]string{"foo": "bar", "route": "GET /foo"}, 1597790835, 1.0),
NewDistributionMetric("distribution", Second(), map[string]string{"$foo$": "%bar%"}, 1597790835, 1.0),
NewGaugeMetric("gauge", Second(), map[string]string{"föö": "bär"}, 1597790835, 1.0),
NewSetMetric[int]("set", Second(), map[string]string{"%{key}": "$value$"}, 1597790835, 1),
NewCounterMetric("no_tags", Second(), nil, 1597790835, 1.0),
)
sentAt := time.Unix(0, 0).UTC()

body := getRequestBodyFromEvent(event)

b, err := envelopeFromBody(event, newTestDSN(t), sentAt, body)
if err != nil {
t.Fatal(err)
}
got := b.String()
want := `{"event_id":"b81c5be4d31e48959103a1f878a1efcb","sent_at":"1970-01-01T00:00:00Z","dsn":"http://public@example.com/sentry/1","sdk":{"name":"sentry.go","version":"0.0.1"}}
{"type":"statsd","length":218}
counter@second:1|c|#foo:bar,route:GET /foo|T1597790835
distribution@second:1|d|#_foo_:bar|T1597790835
gauge@second:1:1:1:1:1|g|#f_:br|T1597790835
set@second:1|s|#_key_:$value$|T1597790835
no_tags@second:1|c|T1597790835
`
if diff := cmp.Diff(want, got); diff != "" {
t.Errorf("Envelope mismatch (-want +got):\n%s", diff)
}
}

0 comments on commit 9a9f285

Please sign in to comment.