diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..a774cfb6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,21 @@ +FROM golang:1.20-alpine3.18 AS BUILD + +RUN apk add make build-base git + +WORKDIR /vegeta + +# cache dependencies +ADD go.mod /vegeta +ADD go.sum /vegeta +RUN go mod download + +ADD . /vegeta + +RUN make generate +RUN make vegeta + +FROM alpine:3.18.0 + +COPY --from=BUILD /vegeta/vegeta /bin/vegeta + +ENTRYPOINT ["vegeta"] diff --git a/README.md b/README.md index 2dbd1564..2cc0daa2 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ Usage: vegeta [global flags] [command flags] global flags: -cpus int - Number of CPUs to use (defaults to the number of CPUs you have) + Number of CPUs to use (default = number of cpus) -profile string Enable profiling of [cpu, heap] -version @@ -126,6 +126,8 @@ attack command: Attack name -output string Output file (default "stdout") + -prometheus-addr string + Prometheus exporter listen address [empty = disabled]. Example: 0.0.0.0:8880 -proxy-header value Proxy CONNECT header -rate value @@ -137,7 +139,7 @@ attack command: -root-certs value TLS root certificate files (comma separated list) -session-tickets - Enable TLS session resumption support using session tickets (default false) + Enable TLS session resumption using session tickets -targets string Targets file (default "stdin") -timeout duration @@ -176,6 +178,7 @@ examples: vegeta report -type=json results.bin > metrics.json cat results.bin | vegeta plot > plot.html cat results.bin | vegeta report -type="hist[0,100ms,200ms,300ms]" + ``` #### `-cpus` @@ -783,6 +786,8 @@ It'll read and sort them by timestamp before generating reports. vegeta report *.bin ``` +Another way to gather results in distributed tests is to use the built-in Prometheus Exporter and configure a Prometheus Server to get test results from all Vegeta instances. See `attack` option "prometheus-addr" for more details and a complete example in the section "Prometheus Support". + ## Usage: Real-time Analysis If you are a happy user of iTerm, you can integrate vegeta with [jplot](https://github.com/rs/jplot) using [jaggr](https://github.com/rs/jaggr) to plot a vegeta report in real-time in the comfort of your terminal: @@ -802,7 +807,7 @@ echo 'GET http://localhost:8080' | \ ![](https://i.imgur.com/ttBDsQS.gif) -## Usage (Library) +## Usage: Library The library versioning follows [SemVer v2.0.0](https://semver.org/spec/v2.0.0.html). Since [lib/v9.0.0](https://github.com/tsenart/vegeta/tree/lib/v9.0.0), the library and cli @@ -859,6 +864,67 @@ $ ulimit -u # processes / threads Just pass a new number as the argument to change it. +## Prometheus Support + +Vegeta has a built-in Prometheus Exporter that may be enabled during attacks so that you can point any Prometheus instance to Vegeta instances and get some metrics about http requests performance and about the Vegeta process itself. + +To enable the Prometheus Exporter on the command line, use the "prometheus-addr" flag. + +A Prometheus HTTP endpoint will be available only during the lifespan of an attack and will be closed right after the attack is finished. + +The following metrics are exposed: + +* `request_bytes_in` - bytes count received from targeted servers by "url", "method" and "status" +* `request_bytes_out` - bytes count sent to targeted server by "url", "method" and "status" +* `request_seconds` - histogram with request latency and counters by "url", "method" and "status" + + + +Check file [lib/prom/grafana.json](lib/prom/grafana.json) with the source of this sample dashboard in Grafana. + +### Samples + +If you want to query P90 quantiles, for example, use "histogram_quantile(0.90, sum(rate(request_seconds_bucket[1m])) by (le, status))" + +### Prometheus Exporter example + +* Create a docker-compose.yml + +``` +version: '3.5' +services: + vegeta: + image: tsenart/vegeta + ports: + - 8880:8880 + command: sh -c 'echo "GET https://www.yahoo.com" | vegeta attack -duration=30s -rate=5 -prometheus-addr=0.0.0.0:8880' + + prometheus: + image: flaviostutz/prometheus:2.19.2.0 + ports: + - 9090:9090 + environment: + - SCRAPE_INTERVAL=10s + - SCRAPE_TIMEOUT=10s + - STATIC_SCRAPE_TARGETS=vegeta@vegeta:8880 +``` + +* Run `docker-compose up -d` + +* Run `curl localhost:8880` to see plain Prometheus Exporter endpoint contents + +* Open Prometheus server instance with your browser at http://localhost:9090 + +* Go to "Graph" and execute query `rate(request_seconds_sum[1m])` and then select the "Graph" tab to see a graph with latency over time + +#### More resources + +* See https://prometheus.io/docs/prometheus/latest/querying/basics/ for query details + +* Use Grafana for creating stateful dashboards. Get a sample dashboard for Vegeta [here](grafana.json) + +* For more elaborated scenarios, see https://github.com/flaviostutz/promster so that you can automatically register new Vegeta Prometheus Exporter instances to Prometheus in elastic scenarios. + ## License See [LICENSE](LICENSE). diff --git a/attack.go b/attack.go index bad0896e..8362e03c 100644 --- a/attack.go +++ b/attack.go @@ -15,8 +15,10 @@ import ( "syscall" "time" + "github.com/prometheus/client_golang/prometheus" "github.com/tsenart/vegeta/v12/internal/resolver" vegeta "github.com/tsenart/vegeta/v12/lib" + prom "github.com/tsenart/vegeta/v12/lib/prom" ) func attackCmd() command { @@ -27,6 +29,7 @@ func attackCmd() command { laddr: localAddr{&vegeta.DefaultLocalAddr}, rate: vegeta.Rate{Freq: 50, Per: time.Second}, maxBody: vegeta.DefaultMaxBody, + promAddr: "0.0.0.0:8880", } fs.StringVar(&opts.name, "name", "", "Attack name") fs.StringVar(&opts.targetsf, "targets", "stdin", "Targets file") @@ -56,6 +59,7 @@ func attackCmd() command { fs.Var(&opts.laddr, "laddr", "Local IP address") fs.BoolVar(&opts.keepalive, "keepalive", true, "Use persistent connections") fs.StringVar(&opts.unixSocket, "unix-socket", "", "Connect over a unix socket. This overrides the host address in target URLs") + fs.StringVar(&opts.promAddr, "prometheus-addr", "", "Prometheus exporter listen address [empty = disabled]. Example: 0.0.0.0:8880") fs.Var(&dnsTTLFlag{&opts.dnsTTL}, "dns-ttl", "Cache DNS lookups for the given duration [-1 = disabled, 0 = forever]") fs.BoolVar(&opts.sessionTickets, "session-tickets", false, "Enable TLS session resumption using session tickets") systemSpecificFlags(fs, opts) @@ -101,6 +105,7 @@ type attackOpts struct { keepalive bool resolvers csl unixSocket string + promAddr string dnsTTL time.Duration sessionTickets bool } @@ -178,6 +183,22 @@ func attack(opts *attackOpts) (err error) { return err } + // Start Prometheus Metrics and Server + var pm *prom.Metrics + if opts.promAddr != "" { + r := prometheus.NewRegistry() + pm, err = prom.NewMetrics(r) + if err != nil { + return err + } + srv, err := prom.StartPromServer(opts.promAddr, r) + if err != nil { + return err + } + + defer srv.Close() + } + atk := vegeta.NewAttacker( vegeta.Redirects(opts.redirects), vegeta.Timeout(opts.timeout), @@ -203,7 +224,7 @@ func attack(opts *attackOpts) (err error) { sig := make(chan os.Signal, 1) signal.Notify(sig, os.Interrupt, syscall.SIGTERM) - return processAttack(atk, res, enc, sig) + return processAttack(atk, res, enc, sig, pm) } func processAttack( @@ -211,6 +232,7 @@ func processAttack( res <-chan *vegeta.Result, enc vegeta.Encoder, sig <-chan os.Signal, + pm *prom.Metrics, ) error { for { select { @@ -223,6 +245,9 @@ func processAttack( if !ok { return nil } + if pm != nil { + pm.Observe(r) + } if err := enc.Encode(r); err != nil { return err } diff --git a/attack_test.go b/attack_test.go index 967044c2..23cff272 100644 --- a/attack_test.go +++ b/attack_test.go @@ -86,7 +86,7 @@ func TestAttackSignalOnce(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - processAttack(atk, res, enc, sig) + processAttack(atk, res, enc, sig, nil) }() // Allow more than one request to have started before stopping. @@ -139,7 +139,7 @@ func TestAttackSignalTwice(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - processAttack(atk, res, enc, sig) + processAttack(atk, res, enc, sig, nil) }() // Exit as soon as possible. diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..5ac11cdc --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,27 @@ +version: '3.5' + +services: + + vegeta: + build: . + image: tsenart/vegeta + ports: + - 8880:8880 + command: sh -c 'echo "GET https://www.yahoo.com" | vegeta attack -duration=300s -rate=5 -prometheus-addr=0.0.0.0:8880 | vegeta report --type=text' + + prometheus: + image: flaviostutz/prometheus + ports: + - 9090:9090 + environment: + - SCRAPE_INTERVAL=10s + - SCRAPE_TIMEOUT=10s + - STATIC_SCRAPE_TARGETS=vegeta@vegeta:8880 + + grafana: + image: flaviostutz/grafana:5.2.4 + ports: + - 3000:3000 + environment: + - GF_SECURITY_ADMIN_PASSWORD=mypass + diff --git a/go.mod b/go.mod index 2d613eeb..f983596c 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/influxdata/tdigest v0.0.1 github.com/mailru/easyjson v0.7.7 github.com/miekg/dns v1.1.55 + github.com/prometheus/client_golang v1.16.0 github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 github.com/streadway/quantile v0.0.0-20220407130108-4246515d968d github.com/tsenart/go-tsz v0.0.0-20180814235614-0bd30b3df1c3 @@ -20,12 +21,20 @@ require ( ) require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/iancoleman/orderedmap v0.3.0 // indirect github.com/josharian/intern v1.0.0 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/prometheus/client_model v0.3.0 // indirect + github.com/prometheus/common v0.42.0 // indirect + github.com/prometheus/procfs v0.10.1 // indirect golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 // indirect golang.org/x/mod v0.12.0 // indirect golang.org/x/sync v0.3.0 // indirect golang.org/x/sys v0.10.0 // indirect golang.org/x/text v0.11.0 // indirect golang.org/x/tools v0.11.0 // indirect + google.golang.org/protobuf v1.30.0 // indirect ) diff --git a/go.sum b/go.sum index 07e38c4d..720e83b5 100644 --- a/go.sum +++ b/go.sum @@ -1,16 +1,28 @@ github.com/alecthomas/jsonschema v0.0.0-20220216202328-9eeeec9d044b h1:doCpXjVwui6HUN+xgNsNS3SZ0/jUZ68Eb+mJRNOZfog= github.com/alecthomas/jsonschema v0.0.0-20220216202328-9eeeec9d044b/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bmizerany/perks v0.0.0-20230307044200-03f9df79da1e h1:mWOqoK5jV13ChKf/aF3plwQ96laasTJgZi4f1aSOu+M= github.com/bmizerany/perks v0.0.0-20230307044200-03f9df79da1e/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q= github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b h1:6+ZFm0flnudZzdSE0JxlhR2hKnGPcNB35BjQf4RYQDY= github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-gk v0.0.0-20200319235926-a69029f61654 h1:XOPLOMn/zT4jIgxfxSsoXPxkrzz0FaCHwp33x5POJ+Q= github.com/dgryski/go-gk v0.0.0-20200319235926-a69029f61654/go.mod h1:qm+vckxRlDt0aOla0RYJJVeqHZlWfOm2UIxHaqPB46E= github.com/dgryski/go-lttb v0.0.0-20230207170358-f8fc36cdbff1 h1:dxwR3CStJdJamsIoMPCmxuIfBAPTgmzvFax+MvFav3M= github.com/dgryski/go-lttb v0.0.0-20230207170358-f8fc36cdbff1/go.mod h1:UwftcHUI/qTYvLAxrWmANuRckf8+08O3C3hwStvkhDU= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA= @@ -20,18 +32,33 @@ github.com/influxdata/tdigest v0.0.1 h1:XpFptwYmnEKUqmkcDjrzffswZ3nvNeevbUSLPP/Z github.com/influxdata/tdigest v0.0.1/go.mod h1:Z0kXnxzbTC2qrx4NaIzYkE1k66+6oEDQTvL95hQFh5Y= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/miekg/dns v1.1.55 h1:GoQ4hpsj0nFLYe+bWiCToyrBEJXkQfOOIvFGFy0lEgo= github.com/miekg/dns v1.1.55/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= +github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= +github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= +github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 h1:Lt9DzQALzHoDwMBGJ6v8ObDPR0dzr2a6sXTB1Fq7IHs= github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417/go.mod h1:qe5TWALJ8/a1Lqznoc5BDHpYX/8HU60Hm2AwRmqzxqA= github.com/streadway/quantile v0.0.0-20220407130108-4246515d968d h1:X4+kt6zM/OVO6gbJdAfJR60MGPsqCzbtXNnjoGqdfAs= github.com/streadway/quantile v0.0.0-20220407130108-4246515d968d/go.mod h1:lbP8tGiBjZ5YWIc2fzuRpTaz0b/53vT6PEs3QuAWzuU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709 h1:Ko2LQMrRU+Oy/+EDBwX7eZ2jp3C47eDBB8EIhKTun+I= github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/tsenart/go-tsz v0.0.0-20180814235614-0bd30b3df1c3 h1:pcQGQzTwCg//7FgVywqge1sW9Yf8VMsMdG58MI5kd8s= github.com/tsenart/go-tsz v0.0.0-20180814235614-0bd30b3df1c3/go.mod h1:SWZznP1z5Ki7hDT2ioqiFKEse8K9tU2OUvaRI0NeGQo= @@ -42,6 +69,7 @@ golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= @@ -52,8 +80,14 @@ golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.11.0 h1:EMCa6U9S2LtZXLAMoWiR/R8dAQFRqbAitmbJ2UKhoi8= golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca h1:PupagGYwj8+I4ubCxcmcBRk3VlUWtTg5huQpZR9flmE= gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= pgregory.net/rapid v1.0.0 h1:iQaM2w5PZ6xvt6x7hbd7tiDS+nk7YPp5uCaEba+T/F4= pgregory.net/rapid v1.0.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= diff --git a/lib/prom/grafana.json b/lib/prom/grafana.json new file mode 100644 index 00000000..331edc5a --- /dev/null +++ b/lib/prom/grafana.json @@ -0,0 +1,645 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "5.2.4" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "5.0.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "5.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(request_seconds_count{status=~\"2..\"}[1m])/rate(request_seconds_count[1m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "status {{status}} {{url}}", + "refId": "A" + }, + { + "expr": "rate(request_seconds_count{status=~\"3..\"}[1m])/rate(request_seconds_count[1m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "status {{status}} {{url}}", + "refId": "B" + }, + { + "expr": "rate(request_seconds_count{status=~\"4..\"}[1m])/rate(request_seconds_count[1m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "status {{status}} {{url}}", + "refId": "C" + }, + { + "expr": "rate(request_seconds_count{status=~\"5..\"}[1m])/rate(request_seconds_count[1m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "status {{status}} {{url}}", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Request Status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(rate(request_seconds_bucket[1m])) by (le, status))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "P50 {{status}}", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.90, sum(rate(request_seconds_bucket[1m])) by (le, status))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "P90 {{status}}", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(request_seconds_bucket[1m])) by (le, status))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "P99 {{status}}", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Average Request Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 4 + }, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(request_seconds_count[1m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "throughtput {{url}} {{status}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Requests per second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 4 + }, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_threads{job=\"vegeta\"} ", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Vegeta Go threads", + "refId": "A" + }, + { + "expr": "go_goroutines{job=\"vegeta\"} ", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Vegeta Go routines", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Vegeta threads", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(request_bytes_in[1m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "bytes_out {{url}} {{status}}", + "refId": "A" + }, + { + "expr": "rate(request_bytes_out[1m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "bytes_out {{url}} {{status}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Data throughput", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_memstats_alloc_bytes{job=\"vegeta\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Vegeta Heap Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 16, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "2023-07-23T13:28:35.330Z", + "to": "2023-07-23T13:33:18.000Z" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Vegeta", + "uid": "6GNY5DGGk", + "version": 3 +} \ No newline at end of file diff --git a/lib/prom/prom.go b/lib/prom/prom.go new file mode 100644 index 00000000..2872eb36 --- /dev/null +++ b/lib/prom/prom.go @@ -0,0 +1,136 @@ +package prom + +import ( + "errors" + "net/http" + "strconv" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + + vegeta "github.com/tsenart/vegeta/v12/lib" +) + +// Metrics encapsulates Prometheus metrics of an attack. +type Metrics struct { + RequestSecondsHistogram *prometheus.HistogramVec + RequestBytesInCounter *prometheus.CounterVec + RequestBytesOutCounter *prometheus.CounterVec + RequestFailCounter *prometheus.CounterVec + Registry prometheus.Registerer +} + +// NewMetrics returns a new Metrics instance and registers all of them in the given Registry. +func NewMetrics(registry prometheus.Registerer) (*Metrics, error) { + if registry == nil { + registry = prometheus.DefaultRegisterer + } + + pm := &Metrics{Registry: registry} + + pm.RequestSecondsHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "request_seconds", + Help: "Request latency", + Buckets: prometheus.DefBuckets, + }, []string{ + "method", + "url", + "status", + }) + err := pm.Registry.Register(pm.RequestSecondsHistogram) + if err != nil { + return nil, err + } + + pm.RequestBytesInCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "request_bytes_in", + Help: "Bytes received from servers as response to requests", + }, []string{ + "method", + "url", + "status", + }) + err = pm.Registry.Register(pm.RequestBytesInCounter) + if err != nil { + return nil, err + } + + pm.RequestBytesOutCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "request_bytes_out", + Help: "Bytes sent to servers during requests", + }, []string{ + "method", + "url", + "status", + }) + err = pm.Registry.Register(pm.RequestBytesOutCounter) + if err != nil { + return nil, err + } + + pm.RequestFailCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "request_fail_count", + Help: "Internal failures that prevented a hit to the target server", + }, []string{ + "method", + "url", + "code", + "message", + }) + err = pm.Registry.Register(pm.RequestFailCounter) + if err != nil { + return nil, err + } + + return pm, nil +} + +// Unregister all prometheus collectors +func (pm *Metrics) Unregister() error { + exists := pm.Registry.Unregister(pm.RequestSecondsHistogram) + if !exists { + return errors.New("'RequestSecondsHistogram' cannot be unregistered because it was not found") + } + + exists = pm.Registry.Unregister(pm.RequestBytesInCounter) + if !exists { + return errors.New("'RequestBytesInCounter' cannot be unregistered because it was not found") + } + + exists = pm.Registry.Unregister(pm.RequestBytesOutCounter) + if !exists { + return errors.New("'RequestBytesOutCounter' cannot be unregistered because it was not found") + } + + exists = pm.Registry.Unregister(pm.RequestFailCounter) + if !exists { + return errors.New("'RequestFailCounter' cannot be unregistered because it was not found") + } + + return nil +} + +// Observe metrics with hit results +func (pm *Metrics) Observe(res *vegeta.Result) { + code := strconv.FormatUint(uint64(res.Code), 10) + pm.RequestBytesInCounter.WithLabelValues(res.Method, res.URL, code).Add(float64(res.BytesIn)) + pm.RequestBytesOutCounter.WithLabelValues(res.Method, res.URL, code).Add(float64(res.BytesOut)) + pm.RequestSecondsHistogram.WithLabelValues(res.Method, res.URL, code).Observe(float64(res.Latency) / float64(time.Second)) + if res.Error != "" { + pm.RequestFailCounter.WithLabelValues(res.Method, res.URL, code, res.Error) + } +} + +// StartPromServer starts a new Prometheus server with metrics present in promRegistry +// launches a http server in a new goroutine and returns the http.Server instance +func StartPromServer(bindAddr string, promRegistry *prometheus.Registry) (*http.Server, error) { + srv := http.Server{ + Addr: bindAddr, + Handler: promhttp.HandlerFor(promRegistry, promhttp.HandlerOpts{}), + } + + go srv.ListenAndServe() + + return &srv, nil +} diff --git a/lib/prom/prom_test.go b/lib/prom/prom_test.go new file mode 100644 index 00000000..acb4332b --- /dev/null +++ b/lib/prom/prom_test.go @@ -0,0 +1,215 @@ +package prom + +import ( + "context" + "io" + "net/http" + "strings" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus" + vegeta "github.com/tsenart/vegeta/v12/lib" +) + +func TestPromMetrics1(t *testing.T) { + pm, err := NewMetrics(nil) + if err != nil { + t.Errorf("Error creating metrics. err=%s", err) + } + + err = pm.Unregister() + if err != nil { + t.Errorf("Cannot unregister metrics. err=%s", err) + } +} + +func TestPromMetrics2(t *testing.T) { + reg := prometheus.NewRegistry() + + pm, err := NewMetrics(reg) + if err != nil { + t.Errorf("Error creating metrics. err=%s", err) + } + + err = pm.Unregister() + if err != nil { + t.Errorf("Cannot unregister metrics. err=%s", err) + } + + // register again to check if registry was cleared correctly + pm, err = NewMetrics(reg) + if err != nil { + t.Errorf("Error creating metrics. err=%s", err) + } + + err = pm.Unregister() + if err != nil { + t.Errorf("Cannot unregister metrics. err=%s", err) + } + + // register again to check if registry was cleared correctly + pm, err = NewMetrics(reg) + if err != nil { + t.Errorf("Error creating metrics. err=%s", err) + } + + err = pm.Unregister() + if err != nil { + t.Errorf("Cannot unregister metrics. err=%s", err) + } + +} + +func TestPromServerBasic1(t *testing.T) { + r := prometheus.NewRegistry() + pm, err := NewMetrics(r) + if err != nil { + t.Errorf("Error creating metrics. err=%s", err) + } + + srv, err := StartPromServer("0.0.0.0:8880", r) + if err != nil { + t.Errorf("Error starting server. err=%s", err) + } + + err = srv.Shutdown(context.Background()) + if err != nil { + t.Errorf("Error shutting down server. err=%s", err) + } + pm.Unregister() +} + +func TestPromServerBasic2(t *testing.T) { + reg := prometheus.NewRegistry() + + pm, err := NewMetrics(reg) + if err != nil { + t.Errorf("Error creating metrics. err=%s", err) + } + + // start/stop 1 + srv, err := StartPromServer("0.0.0.0:8880", reg) + if err != nil { + t.Errorf("Error starting server. err=%s", err) + } + err = srv.Shutdown(context.Background()) + if err != nil { + t.Errorf("Error shutting down server. err=%s", err) + } + + // start/stop 2 + srv, err = StartPromServer("0.0.0.0:8880", reg) + if err != nil { + t.Errorf("Error starting server. err=%s", err) + } + err = srv.Shutdown(context.Background()) + if err != nil { + t.Errorf("Error shutting down server. err=%s", err) + } + + pm.Unregister() + + // start server again after reusing the same registry (sanity check) + _, err = NewMetrics(reg) + if err != nil { + t.Errorf("Error creating metrics. err=%s", err) + } + // start/stop 1 + srv, err = StartPromServer("0.0.0.0:8880", reg) + if err != nil { + t.Errorf("Error starting server. err=%s", err) + } + err = srv.Shutdown(context.Background()) + if err != nil { + t.Errorf("Error shutting down server. err=%s", err) + } + +} + +func TestPromServerObserve(t *testing.T) { + reg := prometheus.NewRegistry() + pm, err := NewMetrics(reg) + if err != nil { + if err != nil { + t.Errorf("Error launching Prometheus http server. err=%s", err) + } + } + + srv, err := StartPromServer("0.0.0.0:8880", reg) + if err != nil { + t.Errorf("Error starting server. err=%s", err) + } + + r := &vegeta.Result{ + URL: "http://test.com/test1", + Method: "GET", + Code: 200, + Error: "", + Latency: 100 * time.Millisecond, + BytesIn: 1000, + BytesOut: 50, + } + pm.Observe(r) + pm.Observe(r) + pm.Observe(r) + pm.Observe(r) + + time.Sleep(3 * time.Second) + resp, err := http.Get("http://localhost:8880") + if err != nil { + t.Errorf("Error calling prometheus metrics. err=%s", err) + } + if resp.StatusCode != 200 { + t.Errorf("Status code should be 200") + } + + data, err := io.ReadAll(resp.Body) + if err != nil { + t.Errorf("Error calling prometheus metrics. err=%s", err) + } + str := string(data) + if len(str) == 0 { + t.Errorf("Body not empty. body=%s", str) + } + if !strings.Contains(str, "request_seconds") { + t.Error("Metrics should contain request_seconds") + } + if !strings.Contains(str, "request_bytes_in") { + t.Error("Metrics should contain request_bytes_in") + } + if !strings.Contains(str, "request_bytes_out") { + t.Error("Metrics should contain request_bytes_out") + } + if strings.Contains(str, "request_fail_count") { + t.Error("Metrics should contain request_fail_count") + } + + r.Code = 500 + r.Error = "REQUEST FAILED" + pm.Observe(r) + + resp, err = http.Get("http://localhost:8880") + if err != nil { + t.Errorf("Error calling prometheus metrics. err=%s", err) + } + if resp.StatusCode != 200 { + t.Errorf("Status code should be 200") + } + + data, err = io.ReadAll(resp.Body) + if err != nil { + t.Errorf("Error calling prometheus metrics. err=%s", err) + } + str = string(data) + + if !strings.Contains(str, "request_fail_count") { + t.Error("Metrics should contain request_fail_count") + } + + err = srv.Shutdown(context.Background()) + if err != nil { + t.Errorf("Error shutting down server. err=%s", err) + } + pm.Unregister() +} diff --git a/prometheus-sample.png b/prometheus-sample.png new file mode 100644 index 00000000..d4e834a0 Binary files /dev/null and b/prometheus-sample.png differ