diff --git a/.gitignore b/.gitignore index c78dc60..b8e501f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ crabby config-*.yaml binaries/ + +# Ignore IDE files +.idea \ No newline at end of file diff --git a/api.go b/api.go new file mode 100644 index 0000000..c24cad3 --- /dev/null +++ b/api.go @@ -0,0 +1,213 @@ +package main + +/* + Portions of this file were derived from Dave Cheney's httpstat: + https://github.com/davecheney/httpstat + + His code is licensed as follows: + + MIT License + + Copyright (c) 2016 Dave Cheney + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + +*/ + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "log" + "net/http" + "net/http/httptrace" + "net/url" + "regexp" + "strings" + "time" +) + +var placeholderRegex = regexp.MustCompile(`{{ *[^}}]* *}}`) + +// RunSimpleTest starts an HTTP/HTTPS API test of a site within crabby. It uses Go's built-in net/http client. +func RunApiTest(ctx context.Context, j Job, storage *Storage, client *http.Client) { + responses := map[string]json.RawMessage{} + for _, s := range j.Steps { + runApiTestStep(ctx, s, storage, client, responses) + } +} + +func runApiTestStep(ctx context.Context, j JobStep, storage *Storage, client *http.Client, responses map[string]json.RawMessage) { + var method = strings.ToUpper(j.Method) + if method == "" { + method = http.MethodGet + } + + body, err := replacePlaceholders(j.Body, responses) + if err != nil { + log.Printf("unable to substitute body variables in body: %v", err) + return + } + + req, err := http.NewRequest(method, j.URL, strings.NewReader(body)) + if err != nil { + log.Printf("unable to create request: %v", err) + return + } + + if err := addHeaders(req, j, responses); err != nil { + log.Printf("unable to process headers: %v", err) + return + } + + var t0, t1, t2, t3, t4 time.Time + + trace := &httptrace.ClientTrace{ + DNSStart: func(_ httptrace.DNSStartInfo) { t0 = time.Now() }, + DNSDone: func(_ httptrace.DNSDoneInfo) { t1 = time.Now() }, + ConnectStart: func(_, _ string) { + if t1.IsZero() { + // connecting to IP + t1 = time.Now() + } + }, + ConnectDone: func(net, addr string, err error) { + if err != nil { + log.Printf("unable to connect to host %v: %v", addr, err) + } + t2 = time.Now() + }, + GotConn: func(_ httptrace.GotConnInfo) { t3 = time.Now() }, + GotFirstResponseByte: func() { t4 = time.Now() }, + } + + // We'll use our Context in this request in case we have to shut down midstream + req = req.WithContext(httptrace.WithClientTrace(ctx, trace)) + + resp, err := client.Do(req) + if err != nil { + log.Println("Failed to read response:", err) + return + } + + // Send our server response code as an event + storage.EventDistributor <- j.makeEvent(resp.StatusCode) + + // unmarshal response and save into map + if responses[j.Name], err = ioutil.ReadAll(resp.Body); err != nil { + log.Println("WARNING!", err) + } else { + out, _ := replacePlaceholders(fmt.Sprintf("%s response:\n{{ %s }}\n", j.Name, j.Name), responses) + log.Println(out) + } + + // Even though we never read the response body, if we don't close it, + // the http.Transport goroutines will terminate and the app will eventually + // crash due to OOM + resp.Body.Close() + + t5 := time.Now() // after read body + if t0.IsZero() { + // we skipped DNS + t0 = t1 + } + + url, err := url.Parse(j.URL) + if err != nil { + log.Println("Failed to parse URL:", err) + return + } + + switch url.Scheme { + case "https": + storage.MetricDistributor <- j.makeMetric("dns_duration_milliseconds", t1.Sub(t0).Seconds()*1000) + storage.MetricDistributor <- j.makeMetric("server_connection_duration_milliseconds", t2.Sub(t1).Seconds()*1000) + storage.MetricDistributor <- j.makeMetric("tls_handshake_duration_milliseconds", t3.Sub(t2).Seconds()*1000) + storage.MetricDistributor <- j.makeMetric("server_processing_duration_milliseconds", t4.Sub(t3).Seconds()*1000) + storage.MetricDistributor <- j.makeMetric("server_response_duration_milliseconds", t5.Sub(t4).Seconds()*1000) + storage.MetricDistributor <- j.makeMetric("time_to_first_byte_milliseconds", t4.Sub(t0).Seconds()*1000) + + case "http": + storage.MetricDistributor <- j.makeMetric("dns_duration_milliseconds", t1.Sub(t0).Seconds()*1000) + storage.MetricDistributor <- j.makeMetric("server_connection_duration_milliseconds", t3.Sub(t1).Seconds()*1000) + storage.MetricDistributor <- j.makeMetric("server_processing_duration_milliseconds", t4.Sub(t3).Seconds()*1000) + storage.MetricDistributor <- j.makeMetric("server_response_duration_milliseconds", t5.Sub(t4).Seconds()*1000) + storage.MetricDistributor <- j.makeMetric("time_to_first_byte_milliseconds", t4.Sub(t0).Seconds()*1000) + } +} + + +func addHeaders(req *http.Request, j JobStep, responses map[string]json.RawMessage) error { + req.Header = http.Header{} + + for key, value := range j.Header { + req.Header.Add(key, value) + } + + if j.ContentType != "" { + req.Header["Content-Type"] = []string{j.ContentType} + } + + // replace placeholders in header values + for key := range req.Header { + for i := range req.Header[key] { + var err error + if req.Header[key][i], err = replacePlaceholders(req.Header[key][i], responses); err != nil { + return err + } + } + } + return nil +} + +// getResponseValue looks at the responses of previous steps for a response value. +// s should be .objectKey1.objectkey2... +// e.g. step1 returns a json { "key": { "subkey": "value" } }. +// step2 can access this by putting {{ step1.key.subkey }} to obtain "value" +// Note: this function will fail if the key contains "." e.g. { "bad.key": value } +func getResponseValue(s string, m map[string]json.RawMessage) (string, error) { + split := strings.SplitN(s, ".",2) + value := m[split[0]] + if len(split) == 1 { + return string(value), nil + } + var submap map[string]json.RawMessage + if err := json.Unmarshal(value, &submap); err != nil { + return "", err + } + return getResponseValue(split[1], submap) +} + +func replacePlaceholders(s string, m map[string]json.RawMessage) (string, error) { + vars := placeholderRegex.FindAll([]byte(s), -1) + varvals := make([]interface{}, len(vars)) + for i, v := range vars { + key := string(v) + key = strings.TrimPrefix(key, "{{") + key = strings.TrimSuffix(key, "}}") + key = strings.TrimSpace(key) + var err error + if varvals[i], err = getResponseValue(key, m); err != nil { + return "", err + } + } + format := placeholderRegex.ReplaceAll([]byte(s), []byte("%s")) + return fmt.Sprintf(string(format), varvals...), nil +} diff --git a/example/config.yaml b/example/config.yaml index b9cf71e..9d1b930 100644 --- a/example/config.yaml +++ b/example/config.yaml @@ -31,6 +31,16 @@ jobs: path: / value: abDijfeiF3290FijEIO30NC9jkqQER secure: false + - name: my_api_endpoint + type: api + url: https://my-api-here/some/resource + method: post + body: >- + { "hello": "world" } + interval: 10 + tags: + region: us-central1 + isp: google # # Selenium Configuration # diff --git a/jobs.go b/jobs.go index 98e78dc..61915c6 100644 --- a/jobs.go +++ b/jobs.go @@ -19,11 +19,11 @@ type Job struct { } type JobStep struct { - Name string `yaml:"name"` - URL string `yaml:"url"` - Method string `yaml:"method"` - Cookies []Cookie `yaml:"cookies,omitempty"` - Header map[string]string `yaml:"header,omitempty"` + Name string `yaml:"name"` + URL string `yaml:"url"` + Method string `yaml:"method"` + Cookies []Cookie `yaml:"cookies,omitempty"` + Header map[string]string `yaml:"header,omitempty"` // if header contains a different content type this overwrites it. ContentType string `yaml:"content-type,omitempty"` Body string `yaml:"body,omitempty"` @@ -65,6 +65,9 @@ func (jr *JobRunner) runJob(wg *sync.WaitGroup, j Job, seleniumServer string, st RunSeleniumTest(j, seleniumServer, storage) case "simple": go RunSimpleTest(jr.ctx, j, storage, client) + case "api": + go RunApiTest(jr.ctx, j, storage, client) + default: // We run Selenium tests by default RunSeleniumTest(j, seleniumServer, storage) @@ -144,8 +147,7 @@ func StartJobs(ctx context.Context, wg *sync.WaitGroup, c *Config, storage *Stor time.Sleep(sleepDur) } - // todo: how to handle this for sequential? - log.Println("Launching job ->", j.Step.Name) + log.Println("Launching job -> ", j.Step.Name, j.Step.URL) go jr.runJob(wg, j, seleniumServer, storage, client) }