Skip to content

Commit

Permalink
Merge pull request #33 from cirnum/feat/timeout-and-status-code-check…
Browse files Browse the repository at this point in the history 
…-added

request timeout added in settings and status code to include in succe…
  • Loading branch information
@manojown
manojown authored Aug 12, 2023
2 parents 862d2e9 + ecb56e9 commit 39074dc
Show file tree
Hide file tree
Showing 19 changed files with 307 additions and 98 deletions.
4 changes: 2 additions & 2 deletions server/app/utils/requestHelper.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ package utils
import (
"context"
"encoding/json"
"io/ioutil"
"io"
"net/http"
"time"

Expand Down Expand Up @@ -95,7 +95,7 @@ func TestRequest(request *models.Request) (helperModels.RequestResponse, error)

defer response.Body.Close()

buf, err := ioutil.ReadAll(response.Body)
buf, err := io.ReadAll(response.Body)

requestResponse.Body = string(buf)

Expand Down
40 changes: 21 additions & 19 deletions server/db/models/request.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,25 +19,27 @@ import "gorm.io/datatypes"
// CreatedAt int64 `json:"created_at" bson:"created_at"`
// }
type Request struct {
ID string `gorm:"primaryKey;type:char(36)" json:"id,omitempty"`
UserID string `json:"userID,omitempty"`
URL string `json:"url"`
Requests int64 `json:"requests"`
Time int `json:"time"`
Clients int `json:"clients"`
Headers datatypes.JSON `json:"headers"`
Cookies datatypes.JSON `json:"cookies"`
QPS int64 `json:"qps"`
Params datatypes.JSON `json:"params"`
KeepAlive bool `json:"keepAlive"`
Method string `json:"method"`
ServerId string `json:"serverId"`
WorkerId string `json:"workerId"`
Ips string `json:"ips" bson:"ips"`
PostData datatypes.JSON `json:"postData,omitempty"`
Created int64 `json:"created" bson:"created"`
UpdatedAt int64 `json:"updated_at" bson:"updated_at"`
CreatedAt int64 `json:"created_at" bson:"created_at"`
ID string `gorm:"primaryKey;type:char(36)" json:"id,omitempty"`
UserID string `json:"userID,omitempty"`
URL string `json:"url"`
Requests int64 `json:"requests"`
Time int `json:"time"`
Clients int `json:"clients"`
Headers datatypes.JSON `json:"headers"`
Cookies datatypes.JSON `json:"cookies"`
QPS int64 `json:"qps"`
Params datatypes.JSON `json:"params"`
KeepAlive bool `json:"keepAlive"`
RequestTimeout int `json:"requestTimeout"`
StatusCodeIncludes string `json:"statusCodeIncludes"`
Method string `json:"method"`
ServerId string `json:"serverId"`
WorkerId string `json:"workerId"`
Ips string `json:"ips" bson:"ips"`
PostData datatypes.JSON `json:"postData,omitempty"`
Created int64 `json:"created" bson:"created"`
UpdatedAt int64 `json:"updated_at" bson:"updated_at"`
CreatedAt int64 `json:"created_at" bson:"created_at"`
}

type RequestList struct {
Expand Down
73 changes: 27 additions & 46 deletions server/pkg/clients/http.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,12 @@ import (
)

type RequestSend struct {
cookies map[string]string
headers map[string]string
body []byte
url string
method string
cookies map[string]string
headers map[string]string
body []byte
url string
method string
statusCodes []int
}

type HttpClient struct {
Expand All @@ -39,62 +40,47 @@ type HttpClient struct {
}

func Initializer(request models.Request) (HttpClient, error) {
httpClient := HttpClient{}
httpClient.title.success = ".http_ok"
httpClient.title.otherFail = ".http_other_fail"
httpClient.title.fail = ".http_fail"
httpClient.title.latency = ".latency"

group := metrics.Group{
httpClient := HttpClient{
title: struct{ success, fail, otherFail, latency string }{
success: ".http_ok",
fail: ".http_fail",
otherFail: ".http_other_fail",
latency: ".latency",
},
}
groups := []metrics.Group{{
Name: "HTTP (" + request.ID + ")",
Graphs: []metrics.Graph{
{
Title: "HTTP Response",
Unit: "N",
Metrics: []metrics.Metric{
{
Title: httpClient.title.success,
Type: metrics.Counter,
},
{
Title: httpClient.title.fail,
Type: metrics.Counter,
},
{
Title: httpClient.title.otherFail,
Type: metrics.Counter,
},
{Title: httpClient.title.success, Type: metrics.Counter},
{Title: httpClient.title.fail, Type: metrics.Counter},
{Title: httpClient.title.otherFail, Type: metrics.Counter},
},
},
{
Title: "Latency",
Unit: "Microsecond",
Metrics: []metrics.Metric{
{
Title: httpClient.title.latency,
Type: metrics.Histogram,
},
{Title: httpClient.title.latency, Type: metrics.Histogram},
},
},
},
}
groups := []metrics.Group{
group,
}
}}

// Create http client with cookies
client, err := utils.GetFormedHttpClient(request)
if err != nil {
return httpClient, err
}
httpClient.client = client

requestedData := &RequestSend{
url: request.URL,
method: utils.GetSelectedMethods(request.Method),
url: request.URL,
method: utils.GetSelectedMethods(request.Method),
statusCodes: utils.GetStatusCodeIncludes(request.StatusCodeIncludes),
}

// Map headers with Http context
requestedData.headers, err = utils.GetFormedHeader(request.Headers)
if err != nil {
return httpClient, err
Expand All @@ -117,14 +103,12 @@ func Initializer(request models.Request) (HttpClient, error) {

func (h *HttpClient) RunScen(ctx context.Context, conf models.Request) {
finished := make(chan error)

go func() {
err := <-finished
if err != nil {
log.Error("Error:", err)
}
}()

h.Manager(ctx, conf, finished)
}

Expand Down Expand Up @@ -154,7 +138,6 @@ func (h *HttpClient) Manager(ctx context.Context, conf models.Request, done chan
}
}()
}

wg.Wait()
done <- nil
}
Expand All @@ -165,7 +148,6 @@ func (h *HttpClient) Request() ([]byte, error) {
return nil, err
}
defer res.Body.Close()

buf, err := ioutil.ReadAll(res.Body)
return buf, err
}
Expand All @@ -182,7 +164,6 @@ func (h *HttpClient) ignoreRes(verb string, url string, body []byte, headers map

func (h *HttpClient) do(method, url string, body []byte, headers map[string]string) (res *http.Response, err error) {
begin := time.Now()

defer func() {
diff := time.Since(begin)
executor.Notify(h.title.latency, diff.Microseconds())
Expand All @@ -192,12 +173,12 @@ func (h *HttpClient) do(method, url string, body []byte, headers map[string]stri
return
}

if res.StatusCode >= 300 || res.StatusCode < 200 {
statusOk := res.StatusCode >= 200 && res.StatusCode < 300
if (len(h.requested.statusCodes) == 0 && statusOk) || utils.IsCodeExist(h.requested.statusCodes, res.StatusCode) {
executor.Notify(h.title.success, 1)
} else {
executor.Notify(h.title.fail, 1)
return
}

executor.Notify(h.title.success, 1)
}()

req, err := http.NewRequest(method, url, strings.NewReader(string(body)))
Expand Down
37 changes: 34 additions & 3 deletions server/pkg/utils/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,11 @@ package utils

import (
"encoding/json"
"log"
"net/http"
"net/http/cookiejar"
"net/url"
"strconv"
"strings"
"time"

Expand Down Expand Up @@ -43,7 +45,7 @@ func Do(method, url string, body []byte, headers map[string]string) (*http.Respo

func GetFormedHttpClient(request models.Request) (*http.Client, error) {
var cookiesBucket []*http.Cookie

var requestTimeout int = 10
// Verify if the URL is correct
parsedUrl, err := url.Parse(request.URL)
if err != nil {
Expand All @@ -70,12 +72,15 @@ func GetFormedHttpClient(request models.Request) (*http.Client, error) {

jar.SetCookies(parsedUrl, cookiesBucket)

if request.RequestTimeout > 0 {
requestTimeout = request.RequestTimeout
}
tr := &http.Transport{
MaxIdleConnsPerHost: 300,
MaxIdleConnsPerHost: 1000,
}
return &http.Client{
Transport: tr,
Timeout: time.Second * 5,
Timeout: time.Second * time.Duration(requestTimeout),
Jar: jar,
CheckRedirect: func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
Expand All @@ -100,6 +105,32 @@ func GetFormedHeader(headers datatypes.JSON) (map[string]string, error) {
return nil, nil
}

func GetStatusCodeIncludes(codes string) []int {
if codes == "" {
return []int{}
}
parts := strings.Split(codes, ",")
numbers := make([]int, len(parts))
for i, part := range parts {
num, err := strconv.Atoi(part)
if err != nil {
log.Printf("Error converting %s to int: %v\n", part, err)
return nil
}
numbers[i] = num
}

return numbers
}

func IsCodeExist(slice []int, target int) bool {
for _, value := range slice {
if value == target {
return true
}
}
return false
}
func GetSelectedMethods(method string) string {
switch method {
case "GET":
Expand Down
33 changes: 23 additions & 10 deletions server/pkg/utils/loadster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,19 @@ import (
"github.com/cirnum/loadtester/server/db/models"
)

const (
latencySuffix = ".latency"
httpOkSuffix = ".http_ok"
httpFailSuffix = ".http_fail"
httpOtherFailSuffix = ".http_other_fail"
loadAvgSuffix = "LA1"
cpuSuffix = "CPU"
ramSuffix = "RAM"
transmitSuffix = "transmit"
receiveSuffix = "receive"
masterServerId = "MASTER"
)

func CalculateRPS(loads []models.Loadster, workers []models.Worker) customModels.CalculatedLoad {
serverMap, isFinish := MapReqByServer(loads)
calculatedInfo := CalculateRPSByTitle(serverMap, workers)
Expand All @@ -22,15 +35,15 @@ func CalculateRPSByTitle(loadsByServer map[string][]models.Loadster, workers []m
calculatedLoads.ServerMap = make(map[string]customModels.WorkerData, len(loadsByServer))
for key, load := range loadsByServer {
loadPayload := customModels.WorkerData{}
latency, lastLatency := HttpReqByType(load, ".latency")
okHTTP, lastOkHttp := HttpReqByType(load, ".http_ok")
failHTTP, lastFailHTTP := HttpReqByType(load, ".http_fail")
otherFailHTTP, lastOtherFailHTTP := HttpReqByType(load, ".http_other_fail")
loadAvg1, lastLoadAvg := HttpReqByType(load, "LA1")
cpu, lastCpuUsage := HttpReqByType(load, "CPU")
ram, lastRamUsage := HttpReqByType(load, "RAM")
outgress, lastOutgress := calcDataTransfer(load, "transmit")
ingress, lastIngress := calcDataTransfer(load, "receive")
latency, lastLatency := HttpReqByType(load, latencySuffix)
okHTTP, lastOkHttp := HttpReqByType(load, httpOkSuffix)
failHTTP, lastFailHTTP := HttpReqByType(load, httpFailSuffix)
otherFailHTTP, lastOtherFailHTTP := HttpReqByType(load, httpOtherFailSuffix)
loadAvg1, lastLoadAvg := HttpReqByType(load, loadAvgSuffix)
cpu, lastCpuUsage := HttpReqByType(load, cpuSuffix)
ram, lastRamUsage := HttpReqByType(load, ramSuffix)
outgress, lastOutgress := calcDataTransfer(load, transmitSuffix)
ingress, lastIngress := calcDataTransfer(load, receiveSuffix)
totalTimeTaken := int64((lastLatency.CreatedAt - lastLatency.StartTime) / 1000)

if lastFailHTTP.Count > 0 {
Expand Down Expand Up @@ -155,7 +168,7 @@ func MapReqByServer(loads []models.Loadster) (map[string][]models.Loadster, bool
serverId := load.ServerId
load.RPS = calculateRpsForEachLoad(load)
if serverId == "" {
serverId = "MASTER"
serverId = masterServerId
load.ServerId = serverId
}
if load.Finish {
Expand Down
Loading

0 comments on commit 39074dc

Please sign in to comment.