diff --git a/performance/config.yml b/performance/config.yml index 90a02273..f551e3bb 100644 --- a/performance/config.yml +++ b/performance/config.yml @@ -1,4 +1,6 @@ modules: + local: + sequential: true jmeter: properties: base-uri: http://localhost diff --git a/performance/test.sh b/performance/test.sh index 2e8c3a32..bac8525b 100755 --- a/performance/test.sh +++ b/performance/test.sh @@ -14,7 +14,7 @@ echo "Running performance tests against: $URL" export BASE_URL="${URL}" docker run --rm -e BASE_URL -v $(pwd):/bzt-configs \ -v $(pwd)/../build/performance-artifacts:/tmp/artifacts \ - blazemeter/taurus:1.10.3 \ + blazemeter/taurus:1.16.35 \ -o modules.jmeter.properties.base-uri="${BASE_URL}" \ -o reporting.2.dump-xml=/tmp/artifacts/stats.xml \ config.yml \ diff --git a/performance/tests/stockCardLineItemReason.yml b/performance/tests/stockCardLineItemReason.yml index 04211d05..ba0961bd 100644 --- a/performance/tests/stockCardLineItemReason.yml +++ b/performance/tests/stockCardLineItemReason.yml @@ -1,15 +1,23 @@ execution: - - concurrency: 1 - hold-for: 2m + - concurrency: 10 + iterations: 10 + ramp-up: 1m + hold-for: 10m scenario: create-reason - - concurrency: 1 - hold-for: 2m + - concurrency: 10 + iterations: 10 + ramp-up: 1m + hold-for: 10m scenario: update-reason - - concurrency: 1 - hold-for: 2m + - concurrency: 10 + iterations: 10 + ramp-up: 1m + hold-for: 10m scenario: get-reason - - concurrency: 1 - hold-for: 2m + - concurrency: 10 + iterations: 10 + ramp-up: 1m + hold-for: 10m scenario: get-reasons scenarios: @@ -81,9 +89,17 @@ scenarios: body: ${reason} reporting: - - module: passfail - criteria: - Get Reason too slow: p90 of GetReason>200ms - Get Reasons too slow: p90 of GetReasons>15000ms - Create Reason too slow: p90 of CreateReason>500ms - Update Reason too slow: p90 of UpdateReason>500ms + - module: passfail + criteria: + - 'CreateReason too high error rate: failures>1%, continue as failed, label=CreateReason, title="CreateReason error rate too high"' + - 'CreateReason is too slow: p90>2000ms, stop as failed, label=CreateReason, title="CreateReason response time too high"' + - 'CreateReason needs improvement: p90>1500ms and p90<=2000ms, continue as passed, label=CreateReason, title="CreateReason needs improvement"' + - 'GetReason too high error rate: failures>1%, continue as failed, label=GetReason, title="GetReason error rate too high"' + - 'GetReason is too slow: p90>2500ms, stop as failed, label=GetReason, title="GetReason response time too high"' + - 'GetReason needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetReason, title="GetReason needs improvement"' + - 'GetReasons too high error rate: failures>1%, continue as failed, label=GetReasons, title="GetReasons error rate too high"' + - 'GetReasons is too slow: p90>2500ms, stop as failed, label=GetReasons, title="GetReasons response time too high"' + - 'GetReasons needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetReasons, title="GetReasons needs improvement"' + - 'UpdateReason too high error rate: failures>1%, continue as failed, label=UpdateReason, title="UpdateReason error rate too high"' + - 'UpdateReason is too slow: p90>4000ms, stop as failed, label=UpdateReason, title="UpdateReason response time too high"' + - 'UpdateReason needs improvement: p90>2500ms and p90<=4000ms, continue as passed, label=UpdateReason, title="UpdateReason needs improvement"' diff --git a/performance/tests/stockCardLineItemReasonTag.yml b/performance/tests/stockCardLineItemReasonTag.yml index b0c00e8d..29cbb76f 100644 --- a/performance/tests/stockCardLineItemReasonTag.yml +++ b/performance/tests/stockCardLineItemReasonTag.yml @@ -1,6 +1,8 @@ execution: - - concurrency: 1 - hold-for: 2m + - concurrency: 10 + iterations: 10 + ramp-up: 1m + hold-for: 10m scenario: get-tags scenarios: @@ -14,6 +16,8 @@ scenarios: Authorization: Bearer ${access_token} reporting: - - module: passfail - criteria: - Get Tags too slow: p90 of GetTags>200ms + - module: passfail + criteria: + - 'GetTags too high error rate: failures>1%, continue as failed, label=GetTags, title="GetTags error rate too high"' + - 'GetTags is too slow: p90>2500ms, stop as failed, label=GetTags, title="GetTags response time too high"' + - 'GetTags needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetTags, title="GetTags needs improvement"' diff --git a/performance/tests/stockCardRangeSummaries.yml b/performance/tests/stockCardRangeSummaries.yml index eac0ae9a..171a2bcd 100644 --- a/performance/tests/stockCardRangeSummaries.yml +++ b/performance/tests/stockCardRangeSummaries.yml @@ -1,6 +1,8 @@ execution: - - concurrency: 1 - hold-for: 2m + - concurrency: 10 + iterations: 10 + ramp-up: 1m + hold-for: 10m scenario: get-stock-card-range-summaries scenarios: @@ -21,6 +23,8 @@ scenarios: Authorization: Bearer ${access_token} reporting: - - module: passfail - criteria: - Get Stock Card Range Summaries too slow: p90 of GetStockCardRangeSummaries>90000ms + - module: passfail + criteria: + - 'GetStockCardRangeSummaries too high error rate: failures>1%, continue as failed, label=GetStockCardRangeSummaries, title="GetStockCardRangeSummaries error rate too high"' + - 'GetStockCardRangeSummaries is too slow: p90>2500ms, stop as failed, label=GetStockCardRangeSummaries, title="GetStockCardRangeSummaries response time too high"' + - 'GetStockCardRangeSummaries needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetStockCardRangeSummaries, title="GetStockCardRangeSummaries needs improvement"' diff --git a/performance/tests/stockCardSummaries.yml b/performance/tests/stockCardSummaries.yml index 60239097..1084461b 100644 --- a/performance/tests/stockCardSummaries.yml +++ b/performance/tests/stockCardSummaries.yml @@ -1,6 +1,8 @@ execution: - - concurrency: 1 - hold-for: 2m + - concurrency: 10 + iterations: 10 + ramp-up: 1m + hold-for: 10m scenario: get-stock-card-summaries-v2 scenarios: @@ -19,6 +21,8 @@ scenarios: Authorization: Bearer ${access_token} reporting: - - module: passfail - criteria: - Get Stock Card Summaries V2 too slow: p90 of GetStockCardSummariesV2>125000ms + - module: passfail + criteria: + - 'GetStockCardSummariesV2 too high error rate: failures>1%, continue as failed, label=GetStockCardSummariesV2, title="GetStockCardSummariesV2 error rate too high"' + - 'GetStockCardSummariesV2 is too slow: p90>2500ms, stop as failed, label=GetStockCardSummariesV2, title="GetStockCardSummariesV2 response time too high"' + - 'GetStockCardSummariesV2 needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetStockCardSummariesV2, title="GetStockCardSummariesV2 needs improvement"'