CQI-21: isolated and extended performance test cases

performance/config.yml

@@ -1,4 +1,6 @@
 modules:
+  local:
+    sequential: true
   jmeter:
     properties:
       base-uri: http://localhost

performance/test.sh

@@ -14,7 +14,7 @@ echo "Running performance tests against: $URL"
 export BASE_URL="${URL}"
 docker run --rm -e BASE_URL -v $(pwd):/bzt-configs \
   -v $(pwd)/../build/performance-artifacts:/tmp/artifacts \
-  blazemeter/taurus:1.10.3 \
+  blazemeter/taurus:1.16.35 \
   -o modules.jmeter.properties.base-uri="${BASE_URL}" \
   -o reporting.2.dump-xml=/tmp/artifacts/stats.xml \
   config.yml \

performance/tests/stockCardLineItemReason.yml

@@ -1,15 +1,23 @@
 execution:
-  - concurrency: 1
-    hold-for: 2m
+  - concurrency: 10
+    iterations: 10
+    ramp-up: 1m
+    hold-for: 10m
     scenario: create-reason
-  - concurrency: 1
-    hold-for: 2m
+  - concurrency: 10
+    iterations: 10
+    ramp-up: 1m
+    hold-for: 10m
     scenario: update-reason
-  - concurrency: 1
-    hold-for: 2m
+  - concurrency: 10
+    iterations: 10
+    ramp-up: 1m
+    hold-for: 10m
     scenario: get-reason
-  - concurrency: 1
-    hold-for: 2m
+  - concurrency: 10
+    iterations: 10
+    ramp-up: 1m
+    hold-for: 10m
     scenario: get-reasons
 
 scenarios:
@@ -81,9 +89,17 @@ scenarios:
         body: ${reason}
 
 reporting:
-    - module: passfail
-      criteria:
-        Get Reason too slow: p90 of GetReason>200ms
-        Get Reasons too slow: p90 of GetReasons>15000ms
-        Create Reason too slow: p90 of CreateReason>500ms
-        Update Reason too slow: p90 of UpdateReason>500ms
+  - module: passfail
+    criteria:
+      - 'CreateReason too high error rate: failures>1%, continue as failed, label=CreateReason, title="CreateReason error rate too high"'
+      - 'CreateReason is too slow: p90>2000ms, stop as failed, label=CreateReason, title="CreateReason response time too high"'
+      - 'CreateReason needs improvement: p90>1500ms and p90<=2000ms, continue as passed, label=CreateReason, title="CreateReason needs improvement"'
+      - 'GetReason too high error rate: failures>1%, continue as failed, label=GetReason, title="GetReason error rate too high"'
+      - 'GetReason is too slow: p90>2500ms, stop as failed, label=GetReason, title="GetReason response time too high"'
+      - 'GetReason needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetReason, title="GetReason needs improvement"'
+      - 'GetReasons too high error rate: failures>1%, continue as failed, label=GetReasons, title="GetReasons error rate too high"'
+      - 'GetReasons is too slow: p90>2500ms, stop as failed, label=GetReasons, title="GetReasons response time too high"'
+      - 'GetReasons needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetReasons, title="GetReasons needs improvement"'
+      - 'UpdateReason too high error rate: failures>1%, continue as failed, label=UpdateReason, title="UpdateReason error rate too high"'
+      - 'UpdateReason is too slow: p90>4000ms, stop as failed, label=UpdateReason, title="UpdateReason response time too high"'
+      - 'UpdateReason needs improvement: p90>2500ms and p90<=4000ms, continue as passed, label=UpdateReason, title="UpdateReason needs improvement"'

performance/tests/stockCardLineItemReasonTag.yml

@@ -1,6 +1,8 @@
 execution:
-  - concurrency: 1
-    hold-for: 2m
+  - concurrency: 10
+    iterations: 10
+    ramp-up: 1m
+    hold-for: 10m
     scenario: get-tags
 
 scenarios:
@@ -14,6 +16,8 @@ scenarios:
           Authorization: Bearer ${access_token}
 
 reporting:
-    - module: passfail
-      criteria:
-        Get Tags too slow: p90 of GetTags>200ms
+  - module: passfail
+    criteria:
+      - 'GetTags too high error rate: failures>1%, continue as failed, label=GetTags, title="GetTags error rate too high"'
+      - 'GetTags is too slow: p90>2500ms, stop as failed, label=GetTags, title="GetTags response time too high"'
+      - 'GetTags needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetTags, title="GetTags needs improvement"'

performance/tests/stockCardRangeSummaries.yml

@@ -1,6 +1,8 @@
 execution:
-  - concurrency: 1
-    hold-for: 2m
+  - concurrency: 10
+    iterations: 10
+    ramp-up: 1m
+    hold-for: 10m
     scenario: get-stock-card-range-summaries
 
 scenarios:
@@ -21,6 +23,8 @@ scenarios:
           Authorization: Bearer ${access_token}
 
 reporting:
-    - module: passfail
-      criteria:
-        Get Stock Card Range Summaries too slow: p90 of GetStockCardRangeSummaries>90000ms
+  - module: passfail
+    criteria:
+      - 'GetStockCardRangeSummaries too high error rate: failures>1%, continue as failed, label=GetStockCardRangeSummaries, title="GetStockCardRangeSummaries error rate too high"'
+      - 'GetStockCardRangeSummaries is too slow: p90>2500ms, stop as failed, label=GetStockCardRangeSummaries, title="GetStockCardRangeSummaries response time too high"'
+      - 'GetStockCardRangeSummaries needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetStockCardRangeSummaries, title="GetStockCardRangeSummaries needs improvement"'

performance/tests/stockCardSummaries.yml

@@ -1,6 +1,8 @@
 execution:
-  - concurrency: 1
-    hold-for: 2m
+  - concurrency: 10
+    iterations: 10
+    ramp-up: 1m
+    hold-for: 10m
     scenario: get-stock-card-summaries-v2
 
 scenarios:
@@ -19,6 +21,8 @@ scenarios:
           Authorization: Bearer ${access_token}
 
 reporting:
-    - module: passfail
-      criteria:
-        Get Stock Card Summaries V2 too slow: p90 of GetStockCardSummariesV2>125000ms
+  - module: passfail
+    criteria:
+      - 'GetStockCardSummariesV2 too high error rate: failures>1%, continue as failed, label=GetStockCardSummariesV2, title="GetStockCardSummariesV2 error rate too high"'
+      - 'GetStockCardSummariesV2 is too slow: p90>2500ms, stop as failed, label=GetStockCardSummariesV2, title="GetStockCardSummariesV2 response time too high"'
+      - 'GetStockCardSummariesV2 needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetStockCardSummariesV2, title="GetStockCardSummariesV2 needs improvement"'