Update pewpew examples and test scripting (#244)

* Update pewpew examples and test (#243)

* Added script to test the examples and script with example variables

* Updated example files to send to stdout so test_examples can view actual errors

* Fixed the loggers for several broken examples

* Updated legacy files in the config-wasm test dir

examples/.gitignore

@@ -1,3 +1,4 @@
 /stats-*.json
 /search-results*.json
 /log-results*
+/*.yaml.out

examples/body_file.yaml

@@ -11,7 +11,7 @@ load_pattern:
 
 loggers:
   test:
-    to: stderr
+    to: !stdout
 
 endpoints:
   - method: POST

examples/body_formdata.yaml

@@ -17,20 +17,20 @@ load_pattern:
 
 loggers:
   test:
-    to: stderr
+    to: !stdout
 
 endpoints:
   - method: POST
     url: http://localhost:${v:port}
     headers:
       Accept-Language: en-us
       Content-Type: application/x-www-form-urlencoded
-    body: !str username=${v:username}&password=${v:password}&grant_type=password&client_id=${v:dev_key}
+    body: !str username=${v:username}&password=${v:password}&grant_type=password&client_id=${v:dev_key}&token=abcd-1234-efgh
     peak_load: 1hps
     logs:
       test:
         select:
           ts: epoch("ms")
           response: request.body
-          token: response.body.token
+          token: match(response.body, "token=(?P<token>[a-zA-Z0-9-]+)").token
     on_demand: true

examples/body_json.yaml

@@ -11,7 +11,7 @@ load_pattern:
 
 loggers:
   test:
-    to: stderr
+    to: !stdout
 
 endpoints:
   - method: POST

examples/body_multipart.yaml

@@ -9,7 +9,7 @@ load_pattern:
 
 loggers:
   test:
-    to: stderr
+    to: !stdout
 
 endpoints:
   - method: POST

examples/delete_search.yaml

@@ -40,7 +40,7 @@ providers:
 
 loggers:
   test:
-    to: !stderr
+    to: !stdout
   test_404s:
     limit: 1
     to: !stdout

examples/delete_sequential_404s.yaml

@@ -35,7 +35,7 @@ providers:
 
 loggers:
   test:
-    to: !stderr
+    to: !stdout
   test_404s:
     limit: 1
     to: !stdout

examples/log_all.yaml

@@ -25,7 +25,7 @@ loggers:
     pretty: false
 
   test:
-    to: !stderr
+    to: !stdout
 
 endpoints:
   - method: POST

examples/log_csv.yaml

@@ -19,7 +19,7 @@ loggers:
     pretty: false
 
   test:
-    to: !stderr
+    to: !stdout
 
 endpoints:
   - method: POST

examples/log_errors.yaml

@@ -29,7 +29,7 @@ loggers:
     pretty: false
 
   test:
-    to: !stderr
+    to: !stdout
 
 endpoints:
   - method: GET

examples/login_for_apis_block.yaml

@@ -29,7 +29,7 @@ providers:
 
 loggers:
   test:
-    to: !stderr
+    to: !stdout
 
 endpoints:
   - method: POST
@@ -64,7 +64,9 @@ endpoints:
     logs:
       test:
         select:
-          a: 'response.body.b'
+          a: a
+          b: 'response.body.b'
+          session: session
 
   - method: POST
     url: 'http://localhost:${v:port}?c'
@@ -75,4 +77,6 @@ endpoints:
     logs:
       test:
         select:
-          a: 'response.body.c'
+          a: a
+          c: 'response.body.c'
+          session: session

examples/login_for_apis_force_new.yaml

@@ -28,7 +28,7 @@ providers:
 
 loggers:
   test:
-    to: !stderr
+    to: !stdout
 
 endpoints:
   - method: POST
@@ -63,7 +63,9 @@ endpoints:
     logs:
       test:
         select:
-          a: 'response.body.a'
+          a: a
+          b: 'response.body.b'
+          session: session
 
   - method: POST
     url: http://localhost:${v:port}?c
@@ -74,4 +76,6 @@ endpoints:
     logs:
       test:
         select:
-          a: 'response.body.a'
+          a: a
+          c: 'response.body.c'
+          session: session

examples/login_for_apis_on_demand.yaml

@@ -27,7 +27,7 @@ providers:
 
 loggers:
   test:
-    to: !stderr
+    to: !stdout
 
 endpoints:
   - method: POST
@@ -62,7 +62,9 @@ endpoints:
     logs:
       test:
         select:
-          a: 'response.body.a'
+          a: a
+          b: 'response.body.b'
+          session: session
 
   - method: POST
     url: http://localhost:${v:port}?c
@@ -73,4 +75,6 @@ endpoints:
     logs:
       test:
         select:
-          a: 'response.body.a'
+          a: a
+          c: 'response.body.c'
+          session: session

examples/provider_chain.yaml

@@ -19,7 +19,7 @@ providers:
 
 loggers:
   test:
-    to: !stderr
+    to: !stdout
 
 endpoints:
   - method: POST

examples/provider_collect.yaml

@@ -19,7 +19,7 @@ providers:
 
 loggers:
   test:
-    to: stderr
+    to: !stdout
 
 endpoints:
   - method: POST

examples/provider_create_and_delete.yaml

@@ -20,7 +20,7 @@ providers:
 
 loggers:
   test:
-    to: !stderr
+    to: !stdout
 
 endpoints:
   - method: POST

examples/provider_grid.yaml

@@ -19,7 +19,7 @@ providers:
 
 loggers:
   test:
-    to: !stderr
+    to: !stdout
 
 endpoints:
   - method: POST

examples/provider_loop.yaml

@@ -17,7 +17,7 @@ providers:
 
 loggers:
   test:
-    to: !stderr
+    to: !stdout
 
 endpoints:
   - method: POST

examples/provider_loop_with_counter.yaml

@@ -17,7 +17,7 @@ providers:
 
 loggers:
   test:
-    to: !stderr
+    to: !stdout
 
 endpoints:
   - method: POST

examples/provider_redirect.yaml

@@ -15,7 +15,7 @@ providers:
 
 loggers:
   test:
-    to: !stderr
+    to: !stdout
 
 endpoints:
   - method: GET

examples/provider_spread.yaml

@@ -27,7 +27,7 @@ providers:
 
 loggers:
   test:
-    to: !stderr
+    to: !stdout
 
 endpoints:
   - method: POST

examples/ramp_and_load.yaml

@@ -15,7 +15,7 @@ load_pattern:
 
 loggers:
   test:
-    to: !stderr
+    to: !stdout
 
 endpoints:
   - method: GET

examples/ramp_api_different.yaml

@@ -11,7 +11,7 @@ load_pattern:
 
 loggers:
   test:
-    to: !stderr
+    to: !stdout
 
 endpoints:
   - method: GET

examples/ramp_basic.yaml

@@ -11,7 +11,7 @@ load_pattern:
 
 loggers:
   test:
-    to: !stderr
+    to: !stdout
 
 endpoints:
   - method: GET

examples/ramp_over_time.yaml

@@ -20,7 +20,7 @@ load_pattern:
 
 loggers:
   test:
-    to: !stderr
+    to: !stdout
 
 endpoints:
   - method: GET

examples/ramp_to_scale.yaml

@@ -16,7 +16,7 @@ load_pattern:
 
 loggers:
   test:
-    to: !stderr
+    to: !stdout
 
 endpoints:
   - method: GET

examples/ramp_up_and_down.yaml

@@ -31,7 +31,7 @@ load_pattern:
 
 loggers:
   test:
-    to: !stderr
+    to: !stdout
 
 endpoints:
   - method: GET

examples/random_search.yaml

@@ -108,7 +108,7 @@ providers:
 
 loggers:
   test:
-    to: !stderr
+    to: !stdout
   http_errors:
     query:
       select:
@@ -124,7 +124,7 @@ loggers:
         responseBody: response.body
       where: response.status >= 400
     limit: 1000
-    to: !stderr
+    to: !stdout
     pretty: false
   search_logger:
     to: !file 'search-results-${x:epoch("ms")}.json' # log to a file with a timestamp. timestamp set at test start

examples/test_examples.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -e
+set +x
+source ./variables.sh
+
+if [ -z ${PORT+x} ]; then
+  echo "PORT is unset. Please start the test-server and specify the PORT variable when running this";
+  exit 1
+else
+  echo "Testing examples on PORT '$PORT'";
+fi
+PEWPEW_PATH=("pewpew")
+if [[ ! -z "$1" ]] ; then
+  PEWPEW_PATH=("$1")
+fi
+PEWPEW_VERSION=$($PEWPEW_PATH --version)
+echo "Testing examples against version $PEWPEW_VERSION"
+
+
+for file in *.yaml; do
+  [ -f "$file" ] || break
+  echo "Running: $PEWPEW_PATH run -f json $file"
+  "$PEWPEW_PATH" run -f json "$file" > "$file.out"
+  echo "Result: $?"
+done
+echo "All examples passed!"

examples/variables.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+export PASSWORD=bogus
+export RAMP_TIME=5s
+export LOAD_TIME=10s
+export SCALE_PERCENTAGE=100

lib/config-wasm/tests/legacy_examples/body_file.yaml

@@ -11,7 +11,7 @@ load_pattern:
 
 loggers:
   test:
-    to: stderr
+    to: stdout
 
 endpoints:
   - method: POST

lib/config-wasm/tests/legacy_examples/body_formdata.yaml

@@ -17,21 +17,21 @@ load_pattern:
 
 loggers:
   test:
-    to: stderr
+    to: stdout
 
 endpoints:
   - method: POST
     url: http://localhost:${port}
     headers:
       Accept-Language: en-us
       Content-Type: application/x-www-form-urlencoded
-    body: username=${username}&password=${password}&grant_type=password&client_id=${dev_key}
+    body: username=${username}&password=${password}&grant_type=password&client_id=${dev_key}&token=abcd-1234-efgh
     peak_load: 1hps
     logs:
       test:
         select:
           ts: epoch("ms")
           response: request.body
-          token: response.body.token
+          token: match(response.body, "token=(?P<token>[a-zA-Z0-9-]+)").token
     on_demand: true