feat(validator): load validations from validations.yaml

This commit is rewrite of existing validator. Changes include

* remove queries.json in favour of validations.yaml
* move prometheus job names - metal and vm under `prometheus` field
* Simplify parsing queries making it testable
* Add gen-report command to regenerate reports
* Add logging that can be controlled by `log_level` in validator.yaml
* Fix use of hard-coded ssh-key to generate VM specs
* Fixes tests

Signed-off-by: Sunil Thaha <[email protected]>

e2e/tools/validator/.gitignore

@@ -1,2 +1,3 @@
 __pycache__
 validator.yaml
+tmp/

e2e/tools/validator/pyproject.toml

@@ -84,3 +84,8 @@ exclude_lines = [
   "if __name__ == .__main__.:",
   "if TYPE_CHECKING:",
 ]
+
+[tool.pytest.ini_options]
+markers = [
+	"focus",  # used in development to mark focused tests
+]

e2e/tools/validator/ruff.toml

@@ -0,0 +1 @@
+line-length = 120

e2e/tools/validator/scripts/stressor.sh

@@ -1,30 +1,43 @@
-#!/bin/bash
-
-# warm up system by waiting for 40 seconds
-# idle
-stress-ng --cpu $(nproc) --cpu-load 0 --timeout 10s
-
-# # Stress at 10% load for 20 seconds
-# stress-ng --cpu $(nproc) --cpu-load 10 --timeout 20s
-#
-# # Stress at 25% load for 20 seconds
-# stress-ng --cpu $(nproc) --cpu-load 25 --timeout 20s
-#
-# # Stress at 50% load for 20 seconds
-# stress-ng --cpu $(nproc) --cpu-load 50 --timeout 20s
-#
-# Stress at 75% load for 10 seconds
-stress-ng --cpu $(nproc) --cpu-load 75 --timeout 20s
-#
-# # Stress at 50% load for 20 seconds
-# stress-ng --cpu $(nproc) --cpu-load 50 --timeout 20s
-#
-# # Stress at 25% load for 20 seconds
-# stress-ng --cpu $(nproc) --cpu-load 25 --timeout 20s
-#
-# # Stress at 10% load for 20 seconds
-# stress-ng --cpu $(nproc) --cpu-load 10 --timeout 20s
-
-# cool off for 40 seconds
-# idle
-stress-ng --cpu $(nproc) --cpu-load 0 --timeout 10s
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+trap exit_all INT
+exit_all() {
+	pkill -P $$
+}
+
+run() {
+	echo "❯ $*"
+	"$@"
+	echo "      ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾"
+}
+
+main() {
+
+	local cpus
+	cpus=$(nproc)
+
+	# load and time
+	local -a load_curve=(
+		0:5
+		10:20
+		25:20
+		50:20
+		75:20
+		100:30
+		75:20
+		50:20
+		25:20
+		10:20
+		0:5
+	)
+
+	for x in "${load_curve[@]}"; do
+		local load="${x%%:*}"
+		local time="${x##*:}s"
+		run stress-ng --cpu "$cpus" --cpu-load "$load" --timeout "$time"
+	done
+}
+
+main "$@"