Merge pull request #1133 from DataDog/cbeauchesne/scenario-set

Add set of scenario feature

run.sh

@@ -6,19 +6,104 @@
 
 set -eu
 
-if [[ ${1:-} =~ ^[A-Z0-9_]+$ ]]; then
-    # Retro comp: if the first argument is a list of capital letters, then we consider it's a scenario name
-    # and we add the -S option, telling pytest that's a scenario name
-    RUNNER_ARGS="-S $@"
-else
-    RUNNER_ARGS=$@
-fi
-
 # clean any pycache folder
 find utils tests -type d -name '__pycache__'  -prune -exec rm -rf {} +
 
 if [[ -z "${IN_NIX_SHELL:-}" ]]; then
    source venv/bin/activate
 fi
 
-pytest $RUNNER_ARGS
+# All the purpose if this script is to handle set of scenarios
+# convention: a set of scenarios must ends with _SCENARIOS
+
+APPSEC_SCENARIOS=(
+    APPSEC_MISSING_RULES
+    APPSEC_CORRUPTED_RULES
+    APPSEC_CUSTOM_RULES
+    APPSEC_BLOCKING
+    APPSEC_RULES_MONITORING_WITH_ERRORS
+    APPSEC_DISABLED
+    APPSEC_CUSTOM_OBFUSCATION
+    APPSEC_RATE_LIMITER
+    APPSEC_WAF_TELEMETRY
+    APPSEC_IP_BLOCKING
+    APPSEC_IP_BLOCKING_MAXED
+    APPSEC_REQUEST_BLOCKING
+    APPSEC_RUNTIME_ACTIVATION
+)
+
+REMOTE_CONFIG_SCENARIOS=(
+    REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD
+    REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD_NOCACHE
+    REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES
+    REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES_NOCACHE
+    REMOTE_CONFIG_MOCKED_BACKEND_LIVE_DEBUGGING
+    REMOTE_CONFIG_MOCKED_BACKEND_LIVE_DEBUGGING_NOCACHE
+)
+
+TELEMETRY_SCENARIOS=(
+    TELEMETRY_MESSAGE_BATCH_EVENT_ORDER
+    TELEMETRY_APP_STARTED_PRODUCTS_DISABLED
+    TELEMETRY_DEPENDENCY_LOADED_TEST_FOR_DEPENDENCY_COLLECTION_DISABLED
+    TELEMETRY_LOG_GENERATION_DISABLED
+    TELEMETRY_METRIC_GENERATION_DISABLED
+)
+
+TRACER_RELEASE_SCENARIOS=(
+    DEFAULT 
+    TRACE_PROPAGATION_STYLE_W3C 
+    PROFILING 
+    LIBRARY_CONF_CUSTOM_HEADERS_SHORT 
+    LIBRARY_CONF_CUSTOM_HEADERS_LONG
+    INTEGRATIONS 
+    CGROUP 
+    APM_TRACING_E2E_SINGLE_SPAN
+    APM_TRACING_E2E 
+    "${APPSEC_SCENARIOS[@]}"
+    "${REMOTE_CONFIG_SCENARIOS[@]}"
+    "${TELEMETRY_SCENARIOS[@]}"
+)
+
+TRACER_ESSENTIAL_SCENARIOS=(
+    DEFAULT
+    APPSEC_BLOCKING
+    APPSEC_CUSTOM_OBFUSCATION
+    APPSEC_RATE_LIMITER
+    APPSEC_WAF_TELEMETRY
+    APPSEC_IP_BLOCKING
+    APPSEC_IP_BLOCKING_MAXED
+    APPSEC_REQUEST_BLOCKING
+    REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD
+    REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES
+    REMOTE_CONFIG_MOCKED_BACKEND_LIVE_DEBUGGING
+    TELEMETRY_MESSAGE_BATCH_EVENT_ORDER
+    INTEGRATIONS
+    APM_TRACING_E2E
+)
+
+readonly SCENARIO=${1:-}
+
+if [[ $SCENARIO == "TRACER_RELEASE_SCENARIOS" ]]; then
+    for scenario in "${TRACER_RELEASE_SCENARIOS[@]}"; do pytest -S $scenario ${@:2}; done
+
+elif [[ $SCENARIO == "TRACER_ESSENTIAL_SCENARIOS" ]]; then
+    for scenario in "${TRACER_ESSENTIAL_SCENARIOS[@]}"; do pytest -S $scenario ${@:2}; done
+
+elif [[ $SCENARIO == "APPSEC_SCENARIOS" ]]; then
+    for scenario in "${APPSEC_SCENARIOS[@]}"; do pytest -S $scenario ${@:2}; done
+
+elif [[ $SCENARIO == "REMOTE_CONFIG_SCENARIOS" ]]; then
+    for scenario in "${REMOTE_CONFIG_SCENARIOS[@]}"; do pytest -S $scenario ${@:2}; done
+
+elif [[ $SCENARIO == "TELEMETRY_SCENARIOS" ]]; then
+    for scenario in "${TELEMETRY_SCENARIOS[@]}"; do pytest -S $scenario ${@:2}; done
+
+elif [[ $SCENARIO =~ ^[A-Z0-9_]+$ ]]; then
+    # If the first argument is a list of capital letters, then we consider it's a scenario name
+    # and we add the -S option, telling pytest that's a scenario name
+    pytest -S $1 ${@:2}
+
+else
+    # otherwise, a simple proxy to pytest
+    pytest $@
+fi

utils/_context/_scenarios.py

@@ -864,3 +864,10 @@ class scenarios:
         "LIBRARY_CONF_CUSTOM_HEADERS_LONG",
         additional_trace_header_tags=("header-tag1:custom.header-tag1", "header-tag2:custom.header-tag2"),
     )
+
+
+if __name__ == "__main__":
+    for name in dir(scenarios):
+        if not name.startswith("_"):
+            scenario = getattr(scenarios, name)
+            print(scenario.name)