Implementation of manifest files

conftest.py

@@ -6,11 +6,13 @@
 import pytest
 from pytest_jsonreport.plugin import JSONReport
 
+from manifests.parser.core import load as load_manifest
 from utils import context
 from utils._context._scenarios import scenarios
 from utils.tools import logger
 from utils.scripts.junit_report import junit_modifyreport
 from utils._context.library_version import LibraryVersion
+from utils._decorators import released, _get_skipped_item, _get_expected_failure_item
 
 # Monkey patch JSON-report plugin to avoid noise in report
 JSONReport.pytest_terminal_summary = lambda *args, **kwargs: None
@@ -127,9 +129,52 @@ def _get_skip_reason_from_marker(marker):
     return None
 
 
+def pytest_pycollect_makemodule(module_path, parent):
+
+    manifest = load_manifest(context.scenario.library.library, context.scenario.weblog_variant)
+
+    relative_path = str(module_path.relative_to(module_path.cwd()))
+
+    if relative_path in manifest:
+        reason = manifest[relative_path]
+        mod: pytest.Module = pytest.Module.from_parent(parent, path=module_path)
+
+        if reason.startswith("not relevant") or reason.startswith("flaky"):
+            mod.add_marker(pytest.mark.skip(reason=reason))
+            logger.debug(f"Module {relative_path} is skipped by manifest file because {reason}")
+        else:
+            mod.add_marker(pytest.mark.xfail(reason=reason))
+            logger.debug(f"Module {relative_path} is xfailed by manifest file because {reason}")
+
+        return mod
+
+
+@pytest.hookimpl(tryfirst=True)
+def pytest_pycollect_makeitem(collector, name, obj):
+
+    if collector.istestclass(obj, name):
+
+        manifest = load_manifest(context.scenario.library.library, context.scenario.weblog_variant)
+        nodeid = f"{collector.nodeid}::{name}"
+
+        if nodeid in manifest:
+            declaration = manifest[nodeid]
+            logger.info(f"Manifest declaration for {nodeid}: {declaration}")
+
+            if declaration.startswith("v"):
+                if not hasattr(obj, "__released__"):  # let priority to inline declaration
+                    released(**{context.scenario.library.library: declaration[1:]})(obj)
+            elif declaration.startswith("not relevant") or declaration.startswith("flaky"):
+                _get_skipped_item(obj, declaration)
+            else:
+                _get_expected_failure_item(obj, declaration)
+
+
 def pytest_collection_modifyitems(session, config, items):
     """unselect items that are not included in the current scenario"""
 
+    logger.debug("pytest_collection_modifyitems")
+
     def get_declared_scenario(item):
         for marker in item.own_markers:
             if marker.name == "scenario":
@@ -181,6 +226,12 @@ def _item_is_skipped(item):
         if marker.name in ("skip",):
             return True
 
+    # for test methods in classes, item.parent.parent is the module
+    if item.parent.parent:
+        for marker in item.parent.parent.own_markers:
+            if marker.name in ("skip",):
+                return True
+
     return False
 
 
@@ -251,6 +302,8 @@ def pytest_runtest_call(item):
 
 @pytest.hookimpl(optionalhook=True)
 def pytest_json_runtest_metadata(item, call):
+    logger.debug("pytest_json_runtest_metadata")
+
     if call.when != "setup":
         return {}
 

manifests/cpp.yml

@@ -0,0 +1,62 @@
+tests/:
+  appsec/:
+    api_security/:
+      test_schemas.py: irrelevant (ASM is not implemented in C++)
+    iast/:
+      sink/:
+        test_command_injection.py: irrelevant (ASM is not implemented in C++)
+        test_insecure_cookie.py: irrelevant (ASM is not implemented in C++)
+        test_ldap_injection.py: irrelevant (ASM is not implemented in C++)
+        test_no_httponly_cookie.py: irrelevant (ASM is not implemented in C++)
+        test_no_samesite_cookie.py: irrelevant (ASM is not implemented in C++)
+        test_path_traversal.py: irrelevant (ASM is not implemented in C++)
+        test_sql_injection.py: irrelevant (ASM is not implemented in C++)
+        test_ssrf.py: irrelevant (ASM is not implemented in C++)
+        test_trust_boundary_violation.py: irrelevant (ASM is not implemented in C++)
+        test_unvalidated_redirect.py: irrelevant (ASM is not implemented in C++)
+        test_unvalidated_redirect_forward.py: irrelevant (ASM is not implemented in C++)
+        test_weak_cipher.py: irrelevant (ASM is not implemented in C++)
+        test_weak_hash.py: irrelevant (ASM is not implemented in C++)
+        test_weak_randomness.py: irrelevant (ASM is not implemented in C++)
+        test_xpath_injection.py: irrelevant (ASM is not implemented in C++)
+      source/:
+        test_body.py: irrelevant (ASM is not implemented in C++)
+        test_cookie_name.py: irrelevant (ASM is not implemented in C++)
+        test_cookie_value.py: irrelevant (ASM is not implemented in C++)
+        test_header_name.py: irrelevant (ASM is not implemented in C++)
+        test_header_value.py: irrelevant (ASM is not implemented in C++)
+        test_parameter_name.py: irrelevant (ASM is not implemented in C++)
+        test_parameter_value.py: irrelevant (ASM is not implemented in C++)
+    waf/:
+      test_addresses.py: irrelevant (ASM is not implemented in C++)
+      test_blocking.py: irrelevant (ASM is not implemented in C++)
+      test_custom_rules.py: irrelevant (ASM is not implemented in C++)
+      test_exclusions.py: irrelevant (ASM is not implemented in C++)
+      test_miscs.py: irrelevant (ASM is not implemented in C++)
+      test_reports.py: irrelevant (ASM is not implemented in C++)
+      test_rules.py: irrelevant (ASM is not implemented in C++)
+      test_telemetry.py: irrelevant (ASM is not implemented in C++)
+    test_PII.py: irrelevant (ASM is not implemented in C++)
+    test_alpha.py: irrelevant (ASM is not implemented in C++)
+    test_automated_login_events.py: irrelevant (ASM is not implemented in C++)
+    test_blocking_addresses.py: irrelevant (ASM is not implemented in C++)
+    test_conf.py: irrelevant (ASM is not implemented in C++)
+    test_customconf.py: irrelevant (ASM is not implemented in C++)
+    test_docs.py: irrelevant (ASM is not implemented in C++)
+    test_endpoint_ognl.py: irrelevant (ASM is not implemented in C++)
+    test_endpoint_sqli.py: irrelevant (ASM is not implemented in C++)
+    test_endpoint_ssrf.py: irrelevant (ASM is not implemented in C++)
+    test_event_tracking.py: irrelevant (ASM is not implemented in C++)
+    test_identify.py: irrelevant (ASM is not implemented in C++)
+    test_ip_blocking_full_denylist.py: irrelevant (ASM is not implemented in C++)
+    test_logs.py: irrelevant (ASM is not implemented in C++)
+    test_rate_limiter.py: irrelevant (ASM is not implemented in C++)
+    test_reports.py: irrelevant (ASM is not implemented in C++)
+    test_request_blocking.py: irrelevant (ASM is not implemented in C++)
+    test_runtime_activation.py: irrelevant (ASM is not implemented in C++)
+    test_traces.py: irrelevant (ASM is not implemented in C++)
+    test_user_blocking_full_denylist.py: irrelevant (ASM is not implemented in C++)
+    test_versions.py: irrelevant (ASM is not implemented in C++)
+  test_identify.py: irrelevant
+  test_scrubbing.py: irrelevant
+  test_standard_tags.py: irrelevant

manifests/dotnet.yml

@@ -0,0 +1,4 @@
+tests/:
+  appsec/:
+    test_alpha.py:
+      Test_Basic: "v1.28.6"

manifests/golang.yml

@@ -0,0 +1,8 @@
+tests/:
+  appsec/:
+    test_alpha.py:
+      Test_Basic:
+        "*": "v1.34.0"
+        chi: "v1.36.0"
+        echo: "v1.36.0"
+        gin: "v1.37.0"

manifests/java.yml

@@ -0,0 +1,8 @@
+tests/:
+  appsec/:
+    test_alpha.py:
+      Test_Basic:
+        "*": "v0.87.0"
+        akka-http: missing_feature (No AppSec support)
+        spring-boot-3-native: missing_feature (GraalVM. Tracing support only)
+        spring-boot-payara: missing_feature (No AppSec support)

manifests/nodejs.yml

@@ -0,0 +1,4 @@
+tests/:
+  appsec/:
+    test_alpha.py:
+      Test_Basic: "v2.0.0"

manifests/parser/core.py

@@ -0,0 +1,105 @@
+from functools import lru_cache
+import json
+import os
+
+from jsonschema import validate
+import yaml
+
+
+def _flatten(base, object):
+    if base.endswith(".py"):
+        base += "::"
+    for key, value in object.items():
+
+        if isinstance(value, str):
+            yield f"{base}{key}", value
+        elif isinstance(value, dict):
+            if base.endswith(".py::"):
+                yield f"{base}{key}", value
+            else:
+                yield from _flatten(f"{base}{key}", value)
+
+
+def _load_file(file):
+
+    try:
+        with open(file) as f:
+            data = yaml.safe_load(f)
+    except FileNotFoundError:
+        return {}
+
+    return {nodeid: value for nodeid, value in _flatten("", data)}
+
+
+def _filter_with_context(data, weblog_variant):
+    def resolve_value(value):
+        if isinstance(value, str):
+            return value
+        elif weblog_variant in value:
+            return value[weblog_variant]
+        elif "*" in value:
+            return value["*"]
+        else:
+            return None
+
+    def fix_irrelevant_legacy(value: str):
+        # in JSON report, the marker for irrelevant is "not relevant", where the decorator is "irrelevant"
+        if value.startswith("irrelevant"):
+            return "not relevant" + value[len("irrelevant") :]
+
+        return value
+
+    # resolve weblogs dicts
+    result = {nodeid: resolve_value(value) for nodeid, value in data.items()}
+
+    # filter nones, and fix irrelevant => not relevant
+    result = {nodeid: fix_irrelevant_legacy(value) for nodeid, value in result.items() if value is not None}
+
+    return result
+
+
+@lru_cache
+def load(library, weblog_variant):
+
+    result = _load_file(f"manifests/{library}.yml")
+    result = _filter_with_context(result, weblog_variant)
+
+    return result
+
+
+def assert_key_order(obj: dict, path=""):
+    last_key = "/"
+
+    for key in obj:
+        if last_key.endswith("/") and not key.endswith("/"):  # transition from folder fo files, nothing to do
+            pass
+        elif not last_key.endswith("/") and key.endswith("/"):  # folder must be before files
+            raise ValueError("Folders must be placed before files at {path}")
+        else:  # otherwise, it must be sorted
+            assert last_key < key, f"Order is not respcted at {path} ({last_key} < {key})"
+
+        if isinstance(obj[key], dict):
+            assert_key_order(obj[key], f"{path}.{key}")
+
+        last_key = key
+
+
+def validate_manifest_files():
+    with open("manifests/parser/schema.json") as f:
+        schema = json.load(f)
+
+    for file in os.listdir("manifests/"):
+        if file.endswith(".yml"):
+            try:
+                with open(f"manifests/{file}") as f:
+                    data = yaml.safe_load(f)
+
+                validate(data, schema)
+                assert_key_order(data)
+
+            except Exception as e:
+                raise ValueError(f"Fail to validate manifests/{file}") from e
+
+
+if __name__ == "__main__":
+    validate_manifest_files()

manifests/parser/schema.json

@@ -0,0 +1,70 @@
+{
+  "type": "object",
+
+  "properties": {
+    "tests/": { "$ref": "#/$defs/folder_content" }
+  },
+
+  "$defs": {
+    "folder_content": {
+      "type": "object",
+      "patternProperties": {
+        ".+/": { "$ref": "#/$defs/folder_content" },
+        "test_.+\\.py": { "$ref": "#/$defs/file_content" }
+      }
+    },
+
+    "file_content": {
+      "allOf": [
+        {
+          "if": { "type": "string" },
+          "then": { "$ref": "#/$defs/skipped_declaration" }
+        },
+        {
+          "if": { "type": "object" },
+          "then": { "$ref": "#/$defs/classes_object" }
+        }
+      ]
+    },
+
+    "classes_object": {
+      "patternProperties": {
+        "Test_.+": {
+          "$comment": "Value can be a version number, a skip reason, or an object with weblog variant as keys",
+          "allOf": [
+            {
+              "if": { "type": "string" },
+              "then": { "$ref": "#/$defs/feature_state" }
+            },
+
+            {
+              "if": { "type": "object" },
+              "then": { "$ref": "#/$defs/declaration_per_weblog" }
+            }
+          ]
+        }
+      }
+    },
+
+    "declaration_per_weblog": {
+      "$comment": "Keys are weblog variant names, values are version, or a skip reason",
+      "patternProperties": {
+        ".+": { "$ref": "#/$defs/feature_state" }
+      }
+    },
+
+    "feature_state": {
+      "oneOf": [
+        { "$ref": "#/$defs/version" },
+        { "$ref": "#/$defs/skipped_declaration" }
+      ]
+    },
+
+    "version": { "type": "string", "pattern": "v\\d.+" },
+
+    "skipped_declaration": {
+      "type": "string",
+      "pattern": "^(bug|flaky|irrelevant|missing_feature)( \\(.+\\))?$"
+    }
+  }
+}

manifests/python.yml

@@ -0,0 +1,4 @@
+tests/:
+  appsec/:
+    test_alpha.py:
+      Test_Basic: "v1.1.0rc2.dev"