chore(weave): Adds schema validation to llm judge (#3004)

* init * init
wandb · Nov 19, 2024 · 900250e · 900250e
1 parent f664268
commit 900250e
Showing 1 changed file with 11 additions and 0 deletions.
diff --git a/weave/trace_server/interface/base_object_classes/actions.py b/weave/trace_server/interface/base_object_classes/actions.py
@@ -1,5 +1,6 @@
 from typing import Any, Literal, Optional, Union
 
+import jsonschema
 from pydantic import BaseModel, Field, field_validator
 
 from weave.trace_server.interface.base_object_classes import base_object_def
@@ -14,6 +15,16 @@ class LlmJudgeActionConfig(BaseModel):
     # Expected to be valid JSON Schema
     response_schema: dict[str, Any]
 
+    @field_validator("response_schema")
+    def validate_response_schema(cls, v: dict) -> dict:
+        try:
+            jsonschema.validate(None, v)
+        except jsonschema.exceptions.SchemaError as e:
+            raise e
+        except jsonschema.exceptions.ValidationError:
+            pass  # we don't care that `None` does not conform
+        return v
+
 
 class ContainsWordsActionConfig(BaseModel):
     action_type: Literal["contains_words"] = "contains_words"