added ragas version and matrix with precision reference

baseclasses/base_classes.py

@@ -409,7 +409,7 @@ def from_dict(self, metrics_dict: Dict[str, str]) -> 'EvaluationMetrics':
         """Convert metrics from DynamoDB format to EvaluationMetrics"""
         return EvaluationMetrics(
             faithfulness_score=float(metrics_dict.get('faithfulness', '0.0')),
-            context_precision_score=float(metrics_dict.get('llm_context_precision_without_reference', '0.0')),
+            context_precision_score=float(metrics_dict.get('llm_context_precision_with_reference', '0.0')),
             aspect_critic_score=float(metrics_dict.get('maliciousness', '0.0')),
             answers_relevancy_score=float(metrics_dict.get('answer_relevancy', '0.0')),
             string_similarity=float(metrics_dict.get('String_Similarity', '0.0')),

config/experimental_config.py

@@ -41,7 +41,7 @@ class ExperimentalConfig(BaseModel):
     eval_service: str = 'ragas'
     eval_embedding_model: str = 'amazon.titan-embed-text-v1'
     eval_retrieval_model: str = 'mistral.mixtral-8x7b-instruct-v0:1'
-    eval_retrieval_temperature: float = float(0.4)
+    eval_retrieval_temperature: float = float(0.1) #float(0.4)
     # Rerank model id
     rerank_model_id: str = Field(alias="rerank_model_id", default="none")
     bedrock_knowledge_base: bool = False

core/eval/ragas/ragas_llm_eval.py

@@ -2,7 +2,7 @@
 from ragas import evaluate
 from ragas.dataset_schema import SingleTurnSample, EvaluationDataset
 from ragas.metrics._string import NonLLMStringSimilarity
-from ragas.metrics import Faithfulness, AspectCritic, LLMContextPrecisionWithoutReference, ResponseRelevancy
+from ragas.metrics import Faithfulness, AspectCritic, LLMContextPrecisionWithoutReference, ResponseRelevancy, LLMContextPrecisionWithReference
 from langchain_aws import ChatBedrockConverse, BedrockEmbeddings
 from ragas.embeddings import LangchainEmbeddingsWrapper
 from ragas.llms import LangchainLLMWrapper
@@ -44,7 +44,7 @@ def _initialize_scorers(self):
                                           definition="Is the submission intended to harm, deceive, or exploit users?",
                                           llm=self.evaluator_llm)
 
-        self.context_precision = LLMContextPrecisionWithoutReference(llm=self.evaluator_llm)
+        self.context_precision = LLMContextPrecisionWithReference(llm=self.evaluator_llm)
 
         self.answers_relevancy = ResponseRelevancy(llm=self.evaluator_llm,
                                                    embeddings=self.embedding_llm)

evaluation/requirements.txt

@@ -6,7 +6,7 @@ opensearch_py==2.7.1
 pydantic==2.9.2
 python-dotenv==1.0.1
 uvicorn==0.32.0
-ragas==0.2.6
+ragas==0.2.13
 RapidFuzz==3.10.1
 rouge_score==0.1.2
 langchain_aws==0.2.7

lambda_handlers/evaluation_handler.py

@@ -47,7 +47,10 @@ def lambda_handler(event: Dict[str, Any], context: Any) -> Dict[str, Any]:
             n_shot_prompts=exp_config_data.get('n_shot_prompts'),
             n_shot_prompt_guide=exp_config_data.get('n_shot_prompt_guide'),
             indexing_algorithm=exp_config_data.get('indexing_algorithm'),
-            knowledge_base=exp_config_data.get('knowledge_base', False)
+            knowledge_base=exp_config_data.get('knowledge_base', False),
+            eval_service=exp_config_data.get('eval_service', "ragas"), 
+            eval_embedding_model=exp_config_data.get('eval_embedding_model', "amazon.titan-embed-text-v1"), #amazon.nova-pro-v1:0
+            eval_retrieval_model=exp_config_data.get('eval_retrieval_model', "mistral.mixtral-8x7b-instruct-v0:1"),
         )
         logger.info("Processing event: %s", json.dumps(event))