v2.1.0 (#271)

* feat: Experimentation withou knowledge base

* feat: UI rehaul - cloudscape design

* chore: ragas version upgrade

app/configuration_validation.py

@@ -24,7 +24,7 @@
 def is_valid_combination(config, data):
     # Define your rules here
     regions = ["us-east-1", "us-west-2"]
-    if config['bedrock_knowledge_base']:
+    if config['bedrock_knowledge_base'] or not config['knowledge_base']:
         return True
     if config["region"] not in regions:
         return False
@@ -274,7 +274,7 @@ def generate_all_combinations(data):
     [num_prompts, num_chars] = read_gt_data(gt_data)
 
     avg_prompt_length = round(num_chars / num_prompts / 4)
-    if parameters_all["bedrock_knowledge_base"][0]:
+    if parameters_all["bedrock_knowledge_base"][0] or not parameters_all['knowledge_base'][0]:
         num_tokens_kb_data = 0
     else:
         num_tokens_kb_data = count_characters_in_file(parameters_all["kb_data"][0]) / 4
@@ -305,6 +305,7 @@ def generate_all_combinations(data):
 
     if len(valid_configurations) > 0:
         for configuration in valid_configurations:
+            configuration["is_opensearch"] = True if configs.opensearch_host else False
             #TODO: Organize the pricing code, break into static methods
             configuration["directional_pricing"] = 0
             configuration["indexing_cost_estimate"] = 0 
@@ -313,12 +314,12 @@ def generate_all_combinations(data):
             configuration["eval_cost_estimate"] = 0
 
             # kb data tokens would be zero if it is Bedrock knowledge bases
-            effective_num_tokens_kb_data = 0 if configuration["bedrock_knowledge_base"] else estimate_effective_kb_tokens(configuration, num_tokens_kb_data)
+            effective_num_tokens_kb_data = 0 if configuration["bedrock_knowledge_base"] or not configuration["knowledge_base"] else estimate_effective_kb_tokens(configuration, num_tokens_kb_data)
 
             indexing_time, retrieval_time, eval_time = estimate_times(effective_num_tokens_kb_data, num_prompts, configuration)
 
             # Bedrock knowledge bases price not supported at the moment
-            if configuration["bedrock_knowledge_base"]:
+            if configuration["bedrock_knowledge_base"] or not configuration["knowledge_base"]:
                 configuration["indexing_cost_estimate"] = 0
             elif configuration['embedding_service'] == "bedrock" :
                 embedding_price = estimate_embedding_model_bedrock_price(bedrock_price_df, configuration, num_tokens_kb_data)
@@ -349,7 +350,7 @@ def generate_all_combinations(data):
             configuration["eval_cost_estimate"] += estimate_fargate_price(eval_time)
 
             # add opensearch provisioned costs
-            if not configuration["bedrock_knowledge_base"]:
+            if not configuration["bedrock_knowledge_base"] or configuration["is_opensearch"]:
                 configuration["indexing_cost_estimate"] += estimate_opensearch_price(indexing_time)
                 configuration["retrieval_cost_estimate"] += estimate_opensearch_price(retrieval_time)
                 configuration["eval_cost_estimate"] += estimate_opensearch_price(eval_time)
@@ -358,7 +359,7 @@ def generate_all_combinations(data):
             configuration["directional_pricing"] +=configuration["directional_pricing"]*0.05 #extra
             configuration["directional_pricing"] = round(configuration["directional_pricing"],2)    
 
-    return valid_configurations
+    return valid_configurations[:1000]
 
 def generate_all_combinations_in_background(execution_id: str, execution_config_data):
     """       

app/main.py

@@ -3,7 +3,7 @@
 from .seed_data import seed_models
 
 from app.dependencies.database import db
-from app.routes import execution, experiment, health, uploads, bedrock_config
+from app.routes import execution, experiment, health, uploads, bedrock_config, config
 from app.dependencies.database import (
     get_execution_model_invocations_db
 )
@@ -32,6 +32,7 @@ async def startup_event():
     app.include_router(experiment.router)
     app.include_router(health.router)
     app.include_router(bedrock_config.router)
+    app.include_router(config.router)
 
     return app
 

app/price_calculator.py

@@ -127,7 +127,7 @@ def estimate_times(no_of_kb_tokens, num_prompts, configuration):
     indexing_service = configuration['embedding_service']
     retrieval_service = configuration['retrieval_service']
 
-    indexing_time = 0 if configuration["bedrock_knowledge_base"] else (no_of_kb_tokens/ 50000) * estimated_time['indexing'][indexing_service]
+    indexing_time = 0 if configuration["bedrock_knowledge_base"] or not configuration["knowledge_base"] else (no_of_kb_tokens/ 50000) * estimated_time['indexing'][indexing_service]
     retrieval_time = (num_prompts / 25) * estimated_time['retrieval'][retrieval_service]
     eval_time = (num_prompts / 25) * estimated_time['eval'][retrieval_service]
 

app/routes/config.py

@@ -0,0 +1,10 @@
+from fastapi import APIRouter
+from util.open_search_config_utils import OpenSearchUtils
+
+router = APIRouter()
+
+@router.get("/config", tags=["config"])
+async def config():
+    return {
+        "opensearch": OpenSearchUtils.opensearch_config()
+    }

baseclasses/base_classes.py

@@ -409,7 +409,7 @@ def from_dict(self, metrics_dict: Dict[str, str]) -> 'EvaluationMetrics':
         """Convert metrics from DynamoDB format to EvaluationMetrics"""
         return EvaluationMetrics(
             faithfulness_score=float(metrics_dict.get('faithfulness', '0.0')),
-            context_precision_score=float(metrics_dict.get('llm_context_precision_without_reference', '0.0')),
+            context_precision_score=float(metrics_dict.get('llm_context_precision_with_reference', '0.0')),
             aspect_critic_score=float(metrics_dict.get('maliciousness', '0.0')),
             answers_relevancy_score=float(metrics_dict.get('answer_relevancy', '0.0')),
             string_similarity=float(metrics_dict.get('String_Similarity', '0.0')),

cfn/ecr-repository-template.yaml

@@ -17,31 +17,37 @@ Resources:
     Type: AWS::ECR::Repository
     Properties:
       RepositoryName: !Sub "flotorch-indexing-${TableSuffix}"
+      EmptyOnDelete: true
 
   RetrieverRepository:
     Type: AWS::ECR::Repository
     Properties:
       RepositoryName: !Sub "flotorch-retriever-${TableSuffix}"
+      EmptyOnDelete: true
 
   AppRepository:
     Type: AWS::ECR::Repository
     Properties:
       RepositoryName: !Sub "flotorch-app-${TableSuffix}"
+      EmptyOnDelete: true
 
   EvaluationRepository:
     Type: AWS::ECR::Repository
     Properties:
       RepositoryName: !Sub "flotorch-evaluation-${TableSuffix}"
+      EmptyOnDelete: true
 
   RuntimeRepository:
     Type: AWS::ECR::Repository
     Properties:
       RepositoryName: !Sub "flotorch-runtime-${TableSuffix}"
+      EmptyOnDelete: true
 
   CostComputeRepository:
     Type: AWS::ECR::Repository
     Properties:
       RepositoryName: !Sub "flotorch-costcompute-${TableSuffix}"
+      EmptyOnDelete: true
 
 Outputs:
   AppRepositoryURI:

cfn/lambda-template.yaml

@@ -78,6 +78,16 @@ Parameters:
   CostComputeImageTag:
     Type: String
     Description: Tag for the AI image
+  PrerequisitesMet:
+    Type: String
+    Description: Whether prerequisites are met (yes/no)
+    AllowedValues: 
+      - "yes"
+      - "no"
+
+Conditions:
+  PrerequisitesMet: !Equals [!Ref PrerequisitesMet, "yes"]
+  PrerequisitesNotMet: !Equals [!Ref PrerequisitesMet, "no"]
 
 Resources:
   LambdaSecurityGroup:
@@ -156,6 +166,7 @@ Resources:
 
   DockerPullPushRole:
     Type: AWS::IAM::Role
+    Condition: PrerequisitesMet
     Properties:
       AssumeRolePolicyDocument:
         Version: '2012-10-17'
@@ -213,6 +224,7 @@ Resources:
 
   DockerPullPushFunction:
     Type: AWS::Lambda::Function
+    Condition: PrerequisitesMet
     Properties:
       Handler: index.handler
       Role: !GetAtt DockerPullPushRole.Arn
@@ -347,10 +359,21 @@ Resources:
                           fi
                       }}
 
+                      install_git() {{
+                          if command_exists git; then
+                              echo "Git is already installed"
+                              git --version
+                          else
+                              echo "Installing Git..."
+                              sudo yum install -y git
+                          fi
+                      }}
+
                       # Detect OS and install requirements
                       detect_os
                       install_docker
                       install_awscli
+                      install_git
 
                       # Configure Docker to use non-interactive mode
                       export DOCKER_CLI_NO_INTERACTIVE=1
@@ -475,6 +498,7 @@ Resources:
 
   DockerPullPushResource:
     Type: Custom::DockerPullPush
+    Condition: PrerequisitesMet
     DependsOn: DockerPullPushFunction
     Properties:
       ServiceToken: !GetAtt DockerPullPushFunction.Arn
@@ -495,14 +519,45 @@ Resources:
       RuntimeImageTag: !Ref RuntimeImageTag
       CostComputeImageTag: !Ref CostComputeImageTag
 
-  RuntimeLambdaFunction:
+  RuntimeLambdaFunctionWithPrereqs:
     Type: AWS::Lambda::Function
+    Condition: PrerequisitesMet
     DependsOn: DockerPullPushResource
     Properties:
       FunctionName: !Sub flotorch-runtime-${TableSuffix}
       Role: !GetAtt LambdaExecutionRole.Arn
       Code:
-        ImageUri: !Sub ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/flotorch-runtime-${TableSuffix}:${RuntimeImageTag}
+        ImageUri: !Sub ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/flotorch-runtime-${TableSuffix}:latest
+      PackageType: Image
+      MemorySize: 4096
+      Timeout: 900
+      VpcConfig:
+        SubnetIds: !Split [",", !Join [",", !Ref PrivateSubnets]]
+        SecurityGroupIds:
+          - !Ref LambdaSecurityGroup
+      Environment:
+        Variables:
+          DATA_BUCKET: !Ref DataBucketName
+          EXECUTION_TABLE: !Ref ExecutionTableName
+          EXPERIMENT_TABLE: !Ref ExperimentTableName
+          METRICS_TABLE: !Ref MetricsTableName
+          MODEL_INVOCATIONS_TABLE: !Ref ModelInvocationsTableName
+          opensearch_host: !Ref OpenSearchEndpoint
+          opensearch_username: !Ref OpenSearchAdminUser
+          opensearch_password: !Ref OpenSearchAdminPassword
+          INFERENCE_SYSTEM_PROMPT: "You are a helpful assistant. Use the provided context to answer questions accurately. If you cannot find the answer in the context, say so"
+          vector_field_name: "vectors"
+          opensearch_serverless: "false"
+          aws_region: !Ref AWS::Region
+
+  RuntimeLambdaFunctionNoPrereqs:
+    Type: AWS::Lambda::Function
+    Condition: PrerequisitesNotMet
+    Properties:
+      FunctionName: !Sub flotorch-runtime-${TableSuffix}
+      Role: !GetAtt LambdaExecutionRole.Arn
+      Code:
+        ImageUri: !Sub ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/flotorch-runtime-${TableSuffix}:latest
       PackageType: Image
       MemorySize: 4096
       Timeout: 900
@@ -525,8 +580,9 @@ Resources:
           opensearch_serverless: "false"
           aws_region: !Ref AWS::Region
 
-  CostComputeLambdaFunction:
+  CostComputeLambdaFunctionWithPrereqs:
     Type: AWS::Lambda::Function
+    Condition: PrerequisitesMet
     DependsOn: DockerPullPushResource
     Properties:
       FunctionName: !Sub flotorch-costcompute-${TableSuffix}
@@ -550,13 +606,48 @@ Resources:
           experiment_question_metrics_index: experiment_id-index
           experiment_question_metrics_table: !Ref MetricsTableName
 
+  CostComputeLambdaFunctionNoPrereqs:
+    Type: AWS::Lambda::Function
+    Condition: PrerequisitesNotMet
+    Properties:
+      FunctionName: !Sub flotorch-costcompute-${TableSuffix}
+      Role: !GetAtt LambdaExecutionRole.Arn
+      Code:
+        ImageUri: !Sub ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/flotorch-costcompute-${TableSuffix}:latest
+      PackageType: Image
+      MemorySize: 4096
+      Timeout: 900
+      VpcConfig:
+        SubnetIds: !Split [",", !Join [",", !Ref PrivateSubnets]]
+        SecurityGroupIds:
+          - !Ref LambdaSecurityGroup
+      Environment:
+        Variables:
+          execution_table: !Ref ExecutionTableName
+          experiment_table: !Ref ExperimentTableName
+          s3_bucket: !Ref DataBucketName
+          bedrock_limit_csv: seed/bedrock_limits_small.csv
+          aws_region: !Ref AWS::Region
+          experiment_question_metrics_index: experiment_id-index
+          experiment_question_metrics_table: !Ref MetricsTableName
+
 Outputs:
-  RuntimeLambdaArn:
+  RuntimeLambdaArnWithPrereqs:
+    Description: ARN of the Runtime Lambda function
+    Condition: PrerequisitesMet
+    Value: !GetAtt RuntimeLambdaFunctionWithPrereqs.Arn
+  CostComputeLambdaArnWithPrereqs:
+    Description: ARN of the cost compute lambda function
+    Condition: PrerequisitesMet
+    Value: !GetAtt CostComputeLambdaFunctionWithPrereqs.Arn
+  RuntimeLambdaArnNoPrereqs:
     Description: ARN of the Runtime Lambda function
-    Value: !GetAtt RuntimeLambdaFunction.Arn
-  CostComputeLambdaArn:
+    Condition: PrerequisitesNotMet
+    Value: !GetAtt RuntimeLambdaFunctionNoPrereqs.Arn
+  CostComputeLambdaArnNoPrereqs:
     Description: ARN of the cost compute lambda function
-    Value: !GetAtt CostComputeLambdaFunction.Arn
+    Condition: PrerequisitesNotMet
+    Value: !GetAtt CostComputeLambdaFunctionNoPrereqs.Arn
   LambdaRoleArn:
     Description: ARN of the Lambda execution role
     Value: !GetAtt LambdaExecutionRole.Arn