From 29e5e78d2463e25b765fac2636a6d38f06e88a44 Mon Sep 17 00:00:00 2001 From: Brian Krabach Date: Wed, 13 Nov 2024 14:28:20 -0800 Subject: [PATCH] adds reporting of response time to explorer chat messages (#243) Shows up in the footer of the chat message: ![image](https://github.com/user-attachments/assets/e7822513-139c-4387-a935-4caf82bb2f21) --- .../explorer-assistant/assistant/chat.py | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/assistants/explorer-assistant/assistant/chat.py b/assistants/explorer-assistant/assistant/chat.py index 1673a48f..8f894ebd 100644 --- a/assistants/explorer-assistant/assistant/chat.py +++ b/assistants/explorer-assistant/assistant/chat.py @@ -7,6 +7,7 @@ import logging import re +import time from typing import Any, Awaitable, Callable import deepmerge @@ -183,6 +184,9 @@ async def respond_to_conversation( # define the metadata key for any metadata created within this method method_metadata_key = "respond_to_conversation" + # track the start time of the response generation + response_start_time = time.time() + # get the list of conversation participants participants_response = await context.get_participants(include_inactive=True) @@ -371,13 +375,22 @@ async def respond_to_conversation( }, ) + # create the footer items for the response footer_items = [] + if completion is not None: # get the total tokens used for the completion completion_total_tokens = completion.usage.total_tokens if completion.usage else 0 footer_items.append(_get_token_usage_message(config.request_config.max_tokens, completion_total_tokens)) - # add the completion to the metadata for debugging + # track the end time of the response generation + response_end_time = time.time() + response_duration = response_end_time - response_start_time + + # add the response duration to the footer items + footer_items.append(_get_response_duration_message(response_duration)) + + # update the metadata with debug information deepmerge.always_merger.merge( metadata, { @@ -593,6 +606,14 @@ async def _get_history_messages( return history +def _get_response_duration_message(response_duration: float) -> str: + """ + Generate a display friendly message for the response duration, to be added to the footer items. + """ + + return f"Response time: {response_duration:.2f} seconds" + + def _get_token_usage_message( max_tokens: int, completion_total_tokens: int,