From 3cc191a45415768f7774118b5fa9372ec94e6a20 Mon Sep 17 00:00:00 2001 From: Mark Waddle Date: Wed, 6 Nov 2024 16:53:54 -0800 Subject: [PATCH] Extracts candidate values from documents (#226) Using a separate LLM call per document. The extracted, "candidate" field values are then provided to the guided conversation. The guided conversation then works through asking the user which values to use. Additionally, removes unneeded truncation of completion messages in message debug, now that it's stored and retrieved separately. --- .../explorer-assistant/assistant/chat.py | 4 +- .../assistant/agents/document_agent.py | 4 +- .../agents/form_fill_agent/__init__.py | 10 - .../assistant/agents/form_fill_agent/agent.py | 158 ------------ .../form_fill_agent/steps/_attachments.py | 57 ----- .../form_fill_agent/steps/fill_form_step.py | 149 ----------- .../agents/form_fill_extension/__init__.py | 9 + .../config.py | 12 +- .../agents/form_fill_extension/extension.py | 163 ++++++++++++ .../inspector.py | 6 +- .../state.py | 11 +- .../steps/__init__.py | 0 .../steps/_guided_conversation.py | 0 .../agents/form_fill_extension/steps/_llm.py | 47 ++++ .../steps/acquire_form_step.py | 90 ++++--- .../steps/extract_form_fields_step.py | 48 +--- .../steps/fill_form_step.py | 237 ++++++++++++++++++ .../steps/types.py | 22 +- .../prospector-assistant/assistant/chat.py | 141 ++++++----- .../prospector-assistant/assistant/config.py | 6 +- 20 files changed, 628 insertions(+), 546 deletions(-) delete mode 100644 assistants/prospector-assistant/assistant/agents/form_fill_agent/__init__.py delete mode 100644 assistants/prospector-assistant/assistant/agents/form_fill_agent/agent.py delete mode 100644 assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/_attachments.py delete mode 100644 assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/fill_form_step.py create mode 100644 assistants/prospector-assistant/assistant/agents/form_fill_extension/__init__.py rename assistants/prospector-assistant/assistant/agents/{form_fill_agent => form_fill_extension}/config.py (73%) create mode 100644 assistants/prospector-assistant/assistant/agents/form_fill_extension/extension.py rename assistants/prospector-assistant/assistant/agents/{form_fill_agent => form_fill_extension}/inspector.py (90%) rename assistants/prospector-assistant/assistant/agents/{form_fill_agent => form_fill_extension}/state.py (85%) rename assistants/prospector-assistant/assistant/agents/{form_fill_agent => form_fill_extension}/steps/__init__.py (100%) rename assistants/prospector-assistant/assistant/agents/{form_fill_agent => form_fill_extension}/steps/_guided_conversation.py (100%) create mode 100644 assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/_llm.py rename assistants/prospector-assistant/assistant/agents/{form_fill_agent => form_fill_extension}/steps/acquire_form_step.py (79%) rename assistants/prospector-assistant/assistant/agents/{form_fill_agent => form_fill_extension}/steps/extract_form_fields_step.py (66%) create mode 100644 assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/fill_form_step.py rename assistants/prospector-assistant/assistant/agents/{form_fill_agent => form_fill_extension}/steps/types.py (87%) diff --git a/assistants/explorer-assistant/assistant/chat.py b/assistants/explorer-assistant/assistant/chat.py index 363ea4e0..03bf4412 100644 --- a/assistants/explorer-assistant/assistant/chat.py +++ b/assistants/explorer-assistant/assistant/chat.py @@ -334,7 +334,7 @@ async def respond_to_conversation( method_metadata_key: { "request": { "model": config.request_config.openai_model, - "messages": openai_client.truncate_messages_for_logging(completion_messages), + "messages": completion_messages, "max_tokens": config.request_config.response_tokens, }, "response": completion.model_dump() if completion else "[no response from openai]", @@ -358,7 +358,7 @@ async def respond_to_conversation( method_metadata_key: { "request": { "model": config.request_config.openai_model, - "messages": openai_client.truncate_messages_for_logging(completion_messages), + "messages": completion_messages, }, "error": str(e), }, diff --git a/assistants/prospector-assistant/assistant/agents/document_agent.py b/assistants/prospector-assistant/assistant/agents/document_agent.py index bf7b7beb..2629ecba 100644 --- a/assistants/prospector-assistant/assistant/agents/document_agent.py +++ b/assistants/prospector-assistant/assistant/agents/document_agent.py @@ -919,7 +919,7 @@ def _on_success_metadata_update( f"{method_metadata_key}": { "request": { "model": config.request_config.openai_model, - "messages": openai_client.truncate_messages_for_logging(chat_completion_messages), + "messages": chat_completion_messages, "max_tokens": config.request_config.response_tokens, }, "response": completion.model_dump() if completion else "[no response from openai]", @@ -943,7 +943,7 @@ def _on_error_metadata_update( f"{method_metadata_key}": { "request": { "model": config.request_config.openai_model, - "messages": openai_client.truncate_messages_for_logging(chat_completion_messages), + "messages": chat_completion_messages, }, "error": str(e), }, diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_agent/__init__.py b/assistants/prospector-assistant/assistant/agents/form_fill_agent/__init__.py deleted file mode 100644 index b28e952c..00000000 --- a/assistants/prospector-assistant/assistant/agents/form_fill_agent/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -from .agent import execute, extend -from .config import FormFillAgentConfig -from .steps.types import LLMConfig - -__all__ = [ - "execute", - "extend", - "LLMConfig", - "FormFillAgentConfig", -] diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_agent/agent.py b/assistants/prospector-assistant/assistant/agents/form_fill_agent/agent.py deleted file mode 100644 index 8a350e41..00000000 --- a/assistants/prospector-assistant/assistant/agents/form_fill_agent/agent.py +++ /dev/null @@ -1,158 +0,0 @@ -import logging -from typing import Awaitable, Callable, Sequence - -from openai.types.chat import ChatCompletionMessageParam -from semantic_workbench_api_model.workbench_model import MessageType, NewConversationMessage -from semantic_workbench_assistant.assistant_app.context import ConversationContext -from semantic_workbench_assistant.assistant_app.protocol import AssistantAppProtocol - -from . import state -from .config import FormFillAgentConfig -from .steps import acquire_form_step, extract_form_fields_step, fill_form_step -from .steps.types import ConfigT, Context, IncompleteErrorResult, IncompleteResult, LLMConfig - -logger = logging.getLogger(__name__) - - -async def execute( - context: ConversationContext, - llm_config: LLMConfig, - config: FormFillAgentConfig, - latest_user_message: str | None, - get_attachment_messages: Callable[[Sequence[str]], Awaitable[Sequence[ChatCompletionMessageParam]]], -) -> None: - user_messages = [latest_user_message] - - def build_step_context(config: ConfigT) -> Context[ConfigT]: - return Context( - context=context, llm_config=llm_config, config=config, get_attachment_messages=get_attachment_messages - ) - - async with state.agent_state(context) as agent_state: - while True: - logger.info("form-fill-agent execute loop; mode: %s", agent_state.mode) - - match agent_state.mode: - case state.FormFillAgentMode.acquire_form_step: - result = await acquire_form_step.execute( - step_context=build_step_context(config.acquire_form_config), - latest_user_message=user_messages.pop() if user_messages else None, - ) - - match result: - case acquire_form_step.CompleteResult(): - await _send_message(context, result.ai_message, result.debug) - - agent_state.form_filename = result.filename - agent_state.mode = state.FormFillAgentMode.extract_form_fields_step - - case IncompleteResult() | IncompleteErrorResult(): - await _handle_incomplete_results(context, result) - return - - case _: - raise ValueError(f"Unexpected result: {result}") - - case state.FormFillAgentMode.extract_form_fields_step: - result = await extract_form_fields_step.execute( - step_context=build_step_context(config.extract_form_fields_config), - filename=agent_state.form_filename, - ) - - match result: - case extract_form_fields_step.CompleteResult(): - await _send_message(context, result.ai_message, result.debug) - - agent_state.extracted_form_fields = result.extracted_form_fields - agent_state.mode = state.FormFillAgentMode.fill_form_step - - case IncompleteResult() | IncompleteErrorResult(): - await _handle_incomplete_results(context, result) - return - - case _: - raise ValueError(f"Unexpected result: {result}") - - case state.FormFillAgentMode.fill_form_step: - result = await fill_form_step.execute( - step_context=build_step_context(config.fill_form_config), - latest_user_message=user_messages.pop() if user_messages else None, - form_fields=agent_state.extracted_form_fields, - ) - - match result: - case fill_form_step.CompleteResult(): - await _send_message(context, result.ai_message, result.debug) - - agent_state.fill_form_gc_artifact = result.artifact - agent_state.mode = state.FormFillAgentMode.generate_filled_form_step - - case IncompleteResult() | IncompleteErrorResult(): - await _handle_incomplete_results(context, result) - return - - case _: - raise ValueError(f"Unexpected result: {result}") - - case state.FormFillAgentMode.generate_filled_form_step: - await _send_message( - context, "I'd love to generate the filled-out form now, but it's not yet implemented. :)", {} - ) - return - - case state.FormFillAgentMode.end_conversation: - await _send_message(context, "Conversation has ended.", {}) - return - - case _: - raise ValueError(f"Unexpected mode: {state.mode}") - - -async def _handle_incomplete_results( - context: ConversationContext, result: IncompleteErrorResult | IncompleteResult -) -> None: - match result: - case IncompleteResult(): - await _send_message(context, result.ai_message, result.debug) - - case IncompleteErrorResult(): - await _send_error_message(context, result.error_message, result.debug) - - case _: - raise ValueError(f"Unexpected incomplete result: {result}") - - -async def _send_message(context: ConversationContext, message: str, debug: dict) -> None: - if not message: - return - - await context.send_messages( - NewConversationMessage( - content=message, - message_type=MessageType.chat, - metadata={"debug": debug}, - ) - ) - - -async def _send_error_message(context: ConversationContext, message: str, debug: dict) -> None: - await context.send_messages( - NewConversationMessage( - content=message, - message_type=MessageType.notice, - metadata={"debug": debug}, - ) - ) - - -def extend(app: AssistantAppProtocol) -> None: - """ - Extend the assistant app with the form-fill agent inspectors. - """ - - # for agent level state - app.add_inspector_state_provider(state.inspector.state_id, state.inspector) - - # for step level states - acquire_form_step.extend(app) - fill_form_step.extend(app) diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/_attachments.py b/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/_attachments.py deleted file mode 100644 index 0f6424e8..00000000 --- a/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/_attachments.py +++ /dev/null @@ -1,57 +0,0 @@ -""" -Utility functions for handling attachments in chat messages. -""" - -from typing import Awaitable, Callable, Sequence - -from openai.types.chat import ChatCompletionMessageParam -from semantic_workbench_assistant.assistant_app.context import ConversationContext - -from .. import state - - -async def message_with_recent_attachments( - context: ConversationContext, - latest_user_message: str | None, - get_attachment_messages: Callable[[Sequence[str]], Awaitable[Sequence[ChatCompletionMessageParam]]], -) -> str: - files = await context.get_files() - - new_filenames = set() - - async with state.agent_state(context) as agent_state: - max_timestamp = agent_state.most_recent_attachment_timestamp - for file in files.files: - if file.updated_datetime.timestamp() <= agent_state.most_recent_attachment_timestamp: - continue - - max_timestamp = max(file.updated_datetime.timestamp(), max_timestamp) - new_filenames.add(file.filename) - - agent_state.most_recent_attachment_timestamp = max_timestamp - - attachment_messages = await get_attachment_messages(list(new_filenames)) - - return "\n\n".join( - ( - latest_user_message or "", - *( - str(attachment.get("content")) - for attachment in attachment_messages - if "" in str(attachment.get("content", "")) - ), - ), - ) - - -async def attachment_for_filename( - filename: str, get_attachment_messages: Callable[[Sequence[str]], Awaitable[Sequence[ChatCompletionMessageParam]]] -) -> str: - attachment_messages = await get_attachment_messages([filename]) - return "\n\n".join( - ( - str(attachment.get("content")) - for attachment in attachment_messages - if "" in str(attachment.get("content", "")) - ) - ) diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/fill_form_step.py b/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/fill_form_step.py deleted file mode 100644 index e79678cb..00000000 --- a/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/fill_form_step.py +++ /dev/null @@ -1,149 +0,0 @@ -import logging -from dataclasses import dataclass -from pathlib import Path -from typing import Any, Literal - -from guided_conversation.utils.resources import ResourceConstraintMode, ResourceConstraintUnit -from pydantic import Field, create_model -from semantic_workbench_assistant.assistant_app.context import ConversationContext -from semantic_workbench_assistant.assistant_app.protocol import AssistantAppProtocol - -from .. import state -from ..inspector import FileStateInspector -from . import _attachments, _guided_conversation -from .types import ( - Context, - GuidedConversationDefinition, - IncompleteErrorResult, - IncompleteResult, - ResourceConstraintDefinition, - Result, -) - -logger = logging.getLogger(__name__) - - -def extend(app: AssistantAppProtocol) -> None: - app.add_inspector_state_provider(_inspector.state_id, _inspector) - - -@dataclass -class CompleteResult(Result): - ai_message: str - artifact: dict - - -async def execute( - step_context: Context[_guided_conversation.GuidedConversationDefinition], - latest_user_message: str | None, - form_fields: list[state.FormField], -) -> IncompleteResult | IncompleteErrorResult | CompleteResult: - """ - Step: fill out the form with the user - Approach: Guided conversation - """ - message_with_attachments = await _attachments.message_with_recent_attachments( - step_context.context, latest_user_message, step_context.get_attachment_messages - ) - - artifact_type = _form_fields_to_artifact(form_fields) - - definition = step_context.config.model_copy() - definition.resource_constraint.quantity = int(len(form_fields) * 1.5) - async with _guided_conversation.engine( - definition=definition, - artifact_type=artifact_type, - state_file_path=_get_state_file_path(step_context.context), - openai_client=step_context.llm_config.openai_client_factory(), - openai_model=step_context.llm_config.openai_model, - context=step_context.context, - state_id=_inspector.state_id, - ) as gce: - try: - result = await gce.step_conversation(message_with_attachments) - except Exception as e: - logger.exception("failed to execute guided conversation") - return IncompleteErrorResult( - error_message=f"Failed to execute guided conversation: {e}", - debug={"error": str(e)}, - ) - - debug = {"guided-conversation": gce.to_json()} - - logger.info("guided-conversation result: %s", result) - - fill_form_gc_artifact = gce.artifact.artifact.model_dump(mode="json") - logger.info("guided-conversation artifact: %s", gce.artifact) - - if result.is_conversation_over: - return CompleteResult( - ai_message="", - artifact=fill_form_gc_artifact, - debug=debug, - ) - - return IncompleteResult(ai_message=result.ai_message or "", debug=debug) - - -def _form_fields_to_artifact(form_fields: list[state.FormField]): - field_definitions: dict[str, tuple[Any, Any]] = {} - required_fields = [] - for field in form_fields: - if field.required: - required_fields.append(field.id) - - match field.type: - case "string": - field_definitions[field.id] = (str, Field(title=field.name, description=field.description)) - - case "bool": - field_definitions[field.id] = (bool, Field(title=field.name, description=field.description)) - - case "multiple_choice": - field_definitions[field.id] = ( - Literal[tuple(field.options)], - Field(title=field.name, description=field.description), - ) - - return create_model( - "FilledFormArtifact", - **field_definitions, # type: ignore - ) # type: ignore - - -def _get_state_file_path(context: ConversationContext) -> Path: - return _guided_conversation.path_for_state(context, "fill_form") - - -_inspector = FileStateInspector( - display_name="Fill Form Guided Conversation State", - file_path_source=_get_state_file_path, - state_id="fill_form", -) - - -definition = GuidedConversationDefinition( - rules=[ - "For fields that are not in the provided files, collect the data from the user through conversation.", - "When providing options for a multiple choice field, provide the options in a numbered-list, so the user can refer to them by number.", - "When listing anything other than options, like document types, provide them in a bulleted list for improved readability.", - "When updating the agenda, the data-collection for each form field must be in a separate step.", - "When asking for data to fill the form, always ask for a single piece of information at a time. Never ask for multiple pieces of information in a single prompt, ex: 'Please provide field Y, and additionally, field X'.", - "Terminate conversation if inappropriate content is requested.", - ], - conversation_flow=""" -1. Inform the user that we've received the form and determined the fields in the form. -2. Inform the user that our goal is help them fill out the form. -3. Ask the user to provide one or more files that might contain data relevant to fill out the form. The files can be PDF, TXT, or DOCX. -4. When asking for files, suggest types of documents that might contain the data. -5. For each field in the form, check if the data is available in the provided files. -6. If the data is not available in the files, ask the user for the data. -8. When the form is filled out, inform the user that you will now generate a document containing the filled form. -""", - context="", - resource_constraint=ResourceConstraintDefinition( - quantity=15, - unit=ResourceConstraintUnit.TURNS, - mode=ResourceConstraintMode.MAXIMUM, - ), -) diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_extension/__init__.py b/assistants/prospector-assistant/assistant/agents/form_fill_extension/__init__.py new file mode 100644 index 00000000..bdf69b23 --- /dev/null +++ b/assistants/prospector-assistant/assistant/agents/form_fill_extension/__init__.py @@ -0,0 +1,9 @@ +from .config import FormFillConfig +from .extension import FormFillExtension +from .steps.types import LLMConfig + +__all__ = [ + "FormFillExtension", + "LLMConfig", + "FormFillConfig", +] diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_agent/config.py b/assistants/prospector-assistant/assistant/agents/form_fill_extension/config.py similarity index 73% rename from assistants/prospector-assistant/assistant/agents/form_fill_agent/config.py rename to assistants/prospector-assistant/assistant/agents/form_fill_extension/config.py index 8e209b2e..c31c87b6 100644 --- a/assistants/prospector-assistant/assistant/agents/form_fill_agent/config.py +++ b/assistants/prospector-assistant/assistant/agents/form_fill_extension/config.py @@ -2,14 +2,14 @@ from pydantic import BaseModel, Field -from .steps import acquire_form_step, extract_form_fields_step, fill_form_step, types +from .steps import acquire_form_step, extract_form_fields_step, fill_form_step -class FormFillAgentConfig(BaseModel): +class FormFillConfig(BaseModel): acquire_form_config: Annotated[ - types.GuidedConversationDefinition, + acquire_form_step.AcquireFormConfig, Field(title="Form Acquisition", description="Guided conversation for acquiring a form from the user."), - ] = acquire_form_step.definition.model_copy() + ] = acquire_form_step.AcquireFormConfig() extract_form_fields_config: Annotated[ extract_form_fields_step.ExtractFormFieldsConfig, @@ -17,6 +17,6 @@ class FormFillAgentConfig(BaseModel): ] = extract_form_fields_step.ExtractFormFieldsConfig() fill_form_config: Annotated[ - types.GuidedConversationDefinition, + fill_form_step.FillFormConfig, Field(title="Fill Form", description="Guided conversation for filling out the form."), - ] = fill_form_step.definition.model_copy() + ] = fill_form_step.FillFormConfig() diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_extension/extension.py b/assistants/prospector-assistant/assistant/agents/form_fill_extension/extension.py new file mode 100644 index 00000000..9c283258 --- /dev/null +++ b/assistants/prospector-assistant/assistant/agents/form_fill_extension/extension.py @@ -0,0 +1,163 @@ +import logging +from typing import AsyncIterable, Awaitable, Callable, Sequence + +from semantic_workbench_api_model.workbench_model import MessageType, NewConversationMessage +from semantic_workbench_assistant.assistant_app.context import ConversationContext +from semantic_workbench_assistant.assistant_app.protocol import AssistantAppProtocol + +from . import state +from .config import FormFillConfig +from .steps import acquire_form_step, extract_form_fields_step, fill_form_step +from .steps.types import ConfigT, Context, IncompleteErrorResult, IncompleteResult, LLMConfig, UserAttachment, UserInput + +logger = logging.getLogger(__name__) + + +class FormFillExtension: + def __init__(self, assistant_app: AssistantAppProtocol) -> None: + """ + Extend the assistant app with the form-fill agent inspectors. + """ + + # for agent level state + assistant_app.add_inspector_state_provider(state.inspector.state_id, state.inspector) + + # for step level states + acquire_form_step.extend(assistant_app) + fill_form_step.extend(assistant_app) + + async def execute( + self, + context: ConversationContext, + llm_config: LLMConfig, + config: FormFillConfig, + latest_user_message: str | None, + latest_attachment_filenames: Sequence[str], + get_attachment_content: Callable[[str], Awaitable[str]], + ) -> None: + user_messages = [latest_user_message] + + async def latest_attachments() -> AsyncIterable[UserAttachment]: + for filename in latest_attachment_filenames: + content = await get_attachment_content(filename) + yield UserAttachment(filename=filename, content=content) + + def build_step_context(config: ConfigT) -> Context[ConfigT]: + return Context( + context=context, + llm_config=llm_config, + config=config, + latest_user_input=UserInput( + message=user_messages.pop() if user_messages else None, + attachments=latest_attachments(), + ), + ) + + async with state.agent_state(context) as agent_state: + while True: + logger.info("form-fill-agent execute loop; mode: %s", agent_state.mode) + + match agent_state.mode: + case state.FormFillAgentMode.acquire_form_step: + result = await acquire_form_step.execute( + step_context=build_step_context(config.acquire_form_config), + ) + + match result: + case acquire_form_step.CompleteResult(): + await _send_message(context, result.ai_message, result.debug) + + agent_state.form_filename = result.filename + agent_state.mode = state.FormFillAgentMode.extract_form_fields + + continue + + case _: + await _handle_incomplete_result(context, result) + return + + case state.FormFillAgentMode.extract_form_fields: + file_content = await get_attachment_content(agent_state.form_filename) + result = await extract_form_fields_step.execute( + step_context=build_step_context(config.extract_form_fields_config), + file_content=file_content, + ) + + match result: + case extract_form_fields_step.CompleteResult(): + await _send_message(context, result.ai_message, result.debug) + + agent_state.extracted_form_fields = result.extracted_form_fields + agent_state.mode = state.FormFillAgentMode.fill_form_step + + continue + + case _: + await _handle_incomplete_result(context, result) + return + + case state.FormFillAgentMode.fill_form_step: + result = await fill_form_step.execute( + step_context=build_step_context(config.fill_form_config), + form_filename=agent_state.form_filename, + form_fields=agent_state.extracted_form_fields, + ) + + match result: + case fill_form_step.CompleteResult(): + await _send_message(context, result.ai_message, result.debug) + + agent_state.fill_form_gc_artifact = result.artifact + agent_state.mode = state.FormFillAgentMode.generate_filled_form_step + + continue + + case _: + await _handle_incomplete_result(context, result) + return + + case state.FormFillAgentMode.generate_filled_form_step: + await _send_message( + context, + "I'd love to generate the filled-out form now, but it's not yet implemented. :)", + {}, + ) + return + + case _: + raise ValueError(f"Unexpected mode: {agent_state.mode}") + + +async def _handle_incomplete_result(context: ConversationContext, result: IncompleteResult) -> None: + match result: + case IncompleteResult(): + await _send_message(context, result.message, result.debug) + + case IncompleteErrorResult(): + await _send_error_message(context, result.error_message, result.debug) + + case _: + raise ValueError(f"Unexpected incomplete result type: {result}") + + +async def _send_message(context: ConversationContext, message: str, debug: dict) -> None: + if not message: + return + + await context.send_messages( + NewConversationMessage( + content=message, + message_type=MessageType.chat, + debug_data=debug, + ) + ) + + +async def _send_error_message(context: ConversationContext, message: str, debug: dict) -> None: + await context.send_messages( + NewConversationMessage( + content=message, + message_type=MessageType.notice, + debug_data=debug, + ) + ) diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_agent/inspector.py b/assistants/prospector-assistant/assistant/agents/form_fill_extension/inspector.py similarity index 90% rename from assistants/prospector-assistant/assistant/agents/form_fill_agent/inspector.py rename to assistants/prospector-assistant/assistant/agents/form_fill_extension/inspector.py index e35ce6d5..a8b0192f 100644 --- a/assistants/prospector-assistant/assistant/agents/form_fill_agent/inspector.py +++ b/assistants/prospector-assistant/assistant/agents/form_fill_extension/inspector.py @@ -1,5 +1,6 @@ import contextlib import json +from hashlib import md5 from pathlib import Path from typing import Callable @@ -20,10 +21,11 @@ def __init__( self, display_name: str, file_path_source: Callable[[ConversationContext], Path], - state_id: str, description: str = "", ) -> None: - self._state_id = state_id + self._state_id = md5( + (type(self).__name__ + "_" + display_name).encode("utf-8"), usedforsecurity=False + ).hexdigest() self._display_name = display_name self._file_path_source = file_path_source self._description = description diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_agent/state.py b/assistants/prospector-assistant/assistant/agents/form_fill_extension/state.py similarity index 85% rename from assistants/prospector-assistant/assistant/agents/form_fill_agent/state.py rename to assistants/prospector-assistant/assistant/agents/form_fill_extension/state.py index a31dcee6..3d49eae1 100644 --- a/assistants/prospector-assistant/assistant/agents/form_fill_agent/state.py +++ b/assistants/prospector-assistant/assistant/agents/form_fill_extension/state.py @@ -12,7 +12,7 @@ class FormField(BaseModel): - id: str = Field(description="The unique identifier of the field.") + id: str = Field(description="The descriptive, unique identifier of the field as a snake_case_english_string.") name: str = Field(description="The name of the field.") description: str = Field(description="The description of the field.") type: Literal["string", "bool", "multiple_choice"] = Field(description="The type of the field.") @@ -22,16 +22,13 @@ class FormField(BaseModel): class FormFillAgentMode(StrEnum): acquire_form_step = "acquire_form" - extract_form_fields_step = "extract_form_fields" + extract_form_fields = "extract_form_fields" fill_form_step = "fill_form" generate_filled_form_step = "generate_filled_form" - end_conversation = "end_conversation" - class FormFillAgentState(BaseModel): mode: FormFillAgentMode = FormFillAgentMode.acquire_form_step - most_recent_attachment_timestamp: float = 0 form_filename: str = "" extracted_form_fields: list[FormField] = [] fill_form_gc_artifact: dict | None = None @@ -63,6 +60,4 @@ async def agent_state(context: ConversationContext) -> AsyncIterator[FormFillAge current_state.set(None) -inspector = FileStateInspector( - display_name="Form Fill Agent State", file_path_source=path_for_state, state_id="form_fill_agent" -) +inspector = FileStateInspector(display_name="FormFill Agent", file_path_source=path_for_state) diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/__init__.py b/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/__init__.py similarity index 100% rename from assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/__init__.py rename to assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/__init__.py diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/_guided_conversation.py b/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/_guided_conversation.py similarity index 100% rename from assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/_guided_conversation.py rename to assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/_guided_conversation.py diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/_llm.py b/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/_llm.py new file mode 100644 index 00000000..740dbc6d --- /dev/null +++ b/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/_llm.py @@ -0,0 +1,47 @@ +from typing import Any, TypeVar + +from openai.types.chat import ChatCompletionMessageParam +from pydantic import BaseModel + +from .types import LLMConfig + + +class NoResponseChoicesError(Exception): + pass + + +class NoParsedMessageError(Exception): + pass + + +ResponseModelT = TypeVar("ResponseModelT", bound=BaseModel) + + +async def structured_completion( + llm_config: LLMConfig, messages: list[ChatCompletionMessageParam], response_model: type[ResponseModelT] +) -> tuple[ResponseModelT, dict[str, Any]]: + async with llm_config.openai_client_factory() as client: + response = await client.beta.chat.completions.parse( + messages=messages, + model=llm_config.openai_model, + response_format=response_model, + max_tokens=llm_config.max_response_tokens, + ) + + if not response.choices: + raise NoResponseChoicesError() + + if not response.choices[0].message.parsed: + raise NoParsedMessageError() + + metadata = { + "request": { + "model": llm_config.openai_model, + "messages": messages, + "max_tokens": llm_config.max_response_tokens, + "response_format": response_model.model_json_schema(), + }, + "response": response.model_dump(), + } + + return response.choices[0].message.parsed, metadata diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/acquire_form_step.py b/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/acquire_form_step.py similarity index 79% rename from assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/acquire_form_step.py rename to assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/acquire_form_step.py index 10de7789..c2c2fbc7 100644 --- a/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/acquire_form_step.py +++ b/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/acquire_form_step.py @@ -1,6 +1,7 @@ import logging from dataclasses import dataclass from pathlib import Path +from textwrap import dedent from guided_conversation.utils.resources import ResourceConstraintMode, ResourceConstraintUnit from pydantic import BaseModel, Field @@ -8,7 +9,7 @@ from semantic_workbench_assistant.assistant_app.protocol import AssistantAppProtocol from ..inspector import FileStateInspector -from . import _attachments, _guided_conversation +from . import _guided_conversation from .types import ( Context, GuidedConversationDefinition, @@ -16,6 +17,7 @@ IncompleteResult, ResourceConstraintDefinition, Result, + UserInput, ) logger = logging.getLogger(__name__) @@ -25,6 +27,38 @@ def extend(app: AssistantAppProtocol) -> None: app.add_inspector_state_provider(_inspector.state_id, _inspector) +class FormArtifact(BaseModel): + title: str = Field(description="The title of the form.", default="") + filename: str = Field(description="The filename of the form.", default="") + + +definition = GuidedConversationDefinition( + rules=[ + "DO NOT suggest forms or create a form for the user.", + "Politely request another file if the provided file is not a form.", + "Terminate conversation if inappropriate content is requested.", + ], + conversation_flow=dedent(""" + 1. Inform the user that our goal is to help the user fill out a form. + 2. Ask the user to provide a file that contains a form. The file can be PDF, TXT, or DOCX. + 3. When you receive a file, determine if the file looks to be a form. + 4. If the file is not a form, inform the user that the file is not a form. Ask them to provide a different file. + 5. If the form is a file, update the artifcat with the title and filename of the form. + 6. Inform the user that you will now extract the form fields, so that you can assist them in filling it out. + """).strip(), + context="", + resource_constraint=ResourceConstraintDefinition( + quantity=5, + unit=ResourceConstraintUnit.MINUTES, + mode=ResourceConstraintMode.MAXIMUM, + ), +) + + +class AcquireFormConfig(BaseModel): + definition: GuidedConversationDefinition = definition + + @dataclass class CompleteResult(Result): ai_message: str @@ -32,19 +66,16 @@ class CompleteResult(Result): async def execute( - step_context: Context[GuidedConversationDefinition], - latest_user_message: str | None, + step_context: Context[AcquireFormConfig], ) -> IncompleteResult | IncompleteErrorResult | CompleteResult: """ Step: acquire a form from the user Approach: Guided conversation """ - message_with_attachments = await _attachments.message_with_recent_attachments( - step_context.context, latest_user_message, step_context.get_attachment_messages - ) + message_with_attachments = await input_to_message(step_context.latest_user_input) async with _guided_conversation.engine( - definition=step_context.config, + definition=step_context.config.definition, artifact_type=FormArtifact, state_file_path=_get_state_file_path(step_context.context), openai_client=step_context.llm_config.openai_client_factory(), @@ -57,7 +88,7 @@ async def execute( except Exception as e: logger.exception("failed to execute guided conversation") return IncompleteErrorResult( - error_message=f"Failed to execute guided conversation: {e}", + message=f"Failed to execute guided conversation: {e}", debug={"error": str(e)}, ) @@ -77,7 +108,7 @@ async def execute( debug=debug, ) - return IncompleteResult(ai_message=result.ai_message or "", debug=debug) + return IncompleteResult(message=result.ai_message or "", debug=debug) def _get_state_file_path(context: ConversationContext) -> Path: @@ -85,37 +116,22 @@ def _get_state_file_path(context: ConversationContext) -> Path: _inspector = FileStateInspector( - display_name="Acquire Form Guided Conversation State", + display_name="Acquire-Form Guided-Conversation", file_path_source=_get_state_file_path, - state_id="acquire_form", ) -class FormArtifact(BaseModel): - title: str = Field(description="The title of the form.", default="") - filename: str = Field(description="The filename of the form.", default="") +async def input_to_message(input: UserInput) -> str | None: + attachments = [] + async for attachment in input.attachments: + attachments.append(attachment.content) + if not attachments: + return input.message -definition = GuidedConversationDefinition( - rules=[ - "DO NOT suggest forms or create a form for the user.", - "Politely request another file if the provided file is not a form.", - "Terminate conversation if inappropriate content is requested.", - ], - conversation_flow=( - """ - 1. Inform the user that our goal is to help the user fill out a form. - 2. Ask the user to provide a file that contains a form. The file can be PDF, TXT, or DOCX. - 3. When you receive a file, determine if the file looks to be a form. - 4. If the file is not a form, inform the user that the file is not a form. Ask them to provide a different file. - 5. If the form is a file, update the artifcat with the title and filename of the form. - 6. Inform the user that you will now extract the form fields, so that you can assist them in filling it out. - """ - ), - context="", - resource_constraint=ResourceConstraintDefinition( - quantity=5, - unit=ResourceConstraintUnit.MINUTES, - mode=ResourceConstraintMode.MAXIMUM, - ), -) + return "\n\n".join( + ( + input.message or "", + *attachments, + ), + ) diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/extract_form_fields_step.py b/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/extract_form_fields_step.py similarity index 66% rename from assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/extract_form_fields_step.py rename to assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/extract_form_fields_step.py index 44cc3789..64d0ce39 100644 --- a/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/extract_form_fields_step.py +++ b/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/extract_form_fields_step.py @@ -2,12 +2,11 @@ from dataclasses import dataclass from typing import Annotated, Any -import openai_client from openai.types.chat import ChatCompletionMessageParam from pydantic import BaseModel, Field from .. import state -from . import _attachments +from . import _llm from .types import Context, IncompleteErrorResult, IncompleteResult, LLMConfig, Result logger = logging.getLogger(__name__) @@ -31,14 +30,13 @@ class CompleteResult(Result): async def execute( step_context: Context[ExtractFormFieldsConfig], - filename: str, + file_content: str, ) -> IncompleteResult | IncompleteErrorResult | CompleteResult: """ Step: extract form fields from the form file content Approach: Chat completion with LLM """ - file_content = await _attachments.attachment_for_filename(filename, step_context.get_attachment_messages) async with step_context.context.set_status("inspecting form ..."): try: extracted_form_fields, metadata = await _extract( @@ -50,13 +48,13 @@ async def execute( except Exception as e: logger.exception("failed to extract form fields") return IncompleteErrorResult( - error_message=f"Failed to extract form fields: {e}", + message=f"Failed to extract form fields: {e}", debug={"error": str(e)}, ) if extracted_form_fields.error_message: return IncompleteResult( - ai_message=extracted_form_fields.error_message, + message=extracted_form_fields.error_message, debug=metadata, ) @@ -72,14 +70,6 @@ class FormFields(BaseModel): fields: list[state.FormField] = Field(description="The fields in the form.") -class NoResponseChoicesError(Exception): - pass - - -class NoParsedMessageError(Exception): - pass - - async def _extract( llm_config: LLMConfig, config: ExtractFormFieldsConfig, form_content: str ) -> tuple[FormFields, dict[str, Any]]: @@ -94,28 +84,8 @@ async def _extract( }, ] - async with llm_config.openai_client_factory() as client: - response = await client.beta.chat.completions.parse( - messages=messages, - model=llm_config.openai_model, - response_format=FormFields, - max_tokens=llm_config.max_response_tokens, - ) - - if not response.choices: - raise NoResponseChoicesError() - - if not response.choices[0].message.parsed: - raise NoParsedMessageError() - - metadata = { - "request": { - "model": llm_config.openai_model, - "messages": openai_client.truncate_messages_for_logging(messages), - "max_tokens": llm_config.max_response_tokens, - "response_format": FormFields.model_json_schema(), - }, - "response": response.model_dump(), - } - - return response.choices[0].message.parsed, metadata + return await _llm.structured_completion( + llm_config=llm_config, + messages=messages, + response_model=FormFields, + ) diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/fill_form_step.py b/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/fill_form_step.py new file mode 100644 index 00000000..5297414e --- /dev/null +++ b/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/fill_form_step.py @@ -0,0 +1,237 @@ +import logging +from dataclasses import dataclass +from pathlib import Path +from textwrap import dedent +from typing import Annotated, Any, Literal + +from guided_conversation.utils.resources import ResourceConstraintMode, ResourceConstraintUnit +from openai.types.chat import ChatCompletionMessageParam +from pydantic import BaseModel, Field, create_model +from semantic_workbench_assistant.assistant_app.context import ConversationContext +from semantic_workbench_assistant.assistant_app.protocol import AssistantAppProtocol + +from .. import state +from ..inspector import FileStateInspector +from . import _guided_conversation, _llm +from .types import ( + Context, + GuidedConversationDefinition, + IncompleteErrorResult, + IncompleteResult, + LLMConfig, + ResourceConstraintDefinition, + Result, +) + +logger = logging.getLogger(__name__) + + +def extend(app: AssistantAppProtocol) -> None: + app.add_inspector_state_provider(_inspector.state_id, _inspector) + + +definition = GuidedConversationDefinition( + rules=[ + "For fields that are not in the provided files, collect the data from the user through conversation.", + "When providing options for a multiple choice field, provide the options in a numbered-list, so the user can refer to them by number.", + "When listing anything other than options, like document types, provide them in a bulleted list for improved readability.", + "When updating the agenda, the data-collection for each form field must be in a separate step.", + "When asking for data to fill the form, always ask for a single piece of information at a time. Never ask for multiple pieces of information in a single prompt, ex: 'Please provide field Y, and additionally, field X'.", + "Terminate conversation if inappropriate content is requested.", + ], + conversation_flow=dedent(""" + 1. Inform the user that we've received the form and determined the fields in the form. + 2. Inform the user that our goal is help them fill out the form. + 3. Ask the user to provide one or more files that might contain data relevant to fill out the form. The files can be PDF, TXT, or DOCX. + 4. When asking for files, suggest types of documents that might contain the data. + 5. For each field in the form, check if the data is available in the provided files. + 6. If the data is not available in the files, ask the user for the data. + 7. When the form is filled out, inform the user that you will now generate a document containing the filled form. + """).strip(), + context="", + resource_constraint=ResourceConstraintDefinition( + quantity=15, + unit=ResourceConstraintUnit.TURNS, + mode=ResourceConstraintMode.MAXIMUM, + ), +) + + +class ExtractCandidateFieldValuesConfig(BaseModel): + instruction: Annotated[ + str, + Field( + title="Instruction", + description="The instruction for extracting candidate form-field values from an uploaded file", + ), + ] = dedent(""" + Given the field definitions below, extract candidate values for these fields from the user provided + attachment. + + Only include values that are in the provided attachment. + It is possible that there are multiple candidates for a single field, in which case you should provide + all the candidates and an explanation for each candidate. + + Field definitions: + {{form_fields}} + """) + + +class FillFormConfig(BaseModel): + extract_config: ExtractCandidateFieldValuesConfig = ExtractCandidateFieldValuesConfig() + definition: GuidedConversationDefinition = definition + + +class FieldValueCandidate(BaseModel): + field_id: str = Field(description="The ID of the field that the value is a candidate for.") + value: str = Field(description="The value from the document for this field.") + explanation: str = Field(description="The explanation of why this value is a candidate for the field.") + + +class FieldValueCandidates(BaseModel): + response: str = Field(description="The natural language response to send to the user.") + fields: list[FieldValueCandidate] = Field(description="The fields in the form.") + + +class FieldValueCandidatesFromDocument(BaseModel): + filename: str + candidates: FieldValueCandidates + + +@dataclass +class CompleteResult(Result): + ai_message: str + artifact: dict + + +async def execute( + step_context: Context[FillFormConfig], + form_filename: str, + form_fields: list[state.FormField], +) -> IncompleteResult | IncompleteErrorResult | CompleteResult: + """ + Step: fill out the form with the user + Approach: Guided conversation + """ + message = step_context.latest_user_input.message + debug = {"document-extractions": {}} + + async for attachment in step_context.latest_user_input.attachments: + if attachment.filename == form_filename: + continue + + candidate_values, metadata = await _extract( + llm_config=step_context.llm_config, + config=step_context.config.extract_config, + form_fields=form_fields, + document_content=attachment.content, + ) + message = f"{message}\n\n" if message else "" + message = f"{message}{candidate_values.response}\n\nFilename: {attachment.filename}" + for candidate in candidate_values.fields: + message += f"\nField id: {candidate.field_id}:\n Value: {candidate.value}\n Explanation: {candidate.explanation}" + + debug["document-extractions"][attachment.filename] = metadata + + artifact_type = _form_fields_to_artifact(form_fields) + + definition = step_context.config.definition.model_copy() + definition.resource_constraint.quantity = int(len(form_fields) * 1.5) + async with _guided_conversation.engine( + definition=definition, + artifact_type=artifact_type, + state_file_path=_get_state_file_path(step_context.context), + openai_client=step_context.llm_config.openai_client_factory(), + openai_model=step_context.llm_config.openai_model, + context=step_context.context, + state_id=_inspector.state_id, + ) as gce: + try: + result = await gce.step_conversation(message) + except Exception as e: + logger.exception("failed to execute guided conversation") + return IncompleteErrorResult( + message=f"Failed to execute guided conversation: {e}", + debug={"error": str(e)}, + ) + + debug["guided-conversation"] = gce.to_json() + + logger.info("guided-conversation result: %s", result) + + fill_form_gc_artifact = gce.artifact.artifact.model_dump(mode="json") + logger.info("guided-conversation artifact: %s", gce.artifact) + + if result.is_conversation_over: + return CompleteResult( + ai_message="", + artifact=fill_form_gc_artifact, + debug=debug, + ) + + return IncompleteResult(message=result.ai_message or "", debug=debug) + + +def _form_fields_to_artifact(form_fields: list[state.FormField]): + field_definitions: dict[str, tuple[Any, Any]] = {} + required_fields = [] + for field in form_fields: + if field.required: + required_fields.append(field.id) + + match field.type: + case "string": + field_definitions[field.id] = (str, Field(title=field.name, description=field.description)) + + case "bool": + field_definitions[field.id] = (bool, Field(title=field.name, description=field.description)) + + case "multiple_choice": + field_definitions[field.id] = ( + Literal[tuple(field.options)], + Field(title=field.name, description=field.description), + ) + + return create_model( + "FilledFormArtifact", + **field_definitions, # type: ignore + ) # type: ignore + + +def _get_state_file_path(context: ConversationContext) -> Path: + return _guided_conversation.path_for_state(context, "fill_form") + + +_inspector = FileStateInspector( + display_name="Fill-Form Guided-Conversation", + file_path_source=_get_state_file_path, +) + + +async def _extract( + llm_config: LLMConfig, + config: ExtractCandidateFieldValuesConfig, + form_fields: list[state.FormField], + document_content: str, +) -> tuple[FieldValueCandidates, dict[str, Any]]: + class _SerializationModel(BaseModel): + fields: list[state.FormField] + + messages: list[ChatCompletionMessageParam] = [ + { + "role": "system", + "content": config.instruction.replace( + "{{form_fields}}", _SerializationModel(fields=form_fields).model_dump_json(indent=4) + ), + }, + { + "role": "user", + "content": document_content, + }, + ] + + return await _llm.structured_completion( + llm_config=llm_config, + messages=messages, + response_model=FieldValueCandidates, + ) diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/types.py b/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/types.py similarity index 87% rename from assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/types.py rename to assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/types.py index f4cdb8a3..4e11aa71 100644 --- a/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/types.py +++ b/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/types.py @@ -1,9 +1,8 @@ from dataclasses import dataclass -from typing import Annotated, Awaitable, Callable, Generic, Sequence, TypeVar +from typing import Annotated, AsyncIterable, Callable, Generic, TypeVar from guided_conversation.utils.resources import ResourceConstraint, ResourceConstraintMode, ResourceConstraintUnit from openai import AsyncOpenAI -from openai.types.chat import ChatCompletionMessageParam from pydantic import BaseModel, ConfigDict, Field from semantic_workbench_assistant.assistant_app.context import ConversationContext @@ -18,12 +17,24 @@ class LLMConfig: ConfigT = TypeVar("ConfigT", bound=BaseModel) +@dataclass +class UserAttachment: + filename: str + content: str + + +@dataclass +class UserInput: + message: str | None + attachments: AsyncIterable[UserAttachment] + + @dataclass class Context(Generic[ConfigT]): context: ConversationContext llm_config: LLMConfig config: ConfigT - get_attachment_messages: Callable[[Sequence[str]], Awaitable[Sequence[ChatCompletionMessageParam]]] + latest_user_input: UserInput @dataclass @@ -33,12 +44,11 @@ class Result: @dataclass class IncompleteResult(Result): - ai_message: str + message: str @dataclass -class IncompleteErrorResult(Result): - error_message: str +class IncompleteErrorResult(IncompleteResult): ... class ResourceConstraintDefinition(BaseModel): diff --git a/assistants/prospector-assistant/assistant/chat.py b/assistants/prospector-assistant/assistant/chat.py index 4d26c45e..2e90cd1e 100644 --- a/assistants/prospector-assistant/assistant/chat.py +++ b/assistants/prospector-assistant/assistant/chat.py @@ -9,7 +9,8 @@ import logging import re import traceback -from typing import Any, Sequence +from contextlib import asynccontextmanager +from typing import Any, Awaitable, Callable import deepmerge import openai_client @@ -34,9 +35,9 @@ ) from . import legacy -from .agents import form_fill_agent from .agents.artifact_agent import Artifact, ArtifactAgent, ArtifactConversationInspectorStateProvider from .agents.document_agent import DocumentAgent +from .agents.form_fill_extension import FormFillExtension, LLMConfig from .config import AssistantConfigModel logger = logging.getLogger(__name__) @@ -78,9 +79,8 @@ async def content_evaluator_factory(context: ConversationContext) -> ContentSafe }, ) -form_fill_agent.extend(assistant) - attachments_extension = AttachmentsExtension(assistant) +form_fill_extension = FormFillExtension(assistant) # # create the FastAPI app instance @@ -150,7 +150,7 @@ async def on_chat_message_created( """ # update the participant status to indicate the assistant is thinking - async with context.set_status("thinking..."): + async with send_error_message_on_exception(context), context.set_status("thinking..."): config = await assistant_config.get(context.assistant) metadata: dict[str, Any] = {"debug": {"content_safety": event.data.get(content_safety.metadata_key, {})}} @@ -165,43 +165,12 @@ async def on_chat_message_created( is_doc_agent_running = await document_agent_respond_to_conversation(config, context, message, metadata) return - try: - await form_fill_agent.execute( - llm_config=form_fill_agent.LLMConfig( - openai_client_factory=lambda: openai_client.create_client(config.service_config), - openai_model=config.request_config.openai_model, - max_response_tokens=config.request_config.response_tokens, - ), - config=config.agents_config.form_fill_agent, - context=context, - latest_user_message=message.content, - get_attachment_messages=form_fill_agent_get_attachments(context, config), - ) - - except Exception as e: - await context.send_messages( - NewConversationMessage( - content=f"An error occurred: {e}", - message_type=MessageType.notice, - metadata={"debug": {"stack_trace": traceback.format_exc()}}, - ) - ) + await form_fill_execute(context, message) # # Prospector assistant response # await respond_to_conversation(context, config, message, metadata) -def form_fill_agent_get_attachments(context: ConversationContext, config: AssistantConfigModel): - async def get(filenames: Sequence[str]): - return await attachments_extension.get_completion_messages_for_attachments( - context, - config.agents_config.attachment_agent, - include_filenames=list(filenames), - ) - - return get - - background_tasks: set[asyncio.Task] = set() @@ -216,6 +185,10 @@ async def on_conversation_created(context: ConversationContext) -> None: if welcome_sent_before: return + task = asyncio.create_task(welcome_message(context)) + background_tasks.add(task) + task.add_done_callback(background_tasks.remove) + # send a welcome message to the conversation # welcome_message = config.welcome_message # await context.send_messages( @@ -226,36 +199,72 @@ async def on_conversation_created(context: ConversationContext) -> None: # ) # ) - task = asyncio.create_task(welcome_message(context)) - background_tasks.add(task) - task.add_done_callback(background_tasks.remove) - async def welcome_message(context: ConversationContext) -> None: + async with send_error_message_on_exception(context), context.set_status("thinking..."): + await form_fill_execute(context, None) + + +@asynccontextmanager +async def send_error_message_on_exception(context: ConversationContext): + try: + yield + except Exception as e: + await context.send_messages( + NewConversationMessage( + content=f"An error occurred: {e}", + message_type=MessageType.notice, + metadata={"debug": {"stack_trace": traceback.format_exc()}}, + ) + ) + + +# endregion + +# +# region Form fill extension helpers +# + + +async def form_fill_execute(context: ConversationContext, message: ConversationMessage | None) -> None: + """ + Execute the form fill agent to respond to the conversation message. + """ config = await assistant_config.get(context.assistant) + await form_fill_extension.execute( + llm_config=LLMConfig( + openai_client_factory=lambda: openai_client.create_client(config.service_config), + openai_model=config.request_config.openai_model, + max_response_tokens=config.request_config.response_tokens, + ), + config=config.agents_config.form_fill_agent, + context=context, + latest_user_message=message.content if message else None, + latest_attachment_filenames=message.filenames if message else [], + get_attachment_content=form_fill_extension_get_attachment(context, config), + ) - async with context.set_status("thinking..."): - try: - await form_fill_agent.execute( - llm_config=form_fill_agent.LLMConfig( - openai_client_factory=lambda: openai_client.create_client(config.service_config), - openai_model=config.request_config.openai_model, - max_response_tokens=config.request_config.response_tokens, - ), - config=config.agents_config.form_fill_agent, - context=context, - latest_user_message=None, - get_attachment_messages=form_fill_agent_get_attachments(context, config), - ) - except Exception as e: - await context.send_messages( - NewConversationMessage( - content=f"An error occurred: {e}", - message_type=MessageType.notice, - metadata={"debug": {"stack_trace": traceback.format_exc()}}, - ) - ) +def form_fill_extension_get_attachment( + context: ConversationContext, config: AssistantConfigModel +) -> Callable[[str], Awaitable[str]]: + """Helper function for the form_fill_extension to get the content of an attachment by filename.""" + + async def get(filename: str) -> str: + messages = await attachments_extension.get_completion_messages_for_attachments( + context, + config.agents_config.attachment_agent, + include_filenames=[filename], + ) + if not messages: + return "" + + # filter down to the messages that contain the attachment (ie. don't include the system messages) + return "\n\n".join( + (str(message.get("content")) for message in messages if "" in str(message.get("content"))) + ) + + return get # endregion @@ -487,7 +496,7 @@ class StructuredResponseFormat(BaseModel): method_metadata_key: { "request": { "model": config.request_config.openai_model, - "messages": openai_client.truncate_messages_for_logging(completion_messages), + "messages": completion_messages, "max_tokens": config.request_config.response_tokens, "response_format": StructuredResponseFormat.model_json_schema(), }, @@ -510,7 +519,7 @@ class StructuredResponseFormat(BaseModel): method_metadata_key: { "request": { "model": config.request_config.openai_model, - "messages": openai_client.truncate_messages_for_logging(completion_messages), + "messages": completion_messages, }, "error": str(e), }, @@ -543,7 +552,7 @@ class StructuredResponseFormat(BaseModel): method_metadata_key: { "request": { "model": config.request_config.openai_model, - "messages": openai_client.truncate_messages_for_logging(completion_messages), + "messages": completion_messages, "max_tokens": config.request_config.response_tokens, }, "response": completion.model_dump() if completion else "[no response from openai]", @@ -566,7 +575,7 @@ class StructuredResponseFormat(BaseModel): method_metadata_key: { "request": { "model": config.request_config.openai_model, - "messages": openai_client.truncate_messages_for_logging(completion_messages), + "messages": completion_messages, }, "error": str(e), }, diff --git a/assistants/prospector-assistant/assistant/config.py b/assistants/prospector-assistant/assistant/config.py index 7c09b5a4..354cca29 100644 --- a/assistants/prospector-assistant/assistant/config.py +++ b/assistants/prospector-assistant/assistant/config.py @@ -8,7 +8,7 @@ from . import helpers from .agents.artifact_agent import ArtifactAgentConfigModel -from .agents.form_fill_agent import FormFillAgentConfig +from .agents.form_fill_extension import FormFillConfig # The semantic workbench app uses react-jsonschema-form for rendering # dynamic configuration forms based on the configuration model and UI schema @@ -26,9 +26,7 @@ class AgentsConfigModel(BaseModel): - form_fill_agent: Annotated[FormFillAgentConfig, Field(title="Form Fill Agent Configuration")] = ( - FormFillAgentConfig() - ) + form_fill_agent: Annotated[FormFillConfig, Field(title="Form Fill Agent Configuration")] = FormFillConfig() artifact_agent: Annotated[ ArtifactAgentConfigModel,