diff --git a/assistants/explorer-assistant/assistant/chat.py b/assistants/explorer-assistant/assistant/chat.py index 363ea4e0..03bf4412 100644 --- a/assistants/explorer-assistant/assistant/chat.py +++ b/assistants/explorer-assistant/assistant/chat.py @@ -334,7 +334,7 @@ async def respond_to_conversation( method_metadata_key: { "request": { "model": config.request_config.openai_model, - "messages": openai_client.truncate_messages_for_logging(completion_messages), + "messages": completion_messages, "max_tokens": config.request_config.response_tokens, }, "response": completion.model_dump() if completion else "[no response from openai]", @@ -358,7 +358,7 @@ async def respond_to_conversation( method_metadata_key: { "request": { "model": config.request_config.openai_model, - "messages": openai_client.truncate_messages_for_logging(completion_messages), + "messages": completion_messages, }, "error": str(e), }, diff --git a/assistants/prospector-assistant/assistant/agents/document_agent.py b/assistants/prospector-assistant/assistant/agents/document_agent.py index bf7b7beb..2629ecba 100644 --- a/assistants/prospector-assistant/assistant/agents/document_agent.py +++ b/assistants/prospector-assistant/assistant/agents/document_agent.py @@ -919,7 +919,7 @@ def _on_success_metadata_update( f"{method_metadata_key}": { "request": { "model": config.request_config.openai_model, - "messages": openai_client.truncate_messages_for_logging(chat_completion_messages), + "messages": chat_completion_messages, "max_tokens": config.request_config.response_tokens, }, "response": completion.model_dump() if completion else "[no response from openai]", @@ -943,7 +943,7 @@ def _on_error_metadata_update( f"{method_metadata_key}": { "request": { "model": config.request_config.openai_model, - "messages": openai_client.truncate_messages_for_logging(chat_completion_messages), + "messages": chat_completion_messages, }, "error": str(e), }, diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_agent/__init__.py b/assistants/prospector-assistant/assistant/agents/form_fill_agent/__init__.py deleted file mode 100644 index b28e952c..00000000 --- a/assistants/prospector-assistant/assistant/agents/form_fill_agent/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -from .agent import execute, extend -from .config import FormFillAgentConfig -from .steps.types import LLMConfig - -__all__ = [ - "execute", - "extend", - "LLMConfig", - "FormFillAgentConfig", -] diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_agent/agent.py b/assistants/prospector-assistant/assistant/agents/form_fill_agent/agent.py deleted file mode 100644 index 8a350e41..00000000 --- a/assistants/prospector-assistant/assistant/agents/form_fill_agent/agent.py +++ /dev/null @@ -1,158 +0,0 @@ -import logging -from typing import Awaitable, Callable, Sequence - -from openai.types.chat import ChatCompletionMessageParam -from semantic_workbench_api_model.workbench_model import MessageType, NewConversationMessage -from semantic_workbench_assistant.assistant_app.context import ConversationContext -from semantic_workbench_assistant.assistant_app.protocol import AssistantAppProtocol - -from . import state -from .config import FormFillAgentConfig -from .steps import acquire_form_step, extract_form_fields_step, fill_form_step -from .steps.types import ConfigT, Context, IncompleteErrorResult, IncompleteResult, LLMConfig - -logger = logging.getLogger(__name__) - - -async def execute( - context: ConversationContext, - llm_config: LLMConfig, - config: FormFillAgentConfig, - latest_user_message: str | None, - get_attachment_messages: Callable[[Sequence[str]], Awaitable[Sequence[ChatCompletionMessageParam]]], -) -> None: - user_messages = [latest_user_message] - - def build_step_context(config: ConfigT) -> Context[ConfigT]: - return Context( - context=context, llm_config=llm_config, config=config, get_attachment_messages=get_attachment_messages - ) - - async with state.agent_state(context) as agent_state: - while True: - logger.info("form-fill-agent execute loop; mode: %s", agent_state.mode) - - match agent_state.mode: - case state.FormFillAgentMode.acquire_form_step: - result = await acquire_form_step.execute( - step_context=build_step_context(config.acquire_form_config), - latest_user_message=user_messages.pop() if user_messages else None, - ) - - match result: - case acquire_form_step.CompleteResult(): - await _send_message(context, result.ai_message, result.debug) - - agent_state.form_filename = result.filename - agent_state.mode = state.FormFillAgentMode.extract_form_fields_step - - case IncompleteResult() | IncompleteErrorResult(): - await _handle_incomplete_results(context, result) - return - - case _: - raise ValueError(f"Unexpected result: {result}") - - case state.FormFillAgentMode.extract_form_fields_step: - result = await extract_form_fields_step.execute( - step_context=build_step_context(config.extract_form_fields_config), - filename=agent_state.form_filename, - ) - - match result: - case extract_form_fields_step.CompleteResult(): - await _send_message(context, result.ai_message, result.debug) - - agent_state.extracted_form_fields = result.extracted_form_fields - agent_state.mode = state.FormFillAgentMode.fill_form_step - - case IncompleteResult() | IncompleteErrorResult(): - await _handle_incomplete_results(context, result) - return - - case _: - raise ValueError(f"Unexpected result: {result}") - - case state.FormFillAgentMode.fill_form_step: - result = await fill_form_step.execute( - step_context=build_step_context(config.fill_form_config), - latest_user_message=user_messages.pop() if user_messages else None, - form_fields=agent_state.extracted_form_fields, - ) - - match result: - case fill_form_step.CompleteResult(): - await _send_message(context, result.ai_message, result.debug) - - agent_state.fill_form_gc_artifact = result.artifact - agent_state.mode = state.FormFillAgentMode.generate_filled_form_step - - case IncompleteResult() | IncompleteErrorResult(): - await _handle_incomplete_results(context, result) - return - - case _: - raise ValueError(f"Unexpected result: {result}") - - case state.FormFillAgentMode.generate_filled_form_step: - await _send_message( - context, "I'd love to generate the filled-out form now, but it's not yet implemented. :)", {} - ) - return - - case state.FormFillAgentMode.end_conversation: - await _send_message(context, "Conversation has ended.", {}) - return - - case _: - raise ValueError(f"Unexpected mode: {state.mode}") - - -async def _handle_incomplete_results( - context: ConversationContext, result: IncompleteErrorResult | IncompleteResult -) -> None: - match result: - case IncompleteResult(): - await _send_message(context, result.ai_message, result.debug) - - case IncompleteErrorResult(): - await _send_error_message(context, result.error_message, result.debug) - - case _: - raise ValueError(f"Unexpected incomplete result: {result}") - - -async def _send_message(context: ConversationContext, message: str, debug: dict) -> None: - if not message: - return - - await context.send_messages( - NewConversationMessage( - content=message, - message_type=MessageType.chat, - metadata={"debug": debug}, - ) - ) - - -async def _send_error_message(context: ConversationContext, message: str, debug: dict) -> None: - await context.send_messages( - NewConversationMessage( - content=message, - message_type=MessageType.notice, - metadata={"debug": debug}, - ) - ) - - -def extend(app: AssistantAppProtocol) -> None: - """ - Extend the assistant app with the form-fill agent inspectors. - """ - - # for agent level state - app.add_inspector_state_provider(state.inspector.state_id, state.inspector) - - # for step level states - acquire_form_step.extend(app) - fill_form_step.extend(app) diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/_attachments.py b/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/_attachments.py deleted file mode 100644 index 0f6424e8..00000000 --- a/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/_attachments.py +++ /dev/null @@ -1,57 +0,0 @@ -""" -Utility functions for handling attachments in chat messages. -""" - -from typing import Awaitable, Callable, Sequence - -from openai.types.chat import ChatCompletionMessageParam -from semantic_workbench_assistant.assistant_app.context import ConversationContext - -from .. import state - - -async def message_with_recent_attachments( - context: ConversationContext, - latest_user_message: str | None, - get_attachment_messages: Callable[[Sequence[str]], Awaitable[Sequence[ChatCompletionMessageParam]]], -) -> str: - files = await context.get_files() - - new_filenames = set() - - async with state.agent_state(context) as agent_state: - max_timestamp = agent_state.most_recent_attachment_timestamp - for file in files.files: - if file.updated_datetime.timestamp() <= agent_state.most_recent_attachment_timestamp: - continue - - max_timestamp = max(file.updated_datetime.timestamp(), max_timestamp) - new_filenames.add(file.filename) - - agent_state.most_recent_attachment_timestamp = max_timestamp - - attachment_messages = await get_attachment_messages(list(new_filenames)) - - return "\n\n".join( - ( - latest_user_message or "", - *( - str(attachment.get("content")) - for attachment in attachment_messages - if "" in str(attachment.get("content", "")) - ), - ), - ) - - -async def attachment_for_filename( - filename: str, get_attachment_messages: Callable[[Sequence[str]], Awaitable[Sequence[ChatCompletionMessageParam]]] -) -> str: - attachment_messages = await get_attachment_messages([filename]) - return "\n\n".join( - ( - str(attachment.get("content")) - for attachment in attachment_messages - if "" in str(attachment.get("content", "")) - ) - ) diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/fill_form_step.py b/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/fill_form_step.py deleted file mode 100644 index e79678cb..00000000 --- a/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/fill_form_step.py +++ /dev/null @@ -1,149 +0,0 @@ -import logging -from dataclasses import dataclass -from pathlib import Path -from typing import Any, Literal - -from guided_conversation.utils.resources import ResourceConstraintMode, ResourceConstraintUnit -from pydantic import Field, create_model -from semantic_workbench_assistant.assistant_app.context import ConversationContext -from semantic_workbench_assistant.assistant_app.protocol import AssistantAppProtocol - -from .. import state -from ..inspector import FileStateInspector -from . import _attachments, _guided_conversation -from .types import ( - Context, - GuidedConversationDefinition, - IncompleteErrorResult, - IncompleteResult, - ResourceConstraintDefinition, - Result, -) - -logger = logging.getLogger(__name__) - - -def extend(app: AssistantAppProtocol) -> None: - app.add_inspector_state_provider(_inspector.state_id, _inspector) - - -@dataclass -class CompleteResult(Result): - ai_message: str - artifact: dict - - -async def execute( - step_context: Context[_guided_conversation.GuidedConversationDefinition], - latest_user_message: str | None, - form_fields: list[state.FormField], -) -> IncompleteResult | IncompleteErrorResult | CompleteResult: - """ - Step: fill out the form with the user - Approach: Guided conversation - """ - message_with_attachments = await _attachments.message_with_recent_attachments( - step_context.context, latest_user_message, step_context.get_attachment_messages - ) - - artifact_type = _form_fields_to_artifact(form_fields) - - definition = step_context.config.model_copy() - definition.resource_constraint.quantity = int(len(form_fields) * 1.5) - async with _guided_conversation.engine( - definition=definition, - artifact_type=artifact_type, - state_file_path=_get_state_file_path(step_context.context), - openai_client=step_context.llm_config.openai_client_factory(), - openai_model=step_context.llm_config.openai_model, - context=step_context.context, - state_id=_inspector.state_id, - ) as gce: - try: - result = await gce.step_conversation(message_with_attachments) - except Exception as e: - logger.exception("failed to execute guided conversation") - return IncompleteErrorResult( - error_message=f"Failed to execute guided conversation: {e}", - debug={"error": str(e)}, - ) - - debug = {"guided-conversation": gce.to_json()} - - logger.info("guided-conversation result: %s", result) - - fill_form_gc_artifact = gce.artifact.artifact.model_dump(mode="json") - logger.info("guided-conversation artifact: %s", gce.artifact) - - if result.is_conversation_over: - return CompleteResult( - ai_message="", - artifact=fill_form_gc_artifact, - debug=debug, - ) - - return IncompleteResult(ai_message=result.ai_message or "", debug=debug) - - -def _form_fields_to_artifact(form_fields: list[state.FormField]): - field_definitions: dict[str, tuple[Any, Any]] = {} - required_fields = [] - for field in form_fields: - if field.required: - required_fields.append(field.id) - - match field.type: - case "string": - field_definitions[field.id] = (str, Field(title=field.name, description=field.description)) - - case "bool": - field_definitions[field.id] = (bool, Field(title=field.name, description=field.description)) - - case "multiple_choice": - field_definitions[field.id] = ( - Literal[tuple(field.options)], - Field(title=field.name, description=field.description), - ) - - return create_model( - "FilledFormArtifact", - **field_definitions, # type: ignore - ) # type: ignore - - -def _get_state_file_path(context: ConversationContext) -> Path: - return _guided_conversation.path_for_state(context, "fill_form") - - -_inspector = FileStateInspector( - display_name="Fill Form Guided Conversation State", - file_path_source=_get_state_file_path, - state_id="fill_form", -) - - -definition = GuidedConversationDefinition( - rules=[ - "For fields that are not in the provided files, collect the data from the user through conversation.", - "When providing options for a multiple choice field, provide the options in a numbered-list, so the user can refer to them by number.", - "When listing anything other than options, like document types, provide them in a bulleted list for improved readability.", - "When updating the agenda, the data-collection for each form field must be in a separate step.", - "When asking for data to fill the form, always ask for a single piece of information at a time. Never ask for multiple pieces of information in a single prompt, ex: 'Please provide field Y, and additionally, field X'.", - "Terminate conversation if inappropriate content is requested.", - ], - conversation_flow=""" -1. Inform the user that we've received the form and determined the fields in the form. -2. Inform the user that our goal is help them fill out the form. -3. Ask the user to provide one or more files that might contain data relevant to fill out the form. The files can be PDF, TXT, or DOCX. -4. When asking for files, suggest types of documents that might contain the data. -5. For each field in the form, check if the data is available in the provided files. -6. If the data is not available in the files, ask the user for the data. -8. When the form is filled out, inform the user that you will now generate a document containing the filled form. -""", - context="", - resource_constraint=ResourceConstraintDefinition( - quantity=15, - unit=ResourceConstraintUnit.TURNS, - mode=ResourceConstraintMode.MAXIMUM, - ), -) diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_extension/__init__.py b/assistants/prospector-assistant/assistant/agents/form_fill_extension/__init__.py new file mode 100644 index 00000000..bdf69b23 --- /dev/null +++ b/assistants/prospector-assistant/assistant/agents/form_fill_extension/__init__.py @@ -0,0 +1,9 @@ +from .config import FormFillConfig +from .extension import FormFillExtension +from .steps.types import LLMConfig + +__all__ = [ + "FormFillExtension", + "LLMConfig", + "FormFillConfig", +] diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_agent/config.py b/assistants/prospector-assistant/assistant/agents/form_fill_extension/config.py similarity index 73% rename from assistants/prospector-assistant/assistant/agents/form_fill_agent/config.py rename to assistants/prospector-assistant/assistant/agents/form_fill_extension/config.py index 8e209b2e..c31c87b6 100644 --- a/assistants/prospector-assistant/assistant/agents/form_fill_agent/config.py +++ b/assistants/prospector-assistant/assistant/agents/form_fill_extension/config.py @@ -2,14 +2,14 @@ from pydantic import BaseModel, Field -from .steps import acquire_form_step, extract_form_fields_step, fill_form_step, types +from .steps import acquire_form_step, extract_form_fields_step, fill_form_step -class FormFillAgentConfig(BaseModel): +class FormFillConfig(BaseModel): acquire_form_config: Annotated[ - types.GuidedConversationDefinition, + acquire_form_step.AcquireFormConfig, Field(title="Form Acquisition", description="Guided conversation for acquiring a form from the user."), - ] = acquire_form_step.definition.model_copy() + ] = acquire_form_step.AcquireFormConfig() extract_form_fields_config: Annotated[ extract_form_fields_step.ExtractFormFieldsConfig, @@ -17,6 +17,6 @@ class FormFillAgentConfig(BaseModel): ] = extract_form_fields_step.ExtractFormFieldsConfig() fill_form_config: Annotated[ - types.GuidedConversationDefinition, + fill_form_step.FillFormConfig, Field(title="Fill Form", description="Guided conversation for filling out the form."), - ] = fill_form_step.definition.model_copy() + ] = fill_form_step.FillFormConfig() diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_extension/extension.py b/assistants/prospector-assistant/assistant/agents/form_fill_extension/extension.py new file mode 100644 index 00000000..9c283258 --- /dev/null +++ b/assistants/prospector-assistant/assistant/agents/form_fill_extension/extension.py @@ -0,0 +1,163 @@ +import logging +from typing import AsyncIterable, Awaitable, Callable, Sequence + +from semantic_workbench_api_model.workbench_model import MessageType, NewConversationMessage +from semantic_workbench_assistant.assistant_app.context import ConversationContext +from semantic_workbench_assistant.assistant_app.protocol import AssistantAppProtocol + +from . import state +from .config import FormFillConfig +from .steps import acquire_form_step, extract_form_fields_step, fill_form_step +from .steps.types import ConfigT, Context, IncompleteErrorResult, IncompleteResult, LLMConfig, UserAttachment, UserInput + +logger = logging.getLogger(__name__) + + +class FormFillExtension: + def __init__(self, assistant_app: AssistantAppProtocol) -> None: + """ + Extend the assistant app with the form-fill agent inspectors. + """ + + # for agent level state + assistant_app.add_inspector_state_provider(state.inspector.state_id, state.inspector) + + # for step level states + acquire_form_step.extend(assistant_app) + fill_form_step.extend(assistant_app) + + async def execute( + self, + context: ConversationContext, + llm_config: LLMConfig, + config: FormFillConfig, + latest_user_message: str | None, + latest_attachment_filenames: Sequence[str], + get_attachment_content: Callable[[str], Awaitable[str]], + ) -> None: + user_messages = [latest_user_message] + + async def latest_attachments() -> AsyncIterable[UserAttachment]: + for filename in latest_attachment_filenames: + content = await get_attachment_content(filename) + yield UserAttachment(filename=filename, content=content) + + def build_step_context(config: ConfigT) -> Context[ConfigT]: + return Context( + context=context, + llm_config=llm_config, + config=config, + latest_user_input=UserInput( + message=user_messages.pop() if user_messages else None, + attachments=latest_attachments(), + ), + ) + + async with state.agent_state(context) as agent_state: + while True: + logger.info("form-fill-agent execute loop; mode: %s", agent_state.mode) + + match agent_state.mode: + case state.FormFillAgentMode.acquire_form_step: + result = await acquire_form_step.execute( + step_context=build_step_context(config.acquire_form_config), + ) + + match result: + case acquire_form_step.CompleteResult(): + await _send_message(context, result.ai_message, result.debug) + + agent_state.form_filename = result.filename + agent_state.mode = state.FormFillAgentMode.extract_form_fields + + continue + + case _: + await _handle_incomplete_result(context, result) + return + + case state.FormFillAgentMode.extract_form_fields: + file_content = await get_attachment_content(agent_state.form_filename) + result = await extract_form_fields_step.execute( + step_context=build_step_context(config.extract_form_fields_config), + file_content=file_content, + ) + + match result: + case extract_form_fields_step.CompleteResult(): + await _send_message(context, result.ai_message, result.debug) + + agent_state.extracted_form_fields = result.extracted_form_fields + agent_state.mode = state.FormFillAgentMode.fill_form_step + + continue + + case _: + await _handle_incomplete_result(context, result) + return + + case state.FormFillAgentMode.fill_form_step: + result = await fill_form_step.execute( + step_context=build_step_context(config.fill_form_config), + form_filename=agent_state.form_filename, + form_fields=agent_state.extracted_form_fields, + ) + + match result: + case fill_form_step.CompleteResult(): + await _send_message(context, result.ai_message, result.debug) + + agent_state.fill_form_gc_artifact = result.artifact + agent_state.mode = state.FormFillAgentMode.generate_filled_form_step + + continue + + case _: + await _handle_incomplete_result(context, result) + return + + case state.FormFillAgentMode.generate_filled_form_step: + await _send_message( + context, + "I'd love to generate the filled-out form now, but it's not yet implemented. :)", + {}, + ) + return + + case _: + raise ValueError(f"Unexpected mode: {agent_state.mode}") + + +async def _handle_incomplete_result(context: ConversationContext, result: IncompleteResult) -> None: + match result: + case IncompleteResult(): + await _send_message(context, result.message, result.debug) + + case IncompleteErrorResult(): + await _send_error_message(context, result.error_message, result.debug) + + case _: + raise ValueError(f"Unexpected incomplete result type: {result}") + + +async def _send_message(context: ConversationContext, message: str, debug: dict) -> None: + if not message: + return + + await context.send_messages( + NewConversationMessage( + content=message, + message_type=MessageType.chat, + debug_data=debug, + ) + ) + + +async def _send_error_message(context: ConversationContext, message: str, debug: dict) -> None: + await context.send_messages( + NewConversationMessage( + content=message, + message_type=MessageType.notice, + debug_data=debug, + ) + ) diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_agent/inspector.py b/assistants/prospector-assistant/assistant/agents/form_fill_extension/inspector.py similarity index 90% rename from assistants/prospector-assistant/assistant/agents/form_fill_agent/inspector.py rename to assistants/prospector-assistant/assistant/agents/form_fill_extension/inspector.py index e35ce6d5..a8b0192f 100644 --- a/assistants/prospector-assistant/assistant/agents/form_fill_agent/inspector.py +++ b/assistants/prospector-assistant/assistant/agents/form_fill_extension/inspector.py @@ -1,5 +1,6 @@ import contextlib import json +from hashlib import md5 from pathlib import Path from typing import Callable @@ -20,10 +21,11 @@ def __init__( self, display_name: str, file_path_source: Callable[[ConversationContext], Path], - state_id: str, description: str = "", ) -> None: - self._state_id = state_id + self._state_id = md5( + (type(self).__name__ + "_" + display_name).encode("utf-8"), usedforsecurity=False + ).hexdigest() self._display_name = display_name self._file_path_source = file_path_source self._description = description diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_agent/state.py b/assistants/prospector-assistant/assistant/agents/form_fill_extension/state.py similarity index 85% rename from assistants/prospector-assistant/assistant/agents/form_fill_agent/state.py rename to assistants/prospector-assistant/assistant/agents/form_fill_extension/state.py index a31dcee6..3d49eae1 100644 --- a/assistants/prospector-assistant/assistant/agents/form_fill_agent/state.py +++ b/assistants/prospector-assistant/assistant/agents/form_fill_extension/state.py @@ -12,7 +12,7 @@ class FormField(BaseModel): - id: str = Field(description="The unique identifier of the field.") + id: str = Field(description="The descriptive, unique identifier of the field as a snake_case_english_string.") name: str = Field(description="The name of the field.") description: str = Field(description="The description of the field.") type: Literal["string", "bool", "multiple_choice"] = Field(description="The type of the field.") @@ -22,16 +22,13 @@ class FormField(BaseModel): class FormFillAgentMode(StrEnum): acquire_form_step = "acquire_form" - extract_form_fields_step = "extract_form_fields" + extract_form_fields = "extract_form_fields" fill_form_step = "fill_form" generate_filled_form_step = "generate_filled_form" - end_conversation = "end_conversation" - class FormFillAgentState(BaseModel): mode: FormFillAgentMode = FormFillAgentMode.acquire_form_step - most_recent_attachment_timestamp: float = 0 form_filename: str = "" extracted_form_fields: list[FormField] = [] fill_form_gc_artifact: dict | None = None @@ -63,6 +60,4 @@ async def agent_state(context: ConversationContext) -> AsyncIterator[FormFillAge current_state.set(None) -inspector = FileStateInspector( - display_name="Form Fill Agent State", file_path_source=path_for_state, state_id="form_fill_agent" -) +inspector = FileStateInspector(display_name="FormFill Agent", file_path_source=path_for_state) diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/__init__.py b/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/__init__.py similarity index 100% rename from assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/__init__.py rename to assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/__init__.py diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/_guided_conversation.py b/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/_guided_conversation.py similarity index 100% rename from assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/_guided_conversation.py rename to assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/_guided_conversation.py diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/_llm.py b/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/_llm.py new file mode 100644 index 00000000..740dbc6d --- /dev/null +++ b/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/_llm.py @@ -0,0 +1,47 @@ +from typing import Any, TypeVar + +from openai.types.chat import ChatCompletionMessageParam +from pydantic import BaseModel + +from .types import LLMConfig + + +class NoResponseChoicesError(Exception): + pass + + +class NoParsedMessageError(Exception): + pass + + +ResponseModelT = TypeVar("ResponseModelT", bound=BaseModel) + + +async def structured_completion( + llm_config: LLMConfig, messages: list[ChatCompletionMessageParam], response_model: type[ResponseModelT] +) -> tuple[ResponseModelT, dict[str, Any]]: + async with llm_config.openai_client_factory() as client: + response = await client.beta.chat.completions.parse( + messages=messages, + model=llm_config.openai_model, + response_format=response_model, + max_tokens=llm_config.max_response_tokens, + ) + + if not response.choices: + raise NoResponseChoicesError() + + if not response.choices[0].message.parsed: + raise NoParsedMessageError() + + metadata = { + "request": { + "model": llm_config.openai_model, + "messages": messages, + "max_tokens": llm_config.max_response_tokens, + "response_format": response_model.model_json_schema(), + }, + "response": response.model_dump(), + } + + return response.choices[0].message.parsed, metadata diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/acquire_form_step.py b/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/acquire_form_step.py similarity index 79% rename from assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/acquire_form_step.py rename to assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/acquire_form_step.py index 10de7789..c2c2fbc7 100644 --- a/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/acquire_form_step.py +++ b/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/acquire_form_step.py @@ -1,6 +1,7 @@ import logging from dataclasses import dataclass from pathlib import Path +from textwrap import dedent from guided_conversation.utils.resources import ResourceConstraintMode, ResourceConstraintUnit from pydantic import BaseModel, Field @@ -8,7 +9,7 @@ from semantic_workbench_assistant.assistant_app.protocol import AssistantAppProtocol from ..inspector import FileStateInspector -from . import _attachments, _guided_conversation +from . import _guided_conversation from .types import ( Context, GuidedConversationDefinition, @@ -16,6 +17,7 @@ IncompleteResult, ResourceConstraintDefinition, Result, + UserInput, ) logger = logging.getLogger(__name__) @@ -25,6 +27,38 @@ def extend(app: AssistantAppProtocol) -> None: app.add_inspector_state_provider(_inspector.state_id, _inspector) +class FormArtifact(BaseModel): + title: str = Field(description="The title of the form.", default="") + filename: str = Field(description="The filename of the form.", default="") + + +definition = GuidedConversationDefinition( + rules=[ + "DO NOT suggest forms or create a form for the user.", + "Politely request another file if the provided file is not a form.", + "Terminate conversation if inappropriate content is requested.", + ], + conversation_flow=dedent(""" + 1. Inform the user that our goal is to help the user fill out a form. + 2. Ask the user to provide a file that contains a form. The file can be PDF, TXT, or DOCX. + 3. When you receive a file, determine if the file looks to be a form. + 4. If the file is not a form, inform the user that the file is not a form. Ask them to provide a different file. + 5. If the form is a file, update the artifcat with the title and filename of the form. + 6. Inform the user that you will now extract the form fields, so that you can assist them in filling it out. + """).strip(), + context="", + resource_constraint=ResourceConstraintDefinition( + quantity=5, + unit=ResourceConstraintUnit.MINUTES, + mode=ResourceConstraintMode.MAXIMUM, + ), +) + + +class AcquireFormConfig(BaseModel): + definition: GuidedConversationDefinition = definition + + @dataclass class CompleteResult(Result): ai_message: str @@ -32,19 +66,16 @@ class CompleteResult(Result): async def execute( - step_context: Context[GuidedConversationDefinition], - latest_user_message: str | None, + step_context: Context[AcquireFormConfig], ) -> IncompleteResult | IncompleteErrorResult | CompleteResult: """ Step: acquire a form from the user Approach: Guided conversation """ - message_with_attachments = await _attachments.message_with_recent_attachments( - step_context.context, latest_user_message, step_context.get_attachment_messages - ) + message_with_attachments = await input_to_message(step_context.latest_user_input) async with _guided_conversation.engine( - definition=step_context.config, + definition=step_context.config.definition, artifact_type=FormArtifact, state_file_path=_get_state_file_path(step_context.context), openai_client=step_context.llm_config.openai_client_factory(), @@ -57,7 +88,7 @@ async def execute( except Exception as e: logger.exception("failed to execute guided conversation") return IncompleteErrorResult( - error_message=f"Failed to execute guided conversation: {e}", + message=f"Failed to execute guided conversation: {e}", debug={"error": str(e)}, ) @@ -77,7 +108,7 @@ async def execute( debug=debug, ) - return IncompleteResult(ai_message=result.ai_message or "", debug=debug) + return IncompleteResult(message=result.ai_message or "", debug=debug) def _get_state_file_path(context: ConversationContext) -> Path: @@ -85,37 +116,22 @@ def _get_state_file_path(context: ConversationContext) -> Path: _inspector = FileStateInspector( - display_name="Acquire Form Guided Conversation State", + display_name="Acquire-Form Guided-Conversation", file_path_source=_get_state_file_path, - state_id="acquire_form", ) -class FormArtifact(BaseModel): - title: str = Field(description="The title of the form.", default="") - filename: str = Field(description="The filename of the form.", default="") +async def input_to_message(input: UserInput) -> str | None: + attachments = [] + async for attachment in input.attachments: + attachments.append(attachment.content) + if not attachments: + return input.message -definition = GuidedConversationDefinition( - rules=[ - "DO NOT suggest forms or create a form for the user.", - "Politely request another file if the provided file is not a form.", - "Terminate conversation if inappropriate content is requested.", - ], - conversation_flow=( - """ - 1. Inform the user that our goal is to help the user fill out a form. - 2. Ask the user to provide a file that contains a form. The file can be PDF, TXT, or DOCX. - 3. When you receive a file, determine if the file looks to be a form. - 4. If the file is not a form, inform the user that the file is not a form. Ask them to provide a different file. - 5. If the form is a file, update the artifcat with the title and filename of the form. - 6. Inform the user that you will now extract the form fields, so that you can assist them in filling it out. - """ - ), - context="", - resource_constraint=ResourceConstraintDefinition( - quantity=5, - unit=ResourceConstraintUnit.MINUTES, - mode=ResourceConstraintMode.MAXIMUM, - ), -) + return "\n\n".join( + ( + input.message or "", + *attachments, + ), + ) diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/extract_form_fields_step.py b/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/extract_form_fields_step.py similarity index 66% rename from assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/extract_form_fields_step.py rename to assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/extract_form_fields_step.py index 44cc3789..64d0ce39 100644 --- a/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/extract_form_fields_step.py +++ b/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/extract_form_fields_step.py @@ -2,12 +2,11 @@ from dataclasses import dataclass from typing import Annotated, Any -import openai_client from openai.types.chat import ChatCompletionMessageParam from pydantic import BaseModel, Field from .. import state -from . import _attachments +from . import _llm from .types import Context, IncompleteErrorResult, IncompleteResult, LLMConfig, Result logger = logging.getLogger(__name__) @@ -31,14 +30,13 @@ class CompleteResult(Result): async def execute( step_context: Context[ExtractFormFieldsConfig], - filename: str, + file_content: str, ) -> IncompleteResult | IncompleteErrorResult | CompleteResult: """ Step: extract form fields from the form file content Approach: Chat completion with LLM """ - file_content = await _attachments.attachment_for_filename(filename, step_context.get_attachment_messages) async with step_context.context.set_status("inspecting form ..."): try: extracted_form_fields, metadata = await _extract( @@ -50,13 +48,13 @@ async def execute( except Exception as e: logger.exception("failed to extract form fields") return IncompleteErrorResult( - error_message=f"Failed to extract form fields: {e}", + message=f"Failed to extract form fields: {e}", debug={"error": str(e)}, ) if extracted_form_fields.error_message: return IncompleteResult( - ai_message=extracted_form_fields.error_message, + message=extracted_form_fields.error_message, debug=metadata, ) @@ -72,14 +70,6 @@ class FormFields(BaseModel): fields: list[state.FormField] = Field(description="The fields in the form.") -class NoResponseChoicesError(Exception): - pass - - -class NoParsedMessageError(Exception): - pass - - async def _extract( llm_config: LLMConfig, config: ExtractFormFieldsConfig, form_content: str ) -> tuple[FormFields, dict[str, Any]]: @@ -94,28 +84,8 @@ async def _extract( }, ] - async with llm_config.openai_client_factory() as client: - response = await client.beta.chat.completions.parse( - messages=messages, - model=llm_config.openai_model, - response_format=FormFields, - max_tokens=llm_config.max_response_tokens, - ) - - if not response.choices: - raise NoResponseChoicesError() - - if not response.choices[0].message.parsed: - raise NoParsedMessageError() - - metadata = { - "request": { - "model": llm_config.openai_model, - "messages": openai_client.truncate_messages_for_logging(messages), - "max_tokens": llm_config.max_response_tokens, - "response_format": FormFields.model_json_schema(), - }, - "response": response.model_dump(), - } - - return response.choices[0].message.parsed, metadata + return await _llm.structured_completion( + llm_config=llm_config, + messages=messages, + response_model=FormFields, + ) diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/fill_form_step.py b/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/fill_form_step.py new file mode 100644 index 00000000..5297414e --- /dev/null +++ b/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/fill_form_step.py @@ -0,0 +1,237 @@ +import logging +from dataclasses import dataclass +from pathlib import Path +from textwrap import dedent +from typing import Annotated, Any, Literal + +from guided_conversation.utils.resources import ResourceConstraintMode, ResourceConstraintUnit +from openai.types.chat import ChatCompletionMessageParam +from pydantic import BaseModel, Field, create_model +from semantic_workbench_assistant.assistant_app.context import ConversationContext +from semantic_workbench_assistant.assistant_app.protocol import AssistantAppProtocol + +from .. import state +from ..inspector import FileStateInspector +from . import _guided_conversation, _llm +from .types import ( + Context, + GuidedConversationDefinition, + IncompleteErrorResult, + IncompleteResult, + LLMConfig, + ResourceConstraintDefinition, + Result, +) + +logger = logging.getLogger(__name__) + + +def extend(app: AssistantAppProtocol) -> None: + app.add_inspector_state_provider(_inspector.state_id, _inspector) + + +definition = GuidedConversationDefinition( + rules=[ + "For fields that are not in the provided files, collect the data from the user through conversation.", + "When providing options for a multiple choice field, provide the options in a numbered-list, so the user can refer to them by number.", + "When listing anything other than options, like document types, provide them in a bulleted list for improved readability.", + "When updating the agenda, the data-collection for each form field must be in a separate step.", + "When asking for data to fill the form, always ask for a single piece of information at a time. Never ask for multiple pieces of information in a single prompt, ex: 'Please provide field Y, and additionally, field X'.", + "Terminate conversation if inappropriate content is requested.", + ], + conversation_flow=dedent(""" + 1. Inform the user that we've received the form and determined the fields in the form. + 2. Inform the user that our goal is help them fill out the form. + 3. Ask the user to provide one or more files that might contain data relevant to fill out the form. The files can be PDF, TXT, or DOCX. + 4. When asking for files, suggest types of documents that might contain the data. + 5. For each field in the form, check if the data is available in the provided files. + 6. If the data is not available in the files, ask the user for the data. + 7. When the form is filled out, inform the user that you will now generate a document containing the filled form. + """).strip(), + context="", + resource_constraint=ResourceConstraintDefinition( + quantity=15, + unit=ResourceConstraintUnit.TURNS, + mode=ResourceConstraintMode.MAXIMUM, + ), +) + + +class ExtractCandidateFieldValuesConfig(BaseModel): + instruction: Annotated[ + str, + Field( + title="Instruction", + description="The instruction for extracting candidate form-field values from an uploaded file", + ), + ] = dedent(""" + Given the field definitions below, extract candidate values for these fields from the user provided + attachment. + + Only include values that are in the provided attachment. + It is possible that there are multiple candidates for a single field, in which case you should provide + all the candidates and an explanation for each candidate. + + Field definitions: + {{form_fields}} + """) + + +class FillFormConfig(BaseModel): + extract_config: ExtractCandidateFieldValuesConfig = ExtractCandidateFieldValuesConfig() + definition: GuidedConversationDefinition = definition + + +class FieldValueCandidate(BaseModel): + field_id: str = Field(description="The ID of the field that the value is a candidate for.") + value: str = Field(description="The value from the document for this field.") + explanation: str = Field(description="The explanation of why this value is a candidate for the field.") + + +class FieldValueCandidates(BaseModel): + response: str = Field(description="The natural language response to send to the user.") + fields: list[FieldValueCandidate] = Field(description="The fields in the form.") + + +class FieldValueCandidatesFromDocument(BaseModel): + filename: str + candidates: FieldValueCandidates + + +@dataclass +class CompleteResult(Result): + ai_message: str + artifact: dict + + +async def execute( + step_context: Context[FillFormConfig], + form_filename: str, + form_fields: list[state.FormField], +) -> IncompleteResult | IncompleteErrorResult | CompleteResult: + """ + Step: fill out the form with the user + Approach: Guided conversation + """ + message = step_context.latest_user_input.message + debug = {"document-extractions": {}} + + async for attachment in step_context.latest_user_input.attachments: + if attachment.filename == form_filename: + continue + + candidate_values, metadata = await _extract( + llm_config=step_context.llm_config, + config=step_context.config.extract_config, + form_fields=form_fields, + document_content=attachment.content, + ) + message = f"{message}\n\n" if message else "" + message = f"{message}{candidate_values.response}\n\nFilename: {attachment.filename}" + for candidate in candidate_values.fields: + message += f"\nField id: {candidate.field_id}:\n Value: {candidate.value}\n Explanation: {candidate.explanation}" + + debug["document-extractions"][attachment.filename] = metadata + + artifact_type = _form_fields_to_artifact(form_fields) + + definition = step_context.config.definition.model_copy() + definition.resource_constraint.quantity = int(len(form_fields) * 1.5) + async with _guided_conversation.engine( + definition=definition, + artifact_type=artifact_type, + state_file_path=_get_state_file_path(step_context.context), + openai_client=step_context.llm_config.openai_client_factory(), + openai_model=step_context.llm_config.openai_model, + context=step_context.context, + state_id=_inspector.state_id, + ) as gce: + try: + result = await gce.step_conversation(message) + except Exception as e: + logger.exception("failed to execute guided conversation") + return IncompleteErrorResult( + message=f"Failed to execute guided conversation: {e}", + debug={"error": str(e)}, + ) + + debug["guided-conversation"] = gce.to_json() + + logger.info("guided-conversation result: %s", result) + + fill_form_gc_artifact = gce.artifact.artifact.model_dump(mode="json") + logger.info("guided-conversation artifact: %s", gce.artifact) + + if result.is_conversation_over: + return CompleteResult( + ai_message="", + artifact=fill_form_gc_artifact, + debug=debug, + ) + + return IncompleteResult(message=result.ai_message or "", debug=debug) + + +def _form_fields_to_artifact(form_fields: list[state.FormField]): + field_definitions: dict[str, tuple[Any, Any]] = {} + required_fields = [] + for field in form_fields: + if field.required: + required_fields.append(field.id) + + match field.type: + case "string": + field_definitions[field.id] = (str, Field(title=field.name, description=field.description)) + + case "bool": + field_definitions[field.id] = (bool, Field(title=field.name, description=field.description)) + + case "multiple_choice": + field_definitions[field.id] = ( + Literal[tuple(field.options)], + Field(title=field.name, description=field.description), + ) + + return create_model( + "FilledFormArtifact", + **field_definitions, # type: ignore + ) # type: ignore + + +def _get_state_file_path(context: ConversationContext) -> Path: + return _guided_conversation.path_for_state(context, "fill_form") + + +_inspector = FileStateInspector( + display_name="Fill-Form Guided-Conversation", + file_path_source=_get_state_file_path, +) + + +async def _extract( + llm_config: LLMConfig, + config: ExtractCandidateFieldValuesConfig, + form_fields: list[state.FormField], + document_content: str, +) -> tuple[FieldValueCandidates, dict[str, Any]]: + class _SerializationModel(BaseModel): + fields: list[state.FormField] + + messages: list[ChatCompletionMessageParam] = [ + { + "role": "system", + "content": config.instruction.replace( + "{{form_fields}}", _SerializationModel(fields=form_fields).model_dump_json(indent=4) + ), + }, + { + "role": "user", + "content": document_content, + }, + ] + + return await _llm.structured_completion( + llm_config=llm_config, + messages=messages, + response_model=FieldValueCandidates, + ) diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/types.py b/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/types.py similarity index 87% rename from assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/types.py rename to assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/types.py index f4cdb8a3..4e11aa71 100644 --- a/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/types.py +++ b/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/types.py @@ -1,9 +1,8 @@ from dataclasses import dataclass -from typing import Annotated, Awaitable, Callable, Generic, Sequence, TypeVar +from typing import Annotated, AsyncIterable, Callable, Generic, TypeVar from guided_conversation.utils.resources import ResourceConstraint, ResourceConstraintMode, ResourceConstraintUnit from openai import AsyncOpenAI -from openai.types.chat import ChatCompletionMessageParam from pydantic import BaseModel, ConfigDict, Field from semantic_workbench_assistant.assistant_app.context import ConversationContext @@ -18,12 +17,24 @@ class LLMConfig: ConfigT = TypeVar("ConfigT", bound=BaseModel) +@dataclass +class UserAttachment: + filename: str + content: str + + +@dataclass +class UserInput: + message: str | None + attachments: AsyncIterable[UserAttachment] + + @dataclass class Context(Generic[ConfigT]): context: ConversationContext llm_config: LLMConfig config: ConfigT - get_attachment_messages: Callable[[Sequence[str]], Awaitable[Sequence[ChatCompletionMessageParam]]] + latest_user_input: UserInput @dataclass @@ -33,12 +44,11 @@ class Result: @dataclass class IncompleteResult(Result): - ai_message: str + message: str @dataclass -class IncompleteErrorResult(Result): - error_message: str +class IncompleteErrorResult(IncompleteResult): ... class ResourceConstraintDefinition(BaseModel): diff --git a/assistants/prospector-assistant/assistant/chat.py b/assistants/prospector-assistant/assistant/chat.py index 4d26c45e..2e90cd1e 100644 --- a/assistants/prospector-assistant/assistant/chat.py +++ b/assistants/prospector-assistant/assistant/chat.py @@ -9,7 +9,8 @@ import logging import re import traceback -from typing import Any, Sequence +from contextlib import asynccontextmanager +from typing import Any, Awaitable, Callable import deepmerge import openai_client @@ -34,9 +35,9 @@ ) from . import legacy -from .agents import form_fill_agent from .agents.artifact_agent import Artifact, ArtifactAgent, ArtifactConversationInspectorStateProvider from .agents.document_agent import DocumentAgent +from .agents.form_fill_extension import FormFillExtension, LLMConfig from .config import AssistantConfigModel logger = logging.getLogger(__name__) @@ -78,9 +79,8 @@ async def content_evaluator_factory(context: ConversationContext) -> ContentSafe }, ) -form_fill_agent.extend(assistant) - attachments_extension = AttachmentsExtension(assistant) +form_fill_extension = FormFillExtension(assistant) # # create the FastAPI app instance @@ -150,7 +150,7 @@ async def on_chat_message_created( """ # update the participant status to indicate the assistant is thinking - async with context.set_status("thinking..."): + async with send_error_message_on_exception(context), context.set_status("thinking..."): config = await assistant_config.get(context.assistant) metadata: dict[str, Any] = {"debug": {"content_safety": event.data.get(content_safety.metadata_key, {})}} @@ -165,43 +165,12 @@ async def on_chat_message_created( is_doc_agent_running = await document_agent_respond_to_conversation(config, context, message, metadata) return - try: - await form_fill_agent.execute( - llm_config=form_fill_agent.LLMConfig( - openai_client_factory=lambda: openai_client.create_client(config.service_config), - openai_model=config.request_config.openai_model, - max_response_tokens=config.request_config.response_tokens, - ), - config=config.agents_config.form_fill_agent, - context=context, - latest_user_message=message.content, - get_attachment_messages=form_fill_agent_get_attachments(context, config), - ) - - except Exception as e: - await context.send_messages( - NewConversationMessage( - content=f"An error occurred: {e}", - message_type=MessageType.notice, - metadata={"debug": {"stack_trace": traceback.format_exc()}}, - ) - ) + await form_fill_execute(context, message) # # Prospector assistant response # await respond_to_conversation(context, config, message, metadata) -def form_fill_agent_get_attachments(context: ConversationContext, config: AssistantConfigModel): - async def get(filenames: Sequence[str]): - return await attachments_extension.get_completion_messages_for_attachments( - context, - config.agents_config.attachment_agent, - include_filenames=list(filenames), - ) - - return get - - background_tasks: set[asyncio.Task] = set() @@ -216,6 +185,10 @@ async def on_conversation_created(context: ConversationContext) -> None: if welcome_sent_before: return + task = asyncio.create_task(welcome_message(context)) + background_tasks.add(task) + task.add_done_callback(background_tasks.remove) + # send a welcome message to the conversation # welcome_message = config.welcome_message # await context.send_messages( @@ -226,36 +199,72 @@ async def on_conversation_created(context: ConversationContext) -> None: # ) # ) - task = asyncio.create_task(welcome_message(context)) - background_tasks.add(task) - task.add_done_callback(background_tasks.remove) - async def welcome_message(context: ConversationContext) -> None: + async with send_error_message_on_exception(context), context.set_status("thinking..."): + await form_fill_execute(context, None) + + +@asynccontextmanager +async def send_error_message_on_exception(context: ConversationContext): + try: + yield + except Exception as e: + await context.send_messages( + NewConversationMessage( + content=f"An error occurred: {e}", + message_type=MessageType.notice, + metadata={"debug": {"stack_trace": traceback.format_exc()}}, + ) + ) + + +# endregion + +# +# region Form fill extension helpers +# + + +async def form_fill_execute(context: ConversationContext, message: ConversationMessage | None) -> None: + """ + Execute the form fill agent to respond to the conversation message. + """ config = await assistant_config.get(context.assistant) + await form_fill_extension.execute( + llm_config=LLMConfig( + openai_client_factory=lambda: openai_client.create_client(config.service_config), + openai_model=config.request_config.openai_model, + max_response_tokens=config.request_config.response_tokens, + ), + config=config.agents_config.form_fill_agent, + context=context, + latest_user_message=message.content if message else None, + latest_attachment_filenames=message.filenames if message else [], + get_attachment_content=form_fill_extension_get_attachment(context, config), + ) - async with context.set_status("thinking..."): - try: - await form_fill_agent.execute( - llm_config=form_fill_agent.LLMConfig( - openai_client_factory=lambda: openai_client.create_client(config.service_config), - openai_model=config.request_config.openai_model, - max_response_tokens=config.request_config.response_tokens, - ), - config=config.agents_config.form_fill_agent, - context=context, - latest_user_message=None, - get_attachment_messages=form_fill_agent_get_attachments(context, config), - ) - except Exception as e: - await context.send_messages( - NewConversationMessage( - content=f"An error occurred: {e}", - message_type=MessageType.notice, - metadata={"debug": {"stack_trace": traceback.format_exc()}}, - ) - ) +def form_fill_extension_get_attachment( + context: ConversationContext, config: AssistantConfigModel +) -> Callable[[str], Awaitable[str]]: + """Helper function for the form_fill_extension to get the content of an attachment by filename.""" + + async def get(filename: str) -> str: + messages = await attachments_extension.get_completion_messages_for_attachments( + context, + config.agents_config.attachment_agent, + include_filenames=[filename], + ) + if not messages: + return "" + + # filter down to the messages that contain the attachment (ie. don't include the system messages) + return "\n\n".join( + (str(message.get("content")) for message in messages if "" in str(message.get("content"))) + ) + + return get # endregion @@ -487,7 +496,7 @@ class StructuredResponseFormat(BaseModel): method_metadata_key: { "request": { "model": config.request_config.openai_model, - "messages": openai_client.truncate_messages_for_logging(completion_messages), + "messages": completion_messages, "max_tokens": config.request_config.response_tokens, "response_format": StructuredResponseFormat.model_json_schema(), }, @@ -510,7 +519,7 @@ class StructuredResponseFormat(BaseModel): method_metadata_key: { "request": { "model": config.request_config.openai_model, - "messages": openai_client.truncate_messages_for_logging(completion_messages), + "messages": completion_messages, }, "error": str(e), }, @@ -543,7 +552,7 @@ class StructuredResponseFormat(BaseModel): method_metadata_key: { "request": { "model": config.request_config.openai_model, - "messages": openai_client.truncate_messages_for_logging(completion_messages), + "messages": completion_messages, "max_tokens": config.request_config.response_tokens, }, "response": completion.model_dump() if completion else "[no response from openai]", @@ -566,7 +575,7 @@ class StructuredResponseFormat(BaseModel): method_metadata_key: { "request": { "model": config.request_config.openai_model, - "messages": openai_client.truncate_messages_for_logging(completion_messages), + "messages": completion_messages, }, "error": str(e), }, diff --git a/assistants/prospector-assistant/assistant/config.py b/assistants/prospector-assistant/assistant/config.py index 7c09b5a4..354cca29 100644 --- a/assistants/prospector-assistant/assistant/config.py +++ b/assistants/prospector-assistant/assistant/config.py @@ -8,7 +8,7 @@ from . import helpers from .agents.artifact_agent import ArtifactAgentConfigModel -from .agents.form_fill_agent import FormFillAgentConfig +from .agents.form_fill_extension import FormFillConfig # The semantic workbench app uses react-jsonschema-form for rendering # dynamic configuration forms based on the configuration model and UI schema @@ -26,9 +26,7 @@ class AgentsConfigModel(BaseModel): - form_fill_agent: Annotated[FormFillAgentConfig, Field(title="Form Fill Agent Configuration")] = ( - FormFillAgentConfig() - ) + form_fill_agent: Annotated[FormFillConfig, Field(title="Form Fill Agent Configuration")] = FormFillConfig() artifact_agent: Annotated[ ArtifactAgentConfigModel,