From e4537e4567cf3ad6eb2219738af72316a223f119 Mon Sep 17 00:00:00 2001 From: Mark Waddle Date: Tue, 5 Nov 2024 15:25:50 -0800 Subject: [PATCH] Re-organize fill-form agent (#214) To ease reading/understanding/debugging of code --- .../agents/form_fill_agent/__init__.py | 2 +- .../assistant/agents/form_fill_agent/agent.py | 124 +++++++----------- .../agents/form_fill_agent/config.py | 15 +-- .../agents/form_fill_agent/inspector.py | 5 + .../agents/form_fill_agent/llm_config.py | 0 .../assistant/agents/form_fill_agent/state.py | 6 +- .../assistant/agents/form_fill_agent/step.py | 40 ------ .../form_fill_agent/steps/_attachments.py | 57 ++++++++ .../steps/{gce.py => _guided_conversation.py} | 69 ++++------ .../{acquire_form.py => acquire_form_step.py} | 97 +++++++------- ..._fields.py => extract_form_fields_step.py} | 106 +++++++-------- .../steps/{fill_form.py => fill_form_step.py} | 92 +++++++------ .../{gce_config.py => steps/types.py} | 39 +++++- 13 files changed, 329 insertions(+), 323 deletions(-) delete mode 100644 assistants/prospector-assistant/assistant/agents/form_fill_agent/llm_config.py delete mode 100644 assistants/prospector-assistant/assistant/agents/form_fill_agent/step.py create mode 100644 assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/_attachments.py rename assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/{gce.py => _guided_conversation.py} (58%) rename assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/{acquire_form.py => acquire_form_step.py} (76%) rename assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/{extract_form_fields.py => extract_form_fields_step.py} (84%) rename assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/{fill_form.py => fill_form_step.py} (82%) rename assistants/prospector-assistant/assistant/agents/form_fill_agent/{gce_config.py => steps/types.py} (68%) diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_agent/__init__.py b/assistants/prospector-assistant/assistant/agents/form_fill_agent/__init__.py index 21cba969..b28e952c 100644 --- a/assistants/prospector-assistant/assistant/agents/form_fill_agent/__init__.py +++ b/assistants/prospector-assistant/assistant/agents/form_fill_agent/__init__.py @@ -1,6 +1,6 @@ from .agent import execute, extend from .config import FormFillAgentConfig -from .step import LLMConfig +from .steps.types import LLMConfig __all__ = [ "execute", diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_agent/agent.py b/assistants/prospector-assistant/assistant/agents/form_fill_agent/agent.py index eceaf9a5..8a350e41 100644 --- a/assistants/prospector-assistant/assistant/agents/form_fill_agent/agent.py +++ b/assistants/prospector-assistant/assistant/agents/form_fill_agent/agent.py @@ -8,8 +8,8 @@ from . import state from .config import FormFillAgentConfig -from .step import Context, IncompleteErrorResult, IncompleteResult, LLMConfig -from .steps import acquire_form, extract_form_fields, fill_form +from .steps import acquire_form_step, extract_form_fields_step, fill_form_step +from .steps.types import ConfigT, Context, IncompleteErrorResult, IncompleteResult, LLMConfig logger = logging.getLogger(__name__) @@ -23,133 +23,105 @@ async def execute( ) -> None: user_messages = [latest_user_message] - async with state.agent_state(context) as agent_state: - for mode in state.FormFillAgentMode: - if mode in agent_state.mode_debug_log: - continue - - agent_state.mode_debug_log[mode] = [] + def build_step_context(config: ConfigT) -> Context[ConfigT]: + return Context( + context=context, llm_config=llm_config, config=config, get_attachment_messages=get_attachment_messages + ) + async with state.agent_state(context) as agent_state: while True: - logger.info("form-fill-agent step; mode: %s", agent_state.mode) + logger.info("form-fill-agent execute loop; mode: %s", agent_state.mode) match agent_state.mode: case state.FormFillAgentMode.acquire_form_step: - step_context = Context( - context=context, - llm_config=llm_config, - config=config.acquire_form_config, - get_attachment_messages=get_attachment_messages, - ) - - result = await acquire_form.execute( - step_context=step_context, + result = await acquire_form_step.execute( + step_context=build_step_context(config.acquire_form_config), latest_user_message=user_messages.pop() if user_messages else None, ) - agent_state.mode_debug_log[agent_state.mode].insert(0, result.debug) match result: - case IncompleteResult(): - await _send_message(context, result.ai_message, result.debug) - return - - case IncompleteErrorResult(): - await _send_error_message(context, result.error_message, result.debug) - return - - case acquire_form.CompleteResult(): + case acquire_form_step.CompleteResult(): await _send_message(context, result.ai_message, result.debug) agent_state.form_filename = result.filename agent_state.mode = state.FormFillAgentMode.extract_form_fields_step - continue + + case IncompleteResult() | IncompleteErrorResult(): + await _handle_incomplete_results(context, result) + return case _: raise ValueError(f"Unexpected result: {result}") case state.FormFillAgentMode.extract_form_fields_step: - step_context = Context( - context=context, - llm_config=llm_config, - config=config.extract_form_fields_config, - get_attachment_messages=get_attachment_messages, - ) - - result = await extract_form_fields.execute( - step_context=step_context, + result = await extract_form_fields_step.execute( + step_context=build_step_context(config.extract_form_fields_config), filename=agent_state.form_filename, ) - agent_state.mode_debug_log[agent_state.mode].insert(0, result.debug) match result: - case IncompleteErrorResult(): - await _send_error_message(context, result.error_message, result.debug) - return - - case IncompleteResult(): - await _send_message(context, result.ai_message, result.debug) - return - - case extract_form_fields.CompleteResult(): + case extract_form_fields_step.CompleteResult(): await _send_message(context, result.ai_message, result.debug) agent_state.extracted_form_fields = result.extracted_form_fields agent_state.mode = state.FormFillAgentMode.fill_form_step - continue + + case IncompleteResult() | IncompleteErrorResult(): + await _handle_incomplete_results(context, result) + return case _: raise ValueError(f"Unexpected result: {result}") case state.FormFillAgentMode.fill_form_step: - step_context = Context( - context=context, - llm_config=llm_config, - config=config.fill_form_config, - get_attachment_messages=get_attachment_messages, - ) - - result = await fill_form.execute( - step_context=step_context, + result = await fill_form_step.execute( + step_context=build_step_context(config.fill_form_config), latest_user_message=user_messages.pop() if user_messages else None, form_fields=agent_state.extracted_form_fields, ) - agent_state.mode_debug_log[agent_state.mode].insert(0, result.debug) match result: - case IncompleteResult(): - await _send_message(context, result.ai_message, result.debug) - return - - case IncompleteErrorResult(): - await _send_error_message(context, result.error_message, result.debug) - return - - case fill_form.CompleteResult(): + case fill_form_step.CompleteResult(): await _send_message(context, result.ai_message, result.debug) agent_state.fill_form_gc_artifact = result.artifact agent_state.mode = state.FormFillAgentMode.generate_filled_form_step - continue + + case IncompleteResult() | IncompleteErrorResult(): + await _handle_incomplete_results(context, result) + return case _: raise ValueError(f"Unexpected result: {result}") case state.FormFillAgentMode.generate_filled_form_step: - await context.send_messages( - NewConversationMessage( - content="I'd love to generate the fill form now, but it's not yet implemented. :)" - ) + await _send_message( + context, "I'd love to generate the filled-out form now, but it's not yet implemented. :)", {} ) return case state.FormFillAgentMode.end_conversation: - await context.send_messages(NewConversationMessage(content="Conversation has ended.")) + await _send_message(context, "Conversation has ended.", {}) return case _: raise ValueError(f"Unexpected mode: {state.mode}") +async def _handle_incomplete_results( + context: ConversationContext, result: IncompleteErrorResult | IncompleteResult +) -> None: + match result: + case IncompleteResult(): + await _send_message(context, result.ai_message, result.debug) + + case IncompleteErrorResult(): + await _send_error_message(context, result.error_message, result.debug) + + case _: + raise ValueError(f"Unexpected incomplete result: {result}") + + async def _send_message(context: ConversationContext, message: str, debug: dict) -> None: if not message: return @@ -182,5 +154,5 @@ def extend(app: AssistantAppProtocol) -> None: app.add_inspector_state_provider(state.inspector.state_id, state.inspector) # for step level states - acquire_form.extend(app) - fill_form.extend(app) + acquire_form_step.extend(app) + fill_form_step.extend(app) diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_agent/config.py b/assistants/prospector-assistant/assistant/agents/form_fill_agent/config.py index 4d55df09..8e209b2e 100644 --- a/assistants/prospector-assistant/assistant/agents/form_fill_agent/config.py +++ b/assistants/prospector-assistant/assistant/agents/form_fill_agent/config.py @@ -2,22 +2,21 @@ from pydantic import BaseModel, Field -from . import gce_config -from .steps import acquire_form, extract_form_fields, fill_form +from .steps import acquire_form_step, extract_form_fields_step, fill_form_step, types class FormFillAgentConfig(BaseModel): acquire_form_config: Annotated[ - gce_config.GuidedConversationDefinition, + types.GuidedConversationDefinition, Field(title="Form Acquisition", description="Guided conversation for acquiring a form from the user."), - ] = acquire_form.definition.model_copy() + ] = acquire_form_step.definition.model_copy() extract_form_fields_config: Annotated[ - extract_form_fields.ExtractFormFieldsConfig, + extract_form_fields_step.ExtractFormFieldsConfig, Field(title="Extract Form Fields", description="Configuration for extracting form fields from the form."), - ] = extract_form_fields.ExtractFormFieldsConfig() + ] = extract_form_fields_step.ExtractFormFieldsConfig() fill_form_config: Annotated[ - gce_config.GuidedConversationDefinition, + types.GuidedConversationDefinition, Field(title="Fill Form", description="Guided conversation for filling out the form."), - ] = fill_form.definition.model_copy() + ] = fill_form_step.definition.model_copy() diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_agent/inspector.py b/assistants/prospector-assistant/assistant/agents/form_fill_agent/inspector.py index a67af357..19f790d8 100644 --- a/assistants/prospector-assistant/assistant/agents/form_fill_agent/inspector.py +++ b/assistants/prospector-assistant/assistant/agents/form_fill_agent/inspector.py @@ -13,6 +13,10 @@ class FileStateInspector(ReadOnlyAssistantConversationInspectorStateProvider): + """ + A conversation inspector state provider that reads the state from a file and displays it as a yaml code block. + """ + def __init__( self, display_name: str, @@ -53,6 +57,7 @@ def read_state(path: Path) -> dict: @contextlib.asynccontextmanager async def state_change_event_after(context: ConversationContext, state_id: str, set_focus=False) -> AsyncIterator[None]: + """Raise a state change event after the context manager block is executed (optionally set focus as well)""" yield if set_focus: await context.send_conversation_state_event(AssistantStateEvent(state_id=state_id, event="focus", state=None)) diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_agent/llm_config.py b/assistants/prospector-assistant/assistant/agents/form_fill_agent/llm_config.py deleted file mode 100644 index e69de29b..00000000 diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_agent/state.py b/assistants/prospector-assistant/assistant/agents/form_fill_agent/state.py index d14e0dfa..e5f445fc 100644 --- a/assistants/prospector-assistant/assistant/agents/form_fill_agent/state.py +++ b/assistants/prospector-assistant/assistant/agents/form_fill_agent/state.py @@ -36,8 +36,6 @@ class FormFillAgentState(BaseModel): extracted_form_fields: list[FormField] = [] fill_form_gc_artifact: dict | None = None - mode_debug_log: dict[FormFillAgentMode, list[dict]] = {} - def path_for_state(context: ConversationContext) -> Path: return storage_directory_for_context(context) / "state.json" @@ -48,6 +46,10 @@ def path_for_state(context: ConversationContext) -> Path: @asynccontextmanager async def agent_state(context: ConversationContext) -> AsyncIterator[FormFillAgentState]: + """ + Context manager that provides the agent state, reading it from disk, and saving back + to disk after the context manager block is executed. + """ state = current_state.get() if state is not None: yield state diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_agent/step.py b/assistants/prospector-assistant/assistant/agents/form_fill_agent/step.py deleted file mode 100644 index c9c127d7..00000000 --- a/assistants/prospector-assistant/assistant/agents/form_fill_agent/step.py +++ /dev/null @@ -1,40 +0,0 @@ -from dataclasses import dataclass -from typing import Awaitable, Callable, Generic, Sequence, TypeVar - -from openai import AsyncOpenAI -from openai.types.chat import ChatCompletionMessageParam -from pydantic import BaseModel -from semantic_workbench_assistant.assistant_app.context import ConversationContext - - -@dataclass -class LLMConfig: - openai_client_factory: Callable[[], AsyncOpenAI] - openai_model: str - max_response_tokens: int - - -ConfigT = TypeVar("ConfigT", bound=BaseModel) - - -@dataclass -class Context(Generic[ConfigT]): - context: ConversationContext - llm_config: LLMConfig - config: ConfigT - get_attachment_messages: Callable[[Sequence[str]], Awaitable[Sequence[ChatCompletionMessageParam]]] - - -@dataclass -class Result: - debug: dict - - -@dataclass -class IncompleteResult(Result): - ai_message: str - - -@dataclass -class IncompleteErrorResult(Result): - error_message: str diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/_attachments.py b/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/_attachments.py new file mode 100644 index 00000000..0f6424e8 --- /dev/null +++ b/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/_attachments.py @@ -0,0 +1,57 @@ +""" +Utility functions for handling attachments in chat messages. +""" + +from typing import Awaitable, Callable, Sequence + +from openai.types.chat import ChatCompletionMessageParam +from semantic_workbench_assistant.assistant_app.context import ConversationContext + +from .. import state + + +async def message_with_recent_attachments( + context: ConversationContext, + latest_user_message: str | None, + get_attachment_messages: Callable[[Sequence[str]], Awaitable[Sequence[ChatCompletionMessageParam]]], +) -> str: + files = await context.get_files() + + new_filenames = set() + + async with state.agent_state(context) as agent_state: + max_timestamp = agent_state.most_recent_attachment_timestamp + for file in files.files: + if file.updated_datetime.timestamp() <= agent_state.most_recent_attachment_timestamp: + continue + + max_timestamp = max(file.updated_datetime.timestamp(), max_timestamp) + new_filenames.add(file.filename) + + agent_state.most_recent_attachment_timestamp = max_timestamp + + attachment_messages = await get_attachment_messages(list(new_filenames)) + + return "\n\n".join( + ( + latest_user_message or "", + *( + str(attachment.get("content")) + for attachment in attachment_messages + if "" in str(attachment.get("content", "")) + ), + ), + ) + + +async def attachment_for_filename( + filename: str, get_attachment_messages: Callable[[Sequence[str]], Awaitable[Sequence[ChatCompletionMessageParam]]] +) -> str: + attachment_messages = await get_attachment_messages([filename]) + return "\n\n".join( + ( + str(attachment.get("content")) + for attachment in attachment_messages + if "" in str(attachment.get("content", "")) + ) + ) diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/gce.py b/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/_guided_conversation.py similarity index 58% rename from assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/gce.py rename to assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/_guided_conversation.py index 4033952b..81884f5d 100644 --- a/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/gce.py +++ b/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/_guided_conversation.py @@ -1,39 +1,48 @@ +""" +Utility functions for working with guided conversations. +""" + import asyncio import contextlib import json from collections import defaultdict from contextlib import asynccontextmanager from pathlib import Path -from typing import AsyncIterator, Awaitable, Callable, Sequence +from typing import AsyncIterator -from assistant.agents.form_fill_agent.inspector import state_change_event_after from guided_conversation.guided_conversation_agent import GuidedConversation from openai import AsyncOpenAI -from openai.types.chat import ChatCompletionMessageParam from pydantic import BaseModel from semantic_kernel import Kernel from semantic_kernel.connectors.ai.open_ai import OpenAIChatCompletion from semantic_workbench_assistant.assistant_app.context import ConversationContext, storage_directory_for_context -from .. import gce_config, state +from ..inspector import state_change_event_after +from .types import GuidedConversationDefinition -guided_conversation_locks: dict[Path, asyncio.Lock] = defaultdict(asyncio.Lock) +_state_locks: dict[Path, asyncio.Lock] = defaultdict(asyncio.Lock) @asynccontextmanager -async def guided_conversation_with_state( +async def engine( openai_client: AsyncOpenAI, openai_model: str, - definition: gce_config.GuidedConversationDefinition, + definition: GuidedConversationDefinition, artifact_type: type[BaseModel], state_file_path: Path, context: ConversationContext, state_id: str, ) -> AsyncIterator[GuidedConversation]: - # ensure that only one guided conversation is executed at a time for any given state file - # ie. require them to run sequentially - async with guided_conversation_locks[state_file_path], state_change_event_after(context, state_id, set_focus=True): - kernel, service_id = build_kernel_with_service(openai_client, openai_model) + """ + Context manager that provides a guided conversation engine with state, reading it from disk, and saving back + to disk after the context manager block is executed. + + NOTE: This context manager uses a lock to ensure that only one guided conversation is executed at a time for any + given state file. + """ + + async with _state_locks[state_file_path], state_change_event_after(context, state_id, set_focus=True): + kernel, service_id = _build_kernel_with_service(openai_client, openai_model) state: dict | None = None with contextlib.suppress(FileNotFoundError): @@ -82,7 +91,7 @@ async def guided_conversation_with_state( state_file_path.write_text(json.dumps(state), encoding="utf-8") -def build_kernel_with_service(openai_client: AsyncOpenAI, openai_model: str) -> tuple[Kernel, str]: +def _build_kernel_with_service(openai_client: AsyncOpenAI, openai_model: str) -> tuple[Kernel, str]: kernel = Kernel() service_id = "gc_main" chat_service = OpenAIChatCompletion( @@ -94,41 +103,7 @@ def build_kernel_with_service(openai_client: AsyncOpenAI, openai_model: str) -> return kernel, service_id -async def message_with_recent_attachments( - context: ConversationContext, - latest_user_message: str | None, - get_attachment_messages: Callable[[Sequence[str]], Awaitable[Sequence[ChatCompletionMessageParam]]], -) -> str: - files = await context.get_files() - - new_filenames = set() - - async with state.agent_state(context) as agent_state: - max_timestamp = agent_state.most_recent_attachment_timestamp - for file in files.files: - if file.updated_datetime.timestamp() <= agent_state.most_recent_attachment_timestamp: - continue - - max_timestamp = max(file.updated_datetime.timestamp(), max_timestamp) - new_filenames.add(file.filename) - - agent_state.most_recent_attachment_timestamp = max_timestamp - - attachment_messages = await get_attachment_messages(list(new_filenames)) - - return "\n\n".join( - ( - latest_user_message or "", - *( - str(attachment.get("content")) - for attachment in attachment_messages - if "" in str(attachment.get("content", "")) - ), - ), - ) - - -def path_for_guided_conversation_state(context: ConversationContext, dir: str) -> Path: +def path_for_state(context: ConversationContext, dir: str) -> Path: dir_path = storage_directory_for_context(context) / dir dir_path.mkdir(parents=True, exist_ok=True) return dir_path / "guided_conversation_state.json" diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/acquire_form.py b/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/acquire_form_step.py similarity index 76% rename from assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/acquire_form.py rename to assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/acquire_form_step.py index 1b806a2f..10de7789 100644 --- a/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/acquire_form.py +++ b/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/acquire_form_step.py @@ -7,42 +7,22 @@ from semantic_workbench_assistant.assistant_app.context import ConversationContext from semantic_workbench_assistant.assistant_app.protocol import AssistantAppProtocol -from .. import gce_config from ..inspector import FileStateInspector -from ..step import Context, IncompleteErrorResult, IncompleteResult, Result -from . import gce +from . import _attachments, _guided_conversation +from .types import ( + Context, + GuidedConversationDefinition, + IncompleteErrorResult, + IncompleteResult, + ResourceConstraintDefinition, + Result, +) logger = logging.getLogger(__name__) -class FormArtifact(BaseModel): - title: str = Field(description="The title of the form.", default="") - filename: str = Field(description="The filename of the form.", default="") - - -definition = gce_config.GuidedConversationDefinition( - rules=[ - "DO NOT suggest forms or create a form for the user.", - "Politely request another file if the provided file is not a form.", - "Terminate conversation if inappropriate content is requested.", - ], - conversation_flow=( - """ - 1. Inform the user that our goal is to help the user fill out a form. - 2. Ask the user to provide a file that contains a form. The file can be PDF, TXT, or DOCX. - 3. When you receive a file, determine if the file looks to be a form. - 4. If the file is not a form, inform the user that the file is not a form. Ask them to provide a different file. - 5. If the form is a file, update the artifcat with the title and filename of the form. - 6. Inform the user that you will now extract the form fields, so that you can assist them in filling it out. - """ - ), - context="", - resource_constraint=gce_config.ResourceConstraintDefinition( - quantity=5, - unit=ResourceConstraintUnit.MINUTES, - mode=ResourceConstraintMode.MAXIMUM, - ), -) +def extend(app: AssistantAppProtocol) -> None: + app.add_inspector_state_provider(_inspector.state_id, _inspector) @dataclass @@ -52,18 +32,18 @@ class CompleteResult(Result): async def execute( - step_context: Context[gce_config.GuidedConversationDefinition], + step_context: Context[GuidedConversationDefinition], latest_user_message: str | None, ) -> IncompleteResult | IncompleteErrorResult | CompleteResult: """ Step: acquire a form from the user Approach: Guided conversation """ - message_with_attachments = await gce.message_with_recent_attachments( + message_with_attachments = await _attachments.message_with_recent_attachments( step_context.context, latest_user_message, step_context.get_attachment_messages ) - async with gce.guided_conversation_with_state( + async with _guided_conversation.engine( definition=step_context.config, artifact_type=FormArtifact, state_file_path=_get_state_file_path(step_context.context), @@ -71,9 +51,9 @@ async def execute( openai_model=step_context.llm_config.openai_model, context=step_context.context, state_id=_inspector.state_id, - ) as guided_conversation: + ) as gce: try: - result = await guided_conversation.step_conversation(message_with_attachments) + result = await gce.step_conversation(message_with_attachments) except Exception as e: logger.exception("failed to execute guided conversation") return IncompleteErrorResult( @@ -81,10 +61,12 @@ async def execute( debug={"error": str(e)}, ) + debug = {"guided-conversation": gce.to_json()} + logger.info("guided-conversation result: %s", result) - acquire_form_gc_artifact = guided_conversation.artifact.artifact.model_dump(mode="json") - logger.info("guided-conversation artifact: %s", guided_conversation.artifact) + acquire_form_gc_artifact = gce.artifact.artifact.model_dump(mode="json") + logger.info("guided-conversation artifact: %s", gce.artifact) form_filename = acquire_form_gc_artifact.get("filename", "") @@ -92,17 +74,14 @@ async def execute( return CompleteResult( ai_message=result.ai_message or "", filename=form_filename, - debug={"artifact": acquire_form_gc_artifact}, + debug=debug, ) - return IncompleteResult( - ai_message=result.ai_message or "", - debug={"artifact": acquire_form_gc_artifact}, - ) + return IncompleteResult(ai_message=result.ai_message or "", debug=debug) def _get_state_file_path(context: ConversationContext) -> Path: - return gce.path_for_guided_conversation_state(context, "acquire_form") + return _guided_conversation.path_for_state(context, "acquire_form") _inspector = FileStateInspector( @@ -112,5 +91,31 @@ def _get_state_file_path(context: ConversationContext) -> Path: ) -def extend(app: AssistantAppProtocol) -> None: - app.add_inspector_state_provider(_inspector.state_id, _inspector) +class FormArtifact(BaseModel): + title: str = Field(description="The title of the form.", default="") + filename: str = Field(description="The filename of the form.", default="") + + +definition = GuidedConversationDefinition( + rules=[ + "DO NOT suggest forms or create a form for the user.", + "Politely request another file if the provided file is not a form.", + "Terminate conversation if inappropriate content is requested.", + ], + conversation_flow=( + """ + 1. Inform the user that our goal is to help the user fill out a form. + 2. Ask the user to provide a file that contains a form. The file can be PDF, TXT, or DOCX. + 3. When you receive a file, determine if the file looks to be a form. + 4. If the file is not a form, inform the user that the file is not a form. Ask them to provide a different file. + 5. If the form is a file, update the artifcat with the title and filename of the form. + 6. Inform the user that you will now extract the form fields, so that you can assist them in filling it out. + """ + ), + context="", + resource_constraint=ResourceConstraintDefinition( + quantity=5, + unit=ResourceConstraintUnit.MINUTES, + mode=ResourceConstraintMode.MAXIMUM, + ), +) diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/extract_form_fields.py b/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/extract_form_fields_step.py similarity index 84% rename from assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/extract_form_fields.py rename to assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/extract_form_fields_step.py index f98f975a..44cc3789 100644 --- a/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/extract_form_fields.py +++ b/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/extract_form_fields_step.py @@ -1,13 +1,14 @@ import logging from dataclasses import dataclass -from typing import Annotated, Any, Awaitable, Callable, Sequence +from typing import Annotated, Any import openai_client from openai.types.chat import ChatCompletionMessageParam from pydantic import BaseModel, Field from .. import state -from ..step import Context, IncompleteErrorResult, IncompleteResult, LLMConfig, Result +from . import _attachments +from .types import Context, IncompleteErrorResult, IncompleteResult, LLMConfig, Result logger = logging.getLogger(__name__) @@ -22,6 +23,50 @@ class ExtractFormFieldsConfig(BaseModel): ) +@dataclass +class CompleteResult(Result): + ai_message: str + extracted_form_fields: list[state.FormField] + + +async def execute( + step_context: Context[ExtractFormFieldsConfig], + filename: str, +) -> IncompleteResult | IncompleteErrorResult | CompleteResult: + """ + Step: extract form fields from the form file content + Approach: Chat completion with LLM + """ + + file_content = await _attachments.attachment_for_filename(filename, step_context.get_attachment_messages) + async with step_context.context.set_status("inspecting form ..."): + try: + extracted_form_fields, metadata = await _extract( + llm_config=step_context.llm_config, + config=step_context.config, + form_content=file_content, + ) + + except Exception as e: + logger.exception("failed to extract form fields") + return IncompleteErrorResult( + error_message=f"Failed to extract form fields: {e}", + debug={"error": str(e)}, + ) + + if extracted_form_fields.error_message: + return IncompleteResult( + ai_message=extracted_form_fields.error_message, + debug=metadata, + ) + + return CompleteResult( + ai_message="", + extracted_form_fields=extracted_form_fields.fields, + debug=metadata, + ) + + class FormFields(BaseModel): error_message: str = Field(description="The error message in the case that the form fields could not be extracted.") fields: list[state.FormField] = Field(description="The fields in the form.") @@ -74,60 +119,3 @@ async def _extract( } return response.choices[0].message.parsed, metadata - - -@dataclass -class CompleteResult(Result): - ai_message: str - extracted_form_fields: list[state.FormField] - - -async def execute( - step_context: Context[ExtractFormFieldsConfig], - filename: str, -) -> IncompleteResult | IncompleteErrorResult | CompleteResult: - """ - Step: extract form fields from the form file content - Approach: Chat completion with LLM - """ - - file_content = await attachment_for_filename(filename, step_context.get_attachment_messages) - async with step_context.context.set_status("inspecting form ..."): - try: - extracted_form_fields, metadata = await _extract( - llm_config=step_context.llm_config, - config=step_context.config, - form_content=file_content, - ) - - except Exception as e: - logger.exception("failed to extract form fields") - return IncompleteErrorResult( - error_message=f"Failed to extract form fields: {e}", - debug={"error": str(e)}, - ) - - if extracted_form_fields.error_message: - return IncompleteResult( - ai_message=extracted_form_fields.error_message, - debug=metadata, - ) - - return CompleteResult( - ai_message="", - extracted_form_fields=extracted_form_fields.fields, - debug=metadata, - ) - - -async def attachment_for_filename( - filename: str, get_attachment_messages: Callable[[Sequence[str]], Awaitable[Sequence[ChatCompletionMessageParam]]] -) -> str: - attachment_messages = await get_attachment_messages([filename]) - return "\n\n".join( - ( - str(attachment.get("content")) - for attachment in attachment_messages - if "" in str(attachment.get("content", "")) - ) - ) diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/fill_form.py b/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/fill_form_step.py similarity index 82% rename from assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/fill_form.py rename to assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/fill_form_step.py index 639d67d3..878a70ad 100644 --- a/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/fill_form.py +++ b/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/fill_form_step.py @@ -8,39 +8,23 @@ from semantic_workbench_assistant.assistant_app.context import ConversationContext from semantic_workbench_assistant.assistant_app.protocol import AssistantAppProtocol -from .. import gce_config, state +from .. import state from ..inspector import FileStateInspector -from ..step import Context, IncompleteErrorResult, IncompleteResult, Result -from . import gce +from . import _attachments, _guided_conversation +from .types import ( + Context, + GuidedConversationDefinition, + IncompleteErrorResult, + IncompleteResult, + ResourceConstraintDefinition, + Result, +) logger = logging.getLogger(__name__) -definition = gce_config.GuidedConversationDefinition( - rules=[ - "For fields that are not in the provided files, collect the data from the user through conversation.", - "When providing options for a multiple choice field, provide the options in a numbered-list, so the user can refer to them by number.", - "When listing anything other than options, like document types, provide them in a bulleted list for improved readability.", - "When updating the agenda, the data-collection for each form field must be in a separate step.", - "Terminate conversation if inappropriate content is requested.", - ], - conversation_flow=""" -1. Inform the user that we've received the form and determined the fields in the form. -2. Inform the user that our goal is help them fill out the form. -3. Ask the user to provide one or more files that might contain data relevant to fill out the form. The files can be PDF, TXT, or DOCX. -4. When asking for files, suggest types of documents that might contain the data. -5. For each field in the form, check if the data is available in the provided files. -6. If the data is not available in the files, ask the user for the data. -7. When asking for data to fill the form, ask for a single piece of information at a time. -8. When the form is filled out, inform the user that you will now generate a document containing the filled form. -""", - context="", - resource_constraint=gce_config.ResourceConstraintDefinition( - quantity=15, - unit=ResourceConstraintUnit.TURNS, - mode=ResourceConstraintMode.MAXIMUM, - ), -) +def extend(app: AssistantAppProtocol) -> None: + app.add_inspector_state_provider(_inspector.state_id, _inspector) @dataclass @@ -50,7 +34,7 @@ class CompleteResult(Result): async def execute( - step_context: Context[gce_config.GuidedConversationDefinition], + step_context: Context[_guided_conversation.GuidedConversationDefinition], latest_user_message: str | None, form_fields: list[state.FormField], ) -> IncompleteResult | IncompleteErrorResult | CompleteResult: @@ -58,7 +42,7 @@ async def execute( Step: fill out the form with the user Approach: Guided conversation """ - message_with_attachments = await gce.message_with_recent_attachments( + message_with_attachments = await _attachments.message_with_recent_attachments( step_context.context, latest_user_message, step_context.get_attachment_messages ) @@ -66,7 +50,7 @@ async def execute( definition = step_context.config.model_copy() definition.resource_constraint.quantity = int(len(form_fields) * 1.5) - async with gce.guided_conversation_with_state( + async with _guided_conversation.engine( definition=definition, artifact_type=artifact_type, state_file_path=_get_state_file_path(step_context.context), @@ -74,9 +58,9 @@ async def execute( openai_model=step_context.llm_config.openai_model, context=step_context.context, state_id=_inspector.state_id, - ) as guided_conversation: + ) as gce: try: - result = await guided_conversation.step_conversation(message_with_attachments) + result = await gce.step_conversation(message_with_attachments) except Exception as e: logger.exception("failed to execute guided conversation") return IncompleteErrorResult( @@ -84,22 +68,21 @@ async def execute( debug={"error": str(e)}, ) + debug = {"guided-conversation": gce.to_json()} + logger.info("guided-conversation result: %s", result) - fill_form_gc_artifact = guided_conversation.artifact.artifact.model_dump(mode="json") - logger.info("guided-conversation artifact: %s", guided_conversation.artifact) + fill_form_gc_artifact = gce.artifact.artifact.model_dump(mode="json") + logger.info("guided-conversation artifact: %s", gce.artifact) if result.is_conversation_over: return CompleteResult( ai_message="", artifact=fill_form_gc_artifact, - debug={"artifact": fill_form_gc_artifact}, + debug=debug, ) - return IncompleteResult( - ai_message=result.ai_message or "", - debug={"artifact": fill_form_gc_artifact}, - ) + return IncompleteResult(ai_message=result.ai_message or "", debug=debug) def _form_fields_to_artifact(form_fields: list[state.FormField]): @@ -129,7 +112,7 @@ def _form_fields_to_artifact(form_fields: list[state.FormField]): def _get_state_file_path(context: ConversationContext) -> Path: - return gce.path_for_guided_conversation_state(context, "fill_form") + return _guided_conversation.path_for_state(context, "fill_form") _inspector = FileStateInspector( @@ -139,5 +122,28 @@ def _get_state_file_path(context: ConversationContext) -> Path: ) -def extend(app: AssistantAppProtocol) -> None: - app.add_inspector_state_provider(_inspector.state_id, _inspector) +definition = GuidedConversationDefinition( + rules=[ + "For fields that are not in the provided files, collect the data from the user through conversation.", + "When providing options for a multiple choice field, provide the options in a numbered-list, so the user can refer to them by number.", + "When listing anything other than options, like document types, provide them in a bulleted list for improved readability.", + "When updating the agenda, the data-collection for each form field must be in a separate step.", + "Terminate conversation if inappropriate content is requested.", + ], + conversation_flow=""" +1. Inform the user that we've received the form and determined the fields in the form. +2. Inform the user that our goal is help them fill out the form. +3. Ask the user to provide one or more files that might contain data relevant to fill out the form. The files can be PDF, TXT, or DOCX. +4. When asking for files, suggest types of documents that might contain the data. +5. For each field in the form, check if the data is available in the provided files. +6. If the data is not available in the files, ask the user for the data. +7. When asking for data to fill the form, ask for a single piece of information at a time. +8. When the form is filled out, inform the user that you will now generate a document containing the filled form. +""", + context="", + resource_constraint=ResourceConstraintDefinition( + quantity=15, + unit=ResourceConstraintUnit.TURNS, + mode=ResourceConstraintMode.MAXIMUM, + ), +) diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_agent/gce_config.py b/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/types.py similarity index 68% rename from assistants/prospector-assistant/assistant/agents/form_fill_agent/gce_config.py rename to assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/types.py index dc599ecd..f4cdb8a3 100644 --- a/assistants/prospector-assistant/assistant/agents/form_fill_agent/gce_config.py +++ b/assistants/prospector-assistant/assistant/agents/form_fill_agent/steps/types.py @@ -1,7 +1,44 @@ -from typing import Annotated +from dataclasses import dataclass +from typing import Annotated, Awaitable, Callable, Generic, Sequence, TypeVar from guided_conversation.utils.resources import ResourceConstraint, ResourceConstraintMode, ResourceConstraintUnit +from openai import AsyncOpenAI +from openai.types.chat import ChatCompletionMessageParam from pydantic import BaseModel, ConfigDict, Field +from semantic_workbench_assistant.assistant_app.context import ConversationContext + + +@dataclass +class LLMConfig: + openai_client_factory: Callable[[], AsyncOpenAI] + openai_model: str + max_response_tokens: int + + +ConfigT = TypeVar("ConfigT", bound=BaseModel) + + +@dataclass +class Context(Generic[ConfigT]): + context: ConversationContext + llm_config: LLMConfig + config: ConfigT + get_attachment_messages: Callable[[Sequence[str]], Awaitable[Sequence[ChatCompletionMessageParam]]] + + +@dataclass +class Result: + debug: dict + + +@dataclass +class IncompleteResult(Result): + ai_message: str + + +@dataclass +class IncompleteErrorResult(Result): + error_message: str class ResourceConstraintDefinition(BaseModel):