diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/extract_form_fields_step.py b/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/extract_form_fields_step.py deleted file mode 100644 index 8a4e7e5b..00000000 --- a/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/extract_form_fields_step.py +++ /dev/null @@ -1,100 +0,0 @@ -import logging -from dataclasses import dataclass -from typing import Annotated, Any - -from openai.types.chat import ChatCompletionMessageParam -from pydantic import BaseModel, Field -from semantic_workbench_assistant.config import UISchema - -from .. import state -from . import _llm -from .types import Context, IncompleteErrorResult, IncompleteResult, LLMConfig, Result - -logger = logging.getLogger(__name__) - - -class ExtractFormFieldsConfig(BaseModel): - instruction: Annotated[ - str, - Field(title="Instruction", description="The instruction for extracting form fields from the file content."), - UISchema(widget="textarea"), - ] = ( - "Read the user provided form attachment and determine what fields are in the form. Any type of form is allowed, including" - " tax forms, address forms, surveys, and other official or unofficial form-types. If the content is not a form," - " or the fields cannot be determined, then explain the reason why in the error_message. If the fields can be determined," - " leave the error_message empty." - ) - - -@dataclass -class CompleteResult(Result): - message: str - extracted_form_title: str - extracted_form_fields: list[state.FormField] - - -async def execute( - step_context: Context[ExtractFormFieldsConfig], - file_content: str, -) -> IncompleteResult | IncompleteErrorResult | CompleteResult: - """ - Step: extract form fields from the form file content - Approach: Chat completion with LLM - """ - - async with step_context.context.set_status("inspecting form ..."): - try: - extracted_form_fields, metadata = await _extract( - llm_config=step_context.llm_config, - config=step_context.config, - form_content=file_content, - ) - - except Exception as e: - logger.exception("failed to extract form fields") - return IncompleteErrorResult( - message=f"Failed to extract form fields: {e}", - debug={"error": str(e)}, - ) - - if extracted_form_fields.error_message: - return IncompleteResult( - message=extracted_form_fields.error_message, - debug=metadata, - ) - - return CompleteResult( - message="", - extracted_form_title=extracted_form_fields.title, - extracted_form_fields=extracted_form_fields.fields, - debug=metadata, - ) - - -class FormFields(BaseModel): - error_message: str = Field( - description="The error message in the case that the form fields could not be determined." - ) - title: str = Field(description="The title of the form.") - fields: list[state.FormField] = Field(description="The fields in the form.") - - -async def _extract( - llm_config: LLMConfig, config: ExtractFormFieldsConfig, form_content: str -) -> tuple[FormFields, dict[str, Any]]: - messages: list[ChatCompletionMessageParam] = [ - { - "role": "system", - "content": config.instruction, - }, - { - "role": "user", - "content": form_content, - }, - ] - - return await _llm.structured_completion( - llm_config=llm_config, - messages=messages, - response_model=FormFields, - ) diff --git a/assistants/prospector-assistant/assistant/chat.py b/assistants/prospector-assistant/assistant/chat.py index a20cb10b..3fc3d40b 100644 --- a/assistants/prospector-assistant/assistant/chat.py +++ b/assistants/prospector-assistant/assistant/chat.py @@ -14,6 +14,7 @@ import deepmerge import openai_client +from assistant_extensions.ai_clients.model import CompletionMessageImageContent from assistant_extensions.attachments import AttachmentsExtension from content_safety.evaluators import CombinedContentSafetyEvaluator from openai.types.chat import ChatCompletionMessageParam @@ -37,8 +38,8 @@ from . import legacy from .agents.artifact_agent import Artifact, ArtifactAgent, ArtifactConversationInspectorStateProvider from .agents.document_agent import DocumentAgent -from .agents.form_fill_extension import FormFillExtension, LLMConfig from .config import AssistantConfigModel +from .form_fill_extension import FormFillExtension, LLMConfig logger = logging.getLogger(__name__) @@ -131,8 +132,8 @@ async def on_chat_message_created( - @assistant.events.conversation.message.on_created """ - # update the participant status to indicate the assistant is thinking - async with send_error_message_on_exception(context), context.set_status("thinking..."): + # update the participant status to indicate the assistant is responding + async with send_error_message_on_exception(context), context.set_status("responding..."): # # NOTE: we're experimenting with agents, if they are enabled, use them to respond to the conversation # @@ -183,7 +184,7 @@ async def on_conversation_created(context: ConversationContext) -> None: async def welcome_message_form_fill(context: ConversationContext) -> None: - async with send_error_message_on_exception(context), context.set_status("thinking..."): + async with send_error_message_on_exception(context), context.set_status("responding..."): await form_fill_execute(context, None) @@ -193,7 +194,7 @@ async def welcome_message_create_document( message: ConversationMessage | None, metadata: dict[str, Any], ) -> None: - async with send_error_message_on_exception(context), context.set_status("thinking..."): + async with send_error_message_on_exception(context), context.set_status("responding..."): await create_document_execute(config, context, message, metadata) @@ -223,6 +224,7 @@ async def form_fill_execute(context: ConversationContext, message: ConversationM Execute the form fill agent to respond to the conversation message. """ config = await assistant_config.get(context.assistant) + participants = await context.get_participants(include_inactive=True) await form_fill_extension.execute( llm_config=LLMConfig( openai_client_factory=lambda: openai_client.create_client(config.service_config), @@ -231,7 +233,7 @@ async def form_fill_execute(context: ConversationContext, message: ConversationM ), config=config.agents_config.form_fill_agent, context=context, - latest_user_message=message.content if message else None, + latest_user_message=_format_message(message, participants.participants) if message else None, latest_attachment_filenames=message.filenames if message else [], get_attachment_content=form_fill_extension_get_attachment(context, config), ) @@ -251,8 +253,26 @@ async def get(filename: str) -> str: if not messages: return "" - # filter down to the messages that contain the attachment (ie. don't include the system messages) - return "\n\n".join((str(message.content) for message in messages if "" in str(message.content))) + # filter down to the message with the attachment + user_message = next( + (message for message in messages if "" in str(message)), + None, + ) + if not user_message: + return "" + + content = user_message.content + match content: + case str(): + return content + + case list(): + for part in content: + match part: + case CompletionMessageImageContent(): + return part.data + + return "" return get diff --git a/assistants/prospector-assistant/assistant/config.py b/assistants/prospector-assistant/assistant/config.py index f751eb78..2b260ac5 100644 --- a/assistants/prospector-assistant/assistant/config.py +++ b/assistants/prospector-assistant/assistant/config.py @@ -8,7 +8,7 @@ from . import helpers from .agents.artifact_agent import ArtifactAgentConfigModel -from .agents.form_fill_extension import FormFillConfig +from .form_fill_extension import FormFillConfig # The semantic workbench app uses react-jsonschema-form for rendering # dynamic configuration forms based on the configuration model and UI schema diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_extension/__init__.py b/assistants/prospector-assistant/assistant/form_fill_extension/__init__.py similarity index 100% rename from assistants/prospector-assistant/assistant/agents/form_fill_extension/__init__.py rename to assistants/prospector-assistant/assistant/form_fill_extension/__init__.py diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_extension/config.py b/assistants/prospector-assistant/assistant/form_fill_extension/config.py similarity index 100% rename from assistants/prospector-assistant/assistant/agents/form_fill_extension/config.py rename to assistants/prospector-assistant/assistant/form_fill_extension/config.py diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_extension/extension.py b/assistants/prospector-assistant/assistant/form_fill_extension/extension.py similarity index 89% rename from assistants/prospector-assistant/assistant/agents/form_fill_extension/extension.py rename to assistants/prospector-assistant/assistant/form_fill_extension/extension.py index 78a1952b..bc876588 100644 --- a/assistants/prospector-assistant/assistant/agents/form_fill_extension/extension.py +++ b/assistants/prospector-assistant/assistant/form_fill_extension/extension.py @@ -78,31 +78,35 @@ def build_step_context(config: ConfigT) -> Context[ConfigT]: case state.FormFillExtensionMode.extract_form_fields: file_content = await get_attachment_content(agent_state.form_filename) + attachment = UserAttachment(filename=agent_state.form_filename, content=file_content) result = await extract_form_fields_step.execute( step_context=build_step_context(config.extract_form_fields_config), - file_content=file_content, + potential_form_attachment=attachment, ) match result: case extract_form_fields_step.CompleteResult(): - await _send_message(context, result.message, result.debug) + await _send_message(context, result.message, result.debug, MessageType.notice) - agent_state.extracted_form_title = result.extracted_form_title - agent_state.extracted_form_fields = result.extracted_form_fields + agent_state.extracted_form = result.extracted_form agent_state.mode = state.FormFillExtensionMode.fill_form_step continue case _: await _handle_incomplete_result(context, result) + + agent_state.mode = state.FormFillExtensionMode.acquire_form_step return case state.FormFillExtensionMode.fill_form_step: + if agent_state.extracted_form is None: + raise ValueError("extracted_form is None") + result = await fill_form_step.execute( step_context=build_step_context(config.fill_form_config), form_filename=agent_state.form_filename, - form_title=agent_state.extracted_form_title, - form_fields=agent_state.extracted_form_fields, + form=agent_state.extracted_form, ) match result: @@ -143,14 +147,16 @@ async def _handle_incomplete_result(context: ConversationContext, result: Incomp raise ValueError(f"Unexpected incomplete result type: {result}") -async def _send_message(context: ConversationContext, message: str, debug: dict) -> None: +async def _send_message( + context: ConversationContext, message: str, debug: dict, message_type: MessageType = MessageType.chat +) -> None: if not message: return await context.send_messages( NewConversationMessage( content=message, - message_type=MessageType.chat, + message_type=message_type, debug_data=debug, ) ) diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_extension/inspector.py b/assistants/prospector-assistant/assistant/form_fill_extension/inspector.py similarity index 100% rename from assistants/prospector-assistant/assistant/agents/form_fill_extension/inspector.py rename to assistants/prospector-assistant/assistant/form_fill_extension/inspector.py diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_extension/state.py b/assistants/prospector-assistant/assistant/form_fill_extension/state.py similarity index 72% rename from assistants/prospector-assistant/assistant/agents/form_fill_extension/state.py rename to assistants/prospector-assistant/assistant/form_fill_extension/state.py index 121f6414..52ed879d 100644 --- a/assistants/prospector-assistant/assistant/agents/form_fill_extension/state.py +++ b/assistants/prospector-assistant/assistant/form_fill_extension/state.py @@ -39,6 +39,21 @@ class FormField(BaseModel): ) +class Section(BaseModel): + title: str = Field(description="The title of the section if one is provided on the form.") + description: str = Field(description="The description of the section if one is provided on the form.") + instructions: str = Field(description="The instructions for the section if they are provided on the form.") + fields: list[FormField] = Field(description="The fields of the section.") + + +class Form(BaseModel): + title: str = Field(description="The title of the form.") + description: str = Field(description="The description of the form if one is provided on the form.") + instructions: str = Field(description="The instructions for the form if they are provided on the form.") + fields: list[FormField] = Field(description="The fields of the form, if there are any at the top level.") + sections: list[Section] = Field(description="The sections of the form, if there are any.") + + class FormFillExtensionMode(StrEnum): acquire_form_step = "acquire_form" extract_form_fields = "extract_form_fields" @@ -49,8 +64,7 @@ class FormFillExtensionMode(StrEnum): class FormFillExtensionState(BaseModel): mode: FormFillExtensionMode = FormFillExtensionMode.acquire_form_step form_filename: str = "" - extracted_form_title: str = "" - extracted_form_fields: list[FormField] = [] + extracted_form: Form | None = None populated_form_markdown: str = "" fill_form_gc_artifact: dict | None = None @@ -81,4 +95,4 @@ async def extension_state(context: ConversationContext) -> AsyncIterator[FormFil current_state.set(None) -inspector = FileStateInspector(display_name="FormFill Agent", file_path_source=path_for_state) +inspector = FileStateInspector(display_name="Debug: FormFill Agent", file_path_source=path_for_state) diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/__init__.py b/assistants/prospector-assistant/assistant/form_fill_extension/steps/__init__.py similarity index 100% rename from assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/__init__.py rename to assistants/prospector-assistant/assistant/form_fill_extension/steps/__init__.py diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/_guided_conversation.py b/assistants/prospector-assistant/assistant/form_fill_extension/steps/_guided_conversation.py similarity index 100% rename from assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/_guided_conversation.py rename to assistants/prospector-assistant/assistant/form_fill_extension/steps/_guided_conversation.py diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/_llm.py b/assistants/prospector-assistant/assistant/form_fill_extension/steps/_llm.py similarity index 100% rename from assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/_llm.py rename to assistants/prospector-assistant/assistant/form_fill_extension/steps/_llm.py diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/acquire_form_step.py b/assistants/prospector-assistant/assistant/form_fill_extension/steps/acquire_form_step.py similarity index 87% rename from assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/acquire_form_step.py rename to assistants/prospector-assistant/assistant/form_fill_extension/steps/acquire_form_step.py index 516bcd97..c3d9b8af 100644 --- a/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/acquire_form_step.py +++ b/assistants/prospector-assistant/assistant/form_fill_extension/steps/acquire_form_step.py @@ -28,7 +28,6 @@ def extend(app: AssistantAppProtocol) -> None: class FormArtifact(BaseModel): - title: str = Field(description="The title of the form.", default="") filename: str = Field(description="The filename of the form.", default="") @@ -40,11 +39,9 @@ class FormArtifact(BaseModel): ], conversation_flow=dedent(""" 1. Inform the user that our goal is to help the user fill out a form. - 2. Ask the user to provide a file that contains a form. The file can be PDF, TXT, or DOCX. - 3. When you receive a file, determine if the file looks to be a form. - 4. If the file is not a form, inform the user that the file is not a form. Ask them to provide a different file. - 5. If the form is a file, update the artifcat with the title and filename of the form. - 6. Inform the user that you will now extract the form fields, so that you can assist them in filling it out. + 2. Ask the user to provide a file that contains a form. The file can be PDF, TXT, DOCX, or PNG. + 3. When you receive a file, set the filename field in the artifact. + 4. Inform the user that you will now extract the form fields, so that you can assist them in filling it out. """).strip(), context="", resource_constraint=ResourceConstraintDefinition( @@ -116,7 +113,7 @@ def _get_state_file_path(context: ConversationContext) -> Path: _inspector = FileStateInspector( - display_name="Acquire-Form Guided-Conversation", + display_name="Debug: Acquire-Form Guided-Conversation", file_path_source=_get_state_file_path, ) @@ -124,7 +121,7 @@ def _get_state_file_path(context: ConversationContext) -> Path: async def input_to_message(input: UserInput) -> str | None: attachments = [] async for attachment in input.attachments: - attachments.append(attachment.content) + attachments.append(f"{attachment.filename}") if not attachments: return input.message diff --git a/assistants/prospector-assistant/assistant/form_fill_extension/steps/extract_form_fields_step.py b/assistants/prospector-assistant/assistant/form_fill_extension/steps/extract_form_fields_step.py new file mode 100644 index 00000000..0ed85cc8 --- /dev/null +++ b/assistants/prospector-assistant/assistant/form_fill_extension/steps/extract_form_fields_step.py @@ -0,0 +1,126 @@ +import logging +from dataclasses import dataclass +from typing import Annotated, Any + +from openai.types.chat import ChatCompletionContentPartImageParam, ChatCompletionMessageParam +from pydantic import BaseModel, Field +from semantic_workbench_assistant.config import UISchema + +from .. import state +from . import _llm +from .types import Context, IncompleteErrorResult, IncompleteResult, LLMConfig, Result, UserAttachment + +logger = logging.getLogger(__name__) + + +class ExtractFormFieldsConfig(BaseModel): + instruction: Annotated[ + str, + Field(title="Instruction", description="The instruction for extracting form fields from the file content."), + UISchema(widget="textarea"), + ] = ( + "The user has provided a file that might be a form document. {text_or_image}. Determine if the provided file is a form." + " Determine what sections and fields are in the user provided document. Any type of form is allowed, including tax forms," + " address forms, surveys, and other official or unofficial form-types. The goal is to analyze the user provided form, and" + " report what you find. Do not make up a form or populate the form details with a random form. If the user provided document" + " is not a form, or the fields cannot be determined, then explain the reason why in the error_message. If the fields can be" + " determined, leave the error_message empty." + ) + + +@dataclass +class CompleteResult(Result): + message: str + extracted_form: state.Form + + +async def execute( + step_context: Context[ExtractFormFieldsConfig], + potential_form_attachment: UserAttachment, +) -> IncompleteResult | IncompleteErrorResult | CompleteResult: + """ + Step: extract form fields from the form file content + Approach: Chat completion with LLM + """ + + async with step_context.context.set_status("inspecting file ..."): + try: + extracted_form_fields, metadata = await _extract( + llm_config=step_context.llm_config, + config=step_context.config, + potential_form_attachment=potential_form_attachment, + ) + + except Exception as e: + logger.exception("failed to extract form fields") + return IncompleteErrorResult( + message=f"Failed to extract form fields: {e}", + debug={"error": str(e)}, + ) + + if not extracted_form_fields.form: + return IncompleteResult( + message=extracted_form_fields.error_message, + debug=metadata, + ) + + return CompleteResult( + message="The form fields have been extracted.", + extracted_form=extracted_form_fields.form, + debug=metadata, + ) + + +class FormDetails(BaseModel): + error_message: str = Field( + description="The error message in the case that the form fields could not be determined." + ) + form: state.Form | None = Field( + description="The form and it's details, if they can be determined from the user provided file." + ) + + +async def _extract( + llm_config: LLMConfig, config: ExtractFormFieldsConfig, potential_form_attachment: UserAttachment +) -> tuple[FormDetails, dict[str, Any]]: + match potential_form_attachment.filename.split(".")[-1].lower(): + case "png": + messages: list[ChatCompletionMessageParam] = [ + { + "role": "system", + "content": config.instruction.replace( + "{text_or_image}", "The provided message is a screenshot of the potential form." + ), + }, + { + "role": "user", + "content": [ + ChatCompletionContentPartImageParam( + image_url={ + "url": potential_form_attachment.content, + }, + type="image_url", + ) + ], + }, + ] + + case _: + messages: list[ChatCompletionMessageParam] = [ + { + "role": "system", + "content": config.instruction.replace( + "{text_or_image}", "The form has been provided as a text document." + ), + }, + { + "role": "user", + "content": potential_form_attachment.content, + }, + ] + + return await _llm.structured_completion( + llm_config=llm_config, + messages=messages, + response_model=FormDetails, + ) diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/fill_form_step.py b/assistants/prospector-assistant/assistant/form_fill_extension/steps/fill_form_step.py similarity index 74% rename from assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/fill_form_step.py rename to assistants/prospector-assistant/assistant/form_fill_extension/steps/fill_form_step.py index 412bf9e7..b47f4404 100644 --- a/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/fill_form_step.py +++ b/assistants/prospector-assistant/assistant/form_fill_extension/steps/fill_form_step.py @@ -46,7 +46,7 @@ def extend(app: AssistantAppProtocol) -> None: conversation_flow=dedent(""" 1. Inform the user that we've received the form and determined the fields in the form. 2. Inform the user that our goal is help them fill out the form. - 3. Ask the user to provide one or more files that might contain data relevant to fill out the form. The files can be PDF, TXT, or DOCX. + 3. Ask the user to provide one or more files that might contain data relevant to fill out the form. The files can be PDF, TXT, DOCX, or PNG. 4. When asking for files, suggest types of documents that might contain the data. 5. For each field in the form, check if the data is available in the provided files. 6. If the data is not available in the files, ask the user for the data. @@ -54,7 +54,7 @@ def extend(app: AssistantAppProtocol) -> None: """).strip(), context="", resource_constraint=ResourceConstraintDefinition( - quantity=15, + quantity=1000, unit=ResourceConstraintUnit.TURNS, mode=ResourceConstraintMode.MAXIMUM, ), @@ -77,6 +77,13 @@ class ExtractCandidateFieldValuesConfig(BaseModel): It is possible that there are multiple candidates for a single field, in which case you should provide all the candidates and an explanation for each candidate. + For example, if their is a field for an individual's name, 'name', and there are multiple names in the + attachment, you should provide all the names in the attachment as candidates for the 'name' field. + + Also, if their are multiple fields for individual's names in the form, 'name_one' and 'name_two', and + there are one or more names in the attachment, you should provide all the names in the attachment as + candidates for the 'name_one' and 'name_two' field. + Field definitions: {{form_fields}} """) @@ -117,26 +124,31 @@ class CompleteResult(Result): async def execute( step_context: Context[FillFormConfig], form_filename: str, - form_title: str, - form_fields: list[state.FormField], + form: state.Form, ) -> IncompleteResult | IncompleteErrorResult | CompleteResult: """ Step: fill out the form with the user through conversation and pulling values from uploaded attachments. Approach: Guided conversation / direct chat-completion (for document extraction) """ - message_part, debug = await _candidate_values_from_attachments_as_message_part( + form_fields = form.fields.copy() + for section in form.sections: + form_fields.extend(section.fields) + + debug = {} + + message_part, message_debug = await _candidate_values_from_attachments_as_message_part( step_context, form_filename, form_fields ) + message = "\n".join((step_context.latest_user_input.message or "", message_part)) - debug = {"document-extractions": debug} + if message_debug: + debug["document-extractions"] = message_debug - definition = step_context.config.definition.model_copy() - definition.resource_constraint.quantity = int(len(form_fields) * 1.5) artifact_type = _form_fields_to_artifact_basemodel(form_fields) async with _guided_conversation.engine( - definition=definition, + definition=step_context.config.definition, artifact_type=artifact_type, state_file_path=_get_guided_conversation_state_file_path(step_context.context), openai_client=step_context.llm_config.openai_client_factory(), @@ -161,23 +173,22 @@ async def execute( logger.info("guided-conversation artifact: %s", gce.artifact) populated_form_markdown = _generate_populated_form( - form_title=form_title, - form_fields=form_fields, + form=form, populated_fields=fill_form_gc_artifact, ) async with step_state(step_context.context) as state: state.populated_form_markdown = populated_form_markdown - if result.is_conversation_over: - return CompleteResult( - message=populated_form_markdown, - artifact=fill_form_gc_artifact, - populated_form_markdown=populated_form_markdown, - debug=debug, - ) + if result.is_conversation_over: + return CompleteResult( + message=state.populated_form_markdown, + artifact=fill_form_gc_artifact, + populated_form_markdown=state.populated_form_markdown, + debug=debug, + ) - return IncompleteResult(message=result.ai_message or "", debug=debug) + return IncompleteResult(message=result.ai_message or "", debug=debug) async def _candidate_values_from_attachments_as_message_part( @@ -187,21 +198,22 @@ async def _candidate_values_from_attachments_as_message_part( debug_per_file = {} attachment_candidate_value_parts = [] - async for attachment in step_context.latest_user_input.attachments: - if attachment.filename == form_filename: - continue - - candidate_values, metadata = await _extract( - llm_config=step_context.llm_config, - config=step_context.config.extract_config, - form_fields=form_fields, - document_content=attachment.content, - ) + async with step_context.context.set_status("inspecting attachments ..."): + async for attachment in step_context.latest_user_input.attachments: + if attachment.filename == form_filename: + continue + + candidate_values, metadata = await _extract_field_candidates( + llm_config=step_context.llm_config, + config=step_context.config.extract_config, + form_fields=form_fields, + document_content=attachment.content, + ) - message_part = _candidate_values_to_message_part(attachment.filename, candidate_values) - attachment_candidate_value_parts.append(message_part) + message_part = _candidate_values_to_message_part(attachment.filename, candidate_values) + attachment_candidate_value_parts.append(message_part) - debug_per_file[attachment.filename] = metadata + debug_per_file[attachment.filename] = metadata return "\n".join(attachment_candidate_value_parts), debug_per_file @@ -268,7 +280,7 @@ def _get_guided_conversation_state_file_path(context: ConversationContext) -> Pa _guided_conversation_inspector = FileStateInspector( - display_name="Fill-Form Guided-Conversation", + display_name="Debug: Fill-Form Guided-Conversation", file_path_source=_get_guided_conversation_state_file_path, ) @@ -285,7 +297,7 @@ def _get_step_state_file_path(context: ConversationContext) -> Path: ) -async def _extract( +async def _extract_field_candidates( llm_config: LLMConfig, config: ExtractCandidateFieldValuesConfig, form_fields: list[state.FormField], @@ -315,49 +327,67 @@ class _SerializationModel(BaseModel): def _generate_populated_form( - form_title: str, - form_fields: list[state.FormField], + form: state.Form, populated_fields: dict, ) -> str: def field_value(field_id: str) -> str: value = populated_fields.get(field_id) or "" if value == "Unanswered": - return "_" * 20 + return "" if value == "null": return "" return value - markdown_fields: list[str] = [] - for field in form_fields: - value = field_value(field.id) - match field.type: - case state.FieldType.text | state.FieldType.signature | state.FieldType.date: - markdown_fields.append(f"*{field.name}:*\n\n{value}") + def field_values(fields: list[state.FormField]) -> str: + markdown_fields: list[str] = [] + for field in fields: + value = field_value(field.id) - case state.FieldType.multiple_choice: - markdown_fields.append(f"*{field.name}:*\n") - for option in field.options: - if option in value: - markdown_fields.append(f"- [x] {option}\n") - continue - markdown_fields.append(f"- [ ] {option}\n") + markdown_fields.append("_" * 20) + markdown_fields.append(f"{field.name}:") + if field.description: + markdown_fields.append(f'ℹ️ {field.description}\n') - case _: - raise ValueError(f"Unsupported field type: {field.type}") + match field.type: + case state.FieldType.text | state.FieldType.signature | state.FieldType.date: + markdown_fields.append(f"{value}") + + case state.FieldType.multiple_choice: + for option in field.options: + if option == value: + markdown_fields.append(f"- [x] {option}\n") + continue + markdown_fields.append(f"- [ ] {option}\n") + + case _: + raise ValueError(f"Unsupported field type: {field.type}") + + return "\n\n".join(markdown_fields) + + top_level_fields = field_values(form.fields) + + sections = ( + f"## {section.title}\n{section.description}\n{section.instructions}\n{field_values(section.fields)}" + for section in form.sections + ) - all_fields = "\n\n".join(markdown_fields) return "\n".join(( "```markdown", - f"## {form_title}", + f"# {form.title}", + form.description, + form.instructions, + "", + top_level_fields, "", - all_fields, + *sections, "```", )) @asynccontextmanager async def step_state(context: ConversationContext) -> AsyncIterator[FillFormState]: - step_state = state.read_model(_get_step_state_file_path(context), FillFormState) or FillFormState() + state_file_path = _get_step_state_file_path(context) + step_state = state.read_model(state_file_path, FillFormState) or FillFormState() async with context.state_updated_event_after(_populated_form_state_inspector.state_id, focus_event=True): yield step_state - state.write_model(_get_step_state_file_path(context), step_state) + state.write_model(state_file_path, step_state) diff --git a/assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/types.py b/assistants/prospector-assistant/assistant/form_fill_extension/steps/types.py similarity index 100% rename from assistants/prospector-assistant/assistant/agents/form_fill_extension/steps/types.py rename to assistants/prospector-assistant/assistant/form_fill_extension/steps/types.py diff --git a/workbench-app/src/Constants.ts b/workbench-app/src/Constants.ts index 4627564c..60a623e8 100644 --- a/workbench-app/src/Constants.ts +++ b/workbench-app/src/Constants.ts @@ -9,7 +9,6 @@ const serviceUrl = (window.VITE_SEMANTIC_WORKBENCH_SERVICE_URL && window.VITE_SE export const Constants = { app: { name: 'Semantic Workbench', - conversationRedirectPath: '', defaultTheme: 'light', defaultBrand: 'local', autoScrollThreshold: 100, diff --git a/workbench-app/src/libs/useConversationUtility.ts b/workbench-app/src/libs/useConversationUtility.ts index f27880f4..394cf0bd 100644 --- a/workbench-app/src/libs/useConversationUtility.ts +++ b/workbench-app/src/libs/useConversationUtility.ts @@ -34,7 +34,7 @@ export const useConversationUtility = () => { const navigateToConversation = React.useCallback( (conversationId?: string, hash?: string) => { - let path = conversationId ? [Constants.app.conversationRedirectPath, conversationId].join('/') : ''; + let path = conversationId ? '/' + conversationId : ''; if (hash) { path += `#${hash}`; } diff --git a/workbench-app/src/routes/ShareRedeem.tsx b/workbench-app/src/routes/ShareRedeem.tsx index 488b3186..108b57e5 100644 --- a/workbench-app/src/routes/ShareRedeem.tsx +++ b/workbench-app/src/routes/ShareRedeem.tsx @@ -6,7 +6,6 @@ import { useNavigate, useParams } from 'react-router-dom'; import { AppView } from '../components/App/AppView'; import { DialogControl } from '../components/App/DialogControl'; import { Loading } from '../components/App/Loading'; -import { Constants } from '../Constants'; import { ConversationShareType, useConversationUtility } from '../libs/useConversationUtility'; import { useSiteUtility } from '../libs/useSiteUtility'; import { useWorkbenchService } from '../libs/useWorkbenchService'; @@ -235,7 +234,7 @@ export const ShareRedeem: React.FC = () => { {existingDuplicateConversations.map((conversation) => (
  • {conversation.title}