o3 mini support in openai-client and codespace-assistant, improved to…

…ols support in codespace-assistant extension (microsoft#313)

assistants/codespace-assistant/assistant/extensions/tools/__mcp_server_configs.py

@@ -7,18 +7,19 @@
 # Each configuration includes the server name, command, arguments, and environment variables.
 # Tested with 'npx' commands and _should_ work with 'uvx' as well.
 # Can also use 'node' (and probably 'python'), but requires the full path to the lib/script called.
+# The 'key' is used to identify the server in config, logs, etc.
 
 
 def get_mcp_server_configs(tools_config: ToolsConfigModel) -> List[MCPServerConfig]:
     file_system_paths = [f"/workspaces/{file_system_path}" for file_system_path in tools_config.file_system_paths]
     return [
         MCPServerConfig(
-            name="Filesystem MCP Server",
+            key="filesystem",
             command="npx",
             args=["-y", "@modelcontextprotocol/server-filesystem", *file_system_paths],
         ),
         MCPServerConfig(
-            name="Memory MCP Server",
+            key="memory",
             command="npx",
             args=["-y", "@modelcontextprotocol/server-memory"],
             prompt=dedent("""
@@ -45,15 +46,15 @@ def get_mcp_server_configs(tools_config: ToolsConfigModel) -> List[MCPServerConf
         """),
         ),
         MCPServerConfig(
-            name="GIPHY MCP Server",
+            key="giphy",
             command="http://127.0.0.1:6000/sse",
             args=[],
         ),
-        # MCPServerConfig(
-        #     name="Sequential Thinking MCP Server",
-        #     command="npx",
-        #     args=["-y", "@modelcontextprotocol/server-sequential-thinking"],
-        # ),
+        MCPServerConfig(
+            key="sequential_thinking",
+            command="npx",
+            args=["-y", "@modelcontextprotocol/server-sequential-thinking"],
+        ),
         # MCPServerConfig(
         #     name="Web Research MCP Server",
         #     command="npx",

assistants/codespace-assistant/assistant/extensions/tools/__mcp_server_utils.py

@@ -30,14 +30,14 @@ async def connect_to_mcp_server_stdio(server_config: MCPServerConfig) -> AsyncIt
     server_params = StdioServerParameters(command=server_config.command, args=server_config.args, env=server_config.env)
     try:
         logger.debug(
-            f"Attempting to connect to {server_config.name} with command: {server_config.command} {' '.join(server_config.args)}"
+            f"Attempting to connect to {server_config.key} with command: {server_config.command} {' '.join(server_config.args)}"
         )
         async with stdio_client(server_params) as (read_stream, write_stream):
             async with ClientSession(read_stream, write_stream) as client_session:
                 await client_session.initialize()
                 yield client_session  # Yield the session for use
     except Exception as e:
-        logger.exception(f"Error connecting to {server_config.name}: {e}")
+        logger.exception(f"Error connecting to {server_config.key}: {e}")
         yield None  # Yield None if connection fails
 
 
@@ -46,33 +46,43 @@ async def connect_to_mcp_server_sse(server_config: MCPServerConfig) -> AsyncIter
     """Connect to a single MCP server defined in the config using SSE transport."""
 
     try:
-        logger.debug(f"Attempting to connect to {server_config.name} with SSE transport: {server_config.command}")
+        logger.debug(f"Attempting to connect to {server_config.key} with SSE transport: {server_config.command}")
         async with sse_client(url=server_config.command, headers=server_config.env) as (read_stream, write_stream):
             async with ClientSession(read_stream, write_stream) as client_session:
                 await client_session.initialize()
                 yield client_session  # Yield the session for use
     except Exception as e:
-        logger.exception(f"Error connecting to {server_config.name}: {e}")
+        logger.exception(f"Error connecting to {server_config.key}: {e}")
         yield None
 
 
+def is_mcp_server_enabled(server_config: MCPServerConfig, tools_config: ToolsConfigModel) -> bool:
+    """Check if an MCP server is enabled."""
+    return tools_config.tools_enabled.model_dump().get(f"{server_config.key}_enabled", False)
+
+
 async def establish_mcp_sessions(tools_config: ToolsConfigModel, stack: AsyncExitStack) -> List[MCPSession]:
     """
     Establish connections to MCP servers using the provided AsyncExitStack.
     """
 
     mcp_sessions: List[MCPSession] = []
     for server_config in get_mcp_server_configs(tools_config):
+        # Check to see if the server is enabled
+        if not is_mcp_server_enabled(server_config, tools_config):
+            logger.debug(f"Skipping disabled server: {server_config.key}")
+            continue
+
         client_session: ClientSession | None = await stack.enter_async_context(connect_to_mcp_server(server_config))
         if client_session:
             # Create an MCP session with the client session
-            mcp_session = MCPSession(name=server_config.name, client_session=client_session)
+            mcp_session = MCPSession(name=server_config.key, client_session=client_session)
             # Initialize the session to load tools, resources, etc.
             await mcp_session.initialize()
             # Add the session to the list of established sessions
             mcp_sessions.append(mcp_session)
         else:
-            logger.warning(f"Could not establish session with {server_config.name}")
+            logger.warning(f"Could not establish session with {server_config.key}")
     return mcp_sessions
 
 

assistants/codespace-assistant/assistant/extensions/tools/__model.py

@@ -14,7 +14,7 @@
 
 @dataclass
 class MCPServerConfig:
-    name: str
+    key: str
     command: str
     args: List[str]
     env: Optional[dict[str, str]] = None
@@ -67,6 +67,42 @@ class ToolCallResult:
     metadata: dict[str, Any]
 
 
+class MCPServersEnabledConfigModel(BaseModel):
+    # NOTE: create a property for each of the mcp servers following the convention of: {server_key}_enabled
+
+    filesystem_enabled: Annotated[
+        bool,
+        Field(
+            title="File System Enabled",
+            description="Enable file system tools, granting access to defined file system paths for read/write.",
+        ),
+    ] = True
+
+    memory_enabled: Annotated[
+        bool,
+        Field(
+            title="Memory Enabled",
+            description="Enable memory tools, allowing for storing and retrieving data in memory.",
+        ),
+    ] = True
+
+    sequential_thinking_enabled: Annotated[
+        bool,
+        Field(
+            title="Sequential Thinking Enabled",
+            description="Enable sequential thinking tools, supporting sequential processing of information.",
+        ),
+    ] = False
+
+    giphy_enabled: Annotated[
+        bool,
+        Field(
+            title="Giphy Enabled",
+            description="Enable Giphy tools for searching and retrieving GIFs. Must start the Giphy server.",
+        ),
+    ] = False
+
+
 class ToolsConfigModel(BaseModel):
     enabled: Annotated[
         bool,
@@ -105,37 +141,44 @@ class ToolsConfigModel(BaseModel):
         ),
         UISchema(widget="textarea", enable_markdown_in_description=True),
     ] = dedent("""
+        - Use the available tools to assist with specific tasks.
+        - Before performing any file operations, use the `allowed_directories` tool to get a list of directories
+            that are allowed for file operations.
         - When searching or browsing for files, consider the kinds of folders and files that should be avoided:
             - For example, for coding projects exclude folders like `.git`, `.vscode`, `node_modules`, and `dist`.
         - For each turn, always re-read a file before using it to ensure the most up-to-date information, especially
             when writing or editing files.
+        - Either use search or specific list files in directories instead of using `directory_tree` to avoid
+            issues with large directory trees as that tool is not optimized for large trees nor does it allow
+            for filtering.
+        - The search tool does not appear to support wildcards, but does work with partial file names.
     """).strip()
 
-    instructions_for_non_tool_models: Annotated[
-        str,
-        Field(
-            title="Tools Instructions for Models Without Tools Support",
-            description=dedent("""
-                Some models don't support tools (like OpenAI reasoning models), so these instructions
-                are only used to implement tool support through custom instruction and injection of
-                the tool definitions.  Make sure to include {{tools}} in the instructions.
-            """),
-        ),
-        UISchema(widget="textarea", enable_markdown_in_description=True),
-    ] = dedent("""
-        You can perform specific tasks using available tools. When you need to use a tool, respond
-        with a strict JSON object containing only the tool's `id` and function name and arguments.
-
-        Available Tools:
-        {{tools}}
-
-        ### Instructions:
-        - If you need to use a tool to answer the user's query, respond with **ONLY** a JSON object.
-        - If you can answer without using a tool, provide the answer directly.
-        - **No code, no text, no markdown** within the JSON.
-        - Ensure that all values are plain data types (e.g., strings, numbers).
-        - **Do not** include any additional characters, functions, or expressions within the JSON.
-    """).strip()
+    # instructions_for_non_tool_models: Annotated[
+    #     str,
+    #     Field(
+    #         title="Tools Instructions for Models Without Tools Support",
+    #         description=dedent("""
+    #             Some models don't support tools (like OpenAI reasoning models), so these instructions
+    #             are only used to implement tool support through custom instruction and injection of
+    #             the tool definitions.  Make sure to include {{tools}} in the instructions.
+    #         """),
+    #     ),
+    #     UISchema(widget="textarea", enable_markdown_in_description=True),
+    # ] = dedent("""
+    #     You can perform specific tasks using available tools. When you need to use a tool, respond
+    #     with a strict JSON object containing only the tool's `id` and function name and arguments.
+    #     \n\n
+    #     Available Tools:
+    #     {{tools}}
+    #     \n\n
+    #     ### How to Use Tools:
+    #     - If you need to use a tool to answer the user's query, respond with **ONLY** a JSON object.
+    #     - If you can answer without using a tool, provide the answer directly.
+    #     - **No code, no text, no markdown** within the JSON.
+    #     - Ensure that all values are plain data types (e.g., strings, numbers).
+    #     - **Do not** include any additional characters, functions, or expressions within the JSON.
+    # """).strip()
 
     file_system_paths: Annotated[
         list[str],
@@ -144,3 +187,7 @@ class ToolsConfigModel(BaseModel):
             description="Paths to the file system for tools to use, relative to `/workspaces/` in the container.",
         ),
     ] = ["semanticworkbench"]
+
+    tools_enabled: Annotated[MCPServersEnabledConfigModel, Field(title="Tools Enabled")] = (
+        MCPServersEnabledConfigModel()
+    )

assistants/codespace-assistant/assistant/response/request_builder.py

@@ -32,19 +32,23 @@ async def build_request(
     participants_response = await context.get_participants(include_inactive=True)
     participants = participants_response.participants
 
-    # Build system message content
-    system_message_content = build_system_message_content(config, context, participants, silence_token)
+    additional_system_message_content: list[tuple[str, str]] = []
 
     # Add any additional tools instructions to the system message content
     if config.extensions_config.tools.enabled:
-        system_message_content = "\n\n".join([
-            system_message_content,
+        additional_system_message_content.append((
+            "Tool Instructions",
             config.extensions_config.tools.additional_instructions,
-        ])
+        ))
 
     # Add MCP Server prompts to the system message content
     if len(mcp_prompts) > 0:
-        system_message_content = "\n\n".join([system_message_content, *mcp_prompts])
+        additional_system_message_content.append(("Specific Tool Guidance", "\n\n".join(mcp_prompts)))
+
+    # Build system message content
+    system_message_content = build_system_message_content(
+        config, context, participants, silence_token, additional_system_message_content
+    )
 
     completion_messages: List[ChatCompletionMessageParam] = [
         ChatCompletionSystemMessageParam(

assistants/codespace-assistant/assistant/response/utils/message_utils.py

@@ -1,5 +1,6 @@
 import json
 import logging
+from textwrap import dedent
 from typing import Any
 
 import openai_client
@@ -29,30 +30,37 @@ def build_system_message_content(
     context: ConversationContext,
     participants: list[ConversationParticipant],
     silence_token: str,
+    additional_content: list[tuple[str, str]] | None = None,
 ) -> str:
     """
     Construct the system message content with tool descriptions and instructions.
     """
 
-    system_message_content = f'{config.instruction_prompt}\n\nYour name is "{context.assistant.name}".\n'
+    system_message_content = f'{config.instruction_prompt}\n\nYour name is "{context.assistant.name}".'
 
     if len(participants) > 2:
         participant_names = ", ".join([
             f'"{participant.name}"' for participant in participants if participant.id != context.assistant.id
         ])
-        system_message_content += (
-            "\n\n"
-            f"There are {len(participants)} participants in the conversation, "
-            f"including you as the assistant and the following users: {participant_names}."
-            "\n\nYou do not need to respond to every message. Do not respond if the last thing said was a closing "
-            f'statement such as "bye" or "goodbye", or just a general acknowledgement like "ok" or "thanks". Do not '
-            f'respond as another user in the conversation, only as "{context.assistant.name}". '
-            "Sometimes the other users need to talk amongst themselves and that is okay. If the conversation seems to "
-            "be directed at you or the general audience, go ahead and respond."
-            f'\n\nSay "{silence_token}" to skip your turn.'
-        )
-
-    system_message_content += f"\n\n{config.guardrails_prompt}"
+        system_message_content += dedent(f"""
+            \n\n
+            There are {len(participants)} participants in the conversation,
+            including you as the assistant and the following users: {participant_names}.
+            \n\n
+            You do not need to respond to every message. Do not respond if the last thing said was a closing
+            statement such as "bye" or "goodbye", or just a general acknowledgement like "ok" or "thanks". Do not
+            respond as another user in the conversation, only as "{context.assistant.name}".
+            Sometimes the other users need to talk amongst themselves and that is okay. If the conversation seems to
+            be directed at you or the general audience, go ahead and respond.
+            \n\n
+            Say "{silence_token}" to skip your turn.
+        """).strip()
+
+    system_message_content += f"\n\n# Safety Guardrails:\n{config.guardrails_prompt}"
+
+    if additional_content:
+        for section in additional_content:
+            system_message_content += f"\n\n# {section[0]}:\n{section[1]}"
 
     return system_message_content
 

assistants/codespace-assistant/assistant/response/utils/openai_utils.py

@@ -38,12 +38,18 @@ async def get_completion(
 
     if request_config.is_reasoning_model:
         # reasoning models
-        # note: tools are not supported by reasoning models currently
         completion_args["max_completion_tokens"] = request_config.response_tokens
+        completion_args["reasoning_effort"] = request_config.reasoning_effort
 
     else:
         # all other models
         completion_args["max_tokens"] = request_config.response_tokens
+
+    # list of models that do not support tools
+    no_tools_support = ["o1-preview", "o1-mini"]
+
+    # add tools to completion args if model supports tools
+    if request_config.model not in no_tools_support:
         completion_args["tools"] = tools or NotGiven()
         if tools is not None:
             completion_args["tool_choice"] = "auto"

assistants/codespace-assistant/pyproject.toml

@@ -15,7 +15,7 @@ dependencies = [
     "html2docx>=1.6.0",
     "markdown>=3.6",
     "mcp>=1.2.0",
-    "openai>=1.60.2",
+    "openai>=1.61.0",
     "openai-client>=0.1.0",
     "tiktoken>=0.8.0",
 ]