-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
84ead84
commit 14c03c3
Showing
8 changed files
with
458 additions
and
23 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,188 @@ | ||
{ | ||
"cells": [ | ||
{ | ||
"cell_type": "markdown", | ||
"metadata": {}, | ||
"source": [ | ||
"# Streaming APIs\n", | ||
"\n", | ||
"This notebooks shows examples for how to use the streaming APIs for both OpenAI and Ollama.\n", | ||
"\n", | ||
"When instantiating the client, set `async_client=True` to use the async client which supports streaming.\n" | ||
] | ||
}, | ||
{ | ||
"cell_type": "code", | ||
"execution_count": 1, | ||
"metadata": {}, | ||
"outputs": [], | ||
"source": [ | ||
"from not_again_ai.llm.chat_completion import chat_completion_stream\n", | ||
"from not_again_ai.llm.chat_completion.providers.ollama_api import ollama_client\n", | ||
"from not_again_ai.llm.chat_completion.providers.openai_api import openai_client\n", | ||
"from not_again_ai.llm.chat_completion.types import ChatCompletionRequest, SystemMessage, UserMessage\n", | ||
"\n", | ||
"openai_client = openai_client(async_client=True)\n", | ||
"ollama_client = ollama_client(async_client=True)" | ||
] | ||
}, | ||
{ | ||
"cell_type": "code", | ||
"execution_count": 2, | ||
"metadata": {}, | ||
"outputs": [ | ||
{ | ||
"name": "stdout", | ||
"output_type": "stream", | ||
"text": [ | ||
"{'choices': [{'delta': {'content': '', 'role': 'assistant'}, 'index': 0}], 'errors': '', 'response_duration': 1.5805}\n", | ||
"{'choices': [{'delta': {'content': '', 'role': 'assistant', 'tool_calls': [{'id': 'call_W5ScLS0Atqtt4awudPS8pA6G', 'function': {'name': 'get_current_weather', 'arguments': ''}, 'type': 'function'}]}, 'index': 0}], 'errors': '', 'response_duration': 1.5835}\n", | ||
"{'choices': [{'delta': {'content': '', 'role': 'assistant', 'tool_calls': [{'function': {'name': '', 'arguments': '{\"lo'}, 'type': 'function'}]}, 'index': 0}], 'errors': '', 'response_duration': 1.5846}\n", | ||
"{'choices': [{'delta': {'content': '', 'role': 'assistant', 'tool_calls': [{'function': {'name': '', 'arguments': 'catio'}, 'type': 'function'}]}, 'index': 0}], 'errors': '', 'response_duration': 1.5858}\n", | ||
"{'choices': [{'delta': {'content': '', 'role': 'assistant', 'tool_calls': [{'function': {'name': '', 'arguments': 'n\": \"B'}, 'type': 'function'}]}, 'index': 0}], 'errors': '', 'response_duration': 1.5868}\n", | ||
"{'choices': [{'delta': {'content': '', 'role': 'assistant', 'tool_calls': [{'function': {'name': '', 'arguments': 'osto'}, 'type': 'function'}]}, 'index': 0}], 'errors': '', 'response_duration': 1.589}\n", | ||
"{'choices': [{'delta': {'content': '', 'role': 'assistant', 'tool_calls': [{'function': {'name': '', 'arguments': 'n, MA'}, 'type': 'function'}]}, 'index': 0}], 'errors': '', 'response_duration': 1.5899}\n", | ||
"{'choices': [{'delta': {'content': '', 'role': 'assistant', 'tool_calls': [{'function': {'name': '', 'arguments': '\", \"fo'}, 'type': 'function'}]}, 'index': 0}], 'errors': '', 'response_duration': 1.5909}\n", | ||
"{'choices': [{'delta': {'content': '', 'role': 'assistant', 'tool_calls': [{'function': {'name': '', 'arguments': 'rmat'}, 'type': 'function'}]}, 'index': 0}], 'errors': '', 'response_duration': 1.5918}\n", | ||
"{'choices': [{'delta': {'content': '', 'role': 'assistant', 'tool_calls': [{'function': {'name': '', 'arguments': '\": \"f'}, 'type': 'function'}]}, 'index': 0}], 'errors': '', 'response_duration': 1.5934}\n", | ||
"{'choices': [{'delta': {'content': '', 'role': 'assistant', 'tool_calls': [{'function': {'name': '', 'arguments': 'ahrenh'}, 'type': 'function'}]}, 'index': 0}], 'errors': '', 'response_duration': 1.5942}\n", | ||
"{'choices': [{'delta': {'content': '', 'role': 'assistant', 'tool_calls': [{'function': {'name': '', 'arguments': 'eit\"'}, 'type': 'function'}]}, 'index': 0}], 'errors': '', 'response_duration': 1.5955}\n", | ||
"{'choices': [{'delta': {'content': '', 'role': 'assistant', 'tool_calls': [{'function': {'name': '', 'arguments': '}'}, 'type': 'function'}]}, 'index': 0}], 'errors': '', 'response_duration': 1.5964}\n", | ||
"{'choices': [{'delta': {'content': '', 'role': 'assistant', 'tool_calls': [{'id': 'call_sETMkonfyDQP5ndr8kGpHVcY', 'function': {'name': 'get_current_weather', 'arguments': ''}, 'type': 'function'}]}, 'index': 0}], 'errors': '', 'response_duration': 1.5974}\n", | ||
"{'choices': [{'delta': {'content': '', 'role': 'assistant', 'tool_calls': [{'function': {'name': '', 'arguments': '{\"lo'}, 'type': 'function'}]}, 'index': 0}], 'errors': '', 'response_duration': 1.5983}\n", | ||
"{'choices': [{'delta': {'content': '', 'role': 'assistant', 'tool_calls': [{'function': {'name': '', 'arguments': 'catio'}, 'type': 'function'}]}, 'index': 0}], 'errors': '', 'response_duration': 1.5998}\n", | ||
"{'choices': [{'delta': {'content': '', 'role': 'assistant', 'tool_calls': [{'function': {'name': '', 'arguments': 'n\": \"N'}, 'type': 'function'}]}, 'index': 0}], 'errors': '', 'response_duration': 1.6006}\n", | ||
"{'choices': [{'delta': {'content': '', 'role': 'assistant', 'tool_calls': [{'function': {'name': '', 'arguments': 'ew Y'}, 'type': 'function'}]}, 'index': 0}], 'errors': '', 'response_duration': 1.6383}\n", | ||
"{'choices': [{'delta': {'content': '', 'role': 'assistant', 'tool_calls': [{'function': {'name': '', 'arguments': 'ork, '}, 'type': 'function'}]}, 'index': 0}], 'errors': '', 'response_duration': 1.6393}\n", | ||
"{'choices': [{'delta': {'content': '', 'role': 'assistant', 'tool_calls': [{'function': {'name': '', 'arguments': 'NY\", \"'}, 'type': 'function'}]}, 'index': 0}], 'errors': '', 'response_duration': 1.6403}\n", | ||
"{'choices': [{'delta': {'content': '', 'role': 'assistant', 'tool_calls': [{'function': {'name': '', 'arguments': 'form'}, 'type': 'function'}]}, 'index': 0}], 'errors': '', 'response_duration': 1.6413}\n", | ||
"{'choices': [{'delta': {'content': '', 'role': 'assistant', 'tool_calls': [{'function': {'name': '', 'arguments': 'at\": '}, 'type': 'function'}]}, 'index': 0}], 'errors': '', 'response_duration': 1.6422}\n", | ||
"{'choices': [{'delta': {'content': '', 'role': 'assistant', 'tool_calls': [{'function': {'name': '', 'arguments': '\"fahre'}, 'type': 'function'}]}, 'index': 0}], 'errors': '', 'response_duration': 1.6431}\n", | ||
"{'choices': [{'delta': {'content': '', 'role': 'assistant', 'tool_calls': [{'function': {'name': '', 'arguments': 'nhei'}, 'type': 'function'}]}, 'index': 0}], 'errors': '', 'response_duration': 1.6449}\n", | ||
"{'choices': [{'delta': {'content': '', 'role': 'assistant', 'tool_calls': [{'function': {'name': '', 'arguments': 't\"}'}, 'type': 'function'}]}, 'index': 0}], 'errors': '', 'response_duration': 1.6458}\n", | ||
"{'choices': [{'delta': {'content': '', 'role': 'assistant'}, 'index': 0, 'finish_reason': 'tool_calls'}], 'errors': '', 'response_duration': 1.647}\n", | ||
"{'choices': [], 'errors': '', 'completion_tokens': 62, 'prompt_tokens': 112, 'response_duration': 1.6504, 'system_fingerprint': 'fp_72ed7ab54c'}\n" | ||
] | ||
} | ||
], | ||
"source": [ | ||
"# OpenAI Streaming Example with tools\n", | ||
"tools = [\n", | ||
" {\n", | ||
" \"type\": \"function\",\n", | ||
" \"function\": {\n", | ||
" \"name\": \"get_current_weather\",\n", | ||
" \"description\": \"Get the current weather\",\n", | ||
" \"parameters\": {\n", | ||
" \"type\": \"object\",\n", | ||
" \"properties\": {\n", | ||
" \"location\": {\n", | ||
" \"type\": \"string\",\n", | ||
" \"description\": \"The city and state, e.g. San Francisco, CA\",\n", | ||
" },\n", | ||
" \"format\": {\n", | ||
" \"type\": \"string\",\n", | ||
" \"enum\": [\"celsius\", \"fahrenheit\"],\n", | ||
" \"description\": \"The temperature unit to use. Infer this from the users location.\",\n", | ||
" },\n", | ||
" },\n", | ||
" \"required\": [\"location\", \"format\"],\n", | ||
" },\n", | ||
" },\n", | ||
" },\n", | ||
"]\n", | ||
"messages = [\n", | ||
" SystemMessage(content=\"Call the get_current_weather function once for each city that the user mentions.\"),\n", | ||
" UserMessage(content=\"What's the current weather like in Boston, MA and New York, NY today?\"),\n", | ||
"]\n", | ||
"request = ChatCompletionRequest(\n", | ||
" model=\"gpt-4o-mini-2024-07-18\",\n", | ||
" messages=messages,\n", | ||
" tools=tools,\n", | ||
" max_completion_tokens=400,\n", | ||
" temperature=0,\n", | ||
")\n", | ||
"\n", | ||
"async for chunk in chat_completion_stream(request, \"openai\", openai_client):\n", | ||
" print(chunk.model_dump(mode=\"json\", exclude_none=True), flush=True)" | ||
] | ||
}, | ||
{ | ||
"cell_type": "code", | ||
"execution_count": 3, | ||
"metadata": {}, | ||
"outputs": [ | ||
{ | ||
"name": "stdout", | ||
"output_type": "stream", | ||
"text": [ | ||
"{'choices': [{'delta': {'content': '', 'role': 'assistant', 'tool_calls': [{'id': '', 'function': {'name': 'get_current_weather', 'arguments': {'location': 'Boston, MA'}}, 'type': 'function'}]}, 'index': 0}], 'errors': '', 'response_duration': 0.4406}\n", | ||
"{'choices': [{'delta': {'content': '', 'role': 'assistant', 'tool_calls': [{'id': '', 'function': {'name': 'get_current_weather', 'arguments': {'location': 'New York, NY'}}, 'type': 'function'}]}, 'index': 0}], 'errors': '', 'response_duration': 0.7869}\n", | ||
"{'choices': [{'delta': {'content': '', 'role': 'assistant'}, 'index': 0, 'finish_reason': 'stop'}], 'errors': '', 'completion_tokens': 47, 'prompt_tokens': 204, 'response_duration': 0.8152}\n" | ||
] | ||
} | ||
], | ||
"source": [ | ||
"# Ollama Streaming Example with tools\n", | ||
"tools = [\n", | ||
" {\n", | ||
" \"type\": \"function\",\n", | ||
" \"function\": {\n", | ||
" \"name\": \"get_current_weather\",\n", | ||
" \"description\": \"Get the current weather\",\n", | ||
" \"parameters\": {\n", | ||
" \"type\": \"object\",\n", | ||
" \"properties\": {\n", | ||
" \"location\": {\n", | ||
" \"type\": \"string\",\n", | ||
" \"description\": \"The city and state, e.g. San Francisco, CA\",\n", | ||
" },\n", | ||
" \"format\": {\n", | ||
" \"type\": \"string\",\n", | ||
" \"enum\": [\"celsius\", \"fahrenheit\"],\n", | ||
" \"description\": \"The temperature unit to use. Infer this from the users location.\",\n", | ||
" },\n", | ||
" },\n", | ||
" \"required\": [\"location\", \"format\"],\n", | ||
" },\n", | ||
" },\n", | ||
" },\n", | ||
"]\n", | ||
"messages = [\n", | ||
" SystemMessage(content=\"Call the get_current_weather function once for each city that the user mentions.\"),\n", | ||
" UserMessage(content=\"What's the current weather like in Boston, MA and New York, NY today?\"),\n", | ||
"]\n", | ||
"request = ChatCompletionRequest(\n", | ||
" model=\"qwen2.5:14b\",\n", | ||
" messages=messages,\n", | ||
" tools=tools,\n", | ||
" max_completion_tokens=400,\n", | ||
" temperature=0,\n", | ||
")\n", | ||
"async for chunk in chat_completion_stream(request, \"ollama\", ollama_client):\n", | ||
" print(chunk.model_dump(mode=\"json\", exclude_none=True))" | ||
] | ||
} | ||
], | ||
"metadata": { | ||
"kernelspec": { | ||
"display_name": ".venv", | ||
"language": "python", | ||
"name": "python3" | ||
}, | ||
"language_info": { | ||
"codemirror_mode": { | ||
"name": "ipython", | ||
"version": 3 | ||
}, | ||
"file_extension": ".py", | ||
"mimetype": "text/x-python", | ||
"name": "python", | ||
"nbconvert_exporter": "python", | ||
"pygments_lexer": "ipython3", | ||
"version": "3.12.3" | ||
} | ||
}, | ||
"nbformat": 4, | ||
"nbformat_minor": 2 | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.