From 74d122eae68c6925dea721869fc93a720d228228 Mon Sep 17 00:00:00 2001 From: Wei Ouyang Date: Tue, 5 Mar 2024 09:59:06 +0100 Subject: [PATCH] Improve development guideline with notebook --- .../bioimage-chatbot-extension-tutorial.ipynb | 382 ++++++++++++++++++ docs/development.md | 52 +-- 2 files changed, 384 insertions(+), 50 deletions(-) create mode 100644 docs/bioimage-chatbot-extension-tutorial.ipynb diff --git a/docs/bioimage-chatbot-extension-tutorial.ipynb b/docs/bioimage-chatbot-extension-tutorial.ipynb new file mode 100644 index 0000000..db015a4 --- /dev/null +++ b/docs/bioimage-chatbot-extension-tutorial.ipynb @@ -0,0 +1,382 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## BioImage.IO Chatbot Extensions" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Documentation: https://github.com/bioimage-io/bioimageio-chatbot/blob/main/docs/development.md" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": { + "trusted": true + }, + "outputs": [ + { + "data": { + "application/javascript": "window.connectPlugin && window.connectPlugin(\"b78046eb-f9bd-432c-a1ea-5776c21ba3be\")", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "<_GatheringFuture pending>" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from imjoy import api\n", + "\n", + "async def setup():\n", + " chatbot = await api.createWindow(\n", + " src=\"http://127.0.0.1:9003/public/apps/bioimageio-chatbot-client/chat\",\n", + " name=\"BioImage.IO Chatbot\",\n", + " )\n", + " \n", + " await api.showMessage(str(await chatbot.getAllExtensions()))\n", + "\n", + "api.export({\"setup\": setup})" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "trusted": true + }, + "outputs": [ + { + "data": { + "application/javascript": "window.connectPlugin && window.connectPlugin(\"b78046eb-f9bd-432c-a1ea-5776c21ba3be\")", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "<_GatheringFuture pending>" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from imjoy_rpc import api\n", + "from pydantic import BaseModel, Field\n", + "\n", + " \n", + "class MoveStageInput(BaseModel):\n", + " \"\"\"Move the microscope stage\"\"\"\n", + " x: float = Field(..., description=\"x offset\")\n", + " y: float = Field(..., description=\"y offset\")\n", + "\n", + "class SnapImageInput(BaseModel):\n", + " \"\"\"Move the microscope stage\"\"\"\n", + " exposure: float = Field(..., description=\"exposure time\")\n", + "\n", + "async def move_stage(kwargs):\n", + " config = MoveStageInput(**kwargs)\n", + " print(config.x, config.y)\n", + "\n", + " return \"success\"\n", + "\n", + "async def snap_image(kwargs):\n", + " config = SnapImageInput(**kwargs)\n", + " print(config.exposure)\n", + " await api.showDialog(src=\"https://bioimage.io\")\n", + " return \"success\"\n", + "\n", + "async def setup():\n", + " chatbot = await api.createWindow(src=\"https://chat.bioimage.io/public/apps/bioimageio-chatbot-client/chat\")\n", + " \n", + " def get_schema():\n", + " return {\n", + " \"move_stage\": MoveStageInput.schema(),\n", + " \"snap_image\": SnapImageInput.schema()\n", + " }\n", + "\n", + " extension = {\n", + " \"_rintf\": True,\n", + " \"id\": \"squid-control\",\n", + " \"name\": \"Squid Microscope Control\",\n", + " \"description\": \"Contorl the squid microscope....\",\n", + " \"get_schema\": get_schema,\n", + " \"tools\": {\n", + " \"move_stage\": move_stage,\n", + " \"snap_image\": snap_image,\n", + " }\n", + " }\n", + " await chatbot.registerExtension(extension)\n", + "\n", + "api.export({\"setup\": setup})" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": { + "trusted": true + }, + "outputs": [ + { + "data": { + "application/javascript": "window.connectPlugin && window.connectPlugin(\"b78046eb-f9bd-432c-a1ea-5776c21ba3be\")", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "<_GatheringFuture pending>" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from imjoy_rpc import api\n", + "import sys\n", + "import io\n", + "from imjoy import api\n", + "from js import fetch\n", + "from pydantic import BaseModel, Field\n", + "from typing import Optional\n", + "from typing import List, Optional, Dict, Any\n", + "\n", + "class ResourceType(str):\n", + " MODEL = \"model\"\n", + " DATASET = \"dataset\"\n", + " APPLICATION = \"application\"\n", + "\n", + "def normalize_text(text: str) -> str:\n", + " return text.replace('_', ' ').lower()\n", + "\n", + "def matches_keywords(text: str, keywords: List[str]) -> bool:\n", + " normalized_text = normalize_text(text)\n", + " return any(keyword in normalized_text for keyword in keywords)\n", + "\n", + "def search_item(item: Dict[str, Any], keywords: List[str]) -> bool:\n", + " search_fields = [item.get('id', ''), item.get('nickname', ''), item.get('name', ''),\n", + " item.get('nickname_icon', ''), item.get('license', ''), item.get('description', '')\n", + " ] + [tag for tag in item.get('tags', [])]\n", + " search_fields += [author['name'] for author in item.get('authors', [])]\n", + " return any(matches_keywords(field, keywords) for field in search_fields)\n", + "\n", + "def search(keywords, type, top_k, resource_items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:\n", + " keywords = [normalize_text(keyword) for keyword in keywords]\n", + " filtered_items = []\n", + " for item in resource_items:\n", + " if type and item.get('type') != type:\n", + " continue\n", + " if search_item(item, keywords):\n", + " filtered_items.append(item)\n", + " if len(filtered_items) == top_k:\n", + " break\n", + " return filtered_items\n", + "\n", + "async def load_model_info():\n", + " response = await fetch(\"https://bioimage-io.github.io/collection-bioimage-io/collection.json\")\n", + " model_info = await response.json()\n", + " model_info = model_info.to_py()\n", + " resource_items = model_info['collection']\n", + " return resource_items\n", + "\n", + "def execute_code(script, context=None):\n", + " if context is None:\n", + " context = {}\n", + "\n", + " # Redirect stdout and stderr to capture their output\n", + " original_stdout = sys.stdout\n", + " original_stderr = sys.stderr\n", + " sys.stdout = io.StringIO()\n", + " sys.stderr = io.StringIO()\n", + "\n", + " try:\n", + " # Create a copy of the context to avoid modifying the original\n", + " local_vars = context.copy()\n", + "\n", + " # Execute the provided Python script with access to context variables\n", + " exec(script, local_vars)\n", + "\n", + " # Capture the output from stdout and stderr\n", + " stdout_output = sys.stdout.getvalue()\n", + " stderr_output = sys.stderr.getvalue()\n", + "\n", + " return {\n", + " \"stdout\": stdout_output,\n", + " \"stderr\": stderr_output,\n", + " # \"context\": local_vars # Include context variables in the result\n", + " }\n", + " except Exception as e:\n", + " return {\n", + " \"stdout\": \"\",\n", + " \"stderr\": str(e),\n", + " # \"context\": context # Include context variables in the result even if an error occurs\n", + " }\n", + " finally:\n", + " # Restore the original stdout and stderr\n", + " sys.stdout = original_stdout\n", + " sys.stderr = original_stderr\n", + "\n", + "async def register_chatbot_extension(register):\n", + " resource_items = await load_model_info()\n", + " types = set()\n", + " tags = set()\n", + " for resource in resource_items:\n", + " types.add(resource['type'])\n", + " tags.update(resource['tags'])\n", + " types = list(types)\n", + " tags = list(tags)[:5]\n", + " resource_item_stats = f\"\"\"- keys: {list(resource_items[0].keys())}\\n- resource types: {types}\\n- Exampletags: {tags}\\n\"\"\" #Here is an example: {resource_items[0]}\n", + "\n", + " class ModelZooInfoScript(BaseModel):\n", + " script: str = Field(..., description=\"\"\"Executable python script (Python runtime: Pyodide) for querying information\"\"\")\n", + " \n", + " ModelZooInfoScript.__doc__ = (\n", + " \"Search the BioImage Model Zoo for statistical information by executing Python3 scripts on the resource items.\"\n", + " \"For exampling counting models, applications, and datasets filtered by tags in the BioImage Model Zoo (bioimage.io). \"\n", + " \"The generated scripts will be executed browser pyodide environment, the script can access data through the 'resources' local variable, containing zoo resources as dictionaries. \"\n", + " \"Handle any missing fields in zoo items, and ensure outputs are directed to stdout. \"\n", + " \"Filter resources by the 'type' key without making remote server requests. 'resources' variable details:\\\\n\"\n", + " ) + resource_item_stats\n", + "\n", + "\n", + " class ModelZooSearchInput(BaseModel):\n", + " \"\"\"Search the BioImage Model Zoo (bioimage.io) resource items such as models, applications, datasets, etc. in the model zoo and return detailed information. The link format to the models etc. is: https://bioimage.io/#/?id=[ResourceID]\"\"\"\n", + " keywords: List[str] = Field(..., description=\"List of keywords to search for in the model zoo.\")\n", + " top_k: int = Field(3, description=\"The maximum number of search results to return. Default is 3. Please be aware each item may contain a large amount of data.\")\n", + " type: Optional[ResourceType] = Field(None, description=\"The type of resource to search for. Options include 'model', 'dataset', 'application'.\")\n", + "\n", + "\n", + " def get_schema():\n", + " return {\n", + " \"run_script\": ModelZooInfoScript.schema(),\n", + " \"search\": ModelZooSearchInput.schema()\n", + " }\n", + "\n", + " async def execute_script(kwargs):\n", + " info_script = ModelZooInfoScript.parse_obj(kwargs)\n", + " result = execute_code(info_script.script, {\"resources\": resource_items})\n", + " return result\n", + "\n", + " async def execute_search(kwargs):\n", + " config = ModelZooSearchInput.parse_obj(kwargs)\n", + " result = search(config.keywords, config.type, config.top_k, resource_items)\n", + " return result\n", + "\n", + " await register({\n", + " \"_rintf\": True,\n", + " \"id\": \"bioimage_model_zoo\",\n", + " \"name\": \"BioImage Model Zoo\",\n", + " \"description\": \"Getting information about models, applications, datasets, etc. in the BioImage Model Zoo. It takes a list of keywords or a python script to query the resources in the BioImage Model Zoo.\",\n", + " \"get_schema\": get_schema,\n", + " \"tools\": {\n", + " \"run_script\": execute_script,\n", + " \"search\": execute_search,\n", + " }\n", + " })\n", + "\n", + "\n", + "async def setup():\n", + " chatbot = await api.createWindow(src=\"https://chat.bioimage.io/public/apps/bioimageio-chatbot-client/chat\")\n", + " await register_chatbot_extension(chatbot.registerExtension)\n", + "\n", + "api.export({\"setup\": setup})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python (Pyodide)", + "language": "python", + "name": "python" + }, + "language_info": { + "codemirror_mode": { + "name": "python", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/docs/development.md b/docs/development.md index ad971cd..4d20845 100644 --- a/docs/development.md +++ b/docs/development.md @@ -36,54 +36,6 @@ chatbot.registerExtension({ }) ``` -Here is an example of a simple extension that can be used to control a microscope. +[Here](./bioimage-chatbot-extension-tutorial.ipynb) you can find a notebook with tutorials on how you can create your own extensions for the chatbot. -```python -from pydantic import BaseModel, Field -from imjoy_rpc import api - -class MoveStageInput(BaseModel): - """Move the microscope stage""" - x: float = Field(..., description="x offset") - y: float = Field(..., description="y offset") - -class SnapImageInput(BaseModel): - """Move the microscope stage""" - exposure: float = Field(..., description="exposure time") - -def move_stage(kwargs): - config = MoveStageInput(**kwargs) - print(config.x, config.y) - - return "success" - -def snap_image(kwargs): - config = SnapImageInput(**kwargs) - print(config.exposure) - - return "success" - -async def setup(): - chatbot = await api.createWindow(src="https://chat.bioimage.io/public/apps/bioimageio-chatbot-client/chat") - - def get_schema(): - return { - "move_stage": MoveStageInput.schema(), - "snap_image": SnapImageInput.schema() - } - - extension = { - "_rintf": True, - "id": "squid-control", - "name": "Squid Microscope Control", - "description": "Contorl the squid microscope....", - "get_schema": get_schema, - "tools": { - "move_stage": move_stage, - "snap_image": snap_image, - } - } - await chatbot.registerExtension(extension) - -api.export({"setup": setup}) -``` +You can also try it directly in your browser without installing anything by using the [ImJoy Jupyter Notebook](https://imjoy-notebook.netlify.app/lab/index.html?load=https://raw.githubusercontent.com/bioimage-io/bioimageio-chatbot/main/docs/bioimage-chatbot-extension-tutorial.ipynb&open=1).