tutorials/vision/vision-search.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d549d1e4-8054-4cc3-b84c-65d52d48bfbb",
   "metadata": {},
   "source": [
    "### Prequisites\n",
    "1.  Ollama Container.\n",
    "\n",
    "    In a separate terminal, run the docker compose file. \n",
    "\n",
    "    `docker compose -f vision-compose.yml up -d`\n",
    "\n",
    "2. Moondream vision model.\n",
    "\n",
    "   execute the load_model function with moondream model using ollama url\n",
    "\n",
    "<!--\n",
    "    Once the docker container is running, download the moondream model via curl\n",
    "\n",
    "\n",
    "   `curl http://localhost:11434/api/pull -d '{ \"name\": \"moondream:latest\" }'`\n",
    "\n",
    "    Verify if model is downloaded with command\n",
    "\n",
    "   `curl http://localhost:11434/api/tags`\n",
    "-->\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "07f69c54-4842-41b3-87d6-876ef6084b9e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import base64\n",
    "\n",
    "def load_model(ollama_url, model_name):\n",
    "    command = \"/api/pull\"\n",
    "    url = ollama_url + command\n",
    "    model = model_name + \":latest\"\n",
    "    payload = {\"name\": model}\n",
    "    \n",
    "    headers = {\n",
    "        \"Content-Type\": \"application/json\"\n",
    "    }\n",
    "    \n",
    "    response = requests.post(url, json=payload, headers=headers)\n",
    "    \n",
    "    if response.status_code == 200:\n",
    "        print(\"Request successful!\")\n",
    "        print(response.text)\n",
    "    else:\n",
    "        print(f\"Request failed with status code: {response.status_code}\")\n",
    "        print(response.text)\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db68443a-0707-4ca3-b899-a8da5e53fc15",
   "metadata": {},
   "outputs": [],
   "source": [
    "def explain_image(image_path, model, prompt, ollama_url):\n",
    "        \n",
    "    with open(image_path, \"rb\") as image_file:\n",
    "        encoded_image = base64.b64encode(image_file.read())\n",
    "\n",
    "    url = ollama_url + \"/api/chat\"\n",
    "    payload = {\n",
    "        \"model\": model,\n",
    "        \"messages\": [\n",
    "            {\n",
    "                \"role\": \"user\",\n",
    "                \"content\": prompt,\n",
    "                \"images\": [encoded_image.decode(\"utf-8\")]\n",
    "            }\n",
    "        ]\n",
    "    }\n",
    "    \n",
    "    headers = {\n",
    "        \"Content-Type\": \"application/json\"\n",
    "    }\n",
    "    \n",
    "    response = requests.post(url, json=payload, headers=headers)\n",
    "    \n",
    "    if response.status_code == 200:\n",
    "        for chunk in response.iter_lines():\n",
    "            if chunk:\n",
    "                data = chunk.decode('utf-8')\n",
    "                print(data)\n",
    "    else:\n",
    "        print(f\"Error: {response.status_code} - {response.text}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3844e544-4784-4cb9-a980-c566218a8026",
   "metadata": {},
   "outputs": [],
   "source": [
    "ollama_url = \"http://localhost:11434\"\n",
    "model_name = \"moondream\"\n",
    "load_model( ollama_url, model_name )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eecb0096-0973-4278-8382-22e6194e386a",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = \"moondream\"\n",
    "prompt = \"What is in this image?\"\n",
    "        \n",
    "url = \"http://localhost:11434\"\n",
    "    \n",
    "explain_image(\"../../docs/speech-inference.png\", model, prompt, url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "053e7758-0242-4c73-97b1-11da5d6a2749",
   "metadata": {},
   "outputs": [],
   "source": [
    "ollama_url = \"http://localhost:11434\"\n",
    "model_name = \"llava\"\n",
    "load_model( ollama_url, model_name )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "686f8c6b-3ac4-433d-a8cf-dfd122b7d963",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = \"llava\"\n",
    "prompt = \"What is in this image?\"\n",
    "        \n",
    "url = \"http://localhost:11434\"\n",
    "    \n",
    "explain_image(\"../../docs/speech-inference.png\", model, prompt, url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f0e75731-9a27-4137-a789-4625216d40d6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}