LangChain_Train_ChatGPT_with_your_own_Data.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyNCFdijehHSW+Qy26EiSn8U",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/psymed/AllureReport/blob/main/LangChain_Train_ChatGPT_with_your_own_Data.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Install Dependencies"
      ],
      "metadata": {
        "id": "0R5Obh8AucA3"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YcB3mtPuuJbX",
        "outputId": "fcbd7766-ca42-42df-efd7-c85121ebf125"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting langchain\n",
            "  Downloading langchain-0.0.174-py3-none-any.whl (869 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m869.7/869.7 kB\u001b[0m \u001b[31m8.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting chromadb\n",
            "  Downloading chromadb-0.3.23-py3-none-any.whl (71 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.3/71.3 kB\u001b[0m \u001b[31m3.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: PyYAML>=5.4.1 in /usr/local/lib/python3.10/dist-packages (from langchain) (6.0)\n",
            "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.0.10)\n",
            "Collecting aiohttp<4.0.0,>=3.8.3 (from langchain)\n",
            "  Downloading aiohttp-3.8.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m17.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting async-timeout<5.0.0,>=4.0.0 (from langchain)\n",
            "  Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n",
            "Collecting dataclasses-json<0.6.0,>=0.5.7 (from langchain)\n",
            "  Downloading dataclasses_json-0.5.7-py3-none-any.whl (25 kB)\n",
            "Requirement already satisfied: numexpr<3.0.0,>=2.8.4 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.8.4)\n",
            "Requirement already satisfied: numpy<2,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain) (1.22.4)\n",
            "Collecting openapi-schema-pydantic<2.0,>=1.2 (from langchain)\n",
            "  Downloading openapi_schema_pydantic-1.2.4-py3-none-any.whl (90 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m90.0/90.0 kB\u001b[0m \u001b[31m6.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pydantic<2,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain) (1.10.7)\n",
            "Requirement already satisfied: requests<3,>=2 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.27.1)\n",
            "Requirement already satisfied: tenacity<9.0.0,>=8.1.0 in /usr/local/lib/python3.10/dist-packages (from langchain) (8.2.2)\n",
            "Requirement already satisfied: pandas>=1.3 in /usr/local/lib/python3.10/dist-packages (from chromadb) (1.5.3)\n",
            "Collecting requests<3,>=2 (from langchain)\n",
            "  Downloading requests-2.30.0-py3-none-any.whl (62 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.5/62.5 kB\u001b[0m \u001b[31m2.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting hnswlib>=0.7 (from chromadb)\n",
            "  Downloading hnswlib-0.7.0.tar.gz (33 kB)\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting clickhouse-connect>=0.5.7 (from chromadb)\n",
            "  Downloading clickhouse_connect-0.5.24-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (922 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m922.6/922.6 kB\u001b[0m \u001b[31m58.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting sentence-transformers>=2.2.2 (from chromadb)\n",
            "  Downloading sentence-transformers-2.2.2.tar.gz (85 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.0/86.0 kB\u001b[0m \u001b[31m10.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: duckdb>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from chromadb) (0.7.1)\n",
            "Collecting fastapi>=0.85.1 (from chromadb)\n",
            "  Downloading fastapi-0.95.2-py3-none-any.whl (56 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.0/57.0 kB\u001b[0m \u001b[31m6.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting uvicorn[standard]>=0.18.3 (from chromadb)\n",
            "  Downloading uvicorn-0.22.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m6.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting posthog>=2.4.0 (from chromadb)\n",
            "  Downloading posthog-3.0.1-py2.py3-none-any.whl (37 kB)\n",
            "Requirement already satisfied: typing-extensions>=4.5.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (4.5.0)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (23.1.0)\n",
            "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (2.0.12)\n",
            "Collecting multidict<7.0,>=4.5 (from aiohttp<4.0.0,>=3.8.3->langchain)\n",
            "  Downloading multidict-6.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (114 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.5/114.5 kB\u001b[0m \u001b[31m13.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting yarl<2.0,>=1.0 (from aiohttp<4.0.0,>=3.8.3->langchain)\n",
            "  Downloading yarl-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (268 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m268.8/268.8 kB\u001b[0m \u001b[31m25.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting frozenlist>=1.1.1 (from aiohttp<4.0.0,>=3.8.3->langchain)\n",
            "  Downloading frozenlist-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (149 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m149.6/149.6 kB\u001b[0m \u001b[31m8.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting aiosignal>=1.1.2 (from aiohttp<4.0.0,>=3.8.3->langchain)\n",
            "  Downloading aiosignal-1.3.1-py3-none-any.whl (7.6 kB)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from clickhouse-connect>=0.5.7->chromadb) (2022.12.7)\n",
            "Requirement already satisfied: urllib3>=1.26 in /usr/local/lib/python3.10/dist-packages (from clickhouse-connect>=0.5.7->chromadb) (1.26.15)\n",
            "Requirement already satisfied: pytz in /usr/local/lib/python3.10/dist-packages (from clickhouse-connect>=0.5.7->chromadb) (2022.7.1)\n",
            "Collecting zstandard (from clickhouse-connect>=0.5.7->chromadb)\n",
            "  Downloading zstandard-0.21.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.7/2.7 MB\u001b[0m \u001b[31m86.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting lz4 (from clickhouse-connect>=0.5.7->chromadb)\n",
            "  Downloading lz4-4.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m65.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting marshmallow<4.0.0,>=3.3.0 (from dataclasses-json<0.6.0,>=0.5.7->langchain)\n",
            "  Downloading marshmallow-3.19.0-py3-none-any.whl (49 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.1/49.1 kB\u001b[0m \u001b[31m5.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting marshmallow-enum<2.0.0,>=1.5.1 (from dataclasses-json<0.6.0,>=0.5.7->langchain)\n",
            "  Downloading marshmallow_enum-1.5.1-py2.py3-none-any.whl (4.2 kB)\n",
            "Collecting typing-inspect>=0.4.0 (from dataclasses-json<0.6.0,>=0.5.7->langchain)\n",
            "  Downloading typing_inspect-0.8.0-py3-none-any.whl (8.7 kB)\n",
            "Collecting starlette<0.28.0,>=0.27.0 (from fastapi>=0.85.1->chromadb)\n",
            "  Downloading starlette-0.27.0-py3-none-any.whl (66 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.0/67.0 kB\u001b[0m \u001b[31m8.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.3->chromadb) (2.8.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from posthog>=2.4.0->chromadb) (1.16.0)\n",
            "Collecting monotonic>=1.5 (from posthog>=2.4.0->chromadb)\n",
            "  Downloading monotonic-1.6-py2.py3-none-any.whl (8.2 kB)\n",
            "Collecting backoff>=1.10.0 (from posthog>=2.4.0->chromadb)\n",
            "  Downloading backoff-2.2.1-py3-none-any.whl (15 kB)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (3.4)\n",
            "Collecting transformers<5.0.0,>=4.6.0 (from sentence-transformers>=2.2.2->chromadb)\n",
            "  Downloading transformers-4.29.2-py3-none-any.whl (7.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.1/7.1 MB\u001b[0m \u001b[31m91.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from sentence-transformers>=2.2.2->chromadb) (4.65.0)\n",
            "Requirement already satisfied: torch>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from sentence-transformers>=2.2.2->chromadb) (2.0.1+cu118)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.10/dist-packages (from sentence-transformers>=2.2.2->chromadb) (0.15.2+cu118)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from sentence-transformers>=2.2.2->chromadb) (1.2.2)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from sentence-transformers>=2.2.2->chromadb) (1.10.1)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from sentence-transformers>=2.2.2->chromadb) (3.8.1)\n",
            "Collecting sentencepiece (from sentence-transformers>=2.2.2->chromadb)\n",
            "  Downloading sentencepiece-0.1.99-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m57.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting huggingface-hub>=0.4.0 (from sentence-transformers>=2.2.2->chromadb)\n",
            "  Downloading huggingface_hub-0.14.1-py3-none-any.whl (224 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m224.5/224.5 kB\u001b[0m \u001b[31m18.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy<3,>=1.4->langchain) (2.0.2)\n",
            "Requirement already satisfied: click>=7.0 in /usr/local/lib/python3.10/dist-packages (from uvicorn[standard]>=0.18.3->chromadb) (8.1.3)\n",
            "Collecting h11>=0.8 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m5.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting httptools>=0.5.0 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading httptools-0.5.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (414 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m414.1/414.1 kB\u001b[0m \u001b[31m32.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting python-dotenv>=0.13 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading python_dotenv-1.0.0-py3-none-any.whl (19 kB)\n",
            "Collecting uvloop!=0.15.0,!=0.15.1,>=0.14.0 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading uvloop-0.17.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.1/4.1 MB\u001b[0m \u001b[31m65.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting watchfiles>=0.13 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading watchfiles-0.19.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m58.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting websockets>=10.4 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading websockets-11.0.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (129 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m129.9/129.9 kB\u001b[0m \u001b[31m12.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence-transformers>=2.2.2->chromadb) (3.12.0)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence-transformers>=2.2.2->chromadb) (2023.4.0)\n",
            "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence-transformers>=2.2.2->chromadb) (23.1)\n",
            "Requirement already satisfied: anyio<5,>=3.4.0 in /usr/local/lib/python3.10/dist-packages (from starlette<0.28.0,>=0.27.0->fastapi>=0.85.1->chromadb) (3.6.2)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence-transformers>=2.2.2->chromadb) (1.11.1)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence-transformers>=2.2.2->chromadb) (3.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence-transformers>=2.2.2->chromadb) (3.1.2)\n",
            "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence-transformers>=2.2.2->chromadb) (2.0.0)\n",
            "Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.6.0->sentence-transformers>=2.2.2->chromadb) (3.25.2)\n",
            "Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.6.0->sentence-transformers>=2.2.2->chromadb) (16.0.5)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers>=2.2.2->chromadb) (2022.10.31)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1 (from transformers<5.0.0,>=4.6.0->sentence-transformers>=2.2.2->chromadb)\n",
            "  Downloading tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m93.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting mypy-extensions>=0.3.0 (from typing-inspect>=0.4.0->dataclasses-json<0.6.0,>=0.5.7->langchain)\n",
            "  Downloading mypy_extensions-1.0.0-py3-none-any.whl (4.7 kB)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk->sentence-transformers>=2.2.2->chromadb) (1.2.0)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->sentence-transformers>=2.2.2->chromadb) (3.1.0)\n",
            "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.10/dist-packages (from torchvision->sentence-transformers>=2.2.2->chromadb) (8.4.0)\n",
            "Requirement already satisfied: sniffio>=1.1 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.4.0->starlette<0.28.0,>=0.27.0->fastapi>=0.85.1->chromadb) (1.3.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.6.0->sentence-transformers>=2.2.2->chromadb) (2.1.2)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.6.0->sentence-transformers>=2.2.2->chromadb) (1.3.0)\n",
            "Building wheels for collected packages: hnswlib, sentence-transformers\n",
            "  Building wheel for hnswlib (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for hnswlib: filename=hnswlib-0.7.0-cp310-cp310-linux_x86_64.whl size=2119843 sha256=39f8ded83fc0ccdaad7291f649596dfeb4e5e3fd55042f9d475c40f5bf3193f3\n",
            "  Stored in directory: /root/.cache/pip/wheels/8a/ae/ec/235a682e0041fbaeee389843670581ec6c66872db856dfa9a4\n",
            "  Building wheel for sentence-transformers (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sentence-transformers: filename=sentence_transformers-2.2.2-py3-none-any.whl size=125926 sha256=037a91ee6ae4b136742eafc81fa33b1b170e9d40c0d456499a1a547630d0fa9f\n",
            "  Stored in directory: /root/.cache/pip/wheels/62/f2/10/1e606fd5f02395388f74e7462910fe851042f97238cbbd902f\n",
            "Successfully built hnswlib sentence-transformers\n",
            "Installing collected packages: tokenizers, sentencepiece, monotonic, zstandard, websockets, uvloop, requests, python-dotenv, mypy-extensions, multidict, marshmallow, lz4, httptools, hnswlib, h11, frozenlist, backoff, async-timeout, yarl, watchfiles, uvicorn, typing-inspect, starlette, posthog, openapi-schema-pydantic, marshmallow-enum, huggingface-hub, clickhouse-connect, aiosignal, transformers, fastapi, dataclasses-json, aiohttp, langchain, sentence-transformers, chromadb\n",
            "  Attempting uninstall: requests\n",
            "    Found existing installation: requests 2.27.1\n",
            "    Uninstalling requests-2.27.1:\n",
            "      Successfully uninstalled requests-2.27.1\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "google-colab 1.0.0 requires requests==2.27.1, but you have requests 2.30.0 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed aiohttp-3.8.4 aiosignal-1.3.1 async-timeout-4.0.2 backoff-2.2.1 chromadb-0.3.23 clickhouse-connect-0.5.24 dataclasses-json-0.5.7 fastapi-0.95.2 frozenlist-1.3.3 h11-0.14.0 hnswlib-0.7.0 httptools-0.5.0 huggingface-hub-0.14.1 langchain-0.0.174 lz4-4.3.2 marshmallow-3.19.0 marshmallow-enum-1.5.1 monotonic-1.6 multidict-6.0.4 mypy-extensions-1.0.0 openapi-schema-pydantic-1.2.4 posthog-3.0.1 python-dotenv-1.0.0 requests-2.30.0 sentence-transformers-2.2.2 sentencepiece-0.1.99 starlette-0.27.0 tokenizers-0.13.3 transformers-4.29.2 typing-inspect-0.8.0 uvicorn-0.22.0 uvloop-0.17.0 watchfiles-0.19.0 websockets-11.0.3 yarl-1.9.2 zstandard-0.21.0\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting openAi\n",
            "  Downloading openai-0.27.7-py3-none-any.whl (71 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m72.0/72.0 kB\u001b[0m \u001b[31m4.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: requests>=2.20 in /usr/local/lib/python3.10/dist-packages (from openAi) (2.30.0)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from openAi) (4.65.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from openAi) (3.8.4)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openAi) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openAi) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openAi) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openAi) (2022.12.7)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openAi) (23.1.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openAi) (6.0.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openAi) (4.0.2)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openAi) (1.9.2)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openAi) (1.3.3)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openAi) (1.3.1)\n",
            "Installing collected packages: openAi\n",
            "Successfully installed openAi-0.27.7\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting tiktoken\n",
            "  Downloading tiktoken-0.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m31.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken) (2022.10.31)\n",
            "Requirement already satisfied: requests>=2.26.0 in /usr/local/lib/python3.10/dist-packages (from tiktoken) (2.30.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.26.0->tiktoken) (2022.12.7)\n",
            "Installing collected packages: tiktoken\n",
            "Successfully installed tiktoken-0.4.0\n"
          ]
        }
      ],
      "source": [
        "!pip install langchain chromadb\n",
        "!pip install openAi\n",
        "!pip install tiktoken"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "from langchain.indexes import VectorstoreIndexCreator\n",
        "from dotenv import dotenv_values\n",
        "os.environ['OPENAI_API_KEY'] = dotenv_values()['openai_api_key'] # set environment variable"
      ],
      "metadata": {
        "id": "Rb-TUuEGuqHb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.document_loaders import TextLoader\n",
        "\n",
        "loader1 = TextLoader('sd_wiki.txt')\n",
        "loader2 = TextLoader('midjourney_wiki.txt')"
      ],
      "metadata": {
        "id": "WAHYnq6Sd8es"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# or simply\n",
        "index = VectorstoreIndexCreator().from_loaders([loader1,loader2])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xqTJodp2eTyH",
        "outputId": "ac510487-dae6-4e19-e764-e59ffc1ef51e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:chromadb:Using embedded DuckDB without persistence: data will be transient\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "index.query('who authored the theoretical paper behind stable defusion?')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "nc-KikpXgMQO",
        "outputId": "f5ca814f-2ff6-4f9e-9cde-b7438049a5e9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "' Patrick Esser of Runway and Robin Rombach of CompVis.'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "index.query('what is midjourney?')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        },
        "id": "jyYKjFBsgaby",
        "outputId": "e6367daa-e1e5-4732-bf1d-f823903dcb53"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "' Midjourney is a generative artificial intelligence program and service created and hosted by a San Francisco-based independent research lab Midjourney, Inc. Midjourney generates images from natural language descriptions, called \"prompts\", similar to OpenAI\\'s DALL-E and Stable Diffusion.'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "index.query('create a list of top 5 pros and cons per each midjourney and Stable difusion and provide a comparison between the 2 tools.')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "LrpXGP8uggMW",
        "outputId": "e874f855-271e-4909-f313-289dfdcfd072"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\" I don't know.\""
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "index.query('What are the main capabilities of midjourney?')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        },
        "id": "ed3jYv2pgw4P",
        "outputId": "cfbf1be4-e11b-48c7-d180-7e9ad7ba6e9b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "' Midjourney is a generative artificial intelligence program and service that generates images from natural language descriptions. It is used by artists for rapid prototyping of artistic concepts, by the advertising industry to create original content and brainstorm ideas quickly, and by other industries for custom ads, special effects, and e-commerce advertising.'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# proving that we can learn chatGPT new data it didn't know"
      ],
      "metadata": {
        "id": "8O20xurKhAPk"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "index.query('what is a controlnet?')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89
        },
        "id": "REaORMKbg_uj",
        "outputId": "7fc114b0-afa3-46d5-c256-e1d4498be0ff"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "' A ControlNet is a neural network architecture designed to manage diffusion models by incorporating additional conditions. It duplicates the weights of neural network blocks into a \"locked\" copy and a \"trainable\" copy. The \"trainable\" copy learns the desired condition, while the \"locked\" copy preserves the original model. This approach ensures that training with small datasets of image pairs does not compromise the integrity of production-ready diffusion models.'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Using QueryResources to ask multiple questions"
      ],
      "metadata": {
        "id": "b5V4tYHlhTHH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# useful when quering multiple documents \n",
        "index.query_with_sources(\"Can you access midjourney via a web app ?\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "THUwinu6hWdt",
        "outputId": "65d98ecd-11e4-401f-e2a6-ceb70088cc60"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'question': 'Can you access midjourney via a web app ?',\n",
              " 'answer': ' Yes, you can access Midjourney via a web app.\\n',\n",
              " 'sources': 'midjourney_wiki.txt'}"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Now we'll use a different an more robust way to load a documents into a db\n",
        "\n",
        "# Step 1 - Load data & split into chunks"
      ],
      "metadata": {
        "id": "9CZnJBQOiXcV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.document_loaders import TextLoader\n",
        "loader1 = TextLoader('sd_wiki.txt')\n",
        "loader2 = TextLoader('midjourney_wiki.txt')\n",
        "documents = loader1.load()\n",
        "documents += loader2.load() # put document objects inside a list"
      ],
      "metadata": {
        "id": "8v6BIWTciqS0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "documents"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cKOWIscBjl44",
        "outputId": "299a4061-3a68-4617-aa23-4bce4da5d722"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[Document(page_content='Stable Diffusion\\n\\nArticle\\nTalk\\nRead\\nEdit\\nView history\\n\\nTools\\nFrom Wikipedia, the free encyclopedia\\nStable Diffusion\\nA photograph of an astronaut riding a horse 2022-08-28.png\\nAn image generated by Stable Diffusion based on the text prompt \"a photograph of an astronaut riding a horse\"\\nOriginal author(s)\\tRunway, CompVis, and Stability AI\\nDeveloper(s)\\tStability AI\\nInitial release\\tAugust 22, 2022\\nStable release\\t\\n2.1 (model)[1] / December 7, 2022\\nRepository\\tgithub.com/Stability-AI/stablediffusion\\nWritten in\\tPython[2]\\nOperating system\\tAny that support CUDA kernels\\nType\\tText-to-image model\\nLicense\\tCreative ML OpenRAIL-M\\nWebsite\\tommer-lab.com/research/latent-diffusion-models/ \\nStable Diffusion is a deep learning, text-to-image model released in 2022. It is primarily used to generate detailed images conditioned on text descriptions, though it can also be applied to other tasks such as inpainting, outpainting, and generating image-to-image translations guided by a text prompt.[3] It was developed by the start-up Stability AI in collaboration with a number of academic researchers and non-profit organizations.[4]\\n\\nStable Diffusion is a latent diffusion model, a kind of deep generative neural network. Its code and model weights have been released publicly,[5] and it can run on most consumer hardware equipped with a modest GPU with at least 8 GB VRAM. This marked a departure from previous proprietary text-to-image models such as DALL-E and Midjourney which were accessible only via cloud services.[6][7]\\n\\nDevelopment\\nThe development of Stable Diffusion was funded and shaped by the start-up company Stability AI.[7][8][9] The technical license for the model was released by the CompVis group at Ludwig Maximilian University of Munich.[7] Development was led by Patrick Esser of Runway and Robin Rombach of CompVis, who were among the researchers who had earlier invented the latent diffusion model architecture used by Stable Diffusion.[4] Stability AI also credited EleutherAI and LAION (a German nonprofit which assembled the dataset on which Stable Diffusion was trained) as supporters of the project.[4]\\n\\nIn October 2022, Stability AI raised US$101 million in a round led by Lightspeed Venture Partners and Coatue Management.[10]\\n\\nTechnology\\n\\nDiagram of the latent diffusion architecture used by Stable Diffusion\\n\\nThe denoising process used by Stable Diffusion. The model generates images by iteratively denoising random noise until a configured number of steps have been reached, guided by the CLIP text encoder pretrained on concepts along with the attention mechanism, resulting in the desired image depicting a representation of the trained concept.\\nArchitecture\\nStable Diffusion uses a kind of diffusion model (DM), called a latent diffusion model (LDM) developed by the CompVis group at LMU Munich.[11][5] Introduced in 2015, diffusion models are trained with the objective of removing successive applications of Gaussian noise on training images, which can be thought of as a sequence of denoising autoencoders. Stable Diffusion consists of 3 parts: the variational autoencoder (VAE), U-Net, and an optional text encoder.[12] The VAE encoder compresses the image from pixel space to a smaller dimensional latent space, capturing a more fundamental semantic meaning of the image.[11] Gaussian noise is iteratively applied to the compressed latent representation during forward diffusion.[12] The U-Net block, composed of a ResNet backbone, denoises the output from forward diffusion backwards to obtain a latent representation. Finally, the VAE decoder generates the final image by converting the representation back into pixel space.[12] The denoising step can be flexibly conditioned on a string of text, an image, or another modality. The encoded conditioning data is exposed to denoising U-Nets via a cross-attention mechanism.[12] For conditioning on text, the fixed, pretrained CLIP ViT-L/14 text encoder is used to transform text prompts to an embedding space.[5] Researchers point to increased computational efficiency for training and generation as an advantage of LDMs.[4][11]\\n\\nTraining data\\nStable Diffusion was trained on pairs of images and captions taken from LAION-5B, a publicly available dataset derived from Common Crawl data scraped from the web, where 5 billion image-text pairs were classified based on language and filtered into separate datasets by resolution, a predicted likelihood of containing a watermark, and predicted \"aesthetic\" score (e.g. subjective visual quality).[13] The dataset was created by LAION, a German non-profit which receives funding from Stability AI.[13][14] The Stable Diffusion model was trained on three subsets of LAION-5B: laion2B-en, laion-high-resolution, and laion-aesthetics v2 5+.[13] A third-party analysis of the model\\'s training data identified that out of a smaller subset of 12 million images taken from the original wider dataset used, approximately 47% of the sample size of images came from 100 different domains, with Pinterest taking up 8.5% of the subset, followed by websites such as WordPress, Blogspot, Flickr, DeviantArt and Wikimedia Commons.[15][13]\\n\\nTraining procedures\\nThe model was initially trained on the laion2B-en and laion-high-resolution subsets, with the last few rounds of training done on LAION-Aesthetics v2 5+, a subset of 600 million captioned images which the LAION-Aesthetics Predictor V2 predicted that humans would, on average, give a score of at least 5 out of 10 when asked to rate how much they liked them.[16][13][17] The LAION-Aesthetics v2 5+ subset also excluded low-resolution images and images which LAION-5B-WatermarkDetection identified as carrying a watermark with greater than 80% probability.[13] Final rounds of training additionally dropped 10% of text conditioning to improve Classifier-Free Diffusion Guidance.[18]\\n\\nThe model was trained using 256 Nvidia A100 GPUs on Amazon Web Services for a total of 150,000 GPU-hours, at a cost of $600,000.[19][20][21]\\n\\nLimitations\\nStable Diffusion has issues with degradation and inaccuracies in certain scenarios. Initial releases of the model were trained on a dataset that consists of 512×512 resolution images, meaning that the quality of generated images noticeably degrades when user specifications deviate from its \"expected\" 512×512 resolution;[22] the version 2.0 update of the Stable Diffusion model later introduced the ability to natively generate images at 768×768 resolution.[23] Another challenge is in generating human limbs due to poor data quality of limbs in the LAION database.[24] The model is insufficiently trained to understand human limbs and faces due to the lack of representative features in the database, and prompting the model to generate images of such type can confound the model.[25]\\n\\nAccessibility for individual developers can also be a problem. In order to customize the model for new use cases that are not included in the dataset, such as generating anime characters (\"waifu diffusion\"),[26] new data and further training are required. Fine-tuned adaptations of Stable Diffusion created through additional retraining have been used for a variety of different use-cases, from medical imaging[27] to algorithmically-generated music.[28] However, this fine-tuning process is sensitive to the quality of new data; low resolution images or different resolutions from the original data can not only fail to learn the new task but degrade the overall performance of the model. Even when the model is additionally trained on high quality images, it is difficult for individuals to run models in consumer electronics. For example, the training process for waifu-diffusion requires a minimum 30 GB of VRAM,[29] which exceeds the usual resource provided in consumer GPUs, such as Nvidia\\'s GeForce 30 series having around 12 GB.[30]\\n\\nThe creators of Stable Diffusion acknowledge the potential for algorithmic bias, as the model was primarily trained on images with English descriptions.[20] As a result, generated images reinforce social biases and are from a western perspective, as the creators note that the model lacks data from other communities and cultures. The model gives more accurate results for prompts that are written in English in comparison to those written in other languages, with western or white cultures often being the default representation.[20]\\n\\nEnd-user fine-tuning\\nTo address the limitations of the model\\'s initial training, end-users may opt to implement additional training to fine-tune generation outputs to match more specific use-cases. There are three methods in which user-accessible fine-tuning can be applied to a Stable Diffusion model checkpoint:\\n\\nAn \"embedding\" can be trained from a collection of user-provided images, and allows the model to generate visually similar images whenever the name of the embedding is used within a generation prompt.[31] Embeddings are based on the \"textual inversion\" concept developed by researchers from Tel Aviv University in 2022 with support from Nvidia, where vector representations for specific tokens used by the model\\'s text encoder are linked to new pseudo-words. Embeddings can be used to reduce biases within the original model, or mimic visual styles.[32]\\nA \"hypernetwork\" is a small pretrained neural network that is applied to various points within a larger neural network, and refers to the technique created by NovelAI developer Kurumuz in 2021, originally intended for text-generation transformer models. Hypernetworks steer results towards a particular direction, allowing Stable Diffusion-based models to imitate the art style of specific artists, even if the artist is not recognised by the original model; they process the image by finding key areas of importance such as hair and eyes, and then patch these areas in secondary latent space.[33]\\nDreamBooth is a deep learning generation model developed by researchers from Google Research and Boston University in 2022 which can fine-tune the model to generate precise, personalised outputs that depict a specific subject, following training via a set of images which depict the subject.[34]\\nCapabilities\\nThe Stable Diffusion model supports the ability to generate new images from scratch through the use of a text prompt describing elements to be included or omitted from the output.[5] Existing images can be re-drawn by the model to incorporate new elements described by a text prompt (a process known as \"guided image synthesis\"[35]) through its diffusion-denoising mechanism.[5] In addition, the model also allows the use of prompts to partially alter existing images via inpainting and outpainting, when used with an appropriate user interface that supports such features, of which numerous different open source implementations exist.[36]\\n\\nStable Diffusion is recommended to be run with 10 GB or more VRAM, however users with less VRAM may opt to load the weights in float16 precision instead of the default float32 to tradeoff model performance with lower VRAM usage.[22]\\n\\nText to image generation\\n\\n\\n\\nDemonstration of the effect of negative prompts on image generation\\nTop: no negative prompt\\nCentre: \"green trees\"\\nBottom: \"round stones, round rocks\"\\nThe text to image sampling script within Stable Diffusion, known as \"txt2img\", consumes a text prompt in addition to assorted option parameters covering sampling types, output image dimensions, and seed values. The script outputs an image file based on the model\\'s interpretation of the prompt.[5] Generated images are tagged with an invisible digital watermark to allow users to identify an image as generated by Stable Diffusion,[5] although this watermark loses its efficacy if the image is resized or rotated.[37]\\n\\nEach txt2img generation will involve a specific seed value which affects the output image. Users may opt to randomize the seed in order to explore different generated outputs, or use the same seed to obtain the same image output as a previously generated image.[22] Users are also able to adjust the number of inference steps for the sampler; a higher value takes a longer duration of time, however a smaller value may result in visual defects.[22] Another configurable option, the classifier-free guidance scale value, allows the user to adjust how closely the output image adheres to the prompt.[18] More experimentative use cases may opt for a lower scale value, while use cases aiming for more specific outputs may use a higher value.[22]\\n\\nAdditional text2img features are provided by front-end implementations of Stable Diffusion, which allow users to modify the weight given to specific parts of the text prompt. Emphasis markers allow users to add or reduce emphasis to keywords by enclosing them with brackets.[38] An alternative method of adjusting weight to parts of the prompt are \"negative prompts\". Negative prompts are a feature included in some front-end implementations, including Stability AI\\'s own DreamStudio cloud service, and allow the user to specify prompts which the model should avoid during image generation. The specified prompts may be undesirable image features that would otherwise be present within image outputs due to the positive prompts provided by the user, or due to how the model was originally trained, with mangled human hands being a common example.[36][1]\\n\\nImage modification\\nStable Diffusion also includes another sampling script, \"img2img\", which consumes a text prompt, path to an existing image, and strength value between 0.0 and 1.0. The script outputs a new image based on the original image that also features elements provided within the text prompt. The strength value denotes the amount of noise added to the output image. A higher strength value produces more variation within the image but may produce an image that is not semantically consistent with the prompt provided.[5]\\n\\nThe ability of img2img to add noise to the original image makes it potentially useful for data anonymization and data augmentation, in which the visual features of image data are changed and anonymized.[39] The same process may also be useful for image upscaling, in which the resolution of an image is increased, with more detail potentially being added to the image.[39] Additionally, Stable Diffusion has been experimented with as a tool for image compression. Compared to JPEG and WebP, the recent methods used for image compression in Stable Diffusion face limitations in preserving small text and faces.[40]\\n\\nAdditional use-cases for image modification via img2img are offered by numerous front-end implementations of the Stable Diffusion model. Inpainting involves selectively modifying a portion of an existing image delineated by a user-provided layer mask, which fills the masked space with newly generated content based on the provided prompt.[36] A dedicated model specifically fine-tuned for inpainting use-cases was created by Stability AI alongside the release of Stable Diffusion 2.0.[23] Conversely, outpainting extends an image beyond its original dimensions, filling the previously empty space with content generated based on the provided prompt.[36]\\n\\nA depth-guided model, named \"depth2img\", was introduced with the release of Stable Diffusion 2.0 on November 24, 2022; this model infers the depth of the provided input image, and generates a new output image based on both the text prompt and the depth information, which allows the coherence and depth of the original input image to be maintained in the generated output.[23]\\n\\nControlNet\\nControlNet[41] is a neural network architecture designed to manage diffusion models by incorporating additional conditions. It duplicates the weights of neural network blocks into a \"locked\" copy and a \"trainable\" copy. The \"trainable\" copy learns the desired condition, while the \"locked\" copy preserves the original model. This approach ensures that training with small datasets of image pairs does not compromise the integrity of production-ready diffusion models. The \"zero convolution\" is a 1×1 convolution with both weight and bias initialized to zero. Before training, all zero convolutions produce zero output, preventing any distortion caused by ControlNet. No layer is trained from scratch; the process is still fine-tuning, keeping the original model secure. This method enables training on small-scale or even personal devices.\\n\\nUsage and controversy\\nStable Diffusion claims no rights on generated images and freely gives users the rights of usage to any generated images from the model provided that the image content is not illegal or harmful to individuals. The freedom provided to users over image usage has caused controversy over the ethics of ownership, as Stable Diffusion and other generative models are trained from copyrighted images without the owner’s consent.[42]\\n\\nAs visual styles and compositions are not subject to copyright, it is often interpreted that users of Stable Diffusion who generate images of artworks should not be considered to be infringing upon the copyright of visually similar works.[43] However, individuals depicted in generated images may be protected by personality rights if their likeness is used,[43] and intellectual property such as recognizable brand logos still remain protected by copyright. Nonetheless, visual artists have expressed concern that widespread usage of image synthesis software such as Stable Diffusion may eventually lead to human artists, along with photographers, models, cinematographers, and actors, gradually losing commercial viability against AI-based competitors.[9]\\n\\nStable Diffusion is notably more permissive in the types of content users may generate, such as violent or sexually explicit imagery, in comparison to other commercial products based on generative AI.[44] Addressing the concerns that the model may be used for abusive purposes, CEO of Stability AI, Emad Mostaque, explains that \"[it is] peoples\\' responsibility as to whether they are ethical, moral, and legal in how they operate this technology\",[7] and that putting the capabilities of Stable Diffusion into the hands of the public would result in the technology providing a net benefit, in spite of the potential negative consequences.[7] In addition, Mostaque argues that the intention behind the open availability of Stable Diffusion is to end corporate control and dominance over such technologies, who have previously only developed closed AI systems for image synthesis.[7][44] This is reflected by the fact that any restrictions Stability AI places on the content that users may generate can easily be bypassed due to the availability of the source code.[42]\\n\\nLitigation\\nIn January of 2023, three artists: Sarah Andersen, Kelly McKernan, and Karla Ortiz filed a copyright infringement lawsuit against Stability AI, Midjourney, and DeviantArt, claiming that these companies have infringed the rights of millions of artists by training AI tools on five billion images scraped from the web without the consent of the original artists.[45] The same month, Stability AI was also sued by Getty Images for using its images in the training data.[8]\\n\\nLicense\\nUnlike models like DALL-E, Stable Diffusion makes its source code available,[46][5] along with the model (pretrained weights). It applies the Creative ML OpenRAIL-M license, a form of Responsible AI License (RAIL), to the model (M).[47] The licence prohibits certain use cases, including crime, libel, harassment, doxing, \"exploiting ... minors\", giving medical advice, automatically creating legal obligations, producing legal evidence, and \"discriminating against or harming individuals or groups based on ... social behavior or ... personal or personality characteristics ... [or] legally protected characteristics or categories\".[48][49] The user owns the rights to their generated output images, and is free to use them commercially.[50]\\n\\nSee also\\n15.ai\\nArtificial intelligence art\\nCraiyon\\nImagen (Google Brain)\\nSynthography\\nHugging Face\\nReferences\\n \"Stable Diffusion v2.1 and DreamStudio Updates 7-Dec 22\". stability.ai. Archived from the original on December 10, 2022.\\n Ryan O\\'Connor (August 23, 2022). \"How to Run Stable Diffusion Locally to Generate Images\". https://www.assemblyai.com/blog/how-to-run-stable-diffusion-locally-to-generate-images/. {{cite web}}: |access-date= requires |url= (help); External link in |website= (help); Missing or empty |url= (help)\\n \"Diffuse The Rest - a Hugging Face Space by huggingface\". huggingface.co. Archived from the original on 2022-09-05. Retrieved 2022-09-05.\\n \"Stable Diffusion Launch Announcement\". Stability.Ai. Archived from the original on 2022-09-05. Retrieved 2022-09-06.\\n \"Stable Diffusion Repository on GitHub\". CompVis - Machine Vision and Learning Research Group, LMU Munich. 17 September 2022. Retrieved 17 September 2022.\\n \"The new killer app: Creating AI art will absolutely crush your PC\". PCWorld. Archived from the original on 2022-08-31. Retrieved 2022-08-31.\\n Vincent, James (15 September 2022). \"Anyone can use this AI art generator — that\\'s the risk\". The Verge.\\n Korn, Jennifer (2023-01-17). \"Getty Images suing the makers of popular AI art tool for allegedly stealing photos\". CNN. Retrieved 2023-01-22.\\n Heikkilä, Melissa (16 September 2022). \"This artist is dominating AI-generated art. And he\\'s not happy about it\". MIT Technology Review.\\n Wiggers, Kyle (17 October 2022). \"Stability AI, the startup behind Stable Diffusion, raises $101M\". Techcrunch. Retrieved 2022-10-17.\\n Rombach; Blattmann; Lorenz; Esser; Ommer (June 2022). High-Resolution Image Synthesis with Latent Diffusion Models (PDF). International Conference on Computer Vision and Pattern Recognition (CVPR). New Orleans, LA. pp. 10684–10695. arXiv:2112.10752.\\n Alammar, Jay. \"The Illustrated Stable Diffusion\". jalammar.github.io. Retrieved 2022-10-31.\\n Baio, Andy (2022-08-30). \"Exploring 12 Million of the 2.3 Billion Images Used to Train Stable Diffusion\\'s Image Generator\". Waxy.org. Retrieved 2022-11-02.\\n \"This artist is dominating AI-generated art. And he\\'s not happy about it\". MIT Technology Review. Retrieved 2022-11-02.\\n Ivanovs, Alex (2022-09-08). \"Stable Diffusion: Tutorials, Resources, and Tools\". Stack Diary. Retrieved 2022-11-02.\\n Schuhmann, Christoph (2022-11-02), CLIP+MLP Aesthetic Score Predictor, retrieved 2022-11-02\\n \"LAION-Aesthetics | LAION\". laion.ai. Archived from the original on 2022-08-26. Retrieved 2022-09-02.\\n Ho, Jonathan; Salimans, Tim (2022-07-25). \"Classifier-Free Diffusion Guidance\". arXiv:2207.12598 [cs.LG].\\n Mostaque, Emad (August 28, 2022). \"Cost of construction\". Twitter. Archived from the original on 2022-09-06. Retrieved 2022-09-06.\\n \"CompVis/stable-diffusion-v1-4 · Hugging Face\". huggingface.co. Retrieved 2022-11-02.\\n Wiggers, Kyle (2022-08-12). \"A startup wants to democratize the tech behind DALL-E 2, consequences be damned\". TechCrunch. Retrieved 2022-11-02.\\n \"Stable Diffusion with 🧨 Diffusers\". huggingface.co. Retrieved 2022-10-31.\\n \"Stable Diffusion 2.0 Release\". stability.ai. Archived from the original on December 10, 2022.\\n \"LAION\". laion.ai. Retrieved 2022-10-31.\\n \"Generating images with Stable Diffusion\". Paperspace Blog. 2022-08-24. Retrieved 2022-10-31.\\n \"hakurei/waifu-diffusion · Hugging Face\". huggingface.co. Retrieved 2022-10-31.\\n Chambon, Pierre; Bluethgen, Christian; Langlotz, Curtis P.; Chaudhari, Akshay (2022-10-09). \"Adapting Pretrained Vision-Language Foundational Models to Medical Imaging Domains\". arXiv:2210.04133 [cs.CV].\\n Seth Forsgren; Hayk Martiros. \"Riffusion - Stable diffusion for real-time music generation\". Riffusion. Archived from the original on December 16, 2022.\\n Mercurio, Anthony (2022-10-31), Waifu Diffusion, retrieved 2022-10-31\\n Smith, Ryan. \"NVIDIA Quietly Launches GeForce RTX 3080 12GB: More VRAM, More Power, More Money\". www.anandtech.com. Retrieved 2022-10-31.\\n Dave James (October 28, 2022). \"I thrashed the RTX 4090 for 8 hours straight training Stable Diffusion to paint like my uncle Hermann\". PC Gamer. Archived from the original on November 9, 2022.\\n Gal, Rinon; Alaluf, Yuval; Atzmon, Yuval; Patashnik, Or; Bermano, Amit H.; Chechik, Gal; Cohen-Or, Daniel (2022-08-02). \"An Image is Worth One Word: Personalizing Text-to-Image Generation using Textual Inversion\". arXiv:2208.01618 [cs.CV].\\n \"NovelAI Improvements on Stable Diffusion\". NovelAI. October 11, 2022. Archived from the original on October 27, 2022.\\n Yuki Yamashita (September 1, 2022). \"愛犬の合成画像を生成できるAI 文章で指示するだけでコスプレ 米Googleが開発\". ITmedia Inc. (in Japanese). Archived from the original on August 31, 2022.\\n Meng, Chenlin; He, Yutong; Song, Yang; Song, Jiaming; Wu, Jiajun; Zhu, Jun-Yan; Ermon, Stefano (August 2, 2021). \"SDEdit: Guided Image Synthesis and Editing with Stochastic Differential Equations\". arXiv:2108.01073 [cs.CV].\\n \"Stable Diffusion web UI\". GitHub. 10 November 2022.\\n invisible-watermark, Shield Mountain, 2022-11-02, retrieved 2022-11-02\\n \"stable-diffusion-tools/emphasis at master · JohannesGaessler/stable-diffusion-tools\". GitHub. Retrieved 2022-11-02.\\n Luzi, Lorenzo; Siahkoohi, Ali; Mayer, Paul M.; Casco-Rodriguez, Josue; Baraniuk, Richard (2022-10-21). \"Boomerang: Local sampling on image manifolds using diffusion models\". arXiv:2210.12100 [cs.CV].\\n Bühlmann, Matthias (2022-09-28). \"Stable Diffusion Based Image Compression\". Medium. Retrieved 2022-11-02.\\n Zhang, Lvmin (10 February 2023). \"Adding Conditional Control to Text-to-Image Diffusion Models\". Retrieved 10 February 2023.\\n Cai, Kenrick. \"Startup Behind AI Image Generator Stable Diffusion Is In Talks To Raise At A Valuation Up To $1 Billion\". Forbes. Retrieved 2022-10-31.\\n \"高性能画像生成AI「Stable Diffusion」無料リリース。「kawaii」までも理解し創造する画像生成AI\". Automaton Media (in Japanese). August 24, 2022.\\n Ryo Shimizu (August 26, 2022). \"Midjourneyを超えた？ 無料の作画AI｢ #StableDiffusion ｣が｢AIを民主化した｣と断言できる理由\". Business Insider Japan (in Japanese).\\n James Vincent \"AI art tools Stable Diffusion and Midjourney targeted with copyright lawsuit\" The Verge, 16 January, 2023.\\n \"Stable Diffusion Public Release\". Stability.Ai. Archived from the original on 2022-08-30. Retrieved 2022-08-31.\\n \"From RAIL to Open RAIL: Topologies of RAIL Licenses\". Responsible AI Licenses (RAIL). Retrieved 2023-02-20.\\n \"Ready or not, mass video deepfakes are coming\". The Washington Post. 2022-08-30. Archived from the original on 2022-08-31. Retrieved 2022-08-31.\\n \"License - a Hugging Face Space by CompVis\". huggingface.co. Archived from the original on 2022-09-04. Retrieved 2022-09-05.\\n Katsuo Ishida (August 26, 2022). \"言葉で指示した画像を凄いAIが描き出す「Stable Diffusion」 ～画像は商用利用も可能\". Impress Corporation (in Japanese).\\nExternal links\\n\\nWikimedia Commons has media related to Stable Diffusion.\\nStable Diffusion Demo\\nInteractive Explanation of Stable Diffusion', metadata={'source': 'sd_wiki.txt'}),\n",
              " Document(page_content='Midjourney\\n\\nArticle\\nTalk\\nRead\\nEdit\\nView history\\n\\nTools\\nFrom Wikipedia, the free encyclopedia\\nMidjourney\\nMidjourney Emblem.png\\nLogo\\nRupert Breheny mechanical dove eca144e7-476d-4976-821d-a49c408e4f36.png\\nA \"mechanical dove\" created with the latest iteration of Midjourney\\'s algorithm.\\nDeveloper(s)\\tMidjourney, Inc.\\nInitial release\\tJuly 12, 2022; 10 months ago (open beta)\\nWebsite\\tmidjourney.com\\nPart of a series on\\nArtificial intelligence\\nMajor goals\\nApproaches\\nPhilosophy\\nHistory\\nTechnology\\nGlossary\\nvte\\nMidjourney is a generative artificial intelligence program and service created and hosted by a San Francisco-based independent research lab Midjourney, Inc. Midjourney generates images from natural language descriptions, called \"prompts\", similar to OpenAI\\'s DALL-E and Stable Diffusion.[1][2]\\n\\nThe tool is currently in open beta, which it entered on July 12, 2022.[3] The Midjourney team is led by David Holz, who co-founded Leap Motion.[4] Holz told The Register in August 2022 that the company was already profitable.[5] Users create artwork with Midjourney using Discord bot commands.[6]\\n\\nHistory\\nMidjourney, Inc. was founded in San Francisco, California by David Holz,[7] previously co-founder of Leap Motion.[8] The Midjourney image generation platform first entered open beta on July 12, 2022.[3] However, on March 14, 2022, the Discord server launched with a request to post high-quality photographs to Twitter/Reddit for system\\'s training.\\n\\nModel versions\\nThe company has been working on improving its algorithms, releasing new model versions every few months. Version 2 of their algorithm was launched in April 2022[9] and version 3 on July 25.[10] On November 5, 2022, the alpha iteration of version 4 was released to users[11][12] and on March 15, 2023, the alpha iteration of version 5 was released.[13] The 5.1 model is more \\'opinionated\\' than version 5, applying more of its own stylization to images, while the 5.1 RAW model adds improvement while working better with more literal prompts.\\n\\nRegular models\\nVersion\\tRelease date\\nV1\\tFebruary 2022[14]\\nV2\\tApril 12, 2022[9]\\nV3\\tJuly 25, 2022[10]\\nV4\\tNovember 5, 2022 (alpha)[11]\\nV5\\tMarch 15, 2023 (alpha)[13]\\nV5.1\\tMay 3, 2023[15]\\nOther models\\nVersion\\tRelease date\\tNotes\\n--beta\\tAugust 22, 2022\\t\\ntest/testp\\tAugust 28, 2022\\t\\nNiji\\tDecember 20, 2022\\tCollaboration between Midjourney and Spellbrush\\ntuned to produce anime and illustrative styles\\nNiji 5\\tApril 2, 2023\\nFunctionality\\nMidjourney is currently only accessible through a Discord bot on their official Discord server, by direct messaging the bot, or by inviting the bot to a third party server. To generate images, users use the /imagine command and type in a prompt; the bot then returns a set of four images. Users may then choose which images they want to upscale. Midjourney is also working on a web interface.[16]\\n\\nUses\\nFounder David Holz says he sees artists as customers, not competitors of Midjourney. Holz told The Register that artists use Midjourney for rapid prototyping of artistic concepts to show to clients before starting work themselves.[5] Some artists have accused Midjourney of devaluing original creative work by using it in the training set;[17] Midjourney\\'s terms of service includes a DMCA takedown policy, allowing artists to request their work to be removed from the set if they believe copyright infringement to be evident.[18]\\n\\nThe advertising industry has been quick to embrace AI tools such as Midjourney, DALL-E, and Stable Diffusion, among others. The tools, which enable advertisers to create original content and brainstorm ideas quickly are providing new opportunities such as \"custom ads created for individuals, a new way to create special effects, or even making e-commerce advertising more efficient\", according to Ad Age.[19]\\n\\nNotable usage and controversy\\nExternal image\\nimage icon Fake images involving personalities and created with Midjourney become viral (March 2023) : President Macron collecting rubbish, Donald Trump stopped by police officers, the Pope dressed in a white down jacket....[20]\\nThe program was used by the British magazine The Economist to create the front cover for an issue in June 2022.[21][22] In Italy, the leading newspaper Corriere della Sera published a comic created with Midjourney by writer Vanni Santoni in August 2022.[23] Charlie Warzel used Midjourney to generate two images of Alex Jones for Warzel\\'s newsletter in The Atlantic. The use of an AI-generated cover was criticised by people who felt it was taking jobs from artists. Warzel called his action a \"mistake\" in an article about his decision to use generated images.[24] Last Week Tonight with John Oliver included a 10-minute segment on Midjourney in an episode broadcast in August 2022.[25][26]\\n\\nA Midjourney image called Théâtre d\\'Opéra Spatial won first place in the digital art competition at the 2022 Colorado State Fair. Jason Allen, who wrote the prompt that led Midjourney to generate the image, printed the image onto a canvas and entered it into the competition using the name \"Jason M. Allen via Midjourney\". Other digital artists were upset by the news.[17] Allen was unapologetic, insisting that he followed the competition\\'s rules. The two category judges were unaware that Midjourney used AI to generate images, although they later said that had they known this, they would have awarded Allen the top prize anyway.[27]\\n\\nIn December 2022, Midjourney was used to create the images in an AI-generated children\\'s book in the span of a weekend. Titled Alice and Sparkle, the book features a young girl who builds a robot that becomes self-aware. The creator, Ammaar Reeshi, spent hours tweaking Midjourney prompts, rejecting hundreds of generated results to ultimately choose 13 illustrations for the book.[28] Both the product and process drew criticism: “the main problem... is that it was trained off of artists’ work. It’s our creations, our distinct styles that we created, that we did not consent to being used,\" one artist wrote.[29]\\n\\nXi Jinping\\nFurther information: Chinese censorship abroad\\nIn March 2023, the CEO David Holz stated that Midjourney has blocked the generation of images of Xi Jinping, the general secretary of the Chinese Communist Party and president of the People\\'s Republic of China, in order to prevent the Chinese government from potentially censoring Midjourney completely over satirical images of Xi. Holz said that \"the ability for people in China to use this tech is more important than your ability to generate satire.\"[30][31]\\n\\nLitigation\\nOn January 13, 2023, three artists – Sarah Andersen, Kelly McKernan, and Karla Ortiz – filed a copyright infringement lawsuit against Stability AI, Midjourney, and DeviantArt, claiming that these companies have infringed the rights of millions of artists, by training AI tools on five billion images scraped from the web, without the consent of the original artists.[32]\\n\\nSubscription service\\nMidjourney has three subscription tiers,[33] and previously offered a free trial, but withdrew this service citing high demand and trial abuse.\\n\\nWhen the free trial was available, generating an image activated it. Trial users could make roughly 25 jobs before needing to subscribe to continue using it.[34]\\n\\nSee also\\n\\nWikimedia Commons has media related to Images generated by Midjourney.\\nReferences\\n \"Huge \"foundation models\" are turbo-charging AI progress\". The Economist. Retrieved 26 June 2022.\\n Hertzmann, Aaron. \"Give this AI a few words of description and it produces a stunning image – but is it art?\". The Conversation. Retrieved 26 June 2022.\\n @midjourney (July 12, 2022). \"We\\'re officially moving to open-beta! Join now at discord.gg/midjourney. **Please read our directions carefully** or check out our detailed how-to guides here: midjourney.gitbook.io/docs. Most importantly, have fun!\" (Tweet). Retrieved August 31, 2022 – via Twitter.\\n Rose, Janus (July 18, 2022). \"Inside Midjourney, The Generative Art AI That Rivals DALL-E\". Vice.\\n Claburn, Thomas (August 1, 2022). \"Holz, founder of AI art service Midjourney, on future images\". The Register. Retrieved September 5, 2022.\\n Hachman, Mark (July 26, 2022). \"Midjourney\\'s enthralling AI art generator goes live for everyone\". PCWorld. Retrieved September 5, 2022.\\n Salkowitz, Rob (2022-09-16). \"Midjourney Founder David Holz On The Impact Of AI On Art, Imagination And The Creative Economy\". Forbes. Retrieved 2023-03-19.\\n Vincent, James (2022-08-02). \"\"An engine for the imagination\": an interview with David Holz, CEO of AI image generator Midjourney\". The Verge. Retrieved 2023-03-19.\\n @midjourney (2022-04-18). \"We recently started testing a V2 algorithm, it\\'s much better with characters and animals\" (Tweet). Retrieved 2023-03-19 – via Twitter.\\n @midjourney (2022-07-25). \"Today we\\'re starting to test our V3 image generation algorithms\" (Tweet). Retrieved 2023-03-19 – via Twitter.\\n \"David Holz on the official Midjourney Discord server\". Discord. 2022-11-05. Retrieved 2023-03-19.\\n \"Midjourney v4 greatly improves the award-winning image creation AI\". TechSpot. Retrieved 2022-11-17.\\n \"Midjourney V5 Creates Better Images, Fewer Nightmare Hands\". HowToGeek. Retrieved 2023-03-16.\\n \"Midjourney Model Versions\". docs.midjourney.com. Retrieved 2023-04-07.\\n Collins, Barry (2023-05-03). \"Midjourney 5.1 Arrives - And It\\'s Another Leap Forward For AI Art\". Forbes. Retrieved 2023-05-05.\\n Steen, Charles (January 4, 2023). \"\"Midjourney Is Allegedly Working on a Web Interface\"\". Easy With AI.\\n Gault, Matthew (August 31, 2022). \"An AI-Generated Artwork Won First Place at a State Fair Fine Arts Competition, and Artists Are Pissed\". Motherboard. Vice. Retrieved August 31, 2022.\\n \"Terms of Service\". Midjourney. Retrieved 2023-03-19.\\n Bonilla, Brian (2022-09-22). \"How ad agencies are using AI image generators—and how they could be used in the future\". Ad Age. Retrieved 2023-03-19.\\n Photos créées par des IA : une bascule vertigineuse et dangereuse, Jonathan Bouchet-Petersen, 31 March 2023, Libération.\\n \"How a computer designed this week\\'s cover\". The Economist. Retrieved 26 June 2022.\\n Liu, Gloria (21 June 2022). \"DALL-E 2 Made Its First Magazine Cover\". Cosmopolitan. Retrieved 26 June 2022.\\n Bozzi, Ida (2022-08-26). \"Su \"La Lettura\", Highsmith inedita e le città che mutano\". Corriere della Sera (in Italian). Retrieved 2023-03-19.\\n \"I Went Viral in the Bad Way\". Galaxy Brain. 2022-08-17. Retrieved 2022-08-31.\\n SFGATE, Dan Gentile (August 16, 2022). \"John Oliver is weirdly popular on this SF-based AI image app\". SFGATE. Retrieved August 31, 2022.\\n Brathwaite, Lester Fabian (August 29, 2022). \"John Oliver marries a cabbage in ceremony officiated by Steve Buscemi on \\'Last Week Tonight\\'\". EW.com. Retrieved August 31, 2022.\\n Roose, Kevin (September 2, 2022). \"An A.I.-Generated Picture Won an Art Prize. Artists Aren\\'t Happy\". The New York Times. Retrieved September 2, 2022.\\n Stokel-Walker, Chris (13 December 2022). \"A Tech Worker Is Selling A Children\\'s Book He Made Using AI. Professional Illustrators Are Pissed\". BuzzFeed News. Retrieved 19 December 2022.\\n Popli, Nic (14 December 2022). \"He Used AI to Publish a Children\\'s Book in a Weekend. Artists Are Not Happy About It\". Time. Retrieved 19 December 2022.\\n \"How a tiny company with few rules is making fake images go mainstream\". The Washington Post. ISSN 0190-8286. Retrieved 2023-04-05.\\n McFadden, Christopher (2023-04-03). \"Midjourney will no longer let you generate images of Xi Jinping\". interestingengineering.com. Retrieved 2023-04-05.\\n Vincent, James (2023-01-16). \"AI art tools Stable Diffusion and Midjourney targeted with copyright lawsuit\". The Verge.\\n \"Midjourney Subscription Plans\". docs.midjourney.com. Retrieved 2023-03-19.\\n \"Midjourney Quick Start Guide\". docs.midjourney.com. Retrieved 2023-03-19.\\nExternal links\\n\\nWikimedia Commons has media related to Midjourney.\\nOfficial website\\nOfficial Twitter\\nOfficial Discord server', metadata={'source': 'midjourney_wiki.txt'})]"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#split data\n",
        "from langchain.text_splitter import CharacterTextSplitter\n",
        "text_splitter = CharacterTextSplitter(chunk_size=4000, chunk_overlap=0) #default 4000\n",
        "texts = text_splitter.split_documents(documents)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gzQN4F1-jory",
        "outputId": "5c0c4556-33f0-48ee-c4b9-e6c5a534f0fd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:langchain.text_splitter:Created a chunk of size 6746, which is longer than the specified 4000\n",
            "WARNING:langchain.text_splitter:Created a chunk of size 4555, which is longer than the specified 4000\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(texts)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "E1t1kgpJj_dF",
        "outputId": "dd7bba20-642d-45c1-c1bb-94033b5c0ceb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "12"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(texts[1].page_content)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CMvupC5nkDuQ",
        "outputId": "3bed4849-507c-44a0-9a0d-cbd96e11552d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3661"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Print the first chunk's page content\n",
        "texts[0].page_content"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 161
        },
        "id": "fTW2ygLCkNnF",
        "outputId": "44779072-93b0-4fa4-bb97-5c06eae9c5d8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Stable Diffusion\\n\\nArticle\\nTalk\\nRead\\nEdit\\nView history\\n\\nTools\\nFrom Wikipedia, the free encyclopedia\\nStable Diffusion\\nA photograph of an astronaut riding a horse 2022-08-28.png\\nAn image generated by Stable Diffusion based on the text prompt \"a photograph of an astronaut riding a horse\"\\nOriginal author(s)\\tRunway, CompVis, and Stability AI\\nDeveloper(s)\\tStability AI\\nInitial release\\tAugust 22, 2022\\nStable release\\t\\n2.1 (model)[1] / December 7, 2022\\nRepository\\tgithub.com/Stability-AI/stablediffusion\\nWritten in\\tPython[2]\\nOperating system\\tAny that support CUDA kernels\\nType\\tText-to-image model\\nLicense\\tCreative ML OpenRAIL-M\\nWebsite\\tommer-lab.com/research/latent-diffusion-models/ \\nStable Diffusion is a deep learning, text-to-image model released in 2022. It is primarily used to generate detailed images conditioned on text descriptions, though it can also be applied to other tasks such as inpainting, outpainting, and generating image-to-image translations guided by a text prompt.[3] It was developed by the start-up Stability AI in collaboration with a number of academic researchers and non-profit organizations.[4]\\n\\nStable Diffusion is a latent diffusion model, a kind of deep generative neural network. Its code and model weights have been released publicly,[5] and it can run on most consumer hardware equipped with a modest GPU with at least 8 GB VRAM. This marked a departure from previous proprietary text-to-image models such as DALL-E and Midjourney which were accessible only via cloud services.[6][7]\\n\\nDevelopment\\nThe development of Stable Diffusion was funded and shaped by the start-up company Stability AI.[7][8][9] The technical license for the model was released by the CompVis group at Ludwig Maximilian University of Munich.[7] Development was led by Patrick Esser of Runway and Robin Rombach of CompVis, who were among the researchers who had earlier invented the latent diffusion model architecture used by Stable Diffusion.[4] Stability AI also credited EleutherAI and LAION (a German nonprofit which assembled the dataset on which Stable Diffusion was trained) as supporters of the project.[4]\\n\\nIn October 2022, Stability AI raised US$101 million in a round led by Lightspeed Venture Partners and Coatue Management.[10]\\n\\nTechnology\\n\\nDiagram of the latent diffusion architecture used by Stable Diffusion'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Step 2 create text embeddings, save into a vectorstore (database / index)"
      ],
      "metadata": {
        "id": "fZh6uECjlqHC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#create text embedding & index import os\n",
        "from langchain.embeddings import OpenAIEmbeddings\n",
        "from langchain.vectorstores import Chroma\n",
        "from dotenv import dotenv_values\n",
        "import os\n",
        "\n",
        "os.environ['OPENAI_API_KEY'] = dotenv_values() ['openai_api_key'] #set environment variable\n",
        "embeddings = OpenAIEmbeddings ()\n",
        "db = Chroma. from_documents (texts, embeddings)\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jtV_L6B4los9",
        "outputId": "8d928de3-3f43-46d4-c49f-fc87caf414c9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:chromadb:Using embedded DuckDB without persistence: data will be transient\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Step 3 creater a retrieve from the db, create chain & ask questions"
      ],
      "metadata": {
        "id": "NhQC3EGPly9K"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.chains import RetrievalQA\n",
        "from langchain.llms import OpenAI\n",
        "retriever = db.as_retriever()\n",
        "qa = RetrievalQA.from_chain_type(llm=OpenAI(model_name='gpt-3.5-turbo'),\n",
        "                                  chain_type=\"stuff\",\n",
        "                                  retriever=retriever)\n",
        "#chain type\n",
        "# stuff\n",
        "# map_reduce\n",
        "# refine\n",
        "# map-rerankI"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Nl16CmLvlzKt",
        "outputId": "b8fd3f19-ab28-49f5-bcbe-d5fad815af1a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/langchain/llms/openai.py:169: UserWarning: You are trying to use a chat model. This way of initializing it is no longer supported. Instead, please use: `from langchain.chat_models import ChatOpenAI`\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/langchain/llms/openai.py:695: UserWarning: You are trying to use a chat model. This way of initializing it is no longer supported. Instead, please use: `from langchain.chat_models import ChatOpenAI`\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "qa.run('I want to use midjourney, how do i use it?')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89
        },
        "id": "euLxv42nngv9",
        "outputId": "463ea918-82a8-42f9-c983-fd17422e9caa"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Midjourney is currently accessible through a Discord bot on their official Discord server. Users can generate images by using the /imagine command and typing in a prompt. The bot will then return a set of four images and users can choose which images they want to upscale. Midjourney is also working on a web interface, but currently, it is only accessible through the Discord bot. It is important to note that Midjourney is currently in open beta and has three subscription tiers.'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 65
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Company Policy Test"
      ],
      "metadata": {
        "id": "3H0MbZeis40e"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.document_loaders import TextLoader\n",
        "loader1 = TextLoader('./company_policies/Alpha.txt')\n",
        "loader2 = TextLoader('./company_policies/Beta.txt')\n",
        "loader3 = TextLoader('./company_policies/Gamma.txt')\n",
        "documents = loader1.load()\n",
        "documents += loader2.load() \n",
        "documents += loader3.load() \n",
        "# put document objects inside a list\n",
        "\n",
        "#split data\n",
        "from langchain.text_splitter import CharacterTextSplitter\n",
        "text_splitter = CharacterTextSplitter(chunk_size=4000, chunk_overlap=0) #default 4000\n",
        "texts = text_splitter.split_documents(documents)\n",
        "\n",
        "#create text embedding & index import os\n",
        "from langchain.embeddings import OpenAIEmbeddings\n",
        "from langchain.vectorstores import Chroma\n",
        "from dotenv import dotenv_values\n",
        "import os\n",
        "\n",
        "os.environ['OPENAI_API_KEY'] = dotenv_values() ['openai_api_key'] #set environment variable\n",
        "embeddings = OpenAIEmbeddings ()\n",
        "db = Chroma. from_documents (texts, embeddings)\n",
        "\n",
        "from langchain.chains import RetrievalQA\n",
        "from langchain.llms import OpenAI\n",
        "retriever = db.as_retriever()\n",
        "qa = RetrievalQA.from_chain_type(llm=OpenAI(model_name='gpt-3.5-turbo'),\n",
        "                                  chain_type=\"stuff\",\n",
        "                                  retriever=retriever)\n",
        "#chain type\n",
        "# stuff\n",
        "# map_reduce\n",
        "# refine\n",
        "# map-rerankI\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DK_M3J0Ss_Es",
        "outputId": "a8a29a8c-66a0-4dad-e55b-55fa94c55f00"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:chromadb:Using embedded DuckDB without persistence: data will be transient\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "qa.run('What is the Car policies for an employee of company Gamma?')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        },
        "id": "VX1mvGRfu4eP",
        "outputId": "2298b98a-af41-47dd-b26f-c182aebfb7fe"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "ERROR:root:Chroma collection langchain contains fewer than 4 elements.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'The Car policy for an employee of company Gamma is that the use of personal vehicles for business travel will be reimbursed at a rate lower than the current IRS mileage rate.'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "qa.run('Compare air policie for companies Alpha & Beta')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        },
        "id": "M7B1NJRRvS8h",
        "outputId": "dd0eb700-7dde-4b3f-b923-f38a8d526e69"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "ERROR:root:Chroma collection langchain contains fewer than 4 elements.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Company Alpha allows employees to travel in business class for international flights and economy class for domestic flights, while Company Beta allows employees to travel in business class for both international and domestic flights.'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "qa.run('Compare air policie for companies Alpha & Beta - Return comparison in a table format')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        },
        "id": "441yojDgvgWb",
        "outputId": "2e75a21c-3615-4842-d609-184abcc52c72"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "ERROR:root:Chroma collection langchain contains fewer than 4 elements.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'| Company | International Flights | Domestic Flights |\\n| --- | --- | --- |\\n| Alpha | Business Class | Economy Class |\\n| Beta | Business Class | Business Class for employees |'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Asking for Non-Data\n",
        "qa.run('Compare Space Booking policies for all companies - Return comparison in a table format')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        },
        "id": "7_JPNhtzxTji",
        "outputId": "c656aa9f-edb7-452c-e6c9-4c52d99a3cba"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "ERROR:root:Chroma collection langchain contains fewer than 4 elements.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Unfortunately, there is no information provided about the space booking policies for any of the companies mentioned, so a comparison in a table format cannot be made.'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "qa.run('Act as a travel adviser for an Alpha company traveler. Create a booking template or guidlines that will compily with the company policy')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 178
        },
        "id": "DqHAez0cxilq",
        "outputId": "a74ae78b-2105-41e3-e1bb-356373c17e19"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "ERROR:root:Chroma collection langchain contains fewer than 4 elements.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'As a travel adviser for an Alpha company traveler, here are some guidelines to follow when booking travel:\\n\\n1. Enroll in frequent traveler programs to earn benefits for personal travel, but remember that personal travel expenses will not be reimbursed by the company.\\n2. Book your flights at least two weeks in advance, and try to use the lowest logical airfare. Non-refundable fares are only acceptable if they are cheaper than the lowest available refundable fare.\\n3. For international flights, you are eligible to travel in business class, while for domestic flights, only economy class is allowed.\\n4. If you have a colleague from the same department who needs to travel, ensure that no more than two of you are booked on the same flight.\\n5. Avoid late arrival guarantees when booking hotels, and ensure that the rates do not exceed the maximum limit set by the company.\\n6. For short-distance travel, it is encouraged to use rail transportation. However, if you need to rent a car for business travel, the expenses will be covered by the company.\\n7. Remember to keep all receipts and submit them to the HR department for reimbursement.\\n\\nFollowing these guidelines will ensure that your travel plans comply with the Company Alpha policy.'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "qa.run('Provide 2 booking requests for an Alpha Travelers - 1 that fully complies with the company policy, and a second booking request that does not comply. Return results in bullets')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 160
        },
        "id": "h-64s9cIx-jf",
        "outputId": "e3beae37-1217-4c08-dae5-d11dae424fb8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "ERROR:root:Chroma collection langchain contains fewer than 4 elements.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\"Booking Request 1 (Compliant with Company Alpha Policy)\\n- Economy class flight booked at least two weeks in advance using the company's travel agency\\n- Non-refundable fare chosen only because it is cheaper than the lowest refundable fare\\n- Soft dollars balanced across different airlines\\n- Hotel booked through the company's travel agency within the maximum limit set by the company\\n\\nBooking Request 2 (Non-Compliant with Company Alpha Policy)\\n- Business class flight booked for a domestic flight\\n- Flight booked less than two weeks in advance\\n- Non-direct routing chosen even though it does not result in significant cost savings\\n- Personal travel expenses, such as a frequent traveler membership, added onto the booking and requested to be reimbursed by the company\""
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "ZsKRFOqKxdgA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# !zip -r /content/LangChain-Train_ChatGPT_With_Your_Own_Data.zip /content/\n",
        "# #How to Download Files and Folders from Colab 2\n",
        "# #The command is !zip followed by r which means “recursive”, then we write the file path of the zipped file (i.e. /content/sample_data.zip) and finally, we write the folder that we want to zip (i.e. /content/sample_data) and voila, the zip file is generated :-).\n",
        "# #Lastly, we can download the zip file as before:\n",
        "# files.download('/content/sample_data.zip')\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 919
        },
        "id": "D2rXeF5kwTc6",
        "outputId": "96d97a08-695f-4076-ee38-738ae6230016"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "updating: content/company_policies/ (stored 0%)\n",
            "updating: content/company_policies/Gamma.txt (deflated 47%)\n",
            "updating: content/company_policies/Alpha.txt (deflated 57%)\n",
            "updating: content/company_policies/Beta.txt (deflated 47%)\n",
            "updating: content/company_policies/.ipynb_checkpoints/ (stored 0%)\n",
            "updating: content/company_policies/untitled (stored 0%)\n",
            "updating: content/midjourney_wiki.txt (deflated 56%)\n",
            "updating: content/sample_data/ (stored 0%)\n",
            "updating: content/sample_data/anscombe.json (deflated 83%)\n",
            "updating: content/sample_data/README.md (deflated 42%)\n",
            "updating: content/sample_data/california_housing_test.csv (deflated 76%)\n",
            "updating: content/sample_data/mnist_test.csv (deflated 88%)\n",
            "updating: content/sample_data/mnist_train_small.csv (deflated 88%)\n",
            "updating: content/sample_data/california_housing_train.csv (deflated 79%)\n",
            "updating: content/sample_data.zip (stored 0%)\n",
            "updating: content/sd_wiki.txt (deflated 60%)\n",
            "  adding: content/ (stored 0%)\n",
            "  adding: content/.config/ (stored 0%)\n",
            "  adding: content/.config/logs/ (stored 0%)\n",
            "  adding: content/.config/logs/2023.05.18/ (stored 0%)\n",
            "  adding: content/.config/logs/2023.05.18/13.37.21.235628.log (deflated 86%)\n",
            "  adding: content/.config/logs/2023.05.18/13.37.54.305155.log (deflated 57%)\n",
            "  adding: content/.config/logs/2023.05.18/13.37.54.986582.log (deflated 56%)\n",
            "  adding: content/.config/logs/2023.05.18/13.37.28.914432.log (deflated 58%)\n",
            "  adding: content/.config/logs/2023.05.18/13.36.31.856869.log (deflated 91%)\n",
            "  adding: content/.config/logs/2023.05.18/13.36.56.564357.log (deflated 58%)\n",
            "  adding: content/.config/.last_opt_in_prompt.yaml (stored 0%)\n",
            "  adding: content/.config/configurations/ (stored 0%)\n",
            "  adding: content/.config/configurations/config_default (deflated 15%)\n",
            "  adding: content/.config/.last_update_check.json (deflated 22%)\n",
            "  adding: content/.config/active_config (stored 0%)\n",
            "  adding: content/.config/gce (stored 0%)\n",
            "  adding: content/.config/config_sentinel (stored 0%)\n",
            "  adding: content/.config/.last_survey_prompt.yaml (stored 0%)\n",
            "  adding: content/.chroma/ (stored 0%)\n",
            "  adding: content/.chroma/index/ (stored 0%)\n",
            "  adding: content/.ipynb_checkpoints/ (stored 0%)\n",
            "  adding: content/.env (deflated 4%)\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-84-a76b347e37c6>\u001b[0m in \u001b[0;36m<cell line: 5>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;31m#The command is !zip followed by r which means “recursive”, then we write the file path of the zipped file (i.e. /content/sample_data.zip) and finally, we write the folder that we want to zip (i.e. /content/sample_data) and voila, the zip file is generated :-).\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;31m#Lastly, we can download the zip file as before:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mfiles\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdownload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'/content/sample_data.zip'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m: name 'files' is not defined"
          ]
        }
      ]
    }
  ]
}