search engine.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "0a9cfbf8-e62b-4798-b443-332645cc835f",
   "metadata": {},
   "source": [
    "https://www.youtube.com/watch?v=nMrGK5QgPVE&t=1981s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "edbc7d25-cff6-4e36-b053-35f5a1547184",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests \n",
    "\n",
    "docs_url = 'https://github.com/alexeygrigorev/llm-rag-workshop/raw/main/notebooks/documents.json'\n",
    "docs_response = requests.get(docs_url)\n",
    "documents_raw = docs_response.json()\n",
    "\n",
    "documents = []\n",
    "\n",
    "for course in documents_raw:\n",
    "    course_name = course['course']\n",
    "\n",
    "    for doc in course['documents']:\n",
    "        doc['course'] = course_name\n",
    "        documents.append(doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "aa6d7c72-1723-496d-99bb-15299f57072a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'text': \"Yes, even if you don't register, you're still eligible to submit the homeworks.\\nBe aware, however, that there will be deadlines for turning in the final projects. So don't leave everything for the last minute.\",\n",
       " 'section': 'General course-related questions',\n",
       " 'question': 'Course - Can I still join the course after the start date?',\n",
       " 'course': 'data-engineering-zoomcamp'}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "documents[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "76762ac7-09d8-44a4-994c-4cda92c044cc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>course</th>\n",
       "      <th>section</th>\n",
       "      <th>question</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>data-engineering-zoomcamp</td>\n",
       "      <td>General course-related questions</td>\n",
       "      <td>Course - When will the course start?</td>\n",
       "      <td>The purpose of this document is to capture fre...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>data-engineering-zoomcamp</td>\n",
       "      <td>General course-related questions</td>\n",
       "      <td>Course - What are the prerequisites for this c...</td>\n",
       "      <td>GitHub - DataTalksClub data-engineering-zoomca...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>data-engineering-zoomcamp</td>\n",
       "      <td>General course-related questions</td>\n",
       "      <td>Course - Can I still join the course after the...</td>\n",
       "      <td>Yes, even if you don't register, you're still ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>data-engineering-zoomcamp</td>\n",
       "      <td>General course-related questions</td>\n",
       "      <td>Course - I have registered for the Data Engine...</td>\n",
       "      <td>You don't need it. You're accepted. You can al...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>data-engineering-zoomcamp</td>\n",
       "      <td>General course-related questions</td>\n",
       "      <td>Course - What can I do before the course starts?</td>\n",
       "      <td>You can start by installing and setting up all...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      course                           section  \\\n",
       "0  data-engineering-zoomcamp  General course-related questions   \n",
       "1  data-engineering-zoomcamp  General course-related questions   \n",
       "2  data-engineering-zoomcamp  General course-related questions   \n",
       "3  data-engineering-zoomcamp  General course-related questions   \n",
       "4  data-engineering-zoomcamp  General course-related questions   \n",
       "\n",
       "                                            question  \\\n",
       "0               Course - When will the course start?   \n",
       "1  Course - What are the prerequisites for this c...   \n",
       "2  Course - Can I still join the course after the...   \n",
       "3  Course - I have registered for the Data Engine...   \n",
       "4   Course - What can I do before the course starts?   \n",
       "\n",
       "                                                text  \n",
       "0  The purpose of this document is to capture fre...  \n",
       "1  GitHub - DataTalksClub data-engineering-zoomca...  \n",
       "2  Yes, even if you don't register, you're still ...  \n",
       "3  You don't need it. You're accepted. You can al...  \n",
       "4  You can start by installing and setting up all...  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.DataFrame(documents, columns=['course', 'section', 'question', 'text'])\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "828828a6-075e-4c45-8b37-29cd2c3295a0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>course</th>\n",
       "      <th>section</th>\n",
       "      <th>question</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>943</th>\n",
       "      <td>mlops-zoomcamp</td>\n",
       "      <td>Module 6: Best practices</td>\n",
       "      <td>Github actions: Permission denied error when e...</td>\n",
       "      <td>Problem description\\nThis is the step in the c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>944</th>\n",
       "      <td>mlops-zoomcamp</td>\n",
       "      <td>Module 6: Best practices</td>\n",
       "      <td>Managing Multiple Docker Containers with docke...</td>\n",
       "      <td>Problem description\\nWhen a docker-compose fil...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>945</th>\n",
       "      <td>mlops-zoomcamp</td>\n",
       "      <td>Module 6: Best practices</td>\n",
       "      <td>AWS regions need to match docker-compose</td>\n",
       "      <td>Problem description\\nIf you are having problem...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>946</th>\n",
       "      <td>mlops-zoomcamp</td>\n",
       "      <td>Module 6: Best practices</td>\n",
       "      <td>Isort Pre-commit</td>\n",
       "      <td>Problem description\\nPre-commit command was fa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>947</th>\n",
       "      <td>mlops-zoomcamp</td>\n",
       "      <td>Module 6: Best practices</td>\n",
       "      <td>How to destroy infrastructure created via GitH...</td>\n",
       "      <td>Problem description\\nInfrastructure created in...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             course                   section  \\\n",
       "943  mlops-zoomcamp  Module 6: Best practices   \n",
       "944  mlops-zoomcamp  Module 6: Best practices   \n",
       "945  mlops-zoomcamp  Module 6: Best practices   \n",
       "946  mlops-zoomcamp  Module 6: Best practices   \n",
       "947  mlops-zoomcamp  Module 6: Best practices   \n",
       "\n",
       "                                              question  \\\n",
       "943  Github actions: Permission denied error when e...   \n",
       "944  Managing Multiple Docker Containers with docke...   \n",
       "945           AWS regions need to match docker-compose   \n",
       "946                                   Isort Pre-commit   \n",
       "947  How to destroy infrastructure created via GitH...   \n",
       "\n",
       "                                                  text  \n",
       "943  Problem description\\nThis is the step in the c...  \n",
       "944  Problem description\\nWhen a docker-compose fil...  \n",
       "945  Problem description\\nIf you are having problem...  \n",
       "946  Problem description\\nPre-commit command was fa...  \n",
       "947  Problem description\\nInfrastructure created in...  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8db460b7-1065-4ccd-b572-657415367ae8",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>course</th>\n",
       "      <th>section</th>\n",
       "      <th>question</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>data-engineering-zoomcamp</td>\n",
       "      <td>General course-related questions</td>\n",
       "      <td>Course - When will the course start?</td>\n",
       "      <td>The purpose of this document is to capture fre...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>data-engineering-zoomcamp</td>\n",
       "      <td>General course-related questions</td>\n",
       "      <td>Course - What are the prerequisites for this c...</td>\n",
       "      <td>GitHub - DataTalksClub data-engineering-zoomca...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>data-engineering-zoomcamp</td>\n",
       "      <td>General course-related questions</td>\n",
       "      <td>Course - Can I still join the course after the...</td>\n",
       "      <td>Yes, even if you don't register, you're still ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>data-engineering-zoomcamp</td>\n",
       "      <td>General course-related questions</td>\n",
       "      <td>Course - I have registered for the Data Engine...</td>\n",
       "      <td>You don't need it. You're accepted. You can al...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>data-engineering-zoomcamp</td>\n",
       "      <td>General course-related questions</td>\n",
       "      <td>Course - What can I do before the course starts?</td>\n",
       "      <td>You can start by installing and setting up all...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      course                           section  \\\n",
       "0  data-engineering-zoomcamp  General course-related questions   \n",
       "1  data-engineering-zoomcamp  General course-related questions   \n",
       "2  data-engineering-zoomcamp  General course-related questions   \n",
       "3  data-engineering-zoomcamp  General course-related questions   \n",
       "4  data-engineering-zoomcamp  General course-related questions   \n",
       "\n",
       "                                            question  \\\n",
       "0               Course - When will the course start?   \n",
       "1  Course - What are the prerequisites for this c...   \n",
       "2  Course - Can I still join the course after the...   \n",
       "3  Course - I have registered for the Data Engine...   \n",
       "4   Course - What can I do before the course starts?   \n",
       "\n",
       "                                                text  \n",
       "0  The purpose of this document is to capture fre...  \n",
       "1  GitHub - DataTalksClub data-engineering-zoomca...  \n",
       "2  Yes, even if you don't register, you're still ...  \n",
       "3  You don't need it. You're accepted. You can al...  \n",
       "4  You can start by installing and setting up all...  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.course == 'data-engineering-zoomcamp'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "49a052fa-1d62-479d-bc8d-19b558a8915b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "7281b4d1-0724-461f-b53e-3be12319dad3",
   "metadata": {},
   "outputs": [],
   "source": [
    "cv = CountVectorizer(min_df=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "2cfaf921-2685-449d-94cf-ddfddf5fc9f9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-2 {\n",
       "  /* Definition of color scheme common for light and dark mode */\n",
       "  --sklearn-color-text: black;\n",
       "  --sklearn-color-line: gray;\n",
       "  /* Definition of color scheme for unfitted estimators */\n",
       "  --sklearn-color-unfitted-level-0: #fff5e6;\n",
       "  --sklearn-color-unfitted-level-1: #f6e4d2;\n",
       "  --sklearn-color-unfitted-level-2: #ffe0b3;\n",
       "  --sklearn-color-unfitted-level-3: chocolate;\n",
       "  /* Definition of color scheme for fitted estimators */\n",
       "  --sklearn-color-fitted-level-0: #f0f8ff;\n",
       "  --sklearn-color-fitted-level-1: #d4ebff;\n",
       "  --sklearn-color-fitted-level-2: #b3dbfd;\n",
       "  --sklearn-color-fitted-level-3: cornflowerblue;\n",
       "\n",
       "  /* Specific color for light theme */\n",
       "  --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
       "  --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, white)));\n",
       "  --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
       "  --sklearn-color-icon: #696969;\n",
       "\n",
       "  @media (prefers-color-scheme: dark) {\n",
       "    /* Redefinition of color scheme for dark theme */\n",
       "    --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
       "    --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, #111)));\n",
       "    --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
       "    --sklearn-color-icon: #878787;\n",
       "  }\n",
       "}\n",
       "\n",
       "#sk-container-id-2 {\n",
       "  color: var(--sklearn-color-text);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 pre {\n",
       "  padding: 0;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 input.sk-hidden--visually {\n",
       "  border: 0;\n",
       "  clip: rect(1px 1px 1px 1px);\n",
       "  clip: rect(1px, 1px, 1px, 1px);\n",
       "  height: 1px;\n",
       "  margin: -1px;\n",
       "  overflow: hidden;\n",
       "  padding: 0;\n",
       "  position: absolute;\n",
       "  width: 1px;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-dashed-wrapped {\n",
       "  border: 1px dashed var(--sklearn-color-line);\n",
       "  margin: 0 0.4em 0.5em 0.4em;\n",
       "  box-sizing: border-box;\n",
       "  padding-bottom: 0.4em;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-container {\n",
       "  /* jupyter's `normalize.less` sets `[hidden] { display: none; }`\n",
       "     but bootstrap.min.css set `[hidden] { display: none !important; }`\n",
       "     so we also need the `!important` here to be able to override the\n",
       "     default hidden behavior on the sphinx rendered scikit-learn.org.\n",
       "     See: https://github.com/scikit-learn/scikit-learn/issues/21755 */\n",
       "  display: inline-block !important;\n",
       "  position: relative;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-text-repr-fallback {\n",
       "  display: none;\n",
       "}\n",
       "\n",
       "div.sk-parallel-item,\n",
       "div.sk-serial,\n",
       "div.sk-item {\n",
       "  /* draw centered vertical line to link estimators */\n",
       "  background-image: linear-gradient(var(--sklearn-color-text-on-default-background), var(--sklearn-color-text-on-default-background));\n",
       "  background-size: 2px 100%;\n",
       "  background-repeat: no-repeat;\n",
       "  background-position: center center;\n",
       "}\n",
       "\n",
       "/* Parallel-specific style estimator block */\n",
       "\n",
       "#sk-container-id-2 div.sk-parallel-item::after {\n",
       "  content: \"\";\n",
       "  width: 100%;\n",
       "  border-bottom: 2px solid var(--sklearn-color-text-on-default-background);\n",
       "  flex-grow: 1;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-parallel {\n",
       "  display: flex;\n",
       "  align-items: stretch;\n",
       "  justify-content: center;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  position: relative;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-parallel-item {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-parallel-item:first-child::after {\n",
       "  align-self: flex-end;\n",
       "  width: 50%;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-parallel-item:last-child::after {\n",
       "  align-self: flex-start;\n",
       "  width: 50%;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-parallel-item:only-child::after {\n",
       "  width: 0;\n",
       "}\n",
       "\n",
       "/* Serial-specific style estimator block */\n",
       "\n",
       "#sk-container-id-2 div.sk-serial {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "  align-items: center;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  padding-right: 1em;\n",
       "  padding-left: 1em;\n",
       "}\n",
       "\n",
       "\n",
       "/* Toggleable style: style used for estimator/Pipeline/ColumnTransformer box that is\n",
       "clickable and can be expanded/collapsed.\n",
       "- Pipeline and ColumnTransformer use this feature and define the default style\n",
       "- Estimators will overwrite some part of the style using the `sk-estimator` class\n",
       "*/\n",
       "\n",
       "/* Pipeline and ColumnTransformer style (default) */\n",
       "\n",
       "#sk-container-id-2 div.sk-toggleable {\n",
       "  /* Default theme specific background. It is overwritten whether we have a\n",
       "  specific estimator or a Pipeline/ColumnTransformer */\n",
       "  background-color: var(--sklearn-color-background);\n",
       "}\n",
       "\n",
       "/* Toggleable label */\n",
       "#sk-container-id-2 label.sk-toggleable__label {\n",
       "  cursor: pointer;\n",
       "  display: block;\n",
       "  width: 100%;\n",
       "  margin-bottom: 0;\n",
       "  padding: 0.5em;\n",
       "  box-sizing: border-box;\n",
       "  text-align: center;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 label.sk-toggleable__label-arrow:before {\n",
       "  /* Arrow on the left of the label */\n",
       "  content: \"▸\";\n",
       "  float: left;\n",
       "  margin-right: 0.25em;\n",
       "  color: var(--sklearn-color-icon);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 label.sk-toggleable__label-arrow:hover:before {\n",
       "  color: var(--sklearn-color-text);\n",
       "}\n",
       "\n",
       "/* Toggleable content - dropdown */\n",
       "\n",
       "#sk-container-id-2 div.sk-toggleable__content {\n",
       "  max-height: 0;\n",
       "  max-width: 0;\n",
       "  overflow: hidden;\n",
       "  text-align: left;\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-toggleable__content.fitted {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-toggleable__content pre {\n",
       "  margin: 0.2em;\n",
       "  border-radius: 0.25em;\n",
       "  color: var(--sklearn-color-text);\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-toggleable__content.fitted pre {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 input.sk-toggleable__control:checked~div.sk-toggleable__content {\n",
       "  /* Expand drop-down */\n",
       "  max-height: 200px;\n",
       "  max-width: 100%;\n",
       "  overflow: auto;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {\n",
       "  content: \"▾\";\n",
       "}\n",
       "\n",
       "/* Pipeline/ColumnTransformer-specific style */\n",
       "\n",
       "#sk-container-id-2 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-label.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Estimator-specific style */\n",
       "\n",
       "/* Colorize estimator box */\n",
       "#sk-container-id-2 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-estimator.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-label label.sk-toggleable__label,\n",
       "#sk-container-id-2 div.sk-label label {\n",
       "  /* The background is the default theme color */\n",
       "  color: var(--sklearn-color-text-on-default-background);\n",
       "}\n",
       "\n",
       "/* On hover, darken the color of the background */\n",
       "#sk-container-id-2 div.sk-label:hover label.sk-toggleable__label {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "/* Label box, darken color on hover, fitted */\n",
       "#sk-container-id-2 div.sk-label.fitted:hover label.sk-toggleable__label.fitted {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Estimator label */\n",
       "\n",
       "#sk-container-id-2 div.sk-label label {\n",
       "  font-family: monospace;\n",
       "  font-weight: bold;\n",
       "  display: inline-block;\n",
       "  line-height: 1.2em;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-label-container {\n",
       "  text-align: center;\n",
       "}\n",
       "\n",
       "/* Estimator-specific */\n",
       "#sk-container-id-2 div.sk-estimator {\n",
       "  font-family: monospace;\n",
       "  border: 1px dotted var(--sklearn-color-border-box);\n",
       "  border-radius: 0.25em;\n",
       "  box-sizing: border-box;\n",
       "  margin-bottom: 0.5em;\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-estimator.fitted {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "/* on hover */\n",
       "#sk-container-id-2 div.sk-estimator:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-2 div.sk-estimator.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Specification for estimator info (e.g. \"i\" and \"?\") */\n",
       "\n",
       "/* Common style for \"i\" and \"?\" */\n",
       "\n",
       ".sk-estimator-doc-link,\n",
       "a:link.sk-estimator-doc-link,\n",
       "a:visited.sk-estimator-doc-link {\n",
       "  float: right;\n",
       "  font-size: smaller;\n",
       "  line-height: 1em;\n",
       "  font-family: monospace;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  border-radius: 1em;\n",
       "  height: 1em;\n",
       "  width: 1em;\n",
       "  text-decoration: none !important;\n",
       "  margin-left: 1ex;\n",
       "  /* unfitted */\n",
       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-unfitted-level-1);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link.fitted,\n",
       "a:link.sk-estimator-doc-link.fitted,\n",
       "a:visited.sk-estimator-doc-link.fitted {\n",
       "  /* fitted */\n",
       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-fitted-level-1);\n",
       "}\n",
       "\n",
       "/* On hover */\n",
       "div.sk-estimator:hover .sk-estimator-doc-link:hover,\n",
       ".sk-estimator-doc-link:hover,\n",
       "div.sk-label-container:hover .sk-estimator-doc-link:hover,\n",
       ".sk-estimator-doc-link:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "div.sk-estimator.fitted:hover .sk-estimator-doc-link.fitted:hover,\n",
       ".sk-estimator-doc-link.fitted:hover,\n",
       "div.sk-label-container:hover .sk-estimator-doc-link.fitted:hover,\n",
       ".sk-estimator-doc-link.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "/* Span, style for the box shown on hovering the info icon */\n",
       ".sk-estimator-doc-link span {\n",
       "  display: none;\n",
       "  z-index: 9999;\n",
       "  position: relative;\n",
       "  font-weight: normal;\n",
       "  right: .2ex;\n",
       "  padding: .5ex;\n",
       "  margin: .5ex;\n",
       "  width: min-content;\n",
       "  min-width: 20ex;\n",
       "  max-width: 50ex;\n",
       "  color: var(--sklearn-color-text);\n",
       "  box-shadow: 2pt 2pt 4pt #999;\n",
       "  /* unfitted */\n",
       "  background: var(--sklearn-color-unfitted-level-0);\n",
       "  border: .5pt solid var(--sklearn-color-unfitted-level-3);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link.fitted span {\n",
       "  /* fitted */\n",
       "  background: var(--sklearn-color-fitted-level-0);\n",
       "  border: var(--sklearn-color-fitted-level-3);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link:hover span {\n",
       "  display: block;\n",
       "}\n",
       "\n",
       "/* \"?\"-specific style due to the `<a>` HTML tag */\n",
       "\n",
       "#sk-container-id-2 a.estimator_doc_link {\n",
       "  float: right;\n",
       "  font-size: 1rem;\n",
       "  line-height: 1em;\n",
       "  font-family: monospace;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  border-radius: 1rem;\n",
       "  height: 1rem;\n",
       "  width: 1rem;\n",
       "  text-decoration: none;\n",
       "  /* unfitted */\n",
       "  color: var(--sklearn-color-unfitted-level-1);\n",
       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 a.estimator_doc_link.fitted {\n",
       "  /* fitted */\n",
       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-fitted-level-1);\n",
       "}\n",
       "\n",
       "/* On hover */\n",
       "#sk-container-id-2 a.estimator_doc_link:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "#sk-container-id-2 a.estimator_doc_link.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-3);\n",
       "}\n",
       "</style><div id=\"sk-container-id-2\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>CountVectorizer(min_df=5)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator fitted sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" checked><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label fitted sk-toggleable__label-arrow fitted\">&nbsp;&nbsp;CountVectorizer<a class=\"sk-estimator-doc-link fitted\" rel=\"noreferrer\" target=\"_blank\" href=\"https://scikit-learn.org/1.5/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html\">?<span>Documentation for CountVectorizer</span></a><span class=\"sk-estimator-doc-link fitted\">i<span>Fitted</span></span></label><div class=\"sk-toggleable__content fitted\"><pre>CountVectorizer(min_df=5)</pre></div> </div></div></div></div>"
      ],
      "text/plain": [
       "CountVectorizer(min_df=5)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv.fit(df.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "e53f309f-6d00-4932-8b02-9ad28613fd8a",
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_examples = [\n",
    "    \"Course starts on 15th Jan 2024\",\n",
    "    \"Prerequisites listed on GitHub\",\n",
    "    \"Submit homeworks after start date\",\n",
    "    \"Registration not required for participation\",\n",
    "    \"Setup Google Cloud and Python before course\"\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "33108e27-c9dd-448e-8971-e8566ac7bf39",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-4 {\n",
       "  /* Definition of color scheme common for light and dark mode */\n",
       "  --sklearn-color-text: black;\n",
       "  --sklearn-color-line: gray;\n",
       "  /* Definition of color scheme for unfitted estimators */\n",
       "  --sklearn-color-unfitted-level-0: #fff5e6;\n",
       "  --sklearn-color-unfitted-level-1: #f6e4d2;\n",
       "  --sklearn-color-unfitted-level-2: #ffe0b3;\n",
       "  --sklearn-color-unfitted-level-3: chocolate;\n",
       "  /* Definition of color scheme for fitted estimators */\n",
       "  --sklearn-color-fitted-level-0: #f0f8ff;\n",
       "  --sklearn-color-fitted-level-1: #d4ebff;\n",
       "  --sklearn-color-fitted-level-2: #b3dbfd;\n",
       "  --sklearn-color-fitted-level-3: cornflowerblue;\n",
       "\n",
       "  /* Specific color for light theme */\n",
       "  --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
       "  --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, white)));\n",
       "  --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
       "  --sklearn-color-icon: #696969;\n",
       "\n",
       "  @media (prefers-color-scheme: dark) {\n",
       "    /* Redefinition of color scheme for dark theme */\n",
       "    --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
       "    --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, #111)));\n",
       "    --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
       "    --sklearn-color-icon: #878787;\n",
       "  }\n",
       "}\n",
       "\n",
       "#sk-container-id-4 {\n",
       "  color: var(--sklearn-color-text);\n",
       "}\n",
       "\n",
       "#sk-container-id-4 pre {\n",
       "  padding: 0;\n",
       "}\n",
       "\n",
       "#sk-container-id-4 input.sk-hidden--visually {\n",
       "  border: 0;\n",
       "  clip: rect(1px 1px 1px 1px);\n",
       "  clip: rect(1px, 1px, 1px, 1px);\n",
       "  height: 1px;\n",
       "  margin: -1px;\n",
       "  overflow: hidden;\n",
       "  padding: 0;\n",
       "  position: absolute;\n",
       "  width: 1px;\n",
       "}\n",
       "\n",
       "#sk-container-id-4 div.sk-dashed-wrapped {\n",
       "  border: 1px dashed var(--sklearn-color-line);\n",
       "  margin: 0 0.4em 0.5em 0.4em;\n",
       "  box-sizing: border-box;\n",
       "  padding-bottom: 0.4em;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "}\n",
       "\n",
       "#sk-container-id-4 div.sk-container {\n",
       "  /* jupyter's `normalize.less` sets `[hidden] { display: none; }`\n",
       "     but bootstrap.min.css set `[hidden] { display: none !important; }`\n",
       "     so we also need the `!important` here to be able to override the\n",
       "     default hidden behavior on the sphinx rendered scikit-learn.org.\n",
       "     See: https://github.com/scikit-learn/scikit-learn/issues/21755 */\n",
       "  display: inline-block !important;\n",
       "  position: relative;\n",
       "}\n",
       "\n",
       "#sk-container-id-4 div.sk-text-repr-fallback {\n",
       "  display: none;\n",
       "}\n",
       "\n",
       "div.sk-parallel-item,\n",
       "div.sk-serial,\n",
       "div.sk-item {\n",
       "  /* draw centered vertical line to link estimators */\n",
       "  background-image: linear-gradient(var(--sklearn-color-text-on-default-background), var(--sklearn-color-text-on-default-background));\n",
       "  background-size: 2px 100%;\n",
       "  background-repeat: no-repeat;\n",
       "  background-position: center center;\n",
       "}\n",
       "\n",
       "/* Parallel-specific style estimator block */\n",
       "\n",
       "#sk-container-id-4 div.sk-parallel-item::after {\n",
       "  content: \"\";\n",
       "  width: 100%;\n",
       "  border-bottom: 2px solid var(--sklearn-color-text-on-default-background);\n",
       "  flex-grow: 1;\n",
       "}\n",
       "\n",
       "#sk-container-id-4 div.sk-parallel {\n",
       "  display: flex;\n",
       "  align-items: stretch;\n",
       "  justify-content: center;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  position: relative;\n",
       "}\n",
       "\n",
       "#sk-container-id-4 div.sk-parallel-item {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "}\n",
       "\n",
       "#sk-container-id-4 div.sk-parallel-item:first-child::after {\n",
       "  align-self: flex-end;\n",
       "  width: 50%;\n",
       "}\n",
       "\n",
       "#sk-container-id-4 div.sk-parallel-item:last-child::after {\n",
       "  align-self: flex-start;\n",
       "  width: 50%;\n",
       "}\n",
       "\n",
       "#sk-container-id-4 div.sk-parallel-item:only-child::after {\n",
       "  width: 0;\n",
       "}\n",
       "\n",
       "/* Serial-specific style estimator block */\n",
       "\n",
       "#sk-container-id-4 div.sk-serial {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "  align-items: center;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  padding-right: 1em;\n",
       "  padding-left: 1em;\n",
       "}\n",
       "\n",
       "\n",
       "/* Toggleable style: style used for estimator/Pipeline/ColumnTransformer box that is\n",
       "clickable and can be expanded/collapsed.\n",
       "- Pipeline and ColumnTransformer use this feature and define the default style\n",
       "- Estimators will overwrite some part of the style using the `sk-estimator` class\n",
       "*/\n",
       "\n",
       "/* Pipeline and ColumnTransformer style (default) */\n",
       "\n",
       "#sk-container-id-4 div.sk-toggleable {\n",
       "  /* Default theme specific background. It is overwritten whether we have a\n",
       "  specific estimator or a Pipeline/ColumnTransformer */\n",
       "  background-color: var(--sklearn-color-background);\n",
       "}\n",
       "\n",
       "/* Toggleable label */\n",
       "#sk-container-id-4 label.sk-toggleable__label {\n",
       "  cursor: pointer;\n",
       "  display: block;\n",
       "  width: 100%;\n",
       "  margin-bottom: 0;\n",
       "  padding: 0.5em;\n",
       "  box-sizing: border-box;\n",
       "  text-align: center;\n",
       "}\n",
       "\n",
       "#sk-container-id-4 label.sk-toggleable__label-arrow:before {\n",
       "  /* Arrow on the left of the label */\n",
       "  content: \"▸\";\n",
       "  float: left;\n",
       "  margin-right: 0.25em;\n",
       "  color: var(--sklearn-color-icon);\n",
       "}\n",
       "\n",
       "#sk-container-id-4 label.sk-toggleable__label-arrow:hover:before {\n",
       "  color: var(--sklearn-color-text);\n",
       "}\n",
       "\n",
       "/* Toggleable content - dropdown */\n",
       "\n",
       "#sk-container-id-4 div.sk-toggleable__content {\n",
       "  max-height: 0;\n",
       "  max-width: 0;\n",
       "  overflow: hidden;\n",
       "  text-align: left;\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-4 div.sk-toggleable__content.fitted {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-4 div.sk-toggleable__content pre {\n",
       "  margin: 0.2em;\n",
       "  border-radius: 0.25em;\n",
       "  color: var(--sklearn-color-text);\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-4 div.sk-toggleable__content.fitted pre {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-4 input.sk-toggleable__control:checked~div.sk-toggleable__content {\n",
       "  /* Expand drop-down */\n",
       "  max-height: 200px;\n",
       "  max-width: 100%;\n",
       "  overflow: auto;\n",
       "}\n",
       "\n",
       "#sk-container-id-4 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {\n",
       "  content: \"▾\";\n",
       "}\n",
       "\n",
       "/* Pipeline/ColumnTransformer-specific style */\n",
       "\n",
       "#sk-container-id-4 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-4 div.sk-label.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Estimator-specific style */\n",
       "\n",
       "/* Colorize estimator box */\n",
       "#sk-container-id-4 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-4 div.sk-estimator.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-4 div.sk-label label.sk-toggleable__label,\n",
       "#sk-container-id-4 div.sk-label label {\n",
       "  /* The background is the default theme color */\n",
       "  color: var(--sklearn-color-text-on-default-background);\n",
       "}\n",
       "\n",
       "/* On hover, darken the color of the background */\n",
       "#sk-container-id-4 div.sk-label:hover label.sk-toggleable__label {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "/* Label box, darken color on hover, fitted */\n",
       "#sk-container-id-4 div.sk-label.fitted:hover label.sk-toggleable__label.fitted {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Estimator label */\n",
       "\n",
       "#sk-container-id-4 div.sk-label label {\n",
       "  font-family: monospace;\n",
       "  font-weight: bold;\n",
       "  display: inline-block;\n",
       "  line-height: 1.2em;\n",
       "}\n",
       "\n",
       "#sk-container-id-4 div.sk-label-container {\n",
       "  text-align: center;\n",
       "}\n",
       "\n",
       "/* Estimator-specific */\n",
       "#sk-container-id-4 div.sk-estimator {\n",
       "  font-family: monospace;\n",
       "  border: 1px dotted var(--sklearn-color-border-box);\n",
       "  border-radius: 0.25em;\n",
       "  box-sizing: border-box;\n",
       "  margin-bottom: 0.5em;\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-4 div.sk-estimator.fitted {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "/* on hover */\n",
       "#sk-container-id-4 div.sk-estimator:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-4 div.sk-estimator.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Specification for estimator info (e.g. \"i\" and \"?\") */\n",
       "\n",
       "/* Common style for \"i\" and \"?\" */\n",
       "\n",
       ".sk-estimator-doc-link,\n",
       "a:link.sk-estimator-doc-link,\n",
       "a:visited.sk-estimator-doc-link {\n",
       "  float: right;\n",
       "  font-size: smaller;\n",
       "  line-height: 1em;\n",
       "  font-family: monospace;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  border-radius: 1em;\n",
       "  height: 1em;\n",
       "  width: 1em;\n",
       "  text-decoration: none !important;\n",
       "  margin-left: 1ex;\n",
       "  /* unfitted */\n",
       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-unfitted-level-1);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link.fitted,\n",
       "a:link.sk-estimator-doc-link.fitted,\n",
       "a:visited.sk-estimator-doc-link.fitted {\n",
       "  /* fitted */\n",
       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-fitted-level-1);\n",
       "}\n",
       "\n",
       "/* On hover */\n",
       "div.sk-estimator:hover .sk-estimator-doc-link:hover,\n",
       ".sk-estimator-doc-link:hover,\n",
       "div.sk-label-container:hover .sk-estimator-doc-link:hover,\n",
       ".sk-estimator-doc-link:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "div.sk-estimator.fitted:hover .sk-estimator-doc-link.fitted:hover,\n",
       ".sk-estimator-doc-link.fitted:hover,\n",
       "div.sk-label-container:hover .sk-estimator-doc-link.fitted:hover,\n",
       ".sk-estimator-doc-link.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "/* Span, style for the box shown on hovering the info icon */\n",
       ".sk-estimator-doc-link span {\n",
       "  display: none;\n",
       "  z-index: 9999;\n",
       "  position: relative;\n",
       "  font-weight: normal;\n",
       "  right: .2ex;\n",
       "  padding: .5ex;\n",
       "  margin: .5ex;\n",
       "  width: min-content;\n",
       "  min-width: 20ex;\n",
       "  max-width: 50ex;\n",
       "  color: var(--sklearn-color-text);\n",
       "  box-shadow: 2pt 2pt 4pt #999;\n",
       "  /* unfitted */\n",
       "  background: var(--sklearn-color-unfitted-level-0);\n",
       "  border: .5pt solid var(--sklearn-color-unfitted-level-3);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link.fitted span {\n",
       "  /* fitted */\n",
       "  background: var(--sklearn-color-fitted-level-0);\n",
       "  border: var(--sklearn-color-fitted-level-3);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link:hover span {\n",
       "  display: block;\n",
       "}\n",
       "\n",
       "/* \"?\"-specific style due to the `<a>` HTML tag */\n",
       "\n",
       "#sk-container-id-4 a.estimator_doc_link {\n",
       "  float: right;\n",
       "  font-size: 1rem;\n",
       "  line-height: 1em;\n",
       "  font-family: monospace;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  border-radius: 1rem;\n",
       "  height: 1rem;\n",
       "  width: 1rem;\n",
       "  text-decoration: none;\n",
       "  /* unfitted */\n",
       "  color: var(--sklearn-color-unfitted-level-1);\n",
       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
       "}\n",
       "\n",
       "#sk-container-id-4 a.estimator_doc_link.fitted {\n",
       "  /* fitted */\n",
       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-fitted-level-1);\n",
       "}\n",
       "\n",
       "/* On hover */\n",
       "#sk-container-id-4 a.estimator_doc_link:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "#sk-container-id-4 a.estimator_doc_link.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-3);\n",
       "}\n",
       "</style><div id=\"sk-container-id-4\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>CountVectorizer(stop_words=&#x27;english&#x27;)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator fitted sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-4\" type=\"checkbox\" checked><label for=\"sk-estimator-id-4\" class=\"sk-toggleable__label fitted sk-toggleable__label-arrow fitted\">&nbsp;&nbsp;CountVectorizer<a class=\"sk-estimator-doc-link fitted\" rel=\"noreferrer\" target=\"_blank\" href=\"https://scikit-learn.org/1.5/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html\">?<span>Documentation for CountVectorizer</span></a><span class=\"sk-estimator-doc-link fitted\">i<span>Fitted</span></span></label><div class=\"sk-toggleable__content fitted\"><pre>CountVectorizer(stop_words=&#x27;english&#x27;)</pre></div> </div></div></div></div>"
      ],
      "text/plain": [
       "CountVectorizer(stop_words='english')"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv = CountVectorizer(stop_words='english')\n",
    "cv.fit(doc_examples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "7d303dbf-6c1e-410b-8094-ee6cd2dfd5f3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['15th', '2024', 'after', 'and', 'before', 'cloud', 'course',\n",
       "       'date', 'for', 'github', 'google', 'homeworks', 'jan', 'listed',\n",
       "       'not', 'on', 'participation', 'prerequisites', 'python',\n",
       "       'registration', 'required', 'setup', 'start', 'starts', 'submit'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv.get_feature_names_out()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "23228304-ad1f-400d-9b9b-d2dbf12d8a77",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = cv.transform(doc_examples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "a15582f8-1865-46ee-ba6d-453a8c9bf663",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "matrix([[1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0,\n",
       "         0, 0, 1, 0],\n",
       "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0,\n",
       "         0, 0, 0, 0],\n",
       "        [0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 1, 0, 1],\n",
       "        [0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1,\n",
       "         0, 0, 0, 0],\n",
       "        [0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0,\n",
       "         1, 0, 0, 0]])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.todense()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "27101b6a-109d-4b04-a01f-60009f775619",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>15th</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2024</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>after</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>and</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>before</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cloud</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>course</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>for</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>github</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>google</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>homeworks</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>jan</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>listed</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>not</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>on</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>participation</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>prerequisites</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>python</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>registration</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>required</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>setup</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>start</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>starts</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>submit</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               0  1  2  3  4\n",
       "15th           1  0  0  0  0\n",
       "2024           1  0  0  0  0\n",
       "after          0  0  1  0  0\n",
       "and            0  0  0  0  1\n",
       "before         0  0  0  0  1\n",
       "cloud          0  0  0  0  1\n",
       "course         1  0  0  0  1\n",
       "date           0  0  1  0  0\n",
       "for            0  0  0  1  0\n",
       "github         0  1  0  0  0\n",
       "google         0  0  0  0  1\n",
       "homeworks      0  0  1  0  0\n",
       "jan            1  0  0  0  0\n",
       "listed         0  1  0  0  0\n",
       "not            0  0  0  1  0\n",
       "on             1  1  0  0  0\n",
       "participation  0  0  0  1  0\n",
       "prerequisites  0  1  0  0  0\n",
       "python         0  0  0  0  1\n",
       "registration   0  0  0  1  0\n",
       "required       0  0  0  1  0\n",
       "setup          0  0  0  0  1\n",
       "start          0  0  1  0  0\n",
       "starts         1  0  0  0  0\n",
       "submit         0  0  1  0  0"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(X.todense(), columns=cv.get_feature_names_out()).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "0c2f9495-d5f1-4d1a-88fb-818d47d4e19b",
   "metadata": {},
   "outputs": [],
   "source": [
    "cv = CountVectorizer(stop_words='english', min_df=5)\n",
    "X = cv.fit_transform(df.text)\n",
    "\n",
    "names = cv.get_feature_names_out()\n",
    "\n",
    "df_docs = pd.DataFrame(X.toarray(), columns=names).T\n",
    "df_docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "9e6519e9-177d-4af7-90f1-f1b86d00b35f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "7948c9cb-d01d-4e64-9c32-7f2c666f7d2c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>01</th>\n",
       "      <th>02</th>\n",
       "      <th>03</th>\n",
       "      <th>04</th>\n",
       "      <th>05</th>\n",
       "      <th>06</th>\n",
       "      <th>09</th>\n",
       "      <th>10</th>\n",
       "      <th>100</th>\n",
       "      <th>11</th>\n",
       "      <th>...</th>\n",
       "      <th>y_val</th>\n",
       "      <th>yaml</th>\n",
       "      <th>year</th>\n",
       "      <th>yellow</th>\n",
       "      <th>yellow_tripdata_2021</th>\n",
       "      <th>yes</th>\n",
       "      <th>yml</th>\n",
       "      <th>youtube</th>\n",
       "      <th>zip</th>\n",
       "      <th>zoomcamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.28</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>943</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>944</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>945</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.17</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>946</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>947</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.15</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>948 rows × 1333 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      01   02   03   04   05   06   09   10  100   11  ...  y_val  yaml  year  \\\n",
       "0    0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...    0.0  0.00   0.0   \n",
       "1    0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...    0.0  0.00   0.0   \n",
       "2    0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...    0.0  0.00   0.0   \n",
       "3    0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...    0.0  0.00   0.0   \n",
       "4    0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...    0.0  0.00   0.0   \n",
       "..   ...  ...  ...  ...  ...  ...  ...  ...  ...  ...  ...    ...   ...   ...   \n",
       "943  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...    0.0  0.00   0.0   \n",
       "944  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...    0.0  0.00   0.0   \n",
       "945  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...    0.0  0.17   0.0   \n",
       "946  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...    0.0  0.00   0.0   \n",
       "947  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...    0.0  0.00   0.0   \n",
       "\n",
       "     yellow  yellow_tripdata_2021   yes   yml  youtube  zip  zoomcamp  \n",
       "0       0.0                   0.0  0.00  0.00      0.0  0.0      0.00  \n",
       "1       0.0                   0.0  0.00  0.00      0.0  0.0      0.43  \n",
       "2       0.0                   0.0  0.28  0.00      0.0  0.0      0.00  \n",
       "3       0.0                   0.0  0.00  0.00      0.0  0.0      0.00  \n",
       "4       0.0                   0.0  0.00  0.00      0.0  0.0      0.00  \n",
       "..      ...                   ...   ...   ...      ...  ...       ...  \n",
       "943     0.0                   0.0  0.00  0.11      0.0  0.0      0.00  \n",
       "944     0.0                   0.0  0.00  0.00      0.0  0.0      0.00  \n",
       "945     0.0                   0.0  0.00  0.00      0.0  0.0      0.00  \n",
       "946     0.0                   0.0  0.00  0.00      0.0  0.0      0.00  \n",
       "947     0.0                   0.0  0.00  0.00      0.0  0.0      0.15  \n",
       "\n",
       "[948 rows x 1333 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv = TfidfVectorizer(stop_words='english', min_df=5)\n",
    "X = cv.fit_transform(df.text)\n",
    "\n",
    "names = cv.get_feature_names_out()\n",
    "\n",
    "df_docs = pd.DataFrame(X.toarray(), columns=names)\n",
    "df_docs.round(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "90d06f1b-b108-4679-9a5b-f62be613123a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<948x1333 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 23808 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "fc821b39-8641-4cdf-a94c-7f0bc62634f4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-5 {\n",
       "  /* Definition of color scheme common for light and dark mode */\n",
       "  --sklearn-color-text: black;\n",
       "  --sklearn-color-line: gray;\n",
       "  /* Definition of color scheme for unfitted estimators */\n",
       "  --sklearn-color-unfitted-level-0: #fff5e6;\n",
       "  --sklearn-color-unfitted-level-1: #f6e4d2;\n",
       "  --sklearn-color-unfitted-level-2: #ffe0b3;\n",
       "  --sklearn-color-unfitted-level-3: chocolate;\n",
       "  /* Definition of color scheme for fitted estimators */\n",
       "  --sklearn-color-fitted-level-0: #f0f8ff;\n",
       "  --sklearn-color-fitted-level-1: #d4ebff;\n",
       "  --sklearn-color-fitted-level-2: #b3dbfd;\n",
       "  --sklearn-color-fitted-level-3: cornflowerblue;\n",
       "\n",
       "  /* Specific color for light theme */\n",
       "  --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
       "  --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, white)));\n",
       "  --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
       "  --sklearn-color-icon: #696969;\n",
       "\n",
       "  @media (prefers-color-scheme: dark) {\n",
       "    /* Redefinition of color scheme for dark theme */\n",
       "    --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
       "    --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, #111)));\n",
       "    --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
       "    --sklearn-color-icon: #878787;\n",
       "  }\n",
       "}\n",
       "\n",
       "#sk-container-id-5 {\n",
       "  color: var(--sklearn-color-text);\n",
       "}\n",
       "\n",
       "#sk-container-id-5 pre {\n",
       "  padding: 0;\n",
       "}\n",
       "\n",
       "#sk-container-id-5 input.sk-hidden--visually {\n",
       "  border: 0;\n",
       "  clip: rect(1px 1px 1px 1px);\n",
       "  clip: rect(1px, 1px, 1px, 1px);\n",
       "  height: 1px;\n",
       "  margin: -1px;\n",
       "  overflow: hidden;\n",
       "  padding: 0;\n",
       "  position: absolute;\n",
       "  width: 1px;\n",
       "}\n",
       "\n",
       "#sk-container-id-5 div.sk-dashed-wrapped {\n",
       "  border: 1px dashed var(--sklearn-color-line);\n",
       "  margin: 0 0.4em 0.5em 0.4em;\n",
       "  box-sizing: border-box;\n",
       "  padding-bottom: 0.4em;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "}\n",
       "\n",
       "#sk-container-id-5 div.sk-container {\n",
       "  /* jupyter's `normalize.less` sets `[hidden] { display: none; }`\n",
       "     but bootstrap.min.css set `[hidden] { display: none !important; }`\n",
       "     so we also need the `!important` here to be able to override the\n",
       "     default hidden behavior on the sphinx rendered scikit-learn.org.\n",
       "     See: https://github.com/scikit-learn/scikit-learn/issues/21755 */\n",
       "  display: inline-block !important;\n",
       "  position: relative;\n",
       "}\n",
       "\n",
       "#sk-container-id-5 div.sk-text-repr-fallback {\n",
       "  display: none;\n",
       "}\n",
       "\n",
       "div.sk-parallel-item,\n",
       "div.sk-serial,\n",
       "div.sk-item {\n",
       "  /* draw centered vertical line to link estimators */\n",
       "  background-image: linear-gradient(var(--sklearn-color-text-on-default-background), var(--sklearn-color-text-on-default-background));\n",
       "  background-size: 2px 100%;\n",
       "  background-repeat: no-repeat;\n",
       "  background-position: center center;\n",
       "}\n",
       "\n",
       "/* Parallel-specific style estimator block */\n",
       "\n",
       "#sk-container-id-5 div.sk-parallel-item::after {\n",
       "  content: \"\";\n",
       "  width: 100%;\n",
       "  border-bottom: 2px solid var(--sklearn-color-text-on-default-background);\n",
       "  flex-grow: 1;\n",
       "}\n",
       "\n",
       "#sk-container-id-5 div.sk-parallel {\n",
       "  display: flex;\n",
       "  align-items: stretch;\n",
       "  justify-content: center;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  position: relative;\n",
       "}\n",
       "\n",
       "#sk-container-id-5 div.sk-parallel-item {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "}\n",
       "\n",
       "#sk-container-id-5 div.sk-parallel-item:first-child::after {\n",
       "  align-self: flex-end;\n",
       "  width: 50%;\n",
       "}\n",
       "\n",
       "#sk-container-id-5 div.sk-parallel-item:last-child::after {\n",
       "  align-self: flex-start;\n",
       "  width: 50%;\n",
       "}\n",
       "\n",
       "#sk-container-id-5 div.sk-parallel-item:only-child::after {\n",
       "  width: 0;\n",
       "}\n",
       "\n",
       "/* Serial-specific style estimator block */\n",
       "\n",
       "#sk-container-id-5 div.sk-serial {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "  align-items: center;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  padding-right: 1em;\n",
       "  padding-left: 1em;\n",
       "}\n",
       "\n",
       "\n",
       "/* Toggleable style: style used for estimator/Pipeline/ColumnTransformer box that is\n",
       "clickable and can be expanded/collapsed.\n",
       "- Pipeline and ColumnTransformer use this feature and define the default style\n",
       "- Estimators will overwrite some part of the style using the `sk-estimator` class\n",
       "*/\n",
       "\n",
       "/* Pipeline and ColumnTransformer style (default) */\n",
       "\n",
       "#sk-container-id-5 div.sk-toggleable {\n",
       "  /* Default theme specific background. It is overwritten whether we have a\n",
       "  specific estimator or a Pipeline/ColumnTransformer */\n",
       "  background-color: var(--sklearn-color-background);\n",
       "}\n",
       "\n",
       "/* Toggleable label */\n",
       "#sk-container-id-5 label.sk-toggleable__label {\n",
       "  cursor: pointer;\n",
       "  display: block;\n",
       "  width: 100%;\n",
       "  margin-bottom: 0;\n",
       "  padding: 0.5em;\n",
       "  box-sizing: border-box;\n",
       "  text-align: center;\n",
       "}\n",
       "\n",
       "#sk-container-id-5 label.sk-toggleable__label-arrow:before {\n",
       "  /* Arrow on the left of the label */\n",
       "  content: \"▸\";\n",
       "  float: left;\n",
       "  margin-right: 0.25em;\n",
       "  color: var(--sklearn-color-icon);\n",
       "}\n",
       "\n",
       "#sk-container-id-5 label.sk-toggleable__label-arrow:hover:before {\n",
       "  color: var(--sklearn-color-text);\n",
       "}\n",
       "\n",
       "/* Toggleable content - dropdown */\n",
       "\n",
       "#sk-container-id-5 div.sk-toggleable__content {\n",
       "  max-height: 0;\n",
       "  max-width: 0;\n",
       "  overflow: hidden;\n",
       "  text-align: left;\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-5 div.sk-toggleable__content.fitted {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-5 div.sk-toggleable__content pre {\n",
       "  margin: 0.2em;\n",
       "  border-radius: 0.25em;\n",
       "  color: var(--sklearn-color-text);\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-5 div.sk-toggleable__content.fitted pre {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-5 input.sk-toggleable__control:checked~div.sk-toggleable__content {\n",
       "  /* Expand drop-down */\n",
       "  max-height: 200px;\n",
       "  max-width: 100%;\n",
       "  overflow: auto;\n",
       "}\n",
       "\n",
       "#sk-container-id-5 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {\n",
       "  content: \"▾\";\n",
       "}\n",
       "\n",
       "/* Pipeline/ColumnTransformer-specific style */\n",
       "\n",
       "#sk-container-id-5 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-5 div.sk-label.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Estimator-specific style */\n",
       "\n",
       "/* Colorize estimator box */\n",
       "#sk-container-id-5 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-5 div.sk-estimator.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-5 div.sk-label label.sk-toggleable__label,\n",
       "#sk-container-id-5 div.sk-label label {\n",
       "  /* The background is the default theme color */\n",
       "  color: var(--sklearn-color-text-on-default-background);\n",
       "}\n",
       "\n",
       "/* On hover, darken the color of the background */\n",
       "#sk-container-id-5 div.sk-label:hover label.sk-toggleable__label {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "/* Label box, darken color on hover, fitted */\n",
       "#sk-container-id-5 div.sk-label.fitted:hover label.sk-toggleable__label.fitted {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Estimator label */\n",
       "\n",
       "#sk-container-id-5 div.sk-label label {\n",
       "  font-family: monospace;\n",
       "  font-weight: bold;\n",
       "  display: inline-block;\n",
       "  line-height: 1.2em;\n",
       "}\n",
       "\n",
       "#sk-container-id-5 div.sk-label-container {\n",
       "  text-align: center;\n",
       "}\n",
       "\n",
       "/* Estimator-specific */\n",
       "#sk-container-id-5 div.sk-estimator {\n",
       "  font-family: monospace;\n",
       "  border: 1px dotted var(--sklearn-color-border-box);\n",
       "  border-radius: 0.25em;\n",
       "  box-sizing: border-box;\n",
       "  margin-bottom: 0.5em;\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-5 div.sk-estimator.fitted {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "/* on hover */\n",
       "#sk-container-id-5 div.sk-estimator:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-5 div.sk-estimator.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Specification for estimator info (e.g. \"i\" and \"?\") */\n",
       "\n",
       "/* Common style for \"i\" and \"?\" */\n",
       "\n",
       ".sk-estimator-doc-link,\n",
       "a:link.sk-estimator-doc-link,\n",
       "a:visited.sk-estimator-doc-link {\n",
       "  float: right;\n",
       "  font-size: smaller;\n",
       "  line-height: 1em;\n",
       "  font-family: monospace;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  border-radius: 1em;\n",
       "  height: 1em;\n",
       "  width: 1em;\n",
       "  text-decoration: none !important;\n",
       "  margin-left: 1ex;\n",
       "  /* unfitted */\n",
       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-unfitted-level-1);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link.fitted,\n",
       "a:link.sk-estimator-doc-link.fitted,\n",
       "a:visited.sk-estimator-doc-link.fitted {\n",
       "  /* fitted */\n",
       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-fitted-level-1);\n",
       "}\n",
       "\n",
       "/* On hover */\n",
       "div.sk-estimator:hover .sk-estimator-doc-link:hover,\n",
       ".sk-estimator-doc-link:hover,\n",
       "div.sk-label-container:hover .sk-estimator-doc-link:hover,\n",
       ".sk-estimator-doc-link:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "div.sk-estimator.fitted:hover .sk-estimator-doc-link.fitted:hover,\n",
       ".sk-estimator-doc-link.fitted:hover,\n",
       "div.sk-label-container:hover .sk-estimator-doc-link.fitted:hover,\n",
       ".sk-estimator-doc-link.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "/* Span, style for the box shown on hovering the info icon */\n",
       ".sk-estimator-doc-link span {\n",
       "  display: none;\n",
       "  z-index: 9999;\n",
       "  position: relative;\n",
       "  font-weight: normal;\n",
       "  right: .2ex;\n",
       "  padding: .5ex;\n",
       "  margin: .5ex;\n",
       "  width: min-content;\n",
       "  min-width: 20ex;\n",
       "  max-width: 50ex;\n",
       "  color: var(--sklearn-color-text);\n",
       "  box-shadow: 2pt 2pt 4pt #999;\n",
       "  /* unfitted */\n",
       "  background: var(--sklearn-color-unfitted-level-0);\n",
       "  border: .5pt solid var(--sklearn-color-unfitted-level-3);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link.fitted span {\n",
       "  /* fitted */\n",
       "  background: var(--sklearn-color-fitted-level-0);\n",
       "  border: var(--sklearn-color-fitted-level-3);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link:hover span {\n",
       "  display: block;\n",
       "}\n",
       "\n",
       "/* \"?\"-specific style due to the `<a>` HTML tag */\n",
       "\n",
       "#sk-container-id-5 a.estimator_doc_link {\n",
       "  float: right;\n",
       "  font-size: 1rem;\n",
       "  line-height: 1em;\n",
       "  font-family: monospace;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  border-radius: 1rem;\n",
       "  height: 1rem;\n",
       "  width: 1rem;\n",
       "  text-decoration: none;\n",
       "  /* unfitted */\n",
       "  color: var(--sklearn-color-unfitted-level-1);\n",
       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
       "}\n",
       "\n",
       "#sk-container-id-5 a.estimator_doc_link.fitted {\n",
       "  /* fitted */\n",
       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-fitted-level-1);\n",
       "}\n",
       "\n",
       "/* On hover */\n",
       "#sk-container-id-5 a.estimator_doc_link:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "#sk-container-id-5 a.estimator_doc_link.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-3);\n",
       "}\n",
       "</style><div id=\"sk-container-id-5\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>TfidfVectorizer(min_df=5, stop_words=&#x27;english&#x27;)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator fitted sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-5\" type=\"checkbox\" checked><label for=\"sk-estimator-id-5\" class=\"sk-toggleable__label fitted sk-toggleable__label-arrow fitted\">&nbsp;&nbsp;TfidfVectorizer<a class=\"sk-estimator-doc-link fitted\" rel=\"noreferrer\" target=\"_blank\" href=\"https://scikit-learn.org/1.5/modules/generated/sklearn.feature_extraction.text.TfidfVectorizer.html\">?<span>Documentation for TfidfVectorizer</span></a><span class=\"sk-estimator-doc-link fitted\">i<span>Fitted</span></span></label><div class=\"sk-toggleable__content fitted\"><pre>TfidfVectorizer(min_df=5, stop_words=&#x27;english&#x27;)</pre></div> </div></div></div></div>"
      ],
      "text/plain": [
       "TfidfVectorizer(min_df=5, stop_words='english')"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "119aa2b7-600b-415a-bee8-6e5c30720b1f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 0., 0., ..., 0., 0., 0.]])"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query = \"I just discovered the course, is it too late to join?\"\n",
    "\n",
    "q = cv.transform([query])\n",
    "q.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "9d75b67c-a9c4-4a3a-8f89-8fd1c4398783",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'01': 0.0,\n",
       " '02': 0.0,\n",
       " '03': 0.0,\n",
       " '04': 0.0,\n",
       " '05': 0.0,\n",
       " '06': 0.0,\n",
       " '09': 0.0,\n",
       " '10': 0.0,\n",
       " '100': 0.0,\n",
       " '11': 0.0,\n",
       " '12': 0.0,\n",
       " '127': 0.0,\n",
       " '13': 0.0,\n",
       " '14': 0.0,\n",
       " '15': 0.0,\n",
       " '16': 0.0,\n",
       " '17': 0.0,\n",
       " '19': 0.0,\n",
       " '1st': 0.0,\n",
       " '20': 0.0,\n",
       " '2019': 0.0,\n",
       " '2020': 0.0,\n",
       " '2021': 0.0,\n",
       " '2022': 0.0,\n",
       " '2023': 0.0,\n",
       " '2024': 0.0,\n",
       " '21': 0.0,\n",
       " '22': 0.0,\n",
       " '24': 0.0,\n",
       " '25': 0.0,\n",
       " '2pacx': 0.0,\n",
       " '30': 0.0,\n",
       " '35': 0.0,\n",
       " '403': 0.0,\n",
       " '42': 0.0,\n",
       " '50': 0.0,\n",
       " '5000': 0.0,\n",
       " '5431': 0.0,\n",
       " '5432': 0.0,\n",
       " '60': 0.0,\n",
       " '600': 0.0,\n",
       " '7077': 0.0,\n",
       " '80': 0.0,\n",
       " '8080': 0.0,\n",
       " '8888': 0.0,\n",
       " '9696': 0.0,\n",
       " 'abhijit': 0.0,\n",
       " 'able': 0.0,\n",
       " 'abolade': 0.0,\n",
       " 'absolute': 0.0,\n",
       " 'accept': 0.0,\n",
       " 'access': 0.0,\n",
       " 'accordingly': 0.0,\n",
       " 'account': 0.0,\n",
       " 'accuracy': 0.0,\n",
       " 'action': 0.0,\n",
       " 'activate': 0.0,\n",
       " 'actual': 0.0,\n",
       " 'actually': 0.0,\n",
       " 'add': 0.0,\n",
       " 'added': 0.0,\n",
       " 'adding': 0.0,\n",
       " 'addition': 0.0,\n",
       " 'additional': 0.0,\n",
       " 'additionally': 0.0,\n",
       " 'address': 0.0,\n",
       " 'admin': 0.0,\n",
       " 'advani': 0.0,\n",
       " 'ahmed': 0.0,\n",
       " 'ai': 0.0,\n",
       " 'airflow': 0.0,\n",
       " 'alejandro': 0.0,\n",
       " 'alena': 0.0,\n",
       " 'alex': 0.0,\n",
       " 'alexey': 0.0,\n",
       " 'alexeygrigorev': 0.0,\n",
       " 'allocate': 0.0,\n",
       " 'allow': 0.0,\n",
       " 'allowed': 0.0,\n",
       " 'allowing': 0.0,\n",
       " 'allows': 0.0,\n",
       " 'alpha': 0.0,\n",
       " 'alternative': 0.0,\n",
       " 'alternatively': 0.0,\n",
       " 'amazon': 0.0,\n",
       " 'amazonaws': 0.0,\n",
       " 'amd64': 0.0,\n",
       " 'aminat': 0.0,\n",
       " 'anaconda': 0.0,\n",
       " 'anaconda3': 0.0,\n",
       " 'analytics': 0.0,\n",
       " 'anand': 0.0,\n",
       " 'andrii': 0.0,\n",
       " 'ans': 0.0,\n",
       " 'answer': 0.0,\n",
       " 'answered': 0.0,\n",
       " 'answers': 0.0,\n",
       " 'apache': 0.0,\n",
       " 'api': 0.0,\n",
       " 'aponte': 0.0,\n",
       " 'app': 0.0,\n",
       " 'appear': 0.0,\n",
       " 'appears': 0.0,\n",
       " 'append': 0.0,\n",
       " 'application': 0.0,\n",
       " 'apply': 0.0,\n",
       " 'appname': 0.0,\n",
       " 'approach': 0.0,\n",
       " 'apt': 0.0,\n",
       " 'architecture': 0.0,\n",
       " 'archives': 0.0,\n",
       " 'args': 0.0,\n",
       " 'argument': 0.0,\n",
       " 'array': 0.0,\n",
       " 'arrays': 0.0,\n",
       " 'article': 0.0,\n",
       " 'asia': 0.0,\n",
       " 'ask': 0.0,\n",
       " 'asked': 0.0,\n",
       " 'asks': 0.0,\n",
       " 'assigned': 0.0,\n",
       " 'astype': 0.0,\n",
       " 'attempting': 0.0,\n",
       " 'attribute': 0.0,\n",
       " 'attributeerror': 0.0,\n",
       " 'auc': 0.0,\n",
       " 'auth': 0.0,\n",
       " 'authenticate': 0.0,\n",
       " 'authentication': 0.0,\n",
       " 'auto': 0.0,\n",
       " 'automatically': 0.0,\n",
       " 'available': 0.0,\n",
       " 'average': 0.0,\n",
       " 'avila': 0.0,\n",
       " 'avoid': 0.0,\n",
       " 'aws': 0.0,\n",
       " 'azure': 0.0,\n",
       " 'backend': 0.0,\n",
       " 'bad': 0.0,\n",
       " 'bar': 0.0,\n",
       " 'base': 0.0,\n",
       " 'based': 0.0,\n",
       " 'bash': 0.0,\n",
       " 'bashrc': 0.0,\n",
       " 'basically': 0.0,\n",
       " 'batch': 0.0,\n",
       " 'best': 0.0,\n",
       " 'better': 0.0,\n",
       " 'bhaskar': 0.0,\n",
       " 'bhatia': 0.0,\n",
       " 'big': 0.0,\n",
       " 'bigquery': 0.0,\n",
       " 'billing': 0.0,\n",
       " 'bin': 0.0,\n",
       " 'binary': 0.0,\n",
       " 'bind': 0.0,\n",
       " 'bit': 0.0,\n",
       " 'blob': 0.0,\n",
       " 'block': 0.0,\n",
       " 'blog': 0.0,\n",
       " 'box': 0.0,\n",
       " 'bq': 0.0,\n",
       " 'branch': 0.0,\n",
       " 'brew': 0.0,\n",
       " 'brigida': 0.0,\n",
       " 'browser': 0.0,\n",
       " 'bucket': 0.0,\n",
       " 'build': 0.0,\n",
       " 'builder': 0.0,\n",
       " 'building': 0.0,\n",
       " 'built': 0.0,\n",
       " 'calculate': 0.0,\n",
       " 'calderin': 0.0,\n",
       " 'called': 0.0,\n",
       " 'calling': 0.0,\n",
       " 'capstone': 0.0,\n",
       " 'capture': 0.0,\n",
       " 'care': 0.0,\n",
       " 'careful': 0.0,\n",
       " 'case': 0.0,\n",
       " 'cast': 0.0,\n",
       " 'cat': 0.0,\n",
       " 'categorical': 0.0,\n",
       " 'cause': 0.0,\n",
       " 'caused': 0.0,\n",
       " 'causing': 0.0,\n",
       " 'cd': 0.0,\n",
       " 'cell': 0.0,\n",
       " 'certificate': 0.0,\n",
       " 'chakraborty': 0.0,\n",
       " 'change': 0.0,\n",
       " 'changed': 0.0,\n",
       " 'changes': 0.0,\n",
       " 'changing': 0.0,\n",
       " 'channel': 0.0,\n",
       " 'check': 0.0,\n",
       " 'checking': 0.0,\n",
       " 'chizhmak': 0.0,\n",
       " 'chmod': 0.0,\n",
       " 'choice': 0.0,\n",
       " 'choose': 0.0,\n",
       " 'chown': 0.0,\n",
       " 'churn': 0.0,\n",
       " 'ci': 0.0,\n",
       " 'class': 0.0,\n",
       " 'classes': 0.0,\n",
       " 'classification': 0.0,\n",
       " 'clear': 0.0,\n",
       " 'cli': 0.0,\n",
       " 'click': 0.0,\n",
       " 'client': 0.0,\n",
       " 'clone': 0.0,\n",
       " 'close': 0.0,\n",
       " 'closed': 0.0,\n",
       " 'closest': 0.0,\n",
       " 'clothing': 0.0,\n",
       " 'cloud': 0.0,\n",
       " 'cloudfront': 0.0,\n",
       " 'club': 0.0,\n",
       " 'cluster': 0.0,\n",
       " 'cmd': 0.0,\n",
       " 'code': 0.0,\n",
       " 'codespaces': 0.0,\n",
       " 'coding': 0.0,\n",
       " 'cohort': 0.0,\n",
       " 'cohorts': 0.0,\n",
       " 'col': 0.0,\n",
       " 'colab': 0.0,\n",
       " 'column': 0.0,\n",
       " 'columns': 0.0,\n",
       " 'com': 0.0,\n",
       " 'command': 0.0,\n",
       " 'commands': 0.0,\n",
       " 'commit': 0.0,\n",
       " 'common': 0.0,\n",
       " 'community': 0.0,\n",
       " 'compatibility': 0.0,\n",
       " 'compatible': 0.0,\n",
       " 'complete': 0.0,\n",
       " 'completed': 0.0,\n",
       " 'compose': 0.0,\n",
       " 'compute': 0.0,\n",
       " 'computer': 0.0,\n",
       " 'conda': 0.0,\n",
       " 'config': 0.0,\n",
       " 'configuration': 0.0,\n",
       " 'configure': 0.0,\n",
       " 'configured': 0.0,\n",
       " 'confirm': 0.0,\n",
       " 'connect': 0.0,\n",
       " 'connecting': 0.0,\n",
       " 'connection': 0.0,\n",
       " 'connector': 0.0,\n",
       " 'consider': 0.0,\n",
       " 'consistent': 0.0,\n",
       " 'console': 0.0,\n",
       " 'contain': 0.0,\n",
       " 'container': 0.0,\n",
       " 'containers': 0.0,\n",
       " 'contains': 0.0,\n",
       " 'content': 0.0,\n",
       " 'contents': 0.0,\n",
       " 'context': 0.0,\n",
       " 'continue': 0.0,\n",
       " 'control': 0.0,\n",
       " 'convert': 0.0,\n",
       " 'converted': 0.0,\n",
       " 'copy': 0.0,\n",
       " 'core': 0.0,\n",
       " 'correct': 0.0,\n",
       " 'correctly': 0.0,\n",
       " 'correlation': 0.0,\n",
       " 'corresponding': 0.0,\n",
       " 'couldn': 0.0,\n",
       " 'course': 0.49695797492447685,\n",
       " 'cp': 0.0,\n",
       " 'cpu': 0.0,\n",
       " 'create': 0.0,\n",
       " 'create_engine': 0.0,\n",
       " 'created': 0.0,\n",
       " 'creates': 0.0,\n",
       " 'creating': 0.0,\n",
       " 'creation': 0.0,\n",
       " 'credentials': 0.0,\n",
       " 'credit': 0.0,\n",
       " 'crucial': 0.0,\n",
       " 'csv': 0.0,\n",
       " 'ctrl': 0.0,\n",
       " 'curl': 0.0,\n",
       " 'current': 0.0,\n",
       " 'custom': 0.0,\n",
       " 'd37ci6vzurychx': 0.0,\n",
       " 'daemon': 0.0,\n",
       " 'daniel': 0.0,\n",
       " 'dashboard': 0.0,\n",
       " 'data': 0.0,\n",
       " 'database': 0.0,\n",
       " 'dataframe': 0.0,\n",
       " 'dataframes': 0.0,\n",
       " 'dataproc': 0.0,\n",
       " 'dataset': 0.0,\n",
       " 'datasets': 0.0,\n",
       " 'datatalks': 0.0,\n",
       " 'datatalksclub': 0.0,\n",
       " 'datatype': 0.0,\n",
       " 'date': 0.0,\n",
       " 'datetime': 0.0,\n",
       " 'david': 0.0,\n",
       " 'day': 0.0,\n",
       " 'db': 0.0,\n",
       " 'dbt': 0.0,\n",
       " 'dbt_project': 0.0,\n",
       " 'deadline': 0.0,\n",
       " 'dealing': 0.0,\n",
       " 'debian': 0.0,\n",
       " 'debug': 0.0,\n",
       " 'deep': 0.0,\n",
       " 'def': 0.0,\n",
       " 'default': 0.0,\n",
       " 'define': 0.0,\n",
       " 'defined': 0.0,\n",
       " 'definition': 0.0,\n",
       " 'delete': 0.0,\n",
       " 'deleted': 0.0,\n",
       " 'deleting': 0.0,\n",
       " 'denied': 0.0,\n",
       " 'dependencies': 0.0,\n",
       " 'dependency': 0.0,\n",
       " 'deploy': 0.0,\n",
       " 'deploying': 0.0,\n",
       " 'deployment': 0.0,\n",
       " 'deprecated': 0.0,\n",
       " 'description': 0.0,\n",
       " 'desktop': 0.0,\n",
       " 'detailed': 0.0,\n",
       " 'details': 0.0,\n",
       " 'dev': 0.0,\n",
       " 'development': 0.0,\n",
       " 'deviation': 0.0,\n",
       " 'device': 0.0,\n",
       " 'df': 0.0,\n",
       " 'df_train': 0.0,\n",
       " 'dict': 0.0,\n",
       " 'dictionary': 0.0,\n",
       " 'dictvectorizer': 0.0,\n",
       " 'did': 0.0,\n",
       " 'didn': 0.0,\n",
       " 'difference': 0.0,\n",
       " 'different': 0.0,\n",
       " 'dino': 0.0,\n",
       " 'dir': 0.0,\n",
       " 'directly': 0.0,\n",
       " 'directory': 0.0,\n",
       " 'disk': 0.0,\n",
       " 'distribution': 0.0,\n",
       " 'doc': 0.0,\n",
       " 'docker': 0.0,\n",
       " 'dockerfile': 0.0,\n",
       " 'docs': 0.0,\n",
       " 'document': 0.0,\n",
       " 'documentation': 0.0,\n",
       " 'does': 0.0,\n",
       " 'doesn': 0.0,\n",
       " 'doing': 0.0,\n",
       " 'dolocationid': 0.0,\n",
       " 'don': 0.0,\n",
       " 'dot': 0.0,\n",
       " 'double': 0.0,\n",
       " 'downgrade': 0.0,\n",
       " 'download': 0.0,\n",
       " 'downloaded': 0.0,\n",
       " 'downloading': 0.0,\n",
       " 'downloads': 0.0,\n",
       " 'dpage': 0.0,\n",
       " 'dragon': 0.0,\n",
       " 'drive': 0.0,\n",
       " 'drop': 0.0,\n",
       " 'dropoff_datetime': 0.0,\n",
       " 'dropping': 0.0,\n",
       " 'dt': 0.0,\n",
       " 'dtc': 0.0,\n",
       " 'duration': 0.0,\n",
       " 'dv': 0.0,\n",
       " 'easier': 0.0,\n",
       " 'easiest': 0.0,\n",
       " 'easily': 0.0,\n",
       " 'ec2': 0.0,\n",
       " 'echo': 0.0,\n",
       " 'ecr': 0.0,\n",
       " 'edit': 0.0,\n",
       " 'editing': 0.0,\n",
       " 'editor': 0.0,\n",
       " 'email': 0.0,\n",
       " 'en': 0.0,\n",
       " 'enable': 0.0,\n",
       " 'enabled': 0.0,\n",
       " 'encode': 0.0,\n",
       " 'encoding': 0.0,\n",
       " 'encounter': 0.0,\n",
       " 'encountered': 0.0,\n",
       " 'end': 0.0,\n",
       " 'endpoint': 0.0,\n",
       " 'engine': 0.0,\n",
       " 'engineering': 0.0,\n",
       " 'ensure': 0.0,\n",
       " 'enter': 0.0,\n",
       " 'entire': 0.0,\n",
       " 'entries': 0.0,\n",
       " 'env': 0.0,\n",
       " 'environ': 0.0,\n",
       " 'environment': 0.0,\n",
       " 'environments': 0.0,\n",
       " 'equal': 0.0,\n",
       " 'erick': 0.0,\n",
       " 'error': 0.0,\n",
       " 'errors': 0.0,\n",
       " 'escape': 0.0,\n",
       " 'especially': 0.0,\n",
       " 'eu': 0.0,\n",
       " 'europe': 0.0,\n",
       " 'evaluate': 0.0,\n",
       " 'evaluated': 0.0,\n",
       " 'evaluation': 0.0,\n",
       " 'exactly': 0.0,\n",
       " 'example': 0.0,\n",
       " 'exception': 0.0,\n",
       " 'exe': 0.0,\n",
       " 'exec': 0.0,\n",
       " 'executable': 0.0,\n",
       " 'execute': 0.0,\n",
       " 'executed': 0.0,\n",
       " 'executing': 0.0,\n",
       " 'execution': 0.0,\n",
       " 'exist': 0.0,\n",
       " 'existing': 0.0,\n",
       " 'exists': 0.0,\n",
       " 'exit': 0.0,\n",
       " 'expected': 0.0,\n",
       " 'experiment': 0.0,\n",
       " 'explanation': 0.0,\n",
       " 'explicitly': 0.0,\n",
       " 'export': 0.0,\n",
       " 'extension': 0.0,\n",
       " 'external': 0.0,\n",
       " 'extra': 0.0,\n",
       " 'extract': 0.0,\n",
       " 'face': 0.0,\n",
       " 'faced': 0.0,\n",
       " 'fact': 0.0,\n",
       " 'fact_trips': 0.0,\n",
       " 'fail': 0.0,\n",
       " 'failed': 0.0,\n",
       " 'failing': 0.0,\n",
       " 'fails': 0.0,\n",
       " 'false': 0.0,\n",
       " 'faq': 0.0,\n",
       " 'fatal': 0.0,\n",
       " 'feature': 0.0,\n",
       " 'features': 0.0,\n",
       " 'fetch': 0.0,\n",
       " 'fhv': 0.0,\n",
       " 'fhv_tripdata_2019': 0.0,\n",
       " 'field': 0.0,\n",
       " 'figure': 0.0,\n",
       " 'file': 0.0,\n",
       " 'files': 0.0,\n",
       " 'filesystem': 0.0,\n",
       " 'filter': 0.0,\n",
       " 'final': 0.0,\n",
       " 'finally': 0.0,\n",
       " 'findspark': 0.0,\n",
       " 'fine': 0.0,\n",
       " 'fit': 0.0,\n",
       " 'fix': 0.0,\n",
       " 'fixed': 0.0,\n",
       " 'flag': 0.0,\n",
       " 'flask': 0.0,\n",
       " 'float': 0.0,\n",
       " 'float64': 0.0,\n",
       " 'flow': 0.0,\n",
       " 'folder': 0.0,\n",
       " 'folders': 0.0,\n",
       " 'follow': 0.0,\n",
       " 'followed': 0.0,\n",
       " 'following': 0.0,\n",
       " 'follows': 0.0,\n",
       " 'forbidden': 0.0,\n",
       " 'force': 0.0,\n",
       " 'forget': 0.0,\n",
       " 'forgot': 0.0,\n",
       " 'form': 0.0,\n",
       " 'format': 0.0,\n",
       " 'forms': 0.0,\n",
       " 'forward': 0.0,\n",
       " 'fouesnard': 0.0,\n",
       " 'free': 0.0,\n",
       " 'fresh': 0.0,\n",
       " 'fs': 0.0,\n",
       " 'function': 0.0,\n",
       " 'functions': 0.0,\n",
       " 'future': 0.0,\n",
       " 'gateway': 0.0,\n",
       " 'gcloud': 0.0,\n",
       " 'gcp': 0.0,\n",
       " 'gcs': 0.0,\n",
       " 'generate': 0.0,\n",
       " 'generated': 0.0,\n",
       " 'generator': 0.0,\n",
       " 'george': 0.0,\n",
       " 'get_dummies': 0.0,\n",
       " 'get_feature_names_out': 0.0,\n",
       " 'getdbt': 0.0,\n",
       " 'getorcreate': 0.0,\n",
       " 'gets': 0.0,\n",
       " 'getting': 0.0,\n",
       " 'git': 0.0,\n",
       " 'github': 0.0,\n",
       " 'githubusercontent': 0.0,\n",
       " 'gitignore': 0.0,\n",
       " 'given': 0.0,\n",
       " 'gives': 0.0,\n",
       " 'gmail': 0.0,\n",
       " 'going': 0.0,\n",
       " 'good': 0.0,\n",
       " 'google': 0.0,\n",
       " 'google_application_credentials': 0.0,\n",
       " 'googleapi': 0.0,\n",
       " 'googleapis': 0.0,\n",
       " 'got': 0.0,\n",
       " 'gpu': 0.0,\n",
       " 'great': 0.0,\n",
       " 'green': 0.0,\n",
       " 'grigorev': 0.0,\n",
       " 'group': 0.0,\n",
       " 'gs': 0.0,\n",
       " 'guide': 0.0,\n",
       " 'gunicorn': 0.0,\n",
       " 'gz': 0.0,\n",
       " 'gzip': 0.0,\n",
       " 'hadoop': 0.0,\n",
       " 'hadoop3': 0.0,\n",
       " 'hand': 0.0,\n",
       " 'handle': 0.0,\n",
       " 'happen': 0.0,\n",
       " 'happens': 0.0,\n",
       " 'haven': 0.0,\n",
       " 'having': 0.0,\n",
       " 'head': 0.0,\n",
       " 'help': 0.0,\n",
       " 'helped': 0.0,\n",
       " 'helps': 0.0,\n",
       " 'high': 0.0,\n",
       " 'higher': 0.0,\n",
       " 'highly': 0.0,\n",
       " 'home': 0.0,\n",
       " 'homebrew': 0.0,\n",
       " 'homework': 0.0,\n",
       " 'homeworks': 0.0,\n",
       " 'hope': 0.0,\n",
       " 'host': 0.0,\n",
       " 'hostname': 0.0,\n",
       " 'hot': 0.0,\n",
       " 'hours': 0.0,\n",
       " 'housing': 0.0,\n",
       " 'hrithik': 0.0,\n",
       " 'html': 0.0,\n",
       " 'http': 0.0,\n",
       " 'https': 0.0,\n",
       " 'humberto': 0.0,\n",
       " 'hw': 0.0,\n",
       " 'hws': 0.0,\n",
       " 'iam': 0.0,\n",
       " 'icon': 0.0,\n",
       " 'id': 0.0,\n",
       " 'ide': 0.0,\n",
       " 'idea': 0.0,\n",
       " 'ignore': 0.0,\n",
       " 'image': 0.0,\n",
       " 'images': 0.0,\n",
       " 'immediately': 0.0,\n",
       " 'import': 0.0,\n",
       " 'importance': 0.0,\n",
       " 'important': 0.0,\n",
       " 'importerror': 0.0,\n",
       " 'importing': 0.0,\n",
       " 'include': 0.0,\n",
       " 'included': 0.0,\n",
       " 'including': 0.0,\n",
       " 'increase': 0.0,\n",
       " 'index': 0.0,\n",
       " 'info': 0.0,\n",
       " 'information': 0.0,\n",
       " 'init': 0.0,\n",
       " 'initial': 0.0,\n",
       " 'input': 0.0,\n",
       " 'inside': 0.0,\n",
       " 'install': 0.0,\n",
       " 'installation': 0.0,\n",
       " 'installed': 0.0,\n",
       " 'installing': 0.0,\n",
       " 'instance': 0.0,\n",
       " 'instances': 0.0,\n",
       " 'instead': 0.0,\n",
       " 'instruction': 0.0,\n",
       " 'instructions': 0.0,\n",
       " 'insufficient': 0.0,\n",
       " 'int': 0.0,\n",
       " 'int64': 0.0,\n",
       " 'integer': 0.0,\n",
       " 'interesting': 0.0,\n",
       " 'interface': 0.0,\n",
       " 'internal': 0.0,\n",
       " 'internet': 0.0,\n",
       " 'interpreter': 0.0,\n",
       " 'intro': 0.0,\n",
       " 'invalid': 0.0,\n",
       " 'io': 0.0,\n",
       " 'ip': 0.0,\n",
       " 'ipynb': 0.0,\n",
       " 'isn': 0.0,\n",
       " 'issue': 0.0,\n",
       " 'issues': 0.0,\n",
       " 'iteration': 0.0,\n",
       " 'ivan': 0.0,\n",
       " 'jan': 0.0,\n",
       " 'jar': 0.0,\n",
       " 'java': 0.0,\n",
       " 'java_home': 0.0,\n",
       " 'job': 0.0,\n",
       " 'jobs': 0.0,\n",
       " 'join': 0.7433502942325737,\n",
       " 'json': 0.0,\n",
       " 'jupyter': 0.0,\n",
       " 'just': 0.4477310702009737,\n",
       " 'kafka': 0.0,\n",
       " 'kaggle': 0.0,\n",
       " 'keras': 0.0,\n",
       " 'kernel': 0.0,\n",
       " 'key': 0.0,\n",
       " 'keys': 0.0,\n",
       " 'kill': 0.0,\n",
       " 'kind': 0.0,\n",
       " 'kinesis': 0.0,\n",
       " 'kniazeva': 0.0,\n",
       " 'know': 0.0,\n",
       " 'known': 0.0,\n",
       " 'konrad': 0.0,\n",
       " 'krishna': 0.0,\n",
       " 'kubectl': 0.0,\n",
       " 'kubernetes': 0.0,\n",
       " 'kumar': 0.0,\n",
       " 'kwargs': 0.0,\n",
       " 'lambda': 0.0,\n",
       " 'laptop': 0.0,\n",
       " 'large': 0.0,\n",
       " 'larkin': 0.0,\n",
       " 'later': 0.0,\n",
       " 'latest': 0.0,\n",
       " 'launch': 0.0,\n",
       " 'layer': 0.0,\n",
       " 'layers': 0.0,\n",
       " 'lead': 0.0,\n",
       " 'leaderboard': 0.0,\n",
       " 'learn': 0.0,\n",
       " 'learning': 0.0,\n",
       " 'left': 0.0,\n",
       " 'length': 0.0,\n",
       " 'lesson': 0.0,\n",
       " 'lessons': 0.0,\n",
       " 'let': 0.0,\n",
       " 'level': 0.0,\n",
       " 'lib': 0.0,\n",
       " 'libraries': 0.0,\n",
       " 'library': 0.0,\n",
       " 'like': 0.0,\n",
       " 'likely': 0.0,\n",
       " 'line': 0.0,\n",
       " 'linear': 0.0,\n",
       " 'lines': 0.0,\n",
       " 'link': 0.0,\n",
       " 'links': 0.0,\n",
       " 'linux': 0.0,\n",
       " 'list': 0.0,\n",
       " 'listed': 0.0,\n",
       " 'litvinov': 0.0,\n",
       " 'live': 0.0,\n",
       " 'll': 0.0,\n",
       " 'load': 0.0,\n",
       " 'loaded': 0.0,\n",
       " 'loading': 0.0,\n",
       " 'local': 0.0,\n",
       " 'localhost': 0.0,\n",
       " 'locally': 0.0,\n",
       " 'located': 0.0,\n",
       " 'location': 0.0,\n",
       " 'locations': 0.0,\n",
       " 'lock': 0.0,\n",
       " 'log': 0.0,\n",
       " 'logged': 0.0,\n",
       " 'login': 0.0,\n",
       " 'logistic': 0.0,\n",
       " 'logs': 0.0,\n",
       " 'long': 0.0,\n",
       " 'longer': 0.0,\n",
       " 'look': 0.0,\n",
       " 'looking': 0.0,\n",
       " 'looks': 0.0,\n",
       " 'loop': 0.0,\n",
       " 'lot': 0.0,\n",
       " 'lower': 0.0,\n",
       " 'ls': 0.0,\n",
       " 'lukafiardi': 0.0,\n",
       " 'mac': 0.0,\n",
       " 'machine': 0.0,\n",
       " 'macos': 0.0,\n",
       " 'mage': 0.0,\n",
       " 'magic': 0.0,\n",
       " 'main': 0.0,\n",
       " 'make': 0.0,\n",
       " 'makes': 0.0,\n",
       " 'making': 0.0,\n",
       " 'manage': 0.0,\n",
       " 'manually': 0.0,\n",
       " 'map': 0.0,\n",
       " 'marcos': 0.0,\n",
       " 'marcosmjd': 0.0,\n",
       " 'martin': 0.0,\n",
       " 'master': 0.0,\n",
       " 'match': 0.0,\n",
       " 'matplotlib': 0.0,\n",
       " 'matrix': 0.0,\n",
       " 'max': 0.0,\n",
       " 'maybe': 0.0,\n",
       " 'md': 0.0,\n",
       " 'mean': 0.0,\n",
       " 'mean_squared_error': 0.0,\n",
       " 'means': 0.0,\n",
       " 'media': 0.0,\n",
       " 'median_house_value': 0.0,\n",
       " 'memoona': 0.0,\n",
       " 'memory': 0.0,\n",
       " 'mentioned': 0.0,\n",
       " 'merge': 0.0,\n",
       " 'message': 0.0,\n",
       " 'metadata': 0.0,\n",
       " 'method': 0.0,\n",
       " 'metrics': 0.0,\n",
       " 'microsoft': 0.0,\n",
       " 'midterm': 0.0,\n",
       " 'minutes': 0.0,\n",
       " 'missing': 0.0,\n",
       " 'mkdir': 0.0,\n",
       " 'ml': 0.0,\n",
       " 'mlbookcamp': 0.0,\n",
       " 'mlflow': 0.0,\n",
       " 'mlops': 0.0,\n",
       " 'mode': 0.0,\n",
       " 'model': 0.0,\n",
       " 'models': 0.0,\n",
       " 'modified': 0.0,\n",
       " 'modify': 0.0,\n",
       " 'module': 0.0,\n",
       " 'modulenotfounderror': 0.0,\n",
       " 'modules': 0.0,\n",
       " 'month': 0.0,\n",
       " 'mount': 0.0,\n",
       " 'moved': 0.0,\n",
       " 'multiple': 0.0,\n",
       " 'multiplication': 0.0,\n",
       " 'mélanie': 0.0,\n",
       " 'named': 0.0,\n",
       " 'names': 0.0,\n",
       " 'nan': 0.0,\n",
       " 'nano': 0.0,\n",
       " 'native': 0.0,\n",
       " 'navigate': 0.0,\n",
       " 'ndarray': 0.0,\n",
       " 'necessary': 0.0,\n",
       " 'need': 0.0,\n",
       " 'needed': 0.0,\n",
       " 'needs': 0.0,\n",
       " 'negative': 0.0,\n",
       " 'net': 0.0,\n",
       " 'network': 0.0,\n",
       " 'networks': 0.0,\n",
       " 'new': 0.0,\n",
       " 'newly': 0.0,\n",
       " 'non': 0.0,\n",
       " 'normal': 0.0,\n",
       " 'note': 0.0,\n",
       " 'notebook': 0.0,\n",
       " 'notebooks': 0.0,\n",
       " 'notes': 0.0,\n",
       " 'notice': 0.0,\n",
       " 'np': 0.0,\n",
       " 'nukta': 0.0,\n",
       " 'null': 0.0,\n",
       " 'number': 0.0,\n",
       " 'numeric': 0.0,\n",
       " 'numerical': 0.0,\n",
       " 'numpy': 0.0,\n",
       " 'ny': 0.0,\n",
       " 'ny_taxi': 0.0,\n",
       " 'ny_taxi_postgres_data': 0.0,\n",
       " 'nyc': 0.0,\n",
       " 'object': 0.0,\n",
       " 'observations': 0.0,\n",
       " 'occur': 0.0,\n",
       " 'occurred': 0.0,\n",
       " 'occurs': 0.0,\n",
       " 'odimegwu': 0.0,\n",
       " 'office': 0.0,\n",
       " 'official': 0.0,\n",
       " 'ok': 0.0,\n",
       " 'old': 0.0,\n",
       " 'older': 0.0,\n",
       " 'olga': 0.0,\n",
       " 'onehotencoder': 0.0,\n",
       " 'ones': 0.0,\n",
       " 'open': 0.0,\n",
       " 'operation': 0.0,\n",
       " 'opt': 0.0,\n",
       " 'option': 0.0,\n",
       " 'optional': 0.0,\n",
       " 'options': 0.0,\n",
       " 'order': 0.0,\n",
       " 'org': 0.0,\n",
       " 'origin': 0.0,\n",
       " 'original': 0.0,\n",
       " 'os': 0.0,\n",
       " 'output': 0.0,\n",
       " 'outputs': 0.0,\n",
       " 'overfitting': 0.0,\n",
       " 'owner': 0.0,\n",
       " 'package': 0.0,\n",
       " 'packages': 0.0,\n",
       " 'page': 0.0,\n",
       " 'pandas': 0.0,\n",
       " 'parameter': 0.0,\n",
       " 'parameters': 0.0,\n",
       " 'params': 0.0,\n",
       " 'parquet': 0.0,\n",
       " 'parse': 0.0,\n",
       " 'particular': 0.0,\n",
       " 'particularly': 0.0,\n",
       " 'pass': 0.0,\n",
       " 'passed': 0.0,\n",
       " 'passing': 0.0,\n",
       " 'password': 0.0,\n",
       " 'paste': 0.0,\n",
       " 'pastor': 0.0,\n",
       " 'path': 0.0,\n",
       " 'paths': 0.0,\n",
       " 'pay': 0.0,\n",
       " 'payment_type': 0.0,\n",
       " 'pc': 0.0,\n",
       " 'pd': 0.0,\n",
       " 'peer': 0.0,\n",
       " 'perform': 0.0,\n",
       " 'performance': 0.0,\n",
       " 'period': 0.0,\n",
       " 'permission': 0.0,\n",
       " 'permissions': 0.0,\n",
       " 'pg': 0.0,\n",
       " 'pgadmin': 0.0,\n",
       " 'pgadmin4': 0.0,\n",
       " 'pgadmin_default_email': 0.0,\n",
       " 'pgadmin_default_password': 0.0,\n",
       " 'pgcli': 0.0,\n",
       " 'pgdatabase': 0.0,\n",
       " 'pick': 0.0,\n",
       " 'pickup_datetime': 0.0,\n",
       " 'pip': 0.0,\n",
       " 'pipeline': 0.0,\n",
       " 'pipenv': 0.0,\n",
       " 'pipfile': 0.0,\n",
       " 'pl3mmuxubc_hihxl5ji8t4o6lpaophaclr': 0.0,\n",
       " 'place': 0.0,\n",
       " 'platform': 0.0,\n",
       " 'platforms': 0.0,\n",
       " 'plt': 0.0,\n",
       " 'point': 0.0,\n",
       " 'points': 0.0,\n",
       " 'port': 0.0,\n",
       " 'ports': 0.0,\n",
       " 'position': 0.0,\n",
       " 'possible': 0.0,\n",
       " 'post': 0.0,\n",
       " 'postgres': 0.0,\n",
       " 'postgres_db': 0.0,\n",
       " 'postgres_password': 0.0,\n",
       " 'postgres_user': 0.0,\n",
       " 'postgresql': 0.0,\n",
       " 'powershell': 0.0,\n",
       " 'pq': 0.0,\n",
       " 'practice': 0.0,\n",
       " 'pre': 0.0,\n",
       " 'precision': 0.0,\n",
       " 'predict': 0.0,\n",
       " 'prediction': 0.0,\n",
       " 'predictions': 0.0,\n",
       " 'prefect': 0.0,\n",
       " 'prefer': 0.0,\n",
       " 'present': 0.0,\n",
       " 'press': 0.0,\n",
       " 'pretty': 0.0,\n",
       " 'prevent': 0.0,\n",
       " 'previous': 0.0,\n",
       " 'previously': 0.0,\n",
       " 'print': 0.0,\n",
       " 'private': 0.0,\n",
       " 'probably': 0.0,\n",
       " 'problem': 0.0,\n",
       " 'problems': 0.0,\n",
       " 'proceed': 0.0,\n",
       " 'process': 0.0,\n",
       " 'processes': 0.0,\n",
       " 'production': 0.0,\n",
       " 'profile': 0.0,\n",
       " 'program': 0.0,\n",
       " 'programs': 0.0,\n",
       " 'progress': 0.0,\n",
       " 'project': 0.0,\n",
       " 'project_id': 0.0,\n",
       " 'projects': 0.0,\n",
       " 'prompt': 0.0,\n",
       " 'properly': 0.0,\n",
       " 'protobuf': 0.0,\n",
       " 'provide': 0.0,\n",
       " 'provided': 0.0,\n",
       " 'provides': 0.0,\n",
       " 'ps': 0.0,\n",
       " 'psql': 0.0,\n",
       " 'psycopg2': 0.0,\n",
       " 'pubhtml': 0.0,\n",
       " 'public': 0.0,\n",
       " 'pull': 0.0,\n",
       " 'pulocationid': 0.0,\n",
       " 'purpose': 0.0,\n",
       " 'push': 0.0,\n",
       " 'pwd': 0.0,\n",
       " 'py': 0.0,\n",
       " 'py4j': 0.0,\n",
       " 'pyarrow': 0.0,\n",
       " 'pyspark': 0.0,\n",
       " 'python': 0.0,\n",
       " 'python3': 0.0,\n",
       " 'pythonpath': 0.0,\n",
       " 'queries': 0.0,\n",
       " 'query': 0.0,\n",
       " 'question': 0.0,\n",
       " 'questions': 0.0,\n",
       " 'quinn': 0.0,\n",
       " 'quite': 0.0,\n",
       " 'quotes': 0.0,\n",
       " 'radikal': 0.0,\n",
       " 'raised': 0.0,\n",
       " 'ram': 0.0,\n",
       " 'ran': 0.0,\n",
       " 'random': 0.0,\n",
       " 'random_state': 0.0,\n",
       " 'range': 0.0,\n",
       " 'raw': 0.0,\n",
       " 'read': 0.0,\n",
       " 'read_csv': 0.0,\n",
       " 'read_parquet': 0.0,\n",
       " 'reading': 0.0,\n",
       " 'ready': 0.0,\n",
       " 'real': 0.0,\n",
       " 'really': 0.0,\n",
       " 'reason': 0.0,\n",
       " 'recall': 0.0,\n",
       " 'receive': 0.0,\n",
       " 'recent': 0.0,\n",
       " 'recognized': 0.0,\n",
       " 'recommend': 0.0,\n",
       " 'recommended': 0.0,\n",
       " 'record': 0.0,\n",
       " 'recorded': 0.0,\n",
       " 'records': 0.0,\n",
       " 'recreate': 0.0,\n",
       " 'refer': 0.0,\n",
       " 'reference': 0.0,\n",
       " 'regarding': 0.0,\n",
       " 'region': 0.0,\n",
       " 'register': 0.0,\n",
       " 'registry': 0.0,\n",
       " 'regression': 0.0,\n",
       " 'regularization': 0.0,\n",
       " 'reinstall': 0.0,\n",
       " 'related': 0.0,\n",
       " 'releases': 0.0,\n",
       " 'remember': 0.0,\n",
       " 'remote': 0.0,\n",
       " 'remove': 0.0,\n",
       " 'removed': 0.0,\n",
       " 'removing': 0.0,\n",
       " 'rename': 0.0,\n",
       " 'replace': 0.0,\n",
       " 'replaced': 0.0,\n",
       " ...}"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query_dict = dict(zip(names, q.toarray()[0]))\n",
    "query_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "6cbfd481-d9f7-4c34-8695-6b169f8e43fa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['01', '02', '03', ..., 'youtube', 'zip', 'zoomcamp'], dtype=object)"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "37109656-23ea-4667-a286-27c0c0efafa5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0., 0., 0., ..., 0., 0., 0.])"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q.toarray()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "5494f594-4b4f-402f-840a-d803c1750444",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'01': 0.0,\n",
       " '02': 0.0,\n",
       " '03': 0.0,\n",
       " '04': 0.0,\n",
       " '05': 0.0,\n",
       " '06': 0.0,\n",
       " '09': 0.0,\n",
       " '10': 0.0,\n",
       " '100': 0.0,\n",
       " '11': 0.0,\n",
       " '12': 0.0,\n",
       " '127': 0.0,\n",
       " '13': 0.0,\n",
       " '14': 0.0,\n",
       " '15': 0.0,\n",
       " '16': 0.0,\n",
       " '17': 0.0,\n",
       " '19': 0.0,\n",
       " '1st': 0.0,\n",
       " '20': 0.0,\n",
       " '2019': 0.0,\n",
       " '2020': 0.0,\n",
       " '2021': 0.0,\n",
       " '2022': 0.0,\n",
       " '2023': 0.0,\n",
       " '2024': 0.0,\n",
       " '21': 0.0,\n",
       " '22': 0.0,\n",
       " '24': 0.0,\n",
       " '25': 0.0,\n",
       " '2pacx': 0.0,\n",
       " '30': 0.0,\n",
       " '35': 0.0,\n",
       " '403': 0.0,\n",
       " '42': 0.0,\n",
       " '50': 0.0,\n",
       " '5000': 0.0,\n",
       " '5431': 0.0,\n",
       " '5432': 0.0,\n",
       " '60': 0.0,\n",
       " '600': 0.0,\n",
       " '7077': 0.0,\n",
       " '80': 0.0,\n",
       " '8080': 0.0,\n",
       " '8888': 0.0,\n",
       " '9696': 0.0,\n",
       " 'abhijit': 0.0,\n",
       " 'able': 0.0,\n",
       " 'abolade': 0.0,\n",
       " 'absolute': 0.0,\n",
       " 'accept': 0.0,\n",
       " 'access': 0.0,\n",
       " 'accordingly': 0.0,\n",
       " 'account': 0.0,\n",
       " 'accuracy': 0.0,\n",
       " 'action': 0.0,\n",
       " 'activate': 0.0,\n",
       " 'actual': 0.0,\n",
       " 'actually': 0.0,\n",
       " 'add': 0.0,\n",
       " 'added': 0.0,\n",
       " 'adding': 0.0,\n",
       " 'addition': 0.0,\n",
       " 'additional': 0.0,\n",
       " 'additionally': 0.0,\n",
       " 'address': 0.0,\n",
       " 'admin': 0.0,\n",
       " 'advani': 0.0,\n",
       " 'ahmed': 0.0,\n",
       " 'ai': 0.0,\n",
       " 'airflow': 0.0,\n",
       " 'alejandro': 0.0,\n",
       " 'alena': 0.0,\n",
       " 'alex': 0.0,\n",
       " 'alexey': 0.0,\n",
       " 'alexeygrigorev': 0.0,\n",
       " 'allocate': 0.0,\n",
       " 'allow': 0.0,\n",
       " 'allowed': 0.0,\n",
       " 'allowing': 0.0,\n",
       " 'allows': 0.0,\n",
       " 'alpha': 0.0,\n",
       " 'alternative': 0.0,\n",
       " 'alternatively': 0.0,\n",
       " 'amazon': 0.0,\n",
       " 'amazonaws': 0.0,\n",
       " 'amd64': 0.0,\n",
       " 'aminat': 0.0,\n",
       " 'anaconda': 0.0,\n",
       " 'anaconda3': 0.0,\n",
       " 'analytics': 0.0,\n",
       " 'anand': 0.0,\n",
       " 'andrii': 0.0,\n",
       " 'ans': 0.0,\n",
       " 'answer': 0.0,\n",
       " 'answered': 0.0,\n",
       " 'answers': 0.0,\n",
       " 'apache': 0.0,\n",
       " 'api': 0.0,\n",
       " 'aponte': 0.0,\n",
       " 'app': 0.0,\n",
       " 'appear': 0.0,\n",
       " 'appears': 0.0,\n",
       " 'append': 0.0,\n",
       " 'application': 0.0,\n",
       " 'apply': 0.0,\n",
       " 'appname': 0.0,\n",
       " 'approach': 0.0,\n",
       " 'apt': 0.0,\n",
       " 'architecture': 0.0,\n",
       " 'archives': 0.0,\n",
       " 'args': 0.0,\n",
       " 'argument': 0.0,\n",
       " 'array': 0.0,\n",
       " 'arrays': 0.0,\n",
       " 'article': 0.0,\n",
       " 'asia': 0.0,\n",
       " 'ask': 0.0,\n",
       " 'asked': 0.0,\n",
       " 'asks': 0.0,\n",
       " 'assigned': 0.0,\n",
       " 'astype': 0.0,\n",
       " 'attempting': 0.0,\n",
       " 'attribute': 0.0,\n",
       " 'attributeerror': 0.0,\n",
       " 'auc': 0.0,\n",
       " 'auth': 0.0,\n",
       " 'authenticate': 0.0,\n",
       " 'authentication': 0.0,\n",
       " 'auto': 0.0,\n",
       " 'automatically': 0.0,\n",
       " 'available': 0.0,\n",
       " 'average': 0.0,\n",
       " 'avila': 0.0,\n",
       " 'avoid': 0.0,\n",
       " 'aws': 0.0,\n",
       " 'azure': 0.0,\n",
       " 'backend': 0.0,\n",
       " 'bad': 0.0,\n",
       " 'bar': 0.0,\n",
       " 'base': 0.0,\n",
       " 'based': 0.0,\n",
       " 'bash': 0.0,\n",
       " 'bashrc': 0.0,\n",
       " 'basically': 0.0,\n",
       " 'batch': 0.0,\n",
       " 'best': 0.0,\n",
       " 'better': 0.0,\n",
       " 'bhaskar': 0.0,\n",
       " 'bhatia': 0.0,\n",
       " 'big': 0.0,\n",
       " 'bigquery': 0.0,\n",
       " 'billing': 0.0,\n",
       " 'bin': 0.0,\n",
       " 'binary': 0.0,\n",
       " 'bind': 0.0,\n",
       " 'bit': 0.0,\n",
       " 'blob': 0.0,\n",
       " 'block': 0.0,\n",
       " 'blog': 0.0,\n",
       " 'box': 0.0,\n",
       " 'bq': 0.0,\n",
       " 'branch': 0.0,\n",
       " 'brew': 0.0,\n",
       " 'brigida': 0.0,\n",
       " 'browser': 0.0,\n",
       " 'bucket': 0.0,\n",
       " 'build': 0.0,\n",
       " 'builder': 0.0,\n",
       " 'building': 0.0,\n",
       " 'built': 0.0,\n",
       " 'calculate': 0.0,\n",
       " 'calderin': 0.0,\n",
       " 'called': 0.0,\n",
       " 'calling': 0.0,\n",
       " 'capstone': 0.0,\n",
       " 'capture': 0.0,\n",
       " 'care': 0.0,\n",
       " 'careful': 0.0,\n",
       " 'case': 0.0,\n",
       " 'cast': 0.0,\n",
       " 'cat': 0.0,\n",
       " 'categorical': 0.0,\n",
       " 'cause': 0.0,\n",
       " 'caused': 0.0,\n",
       " 'causing': 0.0,\n",
       " 'cd': 0.0,\n",
       " 'cell': 0.0,\n",
       " 'certificate': 0.0,\n",
       " 'chakraborty': 0.0,\n",
       " 'change': 0.0,\n",
       " 'changed': 0.0,\n",
       " 'changes': 0.0,\n",
       " 'changing': 0.0,\n",
       " 'channel': 0.0,\n",
       " 'check': 0.0,\n",
       " 'checking': 0.0,\n",
       " 'chizhmak': 0.0,\n",
       " 'chmod': 0.0,\n",
       " 'choice': 0.0,\n",
       " 'choose': 0.0,\n",
       " 'chown': 0.0,\n",
       " 'churn': 0.0,\n",
       " 'ci': 0.0,\n",
       " 'class': 0.0,\n",
       " 'classes': 0.0,\n",
       " 'classification': 0.0,\n",
       " 'clear': 0.0,\n",
       " 'cli': 0.0,\n",
       " 'click': 0.0,\n",
       " 'client': 0.0,\n",
       " 'clone': 0.0,\n",
       " 'close': 0.0,\n",
       " 'closed': 0.0,\n",
       " 'closest': 0.0,\n",
       " 'clothing': 0.0,\n",
       " 'cloud': 0.0,\n",
       " 'cloudfront': 0.0,\n",
       " 'club': 0.0,\n",
       " 'cluster': 0.0,\n",
       " 'cmd': 0.0,\n",
       " 'code': 0.0,\n",
       " 'codespaces': 0.0,\n",
       " 'coding': 0.0,\n",
       " 'cohort': 0.0,\n",
       " 'cohorts': 0.0,\n",
       " 'col': 0.0,\n",
       " 'colab': 0.0,\n",
       " 'column': 0.0,\n",
       " 'columns': 0.0,\n",
       " 'com': 0.0,\n",
       " 'command': 0.0,\n",
       " 'commands': 0.0,\n",
       " 'commit': 0.0,\n",
       " 'common': 0.0,\n",
       " 'community': 0.0,\n",
       " 'compatibility': 0.0,\n",
       " 'compatible': 0.0,\n",
       " 'complete': 0.0,\n",
       " 'completed': 0.0,\n",
       " 'compose': 0.0,\n",
       " 'compute': 0.0,\n",
       " 'computer': 0.0,\n",
       " 'conda': 0.0,\n",
       " 'config': 0.0,\n",
       " 'configuration': 0.0,\n",
       " 'configure': 0.0,\n",
       " 'configured': 0.0,\n",
       " 'confirm': 0.0,\n",
       " 'connect': 0.0,\n",
       " 'connecting': 0.0,\n",
       " 'connection': 0.0,\n",
       " 'connector': 0.0,\n",
       " 'consider': 0.0,\n",
       " 'consistent': 0.0,\n",
       " 'console': 0.0,\n",
       " 'contain': 0.0,\n",
       " 'container': 0.0,\n",
       " 'containers': 0.0,\n",
       " 'contains': 0.0,\n",
       " 'content': 0.0,\n",
       " 'contents': 0.0,\n",
       " 'context': 0.0,\n",
       " 'continue': 0.0,\n",
       " 'control': 0.0,\n",
       " 'convert': 0.0,\n",
       " 'converted': 0.0,\n",
       " 'copy': 0.0,\n",
       " 'core': 0.0,\n",
       " 'correct': 0.0,\n",
       " 'correctly': 0.0,\n",
       " 'correlation': 0.0,\n",
       " 'corresponding': 0.0,\n",
       " 'couldn': 0.0,\n",
       " 'course': 0.0,\n",
       " 'cp': 0.0,\n",
       " 'cpu': 0.0,\n",
       " 'create': 0.0,\n",
       " 'create_engine': 0.0,\n",
       " 'created': 0.0,\n",
       " 'creates': 0.0,\n",
       " 'creating': 0.0,\n",
       " 'creation': 0.0,\n",
       " 'credentials': 0.0,\n",
       " 'credit': 0.0,\n",
       " 'crucial': 0.0,\n",
       " 'csv': 0.0,\n",
       " 'ctrl': 0.0,\n",
       " 'curl': 0.0,\n",
       " 'current': 0.0,\n",
       " 'custom': 0.0,\n",
       " 'd37ci6vzurychx': 0.0,\n",
       " 'daemon': 0.0,\n",
       " 'daniel': 0.0,\n",
       " 'dashboard': 0.0,\n",
       " 'data': 0.3127766226016382,\n",
       " 'database': 0.0,\n",
       " 'dataframe': 0.0,\n",
       " 'dataframes': 0.0,\n",
       " 'dataproc': 0.0,\n",
       " 'dataset': 0.0,\n",
       " 'datasets': 0.0,\n",
       " 'datatalks': 0.0,\n",
       " 'datatalksclub': 0.5316383823591385,\n",
       " 'datatype': 0.0,\n",
       " 'date': 0.0,\n",
       " 'datetime': 0.0,\n",
       " 'david': 0.0,\n",
       " 'day': 0.0,\n",
       " 'db': 0.0,\n",
       " 'dbt': 0.0,\n",
       " 'dbt_project': 0.0,\n",
       " 'deadline': 0.0,\n",
       " 'dealing': 0.0,\n",
       " 'debian': 0.0,\n",
       " 'debug': 0.0,\n",
       " 'deep': 0.0,\n",
       " 'def': 0.0,\n",
       " 'default': 0.0,\n",
       " 'define': 0.0,\n",
       " 'defined': 0.0,\n",
       " 'definition': 0.0,\n",
       " 'delete': 0.0,\n",
       " 'deleted': 0.0,\n",
       " 'deleting': 0.0,\n",
       " 'denied': 0.0,\n",
       " 'dependencies': 0.0,\n",
       " 'dependency': 0.0,\n",
       " 'deploy': 0.0,\n",
       " 'deploying': 0.0,\n",
       " 'deployment': 0.0,\n",
       " 'deprecated': 0.0,\n",
       " 'description': 0.0,\n",
       " 'desktop': 0.0,\n",
       " 'detailed': 0.0,\n",
       " 'details': 0.0,\n",
       " 'dev': 0.0,\n",
       " 'development': 0.0,\n",
       " 'deviation': 0.0,\n",
       " 'device': 0.0,\n",
       " 'df': 0.0,\n",
       " 'df_train': 0.0,\n",
       " 'dict': 0.0,\n",
       " 'dictionary': 0.0,\n",
       " 'dictvectorizer': 0.0,\n",
       " 'did': 0.0,\n",
       " 'didn': 0.0,\n",
       " 'difference': 0.0,\n",
       " 'different': 0.0,\n",
       " 'dino': 0.0,\n",
       " 'dir': 0.0,\n",
       " 'directly': 0.0,\n",
       " 'directory': 0.0,\n",
       " 'disk': 0.0,\n",
       " 'distribution': 0.0,\n",
       " 'doc': 0.0,\n",
       " 'docker': 0.0,\n",
       " 'dockerfile': 0.0,\n",
       " 'docs': 0.0,\n",
       " 'document': 0.0,\n",
       " 'documentation': 0.0,\n",
       " 'does': 0.0,\n",
       " 'doesn': 0.0,\n",
       " 'doing': 0.0,\n",
       " 'dolocationid': 0.0,\n",
       " 'don': 0.0,\n",
       " 'dot': 0.0,\n",
       " 'double': 0.0,\n",
       " 'downgrade': 0.0,\n",
       " 'download': 0.0,\n",
       " 'downloaded': 0.0,\n",
       " 'downloading': 0.0,\n",
       " 'downloads': 0.0,\n",
       " 'dpage': 0.0,\n",
       " 'dragon': 0.0,\n",
       " 'drive': 0.0,\n",
       " 'drop': 0.0,\n",
       " 'dropoff_datetime': 0.0,\n",
       " 'dropping': 0.0,\n",
       " 'dt': 0.0,\n",
       " 'dtc': 0.0,\n",
       " 'duration': 0.0,\n",
       " 'dv': 0.0,\n",
       " 'easier': 0.0,\n",
       " 'easiest': 0.0,\n",
       " 'easily': 0.0,\n",
       " 'ec2': 0.0,\n",
       " 'echo': 0.0,\n",
       " 'ecr': 0.0,\n",
       " 'edit': 0.0,\n",
       " 'editing': 0.0,\n",
       " 'editor': 0.0,\n",
       " 'email': 0.0,\n",
       " 'en': 0.0,\n",
       " 'enable': 0.0,\n",
       " 'enabled': 0.0,\n",
       " 'encode': 0.0,\n",
       " 'encoding': 0.0,\n",
       " 'encounter': 0.0,\n",
       " 'encountered': 0.0,\n",
       " 'end': 0.0,\n",
       " 'endpoint': 0.0,\n",
       " 'engine': 0.0,\n",
       " 'engineering': 0.5401030373894639,\n",
       " 'ensure': 0.0,\n",
       " 'enter': 0.0,\n",
       " 'entire': 0.0,\n",
       " 'entries': 0.0,\n",
       " 'env': 0.0,\n",
       " 'environ': 0.0,\n",
       " 'environment': 0.0,\n",
       " 'environments': 0.0,\n",
       " 'equal': 0.0,\n",
       " 'erick': 0.0,\n",
       " 'error': 0.0,\n",
       " 'errors': 0.0,\n",
       " 'escape': 0.0,\n",
       " 'especially': 0.0,\n",
       " 'eu': 0.0,\n",
       " 'europe': 0.0,\n",
       " 'evaluate': 0.0,\n",
       " 'evaluated': 0.0,\n",
       " 'evaluation': 0.0,\n",
       " 'exactly': 0.0,\n",
       " 'example': 0.0,\n",
       " 'exception': 0.0,\n",
       " 'exe': 0.0,\n",
       " 'exec': 0.0,\n",
       " 'executable': 0.0,\n",
       " 'execute': 0.0,\n",
       " 'executed': 0.0,\n",
       " 'executing': 0.0,\n",
       " 'execution': 0.0,\n",
       " 'exist': 0.0,\n",
       " 'existing': 0.0,\n",
       " 'exists': 0.0,\n",
       " 'exit': 0.0,\n",
       " 'expected': 0.0,\n",
       " 'experiment': 0.0,\n",
       " 'explanation': 0.0,\n",
       " 'explicitly': 0.0,\n",
       " 'export': 0.0,\n",
       " 'extension': 0.0,\n",
       " 'external': 0.0,\n",
       " 'extra': 0.0,\n",
       " 'extract': 0.0,\n",
       " 'face': 0.0,\n",
       " 'faced': 0.0,\n",
       " 'fact': 0.0,\n",
       " 'fact_trips': 0.0,\n",
       " 'fail': 0.0,\n",
       " 'failed': 0.0,\n",
       " 'failing': 0.0,\n",
       " 'fails': 0.0,\n",
       " 'false': 0.0,\n",
       " 'faq': 0.0,\n",
       " 'fatal': 0.0,\n",
       " 'feature': 0.0,\n",
       " 'features': 0.0,\n",
       " 'fetch': 0.0,\n",
       " 'fhv': 0.0,\n",
       " 'fhv_tripdata_2019': 0.0,\n",
       " 'field': 0.0,\n",
       " 'figure': 0.0,\n",
       " 'file': 0.0,\n",
       " 'files': 0.0,\n",
       " 'filesystem': 0.0,\n",
       " 'filter': 0.0,\n",
       " 'final': 0.0,\n",
       " 'finally': 0.0,\n",
       " 'findspark': 0.0,\n",
       " 'fine': 0.0,\n",
       " 'fit': 0.0,\n",
       " 'fix': 0.0,\n",
       " 'fixed': 0.0,\n",
       " 'flag': 0.0,\n",
       " 'flask': 0.0,\n",
       " 'float': 0.0,\n",
       " 'float64': 0.0,\n",
       " 'flow': 0.0,\n",
       " 'folder': 0.0,\n",
       " 'folders': 0.0,\n",
       " 'follow': 0.0,\n",
       " 'followed': 0.0,\n",
       " 'following': 0.0,\n",
       " 'follows': 0.0,\n",
       " 'forbidden': 0.0,\n",
       " 'force': 0.0,\n",
       " 'forget': 0.0,\n",
       " 'forgot': 0.0,\n",
       " 'form': 0.0,\n",
       " 'format': 0.0,\n",
       " 'forms': 0.0,\n",
       " 'forward': 0.0,\n",
       " 'fouesnard': 0.0,\n",
       " 'free': 0.0,\n",
       " 'fresh': 0.0,\n",
       " 'fs': 0.0,\n",
       " 'function': 0.0,\n",
       " 'functions': 0.0,\n",
       " 'future': 0.0,\n",
       " 'gateway': 0.0,\n",
       " 'gcloud': 0.0,\n",
       " 'gcp': 0.0,\n",
       " 'gcs': 0.0,\n",
       " 'generate': 0.0,\n",
       " 'generated': 0.0,\n",
       " 'generator': 0.0,\n",
       " 'george': 0.0,\n",
       " 'get_dummies': 0.0,\n",
       " 'get_feature_names_out': 0.0,\n",
       " 'getdbt': 0.0,\n",
       " 'getorcreate': 0.0,\n",
       " 'gets': 0.0,\n",
       " 'getting': 0.0,\n",
       " 'git': 0.0,\n",
       " 'github': 0.3792268345828722,\n",
       " 'githubusercontent': 0.0,\n",
       " 'gitignore': 0.0,\n",
       " 'given': 0.0,\n",
       " 'gives': 0.0,\n",
       " 'gmail': 0.0,\n",
       " 'going': 0.0,\n",
       " 'good': 0.0,\n",
       " 'google': 0.0,\n",
       " 'google_application_credentials': 0.0,\n",
       " 'googleapi': 0.0,\n",
       " 'googleapis': 0.0,\n",
       " 'got': 0.0,\n",
       " 'gpu': 0.0,\n",
       " 'great': 0.0,\n",
       " 'green': 0.0,\n",
       " 'grigorev': 0.0,\n",
       " 'group': 0.0,\n",
       " 'gs': 0.0,\n",
       " 'guide': 0.0,\n",
       " 'gunicorn': 0.0,\n",
       " 'gz': 0.0,\n",
       " 'gzip': 0.0,\n",
       " 'hadoop': 0.0,\n",
       " 'hadoop3': 0.0,\n",
       " 'hand': 0.0,\n",
       " 'handle': 0.0,\n",
       " 'happen': 0.0,\n",
       " 'happens': 0.0,\n",
       " 'haven': 0.0,\n",
       " 'having': 0.0,\n",
       " 'head': 0.0,\n",
       " 'help': 0.0,\n",
       " 'helped': 0.0,\n",
       " 'helps': 0.0,\n",
       " 'high': 0.0,\n",
       " 'higher': 0.0,\n",
       " 'highly': 0.0,\n",
       " 'home': 0.0,\n",
       " 'homebrew': 0.0,\n",
       " 'homework': 0.0,\n",
       " 'homeworks': 0.0,\n",
       " 'hope': 0.0,\n",
       " 'host': 0.0,\n",
       " 'hostname': 0.0,\n",
       " 'hot': 0.0,\n",
       " 'hours': 0.0,\n",
       " 'housing': 0.0,\n",
       " 'hrithik': 0.0,\n",
       " 'html': 0.0,\n",
       " 'http': 0.0,\n",
       " 'https': 0.0,\n",
       " 'humberto': 0.0,\n",
       " 'hw': 0.0,\n",
       " 'hws': 0.0,\n",
       " 'iam': 0.0,\n",
       " 'icon': 0.0,\n",
       " 'id': 0.0,\n",
       " 'ide': 0.0,\n",
       " 'idea': 0.0,\n",
       " 'ignore': 0.0,\n",
       " 'image': 0.0,\n",
       " 'images': 0.0,\n",
       " 'immediately': 0.0,\n",
       " 'import': 0.0,\n",
       " 'importance': 0.0,\n",
       " 'important': 0.0,\n",
       " 'importerror': 0.0,\n",
       " 'importing': 0.0,\n",
       " 'include': 0.0,\n",
       " 'included': 0.0,\n",
       " 'including': 0.0,\n",
       " 'increase': 0.0,\n",
       " 'index': 0.0,\n",
       " 'info': 0.0,\n",
       " 'information': 0.0,\n",
       " 'init': 0.0,\n",
       " 'initial': 0.0,\n",
       " 'input': 0.0,\n",
       " 'inside': 0.0,\n",
       " 'install': 0.0,\n",
       " 'installation': 0.0,\n",
       " 'installed': 0.0,\n",
       " 'installing': 0.0,\n",
       " 'instance': 0.0,\n",
       " 'instances': 0.0,\n",
       " 'instead': 0.0,\n",
       " 'instruction': 0.0,\n",
       " 'instructions': 0.0,\n",
       " 'insufficient': 0.0,\n",
       " 'int': 0.0,\n",
       " 'int64': 0.0,\n",
       " 'integer': 0.0,\n",
       " 'interesting': 0.0,\n",
       " 'interface': 0.0,\n",
       " 'internal': 0.0,\n",
       " 'internet': 0.0,\n",
       " 'interpreter': 0.0,\n",
       " 'intro': 0.0,\n",
       " 'invalid': 0.0,\n",
       " 'io': 0.0,\n",
       " 'ip': 0.0,\n",
       " 'ipynb': 0.0,\n",
       " 'isn': 0.0,\n",
       " 'issue': 0.0,\n",
       " 'issues': 0.0,\n",
       " 'iteration': 0.0,\n",
       " 'ivan': 0.0,\n",
       " 'jan': 0.0,\n",
       " 'jar': 0.0,\n",
       " 'java': 0.0,\n",
       " 'java_home': 0.0,\n",
       " 'job': 0.0,\n",
       " 'jobs': 0.0,\n",
       " 'join': 0.0,\n",
       " 'json': 0.0,\n",
       " 'jupyter': 0.0,\n",
       " 'just': 0.0,\n",
       " 'kafka': 0.0,\n",
       " 'kaggle': 0.0,\n",
       " 'keras': 0.0,\n",
       " 'kernel': 0.0,\n",
       " 'key': 0.0,\n",
       " 'keys': 0.0,\n",
       " 'kill': 0.0,\n",
       " 'kind': 0.0,\n",
       " 'kinesis': 0.0,\n",
       " 'kniazeva': 0.0,\n",
       " 'know': 0.0,\n",
       " 'known': 0.0,\n",
       " 'konrad': 0.0,\n",
       " 'krishna': 0.0,\n",
       " 'kubectl': 0.0,\n",
       " 'kubernetes': 0.0,\n",
       " 'kumar': 0.0,\n",
       " 'kwargs': 0.0,\n",
       " 'lambda': 0.0,\n",
       " 'laptop': 0.0,\n",
       " 'large': 0.0,\n",
       " 'larkin': 0.0,\n",
       " 'later': 0.0,\n",
       " 'latest': 0.0,\n",
       " 'launch': 0.0,\n",
       " 'layer': 0.0,\n",
       " 'layers': 0.0,\n",
       " 'lead': 0.0,\n",
       " 'leaderboard': 0.0,\n",
       " 'learn': 0.0,\n",
       " 'learning': 0.0,\n",
       " 'left': 0.0,\n",
       " 'length': 0.0,\n",
       " 'lesson': 0.0,\n",
       " 'lessons': 0.0,\n",
       " 'let': 0.0,\n",
       " 'level': 0.0,\n",
       " 'lib': 0.0,\n",
       " 'libraries': 0.0,\n",
       " 'library': 0.0,\n",
       " 'like': 0.0,\n",
       " 'likely': 0.0,\n",
       " 'line': 0.0,\n",
       " 'linear': 0.0,\n",
       " 'lines': 0.0,\n",
       " 'link': 0.0,\n",
       " 'links': 0.0,\n",
       " 'linux': 0.0,\n",
       " 'list': 0.0,\n",
       " 'listed': 0.0,\n",
       " 'litvinov': 0.0,\n",
       " 'live': 0.0,\n",
       " 'll': 0.0,\n",
       " 'load': 0.0,\n",
       " 'loaded': 0.0,\n",
       " 'loading': 0.0,\n",
       " 'local': 0.0,\n",
       " 'localhost': 0.0,\n",
       " 'locally': 0.0,\n",
       " 'located': 0.0,\n",
       " 'location': 0.0,\n",
       " 'locations': 0.0,\n",
       " 'lock': 0.0,\n",
       " 'log': 0.0,\n",
       " 'logged': 0.0,\n",
       " 'login': 0.0,\n",
       " 'logistic': 0.0,\n",
       " 'logs': 0.0,\n",
       " 'long': 0.0,\n",
       " 'longer': 0.0,\n",
       " 'look': 0.0,\n",
       " 'looking': 0.0,\n",
       " 'looks': 0.0,\n",
       " 'loop': 0.0,\n",
       " 'lot': 0.0,\n",
       " 'lower': 0.0,\n",
       " 'ls': 0.0,\n",
       " 'lukafiardi': 0.0,\n",
       " 'mac': 0.0,\n",
       " 'machine': 0.0,\n",
       " 'macos': 0.0,\n",
       " 'mage': 0.0,\n",
       " 'magic': 0.0,\n",
       " 'main': 0.0,\n",
       " 'make': 0.0,\n",
       " 'makes': 0.0,\n",
       " 'making': 0.0,\n",
       " 'manage': 0.0,\n",
       " 'manually': 0.0,\n",
       " 'map': 0.0,\n",
       " 'marcos': 0.0,\n",
       " 'marcosmjd': 0.0,\n",
       " 'martin': 0.0,\n",
       " 'master': 0.0,\n",
       " 'match': 0.0,\n",
       " 'matplotlib': 0.0,\n",
       " 'matrix': 0.0,\n",
       " 'max': 0.0,\n",
       " 'maybe': 0.0,\n",
       " 'md': 0.0,\n",
       " 'mean': 0.0,\n",
       " 'mean_squared_error': 0.0,\n",
       " 'means': 0.0,\n",
       " 'media': 0.0,\n",
       " 'median_house_value': 0.0,\n",
       " 'memoona': 0.0,\n",
       " 'memory': 0.0,\n",
       " 'mentioned': 0.0,\n",
       " 'merge': 0.0,\n",
       " 'message': 0.0,\n",
       " 'metadata': 0.0,\n",
       " 'method': 0.0,\n",
       " 'metrics': 0.0,\n",
       " 'microsoft': 0.0,\n",
       " 'midterm': 0.0,\n",
       " 'minutes': 0.0,\n",
       " 'missing': 0.0,\n",
       " 'mkdir': 0.0,\n",
       " 'ml': 0.0,\n",
       " 'mlbookcamp': 0.0,\n",
       " 'mlflow': 0.0,\n",
       " 'mlops': 0.0,\n",
       " 'mode': 0.0,\n",
       " 'model': 0.0,\n",
       " 'models': 0.0,\n",
       " 'modified': 0.0,\n",
       " 'modify': 0.0,\n",
       " 'module': 0.0,\n",
       " 'modulenotfounderror': 0.0,\n",
       " 'modules': 0.0,\n",
       " 'month': 0.0,\n",
       " 'mount': 0.0,\n",
       " 'moved': 0.0,\n",
       " 'multiple': 0.0,\n",
       " 'multiplication': 0.0,\n",
       " 'mélanie': 0.0,\n",
       " 'named': 0.0,\n",
       " 'names': 0.0,\n",
       " 'nan': 0.0,\n",
       " 'nano': 0.0,\n",
       " 'native': 0.0,\n",
       " 'navigate': 0.0,\n",
       " 'ndarray': 0.0,\n",
       " 'necessary': 0.0,\n",
       " 'need': 0.0,\n",
       " 'needed': 0.0,\n",
       " 'needs': 0.0,\n",
       " 'negative': 0.0,\n",
       " 'net': 0.0,\n",
       " 'network': 0.0,\n",
       " 'networks': 0.0,\n",
       " 'new': 0.0,\n",
       " 'newly': 0.0,\n",
       " 'non': 0.0,\n",
       " 'normal': 0.0,\n",
       " 'note': 0.0,\n",
       " 'notebook': 0.0,\n",
       " 'notebooks': 0.0,\n",
       " 'notes': 0.0,\n",
       " 'notice': 0.0,\n",
       " 'np': 0.0,\n",
       " 'nukta': 0.0,\n",
       " 'null': 0.0,\n",
       " 'number': 0.0,\n",
       " 'numeric': 0.0,\n",
       " 'numerical': 0.0,\n",
       " 'numpy': 0.0,\n",
       " 'ny': 0.0,\n",
       " 'ny_taxi': 0.0,\n",
       " 'ny_taxi_postgres_data': 0.0,\n",
       " 'nyc': 0.0,\n",
       " 'object': 0.0,\n",
       " 'observations': 0.0,\n",
       " 'occur': 0.0,\n",
       " 'occurred': 0.0,\n",
       " 'occurs': 0.0,\n",
       " 'odimegwu': 0.0,\n",
       " 'office': 0.0,\n",
       " 'official': 0.0,\n",
       " 'ok': 0.0,\n",
       " 'old': 0.0,\n",
       " 'older': 0.0,\n",
       " 'olga': 0.0,\n",
       " 'onehotencoder': 0.0,\n",
       " 'ones': 0.0,\n",
       " 'open': 0.0,\n",
       " 'operation': 0.0,\n",
       " 'opt': 0.0,\n",
       " 'option': 0.0,\n",
       " 'optional': 0.0,\n",
       " 'options': 0.0,\n",
       " 'order': 0.0,\n",
       " 'org': 0.0,\n",
       " 'origin': 0.0,\n",
       " 'original': 0.0,\n",
       " 'os': 0.0,\n",
       " 'output': 0.0,\n",
       " 'outputs': 0.0,\n",
       " 'overfitting': 0.0,\n",
       " 'owner': 0.0,\n",
       " 'package': 0.0,\n",
       " 'packages': 0.0,\n",
       " 'page': 0.0,\n",
       " 'pandas': 0.0,\n",
       " 'parameter': 0.0,\n",
       " 'parameters': 0.0,\n",
       " 'params': 0.0,\n",
       " 'parquet': 0.0,\n",
       " 'parse': 0.0,\n",
       " 'particular': 0.0,\n",
       " 'particularly': 0.0,\n",
       " 'pass': 0.0,\n",
       " 'passed': 0.0,\n",
       " 'passing': 0.0,\n",
       " 'password': 0.0,\n",
       " 'paste': 0.0,\n",
       " 'pastor': 0.0,\n",
       " 'path': 0.0,\n",
       " 'paths': 0.0,\n",
       " 'pay': 0.0,\n",
       " 'payment_type': 0.0,\n",
       " 'pc': 0.0,\n",
       " 'pd': 0.0,\n",
       " 'peer': 0.0,\n",
       " 'perform': 0.0,\n",
       " 'performance': 0.0,\n",
       " 'period': 0.0,\n",
       " 'permission': 0.0,\n",
       " 'permissions': 0.0,\n",
       " 'pg': 0.0,\n",
       " 'pgadmin': 0.0,\n",
       " 'pgadmin4': 0.0,\n",
       " 'pgadmin_default_email': 0.0,\n",
       " 'pgadmin_default_password': 0.0,\n",
       " 'pgcli': 0.0,\n",
       " 'pgdatabase': 0.0,\n",
       " 'pick': 0.0,\n",
       " 'pickup_datetime': 0.0,\n",
       " 'pip': 0.0,\n",
       " 'pipeline': 0.0,\n",
       " 'pipenv': 0.0,\n",
       " 'pipfile': 0.0,\n",
       " 'pl3mmuxubc_hihxl5ji8t4o6lpaophaclr': 0.0,\n",
       " 'place': 0.0,\n",
       " 'platform': 0.0,\n",
       " 'platforms': 0.0,\n",
       " 'plt': 0.0,\n",
       " 'point': 0.0,\n",
       " 'points': 0.0,\n",
       " 'port': 0.0,\n",
       " 'ports': 0.0,\n",
       " 'position': 0.0,\n",
       " 'possible': 0.0,\n",
       " 'post': 0.0,\n",
       " 'postgres': 0.0,\n",
       " 'postgres_db': 0.0,\n",
       " 'postgres_password': 0.0,\n",
       " 'postgres_user': 0.0,\n",
       " 'postgresql': 0.0,\n",
       " 'powershell': 0.0,\n",
       " 'pq': 0.0,\n",
       " 'practice': 0.0,\n",
       " 'pre': 0.0,\n",
       " 'precision': 0.0,\n",
       " 'predict': 0.0,\n",
       " 'prediction': 0.0,\n",
       " 'predictions': 0.0,\n",
       " 'prefect': 0.0,\n",
       " 'prefer': 0.0,\n",
       " 'present': 0.0,\n",
       " 'press': 0.0,\n",
       " 'pretty': 0.0,\n",
       " 'prevent': 0.0,\n",
       " 'previous': 0.0,\n",
       " 'previously': 0.0,\n",
       " 'print': 0.0,\n",
       " 'private': 0.0,\n",
       " 'probably': 0.0,\n",
       " 'problem': 0.0,\n",
       " 'problems': 0.0,\n",
       " 'proceed': 0.0,\n",
       " 'process': 0.0,\n",
       " 'processes': 0.0,\n",
       " 'production': 0.0,\n",
       " 'profile': 0.0,\n",
       " 'program': 0.0,\n",
       " 'programs': 0.0,\n",
       " 'progress': 0.0,\n",
       " 'project': 0.0,\n",
       " 'project_id': 0.0,\n",
       " 'projects': 0.0,\n",
       " 'prompt': 0.0,\n",
       " 'properly': 0.0,\n",
       " 'protobuf': 0.0,\n",
       " 'provide': 0.0,\n",
       " 'provided': 0.0,\n",
       " 'provides': 0.0,\n",
       " 'ps': 0.0,\n",
       " 'psql': 0.0,\n",
       " 'psycopg2': 0.0,\n",
       " 'pubhtml': 0.0,\n",
       " 'public': 0.0,\n",
       " 'pull': 0.0,\n",
       " 'pulocationid': 0.0,\n",
       " 'purpose': 0.0,\n",
       " 'push': 0.0,\n",
       " 'pwd': 0.0,\n",
       " 'py': 0.0,\n",
       " 'py4j': 0.0,\n",
       " 'pyarrow': 0.0,\n",
       " 'pyspark': 0.0,\n",
       " 'python': 0.0,\n",
       " 'python3': 0.0,\n",
       " 'pythonpath': 0.0,\n",
       " 'queries': 0.0,\n",
       " 'query': 0.0,\n",
       " 'question': 0.0,\n",
       " 'questions': 0.0,\n",
       " 'quinn': 0.0,\n",
       " 'quite': 0.0,\n",
       " 'quotes': 0.0,\n",
       " 'radikal': 0.0,\n",
       " 'raised': 0.0,\n",
       " 'ram': 0.0,\n",
       " 'ran': 0.0,\n",
       " 'random': 0.0,\n",
       " 'random_state': 0.0,\n",
       " 'range': 0.0,\n",
       " 'raw': 0.0,\n",
       " 'read': 0.0,\n",
       " 'read_csv': 0.0,\n",
       " 'read_parquet': 0.0,\n",
       " 'reading': 0.0,\n",
       " 'ready': 0.0,\n",
       " 'real': 0.0,\n",
       " 'really': 0.0,\n",
       " 'reason': 0.0,\n",
       " 'recall': 0.0,\n",
       " 'receive': 0.0,\n",
       " 'recent': 0.0,\n",
       " 'recognized': 0.0,\n",
       " 'recommend': 0.0,\n",
       " 'recommended': 0.0,\n",
       " 'record': 0.0,\n",
       " 'recorded': 0.0,\n",
       " 'records': 0.0,\n",
       " 'recreate': 0.0,\n",
       " 'refer': 0.0,\n",
       " 'reference': 0.0,\n",
       " 'regarding': 0.0,\n",
       " 'region': 0.0,\n",
       " 'register': 0.0,\n",
       " 'registry': 0.0,\n",
       " 'regression': 0.0,\n",
       " 'regularization': 0.0,\n",
       " 'reinstall': 0.0,\n",
       " 'related': 0.0,\n",
       " 'releases': 0.0,\n",
       " 'remember': 0.0,\n",
       " 'remote': 0.0,\n",
       " 'remove': 0.0,\n",
       " 'removed': 0.0,\n",
       " 'removing': 0.0,\n",
       " 'rename': 0.0,\n",
       " 'replace': 0.0,\n",
       " 'replaced': 0.0,\n",
       " ...}"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc_dict = dict(zip(names, X.toarray()[1]))\n",
    "doc_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "2e3d9256-b16d-457d-bf1a-8b977f103702",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "matrix([[0.48049682],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.2083882 ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.17557272],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.15870689],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.09680922],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.07529201],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.29986763],\n",
       "        [0.10520675],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.27447476],\n",
       "        [0.12828407],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.05163407],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.03156309],\n",
       "        [0.04914818],\n",
       "        [0.07138962],\n",
       "        [0.        ],\n",
       "        [0.04329773],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.02804374],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.06739038],\n",
       "        [0.        ],\n",
       "        [0.00980845],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.05820102],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.05020173],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.0605701 ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.05101782],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.04327901],\n",
       "        [0.        ],\n",
       "        [0.03089898],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.05106564],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.08232505],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.02129978],\n",
       "        [0.        ],\n",
       "        [0.0308046 ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.16499001],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.03561953],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.07733187],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.12621053],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.01502752],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.05398514],\n",
       "        [0.        ],\n",
       "        [0.05115414],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.03869534],\n",
       "        [0.04112908],\n",
       "        [0.03728758],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.09064943],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.04233182],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.07784175],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.02591152],\n",
       "        [0.        ],\n",
       "        [0.03451997],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.04143129],\n",
       "        [0.        ],\n",
       "        [0.05407899],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.22471032],\n",
       "        [0.04931254],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.03820044],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.02533066],\n",
       "        [0.04085302],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.07593909],\n",
       "        [0.0723982 ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.01328652],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.10202387],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.038628  ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.12084703],\n",
       "        [0.10694586],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.08997023],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.05839441],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.12589674],\n",
       "        [0.196371  ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.06058604],\n",
       "        [0.35656584],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.16267179],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.30485152],\n",
       "        [0.33773583],\n",
       "        [0.1231099 ],\n",
       "        [0.13606474],\n",
       "        [0.26771296],\n",
       "        [0.        ],\n",
       "        [0.10835741],\n",
       "        [0.13944604],\n",
       "        [0.03865125],\n",
       "        [0.18083089],\n",
       "        [0.03179529],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.05501329],\n",
       "        [0.03482028],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.06558528],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.03200722],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.05236864],\n",
       "        [0.0581809 ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.10647026],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.03710508],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.04720167],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.04364401],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.03714607],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.04808267],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.14400322],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.03667548],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.06555259],\n",
       "        [0.13437698],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.05902352],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.01885244],\n",
       "        [0.03485812],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.05249485],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.06591061],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.05969678],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.0284637 ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.06244092],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.02243907],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.15333692],\n",
       "        [0.        ],\n",
       "        [0.03506481],\n",
       "        [0.18859024],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.03423508],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.01588394],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.04924389],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.05098356],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.03294308],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.03026252],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.02958017],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.07023608],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.04109976],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.04644028],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.03833017],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.0364746 ],\n",
       "        [0.08415092],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.02442063],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.12810532],\n",
       "        [0.05637395],\n",
       "        [0.04826177],\n",
       "        [0.        ],\n",
       "        [0.04906345],\n",
       "        [0.        ],\n",
       "        [0.26727441],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.11135001],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.06894246],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.0685693 ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.04106944],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.06551955],\n",
       "        [0.        ],\n",
       "        [0.07186382],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.06333884],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.12685281],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.07782842],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.02660461],\n",
       "        [0.06140912],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.05395122],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.02053581],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.02983282],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.03416571],\n",
       "        [0.03748537],\n",
       "        [0.        ],\n",
       "        [0.05747492],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.13556595],\n",
       "        [0.        ],\n",
       "        [0.10025904],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.03278609],\n",
       "        [0.11688021],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.03662924],\n",
       "        [0.0451592 ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ],\n",
       "        [0.        ]])"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.dot(q.T).todense()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "ed5736c8-cb12-4f67-be4d-0d603609b470",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics.pairwise import cosine_similarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "8443364d-7e78-408d-b577-0dd03d08af33",
   "metadata": {},
   "outputs": [],
   "source": [
    "score = cosine_similarity(X, q).flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "36bb0a21-5050-4c38-83ee-c4cef61205e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "829ec245-3d64-4e94-8959-5f5d6c4c9c4c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 22, 448, 449, 440,   0])"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.argsort(score)[-5:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "9880a8a6-5bc9-42ed-992f-00fd4c5a46f1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "numpy.ndarray"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "401a7bbd-2926-44ed-bc8c-646c3b8ac49f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Yes, you can. You won’t be able to submit some of the homeworks, but you can still take part in the course.\\nIn order to get a certificate, you need to submit 2 out of 3 course projects and review 3 peers’ Projects by the deadline. It means that if you join the course at the end of November and manage to work on two projects, you will still be eligible for a certificate.'"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.iloc[449].text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "id": "4427cd31-e3ef-46a0-a88c-c3b65df7c784",
   "metadata": {},
   "outputs": [],
   "source": [
    "fields = ['section',\t'question','text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "id": "09c16dca-8c80-4994-8c88-99a584f456af",
   "metadata": {},
   "outputs": [],
   "source": [
    "matrices = {}\n",
    "vectorizers = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "id": "b9c953e6-7170-4a99-851c-966ae8d2d992",
   "metadata": {},
   "outputs": [],
   "source": [
    "for f in fields:\n",
    "\n",
    "    cv = TfidfVectorizer(stop_words='english', min_df=5)\n",
    "    X = cv.fit_transform(df[f])\n",
    "    matrices[f] = X\n",
    "    vectorizers[f] = cv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "id": "80a30b47-8d57-4854-b83c-daa819e46804",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'section': <948x66 sparse matrix of type '<class 'numpy.float64'>'\n",
       " \twith 3090 stored elements in Compressed Sparse Row format>,\n",
       " 'question': <948x291 sparse matrix of type '<class 'numpy.float64'>'\n",
       " \twith 3431 stored elements in Compressed Sparse Row format>,\n",
       " 'text': <948x1333 sparse matrix of type '<class 'numpy.float64'>'\n",
       " \twith 23808 stored elements in Compressed Sparse Row format>}"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matrices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "id": "c3b05d8c-0ec6-42ab-97fa-239d3d20f02a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'section': TfidfVectorizer(min_df=5, stop_words='english'),\n",
       " 'question': TfidfVectorizer(min_df=5, stop_words='english'),\n",
       " 'text': TfidfVectorizer(min_df=5, stop_words='english')}"
      ]
     },
     "execution_count": 160,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "id": "d37ca2f1-96a5-4b15-888e-c7743634c36a",
   "metadata": {},
   "outputs": [],
   "source": [
    "n = len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "id": "d0f08dd7-ecbe-4f00-b215-8f009396e8c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "score = np.zeros(n)\n",
    "\n",
    "query = \"I just discovered the course, is it too late to join?\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "id": "eb65b890-9910-4606-a4c1-8be6c19018b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "boosts = {\n",
    "    'question': 3\n",
    "    #,'text': 0.5\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "id": "d2ac921c-e9dd-4fbb-b337-4cbaafe4923c",
   "metadata": {},
   "outputs": [],
   "source": [
    "for field in fields:\n",
    "    q = vectorizers[field].transform([query])\n",
    "    X = matrices[field]\n",
    "\n",
    "    field_score = cosine_similarity(X, q).flatten()\n",
    "\n",
    "    boost = boosts.get(f, 1.0)\n",
    "    \n",
    "    score += (boost * field_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "id": "c4c98c29-278c-4aa3-bb96-51e39d9243bf",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1.82703079, 1.49512426, 1.23253339, 1.45772228, 1.49512426,\n",
       "       1.49512426, 0.97571381, 1.67069698, 1.22089233, 1.49512426,\n",
       "       1.36407773, 1.25620994, 0.49512426, 0.49512426, 0.49512426,\n",
       "       0.59193348, 0.49512426, 1.20932345, 0.57041627, 0.49512426,\n",
       "       0.49512426, 0.49512426, 0.79499188, 0.60033101, 0.49512426,\n",
       "       0.49512426, 0.49512426, 0.76959902, 0.62340833, 0.49512426,\n",
       "       0.49512426, 0.49512426, 0.49512426, 0.96108   , 1.49512426,\n",
       "       0.90368553, 0.49512426, 0.49512426, 0.49512426, 0.52668735,\n",
       "       0.54427244, 1.04472639, 0.49512426, 0.53842198, 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.02804374, 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.06739038, 0.        , 0.00980845,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.05820102,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.05020173, 0.        , 0.        , 0.0605701 , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.05101782, 0.        ,\n",
       "       0.        , 0.04327901, 0.        , 0.03089898, 0.        ,\n",
       "       0.        , 0.05106564, 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.08232505, 0.        , 0.        , 0.        , 0.02129978,\n",
       "       0.        , 0.0308046 , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.16499001, 0.        ,\n",
       "       0.        , 0.        , 0.03561953, 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.07733187, 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.12621053, 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.01502752, 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.05398514, 0.        , 0.05115414, 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.03869534, 0.04112908, 0.03728758,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.09064943, 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.04233182,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.07784175, 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.02591152, 0.        , 0.03451997, 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.04143129, 0.        , 0.05407899, 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.22471032, 0.04931254, 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.03820044, 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.02533066, 0.04085302, 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.07593909, 0.0723982 , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.01328652, 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.10202387, 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.038628  , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.12084703, 0.10694586, 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       1.        , 1.08997023, 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.05839441, 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.621021  , 0.69149525, 0.49512426, 0.49512426, 1.14930942,\n",
       "       0.8516901 , 1.07881798, 0.98377269, 0.49512426, 0.49512426,\n",
       "       0.65779605, 1.19454307, 0.49512426, 1.79997577, 1.45032846,\n",
       "       0.61823416, 0.631189  , 1.41381783, 1.49512426, 0.99196149,\n",
       "       0.92942112, 0.5337755 , 0.67595514, 0.03179529, 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.05501329,\n",
       "       0.03482028, 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.06558528, 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.03200722, 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.05236864, 0.0581809 , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.10647026, 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.03710508, 0.        , 0.        ,\n",
       "       0.04720167, 0.        , 0.        , 0.04364401, 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.03714607, 0.        , 0.        ,\n",
       "       0.        , 0.04808267, 0.        , 0.        , 0.        ,\n",
       "       0.14400322, 0.        , 0.        , 0.03667548, 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.06555259, 0.13437698, 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.05902352, 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.01885244, 0.03485812,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.05249485,\n",
       "       0.        , 0.        , 0.        , 0.06591061, 0.        ,\n",
       "       0.        , 0.05969678, 0.        , 0.        , 0.        ,\n",
       "       0.0284637 , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.06244092,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.02243907, 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.15333692,\n",
       "       0.        , 0.03506481, 0.18859024, 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.03423508,\n",
       "       0.        , 0.        , 0.01588394, 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.04924389, 0.        , 0.        , 0.        ,\n",
       "       0.05098356, 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.03294308, 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.03026252, 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.02958017, 0.        , 0.        , 0.        , 0.07023608,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.04109976, 0.        , 0.        , 0.        , 0.04644028,\n",
       "       0.        , 0.        , 0.03833017, 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.0364746 , 0.08415092, 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.02442063,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.12810532, 0.05637395,\n",
       "       0.04826177, 0.        , 0.04906345, 0.        , 0.26727441,\n",
       "       0.        , 0.        , 0.        , 0.11135001, 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.06894246,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.0685693 , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.04106944, 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.06551955,\n",
       "       0.        , 0.68214193, 0.        , 0.        , 0.        ,\n",
       "       0.63924543, 1.2753254 , 1.03761299, 0.57590658, 0.7027594 ,\n",
       "       0.57590658, 0.57590658, 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.07782842, 0.5012952 , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.02660461, 0.06140912, 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.05395122, 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.02053581, 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.02983282, 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.03416571, 0.03748537, 0.        , 0.05747492, 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.13556595, 0.        ,\n",
       "       0.10025904, 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.03278609, 0.11688021, 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.03662924, 0.0451592 , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        ])"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "id": "9eefd7b0-f077-4bdb-818a-46222a9492b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "filters = {\n",
    "    'course': 'data-engineering-zoomcamp'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "id": "5ed922ce-894e-494e-8599-43d65f1c1a0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "for field, value in filters.items():\n",
    "    mask = (df[field] == value).astype(int).values\n",
    "    score *= mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "id": "c75a246b-25ce-4e31-872a-9c735b456551",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0])"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "id": "ec688824-68f0-4b7a-8b4a-eb0baf271d4b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1.82703079, 1.49512426, 1.23253339, 1.45772228, 1.49512426,\n",
       "       1.49512426, 0.97571381, 1.67069698, 1.22089233, 1.49512426,\n",
       "       1.36407773, 1.25620994, 0.49512426, 0.49512426, 0.49512426,\n",
       "       0.59193348, 0.49512426, 1.20932345, 0.57041627, 0.49512426,\n",
       "       0.49512426, 0.49512426, 0.79499188, 0.60033101, 0.49512426,\n",
       "       0.49512426, 0.49512426, 0.76959902, 0.62340833, 0.49512426,\n",
       "       0.49512426, 0.49512426, 0.49512426, 0.96108   , 1.49512426,\n",
       "       0.90368553, 0.49512426, 0.49512426, 0.49512426, 0.52668735,\n",
       "       0.54427244, 1.04472639, 0.49512426, 0.53842198, 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.02804374, 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.06739038, 0.        , 0.00980845,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.05820102,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.05020173, 0.        , 0.        , 0.0605701 , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.05101782, 0.        ,\n",
       "       0.        , 0.04327901, 0.        , 0.03089898, 0.        ,\n",
       "       0.        , 0.05106564, 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.08232505, 0.        , 0.        , 0.        , 0.02129978,\n",
       "       0.        , 0.0308046 , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.16499001, 0.        ,\n",
       "       0.        , 0.        , 0.03561953, 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.07733187, 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.12621053, 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.01502752, 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.05398514, 0.        , 0.05115414, 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.03869534, 0.04112908, 0.03728758,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.09064943, 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.04233182,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.07784175, 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.02591152, 0.        , 0.03451997, 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.04143129, 0.        , 0.05407899, 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.22471032, 0.04931254, 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.03820044, 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.02533066, 0.04085302, 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.07593909, 0.0723982 , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.01328652, 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.10202387, 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.038628  , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.12084703, 0.10694586, 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       1.        , 1.08997023, 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.05839441, 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        ])"
      ]
     },
     "execution_count": 169,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "id": "52b9d4b8-7819-4a69-9177-fe296e35f0fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "idx = np.argsort(-score)[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "id": "86e232b2-2c3d-4ffd-8e96-546b2aabaf3b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>course</th>\n",
       "      <th>section</th>\n",
       "      <th>question</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>data-engineering-zoomcamp</td>\n",
       "      <td>General course-related questions</td>\n",
       "      <td>Course - When will the course start?</td>\n",
       "      <td>The purpose of this document is to capture fre...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>data-engineering-zoomcamp</td>\n",
       "      <td>General course-related questions</td>\n",
       "      <td>Course - Can I follow the course after it fini...</td>\n",
       "      <td>Yes, we will keep all the materials after the ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>data-engineering-zoomcamp</td>\n",
       "      <td>General course-related questions</td>\n",
       "      <td>Course - What are the prerequisites for this c...</td>\n",
       "      <td>GitHub - DataTalksClub data-engineering-zoomca...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>data-engineering-zoomcamp</td>\n",
       "      <td>General course-related questions</td>\n",
       "      <td>Course - What can I do before the course starts?</td>\n",
       "      <td>You can start by installing and setting up all...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>data-engineering-zoomcamp</td>\n",
       "      <td>General course-related questions</td>\n",
       "      <td>Course - how many Zoomcamps in a year?</td>\n",
       "      <td>There are 3 Zoom Camps in a year, as of 2024. ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      course                           section  \\\n",
       "0  data-engineering-zoomcamp  General course-related questions   \n",
       "7  data-engineering-zoomcamp  General course-related questions   \n",
       "1  data-engineering-zoomcamp  General course-related questions   \n",
       "4  data-engineering-zoomcamp  General course-related questions   \n",
       "5  data-engineering-zoomcamp  General course-related questions   \n",
       "\n",
       "                                            question  \\\n",
       "0               Course - When will the course start?   \n",
       "7  Course - Can I follow the course after it fini...   \n",
       "1  Course - What are the prerequisites for this c...   \n",
       "4   Course - What can I do before the course starts?   \n",
       "5             Course - how many Zoomcamps in a year?   \n",
       "\n",
       "                                                text  \n",
       "0  The purpose of this document is to capture fre...  \n",
       "7  Yes, we will keep all the materials after the ...  \n",
       "1  GitHub - DataTalksClub data-engineering-zoomca...  \n",
       "4  You can start by installing and setting up all...  \n",
       "5  There are 3 Zoom Camps in a year, as of 2024. ...  "
      ]
     },
     "execution_count": 175,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.iloc[idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "id": "79a08080-e89e-4897-9ec0-3da380729263",
   "metadata": {},
   "outputs": [],
   "source": [
    "class TextSearch:\n",
    "\n",
    "    def __init__(self, text_fields):\n",
    "        self.text_fields = text_fields\n",
    "        self.matrices = {}\n",
    "        self.vectorizers = {}\n",
    "\n",
    "    def fit(self, records, vectorizer_params={}):\n",
    "        self.df = pd.DataFrame(records)\n",
    "\n",
    "        for f in self.text_fields:\n",
    "            cv = TfidfVectorizer(**vectorizer_params)\n",
    "            X = cv.fit_transform(self.df[f])\n",
    "            self.matrices[f] = X\n",
    "            self.vectorizers[f] = cv\n",
    "\n",
    "    def search(self, query, n_results=10, boost={}, filters={}):\n",
    "        score = np.zeros(len(self.df))\n",
    "\n",
    "        for f in self.text_fields:\n",
    "            b = boost.get(f, 1.0)\n",
    "            q = self.vectorizers[f].transform([query])\n",
    "            s = cosine_similarity(self.matrices[f], q).flatten()\n",
    "            score = score + b * s\n",
    "\n",
    "        for field, value in filters.items():\n",
    "            mask = (self.df[field] == value).values\n",
    "            score = score * mask\n",
    "\n",
    "        idx = np.argsort(-score)[:n_results]\n",
    "        results = self.df.iloc[idx]\n",
    "        return results.to_dict(orient='records')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "id": "33cd462a-4ca8-4828-893c-c880703af922",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'text': \"Yes, even if you don't register, you're still eligible to submit the homeworks.\\nBe aware, however, that there will be deadlines for turning in the final projects. So don't leave everything for the last minute.\",\n",
       "  'section': 'General course-related questions',\n",
       "  'question': 'Course - Can I still join the course after the start date?',\n",
       "  'course': 'data-engineering-zoomcamp'},\n",
       " {'text': \"The purpose of this document is to capture frequently asked technical questions\\nThe exact day and hour of the course will be 15th Jan 2024 at 17h00. The course will start with the first  “Office Hours'' live.1\\nSubscribe to course public Google Calendar (it works from Desktop only).\\nRegister before the course starts using this link.\\nJoin the course Telegram channel with announcements.\\nDon’t forget to register in DataTalks.Club's Slack and join the channel.\",\n",
       "  'section': 'General course-related questions',\n",
       "  'question': 'Course - When will the course start?',\n",
       "  'course': 'data-engineering-zoomcamp'},\n",
       " {'text': 'Yes, we will keep all the materials after the course finishes, so you can follow the course at your own pace after it finishes.\\nYou can also continue looking at the homeworks and continue preparing for the next cohort. I guess you can also start working on your final capstone project.',\n",
       "  'section': 'General course-related questions',\n",
       "  'question': 'Course - Can I follow the course after it finishes?',\n",
       "  'course': 'data-engineering-zoomcamp'},\n",
       " {'text': 'No, late submissions are not allowed. But if the form is still not closed and it’s after the due date, you can still submit the homework. confirm your submission by the date-timestamp on the Course page.y\\nOlder news:[source1] [source2]',\n",
       "  'section': 'General course-related questions',\n",
       "  'question': 'Homework - Are late submissions of homework allowed?',\n",
       "  'course': 'data-engineering-zoomcamp'},\n",
       " {'text': 'You can start by installing and setting up all the dependencies and requirements:\\nGoogle cloud account\\nGoogle Cloud SDK\\nPython 3 (installed with Anaconda)\\nTerraform\\nGit\\nLook over the prerequisites and syllabus to see if you are comfortable with these subjects.',\n",
       "  'section': 'General course-related questions',\n",
       "  'question': 'Course - What can I do before the course starts?',\n",
       "  'course': 'data-engineering-zoomcamp'}]"
      ]
     },
     "execution_count": 177,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "index = TextSearch(\n",
    "    text_fields=['section', 'question', 'text']\n",
    ")\n",
    "index.fit(documents)\n",
    "\n",
    "index.search(\n",
    "    query='I just singned up. Is it too late to join the course?',\n",
    "    n_results=5,\n",
    "    boost={'question': 3.0},\n",
    "    filters={'course': 'data-engineering-zoomcamp'}\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee1a9793-2225-48d2-8054-e0a65a1005ae",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}