process_summaries.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import sys\n",
    "import csv\n",
    "from time import strptime\n",
    "import json\n",
    "from datetime import datetime as dt\n",
    "from datetime import date\n",
    "from statistics import mean\n",
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('14 ratings.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "metadata = pd.read_csv('MovieSummaries/movie.metadata.tsv', sep='\\t', header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "metadata.columns = ['id_wiki','id_freebase', 'name', 'release', 'revenue', 'runtime', 'langs', 'countries', 'genres']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(81741, 9)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metadata.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "file1 = open(\"MovieSummaries/plot_summaries.txt\",\"r+\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "plots = file1.read().split('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_ids = [s.split('\\t')[0] for s in plots]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_plots = pd.DataFrame(data = {'id_wiki': plot_ids, 'plot': plots}).drop(42306, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_plots['plot'] = [plot.split('\\t', 1)[1] for plot in list(df_plots['plot'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_plots.id_wiki = df_plots.id_wiki.astype(np.int64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(42306, 2)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_plots.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged = df_plots.merge(metadata, how='inner', on=['id_wiki'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged['year'] = [release.split('-', 1)[0] if not pd.isna(release) else np.nan for release in merged.release]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged['year'] = [int(y) if not pd.isna(y) else np.nan for y in merged.year]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_wiki</th>\n",
       "      <th>plot</th>\n",
       "      <th>id_freebase</th>\n",
       "      <th>name</th>\n",
       "      <th>release</th>\n",
       "      <th>revenue</th>\n",
       "      <th>runtime</th>\n",
       "      <th>langs</th>\n",
       "      <th>countries</th>\n",
       "      <th>genres</th>\n",
       "      <th>year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>23890098</td>\n",
       "      <td>Shlykov, a hard-working taxi driver and Lyosha...</td>\n",
       "      <td>/m/076w2lb</td>\n",
       "      <td>Taxi Blues</td>\n",
       "      <td>1990-09-07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>110.0</td>\n",
       "      <td>{\"/m/06b_j\": \"Russian Language\"}</td>\n",
       "      <td>{\"/m/0f8l9c\": \"France\", \"/m/05vz3zq\": \"Soviet ...</td>\n",
       "      <td>{\"/m/07s9rl0\": \"Drama\", \"/m/03q4nz\": \"World ci...</td>\n",
       "      <td>1990.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>31186339</td>\n",
       "      <td>The nation of Panem consists of a wealthy Capi...</td>\n",
       "      <td>/m/0gkz15s</td>\n",
       "      <td>The Hunger Games</td>\n",
       "      <td>2012-03-12</td>\n",
       "      <td>686533290.0</td>\n",
       "      <td>142.0</td>\n",
       "      <td>{\"/m/02h40lc\": \"English Language\"}</td>\n",
       "      <td>{\"/m/09c7w0\": \"United States of America\"}</td>\n",
       "      <td>{\"/m/03btsm8\": \"Action/Adventure\", \"/m/06n90\":...</td>\n",
       "      <td>2012.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>20663735</td>\n",
       "      <td>Poovalli Induchoodan  is sentenced for six yea...</td>\n",
       "      <td>/m/051zjwb</td>\n",
       "      <td>Narasimham</td>\n",
       "      <td>2000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>175.0</td>\n",
       "      <td>{\"/m/0999q\": \"Malayalam Language\"}</td>\n",
       "      <td>{\"/m/03rk0\": \"India\"}</td>\n",
       "      <td>{\"/m/04t36\": \"Musical\", \"/m/02kdv5l\": \"Action\"...</td>\n",
       "      <td>2000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>2231378</td>\n",
       "      <td>The Lemon Drop Kid , a New York City swindler,...</td>\n",
       "      <td>/m/06xtz3</td>\n",
       "      <td>The Lemon Drop Kid</td>\n",
       "      <td>1951-03-08</td>\n",
       "      <td>2300000.0</td>\n",
       "      <td>91.0</td>\n",
       "      <td>{\"/m/02h40lc\": \"English Language\"}</td>\n",
       "      <td>{\"/m/09c7w0\": \"United States of America\"}</td>\n",
       "      <td>{\"/m/06qm3\": \"Screwball comedy\", \"/m/01z4y\": \"...</td>\n",
       "      <td>1951.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>595909</td>\n",
       "      <td>Seventh-day Adventist Church pastor Michael Ch...</td>\n",
       "      <td>/m/02tqm5</td>\n",
       "      <td>A Cry in the Dark</td>\n",
       "      <td>1988-11-03</td>\n",
       "      <td>6908797.0</td>\n",
       "      <td>121.0</td>\n",
       "      <td>{\"/m/02h40lc\": \"English Language\"}</td>\n",
       "      <td>{\"/m/09c7w0\": \"United States of America\", \"/m/...</td>\n",
       "      <td>{\"/m/0lsxr\": \"Crime Fiction\", \"/m/07s9rl0\": \"D...</td>\n",
       "      <td>1988.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    id_wiki                                               plot id_freebase  \\\n",
       "0  23890098  Shlykov, a hard-working taxi driver and Lyosha...  /m/076w2lb   \n",
       "1  31186339  The nation of Panem consists of a wealthy Capi...  /m/0gkz15s   \n",
       "2  20663735  Poovalli Induchoodan  is sentenced for six yea...  /m/051zjwb   \n",
       "3   2231378  The Lemon Drop Kid , a New York City swindler,...   /m/06xtz3   \n",
       "4    595909  Seventh-day Adventist Church pastor Michael Ch...   /m/02tqm5   \n",
       "\n",
       "                 name     release      revenue  runtime  \\\n",
       "0          Taxi Blues  1990-09-07          NaN    110.0   \n",
       "1    The Hunger Games  2012-03-12  686533290.0    142.0   \n",
       "2          Narasimham        2000          NaN    175.0   \n",
       "3  The Lemon Drop Kid  1951-03-08    2300000.0     91.0   \n",
       "4   A Cry in the Dark  1988-11-03    6908797.0    121.0   \n",
       "\n",
       "                                langs  \\\n",
       "0    {\"/m/06b_j\": \"Russian Language\"}   \n",
       "1  {\"/m/02h40lc\": \"English Language\"}   \n",
       "2  {\"/m/0999q\": \"Malayalam Language\"}   \n",
       "3  {\"/m/02h40lc\": \"English Language\"}   \n",
       "4  {\"/m/02h40lc\": \"English Language\"}   \n",
       "\n",
       "                                           countries  \\\n",
       "0  {\"/m/0f8l9c\": \"France\", \"/m/05vz3zq\": \"Soviet ...   \n",
       "1          {\"/m/09c7w0\": \"United States of America\"}   \n",
       "2                              {\"/m/03rk0\": \"India\"}   \n",
       "3          {\"/m/09c7w0\": \"United States of America\"}   \n",
       "4  {\"/m/09c7w0\": \"United States of America\", \"/m/...   \n",
       "\n",
       "                                              genres    year  \n",
       "0  {\"/m/07s9rl0\": \"Drama\", \"/m/03q4nz\": \"World ci...  1990.0  \n",
       "1  {\"/m/03btsm8\": \"Action/Adventure\", \"/m/06n90\":...  2012.0  \n",
       "2  {\"/m/04t36\": \"Musical\", \"/m/02kdv5l\": \"Action\"...  2000.0  \n",
       "3  {\"/m/06qm3\": \"Screwball comedy\", \"/m/01z4y\": \"...  1951.0  \n",
       "4  {\"/m/0lsxr\": \"Crime Fiction\", \"/m/07s9rl0\": \"D...  1988.0  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['budget', 'META__id', 'PROCESS__original_language',\n",
       "       'PROCESS__original_title', 'PROCESS__overview', 'revenue', 'runtime',\n",
       "       'spoken_languages', 'PROCESS__title', 'META__year',\n",
       "       ...\n",
       "       'production_company_3_avg_profitability',\n",
       "       'production_company_3_avg_revenue', 'collection_name_avg_profit',\n",
       "       'collection_name_avg_revenue', 'competition', 'rating__g', 'rating__pg',\n",
       "       'rating__pg-13', 'rating__r', 'rating__nc-17'],\n",
       "      dtype='object', length=305)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7494, 305)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged.columns = ['id_wiki', 'plot', 'id_freebase', 'PROCESS__title', 'release', 'revenue', 'runtime', 'langs', 'countries', 'genres', 'META__year']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.META__year = df.META__year.astype(np.float64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged_df = df.merge(merged, how='inner', on=['PROCESS__title', 'META__year'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4520, 314)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4441"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c_longer = 0\n",
    "for i, row in merged_df.iterrows():\n",
    "    if len(row['plot']) > len(row['PROCESS__overview']):\n",
    "        c_longer+=1\n",
    "c_longer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9825221238938053"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "4441/4520"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, row in merged_df.iterrows():\n",
    "    if len(row['plot']) < len(row['PROCESS__overview']):\n",
    "        merged_df.loc[merged_df.META__id == row.META__id, 'plot'] = row['PROCESS__overview']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4520, 314)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4503, 314)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_df.drop_duplicates('META__id').shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_df = merged_df[['META__id', 'plot']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>META__id</th>\n",
       "      <th>plot</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>The film is set on New Year's Eve, and starts ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>Frank Wyatt  leaves his wife and baby daughter...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>12</td>\n",
       "      <td>Two clownfish, Marlin and his wife Coral are a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>13</td>\n",
       "      <td>As he waits at a bus stop, Forrest Gump  star...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>14</td>\n",
       "      <td>Lester Burnham  is a middle-aged magazine writ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   META__id                                               plot\n",
       "0         5  The film is set on New Year's Eve, and starts ...\n",
       "1         6  Frank Wyatt  leaves his wife and baby daughter...\n",
       "2        12  Two clownfish, Marlin and his wife Coral are a...\n",
       "3        13   As he waits at a bus stop, Forrest Gump  star...\n",
       "4        14  Lester Burnham  is a middle-aged magazine writ..."
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "plot_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/stas/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "plot_df['plot'] = plot_df['plot'].str.replace('$', '')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_df.to_csv('plot_df.csv', mode='w+', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}