recommendation_using_clustering.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no. of users : 943\n",
      "no. of items : 1682\n"
     ]
    }
   ],
   "source": [
    "#!/bin/python\n",
    "from movielens import *\n",
    "import numpy as np\n",
    "import math\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.cluster import KMeans\n",
    "import sys, time\n",
    "\n",
    "# Store data in arrays\n",
    "user = []\n",
    "item = []\n",
    "rating = []\n",
    "rating_test = []\n",
    "\n",
    "# Load the movie lens dataset into arrays\n",
    "d = Dataset()\n",
    "d.load_users(\"data/u.user\", user)\n",
    "d.load_items(\"data/u.item\", item)\n",
    "d.load_ratings(\"data/u.base\", rating)\n",
    "d.load_ratings(\"data/u.test\", rating_test)\n",
    "\n",
    "n_users = len(user)\n",
    "n_items = len(item)\n",
    "\n",
    "print \"no. of users : \"+str(n_users)\n",
    "print \"no. of items : \"+str(n_items)\n",
    "\n",
    "# The utility matrix stores the rating for each user-item pair in the matrix form.\n",
    "# Note that the movielens data is indexed starting from 1 (instead of 0).\n",
    "utility = np.zeros((n_users, n_items))\n",
    "for r in rating:\n",
    "    utility[r.user_id-1][r.item_id-1] = r.rating\n",
    "\n",
    "# Finds the average rating for each user and stores it in the user's object\n",
    "for i in range(n_users):\n",
    "    rated = np.nonzero(utility[i])\n",
    "    n = len(rated[0])\n",
    "    if n != 0:\n",
    "        user[i].avg_r = np.mean(utility[i][rated])\n",
    "    else:\n",
    "        user[i].avg_r = 0.\n",
    "\n",
    "# print utility\n",
    "\n",
    "# Finds the Pearson Correlation Similarity Measure between two users\n",
    "def pcs(x, y,utility):\n",
    "    num = 0.\n",
    "    ct = 0\n",
    "    (den_a , den_b) = (0., 0.)\n",
    "    for i in range(n_clusters_item):\n",
    "        if utility[x-1][i] != 0.0 and utility[y-1][i] != 0.0 :\n",
    "            num += (utility[x-1][i]-user[x-1].avg_r) * (utility[y-1][i]-user[y-1].avg_r)\n",
    "            den_a += (utility[x-1][i]-user[x-1].avg_r) * (utility[x-1][i]-user[x-1].avg_r)\n",
    "            den_b += (utility[y-1][i]-user[y-1].avg_r) * (utility[y-1][i]-user[y-1].avg_r)\n",
    "            ct += 1\n",
    "    den = math.sqrt(den_a) * math.sqrt(den_b)       \n",
    "    if den == 0.:\n",
    "        return 0.\n",
    "    return float(num) / float(den)\n",
    "\n",
    "\n",
    "\n",
    "def number_map_to_sorted_list(map_dict,top_n):\n",
    "    sorted_list = [v[0] for v in sorted(map_dict.iteritems(), key=lambda (k, v): (-v, k))]\n",
    "    return sorted_list[:top_n]\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# Guesses the ratings that user with id, user_id, might give to item with id, i_id.\n",
    "# We will consider the top_n similar users to do this.\n",
    "def guess(user_id, i_id, top_n,utility):\n",
    "    dict = {}\n",
    "    for i in range(n_users):\n",
    "        if i+1 != user_id:\n",
    "            dict[i+1] = pcs(user_id,i+1,utility)\n",
    "\n",
    "    top_n_list = number_map_to_sorted_list(dict,top_n)\n",
    "    # print top_n_list\n",
    "    sum_diff = 0.\n",
    "    count = 0.\n",
    "    for uid in top_n_list:\n",
    "        if utility[uid-1][i_id-1] != 0.0:\n",
    "            sum_diff += utility[uid-1][i_id-1] - user[uid-1].avg_r\n",
    "            count += 1\n",
    "    # print count \n",
    "    # print sum_diff\n",
    "    if count == 0:\n",
    "        return user[user_id-1].avg_r     \n",
    "    guessed_result = user[user_id-1].avg_r + float(sum_diff/count)\n",
    "    if guessed_result < 1.0:\n",
    "        return 1.0\n",
    "    if guessed_result > 5.0:\n",
    "        return 5.0\n",
    "    return guessed_result\n",
    "\n",
    "# to check for predictions using collaborative filtering\n",
    "# guesses = []\n",
    "# actual_rating = []\n",
    "# for ratings_tests in rating_test:\n",
    "#     guessed = guess(ratings_tests.user_id,ratings_tests.item_id,150)\n",
    "#     print \"user-id : \"+str(ratings_tests.user_id)+ \" item-id : \"+str(ratings_tests.item_id) +  \" expected-rating : \"+str(ratings_tests.rating) +  \" guessed-rating : \"+str(guessed)\n",
    "#     guesses.append(guessed)\n",
    "#     actual_rating.append(ratings_tests.rating)\n",
    "\n",
    "#print 'Mean Squared Error using collaborative-filtering is ' + str(mean_squared_error(guesses, actual_rating))\n",
    "## THINGS THAT YOU WILL NEED TO DO:\n",
    "# Perform clustering on users and items\n",
    "# Predict the ratings of the user-item pairs in rating_test\n",
    "# Find mean-squared error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[105  28 261 115  35   0  63  92   0 218 288 288   1  68   0 228  57   0\n",
      "   0  68 127 389 267  64   1   1 235  46  99   0]\n"
     ]
    }
   ],
   "source": [
    "X= []\n",
    "for items in item:\n",
    "    features = [items.unknown, items.action, items.adventure, items.animation, items.childrens, items.comedy, items.crime, items.documentary, items.drama, items.fantasy, items.film_noir, items.horror, items.musical, items.mystery, items.romance, items.sci_fi, items.thriller, items.war, items.western]\n",
    "    X.append(features)\n",
    "# X = np.array(X)    \n",
    "kmeans = KMeans(n_clusters=400).fit(X)\n",
    "clustered_label = kmeans.labels_\n",
    "n_clusters_item = 400\n",
    "print clustered_label[0:30]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[4.18918919 3.58333333 0.         4.         3.         0.\n",
      " 1.         4.8        0.         0.         4.         0.\n",
      " 0.         2.         0.         0.         2.5        0.\n",
      " 0.         0.         2.33333333 0.         0.         4.\n",
      " 0.         2.         4.5        0.         3.         0.\n",
      " 0.         0.         0.         0.         0.         4.\n",
      " 4.         4.         0.         4.5        0.         0.\n",
      " 0.         0.         3.5        0.         3.         0.\n",
      " 3.         0.         5.         3.         0.         0.\n",
      " 5.         0.         0.         3.         0.         0.\n",
      " 0.         0.         0.         4.5        2.         5.\n",
      " 0.         2.5        3.5        0.         4.         4.\n",
      " 1.         0.         4.         4.         3.         0.\n",
      " 0.         4.         0.         0.         4.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         4.         1.         0.         0.         4.\n",
      " 0.         0.         3.         1.         4.         4.\n",
      " 0.         0.         5.         5.         0.         5.\n",
      " 5.         0.         0.         0.         0.         0.\n",
      " 0.         2.         2.         5.         0.         5.\n",
      " 0.         0.         0.         0.         4.         4.\n",
      " 0.         1.         0.         0.         5.         0.\n",
      " 0.         0.         0.         0.         0.         5.\n",
      " 4.         0.         1.         0.         0.         0.\n",
      " 0.         0.         4.         0.         0.         4.\n",
      " 4.         5.         0.         0.         3.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 1.         0.         1.         4.         5.         0.\n",
      " 4.         2.         0.         3.         0.         3.\n",
      " 2.         0.         5.         4.         0.         0.\n",
      " 0.         3.         0.         0.         0.         0.\n",
      " 0.         0.         4.         4.         3.         2.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 5.         0.         0.         0.         0.         0.\n",
      " 4.         0.         4.         4.         0.         0.\n",
      " 0.         0.         3.66666667 0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 3.83333333 0.         0.         0.         1.625      0.\n",
      " 0.         1.66666667 0.         0.         0.         0.\n",
      " 0.         0.         0.         3.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         1.         0.         0.\n",
      " 0.         0.         0.         4.         4.         0.\n",
      " 0.         0.         0.         4.28571429 0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         4.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 3.66666667 0.         0.         0.         0.         5.\n",
      " 0.         0.         0.         0.         3.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         4.14285714 0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         4.14285714 0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         3.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         5.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.         0.         3.5\n",
      " 0.         0.         0.         0.         0.         0.\n",
      " 0.         0.         0.         0.        ]\n"
     ]
    }
   ],
   "source": [
    "utility_item_clustered = np.zeros((n_users, n_clusters_item))\n",
    "for user_id in range(n_users):\n",
    "    label_dict_sum = {}\n",
    "    label_dict_count = {}\n",
    "    for cluster_id in range(n_clusters_item):\n",
    "        label_dict_count[cluster_id] = 0\n",
    "        label_dict_sum[cluster_id] = 0.0\n",
    "    for item_id in range(n_items):\n",
    "        if utility[user_id][item_id] != 0.0:\n",
    "            label_dict_sum[clustered_label[item_id]] += utility[user_id][item_id]\n",
    "            label_dict_count[clustered_label[item_id]] += 1\n",
    "    for cluster_id in range(n_clusters_item):\n",
    "        if label_dict_count[cluster_id] != 0:\n",
    "            utility_item_clustered[user_id][cluster_id] = float(label_dict_sum[cluster_id]/label_dict_count[cluster_id])\n",
    "print utility_item_clustered[0] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "# kmeans_user = KMeans(n_clusters=300).fit(utility_item_clustered) #clustering users\n",
    "# clustered_label_user = kmeans_user.labels_\n",
    "# n_clusters_user = 300\n",
    "# print clustered_label_user[0:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print len(clustered_label_user)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "# utility_user_clustered = np.zeros((n_clusters_user, n_clusters_item))\n",
    "# for i in range(n_clusters_item):\n",
    "#     label_sum = {}   #sum for each label of users\n",
    "#     label_count = {}\n",
    "#     for j in range(n_clusters_user):\n",
    "#         label_sum[j] = 0.0\n",
    "#         label_count[j] = 0\n",
    "#     for k in range(n_users):\n",
    "#         if utility_item_clustered[k][i] != 0.0:\n",
    "#             label_sum[clustered_label_user[k]] += utility_item_clustered[k][i]\n",
    "#             label_count[clustered_label_user[k]] += 1\n",
    "#     for v in range(n_clusters_user):\n",
    "#         if label_count[v] != 0:\n",
    "#             utility_user_clustered[v][i] = float(label_sum[v] / label_count[v])        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print len(utility_user_clustered)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Find the average rating(of clusters) for each user and stores it in the user's object\n",
    "for i in range(0, n_users):\n",
    "    x = utility_item_clustered[i]\n",
    "    user[i].avg_r = sum(a for a in x if a > 0) / sum(a > 0 for a in x)  \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Guessing [User:Rating] = [13:241]"
     ]
    }
   ],
   "source": [
    "utility_copy = np.copy(utility_item_clustered)   # to fill the empty values in the clustered utility2 matrix\n",
    "for i in range(0, n_users):\n",
    "    for j in range(0, n_clusters_item):\n",
    "        if utility_copy[i][j] == 0:\n",
    "            sys.stdout.write(\"\\rGuessing [User:Rating] = [%d:%d]\" % (i, j))\n",
    "            sys.stdout.flush()\n",
    "            time.sleep(0.00005)\n",
    "            utility_copy[i][j] = guess(i+1, j+1, 155,utility_item_clustered)\n",
    "print \"\\rGuessing [User:Rating] = [%d:%d]\" % (i, j)\n",
    "\n",
    "print utility_copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Squared Error using clustering: 1.230389\n"
     ]
    }
   ],
   "source": [
    "test = np.zeros((n_users, n_items))\n",
    "for r in rating_test:\n",
    "    test[r.user_id - 1][r.item_id - 1] = r.rating\n",
    "\n",
    "# Predict ratings for u.test and find the mean squared error\n",
    "y_true = []\n",
    "y_pred = []\n",
    "f = open('test_prediction.txt', 'w')\n",
    "for i in range(0, n_users):\n",
    "    for j in range(0, n_items):\n",
    "        if test[i][j] > 0:\n",
    "            f.write(\"%d, %d, %.5f\\n\" % (i+1, j+1, utility_copy[i][clustered_label[j]]))\n",
    "            y_true.append(test[i][j])\n",
    "            y_pred.append(utility_copy[i][clustered_label[j]])\n",
    "f.close()\n",
    "\n",
    "print \"Mean Squared Error using clustering: %f\" % mean_squared_error(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}