dataset_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import numpy as np\n",
    "import sys\n",
    "import sklearn\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.linear_model import LinearRegression, Lasso\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.feature_selection import VarianceThreshold\n",
    "import pandas as pd\n",
    "from os.path import join\n",
    "from src.dump_results import dump\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "from src.pre_process import process_level2, load_binary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "data = load_binary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analysing the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/tracking.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# df.columns = ['user_id', 'symptom', 'day_in_cycle', 'probability']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "df2 = pd.DataFrame(df.groupby('day_in_cycle').symptom.count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Happy\n",
    "df2.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import scipy as sp\n",
    "def logloss(act, pred):\n",
    "    \"\"\" Vectorised computation of logloss \"\"\"\n",
    "\n",
    "    #cap in official Kaggle implementation,\n",
    "    #per forums/t/1576/r-code-for-logloss\n",
    "    epsilon = 1e-15\n",
    "    pred = sp.maximum(epsilon, pred)\n",
    "    pred = sp.minimum(1-epsilon, pred)\n",
    "\n",
    "    #compute logloss function (vectorised)\n",
    "    ll = sum(   act*sp.log(pred) +\n",
    "                sp.subtract(1,act)*sp.log(sp.subtract(1,pred)))\n",
    "    ll = ll * -1.0/len(act)\n",
    "    return ll"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Experimenting with building a model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Extract features\n",
    "user_feat_matrix = process_level2(data)  # X\n",
    "\n",
    "del user_feat_matrix['X']['user_id']\n",
    "X = user_feat_matrix['X'].values\n",
    "X[np.isnan(X)] = 0\n",
    "Y = user_feat_matrix['Y']\n",
    "Y.fillna(0, inplace=True)\n",
    "del user_feat_matrix['X_all']['user_id']\n",
    "X_all = user_feat_matrix['X_all'].values\n",
    "X_all[np.isnan(X_all)] = 0\n",
    "\n",
    "cols = list(Y.columns.values)\n",
    "symptoms = ['happy', 'pms', 'sad', 'sensitive_emotion', 'energized', 'exhausted',\n",
    "            'high_energy', 'low_energy', 'cramps', 'headache', 'ovulation_pain',\n",
    "            'tender_breasts', 'acne_skin', 'good_skin', 'oily_skin', 'dry_skin']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "symptoms = ['cramps']\n",
    "for symptom in symptoms:\n",
    "    print(symptom)\n",
    "    s_Y = Y[[x for x in cols if x[1] == symptom]]\n",
    "    print(\"Lasso\")\n",
    "    pipeline = Pipeline([\n",
    "        ('remove_low_variance_features', VarianceThreshold(threshold=0.0)),\n",
    "        ('standard_scale', StandardScaler()),\n",
    "        ('estimator', Lasso()),\n",
    "    ])\n",
    "\n",
    "    model = GridSearchCV(pipeline,\n",
    "                         {'estimator__alpha': [0.5]},\n",
    "                         verbose=10,\n",
    "                         cv=2\n",
    "             )\n",
    "\n",
    "    model.fit(X, s_Y.values)\n",
    "\n",
    "    model.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.multioutput import MultiOutputRegressor\n",
    "from sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "symptoms = ['cramps']\n",
    "for symptom in symptoms:\n",
    "    print(symptom)\n",
    "    s_Y = Y[[x for x in cols if x[1] == symptom]]\n",
    "    pipeline = Pipeline([\n",
    "        ('remove_low_variance_features', VarianceThreshold(threshold=0.0)),\n",
    "        ('standard_scale', StandardScaler()),        \n",
    "        ('estimator', DecisionTreeRegressor(max_depth=5))\n",
    "    ])\n",
    "\n",
    "    param_grid = {'estimator__max_depth': [3, 5, 7],\n",
    "                  'estimator__max_features': ['auto', 'sqrt', 'log2']}\n",
    "    model = GridSearchCV(pipeline, param_grid = param_grid, n_jobs = 4,\n",
    "                     verbose=2)\n",
    "    model.fit(X, s_Y.values)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.multioutput import MultiOutputRegressor\n",
    "from sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "symptoms = ['cramps']\n",
    "for symptom in symptoms:\n",
    "    print(symptom)\n",
    "    s_Y = Y[[x for x in cols if x[1] == symptom]]\n",
    "    pipeline = Pipeline([\n",
    "        ('remove_low_variance_features', VarianceThreshold(threshold=0.0)),\n",
    "        ('standard_scale', StandardScaler()),        \n",
    "        ('estimator', Lasso())\n",
    "    ])\n",
    "\n",
    "    param_grid = {'estimator__alpha': [0.1, 0.3, 0.5, 0.7]}\n",
    "    model = GridSearchCV(pipeline, param_grid = param_grid,\n",
    "                     verbose=2, cv=2)\n",
    "    model.fit(X, s_Y.values)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "symptoms = ['happy']\n",
    "for symptom in symptoms:\n",
    "    print(symptom)\n",
    "    s_Y = Y[[x for x in cols if x[1] == symptom]]\n",
    "    print(\"Lasso\")\n",
    "    pipeline = Pipeline([\n",
    "        ('remove_low_variance_features', VarianceThreshold(threshold=0.0)),\n",
    "        ('standard_scale', StandardScaler()),\n",
    "        ('estimator', Lasso()),\n",
    "    ])\n",
    "\n",
    "    model = GridSearchCV(pipeline,\n",
    "                         {'estimator__alpha': np.array(list(range(5))) /5 + 0.1},\n",
    "                         verbose=10,\n",
    "                         cv=2\n",
    "             )\n",
    "\n",
    "    model.fit(X, s_Y.values)\n",
    "\n",
    "    model.best_score_\n",
    "    print(\"dumping...\")\n",
    "    data_dir = 'data'\n",
    "    cycles0 = pd.read_csv(join(data_dir, 'cycles0.csv'))\n",
    "    c_length = {k:v for k,v in zip(cycles0.user_id.values, cycles0.expected_cycle_length)}\n",
    "    %time dump(symptom, pipeline, X_all, c_length, data['users'].user_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}