Geohack112_eelvira.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# МТС Geohack.112"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Построение таблицы признаков"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lat_bl</th>\n",
       "      <th>lon_bl</th>\n",
       "      <th>lat_tr</th>\n",
       "      <th>lon_tr</th>\n",
       "      <th>lat_c</th>\n",
       "      <th>lon_c</th>\n",
       "      <th>is_test</th>\n",
       "      <th>is_target</th>\n",
       "      <th>calls_daily</th>\n",
       "      <th>calls_workday</th>\n",
       "      <th>calls_weekend</th>\n",
       "      <th>calls_wd0</th>\n",
       "      <th>calls_wd1</th>\n",
       "      <th>calls_wd2</th>\n",
       "      <th>calls_wd3</th>\n",
       "      <th>calls_wd4</th>\n",
       "      <th>calls_wd5</th>\n",
       "      <th>calls_wd6</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>zone_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>55.378220</td>\n",
       "      <td>36.999105</td>\n",
       "      <td>55.382744</td>\n",
       "      <td>37.007050</td>\n",
       "      <td>55.380482</td>\n",
       "      <td>37.003077</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.002177</td>\n",
       "      <td>0.001325</td>\n",
       "      <td>0.004213</td>\n",
       "      <td>0.003006</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>6.885802e-04</td>\n",
       "      <td>0.003006</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.001374</td>\n",
       "      <td>0.007052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>55.378252</td>\n",
       "      <td>37.006994</td>\n",
       "      <td>55.382775</td>\n",
       "      <td>37.014938</td>\n",
       "      <td>55.380514</td>\n",
       "      <td>37.010966</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000996</td>\n",
       "      <td>0.000535</td>\n",
       "      <td>0.002096</td>\n",
       "      <td>0.000765</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.176913e-03</td>\n",
       "      <td>0.000765</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.001374</td>\n",
       "      <td>0.002818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>55.378284</td>\n",
       "      <td>37.014883</td>\n",
       "      <td>55.382806</td>\n",
       "      <td>37.022827</td>\n",
       "      <td>55.380545</td>\n",
       "      <td>37.018855</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001284</td>\n",
       "      <td>0.000724</td>\n",
       "      <td>0.002622</td>\n",
       "      <td>0.001322</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.020885e-03</td>\n",
       "      <td>0.001322</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.001374</td>\n",
       "      <td>0.003871</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>55.378315</td>\n",
       "      <td>37.022772</td>\n",
       "      <td>55.382837</td>\n",
       "      <td>37.030715</td>\n",
       "      <td>55.380576</td>\n",
       "      <td>37.026744</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000968</td>\n",
       "      <td>0.000476</td>\n",
       "      <td>0.002143</td>\n",
       "      <td>0.000815</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>7.802124e-04</td>\n",
       "      <td>0.000815</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.001374</td>\n",
       "      <td>0.002912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>55.378345</td>\n",
       "      <td>37.030662</td>\n",
       "      <td>55.382867</td>\n",
       "      <td>37.038603</td>\n",
       "      <td>55.380606</td>\n",
       "      <td>37.034633</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000842</td>\n",
       "      <td>0.000310</td>\n",
       "      <td>0.002113</td>\n",
       "      <td>0.000783</td>\n",
       "      <td>1.821651e-07</td>\n",
       "      <td>7.286604e-07</td>\n",
       "      <td>0.000782</td>\n",
       "      <td>8.602241e-07</td>\n",
       "      <td>0.001374</td>\n",
       "      <td>0.002851</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            lat_bl     lon_bl     lat_tr     lon_tr      lat_c      lon_c  \\\n",
       "zone_id                                                                     \n",
       "0        55.378220  36.999105  55.382744  37.007050  55.380482  37.003077   \n",
       "1        55.378252  37.006994  55.382775  37.014938  55.380514  37.010966   \n",
       "2        55.378284  37.014883  55.382806  37.022827  55.380545  37.018855   \n",
       "3        55.378315  37.022772  55.382837  37.030715  55.380576  37.026744   \n",
       "4        55.378345  37.030662  55.382867  37.038603  55.380606  37.034633   \n",
       "\n",
       "         is_test  is_target  calls_daily  calls_workday  calls_weekend  \\\n",
       "zone_id                                                                  \n",
       "0              0        0.0     0.002177       0.001325       0.004213   \n",
       "1              0        0.0     0.000996       0.000535       0.002096   \n",
       "2              0        0.0     0.001284       0.000724       0.002622   \n",
       "3              0        0.0     0.000968       0.000476       0.002143   \n",
       "4              0        0.0     0.000842       0.000310       0.002113   \n",
       "\n",
       "         calls_wd0     calls_wd1     calls_wd2  calls_wd3     calls_wd4  \\\n",
       "zone_id                                                                   \n",
       "0         0.003006  0.000000e+00  6.885802e-04   0.003006  0.000000e+00   \n",
       "1         0.000765  0.000000e+00  1.176913e-03   0.000765  0.000000e+00   \n",
       "2         0.001322  0.000000e+00  1.020885e-03   0.001322  0.000000e+00   \n",
       "3         0.000815  0.000000e+00  7.802124e-04   0.000815  0.000000e+00   \n",
       "4         0.000783  1.821651e-07  7.286604e-07   0.000782  8.602241e-07   \n",
       "\n",
       "         calls_wd5  calls_wd6  \n",
       "zone_id                        \n",
       "0         0.001374   0.007052  \n",
       "1         0.001374   0.002818  \n",
       "2         0.001374   0.003871  \n",
       "3         0.001374   0.002912  \n",
       "4         0.001374   0.002851  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import collections\n",
    "import warnings\n",
    "\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "df_zones = pd.read_csv('zones.csv', index_col='zone_id')\n",
    "df_zones.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>zone_id</th>\n",
       "      <th>lat_bl</th>\n",
       "      <th>lon_bl</th>\n",
       "      <th>lat_tr</th>\n",
       "      <th>lon_tr</th>\n",
       "      <th>lat_c</th>\n",
       "      <th>lon_c</th>\n",
       "      <th>is_test</th>\n",
       "      <th>is_target</th>\n",
       "      <th>calls_daily</th>\n",
       "      <th>calls_workday</th>\n",
       "      <th>calls_weekend</th>\n",
       "      <th>calls_wd0</th>\n",
       "      <th>calls_wd1</th>\n",
       "      <th>calls_wd2</th>\n",
       "      <th>calls_wd3</th>\n",
       "      <th>calls_wd4</th>\n",
       "      <th>calls_wd5</th>\n",
       "      <th>calls_wd6</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>55.378220</td>\n",
       "      <td>36.999105</td>\n",
       "      <td>55.382744</td>\n",
       "      <td>37.007050</td>\n",
       "      <td>55.380482</td>\n",
       "      <td>37.003077</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.002177</td>\n",
       "      <td>0.001325</td>\n",
       "      <td>0.004213</td>\n",
       "      <td>0.003006</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>6.885802e-04</td>\n",
       "      <td>0.003006</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.001374</td>\n",
       "      <td>0.007052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>55.378252</td>\n",
       "      <td>37.006994</td>\n",
       "      <td>55.382775</td>\n",
       "      <td>37.014938</td>\n",
       "      <td>55.380514</td>\n",
       "      <td>37.010966</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000996</td>\n",
       "      <td>0.000535</td>\n",
       "      <td>0.002096</td>\n",
       "      <td>0.000765</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.176913e-03</td>\n",
       "      <td>0.000765</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.001374</td>\n",
       "      <td>0.002818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>55.378284</td>\n",
       "      <td>37.014883</td>\n",
       "      <td>55.382806</td>\n",
       "      <td>37.022827</td>\n",
       "      <td>55.380545</td>\n",
       "      <td>37.018855</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001284</td>\n",
       "      <td>0.000724</td>\n",
       "      <td>0.002622</td>\n",
       "      <td>0.001322</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.020885e-03</td>\n",
       "      <td>0.001322</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.001374</td>\n",
       "      <td>0.003871</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>55.378315</td>\n",
       "      <td>37.022772</td>\n",
       "      <td>55.382837</td>\n",
       "      <td>37.030715</td>\n",
       "      <td>55.380576</td>\n",
       "      <td>37.026744</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000968</td>\n",
       "      <td>0.000476</td>\n",
       "      <td>0.002143</td>\n",
       "      <td>0.000815</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>7.802124e-04</td>\n",
       "      <td>0.000815</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.001374</td>\n",
       "      <td>0.002912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>55.378345</td>\n",
       "      <td>37.030662</td>\n",
       "      <td>55.382867</td>\n",
       "      <td>37.038603</td>\n",
       "      <td>55.380606</td>\n",
       "      <td>37.034633</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000842</td>\n",
       "      <td>0.000310</td>\n",
       "      <td>0.002113</td>\n",
       "      <td>0.000783</td>\n",
       "      <td>1.821651e-07</td>\n",
       "      <td>7.286604e-07</td>\n",
       "      <td>0.000782</td>\n",
       "      <td>8.602241e-07</td>\n",
       "      <td>0.001374</td>\n",
       "      <td>0.002851</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   zone_id     lat_bl     lon_bl     lat_tr     lon_tr      lat_c      lon_c  \\\n",
       "0        0  55.378220  36.999105  55.382744  37.007050  55.380482  37.003077   \n",
       "1        1  55.378252  37.006994  55.382775  37.014938  55.380514  37.010966   \n",
       "2        2  55.378284  37.014883  55.382806  37.022827  55.380545  37.018855   \n",
       "3        3  55.378315  37.022772  55.382837  37.030715  55.380576  37.026744   \n",
       "4        4  55.378345  37.030662  55.382867  37.038603  55.380606  37.034633   \n",
       "\n",
       "   is_test  is_target  calls_daily  calls_workday  calls_weekend  calls_wd0  \\\n",
       "0        0        0.0     0.002177       0.001325       0.004213   0.003006   \n",
       "1        0        0.0     0.000996       0.000535       0.002096   0.000765   \n",
       "2        0        0.0     0.001284       0.000724       0.002622   0.001322   \n",
       "3        0        0.0     0.000968       0.000476       0.002143   0.000815   \n",
       "4        0        0.0     0.000842       0.000310       0.002113   0.000783   \n",
       "\n",
       "      calls_wd1     calls_wd2  calls_wd3     calls_wd4  calls_wd5  calls_wd6  \n",
       "0  0.000000e+00  6.885802e-04   0.003006  0.000000e+00   0.001374   0.007052  \n",
       "1  0.000000e+00  1.176913e-03   0.000765  0.000000e+00   0.001374   0.002818  \n",
       "2  0.000000e+00  1.020885e-03   0.001322  0.000000e+00   0.001374   0.003871  \n",
       "3  0.000000e+00  7.802124e-04   0.000815  0.000000e+00   0.001374   0.002912  \n",
       "4  1.821651e-07  7.286604e-07   0.000782  8.602241e-07   0.001374   0.002851  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Плотность населения Москвы можно приблизительно оценить, зная площадь жилой недвижимости (данные взяты с сайта www.reformagkh.ru)\n",
    "df_housedata = pd.read_csv('housedata.csv')\n",
    "\n",
    "df_housedata.loc[df_housedata.AREA_LIVE == 'Незаполнено', 'AREA_LIVE' ] = 0\n",
    "df_housedata.AREA_LIVE = df_housedata.AREA_LIVE.apply(lambda x: float(x))\n",
    "\n",
    "df_housedata.loc[df_housedata.AREA_NONLIVE == 'Незаполнено', 'AREA_NONLIVE' ] = 0\n",
    "df_housedata.AREA_NONLIVE = df_housedata.AREA_NONLIVE.apply(lambda x: float(x))\n",
    "\n",
    "df_housedata.loc[df_housedata.ROOM_COUNT_NONLIVE == 'Не заполнено', 'ROOM_COUNT_NONLIVE' ] = 0\n",
    "df_housedata.ROOM_COUNT_NONLIVE = df_housedata.ROOM_COUNT_NONLIVE.apply(lambda x: int(str(x).replace(' ', '')))\n",
    "\n",
    "df_housedata.loc[df_housedata.ROOM_COUNT_LIVE == 'Не заполнено', 'ROOM_COUNT_LIVE' ] = 0\n",
    "df_housedata.ROOM_COUNT_LIVE = df_housedata.ROOM_COUNT_LIVE.apply(lambda x: int(str(x).replace(' ', '')))\n",
    "\n",
    "df_zones_train = df_zones\n",
    "df_zones_train.sort_values(by='lat_bl')\n",
    "df_housedata = df_housedata.sort_values(by='LAT')\n",
    "              \n",
    "df_zones_train.reset_index(inplace=True)\n",
    "\n",
    "df_zones_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "22172it [03:30, 105.17it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "df_zones_train['live_amount'] = 0\n",
    "df_zones_train['nonlive_amount'] = 0\n",
    "df_zones_train['room_live'] = 0\n",
    "df_zones_train['room_nonlive'] = 0\n",
    "\n",
    "for i, row in tqdm(df_zones_train.iterrows()):\n",
    "    tmp = df_housedata.query('(LON >= @row.lon_bl) & (LON < @row.lon_tr)')\n",
    "    tmp = tmp.query('(LAT >= @row.lat_bl) & (LAT < @row.lat_tr)')\n",
    "    if tmp.shape[0]:\n",
    "        df_zones_train.loc[i, 'live_amount'] = tmp.AREA_LIVE.sum()\n",
    "        df_zones_train.loc[i, 'nonlive_amount'] = tmp.AREA_NONLIVE.sum()\n",
    "        df_zones_train.loc[i, 'room_live'] = tmp.ROOM_COUNT_LIVE.sum()\n",
    "        df_zones_train.loc[i, 'room_nonlive'] = tmp.ROOM_COUNT_NONLIVE.sum()        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>zone_id</th>\n",
       "      <th>distance_to_kremlin</th>\n",
       "      <th>tagged_points_in_0.001</th>\n",
       "      <th>tagged_points_in_0.003</th>\n",
       "      <th>tagged_points_in_0.005</th>\n",
       "      <th>tagged_points_in_0.007</th>\n",
       "      <th>tagged_points_in_0.01</th>\n",
       "      <th>tagged_mean_dist_k_3</th>\n",
       "      <th>tagged_max_dist_k_3</th>\n",
       "      <th>tagged_std_dist_k_3</th>\n",
       "      <th>...</th>\n",
       "      <th>public_transport_std_dist_k_5</th>\n",
       "      <th>public_transport_mean_dist_k_10</th>\n",
       "      <th>public_transport_max_dist_k_10</th>\n",
       "      <th>public_transport_std_dist_k_10</th>\n",
       "      <th>public_transport_min</th>\n",
       "      <th>room_nonlive</th>\n",
       "      <th>room_live</th>\n",
       "      <th>calls_daily</th>\n",
       "      <th>is_test</th>\n",
       "      <th>is_target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>56.852292</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0.006777</td>\n",
       "      <td>0.015403</td>\n",
       "      <td>0.006120</td>\n",
       "      <td>...</td>\n",
       "      <td>0.019011</td>\n",
       "      <td>0.102931</td>\n",
       "      <td>0.119075</td>\n",
       "      <td>0.019160</td>\n",
       "      <td>0.066160</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.002177</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>56.511876</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0.008668</td>\n",
       "      <td>0.010560</td>\n",
       "      <td>0.001347</td>\n",
       "      <td>...</td>\n",
       "      <td>0.018316</td>\n",
       "      <td>0.104594</td>\n",
       "      <td>0.124896</td>\n",
       "      <td>0.019435</td>\n",
       "      <td>0.069525</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000996</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>56.173822</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.013769</td>\n",
       "      <td>0.015696</td>\n",
       "      <td>0.002323</td>\n",
       "      <td>...</td>\n",
       "      <td>0.014751</td>\n",
       "      <td>0.106627</td>\n",
       "      <td>0.130933</td>\n",
       "      <td>0.020701</td>\n",
       "      <td>0.073586</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.001284</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>55.838172</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.020577</td>\n",
       "      <td>0.023549</td>\n",
       "      <td>0.003754</td>\n",
       "      <td>...</td>\n",
       "      <td>0.011964</td>\n",
       "      <td>0.108593</td>\n",
       "      <td>0.133372</td>\n",
       "      <td>0.022350</td>\n",
       "      <td>0.078232</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000968</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>55.504970</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.026699</td>\n",
       "      <td>0.030437</td>\n",
       "      <td>0.003545</td>\n",
       "      <td>...</td>\n",
       "      <td>0.010822</td>\n",
       "      <td>0.109584</td>\n",
       "      <td>0.137651</td>\n",
       "      <td>0.023498</td>\n",
       "      <td>0.071765</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000842</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 67 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   zone_id  distance_to_kremlin  tagged_points_in_0.001  \\\n",
       "0        0            56.852292                       0   \n",
       "1        1            56.511876                       0   \n",
       "2        2            56.173822                       0   \n",
       "3        3            55.838172                       0   \n",
       "4        4            55.504970                       0   \n",
       "\n",
       "   tagged_points_in_0.003  tagged_points_in_0.005  tagged_points_in_0.007  \\\n",
       "0                       1                       2                       2   \n",
       "1                       0                       0                       0   \n",
       "2                       0                       0                       0   \n",
       "3                       0                       0                       0   \n",
       "4                       0                       0                       0   \n",
       "\n",
       "   tagged_points_in_0.01  tagged_mean_dist_k_3  tagged_max_dist_k_3  \\\n",
       "0                      2              0.006777             0.015403   \n",
       "1                      2              0.008668             0.010560   \n",
       "2                      0              0.013769             0.015696   \n",
       "3                      0              0.020577             0.023549   \n",
       "4                      0              0.026699             0.030437   \n",
       "\n",
       "   tagged_std_dist_k_3    ...      public_transport_std_dist_k_5  \\\n",
       "0             0.006120    ...                           0.019011   \n",
       "1             0.001347    ...                           0.018316   \n",
       "2             0.002323    ...                           0.014751   \n",
       "3             0.003754    ...                           0.011964   \n",
       "4             0.003545    ...                           0.010822   \n",
       "\n",
       "   public_transport_mean_dist_k_10  public_transport_max_dist_k_10  \\\n",
       "0                         0.102931                        0.119075   \n",
       "1                         0.104594                        0.124896   \n",
       "2                         0.106627                        0.130933   \n",
       "3                         0.108593                        0.133372   \n",
       "4                         0.109584                        0.137651   \n",
       "\n",
       "   public_transport_std_dist_k_10  public_transport_min  room_nonlive  \\\n",
       "0                        0.019160              0.066160             0   \n",
       "1                        0.019435              0.069525             0   \n",
       "2                        0.020701              0.073586             0   \n",
       "3                        0.022350              0.078232             0   \n",
       "4                        0.023498              0.071765             0   \n",
       "\n",
       "   room_live  calls_daily  is_test  is_target  \n",
       "0          0     0.002177        0        0.0  \n",
       "1          0     0.000996        0        0.0  \n",
       "2          0     0.001284        0        0.0  \n",
       "3          0     0.000968        0        0.0  \n",
       "4          0     0.000842        0        0.0  \n",
       "\n",
       "[5 rows x 67 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Также будем использовать признаки, которые были построены на основе координат квадратов и данных из OpenStreetMap (из Geohack112_StarterKit.ipynb)\n",
    "df_features = pd.read_csv('features.csv')\n",
    "\n",
    "# Добавим дополнительные признаки к уже имеющимся\n",
    "df_zoness = df_zones_train[['zone_id', 'room_nonlive', 'room_live', 'calls_daily', 'is_test', 'is_target']]\n",
    "df_features = pd.merge(df_features, df_zoness, on='zone_id')\n",
    "df_features.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Предсказание числа вызовов"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Обучающая выборка"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "df_zones_tr = df_features.query('is_test == 0 & is_target == 1')\n",
    "\n",
    "idx_train, idx_valid = train_test_split(df_zones_tr.index, test_size=0.3, random_state=17)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=None,\n",
       "           max_features='auto', max_leaf_nodes=None,\n",
       "           min_impurity_split=1e-07, min_samples_leaf=1,\n",
       "           min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "           n_estimators=240, n_jobs=-1, oob_score=False, random_state=42,\n",
       "           verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor\n",
    "X_train = df_zones_tr.loc[idx_train, :].drop(['calls_daily', 'zone_id', 'is_test', 'is_target'], axis=1)\n",
    "y_train = df_zones_tr.loc[idx_train, 'calls_daily']\n",
    "\n",
    "model = RandomForestRegressor(n_estimators=240, n_jobs=-1, random_state=42)\n",
    "\n",
    "model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Пример таблицы с предсказаниями (для валидационной выборки)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from scipy.stats import kendalltau\n",
    "\n",
    "X_valid = df_zones_tr.loc[idx_valid, :].drop(['calls_daily', 'zone_id', 'is_test', 'is_target' ], axis=1)\n",
    "y_valid = df_zones_tr.loc[idx_valid, 'calls_daily']\n",
    "\n",
    "y_pred = model.predict(X_valid)\n",
    "\n",
    "target_columns = ['calls_wd{}'.format(d) for d in range(7)]\n",
    "\n",
    "df_valid_target = df_zones_train.loc[idx_valid, target_columns]\n",
    "df_valid_predictions = pd.DataFrame(collections.OrderedDict([\n",
    "    (column_name, y_pred)\n",
    "    for column_name in target_columns\n",
    "]), index=idx_valid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Оценка качества предсказаний на валидационной выборке"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation score: 0.680034993708\n"
     ]
    }
   ],
   "source": [
    "df_comparison = pd.DataFrame({\n",
    "    'target': df_valid_target.unstack(),\n",
    "    'prediction': df_valid_predictions.unstack(),\n",
    "})\n",
    "\n",
    "valid_score = kendalltau(df_comparison['target'], df_comparison['prediction']).correlation\n",
    "print('Validation score:', valid_score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Тестовая выборка"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>calls_wd0</th>\n",
       "      <th>calls_wd1</th>\n",
       "      <th>calls_wd2</th>\n",
       "      <th>calls_wd3</th>\n",
       "      <th>calls_wd4</th>\n",
       "      <th>calls_wd5</th>\n",
       "      <th>calls_wd6</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>0.016428</td>\n",
       "      <td>0.016428</td>\n",
       "      <td>0.016428</td>\n",
       "      <td>0.016428</td>\n",
       "      <td>0.016428</td>\n",
       "      <td>0.016428</td>\n",
       "      <td>0.016428</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>0.020703</td>\n",
       "      <td>0.020703</td>\n",
       "      <td>0.020703</td>\n",
       "      <td>0.020703</td>\n",
       "      <td>0.020703</td>\n",
       "      <td>0.020703</td>\n",
       "      <td>0.020703</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>0.047453</td>\n",
       "      <td>0.047453</td>\n",
       "      <td>0.047453</td>\n",
       "      <td>0.047453</td>\n",
       "      <td>0.047453</td>\n",
       "      <td>0.047453</td>\n",
       "      <td>0.047453</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>0.055695</td>\n",
       "      <td>0.055695</td>\n",
       "      <td>0.055695</td>\n",
       "      <td>0.055695</td>\n",
       "      <td>0.055695</td>\n",
       "      <td>0.055695</td>\n",
       "      <td>0.055695</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>0.061356</td>\n",
       "      <td>0.061356</td>\n",
       "      <td>0.061356</td>\n",
       "      <td>0.061356</td>\n",
       "      <td>0.061356</td>\n",
       "      <td>0.061356</td>\n",
       "      <td>0.061356</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    calls_wd0  calls_wd1  calls_wd2  calls_wd3  calls_wd4  calls_wd5  \\\n",
       "79   0.016428   0.016428   0.016428   0.016428   0.016428   0.016428   \n",
       "80   0.020703   0.020703   0.020703   0.020703   0.020703   0.020703   \n",
       "81   0.047453   0.047453   0.047453   0.047453   0.047453   0.047453   \n",
       "82   0.055695   0.055695   0.055695   0.055695   0.055695   0.055695   \n",
       "83   0.061356   0.061356   0.061356   0.061356   0.061356   0.061356   \n",
       "\n",
       "    calls_wd6  \n",
       "79   0.016428  \n",
       "80   0.020703  \n",
       "81   0.047453  \n",
       "82   0.055695  \n",
       "83   0.061356  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idx_test = df_features.query('is_test == 1').index\n",
    "\n",
    "X_test = df_features.loc[idx_test, :].drop([ 'calls_daily', 'zone_id', 'is_test', 'is_target'], axis=1)\n",
    "y_pred = model.predict(X_test)\n",
    "\n",
    "df_test_predictions = pd.DataFrame(collections.OrderedDict([\n",
    "    (column_name, y_pred)\n",
    "    for column_name in target_columns\n",
    "]), index=idx_test)\n",
    "\n",
    "df_test_predictions.to_csv('sample_submission.csv')\n",
    "df_test_predictions.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}