final_m5_with_GroupKFold.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "final_m5.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "SfCFUMHppL_J",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# General imports\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import os, sys, gc, time, warnings, pickle, psutil, random\n",
        "from sklearn.model_selection import GroupKFold\n",
        "# custom imports\n",
        "from multiprocessing import Pool        # Multiprocess Runs\n",
        "\n",
        "warnings.filterwarnings('ignore')"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PbW8SISU-Hdv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "########################### Helpers\n",
        "#################################################################################\n",
        "## Seeder\n",
        "# :seed to make all processes deterministic     # type: int\n",
        "def seed_everything(seed=0):\n",
        "    random.seed(seed)\n",
        "    np.random.seed(seed)\n",
        "\n",
        "    \n",
        "## Multiprocess Runs\n",
        "def df_parallelize_run(func, t_split):\n",
        "    num_cores = np.min([N_CORES,len(t_split)])\n",
        "    pool = Pool(num_cores)\n",
        "    df = pd.concat(pool.map(func, t_split), axis=1)\n",
        "    pool.close()\n",
        "    pool.join()\n",
        "    return df"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xTYIQ8lpohZr",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        },
        "outputId": "132d336d-3b35-45b6-d904-ae1ab3ed4dc6"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hEI8d26DojDB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "########################### Helper to load data by store ID\n",
        "#################################################################################\n",
        "# Read data\n",
        "def get_data_by_store(store):\n",
        "    \n",
        "    # Read and contact basic feature\n",
        "    df = pd.concat([pd.read_pickle(BASE),\n",
        "                    pd.read_pickle(PRICE).iloc[:,2:],\n",
        "                    pd.read_pickle(CALENDAR).iloc[:,2:]],\n",
        "                    axis=1)\n",
        "    \n",
        "    # Leave only relevant store\n",
        "    df = df[df['store_id']==store]\n",
        "\n",
        "    # With memory limits we have to read \n",
        "    # lags and mean encoding features\n",
        "    # separately and drop items that we don't need.\n",
        "    # As our Features Grids are aligned \n",
        "    # we can use index to keep only necessary rows\n",
        "    # Alignment is good for us as concat uses less memory than merge.\n",
        "    df2 = pd.read_pickle(MEAN_ENC)[mean_features]\n",
        "    df2 = df2[df2.index.isin(df.index)]\n",
        "    \n",
        "    df3 = pd.read_pickle(LAGS).iloc[:,3:]\n",
        "    df3 = df3[df3.index.isin(df.index)]\n",
        "    \n",
        "    df = pd.concat([df, df2], axis=1)\n",
        "    del df2 # to not reach memory limit \n",
        "    \n",
        "    df = pd.concat([df, df3], axis=1)\n",
        "    del df3 # to not reach memory limit \n",
        "    \n",
        "    # Create features list\n",
        "    features = [col for col in list(df) if col not in remove_features]\n",
        "    df = df[['id','d',TARGET]+features]\n",
        "    \n",
        "    # Skipping first n rows\n",
        "    df = df[df['d']>=START_TRAIN].reset_index(drop=True)\n",
        "    \n",
        "    return df, features\n",
        "\n",
        "# Recombine Test set after training\n",
        "def get_base_test():\n",
        "    base_test = pd.DataFrame()\n",
        "\n",
        "    for store_id in STORES_IDS:\n",
        "        temp_df = pd.read_pickle('test_'+store_id+'.pkl')\n",
        "        temp_df['store_id'] = store_id\n",
        "        base_test = pd.concat([base_test, temp_df]).reset_index(drop=True)\n",
        "    \n",
        "    return base_test\n",
        "\n",
        "\n",
        "########################### Helper to make dynamic rolling lags\n",
        "#################################################################################\n",
        "def make_lag(LAG_DAY):\n",
        "    lag_df = base_test[['id','d',TARGET]]\n",
        "    col_name = 'sales_lag_'+str(LAG_DAY)\n",
        "    lag_df[col_name] = lag_df.groupby(['id'])[TARGET].transform(lambda x: x.shift(LAG_DAY)).astype(np.float16)\n",
        "    return lag_df[[col_name]]\n",
        "\n",
        "\n",
        "def make_lag_roll(LAG_DAY):\n",
        "    shift_day = LAG_DAY[0]\n",
        "    roll_wind = LAG_DAY[1]\n",
        "    lag_df = base_test[['id','d',TARGET]]\n",
        "    col_name = 'rolling_mean_tmp_'+str(shift_day)+'_'+str(roll_wind)\n",
        "    lag_df[col_name] = lag_df.groupby(['id'])[TARGET].transform(lambda x: x.shift(shift_day).rolling(roll_wind).mean())\n",
        "    return lag_df[[col_name]]"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9UJzHAoIorHz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "########################### Model params\n",
        "#################################################################################\n",
        "import lightgbm as lgb\n",
        "lgb_params = {\n",
        "                    'boosting_type': 'gbdt',\n",
        "                    'objective': 'tweedie',\n",
        "                    'tweedie_variance_power': 1.1,\n",
        "                    'metric': 'rmse',\n",
        "                    'subsample': 0.5,\n",
        "                    'subsample_freq': 1,\n",
        "                    'learning_rate': 0.03,\n",
        "                    'num_leaves': 2**11-1,\n",
        "                    'min_data_in_leaf': 2**12-1,\n",
        "                    'feature_fraction': 0.5,\n",
        "                    'max_bin': 100,\n",
        "                    'n_estimators': 1400,\n",
        "                    'boost_from_average': False,\n",
        "                    'verbose': -1,\n",
        "                } \n",
        "\n",
        "# Let's look closer on params\n",
        "\n",
        "## 'boosting_type': 'gbdt'\n",
        "# we have 'goss' option for faster training\n",
        "# but it normally leads to underfit.\n",
        "# Also there is good 'dart' mode\n",
        "# but it takes forever to train\n",
        "# and model performance depends \n",
        "# a lot on random factor \n",
        "# https://www.kaggle.com/c/home-credit-default-risk/discussion/60921\n",
        "\n",
        "## 'objective': 'tweedie'\n",
        "# Tweedie Gradient Boosting for Extremely\n",
        "# Unbalanced Zero-inflated Data\n",
        "# https://arxiv.org/pdf/1811.10192.pdf\n",
        "# and many more articles about tweediie\n",
        "#\n",
        "# Strange (for me) but Tweedie is close in results\n",
        "# to my own ugly loss.\n",
        "# My advice here - make OWN LOSS function\n",
        "# https://www.kaggle.com/c/m5-forecasting-accuracy/discussion/140564\n",
        "# https://www.kaggle.com/c/m5-forecasting-accuracy/discussion/143070\n",
        "# I think many of you already using it (after poisson kernel appeared) \n",
        "# (kagglers are very good with \"params\" testing and tuning).\n",
        "# Try to figure out why Tweedie works.\n",
        "# probably it will show you new features options\n",
        "# or data transformation (Target transformation?).\n",
        "\n",
        "## 'tweedie_variance_power': 1.1\n",
        "# default = 1.5\n",
        "# set this closer to 2 to shift towards a Gamma distribution\n",
        "# set this closer to 1 to shift towards a Poisson distribution\n",
        "# my CV shows 1.1 is optimal \n",
        "# but you can make your own choice\n",
        "\n",
        "## 'metric': 'rmse'\n",
        "# Doesn't mean anything to us\n",
        "# as competition metric is different\n",
        "# and we don't use early stoppings here.\n",
        "# So rmse serves just for general \n",
        "# model performance overview.\n",
        "# Also we use \"fake\" validation set\n",
        "# (as it makes part of the training set)\n",
        "# so even general rmse score doesn't mean anything))\n",
        "# https://www.kaggle.com/c/m5-forecasting-accuracy/discussion/133834\n",
        "\n",
        "## 'subsample': 0.5\n",
        "# Serves to fight with overfit\n",
        "# this will randomly select part of data without resampling\n",
        "# Chosen by CV (my CV can be wrong!)\n",
        "# Next kernel will be about CV\n",
        "\n",
        "##'subsample_freq': 1\n",
        "# frequency for bagging\n",
        "# default value - seems ok\n",
        "\n",
        "## 'learning_rate': 0.03\n",
        "# Chosen by CV\n",
        "# Smaller - longer training\n",
        "# but there is an option to stop \n",
        "# in \"local minimum\"\n",
        "# Bigger - faster training\n",
        "# but there is a chance to\n",
        "# not find \"global minimum\" minimum\n",
        "\n",
        "## 'num_leaves': 2**11-1\n",
        "## 'min_data_in_leaf': 2**12-1\n",
        "# Force model to use more features\n",
        "# We need it to reduce \"recursive\"\n",
        "# error impact.\n",
        "# Also it leads to overfit\n",
        "# that's why we use small \n",
        "# 'max_bin': 100\n",
        "\n",
        "## l1, l2 regularizations\n",
        "# https://towardsdatascience.com/l1-and-l2-regularization-methods-ce25e7fc831c\n",
        "# Good tiny explanation\n",
        "# l2 can work with bigger num_leaves\n",
        "# but my CV doesn't show boost\n",
        "                    \n",
        "## 'n_estimators': 1400\n",
        "# CV shows that there should be\n",
        "# different values for each state/store.\n",
        "# Current value was chosen \n",
        "# for general purpose.\n",
        "# As we don't use any early stopings\n",
        "# careful to not overfit Public LB.\n",
        "\n",
        "##'feature_fraction': 0.5\n",
        "# LightGBM will randomly select \n",
        "# part of features on each iteration (tree).\n",
        "# We have maaaany features\n",
        "# and many of them are \"duplicates\"\n",
        "# and many just \"noise\"\n",
        "# good values here - 0.5-0.7 (by CV)\n",
        "\n",
        "## 'boost_from_average': False\n",
        "# There is some \"problem\"\n",
        "# to code boost_from_average for \n",
        "# custom loss\n",
        "# 'True' makes training faster\n",
        "# BUT carefull use it\n",
        "# https://github.com/microsoft/LightGBM/issues/1514\n",
        "# not our case but good to know cons"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vP5qxWbwo-Kf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "########################### Vars\n",
        "#################################################################################\n",
        "VER = 1                          # Our model version\n",
        "SEED = 42                        # We want all things\n",
        "seed_everything(SEED)            # to be as deterministic \n",
        "lgb_params['seed'] = SEED        # as possible\n",
        "N_CORES = psutil.cpu_count()     # Available CPU cores\n",
        "\n",
        "\n",
        "#LIMITS and const\n",
        "TARGET      = 'sales'            # Our target\n",
        "START_TRAIN = 0                  # We can skip some rows (Nans/faster training)\n",
        "END_TRAIN   = 1941               # End day of our train set\n",
        "P_HORIZON   = 28                 # Prediction horizon\n",
        "USE_AUX     = True               # Use or not pretrained models\n",
        "\n",
        "#FEATURES to remove\n",
        "## These features lead to overfit\n",
        "## or values not present in test set\n",
        "remove_features = ['id','state_id','store_id',\n",
        "                   'date','wm_yr_wk','d',TARGET]\n",
        "mean_features   = ['enc_cat_id_mean','enc_cat_id_std',\n",
        "                   'enc_dept_id_mean','enc_dept_id_std',\n",
        "                   'enc_item_id_mean','enc_item_id_std'] \n",
        "\n",
        "#PATHS for Features\n",
        "ORIGINAL = '/content/gdrive/My Drive/m5/'\n",
        "BASE     = '/content/gdrive/My Drive/m5_new/grid_part_1.pkl'\n",
        "PRICE    = '/content/gdrive/My Drive/m5_new/grid_part_2.pkl'\n",
        "CALENDAR = '/content/gdrive/My Drive/m5_new/grid_part_3.pkl'\n",
        "LAGS     = '/content/gdrive/My Drive/m5_new/lags_df_28.pkl'\n",
        "MEAN_ENC = '/content/gdrive/My Drive/m5_new/mean_encoding_df.pkl'\n",
        "\n",
        "\n",
        "# AUX(pretrained) Models paths\n",
        "AUX_MODELS = '/content/gdrive/My Drive/m5_new/'\n",
        "\n",
        "\n",
        "#STORES ids\n",
        "STORES_IDS = pd.read_csv(ORIGINAL+'sales_train_validation.csv')['store_id']\n",
        "STORES_IDS = list(STORES_IDS.unique())\n",
        "\n",
        "#FOLDS\n",
        "CV_FOLDS = [0,1,2,3]\n",
        "\n",
        "\n",
        "#SPLITS for lags creation\n",
        "SHIFT_DAY  = 28\n",
        "N_LAGS     = 15\n",
        "LAGS_SPLIT = [col for col in range(SHIFT_DAY,SHIFT_DAY+N_LAGS)]\n",
        "ROLS_SPLIT = []\n",
        "for i in [1,7,14]:\n",
        "    for j in [7,14,30,60]:\n",
        "        ROLS_SPLIT.append([i,j])"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GQENK0wLIxbu",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "23322b23-5e5d-491d-be93-e5e55d702042"
      },
      "source": [
        "########################### Train Models\n",
        "#################################################################################\n",
        "for store_id in STORES_IDS:\n",
        "    print('Train', store_id)\n",
        "    \n",
        "    # Get grid for current store\n",
        "    grid_df, features_columns = get_data_by_store(store_id)\n",
        "    \n",
        "    # Masks for \n",
        "    # Train (All data less than 1913)\n",
        "    # \"Validation\" (Last 28 days - not real validatio set)\n",
        "    # Test (All data greater than 1913 day, \n",
        "    #       with some gap for recursive features)\n",
        "    train_mask = grid_df['d']<=END_TRAIN\n",
        "    preds_mask = grid_df['d']>(END_TRAIN-100)\n",
        "    \n",
        "    ## Initiating our GroupKFold\n",
        "    folds = GroupKFold(n_splits=4)\n",
        "\n",
        "    # get subgroups for each week, year pair\n",
        "    grid_df['groups'] = grid_df['tm_w'].astype(str) + '_' + grid_df['tm_y'].astype(str)\n",
        "    split_groups = grid_df[train_mask]['groups']\n",
        "\n",
        "    # Main Data\n",
        "    X,y = grid_df[train_mask][features_columns], grid_df[train_mask][TARGET]\n",
        "        \n",
        "    # Saving part of the dataset for later predictions\n",
        "    # Removing features that we need to calculate recursively \n",
        "    grid_df = grid_df[preds_mask].reset_index(drop=True)\n",
        "    keep_cols = [col for col in list(grid_df) if '_tmp_' not in col]\n",
        "    grid_df = grid_df[keep_cols]\n",
        "    grid_df.to_pickle('test_'+store_id+'.pkl')\n",
        "    del grid_df\n",
        "    \n",
        "    # Launch seeder again to make lgb training 100% deterministic\n",
        "    # with each \"code line\" np.random \"evolves\" \n",
        "    # so we need (may want) to \"reset\" it\n",
        "    \n",
        "    for fold_, (trn_idx, val_idx) in enumerate(folds.split(X, y, groups=split_groups)):\n",
        "        print('Fold:',fold_)\n",
        "        print(len(trn_idx),len(val_idx))\n",
        "        tr_x, tr_y = X.iloc[trn_idx,:], y[trn_idx]\n",
        "        v_X, v_y   = X.iloc[val_idx,:], y[val_idx] \n",
        "        train_data = lgb.Dataset(tr_x, label=tr_y)\n",
        "        valid_data = lgb.Dataset(v_X, label=v_y)  \n",
        "\n",
        "        seed_everything(SEED)\n",
        "        estimator = lgb.train(\n",
        "                 {\n",
        "                    'boosting_type': 'gbdt',\n",
        "                    'objective': 'tweedie',\n",
        "                    'tweedie_variance_power': 1.1,\n",
        "                    'metric': 'rmse',\n",
        "                    'subsample': 0.5,\n",
        "                    'subsample_freq': 1,\n",
        "                    'learning_rate': 0.03,\n",
        "                    'num_leaves': 2**11-1,\n",
        "                    'min_data_in_leaf': 2**12-1,\n",
        "                    'feature_fraction': 0.6,\n",
        "                    'max_bin': 100,\n",
        "                    'n_estimators': 1450,\n",
        "                    'early_stopping_rounds': 30,\n",
        "                    'boost_from_average': False,\n",
        "                    'verbose': -1,\n",
        "                } ,\n",
        "                train_data,\n",
        "                valid_sets = [train_data, valid_data],\n",
        "                verbose_eval = 100\n",
        "            )\n",
        "        # Save model - it's not real '.bin' but a pickle file\n",
        "        # estimator = lgb.Booster(model_file='model.txt')\n",
        "        # can only predict with the best iteration (or the saving iteration)\n",
        "        # pickle.dump gives us more flexibility\n",
        "        # like estimator.predict(TEST, num_iteration=100)\n",
        "        # num_iteration - number of iteration want to predict with, \n",
        "        # NULL or <= 0 means use best iteration\n",
        "        model_name = 'lgb_model_'+ store_id +'_'+str(fold_)+'.bin'\n",
        "        pickle.dump(estimator, open(model_name, 'wb'))\n",
        "\n",
        "        # Remove temporary files and objects \n",
        "        # to free some hdd space and ram memory\n",
        "        del train_data, valid_data, estimator\n",
        "        gc.collect()\n",
        "\n",
        "    # \"Keep\" models features for predictions\n",
        "    MODEL_FEATURES = features_columns"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train CA_1\n",
            "Fold: 0\n",
            "3596122 1192145\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 2.56016\tvalid_1's rmse: 2.59594\n",
            "[200]\ttraining's rmse: 2.4615\tvalid_1's rmse: 2.50445\n",
            "[300]\ttraining's rmse: 2.42741\tvalid_1's rmse: 2.48399\n",
            "[400]\ttraining's rmse: 2.40542\tvalid_1's rmse: 2.47598\n",
            "[500]\ttraining's rmse: 2.38839\tvalid_1's rmse: 2.47056\n",
            "[600]\ttraining's rmse: 2.37278\tvalid_1's rmse: 2.46688\n",
            "[700]\ttraining's rmse: 2.35919\tvalid_1's rmse: 2.46333\n",
            "[800]\ttraining's rmse: 2.34717\tvalid_1's rmse: 2.4616\n",
            "[900]\ttraining's rmse: 2.33637\tvalid_1's rmse: 2.46015\n",
            "[1000]\ttraining's rmse: 2.32556\tvalid_1's rmse: 2.45802\n",
            "Early stopping, best iteration is:\n",
            "[1029]\ttraining's rmse: 2.32233\tvalid_1's rmse: 2.45775\n",
            "Fold: 1\n",
            "3587602 1200665\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 2.58892\tvalid_1's rmse: 2.55975\n",
            "[200]\ttraining's rmse: 2.48642\tvalid_1's rmse: 2.46735\n",
            "[300]\ttraining's rmse: 2.45286\tvalid_1's rmse: 2.44081\n",
            "[400]\ttraining's rmse: 2.43182\tvalid_1's rmse: 2.42614\n",
            "[500]\ttraining's rmse: 2.41437\tvalid_1's rmse: 2.41641\n",
            "[600]\ttraining's rmse: 2.39955\tvalid_1's rmse: 2.40851\n",
            "[700]\ttraining's rmse: 2.3872\tvalid_1's rmse: 2.40374\n",
            "[800]\ttraining's rmse: 2.37451\tvalid_1's rmse: 2.39904\n",
            "[900]\ttraining's rmse: 2.36203\tvalid_1's rmse: 2.39491\n",
            "[1000]\ttraining's rmse: 2.3515\tvalid_1's rmse: 2.39235\n",
            "[1100]\ttraining's rmse: 2.34146\tvalid_1's rmse: 2.38853\n",
            "[1200]\ttraining's rmse: 2.33207\tvalid_1's rmse: 2.38676\n",
            "Early stopping, best iteration is:\n",
            "[1227]\ttraining's rmse: 2.32947\tvalid_1's rmse: 2.38565\n",
            "Fold: 2\n",
            "3587361 1200906\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 2.58206\tvalid_1's rmse: 2.54746\n",
            "[200]\ttraining's rmse: 2.47817\tvalid_1's rmse: 2.45259\n",
            "[300]\ttraining's rmse: 2.4443\tvalid_1's rmse: 2.43321\n",
            "[400]\ttraining's rmse: 2.42139\tvalid_1's rmse: 2.42387\n",
            "[500]\ttraining's rmse: 2.40397\tvalid_1's rmse: 2.41926\n",
            "[600]\ttraining's rmse: 2.38768\tvalid_1's rmse: 2.4169\n",
            "Early stopping, best iteration is:\n",
            "[638]\ttraining's rmse: 2.38211\tvalid_1's rmse: 2.41585\n",
            "Fold: 3\n",
            "3593716 1194551\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 2.55305\tvalid_1's rmse: 2.64726\n",
            "[200]\ttraining's rmse: 2.44686\tvalid_1's rmse: 2.57144\n",
            "[300]\ttraining's rmse: 2.41054\tvalid_1's rmse: 2.55252\n",
            "[400]\ttraining's rmse: 2.38714\tvalid_1's rmse: 2.54288\n",
            "[500]\ttraining's rmse: 2.36869\tvalid_1's rmse: 2.5366\n",
            "[600]\ttraining's rmse: 2.35283\tvalid_1's rmse: 2.53358\n",
            "[700]\ttraining's rmse: 2.33808\tvalid_1's rmse: 2.53048\n",
            "[800]\ttraining's rmse: 2.32478\tvalid_1's rmse: 2.52832\n",
            "Early stopping, best iteration is:\n",
            "[809]\ttraining's rmse: 2.32364\tvalid_1's rmse: 2.52795\n",
            "Train CA_2\n",
            "Fold: 0\n",
            "3267848 1093300\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 2.04174\tvalid_1's rmse: 2.08893\n",
            "[200]\ttraining's rmse: 2.00123\tvalid_1's rmse: 2.05327\n",
            "[300]\ttraining's rmse: 1.98614\tvalid_1's rmse: 2.04457\n",
            "[400]\ttraining's rmse: 1.97569\tvalid_1's rmse: 2.04037\n",
            "[500]\ttraining's rmse: 1.96678\tvalid_1's rmse: 2.03742\n",
            "[600]\ttraining's rmse: 1.95893\tvalid_1's rmse: 2.03539\n",
            "[700]\ttraining's rmse: 1.95154\tvalid_1's rmse: 2.03443\n",
            "[800]\ttraining's rmse: 1.94453\tvalid_1's rmse: 2.03337\n",
            "[900]\ttraining's rmse: 1.93797\tvalid_1's rmse: 2.03275\n",
            "[1000]\ttraining's rmse: 1.93168\tvalid_1's rmse: 2.03217\n",
            "[1100]\ttraining's rmse: 1.92587\tvalid_1's rmse: 2.03121\n",
            "Early stopping, best iteration is:\n",
            "[1143]\ttraining's rmse: 1.92336\tvalid_1's rmse: 2.03091\n",
            "Fold: 1\n",
            "3275099 1086049\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 2.05699\tvalid_1's rmse: 2.01837\n",
            "[200]\ttraining's rmse: 2.0161\tvalid_1's rmse: 1.99309\n",
            "[300]\ttraining's rmse: 2.00022\tvalid_1's rmse: 1.98772\n",
            "[400]\ttraining's rmse: 1.98847\tvalid_1's rmse: 1.98503\n",
            "[500]\ttraining's rmse: 1.97832\tvalid_1's rmse: 1.98351\n",
            "[600]\ttraining's rmse: 1.96936\tvalid_1's rmse: 1.98263\n",
            "[700]\ttraining's rmse: 1.96088\tvalid_1's rmse: 1.98206\n",
            "[800]\ttraining's rmse: 1.95312\tvalid_1's rmse: 1.98143\n",
            "[900]\ttraining's rmse: 1.94611\tvalid_1's rmse: 1.98109\n",
            "Early stopping, best iteration is:\n",
            "[870]\ttraining's rmse: 1.94824\tvalid_1's rmse: 1.98105\n",
            "Fold: 2\n",
            "3267365 1093783\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 2.03887\tvalid_1's rmse: 2.08518\n",
            "[200]\ttraining's rmse: 1.99786\tvalid_1's rmse: 2.05395\n",
            "[300]\ttraining's rmse: 1.9818\tvalid_1's rmse: 2.04767\n",
            "[400]\ttraining's rmse: 1.9704\tvalid_1's rmse: 2.04465\n",
            "[500]\ttraining's rmse: 1.96045\tvalid_1's rmse: 2.04322\n",
            "Early stopping, best iteration is:\n",
            "[555]\ttraining's rmse: 1.95538\tvalid_1's rmse: 2.04291\n",
            "Fold: 3\n",
            "3273132 1088016\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 2.0542\tvalid_1's rmse: 2.04349\n",
            "[200]\ttraining's rmse: 2.01384\tvalid_1's rmse: 2.01211\n",
            "[300]\ttraining's rmse: 1.99741\tvalid_1's rmse: 2.00553\n",
            "[400]\ttraining's rmse: 1.98608\tvalid_1's rmse: 2.00187\n",
            "[500]\ttraining's rmse: 1.97638\tvalid_1's rmse: 1.99923\n",
            "[600]\ttraining's rmse: 1.96787\tvalid_1's rmse: 1.9972\n",
            "[700]\ttraining's rmse: 1.95952\tvalid_1's rmse: 1.99577\n",
            "[800]\ttraining's rmse: 1.9522\tvalid_1's rmse: 1.99464\n",
            "[900]\ttraining's rmse: 1.94533\tvalid_1's rmse: 1.99376\n",
            "[1000]\ttraining's rmse: 1.93901\tvalid_1's rmse: 1.99325\n",
            "Early stopping, best iteration is:\n",
            "[991]\ttraining's rmse: 1.93956\tvalid_1's rmse: 1.9932\n",
            "Train CA_3\n",
            "Fold: 0\n",
            "3572933 1184380\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 3.66797\tvalid_1's rmse: 3.72633\n",
            "[200]\ttraining's rmse: 3.44544\tvalid_1's rmse: 3.53828\n",
            "[300]\ttraining's rmse: 3.37045\tvalid_1's rmse: 3.50042\n",
            "[400]\ttraining's rmse: 3.32624\tvalid_1's rmse: 3.48142\n",
            "[500]\ttraining's rmse: 3.29585\tvalid_1's rmse: 3.47027\n",
            "[600]\ttraining's rmse: 3.26828\tvalid_1's rmse: 3.46328\n",
            "Early stopping, best iteration is:\n",
            "[657]\ttraining's rmse: 3.2549\tvalid_1's rmse: 3.45849\n",
            "Fold: 1\n",
            "3564340 1192973\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 3.70054\tvalid_1's rmse: 3.70908\n",
            "[200]\ttraining's rmse: 3.46393\tvalid_1's rmse: 3.52309\n",
            "[300]\ttraining's rmse: 3.3859\tvalid_1's rmse: 3.47921\n",
            "[400]\ttraining's rmse: 3.34468\tvalid_1's rmse: 3.45526\n",
            "[500]\ttraining's rmse: 3.31384\tvalid_1's rmse: 3.4415\n",
            "[600]\ttraining's rmse: 3.28475\tvalid_1's rmse: 3.42996\n",
            "[700]\ttraining's rmse: 3.26199\tvalid_1's rmse: 3.42067\n",
            "[800]\ttraining's rmse: 3.23988\tvalid_1's rmse: 3.41406\n",
            "[900]\ttraining's rmse: 3.22105\tvalid_1's rmse: 3.4065\n",
            "[1000]\ttraining's rmse: 3.20218\tvalid_1's rmse: 3.40269\n",
            "[1100]\ttraining's rmse: 3.18465\tvalid_1's rmse: 3.39974\n",
            "Early stopping, best iteration is:\n",
            "[1118]\ttraining's rmse: 3.18184\tvalid_1's rmse: 3.39854\n",
            "Fold: 2\n",
            "3570495 1186818\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 3.67687\tvalid_1's rmse: 3.76392\n",
            "[200]\ttraining's rmse: 3.45323\tvalid_1's rmse: 3.54811\n",
            "[300]\ttraining's rmse: 3.37477\tvalid_1's rmse: 3.4953\n",
            "[400]\ttraining's rmse: 3.32947\tvalid_1's rmse: 3.48011\n",
            "[500]\ttraining's rmse: 3.29453\tvalid_1's rmse: 3.47407\n",
            "Early stopping, best iteration is:\n",
            "[475]\ttraining's rmse: 3.30219\tvalid_1's rmse: 3.47315\n",
            "Fold: 3\n",
            "3564171 1193142\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 3.70666\tvalid_1's rmse: 3.63351\n",
            "[200]\ttraining's rmse: 3.48221\tvalid_1's rmse: 3.42178\n",
            "[300]\ttraining's rmse: 3.41202\tvalid_1's rmse: 3.37387\n",
            "[400]\ttraining's rmse: 3.37107\tvalid_1's rmse: 3.35206\n",
            "[500]\ttraining's rmse: 3.33819\tvalid_1's rmse: 3.34309\n",
            "Early stopping, best iteration is:\n",
            "[556]\ttraining's rmse: 3.32141\tvalid_1's rmse: 3.33999\n",
            "Train CA_4\n",
            "Fold: 0\n",
            "3491747 1160811\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 1.50596\tvalid_1's rmse: 1.56239\n",
            "[200]\ttraining's rmse: 1.47739\tvalid_1's rmse: 1.53417\n",
            "[300]\ttraining's rmse: 1.46609\tvalid_1's rmse: 1.52653\n",
            "[400]\ttraining's rmse: 1.45858\tvalid_1's rmse: 1.52316\n",
            "[500]\ttraining's rmse: 1.45218\tvalid_1's rmse: 1.52098\n",
            "[600]\ttraining's rmse: 1.44667\tvalid_1's rmse: 1.51968\n",
            "[700]\ttraining's rmse: 1.44126\tvalid_1's rmse: 1.51857\n",
            "[800]\ttraining's rmse: 1.43638\tvalid_1's rmse: 1.51763\n",
            "[900]\ttraining's rmse: 1.43178\tvalid_1's rmse: 1.5171\n",
            "[1000]\ttraining's rmse: 1.42741\tvalid_1's rmse: 1.51666\n",
            "[1100]\ttraining's rmse: 1.42324\tvalid_1's rmse: 1.51611\n",
            "[1200]\ttraining's rmse: 1.41917\tvalid_1's rmse: 1.5156\n",
            "[1300]\ttraining's rmse: 1.41536\tvalid_1's rmse: 1.51542\n",
            "Early stopping, best iteration is:\n",
            "[1306]\ttraining's rmse: 1.41517\tvalid_1's rmse: 1.51539\n",
            "Fold: 1\n",
            "3486335 1166223\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 1.51385\tvalid_1's rmse: 1.54168\n",
            "[200]\ttraining's rmse: 1.48415\tvalid_1's rmse: 1.51632\n",
            "[300]\ttraining's rmse: 1.47333\tvalid_1's rmse: 1.50903\n",
            "[400]\ttraining's rmse: 1.46599\tvalid_1's rmse: 1.50544\n",
            "[500]\ttraining's rmse: 1.45985\tvalid_1's rmse: 1.50304\n",
            "[600]\ttraining's rmse: 1.45455\tvalid_1's rmse: 1.50129\n",
            "[700]\ttraining's rmse: 1.44947\tvalid_1's rmse: 1.5004\n",
            "[800]\ttraining's rmse: 1.44469\tvalid_1's rmse: 1.4993\n",
            "[900]\ttraining's rmse: 1.44022\tvalid_1's rmse: 1.4986\n",
            "[1000]\ttraining's rmse: 1.43594\tvalid_1's rmse: 1.49789\n",
            "[1100]\ttraining's rmse: 1.4319\tvalid_1's rmse: 1.49754\n",
            "[1200]\ttraining's rmse: 1.42815\tvalid_1's rmse: 1.4971\n",
            "Early stopping, best iteration is:\n",
            "[1214]\ttraining's rmse: 1.42757\tvalid_1's rmse: 1.49701\n",
            "Fold: 2\n",
            "3485857 1166701\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 1.52675\tvalid_1's rmse: 1.4931\n",
            "[200]\ttraining's rmse: 1.49711\tvalid_1's rmse: 1.47368\n",
            "[300]\ttraining's rmse: 1.48528\tvalid_1's rmse: 1.46894\n",
            "[400]\ttraining's rmse: 1.47701\tvalid_1's rmse: 1.46637\n",
            "[500]\ttraining's rmse: 1.47043\tvalid_1's rmse: 1.46556\n",
            "Early stopping, best iteration is:\n",
            "[534]\ttraining's rmse: 1.46827\tvalid_1's rmse: 1.46541\n",
            "Fold: 3\n",
            "3493735 1158823\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 1.52545\tvalid_1's rmse: 1.49762\n",
            "[200]\ttraining's rmse: 1.49519\tvalid_1's rmse: 1.47589\n",
            "[300]\ttraining's rmse: 1.48368\tvalid_1's rmse: 1.47143\n",
            "[400]\ttraining's rmse: 1.47536\tvalid_1's rmse: 1.4687\n",
            "[500]\ttraining's rmse: 1.46887\tvalid_1's rmse: 1.46734\n",
            "[600]\ttraining's rmse: 1.46324\tvalid_1's rmse: 1.46655\n",
            "[700]\ttraining's rmse: 1.45785\tvalid_1's rmse: 1.46577\n",
            "[800]\ttraining's rmse: 1.45287\tvalid_1's rmse: 1.46521\n",
            "[900]\ttraining's rmse: 1.44818\tvalid_1's rmse: 1.46482\n",
            "Early stopping, best iteration is:\n",
            "[935]\ttraining's rmse: 1.44647\tvalid_1's rmse: 1.46467\n",
            "Train TX_1\n",
            "Fold: 0\n",
            "3603283 1194672\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 2.16099\tvalid_1's rmse: 2.10158\n",
            "[200]\ttraining's rmse: 2.08713\tvalid_1's rmse: 2.03508\n",
            "[300]\ttraining's rmse: 2.06192\tvalid_1's rmse: 2.01866\n",
            "[400]\ttraining's rmse: 2.04564\tvalid_1's rmse: 2.01163\n",
            "[500]\ttraining's rmse: 2.03157\tvalid_1's rmse: 2.00634\n",
            "[600]\ttraining's rmse: 2.01864\tvalid_1's rmse: 2.00125\n",
            "[700]\ttraining's rmse: 2.00753\tvalid_1's rmse: 1.998\n",
            "[800]\ttraining's rmse: 1.99756\tvalid_1's rmse: 1.99543\n",
            "[900]\ttraining's rmse: 1.98876\tvalid_1's rmse: 1.99409\n",
            "[1000]\ttraining's rmse: 1.97944\tvalid_1's rmse: 1.99283\n",
            "[1100]\ttraining's rmse: 1.96984\tvalid_1's rmse: 1.99101\n",
            "[1200]\ttraining's rmse: 1.96227\tvalid_1's rmse: 1.98956\n",
            "Early stopping, best iteration is:\n",
            "[1230]\ttraining's rmse: 1.96009\tvalid_1's rmse: 1.98912\n",
            "Fold: 1\n",
            "3600993 1196962\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 2.12719\tvalid_1's rmse: 2.21828\n",
            "[200]\ttraining's rmse: 2.0556\tvalid_1's rmse: 2.14681\n",
            "[300]\ttraining's rmse: 2.03002\tvalid_1's rmse: 2.1298\n",
            "[400]\ttraining's rmse: 2.01296\tvalid_1's rmse: 2.12226\n",
            "[500]\ttraining's rmse: 1.99846\tvalid_1's rmse: 2.11696\n",
            "[600]\ttraining's rmse: 1.98675\tvalid_1's rmse: 2.11494\n",
            "[700]\ttraining's rmse: 1.97575\tvalid_1's rmse: 2.11289\n",
            "[800]\ttraining's rmse: 1.96506\tvalid_1's rmse: 2.11136\n",
            "[900]\ttraining's rmse: 1.95673\tvalid_1's rmse: 2.11057\n",
            "[1000]\ttraining's rmse: 1.94811\tvalid_1's rmse: 2.10933\n",
            "Early stopping, best iteration is:\n",
            "[1008]\ttraining's rmse: 1.94751\tvalid_1's rmse: 2.10916\n",
            "Fold: 2\n",
            "3595025 1202930\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 2.13597\tvalid_1's rmse: 2.17775\n",
            "[200]\ttraining's rmse: 2.0668\tvalid_1's rmse: 2.11075\n",
            "[300]\ttraining's rmse: 2.04125\tvalid_1's rmse: 2.0938\n",
            "[400]\ttraining's rmse: 2.02455\tvalid_1's rmse: 2.08401\n",
            "[500]\ttraining's rmse: 2.01164\tvalid_1's rmse: 2.07726\n",
            "[600]\ttraining's rmse: 2.00013\tvalid_1's rmse: 2.07255\n",
            "[700]\ttraining's rmse: 1.98831\tvalid_1's rmse: 2.06868\n",
            "[800]\ttraining's rmse: 1.97909\tvalid_1's rmse: 2.06649\n",
            "[900]\ttraining's rmse: 1.96974\tvalid_1's rmse: 2.06354\n",
            "[1000]\ttraining's rmse: 1.96048\tvalid_1's rmse: 2.06164\n",
            "[1100]\ttraining's rmse: 1.95213\tvalid_1's rmse: 2.06044\n",
            "[1200]\ttraining's rmse: 1.94336\tvalid_1's rmse: 2.05917\n",
            "[1300]\ttraining's rmse: 1.9354\tvalid_1's rmse: 2.05853\n",
            "Early stopping, best iteration is:\n",
            "[1284]\ttraining's rmse: 1.93641\tvalid_1's rmse: 2.05815\n",
            "Fold: 3\n",
            "3594564 1203391\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 2.15004\tvalid_1's rmse: 2.11906\n",
            "[200]\ttraining's rmse: 2.07008\tvalid_1's rmse: 2.07956\n",
            "[300]\ttraining's rmse: 2.04026\tvalid_1's rmse: 2.07524\n",
            "Early stopping, best iteration is:\n",
            "[357]\ttraining's rmse: 2.02927\tvalid_1's rmse: 2.07396\n",
            "Train TX_2\n",
            "Fold: 0\n",
            "3610761 1197120\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 2.59777\tvalid_1's rmse: 2.492\n",
            "[200]\ttraining's rmse: 2.48854\tvalid_1's rmse: 2.3853\n",
            "[300]\ttraining's rmse: 2.45215\tvalid_1's rmse: 2.36051\n",
            "[400]\ttraining's rmse: 2.42925\tvalid_1's rmse: 2.34826\n",
            "[500]\ttraining's rmse: 2.41184\tvalid_1's rmse: 2.3413\n",
            "[600]\ttraining's rmse: 2.39569\tvalid_1's rmse: 2.33512\n",
            "[700]\ttraining's rmse: 2.38007\tvalid_1's rmse: 2.3306\n",
            "[800]\ttraining's rmse: 2.36663\tvalid_1's rmse: 2.32815\n",
            "Early stopping, best iteration is:\n",
            "[772]\ttraining's rmse: 2.37009\tvalid_1's rmse: 2.32743\n",
            "Fold: 1\n",
            "3608411 1199470\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 2.57834\tvalid_1's rmse: 2.55309\n",
            "[200]\ttraining's rmse: 2.46646\tvalid_1's rmse: 2.4635\n",
            "[300]\ttraining's rmse: 2.42458\tvalid_1's rmse: 2.44319\n",
            "[400]\ttraining's rmse: 2.39711\tvalid_1's rmse: 2.43542\n",
            "[500]\ttraining's rmse: 2.37735\tvalid_1's rmse: 2.43148\n",
            "Early stopping, best iteration is:\n",
            "[525]\ttraining's rmse: 2.37281\tvalid_1's rmse: 2.43103\n",
            "Fold: 2\n",
            "3602115 1205766\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 2.56844\tvalid_1's rmse: 2.56972\n",
            "[200]\ttraining's rmse: 2.45649\tvalid_1's rmse: 2.47412\n",
            "[300]\ttraining's rmse: 2.41858\tvalid_1's rmse: 2.45493\n",
            "[400]\ttraining's rmse: 2.3951\tvalid_1's rmse: 2.44752\n",
            "[500]\ttraining's rmse: 2.3739\tvalid_1's rmse: 2.44316\n",
            "[600]\ttraining's rmse: 2.35781\tvalid_1's rmse: 2.44188\n",
            "Early stopping, best iteration is:\n",
            "[620]\ttraining's rmse: 2.35422\tvalid_1's rmse: 2.44112\n",
            "Fold: 3\n",
            "3602356 1205525\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 2.53577\tvalid_1's rmse: 2.71499\n",
            "[200]\ttraining's rmse: 2.41731\tvalid_1's rmse: 2.62609\n",
            "[300]\ttraining's rmse: 2.38057\tvalid_1's rmse: 2.60446\n",
            "[400]\ttraining's rmse: 2.35647\tvalid_1's rmse: 2.59114\n",
            "[500]\ttraining's rmse: 2.33853\tvalid_1's rmse: 2.58322\n",
            "[600]\ttraining's rmse: 2.32411\tvalid_1's rmse: 2.57965\n",
            "[700]\ttraining's rmse: 2.30983\tvalid_1's rmse: 2.57563\n",
            "[800]\ttraining's rmse: 2.2972\tvalid_1's rmse: 2.57346\n",
            "[900]\ttraining's rmse: 2.28407\tvalid_1's rmse: 2.57037\n",
            "[1000]\ttraining's rmse: 2.27381\tvalid_1's rmse: 2.56823\n",
            "[1100]\ttraining's rmse: 2.26338\tvalid_1's rmse: 2.56683\n",
            "[1200]\ttraining's rmse: 2.25427\tvalid_1's rmse: 2.56518\n",
            "Early stopping, best iteration is:\n",
            "[1237]\ttraining's rmse: 2.25066\tvalid_1's rmse: 2.56467\n",
            "Train TX_3\n",
            "Fold: 0\n",
            "3555395 1181772\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 2.22978\tvalid_1's rmse: 2.19734\n",
            "[200]\ttraining's rmse: 2.13559\tvalid_1's rmse: 2.12406\n",
            "[300]\ttraining's rmse: 2.10087\tvalid_1's rmse: 2.10673\n",
            "[400]\ttraining's rmse: 2.0793\tvalid_1's rmse: 2.09763\n",
            "[500]\ttraining's rmse: 2.061\tvalid_1's rmse: 2.09202\n",
            "[600]\ttraining's rmse: 2.04615\tvalid_1's rmse: 2.0887\n",
            "[700]\ttraining's rmse: 2.03287\tvalid_1's rmse: 2.08703\n",
            "[800]\ttraining's rmse: 2.02046\tvalid_1's rmse: 2.08522\n",
            "[900]\ttraining's rmse: 2.0095\tvalid_1's rmse: 2.08384\n",
            "Early stopping, best iteration is:\n",
            "[939]\ttraining's rmse: 2.00548\tvalid_1's rmse: 2.08308\n",
            "Fold: 1\n",
            "3548875 1188292\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 2.19922\tvalid_1's rmse: 2.28504\n",
            "[200]\ttraining's rmse: 2.10989\tvalid_1's rmse: 2.19245\n",
            "[300]\ttraining's rmse: 2.07739\tvalid_1's rmse: 2.17384\n",
            "[400]\ttraining's rmse: 2.05653\tvalid_1's rmse: 2.16753\n",
            "[500]\ttraining's rmse: 2.03894\tvalid_1's rmse: 2.16465\n",
            "[600]\ttraining's rmse: 2.02468\tvalid_1's rmse: 2.16197\n",
            "[700]\ttraining's rmse: 2.01074\tvalid_1's rmse: 2.16105\n",
            "Early stopping, best iteration is:\n",
            "[735]\ttraining's rmse: 2.00646\tvalid_1's rmse: 2.16058\n",
            "Fold: 2\n",
            "3549640 1187527\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 2.21\tvalid_1's rmse: 2.30102\n",
            "[200]\ttraining's rmse: 2.11516\tvalid_1's rmse: 2.21095\n",
            "[300]\ttraining's rmse: 2.08386\tvalid_1's rmse: 2.18677\n",
            "[400]\ttraining's rmse: 2.06398\tvalid_1's rmse: 2.17348\n",
            "[500]\ttraining's rmse: 2.04657\tvalid_1's rmse: 2.16478\n",
            "[600]\ttraining's rmse: 2.0314\tvalid_1's rmse: 2.15952\n",
            "[700]\ttraining's rmse: 2.0174\tvalid_1's rmse: 2.15501\n",
            "[800]\ttraining's rmse: 2.00584\tvalid_1's rmse: 2.15208\n",
            "[900]\ttraining's rmse: 1.995\tvalid_1's rmse: 2.14979\n",
            "[1000]\ttraining's rmse: 1.98521\tvalid_1's rmse: 2.14768\n",
            "[1100]\ttraining's rmse: 1.97595\tvalid_1's rmse: 2.14549\n",
            "Early stopping, best iteration is:\n",
            "[1136]\ttraining's rmse: 1.97268\tvalid_1's rmse: 2.1448\n",
            "Fold: 3\n",
            "3557591 1179576\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 2.23708\tvalid_1's rmse: 2.142\n",
            "[200]\ttraining's rmse: 2.14284\tvalid_1's rmse: 2.07908\n",
            "[300]\ttraining's rmse: 2.11116\tvalid_1's rmse: 2.065\n",
            "[400]\ttraining's rmse: 2.08958\tvalid_1's rmse: 2.05793\n",
            "[500]\ttraining's rmse: 2.07313\tvalid_1's rmse: 2.05357\n",
            "[600]\ttraining's rmse: 2.05783\tvalid_1's rmse: 2.04981\n",
            "[700]\ttraining's rmse: 2.04424\tvalid_1's rmse: 2.04797\n",
            "[800]\ttraining's rmse: 2.0317\tvalid_1's rmse: 2.0453\n",
            "[900]\ttraining's rmse: 2.02092\tvalid_1's rmse: 2.0446\n",
            "[1000]\ttraining's rmse: 2.01035\tvalid_1's rmse: 2.04343\n",
            "[1100]\ttraining's rmse: 2.00118\tvalid_1's rmse: 2.04284\n",
            "Early stopping, best iteration is:\n",
            "[1086]\ttraining's rmse: 2.00244\tvalid_1's rmse: 2.04268\n",
            "Train WI_1\n",
            "Fold: 0\n",
            "3417517 1143250\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 1.71579\tvalid_1's rmse: 1.72767\n",
            "[200]\ttraining's rmse: 1.67127\tvalid_1's rmse: 1.69614\n",
            "[300]\ttraining's rmse: 1.65502\tvalid_1's rmse: 1.69166\n",
            "[400]\ttraining's rmse: 1.64297\tvalid_1's rmse: 1.69033\n",
            "Early stopping, best iteration is:\n",
            "[416]\ttraining's rmse: 1.64132\tvalid_1's rmse: 1.69008\n",
            "Fold: 1\n",
            "3422591 1138176\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 1.72114\tvalid_1's rmse: 1.71221\n",
            "[200]\ttraining's rmse: 1.67746\tvalid_1's rmse: 1.68026\n",
            "[300]\ttraining's rmse: 1.66144\tvalid_1's rmse: 1.67268\n",
            "[400]\ttraining's rmse: 1.65023\tvalid_1's rmse: 1.66815\n",
            "[500]\ttraining's rmse: 1.64134\tvalid_1's rmse: 1.66515\n",
            "[600]\ttraining's rmse: 1.63339\tvalid_1's rmse: 1.66252\n",
            "[700]\ttraining's rmse: 1.62602\tvalid_1's rmse: 1.66061\n",
            "[800]\ttraining's rmse: 1.61881\tvalid_1's rmse: 1.65904\n",
            "[900]\ttraining's rmse: 1.61284\tvalid_1's rmse: 1.65809\n",
            "[1000]\ttraining's rmse: 1.60638\tvalid_1's rmse: 1.65695\n",
            "[1100]\ttraining's rmse: 1.60076\tvalid_1's rmse: 1.65584\n",
            "[1200]\ttraining's rmse: 1.5954\tvalid_1's rmse: 1.65538\n",
            "[1300]\ttraining's rmse: 1.59041\tvalid_1's rmse: 1.65495\n",
            "Early stopping, best iteration is:\n",
            "[1312]\ttraining's rmse: 1.58983\tvalid_1's rmse: 1.6549\n",
            "Fold: 2\n",
            "3417972 1142795\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 1.71456\tvalid_1's rmse: 1.74208\n",
            "[200]\ttraining's rmse: 1.67155\tvalid_1's rmse: 1.70837\n",
            "[300]\ttraining's rmse: 1.65681\tvalid_1's rmse: 1.69926\n",
            "[400]\ttraining's rmse: 1.64673\tvalid_1's rmse: 1.6943\n",
            "[500]\ttraining's rmse: 1.63843\tvalid_1's rmse: 1.69086\n",
            "[600]\ttraining's rmse: 1.63055\tvalid_1's rmse: 1.68765\n",
            "[700]\ttraining's rmse: 1.62358\tvalid_1's rmse: 1.68609\n",
            "[800]\ttraining's rmse: 1.61697\tvalid_1's rmse: 1.68424\n",
            "[900]\ttraining's rmse: 1.61092\tvalid_1's rmse: 1.6833\n",
            "[1000]\ttraining's rmse: 1.60527\tvalid_1's rmse: 1.6823\n",
            "[1100]\ttraining's rmse: 1.60002\tvalid_1's rmse: 1.68174\n",
            "[1200]\ttraining's rmse: 1.595\tvalid_1's rmse: 1.68115\n",
            "[1300]\ttraining's rmse: 1.59035\tvalid_1's rmse: 1.68071\n",
            "[1400]\ttraining's rmse: 1.58579\tvalid_1's rmse: 1.68009\n",
            "Did not meet early stopping. Best iteration is:\n",
            "[1450]\ttraining's rmse: 1.5835\tvalid_1's rmse: 1.67986\n",
            "Fold: 3\n",
            "3424221 1136546\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 1.71703\tvalid_1's rmse: 1.72465\n",
            "[200]\ttraining's rmse: 1.67369\tvalid_1's rmse: 1.68818\n",
            "[300]\ttraining's rmse: 1.65803\tvalid_1's rmse: 1.68107\n",
            "[400]\ttraining's rmse: 1.6471\tvalid_1's rmse: 1.67774\n",
            "[500]\ttraining's rmse: 1.6377\tvalid_1's rmse: 1.67533\n",
            "[600]\ttraining's rmse: 1.62966\tvalid_1's rmse: 1.67346\n",
            "[700]\ttraining's rmse: 1.62217\tvalid_1's rmse: 1.67227\n",
            "[800]\ttraining's rmse: 1.61559\tvalid_1's rmse: 1.67143\n",
            "[900]\ttraining's rmse: 1.60923\tvalid_1's rmse: 1.6707\n",
            "[1000]\ttraining's rmse: 1.60344\tvalid_1's rmse: 1.66998\n",
            "[1100]\ttraining's rmse: 1.5979\tvalid_1's rmse: 1.66954\n",
            "Early stopping, best iteration is:\n",
            "[1097]\ttraining's rmse: 1.59803\tvalid_1's rmse: 1.66953\n",
            "Train WI_2\n",
            "Fold: 0\n",
            "3482372 1164208\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 2.65004\tvalid_1's rmse: 2.8184\n",
            "[200]\ttraining's rmse: 2.58365\tvalid_1's rmse: 2.76573\n",
            "[300]\ttraining's rmse: 2.5557\tvalid_1's rmse: 2.75247\n",
            "[400]\ttraining's rmse: 2.53749\tvalid_1's rmse: 2.74602\n",
            "[500]\ttraining's rmse: 2.52014\tvalid_1's rmse: 2.74087\n",
            "[600]\ttraining's rmse: 2.50501\tvalid_1's rmse: 2.73736\n",
            "[700]\ttraining's rmse: 2.49116\tvalid_1's rmse: 2.73517\n",
            "[800]\ttraining's rmse: 2.47823\tvalid_1's rmse: 2.73323\n",
            "[900]\ttraining's rmse: 2.46644\tvalid_1's rmse: 2.73158\n",
            "Early stopping, best iteration is:\n",
            "[963]\ttraining's rmse: 2.45891\tvalid_1's rmse: 2.73104\n",
            "Fold: 1\n",
            "3488627 1157953\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 2.68765\tvalid_1's rmse: 2.6772\n",
            "[200]\ttraining's rmse: 2.61887\tvalid_1's rmse: 2.6377\n",
            "[300]\ttraining's rmse: 2.58932\tvalid_1's rmse: 2.62924\n",
            "[400]\ttraining's rmse: 2.56902\tvalid_1's rmse: 2.62536\n",
            "[500]\ttraining's rmse: 2.55222\tvalid_1's rmse: 2.62313\n",
            "[600]\ttraining's rmse: 2.53799\tvalid_1's rmse: 2.6223\n",
            "Early stopping, best iteration is:\n",
            "[582]\ttraining's rmse: 2.54025\tvalid_1's rmse: 2.62177\n",
            "Fold: 2\n",
            "3481789 1164791\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 2.69114\tvalid_1's rmse: 2.68112\n",
            "[200]\ttraining's rmse: 2.62695\tvalid_1's rmse: 2.63192\n",
            "[300]\ttraining's rmse: 2.60043\tvalid_1's rmse: 2.61934\n",
            "[400]\ttraining's rmse: 2.58131\tvalid_1's rmse: 2.61288\n",
            "[500]\ttraining's rmse: 2.56392\tvalid_1's rmse: 2.60933\n",
            "Early stopping, best iteration is:\n",
            "[496]\ttraining's rmse: 2.56451\tvalid_1's rmse: 2.60921\n",
            "Fold: 3\n",
            "3486952 1159628\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 2.71083\tvalid_1's rmse: 2.61919\n",
            "[200]\ttraining's rmse: 2.646\tvalid_1's rmse: 2.57056\n",
            "[300]\ttraining's rmse: 2.61738\tvalid_1's rmse: 2.56229\n",
            "[400]\ttraining's rmse: 2.59856\tvalid_1's rmse: 2.55917\n",
            "[500]\ttraining's rmse: 2.58154\tvalid_1's rmse: 2.55653\n",
            "Early stopping, best iteration is:\n",
            "[498]\ttraining's rmse: 2.58193\tvalid_1's rmse: 2.55652\n",
            "Train WI_3\n",
            "Fold: 0\n",
            "3575161 1196880\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 2.45896\tvalid_1's rmse: 2.41404\n",
            "[200]\ttraining's rmse: 2.33658\tvalid_1's rmse: 2.3093\n",
            "[300]\ttraining's rmse: 2.2902\tvalid_1's rmse: 2.28749\n",
            "[400]\ttraining's rmse: 2.25927\tvalid_1's rmse: 2.28272\n",
            "Early stopping, best iteration is:\n",
            "[443]\ttraining's rmse: 2.24734\tvalid_1's rmse: 2.28184\n",
            "Fold: 1\n",
            "3581549 1190492\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 2.44325\tvalid_1's rmse: 2.42422\n",
            "[200]\ttraining's rmse: 2.32781\tvalid_1's rmse: 2.33911\n",
            "[300]\ttraining's rmse: 2.28648\tvalid_1's rmse: 2.31937\n",
            "[400]\ttraining's rmse: 2.25908\tvalid_1's rmse: 2.3081\n",
            "[500]\ttraining's rmse: 2.23852\tvalid_1's rmse: 2.30216\n",
            "[600]\ttraining's rmse: 2.21862\tvalid_1's rmse: 2.29699\n",
            "[700]\ttraining's rmse: 2.20228\tvalid_1's rmse: 2.29264\n",
            "[800]\ttraining's rmse: 2.18913\tvalid_1's rmse: 2.28981\n",
            "[900]\ttraining's rmse: 2.17476\tvalid_1's rmse: 2.28688\n",
            "[1000]\ttraining's rmse: 2.16225\tvalid_1's rmse: 2.28482\n",
            "Early stopping, best iteration is:\n",
            "[976]\ttraining's rmse: 2.1651\tvalid_1's rmse: 2.28437\n",
            "Fold: 2\n",
            "3583774 1188267\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 2.43651\tvalid_1's rmse: 2.48618\n",
            "[200]\ttraining's rmse: 2.32214\tvalid_1's rmse: 2.37402\n",
            "[300]\ttraining's rmse: 2.27647\tvalid_1's rmse: 2.34371\n",
            "[400]\ttraining's rmse: 2.24686\tvalid_1's rmse: 2.32981\n",
            "[500]\ttraining's rmse: 2.22431\tvalid_1's rmse: 2.32199\n",
            "[600]\ttraining's rmse: 2.2065\tvalid_1's rmse: 2.31625\n",
            "[700]\ttraining's rmse: 2.18892\tvalid_1's rmse: 2.31363\n",
            "[800]\ttraining's rmse: 2.17448\tvalid_1's rmse: 2.31025\n",
            "[900]\ttraining's rmse: 2.16128\tvalid_1's rmse: 2.30791\n",
            "[1000]\ttraining's rmse: 2.14894\tvalid_1's rmse: 2.30513\n",
            "[1100]\ttraining's rmse: 2.13779\tvalid_1's rmse: 2.30434\n",
            "Early stopping, best iteration is:\n",
            "[1119]\ttraining's rmse: 2.13583\tvalid_1's rmse: 2.30359\n",
            "Fold: 3\n",
            "3575639 1196402\n",
            "Training until validation scores don't improve for 30 rounds.\n",
            "[100]\ttraining's rmse: 2.43984\tvalid_1's rmse: 2.52146\n",
            "[200]\ttraining's rmse: 2.3195\tvalid_1's rmse: 2.43343\n",
            "[300]\ttraining's rmse: 2.27279\tvalid_1's rmse: 2.40375\n",
            "[400]\ttraining's rmse: 2.24487\tvalid_1's rmse: 2.38902\n",
            "[500]\ttraining's rmse: 2.22572\tvalid_1's rmse: 2.37624\n",
            "[600]\ttraining's rmse: 2.20706\tvalid_1's rmse: 2.36914\n",
            "[700]\ttraining's rmse: 2.19109\tvalid_1's rmse: 2.3612\n",
            "[800]\ttraining's rmse: 2.17573\tvalid_1's rmse: 2.35623\n",
            "[900]\ttraining's rmse: 2.16333\tvalid_1's rmse: 2.35168\n",
            "[1000]\ttraining's rmse: 2.15062\tvalid_1's rmse: 2.34899\n",
            "[1100]\ttraining's rmse: 2.13968\tvalid_1's rmse: 2.34659\n",
            "[1200]\ttraining's rmse: 2.12965\tvalid_1's rmse: 2.34366\n",
            "[1300]\ttraining's rmse: 2.12015\tvalid_1's rmse: 2.34211\n",
            "Early stopping, best iteration is:\n",
            "[1352]\ttraining's rmse: 2.11487\tvalid_1's rmse: 2.3405\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z-9_Ux2_Y4wC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "lgb_model_"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "diB1qBcxLt6A",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "bcfc5f65-f3a7-4ee2-fa24-d202dc720d87"
      },
      "source": [
        "########################### Predict\n",
        "#################################################################################\n",
        "\n",
        "for fold_ in [0,1,2,3]:\n",
        "    print(\"FOLD:\", fold_)\n",
        "    # Join back the Test dataset with \n",
        "    # a small part of the training data \n",
        "    # to make recursive features\n",
        "    all_preds = pd.DataFrame()\n",
        "    base_test = get_base_test()\n",
        "    # Timer to measure predictions time \n",
        "    main_time = time.time()\n",
        "\n",
        "    # Loop over each prediction day\n",
        "    # As rolling lags are the most timeconsuming\n",
        "    # we will calculate it for whole day\n",
        "    for PREDICT_DAY in range(1,29):    \n",
        "        print('Predict | Day:', PREDICT_DAY)\n",
        "        start_time = time.time()\n",
        "\n",
        "        # Make temporary grid to calculate rolling lags\n",
        "        grid_df = base_test.copy()\n",
        "        grid_df = pd.concat([grid_df, df_parallelize_run(make_lag_roll, ROLS_SPLIT)], axis=1)\n",
        "\n",
        "        for store_id in STORES_IDS:\n",
        "        \n",
        "            # Read all our models and make predictions\n",
        "            # for each day/store pairs\n",
        "            model_path = 'lgb_model_'+store_id+'_'+str(fold_)+'.bin' \n",
        "            if USE_AUX:\n",
        "                model_path = '' + model_path\n",
        "\n",
        "            estimator = pickle.load(open(model_path, 'rb'))\n",
        "\n",
        "            day_mask = base_test['d']==(END_TRAIN+PREDICT_DAY)\n",
        "            store_mask = base_test['store_id']==store_id\n",
        "\n",
        "            mask = (day_mask)&(store_mask)\n",
        "            base_test[TARGET][mask] = estimator.predict(grid_df[mask][MODEL_FEATURES])\n",
        "\n",
        "        # Make good column naming and add \n",
        "        # to all_preds DataFrame\n",
        "        temp_df = base_test[day_mask][['id',TARGET]]\n",
        "        temp_df.columns = ['id','F'+str(PREDICT_DAY)]\n",
        "        if 'id' in list(all_preds):\n",
        "            all_preds = all_preds.merge(temp_df, on=['id'], how='left')\n",
        "        else:\n",
        "            all_preds = temp_df.copy()\n",
        "        all_preds = all_preds.reset_index(drop=True)\n",
        "        print('#'*10, ' %0.2f min round |' % ((time.time() - start_time) / 60),\n",
        "                      ' %0.2f min total |' % ((time.time() - main_time) / 60),\n",
        "                      ' %0.2f day sales |' % (temp_df['F'+str(PREDICT_DAY)].sum()))\n",
        "    all_preds.to_csv('all_preds_'+str(fold_)+'.csv',index=False)\n",
        "    !cp 'all_preds_'+str(fold_)+'.csv' /content/gdrive/'My Drive'/m5_new\n",
        "    del temp_df, all_preds"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "FOLD: 0\n",
            "Predict | Day: 1\n",
            "##########  0.74 min round |  0.74 min total |  39840.63 day sales |\n",
            "Predict | Day: 2\n",
            "##########  0.65 min round |  1.39 min total |  37056.26 day sales |\n",
            "Predict | Day: 3\n",
            "##########  0.64 min round |  2.03 min total |  37019.72 day sales |\n",
            "Predict | Day: 4\n",
            "##########  0.63 min round |  2.66 min total |  37172.93 day sales |\n",
            "Predict | Day: 5\n",
            "##########  0.63 min round |  3.29 min total |  42358.75 day sales |\n",
            "Predict | Day: 6\n",
            "##########  0.63 min round |  3.92 min total |  51003.85 day sales |\n",
            "Predict | Day: 7\n",
            "##########  0.62 min round |  4.54 min total |  52010.93 day sales |\n",
            "Predict | Day: 8\n",
            "##########  0.62 min round |  5.16 min total |  44244.34 day sales |\n",
            "Predict | Day: 9\n",
            "##########  0.63 min round |  5.79 min total |  38827.85 day sales |\n",
            "Predict | Day: 10\n",
            "##########  0.64 min round |  6.42 min total |  43805.29 day sales |\n",
            "Predict | Day: 11\n",
            "##########  0.63 min round |  7.05 min total |  45121.14 day sales |\n",
            "Predict | Day: 12\n",
            "##########  0.65 min round |  7.70 min total |  52011.94 day sales |\n",
            "Predict | Day: 13\n",
            "##########  0.64 min round |  8.34 min total |  55678.42 day sales |\n",
            "Predict | Day: 14\n",
            "##########  0.63 min round |  8.98 min total |  58000.56 day sales |\n",
            "Predict | Day: 15\n",
            "##########  0.63 min round |  9.61 min total |  47834.77 day sales |\n",
            "Predict | Day: 16\n",
            "##########  0.64 min round |  10.25 min total |  43230.48 day sales |\n",
            "Predict | Day: 17\n",
            "##########  0.64 min round |  10.89 min total |  42670.58 day sales |\n",
            "Predict | Day: 18\n",
            "##########  0.64 min round |  11.54 min total |  44435.36 day sales |\n",
            "Predict | Day: 19\n",
            "##########  0.64 min round |  12.18 min total |  46505.58 day sales |\n",
            "Predict | Day: 20\n",
            "##########  0.63 min round |  12.81 min total |  57756.26 day sales |\n",
            "Predict | Day: 21\n",
            "##########  0.64 min round |  13.45 min total |  58995.76 day sales |\n",
            "Predict | Day: 22\n",
            "##########  0.65 min round |  14.10 min total |  46069.80 day sales |\n",
            "Predict | Day: 23\n",
            "##########  0.63 min round |  14.72 min total |  43323.75 day sales |\n",
            "Predict | Day: 24\n",
            "##########  0.63 min round |  15.36 min total |  44625.31 day sales |\n",
            "Predict | Day: 25\n",
            "##########  0.63 min round |  15.99 min total |  41130.13 day sales |\n",
            "Predict | Day: 26\n",
            "##########  0.63 min round |  16.61 min total |  45417.02 day sales |\n",
            "Predict | Day: 27\n",
            "##########  0.65 min round |  17.26 min total |  54266.15 day sales |\n",
            "Predict | Day: 28\n",
            "##########  0.66 min round |  17.92 min total |  49903.81 day sales |\n",
            "/bin/bash: -c: line 0: syntax error near unexpected token `('\n",
            "/bin/bash: -c: line 0: `cp 'all_preds_'+str(fold_)+'.csv' /content/gdrive/'My Drive'/m5_new'\n",
            "FOLD: 1\n",
            "Predict | Day: 1\n",
            "##########  0.77 min round |  0.77 min total |  39716.06 day sales |\n",
            "Predict | Day: 2\n",
            "##########  0.72 min round |  1.48 min total |  36982.44 day sales |\n",
            "Predict | Day: 3\n",
            "##########  0.67 min round |  2.16 min total |  36931.43 day sales |\n",
            "Predict | Day: 4\n",
            "##########  0.67 min round |  2.82 min total |  37066.08 day sales |\n",
            "Predict | Day: 5\n",
            "##########  0.64 min round |  3.46 min total |  42117.41 day sales |\n",
            "Predict | Day: 6\n",
            "##########  0.64 min round |  4.10 min total |  50371.81 day sales |\n",
            "Predict | Day: 7\n",
            "##########  0.64 min round |  4.74 min total |  51278.80 day sales |\n",
            "Predict | Day: 8\n",
            "##########  0.63 min round |  5.37 min total |  45831.97 day sales |\n",
            "Predict | Day: 9\n",
            "##########  0.62 min round |  5.99 min total |  39016.96 day sales |\n",
            "Predict | Day: 10\n",
            "##########  0.62 min round |  6.61 min total |  44233.68 day sales |\n",
            "Predict | Day: 11\n",
            "##########  0.63 min round |  7.25 min total |  44767.39 day sales |\n",
            "Predict | Day: 12\n",
            "##########  0.62 min round |  7.87 min total |  52685.31 day sales |\n",
            "Predict | Day: 13\n",
            "##########  0.62 min round |  8.50 min total |  56091.27 day sales |\n",
            "Predict | Day: 14\n",
            "##########  0.63 min round |  9.13 min total |  57822.54 day sales |\n",
            "Predict | Day: 15\n",
            "##########  0.62 min round |  9.75 min total |  47719.33 day sales |\n",
            "Predict | Day: 16\n",
            "##########  0.63 min round |  10.38 min total |  44182.97 day sales |\n",
            "Predict | Day: 17\n",
            "##########  0.64 min round |  11.02 min total |  42777.49 day sales |\n",
            "Predict | Day: 18\n",
            "##########  0.63 min round |  11.65 min total |  44541.48 day sales |\n",
            "Predict | Day: 19\n",
            "##########  0.62 min round |  12.27 min total |  46822.73 day sales |\n",
            "Predict | Day: 20\n",
            "##########  0.63 min round |  12.91 min total |  57535.02 day sales |\n",
            "Predict | Day: 21\n",
            "##########  0.63 min round |  13.54 min total |  58802.10 day sales |\n",
            "Predict | Day: 22\n",
            "##########  0.62 min round |  14.16 min total |  46042.67 day sales |\n",
            "Predict | Day: 23\n",
            "##########  0.63 min round |  14.79 min total |  43010.91 day sales |\n",
            "Predict | Day: 24\n",
            "##########  0.65 min round |  15.43 min total |  44266.11 day sales |\n",
            "Predict | Day: 25\n",
            "##########  0.64 min round |  16.07 min total |  41378.41 day sales |\n",
            "Predict | Day: 26\n",
            "##########  0.64 min round |  16.71 min total |  45868.08 day sales |\n",
            "Predict | Day: 27\n",
            "##########  0.65 min round |  17.36 min total |  54695.22 day sales |\n",
            "Predict | Day: 28\n",
            "##########  0.64 min round |  18.00 min total |  50872.51 day sales |\n",
            "/bin/bash: -c: line 0: syntax error near unexpected token `('\n",
            "/bin/bash: -c: line 0: `cp 'all_preds_'+str(fold_)+'.csv' /content/gdrive/'My Drive'/m5_new'\n",
            "FOLD: 2\n",
            "Predict | Day: 1\n",
            "##########  0.72 min round |  0.72 min total |  39782.43 day sales |\n",
            "Predict | Day: 2\n",
            "##########  0.62 min round |  1.34 min total |  37375.69 day sales |\n",
            "Predict | Day: 3\n",
            "##########  0.63 min round |  1.97 min total |  36781.29 day sales |\n",
            "Predict | Day: 4\n",
            "##########  0.64 min round |  2.61 min total |  37029.11 day sales |\n",
            "Predict | Day: 5\n",
            "##########  0.64 min round |  3.25 min total |  41992.71 day sales |\n",
            "Predict | Day: 6\n",
            "##########  0.64 min round |  3.89 min total |  50427.09 day sales |\n",
            "Predict | Day: 7\n",
            "##########  0.64 min round |  4.53 min total |  51155.27 day sales |\n",
            "Predict | Day: 8\n",
            "##########  0.62 min round |  5.15 min total |  45474.08 day sales |\n",
            "Predict | Day: 9\n",
            "##########  0.64 min round |  5.79 min total |  39608.53 day sales |\n",
            "Predict | Day: 10\n",
            "##########  0.64 min round |  6.42 min total |  43739.06 day sales |\n",
            "Predict | Day: 11\n",
            "##########  0.64 min round |  7.06 min total |  45023.47 day sales |\n",
            "Predict | Day: 12\n",
            "##########  0.63 min round |  7.69 min total |  52808.28 day sales |\n",
            "Predict | Day: 13\n",
            "##########  0.63 min round |  8.32 min total |  55776.88 day sales |\n",
            "Predict | Day: 14\n",
            "##########  0.63 min round |  8.95 min total |  57895.89 day sales |\n",
            "Predict | Day: 15\n",
            "##########  0.63 min round |  9.58 min total |  47642.81 day sales |\n",
            "Predict | Day: 16\n",
            "##########  0.63 min round |  10.20 min total |  43741.26 day sales |\n",
            "Predict | Day: 17\n",
            "##########  0.64 min round |  10.85 min total |  42808.79 day sales |\n",
            "Predict | Day: 18\n",
            "##########  0.63 min round |  11.47 min total |  44599.36 day sales |\n",
            "Predict | Day: 19\n",
            "##########  0.64 min round |  12.11 min total |  46499.22 day sales |\n",
            "Predict | Day: 20\n",
            "##########  0.64 min round |  12.76 min total |  57794.44 day sales |\n",
            "Predict | Day: 21\n",
            "##########  0.63 min round |  13.39 min total |  58869.57 day sales |\n",
            "Predict | Day: 22\n",
            "##########  0.64 min round |  14.03 min total |  45725.06 day sales |\n",
            "Predict | Day: 23\n",
            "##########  0.65 min round |  14.68 min total |  43216.74 day sales |\n",
            "Predict | Day: 24\n",
            "##########  0.64 min round |  15.32 min total |  44110.69 day sales |\n",
            "Predict | Day: 25\n",
            "##########  0.63 min round |  15.95 min total |  41013.37 day sales |\n",
            "Predict | Day: 26\n",
            "##########  0.65 min round |  16.59 min total |  45256.76 day sales |\n",
            "Predict | Day: 27\n",
            "##########  0.64 min round |  17.23 min total |  54126.06 day sales |\n",
            "Predict | Day: 28\n",
            "##########  0.63 min round |  17.87 min total |  51380.19 day sales |\n",
            "/bin/bash: -c: line 0: syntax error near unexpected token `('\n",
            "/bin/bash: -c: line 0: `cp 'all_preds_'+str(fold_)+'.csv' /content/gdrive/'My Drive'/m5_new'\n",
            "FOLD: 3\n",
            "Predict | Day: 1\n",
            "##########  0.73 min round |  0.73 min total |  39939.97 day sales |\n",
            "Predict | Day: 2\n",
            "##########  0.64 min round |  1.37 min total |  37168.50 day sales |\n",
            "Predict | Day: 3\n",
            "##########  0.65 min round |  2.01 min total |  36873.71 day sales |\n",
            "Predict | Day: 4\n",
            "##########  0.65 min round |  2.66 min total |  37030.76 day sales |\n",
            "Predict | Day: 5\n",
            "##########  0.65 min round |  3.31 min total |  42068.41 day sales |\n",
            "Predict | Day: 6\n",
            "##########  0.67 min round |  3.98 min total |  50480.71 day sales |\n",
            "Predict | Day: 7\n",
            "##########  0.69 min round |  4.67 min total |  51431.06 day sales |\n",
            "Predict | Day: 8\n",
            "##########  0.66 min round |  5.32 min total |  44791.98 day sales |\n",
            "Predict | Day: 9\n",
            "##########  0.63 min round |  5.95 min total |  38817.20 day sales |\n",
            "Predict | Day: 10\n",
            "##########  0.63 min round |  6.58 min total |  44100.21 day sales |\n",
            "Predict | Day: 11\n",
            "##########  0.63 min round |  7.21 min total |  44415.90 day sales |\n",
            "Predict | Day: 12\n",
            "##########  0.63 min round |  7.84 min total |  53169.46 day sales |\n",
            "Predict | Day: 13\n",
            "##########  0.64 min round |  8.48 min total |  55852.27 day sales |\n",
            "Predict | Day: 14\n",
            "##########  0.64 min round |  9.12 min total |  57837.81 day sales |\n",
            "Predict | Day: 15\n",
            "##########  0.63 min round |  9.75 min total |  47577.17 day sales |\n",
            "Predict | Day: 16\n",
            "##########  0.63 min round |  10.38 min total |  43274.12 day sales |\n",
            "Predict | Day: 17\n",
            "##########  0.63 min round |  11.01 min total |  42856.74 day sales |\n",
            "Predict | Day: 18\n",
            "##########  0.63 min round |  11.63 min total |  44687.28 day sales |\n",
            "Predict | Day: 19\n",
            "##########  0.63 min round |  12.26 min total |  46583.81 day sales |\n",
            "Predict | Day: 20\n",
            "##########  0.63 min round |  12.89 min total |  57906.23 day sales |\n",
            "Predict | Day: 21\n",
            "##########  0.64 min round |  13.53 min total |  59141.79 day sales |\n",
            "Predict | Day: 22\n",
            "##########  0.64 min round |  14.17 min total |  46348.91 day sales |\n",
            "Predict | Day: 23\n",
            "##########  0.64 min round |  14.81 min total |  43414.42 day sales |\n",
            "Predict | Day: 24\n",
            "##########  0.64 min round |  15.45 min total |  44697.03 day sales |\n",
            "Predict | Day: 25\n",
            "##########  0.64 min round |  16.09 min total |  41439.48 day sales |\n",
            "Predict | Day: 26\n",
            "##########  0.63 min round |  16.72 min total |  45719.60 day sales |\n",
            "Predict | Day: 27\n",
            "##########  0.64 min round |  17.36 min total |  54461.82 day sales |\n",
            "Predict | Day: 28\n",
            "##########  0.64 min round |  18.00 min total |  51539.67 day sales |\n",
            "/bin/bash: -c: line 0: syntax error near unexpected token `('\n",
            "/bin/bash: -c: line 0: `cp 'all_preds_'+str(fold_)+'.csv' /content/gdrive/'My Drive'/m5_new'\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Crt_qlQB7vzn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "all_preds_0=pd.read_csv('all_preds_0'+'.csv')\n",
        "all_preds_1=pd.read_csv('all_preds_1'+'.csv')\n",
        "all_preds_2=pd.read_csv('all_preds_2'+'.csv')\n",
        "all_preds_3=pd.read_csv('all_preds_3'+'.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ncKLw8Q27flW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Create Dummy DataFrame to store predictions\n",
        "final_all_preds = pd.DataFrame()\n",
        "final_all_preds['id'] = all_preds_1['id']\n",
        "for item in all_preds_1:\n",
        "    if item!='id':\n",
        "        final_all_preds[item]=(all_preds_0[item]*(1/4))+(all_preds_1[item]*(1/4))+(all_preds_2[item]*(1/4) + all_preds_3[item]*(1/4))\n",
        "final_all_preds"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FEy0UjYp7fwE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "0\tHOBBIES_1_001_CA_1_evaluation\t0.897567\t0.808433\t0.787248\t0.834224\t1.023035\t1.240877\t1.142573\t1.025858\t0.869062\t0.954067\t0.819747\t1.053971\t1.252962\t1.169393\t0.899686\t0.867141\t0.849904\t0.841423\t0.972701\t1.255027\t1.143760\t0.938854\t0.851907\t0.850253\t0.899863\t1.088019\t1.209305\t1.073520\n",
        "1\tHOBBIES_1_002_CA_1_evaluation\t0.213175\t0.190870\t0.194972\t0.192493\t0.222992\t0.285271\t0.312677\t0.237396\t0.207413\t0.242501\t0.211367\t0.279419\t0.354906\t0.331790\t0.228765\t0.229240\t0.223464\t0.232000\t0.270699\t0.339222\t0.362581\t0.243112\t0.229265\t0.233733\t0.248364\t0.294227\t0.379558\t0.405173\n",
        "2\tHOBBIES_1_003_CA_1_evaluation\t0.520261\t0.471055\t0.484993\t0.504455\t0.672100\t0.799509\t0.801922\t0.553256\t0.526301\t0.513892\t0.486507\t0.718935\t0.842681\t0.777672\t0.509570\t0.502527\t0.497165\t0.507420\t0.669064\t0.811831\t0.802531\t0.527526\t0.471016\t0.481420\t0.506258\t0.684005\t0.781834\t0.785586\n",
        "3\tHOBBIES_1_004_CA_1_evaluation\t1.515989\t1.303852\t1.299618\t1.353299\t1.810563\t2.557639\t2.840314\t1.936896\t1.364120\t1.369339\t1.348411\t2.176968\t2.847142\t2.942518\t1.648374\t1.457491\t1.317094\t1.419085\t1.880304\t2.538570\t2.938713\t1.634545\t1.339442\t1.322486\t1.436190\t1.855930\t2.566287\t2.624366\n",
        "4\tHOBBIES_1_005_CA_1_evaluation\t1.114401\t0.970305\t0.939629\t0.999928\t1.179970\t1.410065\t1.534941\t1.219148\t1.056377\t1.141098\t1.029266\t1.398179\t1.646224\t1.656603\t1.128693\t1.025425\t1.034846\t1.038983\t1.244264\t1.464190\t1.542456\t1.105556\t1.009824\t1.015949\t1.015203\t1.298450\t1.558173\t1.432512\n",
        "...\t...\t...\t...\t...\t...\t...\t...\t...\t...\t...\t...\t...\t...\t...\t...\t...\t...\t...\t...\t...\t...\t...\t...\t...\t...\t...\t...\t...\t...\n",
        "30485\tFOODS_3_823_WI_3_evaluation\t0.500702\t0.462877\t0.433694\t0.447268\t0.511382\t0.555119\t0.664946\t0.539321\t0.468327\t0.455805\t0.654341\t0.701230\t0.670452\t0.749824\t0.650389\t0.533744\t0.557373\t0.590014\t0.583402\t0.729517\t0.828638\t0.585383\t0.625468\t0.593743\t0.480860\t0.513310\t0.595156\t0.655568\n",
        "30486\tFOODS_3_824_WI_3_evaluation\t0.259211\t0.254881\t0.238968\t0.225033\t0.226900\t0.287327\t0.297278\t0.241045\t0.242830\t0.271071\t0.368390\t0.381267\t0.376315\t0.420627\t0.360178\t0.316844\t0.361534\t0.345815\t0.258029\t0.379033\t0.406634\t0.289211\t0.344375\t0.349159\t0.273884\t0.241947\t0.298811\t0.298772\n",
        "30487\tFOODS_3_825_WI_3_evaluation\t0.630791\t0.529853\t0.491117\t0.477037\t0.544271\t0.641009\t0.701201\t0.662344\t0.525250\t0.674044\t1.036723\t1.112819\t0.895466\t1.287720\t1.139596\t0.811612\t0.999034\t0.964596\t0.775849\t1.146324\t1.273331\t0.942830\t1.058475\t1.068908\t0.743412\t0.696659\t0.798183\t0.936363\n",
        "30488\tFOODS_3_826_WI_3_evaluation\t1.014185\t1.014675\t0.968098\t0.941371\t1.094441\t1.173944\t1.154782\t1.163941\t1.041478\t0.993564\t1.318697\t1.492611\t1.347546\t1.528297\t1.231180\t1.068935\t1.204987\t1.188645\t1.066231\t1.481773\t1.546942\t1.180402\t1.334267\t1.270570\t1.031520\t1.117360\t1.197175\t1.297182\n",
        "30489\tFOODS_3_827_WI_3_evaluation\t1.808649\t1.595394\t1.546023\t1.589576\t1.912314\t1.996711\t1.802233\t1.733731\t1.550154\t1.312341\t1.744125\t2.024802\t1.988244\t1.903606\t1.651901\t1.457500\t1.466568\t1.542802\t1.580044\t1.940835\t1.915329\t1.624481\t1.829056\t1.647234\t1.447651\t1.640128\t1.813909\t1.841560\n",
        "30490 rows × 29 columns"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bmz0bxeH7f4v",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "########################### Export\n",
        "#################################################################################\n",
        "# Reading competition sample submission and\n",
        "# merging our predictions\n",
        "# As we have predictions only for \"_validation\" data\n",
        "# we need to do fillna() for \"_evaluation\" items\n",
        "submission = pd.read_csv(ORIGINAL + 'sample_submission.csv')[['id']]\n",
        "submission = submission.merge(final_all_preds, on=['id'], how='left').fillna(0)\n",
        "submission.to_csv('submission'+'.csv', index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y0J_sppvvrIp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!cp 'submission.csv' /content/gdrive/'My Drive'/m5_new"
      ],
      "execution_count": 12,
      "outputs": []
    }
  ]
}