NN_pricing_OLD_torch_changed_setting.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.metrics import mean_squared_error as mse\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "krx = '/data/hun/KRX_marketdata/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tickers = pd.read_csv(krx+'ticker_recent.csv', encoding ='cp949')\n",
    "tickers.columns = ('full_ticker', 'ticker', 'name_full', 'name',\n",
    "              'name_eng', 'listed_data', 'market', 'security_category',\n",
    "              'related_department', 'preferred', 'face_value', 'shares')\n",
    "tickers = tickers.drop(['related_department'], axis=1).dropna().reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "factors= pd.read_csv(krx+'factors.csv')\n",
    "factors['ticker'] = factors.ticker.apply(lambda x: '0'*(6-len(str(x)))+str(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "factors =factors.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "factors.reset_index(drop = True, inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preprocessing\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def xvar(data):\n",
    "    return data.drop(['date','ticker','name'], axis=1).astype('float')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def xy(data):\n",
    "    return data.drop(['ret'], axis=1), data.ret"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dates = factors.date.unique()[::-1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pooling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#preprocessing\n",
    "\n",
    "length = len(dates)\n",
    "ind_train = int(length*0.7)+2\n",
    "ind_tv = int(length*0.4)\n",
    "# pool_train_pre = factors[factors.date.apply(lambda x: x in dates[0:ind_train])]\n",
    "# pool_test = factors[factors.date.apply(lambda x: x in dates[ind_train:])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dates[ind_tv], dates[ind_train]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Time series split\n",
    "\n",
    "pool_train = []\n",
    "pool_val = []\n",
    "pool_test = []\n",
    "cnt = ind_tv\n",
    "start= 0\n",
    "for i in tqdm(range(12)):\n",
    "    train_ind = dates[start:cnt-12]\n",
    "    val_ind = dates[cnt-12:cnt]\n",
    "    test_ind = dates[cnt:min(cnt+12, length)]\n",
    "    \n",
    "    pool_train.append(xvar(factors[factors.date.apply(lambda x: x in train_ind)]))\n",
    "    pool_val.append(xvar(factors[factors.date.apply(lambda x: x in val_ind)]))\n",
    "    pool_test.append(xvar(factors[factors.date.apply(lambda x: x in test_ind)]))\n",
    "    cnt += 12"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Neural net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import Dataset\n",
    "class CustomDataset(Dataset):\n",
    "    def __init__(self, X, Y):\n",
    "        self.x_data=torch.FloatTensor(X.values)\n",
    "        self.y_data=torch.FloatTensor(Y.values.reshape(-1,1))\n",
    "    def __len__(self):     #len(dataset)으로 호출 가능\n",
    "        return len(self.x_data)\n",
    "    def __getitem__(self,idx):   #dataset[]으로 callable하게 된다\n",
    "        x=self.x_data[idx]\n",
    "        y=self.y_data[idx]\n",
    "        return x, y\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "def stdscaler(data_tensor):\n",
    "    scalers = {}\n",
    "    for i in range(data_tensor.shape[0]): #batch값을 넣어주기\n",
    "        scalers[i] = StandardScaler()\n",
    "        data_tensor[i, :, :] = torch.tensor(scalers[i].fit_transform(data_tensor[i, :, :]))\n",
    "    return data_tensor\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cuda = torch.cuda.is_available() # False\n",
    "if cuda:\n",
    "    device = 'cuda:3'\n",
    "else:\n",
    "    device = 'cpu'\n",
    "cuda, device"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FC Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net1(nn.Module):\n",
    "    def __init__(self):\n",
    "        self.name = 'Net1'\n",
    "        super(Net1, self).__init__()\n",
    "        self.fc1 = nn.Linear(19,16)\n",
    "        self.fc2 = nn.Linear(16,1)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = self.fc2(x)\n",
    "        return x\n",
    "\n",
    "class Net2(nn.Module):\n",
    "    def __init__(self):\n",
    "        self.name = 'Net2'\n",
    "        super(Net2, self).__init__()\n",
    "        self.fc1 = nn.Linear(19,16)\n",
    "        self.fc2 = nn.Linear(16,8)\n",
    "        self.fc3 = nn.Linear(8,1)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.relu(self.fc2(x)) \n",
    "        x = self.fc3(x)\n",
    "        return x\n",
    "\n",
    "class Net3(nn.Module):\n",
    "    def __init__(self):\n",
    "        self.name = 'Net3'\n",
    "        super(Net3, self).__init__()\n",
    "        self.fc1 = nn.Linear(19,16)\n",
    "        self.fc2 = nn.Linear(16,8)\n",
    "        self.fc3 = nn.Linear(8,4)\n",
    "        self.fc4 = nn.Linear(4,1)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.relu(self.fc2(x)) \n",
    "        x = F.relu(self.fc3(x)) \n",
    "        x = self.fc4(x)\n",
    "        return x\n",
    "    \n",
    "class Net4(nn.Module):\n",
    "    def __init__(self):\n",
    "        self.name = 'Net4'\n",
    "        super(Net4, self).__init__()\n",
    "        self.fc1 = nn.Linear(19,16)\n",
    "        self.fc2 = nn.Linear(16,8)\n",
    "        self.fc3 = nn.Linear(8,4)\n",
    "        self.fc4 = nn.Linear(4,2)\n",
    "        self.fc5 = nn.Linear(2,1)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.relu(self.fc2(x)) \n",
    "        x = F.relu(self.fc3(x))\n",
    "        x = F.relu(self.fc4(x))\n",
    "        x = self.fc5(x)\n",
    "        return x\n",
    "\n",
    "class Net5(nn.Module):\n",
    "    def __init__(self):\n",
    "        self.name = 'Net5'\n",
    "        super(Net5, self).__init__()\n",
    "        self.fc1 = nn.Linear(19,32)\n",
    "        self.fc2 = nn.Linear(32,16)\n",
    "        self.fc3 = nn.Linear(16,8)\n",
    "        self.fc4 = nn.Linear(8,1)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.relu(self.fc2(x)) \n",
    "        x = F.relu(self.fc3(x))\n",
    "        x = self.fc4(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net6(nn.Module):\n",
    "    def __init__(self):\n",
    "        self.name = 'Net6'\n",
    "        super(Net6, self).__init__()\n",
    "        self.fc1 = nn.Linear(19,64)\n",
    "        self.fc2 = nn.Linear(64,32)\n",
    "        self.fc3 = nn.Linear(32,16)\n",
    "        self.fc4 = nn.Linear(16,8)\n",
    "        self.fc5 = nn.Linear(8,4)\n",
    "        self.fc6 = nn.Linear(4,2)\n",
    "        self.fc7 = nn.Linear(2,1)\n",
    "        \n",
    "        self.fc8 = nn.Linear(19,16)\n",
    "        self.fc9 = nn.Linear(16,8)\n",
    "        self.fc10 = nn.Linear(8,4)\n",
    "        self.fc11 = nn.Linear(4,2)\n",
    "        self.fc12 = nn.Linear(2,1)\n",
    "        \n",
    "        self.fc13 = nn.Linear(8,4)\n",
    "        self.fc14 = nn.Linear(4,2)\n",
    "        self.fc15 = nn.Linear(2,1)\n",
    "        \n",
    "        self.res = nn.Linear(3,1)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x1 = F.relu(self.fc1(x))\n",
    "        x1 = F.relu(self.fc2(x1)) \n",
    "        x1 = F.relu(self.fc3(x1))\n",
    "        x1 = F.relu(self.fc4(x1))\n",
    "        x1 = F.relu(self.fc5(x1))\n",
    "        x2 = F.relu(self.fc6(x1))\n",
    "        x2 = self.fc7(x2)\n",
    "        \n",
    "        y1 = F.relu(self.fc8(x))\n",
    "        y1 = F.relu(self.fc9(y1))\n",
    "        y1 = F.relu(self.fc10(y1))\n",
    "        y2 = F.relu(self.fc11(y1))\n",
    "        y2 = self.fc12(y2)\n",
    "        \n",
    "        z=torch.cat((x1,y1),1)\n",
    "        \n",
    "        z = F.relu(self.fc13(z))\n",
    "        z = F.relu(self.fc14(z))\n",
    "        z = self.fc15(z)\n",
    "        \n",
    "        fin = self.res(torch.cat((x2, y2, z),1))\n",
    "        \n",
    "        return fin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ros_nn(model, CustomData):\n",
    "    model.eval()\n",
    "    pred = model(CustomData.x_data.to(device)).cpu().detach().numpy().reshape(-1)\n",
    "    real = CustomData.y_data.numpy().reshape(-1)\n",
    "    ros = 1- ((pred - real)**2).mean()/(real**2).mean()\n",
    "    return ros, pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_hun(output, real):\n",
    "    temp=output-real\n",
    "#     bear_bull=(real>0)*(output<0)*temp\n",
    "    bull_bear=(real<0)*(output>0)*temp\n",
    "#     bear=(real<0)*temp\n",
    "    bull=(real>0)*temp\n",
    "    return ((bull_bear+bull)**2).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def torch_train(model, train_loader, valid_loader, period, ite = 700, lr = 0.01):\n",
    "#     criterion = torch.nn.MSELoss()\n",
    "    criterion = loss_hun\n",
    "    optimizer = optim.RAdam(model.parameters(), lr=lr ) #, lr_decay=1e-9)\n",
    "    \n",
    "    loss_array_train = []\n",
    "    loss_array_valid = []\n",
    "    \n",
    "    patience = 0\n",
    "    min_loss = np.inf\n",
    "\n",
    "    for i in tqdm(range(ite)):\n",
    "        loss_array_train_tmp = []\n",
    "        \n",
    "        model.train()\n",
    "        for batch_idx, samples in enumerate(train_loader):\n",
    "            trainX_tensor, trainY_tensor=samples\n",
    "            optimizer.zero_grad()\n",
    "            out = model(trainX_tensor.to(device))\n",
    "            loss = criterion(out, trainY_tensor.to(device))\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            loss_array_train_tmp.append(loss.item())\n",
    "        loss_array_train.append(np.mean(loss_array_train_tmp))    \n",
    "\n",
    "        torch.cuda.empty_cache() ## 캐시 비워주기 자주 해줘야함\n",
    "\n",
    "        model.eval()\n",
    "        loss_array_valid_tmp = []\n",
    "        with torch.no_grad():\n",
    "            for batch_idx, val_samples in enumerate(valid_loader):\n",
    "                validX_tensor, validY_tensor=val_samples\n",
    "                val_out = model(validX_tensor.to(device))\n",
    "                val_loss = criterion(val_out, validY_tensor.to(device))\n",
    "                loss_array_valid_tmp.append(val_loss.item())\n",
    "            loss_array_valid.append(np.mean(loss_array_valid_tmp))\n",
    "\n",
    "        ## update the minimum loss\n",
    "        if min_loss > loss_array_valid[-1]:\n",
    "            patience = 0\n",
    "            min_loss = loss_array_valid[-1]\n",
    "        else:\n",
    "            patience += 1\n",
    "        if patience > 20: ## early stop when patience become larger than 10\n",
    "            break\n",
    "\n",
    "        torch.cuda.empty_cache()\n",
    "    \n",
    "    path = f\"./NN_save/{model.name}_{period}\"\n",
    "    torch.save(model, path+\"_RAdam\")\n",
    "    \n",
    "    train_loss, valid_loss = loss_array_train[-1], loss_array_valid[-1]\n",
    "    print('Model: {}, Train Loss: {:.4e}, Valid Loss: {:.4e}'.format(model.name, train_loss, valid_loss))\n",
    "    print(patience)\n",
    "    plt.plot(loss_array_train, label='Train Loss')\n",
    "    plt.plot(loss_array_valid, label='Valid Loss')\n",
    "    plt.ylabel('loss')\n",
    "    plt.xlabel('epoch')\n",
    "    plt.yscale('log')\n",
    "    plt.legend()\n",
    "    plt.show()\n",
    "    \n",
    "    torch.cuda.empty_cache()\n",
    "    return model, valid_loss, train_loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nets = [Net1(), Net2(), Net3(), Net4(), Net5()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "results = pd.DataFrame(columns = ['model','period','r_squared','valid_loss','train_loss'])\n",
    "results.to_csv('NN_test_res_rolling0_11_start0_adagrad.csv', index = False)\n",
    "model_name = [i.name for i in nets]\n",
    "preds = pd.DataFrame(columns = model_name)\n",
    "preds.to_csv('NN_test_pred0_11_start0_adagrad.csv', index = False)\n",
    "\n",
    "for period in range(8,12):\n",
    "    train_set, val_set, test_set = pool_train[period], pool_val[period], pool_test[period]\n",
    "    train_x, train_y = xy(train_set)\n",
    "    val_x, val_y = xy(val_set)\n",
    "    test_x, test_y = xy(test_set)\n",
    "    \n",
    "    train_data, valid_data = CustomDataset(train_x,train_y), CustomDataset(val_x, val_y)\n",
    "    test_data = CustomDataset(test_x, test_y)\n",
    "    \n",
    "    train_loader = DataLoader(train_data, batch_size=120, shuffle=False, pin_memory=True)\n",
    "    valid_loader = DataLoader(valid_data, batch_size=120, shuffle=False, pin_memory=True)\n",
    "    \n",
    "    r_squared, valid_loss, train_loss =[], [], []\n",
    "    predicted = []\n",
    "    for model in nets:\n",
    "        model = model.to(device)\n",
    "        model_trained, val_loss, tr_loss = torch_train(model, train_loader, valid_loader, period)\n",
    "        valid_loss.append(val_loss)\n",
    "        train_loss.append(tr_loss)\n",
    "        \n",
    "        r_sq, pred_res = ros_nn(model, test_data)\n",
    "        r_squared.append(r_sq)\n",
    "        predicted.append(pred_res)\n",
    "        print(r_sq)\n",
    "\n",
    "    res_temp = pd.DataFrame({\"model\": model_name, \"period\": [period]*5, \"r_squared\": r_squared, \n",
    "                             \"valid_loss\": valid_loss, \"train_loss\": train_loss})\n",
    "    \n",
    "    pred_temp = pd.DataFrame({\"Net1\":predicted[0], \"Net2\":predicted[1], \"Net3\":predicted[2], \n",
    "                              \"Net4\":predicted[3], \"Net5\":predicted[4]})\n",
    "    \n",
    "    res_temp.to_csv('NN_test_res_rolling0_11_start0_adagrad.csv', index = False, header = False, mode ='a')\n",
    "    pred_temp.to_csv('NN_test_pred0_11_start0_adagrad.csv', index = False, header = False, mode ='a')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "net6 = Net6()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "results = pd.DataFrame(columns = ['model','period','r_squared','valid_loss','train_loss'])\n",
    "results.to_csv('NN_test_res_net6.csv', index = False)\n",
    "preds = pd.DataFrame(columns =['net6'])\n",
    "preds.to_csv('NN_test_pred_net6.csv', index = False)\n",
    "\n",
    "for period in range(0,12):\n",
    "    train_set, val_set, test_set = pool_train[period], pool_val[period], pool_test[period]\n",
    "    train_x, train_y = xy(train_set)\n",
    "    val_x, val_y = xy(val_set)\n",
    "    test_x, test_y = xy(test_set)\n",
    "    \n",
    "    train_data, valid_data = CustomDataset(train_x,train_y), CustomDataset(val_x, val_y)\n",
    "    test_data = CustomDataset(test_x, test_y)\n",
    "    \n",
    "    train_loader = DataLoader(train_data, batch_size=120, shuffle=False, pin_memory=True)\n",
    "    valid_loader = DataLoader(valid_data, batch_size=120, shuffle=False, pin_memory=True)\n",
    "    \n",
    "    r_squared, valid_loss, train_loss =[], [], []\n",
    "    predicted = []\n",
    "\n",
    "    net6 = net6.to(device)\n",
    "    model_trained, val_loss, tr_loss = torch_train(net6, train_loader, valid_loader, period)\n",
    "    valid_loss.append(val_loss)\n",
    "    train_loss.append(tr_loss)\n",
    "\n",
    "    r_sq, pred_res = ros_nn(net6, test_data)\n",
    "    r_squared.append(r_sq)\n",
    "    predicted.append(pred_res)\n",
    "    print(r_sq)\n",
    "\n",
    "    res_temp = pd.DataFrame({\"model\": net6.name, \"period\": [period], \"r_squared\": r_squared})\n",
    "    pred_temp = pd.DataFrame({'net6': predicted[0]})\n",
    "    \n",
    "    \n",
    "    res_temp.to_csv('NN_test_res_net6_hunloss.csv', index = False, header = False, mode ='a')\n",
    "    pred_temp.to_csv('NN_test_pred_net6_hunloss.csv', index = False, header = False, mode ='a')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pd.read_csv('NN_test_res_net6_hunloss.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}