convlstm1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/sanjaymoto75/.conda/envs/env/lib/python3.7/site-packages/tqdm/autonotebook/__init__.py:14: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
      "  \" (e.g. in jupyter console)\", TqdmExperimentalWarning)\n"
     ]
    }
   ],
   "source": [
    "#!pip install torch===1.2.0 torchvision===0.4.0 -f https://download.pytorch.org/whl/torch_stable.html\n",
    "import torchvision\n",
    "import torch\n",
    "from torch import nn\n",
    "import torch.nn.functional as F\n",
    "import torchvision.models as models\n",
    "import torch.optim as optim\n",
    "import copy\n",
    "import os\n",
    "import torch\n",
    "from tqdm.autonotebook import tqdm\n",
    "from torch.optim.lr_scheduler import _LRScheduler\n",
    "import matplotlib.pyplot as plt\n",
    "from PIL import Image\n",
    "from torch.utils.data import Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def frame_extract(path):\n",
    "    vidObj = cv2.VideoCapture(path) \n",
    "    success = 1\n",
    "    while success:\n",
    "        success, image = vidObj.read()\n",
    "        if success:\n",
    "            yield image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "train_label = pd.read_csv('./DAiSEE/Labels/TrainLabels.csv')\n",
    "valid_label = pd.read_csv('./DAiSEE/Labels/ValidationLabels.csv')\n",
    "test_label = pd.read_csv('./DAiSEE/Labels/TestLabels.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "from torchvision import transforms\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.utils.data.dataset import Dataset\n",
    "import os\n",
    "from tqdm.autonotebook import tqdm\n",
    "import numpy as np\n",
    "from torch.utils.data.sampler import SubsetRandomSampler\n",
    "import cv2\n",
    "import sys\n",
    "im_size = 112"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class video_dataset(Dataset):\n",
    "    def __init__(self,frame_dir,id,label,sequence_length = 60,transform = None):\n",
    "        self.folder = os.listdir(frame_dir)\n",
    "        self.id =id\n",
    "        self.engagement = label\n",
    "        self.frame_dir = frame_dir\n",
    "        self.transform = transform\n",
    "        self.count = sequence_length\n",
    "    def __len__(self):\n",
    "        return len(self.id)\n",
    "    def __getitem__(self,idx):\n",
    "        video_path = self.id[idx]\n",
    "        frames = []\n",
    "        i = np.random.randint(0,int(300/self.count))\n",
    "        a = int(300/self.count)\n",
    "        hp = np.random.randint(0,2)\n",
    "        for idx,frame in enumerate(frame_extract(video_path)):\n",
    "            if(idx%a == i):\n",
    "                if(hp == 1):\n",
    "                    frame = cv2.flip(frame,1)\n",
    "                frames.append(self.transform(frame))\n",
    "        frames = frames[:self.count]\n",
    "        frames = torch.stack(frames)\n",
    "        frames = frames.reshape(-1,3,im_size,im_size)\n",
    "        label = self.engagement[idx]\n",
    "        return frames,label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "id_1 = train_label['ClipID']\n",
    "label = train_label['Engagement']\n",
    "id_train = []\n",
    "label_train = []\n",
    "for id in range(len(id_1)):\n",
    "    path1 = os.path.join('./DataSet/Train',id_1[id][:6])\n",
    "    id_2 = id_1[id][:-4]\n",
    "    video_path = os.path.join(path1,id_2)\n",
    "    if(os.path.exists(video_path)):\n",
    "        p = os.path.join(video_path,os.listdir(video_path)[0])\n",
    "        if(os.path.exists(p)):\n",
    "            cap = cv2.VideoCapture(p)\n",
    "            length = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))\n",
    "            if(length>=300):\n",
    "                id_train.append(os.path.join(video_path,os.listdir(video_path)[0]))\n",
    "                label_train.append(label[id])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "id_1 = valid_label['ClipID']\n",
    "label = valid_label['Engagement']\n",
    "id_val = []\n",
    "label_val = []\n",
    "for id in range(len(id_1)):\n",
    "    path1 = os.path.join('./DataSet/Validation',id_1[id][:6])\n",
    "    id_2 = id_1[id][:-4]\n",
    "    video_path = os.path.join(path1,id_2)\n",
    "    if(os.path.exists(video_path)):\n",
    "        try:\n",
    "            p = os.path.join(video_path,os.listdir(video_path)[0])\n",
    "            if(os.path.exists(p)):\n",
    "                cap = cv2.VideoCapture(p)\n",
    "                length = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))\n",
    "                if(length>=300):\n",
    "                    id_val.append(os.path.join(video_path,os.listdir(video_path)[0]))\n",
    "                    label_val.append(label[id])\n",
    "        except:\n",
    "            pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "id_1 = test_label['ClipID']\n",
    "label = test_label['Engagement']\n",
    "id_test = []\n",
    "label_test = []\n",
    "for id in range(len(id_1)):\n",
    "    path1 = os.path.join('./DataSet/Test',id_1[id][:6])\n",
    "    id_2 = id_1[id][:-4]\n",
    "    video_path = os.path.join(path1,id_2)\n",
    "    if(os.path.exists(video_path)):\n",
    "        try:\n",
    "            p = os.path.join(video_path,os.listdir(video_path)[0])\n",
    "            if(os.path.exists(p)):\n",
    "                cap = cv2.VideoCapture(p)\n",
    "                length = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))\n",
    "                if(length>=300):\n",
    "                    id_test.append(os.path.join(video_path,os.listdir(video_path)[0]))\n",
    "                    label_test.append(label[id])\n",
    "        except:\n",
    "            pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "id_val = id_val + id_test\n",
    "label_val = label_val + label_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "mean = [0.43216, 0.394666, 0.37645]\n",
    "std = [0.22803, 0.22145, 0.216989]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class video_dataset1(Dataset):\n",
    "    def __init__(self,frame_dir,id,label,sequence_length = 60,transform = None):\n",
    "        self.folder = os.listdir(frame_dir)\n",
    "        self.id =id\n",
    "        self.engagement = label\n",
    "        self.frame_dir = frame_dir\n",
    "        self.transform = transform\n",
    "        self.count = sequence_length\n",
    "    def __len__(self):\n",
    "        return len(self.id)-self.count\n",
    "    def __getitem__(self,idx):\n",
    "        video_path = self.id[idx]\n",
    "        frames = []\n",
    "        a = int(300/self.count)\n",
    "        for frame in frame_extract(video_path):\n",
    "            frames.append(self.transform(frame))\n",
    "        frames = torch.stack(frames)\n",
    "        frames = frames[::a,:,:,:]\n",
    "        frames = frames[:self.count]\n",
    "        frames = frames.reshape(self.count,3,im_size,im_size)\n",
    "        #frames = frames.permute(1,0,2,3)\n",
    "        label = self.engagement[idx]\n",
    "        return frames,label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_transforms = transforms.Compose([\n",
    "                                        transforms.ToPILImage(),\n",
    "                                        transforms.Resize((im_size,im_size)),\n",
    "                                        transforms.ToTensor(),\n",
    "                                        transforms.Normalize(mean,std)])\n",
    "train_data = video_dataset('./DataSet/Train',id_train,label_train,sequence_length = 150,transform = train_transforms)\n",
    "val_data = video_dataset1('./DataSet/Validation',id_val,label_val,sequence_length = 150,transform = train_transforms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([150, 3, 112, 112])\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQEAAAD7CAYAAABqkiE2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOy9aawl53km9nxVdfZzz116X9hkb6QkUqI2W7JkS7ItRx5ZsQDDnswCQ3E8NpBlMpkkiD2TH8mfADN/MmMkQABhPAMnMOIlDsYG4tgzkUeTSJZlUZYlSiRFUSS72XvfvuvZT1V9+fE+z3du1b232WKT8h3cegGy+pxTy1df1f3e7Xmf13nvUUkllRxeif6qB1BJJZX81Uq1CFRSySGXahGopJJDLtUiUEklh1yqRaCSSg65VItAJZUccnlLFgHn3E86577tnHvJOferb8U1KqmkkjdH3JuNE3DOxQBeBPATAK4B+AqAv+m9f+5NvVAllVTypkjyFpzzBwG85L1/GQCcc78F4NMA9l0ElpYW/emTJ+ZfuPIe7j6fADh+80Dr2Zuz6M3Xzgc4ny+OeL7wlo519/99v/Xal/6lq+V5DgCYjkYAgMGgD8/v8jwDAGT8rK3Pi+cM23BxV/rFFb51/JdHXhjzrmdGidz8TDGfYxy58BsAJJEZrJG+j4pj0NjCVvcQLu6KY4nKz2N+tjyMv3iXefkaxVPDORtjnNQAAK12CwBQr9fteM73dDrj8XnxfBzzdDa17WTC7/2O+y3K/Lnr/oovyHxs9o/xLF/13h8rn+etWATOAHhtx+drAD5Q3sk598sAfhkATp44jt/85/9TeIF2jB4A4H1U+rr4spjxAZsVX35BdEH+gbi9/5JKuwHlP0B+znLb5trq6Wm/nW+U7iePCqfM0pTnsGOdnDKX8fspz8U/WK9rcj9uc85DphPzRYv5eTweAABe+dY3AQDPfPlLmI3su35/CwAwGGwDADYHYwDAZGInn3EeU97nVC+t7imz7yOvZ2HPQH+wWTbmlveoZ8eh8omhHs1nutu0P6DlTgMA0E7soKW2fe627HXttO0Py+c2j2nG7cy20xG3Kd+BKCncS9Kw4yPeyzSzeZv4CGO+a3lsI5xwTkcT++Odzrh4cj4i3lit3rGxHz0NAHjHe54CAJx77CwAYNi3eb722g07z8QWZr0Lo4md97Xr1wEAV1/5rv0+HKPJ+474ouRhAdGk2u+z8F7ZfdZq9n0tse1zN7evYA95KxaBvZatXX953vvPAvgsALzrHW/zK612mFDPreMLpc9lifi7/op2XkR/tK+nqOd/tFwkwsfiH7deZq3oPiwC5Qvs1MvFRUzXyrKocKwWA+2n74OBU9qvvAjomlnKP1y+1NOJLSbrGxvhPBl/m/EPJg2WAM+ha2VFqyLhv7IwP5ovDkbzk3Ih4n4tPqKm/R2HP/RWw7aYzcLitdKznU4dXQawYzGox7zWhDduf5BSAq32AgCgTi08426TqZ13e2z3uta3P7ykbudNwspkg1wfzrDJP/ZtauQJ/+jL75HuP+YfWMIXZ7C5CgB4/pvPAgAWFpo2tpotPItt+zyr2TX1rNoNLvyzFTtfRotgOkGdf8wJr+XCc9eCa3M/ppWhd7nRaBSOe+7mN7GXvBWBwWsAHtnx+SyAG2/BdSqppJI3Qd4KS+ArAC47584DuA7gbwD4W/c7oBbHONNbQlzjis+VCzQxUfaJ9FmWQLTHb2GV1w/F1TNI2f/OSoPLS7/rM7UXaJJ6mqTBJEvzYDJKq5b966B9qT0zmbVZo7CfD5cumuR5cHns+8nUVOB4aqb4rS2zADbX7tnYJlNs94e2D62EGRW5NFqNZkaSFb+XETLjNaTBqKTRjB239kUzsQMWZOJTI7YbvE4kd6KOXttM6RPHlgAAp44d4Q3bfGRU7X4ms5hmfWLXajVNyzbqOrddc8qbu7tpFsArt2zMqVccw7Z1asxmq4n03iYAYJvzk2emXbNM8Qi7RosuhVyQGS2HOp/V2q1bAIDrr74KAHjySXMPjp2hfuTrmOoZyjXkO6H3yEVJsHhlAUScY7lgcicdz6X9NT+yBP7wT/dO1L3pi4D3PnXO/WcA/hjm+v1z7/233uzrVFJJJW+OvBWWALz3fwjgDx90/+mojytf/1MksSwA+j7BIihq73KcOt8RUNQqGHO1TLit1U07JPQbFSxR8CRhFBe6ZiZfVz4htTBX6Bn9r/HENOtoPOZn01qTyRTTqWmHNFgCJYsgBBnpl2faFiPR4b596Xj+a8axTWmdbAxtTFevWhxoe2PdxjgcYkqfd0Rfd0orRPGXer0Y4GtwvpxiAJyOKLX9etTsC03bf4Gf2w373K7Tb+f3Dc5vM6a2imMsLvQAAEtL2pqPX+M+g0Hf7ndic9ygxVin5o8VZQzxHRtzxjH52D7fWrd5WN204GjiFDgDtzXUEllbtBpkrWXa2vzlIeZkWz1jWQay8q6+ZAG+BrX4UnfRrkVrJo1oBcZmjbSo1tu8foIcITq1K3sEXqt4+yjt7/bNzZhUiMFKKjnk8pZYAt+rxOM+ei99cb7SBZ+3GPXXihhy3Ypky8fGPI2SMjYw0UkVX/BF/0o/ywqpxYpEF8+9K5JPfzVXEKG0X5znaFBDN1G41B5SzLWXExvSOsHn0/xwhzFTpNst86XXtm1s66sWqR5Sk/a3tjCir6ssgXzc4Hcq/63Jpj/ertvvS23Tqr2a3dUy03c9av4m36gYNgZpWfnt8qVrsSyDGrrdth3bst+owOdWWstiBnnNrt2kVdHkOWkwzDEPDJKnetaMzD968hjnT364besNpgp9jsWmzWWX2wEtpakyMpmyJ8XgkZIkw6HmlxmMO7ftfA0733TJ7qWej3le7rdoOJmkY3OxMDWrpTPZCteYZ4u4VVZIn5VVU5YtWJjlQFdRKkugkkoOuRwISyDyKZrjO/ByOL2tmhmUNy/uv8tXLgAGFcUtr2865v7rXhkJFu+zgxBxrqy2tVsO5L7ob+93LUnEc8aOGQdePYsYKda9EWcw473EzjTk2JkWGW4yTrG5Zp+HZgnMsmmIJHcSQ7RNqOnylL7u1K5VJ1BHvv6Znu2/1KGmTGzwXcUAlKjxdp5aQsuAsZh6vRiLUcS6UauhTp+9LmuNWlXWSKNWjNd0O2aFtAQ+4OTnitRHzHykzHTQUjp/0nLwxxZNG6f8fZuxhrX+CMOJneMYLZwxwVOalxHfixE/631TTr7J+ZpMi6jHCMwyzCxjkzqCjxyP84VbRKdvz661wxIoS3h9Svia8HvAfBQxL2WpLIFKKjnkciAsgRzANIoxzqXSBbO1FTwqacy8lNv3ZTWMHT7tDlQ7MId5ouRXl3HoZXjxHDueF/Z3pWvPMQD5PH5QHpu2pZVb2PlOjXEF3oSuGGlsjEMobhHRYvD0/dbumRbZJFJQKL7lxUXUGO0fj00TJSPm/en0TqhF29TkRxfMAji90gUAtBpEo9E66TJS32FWIeYrJTSfctnK1mjbYDamUWuh3bJrtLgVNiFkeEo1A+2WWQJNxgj0EFNaIVGuTAZz/XwCTeXNqeWp9EEIAIazLJyzUbP5qNdtp2isWBAzMjM9W2IWeD8RLag2La0GkZETQpi3Bnx2tHpntPrGMCzDkvz6tPie7SXlKo5du+5Xo1KSyhKopJJDLgfCEtia5PhXLw1xdc3QWl4FN3RgnXxFRdF3VVUp+pmH5TEKVTnS7EWc+LzajXuF6jqtwNnOS+6q7JsX9+SF43fi/FP5YjNp7KIEi4bbywu2x48+dRIA0K4VUYoqGFHmWFZMxrFOvfDmphFVHxBmIvfwHJO2M/rDaWoWAWtVsNCyOV9ZqPOz/VCP7VrCDwjX3wj4At4r5zlywmvQ76d/32xZ/KLdaKNN7H+7VcS6x8wgaPx67I2avldGhhZAGSPC89QDko4nmNl2poKrgMBz4T3JS9WVmuuI91Gj76+5Dlkhzoveg5Tne/XmXdt/POF5iQ7lCxbX7gAAJktmQbzzPOMdu4JSb75UlkAllRxyORCWwHCW4Wu3NvGN67YaSmvU6G/GpZJIbSMmiGs1Rmhr0Xz1p4+WlPxLHburbLmEA1DJqOq7Z7NZ4fdZqBXgZ1XlMcqezjLMQmWdtEvxGnlACBIdNrXff4jXbkXyO4tViJ7xilA7IAVXqgwM2oif0yzDjJqozxqC0cB80RbnsEPN32vZM1hoKp/POQ7zWJy+kLmg3y2NXw++MTU/c/6tpm2bjSYaDcYEWGHXZJVfqCIVjl5WmuI1M5Yrq35jVz298ubiIwC32q9YEeqiKLjRqucXUlDPcnts86XMjJCBs+GY98dYQtPuRRiFrXsWn+kPeV4ZeRxJvWYZnGmD+A3/vZsAZfu4XCG7n1SWQCWVHHI5EJaA9x5pPsUs+FVcuWUB1JRjtu3c56N2Ej69WQv52po0kvLUPIesjJC7L6GvpBVSIr6mUxuDkHaq7pqldp6UWnvMvPF0IkRiBEftMY2LKDOv2gBW3lGRBf8xWAxSF2WMQyQLgKhIxlC2OZYBxzrg543BgPcyw/b2oDCWupMVZZqrw6q/jiwA+frUpvWa8v+MBdBaaxPH36Y273To5zdZB1BXXX2Lx9NCaDbRaFBr8tlBdRzS5KzBmEMBtWXMg7GQPABGosLWoRhT0vwpbKS4zmyWY8pnNmD2ZExOghqtE4zNgpowFuBojabptHBOx3c0C3EccWMIA8N3IaD9bNtQhWSIe90f9w8Azou3YlcarbjdRw7EIuAcUHM5pgTBOKZTchZXTDkRCf8oYv4B833EZGy/jyczNOpmbsmVSLigiHCinhBqGougIYzCNl7kGTKhmTqj2T/ldhLMfxaA8Ps0HB9DfBRpqoWFwJxdVpqOEaUUXxC+vBHTSHl4wwSntmuu07R/4YalBq/fNdPy2qp93hrShM2yUBNcF2iI89QQFJd/zFo041IZq+C+CuL1umbWd2j+LnWtCGhxwcqCYy4uWQjqKnjHBb7egNMfWFwvbuVzCEHDxS1gdDkPiUBlSpUqcOqKi+gcAq5v5Z7xPRtPMBrZNYZMAQ7J+JNGhPdyKH6qUvJiKjQoERZquVQAOBUW0dzXyBRw5C3VI737nIoHWAR20avts91PKnegkkoOuRwMSwBG9SRzbqL0FQEsNWrQwGNHDTATeQLN6lk2w3iqYCH3pbnQiAlnjbWiq3CGGsoXza856EdEFLQERN1FNT+3BHzhc5rNU4QKLu3PS7g3MGS/9TukIbnHgOby6oZBTNc3zRIYjJX+y8L+cyuzCMBpCOZLN0opQBVWyQVrEqrb6dACYMFLVyk/An7qNQb3dHzpnkKQL5sCE36bCoqr+SAYiFoU5F4Mv7OACEoR8n2phyoogYhU/UOLqkTOosCh8xlm5P4bMIC3OVYqT4VEBPvQFdV96JJ1FWTJukuVQpZ1Vxha2Eob6zncX3e/uVJZApVUcsjlgFgCDlEU7yDilDblSs7gS0x/qcaVW2W/on5KMyCiho5VCENLYEqQSxIzmBR8U1oTAdxSZDZWbEAEnGkplTgn7uRWWjf3OywAO9cuurAS7XdZ5iQie29VoCTrRdp6yDRWyrTmzuDS/NpFf1hxUgVU5fMv0udf7JjWVfGOyniVco3LxC+8pyhSgVKpnJUWSDrN4N2U42fQVgHAqWIgGrTOwc+qIVYhkaw6EW7qUhAMm/MnyjD+HvMdabfqWOqS3ENkH2IDDtYaLSLGK6Z58X1KakpBm2QzvQ+aD/5QIgIJVGc6vlw3fB+ZQ933LiAqf1+WyhKopJJDLgfCEgCseCaQJAhk40W+KFinIqe0ALi6yt+Mox2AEMUJUpEuMtUTClpKENNSvCEUEEVK8ZgE8tBd/Qd8Yb+dun2vAqe9xO+zJpePnmcXpCFNG21tWUxAoKE8kFZSIyQ7LIFwjmJfAKVn5ft3uy1uaQE0ipDeecuDomWUKWXmi4CcALtW1U7q4Dn3OYuXaiz8caQwQ6zB8XOonBFSibEDpev47CaMJbBmB2Nus1jFPnwnaB0udju4dJZjIVhn9l0DsL1w1zIwM2WqAuycFiStmGDF8hko3qBnpZjMLtuPD0D0dyU2+YeSyhKopJJK7isHwhJwzgpPpLlStbAScUe5gEZU1FqFQyciP9dogrcy4RqnWvWZey8Vm9SDRVA8pywGnTgrNevQmPxu0Oac7CSaxwmAeQFQpnw2j0hFIiINnhcdR5Wf+uAqk9abwJ1LFy8BALbazA58y0ieV7dGukCQKGhw28ZMVDdIE6YuP62OIL0CYdES0EQzT56p+w/JMxJqep8UC7nm4XERYcTwczwv759kILlwAiIbCQcDACbM5c80X7RGNrbtfu/es6IdNV+Z8b1ZoHWzQlqzOCFYKU7Ratm5Lpw5akMihiFNrwEAXlwdcIwC9RTpzWepgEuMX5FOLQ+NZYolyBIZO3X9Rb4BCyAvwcUftM9oZQlUUskhl4NhCcCj5vMAZU2Rhu+BuesnQhC560KhzduR5fMGDYraZ0UyiyRmHCFYAiR1CBkIWhCKHURC7ZU0fakAJNtjFQ7R/HKUv6QU598XMQrBbwy55Lywn/zLRTa/PL3wGADgJkwDfv156wG7s3tZiEbTKojtULSIq+iRRKTTMy1ZYxbAU+OpWUkgs5SvK2wHc/njckV2qOQuIhDzfDbPLMjnV0YGii8IHm0/r7Nv4s0N6++3tm3X3B5Ybn9720rSJ1PT2mqFFtEq2eDxoghrEio9i2LkaqLCUupLx41MpfaU9RhMv/YyAOA7G2ZtTEVsW+q5KMj7BOpzyHspoYH1DGM+2w4tMpeFh/768j1kEvaSyhKopJJDLgfCEgAAhzwsZCrTDdRdJZRb6Gyrn4Oa2WEJlHx+fU4DzVVpvxALoAVRal4yL0tlrEA4/l1NReercWjPvUPn77gN7NixcGw5l7+LMrXUhERxC+Xul5ZMi6uRiE5Q4JtUwQr36RHz327bdkRVfvOuaVu59jU1KVEtAUfR5hjadZUgk8KrLvwFh6L53hH91hyHBjDM44+HNobVLdPoL98wC+elaxaxv8kmIv1UsRR7nRc7pr27xDbUS4jDZmKfCQrE8nFrgNpodJEkVvCkOo48tvjK6eN2rndfMgvgzrMWI7g1LJKONEJDFNGKEbMQLKhw2wVRc9RWHEwmO8zle+xdkn1+fr2sgKSyBCqp5JDLgbAEHKixhcoj3tpp+YyKWji0Dy8WkwHOz9ublzS0NFBA+tHPjEqWgq5RJ6V2HmeF37X1JUugLHmeh9x5OVobqglLRKT6PCblV92JwEO56L0RYRqTLIGVZdNmJ06YhrtGbe4xB9lJs/cWTGsuLRkd98a65cNf3b5TGJMIXIJRpsYqzBYssYR4hZH3I2wrvswsg+o7RD/WUCORWhx+8+TzHrAG4M664R5euGJNrb/zmjVT2aSVssXMT07CkuVjRssWr1hk39GyEFLw+BG7RwR6ddP2R05fBgDU2t1QBToe2TP4wr/6Y7uvtj2La5tsIc6MRUziVZGAqNZCMQIhSUO1YOn9lNSEGGTtgazHB9PlDyeVJVBJJYdcDoQlADg4F+3GyJdQZuXGinJxyX4NH4XWpHPNLh80EIlKe9ICCMZG0WII0f9clYo6bu+MRVnyfAenQ4m0smwZBEozRqtVBZg4ovXYoCIQZZbakYVmoShiHo4t9jh2O8rlc+tJJfwt8gDcXbVc+mDbtK+qJYcc04CasT9SVaUd3yEN2SItgQW+Ucfa9o/zx41X4OgiW42Fpibcv9MMcz4csxqSuIaXSTf38h3jRRix0WiLmv7iEWu7trhiGn6px9bmK/b90SU2/6TZ88ilC3ZvxIy89E3DUfQ37DrrV1/BxsishCFd+et3OS+smbhy18bWn6gWhVWXfDa1pIgJSWdClKLwfbkxmGICgTxV1YkOmMeSHswueND9JJUlUEklh1zesCXgnHsEwP8K4CRMKX/We/9rzrkVAL8N4DEArwL469779fufzDOyL21dRAyGyGqJclzrnSwCl/uwrAVfVvloL61JnIAyC1770WIoJfHnXI30aXmgF0XVPuSWPp/7+tKaSv26YJUUa8wnai3OPLYovNSey8Vi++GYQzCh6HfGHOsJtfrukiJtPd3hm9p2RA1/lRaA5itmlDvEPiI1bCVCkK/OKBZNuGl6wjIwZh3+6I5ZNUeGNsYOAQriLej1FhG3rVLx5qrl969ftxhAm/iHM+cNCblw3Jp21jp2X6roTEmeOiE+YI1DrrPBaW1iY/7W103zyzLor7NN24ZtJ9MRvLdjrlwx62Bj9Z6NJbb4CuEDwdoSA9CFozYP73vU4hJdWj5/9tx1AMCXX7J4xkBAy/AcmG3hM01o1uZqPpK7EI/xrhi32iUhTLU3P8V+8jCWQArgv/Levx3ABwH8p865dwD4VQCf895fBvA5fq6kkkoOqLxhS8B7fxPATf572zn3PIAzAD4N4GPc7TcAfB7Ar9z/ZIDPUsAJMSg2ljJ2nn6TL66EAUkIBCx7yBiUkVclVp4IxarBUINdQvXJ7tAKrih7vM+q7L0PHARlbjvnixZPqFTkfpOAdiTajLiJJFe9pJiCdItku+EX8i+Xifs/d8J85ntbd0CXN8QGRvTDI+bSW4yo1zgvoaEoK+8abEgasWFItGBaOWI2IA2xFkbZh2ZhfJusR2u37vFe7F7Pno4wiSx7sUkuxIsnzed/8vIpzo+de0QtvS1SV4gdyrZb2+a/32D+/5YzP75GhqVltW+bmHWytmba2ZEzYLHXmls6E8MgnDttc3fhrMUbTq+outA0/FWiFR8/bb+/9xGLQ0z5TDYeM8vgq6+tcV5oxQXMhrS8jWFb9SC8t8R7JMoC6TnzyHILvHmHPX1//5bkxRE8pDjnHgPwHgBfBnCCC4QWiuP7HPPLzrlnnHPPDKbpXrtUUkkl3wd56OyAc64L4PcA/Bfe+60HjUx67z8L4LMAcHax6fMs242k230MgDnLT/heFoJzCLpbJK8CYKn+W5V8ocmlov7aUatu8ZqSOODbbVOuC9+Jqc8DK1FWGLeTDx+yBTyX+BLEa1iKHYSIcakiT0w59cx84uPk/cuOmrZ+6sIjAICrN1axzhGLLyChpj9KlJ3y9xkj6M2a8vy0mOjLN0+Yhqsvm6YUxfaQ63mdTUa6zAa89Px3AADX2WqulZp2PrPcwNa2aeYNKoPWEbMEOos2/nSs7AorG8k92GnYNVZY53Cbz/j2mBYV76HDysgetfy9ezaGzXWzShYXzGI4unQM2yMbw4kFu58ffIdlFI507djtTZu39U07douWQI3WRI3zdH3N8BZfef4qgHlFKJkPwrsu7oMx1fF3+6xJ4HO5ePYMmoxtxGKSYsu4jNySquAUxX2gZBcf4lvZmtw5V4MtAL/pvf8/+fVt59wp/n4KwJ2HuUYllVTy1srDZAccgF8H8Lz3/n/c8dMfAPgMgH/E7e8/wLkQxfEcjVfC0O+C2pctAW6zPN+FogvLXKhmK1K75HO0AX+X9t7nWvqc6KgimnEnBiBwD3jl1jm2oNGLuPMdxfY8RzF2ECyB0I5M90a23ZGtt5E3rX75EdOo46FppUaa49zJI7wStWVuxwptmKVqZkpWZmVPWBffZ5xii9qoQX/73DF6fVRCN1jRdxe2HbHNWYvh9R84amP7qR+4gAbs3L/zxW8DAIZDRv0d+Q2phVswLZxt2jXH23ZffY6hw3jECZYgdMgdeKTBmMumxQAWWEl6pkcLgFiGI70WVtduAwCevHgWAPDocRtD4u0atYlZHY+sWLbgxasW66h7eyEWuhYT+M63yEh0m+3FhFfhM9drKRCoHOJVavU6EYhnlo+hd8TOWU9U4ep5Ls/PZNQWqpUci6m4Jtl+Hn/wJ9hLHsYd+DCAnwfwrHPuL/ndP4T98f+Oc+4XAVwF8HMPcY1KKqnkLZaHyQ58AftDm3/8ezwb4HzwdedRz73znA8Sdyij8vY9pmQRiCFI7D3CDQQUY9C+tklcscpwpxYPGhzFmICutZtEZp9Mw65vXGFMEbMqdW+aMSP2PmdnpnpqGmGp5rB81Hz4F159DQDQoJOahdJMak31I2CLsEuPWKT+Hi2ALVYHLi2YZlyKTP0OxO83tOj6gBZFxIR6t2Hz9cF3PQEAePpdT6JTF7OPxQB+84tftWvQP+6QATif2Wc/IN8fYX0z9h1Q09Q2G5zWBmQCToUNsf0W6GN3G2YVnThh20nmoB4HF5gVaNOCCTwKiZ2j1WZcgtmS00fMmmgSg8BGRphG6r/AeeCLpZiTWrfXhUhV1Sq5BlGrwXN+Zol+IyOx6l7IjNRsq/lrmadQ1u7elkCFGKykkkMuB6N2wDm4mttRo26ShRbcYUfblHAC8xbMO7xqV9SWgZpAnIPS4LoWvbK8FgrfC5eSZaDlVTgBHzjjAudxOCCigyzNH4W+mMxv58X6hdTNCucoGRfIXZFPX3j1TNpD2kSOOVl7j7B6rt1dwNEV8y/fQX/59u27PLdqLWilCDHIa2z3LZcvbXuEY+r0xeJj87dBpOD2bcvZT1nrX8/NNz6zbPGKSxctY9FpLwbr4+mLFie4csM08ys3zT9PHz1v1xiZdVGb2VgWGG+IaV3U1f2oaZ9b/Ly4YH694kQT9jTUO6JZvXXjBp5+8nEAwLGeaXBPSyZ0mKJV12qZ9j25bBbACWZiPM92esXwE93kps2D+DFLjMmJFwu2fa6R6kk9M+tRFPoRBnxM6aV0hGn6zJ5Zrr+gJBS44H5SWQKVVHLI5UBYAs451FwNTfLHn1g5BgDwqul3ihUoe1BkAC6sc1r8hLfW1wzBxwF9JYuAq6euFYk/v4ha1LXlw6k6cR7Zp4UQrpdCa2zMbVuJhTBuruSsOW9ixO/td9W2T3O2Zk/VMVgBC15MPiL9cqEmRWh3nLwCn/prH8E3X7kFAHj7xYsAgAkzB+rdKMx6nTiCNqv9vHo7krE3mZCjgDx+Kf3Scd+07PS2Rc3jjml3P7HP507as62rV4B3oQehY9T+g+98DACw/aWvAwDu3rBc+8mLTwIAOqe+0CMAACAASURBVKcsdy+W4hnz4XW1VSfmQfM4VjtxWhJqI95J7HpHGde4+O4n0CR/hJ/ZMYNU7wMtAr4XslpPsEagzvdtyL6Kp48Rw2DTgi0CASZ69nyEYmRq8f1LiH5sJMJnxAEJKjYs1b2IPHLGjskzWiEN1piEblqvw0pQWQKVVHLI5UBYArHPsZhPcIbcb/2AjS9q2+B/5/L39T3393nIt4qLXh2OU2qLsfD4qaK+tBgS+VNkEkqLEfwmtY569SkOMWG9eIs7NrWiwwfkdkotonMsEQ3WYPwhoaXT4Mq+yqh2syZfkL3uxeXPGv4OGW0R/HkFKoQ/oL9JjffOt53EN154FQAQ04x4lFH/CXH3mtM2uwx3yTykBsAxI/S1XB2bbAzjGcfAeT7LWv8ZARcZfetzR80qkSaFcyFGIl93ibX7H3q3WStff9HGvPpdy0RnrFickgFZyEo+QrQ42CbjGkv8fIpdlI4ete0ykZXNWDUIPvAwTvheDEsYDb2LYmZaZM9GZVG+/qJZLbUFixWwgBMT5fYVc+Fge8xUNNV9ScxL9OeTKJp3w5pzUHFbRLl6V+SrKIWU9pXKEqikkkMuB8ISSJzH0XiC9x8z7bLGarLRTHyAjMym6lZM7U5Vy7Q4ppnHkL+NxSDE7UArPI8h5Dt8nymCylW0Tq0k7a1mOB21iOF5x4w00w1DnVpsKYqQKHrPYwbkDhxtEX3WMb/xzlCxAOIJiKA7yUh6m5HoOrsAqdtwPmVUnBV8gZ3GiRdRWxtrtx7j6KJFn7/yla8AAD78wz9o5zhuvvrWqlW7qTqyxnx1oDekFQLGRvpsBrA5tXsCuxjXhvY53zQL4wmiF08sspORHNtaM2RiooScBlObD/nqTz1qx273LcNQY31DzNoBpV0CtkOM0olYf2h5xeoozArJ8KxVyzFHima0NlMxAWmr/gLQ/Niz7dNyfHXTMhYdViGmis/QRo35Wb0Ma4HtWhkkMhbJCozmWSJZBKG3QVDhqjkpVac+oByIRQDeG9EGgShTAk2m0+Ife2iiUf7j53Y8AwbM9/CVRNak2bVsqbEFpo1GNIJEjiFLazywa9dFlR0ISJW246LAQfUYMGuEltQ2mGGWIiIkt8YVosmXMOVTHAow0rNjVuo06whIWWXBTDIjrdZIoBm+9OQIa0wYdAp06juCbthJYFHDiVMMuj73IgDgpRcNqttpF4Npuj9HABICZHnCczFAyD/mLmHFy7STtyN7AumCne+xE6QBbxKEJDp07+YdOZgeS3hs4m0xWGYxkiOk1vOPgWsfYrlFsYK3ZQOXYw3EHSrksl9nalSa5TvcRTtmNmORTlYsy5XJPeWC/OVnXwYAvLxqC9nbL1pa1tcYWGWxzzzTLEIZBvuUmlbnNVGXJ8m8JV4oeitC1ctjKkPdq4aklVRSyX3lQFgCkwx4ad3jtTVbLde2SLwpEA33K4dHBPKgIkTfAVtqQ82dmjStwZRLjUGgelxaTaku6xGJKKhdIpJpBNuLWkaFQ3Wag3UG+5JEacwMnqZj4ooreaJ24DKDA5CJ7g5TZS/xklf6MiFlxpoW7bDsNb5jgJ+L561V1rmzRwvzlBC6mroYzOCFIFI7Mc3eoh3cY0CwTcZQBcCiWfFemizPbTYEfrHvr982d2K9b27AAklH5F6oJbkXD9lkMn8LqanVTCViyrNRNyBORzRpE7pPAl2JGs6JPr4YvJWLJ9Nb95AzYDt3MzNMSeCipiETQq5n3FdkIWPOl0qHV1iMtDwgCKtv50nrZt2IEORIjy3jTphFJhh1QsugVbexrSyzVLvRCUHERDTn+1ingX5P0ykDKypaBmWpLIFKKjnkciAsgWZ3AZc//COovWYQy+UNCwCJXkspoACf5XEqzR3R9727OUBOkEqNGmfMlf0u21X7dTu33EatnqIo02qb0JdTwUhovhFallPrcCyiig6NQqJ43tac4oLS5Dm0squNOtN2EeGfSlNpbGpIWqMl0aA2ajMF9sNvfycA4PQZS89tbRq/q6MW2px4fJeFQyP14KL10WB6tsdCl54oyembyvqo1xrcUttybPe2LZYigpAZfeqFXpfzUYxPBOq4fIowi67YvTTmtRwDe72Eg4LayAugxWej2IcvWXlhw3gOg3NqIz6dkag0S4MlMKIPPyP5iYLRU1lrPFeXRTtPnDbNPanbcV9+9QbPbcc9ftngyB/5wNMAgJNHLYXYqBVLjGsMDjcEzqrVQsrXB5KQEm1YKLkvBrdlCdTi++v6yhKopJJDLgfCEugtLeETP/Mz2ByQWmkqNsygNgr7B4JNbrUqv3bzDp75hkW9p/pRpbGhZpjpFKUZ6Q8qyqv0W6q0ZEhHZoWtUkV5iTgkAJq8n1OaafyiEQsrexH0oUKRQIkWory8B5ovs0BBbb8eIUHH0tnHAAAnLj4KAFigVp7QP+/OxvjwB98NALjyyqt23yz8SWmViNR0RjBVlyWzTfq2rVivTF6YtyGJPcak/15eMk137vQp3hvLegORivjfsrlZRtkvDRcsJpJlCCQ2bz7rCp8l5aj5lOkAvWdT+v/TLMWIrcnGzArIMpjyfVDjkgEtheVFyzop61Enacr7qPFVNPYIQVlvu/wYAKDDGJJIZ0PLPV5PllMtjoNG94RsZ8pYTPhZbdXCfBQtzFhlyftIZQlUUskhlwNhCSBJ4JaPYWkxMHw8oOSF7aOPnsOHP/BBAHNfVTnlUFIs31RHltqThciqqMsDNbmupPOZBOLSsN1BHloKygYLRg1JSxoqyovR7EAvJoLVUjlzQo3XJeioxbJWsCVY+7hpgvaYltX2HXz6UxaV/otnnrWv1qygiOhnbJOKyrONd4tQ20YgsVS+2+5BvvOwbzGBjFr2BFuDifJqSEDUJrETmyzyWWw10c3NykhZzLS1tcUJY7ZANOalprB6WlGpVDbEXNQgJLR5o8aXtp/K36f1k6chDpWFhqKyeGw7oqWjc3Q4P5m0b9vG9jOfNl6dE0eWdw51jhMogXn1DuzCAuR7kO3IitIzCZT2RWvUqWVeXoohlKSyBCqp5JDLwbAEohhodTDn+ioVSLzuWqX94tCUM6y0815d/PyAZRVF/tEHuPYeElhElGx/vWvuN9ZibGCOF9V5S4Qm+qx5bHG/pIsefftP/MTHAAC/87//NgBgRP+SlaxoswnJiFZETJ+4Lu2MonYdkvJLI0knpinv3DJikI1Nww+0ieBsE9VYdw7NdTb0oJYdMJZxjxBmNf1cIeqz2zUsw8oKI+wNZlOmsmL2zouLun0qf59WTEpfe5pnwSpQ7Efxg2AR8H4zBWSYRZrUTOMfO2mYhrMnLVsghOqcIaZY5CPxocRYWaid1PZ6GQOmuXRnxXOH/UKvmvub1pUlUEklh1wOiCWQAO2jO74oab7dB3BT1upu/u/yob6sHUo75K93zf2kGGu47zV2fV+6ZmhLhuJWkpZW9H0tC7f373ETqJmGeurp9wMAfvd3/yUAa8YJAAmvMaQFpdoAUNuqoKoRUHcqfbXfj7IJqshKbt1k2+9NoyETEnPChiKT8Rg9+tUtEoQqxrHE291gu7Bbt82qOJvYPTQbx3lbxA9wTMr7Z7kyHdra92k+LWyVAUi9x0yIQWY9xtT8QgjO+CfTWDYsxuM/8gkAQHvZ3t/4qrUnq9PiyrwQpDZEYRTU4DY01AnZKtUqpGE7RzRynLyfEEdQe3Rab8K6qNBqL6ranVJZApVUcsjlYFgCcADqr78klVyeXf0WC9VS+2hbX9b48rP48UGXxVCaVypwkDzQeXiO0IZaJynHBFDaLyvuF8ZU9v3y3fuRoCSdsjpQFOHUeFQiGJAz2wWSFdMqohkT2lEIwhMsRW50Lep/b80i/Kub5t+nrF/osyPq8BVD1I1mU1y6cAYAsMhJ22K7dE+N3iDleGjRrvqMqGjxRMQwOOXTVRvAjETKysdMWIWQOeJnzFvHqYpwRPyDiEJHqV37HmsKnnnxFZuP2MhEXiEi8/ZNy7pIshBjoFUSMhWsUWB8QlvhL6azWcCqzIgHkAWQsC18LHQr60H0Z9Bje7beQgv3k8oSqKSSQy4HwhLIpxMMr730ut54HhyrUt30TpenREUWcvGlY7QNef3QBl113rsRgMCcXCJcW0Xp4bpz7eJKx84xBEXUolZ4ocUyRbED2QX3pxaRm56n0njKaYsMk8fLh5Tl4POQM97asLoCkVcMtlhzz1eiptbiNE5q02IVYU2INtJgtdgGvdOz6PiQfmx70fz79YlQenZCkbHc6Y/RWjer5Dpr8R2p0hfIUdCui4qLFZ6MuKv2v6P7L+X4A5bDFbeuVAciHgKf5RhzTqe0Irz4IwToY5bg7h3LXPzff/QnvC/iJpjheOarz6IwCP4j5TuRlprMhmcXmu6yvsFHYZzCR/RI+XbpwmUAwOkz5wEAt+9YzOTPnvkyAGBtfZVXrrIDlVRSyX3kQFgCq6t38C9+/X+ef3H/YGZA5+2l3eeauKht5xj/opaYh21t41Dyy1G+Vul4VxyDCxRfgMt0bKn+QPhzshgliQgz7ZRTjnmbGPkGNWJC7XCE2lVWi7TI9vZ24Za0wiunH0c70HRsOV6nJh9Tk8WsFowDlRXprpz4E5gd4LlnolWnvz6ktbLeN+1+b2ubn03Lj0lH1mKT0Mx79IkmzBgDaMayNkScapuEPAvbxDTcWLWMQ7s5KYxZVF0K1GR6htSo2gZmdj+vgxiOyB+Q6r5tnoTWmzB6f4uW1JUt8/3TtIhS3I/NJ1gl+iJAP8QRoArI+X4FFCqAe+u89muWiYjwpzZ+0tcpBpJ7ZREqxGAllVRyHzkQlsBkNMR3nvvGvr8H3y5QKRex45Kdnk9Z02fBh7dNFFZqruABl170v0KFmlhplIMNXAClmv9Q++8wxzMU+QenbIZBlx1LrLnvMyKfcgVviJ+OFsWpU1aJtkyy0IiasU/cfp/NNXTParRSy0xTuiwLEXQRiC4vW1T/2jXzcUcMAsQhXaKGmeQxFL+j+Bp4D2o6MqF/f4fZAfnYauC5tEj0H0lEFxoNxDnHx+2JY5ZzP7Zk9QfiIBQf3z1yQ6wNmJNvmF3SYiPODq2MHnngxX40x2OyIjAvZQJms0A0mpfQqqKNH7DSdTYTB6PeH86Wm18F2Nmchr+HZjaytHgVL4RgsT7A7cS3BAhM0VqIImZRasX4VKCi5zP8Dtu2laWyBCqp5JDLQ1sCzojOngFw3Xv/KefceQC/BWAFwF8A+Hnv/fT+54hQr7XnEX3RO5fq8F3JnyqDAD38rkho+BQalILn1velwWhlF58fUVgtNpmQJdCg9mmwLnyeu2XFWxwjpX8sLjzFFfqMID/y6LnCIKbOtOjmmtX/t9umLVWBt8jGomL1EW13Tlx/i36+T1W7TzUd/NosYNLFjbjCevgW21/3J5bXn86KVW0T+sgjbmtsNDohQ1GfbclUd69pPbJk2YJaqEkg7TfHsZDU0W6IS9DmcoG1AU029KiRVTlXReKEMRD6732iHddHNvYGtfEZsvccr7EhKd8BMVVxqKFBSp778E6lsj6oTVVbcW/T7lNNTcW4NK9XYK2Fmunq2fNXMVBF5RdvF8YjUF/tOLp0TMlqUKnALiTM68TY3gxL4O8BeH7H538M4J947y8DWAfwi2/CNSqppJK3SB7KEnDOnQXwUwD+BwD/pbPQ5o8B+Fvc5TcA/PcA/pf7ncfDNEheyqcLKbWr4moffvXc54E/oLzvrmitzhHQZmKiVWNOrq4hRmCHjcbFrIFiA3npOh7AiJV0CRlkR/T5J8wR11j3v75m0d5N4u0T+rabzB6cPWMswnVlCTiWmsZAbZUE7THPC+y8R7hoB4c9W3Sxbfe5R08AAJ79ho1lTGScGJNUXRhiIiRhTlmTvzVgQxWyOjdrqjVgoxCx31Brh0xIXEODeW+1l9scWdT/XmZxitBMg3iBBudNrFF6xmKFUjZgMDYLYpKpzbcdlqkOgOcdhUrBNGhNZTs2iV5c3bJnc3dD7MNineY7GxXfi/1qVb/XypT7H6X71+fyu7/392V5WEvgnwL4bzC3uo8A2PDeK892DcCZvQ50zv2yc+4Z59wzo2kZc1tJJZV8v+QNWwLOuU8BuOO9/6pz7mP6eo9d91yGvPefBfBZADi+2PZpliGTBRCQcEVUVdCy+1gC3vt5HraE4Ns3b1tqXeXFxqJycbEHqy00LYbATqM2XYoF7BhjQm0ofMDGlmm4RaLqtrbkR9v9Dkf2eZkMQZEzTXb6hMUC1HCz1CphjngLZHSMHqudenAS/Zx3jts6rY4nLtla/fzzxtE4mxURkpF6HMTM5ZNlV/EbYR8ixyg6c9arfdOcTY6xS23eCrl/hxoRgjOiFoWLCJh51QLw2ptkjFYr7pjnrJN1OSL2YZvZkuGElgNjKaOJeBGJ0+fYJ7MUMftM1GjqTGiNbYzYbyAPzed5v4xjKcgU+lngTZf93n9fyjSUpfx3UpaHcQc+DOCnnXOfBNAE0INZBkvOuYTWwFkANx7iGpVUUslbLG94EfDe/wMA/wAAaAn81977v+2c+10APwvLEHwGwO8/wLmQpukuHPWM0fUy5j5g78sronc7LIHi6rdvP7YQE2DUnxorF2pNKzsthjq1TqY8OavM1CUmUgejJHjogXdPlWPtpu0jNOOAef7ljlkIo037fOniBduffPvS3oF+gLnr4I2GW/SF7c47jwKDTdGyUa+CHtGIN27cs/slL75asI/GjG8ww6ExiZ2nqdoD8QaoY5Oze+42zf9folZerjfRZCWiIuxLPeMkOM7KROX1N8ZmAYxYwTcR/kExF8YbtmlRDYnH2BqTB5CxgQ32SOgzZhOsxSgOWSNF/ze27fne3rRzjUPCaj92n++fhIyErFd+vx/b8n7yVuAEfgUWJHwJFiP49bfgGpVUUsmbJG8KYtB7/3kAn+e/Xwbwg9/b8Tmm4xFmqqVWpZliA3nR9/F7MfsCANwObViUeb+28IVtSx2FUlXHsfY8ZaNDaTZ1+5lyq05FYnGZEu+fJEnwk+/ctbz/gNH/SCwyrEjbJuuOFL0QhDUi6AYb1msw4MuFgef9j4kj8PR1c+Lbg7YScDHyOyj+iSVgNqXB8XdYe+5LiQY9g5nqGth6vMnj8pDJYeyAWYQekYE1kFOQFoA4DHudFrpt0/x1YhUWyCy0QG6CMRmBms6u6TI7V01xH7bzrpNhqE3G3ynHop4IKesjhAvIAnqP1p+PsE1E4Brbx792y5CPGwOyDAcDYJ7550zx/0Wt7Ha8m/b/N9FieJNUeIUYrKSSQy4HonYgyzJsbq1jSmbbOSccK7dKlYCS/L68gG7PT7Im5l1rSv0A6SOTUg8syEJODeZUo8/8eD0y/3PGLMGUUeU4rqFPrXJznVmBZdNww5Fpl+tXzO/uNNQ/wI5dpt883rLejMpMBCIdfeZdZSQJAll001mf96iMB++1Hu9AkSmaz/mgObVAvr9uz8aqmgB1cFI2V/XzoipI1BlYxMaKqbAjj6yiramNLZsxZjAbYYFdhjvkC1gfUdOvs+MSMe8zxxgAMzb1JrszM27RVqNjXnswYY3AWNkFWVI8XtYfLYPBJMN638Z5j+2tU+IcfCKGJfWMYIVeeJ9onYm6QX0D3O64zPciOy3b0DtDsI9g1hX/Lh44HkapLIFKKjnkciAsAWUHpmKHVedX+Zmpov+l40Kktpjb33NnynxR5MouHvhIPi21p4roQith+pGM8EdefQF9cawT5bYnuHPXLABp7uG23d+du8YAs9Q1LXN6xTT/CjnhoqS0souFRr69L/al3807Xzw+4c3UfISauhexmjChpmszt375gqET1zbNAvjOhlXquVz4dGpX5tMzdRJOGKfJNALGUjh9dV53xrFs0Hy5szHD9nW7huccsugPCx3z8Y/Rgjp+xLInvXpXM2PXElbBK25BC5LPYpP+vHdmcURkUxJ+QjiBzWGGjW277wG5EHsLds18U/0Py+/V/azRvfY7eFJZApVUcsjlQFgCgPGzz3JVrKk6zn4LFHllS4Cfs7nDP5f9sgQlZZlF8tlkfdDHo4aPcvl+cRinbckREImZR/EMO+8sizCh3/zoKcvB94lQW1m0aPiFs7btROLv49AUmQ8IMY2Z1ghvIpN1gmKX4jywAoHH2z+atRqOLTMHf8y2VLK4dcsYcv7t//tNAMDAM0K/Yrn6u2TPdQT9R8yeJLx/hnEQ15gdYEWgJrxFP7zdZaUjbDtsZtiS9cCYT5OxkWOsAjx73LaLHWYYRIms50+TcKheAcy69Fk+v0qrJuUzjuusueCYBkQD9gdZyNhcokV0SycpaXxX4qMI/SGJIynzSn4/RWN70GsfiEUg98BkCmz3WSI6EfknilvFRcq1QN+LPSN0Jz+q21PChSdWuklFOrxWQrBMTZTbM5b/pnqD7fdlAl3gUvRaZkqKVLNOM//caTNnG4kWGjN7FciDTGzRZdF0l1uhRh8hPcfUWJO03t7ZH81Ch5TT/OM5ttILKTs1FVnftCDl7/3RnwMArmzajV+88AgAYO3mFRsDg2995sgmmZnHIvIYqaxZiydTqvybR6PJFueiJ+M89loNtPjvTqfJ8TJVSPhwjydp8lpJrKYa/CMm6GdIE36TgcA1Qpbv0h1QObTSlynvYczCrmYS4fHHrLHJ4oLtc/We7VOGrmO/RSAEG+9P6fX9lL8KsFAllVTy75AcCEsgzXLc2RxgfVt0T/a90I8BFrufJbDHQrff4lfuPVJe2EOKi7sl4XsbhbR5ktD8L/UJmWUWDHzs1LFgRagM9/RJo80i0xZimuldauyTJ0yDnz5NU33RLAalztR4U+nLRCZSziBdKhAMgS2pYNe0sAYDTKa2T39o333hKy8AAL7+spXtnnzqXQCAO1sGcOpR+y7TguizWSizn5ixmYng1Amt5xbTc026BWrjtUVAWIeFOLUoQoew3w7NL0+wU19U7LSMRBHXadk5pZXlPgrQc2fDxhgsAZYBj8cim2V6kuc/fZRFVJdP4EjXwFKOv505ZtbcGtPXt+/ZuYcTAbL0UmqueQ+h1b1esPtr49cj/ngYqVKElVRSyX3lYFgCOXBvmEMcZJ6acj8I8K4kzfewigZPrmQBBCBOVNpPMQPO1AKDViL0GCt9KVgyNWKz2wlkmueOm0Y/Rw1/bMk+nzjC7Yppm3ZHVUpFgor5Wl0sDAomEqm6QQKMlNTjk6ECYowtIEJMf/qLz1jJ8B9/yXz+7nG20uaYr9x8GQBw5gRjHFMWTk1I6skh3ab2HRFEVWO0hW496k1aJ7QERDk+SlhwFEfosix5GnglihFSAXGmZDkdUCvL4ukTFjwaMUU4UorQtkO651tjNnOhgXHxlM37e5+wxqYrXYc4BKcZpJyZhfShi9Z6fHbR5ulV0p1/5zVr8HGPJcfjtEgvFonarRSRnr+yalhaeraaif3+CHZIeZfXK58vS2UJVFLJIZcDYQl4ADOHELIva+uyROUA7R7yeovgvMlDaQs1gbCpUYzARbbCj5j2GwsWyu0igS1ve9Q0xY994Em85+2WZnr0pGmcRlvNNKiKZnSs/bg46FiPRTk+akjRUiuHmCkmwPOltAREeqrW1KFJCfDKdStG+tILlvJbufQ2AEBr0YhLpjM7R0y/fIFY3Om2oMu8X8KLPQuRblPdyi+vMZuQOqb1qBmVdWnx96VuBxHvJ50JcMTYkAhFaSKKpCVbM7+8ziIk+fYqQZdVMlBBWiANte0TFyw288HHjVLtSIeWkp+FGA+NBly/Zee4d+/btu8Ru+8L5+0cbzt7FgDwyj078Jnnrtm1OY+5QFZBXZfBRSH/i6J8/0qTK0ugkkoOuRwISwAA4HbwY1Kikq8/b7bw5q2SZb9pjsQlCCYqAnHUqPSR4+Yjfvg9RvzxEx+4BAB46pI1CFlcTABHFUb6rN1oJ15sSg0vyyCjZYBSgEKaP0ScxfZJn3jL/NJNAlwG3D+m+q3VFzElJiE+8ih/M59fQK0BCU16zOsnjMyrECYhtiFlBGelx7bYTAuskSJsk5F5AXSUJQi4C8YG+tMBRoykN0QLtm3RfE8Lod0qlj2nBGY1SEVe47mVDblL2ra7WyINsfO8hxRqP/gEm7g0hdOw/bI0Dk1HxCL22MWLNreR+f6vsRjsta9YC/LTyyRluWTz+Z7LFl8YMAvz/MtWBDbkZzVDjXbheB4Ufvz68nq4gLJUlkAllRxyOVCWgN+1Jv0VQC65iGpiWAmLy+dthf/JH3k7AODjH3gCAHD+BHG3kfx7bkfpvMOJ1IpSDTJxWEKLiWqB6fsTbeYZf3BCyJF0I3QDnRraL9u2kuQpI9S6XqNlY2u22hxbjPURS37bBgceE21XY1xha8POdX6J2AMSqjrGFZpU5SMSiS6yQcoKacBjFuncZUnuBkuR6zXei0hceAu1eIpt+v4dtg0bDgTVZQOYschABO22eaonRWITWVrCB4jo5d0XzW9//2WLASw1Be5gYRaVb+ajAEHXdpEZm3e+yzT9zW0b+eq6XeObL1r7vBf+/BUAwPGePZwf/eF3AADe94TFhf74z6w1x2urfEbloi+9eCF2UMUEKqmkku+THBhLwBbxcg6VUlocfTmX+gCrpg/AANuIKjoE5KlFemxm+d6nLMr/0x97NwDgI+8233+FKz1IdglqYTXVCDzgkUOkUHizVbyBgWlwDM2/RF2DoH4kgaijZkSDx7M2AFt37DT3zN+cjtiQktfpHTE/P6+blp6oziEDRhmJQIkEBJuGiEJ9ypbiLRbvpMIgUJp1tQIjyYj8ed7qEtGNSZ3Uajz/cCzUX3EqEgeo70RrRNx9qQ9FFHGMOpgPrRYR88/3pkfKt7c/YlbbE+csgn/uhGVnOmzYGasAK1B2DkM4YgAAIABJREFUU7yfk62wc54oyBcaLCQjaUzUNavC9z4IAHj2q18FAKxfMaTlUudZAMBHP/QUAOBvf/KHAAB/8P/Y999lA9jMk7AVIlApZquK/v0eJfO2d/mL70kqS6CSSg65HChLoIzik6+mpVqudDn4GZi4y+mFnecv/xQaj9r2NNt9/9LPfQQA8NMfvQwAWG5QE6YWHcZ6XrwoKa4iaVbh+aNozqjBnDvYaHRKYtG6+okl9NlVjytGk/YCr8WJWLfc/vaqWQAZ6cQaDTu+feQEj7MmozF9bZeJFr2Fbs+yGq2m5dpdzly6WqCxFjoJZK7UjHJd+XBabMg64zVa1MJqkiqMQo0NSPtE+W0O7Hos+IPP5/iAMR9sFLAaLFu2XbFCLEa3bdfqsojiFMujz5+wku3TR20+lltqCGMXC8Srep9SEYzaPWU+x4yxERGq1vg8G/w+im17i3PabBvq8/wT7wQAPPt1ixH8fy/aMx5NrTbjUx+1MX3mk+8DAPze56xk+7krRqgyJKbBxypZ30ElxpdX6EHnS3GEkjwoUlBSWQKVVHLI5cBYAm+kbdP8mCIB416iBhZCBLYi0yaXT5u2/U9+1ny2T7zXorlJjX47c/Dy08Ha/VDKp9p/sYkEmu94hwVg55owx5zPBIFjZkHkDzoH/WqIPIPtyzbvWDOnbGoR5lbHNGBrxTQg2N4MKmUvtXHL8xzOFdumxcJBkDh1sUlkoLAKqgkQ2JHW1gI1/zCTT8uqQVo3iq43IyEDbd5OMIsgtz/K3bxCk5mHLu+/G/gFbMwiFem1aAnUxVFAnoUmKxn5iIQrCAjLcu1FkHnQqRxfEmFqIxalGzX1VK3MaL3UzZJcOmNZhCHjPl+7afGbrT+0mMEv/azFEP7eZz4MAPjGd826+zd/bhbDt66Yxbk5EsFrFGjMZQHMP/P2HrLnWWUJVFLJIZd/NyyBMsz6gc5XjK5qBW9QTX7wcYv+/8Nf+kkAwNvO8CKbbJ3o1f5aml1+GC0CofdEFhBL/XBdTWcAa/LHa5ZByJhbT4h0Q4tbVh4GmiNlA/qsXb9jfuNsZJ+bzM23jln0GwtLumteW6ZAqcErskCkqsns6lwLdr+dmlkndagBKWMA9PFF5aV4TZ2oRU/Q/WLb7slFyr4wol0i9czp5zajJPArtMgw2uvY/Xfo+7doddX5e4MWQzNUeBKFVys2Z5FW95FaxynOIQQnf9fN+B3U8ywb1Xuk7/NZkWo9omUlzoYGKyJbR6wWIz9uluWNa68CAD7H6s3/+LzFEn7q/WbFfezpHwUAfOlb1hr+9z9vsYXnX76F6UwxDVHflXAnD4mnqSyBSio55HJgLIE3IrsZhvy8VVepOrDGSPMHLlkE/b/9O7byPvEIDx4zBsAmlkHDCzKozENA/VGLi2hAVsrQtDW2NzDZNl9+RkSgMhFqaQZy6IVMg85F1popcQCTvlkUdTq77S5r/GUxiM9OsQWNlemUGavsxukAjVqRIFUcgR1G2E/w+3xo8zGj8y5DxwdiVlKr0w8fM/rf67DlebnVu6wSVQpS+9YTFzgTW/TxWwQdaKtYgWoLxPMYseFL2CqrUAbflajJQ7v1chDJzZvShMYt2pfvUX9o8ZjtPisP2SotClwFPJ6W1SJrTE4eM2vthVf+EgDwzLes9uCTP2DvQGfB5ufj7zML4j2XfxwA8H998Xn8yRcsk3BzjS3cVddSyna8UaksgUoqOeRyYCyBvQqftE5rpQqsPyU8gdsRNIjCskhtwY/nj5rW/Pt/03AAT5yjFvXEcotMsF7fefhcrUSi0Jbvz/0U6Wc0ON8yn24y2A4NVh21ScxYQMQc+46eYLYV6pBVcGOi9yJeq9mi1mEkGqmOYyRf/r5qDSZi3GFtez3CYpd5b2YeHLMdK2RGPsHqQTey7+/dXeftM//f4JZaSLyGffnlsfgVOAZZAnSisxl9a8YWoiRCnZpclYZhG7AHtAg45kiVeLqHKKQubANlQ+zrLLSxEw+g4kT2uyyn2PkQllHLdp/JkrHPd1dtPtKpPYsux7hN/MNMMY+hXatNBqWEgY/eWas5+dKLVmvw4XdadmWxqffWPh/p2vX+g3/vMXyQdQt/9CWLJ/zbrxm3wd0NxphYGZqGP4giTbd/HURhZQlUUskhl4eyBJxzSwD+GYCnYJ7JfwTg2wB+G8BjAF4F8Ne99+tv6AIlF1cav7zVJvb5nIOABy3TZ/3Mpywv+/7HmVOP6LtLizLqixa1rBLjWlXlt6vSbywLwM4zEa/fwLbZbAwnhp/Q9FQ0wcIYKJMgEjyzJgbMKiiqX2f7c+HYA/xxKP4BjlEJcmY0xiOzctQSPZ95dGLT+Iki4tTCy137/tQR82ET4gQaNTvnFtujN4kjEIgxS9WWjVwAYzELxTtPj4zAAcWxU86jiz0aNfUikM+vKD+nSTn6kKvn74r+B41HUf5cMYBMFkCJj0Ghk5BJipBA2QB7Rjn7LIwIcbx5h/UebE8mbosxuSCmRHu2OQa1Q98O17ZneXXdvl8lM3KP8R3FamQdNhpjPH7B3tmLj/0gAOBHPmho1s99wSoTv/Z1sypubRLtGDJArLp8nezBw1oCvwbgj7z3bwPwNIDnAfwqgM957y8D+Bw/V1JJJQdU3rAl4JzrAfgIgP8QALz3UwBT59ynAXyMu/0GgM8D+JWHGeTri/y2OFQLxtx+9P3nAQCf+JCtnlHOLID8b6H6UvlR0tLFijXVnoN+vrTwmFp2wlr+qTgC8hkS+axOjQa05TWkwQd2jkHfxiYNHqtirVbUhHJ2fcnHjWUhzIRms/MIDz8aeLQW08LtOOb5e0sWvT7/mFXg1XI7domZiKsv0/ffusdbKGpMtJklmBUrAOscuzDxuRCF4kWEDz6/5kv+uHLzD0okJUxIVkJKZqVuQJE6OgWySjH+OrD0IVgyA1oC164Zkm+dpSSnL1qWaUjLRxrce1lvanWvlml24inRi626WRLbI9t/SARiR9RLREWiWQNg71RMpqr3nLfxv+eR9wMAXvyI8UT+8RcsZvDFr74EALjLV32mdxrFZxPmY89vH0wuALgL4F84577mnPtnzrkOgBPe+5sAwO3xvQ52zv2yc+4Z59wz+R5BwUoqqeT7Iw8TE0gAvBfA3/Xef9k592v4Hkx/7/1nAXwWAGqx83shBvdboXypjZDqAbyLkdOHP96zW/vUxywa24m3ebCYebmkj9VKh5VkrBGIm2qiVy/sN9owf33GmICT1qHfKQYen2fz2IUraU1Fs3nOIeMIkxExCry9oAmVuw6YhGLee8fE2HmojRUVF8pvMh5jYdH2qeXm8w/oq2+yO88otc/LTfYsPEorA2ZRXX3Zzj0amG+cRNLadlyH2YYpYyw9AgxiVVTSKsoyZU6yoEVDu/REmrqI+SiL3gPtV9b8aSp+RM0/xxqyDHp/pK1d4BgUULRPK+2br1g7+cYJ1gZw3hS30bNSHKhNOKNQkFkq9CbjHowBaJ6SeZCFd6fsUzrvzKs/CFmUiR37+GP2+dLp9wIAPvS0gV9+//MWM/iz527xQHFXFuVhLIFrAK5577/Mz/8HbFG47Zw7BQDc3nmIa1RSSSVvsbxhS8B7f8s595pz7gnv/bcB/DiA5/jfZwD8I25//8HO98YqCQGEtuEAUKfW+xhZgN9+xvK5NVUDeuHGaQmQUUeWwJjIL7XSjugvZswijLfNV85Di27FEqgRxIUQuRCXCOhFceFRewxDXIGdb1WxVytW+kUlngQvjVdqPR1y8erhl4cwuV1vMoXjPHTIiXhn6PibcRB89zVbsxdOmSVQZ4+9Ji2i48eMm/A64w2eWle4/l7Xjttm1qTOuEYcEITsp0gexAh+zhhFLTqvcCxlgV5HNM+yDJLS2y0LIw6xGv7gZUk6OMVbaEXduEVcQGLvUb1l83Rv2+av17Rof43auaFnx5iSDKCI14j5jLst4im6tr8n3kTP0PE9Q5TMLUfVmIiJSlYtmZ8j9sd83zssw/P4o4aK/bNvmRXzr/+7f4m95GHBQn8XwG86iy69DOAXYNbF7zjnfhHAVQA/95DXqKSSSt5CeahFwHv/lwDev8dPP/69nss53K/M264X9i3iAwLTis9xdNlu6ZM/dA4AUGNNgGvqpPS7xQ8QQAa22k6Y7/X0z9v8Wdp6Fth3i4c78gpo0Y4RzX1aJcupiTwjwRl76Hnl2rl/TVj44CsXK//Eg6etvLoZq+NyWjvq5JMT/5+ngCe78DFSGXzznt3X+qZx3r2wbb8fT0ybnKKmyoPvapq9TSbjPvEREWsq1GF40KdfTz9fMYGE+yWY1zD4Up3/frz5u9r1lbZ6L5JI2RR1kRKaUWpZjEI8PPANZAFVuE1NfJU9B5unrP/AmDENtjnEUO8mLcI4ZsxkZPPSI/oxgywDe9aPnTSLaaljJ5gNbf+m3hWRQkSzHSaNzEzFMEKKh59p5dJ6W2jbtX7ifaoy3VsqxGAllRxyOTC1A1G2u8GO9Nwc462KLkWF7Wux2kTI8fRl6zLz6FHmpVmDL7y+tK58v5Czp9ZNhT6TxufFp9SgwoaHKETon0hfTh2A43o4d6R+AfxtwkrFTPQ6vGZSwsKrCm7ecUmqT1BKanri29NQq0/NT/8zYyYkSqPAaXCcTD+OvuiEuIkaz722bvO2GFsUW/GHnNqnxbr5IavqYmr4OqsUa7yXKWMtbfrAMYpxjihy4a5kCe3O92eFeVIEvrwNGr9efK3LFaXC0meqNdjJMUjcyPU7ZhlNE3t2USJEn+I17J8YKhcZz+G11ZdhxPiPOAt7DTv/By6Zdm7Hhj/wrMEYp2Jy4nw1mwDjKnPKJFkLFGUPFAsKRTe0DFTnso9UlkAllRxyOTCWQIY5J31w8fcJCgcQn7bUUit14OPsDxBPaQFQS4yJ6PO5fd/lih0i9k5oNWp++u25tBG1bNBOpTElAWHITRIHrVFn9WDODMOE+ICUWmdu2bjCfYfbL8VGJMqLp2lRc4b6ebH6QNiFGYa0Qk6yq/DRmroJk7mYPuydLRvrCeIt6mJWoiWgDEaD0XGEOgn1D7R7nkzlC4upJ5RnAiDHQujwXLQEJLtqA15HdLwPeIq8sFVFYMpYgJiFJn6MrS2bn2v3iINgf4GM466F5xuaVgIApmRWarctVrLJCtAhrT1ZBj/wiM3L+QVqafZ1UHxiIry/LKhpNrcEBSFQxaveOT33UE0qK1e1JEXEZFkOxCIQOaDTiubglkBvza32C2m74ucaTcunH1nGuy4YKYOf0MxiUOXGLSPoEMHH8SOW6lleoJkXzFM795RmcpbpReELJHotDlIFJ/NiF73IUQDGhKAjA4EzPngVtsQi6ghma/H+AkVaCf46p1ATSEYvuYKUTJkJJBN59PmiXDxu+7zrEZufP3nF5mU9tVTYrZEd+yhf7mMt/pEr50UwS69rL/2QoBq1C1/i4rG6adcTDVc5D+zyuXle/uOfE3wUDVaRs5S/V2vyMkx4vggUo7nZjtQyAKT5DBtsLzbI7T7ylsGm1QpOoB6lhsckSUkJHkoaxUBqn0Vhjx21sf7wJQsIdvJ7hbEKAt5gqlVp0eFoDLC5jEBNCYOH9eA+8gai0h+73ISspD1KUrkDlVRyyOVAWAJHl7r4hU+/dxdMVBpfKR+BSMqFJoKuXj6ZoBETBjy2fcappV5GY9tubxI+TONSDUBCGk7XVDpP1slMwbas8L2IHBKNOXQ5ieduAIM9avUlaLI0uCtpvF2BLJm3skbKPdv3kTl4RsUsDtNMbb5NQz/5iGmVV/u273fWbIy3xzb2m9u2/5EWNaEKYwj6kXks10aBsR414t11u05G8zZmm7d8B2Hpbj1VNOfLGh/BChY8eB+ykOLuQfS7GN0VUBwN8tBaHW0jcfV1s4xabOraaBThwkO+F0M2YampUYiC1VMrwf74kwY3frTNpi8EbYXiL6b16iJM0buex0CwfAQbL87hjPTkns+2Scp21xSxSymQWJLKEqikkkMuB8ISaDVrePel07u+L/t+rqSlRQbRYP3n2aUU0dBgnjkLZBLSNS1whd2iBt/ctpV7sWfaZGWBtFoMsolCu+aUPipqY2kCx5U/UUxBlkXkECfmV05YXpzNRFfONGOpMMgreBapcIhpTsGEGZRTOyrFhzIG7VIoQKjUGbU2ZGmkoVeKgkrHCSj54csGDlr7ho313sR2vLphvv7Fo1b62mPTU6US45rNb50goZxWSp1mXJPaaMT5XGBbMrn/uXOIQwS4SIul1F/Ymc9C8zFT8DYXUIexj1CstLdlVd4qIn13a4z1yDQ/WpbCq7PQR41NckJ1BxPRidnuNT73bRK05gwEvu2kHffO4/T5U5aeCzAWqN1Ji+4JFAsWxTym42S6hAg675dkp5ix+SvTtipQU5u6/aSyBCqp5JDLgbAEHOaAn52i9J5cYAFMktBSzKRFLdRweVgkU4I3RF6hBhV1psC2J/b7FgtBWqK00uqqa8YqQ7XPM0aDRwQPqdRYtRxiIm/EUYgfpCw79io6kiZXg4u4mKaUJaCMRRai/yqNtd1DKjFozrxwnUDHpeg7smB9KODSYPHJhZ5pqI+/wwqE/uh50ybrI7vf9Rl9ffmsAskQiOJIgZYyKt4kOGh5yTTpZt/mq9MRRZqe4Q4HXsVY2io2wI95ABERgMMfVKgVCo+YvnQhki+LQEClItnptVUDBl1dnyLvWVOaXsMsnza17IjxnDWmTgeiYqe1l9DS2Vy3c73trGVNnjxr959uWywg7ciCEFScBVgikJG/n6l5SzQnk9GzE7mtzDpNIZu1tBNmLvoiqyEl2j5SWQKVVHLI5UBYApJyNHwuZfAI9+cqqoit9yMk1HBTHjOjaaBmGG0SiW6RhntM32yThTB1amVpOGnfMmVV7Bgd5wo+UWvpkB2IMKHfKDiv6KvD/aKk0ktx8jLoxZd85NC6PTTIKFbUzM82hx0HQJK0qNqR5UaW8j6WEK+ObPu1K8wWMHtwdllxDI6J9GQJfecx59FzLF1Sbd9eI/0aYy4NEXp4zGnJfdECCFHwQBiqZ6EybkXSVaxUL34ONOjyr4X5sO2NW1Y2/fUrJFE9/QRqDZaeM/ahuMPG0OZhRDqwKa/dYmBmk+CipbqN8SfZXuwU39EXr9q14kcIpiKkt04asZiRf0Gk9STjyM/L1cPzLc2TCqBmIpy1czWJ4aiN96YVk1SWQCWVHHI50JbA7ihB8ZskUc6W1NrjdSyKKlo5VEbxFVgV3DXPzV+ao8yYBXDyrxl5Di2sMu7PyD6vk8rvn8lvj8JnlZWmJchxLHShK2o6FTUpG5LuskKk8TUdRY2vGAJCPER+5TwanshfDnEC+d2M8qemFT90/hQAYL1v57zet/m4NDYLYaGneWKTUNKKjeiHzjhfCdhajBrvHim2Ty7toNjmY51xDoWolObTMxRxh8YujZ8ECjNuRR+WKPYiaK+N6cprRrLxtW9fs99PPWb30D2COs8xZEFVv2+WzWBolqPY4ZWJ2GI8IR7Z9tMftSzXO4/YO9lftWt/7iU7j1qJffRtZnG0Wcg1DuXBRfxKjnxeMiwrM55bdrYTNb0MhFCIx/hWvcIJVFJJJfeRA2MJmLYq+sZzD5fauPS5VaPmpFbf7m+hu2A1AdKaM26bIX9fRKGlqgGgb99iQYxqB9SQU2ORNmkxKzAcMRfL6HhO/yxN83nJqjS5imxEcV2qjQi58zJt2K6CmlKxkmIAZZp0ZVcCAtPt8LtNFFEfM+Je5zFHE/Nxf+hRww/861dtv1cJuDy5bPfbIHll25NgtMlagqk9kxa19VLHvr9KP3ydGPnFdh2zgZ3j6jUjxByN7drHjpm1oM5wUcPOEYdWcDYGzzqGMEuhCapth6SMe5l++V8+Z81AG6eMhr7WM6tnNJpizEKqbWUDBiSYVUtyPd+pxVCWnOECPvERI9V++jRbx7Fg7cZNm+/nrtnnF1dFHGPX+eSSfW7Uipmf2KnIKZ8XCEVFVOtuU9kXvw9plfvr+soSqKSSQy4HxhIAEJbB0ECRvM8ZiithTJRVN7bVesgW4JubWzhKTd6gZl9ft3N0hTLkuWqR6MSI45ejzv1Soq9S5mKVDZgxui1YttB5EWm4plDk2qPFa4moQppJuPtQLRlIQoSQLCHaSuLLK70TQpDHKTMif1+/wwULSC3K6zVhEahNibTsOPP9L/bMhx8+ar7ud/umhTfGdr9niAeQNhEOoL9hmm9E3zohgOIoW6DfXrcqT0QLuMno/EuvmI9+gZWNHaI9pdnSqZ0zYwzAe1kGpH5ji69EkXY2P33pql3r2Vescm/5jDXraB8xC2Az+PtpqIEY871QrKfJbMjRBXv3LpAe7MmTds3jkd1DRCr2wdBmZPWeaOV5vsiyBl980WIIH36HVb2eWJT1wvgFYyqRw/yBl8trQyNW8HMxOxSOK1l/ZaksgUoqOeRyICwBDyD1WVjIsuAcF/H6Gf20mG2yY0bwp8xND/sDbLZMcy31bIXtD21lX2wVyTFarHK7zfzugCv1YNu0xTbbi7WaopLWKistbudpyGENNQasjovcruh+Qq27q6pN81BqKiIff1esIPyuKkRRfzHbIGIK5ddzNabMsNBQo1AhB2md8L5EqqJof8L7vujNB76xZed64Yp9v/I4MfaMLdR53CKpxzc3bX6V+aix7LCrGMHVbVy/waauHOdSl5mDjP444zKiJEhEs+XMMkhppeUZv8/t3BkzF8eP2rV/7IxV8jUXDBWpzu79EYla0wgewonYd002+Fhp2+de3ayG5swyDDVWqca0GIdsXDrs0wpkzKjN3+/wvVrlzTx/3T4fXWCzFz4PUcbFtfo8eCQ+hMBnomfI90KWQimm9HqM7ZUlUEklh1wOhCUAADkcZmrjzMjqkKi+lP6Z/PIzy7aKdk4Sl82qsa0oxtUb5v+1WQU2JY5+kxHo44tsftFgDTkthRs3NjgS+3znrq3wx44ZhvzIkmmXOSJQmq2IS4io55PIYTIWOxHx4aGCTpq8hIsoEWyWYwLl/eZU68o+6HyyQDznQFwIOerECXhqlVQNS9h4MyEHAhgjiIk5WKmZtr5gyRd8mX2lXrpr5zu/QuQg/fUOKclzciesrdv8RsTat9qm+SbpAK8Rg3Bn0+Zn9pzN2+MnzN9WkVydeHthHRJmJhw5JBJq20aL1h5z8Kobce4O92M8wvH96bDuwQM58RIuZps00njXlNlhjUidz1kYkAmt1Nmk2LxkkVmkBWca/wZrCMYNm8gvv0Sk5qUO9y9ag97vQMgGkxAo/mM3okbH2n3v+XOQyhKopJJDLgfCEphMZnjlym0MmJNVHb1acgdqSjbXcMyt1onLXunaaru23cTdVeMTuHrjBoA5y8qduxaNXWlbdFaFWTFRaHdXbYU+dtS0Q57ZDmurtsJ3SZwZmINolUgrqz68Q3rxWZoFmvI5JWVR8++HBygz6ezKEoR0MM8TiDiLWQVV26WzOYJQsYypWqwzrqC4y1BNUZ1pqOHQ5iVhbOQirbAbx81Ceo6R94azeT2zpNbuzELUbT6bdZuLW2zmcYfZg29dWcOrI3uO6zXb9+VbNoY/vW1jPHvMzv2Rp6257OULRv4ZDS1+c/uKNd6MiBTsZXw2tCBVjRoHvgLGQYgzUUOQKI4DX0DGLImq+TJaU2NaNgNBB7nRfiKorZHN59xJy4Y8cdd+v/syqxBpoV7ZsGd1Z5MNQ47xHcmLXBJ7Sjn6X4op5SUE6n5SWQKVVHLI5UBYAmma487qNlqs3PIBlcXod8ChEynYYM5+ZNpkgfx3y+0aNhbMgXyNyLTHHjGtMaavdpMav0fs+kLP9u/fWOe1bSU+cdyOu3bNNN2U1Nmxo3YPHADFSkD588NhP9DOtDk+ZT2iEN0vr8F7r/plSyC0PC/vWKpGdKXPERyy4MPSqqI1FQWuOxvrJttw9cnOfKxt87TSME33nnM29iu3TbM9/5pp5ZWmcfN1qJVrxOK3SMW9sWFYgBdftrz6jX4dd9gWPWUuPmK8YGti4z73jg8BAMY1i9Os1yze8/QHPwEA+N8+/0/tnmb2zN/Rs3n/oSft2pdPMyMSqzKU2AjGTHKiG2fpDBlx+EJWzmbKuKg9OOM1Ur6pOCg5j8yudLt2zWXGklpLFgNIm4aK/OILZmlt1mxeN/s8/xFaMeFVcDtc/lL+v9ya/g1KZQlUUskhlwNhCbjIoVlv7GKNDQ4XE7rtjq1Z3ZZpMTXR7JH/rtnw6DRt31v3cu5jq3y3ZyvyzXum2eOWocUaTVuxu4wQ9/t27uMnbeVWtPz2PbMAegs2pl6qKDIrGRlN3yDPfKsB1EJ7K1aEcbF3gQmHW9USCCexT3+B8ufQutsX+fOjcD5VKdr3cZKgliSF7yaZ3a+YlDq0xtZWTVMNBvbDubOmnSNmC84tmAXwUx+wefw3X7V5vbVm2vrS0bYGAwBYJ3f+q3ftebzC3PzNSYYpoz7KUKiCrs2x1Nkb4uP/vmn+JVpWA7Ie3aQWXh3YM1Z7+cvnbQhn2T10ymlqJNKo5Aagnz9LZ4FzIDSOaYqLwJ7vlBWOaaamIfS7iTitE3/S6fKdZPXkGcZU/tqHrE1ezRvO4Jvrqiwt8h3mQf27YLEIQ7Cr+04oIlFfDsUC7t90RFJZApVUcsjloSwB59zfB/B3YM7KswB+AcApAL8FYAXAXwD4ee/VM3lv8bnHdDqbs+oKaSf/jFHeCyeITouYeyba72jHVtlmM0GraccudkyD31s1zXTmuGn2BrXLHTacPLJk0Vt1I9smz/42edmWV8xCuPaafY6pIcZCsdUVA1AmQI09XbAiUtX1z+fN7ksToPrwXNH9oq9XjhCH47m/2JRUg+BClyRZVupIVAsNV9W6TN2KdM5ZZhpVy2HeAAAgAElEQVReefx7a+Zn9xkjaFILx7SEzjRtXj7+tKHwvv0qmXYYQ+nU7XrbRHX2aTFt0ezp57NQ5KaskLgIwLjFF7/8jI2V03TxnPEA/vlXv27nElsRm4eePWnHv/28WQZHOtLejAWoLkAsUbx+o9sKbdgc53Q0sGP7zFwFK62E1hOL0QKRf+2ObdV1LWHW4Qy7hP8Uqw6PftvOO2XfgjRjDUdoNpsj1XMN/Tfm7FUF0QsmCyFksu4fO3jDloBz7gyA/xzA+733T8EyeX8DwD8G8E+895cBrAP/f3vfGitJep71fFV973O6z2WuOzO7M7Pei2+Y2AtZO0GybKPEFsIIgoQBZUGW/CcS4SIRW/ww/AtSBCEisrBIiIOQAzYGWw4CRctKRkJe2CXGWWcvs+zs7szumZkz59anr9Xd9fHjfZ6vT9Wcs+PMerdbOvVKo5qurq766qs633t73ufF5+/1GoUUUsg7L283JlACUHfOjQE0AGwA+ASAv87vvwbgHwP4yludxHuP0Wh0hyUw4rZMxuDVZVvhhl3zV7e2TQs1iEe/79QyTrZt9U8YWe7s2Up+iz7u+oqt0BN2gFGDzhXu7/ftmnt75Akg3/yQPG0kmsFoRB+QrDU9aoz2EtfVdHonwi+f7hdCUJrPHW4J5PkE8paEGJIDF11AFPKj8sbRFH1i19NEPAvkEWCHnYS8+OLbv3jJ+AQmE2rCkVhu6KeSy+FMldx4F0xLv3iN2pcWw5Vbdp3bE9N0A1p5ruQP8CHYqWO9lrxGj/j7p/7n97nljceqdzCL6zQzFx97yLR5I7X4zB4xCUrtV5nZabC+oU4MSDIeo9u17FGHyL5kSLRqLE5JVqPKEKAeXWoy20RevzjH/uPVp4G3dopYl59+2CzRH1239/M2qxZbZNAuuzFSNVCd6n3KjeGIDk0zHME7ZAl4798A8GsAXof98e8BeBbArvdezIbXAZw77PfOuS84555xzj2jzq2FFFLIuy/3bAk451YBfBbAJQC7AL4B4NOHHHroMuS9/yqArwLAqVbNH7QExkRfDZmDbZXJH8DOuPUKcdv08V7bsBx/vVzCClf1syfs1hL2CbjKmvWbRKEtL6l+nisu4wyXLpmvtr1rlsJOx86t/G+PtQZxRAYjjmFEDemWeV4/uaNGIF8rcAdikNtZq/LD1+hgQeh4WhKBYvAOzDj9/clYbjbiGnPm7L9Qp09bpjbtdEwzNcllL96+gFGI2ZKc1kedNRfnmsx/nzK8wO8/a1Hw779u87llChZDRttd7BAp9qPuTmR10ucRrY0qx6JaiIjVhKu89iOX7ZnfzxiR43nUR7DSsDGXyZAsnoetLfFR9NBn52jNWY0xgkbDrlVWfwrObZn1IEsN9SrMIknzuAOVQioLscq6iCqzCG9u2/HNszzOpTMuSr03un9xCPI9iUPrdXD+iBhN3zpL8HayA58CcNV7v+m9HwP4FoCPAVhxzmlxOQ/gzbdxjUIKKeQdlrcTE3gdwOPOuQaAAYBPAngGwFMAfgGWIXgCwLfvdiLvPUZJEiqyxtwOGPVdYV+BJiuslKtuLlsudm/PtM+rb27hwrotrS0iAS/cZ8cIg7CxYX5it2srfqWijjrK95rmXz9hvnC9bpZBZ8f23ybicDgSqlH98HgvAb9fCszG0R098Wb3rV8BMwBYYAA+IhYQVm5phsA6m9kdKOmqNXVJmuDGhgU1Xt8xX319xcZY9oysn6OGb9irsU3cQ8qsitCZZT6jipxcaro6ux0/uGba9aH77Tk8+ceGlNvd4/k42GnqMGXyKJXTHlzZLF5k1OM1laFwNvePWGkBPv5+e9YPs7pUHJTCQnRYB9HtEBdApOKsP8QkdKhqEXkqi7HMqtMy61kqqmsRIVWkngYc8SSX0eFWFmVJMQOyZ50gBuaNDYt/XFy3e0B19j7gAEMUgPDCpKoiDTE13g+xDHf28cjK24kJPA3gm7A04B/xXF8F8CsA/r5z7mUA6wB+616vUUghhbzz8rayA977LwP4cm73KwD+7J/wPBiNp6EHwIgRUvkyy8y3r8oSGLP7C1lypi3zx9LpFNvivSfWu9UyzXaRFsES/cOtLdNIqgMvBQ571vynpjXUVffSg6ZuXr9lq+wr1+z3EX3AWlVc+IwC1+ohFDwNKl6QQdvI5Vdl38wwkN7IVoFFs5A0v2YuOSDEstmFKFYMxXFbx5a3SPkrV01jX2FN/g6r2Mo1i+OeZjXliZadY0DEn2oOUt0LMzOpOkez10IMm7+PXjT8wKsftBjK1zfNItgmk1PqY6SeHaQjsR7ZM/IlsjpxbgX0K/H+Hj1n2vqvfMTO/TC5/vt9Rtqp+QfkpRiNxbNg5/FTaWUb+4m1pYAbEfeiYtzqXSALoFTOxkgkM8wDn02aDdDIitWBZV77fMvelZdfse8391gh2yyHvL8nX0IUOleJUYrxB1oEyZiZHMY84tJb/5kvBGw49R794TjQcakJhTAPq2xl1aB5K/htmbDIEie8HFXgCK3sE3wBmv010oCdOGHbtujHunq5uRiImjxWkErwUJvYJtOUb9ywz7WmTeF5NqAUaCQulTBKVGea+6PmS6w0UgjgBaLQrIGWpxiPwiKRpaIKgcgQIBR4iC+7S7HESVxlO7FrLHF9mqCV17auAgA+9LAteh8gi8hyXbzfDEJykVVh1VhAJ77UKU3x5dj+MP/yh82t2utdBAB843+8CAAYJB6VCttlka48IpV4iUG3mC7bEt2CtbI90/eu2/52xD/6m7bApMw2ybvQ/IQSdc6bGn+cXLc//OVGLfzxIhdMq5SlJLjY5Sxsn4t/z6DeeVPcZ/aK+LbF53LmlC1sb/L9es/ZKso5Ny/Q7QUyGZWK87PatEWlzPYoKWDDhRRyzGVBLAEgGU9Co0ltpcnqNDljlRSrwITuwB5Ny05/EPJmjpp8eVTisSR7KLPdFoN2VVFmMwWmBpQJwUGdXRJD7pAqjDTWxCmhyVX4PTUV5thy3ekM4CfCP2QLhpRmKkEtyO2oWG3Ro2xqMa8B0lBKnE01hoCg4oR5olYPVNn04twZ0+xVBqR2CJfe7dj33yN/2IsvWWr1kYsWKL14v1kGJ1kgtCzzjJaBZzFOAMfQujsfm2n+N3/W3I3e0K771I/2MBK2liayzhgpFUYymdN0TT52P2nl2QBk45ppzfUG6cdkkqughpqwzgDpMguSWi3bNlhElk4TjKciBZH5b9dSwFAuWT7NK2h2vhj8DuCXXLrgRRBkRQvrgfvMKvnBi0xbDqY41ZAJqFZlnOPQnk6WT9Y9CMAmFG3ICimkkLeQhbAEvPeYTNJgAUilxVxFy6LFlsbkKl2nRq1zxesmQwyTbNqsR4LRZZV4UhuUytJY2bRkQtCP4K5Jn7RbZN26zdLPGxYXxOUVrex2XIdw43QKRF4Eo6TaIh13rBbZyGqV4NPn2km5ADrKooHypCKynBSIVBGKClBKHqiSR7RJ6uyobdpF5blDwqd7Qzt7l4VR/+cFK7j6v8/btsFYyMXzVhFziZbC+TPm066TtEXFXuDmMhuF/K1PPWjXGb2K771kvrwMJ5HIiBasTSLRP/OgDf7+kqU5O0wZ7ncYMyB4qNrQvDNoW1McyILDdaUYY8GvGTtxHuUQE2IAUCCpI9g6gyUg6zWo1cNjArLiYuQAQLSkanwvq02bx539BCcZjvFqtU4yVJ+PHYX/6T3R99lS87wUlkAhhRxzWQhLAN4oxgKMVpoxKL5suWuVFkCLZBpTFg1NRin22FBS1NYV9XAkgCIlmLE/sJV3wBZUfcKBk5EYOGyz1rYV+b5z5qv16NNt/MBoskQaoTLeLq8P71Bhii5Wqy81SZXFE2W1RRQix1nYsA5TfVCwCHL+5h2WgjJJEJzUo05TQOnVlH5lVLJjqrxmmxDdQcuO39knVJvXYpc2/PB5IxN5/gqp3gkyOnPasi8PX7R5e/B++7yyZGe41DKL4jOPreLZK5aRuDHgM2Js4L66je1nHrSxPLhi5liNqbKh0nSEASsb127T9+dYanVBfllwFGUtL8WR4rgSrAPFFQJpa5p9ByUzTc7jc9bdTLI0dD4H/NHvSwRAtWgJ7HYTJGt8fmq+w2O87lvXDGQ8orrjfRVEo4UUUshbyUJYAh7A2PtAxKm8bpnrY5WAnZK0GH26VC2bmKP26x4V5pD3+wIBkfCSwBO1FhcpiCyAcV/U2zYmVi9jc0TK7bJpoUZMOCmxCy2OZTRUoQh4vEOJPmlEsIZaW2m9ToO2kIrnfMjyCa3Mld/OHA0XVv6s5RAsglQ+pMAkQMzCn0bd7iNhk0/BpgVpEJSXgXMsV7KR5+oy57PNqDnPOySY6JU3LLvwhy/adr1tMYIPvdcsgkcftgKjB1fXcGHN4gm7PbMOanyuf+6yzekHz7MdPIPcddKYcwN4tYUnpLdu23pVOAmbh7Go1x2zDhBYSMCfGBHnegZY1mwHXHhGlNhQnCfUhAVoh545NT7VsxcOoaTYgK5CS6BqY7y+7TFU1mei8TMrwIyCD3TnBDaRUFUv1F1Qw4UlUEghx10WwxLw3hpj5PABVcJd2/TLb3eI/qvZ54hQqgpz/61mirLIO7ly73ZMg4+4igqNOBop10osQlWDsc04t7I3SSG9RHRbrWYaTw0794k8VMlppRIHpJsP7cGyUf07CohyaLLQatzJIkD2e/zJZJpOAzVVk4Qaapahxi85cGKIYqtQxgtIx9y04LVV0rrVyC8qmq3OwObt6qad/z997w0AwPtfsxz/4x95FKfabDfWsej9BUb3P/Qg4wsidyVxR2upxcHYubt9UnQNTAPu8z2R5eBYaiy0Y5kTGd1B+T6781k5dij9OeTYA78KD1UW5R2QQgAzYlKBOTxNCTWyFalovaIsFdAlcepyDjo4DVYt3+U0O1ZZhnni2rwUlkAhhRxzWQxLANbYMbRN4rLaarKMk1HdF1/dAADsd01jrLWY8xc9FTwa9MNPrbKBKFfBLZI1pIGU0n7bIOKtXsnmVD0JRUskr4xIbPLcFZbCpsLOizJaKC3+Lp4VBkW5XHPILediAtL4LhahJL8OdGKKastiylkSIjQNaulOn1AYhKrIMIifV8sytdb2wtuHfLkorrJFOKHZJ5VUVZYYW3o32tS+RNy9zvPd3LRYxKtXX8Vawxq9qICsGpnmvkTMwdnVLOY/ClkRzb196tGf3t21e+owA9ReDhVDnAS+9iGGomh6Ck80niSgMSGyVjVVYS1BSSQih2vb8HvOz0QNaMIBvLZiMqnukbRmiAKas9GehHECQMJnNiZVe5nvtEryVVOjGMJRUlgChRRyzGUhLAF4jzRNA+pKdM+rxHYL+TYmweKbt02LJCNW8lEbVcoOlXJW+y2R+LFEDHig9VYLcQ5B0fEZoYlpo72OYbh3O3atW7foQ5Mma9e4RlA/bxqiyaxB2aXBzzvANc6xZTX4TPJVhHLEqS28ylGFAzgcOZg7XYbERGMq0zpp1GUJEF2oyjuVDOcyD8F2UdsyWk6itoqpdWJqRsUUlql9zzZZLs7YQbe/h3pkqMMSNZoXdmOPFg6fYbmq+ows+Uhd2H+hPEn2ensnOPYAgFY9VCVk7kUW1nSawk+yVlWiluOMKSnnvsx+6bIkZ+Xf/JirLZDlFAhgohxBTP531PLVUhlbrOc41aQVRmNlKquFcYVgWao1vQJaoYblcCksgUIKOeayGJYAbAVUsFZorSUSRu7skwCky5r+ivLAXAF5/HgKJKqkon8p7Ps+24ulY+VW6V8rI6FO01w9B3If1RyVrcpbjEOcg2mC3r7ltqlYkZCqKi7HiEUvFstvVEzAxuainEYPSMCsJZAqS0DtG/zKnMEgCX5ojsZsmqaBeCSl9qiwnqFJpaHsiRByySRnteRSE2raEurwXS7eIe6EElu315n5Cc07PBw1eCT+A5ojw5593tsVYSir5SZZTddk7KhOlOMUlhHysVkYt7es1qB6WjEmzncg4xDBiT+QPSKClNgCaXCGnEJEXjEBxUZ0nCyuSS7+4wPxC7KS6t2wjwkt0Ua5jFv72fjTRGELWXWMtwTumiwQMpCdHiWFJVBIIcdcFsIS8DAstYK+Faq4dVKDjemfJhOtlmwMQtx+zLbXceTDihurLdSSnUPnHtAiGGmFF9rQK1JsU1ItZ1FlZTa4uHzWItmVa3b8lZctKFBmBVvC5peTaYK66KVD4CHAwmyMgQkoi/W/g7IqwMmy2QFxVQpRGBCJuRoM1TV47zHJWRPKQIguS23GplO7H98nv4L4Aag2KkRx1oifCP00SdklXgFZdaEOJBVtOOsoUj+rdo8VE1IGwo7Z3bdzsRMaJqGza7ZJzToj9zVmenoj40Ko1iyb9MamxRounLF7a5R1HXJFTIEJKerFCiVDqMRsgCc3geZP3ytzMVVVKi2p8TTLKRHiPfxdHFihsnj/NNCalYMF0GO2Y8J3t6JMTEW/4aV4LmVy0lzGIy+FJVBIIcdcFsISADy8S+EYaV6iljl3yjQ8uPK1qraix7CVvBSz1RX9/2TiQ112kxptmU5cixbBlKvqPuML3X07h9qMqZlIpdnMjKVJa4Ml7OgNRE1OUswQDZ5p+zFppx2v6QOaURaAHRor6n9Ek5LAXydnTz6/IsxqPhELKTbLe2e+j6KAYZ9RN2StEXEf+IYq88i4xEYeE34WO0+zKStECDdlWTQ2u466ZKsOhKfDYDRBbcjqPvrqYisS4lKVncIRKGdfJgFpr6eGMGYqrHJsDfrrSSzyWdv/6oZ9Prum15/PfuyRTrNzHxFdWK2SYaikWhRajjyDnpGbmXP8LNQoD4SedZz5XhTlamzraXn2khRtxjwisTdB1m72HEqaxDTvVLXaH75lP+DCEiikkOMuC2IJOMBHIae8QtaaFTL7Vp1pW/m+ykGnbKM9HppW7/eGSEbMBlRsu7pEvDnz4eIWXGX1W5MR5y4ZiMaJ/EyuzAPLTDhyA9xi7nnjhmUF1lZtrL0uYwhksamVymGFnvHSZSPv0iLl0FgzzmxDpFl53hBwV9yBjEHiBIiylkSeqjyKohATCO3fpamlXWgRxMRbVJmDb6Q2fxG1cpm1FlNlOpAZ4qxqjp8dtXy9Ri4IIjFHI4ebN+3/SuOfWw3lgXafakumCL1ag9XVRpzxBSq8HitCVW1arxANObJnPCRr0hs3bf9KW5miBJiIoVcYfmphPktlA5TRkd9dCghCPrtQTagYfZaNWe+GcBnjURYNGFVsTjqDEU40ss+3wYxOtayKxyxXZcgOMGChprxHSWEJFFLIMZcFsQQAeBfy4CdWbRVcY0MGCElGLS2/fkQugAFbV/d649AwdMLa6lvO/MSamH4rwn4zqp1j9+mzIs0zZH3+jNW6J6w6fPO6fb+/Z2NaXdUKbr9XTXe5UjpQkae1NmsBCBkZyZePFWmXjycGX8YbchyEKhqbBHI+1scLtKDrHPi/MASKbo+p0Qi5CIS2JcYpKmpcovbnrGQrVbLWiBqyEmCHwShbS1FjM1lfEd6ftQVLNQyuUYWzacbJuj1/ofWq1MqtJTUjEUZeqD5eWzgMtt8Sd0Kd52tMLRbQ5/H9np1nmNh1260IJTIb15jtkWUzYXJ+FOo2+JwVIxAjdpx9v2ZsT1neAD5SDIc21gG3dcYeulPyM4x2ENfNghE+oFFTs9IsjkQW34RYh87+7O/iraSwBAop5JjLwlgCaeoQM0/8yOUzAICVJeO472xvAQD2uqbV94gA29s1XP+QlsAoOXg+29I9DLX4cZTVxgecWAAA3U3cf5ldLhkNv3nDOPSGw6wvpxSs0GqNutqSlWbdalSBply7/G1i4SvqtCMNriSAcOzTHI6A/1GOWjUX6sgUYgDhHmfZiFnvAv425PGz8QN1YJqy91dZqovap8oIfjk2/30ytol76ao9k1dumWa7/MhlAMDyCuc9MWbhESPWIw9sMH+/Rp7D9SW71hLndJV4kWoty6acDLPzItaoAa0x35nyd8qri3GaVgkNqO6AbNV9j2X2YZjypKrQG9NKiXKOt553zBfMp4rv6NmLl0IZHsaHZMWQVzFgQxgf2tm175crY5RpfUSQ9Zptf66t0Indns397h5buiuAg8ORg4UlUEghx1wWxBJwcIiw3LAV7dyardhbty0Xf3PjJgBg84ZpmRF9nDH9dFVVRfGM6Ubcb4ocxyFXroajzDBQpdeplc+fOckx2e+uXzfUWbcnFqNsS2rh22P29quT4bZciYM20LmkRdTXLnSIibM8A8LGy9cPPAuRLABVFXK/igSU63chNG1zEBCGIVk900g6V+DDV/Ra8wZ+5jmFQhsSm+GyGIUEjGorz83moo8+cMHGfl0VoJZ16XeBG7fNEri/xR4IVpaBFfISNht6TZPM2CqsLSmpgIHzm3D+RiPVGmRrBZrEjiRkpFomsrPTGyFWpqbJuR4L+UcmqZwfrg7kLhrxGmLF0rPlPMt6kxWjlzbUYNi9D8iTqNjWyRoQCQBAjosJn1FJ/BS85oAMRJ0OsQY084SOPUruagk4537bOXfLOffcgX1rzrk/cM5d4XaV+51z7jeccy87537onPvw3c5fSCGFzFd+HEvgdwD8SwC/e2DfFwE86b3/VefcF/n5VwB8GsBD/PfTAL7C7V3FAVhi9HfYM43/4htmAezvmfYYERmm1Kt6+S0TC9Bu19BUrYCYWMmaq9y8qsTSKXsUMufabqkq0IgCtm+ZFaIFuxT6IUqbc+CKPXDlrinZ7eIDmQfx9NmaK+bjUqAOYtaDakXaZ9ZjUGhEOzrgASKh9Wz/AT1/8LIzFNtBQyDnTx5FXCgtoXOUhHlgjED8C1GJfv1pYxF+/KGH7Z6J6uyTj2HEFucTcgYMenUM2DXqg2ftmQgxWK5pMII3RpkxyMpTBWNM7VsmVqPKh9RnSajiEIopVKrKxtj3p1aXcJt+dFnMUqFDFfEVTnEaG5Ly+urSHOI5HHpV/js/q6ZAXbAjtUCPhBjknyQtj0Y8CTGh0Oo+8BJS05NXo8M57rPeQ3UwScImEUfIXS0B7/33AGzndn8WwNf4/68B+EsH9v+uN/k+gBXn3Nm7XaOQQgqZn9xrTOC0934DALz3G865U9x/DsC1A8dd576N/Amcc18A8AVA1VA+VP5tk66nt2NaWVpnZdVqB5o1VYtxy47AlVIUtKTQZZM0q+li+qplItYGxAXcZG/7HivWpMSFcJvSv+oNs/z8sfoZsL5ctQg+dSFqHVHjiF9AGk157tAhKM3iz/X7NM9cS5nVGIi9hpiFYHHQfz1QvK7/Cek2cYo38Np6I3x2LIG5VrcQ4hSMnrPSU92fvTM2Yce6+JuvWf+BSmLPVLgEpFVUnGnfM+QSFKpTdfCeg1JOfsZyJIuJ2nogDgAbU69rYxkOFWuhxUA/vUS8gTAP5SjBCTIf7+0ob6+ZE4pT8Rv548SXuCyqT5iPQJvAMSecpxEtKHXTUppmv0c8CxGWsUsxpcU3Uddmzof6bnb5znY66pyt94vWhspN77QV7ftD9967HMZ0deiVvfdf9d4/5r1/rPSTHkUhhRTyY8u9WgI3nXNnaQWcBXCL+68DuHDguPMA3vzxTulDhFQqcH3dNH+V/vgqO92uLJMBmKvwcGAxg3EyDnhpofHUrUbsK2OCCbY3zdrYZxdhLVXqKNRqC79usrXd49CIsJMfLlQf3dbhQL7hMNSp+6ZYkU0Un1A9fGDPzbHORMHaUA46m0UISYDAF0CNx3hIGufXZHcH4/F4rPx/Nhc9yyRkKxMV7Z6qbwOBEmNF0fv2KqS7t8M1AYS+jBUi8ir02/ujEtpVIkVbunHw3PS/Z0UTB08J55XpYXxC1o0aITPH3++px6HtV2/CUsCGcE4mIzRpFfhlG9/tLcti1Gi6lDSn/O04FWKUmQgyUAlBKm1dIxKwzwi+LEjH7YQ1C11aM2vtWe5/zAIPxwrHiJaNYgAdMi8xNBA6LcnwXFG6Bfs4TO5VB38HwBP8/xMAvn1g/y8yS/A4gD25DYUUUshiyl0tAefc1wF8HMAJ59x1AF8G8KsA/oNz7vMAXgfwV3n4fwHwGQAvA+gD+Ns/3jA8gAlirqqNZeswcx+1ccroZq9rfmY0sZV8mTX/JeZ5py6CU786dQvmwt3v2yo44kqruvmzp2ltiLeQWiKhVtnZNQugShbhVWqCP0U2l5u3iSTTnUgDDMYQqcw08NYpKk3fjmOVBaS6BgEHQ1fjEOfgNQKzrfxHchmGrIF9K0MhZoS65IEoUs44i/2PqD0nUsayyg70dACApTKrJJesp2DK9EkS8Pg2v51926onpFLVqlpMqK5fu9EFW0Qo+B0wHELfBUxDKElULEQjpKbkDnUYWl4S74T88ax142RRKQ2fulkPRj6jJs+xuW3v4CqzTyVn95VGWYtKTEIJK1xrI/WCoDXCm2i3GWxgVeIkZSZjSK6Dkxxz14NwhzA/t7Y73O7zvhmXYv2GOisvr5oFUAksy4fLXRcB7/3njvjqk4cc6wH80t3OeedFHJDGSEjs0WNZ75CQzBU+CMFsO3sWXEp2bRIaCvCUohAk6jEFpQCY2pI1CQKp1w3EUgtQXaXpWCjCsays2UOPmTKSu1FhO7LnXrDEyfU3WXIcwDUz81PEoipT1kJV5kLUYIpzKVLKisASJ5CRgnQhYWfnFVGmAl50efSqJzST1eAzjuNQhisizREXrXiapcae5BpzhJRiKgATL8I/gmpNBVp01fhMxtv2Ug936LIJ6kuY8WZniAtns8U1I/5Bce1CPFVhEIuXBP1W05BcmlNLZDDhuSiInCRJdK8inZ0FPwNxCf9oG/wDqtbVot2+P71C+nOmBgXhdgGXkwvWTtWERUAxW0QUIBwktl/kLRFfksHUY6j6MKATIaEAAA3sSURBVKb+Nm/b+x+4QuiSqfhrqW3vdp1uqGjbjpIiJFdIIcdcFgY2DMQhhbO3a1q8RI0wPalGlLaytWnmjLmyT7l6j/o9jKnJQ+yN6ZRZxbAgpGzhJKrsANWlRUCtW6lmCT8CQpXaqNk0DcFMI8rqcBXf2aY6gH+4so/oJpRIeJkyHUnkaCg3TXgz5ThbxBPII0KjjOzYBQGOWZgTlaLgWsSCLItiXOXKwXqRGZslyRAUd8Dy2zRHmhHTx1HrtdNnzbU7sWZu1y7dq+3OrKR4fZmAGp5DhUABoKWGrmoQ47OWQCisChZPtqQ6tAyLgo2UuRcFVJ2LUK3SUhKZJ6219RXbv0GXYrdvY2gzgFimNlZJtYq/hrxWSEvyPRrSIoo475t7tABiFS7RGk7SYBmlPpsCDG3OeVdLbBe/smLvpIraQrOWI6SwBAop5JjLQlgCznlE0TSsutK61bJZAPWS+d/pREEnCxB2uXomA2qzSYopI1uhUUNqK2qDPmukVJjSbxpDlqV75uPxACF8RR6hVNlwbP6XFlvFnsoeR2EzAtBI5c2pGpzQQggKK9ckI+UXSlNJaygYKStGvrUjAGN6kKA0EuyZqSlaCWmS9fWnoXxZYCClIampNE+pLIgsWKZUMc1Yo3XTZIrs/H0WUIwbtn9t4xZOtkWSQUtGUN1xZhpCbGU6yVofavxRDu9PNq0pSrAAiKLqU+p1cqDARjGTmIHRfiLiWvv+5Kp9/8ZNG4ta0YuWTQHUcSgDt9/VGy4zdhpScEwNbu/Zdc4TOi1S0XQaBSLddCLiVT5vzkuTbdeWWXLN2C0ipWUZpzlKCkugkEKOuSyEJRBFQKPh8MjD5wAAH3yPaYvK2CKomzeuAwB2d1hK3GfElVp/qthBejAWYN9duGD0YO22rewjkpL2SFCS5lbsQA0dKL5Y+snVd8BIfb0hP81nficXOY0BKq5gEEjzpDlIrq6trIgXzFgpP5kh8o0VwafaFkBpyPSncoNNYV6XCKuuxiiHVuOizmZUWhDmEMcQRFeNS5mG4lxLe+repoHA1MY45HE9UrsPaBk0mNZNphbXOblWxTIzNmMV5UBjyGYF0kCwqqYhtD4mtByZUnRKiVLrulymo6IyX87vSACo6TSAzGpUp57fKbXcJLCpSX+7Qyt0nf64sgqykEqVgGzivSj7YMfv79v3vaE9+3rNtHaAkCMOQSS9731es8rU6sqKWaOigS+VFSNimpZpx6OksAQKKeSYy0JYAsuNKj7xkUv4yMNG6NG5+ToAYPum0YolXMnGI+V17Xfyv6RhYxehQZDQA+etpmmNDSYmE8utThNbaSfUjsr7zxpR2krdH4lMwr5vkLZ6dZ0NPFs21teumXVy/oIty31aGJNR4EcNPvo4BBpsIxjwdMYTZr8VeYii3NTGagOuFllD5voF0BmoTJcGQadKGvWTNraVlWpopiGykJIqVwUwCvGFgMwBMIPWtqgBKxORtFQzx8kikD8/ZG57QItga2jz1SVd12orCrRyIac+yWYBIsYvpMGJqcKI8NmhqMoYVJlSy05o1tRFmsqbjZ0sA86BiD+8D1mRWRk2x6JW4aQgXycZ7vVb1LKe75PP0pMpzuVilvfymSeRnWejY/vPnKbFVVF2y45r1mYNYHYIwBJo6Ox9lnE5cXKZgyXmgFZLn1uVph8lhSVQSCHHXBbCEihFU6w1urj6kpXz9rdNWziqpxUioNotahk2GBGSrlE3X2i52Qz+UUTPstcTOQi5sKdZH155cJGEKDpcE3UVtcdSQ6hFW8FfuUayDMK53v8Bi2P0OqYhNm/sY2tPzU8zlw5a0g1UjsuctCpaVIySUKtQ645Zdhpy0FzpB9SE41A4Y9vBgH6674briz6rUVMOnZkJatcQ1Q4+PotwBqbxGnxlBLuQFhZVdpVatUG02hoj1tMRcQIdDY6aseJD2/IKVLYt/AfHxImrKGtEym3FNzzsWXSJEh13aNXRutM91VlyXheYY9YbDIChQpVhUWn0ONe6fjjRs7J3TvTwg4nKnoneDEhUxY7Yzo3dQ2/uyJKymzx/klZKoKIjeWx9Rp8Ws3XrA5cIl2eDneFgj/eZbWAii6dC6xg4nHq8sAQKKeSYy0JYAmmaYtDror9tGmvcZfRTraCqtsK1T7CUuEUfkVqsweYS9WoJaWoaK2EWQEUiQWu4bGRZUVsfik9IWdagZqtm21DtUru/cMXiFa0TdlyLJJkn2oaQW2lX0LpFdNyOjanTYzssttgek/VpHHMsbMwxpAYvB1+VxU0sapGGlO8nzZ/rSB7qAHZ3uT/to8z4QoXNXnW/CednfKCNOXDAcqA2GZOoIvjM1CMjkqrM5tXuWfUOJUdLilbbCnPY/fEgIPaGiVB32eawAefHdH4p0IrZ9yL1TFhYpuay/a4ajXJ+lC1gW6+SSEUjbYFJ7v4j+vhVlvzudBjzEdkKRITDtndrfCf5vYp/prD52+5wrIznfOARe3+aFfUjUwbIfhjXaiHO0m6LvFTl23yB+OCrxEuU6qqlUZZJaBi+eDkpLIFCCjnmshCWALyHS6eoMPJab4kuTNhurXzERrfM72+SmFRNPQaDfUypTaRF61XTPCKddFFIhPPc08zngAyUTyeCSOa9B6TadvQJN3fZlowEDydajMSvVgN2e5VY7i3WROzt2n3dZhkyQrsx4R5s97Ar1U5sO+dFCEGVKgfXNrcVBwf7cGJnmiJyHc6P7VtfI+kK/U5ZJbNyZZ7LqWzZ7kl+p8haOiRo1X4NpVphI9immFmpjaemlRoVF6od9SNpsECtPhE2geemxlO1pWjpVGWqMahqU1WDTjUH3NaqQtQxCxNHMyquHG5EGSjdr1p7ldiubItt6c6csfmscR7jsX2+fjtbnfreh+0dbjVlKmQJRmVhjccOPVqQa2zUusTMFmhdif48zhHPOpd/MbZwmBSWQCGFHHNZCEvAOaBUSnGG6L61lq1wjQqr7DjMsgrxvWnQKf1/ETGMehNMRNwhembmykX11aB2VmWZ1GVg/4aou0Q1rdZOrDqcCjFoGv+lK8ZpoBrvNhFftXIdDbY9r1W46q8KS5ByLJa52NiwcwRqaSc6tQbHJMJMtTOTtsZbSqAp41I/ToC9PSLe6qaJ68wSLNcVAyFCMtQSqLHFkJ9VbZlmvpfMqi2JJ6AlIPISNRmd8vwRokDDPasKVH2/qNCymQplD+IcHZseabWm2gHGFHJjlWUxFT4h1NunB+ZUY9BH+0L09uIBECZD+AvRhAn7sL1JNGdiv3vfIyTHLdv8D1hNWGb9SJ3niWjd7HUT3Nzs8tykq/M2t0ssaKgSOqiMRKCrU0GMe+sXpbAECinkmIvzd1Mn78YgnNuEhS5v3+3YOckJFGO7F1nUsS3quIB3dmwPeO9P5ncuxCIAAM65Z7z3j817HIdJMbZ7k0Ud26KOC5jP2Ap3oJBCjrkUi0AhhRxzWaRF4KvzHsBbSDG2e5NFHduijguYw9gWJiZQSCGFzEcWyRIopJBC5iDFIlBIIcdcFmIRcM79vHPuRefcy865L85xHBecc0855553zv3IOffL3L/mnPsD59wVblfnOMbYOfeHzrnv8vMl59zTHNu/d85V7naOd2hcK865bzrnXuD8fXRR5s059/f4PJ9zzn3dOVeb17w5537bOXfLOffcgX2HzhN7ev4G/y5+6Jz78DsxprkvAs5YPX4TwKcBvA/A55xz75vTcCYA/oH3/r0AHgfwSxzLFwE86b1/CMCT/Dwv+WUAzx/4/E8B/HOObQfA5+cyKuBfAPiv3vtHAXwINsa5z5tz7hyAvwPgMe/9B2As838N85u33wHw87l9R83TpwE8xH9fAPCVd2RE3vu5/gPwUQD/7cDnLwH40rzHxbF8G8CfB/AigLPcdxbAi3Maz3m+JJ8A8F0YwP02gNJhc/kujqsF4CoYaD6wf+7zBuAcgGsA1mC1Mt8F8HPznDcAFwE8d7d5AvCvAHzusON+kv/mbglg9pAk17lvruKcuwjgpwA8DeC0Z4t1bk/NaVi/DuAfYtZweB3ArvdqbDa3ubsMYBPAv6Gr8q+dc00swLx5798A8Guw7tkbAPYAPIvFmDfJUfP0rvxtLMIi4A7ZN9e8pXNuCcB/BPB3vfedeY5F4pz7CwBuee+fPbj7kEPnMXclAB8G8BXv/U/B6kDm6TIFoX/9WQCXANwHoAkzs/OyiLnyd+X5LsIicB3AhQOfzwN4c05jgXOuDFsA/p33/lvcfdM5d5bfnwVwaw5D+xkAf9E59yqA34O5BL8OYMWp6+r85u46gOve+6f5+ZuwRWER5u1TAK567ze9sdN8C8DHsBjzJjlqnt6Vv41FWAT+N4CHGK2twII235nHQJwVsv8WgOe99//swFffAfAE//8ELFbwror3/kve+/Pe+4uwOfrv3vu/AeApAL8w57HdAHDNOfcId30SwB9jAeYN5gY87pxr8PlqbHOftwNy1Dx9B8AvMkvwOIA9uQ0/UXm3AzVHBEo+A+AlAP8PwD+a4zh+FmZu/RDAD/jvMzDf+0kAV7hdm/N8fRzAd/n/ywD+F4CXAXwDQHVOY/rTAJ7h3P1nAKuLMm8A/gmAFwA8B+DfAqjOa94AfB0WmxjDNP3nj5onmDvwm/y7+CNYhuMnPqYCNlxIIcdcFsEdKKSQQuYoxSJQSCHHXIpFoJBCjrkUi0AhhRxzKRaBQgo55lIsAoUUcsylWAQKKeSYy/8Hya92lmOAgi8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import cv2\n",
    "import random\n",
    "image,_ = train_data[3900]\n",
    "def im_plot(tensor):\n",
    "    image = tensor.cpu().numpy().transpose(1,2,0)\n",
    "    b,g,r = cv2.split(image)\n",
    "    image = cv2.merge((r,g,b))\n",
    "    image = image*[0.22803, 0.22145, 0.216989] +  [0.43216, 0.394666, 0.37645]\n",
    "    image = image*255.0\n",
    "    plt.imshow(image.astype(int))\n",
    "    plt.show()\n",
    "print(image.shape)\n",
    "im_plot(image[10,:,:,:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_lr(optimizer, lr):\n",
    "    for g in optimizer.param_groups:\n",
    "        g['lr'] = lr\n",
    "def update_mom(optimizer, mom):\n",
    "    for g in optimizer.param_groups:\n",
    "        g['momentum'] = mom"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from Modular_code.lr_finder import *\n",
    "from RAdam.radam import RAdam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lr_finder(model,train_loader):\n",
    "    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "    criterion = nn.CrossEntropyLoss()\n",
    "    optimizer_ft = RAdam(model.parameters(), lr=1e-6)\n",
    "    lr_finder = LRFinder(model, optimizer_ft, criterion, device=device)\n",
    "    lr_finder.range_test(train_loader, end_lr=1, num_iter=1000)\n",
    "    lr_finder.reset()\n",
    "    lr_finder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!git clone https://github.com/ndrplz/ConvLSTM_pytorch\n",
    "from ConvLSTM_pytorch.convlstm import ConvLSTM\n",
    "size = 3\n",
    "\n",
    "class Model(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Model, self).__init__()\n",
    "        model = models.densenet121(pretrained = True).to('cuda')\n",
    "        modules= list(model.children())[0]\n",
    "        model = nn.Sequential(*modules)\n",
    "        self.model = model\n",
    "        self.c1 = ConvLSTM(  input_size=(size,size),\n",
    "                             input_dim= 1024,\n",
    "                             hidden_dim=[128, 512],\n",
    "                             kernel_size=(5,5),\n",
    "                             num_layers=2,\n",
    "                             batch_first = True,\n",
    "                             bias=True,\n",
    "                             return_all_layers=False)\n",
    "        self.fc1 = nn.Linear(512*size*size,1000)\n",
    "        self.relu = nn.ReLU()\n",
    "        self.fc2 = nn.Linear(1000,256)\n",
    "        self.fc3 = nn.Linear(256,4)\n",
    "        self.dp = nn.Dropout(0.2)\n",
    "        self.bn1 = nn.BatchNorm1d(1000)\n",
    "        self.bn2 = nn.BatchNorm1d(256)\n",
    "        self.bn3 = nn.BatchNorm1d(10)\n",
    "    def forward(self, x):\n",
    "        x = self.model(x)\n",
    "        x = x.view(-1,150,1024,size,size).detach()\n",
    "        x = self.c1(x)\n",
    "        x = x[0][0][:,-1,:,:,:]\n",
    "        x = x.view(-1,512*size*size)\n",
    "        x = self.dp(self.relu(self.fc1(x)))\n",
    "        x = self.dp(self.relu(self.fc2(x)))\n",
    "        x = self.dp(self.fc3(x))\n",
    "        return x    \n",
    "model = Model().to('cuda')\n",
    "model.load_state_dict(torch.load('./weights_att1/convlstmdense.h5')['dict'])\n",
    "model = nn.DataParallel(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "class AverageMeter(object):\n",
    "    \"\"\"Computes and stores the average and current value\"\"\"\n",
    "    def __init__(self):\n",
    "        self.reset()\n",
    "    def reset(self):\n",
    "        self.val = 0\n",
    "        self.avg = 0\n",
    "        self.sum = 0\n",
    "        self.count = 0\n",
    "\n",
    "    def update(self, val, n=1):\n",
    "        self.val = val\n",
    "        self.sum += val * n\n",
    "        self.count += n\n",
    "        self.avg = self.sum / self.count\n",
    "\n",
    "\n",
    "class Logger(object):\n",
    "    def __init__(self, path, header):\n",
    "        self.log_file = open(path, 'w')\n",
    "        self.logger = csv.writer(self.log_file, delimiter='\\t')\n",
    "        self.logger.writerow(header)\n",
    "        self.header = header\n",
    "\n",
    "    def __del(self):\n",
    "        self.log_file.close()\n",
    "\n",
    "    def log(self, values):\n",
    "        write_values = []\n",
    "        for col in self.header:\n",
    "            assert col in values\n",
    "            write_values.append(values[col])\n",
    "\n",
    "        self.logger.writerow(write_values)\n",
    "        self.log_file.flush()\n",
    "\n",
    "\n",
    "def load_value_file(file_path):\n",
    "    with open(file_path, 'r') as input_file:\n",
    "        value = float(input_file.read().rstrip('\\n\\r'))\n",
    "    return value\n",
    "\n",
    "\n",
    "def calculate_accuracy(outputs, targets):\n",
    "    batch_size = targets.size(0)\n",
    "\n",
    "    _, pred = outputs.topk(1, 1, True)\n",
    "    pred = pred.t()\n",
    "    correct = pred.eq(targets.view(1, -1))\n",
    "    n_correct_elems = correct.float().sum().item()\n",
    "    return 100* n_correct_elems / batch_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.autograd import Variable\n",
    "import time\n",
    "import os\n",
    "import sys\n",
    "\n",
    "def train_epoch(epoch, data_loader, model, criterion, optimizer, epoch_logger, batch_logger):\n",
    "    print('train at epoch {}'.format(epoch))\n",
    "    model.train()\n",
    "    batch_time = AverageMeter()\n",
    "    data_time = AverageMeter()\n",
    "    losses = AverageMeter()\n",
    "    accuracies = AverageMeter()\n",
    "    end_time = time.time()\n",
    "    for i, (inputs, targets) in enumerate(data_loader):\n",
    "        data_time.update(time.time() - end_time)\n",
    "        if(inputs.size(0)<8):\n",
    "            continue\n",
    "\n",
    "        if torch.cuda.is_available():\n",
    "            targets = targets.cuda()\n",
    "                \n",
    "        inputs = Variable(inputs)\n",
    "        with torch.no_grad():\n",
    "            inputs = inputs.reshape(-1,3,im_size,im_size)\n",
    "        \n",
    "        targets = Variable(targets)\n",
    "        outputs = model(inputs)\n",
    "        loss = criterion(outputs, targets)\n",
    "        acc = calculate_accuracy(outputs, targets)\n",
    "        losses.update(loss.item(), inputs.size(0))\n",
    "        accuracies.update(acc, inputs.size(0))\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        batch_time.update(time.time() - end_time)\n",
    "        end_time = time.time()\n",
    "\n",
    "        batch_logger.log({\n",
    "            'epoch': epoch,\n",
    "            'batch': i + 1,\n",
    "            'iter': (epoch - 1) * len(data_loader) + (i + 1),\n",
    "            'loss': losses.val,\n",
    "            'acc': accuracies.val,\n",
    "            'lr': optimizer.param_groups[0]['lr']\n",
    "        })\n",
    "        sys.stdout.write(\n",
    "                \"\\r[Epoch %d/%d] [Batch %d / %d] [Time %.2f %.2f] [Data %.2f %.2f] [Loss: %f, Acc: %.2f%%]\"\n",
    "                % (\n",
    "                    epoch,\n",
    "                    num_epochs,\n",
    "                    i,\n",
    "                    len(data_loader),\n",
    "                    batch_time.val,\n",
    "                    batch_time.avg,\n",
    "                    data_time.val,\n",
    "                    data_time.avg,\n",
    "                    losses.avg,\n",
    "                    accuracies.avg\n",
    "                    )\n",
    "                )\n",
    "    epoch_logger.log({\n",
    "        'epoch': epoch,\n",
    "        'loss': losses.avg,\n",
    "        'acc': accuracies.avg,\n",
    "        'lr': optimizer.param_groups[0]['lr']\n",
    "    })\n",
    "\n",
    "    if epoch % checkpoint == 0:\n",
    "        save_file_path = os.path.join(result_path,\n",
    "                                      'save_{}.pth'.format(epoch))\n",
    "        states = {\n",
    "            'epoch': epoch + 1,\n",
    "            'arch': arch,\n",
    "            'state_dict': model.state_dict(),\n",
    "            'optimizer': optimizer.state_dict(),\n",
    "        }\n",
    "        torch.save(states, save_file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "def val_epoch(epoch, data_loader, model, criterion,  logger):\n",
    "    print('validation at epoch {}'.format(epoch))\n",
    "\n",
    "    model.eval()\n",
    "    batch_time = AverageMeter()\n",
    "    data_time = AverageMeter()\n",
    "    losses = AverageMeter()\n",
    "    accuracies = AverageMeter()\n",
    "\n",
    "    end_time = time.time()\n",
    "    for i, (inputs, targets) in enumerate(data_loader):\n",
    "        if(inputs.size(0)<8):\n",
    "            continue\n",
    "        data_time.update(time.time() - end_time)\n",
    "        if torch.cuda.is_available():\n",
    "            targets = targets.cuda()\n",
    "        inputs = Variable(inputs)\n",
    "        with torch.no_grad():\n",
    "            inputs = inputs.reshape(-1,3,im_size,im_size)\n",
    "        targets = Variable(targets)\n",
    "        outputs = model(inputs)\n",
    "        loss = criterion(outputs, targets)\n",
    "        acc = calculate_accuracy(outputs, targets)\n",
    "\n",
    "        losses.update(loss.item(), inputs.size(0))\n",
    "        accuracies.update(acc, inputs.size(0))\n",
    "\n",
    "        batch_time.update(time.time() - end_time)\n",
    "        end_time = time.time()\n",
    "        sys.stdout.write(\n",
    "                \"\\r[Epoch %d/%d] [Batch %d / %d] [Time %.2f %.2f] [Data %.2f %.2f] [Loss: %f, Acc: %.2f%%]\"\n",
    "                % (\n",
    "                    epoch,\n",
    "                    num_epochs,\n",
    "                    i,\n",
    "                    len(data_loader),\n",
    "                    batch_time.val,\n",
    "                    batch_time.avg,\n",
    "                    data_time.val,\n",
    "                    data_time.avg,\n",
    "                    losses.avg,\n",
    "                    accuracies.avg\n",
    "                    )\n",
    "                )\n",
    "        \n",
    "    logger.log({'epoch': epoch, 'loss': losses.avg, 'acc': accuracies.avg})\n",
    "    return losses.avg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mkdir: cannot create directory ‘results’: File exists\r\n"
     ]
    }
   ],
   "source": [
    "num_epochs = 10\n",
    "result_path = './results'\n",
    "arch = 'dense121-150'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_logger = Logger(\n",
    "            os.path.join(result_path, 'train{}.log'.format(arch)),\n",
    "            ['epoch', 'loss', 'acc', 'lr'])\n",
    "train_batch_logger = Logger(\n",
    "            os.path.join(result_path, 'train_batch{}.log'.format(arch)),\n",
    "            ['epoch', 'batch', 'iter', 'loss', 'acc', 'lr'])    \n",
    "val_logger = Logger(\n",
    "            os.path.join(result_path, 'val{}.log'.format(arch)), ['epoch', 'loss', 'acc'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train at epoch 0\n",
      "[Epoch 0/10] [Batch 51 / 670] [Time 2.88 3.02] [Data 0.02 0.15] [Loss: 0.683905, Acc: 62.74%]"
     ]
    }
   ],
   "source": [
    "optimizer = RAdam(model.parameters(), lr=1e-3,weight_decay = 1e-3)\n",
    "train_loader = DataLoader(train_data,batch_size = 8,num_workers = 12,shuffle = True) \n",
    "valid_loader = DataLoader(val_data,batch_size = 8,num_workers = 12)\n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "    train_epoch(epoch,train_loader, model, criterion, optimizer, train_logger, train_batch_logger)\n",
    "    val_epoch(epoch, valid_loader, model, criterion, val_logger)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "from efficientnet_pytorch import EfficientNet\n",
    "from ConvLSTM_pytorch.convlstm import ConvLSTM\n",
    "size = 7\n",
    "class Model(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Model, self).__init__()\n",
    "        model = models.vgg16_bn(pretrained = True)\n",
    "        model = nn.Sequential(*[model.features[i] for i in range(43)])\n",
    "        for param in model.parameters():\n",
    "            param.requires_grad = False\n",
    "        self.model = model\n",
    "        self.c1 = ConvLSTM(  input_size=(size,size),\n",
    "                             input_dim=512,\n",
    "                             hidden_dim=[128, 51],\n",
    "                             kernel_size=(5,5),\n",
    "                             num_layers=2,\n",
    "                             batch_first = True,\n",
    "                             bias=True,\n",
    "                             return_all_layers = False)\n",
    "        self.fc1 = nn.Linear(512*size*size,1000)\n",
    "        self.relu = nn.ReLU()\n",
    "        self.fc2 = nn.Linear(1000,256)\n",
    "        self.fc3 = nn.Linear(256,4)\n",
    "        self.dp = nn.Dropout(0.2)\n",
    "    def forward(self, x):\n",
    "        x = self.model(x)\n",
    "        x = x.view(-1,150,512,size,size).detach()\n",
    "        x = self.c1(x)\n",
    "        x = x[0][0][:,-1,:,:,:]\n",
    "        x = x.view(-1,512*size*size)\n",
    "        x = self.dp(self.relu(self.fc1(x)))\n",
    "        x = self.dp(self.relu(self.fc2(x)))\n",
    "        x = self.dp(self.fc3(x))\n",
    "        return x\n",
    "model = Model().to('cuda')\n",
    "model = nn.DataParallel(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#regression\n",
    "os.makedirs('./weights_att1',exist_ok = True)\n",
    "import sys\n",
    "import cv2\n",
    "from torch.autograd import Variable\n",
    "device = 'cuda'\n",
    "criterion = nn.MSELoss().to(device)\n",
    "def update_lr(optimizer, lr):\n",
    "    for g in optimizer.param_groups:\n",
    "        g['lr'] = lr\n",
    "def update_mom(optimizer, mom):\n",
    "    for g in optimizer.param_groups:\n",
    "        g['momentum'] = mom\n",
    "\n",
    "def train(model,dataloaders,device,num_epochs,lr,batch_size,patience):\n",
    "    i = 0\n",
    "    phase1 = dataloaders.keys()\n",
    "    losses = list()\n",
    "    acc_all = list()\n",
    "    train_loader = dataloaders['train']\n",
    "    iteration = 0\n",
    "    iteration1 = 0\n",
    "    onecyc = OneCycle(len(train_loader)*num_epochs,lr)\n",
    "    optimizer = RAdam(model.parameters(), lr=lr)\n",
    "    for epoch in range(num_epochs):\n",
    "        print('------------------Epoch: {} ------------------'.format(epoch))\n",
    "        if(epoch%20==0 and epoch!=0):\n",
    "            lr = 0.9*lr\n",
    "            num_epochs = num_epochs-20\n",
    "            onecyc = OneCycle(len(train_loader)*num_epochs,lr)\n",
    "            optimizer = RAdam(model.parameters(), lr=lr)\n",
    "        \n",
    "        epoch_metrics = {\"loss\": [], \"acc\": []}\n",
    "        for phase in phase1:\n",
    "            print('')\n",
    "            print('-------------------{}------------------'.format(phase))\n",
    "            if(phase  == 'train'):\n",
    "                model = model.train()\n",
    "            else:\n",
    "                model = model.eval()\n",
    "            for  batch_idx, (data, target) in enumerate(dataloaders[phase]):\n",
    "                bs = data.size(0)\n",
    "                data, target = Variable(data), Variable(target)\n",
    "                data = data.type(torch.FloatTensor).to(device)\n",
    "                target = target.type(torch.FloatTensor).to(device)\n",
    "                optimizer.zero_grad()\n",
    "                output = model(data)\n",
    "                del data\n",
    "                loss = criterion(output.squeeze(), target)\n",
    "                #acc = 100 * (output.detach().argmax(1) == target).cpu().numpy().mean()\n",
    "                del output,target\n",
    "                epoch_metrics[\"loss\"].append(loss.item())\n",
    "                #epoch_metrics[\"acc\"].append(acc)\n",
    "                sys.stdout.write(\n",
    "                \"\\r[Epoch %d/%d] [Batch %d/%d] [Loss: %f (%f)]\"\n",
    "                    #, Acc: %.2f%% (%.2f%%)\n",
    "                % (\n",
    "                    epoch,\n",
    "                    num_epochs,\n",
    "                    batch_idx,\n",
    "                    len(dataloaders[phase]),\n",
    "                    loss.item(),\n",
    "                    np.mean(epoch_metrics[\"loss\"])#,\n",
    "                    #acc,\n",
    "                   # np.mean(epoch_metrics[\"acc\"]),\n",
    "                    )\n",
    "                )\n",
    "\n",
    "                if(phase =='train'):\n",
    "                    loss.backward()\n",
    "                    optimizer.step()\n",
    "                    lr,_ = onecyc.calc()\n",
    "                    update_lr(optimizer, lr)\n",
    "                    iteration = iteration+1\n",
    "                    #writer.add_scalar('data/acc',np.mean(epoch_metrics[\"acc\"]) , iteration)\n",
    "                    writer.add_scalar('data/loss',np.mean(epoch_metrics[\"loss\"]) , iteration)\n",
    "                if(phase=='val'):\n",
    "                    iteration1 = iteration1+1\n",
    "                    #writer.add_scalar('data/valacc',np.mean(epoch_metrics[\"acc\"]) , iteration1)\n",
    "                    writer.add_scalar('data/valloss',np.mean(epoch_metrics[\"loss\"]) , iteration1)\n",
    "                if torch.cuda.is_available():\n",
    "                    torch.cuda.empty_cache()\n",
    "         #   epoch_acc = np.mean(epoch_metrics[\"acc\"])\n",
    "            epoch_loss = np.mean(epoch_metrics[\"loss\"])\n",
    "            if(phase == 'train'):\n",
    "                losses.append(epoch_loss)\n",
    "                #acc_all.append(epoch_acc)\n",
    "                #writer.add_scalar('data/accep',np.mean(epoch_metrics[\"acc\"]) , epoch)\n",
    "                writer.add_scalar('data/lossep',np.mean(epoch_metrics[\"loss\"]) , epoch)\n",
    "                save = {\n",
    "                    'model':'101-b0',\n",
    "                    'epoch':epoch,\n",
    "                    'train_loss':epoch_loss,\n",
    "                    #'train_acc':epoch_acc,\n",
    "                    'dict':model.module.state_dict(),\n",
    "                    'lr':lr\n",
    "                }\n",
    "                torch.save(save,'./weights_att1/slow_fast101-b0-1.h5')\n",
    "            print('')\n",
    "          #  print('{} Accuracy: {}'.format(phase,epoch_acc.item()))\n",
    "    return losses,acc_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!git clone https://github.com/ndrplz/ConvLSTM_pytorch\n",
    "from ConvLSTM_pytorch.convlstm import ConvLSTM\n",
    "size = 3\n",
    "class Model(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Model, self).__init__()\n",
    "        model = models.densenet121(pretrained = True).to('cuda')\n",
    "        modules= list(model.children())[0]\n",
    "        model = nn.Sequential(*modules)\n",
    "        self.model = model\n",
    "        self.c1 = ConvLSTM(  input_size=(size,size),\n",
    "                             input_dim= 1024,\n",
    "                             hidden_dim=[128, 512],\n",
    "                             kernel_size=(5,5),\n",
    "                             num_layers=2,\n",
    "                             batch_first = True,\n",
    "                             bias=True,\n",
    "                             return_all_layers=False)\n",
    "        self.fc1 = nn.Linear(512*size*size,1000)\n",
    "        self.relu = nn.ReLU()\n",
    "        self.bn1 = nn.BatchNorm1d(1000)\n",
    "        self.bn2 = nn.BatchNorm1d(256)\n",
    "        self.bn3 = nn.BatchNorm1d(10)\n",
    "        self.fc2 = nn.Linear(1000,256)\n",
    "        self.fc3 = nn.Linear(256,4)\n",
    "        self.dp = nn.Dropout(0.2)\n",
    "    def forward(self, x):\n",
    "        x = self.model(x)\n",
    "        x = x.view(-1,150,1024,size,size).detach()\n",
    "        x = self.c1(x)\n",
    "        x = x[0][0][:,-1,:,:,:]\n",
    "        x = x.view(-1,512*size*size)\n",
    "        x = self.dp(self.relu(self.fc1(x)))\n",
    "        x = self.dp(self.relu(self.fc2(x)))\n",
    "        x = self.dp(self.fc3(x))\n",
    "        return x\n",
    "model = Model().to('cuda')\n",
    "model = nn.DataParallel(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "\n",
    "__all__ = ['resnet50', 'resnet101','resnet152', 'resnet200']\n",
    "\n",
    "\n",
    "\n",
    "class Bottleneck(nn.Module):\n",
    "    expansion = 4\n",
    "\n",
    "    def __init__(self, inplanes, planes, stride=1, downsample=None, head_conv=1):\n",
    "        super(Bottleneck, self).__init__()\n",
    "        if head_conv == 1:\n",
    "            self.conv1 = nn.Conv3d(inplanes, planes, kernel_size=1, bias=False)\n",
    "            self.bn1 = nn.BatchNorm3d(planes)\n",
    "        elif head_conv == 3:\n",
    "            self.conv1 = nn.Conv3d(inplanes, planes, kernel_size=(3, 1, 1), bias=False, padding=(1, 0, 0))\n",
    "            self.bn1 = nn.BatchNorm3d(planes)\n",
    "        else:\n",
    "            raise ValueError(\"Unsupported head_conv!\")\n",
    "        self.conv2 = nn.Conv3d(\n",
    "            planes, planes, kernel_size=(1, 3, 3), stride=(1,stride,stride), padding=(0, 1, 1), bias=False)\n",
    "        self.bn2 = nn.BatchNorm3d(planes)\n",
    "        self.conv3 = nn.Conv3d(planes, planes * 4, kernel_size=1, bias=False)\n",
    "        self.bn3 = nn.BatchNorm3d(planes * 4)\n",
    "        self.relu = nn.ReLU(inplace=True)\n",
    "        self.downsample = downsample\n",
    "        self.stride = stride\n",
    "\n",
    "    def forward(self, x):\n",
    "        residual = x\n",
    "\n",
    "        out = self.conv1(x)\n",
    "        out = self.bn1(out)\n",
    "        out = self.relu(out)\n",
    "\n",
    "        out = self.conv2(out)\n",
    "        out = self.bn2(out)\n",
    "        out = self.relu(out)\n",
    "\n",
    "        out = self.conv3(out)\n",
    "        out = self.bn3(out)\n",
    "\n",
    "        if self.downsample is not None:\n",
    "            residual = self.downsample(x)\n",
    "        out += residual\n",
    "        out = self.relu(out)\n",
    "\n",
    "        return out\n",
    "\n",
    "\n",
    "class SlowFast(nn.Module):\n",
    "    def __init__(self, block=Bottleneck, layers=[3, 4, 6, 3], class_num=10, dropout=0.2):\n",
    "        super(SlowFast, self).__init__()\n",
    "\n",
    "        self.fast_inplanes = 8\n",
    "        self.fast_conv1 = nn.Conv3d(3, 8, kernel_size=(5, 7, 7), stride=(1, 2, 2), padding=(2, 3, 3), bias=False)\n",
    "        self.fast_bn1 = nn.BatchNorm3d(8)\n",
    "        self.fast_relu = nn.ReLU(inplace=True)\n",
    "        self.fast_maxpool = nn.MaxPool3d(kernel_size=(1, 3, 3), stride=(1, 2, 2), padding=(0, 1, 1))\n",
    "        self.fast_res2 = self._make_layer_fast(block, 8, layers[0], head_conv=3)\n",
    "        self.fast_res3 = self._make_layer_fast(\n",
    "            block, 16, layers[1], stride=2, head_conv=3)\n",
    "        self.fast_res4 = self._make_layer_fast(\n",
    "            block, 32, layers[2], stride=2, head_conv=3)\n",
    "        self.fast_res5 = self._make_layer_fast(\n",
    "            block, 64, layers[3], stride=2, head_conv=3)\n",
    "        \n",
    "        self.lateral_p1 = nn.Conv3d(8, 8*2, kernel_size=(5, 1, 1), stride=(8, 1 ,1), bias=False, padding=(2, 0, 0))\n",
    "        self.lateral_res2 = nn.Conv3d(32,32*2, kernel_size=(5, 1, 1), stride=(8, 1 ,1), bias=False, padding=(2, 0, 0))\n",
    "        self.lateral_res3 = nn.Conv3d(64,64*2, kernel_size=(5, 1, 1), stride=(8, 1 ,1), bias=False, padding=(2, 0, 0))\n",
    "        self.lateral_res4 = nn.Conv3d(128,128*2, kernel_size=(5, 1, 1), stride=(8, 1 ,1), bias=False, padding=(2, 0, 0))\n",
    "\n",
    "        self.slow_inplanes = 64+64//8*2\n",
    "        self.slow_conv1 = nn.Conv3d(3, 64, kernel_size=(1, 7, 7), stride=(1, 2, 2), padding=(0, 3, 3), bias=False)\n",
    "        self.slow_bn1 = nn.BatchNorm3d(64)\n",
    "        self.slow_relu = nn.ReLU(inplace=True)\n",
    "        self.slow_maxpool = nn.MaxPool3d(kernel_size=(1, 3, 3), stride=(1, 2, 2), padding=(0, 1, 1))\n",
    "        self.slow_res2 = self._make_layer_slow(block, 64, layers[0], head_conv=1)\n",
    "        self.slow_res3 = self._make_layer_slow(\n",
    "            block, 128, layers[1], stride=2, head_conv=1)\n",
    "        self.slow_res4 = self._make_layer_slow(\n",
    "            block, 256, layers[2], stride=2, head_conv=3)\n",
    "        self.slow_res5 = self._make_layer_slow(\n",
    "            block, 512, layers[3], stride=2, head_conv=3)\n",
    "        self.dp = nn.Dropout(0.2)\n",
    "        self.fc = nn.Linear(768, 2, bias=False)\n",
    "        model = models.video.mc3_18(pretrained  = True)\n",
    "        model.fc = Identity()\n",
    "        for param in model.parameters():\n",
    "            param.requires_grad = False\n",
    "        self.SlowPath = model\n",
    "    def forward(self, input):\n",
    "        fast = self.FastPath(input[:, :, :, :, :])\n",
    "        slow = self.SlowPath(input[:, :, ::2, :, :])\n",
    "        x = torch.cat([slow, fast], dim=1)\n",
    "        x = self.fc(x)\n",
    "        return x\n",
    "\n",
    "\n",
    "\n",
    "    def FastPath(self, input):\n",
    "        lateral = []\n",
    "        x = self.fast_conv1(input)\n",
    "        x = self.fast_bn1(x)\n",
    "        x = self.fast_relu(x)\n",
    "        pool1 = self.fast_maxpool(x)\n",
    "        res2 = self.fast_res2(pool1)\n",
    "        \n",
    "        res3 = self.fast_res3(res2)\n",
    "\n",
    "        res4 = self.fast_res4(res3)\n",
    "\n",
    "        res5 = self.fast_res5(res4)\n",
    "        x = nn.AdaptiveAvgPool3d(1)(res5)\n",
    "        x = x.view(-1, x.size(1))\n",
    "\n",
    "        return x\n",
    "\n",
    "    def _make_layer_fast(self, block, planes, blocks, stride=1, head_conv=1):\n",
    "        downsample = None\n",
    "        if stride != 1 or self.fast_inplanes != planes * block.expansion:\n",
    "            downsample = nn.Sequential(\n",
    "                nn.Conv3d(\n",
    "                    self.fast_inplanes,\n",
    "                    planes * block.expansion,\n",
    "                    kernel_size=1,\n",
    "                    stride=(1,stride,stride),\n",
    "                    bias=False), nn.BatchNorm3d(planes * block.expansion))\n",
    "\n",
    "        layers = []\n",
    "        layers.append(block(self.fast_inplanes, planes, stride, downsample, head_conv=head_conv))\n",
    "        self.fast_inplanes = planes * block.expansion\n",
    "        for i in range(1, blocks):\n",
    "            layers.append(block(self.fast_inplanes, planes, head_conv=head_conv))\n",
    "        return nn.Sequential(*layers)\n",
    "\n",
    "    def _make_layer_slow(self, block, planes, blocks, stride=1, head_conv=1):\n",
    "        downsample = None\n",
    "        if stride != 1 or self.slow_inplanes != planes * block.expansion:\n",
    "            downsample = nn.Sequential(\n",
    "                nn.Conv3d(\n",
    "                    self.slow_inplanes,\n",
    "                    planes * block.expansion,\n",
    "                    kernel_size=1,\n",
    "                    stride=(1,stride,stride),\n",
    "                    bias=False), nn.BatchNorm3d(planes * block.expansion))\n",
    "\n",
    "        layers = []\n",
    "        layers.append(block(self.slow_inplanes, planes, stride, downsample, head_conv=head_conv))\n",
    "        self.slow_inplanes = planes * block.expansion\n",
    "        for i in range(1, blocks):\n",
    "            layers.append(block(self.slow_inplanes, planes, head_conv=head_conv))\n",
    "  \n",
    "        self.slow_inplanes = planes * block.expansion + planes * block.expansion//8*2\n",
    "        return nn.Sequential(*layers)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def resnet50(**kwargs):\n",
    "    \"\"\"Constructs a ResNet-50 model.\n",
    "    \"\"\"\n",
    "    model = SlowFast(Bottleneck, [3, 4, 6, 3], **kwargs)\n",
    "    return model\n",
    "\n",
    "\n",
    "def resnet101(**kwargs):\n",
    "    \"\"\"Constructs a ResNet-101 model.\n",
    "    \"\"\"\n",
    "    model = SlowFast(Bottleneck, [3, 4, 23, 3], **kwargs)\n",
    "    return model\n",
    "\n",
    "\n",
    "def resnet152(**kwargs):\n",
    "    \"\"\"Constructs a ResNet-101 model.\n",
    "    \"\"\"\n",
    "    model = SlowFast(Bottleneck, [3, 8, 36, 3], **kwargs)\n",
    "    return model\n",
    "\n",
    "\n",
    "def resnet200(**kwargs):\n",
    "    \"\"\"Constructs a ResNet-101 model.\n",
    "    \"\"\"\n",
    "    model = SlowFast(Bottleneck, [3, 24, 36, 3], **kwargs)\n",
    "    return model"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}