chapter06_optimization/adam-scratch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Adam from scratch\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-22T11:57:30.909039Z",
     "start_time": "2017-10-22T11:57:30.157809Z"
    }
   },
   "outputs": [],
   "source": [
    "# Adam.\n",
    "def adam(params, vs, sqrs, lr, batch_size, t):\n",
    "    beta1 = 0.9\n",
    "    beta2 = 0.999\n",
    "    eps_stable = 1e-8\n",
    "\n",
    "    for param, v, sqr in zip(params, vs, sqrs):      \n",
    "        g = param.grad / batch_size\n",
    "        \n",
    "        v[:] = beta1 * v + (1. - beta1) * g\n",
    "        sqr[:] = beta2 * sqr + (1. - beta2) * nd.square(g)\n",
    "\n",
    "        v_bias_corr = v / (1. - beta1 ** t)\n",
    "        sqr_bias_corr = sqr / (1. - beta2 ** t)\n",
    "        \n",
    "        div = lr * v_bias_corr / (nd.sqrt(sqr_bias_corr) + eps_stable)        \n",
    "        param[:] = param - div\n",
    "\n",
    "import mxnet as mx\n",
    "from mxnet import autograd\n",
    "from mxnet import ndarray as nd\n",
    "from mxnet import gluon\n",
    "import random\n",
    "\n",
    "mx.random.seed(1)\n",
    "random.seed(1)\n",
    "\n",
    "# Generate data.\n",
    "num_inputs = 2\n",
    "num_examples = 1000\n",
    "true_w = [2, -3.4]\n",
    "true_b = 4.2\n",
    "X = nd.random_normal(scale=1, shape=(num_examples, num_inputs))\n",
    "y = true_w[0] * X[:, 0] + true_w[1] * X[:, 1] + true_b\n",
    "y += .01 * nd.random_normal(scale=1, shape=y.shape)\n",
    "dataset = gluon.data.ArrayDataset(X, y)\n",
    "\n",
    "# Construct data iterator.\n",
    "def data_iter(batch_size):\n",
    "    idx = list(range(num_examples))\n",
    "    random.shuffle(idx)\n",
    "    for batch_i, i in enumerate(range(0, num_examples, batch_size)):\n",
    "        j = nd.array(idx[i: min(i + batch_size, num_examples)])\n",
    "        yield batch_i, X.take(j), y.take(j)\n",
    "\n",
    "# Initialize model parameters.\n",
    "def init_params():\n",
    "    w = nd.random_normal(scale=1, shape=(num_inputs, 1))\n",
    "    b = nd.zeros(shape=(1,))\n",
    "    params = [w, b]\n",
    "    vs = []\n",
    "    sqrs = []\n",
    "    for param in params:\n",
    "        param.attach_grad()\n",
    "        vs.append(param.zeros_like())\n",
    "        sqrs.append(param.zeros_like())\n",
    "    return params, vs, sqrs\n",
    "\n",
    "# Linear regression.\n",
    "def net(X, w, b):\n",
    "    return nd.dot(X, w) + b\n",
    "\n",
    "# Loss function.\n",
    "def square_loss(yhat, y):\n",
    "    return (yhat - y.reshape(yhat.shape)) ** 2 / 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-22T11:57:31.103948Z",
     "start_time": "2017-10-22T11:57:30.910774Z"
    }
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib as mpl\n",
    "mpl.rcParams['figure.dpi']= 120\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "def train(batch_size, lr, epochs, period):\n",
    "    assert period >= batch_size and period % batch_size == 0\n",
    "    [w, b], vs, sqrs = init_params()\n",
    "    total_loss = [np.mean(square_loss(net(X, w, b), y).asnumpy())]\n",
    "\n",
    "    t = 0\n",
    "    # Epoch starts from 1.\n",
    "    for epoch in range(1, epochs + 1):\n",
    "        for batch_i, data, label in data_iter(batch_size):\n",
    "            with autograd.record():\n",
    "                output = net(data, w, b)\n",
    "                loss = square_loss(output, label)\n",
    "            loss.backward()\n",
    "            # Increment t before invoking adam.\n",
    "            t += 1\n",
    "            adam([w, b], vs, sqrs, lr, batch_size, t)\n",
    "            if batch_i * batch_size % period == 0:\n",
    "                total_loss.append(np.mean(square_loss(net(X, w, b), y).asnumpy()))\n",
    "        print(\"Batch size %d, Learning rate %f, Epoch %d, loss %.4e\" % \n",
    "              (batch_size, lr, epoch, total_loss[-1]))\n",
    "    print('w:', np.reshape(w.asnumpy(), (1, -1)), \n",
    "          'b:', b.asnumpy()[0], '\\n')\n",
    "    x_axis = np.linspace(0, epochs, len(total_loss), endpoint=True)\n",
    "    plt.semilogy(x_axis, total_loss)\n",
    "    plt.xlabel('epoch')\n",
    "    plt.ylabel('loss')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-22T11:57:32.973996Z",
     "start_time": "2017-10-22T11:57:31.105799Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch size 10, Learning rate 0.100000, Epoch 1, loss 6.7040e-04\n",
      "Batch size 10, Learning rate 0.100000, Epoch 2, loss 5.0751e-05\n",
      "Batch size 10, Learning rate 0.100000, Epoch 3, loss 5.0725e-05\n",
      "w: [[ 1.9997046  -3.39914703]] b: 4.1986 \n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAApUAAAG8CAYAAACPGl7EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAASdAAAEnQB3mYfeAAAIABJREFUeJzs3Xmc3FWd7//Xp/fudPaNJISEAIGQyCphEQUX0GFAFMFd\nEFS8cr0/Zxzxjg6jQXQcGa/XqzM4MoPiNi6AiguLOCrIFtYACSEQICErIXvS6b3P74+qDk2brdNV\n/a2ufj0fj++jqr71rfp+qrqTvHPO95wTKSUkSZKk/qjIugBJkiQNfoZKSZIk9ZuhUpIkSf1mqJQk\nSVK/GSolSZLUb4ZKSZIk9ZuhUpIkSf1mqJQkSVK/GSolSZLUb4ZKSZIk9ZuhUpIkSf1mqJQkSVK/\nGSolSZLUb1VZF1DOImIkcBqwAmjLuBxJkqQ9qQGmAnemlLb09cWGyiKIiHnA57OuQ5IkaT+cC/yq\nry+KlFIRahFARBwLPPLLX/6SQw89NOtyJEmSdmvp0qW87W1vAzgupfRoX19vS2VxtQEceuihzJ49\nO+taJEmS9sV+XbLnQB1JkiT1m6FSkiRJ/WaolCRJUr8ZKiVJktRvhkpJkiT1m6FSkiRJ/WaolCRJ\nUr8ZKncjIj4WEY9ERHt+hRxJkiTthqFy99YA84CbMq5DkiSp5Lmizm6klH4JEBFnZV2LJElSqSuL\nlsqIaIyIKyPitojYGBEpIj64m2NrI+IrEbE6IpojYn5EnDHAJUuSJJWVsgiVwDjgc8As4LG9HHs9\n8EngR8AngE7glog4tZgFFlNKKesSJEnSEFcuoXINMCmlNA24fHcHRcRc4N3AZ1JKl6eUrgXeACwH\nrh6QSgtsydptvONb97Jy046sS5EkSUNYWYTKlFJrSmntPhx6PrmWyWt7vLYFuA44OSKmFqnEoli/\nvZWLvvMAj7ywmfOuuZfFa7ZmXZIkSRqiyiJU9sGxwNMppd7p64H87THdOyKiKiLqgEqgKiLqIqJy\ngOrcJ2MaavjroyYBsG5bK+/89/u479kNGVclSZKGoqEWKieR6yrvrXvf5B77rgCagQ8D/5C//4Hd\nvXFETIiI2T034JDClL1rFRXBP559JP9w1iwAtrV2cNF3HuC3j+/qI0qSJBXPUJtSqB5o3cX+lh7P\nA5BSmkdunsp9dRnw+f0trD8+8roZjB9ey6dueIy2zi4+/uNHeGnbkXzwNQdnUY4kSRqChlpLZTNQ\nu4v9dT2e31/XAHN6bef24/365G3HTuG7F5/AsJpKUoJ5v36Sr9z2lCPDJUnSgBhqoXINuS7w3rr3\nrd7fN04prUspLUopLQIuABYCN+/v++2P1x42np9+9GTGNdYA8K0/Pcvf3fAY7Z1dA1mGJEkagoZa\nqFwAzIyIEb32n9jj+X5LKc1LKQW51soBNWfKSH7+sdcwfWwDAD9/ZBUf/t5DNLV2DHQpkiRpCBlq\nofJGcqO5L+3eERG1wMXA/JTSiqwKK6SDxjZw48dO4egDRwJw59Mv8Z7/uJ/123d1OakkSVL/lU2o\njIiPR8QVwCX5XedExBX5bSRASmk+cAPw5Yi4OiIuBf4ATAc+nUXdxTKusZb/+shJnDZzPACPr9zC\n+d+6lxc2OEm6JEkqvLIJlcCngKuAj+Ufn5d/fBUwusdxFwJfJzc90DeAauDslNJdhSokIuZFRCJ3\nXWVmhtVW8Z8XvZp3HHcgAMs27OC8b93DEyu3ZFmWJEkqQ2UTKlNK01NKsZttWY/jWvJLNE5KKdWl\nlOamlG4vcC2ZXVPZW3VlBV+94CguOz03Zeb67W28+9r7+PMzL2VcmSRJKidlEyq1exHBp99yBFe+\ndTYR0NTWycXffZBfProq69IkSVKZMFQWQal0f/d20SnT+bf3HkdNZQUdXYm/+ekCrr3rWeeylCRJ\n/WaoLIJS6v7u7axXTeL7H5rL8LrcYkr/dMtTfOm3iw2WkiSpXwyVQ9BJM8Zyw/84mYkjcosL/efd\nz3PFLxfS1WWwlCRJ+8dQOUQdccAIfn7Zy5Ok/2j+C3z6psfpNFhKkqT9YKgsglK9prK3KaPq+dlH\nT+bQCY0A3PjwSv72pwvocFlHSZLUR4bKIijlayp7mzCijp9cehJHHDAcgF89tppP3/i4XeGSJKlP\nDJViXGMtP7n0JGZPzi2J/vNHV/G5Xy108I4kSdpnhkoBMKqhhu9fMndnV/gP73+Bf77tKYOlJEna\nJ4ZK7TS2sZYffuhEpo6pB+Dbdz7Ht+58NuOqJEnSYGCoLILBMlBnVw4YWcd/ffgkDhhRB8DVty3h\n14+tzrgqSZJU6gyVRTCYBursytQxDbkJ0mtzE6T/3Q2P8dCyjRlXJUmSSpmhUrs0c+JwvvX+46mq\nCNo6uvjI9x9i2fqmrMuSJEklylCp3Tr1sHF86e25xtZNO9q55PoH2dTUlnFVkiSpFBkqtUfvOuEg\nLjv9EACeW9/ER3/4MK0dnRlXJUmSSo2hUnv1qTMP5+yjJgHwwPMb+fubnnCqIUmS9AqGyiIYzKO/\nd6WiIvjqBUdz/LTRAPzi0VVc8yenGpIkSS8zVBbBYB/9vSt11ZV8+wPHc+Do3ByW/3L7Em5buCbj\nqiRJUqkwVGqfjWus5bqLTqAxP9XQ3/70MRau2pJxVZIkqRQYKtUnhx8wnG++51gqAprbO/nw9x7i\nxa0tWZclSZIyZqhUn73+iAl89qxZAKzd2sJHvv8QzW2OCJckaSgzVGq/fOjUg3nP3KkAPL5yC5++\n6XFHhEuSNIQZKrVfIoIvnDuHk2eMBeDXj63m+/ctz7gqSZKUFUOl9lt1ZQX/9r7jmDSyDoAv/vZJ\nHnlhU8ZVSZKkLBgqi6Dc5qnckzHDavi39x1HdWXQ3pn4+I8eYaNLOUqSNOQYKougHOep3JPjDhrN\nP+QH7qze0sLf/HQBXV1eXylJ0lBiqFRBXHTKdP46v5TjXU+/xH/e/VzGFUmSpIFkqFRBRARfecdR\nTB/bAORW3Fm02onRJUkaKgyVKpjG2ir+77uOobIid33lJ36ygJZ256+UJGkoMFSqoI49aDSfeONh\nACxdt50v37I444okSdJAMFSq4C47/RCOnzYagO/dt5w/LlmXcUWSJKnYDJUquKrKCr7+rmNorK0C\n4PIbHmf99taMq5IkScVkqFRRTB3TwJVvnQ3A+u2t/L3LOEqSVNYMlSqa846bsnOaod8vXsfPHlqR\ncUWSJKlYDJUqmojgn972KiaOqAXgi79ZzOrNzRlXJUmSisFQWQRDaZnGvRnZUM2Xz3sVANtaO/jM\nz5+wG1ySpDJkqCyCobZM49684YiJvOO4AwG48+mXuOGhlRlXJEmSCs1QqQHxubOP3NkNftVvnmTN\nFrvBJUkqJ4ZKDYje3eB/f5Pd4JIklRNDpQbMX3SDP2w3uCRJ5cJQqQH1ubOPZMLwfDf4r+0GlySp\nXBgqNaAcDS5JUnkyVGrAvXHWRM47bgoAf1piN7gkSeXAUKlMfP7s2S93gzsaXJKkQc9QqUyMbKjm\nn96e7wZvsRtckqTBzlC5CxExPiJ+GxFNEbEkIt6YdU3l6E1HTuS8Y1/uBr/RbnBJkgYtQ+Wu/Ruw\nFhgPXA78LCLGZFtSefr8OS93g3/plsVsamrLuCJJkrQ/DJW9REQj8Dbg8ymlHSmlXwFPAOdmW1l5\nGtlQzZVvnQ3A5h3tXH37UxlXJEmS9segD5UR0RgRV0bEbRGxMSJSRHxwN8fWRsRXImJ1RDRHxPyI\nOKPXYYcB21NKPftinwBmF+kjDHlvmXMAr5s5HoCfPLiCR1/YlHFFkiSprwZ9qATGAZ8DZgGP7eXY\n64FPAj8CPgF0ArdExKk9jmkEtvZ63db8fhVBRHDlW2dTU1lBSvCPNy+ks8tBO5IkDSblECrXAJNS\nStPIXf+4SxExF3g38JmU0uUppWuBNwDLgat7HLodGNHr5SPy+1UkB48bxkdPmwHAwlVb+a/5yzOu\nSJIk9cWgD5UppdaU0tp9OPR8ci2T1/Z4bQtwHXByREzN734GaIyIKT1eOwdYVKCStRuXnX4oB46u\nB+Bfbl/Chu2tGVckSZL21aAPlX1wLPB0Sql31/YD+dtjAFJK24GbgSsjoj4izgaOyu/brYiYEBGz\ne27AIYX9COWtvqaSeefkLl3d2tLB/7nj6YwrkiRJ+2oohcpJ5LrKe+veN7nHvsvyjzcAXwPelVLa\nuJf3vwxY2GvbYxDVX3rjrAkvD9p54AWeXN37/wCSJKkUDaVQWQ/sqj+1pcfzAKSUXkopnZVSakgp\nzUwp/X4f3v8act3kPTenIeqjiOBzZ8+isiLoSvCF3yxypR1JkgaBoRQqm4HaXeyv6/H8fksprUsp\nLeq5Ac/25z2HqkMnDOfCk6cBcP9zG7lt4b5cMitJkrI0lELlGnJd4L1171tdqBNFxLyISOS6wLUf\n/uaNMxndUA3kVtppae/MuCJJkrQnQylULgBmRkTv6YJO7PF8QaSU5qWUglwXuPbDyIZqPnnm4QCs\n3NTMdXc/n3FFkiRpT4ZSqLwRqAQu7d4REbXAxcD8lNKKrArTrr3nhKkcPnE4AP/+p2fZ6LrgkiSV\nrLIIlRHx8Yi4Argkv+uciLgiv40ESCnNB24AvhwRV0fEpcAfgOnApwtcj93fBVBVWcHfn3UEANta\nO/jmH57JuCJJkrQ7ZREqgU8BVwEfyz8+L//4KmB0j+MuBL4OfAD4BlANnJ1SuquQxdj9XTinzxzP\nKYeMBeCH9y9n+YamjCuSJEm7UhahMqU0PaUUu9mW9TiuJb9E46SUUl1KaW5K6fYMS9deRASf+atZ\nALR3Jv7l9iUZVyRJknalLEKlyturDhzJucfk5qb/zeNrWLBic8YVSZKk3gyVReA1lYX3qTMPp6Yy\n9+v6z7cudkJ0SZJKjKGyCLymsvCmjmngAz0mRL/v2Q0ZVyRJknoyVGrQuOz0Q2ioqQTga3c8bWul\nJEklxFCpQWNsYy0XnTIdgIeWb+LPz6zPtiBJkrSTobIIvKayeC597QyG2VopSVLJMVQWgddUFs/o\nYTVccurBACxYsZk/LlmXcUWSJAkMlRqEPnzqDIbXVgG2VkqSVCoMlRp0RjZU72ytXLhqK394ytZK\nSZKyZqjUoHTJqQfTmG+t/Padz2VcjSRJMlQWgQN1im9kfTXvO/EgAB5YtpGHl2/KuCJJkoY2Q2UR\nOFBnYFxy6sFUVwYA377z2YyrkSRpaDNUatCaOKKOtx87BYA7Fr/I0nXbM65IkqShy1CpQe3S1x0C\nQErwH3d5baUkSVkxVGpQO3RCI2ccORGAXzy6ihe3tmRckSRJQ5OhUoPe/zgt11rZ1tnFd+9Zlm0x\nkiQNUYbKInD098A6ftpoTpg+GoAf3b+crS3tGVckSdLQY6gsAkd/D7zu1sptrR38eP4LGVcjSdLQ\nY6hUWXj94RM4bEIjANfd/TytHZ0ZVyRJ0tBiqFRZqKgIPppvrVy3rZWbF6zOuCJJkoYWQ6XKxluP\nnsyE4bUAfPeeZaSUMq5IkqShw1CpslFTVcGFJ08DYPGardz/3MaMK5IkaegwVKqsvPfEadRW5X6t\nv3PP8xlXI0nS0GGoVFkZM6yG847LLd34+8UvsnxDU8YVSZI0NBgqi8B5KrN18WsOBnJLN15/77Js\ni5EkaYgwVBaB81Rma+bE4bz2sHEA/OzBFU6GLknSADBUqixdkm+tbGrr5GcPrsi4GkmSyp+hUmXp\ntJnjmTFuGJDrAu/scnohSZKKyVCpslRREVz8mukArNzUzB1PvphtQZIklTlDpcrWeccdyIi6KsDp\nhSRJKjZDpcrWsNoq3jP3IAAeeH4jC1dtybgiSZLKl6FSZe3CU6ZTWREAfM/phSRJKhpDpcralFH1\nnDFrIgA3P7aajU1tGVckSVJ5MlSq7F10ynQA2jq6+KnTC0mSVBSGyiJwRZ3SctKMMcyc2AjAD+9f\nTkdnV8YVSZJUfgyVReCKOqUlIrjw5OkArNrczH8/tS7bgiRJKkOGSg0Jbz92CsPz0wt9/75lmdYi\nSVI5MlRqSBhWW8UFx08F4J6lG3jmxW0ZVyRJUnkxVGrIuPDkaTvvf/++5RlWIklS+TFUasiYPm4Y\npx8+HoCbHlnJ1pb2jCuSJKl8GCo1pFyUH7Czo62Tmx5emW0xkiSVEUOlhpTTZo5n2tgGAH5w33K6\nulLGFUmSVB4MlRpSKiqCD5yUu7byufVN3L10fcYVSZJUHgyVGnIuePVU6qsrAdcDlySpUAyVuxER\nH4uIRyKiPSLmZV2PCmdkfTVvP24KAH9Yso4XNuzIuCJJkgY/Q+XurQHmATdlXIeKoHt6oZTgB/cv\ny7YYSZLKgKFyN1JKv0wp/QrYnHUtKrwjDhjBSTPGAPDTB1fQ3NaZcUWSJA1uJR0qI6IxIq6MiNsi\nYmNEpIj44G6OrY2Ir0TE6ohojoj5EXHGAJesQaR7eqGtLR3cvGBVtsVIkjTIlXSoBMYBnwNmAY/t\n5djrgU8CPwI+AXQCt0TEqcUsUIPXGUdOZNLIOgCuv3cZKTm9kCRJ+6vUQ+UaYFJKaRpw+e4Oioi5\nwLuBz6SULk8pXQu8AVgOXN3r2LvzLZ672r5YxM+iElNVWcH789MLPbV2Gw8u25RxRZIkDV4lHSpT\nSq0ppbX7cOj55Fomr+3x2hbgOuDkiJjaY/+pKaXYzXZFwT+EStq7TphKTWXuj8H37luWaS2SJA1m\nJR0q++BY4OmU0tZe+x/I3x7T1zeMiKqIqAMqgaqIqIuIyn7WqRIzrrGWs4+aBMBtC9eydktLxhVJ\nkjQ4lUuonESuq7y37n2T9+M9rwCagQ8D/5C//4HdHRwREyJids8NOGQ/zqsBdtEp0wHo7Er81/zl\n2RYjSdIgVS6hsh5o3cX+lh7P90lKad4uusev38NLLgMW9tpu7ut5NfCOnjqKo6eOAuC/HniB1g6n\nF5Ikqa/KJVQ2A7W72F/X4/liuwaY02s7dwDOqwK4KD8Z+vrtbdz6xL5cxitJknoql1C5hlwXeG/d\n+1YXu4CU0rqU0qKeG/Bssc+rwvjroyYxdlgN4IAdSZL2R7mEygXAzIgY0Wv/iT2eHzARMS8iErku\ncA0CtVWVvGfuQQA8+sJmHl/pQkqSJPVFuYTKG8mN0r60e0dE1AIXA/NTSisGspju6zHJdYFrkHjf\nSQdRWREAfO9eB+xIktQXVVkXsDcR8XFgFC+P4D4nIg7M3/9mSmlLSml+RNwAfDkiJgBLgYuA6cCH\nBrpmDU6TRtbz5tkTueWJtfz68dV89qwjGNu4q0t1JUlSb4OhpfJTwFXAx/KPz8s/vgoY3eO4C4Gv\nk5v25xtANXB2SumugSs1x+7vwevC/HrgbR1d3PTIymyLkSRpECn5UJlSmr6HFXCW9TiuJb9E46SU\nUl1KaW5K6faMarb7e5A68eAxHDJ+GAA/eWCF64FLkrSPSj5USgMpInYO2HlufRPzn9+YcUWSJA0O\nhkqpl/OOO3DneuA/fuCFjKuRJGlwMFQWgddUDm5jhtXwljkHAHDrE2vZ1NSWcUWSJJU+Q2UReE3l\n4NfdBd7W6YAdSZL2haFS2oWTZozh4HH5ATsPOmBHkqS9MVRKu5AbsDMVgKXrtvPQ8k0ZVyRJUmkz\nVBaB11SWh3ccdyDVlbkVdn483wE7kiTtiaGyCLymsjyMbazlzNm5ATu/eWINm3c4YEeSpN0xVEp7\n8N7uATsdXfzi0VUZVyNJUukyVEp7cPKMsUwb2wC4wo4kSXtiqJT2oKIiePcJudbKJS9u45EXNmdc\nkSRJpclQWQQO1Ckv5x9/IFUV+QE7rrAjSdIuGSqLwIE65WX88FrOOHIiAL95fDVbmtszrkiSpNJj\nqJT2QfcKOy3tXfxqgQN2JEnqzVAp7YNTDx3HgaPrAfjR/BccsCNJUi+GSmkf5Abs5FbYeWrtNhau\n2ppxRZIklRZDpbSP3nH8geTH63DDwyuyLUaSpBJjqCwCR3+Xp0kj6zn1sPEA3LxgNS3tnRlXJElS\n6TBUFoGjv8vXBccfCMCW5nbuePLFjKuRJKl0FC1URs4bIuKvImJ4sc4jDaQzjpzIyPpqAG54eGXG\n1UiSVDoKEioj4ksR8ccejwP4HXAH8FvgiYg4pBDnkrJUV13JucdMBuDPz7zEmi3NGVckSVJpKFRL\n5TuAB3o8Ph94I3AFcDZQCcwr0LmkTF1wfG4UeErw80ecs1KSJChcqJwCLO3x+DzgyZTSl1NKtwDf\nAk4v0LmkTM2ZMoIjDshd0XHDQyucs1KSJAoXKjuAWtjZ9f1G4LYez78IjCvQuaRMRQTn5wfsLNuw\ng4eWb8q4IkmSsleoULkQeH9EjAYuBsaSu5ay2zRgfYHOVfKcUqj8vf3YKVTlJ638+SMO2JEkqVCh\n8gvAMeSC438A96SU/tjj+b8GHizQuUqeUwqVv7GNtZw2Mzdn5W8eX+OclZKkIa8goTKldAdwHPBJ\n4BLgzO7n8q2XdwHfKMS5pFLx9uOmALCtpYM/PLUu42okScpWVaHeKKX0JPDkLvZvAv62UOeRSsWb\nZk1keG0V21o7+PkjqzjrVZOyLkmSpMwUap7K4RExtde+yRHxhYj4SkScUIjzSKWkrrpyZ5D805J1\nbN7RlnFFkiRlp1DXVF4L3ND9ICJGAPeTm6fy74A/R8TpBTqXVDK6J0Lv6Er8frFd4JKkoatQofJU\n4Dc9Hr8fmAycAowGHicXMKWyMvfgMYxuyC3beNvCtRlXI0lSdgoVKscBPZcWeStwd0rp/pTSNuD7\nwNEFOpdUMqoqKzjjyIkA3PXMS2xv7ci4IkmSslGoULkZOAAgIuqB15Jb+7tbB9BQoHNJJeUtcw4A\noK2jiz8tsQtckjQ0FSpU3gtcFhFvB74O1AE393h+Jq9syZTKxmsOHUdjbW4iBbvAJUlDVaFC5f8G\n2oGbgI8AX0spLQKIiErgAuDOAp1LKim1VZW84YgJAPzxqXVOhC5JGpIKNfn5UuBw4FhgRkrp8h5P\nNwAfB75UiHMNBi7TOPR0d4E3tXVyz9IhsyKpJEk7FaqlkpRSe0rpsZTSsl77t6WUbu69v5y5TOPQ\nc9rM8dRW5f443WoXuCRpCCpYqIyIyoi4KCJ+FhHz89vPIuLCfBe4VLaG1Vbxuvxa4L9f/CLtnV0Z\nVyRJ0sAq1Io6I4F7gO+QW/e7Or+dAXwXuDs/IbpUtt4yO9cFvnlHOw88vzHjaiRJGliFaqn8EnA8\n8L+A8Sml41JKxwETyF1P+WqG0DWVGpreNGsiVRUBOApckjT0FCpUvh24JqV0TUqpvXtn/jrLbwHf\nAt5RoHNJJWlkQzUnHzIWgNsXraWrK2VckSRJA6dQoXIssGQPzz8FjCnQuaSS1T0KfN22Vh5dsSnj\naiRJGjiFCpVLyS3NuDtvBZ4t0LmkknXGkROJXA+4XeCSpCGlUKHyGuDMiLglIs6MiOn57c0R8Vty\nA3b+tUDnkkrWhOF1vHraaABuW7SWlOwClyQNDVWFeJOU0jURMQH4e+DNPZ4KoA34Qv7aSqnsvWXO\nJB5ctokVG5t5cs1WZk8emXVJkiQVXSEnP58HHAi8D/hsfnsvcGBK6cpCnWcgRERtRHwnIl6IiK0R\ncX9EnJx1XRoc3jx74s77t9sFLkkaIvarpTIiDtrD0/fmt24N3cenlF7Yn/NloApYBpwKrATeCfw6\nIqanlLZnWZhK34GjG3jVlJE8sWoLty1ayyfPPDzrkiRJKrr97f5eBuzPxWKDYmWdlFIT8IUeu34S\nEV8jt775w9lUpcHkLXMO4IlVW3j6xe08+9J2DhnfmHVJkiQV1f6GykvYv1DZJxHRCFwOnAjMBUYD\nF6eUrt/FsbXkguAH8sc9DlyRUrqjAHUcRm5KpKX9fS8NDW+efQD/cntulq3bFq7lf77+0IwrkiSp\nuPYrVO4q1BXJOOBzwAvAY8Dpezj2euB84OvAM8AHgVsi4vUppbv3t4CIqAd+CHw5pbRlf99HQ8uh\nExo5dEIjS9dt5/ZFhkpJUvkr2ECdIlkDTEopTSPXYrlLETEXeDfwmZTS5Smla4E3AMuBq3sde3dE\npN1sX+x1bDVwA7kWyp7d4dJenXlkbsDO4yu3sH57a8bVSJJUXCUdKlNKrSmlfRk+ez7QCVzb47Ut\nwHXAyRExtcf+U1NKsZvtiu7jIqIC+AG5bv6LkhMOqo9OP3zCzvt3Pf1ShpVIklR8JR0q++BY4OmU\n0tZe+x/I3x6zH+/5bWAScEFKqaM/xWloOvagUQyvzV1hcqehUpJU5goy+XkJmESuq7y37n2T+/Jm\nETEN+DDQAqyP7nX34K9SSn/ezWsmAON77T6kL+dVeamurODUw8Zx68K13PX0S3R2JSorYu8vlCRp\nECqXUFkP7OqitZYez++zlNJycqsB9cVlwOf7+BqVudNmjufWhWvZtKOdJ1Zt4Zipo7IuSZKkoiiX\n7u9moHYX++t6PF9s1wBzem3nDsB5VcJOO/zlxus/LVmXYSWSJBVXuYTKNeS6wHvr3re62AWklNal\nlBallBYBFwALgZuLfV6Vtkkj6znigOGA11VKkspbuYTKBcDMiBjRa/+JPZ4fMCmleSmlINdaqSHu\ntJm51soFKzazqakt42okSSqOcgmVN5JbAvLS7h35FXYuBuanlFZkVZjU3QWeEvx56fqMq5EkqThK\nfqBORHwcGMXLI7jPiYgD8/e/mVLaklKaHxE3AF/Oj8JeClwETAc+NNA1Sz29etoYhtVU0tTWyZ+W\nrOOtR/dpMgJJkgaFkg+VwKeAaT0en5ffILd8YvfSiRcCV/HKtb/PTindNUB17hQR83AkuPJqqio4\n5dBx3PHki9z19Hq6uhIVTi0kSSozJd/9nVKavocVcJb1OK4lv0TjpJRSXUppbkrp9oxq9ppKvUL3\ndZXrt7eyaHXvOfolSRr8Sj5USuXg9Ue8vGTjfz/1YoaVSJJUHIbKIoiIeRGRyE0rJDFlVD2zJuUm\nJ/j9YkOlJKn8GCqLwO5v7cqbZuVaKxeu2sraLS17OVqSpMHFUCkNkDfNmrjzvl3gkqRyY6iUBsir\npoxk/PAH5ugHAAAgAElEQVTcaqK/f9JQKUkqL4bKIvCaSu1KRUXwxvyAnXuf3UBLe2fGFUmSVDiG\nyiLwmkrtzumH50Jla0cX85/fmHE1kiQVjqFSGkCnHDqWyvzE53cueSnjaiRJKhxDpTSARtRVc/xB\nowG46xlDpSSpfBgqpQH2upnjAFi6bjurNjdnXI0kSYVhqCwCB+poT16XX7IR4K6nba2UJJUHQ2UR\nOFBHezJn8kjGDKsBDJWSpPJhqJQGWEVF8NrDcl3gdy9dT0dnV8YVSZLUf1VZFyANRafNHM/NC1az\nraWDBSs28+rpY4p2rpb2Th5atomFq7fQ3NbJrEnDOfHgsYzOt5ZKklQIhkopA6897OXrKu98+qWi\nhMp1W1v49l3PcdMjK9m8o/0Vz9VVV/CeuQfxP19/KOMaawt+bknS0GP3t5SB8cNrOXLSCKDw11V2\ndSX+/c5nOf2rf+K6u5//i0AJ0NLexXfvWcYZX7uTmxesIqVU0BokSUOPLZVFEBHzgM9nXYdK22mH\nj+fJNVt5fNUWNja17Ry80x872jr45E8f47ZFa3fuO3nGWN5z4kGcPGMs9TWVPLRsI9++8znue24D\nm3a084mfLODxlVv4h7NmUZGfmF2SpL6ypbIIHP2tffG6fBd4SnDn0+v6/X7bWzt433/O3xkop49t\n4AcfmsuPLz2Jtx49mfHDa2msreL0wyfw40tP4pr3HcfYfJC97u7n+dSNj9HVZYulJGn/GCqljLx6\n+mhG1lcDcPvCF/v1XjvaOrjkuw/y6AubAXjNoWO5+X+e+oprN3s761WT+NX/OpUZ44cB8PNHVnHV\nb5+0K1yStF8MlVJGqisreOOsCQD86el1NLd17tf7dHYl/r8fL+CBZRsBeNOsCXz3g3MZ2VC919dO\nGVXPDR89mcMmNALw3XuWcd3dz+9XHZKkoc1QKWXozbMPAHIDZ/Z3LfCrb3uK3y/OtXSeeug4/vW9\nx1FTte9/tMc21vK9S+YycURuFPiXb32KR17YtF+1SJKGLkOllKHXHTaeuurcH8PbF67dy9F/6baF\na/n2Xc8BcNiERq55/3HUVVf2+X0mj6rnPy88gerKoLMr8YmfPMrWlr8cNS5J0u4YKqUM1ddUcvrM\nXBf4HYtfpKV937vAX9rWymd/8QQAI+ur+c4HT2BE3d67vHfnVQeO5H+/5QgAVmxs5opfLPT6SknS\nPjNUShk766hJAGxr6djZjb03KSU+8/PH2djUBsAX3zaHqWMa+l3LJa85mNNm5gb3/Oqx1dz0yKp+\nv6ckaWgwVBZBRMyLiAQszLoWlb4zj5zIiLrclLE3PLRyn15zw8Mr+f3i3DRE5xw9mXOOnlyQWioq\ngq9ecPTOVXY+d/NCVmzcUZD3liSVN0NlEThPpfqirrqStx07BYC7nnmJ1Zub93j8io07+MKvnwRg\nwvBarjp3dkHrGT+8lq+982gAdrR18o832w0uSdo7Q6VUAt756qlAbiL0mx7efWtlV1fi7254jO2t\nHQBcff5RjGro/0o8vb1u5nguOP5AAP605CV++8Sagp9DklReDJVSCZg9eQSz8muB/3D+clo7dj1g\n5zv3PM8Dz+fmo3zfiQdx+uETilbTZ8+atXPpyKt+8yQ72jqKdi5J0uBnqJRKQETwoVMPBuDFra38\n8tG/HCDz5OqtXH37EgCmjW3gs2fNKmpNo4fV8Jm/OmJnTdf92UnRJUm7Z6iUSsRbj57MpJF1AHz7\nrufo7LEO98amNi79wUO0dXRREfC1dx7NsNqqotd03nEHcmS+BfXf73yWl7a1Fv2ckqTByVAplYia\nqoqdrZXPvdTEtflJzTc2tfHRHzzEyk25ATyffssRHD9tzIDUVFkRO1tEm9o6+frvnx6Q80qSBh9D\npVRC3jP3IA4YkWut/MptT/HJny3gnG/ezYPLcssmvvXoyXz0dTMGtKZTDxu3c+7Knzy4gqXrtg3o\n+SVJg4OhUiohw2qr+N4lcxlZn1sZ5+ePrGJVfoqhvz5qEleffxQRMeB1ffasWVQEdHYl/vnWpwb8\n/JKk0meolErM4QcM5/qLT2DKqHqG11YxfWwDnz/nSP71Pcfu17reharpguNz0x79fvE67n9uQyZ1\nSJJKV/Gv9JfUZ8ceNJp7/v4NWZfxCp88cyY3P7aKlvYu/vnWp/jFZadk0moqSSpNtlQWgcs0qhxN\nHFG3cyDRghWbuX3R2owrkiSVEkNlEbhMo8rVR087hFENues9/+X2Ja+Y9kiSNLQZKiXtsxF11Vx2\n+iEAPPtSE7cudPlGSVKOoVJSn7z/pGk7l2/8tz8+S0q2VkqSDJWS+qihpopLXjMdgMVrtvKHp9Zl\nW5AkqSQYKiX12QdOns7w/DKR3/zDUlsrJUmGSkl9N7K+mgtPmQbkRoLf+6zzVkrSUGeolLRfLnnN\nwdTnJ2P/1z8szbgaSVLWDJWS9svYxlree+JBANz33AYeXr4p44okSVkyVErab5e+bgbVlblVda67\n+7mMq5EkZclQuRsRcW1ErImIrRHxRESck3VNUqmZOKKOc46eDMBtC9eyYuOOjCuSJGXFULl7XwOm\np5RGAJcAP4yIsRnXJJWc7qUbuxJ8795l2RYjScqMoXI3UkpPpZRaux8CNcCUDEuSStLsySM5eUbu\n/1s/eXAF21raM65IkpSFkg6VEdEYEVdGxG0RsTEiUkR8cDfH1kbEVyJidUQ0R8T8iDijn+e/JiKa\ngQeBPwBP9Of9pHLV3Vq5vbWDnz20MuNqJElZKOlQCYwDPgfMAh7by7HXA58EfgR8AugEbomIU/f3\n5Cmly4BG4E3A75IzPEu79IYjJnDwuGEAXH/v83R2+UdFkoaaUg+Va4BJKaVpwOW7Oygi5gLvBj6T\nUro8pXQt8AZgOXB1r2Pvzrd47mr7Yu/3Til1ppT+G3hTRJxVyA8nlYuKiuDi/NKNKzY2c8eTa7Mt\nSJI04Eo6VKaUWlNK+/Kv0/nkWiav7fHaFuA64OSImNpj/6kppdjNdsUezlEFHLqfH0Uqe+cffyAj\n66sB+M7dy7ItRpI04Eo6VPbBscDTKaWtvfY/kL89pi9vFhEjI+K9+Ws6qyLiAuD1wF17eM2EiJjd\ncwMO6ct5pcGsoaaKd5+Q+//bA8s2smTttowrkiQNpHIJlZPIdZX31r1vch/fLwEfAVYCG4C/B96b\nUlqwh9dcBizstd3cx/NKg1r3CjsAP5q/PMNKJEkDrVxCZT3Quov9LT2e32cppa0ppdenlEallEam\nlI5PKf18Ly+7BpjTazu3L+eVBrtpY4dx2szxAPz8kVU0tXZkXJEkaaCUS6hsBmp3sb+ux/NFlVJa\nl1Ja1HMDni32eaVS84GTpgG56YV+uWBVxtVIkgZKuYTKNeS6wHvr3rd6AGshIuZFRCLXBS4NKa8/\nYgJTRuU6B35w33KciUuShoZyCZULgJkRMaLX/hN7PD9gUkrzUkpBrgtcGlIqK2LntZVPrd3Gw8s3\nZVyRJGkglEuovBGoBC7t3hERtcDFwPyU0oqsCpOGone+eirVlQHAD+93wI4kDQVVWRewNxHxcWAU\nL4/gPiciDszf/2ZKaUtKaX5E3AB8OSImAEuBi4DpwIcyqHke8PmBPq9UKsYPr+Utcybx68dWc8sT\na/nHs1sZ27iry54lSeViMLRUfgq4CvhY/vF5+cdXAaN7HHch8HXgA8A3gGrg7JTSbueWLBa7v6WX\nB+y0dXa5HrgkDQElHypTStP3sALOsh7HteSXaJyUUqpLKc1NKd2eYenSkHbC9NHMnNgI5OasdD1w\nSSpvJR8qJQ1OEbGztXLlpmbufHpdxhVJkorJUFkETikk5bzt2CkMq6kE4If3v5BxNZKkYjJUFoHX\nVEo5w+uqeduxUwD445J1rNi4I+OKJEnFYqiUVFTvz3eBpwT/9YCtlZJUrgyVkopq1qQRnDA9N1HD\nTx9cQWtHZ8YVSZKKwVBZBF5TKb1Sd2vlxqY2blu4NuNqJEnFYKgsAq+plF7pLXMOYOywGiC3Hrgk\nqfwYKiUVXW1VJe88YSoADy3fxOI1WzOuSJJUaIZKSQPivXMPInLLgbseuCSVIUOlpAExdUwDbzh8\nAgC/eHQV21raM65IklRIhsoicKCOtGvdA3Z2tHXyy0dXZVyNJKmQDJVF4EAdaddeN3M8U8fUA/CD\n+5eTkuuBS1K5MFRKGjCVFcH7Tsy1Vj794nYeeH5jxhVJkgrFUClpQF1w/IHUVOb+6vmBA3YkqWwY\nKiUNqLGNtfz1UZMAuHXhWlZucj1wSSoHhkpJA+5Dpx4MQGdX4rq7n8+4GklSIRgqi8DR39KezZky\nklMPHQfATx5YwaamtowrkiT1l6GyCBz9Le3dR0+bAUBze6eToUtSGTBUSsrEqYeOY/bkEQBcf+8y\nWto7M65IktQfhkpJmYgIPnraIQBsaGrjxodXZlyRJKk/DJWSMnPWnAM4cHRuMvT/+PNzdHY5Gbok\nDVaGSkmZqaqs4COvzV1buXzDDn77xJqMK5Ik7S9DpaRMvfPVUxnXWAPAN//7GbpsrZSkQclQKSlT\n9TWVO1srn1m3nVsXrs24IknS/jBUFoHzVEp98/6TpjFmWK618hu2VkrSoGSoLALnqZT6ZlhtFR9+\nbW6VnSUvbuN3T9paKUmDjaFSUkm48OTpjGqoBuD//fdSWyslaZAxVEoqCY21VXw4vyb44jVbuWPx\nixlXJEnqC0OlpJJx0SnTGVmfa638v3c87byVkjSIGCollYzhddVc+rrcSPCn1m7jV4+tyrgiSdK+\nMlRKKikXv2Y644fXAvB/fvc0bR1dGVckSdoXhkpJJaWhpopPvPEwAFZuaubHD7yQcUWSpH1hqJRU\nct51wlSmjW0A4Jt/eIam1o6MK5Ik7Y2hUlLJqa6s4O/OPByA9dvbuO7u5zOuSJK0N4ZKSSXp7FdN\nYvbkEQBce9dzbGxqy7giSdKeGCqLwGUapf6rqAg+/ZYjANje2sE1f1yacUWSpD0xVBaByzRKhfG6\nw8Zx0owxAHz/vuWs2tyccUWSpN0xVEoqWREvt1a2dXbx9TuezrgiSdLuGCollbTjDhrNmUdOBOCm\nR1byzIvbMq5IkrQrhkpJJe/yNx9ORUBXgq/+bknW5UiSdsFQKankHTZxOO847kAAbl/0Ig8v35Rx\nRZKk3gyVkgaFvzljJjVVub+yrvz1Irq6UsYVSZJ6MlRKGhSmjKrn0tfOAODxlVu48eGVGVckSerJ\nUClp0Ljs9YcwaWQdAF+57Sm2NLdnXJEkqZuhci8i4uSI6IqIK7KuRRrqGmqq+OxZswDY0NTG//v9\nMxlXJEnqZqjcg4ioAP4v8GDWtUjKOfuoScw9ODch+vfuW+YUQ5JUIgyVe3YpMB9YnHUhknIignnn\nzKYioLMrMe/Xi0jJQTuSlLWSDpUR0RgRV0bEbRGxMSJSRHxwN8fWRsRXImJ1RDRHxPyIOKMf5x4L\n/A3w+f19D0nFceTkEbz/pGkA3LN0A7cvWptxRZKkkg6VwDjgc8As4LG9HHs98EngR8AngE7glog4\ndT/P/SXg6ymlzfv5eklF9MkzZjKqoRqAq36zmJb2zowrkqShrdRD5RpgUkppGnD57g6KiLnAu4HP\npJQuTyldC7wBWA5c3evYu/Mtnrvavpg/5ljgBOA/ivS5JPXTqIYaPnXm4QCs2tzMt+98LuOKJGlo\nK+lQmVJqTSntS7/W+eRaJq/t8doW4Drg5IiY2mP/qSml2M3WPcL7NOBwYFVErAXeBfzviPhuoT6b\npP57z9yDmDVpBAD/fuezrNnSnHFFkjR0lXSo7INjgadTSlt77X8gf3tMH9/vWuDQ/OuOAX4F/Bvw\nt/0pUlJhVVYEnz/nSACa2zv5yq1PZVyRJA1d5RIqJ5HrKu+te9/kvrxZSmlHSmlt9wY0A9v3dH1l\nREyIiNk9N+CQvpxXUt+dNGMsZ73qAAB+uWC164JLUkbKJVTWA6272N/S4/n9llL6YErpi3s57DJg\nYa/t5v6cV9K++cxfzdq5LvgXXBdckjJRLqGyGajdxf66Hs8X2zXAnF7buQNwXmnImzqmgY+89mAA\nHlu5hV88uirjiiRp6CmXULmGXBd4b937Vhe7gJTSupTSop4b8Gyxzysp57LTD2XC8Nz/Lb98q+uC\nS9JAK5dQuQCYGREjeu0/scfzAyYi5kVEItcFLmkADKt9eV3w9dtb+T+/W5JxRZI0tJRLqLwRqCS3\nrCKQW2EHuBiYn1JaMZDFpJTmpZSCXBe4pAFy7jGTOXnGWAB+cP9yFqxw7QJJGiglHyoj4uMRcQVw\nSX7XORFxRX4bCZBSmg/cAHw5Iq6OiEuBPwDTgU9nUbekgRcRfPHtc6iprCAl+F8/foQtO+wGl6SB\nUPKhEvgUcBXwsfzj8/KPrwJG9zjuQuDrwAeAbwDVwNkppbsGrtQcu7+l7BwyvpFPvXkmACs2NvN3\nNyxwNLgkDYCSD5Uppel7WAFnWY/jWvJLNE5KKdWllOamlG7PqGa7v6UMfeS1MzjzyIkA/H7xOr7w\nmydJyWApScVU8qFSkvoqIvjqO4/m0AmNAFx/7zK+ctsSOm2xlKSiMVRKKksj6qr5/iVzmTwyN13t\nv9/5LBd+Zz6rNrs+uCQVQ1XWBZSjiJgHfD7rOqShbvKoen744RP58Pce4rn1TdyzdANv+Oqf+PBr\nD+aik6czYUTd3t8kL6XEjrZONja1sX57Kxub2tiwvY31Ta00tXbQWFvN8LoqRtTnbkfWVzO+sZYJ\nI2qpraos4qeUpNIQXmdUPPn1vxcuXLiQ2bNnZ12ONGQ1tXbw+V8t4saHV+7cV1URzD14DHOmjGRE\nXRV11ZXUVleyo7UjHxzb2ND0cnjc0NRKS3tXn89dWREcMn4YsyaN2LnNmTyCsY27WgRMkrKzaNEi\n5syZAzAnv4hLnxgqi8hQKZWWB5dt5Eu/XVzw+SsjoK9/lR45aQSnHz6e1x8xgeMOGk1lRRS0Jknq\nK0NlCTNUSqUnpcSCFZu54eGVPLJ8E8+s2/4XA3hqqioYO6yGsY01jB1Wu/P+mGG1+X01jG18eX99\ndSUt7V1sbWlnW0s7W5o72NLcxrqtrTy/oYnFa7axeM1WXtrWusuaJo2s44LjD+SCV09l6piGgfga\nJOkvGCpLUO9rKg2VUulKKdHW2UVLexet7Z3U11TSWFtFROFbDtdvb2Xxmq088PxG/rTkJZ5YteUv\njjlpxhjOetUk3jz7ACb24ZpPSeovQ2UJs6VS0p68tK2V3z25lp89tJLHdtElf+xBozj+oNEcNXUU\nR00ZyUFjGqiwm1xSkfQ3VDr6W5IyMn54Le87cRrvO3EaT63dys8eXMmtC9ewZksLAI++sJlHX3g5\nbNZVVzB97DBmjB/GjHGNzBg/jIPHDWPG+EZG1ldn9TEGxNaWdtZsbmH1lmZWb85tL2xsZltLO1UV\nFVRXBpUVQXVlBbVVFTTWVjG8LjcSf/zwWsY11tJQU0lDTSV11ZX5+1XUVVcUpVVag19XV6KjK1FT\nVfjZF9s7u1i+oYm1W1rZtKMNgOrKoKqigtrqChpqqhhWW8mwmqpB9btqS2UR2VIpqa9SSjy2cgu3\nLlzDPUvXs2TtNto79/739LjGGqaMqqch/49QXU3lzvfr7Ep0pVfe35WKyI1WjwgqAoLcP2CVFbnA\nVlURVORvK3feVlBVGVRX5gJdTVUFNfnb6sqKHvviFftSvp6uBF0p0dHVxcamdjZsb2VDftqm9dvb\nWLO5mTVbWtje2lGw77i3+ureYbN38Mzdb6yr2nkd7eiGGmqqKqjKf/6urtxlFO2difaOLto7u2jr\n7KKtI7+v+35XF52difauREd+X/dxbR1dtHbmLsOICKaMqs9to+sZMyx3ztHDqqmMoLm9k+a2zp23\nrR3d5+qifTc/4IqAysj9DFNKtHfmvvf2zkRH/n5HZ6IqH26qe/zMqipy93PfVRX1NbmA0/1zTOR+\njrnHuftd3efo7KKjK/cddPQ8Z69z7+rY9nxNKeXqr6jI/Vbu/B3NZ6yUIPHygLlEfgfd+3MPuhK0\ndnTS0t5FS3vue+u+bW3P7d/e2sHWlna2t3aQUu73Y+KIWqaOaeCg/DZtbAPjh9fRWJv781ZfU5n7\nzF3QmRLbWtrZ1NTO5h1tbNrRzqYduVkkXtzawjPrtvH8+qZ9+nPdUwQMq6nikX88oyhBF2yplKSy\nEhEcM3UUx0wdBUBLeydL1m7jiVVbePal7Tz3UhPPr29i5aYdrwiH67fnpkEaKioCpoyuZ3RDzSsC\nUUdXoqW9k+2tHexo69yn92puz4UzaVea2ztZtmEHyzbsyLSOlA/ExQqUhWCoLAInP5dUKHXVlRw9\ndRRH50Nmt5b2TlZs3MGz+ZD53EvbWbu1hea2Tna0ddLS0UkAFRG5rSLXslORb+Ghdzdaj1bDzq70\nihaf7tbNjnwrW0dXyrcupnyrW66VqVDLYNZUVTAuP8L+gJF1TB5Zx6RR9UwaWcfkUfVMHlXPhOG1\nVFfu+R/Xjs4utrZ0sG5bCxub2nZ+N92tey/f79jN/k52tHXsbBHc2tJR0KU+KytiZ6tudwtvbVUF\n9TWVtHV0sXpzM037GIzLUUVAVWUF1fnW854t292tj10Jun+To0frOgE9f82DyD8PtdWV1FVVUFtd\nSW3+tufj4bW5RQxG1FdTXRFsbm5nzZZmXti4g+UbdrCtZf9azeurK5kwopaDxw1j5sThHDahkalj\nGhgzrIaAna23rR1dNOX/U7Tztq2DHa2ddJZ477KhsghSSvOAed3d3xmXI6kM1VVXctjE4Rw2cXjW\npezUme+2bOvsor2j+zb16AZ++bnu7svu7syKimBMQ65ruVCj76sqKxgzrIYxw2r6/+HIhZitzR1s\naGpl04522ju7dn7mini5q7imsoLqqth5/+Vu/9jZldx9mcHezrd5RzurtzSzqSnXhbp5RxuJ3M+/\nu9u+vrqS2uoKaiorqa7KXZbwctTa+W50pdzPqLMrURG5Sxaq8vV011dVUZFr9e1KO39m3fe7u4q7\nQ3dKiYjY+Z+X2Bnkcve7z9H9mat6nKP3uasqg+r8pQTd90t1UNqWHe0s39jExqY2mlpzga8lf8lC\nZf73urGuitENNYxqqM5dttBQQ31N+a+sZaiUJBVE7trL3PWI5SgiGNlQzciGgRkUFRGMHlbD6AKF\nYhXGyIZqjmoYtfcDh6DS7ZiXJEnSoGGolCRJUr8ZKiVJktRvhsoiiIh5EZFwkI4kSRoiDJVFkFKa\nl1IKYE7WtUiSJA0EQ6UkSZL6zVApSZKkfjNUSpIkqd8MlZIkSeo3Q6UkSZL6zVBZBE4pJEmShhpD\nZRE4pZAkSRpqDJWSJEnqt6qsCyhzNQBLly7Nug5JkqQ96pFXavbn9ZFSKlw1eoWIeCtwc9Z1SJIk\n9cG5KaVf9fVFhsoiioiRwGnACqCtSKc5hFxwPRd4tkjnGGr8TgvP77Sw/D4Lz++0sPw+C28gvtMa\nYCpwZ0ppS19fbPd3EeV/IH1O+n0REd13n00pLSrmuYYKv9PC8zstLL/PwvM7LSy/z8IbwO/00f19\noQN1JEmS1G+GSkmSJPWboVKSJEn9Zqgc/F4CrszfqjD8TgvP77Sw/D4Lz++0sPw+C6/kv1NHf0uS\nJKnfbKmUJElSvxkqJUmS1G+GSkmSJPWboVKSJEn9ZqiUJElSvxkqS1RE1EbEVyJidUQ0R8T8iDhj\nH187JSJ+FhGbI2JrRNwcETOKXXOp29/vNCLmRUTaxdYyEHWXqohojIgrI+K2iNiY/04+2IfXj4qI\nayPipYhoiog/RsRxRSy55PXnO42ID+7m9zRFxAFFLr0kRcQJEfGvEbEo/zv2Qv7vxpn7+Hp/R3vo\nz/fp7+euRcTsiLghIp6LiB0RsT4i7oqIc/bx9SX1O+ra36XreuB84OvAM8AHgVsi4vUppbt396KI\naAT+CIwE/gloB/4WuDMijkkpbShy3aXsevbjO/3/27v7GLmqMo7j358VqOiWt5byIlB5kWIQS2l5\nEYIoJAqiRmiBhAQRlDeJgBqLIUEShYCgoiBBECikYlgLGGqDBJAKDUjB8k4KbaBABGp56YuUAtbH\nP84ZvEzv7G7n7nRmd36f5GZ3zpxz59wzT7fP3HvumYJTgX8XHq8Z7A4OMaOBc4EXgceAgwbaUNKH\ngNnAZ4CLgdeA04A5kvaKiIWD3tuhoekxLTgXeL6ubFm1bg1Z04D9gT8CjwNbAacD8yXtGxFPNmro\nGC3V9HgWOD4/aAegB7geeBnYGDgSuE3SyRFxVaOGHRmjEeGtwzZgbyCAHxTKRgKLgPv7afvD3HZy\noWw88B/ggnYf2xAd0/Ny29HtPo5O2oCNgK3y75PyGB0/wLZH5fpTCmVjgDeBG9t9bEN0TI/P9Se1\n+zg6ZQM+C2xYV7YLsBqY0U9bx+jgjqfjc+DjPAJ4FFjQT72Oi1Ff/u5MU0hnwd7/hBIRq4FrgP0k\nbddP24ci4qFC2wXA3aQA7FZVxrRGkkZJUov6OKRExDsR8WqTzacAS4BbCvtbCvQCX5O00SB0ccip\nOKbvk9QjacRg9Gkoi4j7I+LdurKFwFPAbv00d4zWqTie73N89i0i1gAvAZv2U7XjYtRJZWfaE3g2\nIlbUlc/LPyeUNcqnwvcAHi55eh6wk6SeQevl0NLUmNZ5DlgOrJQ0Q9LYwexgl9kTmB8R/60rn0e6\n/DOgOW9W6h5gBbBK0m2Sdml3hzpJ/lA4lnSpsC+O0QFYh/GscXyWkPRRSaMl7STpLOBQ0smgvnRc\njDqp7ExbA6+UlNfKtmnQbnPS5bNm2g53zY4ppEsJlwMnkz4Z/g44GrhP0qjB7GQXqfJ+WLlVpHnD\n3wG+DvwMOBi4f4Bn4rvFscC2wE391HOMDsxAx9Px2befk77TexFwCXArab5qXzouRn2jTmf6CPBO\nSdjJQKgAAAdASURBVPnqwvON2tFk2+Gu2TElIn5VV3SzpHnA70mToi8clB52l6bfDysXEb2ky141\nf5J0B3AvcA5wSls61kEkjQd+AzxAujGiL47RfqzLeDo++3UpMJOUCB5Fmle5YT9tOi5GfaayM71N\nOuNYb2Th+UbtaLLtcNfsmJaKiBuBV4FDKvarWw3q+2HlIq1q8CCOU/KyNbNJU1im5HlrfXGM9qGJ\n8VyL4/P/ImJBRNwVETdExOHAx4BZ/czh77gYdVLZmV4hndauVyt7uUG7N0ifWpppO9w1O6Z9eYk0\n5cDWXSveDyvX9XEqaRPgdtKND1+KiIHEl2O0gSbHs5Guj88GZgKT6XteZMfFqJPKzvQo8MmS+Xr7\nFJ5fS56s+wRpKZJ6+wDPRcTKQevl0NLUmDaSPz2OI82BsXX3KDAx31xWtA9p7tWz679Lw9aOdHGc\nShoJzCL953x4RDw9wKaO0RIVxrORro7PPtQuXW/SR52Oi1EnlZ1pJmk+xUm1grw0wDeBByPipVy2\nfZ7TUt92sqRJhba7Al8gLVjbrZoeU0ljSvZ3Kmk9sL+0rMfDhKStJY2XtEGheCbpjtEjCvVGA1OB\nWRFRNk/IsrIxLYtTSYcBe9GlcZqXrbkJ2A+YGhEPNKjnGB2AKuPp+CwnacuSsg2A40iXr5/OZUMi\nRpUXy7QOI6mXdIfcL0l3g32DtID3wRFxb64zB/hcRKjQrgd4hLRC/yWkb9T5HimhmpDXsOpKFcZ0\nFekP6ROkCdAHAMeQvvFk/4hYtR4Po6NIOp10CWwbUqJ9Cyn+AC6LiOWSppPG+hMRsTi3GwHMBXbn\ng98EsT1p4f5n1uNhdJQKY7ow13uYNM9tInAC6RLZ5IhYsh4PoyNIuhQ4g3Rmrbf++YiYketNxzHa\nr4rj6fgsIelWYBTphqV/kr6l6FjSl5Z8PyJ+ketNZyjEaDtWXPfW/0aaaHsx6R/catK6U1+sqzMn\nvYVrtf046azkcmAl6Q/Azu0+pnZvzY4pcDVpcd8VwLukr3i8EOhp9zG1ewMWk77RoWwbl+tMLz4u\ntN2MtDzTa8Bbeey7/ts2mh1T4Kek/7SX5Th9AbgCGNvuY2rjWM7pYyyjUM8x2uLxdHw2HNNjgDtJ\nN36+R7o34k7gq3X1hkSM+kylmZmZmVXmOZVmZmZmVpmTSjMzMzOrzEmlmZmZmVXmpNLMzMzMKnNS\naWZmZmaVOak0MzMzs8qcVJqZmZlZZU4qzczMzKwyJ5VmZmZmVpmTSjOzLiTpPEkhaXS7+2Jmw4OT\nSjMzMzOrzEmlmZmZmVXmpNLMzMzMKnNSaWbWQpK2lXStpCWS3pH0lKQTCs8flOc2Hi3pAkmvSnpL\n0m2StivZ31RJ/5D0tqTXJM2QtG1JvfGSeiUtzXWfkXR+SRc3lTRd0jJJyyVdJ2njQR4GM+sCH253\nB8zMhitJY4G/AwFcDiwFDgWukTQqIi4tVD8n17sI2BI4E7hL0oSIeDvv73jgOuAh4EfAWOAMYH9J\ne0bEslxvD+A+4D3gKmAxsBPwlfw6Rb3A83l/E4FvAf8Cpg3WOJhZd3BSaWbWOucDI4BPR8TruexK\nSX8AzpP020LdzYHdImIlgKT5pITv28CvJW1ASjifBA6MiNW53lzgz8BZwI/zvi4DBEyMiBdrLyDp\n7JI+PhIRJxbqbAGciJNKM1tHvvxtZtYCkgQcCczKD0fXNuAOYBPSmcGaG2oJZTYTeAU4LD+eRDqD\neUUtoQSIiNnAAuDL+XXHAAcC1xYTylw3Srp6Zd3j+4AtJI1al+M1M/OZSjOz1hgDbAqclLcyWwJv\n5t8XFp+IiJC0CBiXi3bIP58p2c8C4ID8+47555MD7OeLdY9r/dkMWDHAfZiZOak0M2uR2pWgGcD1\nDeo8Dnxq/XSnoTUNyrVee2FmQ56TSjOz1lgKrARGRMRdjSpJqiWVu9SVC9iZlHgCvJB/7gr8tW43\nuxaefy7/3L25bpuZNcdzKs3MWiAi1gA3A0dKWivBy3Mfi46T1FN4PAXYGrg9P36YdFf2KZI2Kuzn\nUGA3YHZ+3aXAvcAJkrave02ffTSzlvGZSjOz1jkb+DzwoKSrgadJd3lPBA7Jv9e8AcyVdB1pqaAz\ngUXA1QAR8Z6kaaQlhf6W7yCvLSm0GPhlYV/fBeYC8yVdRVoyaBzpZp4JrThQMzMnlWZmLRIRSyTt\nDZwLHAGcBrwOPMXaS/ZcAOxBWi+yB7gbOC0iVhX2N13SKlKyehHwFnArMK22RmWu95ikfYGfAKcC\nI0mXx3tbcZxmZgAqX2HCzMzWB0kHAfcAUyNiZpu7Y2bWNM+pNDMzM7PKnFSamZmZWWVOKs3MzMys\nMs+pNDMzM7PKfKbSzMzMzCpzUmlmZmZmlTmpNDMzM7PKnFSamZmZWWVOKs3MzMysMieVZmZmZlaZ\nk0ozMzMzq8xJpZmZmZlV5qTSzMzMzCpzUmlmZmZmlTmpNDMzM7PK/gcf8QUoi9tmOQAAAABJRU5E\nrkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10bfa2c50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train(batch_size=10, lr=0.1, epochs=3, period=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Next\n",
    "[Adam with Gluon](../chapter06_optimization/adam-gluon.ipynb)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For whinges or inquiries, [open an issue on  GitHub.](https://github.com/zackchase/mxnet-the-straight-dope)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}