010-basic-pytorch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Numpy/pandas array to torch tensor\n",
    "\n",
    "Array (matix)\n",
    "\n",
    "$$\\left[\\begin{array}{lll}\n",
    "1 & 0 & 1\\\\\n",
    "2 & 1 & 1\n",
    "\\end{array}\\right]\n",
    "$$\n",
    "\n",
    "In `numpy`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 0, 1],\n",
       "       [2, 1, 1]])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = np.array([[1, 0, 1], [2, 1, 1]])\n",
    "a"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can transform it to `pytorch` __tensor__."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1, 0, 1],\n",
       "        [2, 1, 1]])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b = torch.from_numpy(a)\n",
    "b"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### BTW\n",
    "You can do something similar with `pandas.DataFrame`>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0, 2],\n",
       "        [1, 3],\n",
       "        [2, 4]])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame({\"a\": [0, 1, 2], \"b\": [2, 3, 4]})\n",
    "df_torch = torch.from_numpy(df.values)\n",
    "df_torch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exercise\n",
    "\n",
    "Create array and transform it to a tensor:\n",
    "\n",
    "$$\\left[\\begin{array}{lll}\n",
    "1 & 0 & 1 & 1\\\\\n",
    "2 & 1 & 1 & 1\n",
    "\\end{array}\\right]\n",
    "$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# your code"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### They share memory!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[666,   0,   1],\n",
       "        [  2,   1,   1]])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a[0, 0] = 666\n",
    "b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>666</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     a  b\n",
       "0  666  2\n",
       "1    1  3\n",
       "2    2  4"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_torch[0, 0] = 666\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Other ways\n",
    "\n",
    "#### From python list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0, 1, 0],\n",
       "        [1, 1, 1]])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c = torch.tensor([[0, 1, 0], [1, 1, 1]])\n",
    "c"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### tensor of ones and zeros"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 1., 1.],\n",
       "        [1., 1., 1.]])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d = torch.ones(size=[2, 3])\n",
    "d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0., 0., 0.],\n",
       "        [0., 0., 0.]])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.zeros(size=[2, 3])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Eye tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 0., 0.],\n",
       "        [0., 1., 0.],\n",
       "        [0., 0., 1.]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "e = torch.eye(n=3)\n",
    "e"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Sequence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 1,  2,  3,  4,  5,  6,  7,  8,  9, 10])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.arange(1, 11)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### normal sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.5908, -0.7671, -0.3221],\n",
       "        [ 0.0727, -0.0909, -0.6595]])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f = torch.normal(mean=torch.zeros(size=[2, 3]), std=torch.ones(size=[2, 3]))\n",
    "f"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.4609,  0.1923, -0.6975],\n",
       "        [ 0.0827, -1.1623,  0.6475]])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.randn(2, 3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exercise\n",
    "\n",
    "Create few matrices. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "## your code"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## More than two dimensions "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[0, 1, 0],\n",
       "         [1, 1, 1]],\n",
       "\n",
       "        [[1, 2, 3],\n",
       "         [4, 5, 6]]])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f = torch.tensor(\n",
    "    [\n",
    "        [[0, 1, 0], [1, 1, 1]],\n",
    "        [[1, 2, 3], [4, 5, 6]]      \n",
    "    ]\n",
    "\n",
    ")\n",
    "f"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2, 2, 3])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Operations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1, 0, 1],\n",
      "        [2, 1, 1]])\n",
      "tensor([[0, 1, 0],\n",
      "        [1, 1, 1]])\n"
     ]
    }
   ],
   "source": [
    "b[0, 0] = 1\n",
    "print(b)\n",
    "print(c)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### sums\n",
    "\n",
    "#### pointwise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1, 1, 1],\n",
       "        [3, 2, 2]])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b + c"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### total sum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(6)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b.sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### sums of rows (columns...)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([2, 4])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b.sum(dim=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### scalar muliplication"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1, 0, 1],\n",
       "        [2, 1, 1]])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[2, 0, 2],\n",
       "        [4, 2, 2]])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "2 * b"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### pointwise multiplication"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0, 0, 0],\n",
       "        [2, 1, 1]])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b * c"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### dot product"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([2, 1, 1])\n",
      "tensor([1, 1, 1])\n"
     ]
    }
   ],
   "source": [
    "print(b[1])\n",
    "print(c[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(4)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.dot(b[1], c[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### transpose"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0, 1],\n",
       "        [1, 1],\n",
       "        [0, 1]])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c.t()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0, 1],\n",
       "        [1, 1],\n",
       "        [0, 1]])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c.permute([1, 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 0,  1,  0,  1],\n",
       "         [ 1,  1,  1,  1],\n",
       "         [ 2,  2,  3,  3]],\n",
       "\n",
       "        [[ 1,  2,  3,  4],\n",
       "         [ 5,  6,  7,  8],\n",
       "         [ 9, 10, 11, 12]]])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f = torch.tensor(\n",
    "    [\n",
    "        [[0, 1, 0, 1], [1, 1, 1, 1], [2, 2, 3, 3]],\n",
    "        [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]      \n",
    "    ]\n",
    "\n",
    ")\n",
    "f"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 0,  1],\n",
       "         [ 1,  2],\n",
       "         [ 0,  3],\n",
       "         [ 1,  4]],\n",
       "\n",
       "        [[ 1,  5],\n",
       "         [ 1,  6],\n",
       "         [ 1,  7],\n",
       "         [ 1,  8]],\n",
       "\n",
       "        [[ 2,  9],\n",
       "         [ 2, 10],\n",
       "         [ 3, 11],\n",
       "         [ 3, 12]]])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f.permute([1, 2, 0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Matrix multiplication"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0, 2],\n",
       "        [1, 4]])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.matmul(b, c.t())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 3, 2])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c.t().unsqueeze(0).shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### unsqueeze"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[1, 0, 1]],\n",
       "\n",
       "        [[2, 1, 1]]])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b.unsqueeze(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[0, 1],\n",
       "         [1, 1],\n",
       "         [0, 1]]])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c.t().unsqueeze(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 2, 2])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.matmul(b, c.t().unsqueeze(0)).shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### concatenation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1, 0, 1],\n",
      "        [2, 1, 1]])\n",
      "tensor([[0, 1, 0],\n",
      "        [1, 1, 1]])\n"
     ]
    }
   ],
   "source": [
    "print(b)\n",
    "print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1, 0, 1],\n",
       "        [2, 1, 1],\n",
       "        [0, 1, 0],\n",
       "        [1, 1, 1]])"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cat([b, c], 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1, 0, 1, 0, 1, 0],\n",
       "        [2, 1, 1, 1, 1, 1]])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cat([b, c], 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### view"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 1,  2],\n",
       "        [ 3,  4],\n",
       "        [ 5,  6],\n",
       "        [ 7,  8],\n",
       "        [ 9, 10],\n",
       "        [11, 12]])"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g = torch.arange(1, 13)\n",
    "g.view(-1, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 1,  2],\n",
       "         [ 3,  4],\n",
       "         [ 5,  6]],\n",
       "\n",
       "        [[ 7,  8],\n",
       "         [ 9, 10],\n",
       "         [11, 12]]])"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g.view(2, 3, -1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### reshape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 1  2  3  4  5  6  7  8  9 10 11 12]\n",
      "[[ 1  2  3  4]\n",
      " [ 5  6  7  8]\n",
      " [ 9 10 11 12]]\n",
      "[[ 1  2]\n",
      " [ 3  4]\n",
      " [ 5  6]\n",
      " [ 7  8]\n",
      " [ 9 10]\n",
      " [11 12]]\n"
     ]
    }
   ],
   "source": [
    "print(np.arange(1, 13))\n",
    "print(np.arange(1, 13).reshape(3, 4))\n",
    "print(np.arange(1, 13).reshape(-1, 2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exercises\n",
    "\n",
    "1. Create a 5x5 zero tensor with elements on the main diagonal equal to 1, 2, 3, 4, 5.\n",
    "2. Create a matrix\n",
    "\n",
    "$$\\left[\\begin{array}{lll}\n",
    "    1 & 2 & 3 & 4 & 5 & 6\n",
    "\\end{array}\\right]\n",
    "$$\n",
    "\n",
    "and transform it into\n",
    "\n",
    "$$\\left[\\begin{array}{lll}\n",
    "    1 & 2 & 3\\\\\n",
    "    4 & 5 & 6\n",
    "\\end{array}\\right]\n",
    "$$."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Drivatives: How autograd works\n",
    "\n",
    "$$x_1 = \\left[\\begin{array}{l}\n",
    "x_1^1\\\\\n",
    "x_1^2\n",
    "\\end{array}\\right]=\n",
    "\\left[\\begin{array}{l}\n",
    "0.1\\\\\n",
    "0.2 \n",
    "\\end{array}\\right]\n",
    "$$\n",
    "\n",
    "$$ x_2 = \\left[\\begin{array}{l}\n",
    "x_2^1\\\\\n",
    "x_2^2\n",
    "\\end{array}\\right]=\n",
    "\\left[\\begin{array}{l}\n",
    "0.1\\\\\n",
    "0.1 \n",
    "\\end{array}\\right]\n",
    "$$\n",
    "\n",
    "$$y = 3\\cdot x_1 + x_2 +  \n",
    "\\left[\\begin{array}{l}\n",
    "    1\\\\\n",
    "    1\n",
    "\\end{array}\\right] = \n",
    "\\left[\\begin{array}{l}\n",
    "    3x_1^1 + x_1^1 + 1\\\\\n",
    "    3x_1^2 + x_2^2 + 1\n",
    "\\end{array}\\right]$$\n",
    "\n",
    "$$f(x_1, x_2) = \\textrm{mean}(y) = \\frac{1}{2}(3x_1^1 + x_1^1 + 1 + 3x_1^2 + x_2^2 + 1)$$\n",
    "\n",
    "$$(\\nabla f)_{x_1} = \\left[\\begin{array}{l}\n",
    "    \\frac{\\partial f}{\\partial x_1^1}\\\\\n",
    "    \\frac{\\partial f}{\\partial x_1^2}\n",
    "\\end{array}\\right] = \n",
    "\\left[\\begin{array}{l}\n",
    "    1.5\\\\\n",
    "    1.5\n",
    "\\end{array}\\right] \n",
    "$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Now in pytorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([1.4000, 1.7000], grad_fn=<AddBackward0>)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x1 = torch.tensor([0.1, 0.2], requires_grad=True)\n",
    "x2 = torch.tensor([0.1, 0.1], requires_grad=True)\n",
    "y = 3 * x1 + x2 + 1\n",
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([1.5000, 1.5000]) tensor([0.5000, 0.5000])\n"
     ]
    }
   ],
   "source": [
    "output = y.mean()\n",
    "\n",
    "output.backward()\n",
    "print(x1.grad, x2.grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The method `backward` calculates gradients. __It does not put them to zero!__ But there is a convinient method for that."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([3., 3.]) tensor([1., 1.])\n"
     ]
    }
   ],
   "source": [
    "y = 3 * x1 + x2 + 1\n",
    "output = y.mean()\n",
    "output.backward()\n",
    "print(x1.grad, x2.grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Let's check if more complicated derivatives are calculated correctly\n",
    "\n",
    "We start with $x = 2$.\n",
    "\n",
    "$y = f(x) = x^2 - 3$, $f(2) = 2^2 - 3 = 1$.\n",
    "\n",
    "$z = g(y) = y^3 - 2y$, $g(f(2)) = g(1) = 1^3 - 2\\cdot 1 = -1$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x =  tensor([2.], requires_grad=True)\n",
      "y =  tensor([1.], grad_fn=<SubBackward0>)\n",
      "z =  tensor([-1.], grad_fn=<SubBackward0>)\n"
     ]
    }
   ],
   "source": [
    "x = torch.tensor([2.0], requires_grad=True)\n",
    "print(\"x = \", x)\n",
    "y = x**2 - 3\n",
    "print(\"y = \", y)\n",
    "z = y**3 - 2*y\n",
    "print(\"z = \", z)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "#### Chain rule\n",
    "$(g(f(x))' = g'(f(x))\\cdot f'(x)$\n",
    "\n",
    "#### Derivaties\n",
    "$f'(x) = 2x$, $f'(2) = 2\\cdot 2 = 4$\n",
    "\n",
    "$g'(y) = 3y^2 - 2$, $g'(f(2)) = g'(1) = 3 - 2 = 1$\n",
    "\n",
    "$(g(f(2))'=g'(f(2))\\cdot f(2)) = 4 \\cdot 1 = 4$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([4.])"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "z.backward()\n",
    "x.grad"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}