Bee_optimal_route.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Bee_optimal_route.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "JV59E-Xtzc6C"
      },
      "source": [
        "import gym\n",
        "import numpy as np\n",
        "import random\n",
        "import torch\n",
        "import time\n",
        "import matplotlib.pyplot as plt\n",
        "import pandas as pd"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ra0EQ9tizpV4"
      },
      "source": [
        "map = ['FF',\n",
        "        'SG']\n"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dO5TnTwPzs70",
        "outputId": "4fcccdd2-ddf9-4505-946d-48e5bed4e014"
      },
      "source": [
        "env = gym.make('FrozenLake-v0', is_slippery=False, desc = map)\n",
        "\n",
        "\n",
        "# visualize 4x4 frozen lake\n",
        "env.render()"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "FF\n",
            "\u001b[41mS\u001b[0mG\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VBFCpO8OzvNr",
        "outputId": "c54a0bc0-57dc-46ce-da4a-0ef51cfe8f41"
      },
      "source": [
        "# Total number of States and Actions\n",
        "n_states = env.observation_space.n\n",
        "n_actions = env.action_space.n\n",
        "n_rows = 2\n",
        "n_cols = 2\n",
        "print( \"States = \", n_states)\n",
        "print( \"Actions = \", n_actions)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "States =  4\n",
            "Actions =  4\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EZBSdQuX6USH"
      },
      "source": [
        "def choose_action(Q, state, epsilon):\n",
        "  random_for_epsilon = np.random.rand()\n",
        "  if random_for_epsilon <= epsilon:\n",
        "    s = Q.loc[state].notna()\n",
        "    vals = s[s].index.values\n",
        "    action = random.choice(vals)\n",
        "  else: \n",
        "    Q.loc[state] += np.random.rand(n_actions,)/100\n",
        "    action = np.argmax(Q.loc[state])\n",
        "  return action"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7zaFrxWw6Wfc"
      },
      "source": [
        "##assign index to each state using state-matrix\n",
        "\n",
        "state_matrix = np.arange(0,n_states).reshape(n_rows,n_cols)\n",
        "state_matrix\n",
        "\n",
        "def rowsandcols(state):\n",
        "  ''' input: state returned by env\n",
        "      output: location of state as (row,col) tuple'''\n",
        "  return int(np.where(state_matrix ==state)[0]), int(np.where(state_matrix ==state)[1])"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L52ggN096YRQ"
      },
      "source": [
        "reps = 100\n",
        "num_episodes = 100"
      ],
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jEh2N-Rg6Z6C"
      },
      "source": [
        "#n_successes = []\n",
        "\n",
        "steps_all_reps= []\n",
        "\n",
        "for i_rep in range(reps):\n",
        "  \n",
        "\n",
        "  Q = pd.DataFrame([[np.NaN, np.random.rand(), np.random.rand(), np.NaN ],\n",
        "             [np.random.rand(),  np.random.rand(), np.NaN, np.NaN],\n",
        "              [np.NaN, np.NaN, -0.7, np.random.rand()],\n",
        "              [np.random.rand()*0.001, np.random.rand()*0.001, np.random.rand()*0.001, np.random.rand()*0.001]])\n",
        "\n",
        "\n",
        "  steps_total = [] # store number of steps taken in each episode\n",
        "  rewards_total = [] #store reward obtained for each episode\n",
        "  epsilon_total = [] #store epsilon obtained at the end of each episode\n",
        "  terminal_state = [] \n",
        "\n",
        "\n",
        "  epsilon = 0.2\n",
        "  epsilon_final = 0.1\n",
        "  epsilon_decay = 0.99\n",
        "  gamma = 0.90 # discount factor\n",
        "  learning_rate = 0.9 #how important is the difference between q-val from q-table and what's observed\n",
        "\n",
        "  for i_episode in range(num_episodes):\n",
        "    \n",
        "    # resets the environment\n",
        "    state = env.reset()\n",
        "    step = 0\n",
        "    reward = 0\n",
        "\n",
        "  ## as epsilon decays with more timesteps, the prob. of selecting a random val < e decays --> more likely to exploit. \n",
        "    if epsilon > epsilon_final:\n",
        "            epsilon *= epsilon_decay\n",
        "\n",
        "    while True:\n",
        "        \n",
        "        step += 1\n",
        "        \n",
        "        action = choose_action(Q, state, epsilon)\n",
        "  \n",
        "         \n",
        "        ## env gives reward and next state and whether we've reached terminal state upon taking action at current state.. \n",
        "        new_state, reward , done, info = env.step(action)\n",
        "\n",
        "        ##if you want reward penalized at for each timestep\n",
        "        #reward = rewarder(new_state, reward)\n",
        "\n",
        "        # filling the Q Table - \n",
        "        \n",
        "        Q.loc[state, action] = (1- learning_rate)*Q.at[state, action] + learning_rate*(reward + gamma * np.max(Q.loc[new_state]))\n",
        "        \n",
        "        # Setting new state for next action\n",
        "        state = new_state\n",
        "        tile = map[rowsandcols(state)[0]][rowsandcols(state)[1]]\n",
        "        #env.render()\n",
        "        \n",
        "        if done:\n",
        "          #print(Q)\n",
        "          \n",
        "          terminal_state.append(tile)\n",
        "          steps_total.append(step)\n",
        "          rewards_total.append(reward)\n",
        "          epsilon_total.append(epsilon)\n",
        "          #print('Episode: {} Reward: {} Steps Taken: {} Terminal State: {}, Epsilon: {}'.format(i_episode,reward, step, tile, epsilon))\n",
        "          break\n",
        "  steps_all_reps.append(steps_total)\n",
        "  #n_successes.append(terminal_state.count('G'))"
      ],
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HxN-ATdJfuRR"
      },
      "source": [
        "mean_rep_step = np.mean(steps_all_reps, axis = 0)"
      ],
      "execution_count": 42,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NGKnDNpa9WjL",
        "outputId": "695c413e-b76b-4243-b38c-14b8b501a71b"
      },
      "source": [
        "np.std(steps_all_reps, axis = 0)"
      ],
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([9.18718673, 2.05873748, 1.47986486, 1.33626345, 1.30766968,\n",
              "       1.47986486, 1.07777549, 1.08277421, 1.50983443, 1.28934092,\n",
              "       1.39985714, 1.14455231, 1.38260623, 1.36747943, 1.03923048,\n",
              "       1.14175304, 1.        , 1.28436755, 0.86625631, 0.80099938,\n",
              "       0.80274529, 0.71414284, 0.88181631, 0.64992307, 0.7825599 ,\n",
              "       0.84947042, 0.57236352, 0.68673139, 0.6       , 0.72111026,\n",
              "       0.61188234, 0.64992307, 0.47497368, 0.92606695, 0.70738957,\n",
              "       0.9335952 , 0.63843559, 0.72111026, 0.7825599 , 0.58343809,\n",
              "       0.6       , 0.82462113, 0.5425864 , 0.51029403, 0.58343809,\n",
              "       0.51961524, 0.552811  , 0.63843559, 0.5425864 , 0.58343809,\n",
              "       0.34117444, 0.34117444, 0.34117444, 0.61188234, 0.51029403,\n",
              "       0.66332496, 0.51029403, 0.34117444, 0.39191836, 0.47497368,\n",
              "       0.34117444, 0.69828361, 0.57236352, 0.51029403, 0.63843559,\n",
              "       0.69828361, 0.39191836, 0.47497368, 0.51029403, 0.8138796 ,\n",
              "       0.552811  , 0.58343809, 0.69828361, 0.58343809, 0.43588989,\n",
              "       0.43588989, 0.28      , 0.58343809, 0.34117444, 0.39191836,\n",
              "       0.39191836, 0.39191836, 0.62096699, 0.39191836, 0.43588989,\n",
              "       0.6       , 0.61188234, 0.66332496, 0.58343809, 0.48332184,\n",
              "       0.28      , 0.68234888, 0.58343809, 0.5425864 , 0.34117444,\n",
              "       0.57236352, 0.51961524, 0.5425864 , 0.34117444, 0.61188234])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JLP1GYuw9jaq"
      },
      "source": [
        "import plotly.express as px"
      ],
      "execution_count": 46,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 417
        },
        "id": "igtvi2IrTgkO",
        "outputId": "973bcbec-7969-40b5-cf8b-48e9698d51e7"
      },
      "source": [
        "fig = px.bar( x= np.arange(1,101), y= mean_rep_step, width=800, height=400)\n",
        "fig.update_layout(\n",
        "    title=\" Average steps taken in an episode out of 100 reps \",\n",
        "    xaxis_title='episode',\n",
        "    #x_axis_range = [0,101],\n",
        "    yaxis_title=\" average step taken in an episode\",\n",
        "    yaxis_range=[0,11],\n",
        "    font=dict(\n",
        "        family=\"Courier New, monospace\",\n",
        "        size=15))\n",
        "\n",
        "\n",
        "fig.show()"
      ],
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>\n",
              "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
              "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
              "            <div id=\"2b8675b7-4a1c-4601-bdf8-22a457652611\" class=\"plotly-graph-div\" style=\"height:400px; width:800px;\"></div>\n",
              "            <script type=\"text/javascript\">\n",
              "                \n",
              "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
              "                    \n",
              "                if (document.getElementById(\"2b8675b7-4a1c-4601-bdf8-22a457652611\")) {\n",
              "                    Plotly.newPlot(\n",
              "                        '2b8675b7-4a1c-4601-bdf8-22a457652611',\n",
              "                        [{\"alignmentgroup\": \"True\", \"hoverlabel\": {\"namelength\": 0}, \"hovertemplate\": \"x=%{x}<br>y=%{y}\", \"legendgroup\": \"\", \"marker\": {\"color\": \"#636efa\"}, \"name\": \"\", \"offsetgroup\": \"\", \"orientation\": \"v\", \"showlegend\": false, \"textposition\": \"auto\", \"type\": \"bar\", \"x\": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100], \"xaxis\": \"x\", \"y\": [9.66, 2.96, 2.3, 2.12, 2.1, 2.1, 1.72, 1.74, 1.98, 1.76, 2.02, 1.7, 1.78, 1.7, 1.6, 1.58, 1.6, 1.52, 1.36, 1.28, 1.34, 1.3, 1.32, 1.24, 1.26, 1.28, 1.18, 1.22, 1.2, 1.2, 1.16, 1.24, 1.12, 1.32, 1.14, 1.22, 1.18, 1.2, 1.26, 1.14, 1.2, 1.2, 1.16, 1.14, 1.14, 1.1, 1.12, 1.18, 1.16, 1.14, 1.06, 1.06, 1.06, 1.16, 1.14, 1.2, 1.14, 1.06, 1.08, 1.12, 1.06, 1.18, 1.18, 1.14, 1.18, 1.18, 1.08, 1.12, 1.14, 1.24, 1.12, 1.14, 1.18, 1.14, 1.1, 1.1, 1.04, 1.14, 1.06, 1.08, 1.08, 1.08, 1.12, 1.08, 1.1, 1.2, 1.16, 1.2, 1.14, 1.08, 1.04, 1.12, 1.14, 1.16, 1.06, 1.18, 1.1, 1.16, 1.06, 1.16], \"yaxis\": \"y\"}],\n",
              "                        {\"barmode\": \"relative\", \"font\": {\"family\": \"Courier New, monospace\", \"size\": 15}, \"height\": 400, \"legend\": {\"tracegroupgap\": 0}, \"margin\": {\"t\": 60}, \"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"pie\": [{\"automargin\": true, \"type\": \"pie\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"coloraxis\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"title\": {\"standoff\": 15}, \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"title\": {\"standoff\": 15}, \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \" Average steps taken in an episode out of 100 reps \"}, \"width\": 800, \"xaxis\": {\"anchor\": \"y\", \"domain\": [0.0, 1.0], \"title\": {\"text\": \"episode\"}}, \"yaxis\": {\"anchor\": \"x\", \"domain\": [0.0, 1.0], \"range\": [0, 11], \"title\": {\"text\": \" average step taken in an episode\"}}},\n",
              "                        {\"responsive\": true}\n",
              "                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('2b8675b7-4a1c-4601-bdf8-22a457652611');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })\n",
              "                };\n",
              "                \n",
              "            </script>\n",
              "        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Rw5lb5pY9ryT",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173
        },
        "outputId": "599003ad-0990-4007-a9f4-5a5eda961139"
      },
      "source": [
        "Q"
      ],
      "execution_count": 57,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>NaN</td>\n",
              "      <td>0.901572</td>\n",
              "      <td>0.741194</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.812408</td>\n",
              "      <td>0.264836</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>1.001771</td>\n",
              "      <td>0.870048</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.000741</td>\n",
              "      <td>0.000424</td>\n",
              "      <td>0.000702</td>\n",
              "      <td>0.000880</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "          0         1         2         3\n",
              "0       NaN  0.901572  0.741194       NaN\n",
              "1  0.812408  0.264836       NaN       NaN\n",
              "2       NaN       NaN  1.001771  0.870048\n",
              "3  0.000741  0.000424  0.000702  0.000880"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 57
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6rWrgmpSX4Fp"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}