examples/tf8_demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "cfa88313",
   "metadata": {},
   "source": [
    "# Predicting transcription factor binding affinity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bbce78c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import OrderedDict\n",
    "from pprint import pprint\n",
    "import pyro\n",
    "import torch\n",
    "import numpy as np\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from pyroed.datasets.data import load_tf_data\n",
    "from pyroed.constraints import AllDifferent, Iff, IfThen, TakesValue\n",
    "from pyroed.oed import thompson_sample\n",
    "from pyroed.testing import generate_fake_data\n",
    "\n",
    "matplotlib.rcParams[\"figure.facecolor\"] = \"white\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3168d890",
   "metadata": {},
   "source": [
    "### Transcription factor data\n",
    "\n",
    "We consider data from [Survey of variation in human transcription factors reveals prevalent DNA binding changes](https://www.science.org/doi/abs/10.1126/science.aad2257),\n",
    "in particular for the PBX4 transcription factor.\n",
    "The dataset consists of measurements of the binding affinities of PBX4 to all\n",
    "possible DNA sequences of length 8, i.e. for a total of $4^8 = 65536$ sequences.\n",
    "Since this dataset is exhaustive we can use it to do a semi-synthetic experiment in which\n",
    "we first \"measure\" a small number of binding affinities and then do additional \"experiments\" in multiple rounds."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b9c5f1dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = load_tf_data(data_dir=\"../pyroed/datasets\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "dbcf570c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sequences torch.int64 (65792, 8)\n",
      "responses torch.float32 (65792,)\n",
      "batch_ids torch.int64 (65792,)\n"
     ]
    }
   ],
   "source": [
    "for k, v in data.items():\n",
    "    print(f\"{k} {v.dtype} {tuple(v.shape)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "add4b8cb",
   "metadata": {},
   "source": [
    "Note that there are actually more than $65536$ data points due to some repeats."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "63917f48",
   "metadata": {},
   "source": [
    "Sequences take values 0,1,2,3 corresponding to nucleotides."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "bf570325",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0, 1, 2, 3}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(data[\"sequences\"].reshape(-1).tolist())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fb9d3635",
   "metadata": {},
   "source": [
    "The response variable appears to be approximately Gaussian distributed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e1ebbbac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD7CAYAAACG50QgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAZj0lEQVR4nO3df0yd5f3/8ecpTJJ1GO0EdtgNFj0M4Uit9hSLyVyVna3Hr4Np3QmdhsO0IcIWZvbHwj+Lc8nG6ZIlNo6wnYSYw36dhCUDo/Uwp3bLGpEd1G6CP06UGs7ZGdCBQx1ii/f3D7/fs48fYJwWOJRer0fSpFzcF+d6p82LK9d93dftsG3bRkREjLBtswcgIiLZo9AXETGIQl9ExCAKfRERgyj0RUQMotAXETHIqqH/2muvsXv37vSfSy+9lIcffpiZmRm8Xi/l5eV4vV5mZ2fTfTo7O3G5XFRUVDA4OJhuHxkZobq6GpfLRXt7O9otKiKSXY5z2ae/uLjIZz/7WZ5//nm6urrYsWMHHR0dBINBZmdnOXLkCGNjYxw6dIjh4WH+/ve/88UvfpHXX3+dnJwcampqOHr0KPv27eO2226jvb0dn8+3kfWJiMj/kHsuFz/99NNcffXVXHnllQwMDHD8+HEAAoEA+/fv58iRIwwMDNDY2EheXh5lZWW4XC6Gh4fZuXMnc3Nz1NbWAtDU1ER/f/+qoX/FFVewc+fO8ypORMRUp06d4vTp00vazyn0I5EIhw4dAmBychKn0wmA0+lkamoKgGQyyb59+9J9LMsimUzyiU98AsuylrQvJxQKEQqFANi+fTuxWOxchikiYjyPx7Nse8Y3cj/44AMee+wxvva1r/3X65ZbLXI4HCu2L6elpYVYLEYsFqOgoCDTIYqIyCoyDv0nn3ySG264gaKiIgCKiopIpVIApFIpCgsLgY9m8BMTE+l+iUSC4uJiLMsikUgsaRcRkezJOPR/85vfpJd2AOrr6wmHwwCEw2EaGhrS7ZFIhIWFBcbHx4nH49TU1OB0OsnPz2doaAjbtunt7U33ERGR7MhoTf/f//43Tz31FD//+c/TbR0dHfj9fnp6eigtLaWvrw8At9uN3++nqqqK3Nxcurq6yMnJAaC7u5vm5mbm5+fx+XzauSMikmXntGVzM3g8Ht3IFRE5Rytlp57IFRExiEJfRMQgCn0REYMo9EVEDHJOT+SKXIh2djxx3n1PBf/POo5E5MKnmb6IiEEU+iIiBtHyjhhNS0NiGs30RUQMotAXETGIQl9ExCAKfRERgyj0RUQMot07sunWsoNGRM6NQl/kPGm7p2xFWt4RETGIQl9ExCAKfRERgyj0RUQMotAXETGIQl9ExCAKfRERg2QU+m+//TZ33XUX11xzDZWVlTz33HPMzMzg9XopLy/H6/UyOzubvr6zsxOXy0VFRQWDg4Pp9pGREaqrq3G5XLS3t2Pb9vpXJCIiK8oo9L/97W9z4MABXn31VU6ePEllZSXBYJC6ujri8Th1dXUEg0EAxsbGiEQijI6OEo1GaWtrY3FxEYDW1lZCoRDxeJx4PE40Gt24ykREZIlVQ39ubo4//elP3HfffQBccsklXHbZZQwMDBAIBAAIBAL09/cDMDAwQGNjI3l5eZSVleFyuRgeHiaVSjE3N0dtbS0Oh4OmpqZ0HxERyY5VQ//NN9+koKCAb3zjG1x//fUcPnyY9957j8nJSZxOJwBOp5OpqSkAkskkJSUl6f6WZZFMJkkmk1iWtaR9OaFQCI/Hg8fjYXp6ek0FiojIf6wa+mfPnuWFF16gtbWVF198ke3bt6eXcpaz3Dq9w+FYsX05LS0txGIxYrEYBQUFqw1RREQytOqBa5ZlYVkWN954IwB33XUXwWCQoqIiUqkUTqeTVCpFYWFh+vqJiYl0/0QiQXFxMZZlkUgklrSLmGitJ4vqwDY5X6vO9D/zmc9QUlLCa6+9BsDTTz9NVVUV9fX1hMNhAMLhMA0NDQDU19cTiURYWFhgfHyceDxOTU0NTqeT/Px8hoaGsG2b3t7edB8REcmOjI5WfuSRR7j77rv54IMPuOqqq3j00Uf58MMP8fv99PT0UFpaSl9fHwButxu/309VVRW5ubl0dXWRk5MDQHd3N83NzczPz+Pz+fD5fBtXmYiILOGwL/DN8h6Ph1gsttnDkA2kl6icOy3vyGpWyk49kSsiYhCFvoiIQRT6IiIGUeiLiBhEL0aXdaGbsSJbg2b6IiIGUeiLiBhEoS8iYhCFvoiIQRT6IiIGUeiLiBhEoS8iYhCFvoiIQRT6IiIGUeiLiBhEoS8iYhCFvoiIQRT6IiIGUeiLiBhEoS8iYhCdpy+yBa3l/QV6qbrZNNMXETFIRqG/c+dOqqur2b17Nx6PB4CZmRm8Xi/l5eV4vV5mZ2fT13d2duJyuaioqGBwcDDdPjIyQnV1NS6Xi/b2dmzbXudyRETkv8l4pv/ss8/y0ksvEYvFAAgGg9TV1RGPx6mrqyMYDAIwNjZGJBJhdHSUaDRKW1sbi4uLALS2thIKhYjH48TjcaLR6AaUJCIiKznv5Z2BgQECgQAAgUCA/v7+dHtjYyN5eXmUlZXhcrkYHh4mlUoxNzdHbW0tDoeDpqamdB8REcmOjELf4XDwpS99iT179hAKhQCYnJzE6XQC4HQ6mZqaAiCZTFJSUpLua1kWyWSSZDKJZVlL2pcTCoXweDx4PB6mp6fPrzIREVkio907J06coLi4mKmpKbxeL9dcc82K1y63Tu9wOFZsX05LSwstLS0A6XsIIiKydhnN9IuLiwEoLCzkjjvuYHh4mKKiIlKpFACpVIrCwkLgoxn8xMREum8ikaC4uBjLskgkEkvaRUQke1YN/ffee4933nkn/fff//73XHvttdTX1xMOhwEIh8M0NDQAUF9fTyQSYWFhgfHxceLxODU1NTidTvLz8xkaGsK2bXp7e9N9REQkO1Zd3pmcnOSOO+4A4OzZs3z961/nwIED7N27F7/fT09PD6WlpfT19QHgdrvx+/1UVVWRm5tLV1cXOTk5AHR3d9Pc3Mz8/Dw+nw+fz7eBpYmIyP/msC/wzfIejye9TVQuXGt5QlSyS0/kmmGl7NQTuSIiBlHoi4gYRKEvImIQhb6IiEEU+iIiBlHoi4gYRKEvImIQhb6IiEH0ukRJ0wNWIhc/zfRFRAyi0BcRMYhCX0TEIFrTFzHMWu7d6LC2rU8zfRERgyj0RUQMotAXETGIQl9ExCAKfRERgyj0RUQMotAXETGIQl9ExCAKfRERg2Qc+ouLi1x//fXcfvvtAMzMzOD1eikvL8fr9TI7O5u+trOzE5fLRUVFBYODg+n2kZERqqurcblctLe3Y9v2OpYiIiKryTj0jx49SmVlZfrrYDBIXV0d8Xicuro6gsEgAGNjY0QiEUZHR4lGo7S1tbG4uAhAa2sroVCIeDxOPB4nGo2uczkiIvLfZBT6iUSCJ554gsOHD6fbBgYGCAQCAAQCAfr7+9PtjY2N5OXlUVZWhsvlYnh4mFQqxdzcHLW1tTgcDpqamtJ9REQkOzIK/QceeIAf//jHbNv2n8snJydxOp0AOJ1OpqamAEgmk5SUlKSvsyyLZDJJMpnEsqwl7csJhUJ4PB48Hg/T09PnXpWIiCxr1dB//PHHKSwsZM+ePRn9wOXW6R0Ox4rty2lpaSEWixGLxSgoKMjoc0VEZHWrHq184sQJHnvsMY4dO8b777/P3Nwc99xzD0VFRaRSKZxOJ6lUisLCQuCjGfzExES6fyKRoLi4GMuySCQSS9pFRCR7Vp3pd3Z2kkgkOHXqFJFIhFtvvZVf/vKX1NfXEw6HAQiHwzQ0NABQX19PJBJhYWGB8fFx4vE4NTU1OJ1O8vPzGRoawrZtent7031ERCQ7zvslKh0dHfj9fnp6eigtLaWvrw8At9uN3++nqqqK3Nxcurq6yMnJAaC7u5vm5mbm5+fx+Xz4fL71qUJERDLisC/wzfIej4dYLLbZwzDCWt6oJGbQm7O2jpWyU0/kiogYRKEvImIQhb6IiEEU+iIiBlHoi4gYRKEvImIQhb6IiEEU+iIiBlHoi4gYRKEvImIQhb6IiEEU+iIiBlHoi4gYRKEvImKQ8z5PX0TMs5bjt3Us84VBM30REYMo9EVEDKLQFxExiEJfRMQgCn0REYMo9EVEDKLQFxExyKqh//7771NTU8N1112H2+3mwQcfBGBmZgav10t5eTler5fZ2dl0n87OTlwuFxUVFQwODqbbR0ZGqK6uxuVy0d7ejm3bG1CSiIisZNXQz8vL45lnnuHkyZO89NJLRKNRhoaGCAaD1NXVEY/HqaurIxgMAjA2NkYkEmF0dJRoNEpbWxuLi4sAtLa2EgqFiMfjxONxotHoxlYnIiIfs+oTuQ6Hg0996lMAnDlzhjNnzuBwOBgYGOD48eMABAIB9u/fz5EjRxgYGKCxsZG8vDzKyspwuVwMDw+zc+dO5ubmqK2tBaCpqYn+/n58Pt/GVWegtTwxKSIXv4zW9BcXF9m9ezeFhYV4vV5uvPFGJicncTqdADidTqampgBIJpOUlJSk+1qWRTKZJJlMYlnWkvblhEIhPB4PHo+H6enp8y5OREQ+LqPQz8nJ4aWXXiKRSDA8PMzLL7+84rXLrdM7HI4V25fT0tJCLBYjFotRUFCQyRBFRCQD57R757LLLmP//v1Eo1GKiopIpVIApFIpCgsLgY9m8BMTE+k+iUSC4uJiLMsikUgsaRcRkexZNfSnp6d5++23AZifn+cPf/gD11xzDfX19YTDYQDC4TANDQ0A1NfXE4lEWFhYYHx8nHg8Tk1NDU6nk/z8fIaGhrBtm97e3nQfERHJjlVv5KZSKQKBAIuLi3z44Yf4/X5uv/12amtr8fv99PT0UFpaSl9fHwButxu/309VVRW5ubl0dXWRk5MDQHd3N83NzczPz+Pz+XQTV0Qkyxz2Bb5Z3uPxEIvFNnsYW4Z278iFSufpZ9dK2aknckVEDKLQFxExiEJfRMQgCn0REYMo9EVEDKLQFxExiEJfRMQgCn0REYMo9EVEDKLQFxExiEJfRMQgCn0REYMo9EVEDKLQFxExiEJfRMQgCn0REYMo9EVEDKLQFxExiEJfRMQgq74YXURkPazl/c16v+760UxfRMQgCn0REYOsGvoTExPccsstVFZW4na7OXr0KAAzMzN4vV7Ky8vxer3Mzs6m+3R2duJyuaioqGBwcDDdPjIyQnV1NS6Xi/b2dmzb3oCSRERkJauGfm5uLj/5yU945ZVXGBoaoquri7GxMYLBIHV1dcTjcerq6ggGgwCMjY0RiUQYHR0lGo3S1tbG4uIiAK2trYRCIeLxOPF4nGg0urHViYjIx6wa+k6nkxtuuAGA/Px8KisrSSaTDAwMEAgEAAgEAvT39wMwMDBAY2MjeXl5lJWV4XK5GB4eJpVKMTc3R21tLQ6Hg6ampnQfERHJjnNa0z916hQvvvgiN954I5OTkzidTuCjXwxTU1MAJJNJSkpK0n0syyKZTJJMJrEsa0n7ckKhEB6PB4/Hw/T09DkXJSIiy8s49N99910OHjzIww8/zKWXXrridcut0zscjhXbl9PS0kIsFiMWi1FQUJDpEEVEZBUZhf6ZM2c4ePAgd999N3feeScARUVFpFIpAFKpFIWFhcBHM/iJiYl030QiQXFxMZZlkUgklrSLiEj2rBr6tm1z3333UVlZyXe+8510e319PeFwGIBwOExDQ0O6PRKJsLCwwPj4OPF4nJqaGpxOJ/n5+QwNDWHbNr29vek+IiKSHas+kXvixAl+8YtfUF1dze7duwH40Y9+REdHB36/n56eHkpLS+nr6wPA7Xbj9/upqqoiNzeXrq4ucnJyAOju7qa5uZn5+Xl8Ph8+n28DSxMRkf/NYV/gm+U9Hg+xWGyzh7FlrOVRd5ELlY5hOHcrZafO3rkAKbhFZKPoGAYREYMo9EVEDKLQFxExiEJfRMQgCn0REYMo9EVEDKLQFxExiEJfRMQgCn0REYMo9EVEDKLQFxExiEJfRMQgCn0REYMo9EVEDKLQFxExiEJfRMQgCn0REYPozVkicsFb69vk9LrF/9BMX0TEIAp9ERGDKPRFRAyyaujfe++9FBYWcu2116bbZmZm8Hq9lJeX4/V6mZ2dTX+vs7MTl8tFRUUFg4OD6faRkRGqq6txuVy0t7dj2/Y6lyIiIqtZNfSbm5uJRqMfawsGg9TV1RGPx6mrqyMYDAIwNjZGJBJhdHSUaDRKW1sbi4uLALS2thIKhYjH48Tj8SU/U0RENt6qoX/zzTezY8eOj7UNDAwQCAQACAQC9Pf3p9sbGxvJy8ujrKwMl8vF8PAwqVSKubk5amtrcTgcNDU1pfuIiEj2nNea/uTkJE6nEwCn08nU1BQAyWSSkpKS9HWWZZFMJkkmk1iWtaR9JaFQCI/Hg8fjYXp6+nyGKCIiy1jXG7nLrdM7HI4V21fS0tJCLBYjFotRUFCwnkMUETHaeYV+UVERqVQKgFQqRWFhIfDRDH5iYiJ9XSKRoLi4GMuySCQSS9pFRCS7ziv06+vrCYfDAITDYRoaGtLtkUiEhYUFxsfHicfj1NTU4HQ6yc/PZ2hoCNu26e3tTfcREZHsWfUYhkOHDnH8+HFOnz6NZVk89NBDdHR04Pf76enpobS0lL6+PgDcbjd+v5+qqipyc3Pp6uoiJycHgO7ubpqbm5mfn8fn8+Hz+Ta2sk201kfGRUQ2isO+wDfMezweYrHYZg/jnCj0RS4sJp69s1J26olcERGDKPRFRAyi0BcRMYhCX0TEIAp9ERGD6M1ZInLRW8uOuott549m+iIiBlHoi4gYRKEvImIQhb6IiEEU+iIiBlHoi4gYRKEvImIQ7dMXEfkvLrY9/prpi4gYRKEvImIQLe+sQC9CEZGLkWb6IiIGUeiLiBhEoS8iYhCFvoiIQXQjV0Rkg1yIe/yzPtOPRqNUVFTgcrkIBoPZ/ngREaNldaa/uLjIN7/5TZ566iksy2Lv3r3U19dTVVW1IZ+nbZciIh+X1Zn+8PAwLpeLq666iksuuYTGxkYGBgayOQQREaNldaafTCYpKSlJf21ZFs8///yS60KhEKFQCIBXX30Vj8dzXp9nT09TUFBwfoPdoqZVsxFMq9m0egGuvLJtTTWfOnVq2fashr5t20vaHA7HkraWlhZaWlrW/Hkej4dYLLbmn7OVqGYzmFazafXCxtWc1eUdy7KYmJhIf51IJCguLs7mEEREjJbV0N+7dy/xeJzx8XE++OADIpEI9fX12RyCiIjRcr7//e9/P1sftm3bNsrLy7nnnnt45JFHuOeeezh48OCGfuaePXs29OdfiFSzGUyr2bR6YWNqdtjLLbSLiMhFSccwiIgYRKEvImKQiyL0VzvawbZt2tvbcblc7Nq1ixdeeGETRrl+Vqv3V7/6Fbt27WLXrl3cdNNNnDx5chNGub4yPb7jL3/5Czk5Ofz2t7/N4ug2RiY1Hz9+nN27d+N2u/nCF76Q5RGuv9Vq/te//sVXvvIVrrvuOtxuN48++ugmjHL93HvvvRQWFnLttdcu+/0NyS57izt79qx91VVX2W+88Ya9sLBg79q1yx4dHf3YNU888YR94MAB+8MPP7Sfe+45u6amZpNGu3aZ1HvixAl7ZmbGtm3bPnbs2Jau17Yzq/n/X3fLLbfYPp/P7uvr24SRrp9Map6dnbUrKyvtt956y7Zt256cnNyMoa6bTGr+4Q9/aH/3u9+1bdu2p6am7Msvv9xeWFjYjOGuiz/+8Y/2yMiI7Xa7l/3+RmTXlp/pZ3K0w8DAAE1NTTgcDvbt28fbb79NKpXapBGvTSb13nTTTVx++eUA7Nu3j0QisRlDXTeZHt/xyCOPcPDgQQoLCzdhlOsrk5p//etfc+edd1JaWgqw5evOpGaHw8E777yDbdu8++677Nixg9zcrXtY8M0338yOHTtW/P5GZNeWD/3ljnZIJpPnfM1Wca619PT04PP5sjG0DZPpv/Hvfvc77r///mwPb0NkUvPrr7/O7Ows+/fvZ8+ePfT29mZ7mOsqk5q/9a1v8corr1BcXEx1dTVHjx5l27YtH2Mr2ojs2rq/Iv8fO4OjHTK5Zqs4l1qeffZZenp6+POf/7zRw9pQmdT8wAMPcOTIEXJycrI1rA2VSc1nz55lZGSEp59+mvn5eWpra9m3bx+f+9znsjXMdZVJzYODg+zevZtnnnmGN954A6/Xy+c//3kuvfTSbA0zqzYiu7Z86GdytMPFdPxDprX89a9/5fDhwzz55JN8+tOfzuYQ110mNcdiMRobGwE4ffo0x44dIzc3l69+9atZHet6yfT/9RVXXMH27dvZvn07N998MydPntyyoZ9JzY8++igdHR04HA5cLhdlZWW8+uqr1NTUZHu4WbEh2bXmuwKb7MyZM3ZZWZn95ptvpm/+vPzyyx+75vHHH//YzZC9e/du0mjXLpN633rrLfvqq6+2T5w4sUmjXF+Z1Pw/BQKBLX8jN5Oax8bG7FtvvdU+c+aM/d5779lut9v+29/+tkkjXrtMar7//vvtBx980LZt2/7HP/5hFxcX29PT05sw2vUzPj6+4o3cjciuLT/Tz83N5ac//Slf/vKXWVxc5N5778XtdvOzn/0MgPvvv5/bbruNY8eO4XK5+OQnP7mlt3llUu8PfvAD/vnPf9LW1pbus5VPKMyk5otNJjVXVlZy4MABdu3axbZt2zh8+PCKW/+2gkxq/t73vkdzczPV1dXYts2RI0e44oorNnnk5+/QoUMcP36c06dPY1kWDz30EGfOnAE2Lrt0DIOIiEEu3tveIiKyhEJfRMQgCn0REYMo9EVEDKLQFxExiEJfRMQgCn0REYP8X+DB2OLe9oJNAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(data[\"responses\"].numpy(), bins=20)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f0369fac",
   "metadata": {},
   "source": [
    "## Modeling in pyroed\n",
    "\n",
    "Specify the design space via SCHEMA, CONSTRAINTS, FEATURE_BLOCKS, and GIBBS_BLOCKS."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a5568f60",
   "metadata": {},
   "outputs": [],
   "source": [
    "SCHEMA = OrderedDict()\n",
    "for n in range(8):\n",
    "    SCHEMA[f\"Nucleotide{n}\"] = [\"A\", \"C\", \"G\", \"T\"]\n",
    "\n",
    "CONSTRAINTS = []  # No constraints.\n",
    "\n",
    "singletons = [[name] for name in SCHEMA]\n",
    "pairs = [list(ns) for ns in zip(SCHEMA, list(SCHEMA)[1:])]\n",
    "triples = [list(ns) for ns in zip(SCHEMA, list(SCHEMA)[1:], list(SCHEMA)[2:])]\n",
    "\n",
    "FEATURE_BLOCKS = singletons + pairs\n",
    "GIBBS_BLOCKS = triples"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "53014fc6",
   "metadata": {},
   "source": [
    "Let's start with a random subsample of data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "921550d3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best response = 0.574323\n"
     ]
    }
   ],
   "source": [
    "pyro.set_rng_seed(0)\n",
    "full_size = len(data[\"responses\"])\n",
    "batch_size = 10\n",
    "ids = torch.randperm(full_size)[:batch_size]\n",
    "experiment = {k: v[ids] for k, v in data.items()}\n",
    "print(f\"Best response = {experiment['responses'].max():0.6g}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a92d114c",
   "metadata": {},
   "source": [
    "Each step of the OED process we'll test on new data. Let's make a helper to simulate lab work."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b56ba9d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "seq_to_id = {tuple(row): i for i, row in enumerate(data[\"sequences\"].tolist())}\n",
    "\n",
    "def update_experiment(experiment: dict, design: set) -> dict:\n",
    "    batch_ids = experiment[\"batch_ids\"].max().item() + 1\n",
    "    ids = list(map(seq_to_id.__getitem__, sorted(design)))\n",
    "    new_data = {\n",
    "        \"sequences\": data[\"sequences\"][ids],\n",
    "        \"batch_ids\": torch.full((len(ids),), batch_ids),\n",
    "        \"responses\": data[\"responses\"][ids],\n",
    "    }\n",
    "    experiment = {k: torch.cat([v, new_data[k]]) for k, v in experiment.items()}\n",
    "    print(f\"Best response = {experiment['responses'].max():0.6g}\")\n",
    "    return experiment\n",
    "\n",
    "def make_design(experiment: dict) -> set:\n",
    "    return thompson_sample(\n",
    "        SCHEMA,\n",
    "        CONSTRAINTS,\n",
    "        FEATURE_BLOCKS,\n",
    "        GIBBS_BLOCKS,\n",
    "        experiment,\n",
    "        inference=\"svi\",\n",
    "        svi_num_steps=201,\n",
    "        sa_num_steps=201,\n",
    "        log_every=0,\n",
    "        jit_compile=False,\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2d74b628",
   "metadata": {},
   "source": [
    "Initialize our sequence of experiments:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "4b2c2ed7",
   "metadata": {},
   "outputs": [],
   "source": [
    "experiments = [experiment]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6bcabacb",
   "metadata": {},
   "source": [
    "Let's start with a single loop of active learning:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "7b1ffbf8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Design:\n",
      "{(0, 2, 3, 3, 1, 2, 1, 1),\n",
      " (0, 3, 0, 2, 0, 1, 3, 1),\n",
      " (1, 0, 3, 1, 0, 1, 0, 0),\n",
      " (1, 0, 3, 3, 3, 3, 1, 0),\n",
      " (1, 3, 0, 2, 0, 0, 3, 0),\n",
      " (2, 1, 1, 0, 2, 3, 2, 1),\n",
      " (2, 1, 3, 1, 3, 3, 3, 1),\n",
      " (2, 2, 0, 0, 2, 3, 1, 1),\n",
      " (3, 1, 3, 0, 0, 0, 3, 0),\n",
      " (3, 3, 3, 0, 0, 2, 3, 1)}\n",
      "Best response = 0.905897\n",
      "CPU times: user 4.91 s, sys: 17.3 ms, total: 4.93 s\n",
      "Wall time: 4.93 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "design = make_design(experiments[-1])\n",
    "print(\"Design:\")\n",
    "pprint(design)\n",
    "experiments.append(update_experiment(experiments[-1], design))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8047d410",
   "metadata": {},
   "source": [
    "Let's run multiple loops, say 10 more loops."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "cf15268e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best response = 0.905897\n",
      "Best response = 0.905897\n",
      "Best response = 0.905897\n",
      "Best response = 0.94027\n",
      "Best response = 0.94027\n",
      "Best response = 0.959295\n",
      "Best response = 0.959295\n",
      "Best response = 0.959295\n",
      "Best response = 0.959295\n",
      "Best response = 0.959295\n",
      "CPU times: user 53.3 s, sys: 175 ms, total: 53.4 s\n",
      "Wall time: 53.5 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "for step in range(10):\n",
    "    design = make_design(experiments[-1])\n",
    "    experiments.append(update_experiment(experiments[-1], design))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0a479832",
   "metadata": {},
   "source": [
    "How did the response improve over time?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5b1d0c01",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dfVhUZf4/8PfA8CAIKAIqDog4iDyjDJhrQepXwR6GVcsltVJyiUKzb2XRfjftu3WVrd/1oahl+fmQ5iqurUraRpqZ5UOyqOiWDwxPBmPLjAjIgAgznN8frFMjg4PGMMzM+3VdXNfOmfuc+Ryvdt5zzn2f+xYJgiCAiIjoFg6WLoCIiPonBgQRERnFgCAiIqMYEEREZBQDgoiIjBJbuoDe5OPjg6CgIEuXQURkNaqqqnDlyhWj79lUQAQFBaG4uNjSZRARWQ2ZTNbte7zFRERERjEgiIjIKAYEEREZxYAgIiKjGBBERGQUA4KIiIxiQBARkVE29RwEEd05QRBwpqYR35Sq0a7rsHQ5dBfcXMTITBrd68dlQBDZqXK1BgUll/FJiRJVdS0AAJHIwkXRXfEZ6MKAIKJf5t+Nrdh39jIKSi7jX8pGiETAr0YPwbP3S5EcOQxeA5wsXSL1I2YNiMLCQixduhQ6nQ6LFi1Cdna2wfv19fVIT09HeXk5XF1dsXHjRkRGRgLonDbDw8MDjo6OEIvFnEKD6C41Xm9H4Xc/oqDkMo5X1EEQgGiJF37/YBgejvHHUE9XS5dI/ZTZAkKn0yErKwsHDhyARCJBfHw85HI5wsPD9W3eeustxMbGYvfu3bhw4QKysrJw8OBB/fuHDh2Cj4+PuUokslmt7Tp8eUGFghIlDl1Qo03XgaAhbnhuSgjksf4Y7TvQ0iWSFTBbQBQVFUEqlSI4OBgAkJaWhoKCAoOAOHfuHF599VUAwNixY1FVVYXa2loMHTrUXGUR2Sxdh4Dj5XXYU6LE59/9G003tPD1cMH8e0YiNdYf0RIviNjJQHfAbAGhVCoREBCgfy2RSHDixAmDNjExMdi1axfuvfdeFBUV4dKlS6ipqcHQoUMhEokwffp0iEQiPP3008jIyDD6OXl5ecjLywMAqNVqc50OUb8kCALO1jRiT4kS+87+CHXTDXi4iJESOQypsSMwcfQQODowFOjumC0gBEHosu3WXy/Z2dlYunQpYmNjERUVhXHjxkEs7izp6NGj8Pf3h0qlwrRp0zB27FgkJiZ2OWZGRoY+PG43bS2RLalQa7DnZyOQnB0dMGWsH1Jj/TF5rB9cnRwtXSLZALMFhEQiQXV1tf51TU0N/P39Ddp4enpi06ZNADoDZdSoURg1ahQA6Nv6+flh5syZKCoqMhoQRPai9lor9p4xHIE0MZgjkMh8zBYQ8fHxUCgUqKysxIgRI5Cfn49t27YZtGloaICbmxucnZ2xfv16JCYmwtPTE83Nzejo6ICHhweam5uxf/9+LF++3FylEvVbxkYgRY3gCCTqG2YLCLFYjJycHCQnJ0On0yE9PR0RERHIzc0FAGRmZuL8+fN44okn4OjoiPDwcGzYsAEAUFtbi5kzZwIAtFot5s6di5SUFHOVSmRSu64DG45U4uvSvuvn0uoElFQ36EcgLZkSglSOQKI+JBKMdRZYKZlMxuclqNeVVDcg++9nceHfTYjw94Sbc9/d348c4YVfx47gCCQym9t9b/JJaqJuNN/Q4k/7S/HhsUr4ergg7/E4TI8YZumyiPoMA4LIiEMXVfj97u+gbLiO+fcE4uWUsfB0ZScw2RcGBNHPXNHcwBv7zqGg5DKkfgPxceZEyIK8LV0WkUUwIIjQOcz676eUePPTc2i+ocXz/xWCZ+4fDRcxnycg+8WAILv3Q10Lfrf7XzhSdgVxIwdj5awohAz1sHRZRBbHgCC7pdV1YOPRSqw+UAqxgwPe+HUk5iUEwoFTUxABYECQnfpO2YjsXWfxnfIapoUPxR9SIzDca4ClyyLqVxgQZFeut+mw5otSbDhSCW93Z/x53nikRA7jMwZERjAgyG4cUVzB73b/Cz9cbcFjCQHInhHG+YuIboMBQTavvrkNb356Hn8/VYNgH3fkZ9yDe4KHWLoson6PAUE2SxAEfHLmMv6w9xwar7djyRQpsiZLORU2UQ8xIMgm1dS34H92f4fDpWrEBgzCX2dHYewwT0uXRWRVGBBkU3QdAj48VoU/7b8IAFjxcDiemBjEVdWI7gIDgmzG+R+vIfvvZ3GmphFTxvrhjV9HYsQgDl0lulsMCLJ6re06vHtQgbyvKzDIzQnvPTYOD0UP59BVol+IAWEB11rb0abtsHQZNuHCj034/Z5/oaquBY/GSfA/D4ZhkJuzpcsisgkMiD52+od6zPrzMdjOMk2WN3KIG7YtmoBfSX0sXQqRTTFrQBQWFmLp0qXQ6XRYtGgRsrOzDd6vr69Heno6ysvL4erqio0bNyIyMrJH+1qrUz80QBCA3z8YBhexg6XLsXoDnMV4KHo4h64SmYHZAkKn0yErKwsHDhyARCJBfHw85HI5wsPD9W3eeustxMbGYvfu3bhw4QKysrJw8ODBHu1rrcpUGgxyc8JT947iPXIi6tfM9hO2qKgIUqkUwcHBcHZ2RlpaGgoKCgzanDt3DlOnTgUAjB07FlVVVaitre3RvtaqTNWEEL+BDAci6vfMFhBKpRIBAQH61xKJBEql0qBNTEwMdu3aBaAzUC5duoSampoe7XtTXl4eZDIZZDIZ1Gq1Gc6k9wiCAIVKA6kf1xogov7PbAEhGOmFvfVXc3Z2Nurr6xEbG4v33nsP48aNg1gs7tG+N2VkZKC4uBjFxcXw9fXtneLNpK65DQ0t7ZD6DbR0KUREJpmtD0IikaC6ulr/uqamBv7+/gZtPD09sWnTJgCdgTJq1CiMGjUKLS0tJve1RopaDQAghAFBRFbAbFcQ8fHxUCgUqKysRFtbG/Lz8yGXyw3aNDQ0oK2tDQCwfv16JCYmwtPTs0f7WqMy9X8CYigDgoj6P7NdQYjFYuTk5CA5ORk6nQ7p6emIiIhAbm4uACAzMxPnz5/HE088AUdHR4SHh2PDhg233dfaldU2YaCLGMM8XS1dChGRSSLB2A1/KyWTyVBcXGzpMro1b/230NzQoSBrkqVLISICcPvvTT6p1YcUtRpIfXl7iYisAwOijzReb4eq6Qb7H4jIajAg+kiZiiOYiMi6MCD6SJmqCQD4DAQRWQ0GRB8pU2ngInaAZLCbpUshIuoRBkQfUag0CPYdyKUvichqMCD6iKJWw/4HIrIqDIg+0NKmhbLhOgOCiKwKA6IPlKuaAbCDmoisCwOiD5SpO0cw8RkIIrImDIg+oKjVQOwgwsgh7pYuhYioxxgQfUCh0iDIxx1OjvznJiLrwW+sPlCu4ggmIrI+JgPi6tWrfVGHzbqh1aGqrpkd1ERkdUwGxIQJE/Doo4/iH//4h9GlQOn2qq60oEPgCCYisj4mA6K0tBQZGRn46KOPIJVK8bvf/Q6lpaV9UZtNUHAOJiKyUiYDQiQSYdq0adi+fTvWr1+PzZs3IyEhAUlJSTh+/Pht9y0sLERoaCikUilWrlzZ5f3GxkY8/PDDiImJQUREhH59agAICgpCVFQUYmNjIZPJ7uLU+gdFrQYiETCa60AQkZUxueRoXV0dtm7dio8++ghDhw7Fe++9B7lcjpKSEjz66KOorKw0up9Op0NWVhYOHDgAiUSC+Ph4yOVyhIeH69u8//77CA8Px969e6FWqxEaGop58+bB2dkZAHDo0CH4+Pj00qlaRplag0BvN7g6OVq6FCKiO2IyICZOnIjHH38ce/bsgUQi0W+XyWTIzMzsdr+ioiJIpVIEBwcDANLS0lBQUGAQECKRCE1NTRAEARqNBt7e3hCLzbZMtkWUcRU5IrJSJm8xvfnmm3jttdcMwmHnzp0AgFdeeaXb/ZRKJQICAvSvJRIJlEqlQZvFixfj/Pnz8Pf3R1RUFNatWwcHh86SRCIRpk+fjri4OOTl5XX7OXl5eZDJZJDJZFCr1aZOp09pdR2ovNIMKZ+gJiIrZDIgjPUdvP322yYPbGzEk0hkONX1559/jtjYWFy+fBklJSVYvHgxrl27BgA4evQoTp06hc8++wzvv/8+vv76a6Ofk5GRgeLiYhQXF8PX19dkXX3ph6staNN18AqCiKxSt/dzPvvsM/zjH/+AUqnEc889p99+7dq1Ht0GkkgkqK6u1r+uqamBv7+/QZtNmzYhOzsbIpEIUqkUo0aNwoULF5CQkKBv6+fnh5kzZ6KoqAiJiYl3fIKWpLi5zOhQDwtXQkR057q9gvD394dMJoOrqyvi4uL0f3K5HJ9//rnJA8fHx0OhUKCyshJtbW3Iz8+HXC43aBMYGIiDBw8CAGpra3Hx4kUEBwejubkZTU2dw0Obm5uxf/9+REZG/pLztIib61BziCsRWaNuLwViYmIQExODefPm3VXHsVgsRk5ODpKTk6HT6ZCeno6IiAjk5uYCADIzM/Haa69hwYIFiIqKgiAIeOedd+Dj44OKigrMnDkTAKDVajF37lykpKTc5SlaTplKg+FerhjoYlsd70RkH0RCN49Hz5kzB3/7298QFRXVpe8AAM6ePWv24u6UTCZDcXGxpcvQe/i9Ixjk5oSPnppg6VKIiIy63fdmtz9t161bBwDYt2+feaqycR0dAspUGqQlBJhuTETUD3UbEMOHDwcAjBw5ss+KsSXKhuu43q5DiB87qInIOpkc5rpr1y6EhITAy8sLnp6e8PDwgKenZ1/UZtXK1DdHMLGDmoisk8ne05dffhl79+5FWFhYX9RjM8pq/zOCic9AEJGVMnkFMXToUIbDXShTaeAz0BmD3Z0tXQoR0V0xeQUhk8nwm9/8Br/+9a/h4uKi3z5r1iyzFmbtFKomzuBKRFbNZEBcu3YNbm5u2L9/v36bSCRiQNyGIAhQqDRIjfU33ZiIqJ8yGRA/X6OBekbddANNrVqOYCIiq9ZtQPzxj3/Eyy+/jCVLlhh9UO7dd981a2HWTMEpNojIBnQbEDfXbbDm1dws5eYcTCEMCCKyYt0GxI4dO/DQQw+hoaEBS5cu7cuarJ5C1QRPVzF8PVxMNyYi6qe6HeZ68uRJXLp0CRs3bkR9fT2uXr1q8EfdU9RqIPUbaPTWHBGRtej2CiIzMxMpKSmoqKhAXFycwQJAIpEIFRUVfVKgNSpXazB17FBLl0FE9It0ewXx8MMP4/z580hPT0dFRQUqKyv1fwyH7l1tbsMVTRs7qInI6nUbEI888ggAoLS0tM+KsQX6RYI4BxMRWblubzF1dHTgf//3f1FaWorVq1d3ef+FF14wa2HWiiOYiMhWdHsFkZ+fD1dXV2i1WjQ1NXX5I+MUqiYMcHKEv9cAS5dCRPSLdHsFERoaildeeQXR0dGYMWPGXR28sLAQS5cuhU6nw6JFi5CdnW3wfmNjI+bPn48ffvgBWq0WL730EhYuXNijffurMlXnCCYHB45gIiLrZnKqjRkzZuDTTz/F999/j9bWVv325cuX33Y/nU6HrKwsHDhwABKJBPHx8ZDL5foH8ADg/fffR3h4OPbu3Qu1Wo3Q0FDMmzcPjo6OJvftr8pUGtwTPMTSZRAR/WImp/vOzMzEjh078N5770EQBOzcuROXLl0yeeCioiJIpVIEBwfD2dkZaWlpKCgoMGgjEonQ1NQEQRCg0Wjg7e0NsVjco337o6bWdvzY2MoRTERkE0wGxLFjx7BlyxYMHjwYK1aswPHjx1FdXW3ywEqlEgEBP63HLJFIoFQqDdosXrwY58+fh7+/P6KiorBu3To4ODj0aN+b8vLyIJPJIJPJoFarTdZlTuXqZgDsoCYi22AyIAYM6OxsdXNzw+XLl+Hk5ITKykqTB/75g3U33fpk8eeff47Y2FhcvnwZJSUlWLx4Ma5du9ajfW/KyMhAcXExiouL4evra7Iuc1LUdnbe8wqCiGyByYC4OR/TsmXLMH78eAQFBeGxxx4zeWCJRGJwpVFTUwN/f8P1ETZt2oRZs2ZBJBJBKpVi1KhRuHDhQo/27Y/K1Bo4Ozog0NvN0qUQEf1iJjupX3vtNQDA7Nmz8dBDD6G1tRVeXl4mDxwfHw+FQoHKykqMGDEC+fn52LZtm0GbwMBAHDx4EPfddx9qa2tx8eJFBAcHY9CgQSb37Y/KajUY5eMOsaPJ3CUi6vdMBsTPubi4GCw7etsDi8XIyclBcnIydDod0tPTERERgdzcXACdnd+vvfYaFixYgKioKAiCgHfeeQc+Pj4AYHTf/k6h0iBKYjo8iYisgUgwdsPfSslkMhQXF1vks1vbdQhbXoilU0Pw/H+NsUgNRER36nbfm7wX0kvK1RoIAjuoich2mLzFdOrUqS7bvLy8MHLkSIjFd3SHyqb9NAcT16EmIttg8hv+2WefxalTpxAdHQ1BEPDdd98hOjoadXV1yM3NxfTp0/uizn6vTKWBgwgI8uEIJiKyDSZvMQUFBeH06dMoLi7GyZMncfr0aURGRuKLL77Ayy+/3Bc1WgVFrQZBQ9zhIna0dClERL3CZEBcuHDBYARReHg4Tp8+jeDgYLMWZm3K1Br2PxCRTTF5iyk0NBTPPPMM0tLSAAA7duzAmDFjcOPGDTg5OZm9QGvQrutA1ZVmTA/nMqNEZDtMXkF8+OGHkEqlWLt2LdasWYPg4GB8+OGHcHJywqFDh/qixn7vUl0ztB0CQriKHBHZEJNXEAMGDMCLL76IF198sct7AwfyCxHo7H8AAKkvRzARke0wGRBHjx7F66+/jkuXLkGr1eq3V1RUmLUwa6L4zxDX0X7uFq6EiKj3mAyIp556CmvWrEFcXBwcHTlCx5gylQaSwQPg5sznQojIdpj8RvPy8rrrJUfthULFEUxEZHtMBsTkyZOxbNkyzJo1y2CivvHjx5u1MGuh6xBQodbgXimXGSUi22IyIE6cOAEABpM5iUQifPnll+aryorU1LfghraDVxBEZHNMBgSHst6efgQT52AiIhvTbUBs3boV8+fPx+rVq42+/8ILL5itKGtSpr4ZELyCICLb0m1ANDc3AwCampr6rBhrpKjVwM/DBV4D+FQ5EdmWbgPi6aefBgCsWLGiz4qxRmVqDZ+gJiKb1G1APPfcc7fd8d133zV58MLCQixduhQ6nQ6LFi1Cdna2wfurVq3CX//6VwCAVqvF+fPnoVar4e3tjaCgIHh4eMDR0RFisdhiK8XdjiAIKKttwiNxEkuXQkTU67qdiykuLg5xcXFobW3FqVOnEBISgpCQEJSUlPTogTmdToesrCx89tlnOHfuHLZv345z584ZtFm2bBlKSkpQUlKCt99+G0lJSfD29ta/f+jQIZSUlPTLcACAHxtb0dymg3QoO6iJyPZ0ewXx5JNPAuicrO/QoUP6mVszMzN7tEhQUVERpFKpflrwtLQ0FBQUIDw83Gj77du347HHHrvjE7Ckn1aR4y0mIrI9JmdzvXz5skFHtUajweXLl00eWKlUIiAgQP9aIpFAqVQabdvS0oLCwkLMnj1bv00kEmH69OmIi4tDXl5et5+Tl5cHmUwGmUwGtVptsq7edHMOJo5gIiJbZPI5iOzsbIwbNw6TJ08GABw+fBivv/66yQMLgtBlm0gkMtp27969mDRpksHtpaNHj8Lf3x8qlQrTpk3D2LFjkZiY2GXfjIwMZGRkAABkMpnJunpTmUqDwW5OGOLu3KefS0TUF0wGxMKFCzFjxgz9E9UrV67EsGHDTB5YIpGgurpa/7qmpgb+/v5G2+bn53e5vXSzrZ+fH2bOnImioiKjAWFJZaomhPh5dBt8RETWzOQtJqCzw9nX1xeDBw9GaWkpvv76a5P7xMfHQ6FQoLKyEm1tbcjPz4dcLu/SrrGxEYcPH0Zqaqp+W3Nzs/62VnNzM/bv34/IyMienlOfEAQBCpUGo3l7iYhslMkriFdeeQU7duxAREQEHBw680QkEpn8NS8Wi5GTk4Pk5GTodDqkp6cjIiICubm5ADo7uwFg9+7dmD59Otzdf1pLoba2FjNnzgTQOfx17ty5SElJubszNJO65jY0tLSzg5qIbJZIMNZZ8DOhoaE4e/aswUyu/ZVMJuuzIbHHy+vw2P/7FlvSE5A4xrdPPpOIqLfd7nvT5C2m4OBgtLe393pR1u7mHEx8ipqIbJXJW0xubm6IjY3F1KlTDa4ievIktS0rq23CQBcxhnm6WroUIiKzMBkQcrncaOeyvbvZQc0RTERkq0wGxM0nqslQmUrDvgcismndBsScOXPwt7/9DVFRUUZ/JZ89e9ashfVnjS3tUDXd4BPURGTTug2IdevWAQD27dvXZ8VYizJ15zMaHOJKRLas21FMw4cPBwCMHDkSLi4uOHPmjH6468iRI/uswP7op0n6OIsrEdkuk8Nc169fj4SEBOzatQsff/wx7rnnHmzcuLEvauu3FLUauIgdMGLwAEuXQkRkNiY7qVetWoXTp09jyJAhAIC6ujr86le/Qnp6utmL66/K1BqM9h0IRweOYCIi22XyCkIikcDD46dbKR4eHgbTeNsjRa2GHdREZPO6vYJYvXo1AGDEiBGYMGECUlNTIRKJUFBQgISEhD4rsL9pvqGFsuE60uLtOySJyPZ1GxA3Z1MdPXo0Ro8erd/+81lX7VGFuhkAp9ggItvXbUCsWLGiL+uwGgpVZ3DyFhMR2boerQdBPylTaSB2EGHkEHfTjYmIrBgD4g4pVBoE+bjDyZH/dERk20x+yx09erRH2+xFmUrDJ6iJyC6YDIglS5b0aJsxhYWFCA0NhVQqxcqVK7u8v2rVKsTGxiI2NhaRkZFwdHTE1atXe7SvJdzQ6nCprpkBQUR2odtO6uPHj+PYsWNQq9X6Ia8AcO3aNeh0OpMH1ul0yMrKwoEDByCRSBAfHw+5XI7w8HB9m2XLlmHZsmUAgL1792LNmjXw9vbu0b6WUHmlGR0CuA41EdmFbq8g2traoNFooNVq0dTUpP/z9PTExx9/bPLARUVFkEqlCA4OhrOzM9LS0lBQUNBt++3bt+Oxxx67q337CudgIiJ70u0VRFJSEpKSkrBgwQL95HwdHR3QaDTw9PQ0eWClUmnwxLVEIsGJEyeMtm1paUFhYSFycnLueN+8vDzk5eUBANRqtcm6fglFrQYiERDsyxFMRGT7TPZBvPrqq7h27Rqam5sRHh6O0NBQrFq1yuSBBUHosq271df27t2LSZMmwdvb+473zcjIQHFxMYqLi+Hra94FfMpUGgR6u8HVydGsn0NE1B+YDIhz587B09MTe/bswQMPPIAffvgBH330kckDSyQSVFdX61/X1NTA39/faNv8/Hz97aU73bcvcQQTEdkTkwHR3t6O9vZ27NmzB6mpqXBycurROszx8fFQKBSorKxEW1sb8vPzja5t3djYiMOHDxtM4dHTffuSVteBiisadlATkd0wOd33008/jaCgIMTExCAxMRGXLl3qUR+EWCxGTk4OkpOTodPpkJ6ejoiICOTm5gIAMjMzAQC7d+/G9OnT4e7ubnJfS/rhagvadQI7qInIbogEYzf8TdBqtRCLTWZLn5PJZCguLjbLsT///t94+qOT2JM1CbEBg8zyGUREfe1235smbzHV1tbiqaeewowZMwB09kls3ry5dyu0AjeHuHKSPiKyFyYDYsGCBUhOTsbly5cBAGPGjMHatWvNXlh/U6bSwN/LFQNd+t+VExGROZgMiCtXrmDOnDlwcOhsKhaL4ehof8M8FaomdlATkV0xGRDu7u6oq6vTj1z69ttv4eXlZfbC+pOODgHlqmZ2UBORXTF5v2T16tWQy+UoLy/HpEmToFarezTVhi1RNlzH9XYd+x+IyK6YDIjx48fj8OHDuHjxIgRBQGhoKJycnPqitn5DPwcTlxklIjtiMiBaW1vxwQcf4MiRIxCJRLjvvvuQmZkJV1fXvqivX9CPYPJlQBCR/TAZEE888QQ8PDz0a0Bs374djz/+OHbu3Gn24voLhaoJPgOdMdjd2dKlEBH1GZMBcfHiRZw5c0b/evLkyYiJiTFrUf1NmUrD/gcisjsmRzGNGzcO3377rf71iRMnMGnSJLMW1Z8IggCFSsMRTERkd7q9goiKioJIJEJ7ezu2bNmCwMBAiEQiXLp0yeIru/UlVdMNNLVqeQVBRHan24DYt29fX9bRb/20ihwDgojsS7cBcXMVOXunqG0CwDmYiMj+mOyDsHdlag08XcXw9XCxdClERH2KAWGColaDkKEePVokiYjIljAgTChTafiAHBHZJQbEbVxtbkNdcxun2CAiu2TWgCgsLERoaCikUilWrlxptM1XX32F2NhYREREICkpSb89KCgIUVFRiI2NhUwmM2eZ3bo5gonTfBORPTLb6jc6nQ5ZWVk4cOAAJBIJ4uPjIZfLDZ6haGhowLPPPovCwkIEBgZCpVIZHOPQoUPw8fExV4kmcYgrEdkzs11BFBUVQSqVIjg4GM7OzkhLS0NBQYFBm23btmHWrFkIDAwEAPj5+ZmrnLuiUDXBzdkR/l4DLF0KEVGfM1tAKJVKBAQE6F9LJBIolUqDNqWlpaivr8f999+PuLg4bNmyRf+eSCTC9OnTERcXh7y8vG4/Jy8vDzKZDDKZDGq1ulfPoUylwWjfgXBw4AgmIrI/ZrvFJAhCl223DhXVarU4efIkDh48iOvXr2PixIm45557MGbMGBw9ehT+/v5QqVSYNm0axo4di8TExC7HzMjIQEZGBgD0el9FmUqDicFDevWYRETWwmxXEBKJBNXV1frXNTU18Pf379ImJSUF7u7u8PHxQWJion7m2Jtt/fz8MHPmTBQVFZmrVKOaWtvxY2MrO6iJyG6ZLSDi4+OhUChQWVmJtrY25OfnQy6XG7RJTU3FN998A61Wi5aWFpw4cQJhYWFobm5GU1PnFBfNzc3Yv38/IiMjzVWqUeXqZgDsoCYi+2W2W0xisRg5OTlITk6GTqdDeno6IiIikJubCwDIzMxEWFgYUlJSEB0dDQcHByxatAiRkZGoqKjAzJkzAXTehpo7dy5SUlLMVapRN+dgChnKab6JyCyiVKkAAA78SURBVD6JBGOdBVZKJpOhuLi4V4719j/OY9PRKpz7QzLEjnyekIhs0+2+N/nN140ylQbBvu4MByKyW/z264ZCpWEHNRHZNQaEEa3tOlTXt7CDmojsGgPCiHK1BoIArkNNRHaNAWHEzTmYuIocEdkzBoQRZSoNHB1ECPJxs3QpREQWw4AwQlGrwUhvN7iIHS1dChGRxTAgjFComnh7iYjsHgPiFm3aDlyqa+EqckRk9xgQt7hU1wxth8ArCCKyewyIW/y0ihyHuBKRfWNA3ELxn4AI9nW3cCVERJbFgLiFQqWBZPAAuDmbbaJbIiKrwIC4RZlKwyk2iIjAgDCg6xBQrtawg5qICAwIAzX1LWjTdrCDmogIDAgDitrODmpO801EZOaAKCwsRGhoKKRSKVauXGm0zVdffYXY2FhEREQgKSnpjvbtbQpO0kdEpGe2oTo6nQ5ZWVk4cOAAJBIJ4uPjIZfLER4erm/T0NCAZ599FoWFhQgMDIRKperxvuZQptJgqKcLvAY4mfVziIisgdmuIIqKiiCVShEcHAxnZ2ekpaWhoKDAoM22bdswa9YsBAYGAgD8/Px6vK85lHEOJiIiPbMFhFKpREBAgP61RCKBUqk0aFNaWor6+nrcf//9iIuLw5YtW3q87015eXmQyWSQyWRQq9V3Xa8gCP8Z4soOaiIiwIy3mARB6LJNJBIZvNZqtTh58iQOHjyI69evY+LEibjnnnt6tO9NGRkZyMjIAADIZLK7rvfHxlY0t+l4BUFE9B9mCwiJRILq6mr965qaGvj7+3dp4+PjA3d3d7i7uyMxMRFnzpzp0b69jR3URESGzHaLKT4+HgqFApWVlWhra0N+fj7kcrlBm9TUVHzzzTfQarVoaWnBiRMnEBYW1qN9e9tPk/QxIIiIADNeQYjFYuTk5CA5ORk6nQ7p6emIiIhAbm4uACAzMxNhYWFISUlBdHQ0HBwcsGjRIkRGRgKA0X3NqUzVhMFuThgy0MWsn0NEZC1EgrEb/lZKJpOhuLj4rvZ9NPcYRBDhb5kTe7kqIqL+63bfm3ySGp0d6qW1Gki5ihwRkR4DAsAVTRsar7dD6suAICK6iQGBn3VQ8wqCiEiPAYHODmqAQ1yJiH6OAYHOK4iBLmIM83S1dClERP0GAwKdD8lJ/QZ2+7Q2EZE9YkDgp4AgIqKf2H1AaHUdSAzxxSTpEEuXQkTUr5jtSWprIXZ0wJ/mxFi6DCKifsfuryCIiMg4BgQRERnFgCAiIqMYEEREZBQDgoiIjGJAEBGRUQwIIiIyigFBRERG2dSKcj4+PggKCrqrfdVqNXx9fXu3oH6C52a9bPn8eG79Q1VVFa5cuWL0PZsKiF/ilyxX2t/x3KyXLZ8fz63/4y0mIiIyigFBRERGOb7++uuvW7qI/iIuLs7SJZgNz8162fL58dz6N/ZBEBGRUbzFRERERjEgiIjIKLsPiMLCQoSGhkIqlWLlypWWLqdXVVdXY/LkyQgLC0NERATWrVtn6ZJ6nU6nw7hx4/DQQw9ZupRe1dDQgEceeQRjx45FWFgYjh8/bumSetWaNWsQERGByMhIPPbYY2htbbV0SXctPT0dfn5+iIyM1G+7evUqpk2bhpCQEEybNg319fUWrPDu2XVA6HQ6ZGVl4bPPPsO5c+ewfft2nDt3ztJl9RqxWIw//elPOH/+PL799lu8//77NnV+ALBu3TqEhYVZuoxet3TpUqSkpODChQs4c+aMTZ2jUqnEu+++i+LiYnz33XfQ6XTIz8+3dFl3bcGCBSgsLDTYtnLlSkydOhUKhQJTp0612h+fdh0QRUVFkEqlCA4OhrOzM9LS0lBQUGDpsnrN8OHDMX78eACAh4cHwsLCoFQqLVxV76mpqcGnn36KRYsWWbqUXnXt2jV8/fXXeOqppwAAzs7OGDRokIWr6l1arRbXr1+HVqtFS0sL/P39LV3SXUtMTIS3t7fBtoKCAjz55JMAgCeffBJ79uyxRGm/mF0HhFKpREBAgP61RCKxqS/Qn6uqqsLp06cxYcIES5fSa55//nn88Y9/hIODbf1nXFFRAV9fXyxcuBDjxo3DokWL0NzcbOmyes2IESPw0ksvITAwEMOHD4eXlxemT59u6bJ6VW1tLYYPHw6g84eaSqWycEV3x7b+n3WHjI3wFYlEFqjEvDQaDWbPno21a9fC09PT0uX0in379sHPz88mxprfSqvV4tSpU3jmmWdw+vRpuLu7W+0tCmPq6+tRUFCAyspKXL58Gc3Nzdi6daulyyIj7DogJBIJqqur9a9ramqs+lLXmPb2dsyePRvz5s3DrFmzLF1Orzl69Cg++eQTBAUFIS0tDV9++SXmz59v6bJ6hUQigUQi0V/tPfLIIzh16pSFq+o9X3zxBUaNGgVfX184OTlh1qxZOHbsmKXL6lVDhw7Fjz/+CAD48ccf4efnZ+GK7o5dB0R8fDwUCgUqKyvR1taG/Px8yOVyS5fVawRBwFNPPYWwsDC88MILli6nV7399tuoqalBVVUV8vPzMWXKFJv5FTps2DAEBATg4sWLAICDBw8iPDzcwlX1nsDAQHz77bdoaWmBIAg4ePCgTXXCA4BcLsfmzZsBAJs3b0ZqaqqFK7pLgp379NNPhZCQECE4OFh48803LV1Or/rmm28EAEJUVJQQExMjxMTECJ9++qmly+p1hw4dEh588EFLl9GrTp8+LcTFxQlRUVFCamqqcPXqVUuX1KuWL18uhIaGChEREcL8+fOF1tZWS5d019LS0oRhw4YJYrFYGDFihLB+/XrhypUrwpQpUwSpVCpMmTJFqKurs3SZd4VTbRARkVF2fYuJiIi6x4AgIiKjGBBERGQUA4KIiIxiQBARkVEMCOp3RCIRXnzxRf3r//u//0NvLXy4YMECfPzxx71yrNvZuXMnwsLCMHny5C7vff/995gyZQrGjBmDkJAQvPHGG/qn+j/88EP4+voiNjZW/3fu3DlUVVVhwIABGDduHMLCwpCQkKAfZ98TDQ0N+OCDD3rt/Mg+MCCo33FxccGuXbtw5coVS5diQKfT9bjthg0b8MEHH+DQoUMG269fvw65XI7s7GyUlpbizJkzOHbsmMGX929+8xuUlJTo/24+JDd69GicPn0a58+fR35+PtasWYNNmzb1qB4GBN0NBgT1O2KxGBkZGVizZk2X9269Ahg4cCAA4KuvvkJSUhLmzJmDMWPGIDs7G3/961+RkJCAqKgolJeX6/f54osvcN9992HMmDHYt28fgM4v/2XLliE+Ph7R0dH4y1/+oj/u5MmTMXfuXERFRXWpZ/v27YiKikJkZCReeeUVAMAf/vAHHDlyBJmZmVi2bJlB+23btmHSpEn6yenc3NyQk5Nzx3MtBQcHY/Xq1Xj33Xe7vPf9998jISEBsbGxiI6OhkKhQHZ2NsrLyxEbG6uvadWqVfrzXbFiBYDOSR3Hjh2LJ598EtHR0XjkkUfQ0tICAMjOzkZ4eDiio6Px0ksv3VG9ZKUs/KAeURfu7u5CY2OjMHLkSKGhoUFYtWqVsGLFCkEQBOHJJ58Udu7cadBWEDqfpvby8hIuX74stLa2Cv7+/sLy5csFQRCEtWvXCkuXLtXvn5ycLOh0OqG0tFQYMWKEcP36deEvf/mL8MYbbwiCIAitra1CXFycUFFRIRw6dEhwc3MTKioqutSpVCqFgIAAQaVSCe3t7cLkyZOF3bt3C4IgCElJScI///nPLvv893//t7B27dou2wcNGiQ0NjYKmzZtEnx8fPRPvsfExAgtLS1CZWWlEBERYbBPfX294Orq2uVYixcvFrZu3SoIgiDcuHHD6P6ff/658Nvf/lbo6OgQdDqd8OCDDwqHDx8WKisrBQDCkSNHBEEQhIULFwqrVq0S6urqhDFjxggdHR36zybbxysI6pc8PT3xxBNPGP2F3J34+HgMHz4cLi4uGD16tP5XelRUFKqqqvTt5syZAwcHB4SEhCA4OBgXLlzA/v37sWXLFsTGxmLChAmoq6uDQqEAACQkJGDUqFFdPu+f//wn7r//fvj6+kIsFmPevHn4+uuvb1ujIAjdzhh8c/utt5gGDBjQ7bGMmThxIt566y288847uHTpktH99+/fj/3792PcuHEYP348Lly4oD/fgIAATJo0CQAwf/58HDlyBJ6ennB1dcWiRYuwa9cuuLm53fY8yTYwIKjfev7557FhwwaDtRDEYjE6OjoAdH5BtrW16d9zcXHR/28HBwf9awcHB2i1Wv17t35Bi0QiCIKA9957T/+lXFlZqQ8Yd3d3o/V19wV9OxERESguLjbYVlFRgYEDB8LDw+OOjnX69Gmjk9zNnTsXn3zyCQYMGIDk5GR8+eWXXdoIgoBXX31Vf75lZWX6BYqM/fuIxWIUFRVh9uzZ2LNnD1JSUu6oVrJODAjqt7y9vTFnzhxs2LBBvy0oKAgnT54E0LlqV3t7+x0fd+fOnejo6EB5eTkqKioQGhqK5ORk/PnPf9Yfr7S01OQiPRMmTMDhw4dx5coV6HQ6bN++HUlJSbfdZ968eThy5Ai++OILAJ2d1s899xxefvnlOzqHqqoqvPTSS1iyZEmX9yoqKhAcHIznnnsOcrkcZ8+ehYeHB5qamvRtkpOTsXHjRmg0GgCdi2fdXNTmhx9+0K+BvX37dtx7773QaDRobGzEAw88gLVr16KkpOSO6iXrJLZ0AUS38+KLLyInJ0f/+re//S1SU1ORkJCAqVOndvvr/nZCQ0ORlJSE2tpa5Obm6m+dVFVVYfz48RAEAb6+viaXiRw+fDjefvttTJ48GYIg4IEHHjA5rfOAAQNQUFCAJUuWICsrCzqdDo8//jgWL16sb7Njxw4cOXJE//qDDz6Av78/ysvLMW7cOLS2tsLDwwNLlizBwoULu3zGjh07sHXrVjg5OWHYsGFYvnw5vL29MWnSJERGRmLGjBlYtWoVzp8/j4kTJwLo7OzfunUrHB0dERYWhs2bN+Ppp59GSEgInnnmGTQ2NiI1NRWtra0QBMHoAAKyPZzNlYj0qqqq8NBDD+G7776zdCnUD/AWExERGcUrCCIiMopXEEREZBQDgoiIjGJAEBGRUQwIIiIyigFBRERG/X9YkVsgemePxwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "responses = [e[\"responses\"].max().item() for e in experiments]\n",
    "plt.plot(responses)\n",
    "plt.xlabel(\"Number of OED steps\")\n",
    "plt.ylabel(\"best binding affinity\");"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}