CalculateMetricsOverGlobalSeeds.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9bda1422-1a68-4aef-99bb-3bf20d403303",
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import pandas as pd\n",
    "import wandb\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0ca52615-088f-41c0-8d8d-26a9bc222544",
   "metadata": {},
   "outputs": [],
   "source": [
    "SWEEP_ID = \"no4iaihj\"\n",
    "SWEEP_PATH = \"neuroevolution-fzi/AST2023/\" + SWEEP_ID\n",
    "NUMBER_OF_DIFFERENT_RANDOM_SEEDS = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "928f44c3-d562-457e-9178-c611caa31c74",
   "metadata": {},
   "outputs": [],
   "source": [
    "api = wandb.Api()\n",
    "\n",
    "sweep_data = api.sweep(SWEEP_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ec7339fc-da35-487f-b53f-51cc496da6d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "run_data = []\n",
    "\n",
    "for run in sweep_data.runs:\n",
    "    if run.state != \"finished\":\n",
    "        raise RuntimeError(f\"Run with ID {run.id} has state {run.state} which is therefore not 'finished'. Maybe it needs to be \"\n",
    "                           \"restarted, do not use it here to calculate summary metrics!\")\n",
    "    run_data.append((pd.DataFrame(run.scan_history()), run.config))\n",
    "    \n",
    "print(f\"Number runs: {len(run_data)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e59edf18-5b78-4d14-b1dc-4bd55e11d138",
   "metadata": {},
   "outputs": [],
   "source": [
    "duplicates = {}\n",
    "excludes = []\n",
    "for i, (run, config) in enumerate(run_data):\n",
    "    \n",
    "    # Duplicate dict, but remove the entry with \"global_seed\" as the key, because we want to find duplicates __except__ for\n",
    "    # the \"global_seed\"\n",
    "    temp_dict = {k: v for k, v in config.items() if k != \"global_seed\"}\n",
    "    \n",
    "    if i not in excludes:\n",
    "        for j, (run2, config2) in enumerate(run_data):\n",
    "            # i != j to exclude the same run\n",
    "            if i != j:\n",
    "                # Same as above: Duplicate config\n",
    "                temp_dict2 = {k: v for k, v in config2.items() if k != \"global_seed\"}\n",
    "                \n",
    "                # Same config found?\n",
    "                if temp_dict == temp_dict2:\n",
    "                    # Try-except simply for creating a list if there is not one already\n",
    "                    try:\n",
    "                        duplicates[i].append(j)\n",
    "                    except KeyError:\n",
    "                        duplicates[i] = [j]\n",
    "                    \n",
    "                    # Run with ID j was now processed, so we do not need to look at it again in the outer loop\n",
    "                    excludes.append(j)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0540bbcf-dd65-472e-931b-eba1f4018c1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(duplicates)\n",
    "\n",
    "# NUMBER_OF_DIFFERENT_RANDOM_SEEDS - 1 because the key is the first run, and the list should contain two identical runs with a\n",
    "# different random seed\n",
    "assert all(len(v) == (NUMBER_OF_DIFFERENT_RANDOM_SEEDS - 1) for v in duplicates.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e2d831cc-519c-4f21-b146-6c9290164e60",
   "metadata": {},
   "outputs": [],
   "source": [
    "columns_to_extract = [\"min_train\", \"mean_train\", \"max_train\", \"min_val\", \"mean_val\", \"max_val\", \"best\", \"elapsed_time\"]\n",
    "\n",
    "data_of_all_runs = []\n",
    "for k, v in duplicates.items():\n",
    "    _, config = run_data[k]\n",
    "    \n",
    "    # Exemplary config for that group of runs, where only the global_seed is different\n",
    "    exemplary_config = {k: v for k, v in config.items() if k != \"global_seed\"}\n",
    "    \n",
    "    run_ids = [k] + v\n",
    "    \n",
    "    data = []\n",
    "    for run_id in run_ids:\n",
    "        run, config = run_data[run_id]\n",
    "        \n",
    "        run_config = {k: v for k, v in config.items() if k != \"global_seed\"}\n",
    "        \n",
    "        # Check again, if all configs of that group are identical\n",
    "        if run_config != exemplary_config:\n",
    "            raise RuntimeError(f\"Runs that are averaged have not the same config! run_id: {run_id}\")\n",
    "        \n",
    "        data.append(pd.DataFrame(run[columns_to_extract]))\n",
    "    \n",
    "    # Calculate mean and std per generation, then rename the columns to indicate the \"new\" metric\n",
    "    averaged_data = pd.concat(data).groupby(level=0).mean().rename(columns=lambda x: x + \"_average\")\n",
    "    std_data = pd.concat(data).groupby(level=0).std().rename(columns=lambda x: x + \"_std\")\n",
    "    \n",
    "    data_of_all_runs.append((pd.concat([averaged_data, std_data], axis=1), exemplary_config))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c3690f77-f4e2-4052-951b-07f7be18ffae",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Now simply create a new WandB experiment per group of identical experiments where only the global_seed was different\n",
    "for averaged_data, config in data_of_all_runs:\n",
    "    wandb.init(\n",
    "        entity=\"neuroevolution-fzi\",\n",
    "        project=\"AST2023\",\n",
    "        config=config,\n",
    "        tags=[\"averaged-run\", \"sweep-\" + SWEEP_ID]\n",
    "    )\n",
    "    \n",
    "    for i, row in averaged_data.iterrows():\n",
    "        log_values = row.to_dict()\n",
    "        log_values[\"gen\"] = i\n",
    "        wandb.log(log_values)\n",
    "\n",
    "wandb.finish()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}