2_data_preparation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import textwrap\n",
    "\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "from matplotlib import pyplot as plt\n",
    "from numpy import nanmean as np_nanmean\n",
    "from numpy import nanmedian as np_nanmedian\n",
    "from sklearn import model_selection\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compute Missing Rates\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "patientids = list()\n",
    "with open(\n",
    "    \"./results/patients_18+_1d+_2+lactate_measurements_in_first2d_binned.txt\", \"r\"\n",
    ") as f:\n",
    "    patientids.extend(int(x) for x in f.readlines())\n",
    "\n",
    "os.makedirs(\"./results/missing_rate_plots/data\", exist_ok=True)\n",
    "df_missing_rates = pd.DataFrame()\n",
    "for pid in patientids:\n",
    "    df_binned = pd.read_csv(f\"./results/{pid}/{pid}_dynamic_binned2h_48h.csv\")\n",
    "    pid_rates = dict()\n",
    "    pid_rates[\"Total Records\"] = len(df_binned[\"Patient ID\"])\n",
    "    # skip Patient ID and Minutes of Stay\n",
    "    for col in df_binned.columns[2:]:\n",
    "        # compute missing values\n",
    "        pid_rates[col] = df_binned[col].isnull().sum()\n",
    "    df_missing_rates = df_missing_rates.append(pid_rates, ignore_index=True)\n",
    "# set pids as index\n",
    "df_missing_rates.set_index(\n",
    "    pd.Index(patientids, name=\"Patient ID\"), drop=True, inplace=True\n",
    ")\n",
    "df_missing_rates.to_csv(\"./results/missing_rate_plots/data/missing_rates.csv\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Drop Columns Having >40% Missing Rate (Patient-Level)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_missing_rates = pd.read_csv(\n",
    "    f\"./results/missing_rate_plots/data/missing_rates.csv\", index_col=0\n",
    ")\n",
    "total_pid = len(df_missing_rates)\n",
    "missingness = dict()\n",
    "# skip Total Records\n",
    "for col in df_missing_rates.columns[1:]:\n",
    "    # sum the number of patients not having any records of that variable\n",
    "    missing_pid = len(\n",
    "        df_missing_rates[df_missing_rates[col] == df_missing_rates[\"Total Records\"]]\n",
    "    )\n",
    "    missingness[col] = (missing_pid, total_pid - missing_pid)\n",
    "\n",
    "df_availability = pd.DataFrame(\n",
    "    data=[(k, (total_pid - v[0]) / total_pid) for k, v in missingness.items()],\n",
    "    columns=[\"Variables\", \"Available %\"],\n",
    ")\n",
    "# drop variables having less than 60% availability\n",
    "columns = df_availability[df_availability[\"Available %\"] > 0.60][\"Variables\"].tolist()\n",
    "columns.insert(0, \"Minutes of Stay\")\n",
    "columns.insert(1, \"Patient ID\")\n",
    "\n",
    "patientids = list()\n",
    "with open(\n",
    "    \"./results/patients_18+_1d+_2+lactate_measurements_in_first2d_binned.txt\", \"r\"\n",
    ") as f:\n",
    "    patientids.extend(int(x) for x in f.readlines())\n",
    "\n",
    "for pid in patientids:\n",
    "    df_binned = pd.read_csv(f\"./results/{pid}/{pid}_dynamic_binned2h_48h.csv\")\n",
    "    df_binned = df_binned[columns]\n",
    "    df_binned.set_index(\n",
    "        \"Minutes of Stay\",\n",
    "        drop=True,\n",
    "        inplace=True,\n",
    "    )\n",
    "    df_binned.to_csv(f\"./results/{pid}/{pid}_dynamic_binned2h_48h_dropped.csv\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Drop Empty Rows\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "138 had 1 empty rows which have been removed.\n",
      "660 had 1 empty rows which have been removed.\n",
      "808 had 1 empty rows which have been removed.\n",
      "809 had 1 empty rows which have been removed.\n",
      "889 had 1 empty rows which have been removed.\n",
      "1617 had 1 empty rows which have been removed.\n",
      "2136 had 1 empty rows which have been removed.\n",
      "2340 had 1 empty rows which have been removed.\n",
      "2370 had 1 empty rows which have been removed.\n",
      "2489 had 1 empty rows which have been removed.\n",
      "2611 had 1 empty rows which have been removed.\n",
      "2647 had 1 empty rows which have been removed.\n",
      "2768 had 3 empty rows which have been removed.\n",
      "3060 had 1 empty rows which have been removed.\n",
      "3235 had 2 empty rows which have been removed.\n",
      "3714 had 1 empty rows which have been removed.\n",
      "3861 had 1 empty rows which have been removed.\n",
      "3877 had 1 empty rows which have been removed.\n",
      "4198 had 1 empty rows which have been removed.\n",
      "4512 had 1 empty rows which have been removed.\n",
      "4621 had 1 empty rows which have been removed.\n",
      "5167 had 1 empty rows which have been removed.\n",
      "5196 had 1 empty rows which have been removed.\n",
      "5220 had 1 empty rows which have been removed.\n",
      "5285 had 1 empty rows which have been removed.\n",
      "5370 had 1 empty rows which have been removed.\n",
      "5386 had 1 empty rows which have been removed.\n",
      "5730 had 1 empty rows which have been removed.\n",
      "6706 had 1 empty rows which have been removed.\n",
      "7544 had 2 empty rows which have been removed.\n",
      "8169 had 1 empty rows which have been removed.\n",
      "9260 had 1 empty rows which have been removed.\n",
      "9725 had 1 empty rows which have been removed.\n",
      "9732 had 1 empty rows which have been removed.\n",
      "9992 had 1 empty rows which have been removed.\n",
      "10325 had 1 empty rows which have been removed.\n",
      "10369 had 1 empty rows which have been removed.\n",
      "10729 had 1 empty rows which have been removed.\n",
      "10915 had 1 empty rows which have been removed.\n",
      "11259 had 1 empty rows which have been removed.\n",
      "11351 had 1 empty rows which have been removed.\n",
      "11555 had 2 empty rows which have been removed.\n",
      "12130 had 1 empty rows which have been removed.\n",
      "12494 had 1 empty rows which have been removed.\n",
      "12609 had 1 empty rows which have been removed.\n",
      "12797 had 1 empty rows which have been removed.\n",
      "13079 had 1 empty rows which have been removed.\n",
      "13120 had 1 empty rows which have been removed.\n",
      "13752 had 1 empty rows which have been removed.\n",
      "13880 had 1 empty rows which have been removed.\n",
      "13914 had 1 empty rows which have been removed.\n",
      "13971 had 1 empty rows which have been removed.\n",
      "14244 had 1 empty rows which have been removed.\n",
      "14355 had 1 empty rows which have been removed.\n",
      "14658 had 1 empty rows which have been removed.\n",
      "15023 had 1 empty rows which have been removed.\n",
      "15180 had 1 empty rows which have been removed.\n",
      "15248 had 1 empty rows which have been removed.\n",
      "15583 had 1 empty rows which have been removed.\n",
      "15734 had 1 empty rows which have been removed.\n",
      "15863 had 1 empty rows which have been removed.\n",
      "16422 had 1 empty rows which have been removed.\n",
      "16428 had 1 empty rows which have been removed.\n",
      "16488 had 2 empty rows which have been removed.\n",
      "16752 had 1 empty rows which have been removed.\n",
      "16979 had 1 empty rows which have been removed.\n",
      "17161 had 1 empty rows which have been removed.\n",
      "17174 had 1 empty rows which have been removed.\n",
      "17516 had 1 empty rows which have been removed.\n",
      "18387 had 1 empty rows which have been removed.\n",
      "18489 had 1 empty rows which have been removed.\n",
      "19197 had 1 empty rows which have been removed.\n",
      "19467 had 1 empty rows which have been removed.\n",
      "19799 had 1 empty rows which have been removed.\n",
      "19935 had 1 empty rows which have been removed.\n",
      "20110 had 1 empty rows which have been removed.\n",
      "20135 had 1 empty rows which have been removed.\n",
      "20374 had 1 empty rows which have been removed.\n",
      "20393 had 1 empty rows which have been removed.\n",
      "20697 had 1 empty rows which have been removed.\n",
      "20733 had 1 empty rows which have been removed.\n",
      "21467 had 2 empty rows which have been removed.\n",
      "21784 had 2 empty rows which have been removed.\n",
      "21791 had 1 empty rows which have been removed.\n",
      "22178 had 1 empty rows which have been removed.\n",
      "22237 had 1 empty rows which have been removed.\n",
      "22460 had 1 empty rows which have been removed.\n",
      "22480 had 1 empty rows which have been removed.\n",
      "22865 had 1 empty rows which have been removed.\n",
      "23375 had 1 empty rows which have been removed.\n",
      "23476 had 1 empty rows which have been removed.\n",
      "23600 had 1 empty rows which have been removed.\n",
      "24114 had 1 empty rows which have been removed.\n",
      "24925 had 1 empty rows which have been removed.\n",
      "25013 had 1 empty rows which have been removed.\n",
      "25076 had 1 empty rows which have been removed.\n",
      "25346 had 1 empty rows which have been removed.\n",
      "25351 had 1 empty rows which have been removed.\n",
      "25513 had 1 empty rows which have been removed.\n",
      "25567 had 1 empty rows which have been removed.\n",
      "25886 had 1 empty rows which have been removed.\n",
      "25914 had 1 empty rows which have been removed.\n",
      "26532 had 1 empty rows which have been removed.\n",
      "27081 had 2 empty rows which have been removed.\n",
      "27114 had 1 empty rows which have been removed.\n",
      "27504 had 2 empty rows which have been removed.\n",
      "27566 had 1 empty rows which have been removed.\n",
      "27613 had 1 empty rows which have been removed.\n",
      "27791 had 1 empty rows which have been removed.\n",
      "28269 had 1 empty rows which have been removed.\n",
      "28592 had 2 empty rows which have been removed.\n",
      "28678 had 1 empty rows which have been removed.\n",
      "28871 had 1 empty rows which have been removed.\n",
      "29211 had 1 empty rows which have been removed.\n",
      "29263 had 1 empty rows which have been removed.\n",
      "29527 had 1 empty rows which have been removed.\n",
      "29816 had 1 empty rows which have been removed.\n",
      "29868 had 1 empty rows which have been removed.\n",
      "30121 had 1 empty rows which have been removed.\n",
      "30184 had 1 empty rows which have been removed.\n",
      "30230 had 1 empty rows which have been removed.\n",
      "30432 had 1 empty rows which have been removed.\n",
      "30473 had 1 empty rows which have been removed.\n",
      "30561 had 1 empty rows which have been removed.\n",
      "30800 had 1 empty rows which have been removed.\n",
      "31136 had 1 empty rows which have been removed.\n",
      "31680 had 1 empty rows which have been removed.\n",
      "31838 had 1 empty rows which have been removed.\n",
      "32806 had 1 empty rows which have been removed.\n",
      "32852 had 1 empty rows which have been removed.\n",
      "33312 had 1 empty rows which have been removed.\n",
      "33334 had 1 empty rows which have been removed.\n",
      "33391 had 1 empty rows which have been removed.\n",
      "33887 had 1 empty rows which have been removed.\n"
     ]
    }
   ],
   "source": [
    "patientids = list()\n",
    "with open(\n",
    "    \"./results/patients_18+_1d+_2+lactate_measurements_in_first2d_binned.txt\", \"r\"\n",
    ") as f:\n",
    "    patientids.extend(int(x) for x in f.readlines())\n",
    "\n",
    "for pid in patientids:\n",
    "    df_binned_pre = pd.read_csv(\n",
    "        f\"./results/{pid}/{pid}_dynamic_binned2h_48h_dropped.csv\"\n",
    "    )\n",
    "    df_binned_post = df_binned_pre.dropna(how=\"all\", subset=df_binned_pre.columns[2:])\n",
    "    if not df_binned_pre.equals(df_binned_post):\n",
    "        print(\n",
    "            f\"{pid} had {len(df_binned_pre) - len(df_binned_post)} empty rows which have been removed.\"\n",
    "        )\n",
    "        df_binned_post.set_index(\n",
    "            \"Minutes of Stay\",\n",
    "            drop=True,\n",
    "            inplace=True,\n",
    "        )\n",
    "        df_binned_post.to_csv(f\"./results/{pid}/{pid}_dynamic_binned2h_48h_dropped.csv\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RECompute Missing Rates\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "patientids = list()\n",
    "with open(\n",
    "    \"./results/patients_18+_1d+_2+lactate_measurements_in_first2d_binned.txt\", \"r\"\n",
    ") as f:\n",
    "    patientids.extend(int(x) for x in f.readlines())\n",
    "\n",
    "os.makedirs(\"./results/missing_rate_plots/data\", exist_ok=True)\n",
    "df_missing_rates = pd.DataFrame()\n",
    "for pid in patientids:\n",
    "    df_binned = pd.read_csv(f\"./results/{pid}/{pid}_dynamic_binned2h_48h_dropped.csv\")\n",
    "    pid_rates = dict()\n",
    "    pid_rates[f\"Total Records\"] = len(df_binned[\"Patient ID\"])\n",
    "    # skip Patient ID and Minutes of Stay\n",
    "    for col in df_binned.columns[2:]:\n",
    "        # compute missing values\n",
    "        pid_rates[col] = df_binned[col].isnull().sum()\n",
    "    df_missing_rates = df_missing_rates.append(pid_rates, ignore_index=True)\n",
    "# set pids as index\n",
    "df_missing_rates.set_index(\n",
    "    pd.Index(patientids, name=\"Patient ID\"), drop=True, inplace=True\n",
    ")\n",
    "df_missing_rates.to_csv(\"./results/missing_rate_plots/data/missing_rates_dropped.csv\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Plot Variables' Missing Rates\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# retrieved from: https://stackoverflow.com/questions/6170246/how-do-i-use-matplotlib-autopct/6170354#6170354\n",
    "def make_autopct(values):\n",
    "    def my_autopct(pct):\n",
    "        return f\"{pct:.2f}%  ({round(pct * sum(values) / 100.0):d})\"\n",
    "\n",
    "    return my_autopct\n",
    "\n",
    "\n",
    "df = pd.read_csv(\"./results/missing_rate_plots/data/missing_rates.csv\", index_col=0)\n",
    "df_dropped = pd.read_csv(\n",
    "    \"./results/missing_rate_plots/data/missing_rates_dropped.csv\", index_col=0\n",
    ")\n",
    "total_pid = len(df)\n",
    "missingness = dict()\n",
    "# skip Total Records\n",
    "for col in df.columns[1:]:\n",
    "    # sum the number of missing values\n",
    "    missing_ds = df[col].sum()\n",
    "    # sum the number of total records\n",
    "    total_ds = df[\"Total Records\"].sum()\n",
    "    # sum the number of patients not having any record of that variable\n",
    "    missing_pid = len(df[df[col] == df[\"Total Records\"]])\n",
    "\n",
    "    # remove invalid characters from filename\n",
    "    filename = col\n",
    "    for c in (\"\\\\\", \"/\", \":\", \"*\", \"?\", '\"', \"<\", \">\", \"|\"):\n",
    "        filename = filename.replace(c, \"_\")\n",
    "\n",
    "    missingness[col] = dict(\n",
    "        ds=(missing_ds, total_ds - missing_ds),\n",
    "        pid=(missing_pid, total_pid - missing_pid),\n",
    "    )\n",
    "    if col in df_dropped.columns:\n",
    "        # sum the number of missing values\n",
    "        missing_dropped_ds = df_dropped[col].sum()\n",
    "        # sum the number of total records\n",
    "        total_dropped_ds = df_dropped[\"Total Records\"].sum()\n",
    "        missingness[col][\"dropped_ds\"] = (\n",
    "            missing_dropped_ds,\n",
    "            total_dropped_ds - missing_dropped_ds,\n",
    "        )\n",
    "        # plots\n",
    "        f, (ax1, ax2, ax3) = plt.subplots(ncols=3, figsize=(30, 10))\n",
    "        # https://stackoverflow.com/questions/10351565/how-do-i-fit-long-title\n",
    "        ax1.set_title(\n",
    "            \"\\n\".join(textwrap.wrap(f\"{col} Missing Rate (Patients)\", 45)), fontsize=25\n",
    "        )\n",
    "        ax2.set_title(\n",
    "            \"\\n\".join(textwrap.wrap(f\"{col} Missing Rate (Dataset)\", 45)), fontsize=25\n",
    "        )\n",
    "        ax3.set_title(\n",
    "            \"\\n\".join(textwrap.wrap(f\"{col} Missing Rate (Dataset) AFTER Drop\", 45)),\n",
    "            fontsize=25,\n",
    "        )\n",
    "        ax1.pie(\n",
    "            x=missingness[col][\"pid\"],\n",
    "            labels=[\"Missing\", \"Present\"],\n",
    "            explode=[0.05, 0.05],\n",
    "            autopct=make_autopct(missingness[col][\"pid\"]),\n",
    "            textprops=dict(fontsize=22),\n",
    "        )\n",
    "        ax2.pie(\n",
    "            x=missingness[col][\"ds\"],\n",
    "            labels=[\"Missing\", \"Present\"],\n",
    "            explode=[0.05, 0.05],\n",
    "            autopct=make_autopct(missingness[col][\"ds\"]),\n",
    "            textprops=dict(fontsize=22),\n",
    "        )\n",
    "        ax3.pie(\n",
    "            x=missingness[col][\"dropped_ds\"],\n",
    "            labels=[\"Missing\", \"Present\"],\n",
    "            explode=[0.05, 0.05],\n",
    "            autopct=make_autopct(missingness[col][\"dropped_ds\"]),\n",
    "            textprops=dict(fontsize=22),\n",
    "        )\n",
    "        ax1.axis(\"equal\")\n",
    "        ax2.axis(\"equal\")\n",
    "        ax3.axis(\"equal\")\n",
    "        plt.tight_layout()\n",
    "        plt.savefig(f\"./results/missing_rate_plots/{filename}.png\")\n",
    "        plt.close()\n",
    "    else:\n",
    "        # plots\n",
    "        f, (ax1, ax2) = plt.subplots(ncols=2, figsize=(20, 10))\n",
    "        # https://stackoverflow.com/questions/10351565/how-do-i-fit-long-title\n",
    "        ax1.set_title(\n",
    "            \"\\n\".join(textwrap.wrap(f\"{col} Missing Rate (Patients)\", 45)), fontsize=25\n",
    "        )\n",
    "        ax2.set_title(\n",
    "            \"\\n\".join(textwrap.wrap(f\"{col} Missing Rate (Dataset)\", 45)), fontsize=25\n",
    "        )\n",
    "        ax1.pie(\n",
    "            x=missingness[col][\"pid\"],\n",
    "            labels=[\"Missing\", \"Present\"],\n",
    "            explode=[0.05, 0.05],\n",
    "            autopct=make_autopct(missingness[col][\"pid\"]),\n",
    "            textprops=dict(fontsize=22),\n",
    "        )\n",
    "        ax2.pie(\n",
    "            x=missingness[col][\"ds\"],\n",
    "            labels=[\"Missing\", \"Present\"],\n",
    "            explode=[0.05, 0.05],\n",
    "            autopct=make_autopct(missingness[col][\"ds\"]),\n",
    "            textprops=dict(fontsize=22),\n",
    "        )\n",
    "        ax1.axis(\"equal\")\n",
    "        ax2.axis(\"equal\")\n",
    "        plt.tight_layout()\n",
    "        plt.savefig(f\"./results/missing_rate_plots/{filename}.png\")\n",
    "        plt.close()\n",
    "\n",
    "\n",
    "def plot_total_missingness(data, title, ylim):\n",
    "    data.sort_values(by=\"Available %\", ascending=False, inplace=True)\n",
    "\n",
    "    tmp_title = f\"{title} 1st HALF\"\n",
    "    tmp_data = data.copy()\n",
    "    tmp_data = tmp_data.iloc[: len(data) // 2]\n",
    "    f1, ax1 = plt.subplots(figsize=(50, 27))\n",
    "    sns.barplot(data=tmp_data, x=\"Variables\", y=\"Available %\", color=\"blue\", ax=ax1)\n",
    "    ax1.grid(axis=\"y\")\n",
    "    plt.title(tmp_title, fontsize=35)\n",
    "    plt.ylim(ylim)\n",
    "    plt.xlabel(\"Variables\", fontsize=27)\n",
    "    plt.ylabel(\"Availability %\", fontsize=27)\n",
    "    # disable scientific notation\n",
    "    plt.ticklabel_format(style=\"plain\", axis=\"y\")\n",
    "    # https://stackoverflow.com/questions/10351565/how-do-i-fit-long-title\n",
    "    plt.xticks(\n",
    "        ticks=list(range(len(tmp_data[\"Variables\"]))),\n",
    "        labels=[\n",
    "            \"\\n\".join(textwrap.wrap(f\"{x}\", 60)) for x in tmp_data[\"Variables\"].tolist()\n",
    "        ],\n",
    "        rotation=90,\n",
    "        fontsize=25,\n",
    "    )\n",
    "    plt.yticks(ticks=[x / 10 for x in range(11)], fontsize=25)\n",
    "    plt.tight_layout()\n",
    "    plt.savefig(f\"./results/missing_rate_plots/{tmp_title}.png\")\n",
    "    plt.close()\n",
    "\n",
    "    tmp_title = f\"{title} 2nd HALF\"\n",
    "    tmp_data = data.copy()\n",
    "    tmp_data = tmp_data.iloc[len(data) // 2 :]\n",
    "    f2, ax2 = plt.subplots(figsize=(50, 27))\n",
    "    sns.barplot(data=tmp_data, x=\"Variables\", y=\"Available %\", color=\"blue\", ax=ax2)\n",
    "    ax2.grid(axis=\"y\")\n",
    "    plt.title(tmp_title, fontsize=35)\n",
    "    plt.ylim(ylim)\n",
    "    plt.xlabel(\"Variables\", fontsize=27)\n",
    "    plt.ylabel(\"Availability %\", fontsize=27)\n",
    "    # disable scientific notation\n",
    "    plt.ticklabel_format(style=\"plain\", axis=\"y\")\n",
    "    # https://stackoverflow.com/questions/10351565/how-do-i-fit-long-title\n",
    "    plt.xticks(\n",
    "        ticks=list(range(len(tmp_data[\"Variables\"]))),\n",
    "        labels=[\n",
    "            \"\\n\".join(textwrap.wrap(f\"{x}\", 60)) for x in tmp_data[\"Variables\"].tolist()\n",
    "        ],\n",
    "        rotation=90,\n",
    "        fontsize=25,\n",
    "    )\n",
    "    plt.yticks(ticks=[x / 10 for x in range(11)], fontsize=25)\n",
    "    plt.tight_layout()\n",
    "    plt.savefig(f\"./results/missing_rate_plots/{tmp_title}.png\")\n",
    "    plt.close()\n",
    "\n",
    "\n",
    "plot_total_missingness(\n",
    "    data=pd.DataFrame(\n",
    "        data=[\n",
    "            (k, (total_pid - v[\"pid\"][0]) / total_pid) for k, v in missingness.items()\n",
    "        ],\n",
    "        columns=[\"Variables\", \"Available %\"],\n",
    "    ),\n",
    "    title=\"AVAILABILITY BY PATIENTS (PERCENTAGE)\",\n",
    "    ylim=(0, 1),\n",
    ")\n",
    "plot_total_missingness(\n",
    "    data=pd.DataFrame(\n",
    "        data=[\n",
    "            (k, v[\"ds\"][0] / (v[\"ds\"][0] + v[\"ds\"][1])) for k, v in missingness.items()\n",
    "        ],\n",
    "        columns=[\"Variables\", \"Available %\"],\n",
    "    ),\n",
    "    title=\"AVAILABILITY BY RECORDS (PERCENTAGE)\",\n",
    "    ylim=(0, 1),\n",
    ")\n",
    "plot_total_missingness(\n",
    "    data=pd.DataFrame(\n",
    "        data=[\n",
    "            (k, v[\"dropped_ds\"][0] / (v[\"dropped_ds\"][0] + v[\"dropped_ds\"][1]))\n",
    "            for k, v in missingness.items()\n",
    "            if \"dropped_ds\" in v\n",
    "        ],\n",
    "        columns=[\"Variables\", \"Available %\"],\n",
    "    ),\n",
    "    title=\"AVAILABILITY BY RECORDS (PERCENTAGE) AFTER DROP\",\n",
    "    ylim=(0, 1),\n",
    ")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create Class/Target Labels\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "patientids = list()\n",
    "with open(\n",
    "    \"./results/patients_18+_1d+_2+lactate_measurements_in_first2d_binned.txt\", \"r\"\n",
    ") as f:\n",
    "    patientids.extend(int(x) for x in f.readlines())\n",
    "\n",
    "\n",
    "def get_class(row):\n",
    "    if pd.notna(row[\"Lactate\"]):\n",
    "        if row[\"Lactate\"] < 2:\n",
    "            return 1\n",
    "        elif row[\"Lactate\"] < 4:  # and Lactate >= 2\n",
    "            return 2\n",
    "        else:  # Lactate >= 4\n",
    "            return 3\n",
    "\n",
    "\n",
    "for pid in patientids:\n",
    "    df = pd.read_csv(f\"./results/{pid}/{pid}_dynamic_binned2h_48h_dropped.csv\")\n",
    "    # label current lactate according to class\n",
    "    df[\"Lactate_Class\"] = df.apply(get_class, axis=1)\n",
    "    df[\"Lactate_Class\"].ffill(inplace=True)\n",
    "\n",
    "    last_class = None\n",
    "    last_outcome = None\n",
    "    outcomes = list()\n",
    "    # start from the bottom\n",
    "    df_reversed = df.iloc[::-1].copy()\n",
    "    # compute outcomes\n",
    "    for tup in df_reversed.itertuples():\n",
    "        last_outcome = (\n",
    "            None\n",
    "            if last_class is None\n",
    "            else (\n",
    "                0\n",
    "                if last_class < tup.Lactate_Class\n",
    "                or (last_class == 1 and tup.Lactate_Class == 1)\n",
    "                else 1\n",
    "            )\n",
    "        )\n",
    "        outcomes.append(last_outcome)\n",
    "        if pd.notna(tup.Lactate):\n",
    "            last_class = tup.Lactate_Class\n",
    "    # label current lactate outcome according to previous one (next in terms of time)\n",
    "    df_reversed[\"Lactate_Outcome\"] = outcomes\n",
    "    df_labelled = df_reversed.iloc[::-1].copy()\n",
    "    df_labelled.set_index(\n",
    "        \"Minutes of Stay\",\n",
    "        drop=True,\n",
    "        inplace=True,\n",
    "    )\n",
    "    # fill forward everything except Lactate\n",
    "    columns_no_lactate = df_labelled.columns.tolist()\n",
    "    columns_no_lactate.remove(\"Lactate\")\n",
    "    df_labelled[columns_no_lactate] = df_labelled[columns_no_lactate].ffill()\n",
    "    # drop last row as it cannot be used for prediction\n",
    "    df_labelled.drop(df_labelled.tail(1).index, inplace=True)\n",
    "    df_labelled.to_csv(\n",
    "        f\"./results/{pid}/{pid}_dynamic_binned2h_48h_dropped_labelled.csv\"\n",
    "    )\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Merge Everything and Split According to Class\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "patientids = list()\n",
    "with open(\n",
    "    \"./results/patients_18+_1d+_2+lactate_measurements_in_first2d_binned.txt\", \"r\"\n",
    ") as f:\n",
    "    patientids.extend(int(x) for x in f.readlines())\n",
    "\n",
    "df_complete = pd.concat(\n",
    "    [\n",
    "        pd.read_csv(f\"./results/{pid}/{pid}_dynamic_binned2h_48h_dropped_labelled.csv\")\n",
    "        for pid in patientids\n",
    "    ],\n",
    "    ignore_index=True,\n",
    ")\n",
    "\n",
    "for cl, cl_name in ((1, \"Normal\"), (2, \"Mild\"), (3, \"Severe\")):\n",
    "    df_tmp = df_complete[df_complete[\"Lactate_Class\"] == cl]\n",
    "    df_tmp.reset_index(drop=True, inplace=True)\n",
    "    df_tmp.to_csv(f\"./results/class_{cl}_{cl_name}.csv\", index=False)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Add Sequential ID to Lactates' Values and Apply Fill-Forward\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# global counter\n",
    "i = 0\n",
    "\n",
    "\n",
    "def set_sequential_id(row):\n",
    "    global i\n",
    "    if pd.notna(row[\"Lactate\"]):\n",
    "        i = i + 1\n",
    "        return i\n",
    "\n",
    "\n",
    "for cl, cl_name in ((1, \"Normal\"), (2, \"Mild\"), (3, \"Severe\")):\n",
    "    df_class = pd.read_csv(f\"./results/class_{cl}_{cl_name}.csv\")\n",
    "    df_class[\"Sequential ID\"] = df_class.apply(set_sequential_id, axis=1)\n",
    "    df_class[\"Sequential ID\"].ffill(inplace=True)\n",
    "    df_class[\"Lactate\"].ffill(inplace=True)\n",
    "    df_class.to_csv(f\"./results/class_{cl}_{cl_name}_sequentialised.csv\", index=False)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Split Datasets into Train-Test and Impute Missing Values\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Imputation Strategies:\n",
      "Categorical: mode\n",
      "Numerical: median\n",
      "Normal\n",
      "{'Glasgow Coma Score - Verbal Response': 5.0, 'Glasgow Coma Score - Motor Response': 6.0, 'Glasgow Coma Score - Eye Opening': 4.0, 'Glasgow Coma Score - Total': 15.0, 'Circadian rhythm': 1.0, 'Richmond agitation-sedation scale': -1.0, 'Ventilator Airway Code': 1.0, 'Alanine aminotransferase': 28.0, 'Anion Gap': 4.800000000000011, 'Aspartate aminotransferase': 43.0, 'Base excess': 0.1, 'Bicarbonate': 24.2, 'Body Height': 170.0, 'Body Weight': 75.0, 'Calcium.ionized': 1.15, 'Carbon dioxide [Partial pressure]': 36.3, 'Central Venous Pressure': 8.183333333333334, 'Chloride': 108.0, 'Creatinine': 80.0, 'Diastolic Arterial Blood Pressure': 55.27272727272727, 'Fraction of Inspired Oxygen': 43.7, 'Glucose': 7.4, 'Heart rate': 84.30471628592483, 'Hemoglobin': 102.0, 'Invasive Diastolic Arterial Blood Pressure': 55.21666666666667, 'Invasive Mean Arterial Blood Pressure': 75.06666666666666, 'Invasive Systolic Arterial Blood Pressure': 115.01653005464482, 'Lactate': 1.1, 'Leukocytes': 10.2, 'Magnesium': 0.83, 'Mean Arterial Blood Pressure': 75.13793103448276, 'Mean Cell Hemoglobin': 31.0, 'Mean Cell Hemoglobin Concentration': 338.0, 'Mean Corpuscular Volume': 91.0, 'Oxygen [Partial pressure]': 94.0, 'Oxygen Saturation': 96.66666666666669, 'pH': 7.427, 'Phosphate': 1.07, 'Platelets': 168.0, 'Potassium': 4.0, 'Respiratory rate': 17.470967741935485, 'Sodium': 137.0, 'Systolic Arterial Blood Pressure': 115.06666666666666, 'Temperature': 37.0, 'Urea': 7.2, 'Urine Output': 68.965515, 'C reactive protein [Mass/volume] in Serum or Plasma': 74.0, 'Carboxyhemoglobin/Hemoglobin.total in Arterial blood': 1.4, 'Creatine kinase panel - Serum or Plasma': 291.0, 'End tidal carbon dioxide concentration': 29.913631487005645, 'Expiratory tidal volume': 537.0833333333334, 'Fluid balance in': 1039.9158305091403, 'Fluid balance out': 1220.8531, 'Infusion of saline solution': 534.1823620655173, 'INR in Blood by Coagulation assay': 1.11, 'Intravenous fluid colloid administration': 0.0, 'Mean inspiratory airway pressure': 9.283333333333331, 'Methemoglobin/Hemoglobin.total in Arterial blood': 0.9, 'Oxygen administration by nasal cannula': 4.0, 'Peak inspiratory pressure': 19.589189189189188, 'Plateau pressure': 19.37804555084746, 'Positive end expiratory pressure': 4.976666666666668, 'Positive end expiratory pressure setting': 5.0, 'ST elevation': 0.0508474576271186, 'Tidal volume setting': 500.0, 'Ventilator rate': 13.666666666666666}\n",
      "Mild\n",
      "{'Glasgow Coma Score - Verbal Response': 5.0, 'Glasgow Coma Score - Motor Response': 6.0, 'Glasgow Coma Score - Eye Opening': 4.0, 'Glasgow Coma Score - Total': 15.0, 'Circadian rhythm': 1.0, 'Richmond agitation-sedation scale': -4.0, 'Ventilator Airway Code': 1.0, 'Alanine aminotransferase': 41.0, 'Anion Gap': 5.25, 'Aspartate aminotransferase': 79.0, 'Base excess': -1.2999999999999998, 'Bicarbonate': 23.0, 'Body Height': 170.0, 'Body Weight': 75.0, 'Calcium.ionized': 1.15, 'Carbon dioxide [Partial pressure]': 34.6, 'Central Venous Pressure': 9.016666666666667, 'Chloride': 108.5, 'Creatinine': 95.0, 'Diastolic Arterial Blood Pressure': 53.2, 'Fraction of Inspired Oxygen': 45.25833333333333, 'Glucose': 8.2, 'Heart rate': 90.0, 'Hemoglobin': 102.5, 'Invasive Diastolic Arterial Blood Pressure': 53.134234234234235, 'Invasive Mean Arterial Blood Pressure': 70.08333333333333, 'Invasive Systolic Arterial Blood Pressure': 104.56666666666666, 'Lactate': 2.5, 'Leukocytes': 10.7, 'Magnesium': 0.82, 'Mean Arterial Blood Pressure': 70.18333333333334, 'Mean Cell Hemoglobin': 31.0, 'Mean Cell Hemoglobin Concentration': 340.0, 'Mean Corpuscular Volume': 90.0, 'Oxygen [Partial pressure]': 95.6, 'Oxygen Saturation': 96.25, 'pH': 7.422, 'Phosphate': 1.15, 'Platelets': 132.0, 'Potassium': 4.15, 'Respiratory rate': 16.24128844839371, 'Sodium': 137.0, 'Systolic Arterial Blood Pressure': 104.68333333333334, 'Temperature': 37.06666666666667, 'Urea': 8.2, 'Urine Output': 60.0, 'C reactive protein [Mass/volume] in Serum or Plasma': 65.0, 'Carboxyhemoglobin/Hemoglobin.total in Arterial blood': 1.3, 'Creatine kinase panel - Serum or Plasma': 452.0, 'End tidal carbon dioxide concentration': 28.00454221257062, 'Expiratory tidal volume': 543.9344262295082, 'Fluid balance in': 1490.978293333333, 'Fluid balance out': 1097.2737499999998, 'Infusion of saline solution': 893.7343563736264, 'INR in Blood by Coagulation assay': 1.18, 'Intravenous fluid colloid administration': 0.0, 'Mean inspiratory airway pressure': 10.063022598870056, 'Methemoglobin/Hemoglobin.total in Arterial blood': 0.9, 'Oxygen administration by nasal cannula': 4.0, 'Peak inspiratory pressure': 23.11, 'Plateau pressure': 20.11839139344262, 'Positive end expiratory pressure': 5.110344827586206, 'Positive end expiratory pressure setting': 5.0, 'ST elevation': 0.0491803278688524, 'Tidal volume setting': 500.0, 'Ventilator rate': 14.0}\n",
      "Severe\n",
      "{'Glasgow Coma Score - Verbal Response': 5.0, 'Glasgow Coma Score - Motor Response': 6.0, 'Glasgow Coma Score - Eye Opening': 4.0, 'Glasgow Coma Score - Total': 15.0, 'Circadian rhythm': 1.0, 'Richmond agitation-sedation scale': -5.0, 'Ventilator Airway Code': 1.0, 'Alanine aminotransferase': 80.0, 'Anion Gap': 8.900000000000006, 'Aspartate aminotransferase': 170.0, 'Base excess': -5.3, 'Bicarbonate': 19.5, 'Body Height': 170.0, 'Body Weight': 75.0, 'Calcium.ionized': 1.15, 'Carbon dioxide [Partial pressure]': 32.0, 'Central Venous Pressure': 10.783333333333331, 'Chloride': 108.0, 'Creatinine': 131.0, 'Diastolic Arterial Blood Pressure': 49.15127118644068, 'Fraction of Inspired Oxygen': 50.185, 'Glucose': 8.5, 'Heart rate': 97.98333333333332, 'Hemoglobin': 100.0, 'Invasive Diastolic Arterial Blood Pressure': 49.08620689655172, 'Invasive Mean Arterial Blood Pressure': 63.22950819672131, 'Invasive Systolic Arterial Blood Pressure': 92.82758620689656, 'Lactate': 5.6, 'Leukocytes': 11.1, 'Magnesium': 0.82, 'Mean Arterial Blood Pressure': 63.31147540983606, 'Mean Cell Hemoglobin': 31.0, 'Mean Cell Hemoglobin Concentration': 340.0, 'Mean Corpuscular Volume': 90.0, 'Oxygen [Partial pressure]': 96.7, 'Oxygen Saturation': 91.79603235747304, 'pH': 7.379, 'Phosphate': 1.44, 'Platelets': 105.0, 'Potassium': 4.3, 'Respiratory rate': 15.0, 'Sodium': 137.0, 'Systolic Arterial Blood Pressure': 93.0, 'Temperature': 36.90819672131148, 'Urea': 9.8, 'Urine Output': 42.857143, 'C reactive protein [Mass/volume] in Serum or Plasma': 60.0, 'Carboxyhemoglobin/Hemoglobin.total in Arterial blood': 1.3, 'Creatine kinase panel - Serum or Plasma': 521.0, 'End tidal carbon dioxide concentration': 24.884615384615383, 'Expiratory tidal volume': 547.8770491803278, 'Fluid balance in': 2163.820376485788, 'Fluid balance out': 912.6123749999999, 'Infusion of saline solution': 1333.146052406948, 'INR in Blood by Coagulation assay': 1.32, 'Intravenous fluid colloid administration': 0.0, 'Mean inspiratory airway pressure': 11.96923076923077, 'Methemoglobin/Hemoglobin.total in Arterial blood': 1.0, 'Oxygen administration by nasal cannula': 5.0, 'Peak inspiratory pressure': 28.773333333333333, 'Plateau pressure': 21.648333333333333, 'Positive end expiratory pressure': 5.773770491803279, 'Positive end expiratory pressure setting': 5.666666666666667, 'ST elevation': 0.0249999999999999, 'Tidal volume setting': 507.77777777777777, 'Ventilator rate': 15.3}\n"
     ]
    }
   ],
   "source": [
    "HYPER_categ_fillna = \"mode\"  # possible values: mode\n",
    "HYPER_numer_fillna = \"median\"  # possible values: mean, median\n",
    "print(\n",
    "    f\"Imputation Strategies:\\nCategorical: {HYPER_categ_fillna}\\nNumerical: {HYPER_numer_fillna}\"\n",
    ")\n",
    "\n",
    "\n",
    "def custom_mode(x):\n",
    "    m = pd.Series.mode(x)\n",
    "    return m.iloc[0] if not m.empty else None\n",
    "\n",
    "\n",
    "def get_fillna_method(x):\n",
    "    if x == \"mean\":\n",
    "        return np_nanmean\n",
    "    elif x == \"median\":\n",
    "        return np_nanmedian\n",
    "    elif x == \"mode\":\n",
    "        return custom_mode\n",
    "\n",
    "\n",
    "os.makedirs(\"./results/splits\", exist_ok=True)\n",
    "for cl, cl_name in ((1, \"Normal\"), (2, \"Mild\"), (3, \"Severe\")):\n",
    "    df_class = pd.read_csv(f\"./results/class_{cl}_{cl_name}_sequentialised.csv\")\n",
    "    grouped = df_class.groupby(df_class[\"Sequential ID\"])\n",
    "\n",
    "    train_seq_ids, test_seq_ids = model_selection.train_test_split(\n",
    "        list(grouped.groups.keys()), train_size=0.75, random_state=666\n",
    "    )\n",
    "    df_train = df_class[df_class[\"Sequential ID\"].isin(train_seq_ids)]\n",
    "    df_test = df_class[df_class[\"Sequential ID\"].isin(test_seq_ids)]\n",
    "\n",
    "    X_train = df_train[[x for x in df_train.columns if x != \"Lactate_Outcome\"]].copy()\n",
    "    # remove duplicates of the same sequential id\n",
    "    y_train = (\n",
    "        df_train[[\"Sequential ID\", \"Lactate_Outcome\"]]\n",
    "        .groupby(by=\"Sequential ID\")\n",
    "        .first()[\"Lactate_Outcome\"]\n",
    "    )\n",
    "    y_train.to_csv(f\"./results/splits/y_train_{cl}_{cl_name}.csv\", index=False)\n",
    "\n",
    "    X_test = df_test[[x for x in df_test.columns if x != \"Lactate_Outcome\"]].copy()\n",
    "    # remove duplicates of the same sequential id\n",
    "    y_test = (\n",
    "        df_test[[\"Sequential ID\", \"Lactate_Outcome\"]]\n",
    "        .groupby(by=\"Sequential ID\")\n",
    "        .first()[\"Lactate_Outcome\"]\n",
    "    )\n",
    "    y_test.to_csv(f\"./results/splits/y_test_{cl}_{cl_name}.csv\", index=False)\n",
    "\n",
    "    # extract excluded/categorical/numerical variables\n",
    "    excluded_vars = (\n",
    "        \"Patient ID\",\n",
    "        \"Minutes of Stay\",\n",
    "        \"Lactate_Class\",\n",
    "        \"Lactate_Outcome\",\n",
    "        \"Sequential ID\",\n",
    "    )\n",
    "    categorical_vars = (\n",
    "        \"Glasgow Coma Score - Verbal Response\",\n",
    "        \"Glasgow Coma Score - Motor Response\",\n",
    "        \"Glasgow Coma Score - Eye Opening\",\n",
    "        \"Glasgow Coma Score - Total\",\n",
    "        \"Circadian rhythm\",\n",
    "        \"Richmond agitation-sedation scale\",\n",
    "        \"Ventilator Airway Code\",\n",
    "    )\n",
    "    numerical_vars = [\n",
    "        x\n",
    "        for x in df_class.columns\n",
    "        if x not in excluded_vars and x not in categorical_vars\n",
    "    ]\n",
    "    # create dictionary for fill-forwarding with the appropriate method\n",
    "    fillna_dict = {\n",
    "        x: get_fillna_method(HYPER_categ_fillna)(X_train[x]) for x in categorical_vars\n",
    "    }\n",
    "    fillna_dict.update(\n",
    "        {x: get_fillna_method(HYPER_numer_fillna)(X_train[x]) for x in numerical_vars}\n",
    "    )\n",
    "\n",
    "    X_train.fillna(fillna_dict, inplace=True)\n",
    "    X_train.to_csv(f\"./results/splits/X_train_{cl}_{cl_name}.csv\", index=False)\n",
    "\n",
    "    # impute test with values obtained from train\n",
    "    X_test.fillna(fillna_dict, inplace=True)\n",
    "    X_test.to_csv(f\"./results/splits/X_test_{cl}_{cl_name}.csv\", index=False)\n",
    "\n",
    "    print(cl_name)\n",
    "    print(fillna_dict)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Add Padding/Remove Exceeding Rows and Translate to a Single Row\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "HYPER_max_pad = 10\n",
    "\n",
    "for dataset in (\"train\", \"test\"):\n",
    "    for cl, cl_name in ((1, \"Normal\"), (2, \"Mild\"), (3, \"Severe\")):\n",
    "        X = pd.read_csv(f\"./results/splits/X_{dataset}_{cl}_{cl_name}.csv\")\n",
    "\n",
    "        new_columns = list()\n",
    "        for i in range(HYPER_max_pad):\n",
    "            new_columns.extend([f\"{x}_{i}\" for x in X.columns])\n",
    "\n",
    "        grouped = X.groupby(X[\"Sequential ID\"])\n",
    "\n",
    "        rows = list()\n",
    "        for k, group in grouped.groups.items():\n",
    "            # add row containing both real rows and zero rows\n",
    "            if len(group) > HYPER_max_pad:\n",
    "                group = group[:HYPER_max_pad]\n",
    "            X_sub = X.loc[group]\n",
    "\n",
    "            row = list()\n",
    "            for i in range(len(group)):\n",
    "                row.extend(X_sub.iloc[i].tolist())\n",
    "            for i in range(HYPER_max_pad - len(group)):\n",
    "                row.extend([0] * len(X.columns))\n",
    "            rows.append(row)\n",
    "\n",
    "        new_X = pd.DataFrame(data=rows, columns=new_columns)\n",
    "        new_X.to_csv(\n",
    "            f\"./results/splits/X_{dataset}_{cl}_{cl_name}_padded_translated.csv\",\n",
    "            index=False,\n",
    "        )\n"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "c643c1ae580a7eadaaaf8eb2846b79a7c1c6576dfd78895ee4877fd48cad0027"
  },
  "kernelspec": {
   "display_name": "Python 3.9.5 64-bit ('.venv': venv)",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}