brca_preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import os\n",
    "import polars as pl\n",
    "\n",
    "from src.seqeuencing_data_preprocessing import preprocess_seq_data\n",
    "from src.data_preprocessing import OmicDataSplitter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (10_745, 490)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>gene</th><th>TCGA-BH-A1ET-01</th><th>TCGA-AR-A1AI-01</th><th>TCGA-AN-A0AL-01</th><th>TCGA-BH-A0HK-01</th><th>TCGA-A8-A09G-01</th><th>TCGA-E2-A109-01</th><th>TCGA-AN-A0AM-01</th><th>TCGA-A2-A0CU-01</th><th>TCGA-AR-A1AY-01</th><th>TCGA-AN-A0FF-01</th><th>TCGA-BH-A0B5-01</th><th>TCGA-A8-A06Z-01</th><th>TCGA-C8-A12U-01</th><th>TCGA-C8-A131-01</th><th>TCGA-A2-A0YG-01</th><th>TCGA-A2-A0CV-01</th><th>TCGA-B6-A0RT-01</th><th>TCGA-A7-A0CG-01</th><th>TCGA-C8-A1HI-01</th><th>TCGA-BH-A18S-01</th><th>TCGA-A8-A07L-01</th><th>TCGA-BH-A0BA-01</th><th>TCGA-BH-A0H7-01</th><th>TCGA-BH-A0W5-01</th><th>TCGA-AO-A03T-01</th><th>TCGA-BH-A0BJ-01</th><th>TCGA-A2-A0EM-01</th><th>TCGA-AN-A0XN-01</th><th>TCGA-A8-A084-01</th><th>TCGA-AN-A0FV-01</th><th>TCGA-BH-A0DQ-01</th><th>TCGA-AR-A0TR-01</th><th>TCGA-AN-A0XW-01</th><th>TCGA-BH-A18G-01</th><th>TCGA-BH-A0AY-01</th><th>TCGA-A7-A0D9-01</th><th>&hellip;</th><th>TCGA-AN-A0XU-01</th><th>TCGA-A2-A0EQ-01</th><th>TCGA-D8-A141-01</th><th>TCGA-A8-A07O-01</th><th>TCGA-B6-A0IO-01</th><th>TCGA-E2-A158-01</th><th>TCGA-A8-A07P-01</th><th>TCGA-A8-A099-01</th><th>TCGA-A2-A0YH-01</th><th>TCGA-AN-A04C-01</th><th>TCGA-BH-A0AZ-01</th><th>TCGA-AN-A0AK-01</th><th>TCGA-AO-A12E-01</th><th>TCGA-A2-A04N-01</th><th>TCGA-A2-A0D2-01</th><th>TCGA-A2-A0SW-01</th><th>TCGA-AR-A0U2-01</th><th>TCGA-BH-A0E2-01</th><th>TCGA-D8-A147-01</th><th>TCGA-D8-A13Z-01</th><th>TCGA-E2-A152-01</th><th>TCGA-A8-A06Y-01</th><th>TCGA-A2-A0T4-01</th><th>TCGA-BH-A0HY-01</th><th>TCGA-AQ-A04J-01</th><th>TCGA-BH-A1ES-01</th><th>TCGA-AO-A0J9-01</th><th>TCGA-BH-A0HO-01</th><th>TCGA-BH-A0B4-01</th><th>TCGA-A8-A09B-01</th><th>TCGA-AO-A0J2-01</th><th>TCGA-E2-A1BC-01</th><th>TCGA-A2-A04Q-01</th><th>TCGA-E2-A14O-01</th><th>TCGA-E2-A15F-01</th><th>TCGA-BH-A0DZ-01</th><th>TCGA-E2-A107-01</th></tr><tr><td>str</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>&hellip;</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td></tr></thead><tbody><tr><td>&quot;ARHGEF10L&quot;</td><td>9.471</td><td>11.2755</td><td>10.2441</td><td>9.3233</td><td>8.6366</td><td>9.8626</td><td>9.46</td><td>11.2247</td><td>9.4109</td><td>10.5067</td><td>10.2724</td><td>9.3159</td><td>10.8544</td><td>9.1495</td><td>8.8155</td><td>8.9437</td><td>10.7764</td><td>8.6821</td><td>9.2141</td><td>8.6442</td><td>9.5089</td><td>8.7218</td><td>9.4944</td><td>9.7738</td><td>9.9518</td><td>9.5628</td><td>9.3572</td><td>9.172</td><td>9.6494</td><td>9.0805</td><td>9.2398</td><td>9.9834</td><td>9.3135</td><td>11.0812</td><td>9.1019</td><td>10.9025</td><td>&hellip;</td><td>10.0025</td><td>10.2059</td><td>9.6739</td><td>9.6396</td><td>10.1613</td><td>9.2199</td><td>9.0376</td><td>8.3392</td><td>9.9363</td><td>9.19</td><td>9.4325</td><td>9.9483</td><td>9.8929</td><td>9.9831</td><td>10.1611</td><td>9.1269</td><td>9.4702</td><td>9.6913</td><td>10.4024</td><td>9.5647</td><td>9.8236</td><td>8.7745</td><td>9.1123</td><td>9.34</td><td>9.7674</td><td>9.7386</td><td>9.6978</td><td>8.6016</td><td>9.2534</td><td>10.0525</td><td>8.9836</td><td>9.0303</td><td>10.064</td><td>8.9483</td><td>10.0334</td><td>9.704</td><td>10.3434</td></tr><tr><td>&quot;HIF3A&quot;</td><td>2.4075</td><td>5.553</td><td>2.3294</td><td>5.2633</td><td>3.1131</td><td>0.944</td><td>0.4316</td><td>2.9769</td><td>4.637</td><td>3.4019</td><td>2.5858</td><td>0.5913</td><td>0.8993</td><td>2.1272</td><td>0.0</td><td>3.0291</td><td>7.242</td><td>3.7612</td><td>2.8502</td><td>1.7164</td><td>0.4251</td><td>5.2743</td><td>1.2099</td><td>3.5127</td><td>3.8956</td><td>1.8117</td><td>1.4728</td><td>1.5814</td><td>1.8903</td><td>2.9125</td><td>5.0026</td><td>3.0731</td><td>1.1158</td><td>2.0444</td><td>3.9797</td><td>2.8431</td><td>&hellip;</td><td>6.9861</td><td>1.1658</td><td>1.8964</td><td>1.135</td><td>1.8607</td><td>4.3358</td><td>1.0262</td><td>0.0</td><td>2.8848</td><td>1.2742</td><td>3.4037</td><td>0.0</td><td>4.4619</td><td>1.7558</td><td>2.1132</td><td>3.1533</td><td>2.6489</td><td>0.4777</td><td>4.7636</td><td>2.1727</td><td>2.3097</td><td>0.4999</td><td>4.2523</td><td>0.0</td><td>8.7758</td><td>1.4002</td><td>5.0152</td><td>0.8863</td><td>3.0086</td><td>3.1148</td><td>1.2778</td><td>6.9578</td><td>2.2799</td><td>1.7676</td><td>0.8836</td><td>3.587</td><td>2.6529</td></tr><tr><td>&quot;RNF17&quot;</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.6923</td><td>0.0</td><td>0.0</td><td>1.3043</td><td>0.0</td><td>0.0</td><td>0.3257</td><td>0.8993</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.71</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.4251</td><td>0.0</td><td>0.9891</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>&hellip;</td><td>0.4272</td><td>0.0</td><td>0.0</td><td>0.9238</td><td>0.7294</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.438</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>1.2118</td><td>0.0</td><td>0.0</td><td>0.4173</td><td>1.2324</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.3593</td><td>0.0</td><td>0.0</td><td>0.3617</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td></tr><tr><td>&quot;RNF10&quot;</td><td>11.9416</td><td>11.3922</td><td>12.8216</td><td>11.6027</td><td>12.2614</td><td>11.9256</td><td>11.8298</td><td>11.5801</td><td>11.8127</td><td>12.2998</td><td>11.9089</td><td>11.8353</td><td>11.7444</td><td>11.7985</td><td>12.2153</td><td>12.0265</td><td>11.3035</td><td>11.5211</td><td>12.8131</td><td>11.6078</td><td>11.7502</td><td>12.3758</td><td>12.3283</td><td>11.7013</td><td>12.146</td><td>11.7872</td><td>11.6602</td><td>11.3834</td><td>12.4</td><td>12.09</td><td>12.2941</td><td>12.2632</td><td>11.7749</td><td>12.0761</td><td>11.8532</td><td>11.7959</td><td>&hellip;</td><td>11.1178</td><td>11.9862</td><td>11.9092</td><td>11.2711</td><td>12.5781</td><td>11.8904</td><td>11.8852</td><td>11.7441</td><td>11.8044</td><td>11.6267</td><td>11.9718</td><td>12.3737</td><td>12.0904</td><td>12.185</td><td>11.7749</td><td>12.0454</td><td>12.2348</td><td>11.9049</td><td>11.4056</td><td>12.1598</td><td>12.1153</td><td>12.601</td><td>11.721</td><td>11.4961</td><td>11.3629</td><td>12.2986</td><td>12.4817</td><td>11.9571</td><td>11.6441</td><td>12.023</td><td>11.2162</td><td>11.8271</td><td>11.7065</td><td>12.7112</td><td>12.0135</td><td>11.5199</td><td>12.4371</td></tr><tr><td>&quot;RNF11&quot;</td><td>11.7528</td><td>11.332</td><td>11.1384</td><td>11.2566</td><td>11.8996</td><td>10.862</td><td>10.8835</td><td>11.116</td><td>11.1572</td><td>10.9935</td><td>10.7238</td><td>11.7705</td><td>11.6908</td><td>11.4236</td><td>11.3417</td><td>11.5781</td><td>10.652</td><td>11.5159</td><td>11.7781</td><td>11.4076</td><td>11.2024</td><td>11.4299</td><td>11.2033</td><td>10.8349</td><td>11.2574</td><td>11.3273</td><td>11.7259</td><td>11.5843</td><td>10.9288</td><td>10.8986</td><td>11.6315</td><td>11.3874</td><td>11.4345</td><td>10.9959</td><td>10.7975</td><td>11.164</td><td>&hellip;</td><td>11.1285</td><td>11.3253</td><td>11.0164</td><td>11.0634</td><td>11.0549</td><td>10.6569</td><td>11.9269</td><td>10.5351</td><td>10.5196</td><td>11.7137</td><td>11.0161</td><td>10.7525</td><td>11.301</td><td>11.3242</td><td>10.9105</td><td>11.5032</td><td>10.5823</td><td>11.0442</td><td>10.5655</td><td>11.5122</td><td>11.8669</td><td>10.4113</td><td>10.7345</td><td>11.5667</td><td>11.3039</td><td>11.5836</td><td>10.8825</td><td>11.3023</td><td>10.4376</td><td>10.9447</td><td>10.7171</td><td>11.8211</td><td>10.7574</td><td>10.4735</td><td>10.837</td><td>11.3034</td><td>10.5106</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>&quot;GNGT2&quot;</td><td>3.947</td><td>4.9199</td><td>5.0879</td><td>4.5641</td><td>4.1062</td><td>3.8931</td><td>4.061</td><td>3.8444</td><td>3.1832</td><td>4.6607</td><td>4.1164</td><td>3.6024</td><td>4.5541</td><td>4.3737</td><td>2.9802</td><td>4.042</td><td>6.5023</td><td>4.5706</td><td>4.3694</td><td>3.9724</td><td>2.9091</td><td>4.0966</td><td>2.9808</td><td>4.8465</td><td>4.5305</td><td>4.0375</td><td>3.8574</td><td>4.5494</td><td>3.8226</td><td>3.69</td><td>4.4758</td><td>3.4446</td><td>5.1229</td><td>2.9773</td><td>4.4787</td><td>2.9998</td><td>&hellip;</td><td>4.5902</td><td>5.7674</td><td>5.3513</td><td>4.1249</td><td>3.5676</td><td>1.7652</td><td>4.347</td><td>2.9363</td><td>5.4097</td><td>4.6301</td><td>3.9868</td><td>3.8088</td><td>4.0025</td><td>3.3381</td><td>4.844</td><td>5.1384</td><td>3.8099</td><td>3.8424</td><td>3.9781</td><td>4.5318</td><td>3.4455</td><td>2.765</td><td>4.9435</td><td>3.6319</td><td>5.3346</td><td>2.35</td><td>6.1248</td><td>5.0425</td><td>4.5889</td><td>3.8663</td><td>5.2387</td><td>3.2856</td><td>6.6174</td><td>3.5634</td><td>3.422</td><td>3.7774</td><td>3.9888</td></tr><tr><td>&quot;TULP3&quot;</td><td>9.1785</td><td>9.5478</td><td>9.2774</td><td>9.2738</td><td>9.5246</td><td>9.3404</td><td>10.4162</td><td>9.5108</td><td>10.438</td><td>9.1038</td><td>10.0173</td><td>8.9221</td><td>9.8832</td><td>8.9279</td><td>9.1144</td><td>9.4847</td><td>9.0748</td><td>9.355</td><td>8.4248</td><td>9.1743</td><td>10.5338</td><td>9.4726</td><td>9.0179</td><td>9.2183</td><td>8.8151</td><td>9.3753</td><td>9.1971</td><td>9.5376</td><td>8.9441</td><td>8.805</td><td>8.8631</td><td>8.4917</td><td>9.1998</td><td>9.1684</td><td>9.12</td><td>9.3986</td><td>&hellip;</td><td>10.0633</td><td>9.5737</td><td>9.0835</td><td>9.5946</td><td>9.6786</td><td>9.7393</td><td>8.399</td><td>9.3215</td><td>9.1991</td><td>9.7053</td><td>9.4225</td><td>9.6185</td><td>9.4209</td><td>9.3547</td><td>10.542</td><td>9.2327</td><td>9.0915</td><td>9.4633</td><td>10.7364</td><td>10.5701</td><td>9.3679</td><td>9.1166</td><td>9.406</td><td>9.2747</td><td>10.2631</td><td>9.3301</td><td>9.6605</td><td>8.7205</td><td>9.2443</td><td>9.4361</td><td>10.1325</td><td>8.897</td><td>10.0738</td><td>9.5897</td><td>9.6514</td><td>9.0424</td><td>9.9034</td></tr><tr><td>&quot;PTRF&quot;</td><td>11.434</td><td>10.0178</td><td>12.8431</td><td>12.5212</td><td>11.3567</td><td>11.6336</td><td>12.2257</td><td>12.4225</td><td>11.8505</td><td>11.9616</td><td>12.1276</td><td>10.4666</td><td>10.0608</td><td>12.6</td><td>10.7383</td><td>12.5659</td><td>11.9763</td><td>12.5935</td><td>12.2167</td><td>10.4368</td><td>11.8909</td><td>12.2505</td><td>12.8907</td><td>12.2675</td><td>12.5703</td><td>12.7012</td><td>12.577</td><td>13.2146</td><td>12.1829</td><td>12.633</td><td>13.6699</td><td>9.3613</td><td>13.1219</td><td>11.839</td><td>13.3023</td><td>11.9928</td><td>&hellip;</td><td>11.4742</td><td>11.6713</td><td>13.3273</td><td>10.9943</td><td>10.9355</td><td>11.5021</td><td>11.7454</td><td>11.3534</td><td>11.9217</td><td>11.3933</td><td>13.3697</td><td>10.5919</td><td>12.4369</td><td>12.6766</td><td>13.1791</td><td>12.0932</td><td>10.3675</td><td>12.598</td><td>12.9657</td><td>12.5512</td><td>12.7038</td><td>10.7465</td><td>13.3531</td><td>12.3278</td><td>12.6362</td><td>11.759</td><td>13.9498</td><td>11.2138</td><td>13.2262</td><td>12.2321</td><td>11.3244</td><td>12.9017</td><td>12.1301</td><td>11.1147</td><td>11.3156</td><td>12.7905</td><td>11.4905</td></tr><tr><td>&quot;GSTK1&quot;</td><td>10.511</td><td>11.6433</td><td>10.8662</td><td>11.4611</td><td>10.2741</td><td>10.3287</td><td>10.5127</td><td>11.6343</td><td>9.7031</td><td>10.9937</td><td>10.8841</td><td>11.629</td><td>10.414</td><td>11.2031</td><td>10.8485</td><td>11.8265</td><td>11.4481</td><td>11.0426</td><td>10.9127</td><td>11.6881</td><td>10.2285</td><td>11.0698</td><td>10.8643</td><td>11.0095</td><td>11.8115</td><td>11.7796</td><td>11.477</td><td>9.9953</td><td>11.0149</td><td>11.3842</td><td>11.794</td><td>11.1406</td><td>10.9516</td><td>11.5116</td><td>10.6636</td><td>12.1377</td><td>&hellip;</td><td>10.7191</td><td>11.499</td><td>10.7727</td><td>10.6417</td><td>11.4855</td><td>10.8892</td><td>10.7083</td><td>10.063</td><td>10.8674</td><td>10.9099</td><td>10.922</td><td>10.8782</td><td>11.5536</td><td>11.1675</td><td>11.3601</td><td>11.4435</td><td>9.2171</td><td>11.1389</td><td>10.5617</td><td>10.3353</td><td>10.429</td><td>11.5147</td><td>11.1782</td><td>10.5517</td><td>10.4854</td><td>11.1996</td><td>11.7049</td><td>12.1238</td><td>11.1503</td><td>11.6748</td><td>12.9945</td><td>9.9638</td><td>11.0695</td><td>11.8135</td><td>10.7775</td><td>10.4267</td><td>11.3894</td></tr><tr><td>&quot;SELP&quot;</td><td>7.8144</td><td>5.7845</td><td>8.0978</td><td>8.2477</td><td>6.6639</td><td>5.9535</td><td>3.7971</td><td>8.4096</td><td>5.1953</td><td>6.9957</td><td>6.2298</td><td>3.0611</td><td>4.0124</td><td>5.6183</td><td>5.7678</td><td>8.2715</td><td>9.021</td><td>9.0606</td><td>6.9534</td><td>5.9859</td><td>4.8799</td><td>7.6148</td><td>4.8368</td><td>8.7634</td><td>7.0845</td><td>7.4148</td><td>6.5606</td><td>1.3187</td><td>2.5916</td><td>0.0</td><td>7.9954</td><td>6.8793</td><td>6.0856</td><td>0.0</td><td>7.8754</td><td>6.3448</td><td>&hellip;</td><td>0.4272</td><td>7.6732</td><td>6.9408</td><td>2.9778</td><td>5.3034</td><td>5.2485</td><td>6.0511</td><td>5.5642</td><td>6.7756</td><td>6.6329</td><td>7.3313</td><td>3.2321</td><td>7.7587</td><td>6.4686</td><td>7.4731</td><td>6.0429</td><td>5.5418</td><td>5.8751</td><td>4.9473</td><td>7.061</td><td>1.5747</td><td>5.5913</td><td>9.0802</td><td>5.4838</td><td>9.1992</td><td>3.4378</td><td>7.8102</td><td>6.7798</td><td>7.7695</td><td>7.6048</td><td>2.547</td><td>7.1316</td><td>8.1501</td><td>5.9617</td><td>8.1673</td><td>4.9777</td><td>8.0896</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (10_745, 490)\n",
       "┌───────────┬───────────┬───────────┬───────────┬───┬───────────┬───────────┬───────────┬──────────┐\n",
       "│ gene      ┆ TCGA-BH-A ┆ TCGA-AR-A ┆ TCGA-AN-A ┆ … ┆ TCGA-E2-A ┆ TCGA-E2-A ┆ TCGA-BH-A ┆ TCGA-E2- │\n",
       "│ ---       ┆ 1ET-01    ┆ 1AI-01    ┆ 0AL-01    ┆   ┆ 14O-01    ┆ 15F-01    ┆ 0DZ-01    ┆ A107-01  │\n",
       "│ str       ┆ ---       ┆ ---       ┆ ---       ┆   ┆ ---       ┆ ---       ┆ ---       ┆ ---      │\n",
       "│           ┆ f64       ┆ f64       ┆ f64       ┆   ┆ f64       ┆ f64       ┆ f64       ┆ f64      │\n",
       "╞═══════════╪═══════════╪═══════════╪═══════════╪═══╪═══════════╪═══════════╪═══════════╪══════════╡\n",
       "│ ARHGEF10L ┆ 9.471     ┆ 11.2755   ┆ 10.2441   ┆ … ┆ 8.9483    ┆ 10.0334   ┆ 9.704     ┆ 10.3434  │\n",
       "│ HIF3A     ┆ 2.4075    ┆ 5.553     ┆ 2.3294    ┆ … ┆ 1.7676    ┆ 0.8836    ┆ 3.587     ┆ 2.6529   │\n",
       "│ RNF17     ┆ 0.0       ┆ 0.0       ┆ 0.0       ┆ … ┆ 0.0       ┆ 0.0       ┆ 0.0       ┆ 0.0      │\n",
       "│ RNF10     ┆ 11.9416   ┆ 11.3922   ┆ 12.8216   ┆ … ┆ 12.7112   ┆ 12.0135   ┆ 11.5199   ┆ 12.4371  │\n",
       "│ RNF11     ┆ 11.7528   ┆ 11.332    ┆ 11.1384   ┆ … ┆ 10.4735   ┆ 10.837    ┆ 11.3034   ┆ 10.5106  │\n",
       "│ …         ┆ …         ┆ …         ┆ …         ┆ … ┆ …         ┆ …         ┆ …         ┆ …        │\n",
       "│ GNGT2     ┆ 3.947     ┆ 4.9199    ┆ 5.0879    ┆ … ┆ 3.5634    ┆ 3.422     ┆ 3.7774    ┆ 3.9888   │\n",
       "│ TULP3     ┆ 9.1785    ┆ 9.5478    ┆ 9.2774    ┆ … ┆ 9.5897    ┆ 9.6514    ┆ 9.0424    ┆ 9.9034   │\n",
       "│ PTRF      ┆ 11.434    ┆ 10.0178   ┆ 12.8431   ┆ … ┆ 11.1147   ┆ 11.3156   ┆ 12.7905   ┆ 11.4905  │\n",
       "│ GSTK1     ┆ 10.511    ┆ 11.6433   ┆ 10.8662   ┆ … ┆ 11.8135   ┆ 10.7775   ┆ 10.4267   ┆ 11.3894  │\n",
       "│ SELP      ┆ 7.8144    ┆ 5.7845    ┆ 8.0978    ┆ … ┆ 5.9617    ┆ 8.1673    ┆ 4.9777    ┆ 8.0896   │\n",
       "└───────────┴───────────┴───────────┴───────────┴───┴───────────┴───────────┴───────────┴──────────┘"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mrna = pl.read_csv(\n",
    "    \"tcga_brca_data/raw/mrna.csv\"\n",
    ")\n",
    "mrna"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (489, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>sampleID</th><th>PAM50_mRNA_nature2012</th><th>classes</th></tr><tr><td>str</td><td>str</td><td>i64</td></tr></thead><tbody><tr><td>&quot;TCGA-A1-A0SD-01&quot;</td><td>&quot;Luminal A&quot;</td><td>2</td></tr><tr><td>&quot;TCGA-A1-A0SE-01&quot;</td><td>&quot;Luminal A&quot;</td><td>2</td></tr><tr><td>&quot;TCGA-A1-A0SH-01&quot;</td><td>&quot;Luminal A&quot;</td><td>2</td></tr><tr><td>&quot;TCGA-A1-A0SJ-01&quot;</td><td>&quot;Luminal A&quot;</td><td>2</td></tr><tr><td>&quot;TCGA-A1-A0SK-01&quot;</td><td>&quot;Basal-like&quot;</td><td>0</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>&quot;TCGA-E2-A1B4-01&quot;</td><td>&quot;Luminal A&quot;</td><td>2</td></tr><tr><td>&quot;TCGA-E2-A1B5-01&quot;</td><td>&quot;Basal-like&quot;</td><td>0</td></tr><tr><td>&quot;TCGA-E2-A1B6-01&quot;</td><td>&quot;Luminal A&quot;</td><td>2</td></tr><tr><td>&quot;TCGA-E2-A1BC-01&quot;</td><td>&quot;Luminal A&quot;</td><td>2</td></tr><tr><td>&quot;TCGA-E2-A1BD-01&quot;</td><td>&quot;Luminal A&quot;</td><td>2</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (489, 3)\n",
       "┌─────────────────┬───────────────────────┬─────────┐\n",
       "│ sampleID        ┆ PAM50_mRNA_nature2012 ┆ classes │\n",
       "│ ---             ┆ ---                   ┆ ---     │\n",
       "│ str             ┆ str                   ┆ i64     │\n",
       "╞═════════════════╪═══════════════════════╪═════════╡\n",
       "│ TCGA-A1-A0SD-01 ┆ Luminal A             ┆ 2       │\n",
       "│ TCGA-A1-A0SE-01 ┆ Luminal A             ┆ 2       │\n",
       "│ TCGA-A1-A0SH-01 ┆ Luminal A             ┆ 2       │\n",
       "│ TCGA-A1-A0SJ-01 ┆ Luminal A             ┆ 2       │\n",
       "│ TCGA-A1-A0SK-01 ┆ Basal-like            ┆ 0       │\n",
       "│ …               ┆ …                     ┆ …       │\n",
       "│ TCGA-E2-A1B4-01 ┆ Luminal A             ┆ 2       │\n",
       "│ TCGA-E2-A1B5-01 ┆ Basal-like            ┆ 0       │\n",
       "│ TCGA-E2-A1B6-01 ┆ Luminal A             ┆ 2       │\n",
       "│ TCGA-E2-A1BC-01 ┆ Luminal A             ┆ 2       │\n",
       "│ TCGA-E2-A1BD-01 ┆ Luminal A             ┆ 2       │\n",
       "└─────────────────┴───────────────────────┴─────────┘"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "annotations = pl.read_csv(\"tcga_brca_data/raw/annotations.csv\")\n",
    "annotations\n",
    "\n",
    "# annotations = annotations.with_columns(\n",
    "#     (pl.col(\"PAM50_mRNA_nature2012\").rank(\"dense\") - 1).alias(\"classes\")\n",
    "# )\n",
    "# # annotations\n",
    "# annotations.filter(\n",
    "#     pl.col(\"sampleID\").is_in(mrna.columns)\n",
    "# ).write_csv(\"tcga_brca_data/raw/annotations.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>classes</th><th>count</th></tr><tr><td>i64</td><td>u32</td></tr></thead><tbody><tr><td>3</td><td>122</td></tr><tr><td>4</td><td>8</td></tr><tr><td>2</td><td>219</td></tr><tr><td>1</td><td>55</td></tr><tr><td>0</td><td>85</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 2)\n",
       "┌─────────┬───────┐\n",
       "│ classes ┆ count │\n",
       "│ ---     ┆ ---   │\n",
       "│ i64     ┆ u32   │\n",
       "╞═════════╪═══════╡\n",
       "│ 3       ┆ 122   │\n",
       "│ 4       ┆ 8     │\n",
       "│ 2       ┆ 219   │\n",
       "│ 1       ┆ 55    │\n",
       "│ 0       ┆ 85    │\n",
       "└─────────┴───────┘"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "annotations['classes'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "annotations = annotations.filter(\n",
    "    pl.col('PAM50_mRNA_nature2012') != \"Normal-like\"\n",
    ")\n",
    "nonnormal_samples = annotations['sampleID'].to_list()\n",
    "\n",
    "mrna = mrna.select(['gene'] + nonnormal_samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (4, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>classes</th><th>count</th></tr><tr><td>i64</td><td>u32</td></tr></thead><tbody><tr><td>1</td><td>55</td></tr><tr><td>3</td><td>122</td></tr><tr><td>0</td><td>85</td></tr><tr><td>2</td><td>219</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (4, 2)\n",
       "┌─────────┬───────┐\n",
       "│ classes ┆ count │\n",
       "│ ---     ┆ ---   │\n",
       "│ i64     ┆ u32   │\n",
       "╞═════════╪═══════╡\n",
       "│ 1       ┆ 55    │\n",
       "│ 3       ┆ 122   │\n",
       "│ 0       ┆ 85    │\n",
       "│ 2       ┆ 219   │\n",
       "└─────────┴───────┘"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "annotations['classes'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (10_745, 482)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>gene</th><th>TCGA-A1-A0SD-01</th><th>TCGA-A1-A0SE-01</th><th>TCGA-A1-A0SH-01</th><th>TCGA-A1-A0SJ-01</th><th>TCGA-A1-A0SK-01</th><th>TCGA-A1-A0SM-01</th><th>TCGA-A1-A0SO-01</th><th>TCGA-A2-A04N-01</th><th>TCGA-A2-A04P-01</th><th>TCGA-A2-A04Q-01</th><th>TCGA-A2-A04R-01</th><th>TCGA-A2-A04T-01</th><th>TCGA-A2-A04U-01</th><th>TCGA-A2-A04V-01</th><th>TCGA-A2-A04W-01</th><th>TCGA-A2-A04X-01</th><th>TCGA-A2-A04Y-01</th><th>TCGA-A2-A0CL-01</th><th>TCGA-A2-A0CM-01</th><th>TCGA-A2-A0CP-01</th><th>TCGA-A2-A0CQ-01</th><th>TCGA-A2-A0CS-01</th><th>TCGA-A2-A0CT-01</th><th>TCGA-A2-A0CU-01</th><th>TCGA-A2-A0CV-01</th><th>TCGA-A2-A0CW-01</th><th>TCGA-A2-A0D0-01</th><th>TCGA-A2-A0D1-01</th><th>TCGA-A2-A0D2-01</th><th>TCGA-A2-A0D3-01</th><th>TCGA-A2-A0D4-01</th><th>TCGA-A2-A0EM-01</th><th>TCGA-A2-A0EN-01</th><th>TCGA-A2-A0EO-01</th><th>TCGA-A2-A0EQ-01</th><th>TCGA-A2-A0ER-01</th><th>&hellip;</th><th>TCGA-E2-A14W-01</th><th>TCGA-E2-A14X-01</th><th>TCGA-E2-A14Y-01</th><th>TCGA-E2-A14Z-01</th><th>TCGA-E2-A150-01</th><th>TCGA-E2-A152-01</th><th>TCGA-E2-A153-01</th><th>TCGA-E2-A154-01</th><th>TCGA-E2-A155-01</th><th>TCGA-E2-A156-01</th><th>TCGA-E2-A158-01</th><th>TCGA-E2-A159-01</th><th>TCGA-E2-A15A-01</th><th>TCGA-E2-A15C-01</th><th>TCGA-E2-A15D-01</th><th>TCGA-E2-A15E-01</th><th>TCGA-E2-A15F-01</th><th>TCGA-E2-A15G-01</th><th>TCGA-E2-A15H-01</th><th>TCGA-E2-A15I-01</th><th>TCGA-E2-A15J-01</th><th>TCGA-E2-A15K-01</th><th>TCGA-E2-A15L-01</th><th>TCGA-E2-A15M-01</th><th>TCGA-E2-A15O-01</th><th>TCGA-E2-A15P-01</th><th>TCGA-E2-A15R-01</th><th>TCGA-E2-A15S-01</th><th>TCGA-E2-A15T-01</th><th>TCGA-E2-A1AZ-01</th><th>TCGA-E2-A1B0-01</th><th>TCGA-E2-A1B1-01</th><th>TCGA-E2-A1B4-01</th><th>TCGA-E2-A1B5-01</th><th>TCGA-E2-A1B6-01</th><th>TCGA-E2-A1BC-01</th><th>TCGA-E2-A1BD-01</th></tr><tr><td>str</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>&hellip;</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td></tr></thead><tbody><tr><td>&quot;ARHGEF10L&quot;</td><td>9.1657</td><td>9.7179</td><td>8.8669</td><td>9.7663</td><td>9.8632</td><td>8.8234</td><td>8.6701</td><td>9.9831</td><td>10.5102</td><td>10.064</td><td>9.5425</td><td>10.449</td><td>9.4763</td><td>10.5333</td><td>8.9774</td><td>8.9665</td><td>9.3774</td><td>9.5693</td><td>11.4132</td><td>9.6413</td><td>10.3112</td><td>9.2703</td><td>9.0876</td><td>11.2247</td><td>8.9437</td><td>9.741</td><td>8.3209</td><td>11.1395</td><td>10.1611</td><td>9.2467</td><td>10.9353</td><td>9.3572</td><td>10.0545</td><td>9.778</td><td>10.2059</td><td>10.0794</td><td>&hellip;</td><td>8.9611</td><td>10.7636</td><td>9.9899</td><td>9.8046</td><td>11.2344</td><td>9.8236</td><td>9.984</td><td>9.2405</td><td>9.8415</td><td>10.3731</td><td>9.2199</td><td>10.1219</td><td>9.1705</td><td>10.0851</td><td>9.2262</td><td>9.7065</td><td>10.0334</td><td>10.0949</td><td>9.4848</td><td>9.1406</td><td>9.2531</td><td>9.9412</td><td>9.3713</td><td>8.453</td><td>9.376</td><td>9.1228</td><td>9.1934</td><td>10.0346</td><td>9.8209</td><td>9.1036</td><td>8.5746</td><td>9.3223</td><td>10.4367</td><td>8.9027</td><td>8.6965</td><td>9.0303</td><td>10.7991</td></tr><tr><td>&quot;HIF3A&quot;</td><td>2.4935</td><td>3.3983</td><td>1.6631</td><td>6.1765</td><td>7.497</td><td>1.8177</td><td>8.9699</td><td>1.7558</td><td>6.9166</td><td>2.2799</td><td>1.4911</td><td>9.2573</td><td>6.3214</td><td>2.9105</td><td>2.5516</td><td>1.6612</td><td>3.6294</td><td>3.6849</td><td>0.0</td><td>4.8982</td><td>0.4407</td><td>2.3655</td><td>1.1726</td><td>2.9769</td><td>3.0291</td><td>3.35</td><td>1.7954</td><td>1.9355</td><td>2.1132</td><td>0.0</td><td>4.0158</td><td>1.4728</td><td>4.2849</td><td>2.5901</td><td>1.1658</td><td>1.8923</td><td>&hellip;</td><td>1.7638</td><td>4.8435</td><td>2.8783</td><td>2.9498</td><td>2.9782</td><td>2.3097</td><td>1.6892</td><td>1.9935</td><td>2.1329</td><td>0.7806</td><td>4.3358</td><td>4.1235</td><td>0.7135</td><td>0.848</td><td>3.8032</td><td>4.4089</td><td>0.8836</td><td>2.4616</td><td>1.1422</td><td>3.0117</td><td>0.4366</td><td>1.3397</td><td>1.6929</td><td>4.9786</td><td>0.8672</td><td>1.7922</td><td>0.3056</td><td>5.0409</td><td>0.3936</td><td>5.7418</td><td>1.3134</td><td>4.919</td><td>1.8654</td><td>4.8873</td><td>4.6748</td><td>6.9578</td><td>2.1772</td></tr><tr><td>&quot;RNF17&quot;</td><td>0.0</td><td>0.0</td><td>0.4449</td><td>0.0</td><td>0.7051</td><td>1.178</td><td>0.0</td><td>0.0</td><td>0.6028</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>2.0906</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>7.7299</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>&hellip;</td><td>0.0</td><td>0.8064</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.638</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.4333</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>1.0372</td><td>0.0</td><td>0.0</td><td>1.4776</td><td>0.0</td><td>0.8114</td></tr><tr><td>&quot;RNF10&quot;</td><td>11.7868</td><td>11.3219</td><td>11.8779</td><td>11.502</td><td>12.099</td><td>11.9934</td><td>11.6209</td><td>12.185</td><td>12.1741</td><td>11.7065</td><td>11.9362</td><td>11.3839</td><td>12.5415</td><td>12.22</td><td>12.0188</td><td>12.5742</td><td>12.0841</td><td>11.6492</td><td>12.0196</td><td>12.0527</td><td>12.2531</td><td>12.3055</td><td>11.1702</td><td>11.5801</td><td>12.0265</td><td>12.2589</td><td>11.8358</td><td>13.0469</td><td>11.7749</td><td>11.9473</td><td>12.1437</td><td>11.6602</td><td>12.0735</td><td>11.881</td><td>11.9862</td><td>11.965</td><td>&hellip;</td><td>12.3585</td><td>11.8525</td><td>11.7697</td><td>12.3889</td><td>11.3972</td><td>12.1153</td><td>11.7244</td><td>11.5464</td><td>11.9198</td><td>12.0953</td><td>11.8904</td><td>12.0105</td><td>12.2024</td><td>11.9198</td><td>11.9175</td><td>11.7502</td><td>12.0135</td><td>12.2843</td><td>12.3655</td><td>12.0196</td><td>11.9345</td><td>11.9704</td><td>11.88</td><td>11.7094</td><td>12.4611</td><td>12.4242</td><td>11.7385</td><td>12.4475</td><td>11.8324</td><td>11.416</td><td>11.7854</td><td>11.7005</td><td>12.428</td><td>12.6904</td><td>12.3403</td><td>11.8271</td><td>12.2783</td></tr><tr><td>&quot;RNF11&quot;</td><td>11.0131</td><td>11.2617</td><td>11.8526</td><td>11.4405</td><td>9.9933</td><td>11.9332</td><td>11.0278</td><td>11.3242</td><td>11.2632</td><td>10.7574</td><td>10.4421</td><td>10.6897</td><td>10.9603</td><td>11.2698</td><td>11.9258</td><td>11.0061</td><td>10.9864</td><td>11.0977</td><td>11.1327</td><td>11.1117</td><td>11.1687</td><td>10.7255</td><td>11.4965</td><td>11.116</td><td>11.5781</td><td>10.6494</td><td>10.838</td><td>11.5712</td><td>10.9105</td><td>12.0452</td><td>10.1117</td><td>11.7259</td><td>10.9669</td><td>11.3562</td><td>11.3253</td><td>11.5178</td><td>&hellip;</td><td>11.3929</td><td>11.1539</td><td>11.0064</td><td>10.2678</td><td>11.758</td><td>11.8669</td><td>11.1408</td><td>10.3065</td><td>11.696</td><td>11.6226</td><td>10.6569</td><td>12.1943</td><td>10.6067</td><td>11.4468</td><td>11.066</td><td>11.5324</td><td>10.837</td><td>11.112</td><td>10.7207</td><td>11.3718</td><td>10.688</td><td>10.898</td><td>11.183</td><td>11.415</td><td>10.718</td><td>10.5873</td><td>10.1383</td><td>10.8017</td><td>11.1716</td><td>11.2437</td><td>10.9101</td><td>11.6778</td><td>11.0804</td><td>11.5409</td><td>10.4812</td><td>11.8211</td><td>10.791</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>&quot;GNGT2&quot;</td><td>4.3589</td><td>3.3525</td><td>3.9492</td><td>3.118</td><td>2.0537</td><td>4.5492</td><td>3.2707</td><td>3.3381</td><td>5.3183</td><td>6.6174</td><td>3.277</td><td>4.9498</td><td>3.0485</td><td>3.6332</td><td>3.6879</td><td>5.7155</td><td>4.912</td><td>5.745</td><td>5.0972</td><td>3.4664</td><td>4.9713</td><td>4.819</td><td>4.0778</td><td>3.8444</td><td>4.042</td><td>4.3402</td><td>5.4675</td><td>3.5644</td><td>4.844</td><td>3.6188</td><td>2.9859</td><td>3.8574</td><td>5.4894</td><td>4.6249</td><td>5.7674</td><td>3.3278</td><td>&hellip;</td><td>2.1224</td><td>4.2075</td><td>3.0723</td><td>5.5044</td><td>4.1033</td><td>3.4455</td><td>4.0508</td><td>2.9925</td><td>2.9049</td><td>3.466</td><td>1.7652</td><td>4.7006</td><td>4.2169</td><td>3.4594</td><td>3.8032</td><td>2.922</td><td>3.422</td><td>3.4913</td><td>3.4139</td><td>4.3525</td><td>3.0121</td><td>4.1571</td><td>3.2163</td><td>5.2052</td><td>3.6</td><td>3.3655</td><td>3.3503</td><td>3.3508</td><td>2.9821</td><td>3.7817</td><td>4.2345</td><td>5.107</td><td>3.3946</td><td>5.8837</td><td>6.3216</td><td>3.2856</td><td>3.9395</td></tr><tr><td>&quot;TULP3&quot;</td><td>9.3914</td><td>9.7743</td><td>9.5824</td><td>10.2201</td><td>11.2224</td><td>9.4405</td><td>10.466</td><td>9.3547</td><td>8.7375</td><td>10.0738</td><td>9.4699</td><td>11.3293</td><td>8.9696</td><td>9.2296</td><td>9.5557</td><td>9.1948</td><td>9.2301</td><td>9.1724</td><td>9.4938</td><td>9.3849</td><td>9.0734</td><td>9.441</td><td>8.7935</td><td>9.5108</td><td>9.4847</td><td>9.9703</td><td>9.255</td><td>9.7329</td><td>10.542</td><td>8.8173</td><td>8.3932</td><td>9.1971</td><td>9.2226</td><td>9.4185</td><td>9.5737</td><td>9.422</td><td>&hellip;</td><td>9.3481</td><td>9.4645</td><td>9.1375</td><td>8.7028</td><td>10.3121</td><td>9.3679</td><td>9.3652</td><td>9.5315</td><td>8.2522</td><td>8.951</td><td>9.7393</td><td>9.4379</td><td>9.498</td><td>9.3876</td><td>9.4127</td><td>9.0046</td><td>9.6514</td><td>9.2273</td><td>9.6164</td><td>9.1488</td><td>8.7658</td><td>9.1029</td><td>9.2306</td><td>9.4169</td><td>9.924</td><td>8.6822</td><td>9.1865</td><td>9.7848</td><td>9.6613</td><td>11.1041</td><td>9.5507</td><td>9.318</td><td>8.6234</td><td>8.457</td><td>9.4853</td><td>8.897</td><td>9.3381</td></tr><tr><td>&quot;PTRF&quot;</td><td>12.6757</td><td>12.6287</td><td>13.9112</td><td>12.3596</td><td>10.2188</td><td>12.6397</td><td>9.8958</td><td>12.6766</td><td>12.6538</td><td>12.1301</td><td>11.8827</td><td>11.8673</td><td>13.3441</td><td>12.7181</td><td>13.1402</td><td>11.658</td><td>11.74</td><td>12.4704</td><td>12.4068</td><td>13.6079</td><td>11.7696</td><td>11.9541</td><td>11.2962</td><td>12.4225</td><td>12.5659</td><td>10.6686</td><td>12.0769</td><td>11.946</td><td>13.1791</td><td>10.6496</td><td>11.6519</td><td>12.577</td><td>13.4599</td><td>13.0931</td><td>11.6713</td><td>11.0222</td><td>&hellip;</td><td>10.8005</td><td>12.3478</td><td>11.3063</td><td>12.3172</td><td>12.2112</td><td>12.7038</td><td>13.418</td><td>12.1244</td><td>10.3702</td><td>10.6286</td><td>11.5021</td><td>11.7601</td><td>11.621</td><td>12.0128</td><td>12.5135</td><td>12.3567</td><td>11.3156</td><td>11.0672</td><td>12.0522</td><td>11.278</td><td>11.0203</td><td>11.5675</td><td>12.77</td><td>12.4503</td><td>11.0286</td><td>12.0612</td><td>11.233</td><td>10.7325</td><td>10.6287</td><td>12.1553</td><td>11.706</td><td>12.7004</td><td>11.8097</td><td>12.4708</td><td>12.493</td><td>12.9017</td><td>12.2984</td></tr><tr><td>&quot;GSTK1&quot;</td><td>10.7473</td><td>11.0453</td><td>11.3402</td><td>10.7773</td><td>8.2307</td><td>10.6721</td><td>9.1808</td><td>11.1675</td><td>11.9502</td><td>11.0695</td><td>10.9382</td><td>11.289</td><td>10.1368</td><td>11.2695</td><td>10.9499</td><td>10.7056</td><td>11.4594</td><td>10.9795</td><td>10.763</td><td>11.331</td><td>11.7788</td><td>11.9354</td><td>9.4811</td><td>11.6343</td><td>11.8265</td><td>10.7206</td><td>11.4344</td><td>10.1338</td><td>11.3601</td><td>11.3905</td><td>11.0397</td><td>11.477</td><td>11.2706</td><td>11.5253</td><td>11.499</td><td>11.6754</td><td>&hellip;</td><td>10.5982</td><td>11.836</td><td>11.5583</td><td>11.0924</td><td>11.0916</td><td>10.429</td><td>10.9581</td><td>10.8147</td><td>10.5525</td><td>10.7923</td><td>10.8892</td><td>10.9602</td><td>11.4676</td><td>10.7697</td><td>11.3549</td><td>11.4494</td><td>10.7775</td><td>11.3925</td><td>10.5795</td><td>11.3559</td><td>11.8266</td><td>10.9303</td><td>10.8696</td><td>10.4217</td><td>10.5413</td><td>11.0267</td><td>9.7612</td><td>9.6672</td><td>10.2144</td><td>10.8853</td><td>11.354</td><td>10.7588</td><td>12.5893</td><td>11.011</td><td>12.6251</td><td>9.9638</td><td>12.172</td></tr><tr><td>&quot;SELP&quot;</td><td>8.7529</td><td>7.8723</td><td>4.7356</td><td>7.6359</td><td>1.5316</td><td>5.8216</td><td>4.5001</td><td>6.4686</td><td>7.4662</td><td>8.1501</td><td>5.4746</td><td>5.0741</td><td>2.0417</td><td>6.1677</td><td>5.6013</td><td>7.7196</td><td>7.1651</td><td>8.0634</td><td>5.0376</td><td>8.6064</td><td>5.0513</td><td>5.2271</td><td>6.4859</td><td>8.4096</td><td>8.2715</td><td>6.1228</td><td>3.4978</td><td>3.2442</td><td>7.4731</td><td>6.316</td><td>4.4441</td><td>6.5606</td><td>9.322</td><td>8.4468</td><td>7.6732</td><td>6.9883</td><td>&hellip;</td><td>0.0</td><td>7.6704</td><td>2.9785</td><td>8.7931</td><td>5.8083</td><td>1.5747</td><td>5.6261</td><td>2.9925</td><td>6.2055</td><td>2.8993</td><td>5.2485</td><td>6.0209</td><td>5.2762</td><td>6.9841</td><td>8.42</td><td>7.11</td><td>8.1673</td><td>6.3673</td><td>5.2179</td><td>7.5289</td><td>3.4455</td><td>7.644</td><td>2.067</td><td>9.4108</td><td>6.3534</td><td>8.3532</td><td>4.9187</td><td>5.4257</td><td>5.9138</td><td>8.0785</td><td>7.0453</td><td>8.8142</td><td>2.1495</td><td>9.7613</td><td>7.4161</td><td>7.1316</td><td>6.9635</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (10_745, 482)\n",
       "┌───────────┬───────────┬───────────┬───────────┬───┬───────────┬───────────┬───────────┬──────────┐\n",
       "│ gene      ┆ TCGA-A1-A ┆ TCGA-A1-A ┆ TCGA-A1-A ┆ … ┆ TCGA-E2-A ┆ TCGA-E2-A ┆ TCGA-E2-A ┆ TCGA-E2- │\n",
       "│ ---       ┆ 0SD-01    ┆ 0SE-01    ┆ 0SH-01    ┆   ┆ 1B5-01    ┆ 1B6-01    ┆ 1BC-01    ┆ A1BD-01  │\n",
       "│ str       ┆ ---       ┆ ---       ┆ ---       ┆   ┆ ---       ┆ ---       ┆ ---       ┆ ---      │\n",
       "│           ┆ f64       ┆ f64       ┆ f64       ┆   ┆ f64       ┆ f64       ┆ f64       ┆ f64      │\n",
       "╞═══════════╪═══════════╪═══════════╪═══════════╪═══╪═══════════╪═══════════╪═══════════╪══════════╡\n",
       "│ ARHGEF10L ┆ 9.1657    ┆ 9.7179    ┆ 8.8669    ┆ … ┆ 8.9027    ┆ 8.6965    ┆ 9.0303    ┆ 10.7991  │\n",
       "│ HIF3A     ┆ 2.4935    ┆ 3.3983    ┆ 1.6631    ┆ … ┆ 4.8873    ┆ 4.6748    ┆ 6.9578    ┆ 2.1772   │\n",
       "│ RNF17     ┆ 0.0       ┆ 0.0       ┆ 0.4449    ┆ … ┆ 0.0       ┆ 1.4776    ┆ 0.0       ┆ 0.8114   │\n",
       "│ RNF10     ┆ 11.7868   ┆ 11.3219   ┆ 11.8779   ┆ … ┆ 12.6904   ┆ 12.3403   ┆ 11.8271   ┆ 12.2783  │\n",
       "│ RNF11     ┆ 11.0131   ┆ 11.2617   ┆ 11.8526   ┆ … ┆ 11.5409   ┆ 10.4812   ┆ 11.8211   ┆ 10.791   │\n",
       "│ …         ┆ …         ┆ …         ┆ …         ┆ … ┆ …         ┆ …         ┆ …         ┆ …        │\n",
       "│ GNGT2     ┆ 4.3589    ┆ 3.3525    ┆ 3.9492    ┆ … ┆ 5.8837    ┆ 6.3216    ┆ 3.2856    ┆ 3.9395   │\n",
       "│ TULP3     ┆ 9.3914    ┆ 9.7743    ┆ 9.5824    ┆ … ┆ 8.457     ┆ 9.4853    ┆ 8.897     ┆ 9.3381   │\n",
       "│ PTRF      ┆ 12.6757   ┆ 12.6287   ┆ 13.9112   ┆ … ┆ 12.4708   ┆ 12.493    ┆ 12.9017   ┆ 12.2984  │\n",
       "│ GSTK1     ┆ 10.7473   ┆ 11.0453   ┆ 11.3402   ┆ … ┆ 11.011    ┆ 12.6251   ┆ 9.9638    ┆ 12.172   │\n",
       "│ SELP      ┆ 8.7529    ┆ 7.8723    ┆ 4.7356    ┆ … ┆ 9.7613    ┆ 7.4161    ┆ 7.1316    ┆ 6.9635   │\n",
       "└───────────┴───────────┴───────────┴───────────┴───┴───────────┴───────────┴───────────┴──────────┘"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mrna"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "mrna = preprocess_seq_data(\n",
    "    df = mrna,\n",
    "    annotation_cols=['gene'],\n",
    "    min_median_expression=2,\n",
    "    min_detection_rate=0.4,\n",
    "    top_n=10000,\n",
    "    cpm_normalization=False,\n",
    "    log_transform=False,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (9_111, 482)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>gene</th><th>TCGA-A1-A0SD-01</th><th>TCGA-A1-A0SE-01</th><th>TCGA-A1-A0SH-01</th><th>TCGA-A1-A0SJ-01</th><th>TCGA-A1-A0SK-01</th><th>TCGA-A1-A0SM-01</th><th>TCGA-A1-A0SO-01</th><th>TCGA-A2-A04N-01</th><th>TCGA-A2-A04P-01</th><th>TCGA-A2-A04Q-01</th><th>TCGA-A2-A04R-01</th><th>TCGA-A2-A04T-01</th><th>TCGA-A2-A04U-01</th><th>TCGA-A2-A04V-01</th><th>TCGA-A2-A04W-01</th><th>TCGA-A2-A04X-01</th><th>TCGA-A2-A04Y-01</th><th>TCGA-A2-A0CL-01</th><th>TCGA-A2-A0CM-01</th><th>TCGA-A2-A0CP-01</th><th>TCGA-A2-A0CQ-01</th><th>TCGA-A2-A0CS-01</th><th>TCGA-A2-A0CT-01</th><th>TCGA-A2-A0CU-01</th><th>TCGA-A2-A0CV-01</th><th>TCGA-A2-A0CW-01</th><th>TCGA-A2-A0D0-01</th><th>TCGA-A2-A0D1-01</th><th>TCGA-A2-A0D2-01</th><th>TCGA-A2-A0D3-01</th><th>TCGA-A2-A0D4-01</th><th>TCGA-A2-A0EM-01</th><th>TCGA-A2-A0EN-01</th><th>TCGA-A2-A0EO-01</th><th>TCGA-A2-A0EQ-01</th><th>TCGA-A2-A0ER-01</th><th>&hellip;</th><th>TCGA-E2-A14W-01</th><th>TCGA-E2-A14X-01</th><th>TCGA-E2-A14Y-01</th><th>TCGA-E2-A14Z-01</th><th>TCGA-E2-A150-01</th><th>TCGA-E2-A152-01</th><th>TCGA-E2-A153-01</th><th>TCGA-E2-A154-01</th><th>TCGA-E2-A155-01</th><th>TCGA-E2-A156-01</th><th>TCGA-E2-A158-01</th><th>TCGA-E2-A159-01</th><th>TCGA-E2-A15A-01</th><th>TCGA-E2-A15C-01</th><th>TCGA-E2-A15D-01</th><th>TCGA-E2-A15E-01</th><th>TCGA-E2-A15F-01</th><th>TCGA-E2-A15G-01</th><th>TCGA-E2-A15H-01</th><th>TCGA-E2-A15I-01</th><th>TCGA-E2-A15J-01</th><th>TCGA-E2-A15K-01</th><th>TCGA-E2-A15L-01</th><th>TCGA-E2-A15M-01</th><th>TCGA-E2-A15O-01</th><th>TCGA-E2-A15P-01</th><th>TCGA-E2-A15R-01</th><th>TCGA-E2-A15S-01</th><th>TCGA-E2-A15T-01</th><th>TCGA-E2-A1AZ-01</th><th>TCGA-E2-A1B0-01</th><th>TCGA-E2-A1B1-01</th><th>TCGA-E2-A1B4-01</th><th>TCGA-E2-A1B5-01</th><th>TCGA-E2-A1B6-01</th><th>TCGA-E2-A1BC-01</th><th>TCGA-E2-A1BD-01</th></tr><tr><td>str</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>&hellip;</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td></tr></thead><tbody><tr><td>&quot;CYP2A6&quot;</td><td>0.7326</td><td>0.9907</td><td>0.0</td><td>7.0156</td><td>0.3953</td><td>0.0</td><td>0.0</td><td>0.4815</td><td>0.6028</td><td>0.4706</td><td>5.0547</td><td>0.4305</td><td>1.3554</td><td>2.6545</td><td>4.0591</td><td>0.7832</td><td>0.4164</td><td>2.9395</td><td>0.5474</td><td>1.2268</td><td>3.7932</td><td>8.0671</td><td>2.4851</td><td>2.8297</td><td>1.7191</td><td>2.8724</td><td>0.0</td><td>0.0</td><td>1.0765</td><td>0.0</td><td>1.0461</td><td>13.0051</td><td>3.158</td><td>1.6563</td><td>0.0</td><td>4.1354</td><td>&hellip;</td><td>1.5443</td><td>1.1688</td><td>0.0</td><td>11.6415</td><td>0.8555</td><td>0.0</td><td>0.0</td><td>3.0799</td><td>1.09</td><td>0.0</td><td>0.0</td><td>0.0</td><td>4.0318</td><td>1.1375</td><td>2.3794</td><td>3.771</td><td>8.3743</td><td>8.0539</td><td>11.3892</td><td>4.3017</td><td>4.3529</td><td>4.1877</td><td>0.0</td><td>14.5701</td><td>2.3563</td><td>8.4825</td><td>1.6428</td><td>2.9821</td><td>3.0383</td><td>0.0</td><td>1.5143</td><td>2.9513</td><td>0.6123</td><td>15.9084</td><td>0.0</td><td>2.3155</td><td>1.5912</td></tr><tr><td>&quot;CLEC3A&quot;</td><td>5.6235</td><td>4.0415</td><td>0.0</td><td>3.7481</td><td>1.5316</td><td>4.831</td><td>1.2338</td><td>0.0</td><td>0.6028</td><td>5.6543</td><td>0.0</td><td>2.3709</td><td>0.0</td><td>0.0</td><td>12.4971</td><td>0.0</td><td>0.0</td><td>12.0808</td><td>0.5474</td><td>10.22</td><td>0.0</td><td>0.0</td><td>5.2076</td><td>0.0</td><td>4.3765</td><td>5.2724</td><td>0.0</td><td>0.0</td><td>1.686</td><td>4.7264</td><td>5.3924</td><td>13.0392</td><td>0.4116</td><td>3.7606</td><td>7.2546</td><td>0.0</td><td>&hellip;</td><td>3.7504</td><td>0.0</td><td>1.3719</td><td>10.6166</td><td>0.8555</td><td>11.2396</td><td>11.1355</td><td>12.1301</td><td>0.0</td><td>10.5382</td><td>0.0</td><td>1.4781</td><td>0.4004</td><td>1.3785</td><td>2.2788</td><td>9.4778</td><td>0.8836</td><td>15.5583</td><td>0.0</td><td>6.9493</td><td>0.0</td><td>13.9854</td><td>1.1865</td><td>14.4175</td><td>0.0</td><td>11.8291</td><td>1.6428</td><td>1.1045</td><td>0.3936</td><td>2.6442</td><td>0.0</td><td>8.1001</td><td>11.5015</td><td>14.25</td><td>0.0</td><td>0.0</td><td>0.5881</td></tr><tr><td>&quot;FABP7&quot;</td><td>9.6318</td><td>4.5672</td><td>4.9863</td><td>3.7869</td><td>2.6685</td><td>1.178</td><td>5.7254</td><td>0.4815</td><td>7.9864</td><td>11.4959</td><td>0.7861</td><td>9.7849</td><td>10.1427</td><td>0.8598</td><td>0.0</td><td>2.3118</td><td>3.9118</td><td>4.3149</td><td>4.5895</td><td>5.7889</td><td>1.051</td><td>0.0</td><td>1.6277</td><td>0.4665</td><td>5.8585</td><td>1.2851</td><td>10.7898</td><td>11.2974</td><td>7.9493</td><td>4.2025</td><td>2.3951</td><td>0.0</td><td>1.7276</td><td>4.1585</td><td>0.0</td><td>3.7394</td><td>&hellip;</td><td>0.0</td><td>9.4015</td><td>0.0</td><td>3.4417</td><td>5.0073</td><td>0.0</td><td>0.0</td><td>1.3168</td><td>1.09</td><td>0.0</td><td>8.3559</td><td>6.7183</td><td>0.7135</td><td>3.2928</td><td>4.0671</td><td>7.0941</td><td>2.7898</td><td>0.0</td><td>4.4278</td><td>4.5174</td><td>0.0</td><td>0.0</td><td>0.3994</td><td>0.6949</td><td>3.6482</td><td>4.3138</td><td>0.9587</td><td>0.6554</td><td>1.6762</td><td>2.4597</td><td>1.08</td><td>5.1418</td><td>2.5904</td><td>5.8941</td><td>10.0022</td><td>0.0</td><td>4.5789</td></tr><tr><td>&quot;DSG1&quot;</td><td>3.3121</td><td>5.4522</td><td>4.2247</td><td>2.2197</td><td>3.2493</td><td>2.4381</td><td>0.3453</td><td>2.1909</td><td>4.8584</td><td>5.0107</td><td>3.4303</td><td>12.757</td><td>2.0417</td><td>0.493</td><td>0.0</td><td>0.0</td><td>1.7407</td><td>0.8002</td><td>4.176</td><td>7.0039</td><td>1.051</td><td>0.0</td><td>3.7141</td><td>1.7185</td><td>3.8258</td><td>0.0</td><td>0.0</td><td>1.2706</td><td>1.686</td><td>0.4032</td><td>1.0461</td><td>0.7742</td><td>3.4029</td><td>4.516</td><td>0.0</td><td>1.4896</td><td>&hellip;</td><td>0.5648</td><td>8.3924</td><td>8.2062</td><td>2.9498</td><td>7.3831</td><td>1.9147</td><td>0.0</td><td>0.0</td><td>0.3586</td><td>0.0</td><td>8.4437</td><td>3.9001</td><td>1.3783</td><td>2.433</td><td>0.0</td><td>4.9711</td><td>1.8218</td><td>0.0</td><td>0.8518</td><td>3.1898</td><td>0.0</td><td>0.0</td><td>3.1659</td><td>1.7088</td><td>2.468</td><td>3.9801</td><td>3.0136</td><td>0.3647</td><td>0.0</td><td>4.5081</td><td>1.3134</td><td>6.4787</td><td>3.4654</td><td>4.908</td><td>8.1218</td><td>0.8444</td><td>6.2333</td></tr><tr><td>&quot;GRIA2&quot;</td><td>6.0696</td><td>1.5726</td><td>2.5984</td><td>0.4497</td><td>4.0937</td><td>1.9414</td><td>10.284</td><td>4.7628</td><td>2.6292</td><td>1.109</td><td>7.7487</td><td>6.6353</td><td>3.4425</td><td>0.0</td><td>0.0</td><td>0.0</td><td>1.2254</td><td>0.8002</td><td>0.0</td><td>2.5089</td><td>3.7555</td><td>8.6277</td><td>5.287</td><td>7.3066</td><td>0.3635</td><td>1.4459</td><td>0.8669</td><td>0.0</td><td>0.4538</td><td>0.4032</td><td>1.8728</td><td>0.4384</td><td>4.5519</td><td>1.0538</td><td>0.8711</td><td>6.5748</td><td>&hellip;</td><td>1.9542</td><td>1.3207</td><td>0.613</td><td>3.7609</td><td>1.7775</td><td>0.4811</td><td>1.0788</td><td>3.3142</td><td>0.8848</td><td>8.5144</td><td>8.1203</td><td>0.4406</td><td>1.1888</td><td>0.848</td><td>0.4333</td><td>4.0163</td><td>2.131</td><td>9.8821</td><td>0.0</td><td>8.6982</td><td>1.0427</td><td>7.3546</td><td>9.1116</td><td>6.0303</td><td>1.161</td><td>10.3031</td><td>9.6381</td><td>2.9821</td><td>0.3936</td><td>3.615</td><td>0.0</td><td>4.868</td><td>10.1273</td><td>2.8022</td><td>5.7456</td><td>0.8444</td><td>0.3238</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>&quot;ZNF207&quot;</td><td>11.6064</td><td>11.4692</td><td>11.3071</td><td>11.9207</td><td>12.0643</td><td>11.1946</td><td>11.8784</td><td>11.5263</td><td>11.4543</td><td>11.364</td><td>11.7128</td><td>11.6849</td><td>11.7023</td><td>11.8897</td><td>10.7094</td><td>11.3548</td><td>12.0554</td><td>11.5925</td><td>11.7237</td><td>11.5746</td><td>11.6509</td><td>11.5546</td><td>12.0832</td><td>11.831</td><td>11.1209</td><td>11.8231</td><td>11.7338</td><td>11.1374</td><td>11.2092</td><td>11.784</td><td>11.9979</td><td>11.5529</td><td>11.8414</td><td>11.4836</td><td>12.1446</td><td>11.5869</td><td>&hellip;</td><td>12.1376</td><td>11.6169</td><td>12.7391</td><td>11.9682</td><td>11.4712</td><td>11.2298</td><td>11.5042</td><td>11.7005</td><td>11.6206</td><td>11.4812</td><td>11.3905</td><td>11.6064</td><td>11.6779</td><td>11.3914</td><td>11.3505</td><td>11.458</td><td>11.7035</td><td>11.4262</td><td>11.4201</td><td>11.7156</td><td>11.7965</td><td>11.4217</td><td>11.545</td><td>11.6342</td><td>11.502</td><td>11.8724</td><td>11.1577</td><td>11.8729</td><td>11.6893</td><td>11.6281</td><td>11.9752</td><td>11.9964</td><td>11.3448</td><td>11.7366</td><td>11.2059</td><td>11.2276</td><td>11.594</td></tr><tr><td>&quot;PTBP1&quot;</td><td>12.2297</td><td>12.4608</td><td>12.1939</td><td>12.7051</td><td>12.5171</td><td>12.3927</td><td>12.7373</td><td>12.6637</td><td>12.6647</td><td>12.2667</td><td>12.7545</td><td>12.2645</td><td>12.9476</td><td>12.5875</td><td>12.5195</td><td>12.2069</td><td>12.5743</td><td>12.4805</td><td>12.5142</td><td>12.5582</td><td>12.3687</td><td>12.5594</td><td>12.0842</td><td>12.3534</td><td>12.0963</td><td>12.6244</td><td>12.8723</td><td>12.7914</td><td>12.6127</td><td>12.4117</td><td>12.4826</td><td>12.2437</td><td>12.4432</td><td>12.0887</td><td>13.0451</td><td>12.0082</td><td>&hellip;</td><td>12.8021</td><td>12.6171</td><td>12.6542</td><td>12.892</td><td>11.983</td><td>11.9728</td><td>12.409</td><td>12.3426</td><td>12.3697</td><td>12.119</td><td>12.2094</td><td>12.0684</td><td>12.2934</td><td>12.3711</td><td>12.0546</td><td>12.023</td><td>12.3827</td><td>12.4485</td><td>12.3581</td><td>12.0038</td><td>12.5821</td><td>12.8112</td><td>12.2749</td><td>11.7045</td><td>12.7152</td><td>11.9377</td><td>12.2303</td><td>12.3328</td><td>12.4221</td><td>11.6894</td><td>12.3228</td><td>11.4483</td><td>12.5468</td><td>12.0973</td><td>12.2912</td><td>11.9636</td><td>13.0645</td></tr><tr><td>&quot;SF1&quot;</td><td>11.6677</td><td>11.6738</td><td>11.8627</td><td>11.6928</td><td>13.0678</td><td>11.5173</td><td>11.9364</td><td>12.0804</td><td>11.7973</td><td>11.8734</td><td>12.0784</td><td>12.3984</td><td>12.0479</td><td>11.9921</td><td>11.4675</td><td>11.7178</td><td>12.1193</td><td>11.8597</td><td>11.8097</td><td>12.2791</td><td>12.11</td><td>11.9405</td><td>12.0099</td><td>12.3068</td><td>11.9232</td><td>12.0018</td><td>12.2454</td><td>12.2509</td><td>12.537</td><td>12.2236</td><td>11.8484</td><td>11.8827</td><td>12.3006</td><td>11.9526</td><td>12.2296</td><td>12.0491</td><td>&hellip;</td><td>12.1007</td><td>12.2322</td><td>12.2935</td><td>12.0252</td><td>11.7965</td><td>11.6499</td><td>11.867</td><td>12.3606</td><td>12.0732</td><td>12.1711</td><td>12.07</td><td>11.9611</td><td>12.1465</td><td>12.1039</td><td>12.023</td><td>11.9936</td><td>12.3262</td><td>12.2015</td><td>12.4888</td><td>12.2174</td><td>12.262</td><td>11.969</td><td>11.752</td><td>11.6183</td><td>11.8356</td><td>12.0469</td><td>11.6709</td><td>12.233</td><td>11.9269</td><td>11.9432</td><td>11.915</td><td>11.8969</td><td>12.1512</td><td>11.8412</td><td>12.1892</td><td>11.4552</td><td>11.7151</td></tr><tr><td>&quot;KHDRBS1&quot;</td><td>11.7743</td><td>12.1109</td><td>11.7246</td><td>12.1517</td><td>13.1415</td><td>12.2916</td><td>12.4559</td><td>12.0035</td><td>12.2328</td><td>12.0409</td><td>11.7603</td><td>12.0918</td><td>12.7866</td><td>12.4325</td><td>11.7414</td><td>12.0117</td><td>12.0812</td><td>11.9133</td><td>12.1196</td><td>12.1183</td><td>11.9507</td><td>11.9184</td><td>11.9565</td><td>12.0129</td><td>12.1739</td><td>12.1135</td><td>12.5306</td><td>12.5181</td><td>12.145</td><td>12.2239</td><td>12.2403</td><td>12.0142</td><td>11.6874</td><td>12.0787</td><td>12.4206</td><td>12.0526</td><td>&hellip;</td><td>12.1835</td><td>12.2079</td><td>12.2501</td><td>12.019</td><td>11.6209</td><td>11.7945</td><td>11.9039</td><td>11.9007</td><td>12.5443</td><td>11.9424</td><td>12.4371</td><td>11.9566</td><td>12.2874</td><td>11.9216</td><td>11.8369</td><td>11.9594</td><td>12.0699</td><td>12.1084</td><td>11.887</td><td>12.0826</td><td>11.9247</td><td>12.0408</td><td>11.9188</td><td>12.3552</td><td>12.3378</td><td>11.8599</td><td>11.7072</td><td>12.2047</td><td>12.3424</td><td>12.4175</td><td>11.9229</td><td>12.2999</td><td>11.9464</td><td>12.2477</td><td>12.2344</td><td>12.0866</td><td>11.9909</td></tr><tr><td>&quot;TARDBP&quot;</td><td>11.6166</td><td>11.7528</td><td>11.4385</td><td>11.7364</td><td>12.5311</td><td>11.6752</td><td>11.9659</td><td>11.8266</td><td>12.1269</td><td>12.1657</td><td>11.9128</td><td>12.3512</td><td>11.9041</td><td>12.1227</td><td>11.6403</td><td>12.1453</td><td>12.306</td><td>11.7489</td><td>12.2475</td><td>11.8549</td><td>12.0947</td><td>11.7277</td><td>11.8859</td><td>11.9613</td><td>11.5899</td><td>11.8579</td><td>12.4486</td><td>12.1968</td><td>12.1249</td><td>11.8786</td><td>12.1275</td><td>11.7687</td><td>11.5539</td><td>11.7591</td><td>12.2167</td><td>11.8033</td><td>&hellip;</td><td>11.9305</td><td>12.0358</td><td>12.2326</td><td>11.9873</td><td>11.6746</td><td>11.6015</td><td>11.5496</td><td>11.6291</td><td>12.0285</td><td>11.7849</td><td>11.6865</td><td>11.9373</td><td>11.8161</td><td>11.7319</td><td>11.7231</td><td>11.7332</td><td>11.8353</td><td>11.7399</td><td>11.8378</td><td>11.8885</td><td>11.751</td><td>11.9265</td><td>11.8521</td><td>12.0469</td><td>11.5618</td><td>11.819</td><td>11.7391</td><td>11.9024</td><td>11.8127</td><td>12.0934</td><td>11.9501</td><td>12.0976</td><td>11.6261</td><td>11.781</td><td>12.0107</td><td>11.871</td><td>11.7559</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (9_111, 482)\n",
       "┌─────────┬────────────┬───────────┬───────────┬───┬───────────┬───────────┬───────────┬───────────┐\n",
       "│ gene    ┆ TCGA-A1-A0 ┆ TCGA-A1-A ┆ TCGA-A1-A ┆ … ┆ TCGA-E2-A ┆ TCGA-E2-A ┆ TCGA-E2-A ┆ TCGA-E2-A │\n",
       "│ ---     ┆ SD-01      ┆ 0SE-01    ┆ 0SH-01    ┆   ┆ 1B5-01    ┆ 1B6-01    ┆ 1BC-01    ┆ 1BD-01    │\n",
       "│ str     ┆ ---        ┆ ---       ┆ ---       ┆   ┆ ---       ┆ ---       ┆ ---       ┆ ---       │\n",
       "│         ┆ f64        ┆ f64       ┆ f64       ┆   ┆ f64       ┆ f64       ┆ f64       ┆ f64       │\n",
       "╞═════════╪════════════╪═══════════╪═══════════╪═══╪═══════════╪═══════════╪═══════════╪═══════════╡\n",
       "│ CYP2A6  ┆ 0.7326     ┆ 0.9907    ┆ 0.0       ┆ … ┆ 15.9084   ┆ 0.0       ┆ 2.3155    ┆ 1.5912    │\n",
       "│ CLEC3A  ┆ 5.6235     ┆ 4.0415    ┆ 0.0       ┆ … ┆ 14.25     ┆ 0.0       ┆ 0.0       ┆ 0.5881    │\n",
       "│ FABP7   ┆ 9.6318     ┆ 4.5672    ┆ 4.9863    ┆ … ┆ 5.8941    ┆ 10.0022   ┆ 0.0       ┆ 4.5789    │\n",
       "│ DSG1    ┆ 3.3121     ┆ 5.4522    ┆ 4.2247    ┆ … ┆ 4.908     ┆ 8.1218    ┆ 0.8444    ┆ 6.2333    │\n",
       "│ GRIA2   ┆ 6.0696     ┆ 1.5726    ┆ 2.5984    ┆ … ┆ 2.8022    ┆ 5.7456    ┆ 0.8444    ┆ 0.3238    │\n",
       "│ …       ┆ …          ┆ …         ┆ …         ┆ … ┆ …         ┆ …         ┆ …         ┆ …         │\n",
       "│ ZNF207  ┆ 11.6064    ┆ 11.4692   ┆ 11.3071   ┆ … ┆ 11.7366   ┆ 11.2059   ┆ 11.2276   ┆ 11.594    │\n",
       "│ PTBP1   ┆ 12.2297    ┆ 12.4608   ┆ 12.1939   ┆ … ┆ 12.0973   ┆ 12.2912   ┆ 11.9636   ┆ 13.0645   │\n",
       "│ SF1     ┆ 11.6677    ┆ 11.6738   ┆ 11.8627   ┆ … ┆ 11.8412   ┆ 12.1892   ┆ 11.4552   ┆ 11.7151   │\n",
       "│ KHDRBS1 ┆ 11.7743    ┆ 12.1109   ┆ 11.7246   ┆ … ┆ 12.2477   ┆ 12.2344   ┆ 12.0866   ┆ 11.9909   │\n",
       "│ TARDBP  ┆ 11.6166    ┆ 11.7528   ┆ 11.4385   ┆ … ┆ 11.781    ┆ 12.0107   ┆ 11.871    ┆ 11.7559   │\n",
       "└─────────┴────────────┴───────────┴───────────┴───┴───────────┴───────────┴───────────┴───────────┘"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mrna"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Only 481 samples out of 481 found in y_df\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2000/2000 [15:40<00:00,  2.13it/s]old/s]\n",
      "100%|██████████| 2000/2000 [15:38<00:00,  2.13it/s]55, 943.99s/fold]\n",
      "100%|██████████| 2000/2000 [15:44<00:00,  2.12it/s], 942.13s/fold]  \n",
      "100%|██████████| 2000/2000 [15:45<00:00,  2.11it/s], 944.09s/fold]\n",
      "100%|██████████| 2000/2000 [15:40<00:00,  2.13it/s]45, 945.61s/fold]\n",
      "Processing folds: 100%|██████████| 5/5 [1:18:42<00:00, 944.43s/fold]\n"
     ]
    }
   ],
   "source": [
    "y = annotations.select(\n",
    "    pl.col(\"sampleID\").alias(\"sample_ids\"),\n",
    "    pl.col(\"classes\").alias(\"class\"),\n",
    ")\n",
    "output_dir = \"tcga_brca_data/splits\"\n",
    "OmicDataSplitter(\n",
    "    df=mrna,\n",
    "    annotation_cols=['gene'],\n",
    "    y_df=y,\n",
    "    n_features=2000,\n",
    "    n_splits=5,\n",
    "    random_state=3,\n",
    "    output_dir=output_dir + \"/mrna_genes\",\n",
    ").process_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (1_881, 482)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>miRNA_ID</th><th>TCGA-A1-A0SD-01</th><th>TCGA-A1-A0SE-01</th><th>TCGA-A1-A0SH-01</th><th>TCGA-A1-A0SJ-01</th><th>TCGA-A1-A0SK-01</th><th>TCGA-A1-A0SM-01</th><th>TCGA-A1-A0SO-01</th><th>TCGA-A2-A04N-01</th><th>TCGA-A2-A04P-01</th><th>TCGA-A2-A04Q-01</th><th>TCGA-A2-A04R-01</th><th>TCGA-A2-A04T-01</th><th>TCGA-A2-A04U-01</th><th>TCGA-A2-A04V-01</th><th>TCGA-A2-A04W-01</th><th>TCGA-A2-A04X-01</th><th>TCGA-A2-A04Y-01</th><th>TCGA-A2-A0CL-01</th><th>TCGA-A2-A0CM-01</th><th>TCGA-A2-A0CP-01</th><th>TCGA-A2-A0CQ-01</th><th>TCGA-A2-A0CS-01</th><th>TCGA-A2-A0CT-01</th><th>TCGA-A2-A0CU-01</th><th>TCGA-A2-A0CV-01</th><th>TCGA-A2-A0CW-01</th><th>TCGA-A2-A0D0-01</th><th>TCGA-A2-A0D1-01</th><th>TCGA-A2-A0D2-01</th><th>TCGA-A2-A0D3-01</th><th>TCGA-A2-A0D4-01</th><th>TCGA-A2-A0EM-01</th><th>TCGA-A2-A0EN-01</th><th>TCGA-A2-A0EO-01</th><th>TCGA-A2-A0EQ-01</th><th>TCGA-A2-A0ER-01</th><th>&hellip;</th><th>TCGA-E2-A14W-01</th><th>TCGA-E2-A14X-01</th><th>TCGA-E2-A14Y-01</th><th>TCGA-E2-A14Z-01</th><th>TCGA-E2-A150-01</th><th>TCGA-E2-A152-01</th><th>TCGA-E2-A153-01</th><th>TCGA-E2-A154-01</th><th>TCGA-E2-A155-01</th><th>TCGA-E2-A156-01</th><th>TCGA-E2-A158-01</th><th>TCGA-E2-A159-01</th><th>TCGA-E2-A15A-01</th><th>TCGA-E2-A15C-01</th><th>TCGA-E2-A15D-01</th><th>TCGA-E2-A15E-01</th><th>TCGA-E2-A15F-01</th><th>TCGA-E2-A15G-01</th><th>TCGA-E2-A15H-01</th><th>TCGA-E2-A15I-01</th><th>TCGA-E2-A15J-01</th><th>TCGA-E2-A15K-01</th><th>TCGA-E2-A15L-01</th><th>TCGA-E2-A15M-01</th><th>TCGA-E2-A15O-01</th><th>TCGA-E2-A15P-01</th><th>TCGA-E2-A15R-01</th><th>TCGA-E2-A15S-01</th><th>TCGA-E2-A15T-01</th><th>TCGA-E2-A1AZ-01</th><th>TCGA-E2-A1B0-01</th><th>TCGA-E2-A1B1-01</th><th>TCGA-E2-A1B4-01</th><th>TCGA-E2-A1B5-01</th><th>TCGA-E2-A1B6-01</th><th>TCGA-E2-A1BC-01</th><th>TCGA-E2-A1BD-01</th></tr><tr><td>str</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>&hellip;</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td></tr></thead><tbody><tr><td>&quot;hsa-let-7a-1&quot;</td><td>14.03039</td><td>13.034042</td><td>12.962609</td><td>13.091325</td><td>12.207688</td><td>13.126299</td><td>11.735671</td><td>11.72943</td><td>13.141287</td><td>12.213246</td><td>12.856138</td><td>13.33289</td><td>13.249776</td><td>13.107813</td><td>12.327013</td><td>12.668052</td><td>13.00873</td><td>13.023521</td><td>12.538091</td><td>12.974924</td><td>13.591946</td><td>13.682025</td><td>12.963904</td><td>12.982559</td><td>13.780127</td><td>13.002531</td><td>12.959375</td><td>12.290943</td><td>12.548272</td><td>14.355641</td><td>13.386397</td><td>13.517807</td><td>12.563069</td><td>13.073552</td><td>13.055749</td><td>14.097343</td><td>&hellip;</td><td>12.933047</td><td>13.412834</td><td>13.295752</td><td>13.380763</td><td>12.924364</td><td>13.338225</td><td>13.589713</td><td>13.01339</td><td>13.934445</td><td>14.307726</td><td>13.260388</td><td>11.847683</td><td>13.721022</td><td>13.791229</td><td>13.285355</td><td>13.630036</td><td>13.679882</td><td>13.258044</td><td>12.769781</td><td>14.056884</td><td>14.205192</td><td>13.438621</td><td>13.580108</td><td>13.102478</td><td>13.71765</td><td>13.701193</td><td>13.498005</td><td>14.709119</td><td>14.608977</td><td>12.666225</td><td>13.206409</td><td>13.563254</td><td>15.086711</td><td>13.662556</td><td>13.701712</td><td>13.904604</td><td>14.026616</td></tr><tr><td>&quot;hsa-let-7a-2&quot;</td><td>14.018248</td><td>13.017645</td><td>12.971607</td><td>13.071205</td><td>12.2287</td><td>13.131248</td><td>11.72821</td><td>11.711106</td><td>13.121908</td><td>12.222766</td><td>12.859981</td><td>13.320862</td><td>13.247731</td><td>13.109839</td><td>12.327817</td><td>12.6326</td><td>13.010961</td><td>12.999008</td><td>12.517011</td><td>13.013343</td><td>13.569417</td><td>13.681565</td><td>12.969242</td><td>12.947125</td><td>13.785213</td><td>13.000689</td><td>12.943465</td><td>12.299641</td><td>12.553494</td><td>14.356464</td><td>13.367273</td><td>13.510865</td><td>12.550192</td><td>13.071606</td><td>13.02276</td><td>14.067662</td><td>&hellip;</td><td>12.921891</td><td>13.409076</td><td>13.305842</td><td>13.371711</td><td>12.934441</td><td>13.332303</td><td>13.57188</td><td>13.028045</td><td>13.934515</td><td>14.304563</td><td>13.267611</td><td>11.847515</td><td>13.697301</td><td>13.778067</td><td>13.2757</td><td>13.633115</td><td>13.661747</td><td>13.222523</td><td>12.760095</td><td>14.052668</td><td>14.199054</td><td>13.437686</td><td>13.566287</td><td>13.095546</td><td>13.7218</td><td>13.701746</td><td>13.484103</td><td>14.702159</td><td>14.60681</td><td>12.680116</td><td>13.205629</td><td>13.548641</td><td>15.097745</td><td>13.660415</td><td>13.709611</td><td>13.897027</td><td>14.033012</td></tr><tr><td>&quot;hsa-let-7a-3&quot;</td><td>14.024003</td><td>13.036697</td><td>12.983236</td><td>13.101082</td><td>12.232114</td><td>13.176328</td><td>11.769435</td><td>11.75483</td><td>13.1455</td><td>12.273188</td><td>12.846426</td><td>13.383369</td><td>13.245801</td><td>13.136332</td><td>12.357711</td><td>12.630557</td><td>13.020616</td><td>13.045014</td><td>12.534349</td><td>13.009295</td><td>13.631521</td><td>13.677169</td><td>12.984002</td><td>12.992206</td><td>13.784367</td><td>13.006994</td><td>12.954434</td><td>12.33224</td><td>12.553059</td><td>14.362705</td><td>13.396629</td><td>13.519213</td><td>12.582631</td><td>13.101982</td><td>13.083498</td><td>14.098052</td><td>&hellip;</td><td>12.956874</td><td>13.417069</td><td>13.321482</td><td>13.410418</td><td>12.94685</td><td>13.340087</td><td>13.593498</td><td>13.023757</td><td>13.950362</td><td>14.317231</td><td>13.268565</td><td>11.921041</td><td>13.745192</td><td>13.793379</td><td>13.304198</td><td>13.650106</td><td>13.685244</td><td>13.219345</td><td>12.758886</td><td>14.066863</td><td>14.209779</td><td>13.466337</td><td>13.58083</td><td>13.100015</td><td>13.726457</td><td>13.70946</td><td>13.506195</td><td>14.716166</td><td>14.622777</td><td>12.703487</td><td>13.214104</td><td>13.572047</td><td>15.093364</td><td>13.645393</td><td>13.720637</td><td>13.917199</td><td>14.030702</td></tr><tr><td>&quot;hsa-let-7b&quot;</td><td>15.632852</td><td>15.423974</td><td>15.481879</td><td>15.589285</td><td>14.494925</td><td>16.019204</td><td>12.786965</td><td>13.28372</td><td>14.062908</td><td>13.713737</td><td>14.890818</td><td>14.077445</td><td>13.991182</td><td>14.544838</td><td>13.679741</td><td>14.591142</td><td>15.459013</td><td>13.86464</td><td>13.418417</td><td>14.286339</td><td>16.013266</td><td>15.444711</td><td>14.981333</td><td>14.237693</td><td>15.258472</td><td>13.144742</td><td>15.787212</td><td>13.711375</td><td>13.470114</td><td>15.768565</td><td>14.470983</td><td>15.224578</td><td>14.033874</td><td>14.969877</td><td>14.450751</td><td>15.388345</td><td>&hellip;</td><td>13.600547</td><td>14.456838</td><td>14.39629</td><td>13.569699</td><td>13.42948</td><td>14.246661</td><td>15.184422</td><td>13.698281</td><td>14.871314</td><td>15.782323</td><td>13.565189</td><td>13.61082</td><td>14.327279</td><td>15.245153</td><td>15.533047</td><td>14.339155</td><td>13.908983</td><td>15.407072</td><td>13.776725</td><td>14.69652</td><td>15.48782</td><td>14.316291</td><td>15.64905</td><td>13.846049</td><td>14.371569</td><td>15.015832</td><td>14.652597</td><td>15.347964</td><td>16.638614</td><td>13.494483</td><td>13.912976</td><td>14.373651</td><td>15.608129</td><td>13.884577</td><td>14.582151</td><td>14.850094</td><td>15.234393</td></tr><tr><td>&quot;hsa-let-7c&quot;</td><td>11.397605</td><td>12.028745</td><td>12.813933</td><td>11.400662</td><td>9.579606</td><td>10.793825</td><td>11.579349</td><td>9.882908</td><td>12.686604</td><td>11.425357</td><td>10.469417</td><td>11.06647</td><td>9.836266</td><td>9.059985</td><td>11.910488</td><td>9.49292</td><td>11.27446</td><td>10.911595</td><td>10.090353</td><td>11.989837</td><td>10.099255</td><td>11.006428</td><td>11.717469</td><td>10.845684</td><td>12.271354</td><td>10.487615</td><td>10.2519</td><td>10.497376</td><td>11.910492</td><td>10.750879</td><td>11.231752</td><td>11.360646</td><td>11.15626</td><td>11.632472</td><td>10.735934</td><td>10.888291</td><td>&hellip;</td><td>10.996854</td><td>11.385978</td><td>10.536768</td><td>10.704577</td><td>11.866757</td><td>11.050312</td><td>11.49002</td><td>11.448748</td><td>8.835315</td><td>8.589226</td><td>11.195719</td><td>10.760113</td><td>9.65478</td><td>10.978652</td><td>11.842709</td><td>11.702283</td><td>11.904006</td><td>10.507735</td><td>10.719633</td><td>10.912193</td><td>10.068174</td><td>10.095917</td><td>10.284644</td><td>11.006828</td><td>8.858466</td><td>11.848985</td><td>10.709519</td><td>10.493422</td><td>9.85582</td><td>12.313111</td><td>11.048425</td><td>12.231829</td><td>12.04651</td><td>11.43789</td><td>12.117159</td><td>11.827816</td><td>12.028588</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>&quot;hsa-mir-9500&quot;</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>&hellip;</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td><td>0.0</td></tr><tr><td>&quot;hsa-mir-96&quot;</td><td>4.392142</td><td>5.051083</td><td>3.688321</td><td>5.18143</td><td>6.913881</td><td>5.187847</td><td>6.157048</td><td>2.526444</td><td>6.533742</td><td>4.966971</td><td>5.577324</td><td>4.686394</td><td>4.951953</td><td>5.508314</td><td>6.031446</td><td>3.521336</td><td>4.85597</td><td>4.392065</td><td>5.09264</td><td>4.557138</td><td>5.99532</td><td>5.167516</td><td>1.455335</td><td>4.643448</td><td>4.408819</td><td>5.354894</td><td>5.347107</td><td>7.220928</td><td>4.767787</td><td>4.833912</td><td>5.261017</td><td>5.936909</td><td>4.537918</td><td>4.137711</td><td>7.002225</td><td>5.987481</td><td>&hellip;</td><td>5.294216</td><td>4.279794</td><td>6.058557</td><td>2.399542</td><td>4.901448</td><td>6.649213</td><td>4.517084</td><td>5.046006</td><td>5.604424</td><td>6.263446</td><td>7.053359</td><td>6.324395</td><td>5.375675</td><td>5.362537</td><td>4.882566</td><td>4.170264</td><td>4.508994</td><td>6.495234</td><td>5.352903</td><td>6.319429</td><td>5.824971</td><td>5.25143</td><td>4.347182</td><td>4.601561</td><td>5.631206</td><td>3.863491</td><td>7.013319</td><td>6.485111</td><td>4.59337</td><td>4.759042</td><td>4.70149</td><td>3.808518</td><td>5.595697</td><td>5.394588</td><td>4.969625</td><td>4.122096</td><td>6.174561</td></tr><tr><td>&quot;hsa-mir-98&quot;</td><td>5.675821</td><td>5.389651</td><td>5.399958</td><td>6.565237</td><td>6.37406</td><td>4.477029</td><td>7.264101</td><td>3.094073</td><td>6.714751</td><td>7.218057</td><td>6.397541</td><td>7.853919</td><td>7.494391</td><td>5.865114</td><td>5.557322</td><td>4.951836</td><td>6.519574</td><td>6.328413</td><td>7.052227</td><td>5.639234</td><td>5.182729</td><td>5.385771</td><td>6.311051</td><td>5.772635</td><td>5.504413</td><td>6.80159</td><td>7.225037</td><td>6.542862</td><td>6.739779</td><td>6.112746</td><td>6.90567</td><td>5.389135</td><td>5.279883</td><td>5.142082</td><td>6.122811</td><td>6.362625</td><td>&hellip;</td><td>6.192767</td><td>6.319332</td><td>6.518545</td><td>6.884746</td><td>7.778881</td><td>6.516983</td><td>5.570797</td><td>6.204103</td><td>6.944905</td><td>5.782378</td><td>8.011937</td><td>6.13432</td><td>6.784284</td><td>5.603334</td><td>5.395868</td><td>6.147842</td><td>6.070288</td><td>6.034549</td><td>5.947812</td><td>5.771887</td><td>6.208502</td><td>7.305855</td><td>5.379956</td><td>6.208221</td><td>6.237177</td><td>5.380815</td><td>5.89111</td><td>6.361044</td><td>6.082383</td><td>5.951633</td><td>6.209735</td><td>5.135643</td><td>6.019908</td><td>6.586884</td><td>6.460651</td><td>5.858322</td><td>5.453821</td></tr><tr><td>&quot;hsa-mir-99a&quot;</td><td>10.137212</td><td>10.460951</td><td>10.740593</td><td>9.801765</td><td>7.74077</td><td>8.806292</td><td>9.350125</td><td>9.611545</td><td>9.421005</td><td>8.819037</td><td>8.596035</td><td>7.82393</td><td>8.2332</td><td>5.994597</td><td>10.438461</td><td>6.943025</td><td>9.229815</td><td>9.415873</td><td>8.063494</td><td>9.714326</td><td>7.843009</td><td>9.492261</td><td>8.931435</td><td>8.762002</td><td>10.900338</td><td>9.159716</td><td>7.743276</td><td>8.075346</td><td>9.62421</td><td>9.396281</td><td>8.76632</td><td>8.791372</td><td>9.545945</td><td>9.464765</td><td>8.410212</td><td>8.839354</td><td>&hellip;</td><td>9.594988</td><td>10.311109</td><td>7.973238</td><td>8.984319</td><td>9.517716</td><td>8.95738</td><td>9.445306</td><td>10.1451</td><td>6.842346</td><td>6.340045</td><td>9.400984</td><td>9.20302</td><td>8.32527</td><td>9.15868</td><td>10.489001</td><td>10.079103</td><td>10.154372</td><td>9.034247</td><td>8.819203</td><td>9.438913</td><td>7.978154</td><td>8.459404</td><td>8.793977</td><td>9.230094</td><td>7.422085</td><td>10.419741</td><td>9.147799</td><td>9.391456</td><td>8.444477</td><td>10.862646</td><td>8.99452</td><td>10.28405</td><td>9.677809</td><td>10.032607</td><td>10.808291</td><td>9.894958</td><td>10.0091</td></tr><tr><td>&quot;hsa-mir-99b&quot;</td><td>15.615404</td><td>15.032799</td><td>15.423429</td><td>16.195341</td><td>15.669493</td><td>14.665012</td><td>16.447575</td><td>15.552656</td><td>15.611705</td><td>15.080196</td><td>15.055132</td><td>15.342601</td><td>15.723402</td><td>14.660829</td><td>16.282966</td><td>14.19662</td><td>16.604994</td><td>14.974622</td><td>15.939962</td><td>14.268735</td><td>15.031533</td><td>15.150034</td><td>15.851494</td><td>15.09133</td><td>15.170902</td><td>15.37158</td><td>16.050275</td><td>13.254587</td><td>15.782359</td><td>15.209602</td><td>16.101107</td><td>14.210575</td><td>14.198866</td><td>14.742741</td><td>15.322417</td><td>14.878092</td><td>&hellip;</td><td>15.787643</td><td>16.505299</td><td>15.29004</td><td>15.079417</td><td>16.116626</td><td>15.330432</td><td>15.283936</td><td>15.385487</td><td>16.423958</td><td>15.711319</td><td>15.983639</td><td>16.497562</td><td>16.74407</td><td>15.262859</td><td>15.17501</td><td>15.714665</td><td>15.525617</td><td>15.368869</td><td>15.13195</td><td>14.870743</td><td>15.460908</td><td>16.539252</td><td>14.771636</td><td>15.942544</td><td>15.783722</td><td>14.4397</td><td>16.104439</td><td>17.393903</td><td>15.448666</td><td>15.951523</td><td>15.102015</td><td>14.927887</td><td>14.737701</td><td>15.205029</td><td>14.508937</td><td>15.952149</td><td>15.050555</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (1_881, 482)\n",
       "┌───────────┬───────────┬───────────┬───────────┬───┬───────────┬───────────┬───────────┬──────────┐\n",
       "│ miRNA_ID  ┆ TCGA-A1-A ┆ TCGA-A1-A ┆ TCGA-A1-A ┆ … ┆ TCGA-E2-A ┆ TCGA-E2-A ┆ TCGA-E2-A ┆ TCGA-E2- │\n",
       "│ ---       ┆ 0SD-01    ┆ 0SE-01    ┆ 0SH-01    ┆   ┆ 1B5-01    ┆ 1B6-01    ┆ 1BC-01    ┆ A1BD-01  │\n",
       "│ str       ┆ ---       ┆ ---       ┆ ---       ┆   ┆ ---       ┆ ---       ┆ ---       ┆ ---      │\n",
       "│           ┆ f64       ┆ f64       ┆ f64       ┆   ┆ f64       ┆ f64       ┆ f64       ┆ f64      │\n",
       "╞═══════════╪═══════════╪═══════════╪═══════════╪═══╪═══════════╪═══════════╪═══════════╪══════════╡\n",
       "│ hsa-let-7 ┆ 14.03039  ┆ 13.034042 ┆ 12.962609 ┆ … ┆ 13.662556 ┆ 13.701712 ┆ 13.904604 ┆ 14.02661 │\n",
       "│ a-1       ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ 6        │\n",
       "│ hsa-let-7 ┆ 14.018248 ┆ 13.017645 ┆ 12.971607 ┆ … ┆ 13.660415 ┆ 13.709611 ┆ 13.897027 ┆ 14.03301 │\n",
       "│ a-2       ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ 2        │\n",
       "│ hsa-let-7 ┆ 14.024003 ┆ 13.036697 ┆ 12.983236 ┆ … ┆ 13.645393 ┆ 13.720637 ┆ 13.917199 ┆ 14.03070 │\n",
       "│ a-3       ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ 2        │\n",
       "│ hsa-let-7 ┆ 15.632852 ┆ 15.423974 ┆ 15.481879 ┆ … ┆ 13.884577 ┆ 14.582151 ┆ 14.850094 ┆ 15.23439 │\n",
       "│ b         ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ 3        │\n",
       "│ hsa-let-7 ┆ 11.397605 ┆ 12.028745 ┆ 12.813933 ┆ … ┆ 11.43789  ┆ 12.117159 ┆ 11.827816 ┆ 12.02858 │\n",
       "│ c         ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ 8        │\n",
       "│ …         ┆ …         ┆ …         ┆ …         ┆ … ┆ …         ┆ …         ┆ …         ┆ …        │\n",
       "│ hsa-mir-9 ┆ 0.0       ┆ 0.0       ┆ 0.0       ┆ … ┆ 0.0       ┆ 0.0       ┆ 0.0       ┆ 0.0      │\n",
       "│ 500       ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆          │\n",
       "│ hsa-mir-9 ┆ 4.392142  ┆ 5.051083  ┆ 3.688321  ┆ … ┆ 5.394588  ┆ 4.969625  ┆ 4.122096  ┆ 6.174561 │\n",
       "│ 6         ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆          │\n",
       "│ hsa-mir-9 ┆ 5.675821  ┆ 5.389651  ┆ 5.399958  ┆ … ┆ 6.586884  ┆ 6.460651  ┆ 5.858322  ┆ 5.453821 │\n",
       "│ 8         ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆          │\n",
       "│ hsa-mir-9 ┆ 10.137212 ┆ 10.460951 ┆ 10.740593 ┆ … ┆ 10.032607 ┆ 10.808291 ┆ 9.894958  ┆ 10.0091  │\n",
       "│ 9a        ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆          │\n",
       "│ hsa-mir-9 ┆ 15.615404 ┆ 15.032799 ┆ 15.423429 ┆ … ┆ 15.205029 ┆ 14.508937 ┆ 15.952149 ┆ 15.05055 │\n",
       "│ 9b        ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ 5        │\n",
       "└───────────┴───────────┴───────────┴───────────┴───┴───────────┴───────────┴───────────┴──────────┘"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# based on what genes are selected for mrna, select the same genes for meth and cnv\n",
    "# perform selection for mirnas\n",
    "mirna = pl.read_csv(\n",
    "    \"tcga_brca_data/raw/mirna.csv\"\n",
    ")\n",
    "mirna = mirna.select(['miRNA_ID'] + nonnormal_samples)\n",
    "mirna"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (481, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>sample_ids</th><th>class</th></tr><tr><td>str</td><td>i64</td></tr></thead><tbody><tr><td>&quot;TCGA-A1-A0SD-01&quot;</td><td>2</td></tr><tr><td>&quot;TCGA-A1-A0SE-01&quot;</td><td>2</td></tr><tr><td>&quot;TCGA-A1-A0SH-01&quot;</td><td>2</td></tr><tr><td>&quot;TCGA-A1-A0SJ-01&quot;</td><td>2</td></tr><tr><td>&quot;TCGA-A1-A0SK-01&quot;</td><td>0</td></tr><tr><td>&hellip;</td><td>&hellip;</td></tr><tr><td>&quot;TCGA-E2-A1B4-01&quot;</td><td>2</td></tr><tr><td>&quot;TCGA-E2-A1B5-01&quot;</td><td>0</td></tr><tr><td>&quot;TCGA-E2-A1B6-01&quot;</td><td>2</td></tr><tr><td>&quot;TCGA-E2-A1BC-01&quot;</td><td>2</td></tr><tr><td>&quot;TCGA-E2-A1BD-01&quot;</td><td>2</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (481, 2)\n",
       "┌─────────────────┬───────┐\n",
       "│ sample_ids      ┆ class │\n",
       "│ ---             ┆ ---   │\n",
       "│ str             ┆ i64   │\n",
       "╞═════════════════╪═══════╡\n",
       "│ TCGA-A1-A0SD-01 ┆ 2     │\n",
       "│ TCGA-A1-A0SE-01 ┆ 2     │\n",
       "│ TCGA-A1-A0SH-01 ┆ 2     │\n",
       "│ TCGA-A1-A0SJ-01 ┆ 2     │\n",
       "│ TCGA-A1-A0SK-01 ┆ 0     │\n",
       "│ …               ┆ …     │\n",
       "│ TCGA-E2-A1B4-01 ┆ 2     │\n",
       "│ TCGA-E2-A1B5-01 ┆ 0     │\n",
       "│ TCGA-E2-A1B6-01 ┆ 2     │\n",
       "│ TCGA-E2-A1BC-01 ┆ 2     │\n",
       "│ TCGA-E2-A1BD-01 ┆ 2     │\n",
       "└─────────────────┴───────┘"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Only 481 samples out of 481 found in y_df\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]  0%|          | 0/5 [00:00<?, ?fold/s]\n",
      "0it [00:00, ?it/s]\n",
      "0it [00:00, ?it/s] 40%|████      | 2/5 [00:00<00:00, 15.41fold/s]\n",
      "0it [00:00, ?it/s]\n",
      "0it [00:00, ?it/s] 80%|████████  | 4/5 [00:00<00:00, 14.92fold/s]\n",
      "Processing folds: 100%|██████████| 5/5 [00:00<00:00, 14.83fold/s]\n"
     ]
    }
   ],
   "source": [
    "mirna = preprocess_seq_data(\n",
    "    df = mirna,\n",
    "    annotation_cols=['miRNA_ID'],\n",
    "    min_median_expression=2,\n",
    "    min_detection_rate=0.4,\n",
    "    top_n=10000,\n",
    "    cpm_normalization=False,\n",
    "    log_transform=False,\n",
    ")\n",
    "mirna\n",
    "OmicDataSplitter(\n",
    "    df=mirna,\n",
    "    annotation_cols=['miRNA_ID'],\n",
    "    y_df=y,\n",
    "    n_features=-1,\n",
    "    n_splits=5,\n",
    "    random_state=3,\n",
    "    output_dir=output_dir + \"/mirna\",\n",
    ").process_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (9_111, 482)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>gene</th><th>TCGA-A1-A0SD-01</th><th>TCGA-A1-A0SE-01</th><th>TCGA-A1-A0SH-01</th><th>TCGA-A1-A0SJ-01</th><th>TCGA-A1-A0SK-01</th><th>TCGA-A1-A0SM-01</th><th>TCGA-A1-A0SO-01</th><th>TCGA-A2-A04N-01</th><th>TCGA-A2-A04P-01</th><th>TCGA-A2-A04Q-01</th><th>TCGA-A2-A04R-01</th><th>TCGA-A2-A04T-01</th><th>TCGA-A2-A04U-01</th><th>TCGA-A2-A04V-01</th><th>TCGA-A2-A04W-01</th><th>TCGA-A2-A04X-01</th><th>TCGA-A2-A04Y-01</th><th>TCGA-A2-A0CL-01</th><th>TCGA-A2-A0CM-01</th><th>TCGA-A2-A0CP-01</th><th>TCGA-A2-A0CQ-01</th><th>TCGA-A2-A0CS-01</th><th>TCGA-A2-A0CT-01</th><th>TCGA-A2-A0CU-01</th><th>TCGA-A2-A0CV-01</th><th>TCGA-A2-A0CW-01</th><th>TCGA-A2-A0D0-01</th><th>TCGA-A2-A0D1-01</th><th>TCGA-A2-A0D2-01</th><th>TCGA-A2-A0D3-01</th><th>TCGA-A2-A0D4-01</th><th>TCGA-A2-A0EM-01</th><th>TCGA-A2-A0EN-01</th><th>TCGA-A2-A0EO-01</th><th>TCGA-A2-A0EQ-01</th><th>TCGA-A2-A0ER-01</th><th>&hellip;</th><th>TCGA-E2-A14W-01</th><th>TCGA-E2-A14X-01</th><th>TCGA-E2-A14Y-01</th><th>TCGA-E2-A14Z-01</th><th>TCGA-E2-A150-01</th><th>TCGA-E2-A152-01</th><th>TCGA-E2-A153-01</th><th>TCGA-E2-A154-01</th><th>TCGA-E2-A155-01</th><th>TCGA-E2-A156-01</th><th>TCGA-E2-A158-01</th><th>TCGA-E2-A159-01</th><th>TCGA-E2-A15A-01</th><th>TCGA-E2-A15C-01</th><th>TCGA-E2-A15D-01</th><th>TCGA-E2-A15E-01</th><th>TCGA-E2-A15F-01</th><th>TCGA-E2-A15G-01</th><th>TCGA-E2-A15H-01</th><th>TCGA-E2-A15I-01</th><th>TCGA-E2-A15J-01</th><th>TCGA-E2-A15K-01</th><th>TCGA-E2-A15L-01</th><th>TCGA-E2-A15M-01</th><th>TCGA-E2-A15O-01</th><th>TCGA-E2-A15P-01</th><th>TCGA-E2-A15R-01</th><th>TCGA-E2-A15S-01</th><th>TCGA-E2-A15T-01</th><th>TCGA-E2-A1AZ-01</th><th>TCGA-E2-A1B0-01</th><th>TCGA-E2-A1B1-01</th><th>TCGA-E2-A1B4-01</th><th>TCGA-E2-A1B5-01</th><th>TCGA-E2-A1B6-01</th><th>TCGA-E2-A1BC-01</th><th>TCGA-E2-A1BD-01</th></tr><tr><td>str</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>&hellip;</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td></tr></thead><tbody><tr><td>&quot;CYP2A6&quot;</td><td>0.7326</td><td>0.9907</td><td>0.0</td><td>7.0156</td><td>0.3953</td><td>0.0</td><td>0.0</td><td>0.4815</td><td>0.6028</td><td>0.4706</td><td>5.0547</td><td>0.4305</td><td>1.3554</td><td>2.6545</td><td>4.0591</td><td>0.7832</td><td>0.4164</td><td>2.9395</td><td>0.5474</td><td>1.2268</td><td>3.7932</td><td>8.0671</td><td>2.4851</td><td>2.8297</td><td>1.7191</td><td>2.8724</td><td>0.0</td><td>0.0</td><td>1.0765</td><td>0.0</td><td>1.0461</td><td>13.0051</td><td>3.158</td><td>1.6563</td><td>0.0</td><td>4.1354</td><td>&hellip;</td><td>1.5443</td><td>1.1688</td><td>0.0</td><td>11.6415</td><td>0.8555</td><td>0.0</td><td>0.0</td><td>3.0799</td><td>1.09</td><td>0.0</td><td>0.0</td><td>0.0</td><td>4.0318</td><td>1.1375</td><td>2.3794</td><td>3.771</td><td>8.3743</td><td>8.0539</td><td>11.3892</td><td>4.3017</td><td>4.3529</td><td>4.1877</td><td>0.0</td><td>14.5701</td><td>2.3563</td><td>8.4825</td><td>1.6428</td><td>2.9821</td><td>3.0383</td><td>0.0</td><td>1.5143</td><td>2.9513</td><td>0.6123</td><td>15.9084</td><td>0.0</td><td>2.3155</td><td>1.5912</td></tr><tr><td>&quot;CLEC3A&quot;</td><td>5.6235</td><td>4.0415</td><td>0.0</td><td>3.7481</td><td>1.5316</td><td>4.831</td><td>1.2338</td><td>0.0</td><td>0.6028</td><td>5.6543</td><td>0.0</td><td>2.3709</td><td>0.0</td><td>0.0</td><td>12.4971</td><td>0.0</td><td>0.0</td><td>12.0808</td><td>0.5474</td><td>10.22</td><td>0.0</td><td>0.0</td><td>5.2076</td><td>0.0</td><td>4.3765</td><td>5.2724</td><td>0.0</td><td>0.0</td><td>1.686</td><td>4.7264</td><td>5.3924</td><td>13.0392</td><td>0.4116</td><td>3.7606</td><td>7.2546</td><td>0.0</td><td>&hellip;</td><td>3.7504</td><td>0.0</td><td>1.3719</td><td>10.6166</td><td>0.8555</td><td>11.2396</td><td>11.1355</td><td>12.1301</td><td>0.0</td><td>10.5382</td><td>0.0</td><td>1.4781</td><td>0.4004</td><td>1.3785</td><td>2.2788</td><td>9.4778</td><td>0.8836</td><td>15.5583</td><td>0.0</td><td>6.9493</td><td>0.0</td><td>13.9854</td><td>1.1865</td><td>14.4175</td><td>0.0</td><td>11.8291</td><td>1.6428</td><td>1.1045</td><td>0.3936</td><td>2.6442</td><td>0.0</td><td>8.1001</td><td>11.5015</td><td>14.25</td><td>0.0</td><td>0.0</td><td>0.5881</td></tr><tr><td>&quot;FABP7&quot;</td><td>9.6318</td><td>4.5672</td><td>4.9863</td><td>3.7869</td><td>2.6685</td><td>1.178</td><td>5.7254</td><td>0.4815</td><td>7.9864</td><td>11.4959</td><td>0.7861</td><td>9.7849</td><td>10.1427</td><td>0.8598</td><td>0.0</td><td>2.3118</td><td>3.9118</td><td>4.3149</td><td>4.5895</td><td>5.7889</td><td>1.051</td><td>0.0</td><td>1.6277</td><td>0.4665</td><td>5.8585</td><td>1.2851</td><td>10.7898</td><td>11.2974</td><td>7.9493</td><td>4.2025</td><td>2.3951</td><td>0.0</td><td>1.7276</td><td>4.1585</td><td>0.0</td><td>3.7394</td><td>&hellip;</td><td>0.0</td><td>9.4015</td><td>0.0</td><td>3.4417</td><td>5.0073</td><td>0.0</td><td>0.0</td><td>1.3168</td><td>1.09</td><td>0.0</td><td>8.3559</td><td>6.7183</td><td>0.7135</td><td>3.2928</td><td>4.0671</td><td>7.0941</td><td>2.7898</td><td>0.0</td><td>4.4278</td><td>4.5174</td><td>0.0</td><td>0.0</td><td>0.3994</td><td>0.6949</td><td>3.6482</td><td>4.3138</td><td>0.9587</td><td>0.6554</td><td>1.6762</td><td>2.4597</td><td>1.08</td><td>5.1418</td><td>2.5904</td><td>5.8941</td><td>10.0022</td><td>0.0</td><td>4.5789</td></tr><tr><td>&quot;DSG1&quot;</td><td>3.3121</td><td>5.4522</td><td>4.2247</td><td>2.2197</td><td>3.2493</td><td>2.4381</td><td>0.3453</td><td>2.1909</td><td>4.8584</td><td>5.0107</td><td>3.4303</td><td>12.757</td><td>2.0417</td><td>0.493</td><td>0.0</td><td>0.0</td><td>1.7407</td><td>0.8002</td><td>4.176</td><td>7.0039</td><td>1.051</td><td>0.0</td><td>3.7141</td><td>1.7185</td><td>3.8258</td><td>0.0</td><td>0.0</td><td>1.2706</td><td>1.686</td><td>0.4032</td><td>1.0461</td><td>0.7742</td><td>3.4029</td><td>4.516</td><td>0.0</td><td>1.4896</td><td>&hellip;</td><td>0.5648</td><td>8.3924</td><td>8.2062</td><td>2.9498</td><td>7.3831</td><td>1.9147</td><td>0.0</td><td>0.0</td><td>0.3586</td><td>0.0</td><td>8.4437</td><td>3.9001</td><td>1.3783</td><td>2.433</td><td>0.0</td><td>4.9711</td><td>1.8218</td><td>0.0</td><td>0.8518</td><td>3.1898</td><td>0.0</td><td>0.0</td><td>3.1659</td><td>1.7088</td><td>2.468</td><td>3.9801</td><td>3.0136</td><td>0.3647</td><td>0.0</td><td>4.5081</td><td>1.3134</td><td>6.4787</td><td>3.4654</td><td>4.908</td><td>8.1218</td><td>0.8444</td><td>6.2333</td></tr><tr><td>&quot;GRIA2&quot;</td><td>6.0696</td><td>1.5726</td><td>2.5984</td><td>0.4497</td><td>4.0937</td><td>1.9414</td><td>10.284</td><td>4.7628</td><td>2.6292</td><td>1.109</td><td>7.7487</td><td>6.6353</td><td>3.4425</td><td>0.0</td><td>0.0</td><td>0.0</td><td>1.2254</td><td>0.8002</td><td>0.0</td><td>2.5089</td><td>3.7555</td><td>8.6277</td><td>5.287</td><td>7.3066</td><td>0.3635</td><td>1.4459</td><td>0.8669</td><td>0.0</td><td>0.4538</td><td>0.4032</td><td>1.8728</td><td>0.4384</td><td>4.5519</td><td>1.0538</td><td>0.8711</td><td>6.5748</td><td>&hellip;</td><td>1.9542</td><td>1.3207</td><td>0.613</td><td>3.7609</td><td>1.7775</td><td>0.4811</td><td>1.0788</td><td>3.3142</td><td>0.8848</td><td>8.5144</td><td>8.1203</td><td>0.4406</td><td>1.1888</td><td>0.848</td><td>0.4333</td><td>4.0163</td><td>2.131</td><td>9.8821</td><td>0.0</td><td>8.6982</td><td>1.0427</td><td>7.3546</td><td>9.1116</td><td>6.0303</td><td>1.161</td><td>10.3031</td><td>9.6381</td><td>2.9821</td><td>0.3936</td><td>3.615</td><td>0.0</td><td>4.868</td><td>10.1273</td><td>2.8022</td><td>5.7456</td><td>0.8444</td><td>0.3238</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>&quot;ZNF207&quot;</td><td>11.6064</td><td>11.4692</td><td>11.3071</td><td>11.9207</td><td>12.0643</td><td>11.1946</td><td>11.8784</td><td>11.5263</td><td>11.4543</td><td>11.364</td><td>11.7128</td><td>11.6849</td><td>11.7023</td><td>11.8897</td><td>10.7094</td><td>11.3548</td><td>12.0554</td><td>11.5925</td><td>11.7237</td><td>11.5746</td><td>11.6509</td><td>11.5546</td><td>12.0832</td><td>11.831</td><td>11.1209</td><td>11.8231</td><td>11.7338</td><td>11.1374</td><td>11.2092</td><td>11.784</td><td>11.9979</td><td>11.5529</td><td>11.8414</td><td>11.4836</td><td>12.1446</td><td>11.5869</td><td>&hellip;</td><td>12.1376</td><td>11.6169</td><td>12.7391</td><td>11.9682</td><td>11.4712</td><td>11.2298</td><td>11.5042</td><td>11.7005</td><td>11.6206</td><td>11.4812</td><td>11.3905</td><td>11.6064</td><td>11.6779</td><td>11.3914</td><td>11.3505</td><td>11.458</td><td>11.7035</td><td>11.4262</td><td>11.4201</td><td>11.7156</td><td>11.7965</td><td>11.4217</td><td>11.545</td><td>11.6342</td><td>11.502</td><td>11.8724</td><td>11.1577</td><td>11.8729</td><td>11.6893</td><td>11.6281</td><td>11.9752</td><td>11.9964</td><td>11.3448</td><td>11.7366</td><td>11.2059</td><td>11.2276</td><td>11.594</td></tr><tr><td>&quot;PTBP1&quot;</td><td>12.2297</td><td>12.4608</td><td>12.1939</td><td>12.7051</td><td>12.5171</td><td>12.3927</td><td>12.7373</td><td>12.6637</td><td>12.6647</td><td>12.2667</td><td>12.7545</td><td>12.2645</td><td>12.9476</td><td>12.5875</td><td>12.5195</td><td>12.2069</td><td>12.5743</td><td>12.4805</td><td>12.5142</td><td>12.5582</td><td>12.3687</td><td>12.5594</td><td>12.0842</td><td>12.3534</td><td>12.0963</td><td>12.6244</td><td>12.8723</td><td>12.7914</td><td>12.6127</td><td>12.4117</td><td>12.4826</td><td>12.2437</td><td>12.4432</td><td>12.0887</td><td>13.0451</td><td>12.0082</td><td>&hellip;</td><td>12.8021</td><td>12.6171</td><td>12.6542</td><td>12.892</td><td>11.983</td><td>11.9728</td><td>12.409</td><td>12.3426</td><td>12.3697</td><td>12.119</td><td>12.2094</td><td>12.0684</td><td>12.2934</td><td>12.3711</td><td>12.0546</td><td>12.023</td><td>12.3827</td><td>12.4485</td><td>12.3581</td><td>12.0038</td><td>12.5821</td><td>12.8112</td><td>12.2749</td><td>11.7045</td><td>12.7152</td><td>11.9377</td><td>12.2303</td><td>12.3328</td><td>12.4221</td><td>11.6894</td><td>12.3228</td><td>11.4483</td><td>12.5468</td><td>12.0973</td><td>12.2912</td><td>11.9636</td><td>13.0645</td></tr><tr><td>&quot;SF1&quot;</td><td>11.6677</td><td>11.6738</td><td>11.8627</td><td>11.6928</td><td>13.0678</td><td>11.5173</td><td>11.9364</td><td>12.0804</td><td>11.7973</td><td>11.8734</td><td>12.0784</td><td>12.3984</td><td>12.0479</td><td>11.9921</td><td>11.4675</td><td>11.7178</td><td>12.1193</td><td>11.8597</td><td>11.8097</td><td>12.2791</td><td>12.11</td><td>11.9405</td><td>12.0099</td><td>12.3068</td><td>11.9232</td><td>12.0018</td><td>12.2454</td><td>12.2509</td><td>12.537</td><td>12.2236</td><td>11.8484</td><td>11.8827</td><td>12.3006</td><td>11.9526</td><td>12.2296</td><td>12.0491</td><td>&hellip;</td><td>12.1007</td><td>12.2322</td><td>12.2935</td><td>12.0252</td><td>11.7965</td><td>11.6499</td><td>11.867</td><td>12.3606</td><td>12.0732</td><td>12.1711</td><td>12.07</td><td>11.9611</td><td>12.1465</td><td>12.1039</td><td>12.023</td><td>11.9936</td><td>12.3262</td><td>12.2015</td><td>12.4888</td><td>12.2174</td><td>12.262</td><td>11.969</td><td>11.752</td><td>11.6183</td><td>11.8356</td><td>12.0469</td><td>11.6709</td><td>12.233</td><td>11.9269</td><td>11.9432</td><td>11.915</td><td>11.8969</td><td>12.1512</td><td>11.8412</td><td>12.1892</td><td>11.4552</td><td>11.7151</td></tr><tr><td>&quot;KHDRBS1&quot;</td><td>11.7743</td><td>12.1109</td><td>11.7246</td><td>12.1517</td><td>13.1415</td><td>12.2916</td><td>12.4559</td><td>12.0035</td><td>12.2328</td><td>12.0409</td><td>11.7603</td><td>12.0918</td><td>12.7866</td><td>12.4325</td><td>11.7414</td><td>12.0117</td><td>12.0812</td><td>11.9133</td><td>12.1196</td><td>12.1183</td><td>11.9507</td><td>11.9184</td><td>11.9565</td><td>12.0129</td><td>12.1739</td><td>12.1135</td><td>12.5306</td><td>12.5181</td><td>12.145</td><td>12.2239</td><td>12.2403</td><td>12.0142</td><td>11.6874</td><td>12.0787</td><td>12.4206</td><td>12.0526</td><td>&hellip;</td><td>12.1835</td><td>12.2079</td><td>12.2501</td><td>12.019</td><td>11.6209</td><td>11.7945</td><td>11.9039</td><td>11.9007</td><td>12.5443</td><td>11.9424</td><td>12.4371</td><td>11.9566</td><td>12.2874</td><td>11.9216</td><td>11.8369</td><td>11.9594</td><td>12.0699</td><td>12.1084</td><td>11.887</td><td>12.0826</td><td>11.9247</td><td>12.0408</td><td>11.9188</td><td>12.3552</td><td>12.3378</td><td>11.8599</td><td>11.7072</td><td>12.2047</td><td>12.3424</td><td>12.4175</td><td>11.9229</td><td>12.2999</td><td>11.9464</td><td>12.2477</td><td>12.2344</td><td>12.0866</td><td>11.9909</td></tr><tr><td>&quot;TARDBP&quot;</td><td>11.6166</td><td>11.7528</td><td>11.4385</td><td>11.7364</td><td>12.5311</td><td>11.6752</td><td>11.9659</td><td>11.8266</td><td>12.1269</td><td>12.1657</td><td>11.9128</td><td>12.3512</td><td>11.9041</td><td>12.1227</td><td>11.6403</td><td>12.1453</td><td>12.306</td><td>11.7489</td><td>12.2475</td><td>11.8549</td><td>12.0947</td><td>11.7277</td><td>11.8859</td><td>11.9613</td><td>11.5899</td><td>11.8579</td><td>12.4486</td><td>12.1968</td><td>12.1249</td><td>11.8786</td><td>12.1275</td><td>11.7687</td><td>11.5539</td><td>11.7591</td><td>12.2167</td><td>11.8033</td><td>&hellip;</td><td>11.9305</td><td>12.0358</td><td>12.2326</td><td>11.9873</td><td>11.6746</td><td>11.6015</td><td>11.5496</td><td>11.6291</td><td>12.0285</td><td>11.7849</td><td>11.6865</td><td>11.9373</td><td>11.8161</td><td>11.7319</td><td>11.7231</td><td>11.7332</td><td>11.8353</td><td>11.7399</td><td>11.8378</td><td>11.8885</td><td>11.751</td><td>11.9265</td><td>11.8521</td><td>12.0469</td><td>11.5618</td><td>11.819</td><td>11.7391</td><td>11.9024</td><td>11.8127</td><td>12.0934</td><td>11.9501</td><td>12.0976</td><td>11.6261</td><td>11.781</td><td>12.0107</td><td>11.871</td><td>11.7559</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (9_111, 482)\n",
       "┌─────────┬────────────┬───────────┬───────────┬───┬───────────┬───────────┬───────────┬───────────┐\n",
       "│ gene    ┆ TCGA-A1-A0 ┆ TCGA-A1-A ┆ TCGA-A1-A ┆ … ┆ TCGA-E2-A ┆ TCGA-E2-A ┆ TCGA-E2-A ┆ TCGA-E2-A │\n",
       "│ ---     ┆ SD-01      ┆ 0SE-01    ┆ 0SH-01    ┆   ┆ 1B5-01    ┆ 1B6-01    ┆ 1BC-01    ┆ 1BD-01    │\n",
       "│ str     ┆ ---        ┆ ---       ┆ ---       ┆   ┆ ---       ┆ ---       ┆ ---       ┆ ---       │\n",
       "│         ┆ f64        ┆ f64       ┆ f64       ┆   ┆ f64       ┆ f64       ┆ f64       ┆ f64       │\n",
       "╞═════════╪════════════╪═══════════╪═══════════╪═══╪═══════════╪═══════════╪═══════════╪═══════════╡\n",
       "│ CYP2A6  ┆ 0.7326     ┆ 0.9907    ┆ 0.0       ┆ … ┆ 15.9084   ┆ 0.0       ┆ 2.3155    ┆ 1.5912    │\n",
       "│ CLEC3A  ┆ 5.6235     ┆ 4.0415    ┆ 0.0       ┆ … ┆ 14.25     ┆ 0.0       ┆ 0.0       ┆ 0.5881    │\n",
       "│ FABP7   ┆ 9.6318     ┆ 4.5672    ┆ 4.9863    ┆ … ┆ 5.8941    ┆ 10.0022   ┆ 0.0       ┆ 4.5789    │\n",
       "│ DSG1    ┆ 3.3121     ┆ 5.4522    ┆ 4.2247    ┆ … ┆ 4.908     ┆ 8.1218    ┆ 0.8444    ┆ 6.2333    │\n",
       "│ GRIA2   ┆ 6.0696     ┆ 1.5726    ┆ 2.5984    ┆ … ┆ 2.8022    ┆ 5.7456    ┆ 0.8444    ┆ 0.3238    │\n",
       "│ …       ┆ …          ┆ …         ┆ …         ┆ … ┆ …         ┆ …         ┆ …         ┆ …         │\n",
       "│ ZNF207  ┆ 11.6064    ┆ 11.4692   ┆ 11.3071   ┆ … ┆ 11.7366   ┆ 11.2059   ┆ 11.2276   ┆ 11.594    │\n",
       "│ PTBP1   ┆ 12.2297    ┆ 12.4608   ┆ 12.1939   ┆ … ┆ 12.0973   ┆ 12.2912   ┆ 11.9636   ┆ 13.0645   │\n",
       "│ SF1     ┆ 11.6677    ┆ 11.6738   ┆ 11.8627   ┆ … ┆ 11.8412   ┆ 12.1892   ┆ 11.4552   ┆ 11.7151   │\n",
       "│ KHDRBS1 ┆ 11.7743    ┆ 12.1109   ┆ 11.7246   ┆ … ┆ 12.2477   ┆ 12.2344   ┆ 12.0866   ┆ 11.9909   │\n",
       "│ TARDBP  ┆ 11.6166    ┆ 11.7528   ┆ 11.4385   ┆ … ┆ 11.781    ┆ 12.0107   ┆ 11.871    ┆ 11.7559   │\n",
       "└─────────┴────────────┴───────────┴───────────┴───┴───────────┴───────────┴───────────┴───────────┘"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mrna"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load each mrna train and test split and select the same features and samples for meth and cnv\n",
    "meth = pl.read_csv(\"tcga_brca_data/raw/meth.csv\")\n",
    "cnv = pl.read_csv(\"tcga_brca_data/raw/cnv.csv\")\n",
    "\n",
    "mrna_path = \"tcga_brca_data/splits/mrna\"\n",
    "meth_path = \"tcga_brca_data/splits/meth\"\n",
    "cnv_path = \"tcga_brca_data/splits/cnv\"\n",
    "mrna_train = mrna_path + \"/train\"\n",
    "mrna_test = mrna_path + \"/test\"\n",
    "\n",
    "mrna_train_files = os.listdir(mrna_train)\n",
    "mrna_test_files = os.listdir(mrna_test)\n",
    "mrna_train_files.sort()\n",
    "mrna_test_files.sort()\n",
    "\n",
    "def format_by_df(df, trn_df, tst_df, gene_col='gene'):\n",
    "\n",
    "    selected_features = mrna_trn_df.columns[:-2]\n",
    "    selected_features_tst = mrna_tst_df.columns[:-2]\n",
    "    assert selected_features == selected_features_tst\n",
    "\n",
    "    train_samples, train_classes = trn_df['sample_ids'].to_list(), trn_df['class'].to_list()\n",
    "    test_samples, test_classes = tst_df['sample_ids'].to_list(), tst_df['class'].to_list()\n",
    "    \n",
    "    df_train = df.select([gene_col] + train_samples)\n",
    "    df_test = df.select([gene_col] + test_samples)\n",
    "\n",
    "    # Get genes as column names\n",
    "    genes_trn = df_train['gene'].to_list()\n",
    "    genes_tst = df_test['gene'].to_list()\n",
    "\n",
    "    # Drop gene column and convert to numpy for transpose\n",
    "    values_trn = df_train.drop('gene').to_numpy().T\n",
    "    values_tst = df_test.drop('gene').to_numpy().T\n",
    "\n",
    "    # Create new dataframe with genes as columns\n",
    "    df_train = pl.DataFrame(\n",
    "        values_trn,\n",
    "        schema=genes_trn,\n",
    "    )\n",
    "    df_test = pl.DataFrame(\n",
    "        values_tst,\n",
    "        schema=genes_tst,\n",
    "    )\n",
    "\n",
    "    # Select features and add sample_ids and class columns\n",
    "    df_train = df_train.select(selected_features).with_columns([\n",
    "        pl.Series(name='sample_ids', values=train_samples),\n",
    "        pl.Series(name='class', values=train_classes)\n",
    "    ])\n",
    "    df_test = df_test.select(selected_features).with_columns([\n",
    "        pl.Series(name='sample_ids', values=test_samples),\n",
    "        pl.Series(name='class', values=test_classes)\n",
    "    ])\n",
    "\n",
    "    return df_train, df_test\n",
    "\n",
    "\n",
    "for i, (tr_f, ts_f) in enumerate(zip(mrna_train_files, mrna_test_files)):\n",
    "    mrna_trn_df = pl.read_csv(f\"{mrna_train}/{tr_f}\")\n",
    "    mrna_tst_df = pl.read_csv(f\"{mrna_test}/{ts_f}\")\n",
    "\n",
    "    meth_train_split, meth_test_split = format_by_df(meth, mrna_trn_df, mrna_tst_df)\n",
    "    cnv_train_split, cnv_test_split = format_by_df(meth, mrna_trn_df, mrna_tst_df)\n",
    "\n",
    "    # write to split files\n",
    "    meth_train_split.write_csv(f\"{meth_path}/train/{tr_f}\")\n",
    "    meth_test_split.write_csv(f\"{meth_path}/test/{ts_f}\")\n",
    "    cnv_train_split.write_csv(f\"{cnv_path}/train/{tr_f}\")\n",
    "    cnv_test_split.write_csv(f\"{cnv_path}/test/{ts_f}\")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}