mismatched_seq_efficiency.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "34afb950",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import numpy as np\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import sys\n",
    "import matplotlib.pyplot as plt\n",
    "import argparse\n",
    "import pickle \n",
    "from src.utils import ReaderWriter, create_directory\n",
    "from src.utils import one_hot_encode\n",
    "from src.utils import get_char, outer_cross_val\n",
    "import itertools\n",
    "from sklearn.model_selection import KFold\n",
    "import xgboost as xgb\n",
    "from src.utils import ReaderWriter, create_directory, perfmetric_report_cont\n",
    "from src.utils import compute_spearman_corr, compute_pearson_corr\n",
    "import matplotlib.pyplot as plt\n",
    "from src.utils import performance_statistics,seq_to_onehot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9cb8147d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>NAME</th>\n",
       "      <th>ORIGINAL_SEQ</th>\n",
       "      <th>MISMATCHED_SEQ</th>\n",
       "      <th>EFFICIENCY</th>\n",
       "      <th>MISMATCH_TYPE</th>\n",
       "      <th>POSITION_FROM_LEFT</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1MM_pur_oRNA_DNMT1_1054_mouse</td>\n",
       "      <td>GTCAGTGTGATAGGATCCGT</td>\n",
       "      <td>ATCAGTGTGATAGGATCCGT</td>\n",
       "      <td>2.801900</td>\n",
       "      <td>G-&gt;A</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1MM_pur_oRNA_DNMT1_1054_mouse</td>\n",
       "      <td>GTCAGTGTGATAGGATCCGT</td>\n",
       "      <td>GCCAGTGTGATAGGATCCGT</td>\n",
       "      <td>6.401032</td>\n",
       "      <td>T-&gt;C</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1MM_pur_oRNA_DNMT1_1054_mouse</td>\n",
       "      <td>GTCAGTGTGATAGGATCCGT</td>\n",
       "      <td>GTTAGTGTGATAGGATCCGT</td>\n",
       "      <td>3.629865</td>\n",
       "      <td>C-&gt;T</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1MM_pur_oRNA_DNMT1_1054_mouse</td>\n",
       "      <td>GTCAGTGTGATAGGATCCGT</td>\n",
       "      <td>GTCGGTGTGATAGGATCCGT</td>\n",
       "      <td>1.973759</td>\n",
       "      <td>A-&gt;G</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1MM_pur_oRNA_DNMT1_1054_mouse</td>\n",
       "      <td>GTCAGTGTGATAGGATCCGT</td>\n",
       "      <td>GTCAATGTGATAGGATCCGT</td>\n",
       "      <td>8.968154</td>\n",
       "      <td>G-&gt;A</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>311</th>\n",
       "      <td>2MM_rev_oRNA_HEKs_18_2_human</td>\n",
       "      <td>GAACGTCTGTCCTTACCCCT</td>\n",
       "      <td>GAACGTCTGTCCTTACCCGA</td>\n",
       "      <td>48.705436</td>\n",
       "      <td>C-&gt;G, T-&gt;A</td>\n",
       "      <td>19, 20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>312</th>\n",
       "      <td>0MM_non_oRNA_DNMT1_1054_mouse</td>\n",
       "      <td>GTCAGTGTGATAGGATCCGT</td>\n",
       "      <td>GTCAGTGTGATAGGATCCGT</td>\n",
       "      <td>62.675635</td>\n",
       "      <td>no_MM</td>\n",
       "      <td>no_MM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>313</th>\n",
       "      <td>0MM_non_oRNA_DNMT1_1050_mouse</td>\n",
       "      <td>GTAGTCAGAATACTTGCGGT</td>\n",
       "      <td>GTAGTCAGAATACTTGCGGT</td>\n",
       "      <td>43.151624</td>\n",
       "      <td>no_MM</td>\n",
       "      <td>no_MM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>314</th>\n",
       "      <td>0MM_non_oRNA_EMX1_human</td>\n",
       "      <td>GTGATGGGAGCCCTTCTTCT</td>\n",
       "      <td>GTGATGGGAGCCCTTCTTCT</td>\n",
       "      <td>31.895831</td>\n",
       "      <td>no_MM</td>\n",
       "      <td>no_MM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>315</th>\n",
       "      <td>0MM_non_oRNA_HEKs_18_2_human</td>\n",
       "      <td>GAACGTCTGTCCTTACCCCT</td>\n",
       "      <td>GAACGTCTGTCCTTACCCCT</td>\n",
       "      <td>37.820113</td>\n",
       "      <td>no_MM</td>\n",
       "      <td>no_MM</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>316 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                              NAME          ORIGINAL_SEQ   \n",
       "0    1MM_pur_oRNA_DNMT1_1054_mouse  GTCAGTGTGATAGGATCCGT  \\\n",
       "1    1MM_pur_oRNA_DNMT1_1054_mouse  GTCAGTGTGATAGGATCCGT   \n",
       "2    1MM_pur_oRNA_DNMT1_1054_mouse  GTCAGTGTGATAGGATCCGT   \n",
       "3    1MM_pur_oRNA_DNMT1_1054_mouse  GTCAGTGTGATAGGATCCGT   \n",
       "4    1MM_pur_oRNA_DNMT1_1054_mouse  GTCAGTGTGATAGGATCCGT   \n",
       "..                             ...                   ...   \n",
       "311   2MM_rev_oRNA_HEKs_18_2_human  GAACGTCTGTCCTTACCCCT   \n",
       "312  0MM_non_oRNA_DNMT1_1054_mouse  GTCAGTGTGATAGGATCCGT   \n",
       "313  0MM_non_oRNA_DNMT1_1050_mouse  GTAGTCAGAATACTTGCGGT   \n",
       "314        0MM_non_oRNA_EMX1_human  GTGATGGGAGCCCTTCTTCT   \n",
       "315   0MM_non_oRNA_HEKs_18_2_human  GAACGTCTGTCCTTACCCCT   \n",
       "\n",
       "           MISMATCHED_SEQ  EFFICIENCY MISMATCH_TYPE POSITION_FROM_LEFT  \n",
       "0    ATCAGTGTGATAGGATCCGT    2.801900          G->A                  1  \n",
       "1    GCCAGTGTGATAGGATCCGT    6.401032          T->C                  2  \n",
       "2    GTTAGTGTGATAGGATCCGT    3.629865          C->T                  3  \n",
       "3    GTCGGTGTGATAGGATCCGT    1.973759          A->G                  4  \n",
       "4    GTCAATGTGATAGGATCCGT    8.968154          G->A                  5  \n",
       "..                    ...         ...           ...                ...  \n",
       "311  GAACGTCTGTCCTTACCCGA   48.705436    C->G, T->A             19, 20  \n",
       "312  GTCAGTGTGATAGGATCCGT   62.675635         no_MM              no_MM  \n",
       "313  GTAGTCAGAATACTTGCGGT   43.151624         no_MM              no_MM  \n",
       "314  GTGATGGGAGCCCTTCTTCT   31.895831         no_MM              no_MM  \n",
       "315  GAACGTCTGTCCTTACCCCT   37.820113         no_MM              no_MM  \n",
       "\n",
       "[316 rows x 6 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#. Reading the data\n",
    "data_name = '20240115_for_ML_mismatch_model_potential.xlsx'\n",
    "data_dir = './data/'\n",
    "df = pd.read_excel(os.path.join(data_dir,data_name ) )\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "38420863",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>NAME</th>\n",
       "      <th>ORIGINAL_SEQ</th>\n",
       "      <th>MISMATCHED_SEQ</th>\n",
       "      <th>EFFICIENCY</th>\n",
       "      <th>MISMATCH_TYPE</th>\n",
       "      <th>POSITION_FROM_LEFT</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1MM_pur_oRNA_DNMT1_1054_mouse</td>\n",
       "      <td>GTCAGTGTGATAGGATCCGT</td>\n",
       "      <td>ATCAGTGTGATAGGATCCGT</td>\n",
       "      <td>2.801900</td>\n",
       "      <td>G-&gt;A</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1MM_pur_oRNA_DNMT1_1054_mouse</td>\n",
       "      <td>GTCAGTGTGATAGGATCCGT</td>\n",
       "      <td>GCCAGTGTGATAGGATCCGT</td>\n",
       "      <td>6.401032</td>\n",
       "      <td>T-&gt;C</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1MM_pur_oRNA_DNMT1_1054_mouse</td>\n",
       "      <td>GTCAGTGTGATAGGATCCGT</td>\n",
       "      <td>GTTAGTGTGATAGGATCCGT</td>\n",
       "      <td>3.629865</td>\n",
       "      <td>C-&gt;T</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1MM_pur_oRNA_DNMT1_1054_mouse</td>\n",
       "      <td>GTCAGTGTGATAGGATCCGT</td>\n",
       "      <td>GTCGGTGTGATAGGATCCGT</td>\n",
       "      <td>1.973759</td>\n",
       "      <td>A-&gt;G</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1MM_pur_oRNA_DNMT1_1054_mouse</td>\n",
       "      <td>GTCAGTGTGATAGGATCCGT</td>\n",
       "      <td>GTCAATGTGATAGGATCCGT</td>\n",
       "      <td>8.968154</td>\n",
       "      <td>G-&gt;A</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>309</th>\n",
       "      <td>2MM_rev_oRNA_HEKs_18_2_human</td>\n",
       "      <td>GAACGTCTGTCCTTACCCCT</td>\n",
       "      <td>GAACGTCTGTCCTTACCCGA</td>\n",
       "      <td>48.705436</td>\n",
       "      <td>C-&gt;G, T-&gt;A</td>\n",
       "      <td>19, 20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>310</th>\n",
       "      <td>0MM_non_oRNA_DNMT1_1054_mouse</td>\n",
       "      <td>GTCAGTGTGATAGGATCCGT</td>\n",
       "      <td>GTCAGTGTGATAGGATCCGT</td>\n",
       "      <td>62.675635</td>\n",
       "      <td>no_MM</td>\n",
       "      <td>no_MM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>311</th>\n",
       "      <td>0MM_non_oRNA_DNMT1_1050_mouse</td>\n",
       "      <td>GTAGTCAGAATACTTGCGGT</td>\n",
       "      <td>GTAGTCAGAATACTTGCGGT</td>\n",
       "      <td>43.151624</td>\n",
       "      <td>no_MM</td>\n",
       "      <td>no_MM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>312</th>\n",
       "      <td>0MM_non_oRNA_EMX1_human</td>\n",
       "      <td>GTGATGGGAGCCCTTCTTCT</td>\n",
       "      <td>GTGATGGGAGCCCTTCTTCT</td>\n",
       "      <td>31.895831</td>\n",
       "      <td>no_MM</td>\n",
       "      <td>no_MM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>313</th>\n",
       "      <td>0MM_non_oRNA_HEKs_18_2_human</td>\n",
       "      <td>GAACGTCTGTCCTTACCCCT</td>\n",
       "      <td>GAACGTCTGTCCTTACCCCT</td>\n",
       "      <td>37.820113</td>\n",
       "      <td>no_MM</td>\n",
       "      <td>no_MM</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>314 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                              NAME          ORIGINAL_SEQ   \n",
       "0    1MM_pur_oRNA_DNMT1_1054_mouse  GTCAGTGTGATAGGATCCGT  \\\n",
       "1    1MM_pur_oRNA_DNMT1_1054_mouse  GTCAGTGTGATAGGATCCGT   \n",
       "2    1MM_pur_oRNA_DNMT1_1054_mouse  GTCAGTGTGATAGGATCCGT   \n",
       "3    1MM_pur_oRNA_DNMT1_1054_mouse  GTCAGTGTGATAGGATCCGT   \n",
       "4    1MM_pur_oRNA_DNMT1_1054_mouse  GTCAGTGTGATAGGATCCGT   \n",
       "..                             ...                   ...   \n",
       "309   2MM_rev_oRNA_HEKs_18_2_human  GAACGTCTGTCCTTACCCCT   \n",
       "310  0MM_non_oRNA_DNMT1_1054_mouse  GTCAGTGTGATAGGATCCGT   \n",
       "311  0MM_non_oRNA_DNMT1_1050_mouse  GTAGTCAGAATACTTGCGGT   \n",
       "312        0MM_non_oRNA_EMX1_human  GTGATGGGAGCCCTTCTTCT   \n",
       "313   0MM_non_oRNA_HEKs_18_2_human  GAACGTCTGTCCTTACCCCT   \n",
       "\n",
       "           MISMATCHED_SEQ  EFFICIENCY MISMATCH_TYPE POSITION_FROM_LEFT  \n",
       "0    ATCAGTGTGATAGGATCCGT    2.801900          G->A                  1  \n",
       "1    GCCAGTGTGATAGGATCCGT    6.401032          T->C                  2  \n",
       "2    GTTAGTGTGATAGGATCCGT    3.629865          C->T                  3  \n",
       "3    GTCGGTGTGATAGGATCCGT    1.973759          A->G                  4  \n",
       "4    GTCAATGTGATAGGATCCGT    8.968154          G->A                  5  \n",
       "..                    ...         ...           ...                ...  \n",
       "309  GAACGTCTGTCCTTACCCGA   48.705436    C->G, T->A             19, 20  \n",
       "310  GTCAGTGTGATAGGATCCGT   62.675635         no_MM              no_MM  \n",
       "311  GTAGTCAGAATACTTGCGGT   43.151624         no_MM              no_MM  \n",
       "312  GTGATGGGAGCCCTTCTTCT   31.895831         no_MM              no_MM  \n",
       "313  GAACGTCTGTCCTTACCCCT   37.820113         no_MM              no_MM  \n",
       "\n",
       "[314 rows x 6 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "### remove the nan\n",
    "df = df.dropna()\n",
    "df = df.reset_index(drop=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "267b0312",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(314, 160)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "##2.prepare the data\n",
    "unique_seq = df['ORIGINAL_SEQ'].apply(get_char)\n",
    "missmatch_seq = df['MISMATCHED_SEQ'].apply(get_char)\n",
    "y = np.array(df['EFFICIENCY'])\n",
    "onehot_x_1 = seq_to_onehot(unique_seq)\n",
    "onehot_x_2 = seq_to_onehot(missmatch_seq)\n",
    "onehot_x_1.shape\n",
    "x = np.concatenate([onehot_x_1, onehot_x_2], axis=1)\n",
    "x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8abe7186",
   "metadata": {},
   "outputs": [],
   "source": [
    "## 3. train by splitting the data according to the pairs (original and missmatched sequence)\n",
    "#output_path = './miss_match/standard_data_split'\n",
    "output_path = './miss_match'\n",
    "data_partitions = outer_cross_val(onehot_x_1.shape[0], 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "136ac25b",
   "metadata": {},
   "outputs": [],
   "source": [
    "other_params = {'subsample': 1.0,'min_child_weight': 10, 'gamma': 2.5,'colsample_bytree': 0.4,'alpha': 0, 'learning_rate':0.1}\n",
    "param_vals = {'n_estimators': [600],'max_depth': [12], 'reg_lambda': [1, 10, 100]}\n",
    "N_innerCV  =2\n",
    "inner_cv = KFold(n_splits=N_innerCV, shuffle=True, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f2d47ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "## 4.train\n",
    "hyper_param = param_vals.values()\n",
    "combinations = list(itertools.product(*hyper_param))\n",
    "test_error = []\n",
    "test_spearm_corr = []\n",
    "test_pearson_corr = []\n",
    "# outer cross validation\n",
    "for i in range(len(data_partitions)):\n",
    "    print(f\"Outer Fold {i}:\")\n",
    "    print(f\"   number of Training instances:\", len(data_partitions[i]['train_index']))\n",
    "    print(f\"   number of test instances:\", len(data_partitions[i]['test_index']))\n",
    "    error = np.empty([len(data_partitions),len(combinations)])\n",
    "    ## inner cross validation for hyper param tuning\n",
    "    for j, (train_index_new, val_index) in enumerate(inner_cv.split(data_partitions[i]['train_index'])):\n",
    "        print(f\"Inner Fold {j}:\")\n",
    "        print(f\"   number of Training instances:\", len(train_index_new))\n",
    "        print(f\"   number of validation instances:\", len(val_index))\n",
    "    \n",
    "        for index, c in enumerate(combinations):\n",
    "            model = xgb.XGBRegressor(n_estimators=c[0], max_depth=c[1], reg_lambda=c[2], **other_params)\n",
    "            model.fit(x[train_index_new,:], y[train_index_new], eval_set = [(x[val_index,:], y[val_index])], early_stopping_rounds=10) \n",
    "            ## train error:\n",
    "            yhat_train = model.predict(x[train_index_new,:])\n",
    "            train_error = np.absolute(y[train_index_new]-yhat_train).mean()\n",
    "            #print('training error:', train_error)\n",
    "            yhat_val = model.predict(x[val_index,:])\n",
    "            #error = MAE(y[val_index], yhat)\n",
    "            val_error = np.absolute(y[val_index]-yhat_val).mean()\n",
    "            #print('validation error:', val_error)\n",
    "            error[j,index] =val_error\n",
    "            \n",
    "    #get the best paramerter values       \n",
    "    best_param_indx = error.mean(axis = 0).argmin()\n",
    "    best_param = combinations[best_param_indx]\n",
    "    #best_param = [100, 12, 10]\n",
    "    print('best parameters:', best_param)\n",
    "    #train the model with the best parameters\n",
    "    model = xgb.XGBRegressor(n_estimators=best_param[0], max_depth=best_param[1], reg_lambda=best_param[2], **other_params)\n",
    "    #predict on the test set \n",
    "    model.fit(x[data_partitions[i]['train_index'],:], y[data_partitions[i]['train_index']])\n",
    "    yhat_test = model.predict(x[data_partitions[i]['test_index'],:])\n",
    "    t_error = np.absolute(y[data_partitions[i]['test_index']]-yhat_test).mean()\n",
    "    ## creat directory to save the model\n",
    "    path_to_save = create_directory('run_'+ str(i), output_path)\n",
    "    ## save the best model for this run\n",
    "    model.save_model(path_to_save +'/best.model')\n",
    "    ReaderWriter.write_csv(path_to_save+'/prediction', y[data_partitions[i]['test_index']],yhat_test, header=['true','predicted'])\n",
    "    #ReaderWriter.write_log()\n",
    "    test_error.append(t_error)\n",
    "    test_spearm_corr.append(compute_spearman_corr(yhat_test,y[data_partitions[i]['test_index']]))\n",
    "    test_pearson_corr.append(compute_pearson_corr(yhat_test,y[data_partitions[i]['test_index']]))\n",
    "    perfmetric_report_cont(yhat_test, y[data_partitions[i]['test_index']], path_to_save +'/test_performance.csv' )\n",
    "    ReaderWriter.write_params(path_to_save,param_vals,  best_param, other_params)\n",
    "    fig, ax = plt.subplots()\n",
    "    ax.scatter(yhat_test,y[data_partitions[i]['test_index']],  vmin=0, vmax=100)\n",
    "\n",
    "performance_statistics(len(data_partitions), output_path)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5ccf223e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Spearman Correlation: Mean = 0.7333764011302861 , Std = 0.07349276836001198\n",
      "Spearman P-Value: Mean = 1.8774856923675426e-09 , Std = 3.350146058360265e-09\n",
      "Pearson Correlation: Mean = 0.9078966800158655 , Std = 0.01793635094431689\n",
      "Pearson P-Value: Mean = 2.1671487893712296e-22 , Std = 3.7807270899142507e-22\n",
      "MAE Score: Mean = 9.437229745743858 , Std = 1.3841802206945841\n"
     ]
    }
   ],
   "source": [
    "performance_statistics(len(data_partitions), output_path)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "50ef5543",
   "metadata": {},
   "source": [
    "## Train the model with leave one out split strategy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "5b68974c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>NAME</th>\n",
       "      <th>ORIGINAL_SEQ</th>\n",
       "      <th>MISMATCHED_SEQ</th>\n",
       "      <th>EFFICIENCY</th>\n",
       "      <th>MISMATCH_TYPE</th>\n",
       "      <th>POSITION_FROM_LEFT</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1MM_pur_oRNA_DNMT1_1054_mouse</td>\n",
       "      <td>GTCAGTGTGATAGGATCCGT</td>\n",
       "      <td>ATCAGTGTGATAGGATCCGT</td>\n",
       "      <td>2.801900</td>\n",
       "      <td>G-&gt;A</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1MM_pur_oRNA_DNMT1_1054_mouse</td>\n",
       "      <td>GTCAGTGTGATAGGATCCGT</td>\n",
       "      <td>GCCAGTGTGATAGGATCCGT</td>\n",
       "      <td>6.401032</td>\n",
       "      <td>T-&gt;C</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1MM_pur_oRNA_DNMT1_1054_mouse</td>\n",
       "      <td>GTCAGTGTGATAGGATCCGT</td>\n",
       "      <td>GTTAGTGTGATAGGATCCGT</td>\n",
       "      <td>3.629865</td>\n",
       "      <td>C-&gt;T</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1MM_pur_oRNA_DNMT1_1054_mouse</td>\n",
       "      <td>GTCAGTGTGATAGGATCCGT</td>\n",
       "      <td>GTCGGTGTGATAGGATCCGT</td>\n",
       "      <td>1.973759</td>\n",
       "      <td>A-&gt;G</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1MM_pur_oRNA_DNMT1_1054_mouse</td>\n",
       "      <td>GTCAGTGTGATAGGATCCGT</td>\n",
       "      <td>GTCAATGTGATAGGATCCGT</td>\n",
       "      <td>8.968154</td>\n",
       "      <td>G-&gt;A</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>309</th>\n",
       "      <td>2MM_rev_oRNA_HEKs_18_2_human</td>\n",
       "      <td>GAACGTCTGTCCTTACCCCT</td>\n",
       "      <td>GAACGTCTGTCCTTACCCGA</td>\n",
       "      <td>48.705436</td>\n",
       "      <td>C-&gt;G, T-&gt;A</td>\n",
       "      <td>19, 20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>310</th>\n",
       "      <td>0MM_non_oRNA_DNMT1_1054_mouse</td>\n",
       "      <td>GTCAGTGTGATAGGATCCGT</td>\n",
       "      <td>GTCAGTGTGATAGGATCCGT</td>\n",
       "      <td>62.675635</td>\n",
       "      <td>no_MM</td>\n",
       "      <td>no_MM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>311</th>\n",
       "      <td>0MM_non_oRNA_DNMT1_1050_mouse</td>\n",
       "      <td>GTAGTCAGAATACTTGCGGT</td>\n",
       "      <td>GTAGTCAGAATACTTGCGGT</td>\n",
       "      <td>43.151624</td>\n",
       "      <td>no_MM</td>\n",
       "      <td>no_MM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>312</th>\n",
       "      <td>0MM_non_oRNA_EMX1_human</td>\n",
       "      <td>GTGATGGGAGCCCTTCTTCT</td>\n",
       "      <td>GTGATGGGAGCCCTTCTTCT</td>\n",
       "      <td>31.895831</td>\n",
       "      <td>no_MM</td>\n",
       "      <td>no_MM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>313</th>\n",
       "      <td>0MM_non_oRNA_HEKs_18_2_human</td>\n",
       "      <td>GAACGTCTGTCCTTACCCCT</td>\n",
       "      <td>GAACGTCTGTCCTTACCCCT</td>\n",
       "      <td>37.820113</td>\n",
       "      <td>no_MM</td>\n",
       "      <td>no_MM</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>314 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                              NAME          ORIGINAL_SEQ   \n",
       "0    1MM_pur_oRNA_DNMT1_1054_mouse  GTCAGTGTGATAGGATCCGT  \\\n",
       "1    1MM_pur_oRNA_DNMT1_1054_mouse  GTCAGTGTGATAGGATCCGT   \n",
       "2    1MM_pur_oRNA_DNMT1_1054_mouse  GTCAGTGTGATAGGATCCGT   \n",
       "3    1MM_pur_oRNA_DNMT1_1054_mouse  GTCAGTGTGATAGGATCCGT   \n",
       "4    1MM_pur_oRNA_DNMT1_1054_mouse  GTCAGTGTGATAGGATCCGT   \n",
       "..                             ...                   ...   \n",
       "309   2MM_rev_oRNA_HEKs_18_2_human  GAACGTCTGTCCTTACCCCT   \n",
       "310  0MM_non_oRNA_DNMT1_1054_mouse  GTCAGTGTGATAGGATCCGT   \n",
       "311  0MM_non_oRNA_DNMT1_1050_mouse  GTAGTCAGAATACTTGCGGT   \n",
       "312        0MM_non_oRNA_EMX1_human  GTGATGGGAGCCCTTCTTCT   \n",
       "313   0MM_non_oRNA_HEKs_18_2_human  GAACGTCTGTCCTTACCCCT   \n",
       "\n",
       "           MISMATCHED_SEQ  EFFICIENCY MISMATCH_TYPE POSITION_FROM_LEFT  \n",
       "0    ATCAGTGTGATAGGATCCGT    2.801900          G->A                  1  \n",
       "1    GCCAGTGTGATAGGATCCGT    6.401032          T->C                  2  \n",
       "2    GTTAGTGTGATAGGATCCGT    3.629865          C->T                  3  \n",
       "3    GTCGGTGTGATAGGATCCGT    1.973759          A->G                  4  \n",
       "4    GTCAATGTGATAGGATCCGT    8.968154          G->A                  5  \n",
       "..                    ...         ...           ...                ...  \n",
       "309  GAACGTCTGTCCTTACCCGA   48.705436    C->G, T->A             19, 20  \n",
       "310  GTCAGTGTGATAGGATCCGT   62.675635         no_MM              no_MM  \n",
       "311  GTAGTCAGAATACTTGCGGT   43.151624         no_MM              no_MM  \n",
       "312  GTGATGGGAGCCCTTCTTCT   31.895831         no_MM              no_MM  \n",
       "313  GAACGTCTGTCCTTACCCCT   37.820113         no_MM              no_MM  \n",
       "\n",
       "[314 rows x 6 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "### Train the model with leave one out split strategy where we leave a target site and all its miss matched pairs\n",
    "## from the training and test on the new target site and its missmatched pairs( testing the model on totally unseen target)\n",
    "#. Reading the data\n",
    "data_name = '20240115_for_ML_mismatch_model_potential.xlsx'\n",
    "data_dir = './data/'\n",
    "df = pd.read_excel(os.path.join(data_dir,data_name ) )\n",
    "### remove the nan\n",
    "df = df.dropna()\n",
    "df = df.reset_index(drop=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "94409e28",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['GTCAGTGTGATAGGATCCGT', 'GTAGTCAGAATACTTGCGGT',\n",
       "       'GTGATGGGAGCCCTTCTTCT', 'GAACGTCTGTCCTTACCCCT'], dtype=object)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['ORIGINAL_SEQ'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "3bf3eb17",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Assuming you have already imported pandas as pd and your DataFrame is named df\n",
    "test_seq = 'GTCAGTGTGATAGGATCCGT'\n",
    "train_df = df[~df['ORIGINAL_SEQ'].str.contains(test_seq)]\n",
    "test_df =  df[df['ORIGINAL_SEQ'].str.contains(test_seq)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "f1c06936",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>NAME</th>\n",
       "      <th>ORIGINAL_SEQ</th>\n",
       "      <th>MISMATCHED_SEQ</th>\n",
       "      <th>EFFICIENCY</th>\n",
       "      <th>MISMATCH_TYPE</th>\n",
       "      <th>POSITION_FROM_LEFT</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>1MM_pur_oRNA_DNMT1_1050_mouse</td>\n",
       "      <td>GTAGTCAGAATACTTGCGGT</td>\n",
       "      <td>ATAGTCAGAATACTTGCGGT</td>\n",
       "      <td>3.957317</td>\n",
       "      <td>G-&gt;A</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>1MM_pur_oRNA_DNMT1_1050_mouse</td>\n",
       "      <td>GTAGTCAGAATACTTGCGGT</td>\n",
       "      <td>GCAGTCAGAATACTTGCGGT</td>\n",
       "      <td>2.459427</td>\n",
       "      <td>T-&gt;C</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>1MM_pur_oRNA_DNMT1_1050_mouse</td>\n",
       "      <td>GTAGTCAGAATACTTGCGGT</td>\n",
       "      <td>GTGGTCAGAATACTTGCGGT</td>\n",
       "      <td>0.844803</td>\n",
       "      <td>A-&gt;G</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>1MM_pur_oRNA_DNMT1_1050_mouse</td>\n",
       "      <td>GTAGTCAGAATACTTGCGGT</td>\n",
       "      <td>GTAATCAGAATACTTGCGGT</td>\n",
       "      <td>8.346463</td>\n",
       "      <td>G-&gt;A</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>1MM_pur_oRNA_DNMT1_1050_mouse</td>\n",
       "      <td>GTAGTCAGAATACTTGCGGT</td>\n",
       "      <td>GTAGCCAGAATACTTGCGGT</td>\n",
       "      <td>7.354122</td>\n",
       "      <td>T-&gt;C</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>308</th>\n",
       "      <td>2MM_rev_oRNA_HEKs_18_2_human</td>\n",
       "      <td>GAACGTCTGTCCTTACCCCT</td>\n",
       "      <td>GAACGTCTGTCCTTACCGGT</td>\n",
       "      <td>41.702427</td>\n",
       "      <td>C-&gt;G, C-&gt;G</td>\n",
       "      <td>18, 19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>309</th>\n",
       "      <td>2MM_rev_oRNA_HEKs_18_2_human</td>\n",
       "      <td>GAACGTCTGTCCTTACCCCT</td>\n",
       "      <td>GAACGTCTGTCCTTACCCGA</td>\n",
       "      <td>48.705436</td>\n",
       "      <td>C-&gt;G, T-&gt;A</td>\n",
       "      <td>19, 20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>311</th>\n",
       "      <td>0MM_non_oRNA_DNMT1_1050_mouse</td>\n",
       "      <td>GTAGTCAGAATACTTGCGGT</td>\n",
       "      <td>GTAGTCAGAATACTTGCGGT</td>\n",
       "      <td>43.151624</td>\n",
       "      <td>no_MM</td>\n",
       "      <td>no_MM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>312</th>\n",
       "      <td>0MM_non_oRNA_EMX1_human</td>\n",
       "      <td>GTGATGGGAGCCCTTCTTCT</td>\n",
       "      <td>GTGATGGGAGCCCTTCTTCT</td>\n",
       "      <td>31.895831</td>\n",
       "      <td>no_MM</td>\n",
       "      <td>no_MM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>313</th>\n",
       "      <td>0MM_non_oRNA_HEKs_18_2_human</td>\n",
       "      <td>GAACGTCTGTCCTTACCCCT</td>\n",
       "      <td>GAACGTCTGTCCTTACCCCT</td>\n",
       "      <td>37.820113</td>\n",
       "      <td>no_MM</td>\n",
       "      <td>no_MM</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>235 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                              NAME          ORIGINAL_SEQ   \n",
       "20   1MM_pur_oRNA_DNMT1_1050_mouse  GTAGTCAGAATACTTGCGGT  \\\n",
       "21   1MM_pur_oRNA_DNMT1_1050_mouse  GTAGTCAGAATACTTGCGGT   \n",
       "22   1MM_pur_oRNA_DNMT1_1050_mouse  GTAGTCAGAATACTTGCGGT   \n",
       "23   1MM_pur_oRNA_DNMT1_1050_mouse  GTAGTCAGAATACTTGCGGT   \n",
       "24   1MM_pur_oRNA_DNMT1_1050_mouse  GTAGTCAGAATACTTGCGGT   \n",
       "..                             ...                   ...   \n",
       "308   2MM_rev_oRNA_HEKs_18_2_human  GAACGTCTGTCCTTACCCCT   \n",
       "309   2MM_rev_oRNA_HEKs_18_2_human  GAACGTCTGTCCTTACCCCT   \n",
       "311  0MM_non_oRNA_DNMT1_1050_mouse  GTAGTCAGAATACTTGCGGT   \n",
       "312        0MM_non_oRNA_EMX1_human  GTGATGGGAGCCCTTCTTCT   \n",
       "313   0MM_non_oRNA_HEKs_18_2_human  GAACGTCTGTCCTTACCCCT   \n",
       "\n",
       "           MISMATCHED_SEQ  EFFICIENCY MISMATCH_TYPE POSITION_FROM_LEFT  \n",
       "20   ATAGTCAGAATACTTGCGGT    3.957317          G->A                  1  \n",
       "21   GCAGTCAGAATACTTGCGGT    2.459427          T->C                  2  \n",
       "22   GTGGTCAGAATACTTGCGGT    0.844803          A->G                  3  \n",
       "23   GTAATCAGAATACTTGCGGT    8.346463          G->A                  4  \n",
       "24   GTAGCCAGAATACTTGCGGT    7.354122          T->C                  5  \n",
       "..                    ...         ...           ...                ...  \n",
       "308  GAACGTCTGTCCTTACCGGT   41.702427    C->G, C->G             18, 19  \n",
       "309  GAACGTCTGTCCTTACCCGA   48.705436    C->G, T->A             19, 20  \n",
       "311  GTAGTCAGAATACTTGCGGT   43.151624         no_MM              no_MM  \n",
       "312  GTGATGGGAGCCCTTCTTCT   31.895831         no_MM              no_MM  \n",
       "313  GAACGTCTGTCCTTACCCCT   37.820113         no_MM              no_MM  \n",
       "\n",
       "[235 rows x 6 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "50a1592a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>NAME</th>\n",
       "      <th>ORIGINAL_SEQ</th>\n",
       "      <th>MISMATCHED_SEQ</th>\n",
       "      <th>EFFICIENCY</th>\n",
       "      <th>MISMATCH_TYPE</th>\n",
       "      <th>POSITION_FROM_LEFT</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1MM_pur_oRNA_DNMT1_1054_mouse</td>\n",
       "      <td>GTCAGTGTGATAGGATCCGT</td>\n",
       "      <td>ATCAGTGTGATAGGATCCGT</td>\n",
       "      <td>2.801900</td>\n",
       "      <td>G-&gt;A</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1MM_pur_oRNA_DNMT1_1054_mouse</td>\n",
       "      <td>GTCAGTGTGATAGGATCCGT</td>\n",
       "      <td>GCCAGTGTGATAGGATCCGT</td>\n",
       "      <td>6.401032</td>\n",
       "      <td>T-&gt;C</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1MM_pur_oRNA_DNMT1_1054_mouse</td>\n",
       "      <td>GTCAGTGTGATAGGATCCGT</td>\n",
       "      <td>GTTAGTGTGATAGGATCCGT</td>\n",
       "      <td>3.629865</td>\n",
       "      <td>C-&gt;T</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1MM_pur_oRNA_DNMT1_1054_mouse</td>\n",
       "      <td>GTCAGTGTGATAGGATCCGT</td>\n",
       "      <td>GTCGGTGTGATAGGATCCGT</td>\n",
       "      <td>1.973759</td>\n",
       "      <td>A-&gt;G</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1MM_pur_oRNA_DNMT1_1054_mouse</td>\n",
       "      <td>GTCAGTGTGATAGGATCCGT</td>\n",
       "      <td>GTCAATGTGATAGGATCCGT</td>\n",
       "      <td>8.968154</td>\n",
       "      <td>G-&gt;A</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>251</th>\n",
       "      <td>2MM_rev_oRNA_DNMT1_1054_mouse</td>\n",
       "      <td>GTCAGTGTGATAGGATCCGT</td>\n",
       "      <td>GTCAGTGTGATAGGAAGCGT</td>\n",
       "      <td>74.963963</td>\n",
       "      <td>T-&gt;A, C-&gt;G</td>\n",
       "      <td>16, 17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>252</th>\n",
       "      <td>2MM_rev_oRNA_DNMT1_1054_mouse</td>\n",
       "      <td>GTCAGTGTGATAGGATCCGT</td>\n",
       "      <td>GTCAGTGTGATAGGATGGGT</td>\n",
       "      <td>80.524939</td>\n",
       "      <td>C-&gt;G, C-&gt;G</td>\n",
       "      <td>17, 18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>253</th>\n",
       "      <td>2MM_rev_oRNA_DNMT1_1054_mouse</td>\n",
       "      <td>GTCAGTGTGATAGGATCCGT</td>\n",
       "      <td>GTCAGTGTGATAGGATCGCT</td>\n",
       "      <td>77.191697</td>\n",
       "      <td>C-&gt;G, G-&gt;C</td>\n",
       "      <td>18, 19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>254</th>\n",
       "      <td>2MM_rev_oRNA_DNMT1_1054_mouse</td>\n",
       "      <td>GTCAGTGTGATAGGATCCGT</td>\n",
       "      <td>GTCAGTGTGATAGGATCCCA</td>\n",
       "      <td>85.023695</td>\n",
       "      <td>G-&gt;C, T-&gt;A</td>\n",
       "      <td>19, 20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>310</th>\n",
       "      <td>0MM_non_oRNA_DNMT1_1054_mouse</td>\n",
       "      <td>GTCAGTGTGATAGGATCCGT</td>\n",
       "      <td>GTCAGTGTGATAGGATCCGT</td>\n",
       "      <td>62.675635</td>\n",
       "      <td>no_MM</td>\n",
       "      <td>no_MM</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>79 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                              NAME          ORIGINAL_SEQ   \n",
       "0    1MM_pur_oRNA_DNMT1_1054_mouse  GTCAGTGTGATAGGATCCGT  \\\n",
       "1    1MM_pur_oRNA_DNMT1_1054_mouse  GTCAGTGTGATAGGATCCGT   \n",
       "2    1MM_pur_oRNA_DNMT1_1054_mouse  GTCAGTGTGATAGGATCCGT   \n",
       "3    1MM_pur_oRNA_DNMT1_1054_mouse  GTCAGTGTGATAGGATCCGT   \n",
       "4    1MM_pur_oRNA_DNMT1_1054_mouse  GTCAGTGTGATAGGATCCGT   \n",
       "..                             ...                   ...   \n",
       "251  2MM_rev_oRNA_DNMT1_1054_mouse  GTCAGTGTGATAGGATCCGT   \n",
       "252  2MM_rev_oRNA_DNMT1_1054_mouse  GTCAGTGTGATAGGATCCGT   \n",
       "253  2MM_rev_oRNA_DNMT1_1054_mouse  GTCAGTGTGATAGGATCCGT   \n",
       "254  2MM_rev_oRNA_DNMT1_1054_mouse  GTCAGTGTGATAGGATCCGT   \n",
       "310  0MM_non_oRNA_DNMT1_1054_mouse  GTCAGTGTGATAGGATCCGT   \n",
       "\n",
       "           MISMATCHED_SEQ  EFFICIENCY MISMATCH_TYPE POSITION_FROM_LEFT  \n",
       "0    ATCAGTGTGATAGGATCCGT    2.801900          G->A                  1  \n",
       "1    GCCAGTGTGATAGGATCCGT    6.401032          T->C                  2  \n",
       "2    GTTAGTGTGATAGGATCCGT    3.629865          C->T                  3  \n",
       "3    GTCGGTGTGATAGGATCCGT    1.973759          A->G                  4  \n",
       "4    GTCAATGTGATAGGATCCGT    8.968154          G->A                  5  \n",
       "..                    ...         ...           ...                ...  \n",
       "251  GTCAGTGTGATAGGAAGCGT   74.963963    T->A, C->G             16, 17  \n",
       "252  GTCAGTGTGATAGGATGGGT   80.524939    C->G, C->G             17, 18  \n",
       "253  GTCAGTGTGATAGGATCGCT   77.191697    C->G, G->C             18, 19  \n",
       "254  GTCAGTGTGATAGGATCCCA   85.023695    G->C, T->A             19, 20  \n",
       "310  GTCAGTGTGATAGGATCCGT   62.675635         no_MM              no_MM  \n",
       "\n",
       "[79 rows x 6 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "6aae4cc3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(235, 160)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "##2.prepare the data\n",
    "unique_seq = train_df['ORIGINAL_SEQ'].apply(get_char)\n",
    "missmatch_seq = train_df['MISMATCHED_SEQ'].apply(get_char)\n",
    "train_y = np.array(train_df['EFFICIENCY'])\n",
    "onehot_x_1 = seq_to_onehot(unique_seq)\n",
    "onehot_x_2 = seq_to_onehot(missmatch_seq)\n",
    "onehot_x_1.shape\n",
    "train_x = np.concatenate([onehot_x_1, onehot_x_2], axis=1)\n",
    "train_x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "1a8dda74",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(79, 160)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "### test:\n",
    "unique_seq_test = test_df['ORIGINAL_SEQ'].apply(get_char)\n",
    "missmatch_seq_test = test_df['MISMATCHED_SEQ'].apply(get_char)\n",
    "test_y = np.array(test_df['EFFICIENCY'])\n",
    "onehot_x_1_test = seq_to_onehot(unique_seq_test)\n",
    "onehot_x_2_test = seq_to_onehot(missmatch_seq_test)\n",
    "test_x = np.concatenate([onehot_x_1_test, onehot_x_2_test], axis=1)\n",
    "test_x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "badc0961",
   "metadata": {},
   "outputs": [],
   "source": [
    "output_path = './miss_match/leav_seq_out'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "8727879b",
   "metadata": {},
   "outputs": [],
   "source": [
    "other_params = {'subsample': 1.0,'min_child_weight': 10, 'gamma': 2.5,'colsample_bytree': 0.4,'alpha': 0, 'learning_rate':0.1}\n",
    "param_vals = {'n_estimators': [600],'max_depth': [12], 'reg_lambda': [1, 10, 100]}\n",
    "N_innerCV  =2\n",
    "inner_cv = KFold(n_splits=N_innerCV, shuffle=True, random_state=0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "70de563f",
   "metadata": {},
   "outputs": [],
   "source": [
    "## 4.train\n",
    "hyper_param = param_vals.values()\n",
    "combinations = list(itertools.product(*hyper_param))\n",
    "test_error = []\n",
    "test_spearm_corr = []\n",
    "test_pearson_corr = []\n",
    "error = np.empty([N_innerCV,len(combinations)]) \n",
    "## inner cross validation for hyper param tuning\n",
    "for j, (train_index_new, val_index) in enumerate(inner_cv.split(np.arange(train_x.shape[0]))):\n",
    "    print(f\"Inner Fold {j}:\")\n",
    "    print(f\"   number of Training instances:\", len(train_index_new))\n",
    "    print(f\"   number of validation instances:\", len(val_index))\n",
    "\n",
    "    for index, c in enumerate(combinations):\n",
    "        model = xgb.XGBRegressor(n_estimators=c[0], max_depth=c[1], reg_lambda=c[2], **other_params)\n",
    "        model.fit(train_x[train_index_new,:], train_y[train_index_new], eval_set = [(train_x[val_index,:], train_y[val_index])], early_stopping_rounds=10) \n",
    "        ## train error:\n",
    "        yhat_train = model.predict(train_x[train_index_new,:])\n",
    "        train_error = np.absolute(train_y[train_index_new]-yhat_train).mean()\n",
    "        #print('training error:', train_error)\n",
    "        yhat_val = model.predict(train_x[val_index,:])\n",
    "        #error = MAE(y[val_index], yhat)\n",
    "        val_error = np.absolute(train_y[val_index]-yhat_val).mean()\n",
    "        #print('validation error:', val_error)\n",
    "        error[j,index] =val_error\n",
    "\n",
    "#get the best paramerter values       \n",
    "best_param_indx = error.mean(axis = 0).argmin()\n",
    "best_param = combinations[best_param_indx]\n",
    "#best_param = [100, 12, 10]\n",
    "print('best parameters:', best_param)\n",
    "#train the model with the best parameters\n",
    "model = xgb.XGBRegressor(n_estimators=best_param[0], max_depth=best_param[1], reg_lambda=best_param[2], **other_params)\n",
    "#predict on the test set \n",
    "model.fit(train_x, train_y)\n",
    "yhat_test = model.predict(test_x)\n",
    "t_error = np.absolute(test_y-yhat_test).mean()\n",
    "## creat directory to save the model\n",
    "path_to_save = create_directory(test_seq, output_path)\n",
    "## save the best model for this run\n",
    "model.save_model(path_to_save +'/best.model')\n",
    "ReaderWriter.write_csv(path_to_save+'/prediction', test_y,yhat_test, header=['true','predicted'])\n",
    "#ReaderWriter.write_log()\n",
    "test_error.append(t_error)\n",
    "test_spearm_corr.append(compute_spearman_corr(yhat_test,test_y))\n",
    "test_pearson_corr.append(compute_pearson_corr(yhat_test,test_y))\n",
    "perfmetric_report_cont(yhat_test, test_y, path_to_save +'/test_performance.csv' )\n",
    "ReaderWriter.write_params(path_to_save,param_vals,  best_param, other_params)\n",
    "fig, ax = plt.subplots()\n",
    "ax.scatter(yhat_test,test_y,  vmin=0, vmax=100)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "f66813a4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[SignificanceResult(statistic=0.15359560910440212, pvalue=0.17654737748471333)]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_spearm_corr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "c3159bb3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[PearsonRResult(statistic=0.30947703540451854, pvalue=0.005514533953972714)]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_pearson_corr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "d65c241f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[24.147825127831045]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0ad4b8d3",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}