1.2 Adding Mature, Primary and Extended Sequences.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get miRNA Sequence\n",
    "\n",
    "The simplest way to get the genomic sequence of these regions is to create a UCSC custom track and then download the sequences associated with that track using the UCSC genome browser.\n",
    "\n",
    "To do this, first a custom track in <a href=\"https://genome.ucsc.edu/FAQ/FAQformat.html#format3\">GFF format</a> was created containing the genomic location of each primary miRNA &plusmn;20 bp."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import data_processing as dp\n",
    "\n",
    "def make_track(track_file_loc, db_name, sql_version=\"MySQL\", firewall=False):\n",
    "    db_con = dp.DatabaseConnection(sql_version, db_name=db_name, firewall=firewall)\n",
    "    rows = db_con.fetch_query(\"SELECT PriID, Chr, ChrStrand, GenomeEnd, GenomeStart FROM PrimaryMicroRNA\")\n",
    "    \n",
    "    with open(track_file_loc, \"w\") as fout:\n",
    "        for row in rows:\n",
    "            if sql_version == \"MSSQL\":\n",
    "                out_str = \"chr{}\\tMiRBase\\tExtSeq\\t{}\\t{}\\t.\\t{}\\t.\\t{}\\n\".format(row.Chr, row.GenomeStart-20, row.GenomeEnd+20, \n",
    "                                                                               row.chrStrand, row.PriID)\n",
    "            else:\n",
    "                priID, chrom, strand, end, start = row\n",
    "                out_str = \"chr{}\\tMiRBase\\tExtSeq\\t{}\\t{}\\t.\\t{}\\t.\\t{}\\n\".format(chrom, start-20, end+20, strand, priID)\n",
    "            fout.write(out_str)\n",
    "    db_con.close_cursor()\n",
    "    db_con.close_connection()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "make_track(\"Extended_miR_track.gff\", \"miR-test\", firewall=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The created track was then uploaded using <a href=\"https://genome.ucsc.edu/cgi-bin/hgCustom\">Add Custom Tracks</a>. The custom track was then viewed in <a href=\"https://genome.ucsc.edu/cgi-bin/hgTables\">Table Browser</a> and the \"sequence\" option under output format was selected. The output sequences in fasta format were downloaded and saved as \"Extended_miR_Sequences.fa\".\n",
    "\n",
    "The extended and primary sequences were then added to the database."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import data_processing as dp\n",
    "\n",
    "def import_pri_seq(fasta_file, db_name, sql_version=\"MySQL\", firewall=False):\n",
    "    db_con = dp.DatabaseConnection(sql_version, db_name=db_name, firewall=firewall)\n",
    "    \n",
    "    pri_re = re.compile(\"_(MI[0-9]{7}) \")\n",
    "    with open(fasta_file, \"r\") as fin:\n",
    "        seq_dict = {\"StemLoopSeq\": [], \"LongSeq\": []}\n",
    "        pri_dict = {\"PriID\": []}\n",
    "        seq = \"\"\n",
    "        for line in fin:\n",
    "            if line[0] == \">\":\n",
    "                if seq != \"\":\n",
    "                    seq_dict[\"LongSeq\"] += [seq]\n",
    "                    seq_dict[\"StemLoopSeq\"] += [seq[20:-20]]\n",
    "                    seq = \"\"\n",
    "                    pri_dict[\"PriID\"] += [priID]\n",
    "                priID = pri_re.search(line).group(1)\n",
    "            else:\n",
    "                seq += line.strip(\"\\n\")\n",
    "        # add last lines\n",
    "        seq_dict[\"LongSeq\"] += [seq]\n",
    "        seq_dict[\"StemLoopSeq\"] += [seq[20:-20]]\n",
    "        pri_dict[\"PriID\"] += [priID]\n",
    "        \n",
    "        db_con.update_many_rows(seq_dict, pri_dict, \"PrimaryMicroRNA\")\n",
    "    db_con.close_cursor()\n",
    "    db_con.close_connection()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import_pri_seq(\"Extended_miR_Sequences.fa\", \"miR-test\", firewall=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Once the primary miRNA sequences were loaded into the database, the mature miRNA sequences were added. These sequences were found from within the primary miRNA sequence."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import data_processing as dp\n",
    "\n",
    "def import_mat_seq(db_name, sql_version=\"MySQL\", firewall=False):\n",
    "    \n",
    "    db_con = dp.DatabaseConnection(sql_version, db_name=db_name, firewall=firewall)\n",
    "    \n",
    "    mat_seq_dict = {\"MatSeq\": []}\n",
    "    mat_id_dict = {\"MatID\": []}\n",
    "    rows = db_con.fetch_query(\"\"\"SELECT p.StemLoopSeq, p.GenomeStart, p.GenomeEnd, p.ChrStrand, m.MatID, m.MatStart, m.MatEnd\n",
    "FROM PrimaryMicroRNA AS p\n",
    "JOIN MatureMicroRNA AS m\n",
    "ON p.PriID = m.PriID;\"\"\")\n",
    "    \n",
    "    for row in rows:\n",
    "        if sql_version == \"MSSQL\":\n",
    "            strand = row.ChrStrand\n",
    "            p_start = row.GenomeStart\n",
    "            p_end = row.GenomeEnd\n",
    "            seq = row.StemLoopSeq\n",
    "            matID = row.MatID\n",
    "            m_start = row.MatStart\n",
    "            m_end = row.MatEnd\n",
    "        else:\n",
    "            seq, p_start, p_end, strand, matID, m_start, m_end = row\n",
    "            # MySQL return bytearrays, not strings, so need to convert\n",
    "            seq = str(seq)\n",
    "            matID = str(matID)\n",
    "\n",
    "        start_index = m_start - p_start\n",
    "        end_index = p_end - m_end\n",
    "        if strand == \"+\":\n",
    "            if end_index != 0:\n",
    "                mat_seq = seq[start_index : -1*end_index]\n",
    "            else:\n",
    "                mat_seq = seq[start_index : ]\n",
    "        else:\n",
    "            if start_index != 0:\n",
    "                mat_seq = seq[end_index : -1*start_index]\n",
    "            else:\n",
    "                mat_seq = seq[end_index : ]\n",
    "        mat_seq_dict[\"MatSeq\"] += [mat_seq]\n",
    "        mat_id_dict[\"MatID\"] += [matID]\n",
    "        \n",
    "    db_con.update_many_rows(mat_seq_dict, mat_id_dict, \"MatureMicroRNA\")\n",
    "    db_con.close_cursor()\n",
    "    db_con.close_connection()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import_mat_seq(\"miR-test\", firewall=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Unit Tests\n",
    "\n",
    "To verify the correct sequences were added, the primary, extended and mature sequences were tested using Python's <a href=\"https://docs.python.org/2/library/unittest.html\">unittest module</a>."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import unittest\n",
    "import data_processing as dp\n",
    "\n",
    "class TestSeq(unittest.TestCase):\n",
    "    def setUp(self):\n",
    "        self.db_con = dp.DatabaseConnection(\"MySQL\", db_name=\"miR-test\", firewall=True)\n",
    "        self.sql_version = \"MySQL\"\n",
    "            \n",
    "    def test_stemloop_pos(self):\n",
    "        rows = self.db_con.fetch_query(\"SELECT StemLoopSeq FROM PrimaryMicroRNA WHERE PriID LIKE 'MI0022666'\")\n",
    "        row = rows[0]\n",
    "        if self.sql_version == \"MSSQL\":\n",
    "            seq = row.StemLoopSeq\n",
    "        else:\n",
    "            seq, = row\n",
    "        self.assertEqual(seq, \"GTGCGTGGTGGCTCGAGGCGGGGGTGGGGGCCTCGCCCTGCTTGGGCCCTCCCTGACCTCTCCGCTCCGCACAG\", \n",
    "                         \"MI0022666 sequence incorrect\")\n",
    "        \n",
    "    def test_stemloop_neg(self):\n",
    "        rows = self.db_con.fetch_query(\"SELECT StemLoopSeq FROM PrimaryMicroRNA WHERE PriID LIKE 'MI0001733'\")\n",
    "        row = rows[0]\n",
    "        if self.sql_version == \"MSSQL\":\n",
    "            seq = row.StemLoopSeq\n",
    "        else:\n",
    "            seq, = row\n",
    "        self.assertEqual(seq, \"GCTAAGCACTTACAACTGTTTGCAGAGGAAACTGAGACTTTGTAACTATGTCTCAGTCTCATCTGCAAAGAAGTAAGTGCTTTGC\", \n",
    "                         \"MI0001733 sequence incorrect\")\n",
    "        \n",
    "    def test_longseq_pos(self):\n",
    "        rows = self.db_con.fetch_query(\"SELECT LongSeq FROM PrimaryMicroRNA WHERE PriID LIKE 'MI0003167'\")\n",
    "        row = rows[0]\n",
    "        if self.sql_version == \"MSSQL\":\n",
    "            seq = row.LongSeq\n",
    "        else:\n",
    "            seq, = row\n",
    "        self.assertEqual(seq, \"ACAGTGCTGGAGCAAGAAGATCTCATGATGTGACCATCTGGAGGTAAGAAGCACTTTGTGTTTTGTGAAAGAAAGTGCTTCCTTTCAGAGGGTTACTCTTTGAGAAAAGCAGCATTGAAGTTGAT\", \n",
    "                         \"MI0003167 long sequence incorrect\")\n",
    "        \n",
    "    def test_longseq_neg(self):\n",
    "        rows = self.db_con.fetch_query(\"SELECT LongSeq FROM PrimaryMicroRNA WHERE PriID LIKE 'MI0005116'\")\n",
    "        row = rows[0]\n",
    "        if self.sql_version == \"MSSQL\":\n",
    "            seq = row.LongSeq\n",
    "        else:\n",
    "            seq, = row\n",
    "        self.assertEqual(seq, \"TACCTACCTGACCTAAGGGCTTTAGGCGCTGATGAAAGTGGAGTTCAGTAGACAGCCCTTTTCAAGCCCTACGAGAAACTGGGGTTTCTGGAGGAGAAGGAAGGTGATGAAGGATCTGTTCTCGTGAGCCTGAACTTTCTAGACAAAACATGTG\", \n",
    "                         \"MI0005116 long sequence incorrect\")\n",
    "        \n",
    "    def test_matseq_pos(self):\n",
    "        rows = self.db_con.fetch_query(\"SELECT MatSeq FROM MatureMicroRNA WHERE MatID LIKE 'MIMAT0002806'\")\n",
    "        row = rows[0]\n",
    "        if self.sql_version == \"MSSQL\":\n",
    "            seq = row.MatSeq\n",
    "        else:\n",
    "            seq, = row\n",
    "        self.assertEqual(seq, \"CAACCTGGAGGACTCCATGCTG\", \"MIMAT0002806 sequence incorrect\")\n",
    "        \n",
    "    def test_matseq_neg(self):\n",
    "        rows = self.db_con.fetch_query(\"SELECT MatSeq FROM MatureMicroRNA WHERE MatID LIKE 'MIMAT0019074'\")\n",
    "        row = rows[0]\n",
    "        if self.sql_version == \"MSSQL\":\n",
    "            seq = row.MatSeq\n",
    "        else:\n",
    "            seq, = row\n",
    "        self.assertEqual(seq, \"ACTGGACTAGGAGTCAGAAGG\", \"MIMAT0019074 sequence incorrect\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "test_longseq_neg (__main__.TestSeq) ... ok\n",
      "test_longseq_pos (__main__.TestSeq) ... ok\n",
      "test_matseq_neg (__main__.TestSeq) ... ok\n",
      "test_matseq_pos (__main__.TestSeq) ... ok\n",
      "test_stemloop_neg (__main__.TestSeq) ... ok\n",
      "test_stemloop_pos (__main__.TestSeq) ... ok\n",
      "\n",
      "----------------------------------------------------------------------\n",
      "Ran 6 tests in 7.087s\n",
      "\n",
      "OK\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<unittest.runner.TextTestResult run=6 errors=0 failures=0>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "suite = unittest.TestLoader().loadTestsFromTestCase(TestSeq)\n",
    "unittest.TextTestRunner(verbosity=2).run(suite)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}