Merge pull request #38 from UMCUGenetics/develop

v3.2.0

ExonCov.py

@@ -14,14 +14,15 @@
 manager.add_command('search_sample', cli.SearchSample())
 manager.add_command('remove_sample', cli.RemoveSample())
 manager.add_command('check_samples', cli.CheckSamples())
+manager.add_command('create_sample_set', cli.CreateSampleSet())
 
 db_manager.add_command('migrate', MigrateCommand)
 db_manager.add_command('stats', cli.PrintStats())
 db_manager.add_command('panel_genes', cli.PrintPanelGenesTable())
-db_manager.add_command('transcript_gene', cli.PrintTranscripts())
+db_manager.add_command('gene_transcripts', cli.PrintTranscripts())
 db_manager.add_command('import_alias_table', cli.ImportAliasTable())
 db_manager.add_command('export_panel_bed', cli.PrintPanelBed())
-#db_manager.add_command('load_design', cli.LoadDesign())
+# db_manager.add_command('load_design', cli.LoadDesign())  # Disabled, can be used to setup a new database from scratch
 
 if __name__ == "__main__":
     manager.run()

ExonCov/__init__.py

@@ -6,7 +6,8 @@
 from flask_security import Security, SQLAlchemyUserDatastore
 from flask_debugtoolbar import DebugToolbarExtension
 
-from utils import url_for_other_page
+
+from utils import url_for_other_page, event_logger
 
 # Setup APP
 app = Flask(__name__)
@@ -37,3 +38,28 @@ def security_context_processor():
         h=flask_admin.helpers,
         get_url=url_for
     )
+
+
+# DB event listeners
+@db.event.listens_for(models.Panel, "after_insert")
+@db.event.listens_for(models.PanelVersion, "after_insert")
+def after_update(mapper, connection, target):
+    event_data = dict(target.__dict__)
+    event_logger(connection, models.EventLog, target.__class__.__name__, 'insert', event_data)
+
+
+@db.event.listens_for(models.Panel, "after_update")
+@db.event.listens_for(models.PanelVersion, "after_update")
+def after_insert(mapper, connection, target):
+    event_data = dict(target.__dict__)
+    event_logger(connection, models.EventLog, target.__class__.__name__, 'update', event_data)
+
+
+# Custom filters
+@app.template_filter('supress_none')
+def supress_none_filter(value):
+    """Jinja2 filter to supress none/empty values."""
+    if not value:
+        return '-'
+    else:
+        return value

ExonCov/admin_views.py

@@ -4,7 +4,7 @@
 from flask_security import current_user
 
 from . import db, admin
-from .models import Exon, Transcript, Gene, Panel, PanelVersion, CustomPanel, Sample, SampleProject, SampleSet, SequencingRun, User, Role
+import models
 
 
 class CustomModelView(ModelView):
@@ -33,21 +33,31 @@ def _handle_view(self, name, **kwargs):
 
 class PanelAdminView(CustomModelView):
     """Panel admin view."""
-    column_list = ['name', 'versions']
+    column_list = ['name', 'versions', 'disease_description_eng', 'clinic_contact', 'staff_member', 'comments']
     column_searchable_list = ['name']
 
-    form_columns = ['name']
+    form_columns = [
+        'name', 'disease_description_nl', 'disease_description_eng', 'comments', 'patientfolder_alissa',
+        'clinic_contact', 'staff_member'
+    ]
 
 
 class PanelVersionAdminView(CustomModelView):
     """Panel version admin view."""
     column_searchable_list = ['panel_name']
 
-    form_columns = ['panel', 'version_year', 'version_revision', 'comments', 'active', 'validated', 'transcripts']
+    form_columns = [
+        'panel', 'version_year', 'version_revision', 'comments', 'coverage_requirement_15', 'active', 'validated',
+        'transcripts', 'core_genes'
+    ]
     form_ajax_refs = {
         'transcripts': {
             'fields': ['name', 'gene_id'],
             'page_size': 10
+        },
+        'core_genes': {
+            'fields': ['id'],
+            'page_size': 10
         }
     }
 
@@ -57,7 +67,10 @@ class CustomPanelAdminView(CustomModelView):
     column_list = ['id', 'created_by', 'date', 'research_number', 'comments', 'validated', 'validated_by', 'validated_date']
     column_searchable_list = ['id', 'comments', 'research_number']
 
-    form_columns = ['created_by', 'date', 'research_number', 'comments', 'validated', 'validated_by', 'validated_date', 'samples', 'transcripts']
+    form_columns = [
+        'created_by', 'date', 'research_number', 'comments', 'validated', 'validated_by', 'validated_date',
+        'samples', 'transcripts'
+    ]
 
     form_ajax_refs = {
         'transcripts': {
@@ -121,11 +134,14 @@ class ExonAdminView(CustomModelView):
 
 class SampleAdminView(CustomModelView):
     """Sample admin view."""
-    column_list = ['name', 'project', 'sequencing_runs', 'import_date']
+    column_list = ['name', 'type', 'project', 'sequencing_runs', 'import_date']
     column_sortable_list = ['name', 'import_date']
     column_searchable_list = ['name']
 
-    form_columns = ['name', 'project', 'sequencing_runs', 'import_date', 'file_name', 'import_command', 'exon_measurement_file']
+    form_columns = [
+        'name', 'type', 'project', 'sequencing_runs', 'import_date', 'file_name', 'import_command',
+        'exon_measurement_file'
+    ]
     form_ajax_refs = {
         'project': {
             'fields': ['name'],
@@ -140,10 +156,10 @@ class SampleAdminView(CustomModelView):
 
 class SampleProjectAdminView(CustomModelView):
     """SequencingRun admin view."""
-    column_list = ['name']
-    column_searchable_list = ['name']
+    column_list = ['name', 'type']
+    column_searchable_list = ['name', 'type']
 
-    form_columns = ['name']
+    form_columns = ['name', 'type']
 
 
 class SampleSetAdminView(CustomModelView):
@@ -167,29 +183,41 @@ class SequencingRunAdminView(CustomModelView):
     form_columns = ['name', 'platform_unit']
 
 
-class UserAdmin(CustomModelView):
+class UserAdminView(CustomModelView):
     """User admin model."""
 
     can_create = False
 
     column_list = ('first_name', 'last_name', 'email', 'roles', 'active')
+    column_searchable_list = ['first_name', 'last_name', 'email']
 
     form_columns = ('first_name', 'last_name', 'email', 'roles', 'active', 'password')
 
 
-# Link view classes and models
-admin.add_view(PanelAdminView(Panel, db.session))
-admin.add_view(PanelVersionAdminView(PanelVersion, db.session))
-admin.add_view(CustomPanelAdminView(CustomPanel, db.session))
+class EventLogAdminView(CustomModelView):
+    """EventLog admin model."""
+    column_filters = ['table', 'action', 'modified_on']
+    column_searchable_list = ['data']
 
-admin.add_view(SampleAdminView(Sample, db.session))
-admin.add_view(SampleProjectAdminView(SampleProject, db.session))
-admin.add_view(SampleSetAdminView(SampleSet, db.session))
-admin.add_view(SequencingRunAdminView(SequencingRun, db.session))
+    can_create = False
+    can_edit = False
+    can_view_details = True
 
-admin.add_view(GeneAdminView(Gene, db.session))
-admin.add_view(TranscriptAdminView(Transcript, db.session))
-admin.add_view(ExonAdminView(Exon, db.session))
 
-admin.add_view(UserAdmin(User, db.session))
-admin.add_view(CustomModelView(Role, db.session))
+# Link view classes and models
+admin.add_view(PanelAdminView(models.Panel, db.session))
+admin.add_view(PanelVersionAdminView(models.PanelVersion, db.session))
+admin.add_view(CustomPanelAdminView(models.CustomPanel, db.session))
+
+admin.add_view(SampleAdminView(models.Sample, db.session))
+admin.add_view(SampleProjectAdminView(models.SampleProject, db.session))
+admin.add_view(SampleSetAdminView(models.SampleSet, db.session))
+admin.add_view(SequencingRunAdminView(models.SequencingRun, db.session))
+
+admin.add_view(GeneAdminView(models.Gene, db.session))
+admin.add_view(TranscriptAdminView(models.Transcript, db.session))
+admin.add_view(ExonAdminView(models.Exon, db.session))
+
+admin.add_view(UserAdminView(models.User, db.session))
+admin.add_view(CustomModelView(models.Role, db.session))
+admin.add_view(EventLogAdminView(models.EventLog, db.session))

ExonCov/cli.py

@@ -7,17 +7,22 @@
 import os
 import shlex
 import urllib
+import datetime
 
 from flask_script import Command, Option
 from sqlalchemy import func
 from sqlalchemy.orm import joinedload
 from sqlalchemy.exc import IntegrityError
+from sqlalchemy.sql.expression import func
 import tempfile
 import shutil
 import pysam
 
 from . import app, db, utils
-from .models import Gene, GeneAlias, Transcript, Exon, SequencingRun, Sample, SampleProject, TranscriptMeasurement, Panel, PanelVersion, CustomPanel
+from .models import (
+    Gene, GeneAlias, Transcript, Exon, SequencingRun, Sample, SampleProject, TranscriptMeasurement, Panel,
+    PanelVersion, CustomPanel, SampleSet
+)
 from .utils import weighted_average
 
 
@@ -65,25 +70,31 @@ def run(self):
 
 
 class PrintTranscripts(Command):
-    """Print tab delimited transcript / gene table"""
+    """Print tab delimited gene / transcript table"""
 
-    def run(self):
-        print('{transcript}\t{gene}'.format(transcript='Transcript', gene='Gene'))
+    option_list = (
+        Option('-p', '--preferred_transcripts', dest='preferred_transcripts', default=False, action='store_true', help="Print preferred transcripts only"),
+    )
 
-        transcripts = Transcript.query.options(joinedload('gene'))
+    def run(self, preferred_transcripts):
+        print('{gene}\t{transcript}'.format(gene='Gene', transcript='Transcript'))
 
-        for transcript in transcripts:
-            print('{transcript}\t{gene}'.format(
-                transcript=transcript.name,
-                gene=transcript.gene
-            ))
+        genes = Gene.query.options(joinedload('transcripts'))
+
+        for gene in genes:
+            if preferred_transcripts:
+                print('{gene}\t{transcript}'.format(gene=gene.id, transcript=gene.default_transcript.name))
+            else:
+                for transcript in gene.transcripts:
+                    print('{gene}\t{transcript}'.format(gene=gene.id, transcript=transcript.name))
 
 
 class ImportBam(Command):
     """Import sample from bam file."""
 
     option_list = (
         Option('project_name'),
+        Option('sample_type', choices=['WES', 'WGS', 'RNA']),
         Option('bam'),
         Option('-b', '--exon_bed', dest='exon_bed_file', default=app.config['EXON_BED_FILE']),
         Option('-t', '--threads', dest='threads', default=1),
@@ -92,7 +103,7 @@ class ImportBam(Command):
         Option('--temp', dest='temp_path', default=None),
     )
 
-    def run(self, bam, project_name, exon_bed_file, threads, overwrite, print_output, temp_path):
+    def run(self, project_name, sample_type, bam, exon_bed_file, threads, overwrite, print_output, temp_path):
         try:
             bam_file = pysam.AlignmentFile(bam, "rb")
         except IOError as e:
@@ -128,7 +139,8 @@ def run(self, bam, project_name, exon_bed_file, threads, overwrite, print_output
         sample_project, sample_project_exists = utils.get_one_or_create(
             db.session,
             SampleProject,
-            name=project_name
+            name=project_name,
+            type=''
         )  # returns object and exists bool
 
         # Look for sample in database
@@ -153,6 +165,7 @@ def run(self, bam, project_name, exon_bed_file, threads, overwrite, print_output
         sample = Sample(
             name=sample_name,
             project=sample_project,
+            type=sample_type,
             file_name=bam,
             import_command=sambamba_command,
             sequencing_runs=sequencing_runs.values(),
@@ -307,7 +320,7 @@ class SearchSample(Command):
     )
 
     def run(self, sample_name):
-        samples = Sample.query.filter_by(name=sample_name).all()
+        samples = Sample.query.filter(Sample.name.like('%{0}%'.format(sample_name))).all()
 
         print("Sample ID\tSample Name\tProject\tSequencing Runs\tCustom Panels")
         for sample in samples:
@@ -357,6 +370,76 @@ def run(self):
             print("No errors found.")
 
 
+class CreateSampleSet(Command):
+    """Create (random) sample set."""
+
+    option_list = (
+        Option('name'),
+        Option('-d', '--max_days', dest='max_days', type=int, default=180),
+        Option('-s', '--sample_filter', dest='sample_filter', default=''),
+        Option('-t', '--sample_type', dest='sample_type', default='WES'),
+        Option('-n', '--sample_number', dest='sample_number', type=int, default=100),
+    )
+
+    def run(self, name, max_days, sample_filter, sample_type, sample_number):
+        description = '{0} random {1} samples. Maximum age: {2} days. Sample name filter: {3}'.format(
+            sample_number, sample_type, max_days, sample_filter
+        )
+        filter_date = datetime.date.today() - datetime.timedelta(days=max_days)
+
+        samples = (
+            Sample.query
+            .filter(Sample.name.like('%{0}%'.format(sample_filter)))
+            .filter_by(type=sample_type)
+            .order_by(func.rand())
+        )
+        sample_count = 0
+
+        sample_set = SampleSet(
+            name=name,
+            description=description,
+        )
+
+        for sample in samples:
+            # Filter sampels: import date, 'special' project type (validation etc), Merge samples,
+            if (
+                sample.import_date > filter_date
+                and not sample.project.type
+                and len(sample.sequencing_runs) == 1
+                and sample.sequencing_runs[0].platform_unit in sample.project.name
+            ):
+                sample_set.samples.append(sample)
+                sample_count += 1
+
+            if sample_count >= sample_number:
+                break
+
+        if len(sample_set.samples) != sample_number:
+            print 'Not enough samples found to create sample set, found {0} samples.'.format(len(sample_set.samples))
+        else:
+            print 'Creating new random sample set:'
+            print '\tName: {0}'.format(sample_set.name)
+            print '\tDescription: {0}'.format(sample_set.description)
+            print '\tSamples:'
+            for sample in sample_set.samples:
+                print '\t\t{0}\t{1}\t{2}\t{3}\t{4}'.format(
+                    sample.name,
+                    sample.type,
+                    sample.project,
+                    sample.sequencing_runs,
+                    sample.import_date
+                )
+
+            confirmation = ''
+            while confirmation not in ['y', 'n']:
+                confirmation = raw_input('Please check samples and press [Y/y] to continue or [N/n] to abort. ').lower()
+
+            if confirmation == 'y':
+                db.session.add(sample_set)
+                db.session.commit()
+                print 'Sample set created, make sure to activate it via the admin page.'
+
+
 class ImportAliasTable(Command):
     """Import gene aliases from HGNC (ftp://ftp.ebi.ac.uk/pub/databases/genenames/new/tsv/hgnc_complete_set.txt)"""