Merge branch 'release/v3.1.0'

.travis.yml

@@ -0,0 +1,34 @@
+notifications:
+  slack: wtsi-cgpit:ptUMR1tkNyZJYd9TpGoss8WR
+  email: false
+
+env:
+ - CC=gcc
+
+addons:
+ apt:
+  packages:
+   - build-essential
+   - autoconf
+   - bsdtar
+   - time
+   - curl
+   - libcurl4-openssl-dev
+   - nettle-dev
+   - zlib1g-dev
+   - libncurses5-dev
+   - libpstreams-dev
+   - unzip
+   - libpng12-dev
+   - libexpat1-dev
+
+install: true
+
+language: perl
+
+perl:
+ - "5.22"
+
+script:
+ - ./setup.sh ~/wtsi-opt
+ - ~/wtsi-opt/bin/samtools view # dump usage to show intact

MANIFEST

@@ -1,3 +1,5 @@
+.travis.yml
+bin/bam2bedgraph
 bin/bam_stats.pl
 bin/bam_to_sra_sub.pl
 bin/bamToBw.pl
@@ -9,6 +11,7 @@ bin/gnos_pull.pl
 bin/monitor.pl
 bin/xam_coverage_bins.pl
 bin/xml_to_bas.pl
+c/bam2bedgraph.c
 c/bam_access.c
 c/bam_access.h
 c/bam_stats.c
@@ -27,6 +30,7 @@ c/dbg.h
 c/khash.h
 c/reheadSQ.c
 CHANGES.md
+dists/patch/Bio-BigFile_build.patch
 dists/snappy-1.1.2.tar.gz
 docs.tar.gz
 examples/gnos_pull.ini

README.md

@@ -1,7 +1,11 @@
 ICGC-TCGA-PCAP
 ==============
 
-NGS reference implementations and helper code for the ICGC/TCGA Pan-Cancer Analysis Project
+NGS reference implementations and helper code for the ICGC/TCGA Pan-Cancer Analysis Project.
+
+| Master | Dev |
+|---|---|
+| [![Build Status](https://travis-ci.org/ICGC-TCGA-PanCancer/PCAP-core.svg?branch=master)](https://travis-ci.org/ICGC-TCGA-PanCancer/PCAP-core) |  [![Build Status](https://travis-ci.org/ICGC-TCGA-PanCancer/PCAP-core.svg?branch=dev)](https://travis-ci.org/ICGC-TCGA-PanCancer/PCAP-core) |
 
 This repository contains code to run genomic alignments of paired end data
 and subsequent calling algorithms.

bin/bamToBw.pl

@@ -82,6 +82,7 @@ sub setup {
               'r|reference=s' => \$opts{'reference'},
               'p|process=s' => \$opts{'process'},
               'i|index=i' => \$opts{'index'},
+              'f|filter=i' => \$opts{'filter'},
               'j|jobs' => \$opts{'jobs'},
   ) or pod2usage(2);
 
@@ -106,6 +107,7 @@ sub setup {
 
   delete $opts{'process'} unless(defined $opts{'process'});
   delete $opts{'index'} unless(defined $opts{'index'});
+  delete $opts{'filter'} unless(defined $opts{'filter'});
 
   # now safe to apply defaults
   $opts{'threads'} = 1 unless(defined $opts{'threads'});
@@ -160,10 +162,13 @@ =head1 SYNOPSIS
   Required parameters:
     -bam       -b   BAM/CRAM file to be processed.
     -outdir    -o   Folder to output result to.
-    -threads   -t   Number of threads to use. [1]
     -reference -r   Path to genome.fa.
                      - Actually using fa.fai but for convention just provide '.fa' file
 
+  Optional parameters:
+    -threads   -t   Number of threads to use. [1]
+    -filter    -f   Ignore reads with the filter flags [int]
+
   Targeted processing:
     -process   -p   Only process this step then exit, optionally set -index
                          bamToBw - Per chromosome BigWigs

bin/bam_stats.pl

@@ -32,11 +32,8 @@
 use Config; # so we can see if threads are enabled
 use Data::Dumper;
 
-BEGIN {
-  if($Config{useithreads}) {
-    require threads;
-  }
-};
+our $CAN_USE_THREADS = 0;
+$CAN_USE_THREADS = eval 'use threads; 1';
 
 use PCAP::Bam::Stats;
 
@@ -54,15 +51,20 @@ BEGIN
     $out_location = 'STDOUT';
   }
 
+  if($opts->{'threads'} > 1 && $CAN_USE_THREADS == 0) {
+    warn "Threading is not available perl component will run as a single process";
+    $opts->{'threads'} = 1;
+  }
+
   try{
     my $stats;
-    if($opts->{'threads'} > 1 && $Config{useithreads}) {
+    if($opts->{'threads'} > 1 && $CAN_USE_THREADS) {
       for my $thread(0..($opts->{'threads'}-1)) {
          my ($thr) = threads->create(\&stat_thread, $opts, $thread);
       }
-      sleep 2 while(threads->list(threads::running) > 0);
+      sleep 2 while(threads->list(threads::running()) > 0);
       my @bas_objs;
-      for my $thr(threads->list(threads::joinable)) {
+      for my $thr(threads->list(threads::joinable())) {
         push @bas_objs, $thr->join;
         if(my $err = $thr->error) { die "Thread error: $err\n"; }
       }

bin/bwa_mem.pl

@@ -97,6 +97,8 @@ sub setup {
               'p|process=s' => \$opts{'process'},
               'i|index=i' => \$opts{'index'},
               'b|bwa=s' => \$opts{'bwa'},
+              'c|cram' => \$opts{'cram'},
+              'sc|scramble=s' => \$opts{'scramble'},
   ) or pod2usage(2);
 
   pod2usage(-verbose => 1, -exitval => 0) if(defined $opts{'h'});
@@ -126,6 +128,9 @@ sub setup {
   delete $opts{'process'} unless(defined $opts{'process'});
   delete $opts{'index'} unless(defined $opts{'index'});
   delete $opts{'bwa'} unless(defined $opts{'bwa'});
+  delete $opts{'scramble'} unless(defined $opts{'scramble'});
+
+  PCAP::Cli::opt_requires_opts('scramble', \%opts, ['cram']);
 
   # now safe to apply defaults
   $opts{'threads'} = 1 unless(defined $opts{'threads'});
@@ -189,10 +194,13 @@ =head1 SYNOPSIS
     -threads   -t   Number of threads to use. [1]
 
   Optional parameters:
-    -fragment  -f   Split input into fragements of X million repairs
+    -fragment  -f   Split input into fragements of X million repairs [10]
     -nomarkdup -n   Don't mark duplicates
-    -bwa       -b   Single quoted string of parameters to pass to BWA
-                     - overrides all defaults except '-t,-p,-R'
+    -cram      -c   Output cram, see '-sc'
+    -scramble  -sc  Single quoted string of parameters to pass to Scramble when '-c' used
+                    - '-I,-O' are used internally and should not be provided
+    -bwa       -b   Single quoted string of additional parameters to pass to BWA
+                     - '-t,-p,-R' are used internally and should not be provided
 
   Targeted processing:
     -process   -p   Only process this step then exit, optionally set -index

bin/gnos_pull.pl

@@ -43,6 +43,9 @@
 use Proc::PID::File;
 use Data::Dumper;
 
+my $CAN_USE_THREADS = 0;
+$CAN_USE_THREADS = eval 'use threads; 1';
+
 use PCAP;
 use PCAP::Cli;
 
@@ -99,7 +102,7 @@ sub load_config {
   }
   if(exists $options->{'COMPOSITE_FILTERS'}->{'not_sanger_workflow'}) {
     my %bl_workflows = map { $_ => 1 } split /[\n,]/, $options->{'COMPOSITE_FILTERS'}->{'not_sanger_workflow'};
-    $options->{'COMPOSITE_FILTERS'}->{'not_sanger_workflow'} = [keys \%bl_workflows];
+    $options->{'COMPOSITE_FILTERS'}->{'not_sanger_workflow'} = [keys %bl_workflows];
   }
 
   croak sprintf q{'KEY_FILE' Ssection is absent from %s}, $options->{'config'} unless($cfg->SectionExists('KEY_FILES'));
@@ -132,8 +135,6 @@ sub load_config {
   return 1;
 }
 
-use threads;
-
 sub pull_data {
   my ($options, $to_process) = @_;
 
@@ -153,6 +154,10 @@ sub pull_data {
   }
 
   my $thread_count = $options->{'threads'};
+  if($CAN_USE_THREADS == 0) {
+    warn "Threading is not available perl component will run as a single process";
+    $thread_count = 1;
+  }
 
 
   while(@{$to_process} > 0) {
@@ -170,13 +175,13 @@ sub pull_data {
 
     warn "Submitting: $donor->{donor_unique_id}\n";
     if($thread_count > 1) {
-      if(threads->list(threads::all) < $thread_count) {
+      if(threads->list(threads::all()) < $thread_count) {
         threads->create($code_ref, $options, $donor, $orig_base, $donor_base);
         # don't sleep if not full yet
-        next if(threads->list(threads::all) < $thread_count);
+        next if(threads->list(threads::all()) < $thread_count);
       }
-      sleep 1 while(threads->list(threads::joinable) == 0);
-      for my $thr(threads->list(threads::joinable)) {
+      sleep 1 while(threads->list(threads::joinable()) == 0);
+      for my $thr(threads->list(threads::joinable())) {
         $thr->join;
         if(my $err = $thr->error) { die "Thread error: $err\n"; }
       }
@@ -187,8 +192,8 @@ sub pull_data {
   }
   if($thread_count > 1) {
     # last gasp for any remaining threads
-    sleep 1 while(threads->list(threads::running) > 0);
-    for my $thr(threads->list(threads::joinable)) {
+    sleep 1 while(threads->list(threads::running()) > 0);
+    for my $thr(threads->list(threads::joinable())) {
       $thr->join;
       if(my $err = $thr->error) { die "Thread error: $err\n"; }
     }

c/Makefile

@@ -42,7 +42,7 @@ LFLAGS?= -L$(HTSTMP)
 # define any libraries to link into executable:
 #   if I want to link in libraries (libx.so or libx.a) I use the -llibname
 #   option, something like (this will link in libmylib.so and libm.so:
-LIBS =-lhts -lpthread -lz -lm
+LIBS =-lhts -lpthread -lz -lm -ldl
 
 # define the C source files
 SRCS = ./bam_access.c ./bam_stats_output.c ./bam_stats_calcs.c
@@ -66,6 +66,7 @@ MD := mkdir
 BAM_STATS_TARGET=../bin/bam_stats
 CAT_TARGET=../bin/bwcat
 SQ_TARGET=../bin/reheadSQ
+BAM2BG_TARGET=../bin/bam2bedgraph
 
 #
 # The following part of the makefile is generic; it can be used to
@@ -77,7 +78,7 @@ SQ_TARGET=../bin/reheadSQ
 
 .NOTPARALLEL: test
 
-all: clean pre make_htslib_tmp $(BAM_STATS_TARGET) test remove_htslib_tmp $(CAT_TARGET) $(SQ_TARGET)
+all: clean pre make_htslib_tmp $(BAM_STATS_TARGET) $(BAM2BG_TARGET) test remove_htslib_tmp $(CAT_TARGET) $(SQ_TARGET)
 	@echo  bam_stats, reheadSQ and bwcat compiled.
 
 $(BAM_STATS_TARGET): $(OBJS)
@@ -86,6 +87,9 @@ $(BAM_STATS_TARGET): $(OBJS)
 $(CAT_TARGET):
 	$(CC) $(CAT_INCLUDES) $(CFLAGS) ./bwcat.c $(CAT_LFLAGS) -lBigWig -lz -lcurl -lm -lgnutls -ltasn1 -lhogweed -lnettle -lgmp -lp11-kit -o $(CAT_TARGET)
 
+$(BAM2BG_TARGET): $(OBJS)
+	$(CC) $(CFLAGS) $(INCLUDES) -o $(BAM2BG_TARGET) $(OBJS) $(LFLAGS) $(LIBS) ./bam2bedgraph.c
+
 $(SQ_TARGET):
 	$(CC) $(CFLAGS) ./reheadSQ.c -o $(SQ_TARGET)