Merge pull request #32 from ICGC-TCGA-PanCancer/dev

Dev - merge 1.0.2 for release

Changes

@@ -1,3 +1,9 @@
+1.0.2
+  Upgrade install to pull biobambam 0.0.138
+   - fastqtobam option 'pairedfile' for where readnames don't have trailing '/1' or '/2'.
+   - fastqtobam option to relax qscore validation without turning off... careful
+  Upgrade install to pull BWA 0.7.8
+   - performance improvements for short read alignment (100bp)
 1.0.1
   Upgrade install to pull biobambam 0.0.135
     - fastqtobam supports Casava v1.8

MYMETA.json

@@ -46,6 +46,7 @@
             "Module::Build" : "0.42",
             "Pod::Coverage" : "0.23",
             "Proc::ProcessTable" : "0.5",
+            "Sub::Exporter::Progressive" : "0.001011",
             "Term::UI" : "0.42",
             "Test::Fatal" : "0.013",
             "Try::Tiny" : "0.19"

MYMETA.yml

@@ -32,6 +32,7 @@ requires:
   Module::Build: 0.42
   Pod::Coverage: 0.23
   Proc::ProcessTable: 0.5
+  Sub::Exporter::Progressive: 0.001011
   Term::UI: 0.42
   Test::Fatal: 0.013
   Try::Tiny: 0.19

Makefile.PL

@@ -53,6 +53,7 @@ WriteMakefile(
                      'Devel::Cover' => 1.09,
                      'Pod::Coverage' => 0.23,
                      'Term::UI' => 0.42, # currently in core but scheduled for removal 5.17, no alternative recommended
+                     'Sub::Exporter::Progressive' => 0.001011,
                    }
 );
 

bin/bwa_mem.pl

@@ -23,7 +23,7 @@
 BEGIN {
   use Cwd qw(abs_path);
   use File::Basename;
-  push (@INC,dirname(abs_path($0)).'/../lib');
+  unshift (@INC,dirname(abs_path($0)).'/../lib');
 };
 
 use strict;
@@ -36,6 +36,7 @@ BEGIN
 use Pod::Usage qw(pod2usage);
 use List::Util qw(first);
 use Const::Fast qw(const);
+use File::Copy qw(copy);
 
 use PCAP::Cli;
 use PCAP::Bam;
@@ -51,6 +52,24 @@ BEGIN
   my $options = setup();
   $options->{'meta_set'} = PCAP::Bwa::Meta::files_to_meta($options->{'tmp'}, $options->{'raw_files'}, $options->{'sample'});
 
+  if($options->{'reference'} =~ m/\.gz$/) {
+    if(exists $options->{'index'}) {
+      my $tmp_ref = $options->{'reference'};
+      $tmp_ref =~ s/\.gz$//;
+      if(-e $tmp_ref) {
+        $options->{'decomp_ref'} = $tmp_ref;
+      }
+      else {
+        die "ERROR: When 'index' is defined you must provide a decompressed reference (colocated is sufficient)\n";
+      }
+    }
+    else {
+      $options->{'decomp_ref'} = "$options->{tmp}/decomp.fa";
+      system([0,2], "(gunzip -c $options->{reference} > $options->{decomp_ref}) >& /dev/null") unless(-e $options->{'decomp_ref'});
+      copy("$options->{reference}.fai", "$options->{tmp}/decomp.fa.fai") unless(-e "$options->{decomp_ref}.fai");
+    }
+  }
+
   my $bam_count = scalar @{$options->{'meta_set'}};
   PCAP::Bwa::bwa_mem($options) if(!exists $options->{'process'} || $options->{'process'} eq 'bwamem');
   if(!exists $options->{'process'} || $options->{'process'} eq 'mark') {

lib/PCAP.pm

@@ -40,7 +40,8 @@ const my %UPGRADE_PATH => ( '0.1.0'  => 'biobambam,samtools,bwa',
                             '0.2.99' => 'biobambam',
                             '0.3.0'  => 'biobambam',
                             '1.0.0'  => 'biobambam',
-                            '1.0.1'  => '',
+                            '1.0.1'  => 'biobambam,bwa',
+                            '1.0.2'  => '',
                           );
 
 sub license {

lib/PCAP/Bwa.pm

@@ -38,7 +38,7 @@ const my $BWA_ALN => q{ aln%s -t %s -f %s_%s.sai %s %s.%s};
 const my $BAMFASTQ => q{ exclude=QCFAIL,SECONDARY,SUPPLEMENTARY T=%s S=%s O=%s O2=%s filename=%s};
 const my $BWA_MEM => q{ mem%s -T 0 -R %s -t %s %s};
 const my $ALN_TO_SORTED => q{ sampe -P -a 1000 -r '%s' %s %s_1.sai %s_2.sai %s.%s %s.%s | %s fixmate=1 inputformat=sam level=1 tmpfile=%s_tmp O=%s_sorted.bam};
-const my $BAMSORT => q{ inputformat=sam level=1 tmpfile=%s_tmp O=%s_sorted.bam inputthreads=%s outputthreads=%s};
+const my $BAMSORT => q{ inputformat=sam level=1 tmpfile=%s_tmp O=%s_sorted.bam inputthreads=%s outputthreads=%s calmdnm=1 calmdnmrecompindetonly=1 calmdnmreference=%s};
 
 sub bwa_version {
   my $bwa = which('bwa');
@@ -115,8 +115,9 @@ sub bwa_mem {
     # uncoverable branch false
     $helpers = 2 if($options->{'threads'} > 3);
 
+    my $ref = exists $options->{'decomp_ref'} ? $options->{'decomp_ref'} : $options->{'reference'};
     my $sort = which('bamsort') || die "Unable to find 'bamsort' in path\n";
-    $sort .= sprintf $BAMSORT, File::Spec->catfile($tmp, "bamsort.$index"), $input->tstub, $helpers, $helpers;
+    $sort .= sprintf $BAMSORT, File::Spec->catfile($tmp, "bamsort.$index"), $input->tstub, $helpers, $helpers, $ref;
 
     $command .= " | $sort";
 

lib/PCAP/Threaded.pm

@@ -39,6 +39,8 @@ BEGIN {
   if($Config{useithreads}) { use threads; }
 };
 
+our $OUT_ERR = 1;
+
 sub new {
   my ($class, $max_threads) = @_;
   unless($Config{useithreads}) {
@@ -51,11 +53,25 @@ sub new {
   }
   croak "Number of threads was NAN: $max_threads" if($max_threads !~ m/^[[:digit:]]+$/xms);
   my $self = {'threads' => $max_threads,
-              'join_interval' => 1, };
+              'join_interval' => 1,};
   bless $self, $class;
   return $self;
 }
 
+sub disable_out_err {
+  $OUT_ERR = 0;
+  return $OUT_ERR;
+}
+
+sub enable_out_err {
+  $OUT_ERR = 1;
+  return $OUT_ERR;
+}
+
+sub use_out_err {
+  return $OUT_ERR;
+}
+
 sub add_function {
   my ($self, $function_name, $function_ref, $divisor) = @_;
   croak "Function $function_name has already been defined.\n" if(exists $self->{'functions'}->{$function_name}->{'code'});
@@ -157,19 +173,33 @@ sub touch_success {
 
 sub external_process_handler {
   my ($tmp, $command, @indexes) = @_;
-  my $caller = (caller(1))[3];
-  my $suffix = join q{.}, @indexes;
-  my $out = File::Spec->catfile($tmp, "$caller.$suffix.out");
-  my $err = File::Spec->catfile($tmp, "$caller.$suffix.err");
-
-  my $out_fh = IO::File->new($out, "w+");
-  my $err_fh = IO::File->new($err, "w+");
-  try {
-    warn "Starting: $command\n";
-    capture { system($command); } stdout => $out_fh, stderr => $err_fh;
-  } catch {
-    die $_ if($_);
-  };
+
+  if(&use_out_err == 0) {
+    # these may be marshalled to different files so output both
+    warn "Errors from command: $command\n";
+    print "Output from command: $command\n";
+    try {
+      system($command);
+    }
+    catch { die $_; };
+  }
+  else {
+    my $caller = (caller(1))[3];
+    my $suffix = join q{.}, @indexes;
+    my $out = File::Spec->catfile($tmp, "$caller.$suffix.out");
+    my $err = File::Spec->catfile($tmp, "$caller.$suffix.err");
+
+    my $out_fh = IO::File->new($out, "w+");
+    my $err_fh = IO::File->new($err, "w+");
+    print $err_fh "Errors from command: $command\n";
+    print $out_fh "Output from command: $command\n";
+    try {
+      capture { system($command); } stdout => $out_fh, stderr => $err_fh;
+    } catch {
+      die $_ if($_);
+    };
+  }
+
   return 1;
 }
 
@@ -226,6 +256,28 @@ callback registered in add_function.
 
 These are non-object methods which are useful to related code
 
+=head3 Configuration
+
+=over 4
+
+=item use_out_err
+
+Determines if stdout/stderr are redirected to file, by default 1/true.
+
+=item disable_out_err
+
+Prevent calls to external_process_handler from redirecting stdout/stderr to files.
+
+=item enable_out_err
+
+Enable redirect of stdout/stderr to files when calling external_process_handler.
+
+=item thread_join_interval
+
+Set/get the number of seconds to wait between thread joins.  Default 1.
+
+=back
+
 =head3 Resume Helpers
 
 These are useful methods to help you program resume from the last successfully completed step.

setup.sh

@@ -1,10 +1,10 @@
 #!/bin/bash
 
-SOURCE_BWA="https://github.com/lh3/bwa/archive/0.7.7.tar.gz"
+SOURCE_BWA="https://github.com/lh3/bwa/archive/0.7.8.tar.gz"
 SOURCE_SNAPPY="https://snappy.googlecode.com/files/snappy-1.1.1.tar.gz"
 SOURCE_IOLIB="http://downloads.sourceforge.net/project/staden/io_lib/1.13.4/io_lib-1.13.4.tar.gz"
-SOURCE_LIBMAUS="https://github.com/gt1/libmaus/archive/0.0.115-release-20140423163910.tar.gz"
-SOURCE_BIOBAMBAM="https://github.com/gt1/biobambam/archive/0.0.135-release-20140423164503.tar.gz"
+SOURCE_LIBMAUS="https://github.com/gt1/libmaus/archive/libmaus_experimental_0_0_118.tar.gz"
+SOURCE_BIOBAMBAM="https://github.com/gt1/biobambam/archive/0.0.138-release-20140501104209.tar.gz"
 SOURCE_SAMTOOLS="https://github.com/samtools/samtools/archive/0.1.19.tar.gz"
 
 done_message () {
@@ -158,12 +158,14 @@ if [[ ",$COMPILE," == *,biobambam,* ]] ; then
     echo -n " previously installed ..."
   else
     (
+      unset PERL5LIB
       get_distro "libmaus" $SOURCE_LIBMAUS
       cd $SETUP_DIR/libmaus
       autoreconf -i -f
       ./configure --prefix=$INST_PATH --with-snappy=$INST_PATH --with-io_lib=$INST_PATH
       make -j$CPU
       make -j$CPU install
+      export PERL5LIB="$PERLROOT:$PERLARCH"
       touch $SETUP_DIR/libmaus.success
     ) >>$INIT_DIR/setup.log 2>&1
   fi

t/3_external_progs.t

@@ -17,19 +17,19 @@ my %EXPECTED_VERSION = (
                         'bamcollate2'       => {
                               'get'   => q{ -h},
                               'match' => qr/This is biobambam version ([[:digit:]\.]+)\./,
-                              'version'       => ['0.0.135']},
+                              'version'       => ['0.0.138']},
                         'bammarkduplicates' => {
                               'get'   => q{ -h},
                               'match' => qr/This is biobambam version ([[:digit:]\.]+)\./,
-                              'version'       => ['0.0.135']},
+                              'version'       => ['0.0.138']},
                         'bamsort'           => {
                               'get'   => q{ -h},
                               'match' => qr/This is biobambam version ([[:digit:]\.]+)\./,
-                              'version'       => ['0.0.135']},
+                              'version'       => ['0.0.138']},
                         'bwa'           => {
                               'get'   => q{},
                               'match' => qr/Version: ([[:digit:]\.]+[[:alpha:]]?)/, # we don't care about the revision number
-                              'version'       => ['0.6.2','0.7.7']},
+                              'version'       => ['0.6.2','0.7.8']},
                         );
 
 subtest 'External programs exist on PATH' => sub {

t/pcapThreaded.t

@@ -59,6 +59,18 @@ subtest 'run checks' => sub {
 #  ok($obj->run(2, 'add_one'), 'Success on multiple interations');
 };
 
+subtest 'thread object coniguration' => sub {
+  is(&PCAP::Threaded::use_out_err, 1, 'Default value for out_err = 1');
+  is(&PCAP::Threaded::disable_out_err, 0, 'Disabling out_err returns 0');
+  is(&PCAP::Threaded::use_out_err, 0, 'Value following disable_out_err 0');
+  is(&PCAP::Threaded::enable_out_err, 1, 'Enabling out_err returns 1');
+  is(&PCAP::Threaded::use_out_err, 1, 'Value following enable_out_err 1');
+  $obj = new_ok($MODULE => [1]);
+  is($obj->thread_join_interval, 1, 'Default value for thread_join_interval = 1');
+  is($obj->thread_join_interval(2), 2, 'Changing value for thread_join_interval = 2');
+  is($obj->thread_join_interval, 2, 'Following change to 2, thread_join_interval = 2');
+};
+
 subtest 'completion utility checks' => sub {
   local $SIG{__WARN__}=sub{};
   my $dir = tempdir( CLEANUP => 1 );