diff --git a/Changes b/Changes index 69a0e01..7829a5d 100644 --- a/Changes +++ b/Changes @@ -1,3 +1,9 @@ +1.0.2 + Upgrade install to pull biobambam 0.0.138 + - fastqtobam option 'pairedfile' for where readnames don't have trailing '/1' or '/2'. + - fastqtobam option to relax qscore validation without turning off... careful + Upgrade install to pull BWA 0.7.8 + - performance improvements for short read alignment (100bp) 1.0.1 Upgrade install to pull biobambam 0.0.135 - fastqtobam supports Casava v1.8 diff --git a/MYMETA.json b/MYMETA.json index 62c3f4d..4f62248 100644 --- a/MYMETA.json +++ b/MYMETA.json @@ -46,6 +46,7 @@ "Module::Build" : "0.42", "Pod::Coverage" : "0.23", "Proc::ProcessTable" : "0.5", + "Sub::Exporter::Progressive" : "0.001011", "Term::UI" : "0.42", "Test::Fatal" : "0.013", "Try::Tiny" : "0.19" diff --git a/MYMETA.yml b/MYMETA.yml index 645eed5..edbe4ce 100644 --- a/MYMETA.yml +++ b/MYMETA.yml @@ -32,6 +32,7 @@ requires: Module::Build: 0.42 Pod::Coverage: 0.23 Proc::ProcessTable: 0.5 + Sub::Exporter::Progressive: 0.001011 Term::UI: 0.42 Test::Fatal: 0.013 Try::Tiny: 0.19 diff --git a/Makefile.PL b/Makefile.PL index 3a9cd73..b2f31b6 100644 --- a/Makefile.PL +++ b/Makefile.PL @@ -53,6 +53,7 @@ WriteMakefile( 'Devel::Cover' => 1.09, 'Pod::Coverage' => 0.23, 'Term::UI' => 0.42, # currently in core but scheduled for removal 5.17, no alternative recommended + 'Sub::Exporter::Progressive' => 0.001011, } ); diff --git a/bin/bwa_mem.pl b/bin/bwa_mem.pl index e046487..dafc371 100755 --- a/bin/bwa_mem.pl +++ b/bin/bwa_mem.pl @@ -23,7 +23,7 @@ BEGIN { use Cwd qw(abs_path); use File::Basename; - push (@INC,dirname(abs_path($0)).'/../lib'); + unshift (@INC,dirname(abs_path($0)).'/../lib'); }; use strict; @@ -36,6 +36,7 @@ BEGIN use Pod::Usage qw(pod2usage); use List::Util qw(first); use Const::Fast qw(const); +use File::Copy qw(copy); use PCAP::Cli; use PCAP::Bam; @@ -51,6 +52,24 @@ BEGIN my $options = setup(); $options->{'meta_set'} = PCAP::Bwa::Meta::files_to_meta($options->{'tmp'}, $options->{'raw_files'}, $options->{'sample'}); + if($options->{'reference'} =~ m/\.gz$/) { + if(exists $options->{'index'}) { + my $tmp_ref = $options->{'reference'}; + $tmp_ref =~ s/\.gz$//; + if(-e $tmp_ref) { + $options->{'decomp_ref'} = $tmp_ref; + } + else { + die "ERROR: When 'index' is defined you must provide a decompressed reference (colocated is sufficient)\n"; + } + } + else { + $options->{'decomp_ref'} = "$options->{tmp}/decomp.fa"; + system([0,2], "(gunzip -c $options->{reference} > $options->{decomp_ref}) >& /dev/null") unless(-e $options->{'decomp_ref'}); + copy("$options->{reference}.fai", "$options->{tmp}/decomp.fa.fai") unless(-e "$options->{decomp_ref}.fai"); + } + } + my $bam_count = scalar @{$options->{'meta_set'}}; PCAP::Bwa::bwa_mem($options) if(!exists $options->{'process'} || $options->{'process'} eq 'bwamem'); if(!exists $options->{'process'} || $options->{'process'} eq 'mark') { diff --git a/docs.tar.gz b/docs.tar.gz index 5af691e..d97eb54 100644 Binary files a/docs.tar.gz and b/docs.tar.gz differ diff --git a/lib/PCAP.pm b/lib/PCAP.pm index cb06d6a..54045f4 100644 --- a/lib/PCAP.pm +++ b/lib/PCAP.pm @@ -40,7 +40,8 @@ const my %UPGRADE_PATH => ( '0.1.0' => 'biobambam,samtools,bwa', '0.2.99' => 'biobambam', '0.3.0' => 'biobambam', '1.0.0' => 'biobambam', - '1.0.1' => '', + '1.0.1' => 'biobambam,bwa', + '1.0.2' => '', ); sub license { diff --git a/lib/PCAP/Bwa.pm b/lib/PCAP/Bwa.pm index 410d36c..aab99fa 100644 --- a/lib/PCAP/Bwa.pm +++ b/lib/PCAP/Bwa.pm @@ -38,7 +38,7 @@ const my $BWA_ALN => q{ aln%s -t %s -f %s_%s.sai %s %s.%s}; const my $BAMFASTQ => q{ exclude=QCFAIL,SECONDARY,SUPPLEMENTARY T=%s S=%s O=%s O2=%s filename=%s}; const my $BWA_MEM => q{ mem%s -T 0 -R %s -t %s %s}; const my $ALN_TO_SORTED => q{ sampe -P -a 1000 -r '%s' %s %s_1.sai %s_2.sai %s.%s %s.%s | %s fixmate=1 inputformat=sam level=1 tmpfile=%s_tmp O=%s_sorted.bam}; -const my $BAMSORT => q{ inputformat=sam level=1 tmpfile=%s_tmp O=%s_sorted.bam inputthreads=%s outputthreads=%s}; +const my $BAMSORT => q{ inputformat=sam level=1 tmpfile=%s_tmp O=%s_sorted.bam inputthreads=%s outputthreads=%s calmdnm=1 calmdnmrecompindetonly=1 calmdnmreference=%s}; sub bwa_version { my $bwa = which('bwa'); @@ -115,8 +115,9 @@ sub bwa_mem { # uncoverable branch false $helpers = 2 if($options->{'threads'} > 3); + my $ref = exists $options->{'decomp_ref'} ? $options->{'decomp_ref'} : $options->{'reference'}; my $sort = which('bamsort') || die "Unable to find 'bamsort' in path\n"; - $sort .= sprintf $BAMSORT, File::Spec->catfile($tmp, "bamsort.$index"), $input->tstub, $helpers, $helpers; + $sort .= sprintf $BAMSORT, File::Spec->catfile($tmp, "bamsort.$index"), $input->tstub, $helpers, $helpers, $ref; $command .= " | $sort"; diff --git a/lib/PCAP/Threaded.pm b/lib/PCAP/Threaded.pm index 90417c7..1869e55 100644 --- a/lib/PCAP/Threaded.pm +++ b/lib/PCAP/Threaded.pm @@ -39,6 +39,8 @@ BEGIN { if($Config{useithreads}) { use threads; } }; +our $OUT_ERR = 1; + sub new { my ($class, $max_threads) = @_; unless($Config{useithreads}) { @@ -51,11 +53,25 @@ sub new { } croak "Number of threads was NAN: $max_threads" if($max_threads !~ m/^[[:digit:]]+$/xms); my $self = {'threads' => $max_threads, - 'join_interval' => 1, }; + 'join_interval' => 1,}; bless $self, $class; return $self; } +sub disable_out_err { + $OUT_ERR = 0; + return $OUT_ERR; +} + +sub enable_out_err { + $OUT_ERR = 1; + return $OUT_ERR; +} + +sub use_out_err { + return $OUT_ERR; +} + sub add_function { my ($self, $function_name, $function_ref, $divisor) = @_; croak "Function $function_name has already been defined.\n" if(exists $self->{'functions'}->{$function_name}->{'code'}); @@ -157,19 +173,33 @@ sub touch_success { sub external_process_handler { my ($tmp, $command, @indexes) = @_; - my $caller = (caller(1))[3]; - my $suffix = join q{.}, @indexes; - my $out = File::Spec->catfile($tmp, "$caller.$suffix.out"); - my $err = File::Spec->catfile($tmp, "$caller.$suffix.err"); - - my $out_fh = IO::File->new($out, "w+"); - my $err_fh = IO::File->new($err, "w+"); - try { - warn "Starting: $command\n"; - capture { system($command); } stdout => $out_fh, stderr => $err_fh; - } catch { - die $_ if($_); - }; + + if(&use_out_err == 0) { + # these may be marshalled to different files so output both + warn "Errors from command: $command\n"; + print "Output from command: $command\n"; + try { + system($command); + } + catch { die $_; }; + } + else { + my $caller = (caller(1))[3]; + my $suffix = join q{.}, @indexes; + my $out = File::Spec->catfile($tmp, "$caller.$suffix.out"); + my $err = File::Spec->catfile($tmp, "$caller.$suffix.err"); + + my $out_fh = IO::File->new($out, "w+"); + my $err_fh = IO::File->new($err, "w+"); + print $err_fh "Errors from command: $command\n"; + print $out_fh "Output from command: $command\n"; + try { + capture { system($command); } stdout => $out_fh, stderr => $err_fh; + } catch { + die $_ if($_); + }; + } + return 1; } @@ -226,6 +256,28 @@ callback registered in add_function. These are non-object methods which are useful to related code +=head3 Configuration + +=over 4 + +=item use_out_err + +Determines if stdout/stderr are redirected to file, by default 1/true. + +=item disable_out_err + +Prevent calls to external_process_handler from redirecting stdout/stderr to files. + +=item enable_out_err + +Enable redirect of stdout/stderr to files when calling external_process_handler. + +=item thread_join_interval + +Set/get the number of seconds to wait between thread joins. Default 1. + +=back + =head3 Resume Helpers These are useful methods to help you program resume from the last successfully completed step. diff --git a/setup.sh b/setup.sh index 398df29..0d176f3 100755 --- a/setup.sh +++ b/setup.sh @@ -1,10 +1,10 @@ #!/bin/bash -SOURCE_BWA="https://github.com/lh3/bwa/archive/0.7.7.tar.gz" +SOURCE_BWA="https://github.com/lh3/bwa/archive/0.7.8.tar.gz" SOURCE_SNAPPY="https://snappy.googlecode.com/files/snappy-1.1.1.tar.gz" SOURCE_IOLIB="http://downloads.sourceforge.net/project/staden/io_lib/1.13.4/io_lib-1.13.4.tar.gz" -SOURCE_LIBMAUS="https://github.com/gt1/libmaus/archive/0.0.115-release-20140423163910.tar.gz" -SOURCE_BIOBAMBAM="https://github.com/gt1/biobambam/archive/0.0.135-release-20140423164503.tar.gz" +SOURCE_LIBMAUS="https://github.com/gt1/libmaus/archive/libmaus_experimental_0_0_118.tar.gz" +SOURCE_BIOBAMBAM="https://github.com/gt1/biobambam/archive/0.0.138-release-20140501104209.tar.gz" SOURCE_SAMTOOLS="https://github.com/samtools/samtools/archive/0.1.19.tar.gz" done_message () { @@ -158,12 +158,14 @@ if [[ ",$COMPILE," == *,biobambam,* ]] ; then echo -n " previously installed ..." else ( + unset PERL5LIB get_distro "libmaus" $SOURCE_LIBMAUS cd $SETUP_DIR/libmaus autoreconf -i -f ./configure --prefix=$INST_PATH --with-snappy=$INST_PATH --with-io_lib=$INST_PATH make -j$CPU make -j$CPU install + export PERL5LIB="$PERLROOT:$PERLARCH" touch $SETUP_DIR/libmaus.success ) >>$INIT_DIR/setup.log 2>&1 fi diff --git a/t/3_external_progs.t b/t/3_external_progs.t index 9e752bb..b91903a 100644 --- a/t/3_external_progs.t +++ b/t/3_external_progs.t @@ -17,19 +17,19 @@ my %EXPECTED_VERSION = ( 'bamcollate2' => { 'get' => q{ -h}, 'match' => qr/This is biobambam version ([[:digit:]\.]+)\./, - 'version' => ['0.0.135']}, + 'version' => ['0.0.138']}, 'bammarkduplicates' => { 'get' => q{ -h}, 'match' => qr/This is biobambam version ([[:digit:]\.]+)\./, - 'version' => ['0.0.135']}, + 'version' => ['0.0.138']}, 'bamsort' => { 'get' => q{ -h}, 'match' => qr/This is biobambam version ([[:digit:]\.]+)\./, - 'version' => ['0.0.135']}, + 'version' => ['0.0.138']}, 'bwa' => { 'get' => q{}, 'match' => qr/Version: ([[:digit:]\.]+[[:alpha:]]?)/, # we don't care about the revision number - 'version' => ['0.6.2','0.7.7']}, + 'version' => ['0.6.2','0.7.8']}, ); subtest 'External programs exist on PATH' => sub { diff --git a/t/pcapThreaded.t b/t/pcapThreaded.t index 5e44248..75ebbd6 100644 --- a/t/pcapThreaded.t +++ b/t/pcapThreaded.t @@ -59,6 +59,18 @@ subtest 'run checks' => sub { # ok($obj->run(2, 'add_one'), 'Success on multiple interations'); }; +subtest 'thread object coniguration' => sub { + is(&PCAP::Threaded::use_out_err, 1, 'Default value for out_err = 1'); + is(&PCAP::Threaded::disable_out_err, 0, 'Disabling out_err returns 0'); + is(&PCAP::Threaded::use_out_err, 0, 'Value following disable_out_err 0'); + is(&PCAP::Threaded::enable_out_err, 1, 'Enabling out_err returns 1'); + is(&PCAP::Threaded::use_out_err, 1, 'Value following enable_out_err 1'); + $obj = new_ok($MODULE => [1]); + is($obj->thread_join_interval, 1, 'Default value for thread_join_interval = 1'); + is($obj->thread_join_interval(2), 2, 'Changing value for thread_join_interval = 2'); + is($obj->thread_join_interval, 2, 'Following change to 2, thread_join_interval = 2'); +}; + subtest 'completion utility checks' => sub { local $SIG{__WARN__}=sub{}; my $dir = tempdir( CLEANUP => 1 );