diff --git a/.gitignore b/.gitignore index ef36fa0..1326e03 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ /MANIFEST.bak /install_tmp /setup.log +/reports diff --git a/Changes b/Changes index 7829a5d..aa29eac 100644 --- a/Changes +++ b/Changes @@ -1,3 +1,9 @@ +1.0.3 + Corrected issue from dynamic de-reference of hash, issue fort pre 5.14 perl + and potentially unstable in future. + Added missing project code to cv terms. + Bug-fixed upgrade path, still needs better solution. + Cleaned up messaging in Threaded module. 1.0.2 Upgrade install to pull biobambam 0.0.138 - fastqtobam option 'pairedfile' for where readnames don't have trailing '/1' or '/2'. diff --git a/MANIFEST b/MANIFEST index 6757b7b..a7a5226 100644 --- a/MANIFEST +++ b/MANIFEST @@ -29,6 +29,36 @@ share/cv_tables/ICGC/dcc_specimen_type.txt t/1_pm_compile.t t/2_pl_compile.t t/3_external_progs.t +t/data/1.fq +t/data/1_1.fq +t/data/1_2.fq +t/data/2_1.fq +t/data/3_2.fq +t/data/data.file +t/data/empty.bam +t/data/empty.bam.bas +t/data/empty.file +t/data/empty.fq +t/data/empty_r1_1.fq +t/data/empty_r1_2.fq +t/data/empty_r2_1.fq +t/data/empty_r2_2.fq +t/data/header.bam +t/data/header.sam +t/data/malformed_head.bam +t/data/malformed_head.sam +t/data/md5.bam +t/data/md5.bam.md5 +t/data/multi_sample.bam +t/data/multi_sample.sam +t/data/no_readgroups.bam +t/data/no_readgroups.sam +t/data/not_really_a.bam +t/data/paired.bam +t/data/Stats.bam +t/data/Stats.bam.bas +t/data/test.bam.bas +t/data/unpaired.bam t/pcap.t t/pcapBam.t t/pcapBamBas.t @@ -38,33 +68,3 @@ t/pcapBwaMeta.t t/pcapCli.t t/pcapSra.t t/pcapThreaded.t -testData/1.fq -testData/1_1.fq -testData/1_2.fq -testData/2_1.fq -testData/3_2.fq -testData/data.file -testData/empty.bam -testData/empty.bam.bas -testData/empty.file -testData/empty.fq -testData/empty_r1_1.fq -testData/empty_r1_2.fq -testData/empty_r2_1.fq -testData/empty_r2_2.fq -testData/header.bam -testData/header.sam -testData/malformed_head.bam -testData/malformed_head.sam -testData/md5.bam -testData/md5.bam.md5 -testData/multi_sample.bam -testData/multi_sample.sam -testData/no_readgroups.bam -testData/no_readgroups.sam -testData/not_really_a.bam -testData/paired.bam -testData/Stats.bam -testData/Stats.bam.bas -testData/test.bam.bas -testData/unpaired.bam diff --git a/MYMETA.json b/MYMETA.json index 4f62248..18cfb45 100644 --- a/MYMETA.json +++ b/MYMETA.json @@ -54,5 +54,5 @@ } }, "release_status" : "stable", - "version" : "v1.0.1" + "version" : "v1.0.3" } diff --git a/MYMETA.yml b/MYMETA.yml index edbe4ce..b1d75d8 100644 --- a/MYMETA.yml +++ b/MYMETA.yml @@ -36,4 +36,4 @@ requires: Term::UI: 0.42 Test::Fatal: 0.013 Try::Tiny: 0.19 -version: v1.0.1 +version: v1.0.3 diff --git a/bin/bwa_aln.pl b/bin/bwa_aln.pl index 04b083f..06a591d 100755 --- a/bin/bwa_aln.pl +++ b/bin/bwa_aln.pl @@ -36,6 +36,7 @@ BEGIN use Pod::Usage qw(pod2usage); use List::Util qw(first); use Const::Fast qw(const); +use File::Copy qw(move); use PCAP::Cli; use PCAP::Threaded; @@ -75,7 +76,9 @@ BEGIN } sub cleanup { - my $tmpdir = shift->{'tmp'}; + my $options = shift; + my $tmpdir = $options->{'tmp'}; + move(File::Spec->catdir($tmpdir, 'logs'), File::Spec->catdir($options->{'outdir'}, 'logs')) || die $!; remove_tree $tmpdir if(-e $tmpdir); return 0; } diff --git a/bin/bwa_mem.pl b/bin/bwa_mem.pl index dafc371..b485194 100755 --- a/bin/bwa_mem.pl +++ b/bin/bwa_mem.pl @@ -36,7 +36,7 @@ BEGIN use Pod::Usage qw(pod2usage); use List::Util qw(first); use Const::Fast qw(const); -use File::Copy qw(copy); +use File::Copy qw(copy move); use PCAP::Cli; use PCAP::Bam; @@ -79,7 +79,9 @@ BEGIN } sub cleanup { - my $tmpdir = shift->{'tmp'}; + my $options = shift; + my $tmpdir = $options->{'tmp'}; + move(File::Spec->catdir($tmpdir, 'logs'), File::Spec->catdir($options->{'outdir'}, 'logs')) || die $!; remove_tree $tmpdir if(-e $tmpdir); return 0; } diff --git a/docs.tar.gz b/docs.tar.gz index d97eb54..a57ae07 100644 Binary files a/docs.tar.gz and b/docs.tar.gz differ diff --git a/lib/PCAP.pm b/lib/PCAP.pm index 54045f4..646d8b1 100644 --- a/lib/PCAP.pm +++ b/lib/PCAP.pm @@ -23,7 +23,7 @@ package PCAP; use strict; use Const::Fast qw(const); -our $VERSION = '1.0.1'; +our $VERSION = '1.0.3'; const my $LICENSE => "################# @@ -35,13 +35,14 @@ const my $LICENSE => const my $DEFAULT_PATH => 'biobambam,samtools,bwa'; const my %UPGRADE_PATH => ( '0.1.0' => 'biobambam,samtools,bwa', '0.1.1' => 'biobambam,bwa', - '0.1.2' => 'biobambam', - '0.2.0' => 'biobambam', - '0.2.99' => 'biobambam', - '0.3.0' => 'biobambam', - '1.0.0' => 'biobambam', + '0.1.2' => 'biobambam,bwa', + '0.2.0' => 'biobambam,bwa', + '0.2.99' => 'biobambam,bwa', + '0.3.0' => 'biobambam,bwa', + '1.0.0' => 'biobambam,bwa', '1.0.1' => 'biobambam,bwa', - '1.0.2' => '', + '1.0.2' => 'biobambam,bwa', + '1.0.3' => '', ); sub license { diff --git a/lib/PCAP/Bam/Bas.pm b/lib/PCAP/Bam/Bas.pm index ef9c663..a98ab21 100644 --- a/lib/PCAP/Bam/Bas.pm +++ b/lib/PCAP/Bam/Bas.pm @@ -81,7 +81,7 @@ sub bas_keys { } sub read_groups { - return (sort keys shift->{'_data'}); + return (sort keys %{shift->{'_data'}}); } sub get { diff --git a/lib/PCAP/Threaded.pm b/lib/PCAP/Threaded.pm index 1869e55..abeb002 100644 --- a/lib/PCAP/Threaded.pm +++ b/lib/PCAP/Threaded.pm @@ -172,14 +172,24 @@ sub touch_success { } sub external_process_handler { - my ($tmp, $command, @indexes) = @_; + my ($tmp, $command_in, @indexes) = @_; + + my @commands; + if(ref $command_in eq 'ARRAY') { + @commands = @{$command_in} + } + else { + @commands = ($command_in); + } if(&use_out_err == 0) { # these may be marshalled to different files so output both - warn "Errors from command: $command\n"; - print "Output from command: $command\n"; try { - system($command); + for my $c(@commands) { + warn "\nErrors from command: $c\n\n"; + print "\nOutput from command: $c\n\n"; + system($c); + } } catch { die $_; }; } @@ -191,10 +201,12 @@ sub external_process_handler { my $out_fh = IO::File->new($out, "w+"); my $err_fh = IO::File->new($err, "w+"); - print $err_fh "Errors from command: $command\n"; - print $out_fh "Output from command: $command\n"; try { - capture { system($command); } stdout => $out_fh, stderr => $err_fh; + for my $c(@commands) { + print $err_fh "\nErrors from command: $c\n\n"; + print $out_fh "\nOutput from command: $c\n\n"; + capture { system($c); } stdout => $out_fh, stderr => $err_fh; + } } catch { die $_ if($_); }; @@ -313,11 +325,17 @@ Requires implementation of L. =item external_process_handler - PCAP::Threaded::external_process_handler(File::Spec->catdir($tmp, 'logs'), $command, $index[, $index_2...]); + PCAP::Threaded::external_process_handler($logdir, $commands, $index[, $index_2...]); + + @params logdir - Path to pre-existing log directory + @params commands - Scalar command or arr-ref of commands + @params index - Which index this is, specifically for log/err files. Wraps up command with stdout and stderr catchalls to keep the output of each threaded process separated from the script itself. Added to simplify interpretation of any issues that may occur. +If you don't want to capture stdout/stderr see . + ($index_2... may be useful for some other implementation, see L). =back diff --git a/share/cv_tables/ICGC/dcc_project_code.txt b/share/cv_tables/ICGC/dcc_project_code.txt index 873a1ad..b7b4035 100644 --- a/share/cv_tables/ICGC/dcc_project_code.txt +++ b/share/cv_tables/ICGC/dcc_project_code.txt @@ -1,72 +1,73 @@ dcc_project_code +ALL-US +BLCA-CN BLCA-US -LAML-US -LAML-KR -CLLE-ES -CMDI-UK -MALY-DE -NHLY-MX BOCA-UK -GBM-US -LGG-US -PBCA-DE -PEME-CA +BRCA-CN +BRCA-EU BRCA-FR BRCA-KR -BRCA-US -BRCA-EU -BRCA-UK BRCA-MX +BRCA-UK +BRCA-US +BTCA-SG CESC-US +CLLE-ES +CMDI-UK COAD-US COCA-CN -READ-US +EOPC-DE ESAD-UK ESCA-CN -HNSC-US -THCA-US -ORCA-IN -THCA-SA +GACA-CN +GBM-CN +GBM-US HNCA-MX +HNSC-US KIRC-US KIRP-US -RECA-EU +LAML-CN +LAML-KR +LAML-US +LGG-US LICA-CN LICA-FR +LIHC-US LINC-JP LIRI-JP -LIHC-US LUAD-US +LUCA-DE +LUSC-CN +LUSC-KR LUSC-US +MALY-DE NACA-CN +NBL-US +NHLY-MX +ORCA-IN OV-AU +OV-CN OV-US +PAAD-US PACA-AU PACA-CA +PACA-CN PACA-IT -PAAD-US -EOPC-DE +PAEN-AU +PBCA-DE +PEME-CA PRAD-CA -PRAD-US +PRAD-CN PRAD-UK -SKCM-US -STAD-US -GACA-CN -UCEC-US -ALL-US -PAEN-AU -LUCA-DE +PRAD-US PRCA-FR -SKCA-BR -BLCA-CN -LAML-CN -GBM-CN -BRCA-CN -LUSC-CN -OV-CN -PACA-CN -PRAD-CN +READ-US RECA-CN +RECA-EU +SKCA-BR +SKCM-US +STAD-US THCA-CN -LUSC-KR -NBL-US +THCA-SA +THCA-US +UCEC-US diff --git a/testData/1.fq b/t/data/1.fq similarity index 100% rename from testData/1.fq rename to t/data/1.fq diff --git a/testData/1_1.fq b/t/data/1_1.fq similarity index 100% rename from testData/1_1.fq rename to t/data/1_1.fq diff --git a/testData/1_2.fq b/t/data/1_2.fq similarity index 100% rename from testData/1_2.fq rename to t/data/1_2.fq diff --git a/testData/2_1.fq b/t/data/2_1.fq similarity index 100% rename from testData/2_1.fq rename to t/data/2_1.fq diff --git a/testData/3_2.fq b/t/data/3_2.fq similarity index 100% rename from testData/3_2.fq rename to t/data/3_2.fq diff --git a/testData/Stats.bam b/t/data/Stats.bam similarity index 100% rename from testData/Stats.bam rename to t/data/Stats.bam diff --git a/testData/Stats.bam.bas b/t/data/Stats.bam.bas similarity index 100% rename from testData/Stats.bam.bas rename to t/data/Stats.bam.bas diff --git a/testData/data.file b/t/data/data.file similarity index 100% rename from testData/data.file rename to t/data/data.file diff --git a/testData/empty.bam b/t/data/empty.bam similarity index 100% rename from testData/empty.bam rename to t/data/empty.bam diff --git a/testData/empty.bam.bas b/t/data/empty.bam.bas similarity index 100% rename from testData/empty.bam.bas rename to t/data/empty.bam.bas diff --git a/testData/empty.file b/t/data/empty.file similarity index 100% rename from testData/empty.file rename to t/data/empty.file diff --git a/testData/empty.fq b/t/data/empty.fq similarity index 100% rename from testData/empty.fq rename to t/data/empty.fq diff --git a/testData/empty_r1_1.fq b/t/data/empty_r1_1.fq similarity index 100% rename from testData/empty_r1_1.fq rename to t/data/empty_r1_1.fq diff --git a/testData/empty_r1_2.fq b/t/data/empty_r1_2.fq similarity index 100% rename from testData/empty_r1_2.fq rename to t/data/empty_r1_2.fq diff --git a/testData/empty_r2_1.fq b/t/data/empty_r2_1.fq similarity index 100% rename from testData/empty_r2_1.fq rename to t/data/empty_r2_1.fq diff --git a/testData/empty_r2_2.fq b/t/data/empty_r2_2.fq similarity index 100% rename from testData/empty_r2_2.fq rename to t/data/empty_r2_2.fq diff --git a/testData/header.bam b/t/data/header.bam similarity index 100% rename from testData/header.bam rename to t/data/header.bam diff --git a/testData/header.sam b/t/data/header.sam similarity index 100% rename from testData/header.sam rename to t/data/header.sam diff --git a/testData/malformed_head.bam b/t/data/malformed_head.bam similarity index 100% rename from testData/malformed_head.bam rename to t/data/malformed_head.bam diff --git a/testData/malformed_head.sam b/t/data/malformed_head.sam similarity index 100% rename from testData/malformed_head.sam rename to t/data/malformed_head.sam diff --git a/testData/md5.bam b/t/data/md5.bam similarity index 100% rename from testData/md5.bam rename to t/data/md5.bam diff --git a/testData/md5.bam.md5 b/t/data/md5.bam.md5 similarity index 100% rename from testData/md5.bam.md5 rename to t/data/md5.bam.md5 diff --git a/testData/multi_sample.bam b/t/data/multi_sample.bam similarity index 100% rename from testData/multi_sample.bam rename to t/data/multi_sample.bam diff --git a/testData/multi_sample.sam b/t/data/multi_sample.sam similarity index 100% rename from testData/multi_sample.sam rename to t/data/multi_sample.sam diff --git a/testData/no_readgroups.bam b/t/data/no_readgroups.bam similarity index 100% rename from testData/no_readgroups.bam rename to t/data/no_readgroups.bam diff --git a/testData/no_readgroups.sam b/t/data/no_readgroups.sam similarity index 100% rename from testData/no_readgroups.sam rename to t/data/no_readgroups.sam diff --git a/testData/not_really_a.bam b/t/data/not_really_a.bam similarity index 100% rename from testData/not_really_a.bam rename to t/data/not_really_a.bam diff --git a/testData/paired.bam b/t/data/paired.bam similarity index 100% rename from testData/paired.bam rename to t/data/paired.bam diff --git a/testData/test.bam.bas b/t/data/test.bam.bas similarity index 100% rename from testData/test.bam.bas rename to t/data/test.bam.bas diff --git a/testData/unpaired.bam b/t/data/unpaired.bam similarity index 100% rename from testData/unpaired.bam rename to t/data/unpaired.bam diff --git a/t/pcapBam.t b/t/pcapBam.t index 02c4bfb..d31c93d 100644 --- a/t/pcapBam.t +++ b/t/pcapBam.t @@ -40,7 +40,7 @@ const my $EXPECTED_MULTI_RG => [ {'CN' => 'SANGER', } ]; use FindBin qw($Bin); -my $test_data = "$Bin/../testData"; +my $test_data = "$Bin/data"; # if you need to change the header.bam # edit header.sam then: diff --git a/t/pcapBamBas.t b/t/pcapBamBas.t index 7b9187b..ec42bb9 100644 --- a/t/pcapBamBas.t +++ b/t/pcapBamBas.t @@ -11,7 +11,7 @@ const my $EXP_MEDIAN => '462.000'; const my $RG_ORDER => [qw(1 2 3 4 5 6)]; use FindBin qw($Bin); -my $test_data = "$Bin/../testData"; +my $test_data = "$Bin/data"; my $bas = File::Spec->catfile($test_data, 'test.bam.bas'); my $empty_bas = File::Spec->catfile($test_data, 'empty.bam.bas'); diff --git a/t/pcapBamStats.t b/t/pcapBamStats.t index b918d11..983fd61 100644 --- a/t/pcapBamStats.t +++ b/t/pcapBamStats.t @@ -30,7 +30,7 @@ use Bio::DB::Sam; const my $MODULE => 'PCAP::Bam::Stats'; -my $test_data = "$Bin/../testData"; +my $test_data = "$Bin/data"; my $test_bam_file = join('/',$test_data,'Stats.bam'); my $test_bas_file = join('/',$test_data,'Stats.bam.bas'); diff --git a/t/pcapBwaMeta.t b/t/pcapBwaMeta.t index 2d58e71..407acba 100644 --- a/t/pcapBwaMeta.t +++ b/t/pcapBwaMeta.t @@ -6,7 +6,7 @@ use File::Spec; use FindBin qw($Bin); use File::Temp qw(tempdir); -my $test_data = "$Bin/../testData"; +my $test_data = "$Bin/data"; const my $MODULE => 'PCAP::Bwa::Meta'; const my $REF_INIT => { 'in' => 'somefile', @@ -18,9 +18,6 @@ const my $RG_STRING => qr/\@RG\\tID:[a-z0-9]{8}\-[a-z0-9]{4}\-[a-z0-9]{4}\-[a-z0 const my $RG_PRINT => qr/\@RG\tID:[a-z0-9]{8}\-[a-z0-9]{4}\-[a-z0-9]{4}\-[a-z0-9]{4}\-[a-z0-9]{12}\tCN:SANGER\tDS:short\tLB:wobble\tPI:500\tPL:HiSeq\tPU:1_1\tSM:wibble/; const my @VALID_FASTQ_EXT => qw(fastq fq fastq.gz fq.gz); - -#my $fastq_paired_one = File::Spec->catfile($test_data, 'test_1.fastq'); - my $bail_out = 0; subtest 'Initialisation checks' => sub { diff --git a/t/pcapCli.t b/t/pcapCli.t index 95563c3..33705a1 100644 --- a/t/pcapCli.t +++ b/t/pcapCli.t @@ -13,7 +13,7 @@ use constant MODULE => 'PCAP::Cli'; use_ok(MODULE); use FindBin qw($Bin); -my $test_data = "$Bin/../testData"; +my $test_data = "$Bin/data"; subtest 'file_for_reading' => sub {