Merge pull request #176 from CCBR/python-paths

fix: upgrade python-based docker containers to use versions with PYTHONPATH unset

CHANGELOG.md

@@ -4,8 +4,9 @@
 
 - Consolidate `build_cluster.json` into `cluster.json`. (#167, @kelly-sovacool)
 - Remove unused miniconda container. (#168, @kelly-sovacool)
-- Fix error in RSEQC rules. (#172, @kelly-sovacool)
 - Fix error in `renee build` where `image_cache()` wasn't imported. (#178, @kelly-sovacool)
+- Upgrade containers to fix $PYTHONPATH clashes in `fc_lane`, `inner_distance`, `rseqc`, `tin`, `tin_merge`, and `rnaseq_multiqc`. (#172, #176, @kelly-sovacool)
+- Fix error that prevented existing output directories to be re-run. (#176, @kelly-sovacool)
 
 ## RENEE 2.6.1
 

config/containers/images.json

@@ -2,17 +2,17 @@
     "images": {
         "arriba": "docker://nciccbr/ccbr_arriba_2.0.0:v0.0.1",
         "bam2strandedbw": "docker://nciccbr/ccbr_bam2strandedbw:v0.0.1",
+        "base": "docker://nciccbr/ccbr_ubuntu_22.04:v4",
         "bbtools": "docker://nciccbr/ccbr_bbtools_38.87:v0.0.1",
         "build_rnaseq": "docker://nciccbr/ccbr_build_rnaseq:v0.0.1",
         "cutadapt": "docker://nciccbr/ccbr_cutadapt_1.18:v032219",
         "fastq_screen": "docker://nciccbr/ccbr_fastq_screen_0.13.0:v2.0",
         "fastqc": "docker://nciccbr/ccbr_fastqc_0.11.9:v1.1",
         "fastqvalidator": "docker://nciccbr/ccbr_fastqvalidator:v0.1.0",
         "kraken": "docker://nciccbr/ccbr_kraken_v2.1.1:v0.0.1",
-        "multiqc": "docker://skchronicles/multiqc:v0.1.0",
+        "multiqc": "docker://nciccbr/ccbr_multiqc_1.15:v2",
         "picard": "docker://nciccbr/ccbr_picard:v0.0.1",
         "preseq": "docker://nciccbr/ccbr_preseq:v0.0.1",
-        "python": "docker://nciccbr/ccbr_python:v0.0.1",
         "qualimap": "docker://nciccbr/ccbr_qualimap:v0.0.1",
         "rna": "docker://nciccbr/ccbr_rna:v0.0.1",
         "rsem": "docker://nciccbr/ccbr_rsem_1.3.3:v1.0",

src/renee/orchestrate.py

@@ -4,6 +4,7 @@
 from ccbr_tools.pipeline.util import (
     get_hpcname,
     get_tmp_dir,
+    _get_file_mtime,
 )
 from ccbr_tools.pipeline.cache import get_singularity_cachedir
 

workflow/rules/common.smk

@@ -27,7 +27,7 @@ rule fc_lane:
         rname='pl:fc_lane',
         get_flowcell_lanes=join("workflow", "scripts", "get_flowcell_lanes.py"),
     envmodules: config['bin'][pfamily]['tool_versions']['PYTHONVER']
-    container: config['images']['python']
+    container: config['images']['base']
     shell: """
     python {params.get_flowcell_lanes} {input.R1} {wildcards.name} > {output.fqinfo}
     """
@@ -201,7 +201,7 @@ rule rsem_merge:
         pythonscript=join("workflow", "scripts", "merge_rsem_results.py"),
         inputdir=join(workpath, degall_dir)
     envmodules: config['bin'][pfamily]['tool_versions']['PYTHONVER'],
-    container: config['images']['python']
+    container: config['images']['base']
     shell: """
     python {params.pythonscript} {params.annotate} {params.inputdir} {params.inputdir}
     sed 's/\\t/|/1' {output.gene_counts_matrix} | \
@@ -301,7 +301,7 @@ rule tin_merge:
         rname="pl:tin_merge",
         create_matrix=join("workflow", "scripts", "create_tin_matrix.py")
     envmodules: config['bin'][pfamily]['tool_versions']['PYTHONVER'],
-    container: config['images']['python']
+    container: config['images']['base']
     shell: """
     python {params.create_matrix} {input.tins} > {output.matrix}
     """

workflow/scripts/get_flowcell_lanes.py

@@ -45,19 +45,6 @@ def usage(message="", exitcode=0):
     sys.exit(exitcode)
 
 
-def reader(fname):
-    """Returns correct file object handler or reader for gzipped
-    or non-gzipped FastQ files based on the file extension. Assumes
-    gzipped files endwith the '.gz' extension.
-    """
-    if fname.endswith(".gz"):
-        # Opens up file with gzip handler
-        return gzip.open
-    else:
-        # Opens up file normal, uncompressed handler
-        return open
-
-
 def get_flowcell_lane(sequence_identifier):
     """Returns flowcell and lane information for different fastq formats.
     FastQ files generated with older versions of Casava or downloaded from
@@ -137,10 +124,9 @@ def md5sum(filename, blocksize=65536):
     md5 = md5sum(filename)
 
     # Get Flowcell and Lane information
-    handle = reader(filename)
     meta = {"flowcell": [], "lane": [], "flowcell_lane": []}
     i = 0  # keeps track of line number
-    with handle(filename, "r") as file:
+    with gzip.open(filename, "rt") as file:
         print(
             "sample_name\ttotal_read_pairs\tflowcell_ids\tlanes\tflowcell_lanes\tmd5_checksum"
         )