implement new min_pvalue report table

Dockerfile

@@ -20,7 +20,7 @@ RUN mkdir -p /conda-envs/6e056d31662ab0bd2fd3fba49416042f
 COPY workflow/envs/bgzip_tabix.yaml /conda-envs/6e056d31662ab0bd2fd3fba49416042f/environment.yaml
 
 # Conda environment:
-#   source: workflow/envs/create_inflation_factors_table.yaml
+#   source: workflow/envs/create_report_table.yaml
 #   prefix: /conda-envs/a160f42d06f9d24b41c5cbece52b682d
 #   name: create_inflation_factors_table
 #   channels:
@@ -32,7 +32,7 @@ COPY workflow/envs/bgzip_tabix.yaml /conda-envs/6e056d31662ab0bd2fd3fba49416042f
 #     - pip:
 #         - click==8.1.7
 RUN mkdir -p /conda-envs/a160f42d06f9d24b41c5cbece52b682d
-COPY workflow/envs/create_inflation_factors_table.yaml /conda-envs/a160f42d06f9d24b41c5cbece52b682d/environment.yaml
+COPY workflow/envs/create_report_table.yaml /conda-envs/a160f42d06f9d24b41c5cbece52b682d/environment.yaml
 
 # Conda environment:
 #   source: workflow/envs/plink-pandas.yml

config/config_annotate_sumstats.yml

@@ -1,11 +1,13 @@
 ---
 # common parameters
-n_cores: &cores 8
+n_cores: &cores 4
 
 # This is an ordered dict that contains the order in which the steps are run.
 run_sequence: !!omap
   #- 1: 'basic_check'
-  - 2: 'write_tsv'
+  - 2: 'sort_alphabetically'
+  - 3: 'write_tsv'
+
 
 steps:
   basic_check: # see https://cloufield.github.io/gwaslab/Standardization/
@@ -19,6 +21,11 @@ steps:
         fixsep: True
         forcefixid: False
         overwrite: False
+  sort_alphabetically:
+    params:
+      run: True
+    gl_params:
+      n_cores: *cores
   write_tsv:
     params:
       run: True
@@ -40,5 +47,4 @@ filename_sep: '.'
 # IO
 root_path: "results"
 log_filename: "gwaspipe.log"
-report_if_filename: "report_if_placeholder.txt"
 formatbook_path: "workflow/scripts/gwaspipe/data/formatbook.json"     # fill this var if you want to provide a custom formatbook

config/config_convert_sumstats_to_metal.yml

@@ -1,6 +1,6 @@
 ---
 # common parameters
-n_cores: &cores 3
+n_cores: &cores 4
 
 # This is an ordered dict that contains the order in which the steps are run.
 run_sequence: !!omap
@@ -30,5 +30,4 @@ filename_sep: '.'
 # IO
 root_path: "results"
 log_filename: "gwaspipe.log"
-report_if_filename: "report_if_placeholder.txt"
 formatbook_path: " "     # fill this var if you want to provide a custom formatbook

config/config_convert_sumstats_to_vcf.yml

@@ -1,6 +1,6 @@
 ---
 # common parameters
-n_cores: &cores 3
+n_cores: &cores 4
 
 # This is an ordered dict that contains the order in which the steps are run.
 run_sequence: !!omap
@@ -27,5 +27,4 @@ filename_sep: '.'
 # IO
 root_path: "results"
 log_filename: "gwaspipe.log"
-report_if_filename: "report_if_placeholder.txt"
 formatbook_path: " "     # fill this var if you want to provide a custom formatbook

config/config_harmonize_sumstats.yml

@@ -5,15 +5,14 @@ n_cores: &cores 4
 # This is an ordered dict that contains the order in which the steps are run.
 run_sequence: !!omap
   - 1: 'basic_check'
-  - 2: 'fill_data'
-  - 3: 'infer_build'
+  - 2: 'infer_build'
+  - 3: 'fill_data'
   - 4: 'harmonize'
-  - 5: 'sort_alphabetically'
-  - 6: 'write_pickle'
-  - 7: 'write_tsv'
-  - 8: 'report_harmonization_summary'
-  - 9: 'report_inflation_factors'
-  - 10: 'qq_manhattan_plots'
+  - 5: 'write_pickle'
+  - 6: 'report_harmonization_summary'
+  - 7: 'report_inflation_factors'
+  - 8: 'qq_manhattan_plots'
+
 
 steps:
   basic_check: # see https://cloufield.github.io/gwaslab/Standardization/
@@ -29,7 +28,7 @@ steps:
     params:
       run: True
     gl_params:
-      to_fill: ['MLOG10P', 'P', 'Z']
+      to_fill: ['MLOG10P']
       overwrite: False
       extreme: True
   harmonize: # see https://cloufield.github.io/gwaslab/Harmonization/
@@ -38,11 +37,11 @@ steps:
     gl_params:
       basic_check: False
       n_cores: *cores
-  sort_alphabetically:
+  write_pickle:
     params:
       run: True
-    gl_params:
-      n_cores: *cores
+      overwrite: False
+      workspace: "pickle"
   report_harmonization_summary:
     params:
       run: True
@@ -59,29 +58,12 @@ steps:
       dist: 2
     gl_params:
       mode: 'mqq'
-      stratified : True
-      maf_bins: [[0, 0.01], [0.01, 0.05], [0.05, 1]]
+      stratified: True
+      maf_bins: [ [ 0, 0.01 ], [ 0.01, 0.05 ], [ 0.05, 1 ] ]
       scaled: True
       build: "19"
       skip: 1
       sig_level: 5e-8
-  write_pickle:
-    params:
-      run: True
-      overwrite: False
-      workspace: "pickle"
-  write_tsv:
-    params:
-      run: True
-      workspace: "outputs"
-      workspace_subfolder: True
-    gl_params:
-      fmt: "gwaslab"
-      to_csvargs:
-        compression:
-          method: "gzip"
-          compresslevel: 1
-          mtime: 1
 
 
 # Filename transformation, e.g.: seq.3007.7.gwas.regenie.gz
@@ -91,5 +73,4 @@ filename_sep: '.'
 # IO
 root_path: "results"
 log_filename: "gwaspipe.log"
-report_if_filename: "report_if_placeholder.txt"
 formatbook_path: "workflow/scripts/gwaspipe/data/formatbook.json"     # fill this var if you want to provide a custom formatbook

config/config_munge_sumstats.yml

@@ -1,6 +1,6 @@
 ---
 # common parameters
-n_cores: &cores 3
+n_cores: &cores 4
 
 # This is an ordered dict that contains the order in which the steps are run.
 run_sequence: !!omap
@@ -30,5 +30,4 @@ filename_sep: '.'
 # IO
 root_path: "results"
 log_filename: "gwaspipe.log"
-report_if_filename: "report_if_placeholder.txt"
 formatbook_path: " "     # fill this var if you want to provide a custom formatbook

config/config_save_min_pvalue.yml

@@ -1,6 +1,6 @@
 ---
 # common parameters
-n_cores: &cores 3
+n_cores: &cores 4
 
 # This is an ordered dict that contains the order in which the steps are run.
 run_sequence: !!omap
@@ -12,14 +12,13 @@ steps:
     params:
       run: True
     gl_params:
-      to_fill: ['MLOG10P', 'P', 'Z']
+      to_fill: ['MLOG10P', 'Z']
       overwrite: False
       extreme: True
   report_min_pvalues:
     params:
       run: True
       workspace: "min_P"
-      nrows: 20
 
 
 # Filename transformation, e.g.: seq.3007.7.gwas.regenie.gz
@@ -30,5 +29,4 @@ filename_sep: '.'
 # IO
 root_path: "results"
 log_filename: "gwaspipe.log"
-report_if_filename: "report_if_placeholder.txt"
 formatbook_path: " "     # fill this var if you want to provide a custom formatbook

workflow/rules/common.smk

@@ -50,12 +50,7 @@ def get_final_output():
         final_output.append(ws_path("inflation_factors_table.tsv"))
 
     if config.get("run").get("save_min_pvalue"):
-        final_output.extend(
-            expand(
-                ws_path("min_P/{seqid}.nsmallest.tsv"),
-                seqid=analytes.seqid,
-            )
-        )
+        final_output.append(ws_path("min_pvalue_table.tsv"))
 
     if config.get("run").get("annotation"):
         final_output.extend(

workflow/rules/harmonization.smk

@@ -4,7 +4,7 @@ rule harmonize_sumstats:
     output:
         ws_path("pickle/{seqid}.pkl"),
         #ws_path("outputs/{seqid}/{seqid}.regenie.tsv.gz"),
-        ws_path("if/report_if_{seqid}.txt"),
+        temp(ws_path("if/{seqid}.if.txt")),
         ws_path("plots/{seqid}.png"),
     conda:
         "../scripts/gwaspipe/environment.yml"
@@ -24,22 +24,22 @@ rule harmonize_sumstats:
 
 rule create_if_table:
     input:
-        expand(ws_path("if/report_if_{seqid}.txt"), seqid=analytes.seqid),
+        expand(ws_path("if/{seqid}.if.txt"), seqid=analytes.seqid),
     output:
         ws_path("inflation_factors_table.tsv"),
     conda:
-        "../envs/create_inflation_factors_table.yaml"
+        "../envs/create_report_table.yaml"
     params:
         input_path=ws_path("if"),
     shell:
-        "python workflow/scripts/create_inflation_factors_table.py -i {params.input_path} -o {output}"
+        "python workflow/scripts/create_report_table.py -i {params.input_path} -o {output}"
 
 
 rule save_min_pvalue:
     input:
         ws_path("pickle/{seqid}.pkl"),
     output:
-        ws_path("min_P/{seqid}.nsmallest.tsv"),
+        temp(ws_path("min_P/{seqid}.nlargest.txt")),
     conda:
         "../scripts/gwaspipe/environment.yml"
     params:
@@ -54,3 +54,16 @@ rule save_min_pvalue:
         "-c {params.config_file} "
         "-i {input} "
         "-o {params.output_path}"
+
+
+rule create_min_pvalue_table:
+    input:
+        expand(ws_path("min_P/{seqid}.nlargest.txt"), seqid=analytes.seqid),
+    output:
+        ws_path("min_pvalue_table.tsv"),
+    conda:
+        "../envs/create_report_table.yaml"
+    params:
+        input_path=ws_path("min_P"),
+    shell:
+        "python workflow/scripts/create_report_table.py -i {params.input_path} -o {output}"