From 50f180846d5792ae70975669220b591fbf7eeee5 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 24 Apr 2024 08:45:39 +0200 Subject: [PATCH 1/7] Replace hardcoded URL to test data with parameter --- CHANGELOG.md | 1 + nf_core/pipeline-template/conf/test.config | 2 +- nf_core/pipeline-template/conf/test_full.config | 4 ++-- nf_core/pipeline-template/nextflow.config | 1 + nf_core/pipeline-template/nextflow_schema.json | 6 ++++++ 5 files changed, 11 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4ae1e115d..761aaf1863 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ - Update python to 3.12 ([#2805](https://github.com/nf-core/tools/pull/2805)) - Remove `pyproject.toml` from template root - Shorten lines in pipeline template ([#2908](https://github.com/nf-core/tools/pull/2908)) +- Add a new hidden `--test_data_base_path` parameter to more easily switch locations of test data in test configs (#2900)[] ### Linting diff --git a/nf_core/pipeline-template/conf/test.config b/nf_core/pipeline-template/conf/test.config index 32b9619ebb..f9f4218cb8 100644 --- a/nf_core/pipeline-template/conf/test.config +++ b/nf_core/pipeline-template/conf/test.config @@ -22,7 +22,7 @@ params { // Input data // TODO nf-core: Specify the paths to your test data on nf-core/test-datasets // TODO nf-core: Give any required params for the test so that command line flags are not needed - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/samplesheet/samplesheet_test_illumina_amplicon.csv' + input = test_data_base_path + 'viralrecon/samplesheet/samplesheet_test_illumina_amplicon.csv' {% if igenomes -%} // Genome references diff --git a/nf_core/pipeline-template/conf/test_full.config b/nf_core/pipeline-template/conf/test_full.config index d92692fa94..93db73de26 100644 --- a/nf_core/pipeline-template/conf/test_full.config +++ b/nf_core/pipeline-template/conf/test_full.config @@ -17,13 +17,13 @@ params { // Input data for full size test // TODO nf-core: Specify the paths to your full test data ( on nf-core/test-datasets or directly in repositories, e.g. SRA) // TODO nf-core: Give any required params for the test so that command line flags are not needed - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/samplesheet/samplesheet_full_illumina_amplicon.csv' + input = test_data_base_path + 'viralrecon/samplesheet/samplesheet_full_illumina_amplicon.csv' {% if igenomes -%} // Genome references genome = 'R64-1-1' {%- else -%} // Fasta references - fasta = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/GCF_009858895.2_ASM985889v3_genomic.200409.fna.gz' + fasta = test_data_base_path + 'viralrecon/genome/NC_045512.2/GCF_009858895.2_ASM985889v3_genomic.200409.fna.gz' {%- endif %} } diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 2f9b349916..1da1e7a96b 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -37,6 +37,7 @@ params { hook_url = null help = false version = false + test_data_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/' // Config options config_profile_name = null diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 77b1a7a070..56d07c764e 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -267,6 +267,12 @@ "description": "Validation of parameters in lenient more.", "hidden": true, "help_text": "Allows string values that are parseable as numbers or booleans. For further information see [JSONSchema docs](https://github.com/everit-org/json-schema#lenient-mode)." + }, + "test_data_base_path":{ + "type": "string", + "fa_icon": "far fa-check-circle", + "description": "Base URL to location of pipeline test dataset files", + "hidden": true, } } } From 8dca26f47d7b31b7abe8460703b105702dad68cc Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 24 Apr 2024 08:57:32 +0200 Subject: [PATCH 2/7] Apply suggestions from code review --- nf_core/pipeline-template/nextflow_schema.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 56d07c764e..533989f3f8 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -268,10 +268,10 @@ "hidden": true, "help_text": "Allows string values that are parseable as numbers or booleans. For further information see [JSONSchema docs](https://github.com/everit-org/json-schema#lenient-mode)." }, - "test_data_base_path":{ + "test_data_base_path": { "type": "string", "fa_icon": "far fa-check-circle", - "description": "Base URL to location of pipeline test dataset files", + "description": "Base URL or local path to location of pipeline test dataset files", "hidden": true, } } From cb4a775d79ea8156a4d3b8372a506ccd6350f686 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 24 Apr 2024 12:50:32 +0200 Subject: [PATCH 3/7] Apply suggestions from code review Co-authored-by: Maxime U Garcia --- CHANGELOG.md | 2 +- nf_core/pipeline-template/conf/test.config | 2 +- nf_core/pipeline-template/conf/test_full.config | 4 ++-- nf_core/pipeline-template/nextflow.config | 2 +- nf_core/pipeline-template/nextflow_schema.json | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 761aaf1863..63d41a258c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ - Update python to 3.12 ([#2805](https://github.com/nf-core/tools/pull/2805)) - Remove `pyproject.toml` from template root - Shorten lines in pipeline template ([#2908](https://github.com/nf-core/tools/pull/2908)) -- Add a new hidden `--test_data_base_path` parameter to more easily switch locations of test data in test configs (#2900)[] +- Add a new hidden `--pipelines_testdata_base_path` parameter to more easily switch locations of test data in test configs (#2931)[https://github.com/nf-core/tools/pull/2931] ### Linting diff --git a/nf_core/pipeline-template/conf/test.config b/nf_core/pipeline-template/conf/test.config index f9f4218cb8..cd12404c8e 100644 --- a/nf_core/pipeline-template/conf/test.config +++ b/nf_core/pipeline-template/conf/test.config @@ -22,7 +22,7 @@ params { // Input data // TODO nf-core: Specify the paths to your test data on nf-core/test-datasets // TODO nf-core: Give any required params for the test so that command line flags are not needed - input = test_data_base_path + 'viralrecon/samplesheet/samplesheet_test_illumina_amplicon.csv' + input = pipelines_testdata_base_path + 'viralrecon/samplesheet/samplesheet_test_illumina_amplicon.csv' {% if igenomes -%} // Genome references diff --git a/nf_core/pipeline-template/conf/test_full.config b/nf_core/pipeline-template/conf/test_full.config index 93db73de26..dac47b9867 100644 --- a/nf_core/pipeline-template/conf/test_full.config +++ b/nf_core/pipeline-template/conf/test_full.config @@ -17,13 +17,13 @@ params { // Input data for full size test // TODO nf-core: Specify the paths to your full test data ( on nf-core/test-datasets or directly in repositories, e.g. SRA) // TODO nf-core: Give any required params for the test so that command line flags are not needed - input = test_data_base_path + 'viralrecon/samplesheet/samplesheet_full_illumina_amplicon.csv' + input = pipelines_testdata_base_path + 'viralrecon/samplesheet/samplesheet_full_illumina_amplicon.csv' {% if igenomes -%} // Genome references genome = 'R64-1-1' {%- else -%} // Fasta references - fasta = test_data_base_path + 'viralrecon/genome/NC_045512.2/GCF_009858895.2_ASM985889v3_genomic.200409.fna.gz' + fasta = pipelines_testdata_base_path + 'viralrecon/genome/NC_045512.2/GCF_009858895.2_ASM985889v3_genomic.200409.fna.gz' {%- endif %} } diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 1da1e7a96b..377c2964b9 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -37,7 +37,7 @@ params { hook_url = null help = false version = false - test_data_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/' + pipelines_testdata_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/' // Config options config_profile_name = null diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 533989f3f8..8c687c74a1 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -268,7 +268,7 @@ "hidden": true, "help_text": "Allows string values that are parseable as numbers or booleans. For further information see [JSONSchema docs](https://github.com/everit-org/json-schema#lenient-mode)." }, - "test_data_base_path": { + "pipelines_testdata_base_path": { "type": "string", "fa_icon": "far fa-check-circle", "description": "Base URL or local path to location of pipeline test dataset files", From 60911aba63917b0900449254e4dbdc5ab13c3959 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 24 Apr 2024 12:51:11 +0200 Subject: [PATCH 4/7] =?UTF-8?q?Harshil=20Align=E2=84=A2=EF=B8=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- nf_core/pipeline-template/nextflow.config | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 377c2964b9..4452207ccc 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -28,16 +28,16 @@ params { multiqc_methods_description = null // Boilerplate options - outdir = null - publish_dir_mode = 'copy' - email = null - email_on_fail = null - plaintext_email = false - monochrome_logs = false - hook_url = null - help = false - version = false - pipelines_testdata_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/' + outdir = null + publish_dir_mode = 'copy' + email = null + email_on_fail = null + plaintext_email = false + monochrome_logs = false + hook_url = null + help = false + version = false + pipelines_testdata_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/' // Config options config_profile_name = null From 720852e76561dd0a3262f04bf8713ca15f0b76fe Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 24 Apr 2024 13:00:02 +0200 Subject: [PATCH 5/7] Fix JSON typo --- nf_core/pipeline-template/nextflow_schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 8c687c74a1..01bff71f7d 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -272,7 +272,7 @@ "type": "string", "fa_icon": "far fa-check-circle", "description": "Base URL or local path to location of pipeline test dataset files", - "hidden": true, + "hidden": true } } } From f08f0e49296105b4b4a201e37723eb691a57d45a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Tue, 7 May 2024 15:16:02 +0200 Subject: [PATCH 6/7] Apply suggestions from code review Co-authored-by: Maxime U Garcia --- nf_core/pipeline-template/conf/test.config | 2 +- nf_core/pipeline-template/conf/test_full.config | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/conf/test.config b/nf_core/pipeline-template/conf/test.config index cd12404c8e..827e21b7b7 100644 --- a/nf_core/pipeline-template/conf/test.config +++ b/nf_core/pipeline-template/conf/test.config @@ -22,7 +22,7 @@ params { // Input data // TODO nf-core: Specify the paths to your test data on nf-core/test-datasets // TODO nf-core: Give any required params for the test so that command line flags are not needed - input = pipelines_testdata_base_path + 'viralrecon/samplesheet/samplesheet_test_illumina_amplicon.csv' + input = params.pipelines_testdata_base_path + 'viralrecon/samplesheet/samplesheet_test_illumina_amplicon.csv' {% if igenomes -%} // Genome references diff --git a/nf_core/pipeline-template/conf/test_full.config b/nf_core/pipeline-template/conf/test_full.config index dac47b9867..aa3b70c1e5 100644 --- a/nf_core/pipeline-template/conf/test_full.config +++ b/nf_core/pipeline-template/conf/test_full.config @@ -17,13 +17,13 @@ params { // Input data for full size test // TODO nf-core: Specify the paths to your full test data ( on nf-core/test-datasets or directly in repositories, e.g. SRA) // TODO nf-core: Give any required params for the test so that command line flags are not needed - input = pipelines_testdata_base_path + 'viralrecon/samplesheet/samplesheet_full_illumina_amplicon.csv' + input = params.pipelines_testdata_base_path + 'viralrecon/samplesheet/samplesheet_full_illumina_amplicon.csv' {% if igenomes -%} // Genome references genome = 'R64-1-1' {%- else -%} // Fasta references - fasta = pipelines_testdata_base_path + 'viralrecon/genome/NC_045512.2/GCF_009858895.2_ASM985889v3_genomic.200409.fna.gz' + fasta = params.pipelines_testdata_base_path + 'viralrecon/genome/NC_045512.2/GCF_009858895.2_ASM985889v3_genomic.200409.fna.gz' {%- endif %} } From d4c8f3620493b5384e0f199cb6fc12e612cd864e Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 7 May 2024 15:28:30 +0200 Subject: [PATCH 7/7] add default value to schema --- nf_core/pipeline-template/nextflow_schema.json | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 01bff71f7d..ae7c0b715f 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -272,6 +272,7 @@ "type": "string", "fa_icon": "far fa-check-circle", "description": "Base URL or local path to location of pipeline test dataset files", + "default": "https://raw.githubusercontent.com/nf-core/test-datasets/", "hidden": true } }