From 87201dbbd0765198e6a8734d5abb6ad1b14997c6 Mon Sep 17 00:00:00 2001 From: VincentBeaud Date: Tue, 26 Nov 2024 10:14:00 -0500 Subject: [PATCH 01/10] Generalize data_per_streamline operations --- scripts/scil_tractogram_add_dps.py | 83 ------------------ scripts/scil_tractogram_dps_math.py | 125 ++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+), 83 deletions(-) delete mode 100755 scripts/scil_tractogram_add_dps.py create mode 100755 scripts/scil_tractogram_dps_math.py diff --git a/scripts/scil_tractogram_add_dps.py b/scripts/scil_tractogram_add_dps.py deleted file mode 100755 index b688efe0b..000000000 --- a/scripts/scil_tractogram_add_dps.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -""" Add information to each streamline from a file. Can be for example -SIFT2 weights, processing information, bundle IDs, etc. - -Output must be a .trk otherwise the data will be lost. -""" - -import argparse -import logging - -from dipy.io.streamline import save_tractogram -import numpy as np - -from scilpy.io.streamlines import load_tractogram_with_reference -from scilpy.io.utils import (add_overwrite_arg, - add_reference_arg, - add_verbose_arg, - assert_inputs_exist, - assert_outputs_exist, - check_tract_trk, - load_matrix_in_any_format) - - -def _build_arg_parser(): - - p = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawTextHelpFormatter) - - p.add_argument('in_tractogram', - help='Input tractogram (.trk or .tck).') - p.add_argument('in_dps_file', - help='File containing the data to add to streamlines.') - p.add_argument('dps_key', - help='Where to store the data in the tractogram.') - p.add_argument('out_tractogram', - help='Output tractogram (.trk).') - - add_reference_arg(p) - add_verbose_arg(p) - add_overwrite_arg(p) - - return p - - -def main(): - parser = _build_arg_parser() - args = parser.parse_args() - logging.getLogger().setLevel(logging.getLevelName(args.verbose)) - - # I/O assertions - assert_inputs_exist(parser, [args.in_tractogram, args.in_dps_file], - args.reference) - assert_outputs_exist(parser, args, args.out_tractogram) - check_tract_trk(parser, args.out_tractogram) - - # Load tractogram - sft = load_tractogram_with_reference(parser, args, args.in_tractogram) - - # Make sure the user is not unwillingly overwritting dps - if (args.dps_key in sft.get_data_per_streamline_keys() and - not args.overwrite): - parser.error('"{}" already in data per streamline. Use -f to force ' - 'overwriting.'.format(args.dps_key)) - - # Load data and remove extraneous dimensions - data = np.squeeze(load_matrix_in_any_format(args.in_dps_file)) - - # Quick check as the built-in error from sft is not too explicit - if len(sft) != data.shape[0]: - raise ValueError('Data must have as many entries ({}) as there are' - ' streamlines ({}).'.format(data.shape[0], len(sft))) - # Add data to tractogram - sft.data_per_streamline[args.dps_key] = data - - # Save the new sft - save_tractogram(sft, args.out_tractogram) - - -if __name__ == '__main__': - main() diff --git a/scripts/scil_tractogram_dps_math.py b/scripts/scil_tractogram_dps_math.py new file mode 100755 index 000000000..d284b9ec8 --- /dev/null +++ b/scripts/scil_tractogram_dps_math.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +Add, extract or delete information to each streamline from a tractogram +file. Can be for example SIFT2 weights, processing information, bundle IDs, +etc. + +Input and output tractograms must always be .trk to simplify filetype checks +for each operation. +""" + +import os +import argparse +import logging + +from dipy.io.streamline import save_tractogram, load_tractogram +import numpy as np + +from scilpy.io.utils import (add_overwrite_arg, + add_verbose_arg, + assert_inputs_exist, + assert_outputs_exist, + check_tract_trk, + load_matrix_in_any_format, + save_matrix_in_any_format) + + +def _build_arg_parser(): + + p = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawTextHelpFormatter) + + p.add_argument('in_tractogram', + help='Input tractogram (.trk).') + p.add_argument('operation', metavar='OPERATION', + choices=['add', 'delete', 'export'], + help='The type of operation to be performed on the\n' + 'tractogram\'s data_per_streamline. Must be one of\n' + 'the following: [%(choices)s]. The arguments\n' + 'required for each operation are specified under\n' + 'each group below.') + + add_args = p.add_argument_group('Add operation', + 'Requires the out_tractogram argument.') + add_args.add_argument('--add_dps_file', + help='File containing the data to add to\n' + 'streamlines.') + add_args.add_argument('--add_dps_key', type=str, + help='Where to store the data in the tractogram.') + + delete_args = p.add_argument_group('Delete operation', + 'Requires the out_tractogram argument.') + delete_args.add_argument('--delete_dps_key', type=str, + help='Where to find the data to be deleted, in\n' + 'the tractogram.') + + p.add_argument('--out_tractogram', + help='Output tractogram (.trk). Required for any mutation.') + + export_args = p.add_argument_group('Export operation') + export_args.add_argument('--export_dps_key', type=str, + help='Where to find the data to be exported,\n' + 'in the tractogram.') + export_args.add_argument('--export_dps_file', + help='File in which the extracted data will be\n' + 'saved.') + + add_verbose_arg(p) + add_overwrite_arg(p) + + return p + + +def main(): + parser = _build_arg_parser() + args = parser.parse_args() + logging.getLogger().setLevel(logging.getLevelName(args.verbose)) + + check_tract_trk(parser, args.in_tractogram) + if args.out_tractogram: + check_tract_trk(parser, args.out_tractogram) + + # I/O assertions + assert_inputs_exist(parser, args.in_tractogram, args.add_dps_file) + assert_outputs_exist(parser, args, [], optional=[args.export_dps_file, + args.out_tractogram]) + + sft = load_tractogram(args.in_tractogram, 'same') + + if args.operation == 'add': + # Make sure the user is not unwillingly overwritting dps + if (args.add_dps_key in sft.get_data_per_streamline_keys() and + not args.overwrite): + parser.error('"{}" already in data per streamline. Use -f to force' + ' overwriting.'.format(args.add_dps_key)) + + # Load data and remove extraneous dimensions + data = np.squeeze(load_matrix_in_any_format(args.add_dps_file)) + + # Quick check as the built-in error from sft is not too explicit + if len(sft) != data.shape[0]: + raise ValueError('Data must have as many entries ({}) as there are' + ' streamlines ({}).'.format(data.shape[0], + len(sft))) + + sft.data_per_streamline[args.add_dps_key] = data + + save_tractogram(sft, args.out_tractogram) + + if args.operation == 'delete': + del sft.data_per_streamline[args.delete_dps_key] + + save_tractogram(sft, args.out_tractogram) + + if args.operation == 'export': + # Extract data and reshape + data = np.squeeze(sft.data_per_streamline[args.export_dps_key]) + + save_matrix_in_any_format(args.export_dps_file, data) + + +if __name__ == '__main__': + main() From bb96b054591fa555c15948dbff10de44175ba19c Mon Sep 17 00:00:00 2001 From: VincentBeaud Date: Tue, 26 Nov 2024 14:25:51 -0500 Subject: [PATCH 02/10] tests for scil_tractogram_dps_math.py --- scripts/scil_tractogram_dps_math.py | 6 +- scripts/tests/test_tractogram_add_dps.py | 77 ----------- scripts/tests/test_tractogram_dps_math.py | 151 ++++++++++++++++++++++ 3 files changed, 156 insertions(+), 78 deletions(-) delete mode 100644 scripts/tests/test_tractogram_add_dps.py create mode 100644 scripts/tests/test_tractogram_dps_math.py diff --git a/scripts/scil_tractogram_dps_math.py b/scripts/scil_tractogram_dps_math.py index d284b9ec8..187532d20 100755 --- a/scripts/scil_tractogram_dps_math.py +++ b/scripts/scil_tractogram_dps_math.py @@ -116,8 +116,12 @@ def main(): if args.operation == 'export': # Extract data and reshape - data = np.squeeze(sft.data_per_streamline[args.export_dps_key]) + if not args.export_dps_key in sft.data_per_streamline.keys(): + raise ValueError('Data does not have any data_per_streamline' + ' entry stored at this key: {}' + .format(args.export_dps_key)) + data = np.squeeze(sft.data_per_streamline[args.export_dps_key]) save_matrix_in_any_format(args.export_dps_file, data) diff --git a/scripts/tests/test_tractogram_add_dps.py b/scripts/tests/test_tractogram_add_dps.py deleted file mode 100644 index b5e7f05d2..000000000 --- a/scripts/tests/test_tractogram_add_dps.py +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -import numpy as np -import os -import tempfile - -from dipy.io.streamline import load_tractogram - -from scilpy import SCILPY_HOME -from scilpy.io.fetcher import fetch_data, get_testing_files_dict - -# If they already exist, this only takes 5 seconds (check md5sum) -fetch_data(get_testing_files_dict(), keys=['filtering.zip']) -tmp_dir = tempfile.TemporaryDirectory() - - -def test_help_option(script_runner): - ret = script_runner.run('scil_tractogram_add_dps.py', - '--help') - assert ret.success - - -def test_execution_add_dps(script_runner, monkeypatch): - monkeypatch.chdir(os.path.expanduser(tmp_dir.name)) - in_bundle = os.path.join(SCILPY_HOME, 'filtering', - 'bundle_4.trk') - sft = load_tractogram(in_bundle, 'same') - filename = 'vals.npy' - outname = 'out.trk' - np.save(filename, np.arange(len(sft))) - ret = script_runner.run('scil_tractogram_add_dps.py', - in_bundle, filename, 'key', outname, '-f') - assert ret.success - - -def test_execution_add_dps_missing_vals(script_runner, monkeypatch): - monkeypatch.chdir(os.path.expanduser(tmp_dir.name)) - in_bundle = os.path.join(SCILPY_HOME, 'filtering', - 'bundle_4.trk') - sft = load_tractogram(in_bundle, 'same') - filename = 'vals.npy' - outname = 'out.trk' - np.save(filename, np.arange(len(sft) - 10)) - ret = script_runner.run('scil_tractogram_add_dps.py', - in_bundle, filename, 'key', outname, '-f') - assert ret.stderr - - -def test_execution_add_dps_existing_key(script_runner, monkeypatch): - monkeypatch.chdir(os.path.expanduser(tmp_dir.name)) - in_bundle = os.path.join(SCILPY_HOME, 'filtering', - 'bundle_4.trk') - sft = load_tractogram(in_bundle, 'same') - filename = 'vals.npy' - outname = 'out.trk' - outname2 = 'out_2.trk' - np.save(filename, np.arange(len(sft))) - ret = script_runner.run('scil_tractogram_add_dps.py', - in_bundle, filename, 'key', outname, '-f') - assert ret.success - ret = script_runner.run('scil_tractogram_add_dps.py', - outname, filename, 'key', outname2) - assert not ret.success - - -def test_execution_add_dps_tck(script_runner, monkeypatch): - monkeypatch.chdir(os.path.expanduser(tmp_dir.name)) - in_bundle = os.path.join(SCILPY_HOME, 'filtering', - 'bundle_4.trk') - sft = load_tractogram(in_bundle, 'same') - filename = 'vals.npy' - outname = 'out.tck' - np.save(filename, np.arange(len(sft))) - ret = script_runner.run('scil_tractogram_add_dps.py', - in_bundle, filename, 'key', outname, '-f') - assert not ret.success diff --git a/scripts/tests/test_tractogram_dps_math.py b/scripts/tests/test_tractogram_dps_math.py new file mode 100644 index 000000000..531fb8459 --- /dev/null +++ b/scripts/tests/test_tractogram_dps_math.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import numpy as np +import os +import tempfile + +from dipy.io.streamline import load_tractogram, save_tractogram + +from scilpy import SCILPY_HOME +from scilpy.io.fetcher import fetch_data, get_testing_files_dict + +# If they already exist, this only takes 5 seconds (check md5sum) +fetch_data(get_testing_files_dict(), keys=['filtering.zip']) +tmp_dir = tempfile.TemporaryDirectory() + + +def test_help_option(script_runner): + ret = script_runner.run('scil_tractogram_dps_math.py', + '--help') + assert ret.success + + +def test_execution_dps_math_add(script_runner, monkeypatch): + monkeypatch.chdir(os.path.expanduser(tmp_dir.name)) + in_bundle = os.path.join(SCILPY_HOME, 'filtering', + 'bundle_4.trk') + sft = load_tractogram(in_bundle, 'same') + filename = 'vals.npy' + outname = 'out.trk' + np.save(filename, np.arange(len(sft))) + ret = script_runner.run('scil_tractogram_dps_math.py', in_bundle, 'add', + '--add_dps_file', filename, + '--add_dps_key', 'key', + '--out_tractogram', outname, + '-f') + assert ret.success + + +def test_execution_dps_math_add_with_missing_vals(script_runner, monkeypatch): + monkeypatch.chdir(os.path.expanduser(tmp_dir.name)) + in_bundle = os.path.join(SCILPY_HOME, 'filtering', + 'bundle_4.trk') + sft = load_tractogram(in_bundle, 'same') + filename = 'vals.npy' + outname = 'out.trk' + np.save(filename, np.arange(len(sft) - 10)) + ret = script_runner.run('scil_tractogram_dps_math.py', in_bundle, 'add', + '--add_dps_file', filename, + '--add_dps_key', 'key', + '--out_tractogram', outname, + '-f') + assert ret.stderr + + +def test_execution_dps_math_add_with_existing_key(script_runner, monkeypatch): + monkeypatch.chdir(os.path.expanduser(tmp_dir.name)) + in_bundle = os.path.join(SCILPY_HOME, 'filtering', + 'bundle_4.trk') + sft = load_tractogram(in_bundle, 'same') + filename = 'vals.npy' + outname = 'out.trk' + outname2 = 'out_2.trk' + np.save(filename, np.arange(len(sft))) + ret = script_runner.run('scil_tractogram_dps_math.py', in_bundle, 'add', + '--add_dps_file', filename, + '--add_dps_key', 'key', + '--out_tractogram', outname, + '-f') + assert ret.success + ret = script_runner.run('scil_tractogram_dps_math.py', outname, 'add', + '--add_dps_file', filename, + '--add_dps_key', 'key', + '--out_tractogram', outname2,) + assert not ret.success + + +def test_execution_dps_math_tck(script_runner, monkeypatch): + monkeypatch.chdir(os.path.expanduser(tmp_dir.name)) + in_bundle = os.path.join(SCILPY_HOME, 'filtering', + 'bundle_4.trk') + sft = load_tractogram(in_bundle, 'same') + filename = 'vals.npy' + outname = 'out.tck' + np.save(filename, np.arange(len(sft))) + ret = script_runner.run('scil_tractogram_dps_math.py', in_bundle, 'add', + '--add_dps_file', filename, + '--add_dps_key', 'key', + '--out_tractogram', outname, + '-f') + assert not ret.success + + +def test_execution_dps_math_delete(script_runner, monkeypatch): + monkeypatch.chdir(os.path.expanduser(tmp_dir.name)) + in_bundle_no_key = os.path.join(SCILPY_HOME, 'filtering', + 'bundle_4.trk') + in_bundle = 'bundle_4.trk' + sft = load_tractogram(in_bundle_no_key, 'same') + sft.data_per_streamline = { + "key": [0] * len(sft) + } + save_tractogram(sft, in_bundle) + outname = 'out.trk' + ret = script_runner.run('scil_tractogram_dps_math.py', in_bundle, 'delete', + '--delete_dps_key', 'key', + '--out_tractogram', outname, + '-f') + assert ret.success + + +def test_execution_dps_math_delete_no_key(script_runner, monkeypatch): + monkeypatch.chdir(os.path.expanduser(tmp_dir.name)) + in_bundle = os.path.join(SCILPY_HOME, 'filtering', + 'bundle_4.trk') + outname = 'out.trk' + ret = script_runner.run('scil_tractogram_dps_math.py', in_bundle, 'delete', + '--delete_dps_key', 'key', + '--out_tractogram', outname, + '-f') + assert not ret.success + + +def test_execution_dps_math_export(script_runner, monkeypatch): + monkeypatch.chdir(os.path.expanduser(tmp_dir.name)) + in_bundle_no_key = os.path.join(SCILPY_HOME, 'filtering', + 'bundle_4.trk') + in_bundle = 'bundle_4.trk' + sft = load_tractogram(in_bundle_no_key, 'same') + sft.data_per_streamline = { + "key": [0] * len(sft) + } + save_tractogram(sft, in_bundle) + filename = 'out.txt' + ret = script_runner.run('scil_tractogram_dps_math.py', in_bundle, 'export', + '--export_dps_key', 'key', + '--export_dps_file', filename, + '-f') + assert ret.success + + +def test_execution_dps_math_export_no_key(script_runner, monkeypatch): + monkeypatch.chdir(os.path.expanduser(tmp_dir.name)) + in_bundle = os.path.join(SCILPY_HOME, 'filtering', + 'bundle_4.trk') + filename = 'out.txt' + ret = script_runner.run('scil_tractogram_dps_math.py', in_bundle, 'export', + '--export_dps_key', 'key', + '--export_dps_file', filename, + '-f') + assert not ret.success From 3883e1daec16e6698edaef3c4000872ded642185 Mon Sep 17 00:00:00 2001 From: VincentBeaud Date: Mon, 2 Dec 2024 12:27:02 -0500 Subject: [PATCH 03/10] Pep8 fix --- scripts/scil_tractogram_dps_math.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/scil_tractogram_dps_math.py b/scripts/scil_tractogram_dps_math.py index 187532d20..3cd142461 100755 --- a/scripts/scil_tractogram_dps_math.py +++ b/scripts/scil_tractogram_dps_math.py @@ -4,7 +4,7 @@ """ Add, extract or delete information to each streamline from a tractogram file. Can be for example SIFT2 weights, processing information, bundle IDs, -etc. +tracking seeds, etc. Input and output tractograms must always be .trk to simplify filetype checks for each operation. @@ -42,11 +42,14 @@ def _build_arg_parser(): 'required for each operation are specified under\n' 'each group below.') + p.add_argument('--out_tractogram', + help='Output tractogram (.trk). Required for any mutation.') + add_args = p.add_argument_group('Add operation', 'Requires the out_tractogram argument.') add_args.add_argument('--add_dps_file', help='File containing the data to add to\n' - 'streamlines.') + 'streamlines (.txt, .npy or .mat).') add_args.add_argument('--add_dps_key', type=str, help='Where to store the data in the tractogram.') @@ -56,16 +59,13 @@ def _build_arg_parser(): help='Where to find the data to be deleted, in\n' 'the tractogram.') - p.add_argument('--out_tractogram', - help='Output tractogram (.trk). Required for any mutation.') - export_args = p.add_argument_group('Export operation') export_args.add_argument('--export_dps_key', type=str, help='Where to find the data to be exported,\n' 'in the tractogram.') export_args.add_argument('--export_dps_file', help='File in which the extracted data will be\n' - 'saved.') + 'saved (.txt or .npy).') add_verbose_arg(p) add_overwrite_arg(p) @@ -116,10 +116,10 @@ def main(): if args.operation == 'export': # Extract data and reshape - if not args.export_dps_key in sft.data_per_streamline.keys(): + if args.export_dps_key not in sft.data_per_streamline.keys(): raise ValueError('Data does not have any data_per_streamline' ' entry stored at this key: {}' - .format(args.export_dps_key)) + .format(args.export_dps_key)) data = np.squeeze(sft.data_per_streamline[args.export_dps_key]) save_matrix_in_any_format(args.export_dps_file, data) From da0c8eb64509e244f8612057e8af2e4099b9270a Mon Sep 17 00:00:00 2001 From: VincentBeaud Date: Mon, 2 Dec 2024 16:06:55 -0500 Subject: [PATCH 04/10] Make dps_key mandatory + tests --- scripts/scil_tractogram_dps_math.py | 37 ++++++++++------------- scripts/tests/test_tractogram_dps_math.py | 36 +++++++++++----------- 2 files changed, 34 insertions(+), 39 deletions(-) diff --git a/scripts/scil_tractogram_dps_math.py b/scripts/scil_tractogram_dps_math.py index 3cd142461..30432b093 100755 --- a/scripts/scil_tractogram_dps_math.py +++ b/scripts/scil_tractogram_dps_math.py @@ -37,10 +37,13 @@ def _build_arg_parser(): p.add_argument('operation', metavar='OPERATION', choices=['add', 'delete', 'export'], help='The type of operation to be performed on the\n' - 'tractogram\'s data_per_streamline. Must be one of\n' - 'the following: [%(choices)s]. The arguments\n' - 'required for each operation are specified under\n' - 'each group below.') + 'tractogram\'s data_per_streamline at the given\n' + 'key. Must be one of the following: [%(choices)s].\n' + 'The additional arguments required for each\n' + 'operation are specified under each group below.') + p.add_argument('dps_key', type=str, + help='Where to find the data to be exported,\n' + 'in the tractogram.') p.add_argument('--out_tractogram', help='Output tractogram (.trk). Required for any mutation.') @@ -50,19 +53,11 @@ def _build_arg_parser(): add_args.add_argument('--add_dps_file', help='File containing the data to add to\n' 'streamlines (.txt, .npy or .mat).') - add_args.add_argument('--add_dps_key', type=str, - help='Where to store the data in the tractogram.') - delete_args = p.add_argument_group('Delete operation', - 'Requires the out_tractogram argument.') - delete_args.add_argument('--delete_dps_key', type=str, - help='Where to find the data to be deleted, in\n' - 'the tractogram.') + _ = p.add_argument_group('Delete operation', + 'Requires the out_tractogram argument.') export_args = p.add_argument_group('Export operation') - export_args.add_argument('--export_dps_key', type=str, - help='Where to find the data to be exported,\n' - 'in the tractogram.') export_args.add_argument('--export_dps_file', help='File in which the extracted data will be\n' 'saved (.txt or .npy).') @@ -91,10 +86,10 @@ def main(): if args.operation == 'add': # Make sure the user is not unwillingly overwritting dps - if (args.add_dps_key in sft.get_data_per_streamline_keys() and + if (args.dps_key in sft.get_data_per_streamline_keys() and not args.overwrite): parser.error('"{}" already in data per streamline. Use -f to force' - ' overwriting.'.format(args.add_dps_key)) + ' overwriting.'.format(args.dps_key)) # Load data and remove extraneous dimensions data = np.squeeze(load_matrix_in_any_format(args.add_dps_file)) @@ -105,23 +100,23 @@ def main(): ' streamlines ({}).'.format(data.shape[0], len(sft))) - sft.data_per_streamline[args.add_dps_key] = data + sft.data_per_streamline[args.dps_key] = data save_tractogram(sft, args.out_tractogram) if args.operation == 'delete': - del sft.data_per_streamline[args.delete_dps_key] + del sft.data_per_streamline[args.dps_key] save_tractogram(sft, args.out_tractogram) if args.operation == 'export': # Extract data and reshape - if args.export_dps_key not in sft.data_per_streamline.keys(): + if args.dps_key not in sft.data_per_streamline.keys(): raise ValueError('Data does not have any data_per_streamline' ' entry stored at this key: {}' - .format(args.export_dps_key)) + .format(args.dps_key)) - data = np.squeeze(sft.data_per_streamline[args.export_dps_key]) + data = np.squeeze(sft.data_per_streamline[args.dps_key]) save_matrix_in_any_format(args.export_dps_file, data) diff --git a/scripts/tests/test_tractogram_dps_math.py b/scripts/tests/test_tractogram_dps_math.py index 531fb8459..6a4e5e007 100644 --- a/scripts/tests/test_tractogram_dps_math.py +++ b/scripts/tests/test_tractogram_dps_math.py @@ -29,9 +29,9 @@ def test_execution_dps_math_add(script_runner, monkeypatch): filename = 'vals.npy' outname = 'out.trk' np.save(filename, np.arange(len(sft))) - ret = script_runner.run('scil_tractogram_dps_math.py', in_bundle, 'add', + ret = script_runner.run('scil_tractogram_dps_math.py', + in_bundle, 'add', 'key', '--add_dps_file', filename, - '--add_dps_key', 'key', '--out_tractogram', outname, '-f') assert ret.success @@ -45,9 +45,9 @@ def test_execution_dps_math_add_with_missing_vals(script_runner, monkeypatch): filename = 'vals.npy' outname = 'out.trk' np.save(filename, np.arange(len(sft) - 10)) - ret = script_runner.run('scil_tractogram_dps_math.py', in_bundle, 'add', + ret = script_runner.run('scil_tractogram_dps_math.py', + in_bundle, 'add', 'key', '--add_dps_file', filename, - '--add_dps_key', 'key', '--out_tractogram', outname, '-f') assert ret.stderr @@ -62,15 +62,15 @@ def test_execution_dps_math_add_with_existing_key(script_runner, monkeypatch): outname = 'out.trk' outname2 = 'out_2.trk' np.save(filename, np.arange(len(sft))) - ret = script_runner.run('scil_tractogram_dps_math.py', in_bundle, 'add', + ret = script_runner.run('scil_tractogram_dps_math.py', + in_bundle, 'add', 'key', '--add_dps_file', filename, - '--add_dps_key', 'key', '--out_tractogram', outname, '-f') assert ret.success - ret = script_runner.run('scil_tractogram_dps_math.py', outname, 'add', + ret = script_runner.run('scil_tractogram_dps_math.py', + outname, 'add', 'key', '--add_dps_file', filename, - '--add_dps_key', 'key', '--out_tractogram', outname2,) assert not ret.success @@ -83,9 +83,9 @@ def test_execution_dps_math_tck(script_runner, monkeypatch): filename = 'vals.npy' outname = 'out.tck' np.save(filename, np.arange(len(sft))) - ret = script_runner.run('scil_tractogram_dps_math.py', in_bundle, 'add', + ret = script_runner.run('scil_tractogram_dps_math.py', + in_bundle, 'add', 'key', '--add_dps_file', filename, - '--add_dps_key', 'key', '--out_tractogram', outname, '-f') assert not ret.success @@ -102,8 +102,8 @@ def test_execution_dps_math_delete(script_runner, monkeypatch): } save_tractogram(sft, in_bundle) outname = 'out.trk' - ret = script_runner.run('scil_tractogram_dps_math.py', in_bundle, 'delete', - '--delete_dps_key', 'key', + ret = script_runner.run('scil_tractogram_dps_math.py', + in_bundle, 'delete', 'key', '--out_tractogram', outname, '-f') assert ret.success @@ -114,8 +114,8 @@ def test_execution_dps_math_delete_no_key(script_runner, monkeypatch): in_bundle = os.path.join(SCILPY_HOME, 'filtering', 'bundle_4.trk') outname = 'out.trk' - ret = script_runner.run('scil_tractogram_dps_math.py', in_bundle, 'delete', - '--delete_dps_key', 'key', + ret = script_runner.run('scil_tractogram_dps_math.py', + in_bundle, 'delete', 'key', '--out_tractogram', outname, '-f') assert not ret.success @@ -132,8 +132,8 @@ def test_execution_dps_math_export(script_runner, monkeypatch): } save_tractogram(sft, in_bundle) filename = 'out.txt' - ret = script_runner.run('scil_tractogram_dps_math.py', in_bundle, 'export', - '--export_dps_key', 'key', + ret = script_runner.run('scil_tractogram_dps_math.py', + in_bundle, 'export', 'key', '--export_dps_file', filename, '-f') assert ret.success @@ -144,8 +144,8 @@ def test_execution_dps_math_export_no_key(script_runner, monkeypatch): in_bundle = os.path.join(SCILPY_HOME, 'filtering', 'bundle_4.trk') filename = 'out.txt' - ret = script_runner.run('scil_tractogram_dps_math.py', in_bundle, 'export', - '--export_dps_key', 'key', + ret = script_runner.run('scil_tractogram_dps_math.py', + in_bundle, 'export', 'key', '--export_dps_file', filename, '-f') assert not ret.success From f9695cd37790e0dd6f18f38638b54b052dcf4753 Mon Sep 17 00:00:00 2001 From: VincentBeaud Date: Tue, 10 Dec 2024 10:52:33 -0500 Subject: [PATCH 05/10] Clarify script doc --- scripts/scil_tractogram_dps_math.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/scripts/scil_tractogram_dps_math.py b/scripts/scil_tractogram_dps_math.py index 30432b093..b915448e3 100755 --- a/scripts/scil_tractogram_dps_math.py +++ b/scripts/scil_tractogram_dps_math.py @@ -2,15 +2,20 @@ # -*- coding: utf-8 -*- """ -Add, extract or delete information to each streamline from a tractogram +Add, extract or delete dps (data_per_streamline) information to a tractogram file. Can be for example SIFT2 weights, processing information, bundle IDs, tracking seeds, etc. -Input and output tractograms must always be .trk to simplify filetype checks -for each operation. +Input and output tractograms must always be .trk. If you wish to add dps to +a .tck file, please convert it to .trk first using scil_tractogram_convert.py. + +Usage examples: + > scil_tractogram_dps_math.py tractogram.trk add "bundle_ids" + --add_dps_file my_bundle_ids.txt + > scil_tractogram_dps_math.py tractogram.trk export "seeds" + --export_dps_file seeds.npy """ -import os import argparse import logging From 0cbfce3b632f5308ad4575cb89871396b512a9ac Mon Sep 17 00:00:00 2001 From: VincentBeaud Date: Wed, 11 Dec 2024 11:04:34 -0500 Subject: [PATCH 06/10] Argument checks, pep8 and tests --- scilpy/io/streamlines.py | 4 +- scripts/scil_tractogram_dps_math.py | 78 ++++++++++++++--------- scripts/tests/test_tractogram_dps_math.py | 34 +++++----- 3 files changed, 69 insertions(+), 47 deletions(-) diff --git a/scilpy/io/streamlines.py b/scilpy/io/streamlines.py index cf26c85cc..0fc196f6a 100644 --- a/scilpy/io/streamlines.py +++ b/scilpy/io/streamlines.py @@ -75,8 +75,8 @@ def load_tractogram_with_reference(parser, args, filepath, arg_name=None): filepath: str Path of the tractogram file. arg_name: str, optional - Name of the reference argument. By default the args.ref is used. If - arg_name is given, then args.arg_name_ref will be used instead. + Name of the reference argument. By default the args.reference is used. + If arg_name is given, then args.arg_name_ref will be used instead. """ if is_argument_set(args, 'bbox_check'): bbox_check = args.bbox_check diff --git a/scripts/scil_tractogram_dps_math.py b/scripts/scil_tractogram_dps_math.py index b915448e3..5b96a5827 100755 --- a/scripts/scil_tractogram_dps_math.py +++ b/scripts/scil_tractogram_dps_math.py @@ -2,24 +2,26 @@ # -*- coding: utf-8 -*- """ -Add, extract or delete dps (data_per_streamline) information to a tractogram +Import, extract or delete dps (data_per_streamline) information to a tractogram file. Can be for example SIFT2 weights, processing information, bundle IDs, tracking seeds, etc. -Input and output tractograms must always be .trk. If you wish to add dps to -a .tck file, please convert it to .trk first using scil_tractogram_convert.py. +Input and output tractograms must be .trk, unless you are using the 'import' +operation, in which case a .tck input tractogram is accepted. Usage examples: - > scil_tractogram_dps_math.py tractogram.trk add "bundle_ids" - --add_dps_file my_bundle_ids.txt + > scil_tractogram_dps_math.py tractogram.trk import "bundle_ids" + --in_dps_file my_bundle_ids.txt > scil_tractogram_dps_math.py tractogram.trk export "seeds" - --export_dps_file seeds.npy + --out_dps_file seeds.npy """ +import nibabel as nib import argparse import logging from dipy.io.streamline import save_tractogram, load_tractogram +from scilpy.io.streamlines import load_tractogram_with_reference import numpy as np from scilpy.io.utils import (add_overwrite_arg, @@ -38,32 +40,29 @@ def _build_arg_parser(): formatter_class=argparse.RawTextHelpFormatter) p.add_argument('in_tractogram', - help='Input tractogram (.trk).') + help='Input tractogram (.trk for all operations,' + '.tck accepted for import).') p.add_argument('operation', metavar='OPERATION', - choices=['add', 'delete', 'export'], + choices=['import', 'delete', 'export'], help='The type of operation to be performed on the\n' 'tractogram\'s data_per_streamline at the given\n' 'key. Must be one of the following: [%(choices)s].\n' 'The additional arguments required for each\n' 'operation are specified under each group below.') p.add_argument('dps_key', type=str, - help='Where to find the data to be exported,\n' - 'in the tractogram.') + help='Key name used for the operation.') p.add_argument('--out_tractogram', - help='Output tractogram (.trk). Required for any mutation.') + help='Output tractogram (.trk). Required for "import" and\n' + '"delete" operations.') - add_args = p.add_argument_group('Add operation', - 'Requires the out_tractogram argument.') - add_args.add_argument('--add_dps_file', - help='File containing the data to add to\n' - 'streamlines (.txt, .npy or .mat).') + import_args = p.add_argument_group('Operation "import" mandatory options') + import_args.add_argument('--in_dps_file', + help='File containing the data to import to\n' + 'streamlines (.txt, .npy or .mat).') - _ = p.add_argument_group('Delete operation', - 'Requires the out_tractogram argument.') - - export_args = p.add_argument_group('Export operation') - export_args.add_argument('--export_dps_file', + export_args = p.add_argument_group('Operation "export" mandatory options') + export_args.add_argument('--out_dps_file', help='File in which the extracted data will be\n' 'saved (.txt or .npy).') @@ -78,18 +77,31 @@ def main(): args = parser.parse_args() logging.getLogger().setLevel(logging.getLevelName(args.verbose)) - check_tract_trk(parser, args.in_tractogram) + if args.operation == 'import': + if not nib.streamlines.is_supported(args.in_tractogram): + parser.error('Invalid input streamline file format (must be trk ' + + 'or tck): {0}'.format(args.in_tractogram)) + else: + check_tract_trk(parser, args.in_tractogram) + if args.out_tractogram: check_tract_trk(parser, args.out_tractogram) - # I/O assertions - assert_inputs_exist(parser, args.in_tractogram, args.add_dps_file) - assert_outputs_exist(parser, args, [], optional=[args.export_dps_file, + assert_inputs_exist(parser, args.in_tractogram, args.in_dps_file) + assert_outputs_exist(parser, args, [], optional=[args.out_dps_file, args.out_tractogram]) - sft = load_tractogram(args.in_tractogram, 'same') + sft = load_tractogram_with_reference(parser, args, args.in_tractogram) + + if args.operation == 'import': + if args.in_dps_file is None: + parser.error('The --in_dps_file option is required for ' + + 'the "import" operation.') + + if args.out_tractogram is None: + parser.error('The --out_tractogram option is required for ' + + 'the "import" operation.') - if args.operation == 'add': # Make sure the user is not unwillingly overwritting dps if (args.dps_key in sft.get_data_per_streamline_keys() and not args.overwrite): @@ -97,7 +109,7 @@ def main(): ' overwriting.'.format(args.dps_key)) # Load data and remove extraneous dimensions - data = np.squeeze(load_matrix_in_any_format(args.add_dps_file)) + data = np.squeeze(load_matrix_in_any_format(args.in_dps_file)) # Quick check as the built-in error from sft is not too explicit if len(sft) != data.shape[0]: @@ -110,11 +122,19 @@ def main(): save_tractogram(sft, args.out_tractogram) if args.operation == 'delete': + if args.out_tractogram is None: + parser.error('The --out_tractogram option is required for ' + + 'the "delete" operation.') + del sft.data_per_streamline[args.dps_key] save_tractogram(sft, args.out_tractogram) if args.operation == 'export': + if args.out_dps_file is None: + parser.error('The --out_dps_file option is required for ' + + 'the "export" operation.') + # Extract data and reshape if args.dps_key not in sft.data_per_streamline.keys(): raise ValueError('Data does not have any data_per_streamline' @@ -122,7 +142,7 @@ def main(): .format(args.dps_key)) data = np.squeeze(sft.data_per_streamline[args.dps_key]) - save_matrix_in_any_format(args.export_dps_file, data) + save_matrix_in_any_format(args.out_dps_file, data) if __name__ == '__main__': diff --git a/scripts/tests/test_tractogram_dps_math.py b/scripts/tests/test_tractogram_dps_math.py index 6a4e5e007..03b758de2 100644 --- a/scripts/tests/test_tractogram_dps_math.py +++ b/scripts/tests/test_tractogram_dps_math.py @@ -21,7 +21,7 @@ def test_help_option(script_runner): assert ret.success -def test_execution_dps_math_add(script_runner, monkeypatch): +def test_execution_dps_math_import(script_runner, monkeypatch): monkeypatch.chdir(os.path.expanduser(tmp_dir.name)) in_bundle = os.path.join(SCILPY_HOME, 'filtering', 'bundle_4.trk') @@ -30,14 +30,15 @@ def test_execution_dps_math_add(script_runner, monkeypatch): outname = 'out.trk' np.save(filename, np.arange(len(sft))) ret = script_runner.run('scil_tractogram_dps_math.py', - in_bundle, 'add', 'key', - '--add_dps_file', filename, + in_bundle, 'import', 'key', + '--in_dps_file', filename, '--out_tractogram', outname, '-f') assert ret.success -def test_execution_dps_math_add_with_missing_vals(script_runner, monkeypatch): +def test_execution_dps_math_import_with_missing_vals(script_runner, + monkeypatch): monkeypatch.chdir(os.path.expanduser(tmp_dir.name)) in_bundle = os.path.join(SCILPY_HOME, 'filtering', 'bundle_4.trk') @@ -46,14 +47,15 @@ def test_execution_dps_math_add_with_missing_vals(script_runner, monkeypatch): outname = 'out.trk' np.save(filename, np.arange(len(sft) - 10)) ret = script_runner.run('scil_tractogram_dps_math.py', - in_bundle, 'add', 'key', - '--add_dps_file', filename, + in_bundle, 'import', 'key', + '--in_dps_file', filename, '--out_tractogram', outname, '-f') assert ret.stderr -def test_execution_dps_math_add_with_existing_key(script_runner, monkeypatch): +def test_execution_dps_math_import_with_existing_key(script_runner, + monkeypatch): monkeypatch.chdir(os.path.expanduser(tmp_dir.name)) in_bundle = os.path.join(SCILPY_HOME, 'filtering', 'bundle_4.trk') @@ -63,19 +65,19 @@ def test_execution_dps_math_add_with_existing_key(script_runner, monkeypatch): outname2 = 'out_2.trk' np.save(filename, np.arange(len(sft))) ret = script_runner.run('scil_tractogram_dps_math.py', - in_bundle, 'add', 'key', - '--add_dps_file', filename, + in_bundle, 'import', 'key', + '--in_dps_file', filename, '--out_tractogram', outname, '-f') assert ret.success ret = script_runner.run('scil_tractogram_dps_math.py', - outname, 'add', 'key', - '--add_dps_file', filename, + outname, 'import', 'key', + '--in_dps_file', filename, '--out_tractogram', outname2,) assert not ret.success -def test_execution_dps_math_tck(script_runner, monkeypatch): +def test_execution_dps_math_tck_output(script_runner, monkeypatch): monkeypatch.chdir(os.path.expanduser(tmp_dir.name)) in_bundle = os.path.join(SCILPY_HOME, 'filtering', 'bundle_4.trk') @@ -84,8 +86,8 @@ def test_execution_dps_math_tck(script_runner, monkeypatch): outname = 'out.tck' np.save(filename, np.arange(len(sft))) ret = script_runner.run('scil_tractogram_dps_math.py', - in_bundle, 'add', 'key', - '--add_dps_file', filename, + in_bundle, 'import', 'key', + '--in_dps_file', filename, '--out_tractogram', outname, '-f') assert not ret.success @@ -134,7 +136,7 @@ def test_execution_dps_math_export(script_runner, monkeypatch): filename = 'out.txt' ret = script_runner.run('scil_tractogram_dps_math.py', in_bundle, 'export', 'key', - '--export_dps_file', filename, + '--out_dps_file', filename, '-f') assert ret.success @@ -146,6 +148,6 @@ def test_execution_dps_math_export_no_key(script_runner, monkeypatch): filename = 'out.txt' ret = script_runner.run('scil_tractogram_dps_math.py', in_bundle, 'export', 'key', - '--export_dps_file', filename, + '--out_dps_file', filename, '-f') assert not ret.success From 36d0aa338abaeca13f892d060401d5f8f8628271 Mon Sep 17 00:00:00 2001 From: VincentBeaud Date: Thu, 12 Dec 2024 09:51:53 -0500 Subject: [PATCH 07/10] Avoid scalar with dim > 0 by squeezing. Passes test and pep8 no warning. --- scilpy/tractograms/dps_and_dpp_management.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scilpy/tractograms/dps_and_dpp_management.py b/scilpy/tractograms/dps_and_dpp_management.py index bf40e4350..5390bf063 100644 --- a/scilpy/tractograms/dps_and_dpp_management.py +++ b/scilpy/tractograms/dps_and_dpp_management.py @@ -217,7 +217,7 @@ def project_dpp_to_map(sft, dpp_key, sum_lines=False, endpoints_only=False): for p in points: x, y, z = sft.streamlines[s][p, :].astype(int) # Or floor count[x, y, z] += 1 - the_map[x, y, z] += sft.data_per_point[dpp_key][s][p] + the_map[x, y, z] += np.squeeze(sft.data_per_point[dpp_key][s][p]) if not sum_lines: count = np.maximum(count, 1e-6) # Avoid division by 0 From 66d36c6d8301dea7149029fc42f7dfc637079cb0 Mon Sep 17 00:00:00 2001 From: karp2601 Date: Thu, 12 Dec 2024 09:58:13 -0500 Subject: [PATCH 08/10] Fixing chekc in script and adding test --- scripts/scil_NODDI_maps.py | 7 ++++--- scripts/tests/test_NODDI_maps.py | 22 ++++++++++++++++++++-- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/scripts/scil_NODDI_maps.py b/scripts/scil_NODDI_maps.py index 96be2f227..2d067c5ff 100755 --- a/scripts/scil_NODDI_maps.py +++ b/scripts/scil_NODDI_maps.py @@ -120,12 +120,13 @@ def main(): shells_centroids, indices_shells = identify_shells(bvals, args.tolerance, round_centroids=True) - nb_shells = len(shells_centroids) + non_b0_shells = shells_centroids[shells_centroids > args.tolerance] + nb_shells = len(non_b0_shells) if nb_shells <= 1: raise ValueError("Amico's NODDI works with data with more than one " "shell, but you seem to have single-shell data (we " - "found shells {}). Change tolerance if necessary." - .format(np.sort(shells_centroids))) + "found shell {}). Change tolerance if necessary." + .format(non_b0_shells[0])) logging.info('Will compute NODDI with AMICO on {} shells at found at {}.' .format(len(shells_centroids), np.sort(shells_centroids))) diff --git a/scripts/tests/test_NODDI_maps.py b/scripts/tests/test_NODDI_maps.py index 105bb72e6..ed7e66b9f 100644 --- a/scripts/tests/test_NODDI_maps.py +++ b/scripts/tests/test_NODDI_maps.py @@ -8,7 +8,8 @@ from scilpy.io.fetcher import fetch_data, get_testing_files_dict # If they already exist, this only takes 5 seconds (check md5sum) -fetch_data(get_testing_files_dict(), keys=['commit_amico.zip']) +fetch_data(get_testing_files_dict(), keys=['commit_amico.zip', + 'processing.zip']) tmp_dir = tempfile.TemporaryDirectory() @@ -32,5 +33,22 @@ def test_execution_commit_amico(script_runner, monkeypatch): '--out_dir', 'noddi', '--tol', '30', '--para_diff', '0.0017', '--iso_diff', '0.003', '--lambda1', '0.5', '--lambda2', '0.001', - '--processes', '1') + '--processes', '1', '-f') assert ret.success + + +def test_single_shell_fail(script_runner, monkeypatch): + monkeypatch.chdir(os.path.expanduser(tmp_dir.name)) + in_dwi = os.path.join(SCILPY_HOME, 'processing', + 'dwi_crop_1000.nii.gz') + in_bval = os.path.join(SCILPY_HOME, 'processing', + '1000.bval') + in_bvec = os.path.join(SCILPY_HOME, 'processing', + '1000.bvec') + ret = script_runner.run('scil_NODDI_maps.py', in_dwi, + in_bval, in_bvec, + '--out_dir', 'noddi', '--tol', '30', + '--para_diff', '0.0017', '--iso_diff', '0.003', + '--lambda1', '0.5', '--lambda2', '0.001', + '--processes', '1', '-f') + assert not ret.success From 60903adb3284e8587bbcb346be52967b629de705 Mon Sep 17 00:00:00 2001 From: VincentBeaud Date: Thu, 12 Dec 2024 10:09:37 -0500 Subject: [PATCH 09/10] Clarify doc to differentiate dpp_math with dps mode and dps_math --- scripts/scil_tractogram_dpp_math.py | 3 ++- scripts/scil_tractogram_dps_math.py | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/scil_tractogram_dpp_math.py b/scripts/scil_tractogram_dpp_math.py index 9303c1d32..5949fddb6 100755 --- a/scripts/scil_tractogram_dpp_math.py +++ b/scripts/scil_tractogram_dpp_math.py @@ -19,7 +19,8 @@ If endpoints_only and dps mode is set operation will be calculated across the data at the endpoints and stored as a single value (or array in the 4D case) -per streamline. +per streamline. If you wish to perform operations on dps values, please use +scil_tractogram_dps_math.py. Endpoint only operation: correlation: correlation calculated between arrays extracted from streamline diff --git a/scripts/scil_tractogram_dps_math.py b/scripts/scil_tractogram_dps_math.py index 5b96a5827..02837e07d 100755 --- a/scripts/scil_tractogram_dps_math.py +++ b/scripts/scil_tractogram_dps_math.py @@ -6,6 +6,10 @@ file. Can be for example SIFT2 weights, processing information, bundle IDs, tracking seeds, etc. +This script is not the same as the dps mode of scil_tractogram_dpp_math.py, +which performs operations on dpp (data_per_point) and saves the result as dps. +Instead this script performs operations directly on dps values. + Input and output tractograms must be .trk, unless you are using the 'import' operation, in which case a .tck input tractogram is accepted. From e63c833787db641cc872b790a1025106aa068a54 Mon Sep 17 00:00:00 2001 From: karp2601 Date: Thu, 12 Dec 2024 10:26:55 -0500 Subject: [PATCH 10/10] Adding ref to freewater --- scripts/scil_freewater_maps.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/scil_freewater_maps.py b/scripts/scil_freewater_maps.py index f24c3a94f..854eb30ab 100755 --- a/scripts/scil_freewater_maps.py +++ b/scripts/scil_freewater_maps.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- """ -Compute Free Water maps [1] using AMICO. +Compute Free Water maps [1] using the AMICO framework [2]. This script supports both single and multi-shell data. Formerly: scil_compute_freewater.py @@ -35,6 +35,9 @@ [1] Pasternak 0, Sochen N, Gur Y, Intrator N, Assaf Y. Free water elimination and mapping from diffusion mri. Magn Reson Med. 62 (3) (2009) 717-730. + [2] Daducci A, et al. Accelerated microstructure imaging + via convex optimization (AMICO) from diffusion MRI data. + Neuroimage 105 (2015) 32-44. """