diff --git a/CHANGES.rst b/CHANGES.rst index d1c92ae..e0072af 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,11 +1,17 @@ CHANGES ========= +v0.1.1 +--------- + +* BUG FIX: Fixed issue for incorrect references to tmp directories. This caused image conversion exceptions to be thrown, and thus no NIFTI BIDS files would be returned. +* BUG FIX: Fixed issue for cases in which hidden indexing files (._) would be included in the file search. + v0.1.0 --------- * BUG FIX: Fixed bug in setup, which prevented proper installation and usage of the ``study_proc`` executable. -* UPDATE: The documentation is now `available `_. +* UPDATE: The documentation is now `available `_. v0.1.rc1 --------- diff --git a/README.md b/README.md index ee706cd..2e998e2 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![CircleCI](https://circleci.com/gh/AdebayoBraimah/convert_source.svg?style=svg)](https://app.circleci.com/pipelines/github/AdebayoBraimah/convert_source) [![Documentation Status](https://readthedocs.org/projects/convert-source/badge/?version=latest)](https://convert-source.readthedocs.io/en/master/) +[![CircleCI](https://circleci.com/gh/AdebayoBraimah/convert_source.svg?style=svg)](https://app.circleci.com/pipelines/github/AdebayoBraimah/convert_source) [![Documentation Status](https://readthedocs.org/projects/convert-source/badge/?version=latest)](https://convert-source.readthedocs.io/en/latest/) # convert_source Convert source `DICOM`, `PAR REC` or `NIFTI` image data to BIDS directory layout. diff --git a/convert_source/batch_convert.py b/convert_source/batch_convert.py index c3d2c22..84a87e7 100644 --- a/convert_source/batch_convert.py +++ b/convert_source/batch_convert.py @@ -111,6 +111,8 @@ def batch_proc(study_img_dir: str, * Corresponding list of bval files. * Corresponding list of bvec files. ''' + study_img_dir: str = os.path.abspath(study_img_dir) + # Check dependencies dcm2niix_cmd: Command = Command("dcm2niix") dcm2niix_cmd.check_dependency(path_envs=path_envs) @@ -118,10 +120,12 @@ def batch_proc(study_img_dir: str, # Write logs misc_dir: str = os.path.join(out_dir,'.misc') if os.path.exists(misc_dir): + out_dir: str = os.path.abspath(out_dir) misc_dir: str = os.path.abspath(misc_dir) else: os.makedirs(misc_dir) misc_dir: str = os.path.abspath(misc_dir) + out_dir: str = os.path.abspath(out_dir) now = datetime.now() dt_string: str = str(now.strftime("%m_%d_%Y_%H_%M")) @@ -786,7 +790,7 @@ def source_to_bids(sub_data: SubDataInfo, # Using TmpDir and TmpFile context managers with TmpDir(tmp_dir=sub_tmp,use_cwd=False) as tmp: with TmpDir.TmpFile(tmp_dir=tmp.tmp_dir) as f: - _ = tmp.mk_tmp_dir() + tmp.mk_tmp_dir() [_path, basename, _ext] = f.file_parts() try: img_data = convert_image_data(file=data, @@ -904,7 +908,7 @@ def source_to_bids(sub_data: SubDataInfo, append_dwi_info=append_dwi_info, zero_pad=zero_pad, cprss_lvl=cprss_lvl) - _ = tmp.rm_tmp_dir() + tmp.rm_tmp_dir() return (imgs, jsons, bvals, @@ -929,7 +933,7 @@ def source_to_bids(sub_data: SubDataInfo, append_dwi_info=append_dwi_info, zero_pad=zero_pad, cprss_lvl=cprss_lvl) - _ = tmp.rm_tmp_dir() + tmp.rm_tmp_dir() return (imgs, jsons, bvals, @@ -991,13 +995,13 @@ def source_to_bids(sub_data: SubDataInfo, else: bvecs.append("") # Clean-up - _ = tmp.rm_tmp_dir() + tmp.rm_tmp_dir() return (imgs, jsons, bvals, bvecs) except ConversionError: - _ = tmp.rm_tmp_dir() + tmp.rm_tmp_dir() return [""],[""],[""],[""] def nifti_to_bids(sub_data: SubDataInfo, @@ -1075,7 +1079,7 @@ def nifti_to_bids(sub_data: SubDataInfo, # Use TmpDir and NiiFile class context managers with TmpDir(tmp_dir=sub_tmp, use_cwd=False) as tmp: - _ = tmp.mk_tmp_dir() + tmp.mk_tmp_dir() with NiiFile(data) as n: [path, basename, ext] = n.file_parts() img_files: List[str] = glob.glob(os.path.join(path,basename + "*" + ext)) @@ -1270,7 +1274,7 @@ def nifti_to_bids(sub_data: SubDataInfo, bvals.append("") bvecs.append("") # Clean-up - _ = tmp.rm_tmp_dir() + tmp.rm_tmp_dir() return (imgs, jsons, @@ -1401,7 +1405,7 @@ def bids_ignore(out_dir: str) -> str: # Write to file using File class context manager with File(new_file) as f: - f.write_txt(".misc \n") + f.write_txt(".misc/* \n") f.write_txt("unknown/* \n") return new_file @@ -1426,9 +1430,68 @@ def log_file(log: str) -> LogFile: log: LogFile = LogFile(log_file=log) now = datetime.now() - dt_string = now.strftime("%A %B %d, %Y %H:%M%:%S") + dt_string = now.strftime("%A %B %d, %Y %H:%M:%S") log.info(dt_string) log.info(f"convert_source v{__version__}") return log + +# This function was added for the Mac OS X case in which hidden +# temporary indexing files are present throughout a given directory. +# +# This function was designed to handle that use case, however: +# +# * Implementation methods are currently unclear +# * This approach is VERY SLOW as each parent directory is recursively searched. +# +# Moreover, this should be included in a later release should this continue to be an +# issue moving forward. +# +# Adebayo Braimah - 12 March 2021 +# +# def dir_clean_up(directory: str) -> str: +# '''Removes temporary indexing files (commonly found on +# Mac OS X). +# +# These files are generally problematic as they cause several of +# ``convert_source``'s core functions to behave unpredictably. +# +# Any identified tempoary indexing files are removed. +# +# NOTE: The implementation here does work, but is VERY SLOW as the number of +# files and directories is assumed to be large. +# +# Usage example: +# >>> directory = dir_clean_up(directory) +# >>> +# +# Arguments: +# directory: Input parent directory to recursively search (for hidden files in). +# +# Returns: +# Absolute path to directory as a string. +# ''' +# from shutil import rmtree +# +# if os.path.exists(directory): +# directory = os.path.abspath(directory) +# else: +# raise FileNotFoundError("Input directory does not appear to exist.") +# +# # This works - but is slow | O(n) space and time +# for root,dirnames,filenames in os.walk(directory): +# if len(dirnames) > 0: +# for dirname in dirnames: +# if '._' in dirname: +# hd_name: str = os.path.join(root,dirname) +# rmtree(hd_name) +# print(hd_name) +# if len(filenames) > 0: +# for file in filenames: +# if '._' in file: +# hf_name: str = os.path.join(root,file) +# # hf_list.append(hf_name) +# os.remove(hf_name) +# print(hf_name) +# return directory diff --git a/convert_source/cs_utils/fileio.py b/convert_source/cs_utils/fileio.py index 4d62f2d..973d529 100644 --- a/convert_source/cs_utils/fileio.py +++ b/convert_source/cs_utils/fileio.py @@ -185,7 +185,8 @@ def file_parts(self, file = self.abs_path() if platform.system().lower() == "windows": - [path, _filename] = os.path.splitdrive(file) + # [path, _filename] = os.path.splitdrive(file) + [path, _filename] = os.path.split(file) else: [path, _filename] = os.path.split(file) @@ -359,7 +360,7 @@ def __init__(self, self.tmp_file: str = tmp_file else: _n: int = 10000 # maximum N for random number generator - self.tmp_file: str = "tmp_file_" + str(random.randint(0,_n)) + self.tmp_file: str = "tmp_file_" + str(random.randint(0,_n)) + ".txt" if ext: self.tmp_file: str = self.tmp_file + f".{ext}" diff --git a/convert_source/cs_utils/img_dir.py b/convert_source/cs_utils/img_dir.py index ecd731e..1e01bd5 100644 --- a/convert_source/cs_utils/img_dir.py +++ b/convert_source/cs_utils/img_dir.py @@ -82,6 +82,9 @@ def img_dir_list(directory: str, for root,dirnames,filenames in os.walk(directory): if len(filenames) > 0: for file in filenames: + if '._' in file: + # Skip hidden files if they exist. + continue if '.dcm' in file.lower() or '.PAR' in file.upper() or '.nii' in file.lower(): file_name = os.path.join(root,file) if '.dcm' in file.lower(): diff --git a/convert_source/cs_utils/utils.py b/convert_source/cs_utils/utils.py index 43b083c..e6de262 100644 --- a/convert_source/cs_utils/utils.py +++ b/convert_source/cs_utils/utils.py @@ -847,7 +847,7 @@ def convert_image_data(file: str, # Create TmpDir object with TmpDir(tmp_dir=out_dir,use_cwd=False) as tmp_dir: # Create temporary output directory - _ = tmp_dir.mk_tmp_dir() + tmp_dir.mk_tmp_dir() # Output directory out_dir: str = os.path.abspath(out_dir) @@ -866,7 +866,7 @@ def convert_image_data(file: str, [imgs, jsons, bvals, bvecs] = img_data.copy_img_data(target_dir=out_dir) # Clean-up - _ = tmp_dir.rm_tmp_dir(rm_parent=False) + tmp_dir.rm_tmp_dir(rm_parent=False) # Image file check if len(imgs) == 0: diff --git a/convert_source/imgio/pario.py b/convert_source/imgio/pario.py index 2880b19..5b51316 100644 --- a/convert_source/imgio/pario.py +++ b/convert_source/imgio/pario.py @@ -178,7 +178,7 @@ def get_echo_time(par_file: str, df: pd.DataFrame = df.dropna(axis=0) with TmpDir(tmp_dir=tmp_dir,use_cwd=False) as tmp: - _ = tmp.mk_tmp_dir() + tmp.mk_tmp_dir() with TmpDir.TmpFile(tmp_dir=tmp.tmp_dir,ext="txt") as f: df.to_csv(f.file,sep=",",header=False,index=False) mat = np.loadtxt(f.file,delimiter=",") @@ -222,7 +222,7 @@ def get_flip_angle(par_file: str, df: pd.DataFrame = df.dropna(axis=0) with TmpDir(tmp_dir=tmp_dir,use_cwd=False) as tmp: - _ = tmp.mk_tmp_dir() + tmp.mk_tmp_dir() with TmpDir.TmpFile(tmp_file="file.tmp.txt",tmp_dir=tmp.tmp_dir) as f: df.to_csv(f.file,sep=",",header=False,index=False) mat = np.loadtxt(f.file,delimiter=",") diff --git a/convert_source/tests/01_fileio_test.py b/convert_source/tests/01_fileio_test.py index d2bc220..aeb9a89 100644 --- a/convert_source/tests/01_fileio_test.py +++ b/convert_source/tests/01_fileio_test.py @@ -30,6 +30,7 @@ def test_file_class(): assert f.file == 'test.file.txt' def test_command_class(): + """NOTE: This test will FAIL on Windows operating systems.""" x: str = 'ls' c = Command(x) c.cmd_list.append(os.getcwd()) @@ -44,7 +45,7 @@ def test_command_class(): def test_tmpdir_class(): x: str = 'tmpdir' with TmpDir(x,True) as tmp: - _ = tmp.mk_tmp_dir() + tmp.mk_tmp_dir() assert os.path.exists(tmp.tmp_dir) == True with TmpDir.TmpFile(tmp.tmp_dir) as f: f.touch() diff --git a/convert_source/tests/04_convert_image_test.py b/convert_source/tests/04_convert_image_test.py index a2e17bd..a1fe82f 100644 --- a/convert_source/tests/04_convert_image_test.py +++ b/convert_source/tests/04_convert_image_test.py @@ -89,6 +89,9 @@ class PlatformInferError(Exception): assert os.path.exists(file_name) == False def test_dependency(): + """NOTE: This test will FAIL if ``dcm2niix`` is already in the + system path. + """ dcm2niix_cmd: Command = Command("dcm2niix") with pytest.raises(DependencyError): @@ -130,6 +133,7 @@ def test_data_conversion(): [img, json, bval, bvec] = convert_image_data(data,'test_img',os.getcwd()) def test_cleanup(): + """NOTE: This test currently FAILS on Windows operating systems.""" del_method: bool = False rm_method: bool = False rm_item_method: bool = False diff --git a/convert_source/tests/07_dcmio_test.py b/convert_source/tests/07_dcmio_test.py index f57768e..09bb337 100644 --- a/convert_source/tests/07_dcmio_test.py +++ b/convert_source/tests/07_dcmio_test.py @@ -127,6 +127,7 @@ def test_get_mb(): assert get_mb(data) == 1 def test_cleanup(): + """NOTE: This test currently FAILS on Windows operating systems.""" del_method: bool = False rm_method: bool = False rm_item_method: bool = False diff --git a/convert_source/tests/08_batch_proc_test.py b/convert_source/tests/08_batch_proc_test.py index a95fa9f..d9c4fc8 100644 --- a/convert_source/tests/08_batch_proc_test.py +++ b/convert_source/tests/08_batch_proc_test.py @@ -517,6 +517,7 @@ def test_batch_proc(): assert len(bvecs) == 11 def test_cleanup(): + """NOTE: This test currently FAILS on Windows operating systems.""" shutil.rmtree(out_dir) assert os.path.exists(out_dir) == False