diff --git a/.circleci/main.yml b/.circleci/main.yml index ee836c0607..af60db3156 100644 --- a/.circleci/main.yml +++ b/.circleci/main.yml @@ -1,4 +1,4 @@ ---- +--- version: 2.1 # Singularity started failing to set up on Circle circa May 2023, so those tests are currently disabled @@ -139,7 +139,7 @@ commands: pip install -r dev/circleci_data/requirements.txt - save_cache: key: pip-ci-requirements-{{ checksum "dev/circleci_data/requirements.txt" }}-3.10.10 - paths: + paths: - /opt/circleci/.pyenv/versions/3.10.10 set-up-variant: parameters: @@ -237,7 +237,7 @@ jobs: path: test-results - save_cache: key: coverage-docker-<< parameters.variant >>-{{ .Revision }} - paths: + paths: - .coverage.docker - .coverage.docker-<< parameters.variant >> pytest-singularity: @@ -266,7 +266,7 @@ jobs: path: test-results - save_cache: key: coverage-singularity-<< parameters.variant >>-{{ .Revision }} - paths: + paths: - .coverage.singularity - .coverage.singularity-<> workflows: diff --git a/.github/Dockerfiles/AFNI.16.2.07.neurodocker-xenial.Dockerfile b/.github/Dockerfiles/AFNI.16.2.07.neurodocker-xenial.Dockerfile index c98f0af760..f0d74e6286 100644 --- a/.github/Dockerfiles/AFNI.16.2.07.neurodocker-xenial.Dockerfile +++ b/.github/Dockerfiles/AFNI.16.2.07.neurodocker-xenial.Dockerfile @@ -29,4 +29,4 @@ USER root COPY --from=AFNI /opt/afni-latest/ /usr/lib/afni/bin/ -USER c-pac_user \ No newline at end of file +USER c-pac_user diff --git a/.github/Dockerfiles/FreeSurfer.6.0.1-min-xenial.Dockerfile b/.github/Dockerfiles/FreeSurfer.6.0.1-min-xenial.Dockerfile index 8e1ca0c1a8..1a4f4ed4a7 100644 --- a/.github/Dockerfiles/FreeSurfer.6.0.1-min-xenial.Dockerfile +++ b/.github/Dockerfiles/FreeSurfer.6.0.1-min-xenial.Dockerfile @@ -9,4 +9,4 @@ LABEL org.opencontainers.image.description "NOT INTENDED FOR USE OTHER THAN AS A FreeSurfer 6.0.1-min stage" LABEL org.opencontainers.image.source https://github.com/FCP-INDI/C-PAC COPY --from=FreeSurfer /opt/freesurfer /opt/freesurfer -COPY dev/docker_data/license.txt /opt/freesurfer/license.txt \ No newline at end of file +COPY dev/docker_data/license.txt /opt/freesurfer/license.txt diff --git a/.github/Dockerfiles/ICA-AROMA.0.4.3-beta-bionic.Dockerfile b/.github/Dockerfiles/ICA-AROMA.0.4.3-beta-bionic.Dockerfile index d864e20742..0e313e9219 100644 --- a/.github/Dockerfiles/ICA-AROMA.0.4.3-beta-bionic.Dockerfile +++ b/.github/Dockerfiles/ICA-AROMA.0.4.3-beta-bionic.Dockerfile @@ -23,4 +23,4 @@ USER c-pac_user FROM scratch LABEL org.opencontainers.image.description "NOT INTENDED FOR USE OTHER THAN AS A STAGE IMAGE IN A MULTI-STAGE BUILD \ ICA-AROMA 0.4.3-beta stage" -COPY --from=ICA-AROMA /opt/ICA-AROMA/ /opt/ICA-AROMA/ \ No newline at end of file +COPY --from=ICA-AROMA /opt/ICA-AROMA/ /opt/ICA-AROMA/ diff --git a/.github/Dockerfiles/ICA-AROMA.0.4.4-beta-jammy.Dockerfile b/.github/Dockerfiles/ICA-AROMA.0.4.4-beta-jammy.Dockerfile index ea5787517a..2759c529eb 100644 --- a/.github/Dockerfiles/ICA-AROMA.0.4.4-beta-jammy.Dockerfile +++ b/.github/Dockerfiles/ICA-AROMA.0.4.4-beta-jammy.Dockerfile @@ -26,4 +26,4 @@ USER c-pac_user FROM scratch LABEL org.opencontainers.image.description "NOT INTENDED FOR USE OTHER THAN AS A STAGE IMAGE IN A MULTI-STAGE BUILD \ ICA-AROMA 0.4.4-beta stage" -COPY --from=ICA-AROMA /opt/ICA-AROMA/ /opt/ICA-AROMA/ \ No newline at end of file +COPY --from=ICA-AROMA /opt/ICA-AROMA/ /opt/ICA-AROMA/ diff --git a/.github/Dockerfiles/ICA-AROMA.0.4.5-xenial.Dockerfile b/.github/Dockerfiles/ICA-AROMA.0.4.5-xenial.Dockerfile index d775562bb3..a16d9b3f6e 100644 --- a/.github/Dockerfiles/ICA-AROMA.0.4.5-xenial.Dockerfile +++ b/.github/Dockerfiles/ICA-AROMA.0.4.5-xenial.Dockerfile @@ -26,4 +26,4 @@ FROM scratch LABEL org.opencontainers.image.description "NOT INTENDED FOR USE OTHER THAN AS A STAGE IMAGE IN A MULTI-STAGE BUILD \ ICA-AROMA 0.4.5 stage" LABEL org.opencontainers.image.source https://github.com/FCP-INDI/C-PAC -COPY --from=ICA-AROMA /opt/ICA-AROMA/ /opt/ICA-AROMA/ \ No newline at end of file +COPY --from=ICA-AROMA /opt/ICA-AROMA/ /opt/ICA-AROMA/ diff --git a/.github/Dockerfiles/Ubuntu.jammy-non-free.Dockerfile b/.github/Dockerfiles/Ubuntu.jammy-non-free.Dockerfile index 4498fa0b92..e434119480 100644 --- a/.github/Dockerfiles/Ubuntu.jammy-non-free.Dockerfile +++ b/.github/Dockerfiles/Ubuntu.jammy-non-free.Dockerfile @@ -80,4 +80,4 @@ RUN ldconfig \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/.cache/pip/* # Set user -USER c-pac_user \ No newline at end of file +USER c-pac_user diff --git a/.github/Dockerfiles/base-ABCD-HCP.Dockerfile b/.github/Dockerfiles/base-ABCD-HCP.Dockerfile index 81726e9128..8ce4b2090b 100644 --- a/.github/Dockerfiles/base-ABCD-HCP.Dockerfile +++ b/.github/Dockerfiles/base-ABCD-HCP.Dockerfile @@ -96,4 +96,4 @@ RUN locale-gen --purge en_US.UTF-8 \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* # set user -USER c-pac_user \ No newline at end of file +USER c-pac_user diff --git a/.github/Dockerfiles/base-fMRIPrep-LTS.Dockerfile b/.github/Dockerfiles/base-fMRIPrep-LTS.Dockerfile index ac069ce62c..e3e9a0ccda 100644 --- a/.github/Dockerfiles/base-fMRIPrep-LTS.Dockerfile +++ b/.github/Dockerfiles/base-fMRIPrep-LTS.Dockerfile @@ -68,7 +68,7 @@ ENV PERL5LIB="$MINC_LIB_DIR/perl5/5.8.5" \ # Installing and setting up AFNI COPY --from=AFNI /lib/x86_64-linux-gnu/ld* /lib/x86_64-linux-gnu/ -COPY --from=AFNI /lib/x86_64-linux-gnu/lib*so* /lib/x86_64-linux-gnu/ +COPY --from=AFNI /lib/x86_64-linux-gnu/lib*so* /lib/x86_64-linux-gnu/ COPY --from=AFNI /lib64/ld* /lib64/ COPY --from=AFNI /usr/lib/afni/bin/ /usr/lib/afni/bin/ COPY --from=AFNI /usr/lib/x86_64-linux-gnu/lib*so* /usr/lib/x86_64-linux-gnu/ @@ -92,7 +92,7 @@ COPY --from=c3d /usr/bin/c3d_* /usr/bin/ COPY --from=c3d /usr/share/doc/convert3d /usr/share/doc/convert3d COPY --from=c3d /usr/lib/c3d_gui-1.1.0/Convert3DGUI /usr/lib/c3d_gui-1.1.0/Convert3DGUI -# Installing and setting up FSL +# Installing and setting up FSL COPY --from=FSL /etc/fsl /etc/fsl COPY --from=FSL /usr/lib/fsl /usr/lib/fsl COPY --from=FSL /usr/lib/libnewmat.so.10 /usr/lib/libnewmat.so.10 diff --git a/.github/Dockerfiles/connectome-workbench.1.3.2-1.neurodebian-bionic.Dockerfile b/.github/Dockerfiles/connectome-workbench.1.3.2-1.neurodebian-bionic.Dockerfile index 3c33cd789d..ff3be3ab06 100644 --- a/.github/Dockerfiles/connectome-workbench.1.3.2-1.neurodebian-bionic.Dockerfile +++ b/.github/Dockerfiles/connectome-workbench.1.3.2-1.neurodebian-bionic.Dockerfile @@ -26,4 +26,4 @@ COPY --from=base /usr/share/applications/connectome-workbench.desktop /usr/share COPY --from=base /usr/share/bash-completion/completions/wb* /usr/share/bash_completion/completions/ COPY --from=base /usr/share/doc/connectome-workbench /usr/share/doc/connectome-workbench COPY --from=base /usr/share/man/man1/wb_* /usr/share/man/man1/ -COPY --from=base /usr/share/pixmaps/connectome-workbench.png /usr/share/pixmaps/connectome-workbench.png \ No newline at end of file +COPY --from=base /usr/share/pixmaps/connectome-workbench.png /usr/share/pixmaps/connectome-workbench.png diff --git a/.github/Dockerfiles/connectome-workbench.1.3.2-2.neurodebian-xenial.Dockerfile b/.github/Dockerfiles/connectome-workbench.1.3.2-2.neurodebian-xenial.Dockerfile index 30838e14b7..4c3b79cc8f 100644 --- a/.github/Dockerfiles/connectome-workbench.1.3.2-2.neurodebian-xenial.Dockerfile +++ b/.github/Dockerfiles/connectome-workbench.1.3.2-2.neurodebian-xenial.Dockerfile @@ -24,4 +24,4 @@ COPY --from=base /usr/share/applications/connectome-workbench.desktop /usr/share COPY --from=base /usr/share/bash-completion/completions/wb* /usr/share/bash-completion/completions/ COPY --from=base /usr/share/doc/connectome-workbench /usr/share/doc/connectome-workbench COPY --from=base /usr/share/man/man1/wb_* /usr/share/man/man1/ -COPY --from=base /usr/share/pixmaps/connectome-workbench.png /usr/share/pixmaps/connectome-workbench.png \ No newline at end of file +COPY --from=base /usr/share/pixmaps/connectome-workbench.png /usr/share/pixmaps/connectome-workbench.png diff --git a/.github/Dockerfiles/msm.2.0-bionic.Dockerfile b/.github/Dockerfiles/msm.2.0-bionic.Dockerfile index 089dca2000..f991043ef0 100644 --- a/.github/Dockerfiles/msm.2.0-bionic.Dockerfile +++ b/.github/Dockerfiles/msm.2.0-bionic.Dockerfile @@ -53,4 +53,4 @@ LABEL org.opencontainers.image.description "NOT INTENDED FOR USE OTHER THAN AS A msm v2.0 stage \ Multimodal Surface Matching with Higher order Clique Reduction Version 2.00 (Feb 2017)" LABEL org.opencontainers.image.source https://github.com/FCP-INDI/C-PAC -COPY --from=MSM /opt/msm/Ubuntu/msm /opt/msm/Ubuntu/msm \ No newline at end of file +COPY --from=MSM /opt/msm/Ubuntu/msm /opt/msm/Ubuntu/msm diff --git a/.github/README/README.md b/.github/README/README.md index 274e9c29a5..e807a8e4d1 100644 --- a/.github/README/README.md +++ b/.github/README/README.md @@ -50,9 +50,9 @@ flowchart LR end subgraph build_and_test.yml ubuntu[[Ubnutu]]-->stages[[stages]]-->build-base[[build-base]]-->build-base-standard[[build-base-standard]] - + Circle_tests[[Circle_tests]] - + build-base-standard-->C-PAC C-PAC[[C-PAC]]-->bCPAC C-PAC-->Circle_tests @@ -164,7 +164,7 @@ in the commit message. For this to work, all of these must be true: ```YAML strategy: matrix: - Dockerfile: + Dockerfile: ``` in a job in a [workflow](../workflows) file. diff --git a/.github/scripts/autoversioning.sh b/.github/scripts/autoversioning.sh index 0543f626a1..64cd698626 100755 --- a/.github/scripts/autoversioning.sh +++ b/.github/scripts/autoversioning.sh @@ -59,4 +59,4 @@ cp .github/Dockerfiles/C-PAC.develop-jammy.Dockerfile Dockerfile cp .github/Dockerfiles/C-PAC.develop-ABCD-HCP-bionic.Dockerfile variant-ABCD-HCP.Dockerfile cp .github/Dockerfiles/C-PAC.develop-fMRIPrep-LTS-xenial.Dockerfile variant-fMRIPrep-LTS.Dockerfile cp .github/Dockerfiles/C-PAC.develop-lite-jammy.Dockerfile variant-lite.Dockerfile -git add *Dockerfile \ No newline at end of file +git add *Dockerfile diff --git a/.github/scripts/freesurfer-prune b/.github/scripts/freesurfer-prune index 1f2bed2777..32034e83f4 100755 --- a/.github/scripts/freesurfer-prune +++ b/.github/scripts/freesurfer-prune @@ -6,4 +6,4 @@ do echo "Deleting $FILE" rm -rf $FILE || true fi -done \ No newline at end of file +done diff --git a/.github/scripts/get_package_id.py b/.github/scripts/get_package_id.py index d5e54cc704..358838f9f4 100644 --- a/.github/scripts/get_package_id.py +++ b/.github/scripts/get_package_id.py @@ -1,16 +1,17 @@ -"""Get Package ID +"""Get Package ID. Script to get GHCR ID string for a given owner + image tag Usage: python get_package_id.py $OWNER $IMAGE_TAG $VERSION_TAG """ import os -import requests import sys +import requests + def get_packages(owner, tag, api_token=None): - """Function to collect GHCR packages for a given owner & tag + """Function to collect GHCR packages for a given owner & tag. Parameters ---------- @@ -28,10 +29,10 @@ def get_packages(owner, tag, api_token=None): list """ if api_token is None: - api_token = os.environ.get('GITHUB_TOKEN', '') + api_token = os.environ.get("GITHUB_TOKEN", "") def fetch(url): - """Method to make API call and return response, given a URL + """Method to make API call and return response, given a URL. Parameters ---------- @@ -42,37 +43,46 @@ def fetch(url): dict or list """ response = requests.get( - url, - headers={'Authorization': f'token {api_token}'}).json() - if isinstance(response, dict) and response.get( - 'message', '' - ) == 'Bad credentials': - raise PermissionError('\n'.join([ - ': '.join([ - response['message'], - api_token if api_token else '[no token provided]' - ]), - 'Either set GITHUB_TOKEN to a personal access token with ' - 'read.packages permissions or explicitly pass one as a fourth ' - 'positional argument:\n' - '`python get_package_id.py $OWNER $IMAGE_TAG ' - '$VERSION_TAG $GITHUB_TOKEN`' - ])) + url, headers={"Authorization": f"token {api_token}"} + ).json() + if ( + isinstance(response, dict) + and response.get("message", "") == "Bad credentials" + ): + raise PermissionError( + "\n".join( + [ + ": ".join( + [ + response["message"], + api_token if api_token else "[no token provided]", + ] + ), + "Either set GITHUB_TOKEN to a personal access token with " + "read.packages permissions or explicitly pass one as a fourth " + "positional argument:\n" + "`python get_package_id.py $OWNER $IMAGE_TAG " + "$VERSION_TAG $GITHUB_TOKEN`", + ] + ) + ) return response + _packages = fetch( - f'https://api.github.com/orgs/{owner}/packages/container/' - f'{tag}/versions') + f"https://api.github.com/orgs/{owner}/packages/container/" f"{tag}/versions" + ) packages = [] for _package in _packages: - if _package.get('message', 'Not Found') == 'Not Found': + if _package.get("message", "Not Found") == "Not Found": packages += fetch( - f'https://api.github.com/users/{owner}/packages/container/' - f'{tag}/versions') + f"https://api.github.com/users/{owner}/packages/container/" + f"{tag}/versions" + ) return packages def id_from_tag(owner, image, tag, api_token=None): - """Function to return a package ID given an image version tag + """Function to return a package ID given an image version tag. Parameters ---------- @@ -89,17 +99,19 @@ def id_from_tag(owner, image, tag, api_token=None): GitHub API personal access token with read.packages permission """ packages = get_packages(owner, image, api_token) - versions = [image['id'] for image in packages if tag in image.get( - 'metadata', {} - ).get('container', {}).get('tags', [])] + versions = [ + image["id"] + for image in packages + if tag in image.get("metadata", {}).get("container", {}).get("tags", []) + ] if len(versions): return versions[0] else: - raise LookupError(f'Image not found: ghcr.io/{owner}/{image}:{tag}') + raise LookupError(f"Image not found: ghcr.io/{owner}/{image}:{tag}") -if __name__ == '__main__': +if __name__ == "__main__": if len(sys.argv) == 4: - print(id_from_tag(*sys.argv[1:])) + pass else: - print(__doc__) + pass diff --git a/.github/scripts/get_pr_base_shas.py b/.github/scripts/get_pr_base_shas.py index 7a108ed9e9..ac20c6933f 100644 --- a/.github/scripts/get_pr_base_shas.py +++ b/.github/scripts/get_pr_base_shas.py @@ -14,16 +14,4 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -"""Get base SHAs for open PRs""" -import os -from github import Github - -print(' '.join([ - pr.base.sha for pr in Github(os.environ.get( - 'GITHUB_TOKEN' - )).get_repo(os.environ.get( - 'GITHUB_REPOSITORY' - )).get_commit(os.environ.get( - 'GITHUB_SHA' - )).get_pulls() -])) +"""Get base SHAs for open PRs.""" diff --git a/.github/scripts/local_ghcr b/.github/scripts/local_ghcr index 92b0b3fef8..3d3da29112 100755 --- a/.github/scripts/local_ghcr +++ b/.github/scripts/local_ghcr @@ -3,22 +3,17 @@ import sys def replace_primary_repo(yaml_string, local_owner): - return yaml_string.replace('ghcr.io/fcp-indi', - f'ghcr.io/{local_owner.lower()}') + return yaml_string.replace("ghcr.io/fcp-indi", f"ghcr.io/{local_owner.lower()}") -if __name__ == '__main__': +if __name__ == "__main__": if len(sys.argv) != 4: - raise SyntaxError('Usage: local_ghcr $FILE $OWNER $DOCKER_TAG') + raise SyntaxError("Usage: local_ghcr $FILE $OWNER $DOCKER_TAG") file, owner, tag = sys.argv[1:] - dockerfiles = '.github/Dockerfiles' - new_Dockerfile = replace_primary_repo( - open(file, 'r').read(), - owner.lower() - ) - if '-lite-' in file: + dockerfiles = ".github/Dockerfiles" + new_Dockerfile = replace_primary_repo(open(file, "r").read(), owner.lower()) + if "-lite-" in file: tag = tag.lower().split("c-pac:")[1][:-5] - if tag != 'latest-lite': - new_Dockerfile = new_Dockerfile.replace(':latest-bionic', - f':{tag}') - open(file, 'w').write(new_Dockerfile) + if tag != "latest-lite": + new_Dockerfile = new_Dockerfile.replace(":latest-bionic", f":{tag}") + open(file, "w").write(new_Dockerfile) diff --git a/.github/stage_requirements/ABCD-HCP.txt b/.github/stage_requirements/ABCD-HCP.txt index 6f133b94fe..efa9a93980 100644 --- a/.github/stage_requirements/ABCD-HCP.txt +++ b/.github/stage_requirements/ABCD-HCP.txt @@ -6,4 +6,4 @@ connectome-workbench.1.3.2-1.neurodebian-bionic FSL.5.0.10-bionic ICA-AROMA.0.4.3-beta-bionic msm.2.0-bionic -Ubuntu.bionic-non-free \ No newline at end of file +Ubuntu.bionic-non-free diff --git a/.github/stage_requirements/fMRIPrep-LTS.txt b/.github/stage_requirements/fMRIPrep-LTS.txt index 2e38e7b579..e25139d432 100644 --- a/.github/stage_requirements/fMRIPrep-LTS.txt +++ b/.github/stage_requirements/fMRIPrep-LTS.txt @@ -6,4 +6,4 @@ connectome-workbench.1.3.2-2.neurodebian-xenial FSL.5.0.9-5.neurodebian-xenial ICA-AROMA.0.4.5-xenial msm.2.0-bionic -Ubuntu.xenial-20200114 \ No newline at end of file +Ubuntu.xenial-20200114 diff --git a/.github/stage_requirements/lite.txt b/.github/stage_requirements/lite.txt index 657d977237..69871e04f5 100644 --- a/.github/stage_requirements/lite.txt +++ b/.github/stage_requirements/lite.txt @@ -1,4 +1,4 @@ base-lite AFNI.23.3.09-jammy ICA-AROMA.0.4.4-beta-jammy -Ubuntu.jammy-non-free \ No newline at end of file +Ubuntu.jammy-non-free diff --git a/.github/stage_requirements/phase_one.txt b/.github/stage_requirements/phase_one.txt index 365ff460fd..7b77e4bb4c 100644 --- a/.github/stage_requirements/phase_one.txt +++ b/.github/stage_requirements/phase_one.txt @@ -1,4 +1,4 @@ FSL.data Ubuntu.bionic-non-free Ubuntu.jammy-non-free -Ubuntu.xenial-20200114 \ No newline at end of file +Ubuntu.xenial-20200114 diff --git a/.github/stage_requirements/phase_three.txt b/.github/stage_requirements/phase_three.txt index b71c08de24..b978d93be3 100644 --- a/.github/stage_requirements/phase_three.txt +++ b/.github/stage_requirements/phase_three.txt @@ -1,3 +1,3 @@ lite ABCD-HCP -fMRIPrep-LTS \ No newline at end of file +fMRIPrep-LTS diff --git a/.github/stage_requirements/phase_two.txt b/.github/stage_requirements/phase_two.txt index 7b78675924..04e5f5f54a 100644 --- a/.github/stage_requirements/phase_two.txt +++ b/.github/stage_requirements/phase_two.txt @@ -13,4 +13,4 @@ FSL.6.0.6.5-jammy ICA-AROMA.0.4.3-beta-bionic ICA-AROMA.0.4.4-beta-jammy ICA-AROMA.0.4.5-xenial -msm.2.0-bionic \ No newline at end of file +msm.2.0-bionic diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 62a7a34f4b..23cbfd8345 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -96,7 +96,7 @@ jobs: ${{ env.DOCKER_TAG }} cache-from: type=gha cache-to: type=gha,mode=min,compression=zstd - + stages: name: Build C-PAC Docker stage images for dependencies needs: Ubuntu diff --git a/.github/workflows/delete_images.yml b/.github/workflows/delete_images.yml index 30561715dd..d141e711e5 100644 --- a/.github/workflows/delete_images.yml +++ b/.github/workflows/delete_images.yml @@ -36,7 +36,7 @@ jobs: fi TAG=${GITHUB_REF_NAME} TAG=$TAG$VARIANT - + VERSION_ID=$(python .github/scripts/get_package_id.py $OWNER $IMAGE $TAG) curl \ -u ${GITHUB_TOKEN}: \ diff --git a/.github/workflows/regression_test_full.yml b/.github/workflows/regression_test_full.yml index 39fe11f5d2..6dba2d1bf2 100644 --- a/.github/workflows/regression_test_full.yml +++ b/.github/workflows/regression_test_full.yml @@ -1,13 +1,13 @@ name: Run Regression Full Test -on: +on: workflow_call: -jobs: +jobs: test: name: Regression Test - Full runs-on: ubuntu-latest - steps: + steps: - name: Get C-PAC branch run: | GITHUB_BRANCH=$(echo ${GITHUB_REF} | cut -d '/' -f 3-) diff --git a/.github/workflows/regression_test_lite.yml b/.github/workflows/regression_test_lite.yml index 380a3cc6b4..4e6b5a46f6 100644 --- a/.github/workflows/regression_test_lite.yml +++ b/.github/workflows/regression_test_lite.yml @@ -23,7 +23,7 @@ on: required: true workflow_dispatch: -jobs: +jobs: test: name: Regression Test - Lite environment: ACCESS @@ -55,7 +55,7 @@ jobs: echo "${{ env.SSH_PRIVATE_KEY }}" > ~/.ssh/id_rsa chmod 600 ~/.ssh/id_rsa ssh-keyscan -H -t rsa "${{ env.SSH_HOST }}" > ~/.ssh/known_hosts - + - name: Initiate check uses: guibranco/github-status-action-v2@v1.1.7 with: diff --git a/.gitignore b/.gitignore index 4c6b1550e9..1c2d0e74a6 100755 --- a/.gitignore +++ b/.gitignore @@ -44,4 +44,4 @@ bids-examples cpac_runs/ -.DS_Store \ No newline at end of file +.DS_Store diff --git a/.markdownlint.yml b/.markdownlint.yml index 2b6d69e558..c14a82e488 100644 --- a/.markdownlint.yml +++ b/.markdownlint.yml @@ -23,4 +23,4 @@ MD024: # allow specific inline HTML elements MD033: allowed_elements: - - span \ No newline at end of file + - span diff --git a/.ruff.toml b/.ruff.toml index 8efb3701a1..cb5eb184c3 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,4 +1,4 @@ -extend-exclude=["dev/docker_data/get-pip_23.0.1.py"] +extend-exclude = ["dev/docker_data/get-pip_23.0.1.py"] extend-select = ["A", "C4", "D", "G", "I", "ICN", "NPY", "PL", "RET", "RSE", "RUF", "Q", "T20", "W"] # variants still use 3.7 target-version = "py37" diff --git a/CHANGELOG.md b/CHANGELOG.md index 02d93e7bd3..446ce62cd5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -229,7 +229,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - In a given pipeline configuration, segmentation probability maps and binary tissue masks are warped to template space, and those warped masks are included in the output directory - if `registration_workflows['functional_registration']['EPI_registration']['run segmentation']` is `On` and `segmentation['tissue_segmentation']['Template_Based']['template_for_segmentation']` includes `EPI_Template` - + and/or - if `registration_workflows['anatomical_registration']['run']` is `On` and `segmentation['tissue_segmentation']['Template_Based']['template_for_segmentation']` includes `T1_Template` - Renamed connectivity matrices from `*_connectome.tsv` to `*_correlations.tsv` diff --git a/COPYING b/COPYING index e72bfddabc..f288702d2f 100644 --- a/COPYING +++ b/COPYING @@ -671,4 +671,4 @@ into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. But first, please read -. \ No newline at end of file +. diff --git a/COPYING.LESSER b/COPYING.LESSER index 153d416dc8..0a041280bd 100644 --- a/COPYING.LESSER +++ b/COPYING.LESSER @@ -162,4 +162,4 @@ General Public License ever published by the Free Software Foundation. whether future versions of the GNU Lesser General Public License shall apply, that proxy's public statement of acceptance of any version is permanent authorization for you to choose that version for the -Library. \ No newline at end of file +Library. diff --git a/CPAC/__init__.py b/CPAC/__init__.py index c2d0828353..43d9bb651b 100644 --- a/CPAC/__init__.py +++ b/CPAC/__init__.py @@ -15,7 +15,7 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . """ -Configurable Pipeline for the Analysis of Connectomes +Configurable Pipeline for the Analysis of Connectomes. ===================================================== CPAC is a configurable, open-source, Nipype-based, automated processing @@ -24,12 +24,14 @@ """ from .info import __version__ + version = __version__ def _docs_prefix() -> str: - """Get the docs URL prefix for this version""" + """Get the docs URL prefix for this version.""" from CPAC.utils.docs import DOCS_URL_PREFIX + return DOCS_URL_PREFIX @@ -39,4 +41,4 @@ def _docs_prefix() -> str: and you are welcome to redistribute it under certain conditions. For details, see {_docs_prefix()}/license or the COPYING and COPYING.LESSER files included in the source code.""" -__all__ = ['license_notice', 'version', '__version__'] +__all__ = ["license_notice", "version", "__version__"] diff --git a/CPAC/__main__.py b/CPAC/__main__.py index 2039a8a568..d05a977261 100644 --- a/CPAC/__main__.py +++ b/CPAC/__main__.py @@ -16,10 +16,10 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . import os -import pkg_resources as p + import click from click_aliases import ClickAliasedGroup -from CPAC.utils.docs import version_report +import pkg_resources as p # CLI tree # @@ -54,90 +54,78 @@ def main(): @main.command() def version(): - """Display environment version information""" - import CPAC - print('\n'.join(['Environment', '===========', version_report(), - f'C-PAC version: {CPAC.__version__}'])) + """Display environment version information.""" @main.command() -@click.argument('data_config') -@click.option('--pipe-config', '--pipe_config') -@click.option('--num-cores', '--num_cores') -@click.option('--ndmg-mode', '--ndmg_mode', is_flag=True) -@click.option('--debug', is_flag=True) -def run(data_config, pipe_config=None, num_cores=None, ndmg_mode=False, - debug=False): +@click.argument("data_config") +@click.option("--pipe-config", "--pipe_config") +@click.option("--num-cores", "--num_cores") +@click.option("--ndmg-mode", "--ndmg_mode", is_flag=True) +@click.option("--debug", is_flag=True) +def run(data_config, pipe_config=None, num_cores=None, ndmg_mode=False, debug=False): if not pipe_config: - pipe_config = \ - p.resource_filename("CPAC", - os.path.join("resources", - "configs", - "pipeline_config_template.yml")) - - if pipe_config == 'benchmark-ants': - pipe_config = \ - p.resource_filename("CPAC", - os.path.join("resources", - "configs", - "pipeline_config_benchmark-ANTS.yml")) - - if pipe_config == 'benchmark-fnirt': - pipe_config = \ - p.resource_filename("CPAC", - os.path.join("resources", - "configs", - "pipeline_config_benchmark-FNIRT.yml")) - - if pipe_config == 'anat-only': - pipe_config = \ - p.resource_filename("CPAC", - os.path.join("resources", - "configs", - "pipeline_config_anat-only.yml")) - - if data_config == 'benchmark-data': - data_config = \ - p.resource_filename("CPAC", - os.path.join("resources", - "configs", - "data_config_cpac_benchmark.yml")) - - if data_config == 'ADHD200': - data_config = \ - p.resource_filename("CPAC", - os.path.join("resources", - "configs", - "data_config_S3-BIDS-ADHD200.yml")) - if data_config == 'ADHD200_2': - data_config = \ - p.resource_filename("CPAC", - os.path.join("resources", - "configs", - "data_config_S3-BIDS-ADHD200_only2.yml")) - if data_config == 'ABIDE': - data_config = \ - p.resource_filename("CPAC", - os.path.join("resources", - "configs", - "data_config_S3-BIDS-ABIDE.yml")) - if data_config == 'NKI-RS': - data_config = \ - p.resource_filename("CPAC", - os.path.join("resources", - "configs", - "data_config_S3-BIDS-NKI-RocklandSample.yml")) + pipe_config = p.resource_filename( + "CPAC", os.path.join("resources", "configs", "pipeline_config_template.yml") + ) + + if pipe_config == "benchmark-ants": + pipe_config = p.resource_filename( + "CPAC", + os.path.join("resources", "configs", "pipeline_config_benchmark-ANTS.yml"), + ) + + if pipe_config == "benchmark-fnirt": + pipe_config = p.resource_filename( + "CPAC", + os.path.join("resources", "configs", "pipeline_config_benchmark-FNIRT.yml"), + ) + + if pipe_config == "anat-only": + pipe_config = p.resource_filename( + "CPAC", + os.path.join("resources", "configs", "pipeline_config_anat-only.yml"), + ) + + if data_config == "benchmark-data": + data_config = p.resource_filename( + "CPAC", + os.path.join("resources", "configs", "data_config_cpac_benchmark.yml"), + ) + + if data_config == "ADHD200": + data_config = p.resource_filename( + "CPAC", + os.path.join("resources", "configs", "data_config_S3-BIDS-ADHD200.yml"), + ) + if data_config == "ADHD200_2": + data_config = p.resource_filename( + "CPAC", + os.path.join( + "resources", "configs", "data_config_S3-BIDS-ADHD200_only2.yml" + ), + ) + if data_config == "ABIDE": + data_config = p.resource_filename( + "CPAC", + os.path.join("resources", "configs", "data_config_S3-BIDS-ABIDE.yml"), + ) + if data_config == "NKI-RS": + data_config = p.resource_filename( + "CPAC", + os.path.join( + "resources", "configs", "data_config_S3-BIDS-NKI-RocklandSample.yml" + ), + ) if ndmg_mode: - pipe_config = \ - p.resource_filename("CPAC", - os.path.join("resources", - "configs", - "pipeline_config_ndmg.yml")) + pipe_config = p.resource_filename( + "CPAC", os.path.join("resources", "configs", "pipeline_config_ndmg.yml") + ) - import CPAC.pipeline.cpac_runner as cpac_runner - cpac_runner.run(data_config, pipe_config, num_subs_at_once=num_cores, - debug=debug) + from CPAC.pipeline import cpac_runner + + cpac_runner.run(data_config, pipe_config, num_subs_at_once=num_cores, debug=debug) # Group analysis @@ -153,149 +141,233 @@ def feat(): @feat.command() -@click.argument('group_config') +@click.argument("group_config") def build_models(group_config): import CPAC.pipeline.cpac_group_runner as cgr + cgr.build_feat_models(group_config) -@feat.command(name='run') -@click.argument('group_config') +@feat.command(name="run") +@click.argument("group_config") def run_feat(group_config): import CPAC.pipeline.cpac_group_runner as cgr + cgr.run_feat(group_config) @feat.command() -@click.argument('group_config') +@click.argument("group_config") def randomise(group_config): import CPAC.pipeline.cpac_group_runner as cgr + cgr.run_feat(group_config, feat=False) -@feat.group(aliases=['load_preset'], cls=ClickAliasedGroup) +@feat.group(aliases=["load_preset"], cls=ClickAliasedGroup) def load_preset(): pass -@load_preset.command(aliases=['single_grp_avg']) -@click.argument('pipeline_dir') -@click.argument('z_thresh') -@click.argument('p_thresh') -@click.argument('model_name') -@click.option('--output-dir', '--output_dir', default=None) -@click.option('--group-participants', '--group_participants', default=None) -def single_grp_avg(pipeline_dir, z_thresh, p_thresh, model_name, - output_dir=None, group_participants=None): + +@load_preset.command(aliases=["single_grp_avg"]) +@click.argument("pipeline_dir") +@click.argument("z_thresh") +@click.argument("p_thresh") +@click.argument("model_name") +@click.option("--output-dir", "--output_dir", default=None) +@click.option("--group-participants", "--group_participants", default=None) +def single_grp_avg( + pipeline_dir, + z_thresh, + p_thresh, + model_name, + output_dir=None, + group_participants=None, +): from CPAC.utils import create_fsl_flame_preset + if not output_dir: - output_dir = os.path.join(os.getcwd(), 'cpac_group_analysis') - create_fsl_flame_preset.run(pipeline_dir, 'all', z_thresh, p_thresh, - 'single_grp', group_participants, - output_dir=output_dir, - model_name=model_name) - -@load_preset.command(aliases=['single_grp_cov']) -@click.argument('pipeline_dir') -@click.argument('z_thresh') -@click.argument('p_thresh') -@click.argument('pheno_file') -@click.argument('pheno_sub') -@click.argument('covariate') -@click.argument('model_name') -@click.option('--output-dir', '--output_dir', default=None) -@click.option('--group-participants', '--group_participants', default=None) -def single_grp_cov(pipeline_dir, z_thresh, p_thresh, pheno_file, - pheno_sub, covariate, model_name, output_dir=None, - group_participants=None): + output_dir = os.path.join(os.getcwd(), "cpac_group_analysis") + create_fsl_flame_preset.run( + pipeline_dir, + "all", + z_thresh, + p_thresh, + "single_grp", + group_participants, + output_dir=output_dir, + model_name=model_name, + ) + + +@load_preset.command(aliases=["single_grp_cov"]) +@click.argument("pipeline_dir") +@click.argument("z_thresh") +@click.argument("p_thresh") +@click.argument("pheno_file") +@click.argument("pheno_sub") +@click.argument("covariate") +@click.argument("model_name") +@click.option("--output-dir", "--output_dir", default=None) +@click.option("--group-participants", "--group_participants", default=None) +def single_grp_cov( + pipeline_dir, + z_thresh, + p_thresh, + pheno_file, + pheno_sub, + covariate, + model_name, + output_dir=None, + group_participants=None, +): from CPAC.utils import create_fsl_flame_preset + if not output_dir: - output_dir = os.path.join(os.getcwd(), 'cpac_group_analysis') - create_fsl_flame_preset.run(pipeline_dir, 'all', z_thresh, p_thresh, - 'single_grp_cov', group_participants, - pheno_file=pheno_file, - pheno_sub_label=pheno_sub, - covariate=covariate, output_dir=output_dir, - model_name=model_name) - -@load_preset.command(aliases=['unpaired_two']) -@click.argument('pipeline_dir') -@click.argument('z_thresh') -@click.argument('p_thresh') -@click.argument('pheno_file') -@click.argument('pheno_sub') -@click.argument('covariate') -@click.argument('model_name') -@click.option('--output-dir', '--output_dir', default=None) -@click.option('--group-participants', '--group_participants', default=None) -def unpaired_two(pipeline_dir, z_thresh, p_thresh, pheno_file, - pheno_sub, covariate, model_name, output_dir=None, - group_participants=None): + output_dir = os.path.join(os.getcwd(), "cpac_group_analysis") + create_fsl_flame_preset.run( + pipeline_dir, + "all", + z_thresh, + p_thresh, + "single_grp_cov", + group_participants, + pheno_file=pheno_file, + pheno_sub_label=pheno_sub, + covariate=covariate, + output_dir=output_dir, + model_name=model_name, + ) + + +@load_preset.command(aliases=["unpaired_two"]) +@click.argument("pipeline_dir") +@click.argument("z_thresh") +@click.argument("p_thresh") +@click.argument("pheno_file") +@click.argument("pheno_sub") +@click.argument("covariate") +@click.argument("model_name") +@click.option("--output-dir", "--output_dir", default=None) +@click.option("--group-participants", "--group_participants", default=None) +def unpaired_two( + pipeline_dir, + z_thresh, + p_thresh, + pheno_file, + pheno_sub, + covariate, + model_name, + output_dir=None, + group_participants=None, +): from CPAC.utils import create_fsl_flame_preset + if not output_dir: - output_dir = os.path.join(os.getcwd(), 'cpac_group_analysis') - create_fsl_flame_preset.run(pipeline_dir, 'all', z_thresh, p_thresh, - 'unpaired_two', group_participants, - pheno_file=pheno_file, - pheno_sub_label=pheno_sub, - covariate=covariate, output_dir=output_dir, - model_name=model_name) - -@load_preset.command(aliases=['paired_two']) -@click.argument('pipeline_dir') -@click.argument('z_thresh') -@click.argument('p_thresh') -@click.argument('conditions') -@click.argument('condition_type') -@click.argument('model_name') -@click.option('--output-dir', '--output_dir', default=None) -@click.option('--group-participants', '--group_participants', default=None) -def paired_two(pipeline_dir, z_thresh, p_thresh, conditions, - condition_type, model_name, output_dir=None, - group_participants=None): + output_dir = os.path.join(os.getcwd(), "cpac_group_analysis") + create_fsl_flame_preset.run( + pipeline_dir, + "all", + z_thresh, + p_thresh, + "unpaired_two", + group_participants, + pheno_file=pheno_file, + pheno_sub_label=pheno_sub, + covariate=covariate, + output_dir=output_dir, + model_name=model_name, + ) + + +@load_preset.command(aliases=["paired_two"]) +@click.argument("pipeline_dir") +@click.argument("z_thresh") +@click.argument("p_thresh") +@click.argument("conditions") +@click.argument("condition_type") +@click.argument("model_name") +@click.option("--output-dir", "--output_dir", default=None) +@click.option("--group-participants", "--group_participants", default=None) +def paired_two( + pipeline_dir, + z_thresh, + p_thresh, + conditions, + condition_type, + model_name, + output_dir=None, + group_participants=None, +): from CPAC.utils import create_fsl_flame_preset + if not output_dir: - output_dir = os.path.join(os.getcwd(), 'cpac_group_analysis') - create_fsl_flame_preset.run(pipeline_dir, 'all', z_thresh, p_thresh, - 'paired_two', group_participants, - covariate=conditions, - condition_type=condition_type, - output_dir=output_dir, model_name=model_name) - - -@load_preset.command(aliases=['tripled_two']) -@click.argument('pipeline_dir') -@click.argument('z_thresh') -@click.argument('p_thresh') -@click.argument('conditions') -@click.argument('condition_type') -@click.argument('model_name') -@click.option('--output-dir', '--output_dir', default=None) -@click.option('--group-participants', '--group_participants', default=None) -def tripled_two(pipeline_dir, z_thresh, p_thresh, conditions, - condition_type, model_name, output_dir=None, - group_participants=None): + output_dir = os.path.join(os.getcwd(), "cpac_group_analysis") + create_fsl_flame_preset.run( + pipeline_dir, + "all", + z_thresh, + p_thresh, + "paired_two", + group_participants, + covariate=conditions, + condition_type=condition_type, + output_dir=output_dir, + model_name=model_name, + ) + + +@load_preset.command(aliases=["tripled_two"]) +@click.argument("pipeline_dir") +@click.argument("z_thresh") +@click.argument("p_thresh") +@click.argument("conditions") +@click.argument("condition_type") +@click.argument("model_name") +@click.option("--output-dir", "--output_dir", default=None) +@click.option("--group-participants", "--group_participants", default=None) +def tripled_two( + pipeline_dir, + z_thresh, + p_thresh, + conditions, + condition_type, + model_name, + output_dir=None, + group_participants=None, +): from CPAC.utils import create_fsl_flame_preset + if not output_dir: - output_dir = os.path.join(os.getcwd(), 'cpac_group_analysis') - create_fsl_flame_preset.run(pipeline_dir, 'all', z_thresh, p_thresh, - 'tripled_two', group_participants, - covariate=conditions, - condition_type=condition_type, - output_dir=output_dir, model_name=model_name) + output_dir = os.path.join(os.getcwd(), "cpac_group_analysis") + create_fsl_flame_preset.run( + pipeline_dir, + "all", + z_thresh, + p_thresh, + "tripled_two", + group_participants, + covariate=conditions, + condition_type=condition_type, + output_dir=output_dir, + model_name=model_name, + ) # Group analysis - PyBASC @group.command() -@click.argument('group_config') +@click.argument("group_config") def basc(group_config): - import CPAC.pipeline.cpac_group_runner as cpac_group_runner - cpac_group_runner.run_basc(group_config) + from CPAC.pipeline import cpac_group_runner as cgr + + cgr.run_basc(group_config) @group.command(name="mdmr") @click.argument("group_config", type=click.Path(exists=True)) def group_mdmr(group_config): from CPAC.pipeline.cpac_group_runner import run_cwas + run_cwas(group_config) @@ -303,13 +375,16 @@ def group_mdmr(group_config): @click.argument("group_config", type=click.Path(exists=True)) def group_isc(group_config): from CPAC.pipeline.cpac_group_runner import run_isc + run_isc(group_config) + # Group analysis - QPP @group.command() -@click.argument('group_config') +@click.argument("group_config") def qpp(group_config): from CPAC.pipeline.cpac_group_runner import run_qpp + run_qpp(group_config) @@ -320,72 +395,93 @@ def utils(): @utils.command() -@click.argument('crash_file') +@click.argument("crash_file") def crash(crash_file): - import mock def accept_all(object, name, value): return value - with mock.patch('nipype.interfaces.base.traits_extension.File.validate', side_effect=accept_all): + with mock.patch( + "nipype.interfaces.base.traits_extension.File.validate", side_effect=accept_all + ): from nipype.scripts.crash_files import display_crash_file + display_crash_file(crash_file, False, False, None) -@utils.group(aliases=['data_config'], cls=ClickAliasedGroup) -@click.option('--tracking-opt-out', '--tracking_opt-out', is_flag=True, - help='Disable usage tracking.') +@utils.group(aliases=["data_config"], cls=ClickAliasedGroup) +@click.option( + "--tracking-opt-out", + "--tracking_opt-out", + is_flag=True, + help="Disable usage tracking.", +) def data_config(tracking_opt_out): if not tracking_opt_out: # pylint: disable=import-outside-toplevel from CPAC.utils.ga import track_config - track_config('cli') + + track_config("cli") -@data_config.command(aliases=['new_settings_template']) +@data_config.command(aliases=["new_settings_template"]) def new_settings_template(): from CPAC.utils.build_data_config import util_copy_template - util_copy_template('data_settings') + + util_copy_template("data_settings") @data_config.command() -@click.argument('data_settings_file') +@click.argument("data_settings_file") def build(data_settings_file): from CPAC.utils.build_data_config import run + run(data_settings_file) -@utils.group(aliases=['pipe_config'], cls=ClickAliasedGroup) -@click.option('--tracking-opt-out', '--tracking_opt-out', is_flag=True, - help='Disable usage tracking.') +@utils.group(aliases=["pipe_config"], cls=ClickAliasedGroup) +@click.option( + "--tracking-opt-out", + "--tracking_opt-out", + is_flag=True, + help="Disable usage tracking.", +) def pipe_config(tracking_opt_out): if not tracking_opt_out: # pylint: disable=import-outside-toplevel from CPAC.utils.ga import track_config - track_config('cli') + track_config("cli") -@pipe_config.command(name='new-template', aliases=['new_template']) + +@pipe_config.command(name="new-template", aliases=["new_template"]) def new_pipeline_template(): from CPAC.utils.build_data_config import util_copy_template - util_copy_template('pipeline_config') + + util_copy_template("pipeline_config") -@utils.group(aliases=['group_config'], cls=ClickAliasedGroup) -@click.option('--tracking-opt-out', '--tracking_opt-out', is_flag=True, - help='Disable usage tracking.') +@utils.group(aliases=["group_config"], cls=ClickAliasedGroup) +@click.option( + "--tracking-opt-out", + "--tracking_opt-out", + is_flag=True, + help="Disable usage tracking.", +) def group_config(tracking_opt_out): if not tracking_opt_out: # pylint: disable=import-outside-toplevel from CPAC.utils.ga import track_config - track_config('cli') + track_config("cli") -@group_config.command(name='new-template', aliases=['new_template']) + +@group_config.command(name="new-template", aliases=["new_template"]) def new_group_template(): from CPAC.utils.build_data_config import util_copy_template - util_copy_template('group_config') + + util_copy_template("group_config") @utils.group(cls=ClickAliasedGroup) @@ -393,23 +489,43 @@ def tools(): pass -@tools.command(aliases=['ants_apply_warp']) -@click.argument('moving_image') -@click.argument('reference') -@click.option('--initial', default=None) -@click.option('--rigid', default=None) -@click.option('--affine', default=None) -@click.option('--nonlinear', default=None) -@click.option('--func-to-anat', '--func_to_anat', default=None) -@click.option('--dim', default=3) -@click.option('--interp', default='Linear') -@click.option('--inverse', default=False) -def ants_apply_warp(moving_image, reference, initial=None, rigid=None, - affine=None, nonlinear=None, func_to_anat=None, dim=3, - interp='Linear', inverse=False): +@tools.command(aliases=["ants_apply_warp"]) +@click.argument("moving_image") +@click.argument("reference") +@click.option("--initial", default=None) +@click.option("--rigid", default=None) +@click.option("--affine", default=None) +@click.option("--nonlinear", default=None) +@click.option("--func-to-anat", "--func_to_anat", default=None) +@click.option("--dim", default=3) +@click.option("--interp", default="Linear") +@click.option("--inverse", default=False) +def ants_apply_warp( + moving_image, + reference, + initial=None, + rigid=None, + affine=None, + nonlinear=None, + func_to_anat=None, + dim=3, + interp="Linear", + inverse=False, +): from CPAC.registration.utils import run_ants_apply_warp - run_ants_apply_warp(moving_image, reference, initial, rigid, affine, - nonlinear, func_to_anat, dim, interp, inverse) + + run_ants_apply_warp( + moving_image, + reference, + initial, + rigid, + affine, + nonlinear, + func_to_anat, + dim, + interp, + inverse, + ) @utils.group() @@ -418,9 +534,10 @@ def workflows(): @utils.command() -@click.argument('directory') +@click.argument("directory") def repickle(directory): from CPAC.utils import repickle as repickle_util + if os.path.exists(directory): repickle_util(directory) @@ -430,52 +547,52 @@ def test(): pass -@test.command(aliases=['run_suite']) -@click.option('--list', '-l', 'show_list', is_flag=True) -@click.option('--filter', '-f', 'pipeline_filter', default='') -def run_suite(show_list=False, pipeline_filter=''): - import CPAC.pipeline.cpac_runner as cpac_runner - - test_config_dir = \ - p.resource_filename("CPAC", - os.path.join("resources", - "configs", "test_configs")) - - data_test = \ - p.resource_filename("CPAC", - os.path.join("resources", - "configs", "test_configs", - "data-test_S3-ADHD200_1.yml")) - - data_test_no_scan_param = \ - p.resource_filename("CPAC", - os.path.join("resources", - "configs", "test_configs", - "data-test_S3-ADHD200_no-params.yml")) - - data_test_fmap = \ - p.resource_filename("CPAC", - os.path.join("resources", - "configs", "test_configs", - "data-test_S3-NKI-RS_fmap.yml")) +@test.command(aliases=["run_suite"]) +@click.option("--list", "-l", "show_list", is_flag=True) +@click.option("--filter", "-f", "pipeline_filter", default="") +def run_suite(show_list=False, pipeline_filter=""): + from CPAC.pipeline import cpac_runner + + test_config_dir = p.resource_filename( + "CPAC", os.path.join("resources", "configs", "test_configs") + ) + + data_test = p.resource_filename( + "CPAC", + os.path.join( + "resources", "configs", "test_configs", "data-test_S3-ADHD200_1.yml" + ), + ) + + data_test_no_scan_param = p.resource_filename( + "CPAC", + os.path.join( + "resources", "configs", "test_configs", "data-test_S3-ADHD200_no-params.yml" + ), + ) + + data_test_fmap = p.resource_filename( + "CPAC", + os.path.join( + "resources", "configs", "test_configs", "data-test_S3-NKI-RS_fmap.yml" + ), + ) if show_list: - print("") - print("Availables pipelines:") + pass no_params = False for config_file in os.listdir(test_config_dir): - if config_file.startswith('pipe-test_'): + if config_file.startswith("pipe-test_"): if pipeline_filter not in config_file: continue if show_list: - print("- " + config_file[len('pipe-test_'):]) continue pipe = os.path.join(test_config_dir, config_file) - if 'DistCorr' in pipe: + if "DistCorr" in pipe: data = data_test_fmap elif not no_params: data = data_test_no_scan_param @@ -487,7 +604,7 @@ def run_suite(show_list=False, pipeline_filter=''): cpac_runner.run(data, pipe) if show_list: - print("") + pass @test.group(cls=ClickAliasedGroup) @@ -495,11 +612,11 @@ def functions(): pass -@functions.command(aliases=['gather_outputs_func']) -@click.argument('pipe_config') +@functions.command(aliases=["gather_outputs_func"]) +@click.argument("pipe_config") def gather_outputs_func(pipe_config): - #from CPAC.pipeline. - #run_gather_outputs_func(pipe_config) + # from CPAC.pipeline. + # run_gather_outputs_func(pipe_config) pass diff --git a/CPAC/alff/__init__.py b/CPAC/alff/__init__.py index 735b3182c1..40a96afc6f 100644 --- a/CPAC/alff/__init__.py +++ b/CPAC/alff/__init__.py @@ -1,3 +1 @@ # -*- coding: utf-8 -*- - -from CPAC.alff.alff import create_alff diff --git a/CPAC/alff/alff.py b/CPAC/alff/alff.py index d3c4815c65..8b7c7f127a 100644 --- a/CPAC/alff/alff.py +++ b/CPAC/alff/alff.py @@ -1,18 +1,20 @@ # -*- coding: utf-8 -*- import os -from CPAC.pipeline import nipype_pipeline_engine as pe -from CPAC.pipeline.nodeblock import nodeblock + from nipype.interfaces.afni import preprocess import nipype.interfaces.utility as util + from CPAC.alff.utils import get_opt_string -from CPAC.utils.utils import check_prov_for_regtool +from CPAC.pipeline import nipype_pipeline_engine as pe +from CPAC.pipeline.nodeblock import nodeblock from CPAC.registration.registration import apply_transform +from CPAC.utils.utils import check_prov_for_regtool -def create_alff(wf_name='alff_workflow'): +def create_alff(wf_name="alff_workflow"): """ - Calculate Amplitude of low frequency oscillations (ALFF) and fractional ALFF maps + Calculate Amplitude of low frequency oscillations (ALFF) and fractional ALFF maps. Parameters ---------- @@ -139,12 +141,10 @@ def create_alff(wf_name='alff_workflow'): References ---------- - .. [1] Zou, Q.-H., Zhu, C.-Z., Yang, Y., Zuo, X.-N., Long, X.-Y., Cao, Q.-J., Wang, Y.-F., et al. (2008). An improved approach to detection of amplitude of low-frequency fluctuation (ALFF) for resting-state fMRI: fractional ALFF. Journal of neuroscience methods, 172(1), 137-41. doi:10.10 Examples -------- - >>> alff_w = create_alff() >>> alff_w.inputs.hp_input.hp = [0.01] >>> alff_w.inputs.lp_input.lp = [0.1] @@ -154,90 +154,89 @@ def create_alff(wf_name='alff_workflow'): >>> alff_w.inputs.inputspec.rest_mask= '/home/data/subject/func/rest_mask.nii.gz' >>> alff_w.run() # doctest: +SKIP """ - wf = pe.Workflow(name=wf_name) - input_node = pe.Node(util.IdentityInterface(fields=['rest_res', - 'rest_mask']), - name='inputspec') + input_node = pe.Node( + util.IdentityInterface(fields=["rest_res", "rest_mask"]), name="inputspec" + ) - input_node_hp = pe.Node(util.IdentityInterface(fields=['hp']), - name='hp_input') + input_node_hp = pe.Node(util.IdentityInterface(fields=["hp"]), name="hp_input") - input_node_lp = pe.Node(util.IdentityInterface(fields=['lp']), - name='lp_input') + input_node_lp = pe.Node(util.IdentityInterface(fields=["lp"]), name="lp_input") - output_node = pe.Node(util.IdentityInterface(fields=['alff_img', - 'falff_img']), - name='outputspec') + output_node = pe.Node( + util.IdentityInterface(fields=["alff_img", "falff_img"]), name="outputspec" + ) # filtering - bandpass = pe.Node(interface=preprocess.Bandpass(), - name='bandpass_filtering') - bandpass.inputs.outputtype = 'NIFTI_GZ' - bandpass.inputs.out_file = os.path.join(os.path.curdir, - 'residual_filtered.nii.gz') + bandpass = pe.Node(interface=preprocess.Bandpass(), name="bandpass_filtering") + bandpass.inputs.outputtype = "NIFTI_GZ" + bandpass.inputs.out_file = os.path.join(os.path.curdir, "residual_filtered.nii.gz") - wf.connect(input_node_hp, 'hp', bandpass, 'highpass') - wf.connect(input_node_lp, 'lp', bandpass, 'lowpass') - wf.connect(input_node, 'rest_res', bandpass, 'in_file') + wf.connect(input_node_hp, "hp", bandpass, "highpass") + wf.connect(input_node_lp, "lp", bandpass, "lowpass") + wf.connect(input_node, "rest_res", bandpass, "in_file") - get_option_string = pe.Node(util.Function(input_names=['mask'], - output_names=['option_string'], - function=get_opt_string), - name='get_option_string') + get_option_string = pe.Node( + util.Function( + input_names=["mask"], + output_names=["option_string"], + function=get_opt_string, + ), + name="get_option_string", + ) - wf.connect(input_node, 'rest_mask', get_option_string, 'mask') + wf.connect(input_node, "rest_mask", get_option_string, "mask") # standard deviation over frequency try: from nipype.interfaces.afni import utils as afni_utils - stddev_filtered = pe.Node(interface=afni_utils.TStat(), - name='stddev_filtered') + + stddev_filtered = pe.Node(interface=afni_utils.TStat(), name="stddev_filtered") except ImportError: - stddev_filtered = pe.Node(interface=preprocess.TStat(), - name='stddev_filtered') + stddev_filtered = pe.Node(interface=preprocess.TStat(), name="stddev_filtered") - stddev_filtered.inputs.outputtype = 'NIFTI_GZ' - stddev_filtered.inputs.out_file = os.path.join(os.path.curdir, - 'alff.nii.gz') + stddev_filtered.inputs.outputtype = "NIFTI_GZ" + stddev_filtered.inputs.out_file = os.path.join(os.path.curdir, "alff.nii.gz") - wf.connect(bandpass, 'out_file', stddev_filtered, 'in_file') - wf.connect(get_option_string, 'option_string', stddev_filtered, 'options') + wf.connect(bandpass, "out_file", stddev_filtered, "in_file") + wf.connect(get_option_string, "option_string", stddev_filtered, "options") - wf.connect(stddev_filtered, 'out_file', output_node, 'alff_img') + wf.connect(stddev_filtered, "out_file", output_node, "alff_img") # standard deviation of the unfiltered nuisance corrected image try: - stddev_unfiltered = pe.Node(interface=afni_utils.TStat(), - name='stddev_unfiltered') + stddev_unfiltered = pe.Node( + interface=afni_utils.TStat(), name="stddev_unfiltered" + ) except UnboundLocalError: - stddev_unfiltered = pe.Node(interface=preprocess.TStat(), - name='stddev_unfiltered') + stddev_unfiltered = pe.Node( + interface=preprocess.TStat(), name="stddev_unfiltered" + ) - stddev_unfiltered.inputs.outputtype = 'NIFTI_GZ' - stddev_unfiltered.inputs.out_file = os.path.join(os.path.curdir, - 'residual_3dT.nii.gz') + stddev_unfiltered.inputs.outputtype = "NIFTI_GZ" + stddev_unfiltered.inputs.out_file = os.path.join( + os.path.curdir, "residual_3dT.nii.gz" + ) - wf.connect(input_node, 'rest_res', stddev_unfiltered, 'in_file') - wf.connect(get_option_string, 'option_string', stddev_unfiltered, - 'options') + wf.connect(input_node, "rest_res", stddev_unfiltered, "in_file") + wf.connect(get_option_string, "option_string", stddev_unfiltered, "options") # falff calculations try: - falff = pe.Node(interface=afni_utils.Calc(), name='falff') + falff = pe.Node(interface=afni_utils.Calc(), name="falff") except UnboundLocalError: - falff = pe.Node(interface=preprocess.Calc(), name='falff') + falff = pe.Node(interface=preprocess.Calc(), name="falff") - falff.inputs.args = '-float' - falff.inputs.expr = '(1.0*bool(a))*((1.0*b)/(1.0*c))' - falff.inputs.outputtype = 'NIFTI_GZ' - falff.inputs.out_file = os.path.join(os.path.curdir, 'falff.nii.gz') + falff.inputs.args = "-float" + falff.inputs.expr = "(1.0*bool(a))*((1.0*b)/(1.0*c))" + falff.inputs.outputtype = "NIFTI_GZ" + falff.inputs.out_file = os.path.join(os.path.curdir, "falff.nii.gz") - wf.connect(input_node, 'rest_mask', falff, 'in_file_a') - wf.connect(stddev_filtered, 'out_file', falff, 'in_file_b') - wf.connect(stddev_unfiltered, 'out_file', falff, 'in_file_c') + wf.connect(input_node, "rest_mask", falff, "in_file_a") + wf.connect(stddev_filtered, "out_file", falff, "in_file_b") + wf.connect(stddev_unfiltered, "out_file", falff, "in_file_c") - wf.connect(falff, 'out_file', output_node, 'falff_img') + wf.connect(falff, "out_file", output_node, "falff_img") return wf @@ -255,26 +254,22 @@ def create_alff(wf_name='alff_workflow'): outputs=["alff", "falff"], ) def alff_falff(wf, cfg, strat_pool, pipe_num, opt=None): + alff = create_alff(f"alff_falff_{pipe_num}") - alff = create_alff(f'alff_falff_{pipe_num}') - - alff.inputs.hp_input.hp = \ - cfg.amplitude_low_frequency_fluctuation['highpass_cutoff'] - alff.inputs.lp_input.lp = \ - cfg.amplitude_low_frequency_fluctuation['lowpass_cutoff'] - alff.get_node('hp_input').iterables = ('hp', alff.inputs.hp_input.hp) - alff.get_node('lp_input').iterables = ('lp', alff.inputs.lp_input.lp) + alff.inputs.hp_input.hp = cfg.amplitude_low_frequency_fluctuation["highpass_cutoff"] + alff.inputs.lp_input.lp = cfg.amplitude_low_frequency_fluctuation["lowpass_cutoff"] + alff.get_node("hp_input").iterables = ("hp", alff.inputs.hp_input.hp) + alff.get_node("lp_input").iterables = ("lp", alff.inputs.lp_input.lp) - node, out = strat_pool.get_data(["desc-denoisedNofilt_bold", - "desc-preproc_bold"]) - wf.connect(node, out, alff, 'inputspec.rest_res') + node, out = strat_pool.get_data(["desc-denoisedNofilt_bold", "desc-preproc_bold"]) + wf.connect(node, out, alff, "inputspec.rest_res") - node, out = strat_pool.get_data('space-bold_desc-brain_mask') - wf.connect(node, out, alff, 'inputspec.rest_mask') + node, out = strat_pool.get_data("space-bold_desc-brain_mask") + wf.connect(node, out, alff, "inputspec.rest_mask") outputs = { - 'alff': (alff, 'outputspec.alff_img'), - 'falff': (alff, 'outputspec.falff_img') + "alff": (alff, "outputspec.alff_img"), + "falff": (alff, "outputspec.falff_img"), } return (wf, outputs) @@ -289,77 +284,94 @@ def alff_falff(wf, cfg, strat_pool, pipe_num, opt=None): [ "space-template_res-derivative_desc-denoisedNofilt_bold", "space-template_res-derivative_desc-preproc_bold", - "space-template_desc-preproc_bold" + "space-template_desc-preproc_bold", ], [ "space-template_res-derivative_desc-bold_mask", - "space-template_desc-bold_mask" + "space-template_desc-bold_mask", ], - "desc-denoisedNofilt_bold", + "desc-denoisedNofilt_bold", "from-bold_to-template_mode-image_xfm", "T1w-brain-template-deriv", ) ], - outputs=["space-template_alff", "space-template_falff", - "space-template_res-derivative_desc-denoisedNofilt_bold"], + outputs=[ + "space-template_alff", + "space-template_falff", + "space-template_res-derivative_desc-denoisedNofilt_bold", + ], ) def alff_falff_space_template(wf, cfg, strat_pool, pipe_num, opt=None): - - outputs = {} + outputs = {} if strat_pool.check_rpool("desc-denoisedNofilt_bold"): xfm_prov = strat_pool.get_cpac_provenance( - 'from-bold_to-template_mode-image_xfm') + "from-bold_to-template_mode-image_xfm" + ) reg_tool = check_prov_for_regtool(xfm_prov) - num_cpus = cfg.pipeline_setup['system_config'][ - 'max_cores_per_participant'] + num_cpus = cfg.pipeline_setup["system_config"]["max_cores_per_participant"] - num_ants_cores = cfg.pipeline_setup['system_config']['num_ants_threads'] + num_ants_cores = cfg.pipeline_setup["system_config"]["num_ants_threads"] - apply_xfm = apply_transform(f'warp_denoisedNofilt_to_T1template_{pipe_num}', reg_tool, - time_series=True, num_cpus=num_cpus, - num_ants_cores=num_ants_cores) + apply_xfm = apply_transform( + f"warp_denoisedNofilt_to_T1template_{pipe_num}", + reg_tool, + time_series=True, + num_cpus=num_cpus, + num_ants_cores=num_ants_cores, + ) - if reg_tool == 'ants': + if reg_tool == "ants": apply_xfm.inputs.inputspec.interpolation = cfg.registration_workflows[ - 'functional_registration']['func_registration_to_template'][ - 'ANTs_pipelines']['interpolation'] - elif reg_tool == 'fsl': + "functional_registration" + ]["func_registration_to_template"]["ANTs_pipelines"]["interpolation"] + elif reg_tool == "fsl": apply_xfm.inputs.inputspec.interpolation = cfg.registration_workflows[ - 'functional_registration']['func_registration_to_template'][ - 'FNIRT_pipelines']['interpolation'] + "functional_registration" + ]["func_registration_to_template"]["FNIRT_pipelines"]["interpolation"] node, out = strat_pool.get_data("desc-denoisedNofilt_bold") - wf.connect(node, out, apply_xfm, 'inputspec.input_image') + wf.connect(node, out, apply_xfm, "inputspec.input_image") node, out = strat_pool.get_data("T1w-brain-template-deriv") - wf.connect(node, out, apply_xfm, 'inputspec.reference') + wf.connect(node, out, apply_xfm, "inputspec.reference") node, out = strat_pool.get_data("from-bold_to-template_mode-image_xfm") - wf.connect(node, out, apply_xfm, 'inputspec.transform') + wf.connect(node, out, apply_xfm, "inputspec.transform") outputs = { - f'space-template_res-derivative_desc-denoisedNofilt_bold': (apply_xfm, 'outputspec.output_image') + "space-template_res-derivative_desc-denoisedNofilt_bold": ( + apply_xfm, + "outputspec.output_image", + ) + } + alff = create_alff(f"alff_falff_{pipe_num}") + + alff.inputs.hp_input.hp = cfg.amplitude_low_frequency_fluctuation["highpass_cutoff"] + alff.inputs.lp_input.lp = cfg.amplitude_low_frequency_fluctuation["lowpass_cutoff"] + alff.get_node("hp_input").iterables = ("hp", alff.inputs.hp_input.hp) + alff.get_node("lp_input").iterables = ("lp", alff.inputs.lp_input.lp) + node, out = strat_pool.get_data( + [ + "space-template_res-derivative_desc-denoisedNofilt_bold", + "space-template_res-derivative_desc-preproc_bold", + "space-template_desc-preproc_bold", + ] + ) + wf.connect(node, out, alff, "inputspec.rest_res") + node, out = strat_pool.get_data( + [ + "space-template_res-derivative_desc-bold_mask", + "space-template_desc-bold_mask", + ] + ) + wf.connect(node, out, alff, "inputspec.rest_mask") + + outputs.update( + { + "space-template_alff": (alff, "outputspec.alff_img"), + "space-template_falff": (alff, "outputspec.falff_img"), } - alff = create_alff(f'alff_falff_{pipe_num}') - - alff.inputs.hp_input.hp = \ - cfg.amplitude_low_frequency_fluctuation['highpass_cutoff'] - alff.inputs.lp_input.lp = \ - cfg.amplitude_low_frequency_fluctuation['lowpass_cutoff'] - alff.get_node('hp_input').iterables = ('hp', alff.inputs.hp_input.hp) - alff.get_node('lp_input').iterables = ('lp', alff.inputs.lp_input.lp) - node, out = strat_pool.get_data(["space-template_res-derivative_desc-denoisedNofilt_bold", - "space-template_res-derivative_desc-preproc_bold", - "space-template_desc-preproc_bold"]) - wf.connect(node, out, alff, 'inputspec.rest_res') - node, out = strat_pool.get_data(["space-template_res-derivative_desc-bold_mask", - "space-template_desc-bold_mask"]) - wf.connect(node, out, alff, 'inputspec.rest_mask') - - outputs.update({ - 'space-template_alff': (alff, 'outputspec.alff_img'), - 'space-template_falff': (alff, 'outputspec.falff_img') - }) + ) return (wf, outputs) diff --git a/CPAC/alff/utils.py b/CPAC/alff/utils.py index 75b10e8154..69f2cd4b8d 100644 --- a/CPAC/alff/utils.py +++ b/CPAC/alff/utils.py @@ -1,21 +1,19 @@ # -*- coding: utf-8 -*- + def get_opt_string(mask): """ - Method to return option string for 3dTstat - + Method to return option string for 3dTstat. + Parameters ---------- mask : string Path to mask file - + Returns ------- opt_str : string Command args - - """ - - opt_str = " -stdev -mask %s" % mask - return opt_str + """ + return " -stdev -mask %s" % mask diff --git a/CPAC/anat_preproc/__init__.py b/CPAC/anat_preproc/__init__.py index 2e9c17d6d2..40a96afc6f 100644 --- a/CPAC/anat_preproc/__init__.py +++ b/CPAC/anat_preproc/__init__.py @@ -1,3 +1 @@ # -*- coding: utf-8 -*- - -from CPAC.anat_preproc.lesion_preproc import create_lesion_preproc diff --git a/CPAC/anat_preproc/anat_preproc.py b/CPAC/anat_preproc/anat_preproc.py index 1dcb5b8323..11ea616378 100644 --- a/CPAC/anat_preproc/anat_preproc.py +++ b/CPAC/anat_preproc/anat_preproc.py @@ -17,407 +17,496 @@ # License along with C-PAC. If not, see . # from copy import deepcopy import os -from CPAC.pipeline.nodeblock import nodeblock -from nipype.interfaces import afni -from nipype.interfaces import ants -from nipype.interfaces import fsl -from nipype.interfaces import freesurfer -import nipype.interfaces.utility as util + +from nipype.interfaces import afni, ants, freesurfer, fsl from nipype.interfaces.fsl import utils as fsl_utils -from CPAC.pipeline import nipype_pipeline_engine as pe +import nipype.interfaces.utility as util + from CPAC.anat_preproc.ants import init_brain_extraction_wf -from CPAC.anat_preproc.utils import create_3dskullstrip_arg_string, \ - freesurfer_hemispheres, \ - fsl_aff_to_rigid, \ - mri_convert, \ - wb_command, \ - fslmaths_command, \ - VolumeRemoveIslands, \ - normalize_wmparc +from CPAC.anat_preproc.utils import ( + VolumeRemoveIslands, + create_3dskullstrip_arg_string, + freesurfer_hemispheres, + fsl_aff_to_rigid, + fslmaths_command, + mri_convert, + normalize_wmparc, + wb_command, +) +from CPAC.pipeline import nipype_pipeline_engine as pe +from CPAC.pipeline.nodeblock import nodeblock from CPAC.utils.interfaces.fsl import Merge as fslMerge -def acpc_alignment(config=None, acpc_target='whole-head', mask=False, - wf_name='acpc_align'): +def acpc_alignment( + config=None, acpc_target="whole-head", mask=False, wf_name="acpc_align" +): preproc = pe.Workflow(name=wf_name) - inputnode = pe.Node(util.IdentityInterface(fields=['anat_leaf', - 'anat_brain', - 'brain_mask', - 'template_brain_only_for_anat', - 'template_brain_for_acpc', - 'template_head_for_acpc']), - name='inputspec') - - output_node = pe.Node(util.IdentityInterface(fields=['acpc_aligned_head', - 'acpc_brain_mask', - 'from-T1w_to-ACPC_mode-image_desc-aff2rig_xfm', - 'acpc_aligned_brain', - 'acpc_brain_mask']), - name='outputspec') - if config.anatomical_preproc['acpc_alignment']['FOV_crop'] == 'robustfov': - robust_fov = pe.Node(interface=fsl_utils.RobustFOV(), - name='anat_acpc_1_robustfov') - robust_fov.inputs.brainsize = config.anatomical_preproc['acpc_alignment']['brain_size'] - robust_fov.inputs.out_transform = 'fov_xfm.mat' - - fov, in_file = (robust_fov, 'in_file') - fov, fov_mtx = (robust_fov, 'out_transform') - fov, fov_outfile = (robust_fov, 'out_roi') - - elif config.anatomical_preproc['acpc_alignment']['FOV_crop'] == 'flirt': + inputnode = pe.Node( + util.IdentityInterface( + fields=[ + "anat_leaf", + "anat_brain", + "brain_mask", + "template_brain_only_for_anat", + "template_brain_for_acpc", + "template_head_for_acpc", + ] + ), + name="inputspec", + ) + + output_node = pe.Node( + util.IdentityInterface( + fields=[ + "acpc_aligned_head", + "acpc_brain_mask", + "from-T1w_to-ACPC_mode-image_desc-aff2rig_xfm", + "acpc_aligned_brain", + "acpc_brain_mask", + ] + ), + name="outputspec", + ) + if config.anatomical_preproc["acpc_alignment"]["FOV_crop"] == "robustfov": + robust_fov = pe.Node( + interface=fsl_utils.RobustFOV(), name="anat_acpc_1_robustfov" + ) + robust_fov.inputs.brainsize = config.anatomical_preproc["acpc_alignment"][ + "brain_size" + ] + robust_fov.inputs.out_transform = "fov_xfm.mat" + + fov, in_file = (robust_fov, "in_file") + fov, fov_mtx = (robust_fov, "out_transform") + fov, fov_outfile = (robust_fov, "out_roi") + + elif config.anatomical_preproc["acpc_alignment"]["FOV_crop"] == "flirt": # robustfov doesn't work on some monkey data. prefer using flirt. # ${FSLDIR}/bin/flirt -in "${Input}" -applyxfm -ref "${Input}" -omat "$WD"/roi2full.mat -out "$WD"/robustroi.nii.gz # adopted from DCAN NHP https://github.com/DCAN-Labs/dcan-macaque-pipeline/blob/master/PreFreeSurfer/scripts/ACPCAlignment.sh#L80-L81 - flirt_fov = pe.Node(interface=fsl.FLIRT(), - name='anat_acpc_1_fov') - flirt_fov.inputs.args = '-applyxfm' + flirt_fov = pe.Node(interface=fsl.FLIRT(), name="anat_acpc_1_fov") + flirt_fov.inputs.args = "-applyxfm" - fov, in_file = (flirt_fov, 'in_file') - fov, ref_file = (flirt_fov, 'reference') - fov, fov_mtx = (flirt_fov, 'out_matrix_file') - fov, fov_outfile = (flirt_fov, 'out_file') + fov, in_file = (flirt_fov, "in_file") + fov, ref_file = (flirt_fov, "reference") + fov, fov_mtx = (flirt_fov, "out_matrix_file") + fov, fov_outfile = (flirt_fov, "out_file") # align head-to-head to get acpc.mat (for human) - if acpc_target == 'whole-head': - preproc.connect(inputnode, 'anat_leaf', fov, in_file) - if config.anatomical_preproc['acpc_alignment']['FOV_crop'] == 'flirt': - preproc.connect(inputnode, 'anat_leaf', fov, ref_file) + if acpc_target == "whole-head": + preproc.connect(inputnode, "anat_leaf", fov, in_file) + if config.anatomical_preproc["acpc_alignment"]["FOV_crop"] == "flirt": + preproc.connect(inputnode, "anat_leaf", fov, ref_file) # align brain-to-brain to get acpc.mat (for monkey) - if acpc_target == 'brain': - preproc.connect(inputnode, 'anat_brain', fov, in_file) - if config.anatomical_preproc['acpc_alignment']['FOV_crop'] == 'flirt': - preproc.connect(inputnode, 'anat_brain', fov, ref_file) + if acpc_target == "brain": + preproc.connect(inputnode, "anat_brain", fov, in_file) + if config.anatomical_preproc["acpc_alignment"]["FOV_crop"] == "flirt": + preproc.connect(inputnode, "anat_brain", fov, ref_file) - convert_fov_xfm = pe.Node(interface=fsl_utils.ConvertXFM(), - name='anat_acpc_2_fov_convertxfm') + convert_fov_xfm = pe.Node( + interface=fsl_utils.ConvertXFM(), name="anat_acpc_2_fov_convertxfm" + ) convert_fov_xfm.inputs.invert_xfm = True - preproc.connect(fov, fov_mtx, - convert_fov_xfm, 'in_file') + preproc.connect(fov, fov_mtx, convert_fov_xfm, "in_file") - align = pe.Node(interface=fsl.FLIRT(), - name='anat_acpc_3_flirt') - align.inputs.interp = 'spline' + align = pe.Node(interface=fsl.FLIRT(), name="anat_acpc_3_flirt") + align.inputs.interp = "spline" align.inputs.searchr_x = [30, 30] align.inputs.searchr_y = [30, 30] align.inputs.searchr_z = [30, 30] - preproc.connect(fov, fov_outfile, align, 'in_file') + preproc.connect(fov, fov_outfile, align, "in_file") # align head-to-head to get acpc.mat (for human) - if acpc_target == 'whole-head': - preproc.connect(inputnode, 'template_head_for_acpc', align, - 'reference') + if acpc_target == "whole-head": + preproc.connect(inputnode, "template_head_for_acpc", align, "reference") # align brain-to-brain to get acpc.mat (for monkey) - if acpc_target == 'brain': - preproc.connect(inputnode, 'template_brain_for_acpc', align, - 'reference') + if acpc_target == "brain": + preproc.connect(inputnode, "template_brain_for_acpc", align, "reference") - concat_xfm = pe.Node(interface=fsl_utils.ConvertXFM(), - name='anat_acpc_4_concatxfm') + concat_xfm = pe.Node(interface=fsl_utils.ConvertXFM(), name="anat_acpc_4_concatxfm") concat_xfm.inputs.concat_xfm = True - preproc.connect(convert_fov_xfm, 'out_file', concat_xfm, 'in_file') - preproc.connect(align, 'out_matrix_file', concat_xfm, 'in_file2') + preproc.connect(convert_fov_xfm, "out_file", concat_xfm, "in_file") + preproc.connect(align, "out_matrix_file", concat_xfm, "in_file2") - aff_to_rig_imports = ['import os', 'from numpy import *'] - aff_to_rig = pe.Node(util.Function(input_names=['in_xfm', 'out_name'], - output_names=['out_mat'], - function=fsl_aff_to_rigid, - imports=aff_to_rig_imports), - name='anat_acpc_5_aff2rigid') - aff_to_rig.inputs.out_name = 'acpc.mat' + aff_to_rig_imports = ["import os", "from numpy import *"] + aff_to_rig = pe.Node( + util.Function( + input_names=["in_xfm", "out_name"], + output_names=["out_mat"], + function=fsl_aff_to_rigid, + imports=aff_to_rig_imports, + ), + name="anat_acpc_5_aff2rigid", + ) + aff_to_rig.inputs.out_name = "acpc.mat" - preproc.connect(concat_xfm, 'out_file', aff_to_rig, 'in_xfm') - preproc.connect(aff_to_rig, 'out_mat', output_node, 'from-T1w_to-ACPC_mode-image_desc-aff2rig_xfm') + preproc.connect(concat_xfm, "out_file", aff_to_rig, "in_xfm") + preproc.connect( + aff_to_rig, + "out_mat", + output_node, + "from-T1w_to-ACPC_mode-image_desc-aff2rig_xfm", + ) - apply_xfm = pe.Node(interface=fsl.ApplyWarp(), - name='anat_acpc_6_applywarp') - apply_xfm.inputs.interp = 'spline' + apply_xfm = pe.Node(interface=fsl.ApplyWarp(), name="anat_acpc_6_applywarp") + apply_xfm.inputs.interp = "spline" apply_xfm.inputs.relwarp = True - preproc.connect(inputnode, 'anat_leaf', apply_xfm, 'in_file') - preproc.connect(inputnode, 'template_head_for_acpc', apply_xfm, - 'ref_file') - preproc.connect(aff_to_rig, 'out_mat', apply_xfm, 'premat') - preproc.connect(apply_xfm, 'out_file', output_node, 'acpc_aligned_head') + preproc.connect(inputnode, "anat_leaf", apply_xfm, "in_file") + preproc.connect(inputnode, "template_head_for_acpc", apply_xfm, "ref_file") + preproc.connect(aff_to_rig, "out_mat", apply_xfm, "premat") + preproc.connect(apply_xfm, "out_file", output_node, "acpc_aligned_head") - if acpc_target == 'brain': - apply_xfm_brain = pe.Node(interface=fsl.ApplyWarp(), - name='anat_acpc_brain_6_applywarp') - apply_xfm_brain.inputs.interp = 'spline' + if acpc_target == "brain": + apply_xfm_brain = pe.Node( + interface=fsl.ApplyWarp(), name="anat_acpc_brain_6_applywarp" + ) + apply_xfm_brain.inputs.interp = "spline" apply_xfm_brain.inputs.relwarp = True - preproc.connect(inputnode, 'anat_brain', apply_xfm_brain, 'in_file') - preproc.connect(inputnode, 'template_brain_for_acpc', apply_xfm_brain, - 'ref_file') - preproc.connect(aff_to_rig, 'out_mat', apply_xfm_brain, 'premat') - preproc.connect(apply_xfm_brain, 'out_file', output_node, 'acpc_aligned_brain') + preproc.connect(inputnode, "anat_brain", apply_xfm_brain, "in_file") + preproc.connect( + inputnode, "template_brain_for_acpc", apply_xfm_brain, "ref_file" + ) + preproc.connect(aff_to_rig, "out_mat", apply_xfm_brain, "premat") + preproc.connect(apply_xfm_brain, "out_file", output_node, "acpc_aligned_brain") if mask: - apply_xfm_mask = pe.Node(interface=fsl.ApplyWarp(), - name='anat_mask_acpc_7_applywarp') - apply_xfm_mask.inputs.interp = 'nn' + apply_xfm_mask = pe.Node( + interface=fsl.ApplyWarp(), name="anat_mask_acpc_7_applywarp" + ) + apply_xfm_mask.inputs.interp = "nn" apply_xfm_mask.inputs.relwarp = True - preproc.connect(inputnode, 'brain_mask', apply_xfm_mask, 'in_file') - preproc.connect(inputnode, 'template_brain_for_acpc', apply_xfm_mask, - 'ref_file') - preproc.connect(aff_to_rig, 'out_mat', apply_xfm_mask, 'premat') - preproc.connect(apply_xfm_mask, 'out_file', output_node, - 'acpc_brain_mask') + preproc.connect(inputnode, "brain_mask", apply_xfm_mask, "in_file") + preproc.connect( + inputnode, "template_brain_for_acpc", apply_xfm_mask, "ref_file" + ) + preproc.connect(aff_to_rig, "out_mat", apply_xfm_mask, "premat") + preproc.connect(apply_xfm_mask, "out_file", output_node, "acpc_brain_mask") return preproc -def T2wToT1wReg(wf_name='T2w_to_T1w_reg'): - +def T2wToT1wReg(wf_name="T2w_to_T1w_reg"): # Adapted from DCAN lab # https://github.com/DCAN-Labs/dcan-macaque-pipeline/blob/master/PreFreeSurfer/scripts/T2wToT1wReg.sh preproc = pe.Workflow(name=wf_name) - inputnode = pe.Node(util.IdentityInterface(fields=['T1w', - 'T1w_brain', - 'T2w', - 'T2w_brain']), - name='inputspec') + inputnode = pe.Node( + util.IdentityInterface(fields=["T1w", "T1w_brain", "T2w", "T2w_brain"]), + name="inputspec", + ) - outputnode = pe.Node(util.IdentityInterface(fields=['T2w_to_T1w']), - name='outputspec') + outputnode = pe.Node( + util.IdentityInterface(fields=["T2w_to_T1w"]), name="outputspec" + ) # ${FSLDIR}/bin/epi_reg --epi="$T2wImageBrain" --t1="$T1wImage" --t1brain="$WD"/"$T1wImageBrainFile" --out="$WD"/T2w2T1w - T2w2T1w_reg = pe.Node(interface=fsl.EpiReg(), - name='T2w2T1w_reg') - T2w2T1w_reg.inputs.out_base = 'T2w2T1w' + T2w2T1w_reg = pe.Node(interface=fsl.EpiReg(), name="T2w2T1w_reg") + T2w2T1w_reg.inputs.out_base = "T2w2T1w" - preproc.connect(inputnode, 'T2w_brain', T2w2T1w_reg ,'epi') - preproc.connect(inputnode, 'T1w', T2w2T1w_reg ,'t1_head') - preproc.connect(inputnode, 'T1w_brain', T2w2T1w_reg ,'t1_brain') + preproc.connect(inputnode, "T2w_brain", T2w2T1w_reg, "epi") + preproc.connect(inputnode, "T1w", T2w2T1w_reg, "t1_head") + preproc.connect(inputnode, "T1w_brain", T2w2T1w_reg, "t1_brain") # ${FSLDIR}/bin/applywarp --rel --interp=spline --in="$T2wImage" --ref="$T1wImage" --premat="$WD"/T2w2T1w.mat --out="$WD"/T2w2T1w - T2w2T1w = pe.Node(interface=fsl.ApplyWarp(), - name='T2w2T1w_applywarp') - T2w2T1w.inputs.interp = 'spline' + T2w2T1w = pe.Node(interface=fsl.ApplyWarp(), name="T2w2T1w_applywarp") + T2w2T1w.inputs.interp = "spline" T2w2T1w.inputs.relwarp = True - preproc.connect(inputnode, 'T2w', T2w2T1w, 'in_file') - preproc.connect(inputnode, 'T1w', T2w2T1w, 'ref_file') - preproc.connect(T2w2T1w_reg, 'epi2str_mat', T2w2T1w, 'premat') + preproc.connect(inputnode, "T2w", T2w2T1w, "in_file") + preproc.connect(inputnode, "T1w", T2w2T1w, "ref_file") + preproc.connect(T2w2T1w_reg, "epi2str_mat", T2w2T1w, "premat") # ${FSLDIR}/bin/fslmaths "$WD"/T2w2T1w -add 1 "$WD"/T2w2T1w -odt float - T2w2T1w_final = pe.Node(interface=fsl.ImageMaths(), - name='T2w2T1w_final') - T2w2T1w_final.inputs.op_string = "-add 1" + T2w2T1w_final = pe.Node(interface=fsl.ImageMaths(), name="T2w2T1w_final") + T2w2T1w_final.inputs.op_string = "-add 1" - preproc.connect(T2w2T1w, 'out_file', T2w2T1w_final, 'in_file') - preproc.connect(T2w2T1w_final, 'out_file', outputnode, 'T2w_to_T1w') + preproc.connect(T2w2T1w, "out_file", T2w2T1w_final, "in_file") + preproc.connect(T2w2T1w_final, "out_file", outputnode, "T2w_to_T1w") return preproc -def BiasFieldCorrection_sqrtT1wXT1w(config=None, wf_name='biasfield_correction_t1t2'): - +def BiasFieldCorrection_sqrtT1wXT1w(config=None, wf_name="biasfield_correction_t1t2"): # Adapted from DCAN lab # https://github.com/DCAN-Labs/dcan-macaque-pipeline/blob/master/PreFreeSurfer/scripts/BiasFieldCorrection_sqrtT1wXT1w.sh preproc = pe.Workflow(name=wf_name) - inputnode = pe.Node(util.IdentityInterface(fields=['T1w', - 'T1w_brain', - 'T2w']), - name='inputspec') + inputnode = pe.Node( + util.IdentityInterface(fields=["T1w", "T1w_brain", "T2w"]), name="inputspec" + ) - outputnode = pe.Node(util.IdentityInterface(fields=['T1w_biascorrected', - 'T1w_brain_biascorrected', - 'T2w_biascorrected', - 'T2w_brain_biascorrected', - 'biasfield']), - name='outputspec') + outputnode = pe.Node( + util.IdentityInterface( + fields=[ + "T1w_biascorrected", + "T1w_brain_biascorrected", + "T2w_biascorrected", + "T2w_brain_biascorrected", + "biasfield", + ] + ), + name="outputspec", + ) # 1. Form sqrt(T1w*T2w), mask this and normalise by the mean # ${FSLDIR}/bin/fslmaths $T1wImage -mul $T2wImage -abs -sqrt $WD/T1wmulT2w.nii.gz -odt float - T1wmulT2w = pe.Node(interface=fsl.MultiImageMaths(), - name='T1wmulT2w') + T1wmulT2w = pe.Node(interface=fsl.MultiImageMaths(), name="T1wmulT2w") T1wmulT2w.inputs.op_string = "-mul %s -abs -sqrt" - - preproc.connect(inputnode, 'T1w', T1wmulT2w, 'in_file') - preproc.connect(inputnode, 'T2w', T1wmulT2w, 'operand_files') + + preproc.connect(inputnode, "T1w", T1wmulT2w, "in_file") + preproc.connect(inputnode, "T2w", T1wmulT2w, "operand_files") # ${FSLDIR}/bin/fslmaths $WD/T1wmulT2w.nii.gz -mas $T1wImageBrain $WD/T1wmulT2w_brain.nii.gz - T1wmulT2w_brain = pe.Node(interface=fsl.MultiImageMaths(), - name='T1wmulT2w_brain') + T1wmulT2w_brain = pe.Node(interface=fsl.MultiImageMaths(), name="T1wmulT2w_brain") T1wmulT2w_brain.inputs.op_string = "-mas %s " - preproc.connect(T1wmulT2w, 'out_file', T1wmulT2w_brain, 'in_file') - preproc.connect(inputnode, 'T1w_brain', T1wmulT2w_brain, 'operand_files') + preproc.connect(T1wmulT2w, "out_file", T1wmulT2w_brain, "in_file") + preproc.connect(inputnode, "T1w_brain", T1wmulT2w_brain, "operand_files") # meanbrainval=`${FSLDIR}/bin/fslstats $WD/T1wmulT2w_brain.nii.gz -M` - meanbrainval = pe.Node(interface=fsl.ImageStats(), - name='image_stats', - iterfield=['in_file']) - meanbrainval.inputs.op_string = '-M' + meanbrainval = pe.Node( + interface=fsl.ImageStats(), name="image_stats", iterfield=["in_file"] + ) + meanbrainval.inputs.op_string = "-M" - preproc.connect(T1wmulT2w_brain, 'out_file', meanbrainval, 'in_file') + preproc.connect(T1wmulT2w_brain, "out_file", meanbrainval, "in_file") # ${FSLDIR}/bin/fslmaths $WD/T1wmulT2w_brain.nii.gz -div $meanbrainval $WD/T1wmulT2w_brain_norm.nii.gz - T1wmulT2w_brain_norm = pe.Node(interface=fsl.ImageMaths(), - name='T1wmulT2w_brain_norm') - - def form_meanbrainval_string(meanbrainval): - return '-div %f' % (meanbrainval) + T1wmulT2w_brain_norm = pe.Node( + interface=fsl.ImageMaths(), name="T1wmulT2w_brain_norm" + ) - preproc.connect(T1wmulT2w_brain, 'out_file', T1wmulT2w_brain_norm, 'in_file') - preproc.connect(meanbrainval, ('out_stat', form_meanbrainval_string), - T1wmulT2w_brain_norm, 'op_string') + def form_meanbrainval_string(meanbrainval): + return "-div %f" % (meanbrainval) + + preproc.connect(T1wmulT2w_brain, "out_file", T1wmulT2w_brain_norm, "in_file") + preproc.connect( + meanbrainval, + ("out_stat", form_meanbrainval_string), + T1wmulT2w_brain_norm, + "op_string", + ) # 2. Smooth the normalised sqrt image, using within-mask smoothing : s(Mask*X)/s(Mask) # ${FSLDIR}/bin/fslmaths $WD/T1wmulT2w_brain_norm.nii.gz -bin -s $BiasFieldSmoothingSigma $WD/SmoothNorm_s${BiasFieldSmoothingSigma}.nii.gz - SmoothNorm = pe.Node(interface=fsl.ImageMaths(), - name='SmoothNorm') - SmoothNorm.inputs.op_string = "-bin -s %f" % (config.anatomical_preproc['t1t2_bias_field_correction']['BiasFieldSmoothingSigma']) + SmoothNorm = pe.Node(interface=fsl.ImageMaths(), name="SmoothNorm") + SmoothNorm.inputs.op_string = ( + "-bin -s %f" + % ( + config.anatomical_preproc["t1t2_bias_field_correction"][ + "BiasFieldSmoothingSigma" + ] + ) + ) - preproc.connect(T1wmulT2w_brain_norm, 'out_file', SmoothNorm, 'in_file') + preproc.connect(T1wmulT2w_brain_norm, "out_file", SmoothNorm, "in_file") # ${FSLDIR}/bin/fslmaths $WD/T1wmulT2w_brain_norm.nii.gz -s $BiasFieldSmoothingSigma -div $WD/SmoothNorm_s${BiasFieldSmoothingSigma}.nii.gz $WD/T1wmulT2w_brain_norm_s${BiasFieldSmoothingSigma}.nii.gz def T1wmulT2w_brain_norm_s_string(sigma, in_file): - return "-s %f -div %s" %(sigma, in_file) - - T1wmulT2w_brain_norm_s_string = pe.Node(util.Function(input_names=['sigma', 'in_file'], - output_names=['out_str'], - function=T1wmulT2w_brain_norm_s_string), - name='T1wmulT2w_brain_norm_s_string') - T1wmulT2w_brain_norm_s_string.inputs.sigma = config.anatomical_preproc['t1t2_bias_field_correction']['BiasFieldSmoothingSigma'] - - preproc.connect(SmoothNorm, 'out_file', T1wmulT2w_brain_norm_s_string, 'in_file') - - T1wmulT2w_brain_norm_s = pe.Node(interface=fsl.ImageMaths(), - name='T1wmulT2w_brain_norm_s') - - preproc.connect(T1wmulT2w_brain_norm, 'out_file', T1wmulT2w_brain_norm_s, 'in_file') - preproc.connect(T1wmulT2w_brain_norm_s_string, 'out_str', T1wmulT2w_brain_norm_s, 'op_string') + return "-s %f -div %s" % (sigma, in_file) + + T1wmulT2w_brain_norm_s_string = pe.Node( + util.Function( + input_names=["sigma", "in_file"], + output_names=["out_str"], + function=T1wmulT2w_brain_norm_s_string, + ), + name="T1wmulT2w_brain_norm_s_string", + ) + T1wmulT2w_brain_norm_s_string.inputs.sigma = config.anatomical_preproc[ + "t1t2_bias_field_correction" + ]["BiasFieldSmoothingSigma"] + + preproc.connect(SmoothNorm, "out_file", T1wmulT2w_brain_norm_s_string, "in_file") + + T1wmulT2w_brain_norm_s = pe.Node( + interface=fsl.ImageMaths(), name="T1wmulT2w_brain_norm_s" + ) + + preproc.connect(T1wmulT2w_brain_norm, "out_file", T1wmulT2w_brain_norm_s, "in_file") + preproc.connect( + T1wmulT2w_brain_norm_s_string, "out_str", T1wmulT2w_brain_norm_s, "op_string" + ) # 3. Divide normalised sqrt image by smoothed version (to do simple bias correction) # ${FSLDIR}/bin/fslmaths $WD/T1wmulT2w_brain_norm.nii.gz -div $WD/T1wmulT2w_brain_norm_s$BiasFieldSmoothingSigma.nii.gz $WD/T1wmulT2w_brain_norm_modulate.nii.gz - T1wmulT2w_brain_norm_modulate = pe.Node(interface=fsl.MultiImageMaths(), - name='T1wmulT2w_brain_norm_modulate') - T1wmulT2w_brain_norm_modulate.inputs.op_string = "-div %s" + T1wmulT2w_brain_norm_modulate = pe.Node( + interface=fsl.MultiImageMaths(), name="T1wmulT2w_brain_norm_modulate" + ) + T1wmulT2w_brain_norm_modulate.inputs.op_string = "-div %s" - preproc.connect(T1wmulT2w_brain_norm, 'out_file', T1wmulT2w_brain_norm_modulate, 'in_file') - preproc.connect(T1wmulT2w_brain_norm_s, 'out_file', T1wmulT2w_brain_norm_modulate, 'operand_files') + preproc.connect( + T1wmulT2w_brain_norm, "out_file", T1wmulT2w_brain_norm_modulate, "in_file" + ) + preproc.connect( + T1wmulT2w_brain_norm_s, + "out_file", + T1wmulT2w_brain_norm_modulate, + "operand_files", + ) # 4. Create a mask using a threshold at Mean - 0.5*Stddev, with filling of holes to remove any non-grey/white tissue. # STD=`${FSLDIR}/bin/fslstats $WD/T1wmulT2w_brain_norm_modulate.nii.gz -S` - STD = pe.Node(interface=fsl.ImageStats(), - name='STD', - iterfield=['in_file']) - STD.inputs.op_string = '-S' + STD = pe.Node(interface=fsl.ImageStats(), name="STD", iterfield=["in_file"]) + STD.inputs.op_string = "-S" - preproc.connect(T1wmulT2w_brain_norm_modulate, 'out_file', STD, 'in_file') + preproc.connect(T1wmulT2w_brain_norm_modulate, "out_file", STD, "in_file") # MEAN=`${FSLDIR}/bin/fslstats $WD/T1wmulT2w_brain_norm_modulate.nii.gz -M` - MEAN = pe.Node(interface=fsl.ImageStats(), - name='MEAN', - iterfield=['in_file']) - MEAN.inputs.op_string = '-M' + MEAN = pe.Node(interface=fsl.ImageStats(), name="MEAN", iterfield=["in_file"]) + MEAN.inputs.op_string = "-M" + + preproc.connect(T1wmulT2w_brain_norm_modulate, "out_file", MEAN, "in_file") - preproc.connect(T1wmulT2w_brain_norm_modulate, 'out_file', MEAN, 'in_file') - # Lower=`echo "$MEAN - ($STD * $Factor)" | bc -l` def form_lower_string(mean, std): - Factor = 0.5 #Leave this at 0.5 for now it is the number of standard deviations below the mean to threshold the non-brain tissues at - lower = str(float(mean)-(float(std)*float(Factor))) - return '-thr %s -bin -ero -mul 255' % (lower) - - form_lower_string = pe.Node(util.Function(input_names=['mean', 'std'], - output_names=['out_str'], - function=form_lower_string), - name='form_lower_string') + Factor = 0.5 # Leave this at 0.5 for now it is the number of standard deviations below the mean to threshold the non-brain tissues at + lower = str(float(mean) - (float(std) * float(Factor))) + return "-thr %s -bin -ero -mul 255" % (lower) + + form_lower_string = pe.Node( + util.Function( + input_names=["mean", "std"], + output_names=["out_str"], + function=form_lower_string, + ), + name="form_lower_string", + ) - preproc.connect(MEAN, 'out_stat', form_lower_string, 'mean') - preproc.connect(STD, 'out_stat', form_lower_string, 'std') + preproc.connect(MEAN, "out_stat", form_lower_string, "mean") + preproc.connect(STD, "out_stat", form_lower_string, "std") # ${FSLDIR}/bin/fslmaths $WD/T1wmulT2w_brain_norm_modulate -thr $Lower -bin -ero -mul 255 $WD/T1wmulT2w_brain_norm_modulate_mask - T1wmulT2w_brain_norm_modulate_mask = pe.Node(interface=fsl.ImageMaths(), - name='T1wmulT2w_brain_norm_modulate_mask') + T1wmulT2w_brain_norm_modulate_mask = pe.Node( + interface=fsl.ImageMaths(), name="T1wmulT2w_brain_norm_modulate_mask" + ) - preproc.connect(T1wmulT2w_brain_norm_modulate, 'out_file', T1wmulT2w_brain_norm_modulate_mask, 'in_file') - preproc.connect(form_lower_string, 'out_str', T1wmulT2w_brain_norm_modulate_mask, 'op_string') + preproc.connect( + T1wmulT2w_brain_norm_modulate, + "out_file", + T1wmulT2w_brain_norm_modulate_mask, + "in_file", + ) + preproc.connect( + form_lower_string, "out_str", T1wmulT2w_brain_norm_modulate_mask, "op_string" + ) # ${CARET7DIR}/wb_command -volume-remove-islands $WD/T1wmulT2w_brain_norm_modulate_mask.nii.gz $WD/T1wmulT2w_brain_norm_modulate_mask.nii.gz - T1wmulT2w_brain_norm_modulate_mask_roi = pe.Node(interface=VolumeRemoveIslands(), - name='remove_islands') + T1wmulT2w_brain_norm_modulate_mask_roi = pe.Node( + interface=VolumeRemoveIslands(), name="remove_islands" + ) - preproc.connect(T1wmulT2w_brain_norm_modulate_mask, 'out_file', T1wmulT2w_brain_norm_modulate_mask_roi, 'in_file') + preproc.connect( + T1wmulT2w_brain_norm_modulate_mask, + "out_file", + T1wmulT2w_brain_norm_modulate_mask_roi, + "in_file", + ) # 5. Extrapolate normalised sqrt image from mask region out to whole FOV # ${FSLDIR}/bin/fslmaths $WD/T1wmulT2w_brain_norm.nii.gz -mas $WD/T1wmulT2w_brain_norm_modulate_mask.nii.gz -dilall $WD/bias_raw.nii.gz -odt float - bias_raw = pe.Node(interface=fsl.MultiImageMaths(), - name='bias_raw') + bias_raw = pe.Node(interface=fsl.MultiImageMaths(), name="bias_raw") bias_raw.inputs.op_string = "-mas %s -dilall " - preproc.connect(T1wmulT2w_brain_norm, 'out_file', bias_raw, 'in_file') - preproc.connect(T1wmulT2w_brain_norm_modulate_mask_roi, 'out_file', bias_raw, 'operand_files') + preproc.connect(T1wmulT2w_brain_norm, "out_file", bias_raw, "in_file") + preproc.connect( + T1wmulT2w_brain_norm_modulate_mask_roi, "out_file", bias_raw, "operand_files" + ) # ${FSLDIR}/bin/fslmaths $WD/bias_raw.nii.gz -s $BiasFieldSmoothingSigma $OutputBiasField - OutputBiasField = pe.Node(interface=fsl.ImageMaths(), - name='OutputBiasField') - OutputBiasField.inputs.op_string = "-s %f " % (config.anatomical_preproc['t1t2_bias_field_correction']['BiasFieldSmoothingSigma']) + OutputBiasField = pe.Node(interface=fsl.ImageMaths(), name="OutputBiasField") + OutputBiasField.inputs.op_string = ( + "-s %f " + % ( + config.anatomical_preproc["t1t2_bias_field_correction"][ + "BiasFieldSmoothingSigma" + ] + ) + ) - preproc.connect(bias_raw, 'out_file', OutputBiasField, 'in_file') + preproc.connect(bias_raw, "out_file", OutputBiasField, "in_file") # 6. Use bias field output to create corrected images def file_to_a_list(infile_1, infile_2): - return list([infile_1,infile_2]) - - file_to_a_list = pe.Node(util.Function(input_names=['infile_1', 'infile_2'], - output_names=['out_list'], - function=file_to_a_list), - name='file_to_a_list') + return [infile_1, infile_2] - preproc.connect(OutputBiasField, 'out_file', file_to_a_list, 'infile_1') - preproc.connect(inputnode, 'T1w_brain', file_to_a_list, 'infile_2') + file_to_a_list = pe.Node( + util.Function( + input_names=["infile_1", "infile_2"], + output_names=["out_list"], + function=file_to_a_list, + ), + name="file_to_a_list", + ) + + preproc.connect(OutputBiasField, "out_file", file_to_a_list, "infile_1") + preproc.connect(inputnode, "T1w_brain", file_to_a_list, "infile_2") # ${FSLDIR}/bin/fslmaths $T1wImage -div $OutputBiasField -mas $T1wImageBrain $OutputT1wRestoredBrainImage -odt float - OutputT1wRestoredBrainImage = pe.Node(interface=fsl.MultiImageMaths(), - name='OutputT1wRestoredBrainImage') - OutputT1wRestoredBrainImage.inputs.op_string = "-div %s -mas %s " + OutputT1wRestoredBrainImage = pe.Node( + interface=fsl.MultiImageMaths(), name="OutputT1wRestoredBrainImage" + ) + OutputT1wRestoredBrainImage.inputs.op_string = "-div %s -mas %s " + + preproc.connect(inputnode, "T1w", OutputT1wRestoredBrainImage, "in_file") + preproc.connect( + file_to_a_list, "out_list", OutputT1wRestoredBrainImage, "operand_files" + ) - preproc.connect(inputnode, 'T1w', OutputT1wRestoredBrainImage, 'in_file') - preproc.connect(file_to_a_list,'out_list',OutputT1wRestoredBrainImage, 'operand_files') - # ${FSLDIR}/bin/fslmaths $T1wImage -div $OutputBiasField $OutputT1wRestoredImage -odt float - OutputT1wRestoredImage = pe.Node(interface=fsl.MultiImageMaths(), - name='OutputT1wRestoredImage') + OutputT1wRestoredImage = pe.Node( + interface=fsl.MultiImageMaths(), name="OutputT1wRestoredImage" + ) OutputT1wRestoredImage.inputs.op_string = "-div %s " - preproc.connect(inputnode, 'T1w', OutputT1wRestoredImage, 'in_file') - preproc.connect(OutputBiasField, 'out_file', OutputT1wRestoredImage, 'operand_files') + preproc.connect(inputnode, "T1w", OutputT1wRestoredImage, "in_file") + preproc.connect( + OutputBiasField, "out_file", OutputT1wRestoredImage, "operand_files" + ) # ${FSLDIR}/bin/fslmaths $T2wImage -div $OutputBiasField -mas $T1wImageBrain $OutputT2wRestoredBrainImage -odt float - OutputT2wRestoredBrainImage = pe.Node(interface=fsl.MultiImageMaths(), - name='OutputT2wRestoredBrainImage') - OutputT2wRestoredBrainImage.inputs.op_string = "-div %s -mas %s " - - preproc.connect(inputnode, 'T2w', OutputT2wRestoredBrainImage, 'in_file') - preproc.connect(file_to_a_list,'out_list',OutputT2wRestoredBrainImage, 'operand_files') + OutputT2wRestoredBrainImage = pe.Node( + interface=fsl.MultiImageMaths(), name="OutputT2wRestoredBrainImage" + ) + OutputT2wRestoredBrainImage.inputs.op_string = "-div %s -mas %s " + + preproc.connect(inputnode, "T2w", OutputT2wRestoredBrainImage, "in_file") + preproc.connect( + file_to_a_list, "out_list", OutputT2wRestoredBrainImage, "operand_files" + ) # ${FSLDIR}/bin/fslmaths $T2wImage -div $OutputBiasField $OutputT2wRestoredImage -odt float - OutputT2wRestoredImage = pe.Node(interface=fsl.MultiImageMaths(), - name='OutputT2wRestoredImage') + OutputT2wRestoredImage = pe.Node( + interface=fsl.MultiImageMaths(), name="OutputT2wRestoredImage" + ) OutputT2wRestoredImage.inputs.op_string = "-div %s " - preproc.connect(inputnode, 'T2w', OutputT2wRestoredImage, 'in_file') - preproc.connect(OutputBiasField, 'out_file', OutputT2wRestoredImage, 'operand_files') + preproc.connect(inputnode, "T2w", OutputT2wRestoredImage, "in_file") + preproc.connect( + OutputBiasField, "out_file", OutputT2wRestoredImage, "operand_files" + ) - preproc.connect(OutputT1wRestoredImage, 'out_file', outputnode, 'T1w_biascorrected') - preproc.connect(OutputT1wRestoredBrainImage, 'out_file', outputnode, 'T1w_brain_biascorrected') - preproc.connect(OutputT2wRestoredImage, 'out_file', outputnode, 'T2w_biascorrected') - preproc.connect(OutputT2wRestoredBrainImage, 'out_file', outputnode, 'T2w_brain_biascorrected') - preproc.connect(OutputBiasField, 'out_file', outputnode, 'biasfield') + preproc.connect(OutputT1wRestoredImage, "out_file", outputnode, "T1w_biascorrected") + preproc.connect( + OutputT1wRestoredBrainImage, "out_file", outputnode, "T1w_brain_biascorrected" + ) + preproc.connect(OutputT2wRestoredImage, "out_file", outputnode, "T2w_biascorrected") + preproc.connect( + OutputT2wRestoredBrainImage, "out_file", outputnode, "T2w_brain_biascorrected" + ) + preproc.connect(OutputBiasField, "out_file", outputnode, "biasfield") return preproc @@ -425,262 +514,285 @@ def file_to_a_list(infile_1, infile_2): def afni_brain_connector(wf, cfg, strat_pool, pipe_num, opt): # Skull-stripping using AFNI 3dSkullStrip inputnode_afni = pe.Node( - util.IdentityInterface(fields=['mask_vol', - 'shrink_factor', - 'var_shrink_fac', - 'shrink_fac_bot_lim', - 'avoid_vent', - 'niter', - 'pushout', - 'touchup', - 'fill_hole', - 'NN_smooth', - 'smooth_final', - 'avoid_eyes', - 'use_edge', - 'exp_frac', - 'push_to_edge', - 'use_skull', - 'perc_int', - 'max_inter_iter', - 'blur_fwhm', - 'fac', - 'monkey']), - name=f'AFNI_options_{pipe_num}') - - skullstrip_args = pe.Node(util.Function(input_names=['spat_norm', - 'spat_norm_dxyz', - 'mask_vol', - 'shrink_fac', - 'var_shrink_fac', - 'shrink_fac_bot_lim', - 'avoid_vent', - 'niter', - 'pushout', - 'touchup', - 'fill_hole', - 'NN_smooth', - 'smooth_final', - 'avoid_eyes', - 'use_edge', - 'exp_frac', - 'push_to_edge', - 'use_skull', - 'perc_int', - 'max_inter_iter', - 'blur_fwhm', - 'fac', - 'monkey'], - output_names=['expr'], - function=create_3dskullstrip_arg_string), - name=f'anat_skullstrip_args_{pipe_num}') + util.IdentityInterface( + fields=[ + "mask_vol", + "shrink_factor", + "var_shrink_fac", + "shrink_fac_bot_lim", + "avoid_vent", + "niter", + "pushout", + "touchup", + "fill_hole", + "NN_smooth", + "smooth_final", + "avoid_eyes", + "use_edge", + "exp_frac", + "push_to_edge", + "use_skull", + "perc_int", + "max_inter_iter", + "blur_fwhm", + "fac", + "monkey", + ] + ), + name=f"AFNI_options_{pipe_num}", + ) + + skullstrip_args = pe.Node( + util.Function( + input_names=[ + "spat_norm", + "spat_norm_dxyz", + "mask_vol", + "shrink_fac", + "var_shrink_fac", + "shrink_fac_bot_lim", + "avoid_vent", + "niter", + "pushout", + "touchup", + "fill_hole", + "NN_smooth", + "smooth_final", + "avoid_eyes", + "use_edge", + "exp_frac", + "push_to_edge", + "use_skull", + "perc_int", + "max_inter_iter", + "blur_fwhm", + "fac", + "monkey", + ], + output_names=["expr"], + function=create_3dskullstrip_arg_string, + ), + name=f"anat_skullstrip_args_{pipe_num}", + ) inputnode_afni.inputs.set( - mask_vol=cfg.anatomical_preproc['brain_extraction'][ - 'AFNI-3dSkullStrip']['mask_vol'], - shrink_factor= - cfg.anatomical_preproc['brain_extraction'][ - 'AFNI-3dSkullStrip']['shrink_factor'], - var_shrink_fac= - cfg.anatomical_preproc['brain_extraction'][ - 'AFNI-3dSkullStrip']['var_shrink_fac'], - shrink_fac_bot_lim= - cfg.anatomical_preproc['brain_extraction'][ - 'AFNI-3dSkullStrip']['shrink_factor_bot_lim'], - avoid_vent= - cfg.anatomical_preproc['brain_extraction'][ - 'AFNI-3dSkullStrip']['avoid_vent'], - niter=cfg.anatomical_preproc['brain_extraction'][ - 'AFNI-3dSkullStrip']['n_iterations'], - pushout=cfg.anatomical_preproc['brain_extraction'][ - 'AFNI-3dSkullStrip']['pushout'], - touchup=cfg.anatomical_preproc['brain_extraction'][ - 'AFNI-3dSkullStrip']['touchup'], - fill_hole=cfg.anatomical_preproc['brain_extraction'][ - 'AFNI-3dSkullStrip']['fill_hole'], - NN_smooth=cfg.anatomical_preproc['brain_extraction'][ - 'AFNI-3dSkullStrip']['NN_smooth'], - smooth_final= - cfg.anatomical_preproc['brain_extraction'][ - 'AFNI-3dSkullStrip']['smooth_final'], - avoid_eyes= - cfg.anatomical_preproc['brain_extraction'][ - 'AFNI-3dSkullStrip']['avoid_eyes'], - use_edge=cfg.anatomical_preproc['brain_extraction'][ - 'AFNI-3dSkullStrip']['use_edge'], - exp_frac=cfg.anatomical_preproc['brain_extraction'][ - 'AFNI-3dSkullStrip']['exp_frac'], - push_to_edge= - cfg.anatomical_preproc['brain_extraction'][ - 'AFNI-3dSkullStrip']['push_to_edge'], - use_skull=cfg.anatomical_preproc['brain_extraction'][ - 'AFNI-3dSkullStrip']['use_skull'], - perc_int=cfg.anatomical_preproc['brain_extraction'][ - 'AFNI-3dSkullStrip']['perc_int'], - max_inter_iter= - cfg.anatomical_preproc['brain_extraction'][ - 'AFNI-3dSkullStrip']['max_inter_iter'], - fac=cfg.anatomical_preproc['brain_extraction'][ - 'AFNI-3dSkullStrip']['fac'], - blur_fwhm=cfg.anatomical_preproc['brain_extraction'][ - 'AFNI-3dSkullStrip']['blur_fwhm'], - monkey=cfg.anatomical_preproc['brain_extraction'][ - 'AFNI-3dSkullStrip']['monkey'], - ) - - wf.connect([ - (inputnode_afni, skullstrip_args, [ - ('mask_vol', 'mask_vol'), - ('shrink_factor', 'shrink_fac'), - ('var_shrink_fac', 'var_shrink_fac'), - ('shrink_fac_bot_lim', 'shrink_fac_bot_lim'), - ('avoid_vent', 'avoid_vent'), - ('niter', 'niter'), - ('pushout', 'pushout'), - ('touchup', 'touchup'), - ('fill_hole', 'fill_hole'), - ('avoid_eyes', 'avoid_eyes'), - ('use_edge', 'use_edge'), - ('exp_frac', 'exp_frac'), - ('NN_smooth', 'NN_smooth'), - ('smooth_final', 'smooth_final'), - ('push_to_edge', 'push_to_edge'), - ('use_skull', 'use_skull'), - ('perc_int', 'perc_int'), - ('max_inter_iter', 'max_inter_iter'), - ('blur_fwhm', 'blur_fwhm'), - ('fac', 'fac'), - ('monkey', 'monkey') - ]) - ]) - - anat_skullstrip = pe.Node(interface=afni.SkullStrip(), - name=f'anat_skullstrip_{pipe_num}') - anat_skullstrip.inputs.outputtype = 'NIFTI_GZ' - - if strat_pool.check_rpool('desc-preproc_T1w'): - node, out = strat_pool.get_data('desc-preproc_T1w') - wf.connect(node, out, anat_skullstrip, 'in_file') - - elif strat_pool.check_rpool('desc-preproc_T2w'): - node, out = strat_pool.get_data('desc-preproc_T2w') - wf.connect(node, out, anat_skullstrip, 'in_file') - - wf.connect(skullstrip_args, 'expr', anat_skullstrip, 'args') + mask_vol=cfg.anatomical_preproc["brain_extraction"]["AFNI-3dSkullStrip"][ + "mask_vol" + ], + shrink_factor=cfg.anatomical_preproc["brain_extraction"]["AFNI-3dSkullStrip"][ + "shrink_factor" + ], + var_shrink_fac=cfg.anatomical_preproc["brain_extraction"]["AFNI-3dSkullStrip"][ + "var_shrink_fac" + ], + shrink_fac_bot_lim=cfg.anatomical_preproc["brain_extraction"][ + "AFNI-3dSkullStrip" + ]["shrink_factor_bot_lim"], + avoid_vent=cfg.anatomical_preproc["brain_extraction"]["AFNI-3dSkullStrip"][ + "avoid_vent" + ], + niter=cfg.anatomical_preproc["brain_extraction"]["AFNI-3dSkullStrip"][ + "n_iterations" + ], + pushout=cfg.anatomical_preproc["brain_extraction"]["AFNI-3dSkullStrip"][ + "pushout" + ], + touchup=cfg.anatomical_preproc["brain_extraction"]["AFNI-3dSkullStrip"][ + "touchup" + ], + fill_hole=cfg.anatomical_preproc["brain_extraction"]["AFNI-3dSkullStrip"][ + "fill_hole" + ], + NN_smooth=cfg.anatomical_preproc["brain_extraction"]["AFNI-3dSkullStrip"][ + "NN_smooth" + ], + smooth_final=cfg.anatomical_preproc["brain_extraction"]["AFNI-3dSkullStrip"][ + "smooth_final" + ], + avoid_eyes=cfg.anatomical_preproc["brain_extraction"]["AFNI-3dSkullStrip"][ + "avoid_eyes" + ], + use_edge=cfg.anatomical_preproc["brain_extraction"]["AFNI-3dSkullStrip"][ + "use_edge" + ], + exp_frac=cfg.anatomical_preproc["brain_extraction"]["AFNI-3dSkullStrip"][ + "exp_frac" + ], + push_to_edge=cfg.anatomical_preproc["brain_extraction"]["AFNI-3dSkullStrip"][ + "push_to_edge" + ], + use_skull=cfg.anatomical_preproc["brain_extraction"]["AFNI-3dSkullStrip"][ + "use_skull" + ], + perc_int=cfg.anatomical_preproc["brain_extraction"]["AFNI-3dSkullStrip"][ + "perc_int" + ], + max_inter_iter=cfg.anatomical_preproc["brain_extraction"]["AFNI-3dSkullStrip"][ + "max_inter_iter" + ], + fac=cfg.anatomical_preproc["brain_extraction"]["AFNI-3dSkullStrip"]["fac"], + blur_fwhm=cfg.anatomical_preproc["brain_extraction"]["AFNI-3dSkullStrip"][ + "blur_fwhm" + ], + monkey=cfg.anatomical_preproc["brain_extraction"]["AFNI-3dSkullStrip"][ + "monkey" + ], + ) + + wf.connect( + [ + ( + inputnode_afni, + skullstrip_args, + [ + ("mask_vol", "mask_vol"), + ("shrink_factor", "shrink_fac"), + ("var_shrink_fac", "var_shrink_fac"), + ("shrink_fac_bot_lim", "shrink_fac_bot_lim"), + ("avoid_vent", "avoid_vent"), + ("niter", "niter"), + ("pushout", "pushout"), + ("touchup", "touchup"), + ("fill_hole", "fill_hole"), + ("avoid_eyes", "avoid_eyes"), + ("use_edge", "use_edge"), + ("exp_frac", "exp_frac"), + ("NN_smooth", "NN_smooth"), + ("smooth_final", "smooth_final"), + ("push_to_edge", "push_to_edge"), + ("use_skull", "use_skull"), + ("perc_int", "perc_int"), + ("max_inter_iter", "max_inter_iter"), + ("blur_fwhm", "blur_fwhm"), + ("fac", "fac"), + ("monkey", "monkey"), + ], + ) + ] + ) + + anat_skullstrip = pe.Node( + interface=afni.SkullStrip(), name=f"anat_skullstrip_{pipe_num}" + ) + anat_skullstrip.inputs.outputtype = "NIFTI_GZ" + + if strat_pool.check_rpool("desc-preproc_T1w"): + node, out = strat_pool.get_data("desc-preproc_T1w") + wf.connect(node, out, anat_skullstrip, "in_file") + + elif strat_pool.check_rpool("desc-preproc_T2w"): + node, out = strat_pool.get_data("desc-preproc_T2w") + wf.connect(node, out, anat_skullstrip, "in_file") + + wf.connect(skullstrip_args, "expr", anat_skullstrip, "args") # Generate anatomical brain mask - anat_brain_mask = pe.Node(interface=afni.Calc(), - name=f'anat_brain_mask_{pipe_num}') + anat_brain_mask = pe.Node(interface=afni.Calc(), name=f"anat_brain_mask_{pipe_num}") - anat_brain_mask.inputs.expr = 'step(a)' - anat_brain_mask.inputs.outputtype = 'NIFTI_GZ' + anat_brain_mask.inputs.expr = "step(a)" + anat_brain_mask.inputs.outputtype = "NIFTI_GZ" - wf.connect(anat_skullstrip, 'out_file', - anat_brain_mask, 'in_file_a') + wf.connect(anat_skullstrip, "out_file", anat_brain_mask, "in_file_a") - if strat_pool.check_rpool('desc-preproc_T1w'): - outputs = { - 'space-T1w_desc-brain_mask': (anat_brain_mask, 'out_file') - } + if strat_pool.check_rpool("desc-preproc_T1w"): + outputs = {"space-T1w_desc-brain_mask": (anat_brain_mask, "out_file")} - elif strat_pool.check_rpool('desc-preproc_T2w'): - outputs = { - 'space-T2w_desc-brain_mask': (anat_brain_mask, 'out_file') - } + elif strat_pool.check_rpool("desc-preproc_T2w"): + outputs = {"space-T2w_desc-brain_mask": (anat_brain_mask, "out_file")} return (wf, outputs) def fsl_brain_connector(wf, cfg, strat_pool, pipe_num, opt): inputnode_bet = pe.Node( - util.IdentityInterface(fields=['frac', - 'mask_boolean', - 'mesh_boolean', - 'outline', - 'padding', - 'radius', - 'reduce_bias', - 'remove_eyes', - 'robust', - 'skull', - 'surfaces', - 'threshold', - 'vertical_gradient']), - name=f'BET_options_{pipe_num}') + util.IdentityInterface( + fields=[ + "frac", + "mask_boolean", + "mesh_boolean", + "outline", + "padding", + "radius", + "reduce_bias", + "remove_eyes", + "robust", + "skull", + "surfaces", + "threshold", + "vertical_gradient", + ] + ), + name=f"BET_options_{pipe_num}", + ) anat_skullstrip = pe.Node( - interface=fsl.BET(), name=f'anat_BET_skullstrip_{pipe_num}') - anat_skullstrip.inputs.output_type = 'NIFTI_GZ' + interface=fsl.BET(), name=f"anat_BET_skullstrip_{pipe_num}" + ) + anat_skullstrip.inputs.output_type = "NIFTI_GZ" inputnode_bet.inputs.set( - frac=cfg.anatomical_preproc['brain_extraction'][ - 'FSL-BET']['frac'], - mask_boolean= - cfg.anatomical_preproc['brain_extraction'][ - 'FSL-BET']['mask_boolean'], - mesh_boolean= - cfg.anatomical_preproc['brain_extraction'][ - 'FSL-BET']['mesh_boolean'], - outline=cfg.anatomical_preproc['brain_extraction'][ - 'FSL-BET']['outline'], - padding=cfg.anatomical_preproc['brain_extraction'][ - 'FSL-BET']['padding'], - radius=cfg.anatomical_preproc['brain_extraction'][ - 'FSL-BET']['radius'], - reduce_bias= - cfg.anatomical_preproc['brain_extraction'][ - 'FSL-BET']['reduce_bias'], - remove_eyes= - cfg.anatomical_preproc['brain_extraction'][ - 'FSL-BET']['remove_eyes'], - robust=cfg.anatomical_preproc['brain_extraction'][ - 'FSL-BET']['robust'], - skull=cfg.anatomical_preproc['brain_extraction'][ - 'FSL-BET']['skull'], - surfaces=cfg.anatomical_preproc['brain_extraction'][ - 'FSL-BET']['surfaces'], - threshold=cfg.anatomical_preproc['brain_extraction'][ - 'FSL-BET']['threshold'], - vertical_gradient= - cfg.anatomical_preproc['brain_extraction'][ - 'FSL-BET']['vertical_gradient'], - ) - - if strat_pool.check_rpool('desc-preproc_T1w'): - node, out = strat_pool.get_data('desc-preproc_T1w') - wf.connect(node, out, anat_skullstrip, 'in_file') - - elif strat_pool.check_rpool('desc-preproc_T2w'): - node, out = strat_pool.get_data('desc-preproc_T2w') - wf.connect(node, out, anat_skullstrip, 'in_file') - - wf.connect([ - (inputnode_bet, anat_skullstrip, [ - ('frac', 'frac'), - ('mask_boolean', 'mask'), - ('mesh_boolean', 'mesh'), - ('outline', 'outline'), - ('padding', 'padding'), - ('radius', 'radius'), - ('reduce_bias', 'reduce_bias'), - ('remove_eyes', 'remove_eyes'), - ('robust', 'robust'), - ('skull', 'skull'), - ('surfaces', 'surfaces'), - ('threshold', 'threshold'), - ('vertical_gradient', 'vertical_gradient'), - ]) - ]) - - if strat_pool.check_rpool('desc-preproc_T1w'): - outputs = { - 'space-T1w_desc-brain_mask': (anat_skullstrip, 'mask_file') - } + frac=cfg.anatomical_preproc["brain_extraction"]["FSL-BET"]["frac"], + mask_boolean=cfg.anatomical_preproc["brain_extraction"]["FSL-BET"][ + "mask_boolean" + ], + mesh_boolean=cfg.anatomical_preproc["brain_extraction"]["FSL-BET"][ + "mesh_boolean" + ], + outline=cfg.anatomical_preproc["brain_extraction"]["FSL-BET"]["outline"], + padding=cfg.anatomical_preproc["brain_extraction"]["FSL-BET"]["padding"], + radius=cfg.anatomical_preproc["brain_extraction"]["FSL-BET"]["radius"], + reduce_bias=cfg.anatomical_preproc["brain_extraction"]["FSL-BET"][ + "reduce_bias" + ], + remove_eyes=cfg.anatomical_preproc["brain_extraction"]["FSL-BET"][ + "remove_eyes" + ], + robust=cfg.anatomical_preproc["brain_extraction"]["FSL-BET"]["robust"], + skull=cfg.anatomical_preproc["brain_extraction"]["FSL-BET"]["skull"], + surfaces=cfg.anatomical_preproc["brain_extraction"]["FSL-BET"]["surfaces"], + threshold=cfg.anatomical_preproc["brain_extraction"]["FSL-BET"]["threshold"], + vertical_gradient=cfg.anatomical_preproc["brain_extraction"]["FSL-BET"][ + "vertical_gradient" + ], + ) - elif strat_pool.check_rpool('desc-preproc_T2w'): - outputs = { - 'space-T2w_desc-brain_mask': (anat_skullstrip, 'mask_file') - } + if strat_pool.check_rpool("desc-preproc_T1w"): + node, out = strat_pool.get_data("desc-preproc_T1w") + wf.connect(node, out, anat_skullstrip, "in_file") + + elif strat_pool.check_rpool("desc-preproc_T2w"): + node, out = strat_pool.get_data("desc-preproc_T2w") + wf.connect(node, out, anat_skullstrip, "in_file") + + wf.connect( + [ + ( + inputnode_bet, + anat_skullstrip, + [ + ("frac", "frac"), + ("mask_boolean", "mask"), + ("mesh_boolean", "mesh"), + ("outline", "outline"), + ("padding", "padding"), + ("radius", "radius"), + ("reduce_bias", "reduce_bias"), + ("remove_eyes", "remove_eyes"), + ("robust", "robust"), + ("skull", "skull"), + ("surfaces", "surfaces"), + ("threshold", "threshold"), + ("vertical_gradient", "vertical_gradient"), + ], + ) + ] + ) + + if strat_pool.check_rpool("desc-preproc_T1w"): + outputs = {"space-T1w_desc-brain_mask": (anat_skullstrip, "mask_file")} + + elif strat_pool.check_rpool("desc-preproc_T2w"): + outputs = {"space-T2w_desc-brain_mask": (anat_skullstrip, "mask_file")} return (wf, outputs) @@ -688,37 +800,44 @@ def fsl_brain_connector(wf, cfg, strat_pool, pipe_num, opt): def niworkflows_ants_brain_connector(wf, cfg, strat_pool, pipe_num, opt): # Skull-stripping using niworkflows-ants anat_skullstrip_ants = init_brain_extraction_wf( - tpl_target_path=cfg.anatomical_preproc['brain_extraction'][ - 'niworkflows-ants'][ - 'template_path'], - tpl_mask_path=cfg.anatomical_preproc['brain_extraction'][ - 'niworkflows-ants'][ - 'mask_path'], - tpl_regmask_path=cfg.anatomical_preproc['brain_extraction'][ - 'niworkflows-ants'][ - 'regmask_path'], - name='anat_skullstrip_ants', - atropos_use_random_seed=cfg.pipeline_setup['system_config'][ - 'random_seed'] is None) - - if strat_pool.check_rpool('desc-preproc_T1w'): - node, out = strat_pool.get_data('desc-preproc_T1w') - wf.connect(node, out, anat_skullstrip_ants, 'inputnode.in_files') - - elif strat_pool.check_rpool('desc-preproc_T2w'): - node, out = strat_pool.get_data('desc-preproc_T2w') - wf.connect(node, out, anat_skullstrip_ants, 'inputnode.in_files') - - if strat_pool.check_rpool('desc-preproc_T1w'): + tpl_target_path=cfg.anatomical_preproc["brain_extraction"]["niworkflows-ants"][ + "template_path" + ], + tpl_mask_path=cfg.anatomical_preproc["brain_extraction"]["niworkflows-ants"][ + "mask_path" + ], + tpl_regmask_path=cfg.anatomical_preproc["brain_extraction"]["niworkflows-ants"][ + "regmask_path" + ], + name="anat_skullstrip_ants", + atropos_use_random_seed=cfg.pipeline_setup["system_config"]["random_seed"] + is None, + ) + + if strat_pool.check_rpool("desc-preproc_T1w"): + node, out = strat_pool.get_data("desc-preproc_T1w") + wf.connect(node, out, anat_skullstrip_ants, "inputnode.in_files") + + elif strat_pool.check_rpool("desc-preproc_T2w"): + node, out = strat_pool.get_data("desc-preproc_T2w") + wf.connect(node, out, anat_skullstrip_ants, "inputnode.in_files") + + if strat_pool.check_rpool("desc-preproc_T1w"): outputs = { - 'space-T1w_desc-brain_mask': (anat_skullstrip_ants, 'atropos_wf.copy_xform.out_mask'), - 'desc-preproc_T1w': (anat_skullstrip_ants, 'copy_xform.out_file') + "space-T1w_desc-brain_mask": ( + anat_skullstrip_ants, + "atropos_wf.copy_xform.out_mask", + ), + "desc-preproc_T1w": (anat_skullstrip_ants, "copy_xform.out_file"), } - elif strat_pool.check_rpool('desc-preproc_T2w'): + elif strat_pool.check_rpool("desc-preproc_T2w"): outputs = { - 'space-T2w_desc-brain_mask': (anat_skullstrip_ants, 'atropos_wf.copy_xform.out_mask'), - 'desc-preproc_T2w': (anat_skullstrip_ants, 'copy_xform.out_file') + "space-T2w_desc-brain_mask": ( + anat_skullstrip_ants, + "atropos_wf.copy_xform.out_mask", + ), + "desc-preproc_T2w": (anat_skullstrip_ants, "copy_xform.out_file"), } return (wf, outputs) @@ -731,145 +850,172 @@ def unet_brain_connector(wf, cfg, strat_pool, pipe_num, opt): input_slice: 3 conv_block: 5 kernel_root: 16 - rescale_dim: 256 + rescale_dim: 256. """ from CPAC.unet.function import predict_volumes - unet_mask = pe.Node(util.Function(input_names=['model_path', 'cimg_in'], - output_names=['out_path'], - function=predict_volumes), - name=f'unet_mask_{pipe_num}') - node, out = strat_pool.get_data('unet-model') - wf.connect(node, out, unet_mask, 'model_path') + unet_mask = pe.Node( + util.Function( + input_names=["model_path", "cimg_in"], + output_names=["out_path"], + function=predict_volumes, + ), + name=f"unet_mask_{pipe_num}", + ) - if strat_pool.check_rpool('desc-preproc_T1w'): - node, out = strat_pool.get_data('desc-preproc_T1w') - wf.connect(node, out, unet_mask, 'cimg_in') + node, out = strat_pool.get_data("unet-model") + wf.connect(node, out, unet_mask, "model_path") - elif strat_pool.check_rpool('desc-preproc_T2w'): - node, out = strat_pool.get_data('desc-preproc_T2w') - wf.connect(node, out, unet_mask, 'cimg_in') + if strat_pool.check_rpool("desc-preproc_T1w"): + node, out = strat_pool.get_data("desc-preproc_T1w") + wf.connect(node, out, unet_mask, "cimg_in") + + elif strat_pool.check_rpool("desc-preproc_T2w"): + node, out = strat_pool.get_data("desc-preproc_T2w") + wf.connect(node, out, unet_mask, "cimg_in") """ Revised mask with ANTs """ # fslmaths -mul brain.nii.gz - unet_masked_brain = pe.Node(interface=fsl.MultiImageMaths(), - name=f'unet_masked_brain_{pipe_num}') + unet_masked_brain = pe.Node( + interface=fsl.MultiImageMaths(), name=f"unet_masked_brain_{pipe_num}" + ) unet_masked_brain.inputs.op_string = "-mul %s" - if strat_pool.check_rpool('desc-preproc_T1w'): - node, out = strat_pool.get_data('desc-preproc_T1w') - wf.connect(node, out, unet_masked_brain, 'in_file') - - elif strat_pool.check_rpool('desc-preproc_T2w'): - node, out = strat_pool.get_data('desc-preproc_T2w') - wf.connect(node, out, unet_masked_brain, 'in_file') + if strat_pool.check_rpool("desc-preproc_T1w"): + node, out = strat_pool.get_data("desc-preproc_T1w") + wf.connect(node, out, unet_masked_brain, "in_file") - wf.connect(unet_mask, 'out_path', unet_masked_brain, 'operand_files') + elif strat_pool.check_rpool("desc-preproc_T2w"): + node, out = strat_pool.get_data("desc-preproc_T2w") + wf.connect(node, out, unet_masked_brain, "in_file") + + wf.connect(unet_mask, "out_path", unet_masked_brain, "operand_files") # flirt -v -dof 6 -in brain.nii.gz -ref NMT_SS_0.5mm.nii.gz -o brain_rot2atl -omat brain_rot2atl.mat -interp sinc - native_brain_to_template_brain = pe.Node(interface=fsl.FLIRT(), - name=f'native_brain_to_template_' - f'brain_{pipe_num}') + native_brain_to_template_brain = pe.Node( + interface=fsl.FLIRT(), name=f"native_brain_to_template_" f"brain_{pipe_num}" + ) native_brain_to_template_brain.inputs.dof = 6 - native_brain_to_template_brain.inputs.interp = 'sinc' - wf.connect(unet_masked_brain, 'out_file', - native_brain_to_template_brain, 'in_file') + native_brain_to_template_brain.inputs.interp = "sinc" + wf.connect(unet_masked_brain, "out_file", native_brain_to_template_brain, "in_file") - node, out = strat_pool.get_data('T1w-brain-template') - wf.connect(node, out, native_brain_to_template_brain, 'reference') + node, out = strat_pool.get_data("T1w-brain-template") + wf.connect(node, out, native_brain_to_template_brain, "reference") # flirt -in head.nii.gz -ref NMT_0.5mm.nii.gz -o head_rot2atl -applyxfm -init brain_rot2atl.mat - native_head_to_template_head = pe.Node(interface=fsl.FLIRT(), - name=f'native_head_to_template_' - f'head_{pipe_num}') + native_head_to_template_head = pe.Node( + interface=fsl.FLIRT(), name=f"native_head_to_template_" f"head_{pipe_num}" + ) native_head_to_template_head.inputs.apply_xfm = True - if strat_pool.check_rpool('desc-preproc_T1w'): - node, out = strat_pool.get_data('desc-preproc_T1w') - wf.connect(node, out, native_head_to_template_head, 'in_file') - - elif strat_pool.check_rpool('desc-preproc_T2w'): - node, out = strat_pool.get_data('desc-preproc_T2w') - wf.connect(node, out, native_head_to_template_head, 'in_file') + if strat_pool.check_rpool("desc-preproc_T1w"): + node, out = strat_pool.get_data("desc-preproc_T1w") + wf.connect(node, out, native_head_to_template_head, "in_file") - wf.connect(native_brain_to_template_brain, 'out_matrix_file', - native_head_to_template_head, 'in_matrix_file') + elif strat_pool.check_rpool("desc-preproc_T2w"): + node, out = strat_pool.get_data("desc-preproc_T2w") + wf.connect(node, out, native_head_to_template_head, "in_file") - node, out = strat_pool.get_data('T1w-template') - wf.connect(node, out, native_head_to_template_head, 'reference') + wf.connect( + native_brain_to_template_brain, + "out_matrix_file", + native_head_to_template_head, + "in_matrix_file", + ) + + node, out = strat_pool.get_data("T1w-template") + wf.connect(node, out, native_head_to_template_head, "reference") # fslmaths NMT_SS_0.5mm.nii.gz -bin templateMask.nii.gz - template_brain_mask = pe.Node(interface=fsl.maths.MathsCommand(), - name=f'template_brain_mask_{pipe_num}') - template_brain_mask.inputs.args = '-bin' + template_brain_mask = pe.Node( + interface=fsl.maths.MathsCommand(), name=f"template_brain_mask_{pipe_num}" + ) + template_brain_mask.inputs.args = "-bin" - node, out = strat_pool.get_data('T1w-brain-template') - wf.connect(node, out, template_brain_mask, 'in_file') + node, out = strat_pool.get_data("T1w-brain-template") + wf.connect(node, out, template_brain_mask, "in_file") # ANTS 3 -m CC[head_rot2atl.nii.gz,NMT_0.5mm.nii.gz,1,5] -t SyN[0.25] -r Gauss[3,0] -o atl2T1rot -i 60x50x20 --use-Histogram-Matching --number-of-affine-iterations 10000x10000x10000x10000x10000 --MI-option 32x16000 - ants_template_head_to_template = pe.Node(interface=ants.Registration(), - name=f'template_head_to_' - f'template_{pipe_num}') - ants_template_head_to_template.inputs.metric = ['CC'] + ants_template_head_to_template = pe.Node( + interface=ants.Registration(), name=f"template_head_to_" f"template_{pipe_num}" + ) + ants_template_head_to_template.inputs.metric = ["CC"] ants_template_head_to_template.inputs.metric_weight = [1, 5] - ants_template_head_to_template.inputs.transforms = ['SyN'] + ants_template_head_to_template.inputs.transforms = ["SyN"] ants_template_head_to_template.inputs.transform_parameters = [(0.25,)] - ants_template_head_to_template.inputs.interpolation = 'NearestNeighbor' - ants_template_head_to_template.inputs.number_of_iterations = [ - [60, 50, 20]] + ants_template_head_to_template.inputs.interpolation = "NearestNeighbor" + ants_template_head_to_template.inputs.number_of_iterations = [[60, 50, 20]] ants_template_head_to_template.inputs.smoothing_sigmas = [[0.6, 0.2, 0.0]] ants_template_head_to_template.inputs.shrink_factors = [[4, 2, 1]] - ants_template_head_to_template.inputs.convergence_threshold = [1.e-8] - wf.connect(native_head_to_template_head, 'out_file', - ants_template_head_to_template, 'fixed_image') + ants_template_head_to_template.inputs.convergence_threshold = [1.0e-8] + wf.connect( + native_head_to_template_head, + "out_file", + ants_template_head_to_template, + "fixed_image", + ) - node, out = strat_pool.get_data('T1w-brain-template') - wf.connect(node, out, ants_template_head_to_template, 'moving_image') + node, out = strat_pool.get_data("T1w-brain-template") + wf.connect(node, out, ants_template_head_to_template, "moving_image") # antsApplyTransforms -d 3 -i templateMask.nii.gz -t atl2T1rotWarp.nii.gz atl2T1rotAffine.txt -r brain_rot2atl.nii.gz -o brain_rot2atl_mask.nii.gz template_head_transform_to_template = pe.Node( interface=ants.ApplyTransforms(), - name=f'template_head_transform_to_template_{pipe_num}') + name=f"template_head_transform_to_template_{pipe_num}", + ) template_head_transform_to_template.inputs.dimension = 3 - wf.connect(template_brain_mask, 'out_file', - template_head_transform_to_template, 'input_image') - wf.connect(native_brain_to_template_brain, 'out_file', - template_head_transform_to_template, 'reference_image') - wf.connect(ants_template_head_to_template, 'forward_transforms', - template_head_transform_to_template, 'transforms') + wf.connect( + template_brain_mask, + "out_file", + template_head_transform_to_template, + "input_image", + ) + wf.connect( + native_brain_to_template_brain, + "out_file", + template_head_transform_to_template, + "reference_image", + ) + wf.connect( + ants_template_head_to_template, + "forward_transforms", + template_head_transform_to_template, + "transforms", + ) - # convert_xfm -omat brain_rot2native.mat -inverse brain_rot2atl.mat  - invt = pe.Node(interface=fsl.ConvertXFM(), name='convert_xfm') + # convert_xfm -omat brain_rot2native.mat -inverse brain_rot2atl.mat + invt = pe.Node(interface=fsl.ConvertXFM(), name="convert_xfm") invt.inputs.invert_xfm = True - wf.connect(native_brain_to_template_brain, 'out_matrix_file', invt, - 'in_file') + wf.connect(native_brain_to_template_brain, "out_matrix_file", invt, "in_file") # flirt -in brain_rot2atl_mask.nii.gz -ref brain.nii.gz -o brain_mask.nii.gz -applyxfm -init brain_rot2native.mat - template_brain_to_native_brain = pe.Node(interface=fsl.FLIRT(), - name=f'template_brain_to_native_' - f'brain_{pipe_num}') + template_brain_to_native_brain = pe.Node( + interface=fsl.FLIRT(), name=f"template_brain_to_native_" f"brain_{pipe_num}" + ) template_brain_to_native_brain.inputs.apply_xfm = True - wf.connect(template_head_transform_to_template, 'output_image', - template_brain_to_native_brain, 'in_file') - wf.connect(unet_masked_brain, 'out_file', template_brain_to_native_brain, - 'reference') - wf.connect(invt, 'out_file', template_brain_to_native_brain, - 'in_matrix_file') + wf.connect( + template_head_transform_to_template, + "output_image", + template_brain_to_native_brain, + "in_file", + ) + wf.connect( + unet_masked_brain, "out_file", template_brain_to_native_brain, "reference" + ) + wf.connect(invt, "out_file", template_brain_to_native_brain, "in_matrix_file") # fslmaths brain_mask.nii.gz -thr .5 -bin brain_mask_thr.nii.gz - refined_mask = pe.Node(interface=fsl.Threshold(), name=f'refined_mask' - f'_{pipe_num}') + refined_mask = pe.Node( + interface=fsl.Threshold(), name=f"refined_mask" f"_{pipe_num}" + ) refined_mask.inputs.thresh = 0.5 - refined_mask.inputs.args = '-bin' - wf.connect(template_brain_to_native_brain, 'out_file', refined_mask, - 'in_file') + refined_mask.inputs.args = "-bin" + wf.connect(template_brain_to_native_brain, "out_file", refined_mask, "in_file") - outputs = { - 'space-T1w_desc-brain_mask': (refined_mask, 'out_file') - } + outputs = {"space-T1w_desc-brain_mask": (refined_mask, "out_file")} return (wf, outputs) @@ -877,365 +1023,374 @@ def unet_brain_connector(wf, cfg, strat_pool, pipe_num, opt): def freesurfer_brain_connector(wf, cfg, strat_pool, pipe_num, opt): # register FS brain mask to native space fs_brain_mask_to_native = pe.Node( - interface=freesurfer.ApplyVolTransform(), - name='fs_brain_mask_to_native') + interface=freesurfer.ApplyVolTransform(), name="fs_brain_mask_to_native" + ) fs_brain_mask_to_native.inputs.reg_header = True - node, out = strat_pool.get_data('pipeline-fs_brainmask') - wf.connect(node, out, fs_brain_mask_to_native, 'source_file') + node, out = strat_pool.get_data("pipeline-fs_brainmask") + wf.connect(node, out, fs_brain_mask_to_native, "source_file") - node, out = strat_pool.get_data('pipeline-fs_raw-average') - wf.connect(node, out, fs_brain_mask_to_native, 'target_file') + node, out = strat_pool.get_data("pipeline-fs_raw-average") + wf.connect(node, out, fs_brain_mask_to_native, "target_file") - node, out = strat_pool.get_data('freesurfer-subject-dir') - wf.connect(node, out, fs_brain_mask_to_native, 'subjects_dir') + node, out = strat_pool.get_data("freesurfer-subject-dir") + wf.connect(node, out, fs_brain_mask_to_native, "subjects_dir") # convert brain mask file from .mgz to .nii.gz - fs_brain_mask_to_nifti = pe.Node(util.Function(input_names=['in_file'], - output_names=['out_file'], - function=mri_convert), - name=f'fs_brainmask_to_nifti_{pipe_num}') - wf.connect(fs_brain_mask_to_native, 'transformed_file', - fs_brain_mask_to_nifti, 'in_file') + fs_brain_mask_to_nifti = pe.Node( + util.Function( + input_names=["in_file"], output_names=["out_file"], function=mri_convert + ), + name=f"fs_brainmask_to_nifti_{pipe_num}", + ) + wf.connect( + fs_brain_mask_to_native, "transformed_file", fs_brain_mask_to_nifti, "in_file" + ) # binarize the brain mask - binarize_fs_brain_mask = pe.Node(interface=fsl.maths.MathsCommand(), - name=f'binarize_fs_brainmask_{pipe_num}') - binarize_fs_brain_mask.inputs.args = '-bin' - wf.connect(fs_brain_mask_to_nifti, 'out_file', - binarize_fs_brain_mask, 'in_file') + binarize_fs_brain_mask = pe.Node( + interface=fsl.maths.MathsCommand(), name=f"binarize_fs_brainmask_{pipe_num}" + ) + binarize_fs_brain_mask.inputs.args = "-bin" + wf.connect(fs_brain_mask_to_nifti, "out_file", binarize_fs_brain_mask, "in_file") # fill holes - fill_fs_brain_mask = pe.Node(interface=afni.MaskTool(), - name=f'fill_fs_brainmask_{pipe_num}') + fill_fs_brain_mask = pe.Node( + interface=afni.MaskTool(), name=f"fill_fs_brainmask_{pipe_num}" + ) fill_fs_brain_mask.inputs.fill_holes = True - fill_fs_brain_mask.inputs.outputtype = 'NIFTI_GZ' - wf.connect(binarize_fs_brain_mask, 'out_file', - fill_fs_brain_mask, 'in_file') + fill_fs_brain_mask.inputs.outputtype = "NIFTI_GZ" + wf.connect(binarize_fs_brain_mask, "out_file", fill_fs_brain_mask, "in_file") - outputs = { - 'space-T1w_desc-brain_mask': (fill_fs_brain_mask, 'out_file') - } + outputs = {"space-T1w_desc-brain_mask": (fill_fs_brain_mask, "out_file")} return (wf, outputs) def freesurfer_abcd_brain_connector(wf, cfg, strat_pool, pipe_num, opt): - ''' - ABCD harmonization - anatomical brain mask generation + """ + ABCD harmonization - anatomical brain mask generation. Ref: https://github.com/DCAN-Labs/DCAN-HCP/blob/master/PostFreeSurfer/PostFreeSurferPipeline.sh#L151-L156 - ''' - - wmparc_to_nifti = pe.Node(util.Function(input_names=['in_file', - 'reslice_like', - 'args'], - output_names=['out_file'], - function=mri_convert), - name=f'wmparc_to_nifti_{pipe_num}') - + """ + wmparc_to_nifti = pe.Node( + util.Function( + input_names=["in_file", "reslice_like", "args"], + output_names=["out_file"], + function=mri_convert, + ), + name=f"wmparc_to_nifti_{pipe_num}", + ) + # Register wmparc file if ingressing FreeSurfer data - if strat_pool.check_rpool('pipeline-fs_xfm'): + if strat_pool.check_rpool("pipeline-fs_xfm"): + wmparc_to_native = pe.Node( + util.Function( + input_names=["source_file", "target_file", "xfm", "out_file"], + output_names=["transformed_file"], + function=normalize_wmparc, + ), + name=f"wmparc_to_native_{pipe_num}", + ) + + wmparc_to_native.inputs.out_file = "wmparc_warped.mgz" - wmparc_to_native = pe.Node(util.Function(input_names=['source_file', - 'target_file', - 'xfm', - 'out_file'], - output_names=['transformed_file'], - function=normalize_wmparc), - name=f'wmparc_to_native_{pipe_num}') - - wmparc_to_native.inputs.out_file = 'wmparc_warped.mgz' + node, out = strat_pool.get_data("pipeline-fs_wmparc") + wf.connect(node, out, wmparc_to_native, "source_file") - node, out = strat_pool.get_data('pipeline-fs_wmparc') - wf.connect(node, out, wmparc_to_native, 'source_file') + node, out = strat_pool.get_data("pipeline-fs_raw-average") + wf.connect(node, out, wmparc_to_native, "target_file") - node, out = strat_pool.get_data('pipeline-fs_raw-average') - wf.connect(node, out, wmparc_to_native, 'target_file') + node, out = strat_pool.get_data("pipeline-fs_xfm") + wf.connect(node, out, wmparc_to_native, "xfm") - node, out = strat_pool.get_data('pipeline-fs_xfm') - wf.connect(node, out, wmparc_to_native, 'xfm') + wf.connect(wmparc_to_native, "transformed_file", wmparc_to_nifti, "in_file") - wf.connect(wmparc_to_native, 'transformed_file', wmparc_to_nifti, 'in_file') - else: - - node, out = strat_pool.get_data('pipeline-fs_wmparc') - wf.connect(node, out, wmparc_to_nifti, 'in_file') + node, out = strat_pool.get_data("pipeline-fs_wmparc") + wf.connect(node, out, wmparc_to_nifti, "in_file") - wmparc_to_nifti.inputs.args = '-rt nearest' + wmparc_to_nifti.inputs.args = "-rt nearest" - node, out = strat_pool.get_data('desc-preproc_T1w') - wf.connect(node, out, wmparc_to_nifti, 'reslice_like') + node, out = strat_pool.get_data("desc-preproc_T1w") + wf.connect(node, out, wmparc_to_nifti, "reslice_like") - binary_mask = pe.Node(interface=fsl.maths.MathsCommand(), - name=f'binarize_wmparc_{pipe_num}') - binary_mask.inputs.args = '-bin -dilD -dilD -dilD -ero -ero' + binary_mask = pe.Node( + interface=fsl.maths.MathsCommand(), name=f"binarize_wmparc_{pipe_num}" + ) + binary_mask.inputs.args = "-bin -dilD -dilD -dilD -ero -ero" - wf.connect(wmparc_to_nifti, 'out_file', binary_mask, 'in_file') + wf.connect(wmparc_to_nifti, "out_file", binary_mask, "in_file") - wb_command_fill_holes = pe.Node(util.Function(input_names=['in_file'], - output_names=['out_file'], - function=wb_command), - name=f'wb_command_fill_holes_{pipe_num}') + wb_command_fill_holes = pe.Node( + util.Function( + input_names=["in_file"], output_names=["out_file"], function=wb_command + ), + name=f"wb_command_fill_holes_{pipe_num}", + ) - wf.connect(binary_mask, 'out_file', wb_command_fill_holes, 'in_file') + wf.connect(binary_mask, "out_file", wb_command_fill_holes, "in_file") - binary_filled_mask = pe.Node(interface=fsl.maths.MathsCommand(), - name=f'binarize_filled_wmparc_{pipe_num}') - binary_filled_mask.inputs.args = '-bin' + binary_filled_mask = pe.Node( + interface=fsl.maths.MathsCommand(), name=f"binarize_filled_wmparc_{pipe_num}" + ) + binary_filled_mask.inputs.args = "-bin" - wf.connect(wb_command_fill_holes, 'out_file', - binary_filled_mask, 'in_file') + wf.connect(wb_command_fill_holes, "out_file", binary_filled_mask, "in_file") - brain_mask_to_t1_restore = pe.Node(interface=fsl.ApplyWarp(), - name=f'brain_mask_to_t1_restore_{pipe_num}') - brain_mask_to_t1_restore.inputs.interp = 'nn' - brain_mask_to_t1_restore.inputs.premat = cfg.registration_workflows['anatomical_registration']['registration']['FSL-FNIRT']['identity_matrix'] + brain_mask_to_t1_restore = pe.Node( + interface=fsl.ApplyWarp(), name=f"brain_mask_to_t1_restore_{pipe_num}" + ) + brain_mask_to_t1_restore.inputs.interp = "nn" + brain_mask_to_t1_restore.inputs.premat = cfg.registration_workflows[ + "anatomical_registration" + ]["registration"]["FSL-FNIRT"]["identity_matrix"] - wf.connect(binary_filled_mask, 'out_file', - brain_mask_to_t1_restore, 'in_file') + wf.connect(binary_filled_mask, "out_file", brain_mask_to_t1_restore, "in_file") - node, out = strat_pool.get_data('desc-preproc_T1w') - wf.connect(node, out, brain_mask_to_t1_restore, 'ref_file') + node, out = strat_pool.get_data("desc-preproc_T1w") + wf.connect(node, out, brain_mask_to_t1_restore, "ref_file") - outputs = { - 'space-T1w_desc-brain_mask': (brain_mask_to_t1_restore, 'out_file') - } + outputs = {"space-T1w_desc-brain_mask": (brain_mask_to_t1_restore, "out_file")} return (wf, outputs) def freesurfer_fsl_brain_connector(wf, cfg, strat_pool, pipe_num, opt): - - node_id = f'{opt.lower()}_{pipe_num}' + node_id = f"{opt.lower()}_{pipe_num}" # mri_convert -it mgz ${SUBJECTS_DIR}/${subject}/mri/brainmask.mgz -ot nii brainmask.nii.gz - convert_fs_brainmask_to_nifti = pe.Node(util.Function(input_names=['in_file'], - output_names=['out_file'], - function=mri_convert), - name=f'convert_fs_brainmask_to_nifti_{node_id}') + convert_fs_brainmask_to_nifti = pe.Node( + util.Function( + input_names=["in_file"], output_names=["out_file"], function=mri_convert + ), + name=f"convert_fs_brainmask_to_nifti_{node_id}", + ) - node, out = strat_pool.get_data('pipeline-fs_brainmask') - wf.connect(node, out, convert_fs_brainmask_to_nifti, 'in_file') + node, out = strat_pool.get_data("pipeline-fs_brainmask") + wf.connect(node, out, convert_fs_brainmask_to_nifti, "in_file") # mri_convert -it mgz ${SUBJECTS_DIR}/${subject}/mri/T1.mgz -ot nii T1.nii.gz - convert_fs_T1_to_nifti = pe.Node(util.Function(input_names=['in_file'], - output_names=['out_file'], - function=mri_convert), - name=f'convert_fs_T1_to_nifti_{node_id}') + convert_fs_T1_to_nifti = pe.Node( + util.Function( + input_names=["in_file"], output_names=["out_file"], function=mri_convert + ), + name=f"convert_fs_T1_to_nifti_{node_id}", + ) - node, out = strat_pool.get_data('pipeline-fs_T1') - wf.connect(node, out, convert_fs_T1_to_nifti, 'in_file') + node, out = strat_pool.get_data("pipeline-fs_T1") + wf.connect(node, out, convert_fs_T1_to_nifti, "in_file") # 3dresample -orient RPI -inset brainmask.nii.gz -prefix brain_fs.nii.gz - reorient_fs_brainmask = pe.Node(interface=afni.Resample(), - name=f'reorient_fs_brainmask_{node_id}', - mem_gb=0, - mem_x=(0.0115, 'in_file', 't')) - reorient_fs_brainmask.inputs.orientation = 'RPI' - reorient_fs_brainmask.inputs.outputtype = 'NIFTI_GZ' + reorient_fs_brainmask = pe.Node( + interface=afni.Resample(), + name=f"reorient_fs_brainmask_{node_id}", + mem_gb=0, + mem_x=(0.0115, "in_file", "t"), + ) + reorient_fs_brainmask.inputs.orientation = "RPI" + reorient_fs_brainmask.inputs.outputtype = "NIFTI_GZ" - wf.connect(convert_fs_brainmask_to_nifti, 'out_file', - reorient_fs_brainmask, 'in_file') + wf.connect( + convert_fs_brainmask_to_nifti, "out_file", reorient_fs_brainmask, "in_file" + ) # fslmaths brain_fs.nii.gz -abs -bin brain_fs_mask.nii.gz - binarize_fs_brain = pe.Node(interface=fsl.maths.MathsCommand(), - name=f'binarize_fs_brain_{node_id}') - binarize_fs_brain.inputs.args = '-abs -bin' + binarize_fs_brain = pe.Node( + interface=fsl.maths.MathsCommand(), name=f"binarize_fs_brain_{node_id}" + ) + binarize_fs_brain.inputs.args = "-abs -bin" - wf.connect(reorient_fs_brainmask, 'out_file', - binarize_fs_brain, 'in_file') + wf.connect(reorient_fs_brainmask, "out_file", binarize_fs_brain, "in_file") # 3dresample -orient RPI -inset T1.nii.gz -prefix head_fs.nii.gz - reorient_fs_T1 = pe.Node(interface=afni.Resample(), - name=f'reorient_fs_T1_{node_id}', - mem_gb=0, - mem_x=(0.0115, 'in_file', 't')) - reorient_fs_T1.inputs.orientation = 'RPI' - reorient_fs_T1.inputs.outputtype = 'NIFTI_GZ' + reorient_fs_T1 = pe.Node( + interface=afni.Resample(), + name=f"reorient_fs_T1_{node_id}", + mem_gb=0, + mem_x=(0.0115, "in_file", "t"), + ) + reorient_fs_T1.inputs.orientation = "RPI" + reorient_fs_T1.inputs.outputtype = "NIFTI_GZ" - wf.connect(convert_fs_T1_to_nifti, 'out_file', - reorient_fs_T1, 'in_file') + wf.connect(convert_fs_T1_to_nifti, "out_file", reorient_fs_T1, "in_file") # flirt -in head_fs.nii.gz -ref ${FSLDIR}/data/standard/MNI152_T1_1mm.nii.gz \ # -out tmp_head_fs2standard.nii.gz -omat tmp_head_fs2standard.mat -bins 256 -cost corratio \ # -searchrx -90 90 -searchry -90 90 -searchrz -90 90 -dof 12 -interp trilinear - convert_head_to_template = pe.Node(interface=fsl.FLIRT(), - name=f'convert_head_to_template_{node_id}') - convert_head_to_template.inputs.cost = 'corratio' - convert_head_to_template.inputs.interp = 'trilinear' + convert_head_to_template = pe.Node( + interface=fsl.FLIRT(), name=f"convert_head_to_template_{node_id}" + ) + convert_head_to_template.inputs.cost = "corratio" + convert_head_to_template.inputs.interp = "trilinear" convert_head_to_template.inputs.bins = 256 convert_head_to_template.inputs.dof = 12 convert_head_to_template.inputs.searchr_x = [-90, 90] convert_head_to_template.inputs.searchr_y = [-90, 90] convert_head_to_template.inputs.searchr_z = [-90, 90] - wf.connect(reorient_fs_T1, 'out_file', - convert_head_to_template, 'in_file') + wf.connect(reorient_fs_T1, "out_file", convert_head_to_template, "in_file") - node, out = strat_pool.get_data('T1w-ACPC-template') - wf.connect(node, out, convert_head_to_template, 'reference') + node, out = strat_pool.get_data("T1w-ACPC-template") + wf.connect(node, out, convert_head_to_template, "reference") # convert_xfm -omat tmp_standard2head_fs.mat -inverse tmp_head_fs2standard.mat - convert_xfm = pe.Node(interface=fsl_utils.ConvertXFM(), - name=f'convert_xfm_{node_id}') + convert_xfm = pe.Node( + interface=fsl_utils.ConvertXFM(), name=f"convert_xfm_{node_id}" + ) convert_xfm.inputs.invert_xfm = True - wf.connect(convert_head_to_template, 'out_matrix_file', - convert_xfm, 'in_file') + wf.connect(convert_head_to_template, "out_matrix_file", convert_xfm, "in_file") # bet tmp_head_fs2standard.nii.gz tmp.nii.gz -f ${bet_thr_tight} -m - skullstrip = pe.Node(interface=fsl.BET(), - name=f'anat_BET_skullstrip_{node_id}') - skullstrip.inputs.output_type = 'NIFTI_GZ' - skullstrip.inputs.mask=True - - if opt == 'FreeSurfer-BET-Tight': - skullstrip.inputs.frac=0.3 - elif opt == 'FreeSurfer-BET-Loose': - skullstrip.inputs.frac=0.1 - - wf.connect(convert_head_to_template, 'out_file', - skullstrip, 'in_file') - + skullstrip = pe.Node(interface=fsl.BET(), name=f"anat_BET_skullstrip_{node_id}") + skullstrip.inputs.output_type = "NIFTI_GZ" + skullstrip.inputs.mask = True + + if opt == "FreeSurfer-BET-Tight": + skullstrip.inputs.frac = 0.3 + elif opt == "FreeSurfer-BET-Loose": + skullstrip.inputs.frac = 0.1 + + wf.connect(convert_head_to_template, "out_file", skullstrip, "in_file") + # fslmaths tmp_mask.nii.gz -mas ${CCSDIR}/templates/MNI152_T1_1mm_first_brain_mask.nii.gz tmp_mask.nii.gz - apply_mask = pe.Node(interface=fsl.maths.ApplyMask(), - name=f'apply_mask_{node_id}') + apply_mask = pe.Node(interface=fsl.maths.ApplyMask(), name=f"apply_mask_{node_id}") - wf.connect(skullstrip, 'out_file', - apply_mask, 'in_file') + wf.connect(skullstrip, "out_file", apply_mask, "in_file") - node, out = strat_pool.get_data('T1w-brain-template-mask-ccs') - wf.connect(node, out, apply_mask, 'mask_file') + node, out = strat_pool.get_data("T1w-brain-template-mask-ccs") + wf.connect(node, out, apply_mask, "mask_file") # flirt -in tmp_mask.nii.gz -applyxfm -init tmp_standard2head_fs.mat -out brain_fsl_mask_tight.nii.gz \ # -paddingsize 0.0 -interp nearestneighbour -ref head_fs.nii.gz - convert_template_mask_to_native = pe.Node(interface=fsl.FLIRT(), - name=f'convert_template_mask_to_native_{node_id}') + convert_template_mask_to_native = pe.Node( + interface=fsl.FLIRT(), name=f"convert_template_mask_to_native_{node_id}" + ) convert_template_mask_to_native.inputs.apply_xfm = True convert_template_mask_to_native.inputs.padding_size = 0 - convert_template_mask_to_native.inputs.interp = 'nearestneighbour' + convert_template_mask_to_native.inputs.interp = "nearestneighbour" - wf.connect(apply_mask, 'out_file', - convert_template_mask_to_native, 'in_file') + wf.connect(apply_mask, "out_file", convert_template_mask_to_native, "in_file") - wf.connect(convert_xfm, 'out_file', - convert_template_mask_to_native, 'in_matrix_file') + wf.connect( + convert_xfm, "out_file", convert_template_mask_to_native, "in_matrix_file" + ) - wf.connect(reorient_fs_T1, 'out_file', - convert_template_mask_to_native, 'reference') + wf.connect(reorient_fs_T1, "out_file", convert_template_mask_to_native, "reference") # fslmaths brain_fs_mask.nii.gz -add brain_fsl_mask_tight.nii.gz -bin brain_mask_tight.nii.gz - # BinaryMaths doesn't use -bin! - combine_mask = pe.Node(interface=fsl.BinaryMaths(), - name=f'combine_mask_{node_id}') + # BinaryMaths doesn't use -bin! + combine_mask = pe.Node(interface=fsl.BinaryMaths(), name=f"combine_mask_{node_id}") - if opt == 'FreeSurfer-BET-Tight': - combine_mask.inputs.operation = 'add' - elif opt == 'FreeSurfer-BET-Loose': - combine_mask.inputs.operation = 'mul' + if opt == "FreeSurfer-BET-Tight": + combine_mask.inputs.operation = "add" + elif opt == "FreeSurfer-BET-Loose": + combine_mask.inputs.operation = "mul" - wf.connect(binarize_fs_brain, 'out_file', - combine_mask, 'in_file') + wf.connect(binarize_fs_brain, "out_file", combine_mask, "in_file") - wf.connect(convert_template_mask_to_native, 'out_file', - combine_mask, 'operand_file') + wf.connect( + convert_template_mask_to_native, "out_file", combine_mask, "operand_file" + ) - binarize_combined_mask = pe.Node(interface=fsl.maths.MathsCommand(), - name=f'binarize_combined_mask_{node_id}') - binarize_combined_mask.inputs.args = '-bin' + binarize_combined_mask = pe.Node( + interface=fsl.maths.MathsCommand(), name=f"binarize_combined_mask_{node_id}" + ) + binarize_combined_mask.inputs.args = "-bin" - wf.connect(combine_mask, 'out_file', - binarize_combined_mask, 'in_file') + wf.connect(combine_mask, "out_file", binarize_combined_mask, "in_file") # CCS brain mask is in FS space, transfer it back to native T1 space - fs_fsl_brain_mask_to_native = pe.Node(interface=freesurfer.ApplyVolTransform(), - name=f'fs_fsl_brain_mask_to_native_{node_id}') + fs_fsl_brain_mask_to_native = pe.Node( + interface=freesurfer.ApplyVolTransform(), + name=f"fs_fsl_brain_mask_to_native_{node_id}", + ) fs_fsl_brain_mask_to_native.inputs.reg_header = True - fs_fsl_brain_mask_to_native.inputs.interp = 'nearest' + fs_fsl_brain_mask_to_native.inputs.interp = "nearest" - wf.connect(binarize_combined_mask, 'out_file', - fs_fsl_brain_mask_to_native, 'source_file') + wf.connect( + binarize_combined_mask, "out_file", fs_fsl_brain_mask_to_native, "source_file" + ) - node, out = strat_pool.get_data('pipeline-fs_raw-average') - wf.connect(node, out, fs_fsl_brain_mask_to_native, 'target_file') + node, out = strat_pool.get_data("pipeline-fs_raw-average") + wf.connect(node, out, fs_fsl_brain_mask_to_native, "target_file") - node, out = strat_pool.get_data('freesurfer-subject-dir') - wf.connect(node, out, fs_fsl_brain_mask_to_native, 'subjects_dir') + node, out = strat_pool.get_data("freesurfer-subject-dir") + wf.connect(node, out, fs_fsl_brain_mask_to_native, "subjects_dir") - if opt == 'FreeSurfer-BET-Tight': + if opt == "FreeSurfer-BET-Tight": outputs = { - 'space-T1w_desc-tight_brain_mask': (fs_fsl_brain_mask_to_native, 'transformed_file') + "space-T1w_desc-tight_brain_mask": ( + fs_fsl_brain_mask_to_native, + "transformed_file", + ) } - elif opt == 'FreeSurfer-BET-Loose': + elif opt == "FreeSurfer-BET-Loose": outputs = { - 'space-T1w_desc-loose_brain_mask': (fs_fsl_brain_mask_to_native, 'transformed_file') + "space-T1w_desc-loose_brain_mask": ( + fs_fsl_brain_mask_to_native, + "transformed_file", + ) } return (wf, outputs) -def mask_T2(wf_name='mask_T2'): +def mask_T2(wf_name="mask_T2"): # create T2 mask based on T1 mask # reference https://github.com/DCAN-Labs/dcan-macaque-pipeline/blob/master/PreliminaryMasking/macaque_masking.py - + preproc = pe.Workflow(name=wf_name) - inputnode = pe.Node(util.IdentityInterface(fields=['T1w', - 'T1w_mask', - 'T2w']), - name='inputspec') + inputnode = pe.Node( + util.IdentityInterface(fields=["T1w", "T1w_mask", "T2w"]), name="inputspec" + ) - outputnode = pe.Node(util.IdentityInterface(fields=['T1w_brain', - 'T2w_mask', - 'T2w_brain']), - name='outputspec') + outputnode = pe.Node( + util.IdentityInterface(fields=["T1w_brain", "T2w_mask", "T2w_brain"]), + name="outputspec", + ) # mask_t1w = 'fslmaths {t1w} -mas {t1w_mask_edit} {t1w_brain}'.format(**kwargs) - mask_t1w = pe.Node(interface=fsl.MultiImageMaths(), - name='mask_t1w') + mask_t1w = pe.Node(interface=fsl.MultiImageMaths(), name="mask_t1w") mask_t1w.inputs.op_string = "-mas %s " - preproc.connect(inputnode, 'T1w', mask_t1w, 'in_file') - preproc.connect(inputnode, 'T1w_mask', mask_t1w, 'operand_files') - + preproc.connect(inputnode, "T1w", mask_t1w, "in_file") + preproc.connect(inputnode, "T1w_mask", mask_t1w, "operand_files") # t1w2t2w_rigid = 'flirt -dof 6 -cost mutualinfo -in {t1w} -ref {t2w} ' \ # '-omat {t1w2t2w}'.format(**kwargs) - t1w2t2w_rigid = pe.Node(interface=fsl.FLIRT(), - name='t1w2t2w_rigid') + t1w2t2w_rigid = pe.Node(interface=fsl.FLIRT(), name="t1w2t2w_rigid") t1w2t2w_rigid.inputs.dof = 6 - t1w2t2w_rigid.inputs.cost = 'mutualinfo' - preproc.connect(inputnode, 'T1w', t1w2t2w_rigid, 'in_file') - preproc.connect(inputnode, 'T2w', t1w2t2w_rigid, 'reference') + t1w2t2w_rigid.inputs.cost = "mutualinfo" + preproc.connect(inputnode, "T1w", t1w2t2w_rigid, "in_file") + preproc.connect(inputnode, "T2w", t1w2t2w_rigid, "reference") # t1w2t2w_mask = 'flirt -in {t1w_mask_edit} -interp nearestneighbour -ref {' \ # 't2w} -o {t2w_brain_mask} -applyxfm -init {' \ # 't1w2t2w}'.format(**kwargs) - t1w2t2w_mask = pe.Node(interface=fsl.FLIRT(), - name='t1w2t2w_mask') + t1w2t2w_mask = pe.Node(interface=fsl.FLIRT(), name="t1w2t2w_mask") t1w2t2w_mask.inputs.apply_xfm = True - t1w2t2w_mask.inputs.interp = 'nearestneighbour' + t1w2t2w_mask.inputs.interp = "nearestneighbour" - preproc.connect(inputnode, 'T1w_mask', t1w2t2w_mask, 'in_file') - preproc.connect(inputnode, 'T2w', t1w2t2w_mask, 'reference') - preproc.connect(t1w2t2w_rigid, 'out_matrix_file', t1w2t2w_mask, 'in_matrix_file') + preproc.connect(inputnode, "T1w_mask", t1w2t2w_mask, "in_file") + preproc.connect(inputnode, "T2w", t1w2t2w_mask, "reference") + preproc.connect(t1w2t2w_rigid, "out_matrix_file", t1w2t2w_mask, "in_matrix_file") # mask_t2w = 'fslmaths {t2w} -mas {t2w_brain_mask} ' \ # '{t2w_brain}'.format(**kwargs) - mask_t2w = pe.Node(interface=fsl.MultiImageMaths(), - name='mask_t2w') + mask_t2w = pe.Node(interface=fsl.MultiImageMaths(), name="mask_t2w") mask_t2w.inputs.op_string = "-mas %s " - preproc.connect(inputnode, 'T2w', mask_t2w, 'in_file') - preproc.connect(t1w2t2w_mask, 'out_file', mask_t2w, 'operand_files') + preproc.connect(inputnode, "T2w", mask_t2w, "in_file") + preproc.connect(t1w2t2w_mask, "out_file", mask_t2w, "operand_files") - preproc.connect(mask_t1w, 'out_file', outputnode, 'T1w_brain') - preproc.connect(mask_t2w, 'out_file', outputnode, 'T2w_brain') - preproc.connect(t1w2t2w_mask, 'out_file', outputnode, 'T2w_mask') + preproc.connect(mask_t1w, "out_file", outputnode, "T1w_brain") + preproc.connect(mask_t2w, "out_file", outputnode, "T2w_brain") + preproc.connect(t1w2t2w_mask, "out_file", outputnode, "T2w_mask") return preproc @@ -1248,29 +1403,32 @@ def mask_T2(wf_name='mask_T2'): outputs=["desc-preproc_T1w", "desc-reorient_T1w", "desc-head_T1w"], ) def anatomical_init(wf, cfg, strat_pool, pipe_num, opt=None): - - anat_deoblique = pe.Node(interface=afni.Refit(), - name=f'anat_deoblique_{pipe_num}') + anat_deoblique = pe.Node(interface=afni.Refit(), name=f"anat_deoblique_{pipe_num}") anat_deoblique.inputs.deoblique = True - node, out = strat_pool.get_data('T1w') - wf.connect(node, out, anat_deoblique, 'in_file') + node, out = strat_pool.get_data("T1w") + wf.connect(node, out, anat_deoblique, "in_file") - anat_reorient = pe.Node(interface=afni.Resample(), - name=f'anat_reorient_{pipe_num}', - mem_gb=0, - mem_x=(0.0115, 'in_file', 't')) - anat_reorient.inputs.orientation = 'RPI' - anat_reorient.inputs.outputtype = 'NIFTI_GZ' + anat_reorient = pe.Node( + interface=afni.Resample(), + name=f"anat_reorient_{pipe_num}", + mem_gb=0, + mem_x=(0.0115, "in_file", "t"), + ) + anat_reorient.inputs.orientation = "RPI" + anat_reorient.inputs.outputtype = "NIFTI_GZ" - wf.connect(anat_deoblique, 'out_file', anat_reorient, 'in_file') + wf.connect(anat_deoblique, "out_file", anat_reorient, "in_file") - outputs = {'desc-preproc_T1w': (anat_reorient, 'out_file'), - 'desc-reorient_T1w': (anat_reorient, 'out_file'), - 'desc-head_T1w': (anat_reorient, 'out_file')} + outputs = { + "desc-preproc_T1w": (anat_reorient, "out_file"), + "desc-reorient_T1w": (anat_reorient, "out_file"), + "desc-head_T1w": (anat_reorient, "out_file"), + } return (wf, outputs) + @nodeblock( name="acpc_alignment_head", switch=[ @@ -1285,25 +1443,26 @@ def anatomical_init(wf, cfg, strat_pool, pipe_num, opt=None): ], ) def acpc_align_head(wf, cfg, strat_pool, pipe_num, opt=None): + acpc_align = acpc_alignment( + config=cfg, + acpc_target=cfg.anatomical_preproc["acpc_alignment"]["acpc_target"], + mask=False, + wf_name=f"acpc_align_{pipe_num}", + ) - acpc_align = acpc_alignment(config=cfg, - acpc_target=cfg.anatomical_preproc[ - 'acpc_alignment']['acpc_target'], - mask=False, - wf_name=f'acpc_align_{pipe_num}') - - node, out = strat_pool.get_data(['desc-preproc_T1w','desc-head_T1w']) - wf.connect(node, out, acpc_align, 'inputspec.anat_leaf') + node, out = strat_pool.get_data(["desc-preproc_T1w", "desc-head_T1w"]) + wf.connect(node, out, acpc_align, "inputspec.anat_leaf") - node, out = strat_pool.get_data('T1w-ACPC-template') - wf.connect(node, out, acpc_align, 'inputspec.template_head_for_acpc') + node, out = strat_pool.get_data("T1w-ACPC-template") + wf.connect(node, out, acpc_align, "inputspec.template_head_for_acpc") outputs = { - 'desc-head_T1w': (acpc_align, 'outputspec.acpc_aligned_head'), - 'desc-preproc_T1w': (acpc_align, 'outputspec.acpc_aligned_head'), - 'from-T1w_to-ACPC_mode-image_desc-aff2rig_xfm': ( + "desc-head_T1w": (acpc_align, "outputspec.acpc_aligned_head"), + "desc-preproc_T1w": (acpc_align, "outputspec.acpc_aligned_head"), + "from-T1w_to-ACPC_mode-image_desc-aff2rig_xfm": ( acpc_align, - 'outputspec.from-T1w_to-ACPC_mode-image_desc-aff2rig_xfm') + "outputspec.from-T1w_to-ACPC_mode-image_desc-aff2rig_xfm", + ), } return (wf, outputs) @@ -1332,33 +1491,34 @@ def acpc_align_head(wf, cfg, strat_pool, pipe_num, opt=None): ], ) def acpc_align_head_with_mask(wf, cfg, strat_pool, pipe_num, opt=None): + acpc_align = acpc_alignment( + config=cfg, + acpc_target=cfg.anatomical_preproc["acpc_alignment"]["acpc_target"], + mask=True, + wf_name=f"acpc_align_{pipe_num}", + ) - acpc_align = acpc_alignment(config=cfg, - acpc_target=cfg.anatomical_preproc[ - 'acpc_alignment']['acpc_target'], - mask=True, - wf_name=f'acpc_align_{pipe_num}') - - node, out = strat_pool.get_data(['desc-head_T1w', 'desc-preproc_T1w']) - wf.connect(node, out, acpc_align, 'inputspec.anat_leaf') + node, out = strat_pool.get_data(["desc-head_T1w", "desc-preproc_T1w"]) + wf.connect(node, out, acpc_align, "inputspec.anat_leaf") - node, out = strat_pool.get_data('T1w-ACPC-template') - wf.connect(node, out, acpc_align, 'inputspec.template_head_for_acpc') + node, out = strat_pool.get_data("T1w-ACPC-template") + wf.connect(node, out, acpc_align, "inputspec.template_head_for_acpc") if strat_pool.check_rpool("space-T1w_desc-brain_mask"): node, out = strat_pool.get_data("space-T1w_desc-brain_mask") - wf.connect(node, out, acpc_align, 'inputspec.brain_mask') + wf.connect(node, out, acpc_align, "inputspec.brain_mask") - node, out = strat_pool.get_data('T1w-brain-ACPC-template') - wf.connect(node, out, acpc_align, 'inputspec.template_brain_for_acpc') + node, out = strat_pool.get_data("T1w-brain-ACPC-template") + wf.connect(node, out, acpc_align, "inputspec.template_brain_for_acpc") outputs = { - 'desc-head_T1w': (acpc_align, 'outputspec.acpc_aligned_head'), - 'desc-preproc_T1w': (acpc_align, 'outputspec.acpc_aligned_head'), - 'space-T1w_desc-brain_mask': ( - acpc_align, 'outputspec.acpc_brain_mask'), - 'from-T1w_to-ACPC_mode-image_desc-aff2rig_xfm': ( - acpc_align, 'outputspec.from-T1w_to-ACPC_mode-image_desc-aff2rig_xfm') + "desc-head_T1w": (acpc_align, "outputspec.acpc_aligned_head"), + "desc-preproc_T1w": (acpc_align, "outputspec.acpc_aligned_head"), + "space-T1w_desc-brain_mask": (acpc_align, "outputspec.acpc_brain_mask"), + "from-T1w_to-ACPC_mode-image_desc-aff2rig_xfm": ( + acpc_align, + "outputspec.from-T1w_to-ACPC_mode-image_desc-aff2rig_xfm", + ), } return (wf, outputs) @@ -1385,30 +1545,32 @@ def acpc_align_head_with_mask(wf, cfg, strat_pool, pipe_num, opt=None): ], ) def acpc_align_brain(wf, cfg, strat_pool, pipe_num, opt=None): + acpc_align = acpc_alignment( + config=cfg, + acpc_target=cfg.anatomical_preproc["acpc_alignment"]["acpc_target"], + mask=False, + wf_name=f"acpc_align_{pipe_num}", + ) - acpc_align = acpc_alignment(config=cfg, - acpc_target=cfg.anatomical_preproc[ - 'acpc_alignment']['acpc_target'], - mask=False, - wf_name=f'acpc_align_{pipe_num}') - - node, out = strat_pool.get_data('desc-preproc_T1w') - wf.connect(node, out, acpc_align, 'inputspec.anat_leaf') + node, out = strat_pool.get_data("desc-preproc_T1w") + wf.connect(node, out, acpc_align, "inputspec.anat_leaf") - node, out = strat_pool.get_data('desc-tempbrain_T1w') - wf.connect(node, out, acpc_align, 'inputspec.anat_brain') + node, out = strat_pool.get_data("desc-tempbrain_T1w") + wf.connect(node, out, acpc_align, "inputspec.anat_brain") - node, out = strat_pool.get_data('T1w-ACPC-template') - wf.connect(node, out, acpc_align, 'inputspec.template_head_for_acpc') + node, out = strat_pool.get_data("T1w-ACPC-template") + wf.connect(node, out, acpc_align, "inputspec.template_head_for_acpc") - node, out = strat_pool.get_data('T1w-brain-ACPC-template') - wf.connect(node, out, acpc_align, 'inputspec.template_brain_for_acpc') + node, out = strat_pool.get_data("T1w-brain-ACPC-template") + wf.connect(node, out, acpc_align, "inputspec.template_brain_for_acpc") outputs = { - 'desc-preproc_T1w': (acpc_align, 'outputspec.acpc_aligned_head'), - 'desc-acpcbrain_T1w': (acpc_align, 'outputspec.acpc_aligned_brain'), - 'from-T1w_to-ACPC_mode-image_desc-aff2rig_xfm': ( - acpc_align, 'outputspec.from-T1w_to-ACPC_mode-image_desc-aff2rig_xfm') + "desc-preproc_T1w": (acpc_align, "outputspec.acpc_aligned_head"), + "desc-acpcbrain_T1w": (acpc_align, "outputspec.acpc_aligned_brain"), + "from-T1w_to-ACPC_mode-image_desc-aff2rig_xfm": ( + acpc_align, + "outputspec.from-T1w_to-ACPC_mode-image_desc-aff2rig_xfm", + ), } return (wf, outputs) @@ -1433,33 +1595,35 @@ def acpc_align_brain(wf, cfg, strat_pool, pipe_num, opt=None): ], ) def acpc_align_brain_with_mask(wf, cfg, strat_pool, pipe_num, opt=None): + acpc_align = acpc_alignment( + config=cfg, + acpc_target=cfg.anatomical_preproc["acpc_alignment"]["acpc_target"], + mask=True, + wf_name=f"acpc_align_{pipe_num}", + ) - acpc_align = acpc_alignment(config=cfg, - acpc_target=cfg.anatomical_preproc[ - 'acpc_alignment']['acpc_target'], - mask=True, - wf_name=f'acpc_align_{pipe_num}') - - node, out = strat_pool.get_data('desc-preproc_T1w') - wf.connect(node, out, acpc_align, 'inputspec.anat_leaf') + node, out = strat_pool.get_data("desc-preproc_T1w") + wf.connect(node, out, acpc_align, "inputspec.anat_leaf") - node, out = strat_pool.get_data('desc-tempbrain_T1w') - wf.connect(node, out, acpc_align, 'inputspec.anat_brain') + node, out = strat_pool.get_data("desc-tempbrain_T1w") + wf.connect(node, out, acpc_align, "inputspec.anat_brain") node, out = strat_pool.get_data("space-T1w_desc-brain_mask") - wf.connect(node, out, acpc_align, 'inputspec.brain_mask') + wf.connect(node, out, acpc_align, "inputspec.brain_mask") - node, out = strat_pool.get_data('T1w-ACPC-template') - wf.connect(node, out, acpc_align, 'inputspec.template_head_for_acpc') + node, out = strat_pool.get_data("T1w-ACPC-template") + wf.connect(node, out, acpc_align, "inputspec.template_head_for_acpc") - node, out = strat_pool.get_data('T1w-brain-ACPC-template') - wf.connect(node, out, acpc_align, 'inputspec.template_brain_for_acpc') + node, out = strat_pool.get_data("T1w-brain-ACPC-template") + wf.connect(node, out, acpc_align, "inputspec.template_brain_for_acpc") outputs = { - 'desc-preproc_T1w': (acpc_align, 'outputspec.acpc_aligned_head'), - 'desc-acpcbrain_T1w': (acpc_align, 'outputspec.acpc_aligned_brain'), - 'space-T1w_desc-brain_mask': (acpc_align, 'outputspec.acpc_brain_mask'), - 'space-T1w_desc-prebrain_mask': (strat_pool.get_data('space-T1_desc-brain_mask')) + "desc-preproc_T1w": (acpc_align, "outputspec.acpc_aligned_head"), + "desc-acpcbrain_T1w": (acpc_align, "outputspec.acpc_aligned_brain"), + "space-T1w_desc-brain_mask": (acpc_align, "outputspec.acpc_brain_mask"), + "space-T1w_desc-prebrain_mask": ( + strat_pool.get_data("space-T1_desc-brain_mask") + ), } return (wf, outputs) @@ -1480,24 +1644,21 @@ def acpc_align_brain_with_mask(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["desc-preproc_T2w"], ) def registration_T2w_to_T1w(wf, cfg, strat_pool, pipe_num, opt=None): + T2_to_T1_reg = T2wToT1wReg(wf_name=f"T2w_to_T1w_Reg_{pipe_num}") - T2_to_T1_reg = T2wToT1wReg(wf_name=f'T2w_to_T1w_Reg_{pipe_num}') + node, out = strat_pool.get_data("desc-preproc_T1w") + wf.connect(node, out, T2_to_T1_reg, "inputspec.T1w") - node, out = strat_pool.get_data('desc-preproc_T1w') - wf.connect(node, out, T2_to_T1_reg, 'inputspec.T1w') + node, out = strat_pool.get_data("desc-preproc_T2w") + wf.connect(node, out, T2_to_T1_reg, "inputspec.T2w") - node, out = strat_pool.get_data('desc-preproc_T2w') - wf.connect(node, out, T2_to_T1_reg, 'inputspec.T2w') + node, out = strat_pool.get_data(["desc-acpcbrain_T1w"]) + wf.connect(node, out, T2_to_T1_reg, "inputspec.T1w_brain") - node, out = strat_pool.get_data(['desc-acpcbrain_T1w']) - wf.connect(node, out, T2_to_T1_reg, 'inputspec.T1w_brain') + node, out = strat_pool.get_data(["desc-acpcbrain_T2w"]) + wf.connect(node, out, T2_to_T1_reg, "inputspec.T2w_brain") - node, out = strat_pool.get_data(['desc-acpcbrain_T2w']) - wf.connect(node, out, T2_to_T1_reg, 'inputspec.T2w_brain') - - outputs = { - 'desc-preproc_T2w': (T2_to_T1_reg, 'outputspec.T2w_to_T1w') - } + outputs = {"desc-preproc_T2w": (T2_to_T1_reg, "outputspec.T2w_to_T1w")} return (wf, outputs) @@ -1512,18 +1673,16 @@ def registration_T2w_to_T1w(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["desc-preproc_T1w"], ) def non_local_means(wf, cfg, strat_pool, pipe_num, opt=None): + denoise = pe.Node(interface=ants.DenoiseImage(), name=f"anat_denoise_{pipe_num}") - denoise = pe.Node(interface=ants.DenoiseImage(), - name=f'anat_denoise_{pipe_num}') + denoise.inputs.noise_model = cfg.anatomical_preproc["non_local_means_filtering"][ + "noise_model" + ] - denoise.inputs.noise_model = cfg.anatomical_preproc['non_local_means_filtering']['noise_model'] + node, out = strat_pool.get_data("desc-preproc_T1w") + wf.connect(node, out, denoise, "input_image") - node, out = strat_pool.get_data('desc-preproc_T1w') - wf.connect(node, out, denoise, 'input_image') - - outputs = { - 'desc-preproc_T1w': (denoise, 'output_image') - } + outputs = {"desc-preproc_T1w": (denoise, "output_image")} return (wf, outputs) @@ -1545,18 +1704,20 @@ def non_local_means(wf, cfg, strat_pool, pipe_num, opt=None): }, ) def n4_bias_correction(wf, cfg, strat_pool, pipe_num, opt=None): + n4 = pe.Node( + interface=ants.N4BiasFieldCorrection(dimension=3, copy_header=True), + name=f"anat_n4_{pipe_num}", + ) + n4.inputs.shrink_factor = cfg.anatomical_preproc["n4_bias_field_correction"][ + "shrink_factor" + ] - n4 = pe.Node(interface=ants.N4BiasFieldCorrection(dimension=3, - copy_header=True), - name=f'anat_n4_{pipe_num}') - n4.inputs.shrink_factor = cfg.anatomical_preproc['n4_bias_field_correction']['shrink_factor'] - - node, out = strat_pool.get_data('desc-preproc_T1w') - wf.connect(node, out, n4, 'input_image') + node, out = strat_pool.get_data("desc-preproc_T1w") + wf.connect(node, out, n4, "input_image") outputs = { - 'desc-preproc_T1w': (n4, 'output_image'), - 'desc-n4_T1w': (n4, 'output_image') + "desc-preproc_T1w": (n4, "output_image"), + "desc-n4_T1w": (n4, "output_image"), } return (wf, outputs) @@ -1576,24 +1737,25 @@ def n4_bias_correction(wf, cfg, strat_pool, pipe_num, opt=None): ], ) def t1t2_bias_correction(wf, cfg, strat_pool, pipe_num, opt=None): + t1t2_bias_correction = BiasFieldCorrection_sqrtT1wXT1w( + config=cfg, wf_name=f"t1t2_bias_correction_{pipe_num}" + ) - t1t2_bias_correction = BiasFieldCorrection_sqrtT1wXT1w(config=cfg, wf_name=f't1t2_bias_correction_{pipe_num}') - - node, out = strat_pool.get_data('desc-preproc_T1w') - wf.connect(node, out, t1t2_bias_correction, 'inputspec.T1w') + node, out = strat_pool.get_data("desc-preproc_T1w") + wf.connect(node, out, t1t2_bias_correction, "inputspec.T1w") - node, out = strat_pool.get_data('desc-preproc_T2w') - wf.connect(node, out, t1t2_bias_correction, 'inputspec.T2w') + node, out = strat_pool.get_data("desc-preproc_T2w") + wf.connect(node, out, t1t2_bias_correction, "inputspec.T2w") node, out = strat_pool.get_data("desc-acpcbrain_T1w") - wf.connect(node, out, t1t2_bias_correction, 'inputspec.T1w_brain') + wf.connect(node, out, t1t2_bias_correction, "inputspec.T1w_brain") outputs = { - 'desc-preproc_T1w': (t1t2_bias_correction, 'outputspec.T1w_biascorrected'), - 'desc-brain_T1w': (t1t2_bias_correction, 'outputspec.T1w_brain_biascorrected'), - 'desc-preproc_T2w': (t1t2_bias_correction, 'outputspec.T2w_biascorrected'), - 'desc-brain_T2w': (t1t2_bias_correction, 'outputspec.T2w_brain_biascorrected'), - 'desc-biasfield_T1wT2w': (t1t2_bias_correction, 'outputspec.biasfield'), + "desc-preproc_T1w": (t1t2_bias_correction, "outputspec.T1w_biascorrected"), + "desc-brain_T1w": (t1t2_bias_correction, "outputspec.T1w_brain_biascorrected"), + "desc-preproc_T2w": (t1t2_bias_correction, "outputspec.T2w_biascorrected"), + "desc-brain_T2w": (t1t2_bias_correction, "outputspec.T2w_brain_biascorrected"), + "desc-biasfield_T1wT2w": (t1t2_bias_correction, "outputspec.biasfield"), } return (wf, outputs) @@ -1611,7 +1773,6 @@ def t1t2_bias_correction(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["space-T1w_desc-brain_mask"], ) def brain_mask_afni(wf, cfg, strat_pool, pipe_num, opt=None): - wf, outputs = afni_brain_connector(wf, cfg, strat_pool, pipe_num, opt) return (wf, outputs) @@ -1629,13 +1790,9 @@ def brain_mask_afni(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["space-T1w_desc-acpcbrain_mask"], ) def brain_mask_acpc_afni(wf, cfg, strat_pool, pipe_num, opt=None): - wf, wf_outputs = afni_brain_connector(wf, cfg, strat_pool, pipe_num, opt) - outputs = { - 'space-T1w_desc-acpcbrain_mask': - wf_outputs['space-T1w_desc-brain_mask'] - } + outputs = {"space-T1w_desc-acpcbrain_mask": wf_outputs["space-T1w_desc-brain_mask"]} return (wf, outputs) @@ -1652,7 +1809,6 @@ def brain_mask_acpc_afni(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["space-T1w_desc-brain_mask"], ) def brain_mask_fsl(wf, cfg, strat_pool, pipe_num, opt=None): - wf, outputs = fsl_brain_connector(wf, cfg, strat_pool, pipe_num, opt) return (wf, outputs) @@ -1670,13 +1826,9 @@ def brain_mask_fsl(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["space-T1w_desc-acpcbrain_mask"], ) def brain_mask_acpc_fsl(wf, cfg, strat_pool, pipe_num, opt=None): - wf, wf_outputs = fsl_brain_connector(wf, cfg, strat_pool, pipe_num, opt) - outputs = { - 'space-T1w_desc-acpcbrain_mask': - wf_outputs['space-T1w_desc-brain_mask'] - } + outputs = {"space-T1w_desc-acpcbrain_mask": wf_outputs["space-T1w_desc-brain_mask"]} return (wf, outputs) @@ -1693,9 +1845,7 @@ def brain_mask_acpc_fsl(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["space-T1w_desc-brain_mask", "desc-preproc_T1w"], ) def brain_mask_niworkflows_ants(wf, cfg, strat_pool, pipe_num, opt=None): - - wf, outputs = niworkflows_ants_brain_connector(wf, cfg, strat_pool, - pipe_num, opt) + wf, outputs = niworkflows_ants_brain_connector(wf, cfg, strat_pool, pipe_num, opt) return (wf, outputs) @@ -1712,15 +1862,13 @@ def brain_mask_niworkflows_ants(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["space-T1w_desc-acpcbrain_mask", "desc-preproc_T1w"], ) def brain_mask_acpc_niworkflows_ants(wf, cfg, strat_pool, pipe_num, opt=None): - - wf, wf_outputs = niworkflows_ants_brain_connector(wf, cfg, strat_pool, - pipe_num, opt) + wf, wf_outputs = niworkflows_ants_brain_connector( + wf, cfg, strat_pool, pipe_num, opt + ) outputs = { - 'space-T1w_desc-acpcbrain_mask': - wf_outputs['space-T1w_desc-brain_mask'], - 'desc-preproc_T1w': - wf_outputs['desc-preproc_T1w'] + "space-T1w_desc-acpcbrain_mask": wf_outputs["space-T1w_desc-brain_mask"], + "desc-preproc_T1w": wf_outputs["desc-preproc_T1w"], } return (wf, outputs) @@ -1738,7 +1886,6 @@ def brain_mask_acpc_niworkflows_ants(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["space-T1w_desc-brain_mask"], ) def brain_mask_unet(wf, cfg, strat_pool, pipe_num, opt=None): - wf, outputs = unet_brain_connector(wf, cfg, strat_pool, pipe_num, opt) return (wf, outputs) @@ -1756,13 +1903,9 @@ def brain_mask_unet(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["space-T1w_desc-acpcbrain_mask"], ) def brain_mask_acpc_unet(wf, cfg, strat_pool, pipe_num, opt=None): - wf, wf_outputs = unet_brain_connector(wf, cfg, strat_pool, pipe_num, opt) - outputs = { - 'space-T1w_desc-acpcbrain_mask': - wf_outputs['space-T1w_desc-brain_mask'] - } + outputs = {"space-T1w_desc-acpcbrain_mask": wf_outputs["space-T1w_desc-brain_mask"]} return (wf, outputs) @@ -1783,9 +1926,7 @@ def brain_mask_acpc_unet(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["space-T1w_desc-brain_mask"], ) def brain_mask_freesurfer(wf, cfg, strat_pool, pipe_num, opt=None): - - wf, outputs = freesurfer_brain_connector(wf, cfg, strat_pool, pipe_num, - opt) + wf, outputs = freesurfer_brain_connector(wf, cfg, strat_pool, pipe_num, opt) return (wf, outputs) @@ -1806,12 +1947,9 @@ def brain_mask_freesurfer(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["space-T1w_desc-acpcbrain_mask"], ) def brain_mask_acpc_freesurfer(wf, cfg, strat_pool, pipe_num, opt=None): + wf, wf_outputs = freesurfer_brain_connector(wf, cfg, strat_pool, pipe_num, opt) - wf, wf_outputs = freesurfer_brain_connector(wf, cfg, strat_pool, pipe_num, - opt) - - outputs = {'space-T1w_desc-acpcbrain_mask': - wf_outputs['space-T1w_desc-brain_mask']} + outputs = {"space-T1w_desc-acpcbrain_mask": wf_outputs["space-T1w_desc-brain_mask"]} return (wf, outputs) @@ -1834,9 +1972,7 @@ def brain_mask_acpc_freesurfer(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["space-T1w_desc-brain_mask"], ) def brain_mask_freesurfer_abcd(wf, cfg, strat_pool, pipe_num, opt=None): - - wf, outputs = freesurfer_abcd_brain_connector(wf, cfg, strat_pool, - pipe_num, opt) + wf, outputs = freesurfer_abcd_brain_connector(wf, cfg, strat_pool, pipe_num, opt) return (wf, outputs) @@ -1860,7 +1996,6 @@ def brain_mask_freesurfer_abcd(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["space-T1w_desc-tight_brain_mask"], ) def brain_mask_freesurfer_fsl_tight(wf, cfg, strat_pool, pipe_num, opt=None): - wf, outputs = freesurfer_fsl_brain_connector(wf, cfg, strat_pool, pipe_num, opt) return (wf, outputs) @@ -1884,11 +2019,9 @@ def brain_mask_freesurfer_fsl_tight(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["space-T1w_desc-acpcbrain_mask"], ) def brain_mask_acpc_freesurfer_abcd(wf, cfg, strat_pool, pipe_num, opt=None): - wf, wf_outputs = freesurfer_abcd_brain_connector(wf, cfg, strat_pool, pipe_num, opt) - outputs = {'space-T1w_desc-acpcbrain_mask': - wf_outputs['space-T1w_desc-brain_mask']} + outputs = {"space-T1w_desc-acpcbrain_mask": wf_outputs["space-T1w_desc-brain_mask"]} return (wf, outputs) @@ -1912,7 +2045,6 @@ def brain_mask_acpc_freesurfer_abcd(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["space-T1w_desc-loose_brain_mask"], ) def brain_mask_freesurfer_fsl_loose(wf, cfg, strat_pool, pipe_num, opt=None): - wf, outputs = freesurfer_fsl_brain_connector(wf, cfg, strat_pool, pipe_num, opt) return (wf, outputs) @@ -1935,12 +2067,13 @@ def brain_mask_freesurfer_fsl_loose(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["space-T1w_desc-tight_acpcbrain_mask"], ) def brain_mask_acpc_freesurfer_fsl_tight(wf, cfg, strat_pool, pipe_num, opt=None): + wf, wf_outputs = freesurfer_fsl_brain_connector(wf, cfg, strat_pool, pipe_num, opt) - wf, wf_outputs = freesurfer_fsl_brain_connector(wf, cfg, strat_pool, - pipe_num, opt) - - outputs = {'space-T1w_desc-tight_acpcbrain_mask': - wf_outputs['space-T1w_desc-tight_brain_mask']} + outputs = { + "space-T1w_desc-tight_acpcbrain_mask": wf_outputs[ + "space-T1w_desc-tight_brain_mask" + ] + } return (wf, outputs) @@ -1962,11 +2095,13 @@ def brain_mask_acpc_freesurfer_fsl_tight(wf, cfg, strat_pool, pipe_num, opt=None outputs=["space-T1w_desc-loose_acpcbrain_mask"], ) def brain_mask_acpc_freesurfer_fsl_loose(wf, cfg, strat_pool, pipe_num, opt=None): - wf, wf_outputs = freesurfer_fsl_brain_connector(wf, cfg, strat_pool, pipe_num, opt) - outputs = {'space-T1w_desc-loose_acpcbrain_mask': - wf_outputs['space-T1w_desc-loose_brain_mask']} + outputs = { + "space-T1w_desc-loose_acpcbrain_mask": wf_outputs[ + "space-T1w_desc-loose_brain_mask" + ] + } return (wf, outputs) @@ -1991,13 +2126,12 @@ def brain_mask_acpc_freesurfer_fsl_loose(wf, cfg, strat_pool, pipe_num, opt=None }, ) def brain_extraction(wf, cfg, strat_pool, pipe_num, opt=None): - - ''' + """ brain_mask_deoblique = pe.Node(interface=afni.Refit(), name='brain_mask_deoblique') brain_mask_deoblique.inputs.deoblique = True wf.connect(inputnode, 'brain_mask', - brain_mask_deoblique, 'in_file') + brain_mask_deoblique, 'in_file'). brain_mask_reorient = pe.Node(interface=afni.Resample(), name='brain_mask_reorient', @@ -2007,25 +2141,26 @@ def brain_extraction(wf, cfg, strat_pool, pipe_num, opt=None): brain_mask_reorient.inputs.outputtype = 'NIFTI_GZ' wf.connect(brain_mask_deoblique, 'out_file', brain_mask_reorient, 'in_file') - ''' - - anat_skullstrip_orig_vol = pe.Node(interface=afni.Calc(), - name=f'brain_extraction_{pipe_num}') + """ + anat_skullstrip_orig_vol = pe.Node( + interface=afni.Calc(), name=f"brain_extraction_{pipe_num}" + ) - anat_skullstrip_orig_vol.inputs.expr = 'a*step(b)' - anat_skullstrip_orig_vol.inputs.outputtype = 'NIFTI_GZ' + anat_skullstrip_orig_vol.inputs.expr = "a*step(b)" + anat_skullstrip_orig_vol.inputs.outputtype = "NIFTI_GZ" - node_T1w, out_T1w = strat_pool.get_data('desc-head_T1w') - wf.connect(node_T1w, out_T1w, anat_skullstrip_orig_vol, 'in_file_a') + node_T1w, out_T1w = strat_pool.get_data("desc-head_T1w") + wf.connect(node_T1w, out_T1w, anat_skullstrip_orig_vol, "in_file_a") - node, out = strat_pool.get_data(['space-T1w_desc-brain_mask', - 'space-T1w_desc-acpcbrain_mask']) - wf.connect(node, out, anat_skullstrip_orig_vol, 'in_file_b') + node, out = strat_pool.get_data( + ["space-T1w_desc-brain_mask", "space-T1w_desc-acpcbrain_mask"] + ) + wf.connect(node, out, anat_skullstrip_orig_vol, "in_file_b") outputs = { - 'desc-preproc_T1w': (anat_skullstrip_orig_vol, 'out_file'), - 'desc-brain_T1w': (anat_skullstrip_orig_vol, 'out_file'), - 'desc-head_T1w': (node_T1w, out_T1w) + "desc-preproc_T1w": (anat_skullstrip_orig_vol, "out_file"), + "desc-brain_T1w": (anat_skullstrip_orig_vol, "out_file"), + "desc-head_T1w": (node_T1w, out_T1w), } return (wf, outputs) @@ -2045,23 +2180,24 @@ def brain_extraction(wf, cfg, strat_pool, pipe_num, opt=None): }, ) def brain_extraction_temp(wf, cfg, strat_pool, pipe_num, opt=None): + anat_skullstrip_orig_vol = pe.Node( + interface=afni.Calc(), name=f"brain_extraction_temp_{pipe_num}" + ) - anat_skullstrip_orig_vol = pe.Node(interface=afni.Calc(), - name=f'brain_extraction_temp_{pipe_num}') - - anat_skullstrip_orig_vol.inputs.expr = 'a*step(b)' - anat_skullstrip_orig_vol.inputs.outputtype = 'NIFTI_GZ' + anat_skullstrip_orig_vol.inputs.expr = "a*step(b)" + anat_skullstrip_orig_vol.inputs.outputtype = "NIFTI_GZ" - node, out = strat_pool.get_data('desc-preproc_T1w') - wf.connect(node, out, anat_skullstrip_orig_vol, 'in_file_a') + node, out = strat_pool.get_data("desc-preproc_T1w") + wf.connect(node, out, anat_skullstrip_orig_vol, "in_file_a") - node, out = strat_pool.get_data(['space-T1w_desc-brain_mask', - 'space-T1w_desc-acpcbrain_mask']) - wf.connect(node, out, anat_skullstrip_orig_vol, 'in_file_b') + node, out = strat_pool.get_data( + ["space-T1w_desc-brain_mask", "space-T1w_desc-acpcbrain_mask"] + ) + wf.connect(node, out, anat_skullstrip_orig_vol, "in_file_b") outputs = { - 'desc-preproc_T1w': (anat_skullstrip_orig_vol, 'out_file'), - 'desc-tempbrain_T1w': (anat_skullstrip_orig_vol, 'out_file') + "desc-preproc_T1w": (anat_skullstrip_orig_vol, "out_file"), + "desc-tempbrain_T1w": (anat_skullstrip_orig_vol, "out_file"), } return (wf, outputs) @@ -2075,26 +2211,28 @@ def brain_extraction_temp(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["desc-preproc_T2w", "desc-reorient_T2w", "desc-head_T2w"], ) def anatomical_init_T2(wf, cfg, strat_pool, pipe_num, opt=None): - - T2_deoblique = pe.Node(interface=afni.Refit(), - name=f'T2_deoblique_{pipe_num}') + T2_deoblique = pe.Node(interface=afni.Refit(), name=f"T2_deoblique_{pipe_num}") T2_deoblique.inputs.deoblique = True - node, out = strat_pool.get_data('T2w') - wf.connect(node, out, T2_deoblique, 'in_file') + node, out = strat_pool.get_data("T2w") + wf.connect(node, out, T2_deoblique, "in_file") - T2_reorient = pe.Node(interface=afni.Resample(), - name=f'T2_reorient_{pipe_num}', - mem_gb=0, - mem_x=(0.0115, 'in_file', 't')) - T2_reorient.inputs.orientation = 'RPI' - T2_reorient.inputs.outputtype = 'NIFTI_GZ' + T2_reorient = pe.Node( + interface=afni.Resample(), + name=f"T2_reorient_{pipe_num}", + mem_gb=0, + mem_x=(0.0115, "in_file", "t"), + ) + T2_reorient.inputs.orientation = "RPI" + T2_reorient.inputs.outputtype = "NIFTI_GZ" - wf.connect(T2_deoblique, 'out_file', T2_reorient, 'in_file') + wf.connect(T2_deoblique, "out_file", T2_reorient, "in_file") - outputs = {'desc-preproc_T2w': (T2_reorient, 'out_file'), - 'desc-reorient_T2w': (T2_reorient, 'out_file'), - 'desc-head_T2w': (T2_reorient, 'out_file')} + outputs = { + "desc-preproc_T2w": (T2_reorient, "out_file"), + "desc-reorient_T2w": (T2_reorient, "out_file"), + "desc-head_T2w": (T2_reorient, "out_file"), + } return (wf, outputs) @@ -2109,22 +2247,20 @@ def anatomical_init_T2(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["desc-preproc_T2w"], ) def acpc_align_head_T2(wf, cfg, strat_pool, pipe_num, opt=None): + acpc_align = acpc_alignment( + config=cfg, + acpc_target=cfg.anatomical_preproc["acpc_alignment"]["acpc_target"], + mask=False, + wf_name=f"acpc_align_T2_{pipe_num}", + ) - acpc_align = acpc_alignment(config=cfg, - acpc_target=cfg.anatomical_preproc[ - 'acpc_alignment']['acpc_target'], - mask=False, - wf_name=f'acpc_align_T2_{pipe_num}') - - node, out = strat_pool.get_data('desc-preproc_T2w') - wf.connect(node, out, acpc_align, 'inputspec.anat_leaf') + node, out = strat_pool.get_data("desc-preproc_T2w") + wf.connect(node, out, acpc_align, "inputspec.anat_leaf") - node, out = strat_pool.get_data('T2w-ACPC-template') - wf.connect(node, out, acpc_align, 'inputspec.template_head_for_acpc') + node, out = strat_pool.get_data("T2w-ACPC-template") + wf.connect(node, out, acpc_align, "inputspec.template_head_for_acpc") - outputs = { - 'desc-preproc_T2w': (acpc_align, 'outputspec.acpc_aligned_head') - } + outputs = {"desc-preproc_T2w": (acpc_align, "outputspec.acpc_aligned_head")} return (wf, outputs) @@ -2139,23 +2275,22 @@ def acpc_align_head_T2(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["desc-preproc_T2w", "space-T2w_desc-brain_mask"], ) def acpc_align_head_with_mask_T2(wf, cfg, strat_pool, pipe_num, opt=None): + acpc_align = acpc_alignment( + config=cfg, + acpc_target=cfg.anatomical_preproc["acpc_alignment"]["acpc_target"], + mask=True, + wf_name=f"acpc_align_T2_{pipe_num}", + ) - acpc_align = acpc_alignment(config=cfg, - acpc_target=cfg.anatomical_preproc[ - 'acpc_alignment']['acpc_target'], - mask=True, - wf_name=f'acpc_align_T2_{pipe_num}') - - node, out = strat_pool.get_data('desc-preproc_T2w') - wf.connect(node, out, acpc_align, 'inputspec.anat_leaf') + node, out = strat_pool.get_data("desc-preproc_T2w") + wf.connect(node, out, acpc_align, "inputspec.anat_leaf") - node, out = strat_pool.get_data('T2w-ACPC-template') - wf.connect(node, out, acpc_align, 'inputspec.template_head_for_acpc') + node, out = strat_pool.get_data("T2w-ACPC-template") + wf.connect(node, out, acpc_align, "inputspec.template_head_for_acpc") outputs = { - 'desc-preproc_T2w': (acpc_align, 'outputspec.acpc_aligned_head'), - 'space-T2w_desc-brain_mask': ( - acpc_align, 'outputspec.acpc_brain_mask') + "desc-preproc_T2w": (acpc_align, "outputspec.acpc_aligned_head"), + "space-T2w_desc-brain_mask": (acpc_align, "outputspec.acpc_brain_mask"), } return (wf, outputs) @@ -2178,28 +2313,28 @@ def acpc_align_head_with_mask_T2(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["desc-preproc_T2w", "desc-acpcbrain_T2w"], ) def acpc_align_brain_T2(wf, cfg, strat_pool, pipe_num, opt=None): + acpc_align = acpc_alignment( + config=cfg, + acpc_target=cfg.anatomical_preproc["acpc_alignment"]["acpc_target"], + mask=False, + wf_name=f"acpc_align_T2_{pipe_num}", + ) - acpc_align = acpc_alignment(config=cfg, - acpc_target=cfg.anatomical_preproc[ - 'acpc_alignment']['acpc_target'], - mask=False, - wf_name=f'acpc_align_T2_{pipe_num}') - - node, out = strat_pool.get_data('desc-preproc_T2w') - wf.connect(node, out, acpc_align, 'inputspec.anat_leaf') + node, out = strat_pool.get_data("desc-preproc_T2w") + wf.connect(node, out, acpc_align, "inputspec.anat_leaf") - node, out = strat_pool.get_data('desc-tempbrain_T2w') - wf.connect(node, out, acpc_align, 'inputspec.anat_brain') + node, out = strat_pool.get_data("desc-tempbrain_T2w") + wf.connect(node, out, acpc_align, "inputspec.anat_brain") - node, out = strat_pool.get_data('T2w-ACPC-template') - wf.connect(node, out, acpc_align, 'inputspec.template_head_for_acpc') + node, out = strat_pool.get_data("T2w-ACPC-template") + wf.connect(node, out, acpc_align, "inputspec.template_head_for_acpc") - node, out = strat_pool.get_data('T2w-brain-ACPC-template') - wf.connect(node, out, acpc_align, 'inputspec.template_brain_for_acpc') + node, out = strat_pool.get_data("T2w-brain-ACPC-template") + wf.connect(node, out, acpc_align, "inputspec.template_brain_for_acpc") outputs = { - 'desc-preproc_T2w': (acpc_align, 'outputspec.acpc_aligned_head'), - 'desc-acpcbrain_T2w': (acpc_align, 'outputspec.acpc_aligned_brain') + "desc-preproc_T2w": (acpc_align, "outputspec.acpc_aligned_head"), + "desc-acpcbrain_T2w": (acpc_align, "outputspec.acpc_aligned_brain"), } return (wf, outputs) @@ -2219,33 +2354,32 @@ def acpc_align_brain_T2(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["desc-preproc_T2w", "desc-acpcbrain_T2w", "space-T2w_desc-brain_mask"], ) def acpc_align_brain_with_mask_T2(wf, cfg, strat_pool, pipe_num, opt=None): + acpc_align = acpc_alignment( + config=cfg, + acpc_target=cfg.anatomical_preproc["acpc_alignment"]["acpc_target"], + mask=True, + wf_name=f"acpc_align_T2_{pipe_num}", + ) - acpc_align = acpc_alignment(config=cfg, - acpc_target=cfg.anatomical_preproc[ - 'acpc_alignment']['acpc_target'], - mask=True, - wf_name=f'acpc_align_T2_{pipe_num}') - - node, out = strat_pool.get_data('desc-preproc_T2w') - wf.connect(node, out, acpc_align, 'inputspec.anat_leaf') + node, out = strat_pool.get_data("desc-preproc_T2w") + wf.connect(node, out, acpc_align, "inputspec.anat_leaf") - node, out = strat_pool.get_data('desc-tempbrain_T2w') - wf.connect(node, out, acpc_align, 'inputspec.anat_brain') + node, out = strat_pool.get_data("desc-tempbrain_T2w") + wf.connect(node, out, acpc_align, "inputspec.anat_brain") - node, out = strat_pool.get_data('space-T2w_desc-brain_mask') - wf.connect(node, out, acpc_align, 'inputspec.brain_mask') + node, out = strat_pool.get_data("space-T2w_desc-brain_mask") + wf.connect(node, out, acpc_align, "inputspec.brain_mask") - node, out = strat_pool.get_data('T2w-ACPC-template') - wf.connect(node, out, acpc_align, 'inputspec.template_head_for_acpc') + node, out = strat_pool.get_data("T2w-ACPC-template") + wf.connect(node, out, acpc_align, "inputspec.template_head_for_acpc") - node, out = strat_pool.get_data('T2w-brain-ACPC-template') - wf.connect(node, out, acpc_align, 'inputspec.template_brain_for_acpc') + node, out = strat_pool.get_data("T2w-brain-ACPC-template") + wf.connect(node, out, acpc_align, "inputspec.template_brain_for_acpc") outputs = { - 'desc-preproc_T2w': (acpc_align, 'outputspec.acpc_aligned_head'), - 'desc-acpcbrain_T2w': (acpc_align, 'outputspec.acpc_aligned_brain'), - 'space-T2w_desc-brain_mask': ( - acpc_align, 'outputspec.acpc_brain_mask') + "desc-preproc_T2w": (acpc_align, "outputspec.acpc_aligned_head"), + "desc-acpcbrain_T2w": (acpc_align, "outputspec.acpc_aligned_brain"), + "space-T2w_desc-brain_mask": (acpc_align, "outputspec.acpc_brain_mask"), } return (wf, outputs) @@ -2261,16 +2395,12 @@ def acpc_align_brain_with_mask_T2(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["desc-preproc_T2w"], ) def non_local_means_T2(wf, cfg, strat_pool, pipe_num, opt=None): + denoise = pe.Node(interface=ants.DenoiseImage(), name=f"anat_denoise_T2_{pipe_num}") - denoise = pe.Node(interface=ants.DenoiseImage(), - name=f'anat_denoise_T2_{pipe_num}') - - node, out = strat_pool.get_data('desc-preproc_T2w') - wf.connect(node, out, denoise, 'input_image') + node, out = strat_pool.get_data("desc-preproc_T2w") + wf.connect(node, out, denoise, "input_image") - outputs = { - 'desc-preproc_T2w': (denoise, 'output_image') - } + outputs = {"desc-preproc_T2w": (denoise, "output_image")} return (wf, outputs) @@ -2285,18 +2415,17 @@ def non_local_means_T2(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["desc-preproc_T2w"], ) def n4_bias_correction_T2(wf, cfg, strat_pool, pipe_num, opt=None): + n4 = pe.Node( + interface=ants.N4BiasFieldCorrection( + dimension=3, shrink_factor=2, copy_header=True + ), + name=f"anat_n4_T2_{pipe_num}", + ) - n4 = pe.Node(interface=ants.N4BiasFieldCorrection(dimension=3, - shrink_factor=2, - copy_header=True), - name=f'anat_n4_T2_{pipe_num}') + node, out = strat_pool.get_data("desc-preproc_T2w") + wf.connect(node, out, n4, "input_image") - node, out = strat_pool.get_data('desc-preproc_T2w') - wf.connect(node, out, n4, 'input_image') - - outputs = { - 'desc-preproc_T2w': (n4, 'output_image') - } + outputs = {"desc-preproc_T2w": (n4, "output_image")} return (wf, outputs) @@ -2324,13 +2453,9 @@ def brain_mask_afni_T2(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["space-T2w_desc-acpcbrain_mask"], ) def brain_mask_acpc_afni_T2(wf, cfg, strat_pool, pipe_num, opt=None): - wf, wf_outputs = afni_brain_connector(wf, cfg, strat_pool, pipe_num, opt) - outputs = { - 'space-T2w_desc-acpcbrain_mask': - wf_outputs['space-T2w_desc-brain_mask'] - } + outputs = {"space-T2w_desc-acpcbrain_mask": wf_outputs["space-T2w_desc-brain_mask"]} return (wf, outputs) @@ -2344,7 +2469,6 @@ def brain_mask_acpc_afni_T2(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["space-T2w_desc-brain_mask"], ) def brain_mask_fsl_T2(wf, cfg, strat_pool, pipe_num, opt=None): - wf, outputs = fsl_brain_connector(wf, cfg, strat_pool, pipe_num, opt) return (wf, outputs) @@ -2359,13 +2483,9 @@ def brain_mask_fsl_T2(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["space-T2w_desc-acpcbrain_mask"], ) def brain_mask_acpc_fsl_T2(wf, cfg, strat_pool, pipe_num, opt=None): - wf, wf_outputs = fsl_brain_connector(wf, cfg, strat_pool, pipe_num, opt) - outputs = { - 'space-T2w_desc-acpcbrain_mask': - wf_outputs['space-T2w_desc-brain_mask'] - } + outputs = {"space-T2w_desc-acpcbrain_mask": wf_outputs["space-T2w_desc-brain_mask"]} return (wf, outputs) @@ -2379,9 +2499,7 @@ def brain_mask_acpc_fsl_T2(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["space-T2w_desc-brain_mask"], ) def brain_mask_niworkflows_ants_T2(wf, cfg, strat_pool, pipe_num, opt=None): - - wf, outputs = niworkflows_ants_brain_connector(wf, cfg, strat_pool, - pipe_num, opt) + wf, outputs = niworkflows_ants_brain_connector(wf, cfg, strat_pool, pipe_num, opt) return (wf, outputs) @@ -2395,14 +2513,11 @@ def brain_mask_niworkflows_ants_T2(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["space-T2w_desc-acpcbrain_mask"], ) def brain_mask_acpc_niworkflows_ants_T2(wf, cfg, strat_pool, pipe_num, opt=None): + wf, wf_outputs = niworkflows_ants_brain_connector( + wf, cfg, strat_pool, pipe_num, opt + ) - wf, wf_outputs = niworkflows_ants_brain_connector(wf, cfg, strat_pool, - pipe_num, opt) - - outputs = { - 'space-T2w_desc-acpcbrain_mask': - wf_outputs['space-T2w_desc-brain_mask'] - } + outputs = {"space-T2w_desc-acpcbrain_mask": wf_outputs["space-T2w_desc-brain_mask"]} return (wf, outputs) @@ -2416,7 +2531,6 @@ def brain_mask_acpc_niworkflows_ants_T2(wf, cfg, strat_pool, pipe_num, opt=None) outputs=["space-T2w_desc-brain_mask"], ) def brain_mask_unet_T2(wf, cfg, strat_pool, pipe_num, opt=None): - wf, outputs = unet_brain_connector(wf, cfg, strat_pool, pipe_num, opt) return (wf, outputs) @@ -2431,13 +2545,9 @@ def brain_mask_unet_T2(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["space-T2w_desc-acpcbrain_mask"], ) def brain_mask_acpc_unet_T2(wf, cfg, strat_pool, pipe_num, opt=None): - wf, wf_outputs = unet_brain_connector(wf, cfg, strat_pool, pipe_num, opt) - outputs = { - 'space-T2w_desc-acpcbrain_mask': - wf_outputs['space-T2w_desc-brain_mask'] - } + outputs = {"space-T2w_desc-acpcbrain_mask": wf_outputs["space-T2w_desc-brain_mask"]} return (wf, outputs) @@ -2457,30 +2567,36 @@ def brain_mask_acpc_unet_T2(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["space-T2w_desc-brain_mask"], ) def brain_mask_T2(wf, cfg, strat_pool, pipe_num, opt=None): + brain_mask_T2 = mask_T2(wf_name=f"brain_mask_T2_{pipe_num}") - brain_mask_T2 = mask_T2(wf_name=f'brain_mask_T2_{pipe_num}') - - if not cfg.anatomical_preproc['acpc_alignment']['run']: - node, out = strat_pool.get_data(['desc-reorient_T1w','T1w','desc-preproc_T1w']) - wf.connect(node, out, brain_mask_T2, 'inputspec.T1w') + if not cfg.anatomical_preproc["acpc_alignment"]["run"]: + node, out = strat_pool.get_data( + ["desc-reorient_T1w", "T1w", "desc-preproc_T1w"] + ) + wf.connect(node, out, brain_mask_T2, "inputspec.T1w") - node, out = strat_pool.get_data(['desc-reorient_T2w', 'T2w', 'desc-preproc_T2w']) - wf.connect(node, out, brain_mask_T2, 'inputspec.T2w') + node, out = strat_pool.get_data( + ["desc-reorient_T2w", "T2w", "desc-preproc_T2w"] + ) + wf.connect(node, out, brain_mask_T2, "inputspec.T2w") else: - node, out = strat_pool.get_data(['desc-preproc_T1w','desc-reorient_T1w','T1w']) - wf.connect(node, out, brain_mask_T2, 'inputspec.T1w') + node, out = strat_pool.get_data( + ["desc-preproc_T1w", "desc-reorient_T1w", "T1w"] + ) + wf.connect(node, out, brain_mask_T2, "inputspec.T1w") - node, out = strat_pool.get_data(['desc-preproc_T2w','desc-reorient_T2w', 'T2w']) - wf.connect(node, out, brain_mask_T2, 'inputspec.T2w') + node, out = strat_pool.get_data( + ["desc-preproc_T2w", "desc-reorient_T2w", "T2w"] + ) + wf.connect(node, out, brain_mask_T2, "inputspec.T2w") - node, out = strat_pool.get_data(["space-T1w_desc-brain_mask", - "space-T1w_desc-acpcbrain_mask"]) - wf.connect(node, out, brain_mask_T2, 'inputspec.T1w_mask') - - outputs = { - 'space-T2w_desc-brain_mask': (brain_mask_T2, 'outputspec.T2w_mask') - } + node, out = strat_pool.get_data( + ["space-T1w_desc-brain_mask", "space-T1w_desc-acpcbrain_mask"] + ) + wf.connect(node, out, brain_mask_T2, "inputspec.T1w_mask") + + outputs = {"space-T2w_desc-brain_mask": (brain_mask_T2, "outputspec.T2w_mask")} return (wf, outputs) @@ -2497,21 +2613,20 @@ def brain_mask_T2(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["space-T2w_desc-acpcbrain_mask"], ) def brain_mask_acpc_T2(wf, cfg, strat_pool, pipe_num, opt=None): + brain_mask_T2 = mask_T2(wf_name=f"brain_mask_acpc_T2_{pipe_num}") - brain_mask_T2 = mask_T2(wf_name=f'brain_mask_acpc_T2_{pipe_num}') + node, out = strat_pool.get_data("desc-reorient_T1w") + wf.connect(node, out, brain_mask_T2, "inputspec.T1w") - node, out = strat_pool.get_data('desc-reorient_T1w') - wf.connect(node, out, brain_mask_T2, 'inputspec.T1w') + node, out = strat_pool.get_data("desc-reorient_T2w") + wf.connect(node, out, brain_mask_T2, "inputspec.T2w") - node, out = strat_pool.get_data('desc-reorient_T2w') - wf.connect(node, out, brain_mask_T2, 'inputspec.T2w') + node, out = strat_pool.get_data( + ["space-T1w_desc-acpcbrain_mask", "space-T1w_desc-prebrain_mask"] + ) + wf.connect(node, out, brain_mask_T2, "inputspec.T1w_mask") - node, out = strat_pool.get_data(["space-T1w_desc-acpcbrain_mask", "space-T1w_desc-prebrain_mask"]) - wf.connect(node, out, brain_mask_T2, 'inputspec.T1w_mask') - - outputs = { - 'space-T2w_desc-acpcbrain_mask': (brain_mask_T2, 'outputspec.T2w_mask') - } + outputs = {"space-T2w_desc-acpcbrain_mask": (brain_mask_T2, "outputspec.T2w_mask")} return (wf, outputs) @@ -2530,27 +2645,26 @@ def brain_mask_acpc_T2(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["desc-brain_T2w"], ) def brain_extraction_T2(wf, cfg, strat_pool, pipe_num, opt=None): - - if cfg.anatomical_preproc['acpc_alignment']['run'] and cfg.anatomical_preproc['acpc_alignment']['acpc_target'] == 'brain': - outputs = { - 'desc-brain_T2w': (strat_pool.get_data(["desc-acpcbrain_T2w"])) - } + if ( + cfg.anatomical_preproc["acpc_alignment"]["run"] + and cfg.anatomical_preproc["acpc_alignment"]["acpc_target"] == "brain" + ): + outputs = {"desc-brain_T2w": (strat_pool.get_data(["desc-acpcbrain_T2w"]))} else: - anat_skullstrip_orig_vol = pe.Node(interface=afni.Calc(), - name=f'brain_extraction_T2_{pipe_num}') + anat_skullstrip_orig_vol = pe.Node( + interface=afni.Calc(), name=f"brain_extraction_T2_{pipe_num}" + ) - anat_skullstrip_orig_vol.inputs.expr = 'a*step(b)' - anat_skullstrip_orig_vol.inputs.outputtype = 'NIFTI_GZ' + anat_skullstrip_orig_vol.inputs.expr = "a*step(b)" + anat_skullstrip_orig_vol.inputs.outputtype = "NIFTI_GZ" - node, out = strat_pool.get_data('desc-preproc_T2w') - wf.connect(node, out, anat_skullstrip_orig_vol, 'in_file_a') + node, out = strat_pool.get_data("desc-preproc_T2w") + wf.connect(node, out, anat_skullstrip_orig_vol, "in_file_a") - node, out = strat_pool.get_data(['space-T2w_desc-brain_mask']) - wf.connect(node, out, anat_skullstrip_orig_vol, 'in_file_b') + node, out = strat_pool.get_data(["space-T2w_desc-brain_mask"]) + wf.connect(node, out, anat_skullstrip_orig_vol, "in_file_b") - outputs = { - 'desc-brain_T2w': (anat_skullstrip_orig_vol, 'out_file') - } + outputs = {"desc-brain_T2w": (anat_skullstrip_orig_vol, "out_file")} return (wf, outputs) @@ -2568,26 +2682,26 @@ def brain_extraction_T2(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["desc-tempbrain_T2w"], ) def brain_extraction_temp_T2(wf, cfg, strat_pool, pipe_num, opt=None): + anat_skullstrip_orig_vol = pe.Node( + interface=afni.Calc(), name=f"brain_extraction_temp_T2_{pipe_num}" + ) - anat_skullstrip_orig_vol = pe.Node(interface=afni.Calc(), - name=f'brain_extraction_temp_T2_{pipe_num}') - - anat_skullstrip_orig_vol.inputs.expr = 'a*step(b)' - anat_skullstrip_orig_vol.inputs.outputtype = 'NIFTI_GZ' + anat_skullstrip_orig_vol.inputs.expr = "a*step(b)" + anat_skullstrip_orig_vol.inputs.outputtype = "NIFTI_GZ" - node, out = strat_pool.get_data('desc-preproc_T2w') - wf.connect(node, out, anat_skullstrip_orig_vol, 'in_file_a') + node, out = strat_pool.get_data("desc-preproc_T2w") + wf.connect(node, out, anat_skullstrip_orig_vol, "in_file_a") - node, out = strat_pool.get_data(['space-T2w_desc-brain_mask', - 'space-T2w_desc-acpcbrain_mask']) - wf.connect(node, out, anat_skullstrip_orig_vol, 'in_file_b') + node, out = strat_pool.get_data( + ["space-T2w_desc-brain_mask", "space-T2w_desc-acpcbrain_mask"] + ) + wf.connect(node, out, anat_skullstrip_orig_vol, "in_file_b") - outputs = { - 'desc-tempbrain_T2w': (anat_skullstrip_orig_vol, 'out_file') - } + outputs = {"desc-tempbrain_T2w": (anat_skullstrip_orig_vol, "out_file")} return (wf, outputs) + @nodeblock( name="freesurfer_abcd_preproc", config=["surface_analysis", "abcd_prefreesurfer_prep"], @@ -2627,105 +2741,130 @@ def brain_extraction_temp_T2(wf, cfg, strat_pool, pipe_num, opt=None): ) def freesurfer_abcd_preproc(wf, cfg, strat_pool, pipe_num, opt=None): # fnirt-based brain extraction - brain_extraction = fnirt_based_brain_extraction(config=cfg, - wf_name=f'fnirt_based_brain_extraction_{pipe_num}') + brain_extraction = fnirt_based_brain_extraction( + config=cfg, wf_name=f"fnirt_based_brain_extraction_{pipe_num}" + ) - node, out = strat_pool.get_data('desc-preproc_T1w') - wf.connect(node, out, brain_extraction, 'inputspec.anat_data') + node, out = strat_pool.get_data("desc-preproc_T1w") + wf.connect(node, out, brain_extraction, "inputspec.anat_data") - node, out = strat_pool.get_data('template-ref-mask-res-2') - wf.connect(node, out, brain_extraction, 'inputspec.template-ref-mask-res-2') + node, out = strat_pool.get_data("template-ref-mask-res-2") + wf.connect(node, out, brain_extraction, "inputspec.template-ref-mask-res-2") - node, out = strat_pool.get_data('T1w-template') - wf.connect(node, out, brain_extraction, 'inputspec.template_skull_for_anat') + node, out = strat_pool.get_data("T1w-template") + wf.connect(node, out, brain_extraction, "inputspec.template_skull_for_anat") - node, out = strat_pool.get_data('T1w-template-res-2') - wf.connect(node, out, brain_extraction, 'inputspec.template_skull_for_anat_2mm') + node, out = strat_pool.get_data("T1w-template-res-2") + wf.connect(node, out, brain_extraction, "inputspec.template_skull_for_anat_2mm") - node, out = strat_pool.get_data('T1w-brain-template-mask') - wf.connect(node, out, brain_extraction, 'inputspec.template_brain_mask_for_anat') + node, out = strat_pool.get_data("T1w-brain-template-mask") + wf.connect(node, out, brain_extraction, "inputspec.template_brain_mask_for_anat") # fast bias field correction - fast_correction = fast_bias_field_correction(config=cfg, - wf_name=f'fast_bias_field_correction_{pipe_num}') + fast_correction = fast_bias_field_correction( + config=cfg, wf_name=f"fast_bias_field_correction_{pipe_num}" + ) - node, out = strat_pool.get_data('desc-preproc_T1w') - wf.connect(node, out, fast_correction, 'inputspec.anat_data') + node, out = strat_pool.get_data("desc-preproc_T1w") + wf.connect(node, out, fast_correction, "inputspec.anat_data") - wf.connect(brain_extraction, 'outputspec.anat_brain', fast_correction, 'inputspec.anat_brain') + wf.connect( + brain_extraction, + "outputspec.anat_brain", + fast_correction, + "inputspec.anat_brain", + ) - wf.connect(brain_extraction, 'outputspec.anat_brain_mask', fast_correction, 'inputspec.anat_brain_mask') + wf.connect( + brain_extraction, + "outputspec.anat_brain_mask", + fast_correction, + "inputspec.anat_brain_mask", + ) ### ABCD Harmonization ### # Ref: https://github.com/DCAN-Labs/DCAN-HCP/blob/master/FreeSurfer/FreeSurferPipeline.sh#L140-L144 # flirt -interp spline -in "$T1wImage" -ref "$T1wImage" -applyisoxfm 1 -out "$T1wImageFile"_1mm.nii.gz - resample_head_1mm = pe.Node(interface=fsl.FLIRT(), - name=f'resample_anat_head_1mm_{pipe_num}') - resample_head_1mm.inputs.interp = 'spline' + resample_head_1mm = pe.Node( + interface=fsl.FLIRT(), name=f"resample_anat_head_1mm_{pipe_num}" + ) + resample_head_1mm.inputs.interp = "spline" resample_head_1mm.inputs.apply_isoxfm = 1 - node, out = strat_pool.get_data('desc-preproc_T1w') - wf.connect(node, out, resample_head_1mm, 'in_file') + node, out = strat_pool.get_data("desc-preproc_T1w") + wf.connect(node, out, resample_head_1mm, "in_file") - wf.connect(node, out, resample_head_1mm, 'reference') + wf.connect(node, out, resample_head_1mm, "reference") # applywarp --rel --interp=spline -i "$T1wImage" -r "$T1wImageFile"_1mm.nii.gz --premat=$FSLDIR/etc/flirtsch/ident.mat -o "$T1wImageFile"_1mm.nii.gz - applywarp_head_to_head_1mm = pe.Node(interface=fsl.ApplyWarp(), - name=f'applywarp_head_to_head_1mm_{pipe_num}') + applywarp_head_to_head_1mm = pe.Node( + interface=fsl.ApplyWarp(), name=f"applywarp_head_to_head_1mm_{pipe_num}" + ) applywarp_head_to_head_1mm.inputs.relwarp = True - applywarp_head_to_head_1mm.inputs.interp = 'spline' - applywarp_head_to_head_1mm.inputs.premat = cfg.registration_workflows['anatomical_registration']['registration']['FSL-FNIRT']['identity_matrix'] + applywarp_head_to_head_1mm.inputs.interp = "spline" + applywarp_head_to_head_1mm.inputs.premat = cfg.registration_workflows[ + "anatomical_registration" + ]["registration"]["FSL-FNIRT"]["identity_matrix"] - wf.connect(node, out, applywarp_head_to_head_1mm, 'in_file') + wf.connect(node, out, applywarp_head_to_head_1mm, "in_file") - wf.connect(resample_head_1mm, 'out_file', - applywarp_head_to_head_1mm, 'ref_file') + wf.connect(resample_head_1mm, "out_file", applywarp_head_to_head_1mm, "ref_file") # applywarp --rel --interp=nn -i "$T1wImageBrain" -r "$T1wImageFile"_1mm.nii.gz --premat=$FSLDIR/etc/flirtsch/ident.mat -o "$T1wImageBrainFile"_1mm.nii.gz - applywarp_brain_to_head_1mm = pe.Node(interface=fsl.ApplyWarp(), - name=f'applywarp_brain_to_head_1mm_{pipe_num}') + applywarp_brain_to_head_1mm = pe.Node( + interface=fsl.ApplyWarp(), name=f"applywarp_brain_to_head_1mm_{pipe_num}" + ) applywarp_brain_to_head_1mm.inputs.relwarp = True - applywarp_brain_to_head_1mm.inputs.interp = 'nn' - applywarp_brain_to_head_1mm.inputs.premat = cfg.registration_workflows['anatomical_registration']['registration']['FSL-FNIRT']['identity_matrix'] - - wf.connect(fast_correction, 'outputspec.anat_brain_restore', - applywarp_brain_to_head_1mm, 'in_file') + applywarp_brain_to_head_1mm.inputs.interp = "nn" + applywarp_brain_to_head_1mm.inputs.premat = cfg.registration_workflows[ + "anatomical_registration" + ]["registration"]["FSL-FNIRT"]["identity_matrix"] + + wf.connect( + fast_correction, + "outputspec.anat_brain_restore", + applywarp_brain_to_head_1mm, + "in_file", + ) - wf.connect(resample_head_1mm, 'out_file', - applywarp_brain_to_head_1mm, 'ref_file') + wf.connect(resample_head_1mm, "out_file", applywarp_brain_to_head_1mm, "ref_file") # fslstats $T1wImageBrain -M - average_brain = pe.Node(interface=fsl.ImageStats(), - name=f'average_brain_{pipe_num}') - average_brain.inputs.op_string = '-M' - average_brain.inputs.output_type = 'NIFTI_GZ' + average_brain = pe.Node( + interface=fsl.ImageStats(), name=f"average_brain_{pipe_num}" + ) + average_brain.inputs.op_string = "-M" + average_brain.inputs.output_type = "NIFTI_GZ" - wf.connect(fast_correction, 'outputspec.anat_brain_restore', - average_brain, 'in_file') + wf.connect( + fast_correction, "outputspec.anat_brain_restore", average_brain, "in_file" + ) # fslmaths "$T1wImageFile"_1mm.nii.gz -div $Mean -mul 150 -abs "$T1wImageFile"_1mm.nii.gz - normalize_head = pe.Node(util.Function(input_names=['in_file', 'number', 'out_file_suffix'], - output_names=['out_file'], - function=fslmaths_command), - name=f'normalize_head_{pipe_num}') - normalize_head.inputs.out_file_suffix = '_norm' + normalize_head = pe.Node( + util.Function( + input_names=["in_file", "number", "out_file_suffix"], + output_names=["out_file"], + function=fslmaths_command, + ), + name=f"normalize_head_{pipe_num}", + ) + normalize_head.inputs.out_file_suffix = "_norm" - wf.connect(applywarp_head_to_head_1mm, 'out_file', - normalize_head, 'in_file') + wf.connect(applywarp_head_to_head_1mm, "out_file", normalize_head, "in_file") - wf.connect(average_brain, 'out_stat', - normalize_head, 'number') + wf.connect(average_brain, "out_stat", normalize_head, "number") outputs = { - 'desc-restore_T1w': (fast_correction, 'outputspec.anat_restore'), - 'desc-restore-brain_T1w': (fast_correction, - 'outputspec.anat_brain_restore'), - 'pipeline-fs_desc-fast_biasfield': (fast_correction, 'outputspec.bias_field'), - 'desc-ABCDpreproc_T1w': (normalize_head, 'out_file') - } + "desc-restore_T1w": (fast_correction, "outputspec.anat_restore"), + "desc-restore-brain_T1w": (fast_correction, "outputspec.anat_brain_restore"), + "pipeline-fs_desc-fast_biasfield": (fast_correction, "outputspec.bias_field"), + "desc-ABCDpreproc_T1w": (normalize_head, "out_file"), + } return (wf, outputs) + @nodeblock( name="freesurfer_reconall", config=["surface_analysis", "freesurfer"], @@ -2738,49 +2877,47 @@ def freesurfer_abcd_preproc(wf, cfg, strat_pool, pipe_num, opt=None): "pipeline-fs_brainmask", "pipeline-fs_wmparc", "pipeline-fs_T1", - *freesurfer_abcd_preproc.outputs + *freesurfer_abcd_preproc.outputs, # we're grabbing the postproc outputs and appending them to # the reconall outputs ], ) def freesurfer_reconall(wf, cfg, strat_pool, pipe_num, opt=None): - - reconall = pe.Node(interface=freesurfer.ReconAll(), - name=f'anat_freesurfer_{pipe_num}', - mem_gb=2.7) + reconall = pe.Node( + interface=freesurfer.ReconAll(), name=f"anat_freesurfer_{pipe_num}", mem_gb=2.7 + ) reconall.skip_timeout = True # this Node could take > 24 hours freesurfer_subject_dir = os.path.join( - cfg.pipeline_setup['working_directory']['path'], - 'cpac_'+cfg['subject_id'], - f'anat_preproc_freesurfer_{pipe_num}', - 'anat_freesurfer') + cfg.pipeline_setup["working_directory"]["path"], + "cpac_" + cfg["subject_id"], + f"anat_preproc_freesurfer_{pipe_num}", + "anat_freesurfer", + ) if not os.path.exists(freesurfer_subject_dir): os.makedirs(freesurfer_subject_dir) - reconall.inputs.directive = 'all' + reconall.inputs.directive = "all" reconall.inputs.subjects_dir = freesurfer_subject_dir - reconall.inputs.openmp = cfg.pipeline_setup['system_config'][ - 'num_OMP_threads'] + reconall.inputs.openmp = cfg.pipeline_setup["system_config"]["num_OMP_threads"] - if cfg.surface_analysis['freesurfer']['reconall_args'] is not None: - reconall.inputs.args = cfg.surface_analysis['freesurfer'][ - 'reconall_args'] + if cfg.surface_analysis["freesurfer"]["reconall_args"] is not None: + reconall.inputs.args = cfg.surface_analysis["freesurfer"]["reconall_args"] - node, out = strat_pool.get_data(["desc-ABCDpreproc_T1w","desc-preproc_T1w"]) - wf.connect(node, out, reconall, 'T1_files') + node, out = strat_pool.get_data(["desc-ABCDpreproc_T1w", "desc-preproc_T1w"]) + wf.connect(node, out, reconall, "T1_files") wf, hemisphere_outputs = freesurfer_hemispheres(wf, reconall, pipe_num) outputs = { - 'freesurfer-subject-dir': (reconall, 'subjects_dir'), + "freesurfer-subject-dir": (reconall, "subjects_dir"), **hemisphere_outputs, - 'pipeline-fs_raw-average': (reconall, 'rawavg'), - 'pipeline-fs_subcortical-seg': (reconall, 'aseg'), - 'pipeline-fs_brainmask': (reconall, 'brainmask'), - 'pipeline-fs_wmparc': (reconall, 'wmparc'), - 'pipeline-fs_T1': (reconall, 'T1') + "pipeline-fs_raw-average": (reconall, "rawavg"), + "pipeline-fs_subcortical-seg": (reconall, "aseg"), + "pipeline-fs_brainmask": (reconall, "brainmask"), + "pipeline-fs_wmparc": (reconall, "wmparc"), + "pipeline-fs_T1": (reconall, "T1"), } # for label, connection in outputs.items(): @@ -2795,39 +2932,41 @@ def freesurfer_reconall(wf, cfg, strat_pool, pipe_num, opt=None): return wf, outputs -def fnirt_based_brain_extraction(config=None, - wf_name='fnirt_based_brain_extraction'): - +def fnirt_based_brain_extraction(config=None, wf_name="fnirt_based_brain_extraction"): ### ABCD Harmonization - FNIRT-based brain extraction ### # Ref: https://github.com/DCAN-Labs/DCAN-HCP/blob/master/PreFreeSurfer/scripts/BrainExtraction_FNIRTbased.sh preproc = pe.Workflow(name=wf_name) - inputnode = pe.Node(util.IdentityInterface(fields=['anat_data', - 'template-ref-mask-res-2', - 'template_skull_for_anat', - 'template_skull_for_anat_2mm', - 'template_brain_mask_for_anat']), - name='inputspec') - - outputnode = pe.Node(util.IdentityInterface(fields=['anat_brain', - 'anat_brain_mask']), - name='outputspec') + inputnode = pe.Node( + util.IdentityInterface( + fields=[ + "anat_data", + "template-ref-mask-res-2", + "template_skull_for_anat", + "template_skull_for_anat_2mm", + "template_brain_mask_for_anat", + ] + ), + name="inputspec", + ) + + outputnode = pe.Node( + util.IdentityInterface(fields=["anat_brain", "anat_brain_mask"]), + name="outputspec", + ) # Register to 2mm reference image (linear then non-linear) # linear registration to 2mm reference # flirt -interp spline -dof 12 -in "$Input" -ref "$Reference2mm" -omat "$WD"/roughlin.mat -out "$WD"/"$BaseName"_to_MNI_roughlin.nii.gz -nosearch - linear_reg = pe.Node(interface=fsl.FLIRT(), - name='linear_reg') + linear_reg = pe.Node(interface=fsl.FLIRT(), name="linear_reg") linear_reg.inputs.dof = 12 - linear_reg.inputs.interp = 'spline' + linear_reg.inputs.interp = "spline" linear_reg.inputs.no_search = True - preproc.connect(inputnode, 'anat_data', - linear_reg, 'in_file') + preproc.connect(inputnode, "anat_data", linear_reg, "in_file") - preproc.connect(inputnode, 'template_skull_for_anat_2mm', - linear_reg, 'reference') + preproc.connect(inputnode, "template_skull_for_anat_2mm", linear_reg, "reference") # non-linear registration to 2mm reference # fnirt --in="$Input" --ref="$Reference2mm" --aff="$WD"/roughlin.mat --refmask="$Reference2mmMask" \ @@ -2835,172 +2974,152 @@ def fnirt_based_brain_extraction(config=None, # --refout="$WD"/IntensityModulatedT1.nii.gz --iout="$WD"/"$BaseName"_to_MNI_nonlin.nii.gz \ # --logout="$WD"/NonlinearReg.txt --intout="$WD"/NonlinearIntensities.nii.gz \ # --cout="$WD"/NonlinearReg.nii.gz --config="$FNIRTConfig" - non_linear_reg = pe.Node(interface=fsl.FNIRT(), - name='non_linear_reg') + non_linear_reg = pe.Node(interface=fsl.FNIRT(), name="non_linear_reg") - non_linear_reg.inputs.field_file = True # --fout - non_linear_reg.inputs.jacobian_file = True # --jout - non_linear_reg.inputs.modulatedref_file = True # --refout + non_linear_reg.inputs.field_file = True # --fout + non_linear_reg.inputs.jacobian_file = True # --jout + non_linear_reg.inputs.modulatedref_file = True # --refout # non_linear_reg.inputs.warped_file = 'T1w_acpc_to_MNI_nonlin.nii.gz' # --iout # non_linear_reg.inputs.log_file = 'NonlinearReg.txt' # --logout - non_linear_reg.inputs.out_intensitymap_file = True # --intout - non_linear_reg.inputs.fieldcoeff_file = True # --cout + non_linear_reg.inputs.out_intensitymap_file = True # --intout + non_linear_reg.inputs.fieldcoeff_file = True # --cout non_linear_reg.inputs.config_file = config.registration_workflows[ - 'anatomical_registration']['registration']['FSL-FNIRT']['fnirt_config'] + "anatomical_registration" + ]["registration"]["FSL-FNIRT"]["fnirt_config"] - preproc.connect(inputnode, 'anat_data', - non_linear_reg, 'in_file') + preproc.connect(inputnode, "anat_data", non_linear_reg, "in_file") - preproc.connect(inputnode, 'template_skull_for_anat_2mm', - non_linear_reg, 'ref_file') + preproc.connect( + inputnode, "template_skull_for_anat_2mm", non_linear_reg, "ref_file" + ) - preproc.connect(linear_reg, 'out_matrix_file', - non_linear_reg, 'affine_file') + preproc.connect(linear_reg, "out_matrix_file", non_linear_reg, "affine_file") - preproc.connect(inputnode, 'template-ref-mask-res-2', - non_linear_reg, 'refmask_file') + preproc.connect( + inputnode, "template-ref-mask-res-2", non_linear_reg, "refmask_file" + ) # Overwrite the image output from FNIRT with a spline interpolated highres version # creating spline interpolated hires version # applywarp --rel --interp=spline --in="$Input" --ref="$Reference" -w "$WD"/str2standard.nii.gz --out="$WD"/"$BaseName"_to_MNI_nonlin.nii.gz - apply_warp = pe.Node(interface=fsl.ApplyWarp(), - name='apply_warp') + apply_warp = pe.Node(interface=fsl.ApplyWarp(), name="apply_warp") - apply_warp.inputs.interp = 'spline' + apply_warp.inputs.interp = "spline" apply_warp.inputs.relwarp = True - preproc.connect(inputnode, 'anat_data', - apply_warp, 'in_file') + preproc.connect(inputnode, "anat_data", apply_warp, "in_file") - preproc.connect(inputnode, 'template_skull_for_anat', - apply_warp, 'ref_file') + preproc.connect(inputnode, "template_skull_for_anat", apply_warp, "ref_file") - preproc.connect(non_linear_reg, 'field_file', - apply_warp, 'field_file') + preproc.connect(non_linear_reg, "field_file", apply_warp, "field_file") # Invert warp and transform dilated brain mask back into native space, and use it to mask input image # Input and reference spaces are the same, using 2mm reference to save time # invwarp --ref="$Reference2mm" -w "$WD"/str2standard.nii.gz -o "$WD"/standard2str.nii.gz - inverse_warp = pe.Node(interface=fsl.InvWarp(), name='inverse_warp') - inverse_warp.inputs.output_type = 'NIFTI_GZ' + inverse_warp = pe.Node(interface=fsl.InvWarp(), name="inverse_warp") + inverse_warp.inputs.output_type = "NIFTI_GZ" - preproc.connect(inputnode, 'template_skull_for_anat_2mm', - inverse_warp, 'reference') + preproc.connect(inputnode, "template_skull_for_anat_2mm", inverse_warp, "reference") - preproc.connect(non_linear_reg, 'field_file', - inverse_warp, 'warp') + preproc.connect(non_linear_reg, "field_file", inverse_warp, "warp") # Apply inverse warp # applywarp --rel --interp=nn --in="$ReferenceMask" --ref="$Input" -w "$WD"/standard2str.nii.gz -o "$OutputBrainMask" - apply_inv_warp = pe.Node(interface=fsl.ApplyWarp(), - name='apply_inv_warp') - apply_inv_warp.inputs.interp = 'nn' + apply_inv_warp = pe.Node(interface=fsl.ApplyWarp(), name="apply_inv_warp") + apply_inv_warp.inputs.interp = "nn" apply_inv_warp.inputs.relwarp = True - preproc.connect(inputnode, 'template_brain_mask_for_anat', - apply_inv_warp, 'in_file') + preproc.connect( + inputnode, "template_brain_mask_for_anat", apply_inv_warp, "in_file" + ) - preproc.connect(inputnode, 'anat_data', - apply_inv_warp, 'ref_file') + preproc.connect(inputnode, "anat_data", apply_inv_warp, "ref_file") - preproc.connect(inverse_warp, 'inverse_warp', - apply_inv_warp, 'field_file') + preproc.connect(inverse_warp, "inverse_warp", apply_inv_warp, "field_file") + + preproc.connect(apply_inv_warp, "out_file", outputnode, "anat_brain_mask") - preproc.connect(apply_inv_warp, 'out_file', - outputnode, 'anat_brain_mask') - # Apply mask to create brain # fslmaths "$Input" -mas "$OutputBrainMask" "$OutputBrainExtractedImage" - apply_mask = pe.Node(interface=fsl.MultiImageMaths(), - name='apply_mask') - apply_mask.inputs.op_string = '-mas %s' + apply_mask = pe.Node(interface=fsl.MultiImageMaths(), name="apply_mask") + apply_mask.inputs.op_string = "-mas %s" - preproc.connect(inputnode, 'anat_data', - apply_mask, 'in_file') + preproc.connect(inputnode, "anat_data", apply_mask, "in_file") - preproc.connect(apply_inv_warp, 'out_file', - apply_mask, 'operand_files') + preproc.connect(apply_inv_warp, "out_file", apply_mask, "operand_files") - preproc.connect(apply_mask, 'out_file', - outputnode, 'anat_brain') - - return preproc + preproc.connect(apply_mask, "out_file", outputnode, "anat_brain") + return preproc -def fast_bias_field_correction(config=None, wf_name='fast_bias_field_correction'): +def fast_bias_field_correction(config=None, wf_name="fast_bias_field_correction"): ### ABCD Harmonization - FAST bias field correction ### # Ref: https://github.com/DCAN-Labs/DCAN-HCP/blob/master/PreFreeSurfer/PreFreeSurferPipeline.sh#L688-L694 preproc = pe.Workflow(name=wf_name) - inputnode = pe.Node(util.IdentityInterface(fields=['anat_data', - 'anat_brain', - 'anat_brain_mask']), - name='inputspec') + inputnode = pe.Node( + util.IdentityInterface(fields=["anat_data", "anat_brain", "anat_brain_mask"]), + name="inputspec", + ) - outputnode = pe.Node(util.IdentityInterface(fields=['anat_restore', - 'anat_brain_restore', - 'bias_field']), - name='outputspec') + outputnode = pe.Node( + util.IdentityInterface( + fields=["anat_restore", "anat_brain_restore", "bias_field"] + ), + name="outputspec", + ) # fast -b -B -o ${T1wFolder}/T1w_fast -t 1 ${T1wFolder}/T1w_acpc_dc_brain.nii.gz - fast_bias_field_correction = pe.Node(interface=fsl.FAST(), - name='fast_bias_field_correction') + fast_bias_field_correction = pe.Node( + interface=fsl.FAST(), name="fast_bias_field_correction" + ) fast_bias_field_correction.inputs.img_type = 1 fast_bias_field_correction.inputs.output_biasfield = True fast_bias_field_correction.inputs.output_biascorrected = True - preproc.connect(inputnode, 'anat_brain', - fast_bias_field_correction, 'in_files') + preproc.connect(inputnode, "anat_brain", fast_bias_field_correction, "in_files") - preproc.connect(fast_bias_field_correction, 'restored_image', - outputnode, 'anat_brain_restore') + preproc.connect( + fast_bias_field_correction, "restored_image", outputnode, "anat_brain_restore" + ) - preproc.connect(fast_bias_field_correction, 'bias_field', - outputnode, 'bias_field') + preproc.connect(fast_bias_field_correction, "bias_field", outputnode, "bias_field") - # FAST does not output a non-brain extracted image so create an inverse mask, - # apply it to T1w_acpc_dc.nii.gz, insert the T1w_fast_restore to the skull of + # FAST does not output a non-brain extracted image so create an inverse mask, + # apply it to T1w_acpc_dc.nii.gz, insert the T1w_fast_restore to the skull of # the T1w_acpc_dc.nii.gz and use that for the T1w_acpc_dc_restore head # fslmaths ${T1wFolder}/T1w_acpc_brain_mask.nii.gz -mul -1 -add 1 ${T1wFolder}/T1w_acpc_inverse_brain_mask.nii.gz - inverse_brain_mask = pe.Node(interface=fsl.ImageMaths(), - name='inverse_brain_mask') - inverse_brain_mask.inputs.op_string = '-mul -1 -add 1' + inverse_brain_mask = pe.Node(interface=fsl.ImageMaths(), name="inverse_brain_mask") + inverse_brain_mask.inputs.op_string = "-mul -1 -add 1" - preproc.connect(inputnode, 'anat_brain_mask', - inverse_brain_mask, 'in_file') + preproc.connect(inputnode, "anat_brain_mask", inverse_brain_mask, "in_file") # fslmaths ${T1wFolder}/T1w_acpc_dc.nii.gz -mul ${T1wFolder}/T1w_acpc_inverse_brain_mask.nii.gz ${T1wFolder}/T1w_acpc_dc_skull.nii.gz - apply_mask = pe.Node(interface=fsl.MultiImageMaths(), - name='apply_mask') - apply_mask.inputs.op_string = '-mul %s' + apply_mask = pe.Node(interface=fsl.MultiImageMaths(), name="apply_mask") + apply_mask.inputs.op_string = "-mul %s" - preproc.connect(inputnode, 'anat_data', - apply_mask, 'in_file') + preproc.connect(inputnode, "anat_data", apply_mask, "in_file") - preproc.connect(inverse_brain_mask, 'out_file', - apply_mask, 'operand_files') + preproc.connect(inverse_brain_mask, "out_file", apply_mask, "operand_files") # fslmaths ${T1wFolder}/T1w_fast_restore.nii.gz -add ${T1wFolder}/T1w_acpc_dc_skull.nii.gz ${T1wFolder}/${T1wImage}_acpc_dc_restore - anat_restore = pe.Node(interface=fsl.MultiImageMaths(), - name='get_anat_restore') - anat_restore.inputs.op_string = '-add %s' + anat_restore = pe.Node(interface=fsl.MultiImageMaths(), name="get_anat_restore") + anat_restore.inputs.op_string = "-add %s" - preproc.connect(fast_bias_field_correction, 'restored_image', - anat_restore, 'in_file') + preproc.connect( + fast_bias_field_correction, "restored_image", anat_restore, "in_file" + ) - preproc.connect(apply_mask, 'out_file', - anat_restore, 'operand_files') + preproc.connect(apply_mask, "out_file", anat_restore, "operand_files") - preproc.connect(anat_restore, 'out_file', - outputnode, 'anat_restore') + preproc.connect(anat_restore, "out_file", outputnode, "anat_restore") return preproc - @nodeblock( name="correct_restore_brain_intensity_abcd", config=["anatomical_preproc", "brain_extraction"], @@ -3019,127 +3138,120 @@ def fast_bias_field_correction(config=None, wf_name='fast_bias_field_correction' ], outputs=["desc-restore-brain_T1w"], ) -def correct_restore_brain_intensity_abcd(wf, cfg, strat_pool, pipe_num, - opt=None): - +def correct_restore_brain_intensity_abcd(wf, cfg, strat_pool, pipe_num, opt=None): ### ABCD Harmonization - Myelin Map ### # Ref: https://github.com/DCAN-Labs/DCAN-HCP/blob/master/PreFreeSurfer/PreFreeSurferPipeline.sh#L655-L656 # fslmerge -t ${T1wFolder}/xfms/${T1wImage}_dc ${T1wFolder}/${T1wImage}_acpc ${T1wFolder}/${T1wImage}_acpc ${T1wFolder}/${T1wImage}_acpc - merge_t1_acpc_to_list = pe.Node(util.Merge(3), - name=f'merge_t1_acpc_to_list_{pipe_num}') + merge_t1_acpc_to_list = pe.Node( + util.Merge(3), name=f"merge_t1_acpc_to_list_{pipe_num}" + ) - node, out = strat_pool.get_data('desc-preproc_T1w') - wf.connect(node, out, merge_t1_acpc_to_list, 'in1') - wf.connect(node, out, merge_t1_acpc_to_list, 'in2') - wf.connect(node, out, merge_t1_acpc_to_list, 'in3') + node, out = strat_pool.get_data("desc-preproc_T1w") + wf.connect(node, out, merge_t1_acpc_to_list, "in1") + wf.connect(node, out, merge_t1_acpc_to_list, "in2") + wf.connect(node, out, merge_t1_acpc_to_list, "in3") - merge_t1_acpc = pe.Node(interface=fslMerge(), - name=f'merge_t1_acpc_{pipe_num}') + merge_t1_acpc = pe.Node(interface=fslMerge(), name=f"merge_t1_acpc_{pipe_num}") - merge_t1_acpc.inputs.dimension = 't' + merge_t1_acpc.inputs.dimension = "t" - wf.connect(merge_t1_acpc_to_list, 'out', - merge_t1_acpc, 'in_files') + wf.connect(merge_t1_acpc_to_list, "out", merge_t1_acpc, "in_files") # fslmaths ${T1wFolder}/xfms/${T1wImage}_dc -mul 0 ${T1wFolder}/xfms/${T1wImage}_dc - multiply_t1_acpc_by_zero = pe.Node(interface=fsl.ImageMaths(), - name=f'multiply_t1_acpc_by_zero_{pipe_num}') - - multiply_t1_acpc_by_zero.inputs.op_string = '-mul 0' + multiply_t1_acpc_by_zero = pe.Node( + interface=fsl.ImageMaths(), name=f"multiply_t1_acpc_by_zero_{pipe_num}" + ) + + multiply_t1_acpc_by_zero.inputs.op_string = "-mul 0" - wf.connect(merge_t1_acpc, 'merged_file', - multiply_t1_acpc_by_zero, 'in_file') + wf.connect(merge_t1_acpc, "merged_file", multiply_t1_acpc_by_zero, "in_file") # Ref: https://github.com/DCAN-Labs/DCAN-HCP/blob/master/PostFreeSurfer/PostFreeSurferPipeline.sh#L157 # convertwarp --relout --rel --ref="$T1wFolder"/"$T1wImageBrainMask" --premat="$T1wFolder"/xfms/"$InitialT1wTransform" \ # --warp1="$T1wFolder"/xfms/"$dcT1wTransform" --out="$T1wFolder"/xfms/"$OutputOrigT1wToT1w" - convertwarp_orig_t1_to_t1 = pe.Node(interface=fsl.ConvertWarp(), - name=f'convertwarp_orig_t1_to_t1_{pipe_num}') + convertwarp_orig_t1_to_t1 = pe.Node( + interface=fsl.ConvertWarp(), name=f"convertwarp_orig_t1_to_t1_{pipe_num}" + ) convertwarp_orig_t1_to_t1.inputs.out_relwarp = True convertwarp_orig_t1_to_t1.inputs.relwarp = True - + node, out = strat_pool.get_data("space-T1w_desc-brain_mask") - wf.connect(node, out, convertwarp_orig_t1_to_t1, 'reference') + wf.connect(node, out, convertwarp_orig_t1_to_t1, "reference") - node, out = strat_pool.get_data('from-T1w_to-ACPC_mode-image_desc-aff2rig_xfm') - wf.connect(node, out, convertwarp_orig_t1_to_t1, 'premat') - wf.connect(multiply_t1_acpc_by_zero, 'out_file', - convertwarp_orig_t1_to_t1, 'warp1') + node, out = strat_pool.get_data("from-T1w_to-ACPC_mode-image_desc-aff2rig_xfm") + wf.connect(node, out, convertwarp_orig_t1_to_t1, "premat") + wf.connect(multiply_t1_acpc_by_zero, "out_file", convertwarp_orig_t1_to_t1, "warp1") # Ref: https://github.com/DCAN-Labs/DCAN-HCP/blob/master/PostFreeSurfer/scripts/CreateMyelinMaps.sh#L72-L73 # applywarp --rel --interp=spline -i "$BiasField" -r "$T1wImageBrain" -w "$AtlasTransform" -o "$BiasFieldOutput" - applywarp_biasfield = pe.Node(interface=fsl.ApplyWarp(), - name=f'applywarp_biasfield_{pipe_num}') + applywarp_biasfield = pe.Node( + interface=fsl.ApplyWarp(), name=f"applywarp_biasfield_{pipe_num}" + ) applywarp_biasfield.inputs.relwarp = True - applywarp_biasfield.inputs.interp = 'spline' + applywarp_biasfield.inputs.interp = "spline" - node, out = strat_pool.get_data('pipeline-fs_desc-fast_biasfield') - wf.connect(node, out, applywarp_biasfield, 'in_file') + node, out = strat_pool.get_data("pipeline-fs_desc-fast_biasfield") + wf.connect(node, out, applywarp_biasfield, "in_file") node, out = strat_pool.get_data("space-T1w_desc-brain_mask") - wf.connect(node, out, applywarp_biasfield, 'ref_file') + wf.connect(node, out, applywarp_biasfield, "ref_file") - node, out = strat_pool.get_data('from-T1w_to-template_mode-image_xfm') - wf.connect(node, out, applywarp_biasfield, 'field_file') + node, out = strat_pool.get_data("from-T1w_to-template_mode-image_xfm") + wf.connect(node, out, applywarp_biasfield, "field_file") # fslmaths "$BiasFieldOutput" -thr 0.1 "$BiasFieldOutput" - threshold_biasfield = pe.Node(interface=fsl.ImageMaths(), - name=f'threshold_biasfield_{pipe_num}') + threshold_biasfield = pe.Node( + interface=fsl.ImageMaths(), name=f"threshold_biasfield_{pipe_num}" + ) - threshold_biasfield.inputs.op_string = '-thr 0.1' - wf.connect(applywarp_biasfield, 'out_file', - threshold_biasfield, 'in_file') + threshold_biasfield.inputs.op_string = "-thr 0.1" + wf.connect(applywarp_biasfield, "out_file", threshold_biasfield, "in_file") # Ref: https://github.com/DCAN-Labs/DCAN-HCP/blob/master/PostFreeSurfer/scripts/CreateMyelinMaps.sh#L67-L70 # applywarp --rel --interp=spline -i "$OrginalT1wImage" -r "$T1wImageBrain" -w "$OutputOrigT1wToT1w" -o "$OutputT1wImage" - applywarp_t1 = pe.Node(interface=fsl.ApplyWarp(), - name=f'applywarp_t1_{pipe_num}') - + applywarp_t1 = pe.Node(interface=fsl.ApplyWarp(), name=f"applywarp_t1_{pipe_num}") + applywarp_t1.inputs.relwarp = True - applywarp_t1.inputs.interp = 'spline' - - node, out = strat_pool.get_data('desc-n4_T1w') - wf.connect(node, out, applywarp_t1, 'in_file') - + applywarp_t1.inputs.interp = "spline" + + node, out = strat_pool.get_data("desc-n4_T1w") + wf.connect(node, out, applywarp_t1, "in_file") + node, out = strat_pool.get_data("space-T1w_desc-brain_mask") - wf.connect(node, out, applywarp_t1, 'ref_file') - - wf.connect(convertwarp_orig_t1_to_t1, 'out_file', - applywarp_t1, 'field_file') + wf.connect(node, out, applywarp_t1, "ref_file") + + wf.connect(convertwarp_orig_t1_to_t1, "out_file", applywarp_t1, "field_file") # fslmaths "$OutputT1wImage" -abs "$OutputT1wImage" -odt float - abs_t1 = pe.Node(interface=fsl.ImageMaths(), - name=f'abs_t1_{pipe_num}') + abs_t1 = pe.Node(interface=fsl.ImageMaths(), name=f"abs_t1_{pipe_num}") - abs_t1.inputs.op_string = '-abs' - wf.connect(applywarp_t1, 'out_file', abs_t1, 'in_file') + abs_t1.inputs.op_string = "-abs" + wf.connect(applywarp_t1, "out_file", abs_t1, "in_file") # fslmaths "$OutputT1wImage" -div "$BiasField" "$OutputT1wImageRestore" - div_t1_by_biasfield = pe.Node(interface=fsl.ImageMaths(), - name=f'div_t1_by_biasfield_{pipe_num}') + div_t1_by_biasfield = pe.Node( + interface=fsl.ImageMaths(), name=f"div_t1_by_biasfield_{pipe_num}" + ) - div_t1_by_biasfield.inputs.op_string = '-div' + div_t1_by_biasfield.inputs.op_string = "-div" - wf.connect(abs_t1, 'out_file', div_t1_by_biasfield, 'in_file') + wf.connect(abs_t1, "out_file", div_t1_by_biasfield, "in_file") - node, out = strat_pool.get_data('pipeline-fs_desc-fast_biasfield') - wf.connect(node, out, div_t1_by_biasfield, 'in_file2') + node, out = strat_pool.get_data("pipeline-fs_desc-fast_biasfield") + wf.connect(node, out, div_t1_by_biasfield, "in_file2") # fslmaths "$OutputT1wImageRestore" -mas "$T1wImageBrain" "$OutputT1wImageRestoreBrain" - apply_mask = pe.Node(interface=fsl.maths.ApplyMask(), - name=f'get_restored_corrected_brain_{pipe_num}') + apply_mask = pe.Node( + interface=fsl.maths.ApplyMask(), name=f"get_restored_corrected_brain_{pipe_num}" + ) - wf.connect(div_t1_by_biasfield, 'out_file', - apply_mask, 'in_file') + wf.connect(div_t1_by_biasfield, "out_file", apply_mask, "in_file") node, out = strat_pool.get_data("space-T1w_desc-brain_mask") - wf.connect(node, out, apply_mask, 'mask_file') + wf.connect(node, out, apply_mask, "mask_file") - outputs = { - 'desc-restore-brain_T1w': (apply_mask, 'out_file') - } + outputs = {"desc-restore-brain_T1w": (apply_mask, "out_file")} return (wf, outputs) - diff --git a/CPAC/anat_preproc/ants.py b/CPAC/anat_preproc/ants.py index ac242e720a..e6599ad7d1 100644 --- a/CPAC/anat_preproc/ants.py +++ b/CPAC/anat_preproc/ants.py @@ -1,7 +1,7 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: """ -Nipype translation of ANTs workflows +Nipype translation of ANTs workflows. ------------------------------------ """ # This functionality is adapted from poldracklab/niworkflows: @@ -13,67 +13,72 @@ # general purpose from collections import OrderedDict from logging import getLogger -from pkg_resources import resource_filename as pkgr_fn -from packaging.version import parse as parseversion, Version -# nipype -import CPAC.pipeline.nipype_pipeline_engine as pe +from packaging.version import Version, parse as parseversion +from pkg_resources import resource_filename as pkgr_fn from nipype.interfaces import utility as niu +from nipype.interfaces.ants import Atropos, MultiplyImages, N4BiasFieldCorrection from nipype.interfaces.fsl.maths import ApplyMask -from nipype.interfaces.ants import N4BiasFieldCorrection, Atropos, MultiplyImages -from ..utils.misc import get_template_specs -# niworkflows -from ..utils.interfaces.ants import ( - ImageMath, - ResampleImageBySpacing, - AI, - ThresholdImage, -) +# nipype +import CPAC.pipeline.nipype_pipeline_engine as pe from CPAC.utils.interfaces.fixes import ( - FixHeaderRegistration as Registration, FixHeaderApplyTransforms as ApplyTransforms, + FixHeaderRegistration as Registration, ) from CPAC.utils.interfaces.utils import CopyXForm - +from ..utils.interfaces.ants import ( # niworkflows + AI, + ImageMath, + ResampleImageBySpacing, + ThresholdImage, +) ATROPOS_MODELS = { - 'T1w': OrderedDict([ - ('nclasses', 3), - ('csf', 1), - ('gm', 2), - ('wm', 3), - ]), - 'T2w': OrderedDict([ - ('nclasses', 3), - ('csf', 3), - ('gm', 2), - ('wm', 1), - ]), - 'FLAIR': OrderedDict([ - ('nclasses', 3), - ('csf', 1), - ('gm', 3), - ('wm', 2), - ]), + "T1w": OrderedDict( + [ + ("nclasses", 3), + ("csf", 1), + ("gm", 2), + ("wm", 3), + ] + ), + "T2w": OrderedDict( + [ + ("nclasses", 3), + ("csf", 3), + ("gm", 2), + ("wm", 1), + ] + ), + "FLAIR": OrderedDict( + [ + ("nclasses", 3), + ("csf", 1), + ("gm", 3), + ("wm", 2), + ] + ), } -def init_brain_extraction_wf(tpl_target_path, - tpl_mask_path, - tpl_regmask_path, - name='brain_extraction_wf', - template_spec=None, - use_float=True, - normalization_quality='precise', - omp_nthreads=None, - mem_gb=3.0, - bids_suffix='T1w', - atropos_refine=True, - atropos_use_random_seed=True, - atropos_model=None, - use_laplacian=True, - bspline_fitting_distance=200): +def init_brain_extraction_wf( + tpl_target_path, + tpl_mask_path, + tpl_regmask_path, + name="brain_extraction_wf", + template_spec=None, + use_float=True, + normalization_quality="precise", + omp_nthreads=None, + mem_gb=3.0, + bids_suffix="T1w", + atropos_refine=True, + atropos_use_random_seed=True, + atropos_model=None, + use_laplacian=True, + bspline_fitting_distance=200, +): """ A Nipype implementation of the official ANTs' ``antsBrainExtraction.sh`` workflow (only for 3D images). @@ -158,7 +163,7 @@ def init_brain_extraction_wf(tpl_target_path, out_segm Output segmentation by ATROPOS out_tpms - Output :abbr:`TPMs (tissue probability maps)` by ATROPOS + Output :abbr:`TPMs (tissue probability maps)` by ATROPOS. """ # from templateflow.api import get as get_template wf = pe.Workflow(name) @@ -166,165 +171,231 @@ def init_brain_extraction_wf(tpl_target_path, template_spec = template_spec or {} # suffix passed via spec takes precedence - template_spec['suffix'] = template_spec.get('suffix', bids_suffix) + template_spec["suffix"] = template_spec.get("suffix", bids_suffix) # # Get probabilistic brain mask if available - inputnode = pe.Node(niu.IdentityInterface(fields=['in_files', 'in_mask']), - name='inputnode') + inputnode = pe.Node( + niu.IdentityInterface(fields=["in_files", "in_mask"]), name="inputnode" + ) # # Try to find a registration mask, set if available if tpl_regmask_path: inputnode.inputs.in_mask = str(tpl_regmask_path) - outputnode = pe.Node(niu.IdentityInterface( - fields=['out_file', 'out_mask', 'bias_corrected', 'bias_image', - 'out_segm', 'out_tpms']), - name='outputnode') + outputnode = pe.Node( + niu.IdentityInterface( + fields=[ + "out_file", + "out_mask", + "bias_corrected", + "bias_image", + "out_segm", + "out_tpms", + ] + ), + name="outputnode", + ) - copy_xform = pe.Node(CopyXForm( - fields=['out_file', 'out_mask', 'bias_corrected', 'bias_image']), - name='copy_xform', run_without_submitting=True, - mem_gb=1.3, mem_x=(3811976743057169 / 302231454903657293676544, - 'hdr_file')) + copy_xform = pe.Node( + CopyXForm(fields=["out_file", "out_mask", "bias_corrected", "bias_image"]), + name="copy_xform", + run_without_submitting=True, + mem_gb=1.3, + mem_x=(3811976743057169 / 302231454903657293676544, "hdr_file"), + ) - trunc = pe.MapNode(ImageMath(operation='TruncateImageIntensity', op2='0.01 0.999 256'), - name='truncate_images', iterfield=['op1']) + trunc = pe.MapNode( + ImageMath(operation="TruncateImageIntensity", op2="0.01 0.999 256"), + name="truncate_images", + iterfield=["op1"], + ) inu_n4 = pe.MapNode( N4BiasFieldCorrection( - dimension=3, save_bias=False, copy_header=True, - n_iterations=[50] * 4, convergence_threshold=1e-7, shrink_factor=4, - bspline_fitting_distance=bspline_fitting_distance), - n_procs=omp_nthreads, name='inu_n4', iterfield=['input_image']) + dimension=3, + save_bias=False, + copy_header=True, + n_iterations=[50] * 4, + convergence_threshold=1e-7, + shrink_factor=4, + bspline_fitting_distance=bspline_fitting_distance, + ), + n_procs=omp_nthreads, + name="inu_n4", + iterfield=["input_image"], + ) - res_tmpl = pe.Node(ResampleImageBySpacing( - out_spacing=(4, 4, 4), apply_smoothing=True), name='res_tmpl') + res_tmpl = pe.Node( + ResampleImageBySpacing(out_spacing=(4, 4, 4), apply_smoothing=True), + name="res_tmpl", + ) res_tmpl.inputs.input_image = tpl_target_path - res_target = pe.Node(ResampleImageBySpacing( - out_spacing=(4, 4, 4), apply_smoothing=True), name='res_target') + res_target = pe.Node( + ResampleImageBySpacing(out_spacing=(4, 4, 4), apply_smoothing=True), + name="res_target", + ) - lap_tmpl = pe.Node(ImageMath(operation='Laplacian', op2='1.5 1'), - name='lap_tmpl') + lap_tmpl = pe.Node(ImageMath(operation="Laplacian", op2="1.5 1"), name="lap_tmpl") lap_tmpl.inputs.op1 = tpl_target_path - lap_target = pe.Node(ImageMath(operation='Laplacian', op2='1.5 1'), - name='lap_target') - mrg_tmpl = pe.Node(niu.Merge(2), name='mrg_tmpl') + lap_target = pe.Node( + ImageMath(operation="Laplacian", op2="1.5 1"), name="lap_target" + ) + mrg_tmpl = pe.Node(niu.Merge(2), name="mrg_tmpl") mrg_tmpl.inputs.in1 = tpl_target_path - mrg_target = pe.Node(niu.Merge(2), name='mrg_target') + mrg_target = pe.Node(niu.Merge(2), name="mrg_target") # Initialize transforms with antsAI - init_aff = pe.Node(AI( - metric=('Mattes', 32, 'Regular', 0.25), - transform=('Affine', 0.1), - search_factor=(15, 0.1), - principal_axes=False, - convergence=(10, 1e-6, 10), - verbose=True), - name='init_aff', - n_procs=omp_nthreads) + init_aff = pe.Node( + AI( + metric=("Mattes", 32, "Regular", 0.25), + transform=("Affine", 0.1), + search_factor=(15, 0.1), + principal_axes=False, + convergence=(10, 1e-6, 10), + verbose=True, + ), + name="init_aff", + n_procs=omp_nthreads, + ) # Tolerate missing ANTs at construction time _ants_version = Registration().version - if _ants_version and parseversion(_ants_version) >= Version('2.3.0'): + if _ants_version and parseversion(_ants_version) >= Version("2.3.0"): init_aff.inputs.search_grid = (40, (0, 40, 40)) # Set up spatial normalization - settings_file = 'antsBrainExtraction_%s.json' if use_laplacian \ - else 'antsBrainExtractionNoLaplacian_%s.json' - norm = pe.Node(Registration(from_file=pkgr_fn( - 'CPAC.anat_preproc', 'data/'+settings_file % normalization_quality)), - name='norm', + settings_file = ( + "antsBrainExtraction_%s.json" + if use_laplacian + else "antsBrainExtractionNoLaplacian_%s.json" + ) + norm = pe.Node( + Registration( + from_file=pkgr_fn( + "CPAC.anat_preproc", "data/" + settings_file % normalization_quality + ) + ), + name="norm", n_procs=omp_nthreads, mem_gb=1.7, - mem_x=(1233286593342025 / 151115727451828646838272, 'moving_image')) + mem_x=(1233286593342025 / 151115727451828646838272, "moving_image"), + ) norm.inputs.float = use_float - fixed_mask_trait = 'fixed_image_mask' - if _ants_version and parseversion(_ants_version) >= Version('2.2.0'): - fixed_mask_trait += 's' + fixed_mask_trait = "fixed_image_mask" + if _ants_version and parseversion(_ants_version) >= Version("2.2.0"): + fixed_mask_trait += "s" map_brainmask = pe.Node( - ApplyTransforms(interpolation='Gaussian', float=True), - name='map_brainmask' + ApplyTransforms(interpolation="Gaussian", float=True), name="map_brainmask" ) map_brainmask.inputs.input_image = str(tpl_mask_path) - thr_brainmask = pe.Node(ThresholdImage( - dimension=3, th_low=0.5, th_high=1.0, inside_value=1, - outside_value=0), name='thr_brainmask') + thr_brainmask = pe.Node( + ThresholdImage( + dimension=3, th_low=0.5, th_high=1.0, inside_value=1, outside_value=0 + ), + name="thr_brainmask", + ) # Morphological dilation, radius=2 - dil_brainmask = pe.Node(ImageMath(operation='MD', op2='2'), - name='dil_brainmask') + dil_brainmask = pe.Node(ImageMath(operation="MD", op2="2"), name="dil_brainmask") # Get largest connected component - get_brainmask = pe.Node(ImageMath(operation='GetLargestComponent'), - name='get_brainmask') + get_brainmask = pe.Node( + ImageMath(operation="GetLargestComponent"), name="get_brainmask" + ) # Refine INU correction inu_n4_final = pe.MapNode( N4BiasFieldCorrection( - dimension=3, save_bias=True, copy_header=True, - n_iterations=[50] * 5, convergence_threshold=1e-7, shrink_factor=4, - bspline_fitting_distance=bspline_fitting_distance), - n_procs=omp_nthreads, name='inu_n4_final', iterfield=['input_image']) + dimension=3, + save_bias=True, + copy_header=True, + n_iterations=[50] * 5, + convergence_threshold=1e-7, + shrink_factor=4, + bspline_fitting_distance=bspline_fitting_distance, + ), + n_procs=omp_nthreads, + name="inu_n4_final", + iterfield=["input_image"], + ) # Apply mask - apply_mask = pe.MapNode(ApplyMask(), iterfield=['in_file'], name='apply_mask') - - wf.connect([ - (inputnode, trunc, [('in_files', 'op1')]), - (inputnode, copy_xform, [(('in_files', _pop), 'hdr_file')]), - (inputnode, inu_n4_final, [('in_files', 'input_image')]), - (inputnode, init_aff, [('in_mask', 'fixed_image_mask')]), - (inputnode, norm, [('in_mask', fixed_mask_trait)]), - (inputnode, map_brainmask, [(('in_files', _pop), 'reference_image')]), - (trunc, inu_n4, [('output_image', 'input_image')]), - (inu_n4, res_target, [ - (('output_image', _pop), 'input_image')]), - (res_tmpl, init_aff, [('output_image', 'fixed_image')]), - (res_target, init_aff, [('output_image', 'moving_image')]), - (init_aff, norm, [('output_transform', 'initial_moving_transform')]), - (norm, map_brainmask, [ - ('reverse_transforms', 'transforms'), - ('reverse_invert_flags', 'invert_transform_flags')]), - (map_brainmask, thr_brainmask, [('output_image', 'input_image')]), - (thr_brainmask, dil_brainmask, [('output_image', 'op1')]), - (dil_brainmask, get_brainmask, [('output_image', 'op1')]), - (inu_n4_final, apply_mask, [('output_image', 'in_file')]), - (get_brainmask, apply_mask, [('output_image', 'mask_file')]), - (get_brainmask, copy_xform, [('output_image', 'out_mask')]), - (apply_mask, copy_xform, [('out_file', 'out_file')]), - (inu_n4_final, copy_xform, [('output_image', 'bias_corrected'), - ('bias_image', 'bias_image')]), - (copy_xform, outputnode, [ - ('out_file', 'out_file'), - ('out_mask', 'out_mask'), - ('bias_corrected', 'bias_corrected'), - ('bias_image', 'bias_image')]), - ]) + apply_mask = pe.MapNode(ApplyMask(), iterfield=["in_file"], name="apply_mask") + + wf.connect( + [ + (inputnode, trunc, [("in_files", "op1")]), + (inputnode, copy_xform, [(("in_files", _pop), "hdr_file")]), + (inputnode, inu_n4_final, [("in_files", "input_image")]), + (inputnode, init_aff, [("in_mask", "fixed_image_mask")]), + (inputnode, norm, [("in_mask", fixed_mask_trait)]), + (inputnode, map_brainmask, [(("in_files", _pop), "reference_image")]), + (trunc, inu_n4, [("output_image", "input_image")]), + (inu_n4, res_target, [(("output_image", _pop), "input_image")]), + (res_tmpl, init_aff, [("output_image", "fixed_image")]), + (res_target, init_aff, [("output_image", "moving_image")]), + (init_aff, norm, [("output_transform", "initial_moving_transform")]), + ( + norm, + map_brainmask, + [ + ("reverse_transforms", "transforms"), + ("reverse_invert_flags", "invert_transform_flags"), + ], + ), + (map_brainmask, thr_brainmask, [("output_image", "input_image")]), + (thr_brainmask, dil_brainmask, [("output_image", "op1")]), + (dil_brainmask, get_brainmask, [("output_image", "op1")]), + (inu_n4_final, apply_mask, [("output_image", "in_file")]), + (get_brainmask, apply_mask, [("output_image", "mask_file")]), + (get_brainmask, copy_xform, [("output_image", "out_mask")]), + (apply_mask, copy_xform, [("out_file", "out_file")]), + ( + inu_n4_final, + copy_xform, + [("output_image", "bias_corrected"), ("bias_image", "bias_image")], + ), + ( + copy_xform, + outputnode, + [ + ("out_file", "out_file"), + ("out_mask", "out_mask"), + ("bias_corrected", "bias_corrected"), + ("bias_image", "bias_image"), + ], + ), + ] + ) if use_laplacian: - lap_tmpl = pe.Node(ImageMath(operation='Laplacian', op2='1.5 1'), - name='lap_tmpl') + lap_tmpl = pe.Node( + ImageMath(operation="Laplacian", op2="1.5 1"), name="lap_tmpl" + ) lap_tmpl.inputs.op1 = tpl_target_path - lap_target = pe.Node(ImageMath(operation='Laplacian', op2='1.5 1'), - name='lap_target') - mrg_tmpl = pe.Node(niu.Merge(2), name='mrg_tmpl') + lap_target = pe.Node( + ImageMath(operation="Laplacian", op2="1.5 1"), name="lap_target" + ) + mrg_tmpl = pe.Node(niu.Merge(2), name="mrg_tmpl") mrg_tmpl.inputs.in1 = tpl_target_path - mrg_target = pe.Node(niu.Merge(2), name='mrg_target') - wf.connect([ - (inu_n4, lap_target, [ - (('output_image', _pop), 'op1')]), - (lap_tmpl, mrg_tmpl, [('output_image', 'in2')]), - (inu_n4, mrg_target, [('output_image', 'in1')]), - (lap_target, mrg_target, [('output_image', 'in2')]), - (mrg_tmpl, norm, [('out', 'fixed_image')]), - (mrg_target, norm, [('out', 'moving_image')]), - ]) + mrg_target = pe.Node(niu.Merge(2), name="mrg_target") + wf.connect( + [ + (inu_n4, lap_target, [(("output_image", _pop), "op1")]), + (lap_tmpl, mrg_tmpl, [("output_image", "in2")]), + (inu_n4, mrg_target, [("output_image", "in1")]), + (lap_target, mrg_target, [("output_image", "in2")]), + (mrg_tmpl, norm, [("out", "fixed_image")]), + (mrg_target, norm, [("out", "moving_image")]), + ] + ) else: norm.inputs.fixed_image = tpl_target_path - wf.connect([ - (inu_n4, norm, [ - (('output_image', _pop), 'moving_image')]), - ]) + wf.connect( + [ + (inu_n4, norm, [(("output_image", _pop), "moving_image")]), + ] + ) if atropos_refine: atropos_model = atropos_model or list(ATROPOS_MODELS[bids_suffix].values()) @@ -334,42 +405,54 @@ def init_brain_extraction_wf(tpl_target_path, mem_gb=mem_gb, in_segmentation_model=atropos_model, ) - sel_wm = pe.Node(niu.Select(index=atropos_model[-1] - 1), - name='sel_wm', - run_without_submitting=True, - mem_gb=0.1, - mem_x=(5157380299430287 / 302231454903657293676544, - 'inlist')) - - wf.disconnect([ - (get_brainmask, apply_mask, [('output_image', 'mask_file')]), - (copy_xform, outputnode, [('out_mask', 'out_mask')]), - ]) - wf.connect([ - (inu_n4, atropos_wf, [ - ('output_image', 'inputnode.in_files')]), - (thr_brainmask, atropos_wf, [ - ('output_image', 'inputnode.in_mask')]), - (get_brainmask, atropos_wf, [ - ('output_image', 'inputnode.in_mask_dilated')]), - (atropos_wf, sel_wm, [('outputnode.out_tpms', 'inlist')]), - (sel_wm, inu_n4_final, [('out', 'weight_image')]), - (atropos_wf, apply_mask, [ - ('outputnode.out_mask', 'mask_file')]), - (atropos_wf, outputnode, [ - ('outputnode.out_mask', 'out_mask'), - ('outputnode.out_segm', 'out_segm'), - ('outputnode.out_tpms', 'out_tpms')]), - ]) + sel_wm = pe.Node( + niu.Select(index=atropos_model[-1] - 1), + name="sel_wm", + run_without_submitting=True, + mem_gb=0.1, + mem_x=(5157380299430287 / 302231454903657293676544, "inlist"), + ) + + wf.disconnect( + [ + (get_brainmask, apply_mask, [("output_image", "mask_file")]), + (copy_xform, outputnode, [("out_mask", "out_mask")]), + ] + ) + wf.connect( + [ + (inu_n4, atropos_wf, [("output_image", "inputnode.in_files")]), + (thr_brainmask, atropos_wf, [("output_image", "inputnode.in_mask")]), + ( + get_brainmask, + atropos_wf, + [("output_image", "inputnode.in_mask_dilated")], + ), + (atropos_wf, sel_wm, [("outputnode.out_tpms", "inlist")]), + (sel_wm, inu_n4_final, [("out", "weight_image")]), + (atropos_wf, apply_mask, [("outputnode.out_mask", "mask_file")]), + ( + atropos_wf, + outputnode, + [ + ("outputnode.out_mask", "out_mask"), + ("outputnode.out_segm", "out_segm"), + ("outputnode.out_tpms", "out_tpms"), + ], + ), + ] + ) return wf -def init_atropos_wf(name='atropos_wf', - use_random_seed=True, - omp_nthreads=None, - mem_gb=3.0, - padding=10, - in_segmentation_model=list(ATROPOS_MODELS['T1w'].values())): +def init_atropos_wf( + name="atropos_wf", + use_random_seed=True, + omp_nthreads=None, + mem_gb=3.0, + padding=10, + in_segmentation_model=list(ATROPOS_MODELS["T1w"].values()), +): """ Implements supersteps 6 and 7 of ``antsBrainExtraction.sh``, which refine the mask previously computed with the spatial @@ -394,7 +477,9 @@ def init_atropos_wf(name='atropos_wf', With this option, you can control :math:`$K$` and tell the script which classes represent CSF, gray and white matter. Format (K, csfLabel, gmLabel, wmLabel). - Examples: + + Examples + -------- - ``(3,1,2,3)`` for T1 with K=3, CSF=1, GM=2, WM=3 (default) - ``(3,3,2,1)`` for T2 with K=3, CSF=3, GM=2, WM=1 - ``(3,1,3,2)`` for FLAIR with K=3, CSF=1 GM=3, WM=2 @@ -412,177 +497,217 @@ def init_atropos_wf(name='atropos_wf', out_segm Output segmentation out_tpms - Output :abbr:`TPMs (tissue probability maps)` + Output :abbr:`TPMs (tissue probability maps)`. """ wf = pe.Workflow(name) - inputnode = pe.Node(niu.IdentityInterface( - fields=['in_files', 'in_mask', 'in_mask_dilated']), name='inputnode') - outputnode = pe.Node(niu.IdentityInterface( - fields=['out_mask', 'out_segm', 'out_tpms']), name='outputnode') + inputnode = pe.Node( + niu.IdentityInterface(fields=["in_files", "in_mask", "in_mask_dilated"]), + name="inputnode", + ) + outputnode = pe.Node( + niu.IdentityInterface(fields=["out_mask", "out_segm", "out_tpms"]), + name="outputnode", + ) - copy_xform = pe.Node(CopyXForm( - fields=['out_mask', 'out_segm', 'out_tpms']), - name='copy_xform', run_without_submitting=True, - mem_gb=1.3, mem_x=(3811976743057169 / 302231454903657293676544, - 'hdr_file')) + copy_xform = pe.Node( + CopyXForm(fields=["out_mask", "out_segm", "out_tpms"]), + name="copy_xform", + run_without_submitting=True, + mem_gb=1.3, + mem_x=(3811976743057169 / 302231454903657293676544, "hdr_file"), + ) # Run atropos (core node) - atropos = pe.Node(Atropos( - dimension=3, - initialization='KMeans', - number_of_tissue_classes=in_segmentation_model[0], - n_iterations=3, - convergence_threshold=0.0, - mrf_radius=[1, 1, 1], - mrf_smoothing_factor=0.1, - likelihood_model='Gaussian', - use_random_seed=use_random_seed), - name='01_atropos', n_procs=omp_nthreads, mem_gb=mem_gb) + atropos = pe.Node( + Atropos( + dimension=3, + initialization="KMeans", + number_of_tissue_classes=in_segmentation_model[0], + n_iterations=3, + convergence_threshold=0.0, + mrf_radius=[1, 1, 1], + mrf_smoothing_factor=0.1, + likelihood_model="Gaussian", + use_random_seed=use_random_seed, + ), + name="01_atropos", + n_procs=omp_nthreads, + mem_gb=mem_gb, + ) if not use_random_seed: - getLogger('random').info('%s # (Atropos constant)', atropos.name) + getLogger("random").info("%s # (Atropos constant)", atropos.name) # massage outputs - pad_segm = pe.Node(ImageMath(operation='PadImage', op2='%d' % padding), - name='02_pad_segm') - pad_mask = pe.Node(ImageMath(operation='PadImage', op2='%d' % padding), - name='03_pad_mask') + pad_segm = pe.Node( + ImageMath(operation="PadImage", op2="%d" % padding), name="02_pad_segm" + ) + pad_mask = pe.Node( + ImageMath(operation="PadImage", op2="%d" % padding), name="03_pad_mask" + ) # Split segmentation in binary masks - sel_labels = pe.Node(niu.Function( - function=_select_labels, output_names=['out_wm', 'out_gm', 'out_csf']), - name='04_sel_labels') + sel_labels = pe.Node( + niu.Function( + function=_select_labels, output_names=["out_wm", "out_gm", "out_csf"] + ), + name="04_sel_labels", + ) sel_labels.inputs.labels = list(reversed(in_segmentation_model[1:])) # Select largest components (GM, WM) # ImageMath ${DIMENSION} ${EXTRACTION_WM} GetLargestComponent ${EXTRACTION_WM} - get_wm = pe.Node(ImageMath(operation='GetLargestComponent'), - name='05_get_wm') - get_gm = pe.Node(ImageMath(operation='GetLargestComponent'), - name='06_get_gm') + get_wm = pe.Node(ImageMath(operation="GetLargestComponent"), name="05_get_wm") + get_gm = pe.Node(ImageMath(operation="GetLargestComponent"), name="06_get_gm") # Fill holes and calculate intersection # ImageMath ${DIMENSION} ${EXTRACTION_TMP} FillHoles ${EXTRACTION_GM} 2 # MultiplyImages ${DIMENSION} ${EXTRACTION_GM} ${EXTRACTION_TMP} ${EXTRACTION_GM} - fill_gm = pe.Node(ImageMath(operation='FillHoles', op2='2'), - name='07_fill_gm', - mem_gb=2.1, - mem_x=(1435097126797993 / 1208925819614629174706176, - 'op1')) - mult_gm = pe.Node(MultiplyImages( - dimension=3, output_product_image='08_mult_gm.nii.gz'), name='08_mult_gm') + fill_gm = pe.Node( + ImageMath(operation="FillHoles", op2="2"), + name="07_fill_gm", + mem_gb=2.1, + mem_x=(1435097126797993 / 1208925819614629174706176, "op1"), + ) + mult_gm = pe.Node( + MultiplyImages(dimension=3, output_product_image="08_mult_gm.nii.gz"), + name="08_mult_gm", + ) # MultiplyImages ${DIMENSION} ${EXTRACTION_WM} ${ATROPOS_WM_CLASS_LABEL} ${EXTRACTION_WM} # ImageMath ${DIMENSION} ${EXTRACTION_TMP} ME ${EXTRACTION_CSF} 10 - relabel_wm = pe.Node(MultiplyImages( - dimension=3, second_input=in_segmentation_model[-1], - output_product_image='09_relabel_wm.nii.gz'), name='09_relabel_wm') - me_csf = pe.Node(ImageMath(operation='ME', op2='10'), name='10_me_csf') + relabel_wm = pe.Node( + MultiplyImages( + dimension=3, + second_input=in_segmentation_model[-1], + output_product_image="09_relabel_wm.nii.gz", + ), + name="09_relabel_wm", + ) + me_csf = pe.Node(ImageMath(operation="ME", op2="10"), name="10_me_csf") # ImageMath ${DIMENSION} ${EXTRACTION_GM} addtozero ${EXTRACTION_GM} ${EXTRACTION_TMP} # MultiplyImages ${DIMENSION} ${EXTRACTION_GM} ${ATROPOS_GM_CLASS_LABEL} ${EXTRACTION_GM} # ImageMath ${DIMENSION} ${EXTRACTION_SEGMENTATION} addtozero ${EXTRACTION_WM} ${EXTRACTION_GM} - add_gm = pe.Node(ImageMath(operation='addtozero'), - name='11_add_gm') - relabel_gm = pe.Node(MultiplyImages( - dimension=3, second_input=in_segmentation_model[-2], - output_product_image='12_relabel_gm.nii.gz'), name='12_relabel_gm') - add_gm_wm = pe.Node(ImageMath(operation='addtozero'), - name='13_add_gm_wm') + add_gm = pe.Node(ImageMath(operation="addtozero"), name="11_add_gm") + relabel_gm = pe.Node( + MultiplyImages( + dimension=3, + second_input=in_segmentation_model[-2], + output_product_image="12_relabel_gm.nii.gz", + ), + name="12_relabel_gm", + ) + add_gm_wm = pe.Node(ImageMath(operation="addtozero"), name="13_add_gm_wm") # Superstep 7 # Split segmentation in binary masks - sel_labels2 = pe.Node(niu.Function( - function=_select_labels, output_names=['out_gm', 'out_wm']), - name='14_sel_labels2') + sel_labels2 = pe.Node( + niu.Function(function=_select_labels, output_names=["out_gm", "out_wm"]), + name="14_sel_labels2", + ) sel_labels2.inputs.labels = in_segmentation_model[2:] # ImageMath ${DIMENSION} ${EXTRACTION_MASK} addtozero ${EXTRACTION_MASK} ${EXTRACTION_TMP} - add_7 = pe.Node(ImageMath(operation='addtozero'), name='15_add_7') + add_7 = pe.Node(ImageMath(operation="addtozero"), name="15_add_7") # ImageMath ${DIMENSION} ${EXTRACTION_MASK} ME ${EXTRACTION_MASK} 2 - me_7 = pe.Node(ImageMath(operation='ME', op2='2'), name='16_me_7') + me_7 = pe.Node(ImageMath(operation="ME", op2="2"), name="16_me_7") # ImageMath ${DIMENSION} ${EXTRACTION_MASK} GetLargestComponent ${EXTRACTION_MASK} - comp_7 = pe.Node(ImageMath(operation='GetLargestComponent'), - name='17_comp_7') + comp_7 = pe.Node(ImageMath(operation="GetLargestComponent"), name="17_comp_7") # ImageMath ${DIMENSION} ${EXTRACTION_MASK} MD ${EXTRACTION_MASK} 4 - md_7 = pe.Node(ImageMath(operation='MD', op2='4'), name='18_md_7') + md_7 = pe.Node(ImageMath(operation="MD", op2="4"), name="18_md_7") # ImageMath ${DIMENSION} ${EXTRACTION_MASK} FillHoles ${EXTRACTION_MASK} 2 - fill_7 = pe.Node(ImageMath(operation='FillHoles', op2='2'), - name='19_fill_7', - mem_gb=2.1, - mem_x=(1435097126797993 / 1208925819614629174706176, - 'op1')) + fill_7 = pe.Node( + ImageMath(operation="FillHoles", op2="2"), + name="19_fill_7", + mem_gb=2.1, + mem_x=(1435097126797993 / 1208925819614629174706176, "op1"), + ) # ImageMath ${DIMENSION} ${EXTRACTION_MASK} addtozero ${EXTRACTION_MASK} \ # ${EXTRACTION_MASK_PRIOR_WARPED} - add_7_2 = pe.Node(ImageMath(operation='addtozero'), name='20_add_7_2') + add_7_2 = pe.Node(ImageMath(operation="addtozero"), name="20_add_7_2") # ImageMath ${DIMENSION} ${EXTRACTION_MASK} MD ${EXTRACTION_MASK} 5 - md_7_2 = pe.Node(ImageMath(operation='MD', op2='5'), name='21_md_7_2') + md_7_2 = pe.Node(ImageMath(operation="MD", op2="5"), name="21_md_7_2") # ImageMath ${DIMENSION} ${EXTRACTION_MASK} ME ${EXTRACTION_MASK} 5 - me_7_2 = pe.Node(ImageMath(operation='ME', op2='5'), name='22_me_7_2') + me_7_2 = pe.Node(ImageMath(operation="ME", op2="5"), name="22_me_7_2") # De-pad - depad_mask = pe.Node(ImageMath(operation='PadImage', op2='-%d' % padding), - name='23_depad_mask') - depad_segm = pe.Node(ImageMath(operation='PadImage', op2='-%d' % padding), - name='24_depad_segm') - depad_gm = pe.Node(ImageMath(operation='PadImage', op2='-%d' % padding), - name='25_depad_gm') - depad_wm = pe.Node(ImageMath(operation='PadImage', op2='-%d' % padding), - name='26_depad_wm') - depad_csf = pe.Node(ImageMath(operation='PadImage', op2='-%d' % padding), - name='27_depad_csf') - - msk_conform = pe.Node(niu.Function(function=_conform_mask), name='msk_conform') - merge_tpms = pe.Node(niu.Merge(in_segmentation_model[0]), name='merge_tpms') - wf.connect([ - (inputnode, copy_xform, [(('in_files', _pop), 'hdr_file')]), - (inputnode, pad_mask, [('in_mask', 'op1')]), - (inputnode, atropos, [('in_files', 'intensity_images'), - ('in_mask_dilated', 'mask_image')]), - (inputnode, msk_conform, [(('in_files', _pop), 'in_reference')]), - (atropos, pad_segm, [('classified_image', 'op1')]), - (pad_segm, sel_labels, [('output_image', 'in_segm')]), - (sel_labels, get_wm, [('out_wm', 'op1')]), - (sel_labels, get_gm, [('out_gm', 'op1')]), - (get_gm, fill_gm, [('output_image', 'op1')]), - (get_gm, mult_gm, [('output_image', 'first_input')]), - (fill_gm, mult_gm, [('output_image', 'second_input')]), - (get_wm, relabel_wm, [('output_image', 'first_input')]), - (sel_labels, me_csf, [('out_csf', 'op1')]), - (mult_gm, add_gm, [('output_product_image', 'op1')]), - (me_csf, add_gm, [('output_image', 'op2')]), - (add_gm, relabel_gm, [('output_image', 'first_input')]), - (relabel_wm, add_gm_wm, [('output_product_image', 'op1')]), - (relabel_gm, add_gm_wm, [('output_product_image', 'op2')]), - (add_gm_wm, sel_labels2, [('output_image', 'in_segm')]), - (sel_labels2, add_7, [('out_wm', 'op1'), - ('out_gm', 'op2')]), - (add_7, me_7, [('output_image', 'op1')]), - (me_7, comp_7, [('output_image', 'op1')]), - (comp_7, md_7, [('output_image', 'op1')]), - (md_7, fill_7, [('output_image', 'op1')]), - (fill_7, add_7_2, [('output_image', 'op1')]), - (pad_mask, add_7_2, [('output_image', 'op2')]), - (add_7_2, md_7_2, [('output_image', 'op1')]), - (md_7_2, me_7_2, [('output_image', 'op1')]), - (me_7_2, depad_mask, [('output_image', 'op1')]), - (add_gm_wm, depad_segm, [('output_image', 'op1')]), - (relabel_wm, depad_wm, [('output_product_image', 'op1')]), - (relabel_gm, depad_gm, [('output_product_image', 'op1')]), - (sel_labels, depad_csf, [('out_csf', 'op1')]), - (depad_csf, merge_tpms, [('output_image', 'in1')]), - (depad_gm, merge_tpms, [('output_image', 'in2')]), - (depad_wm, merge_tpms, [('output_image', 'in3')]), - (depad_mask, msk_conform, [('output_image', 'in_mask')]), - (msk_conform, copy_xform, [('out', 'out_mask')]), - (depad_segm, copy_xform, [('output_image', 'out_segm')]), - (merge_tpms, copy_xform, [('out', 'out_tpms')]), - (copy_xform, outputnode, [ - ('out_mask', 'out_mask'), - ('out_segm', 'out_segm'), - ('out_tpms', 'out_tpms')]), - ]) + depad_mask = pe.Node( + ImageMath(operation="PadImage", op2="-%d" % padding), name="23_depad_mask" + ) + depad_segm = pe.Node( + ImageMath(operation="PadImage", op2="-%d" % padding), name="24_depad_segm" + ) + depad_gm = pe.Node( + ImageMath(operation="PadImage", op2="-%d" % padding), name="25_depad_gm" + ) + depad_wm = pe.Node( + ImageMath(operation="PadImage", op2="-%d" % padding), name="26_depad_wm" + ) + depad_csf = pe.Node( + ImageMath(operation="PadImage", op2="-%d" % padding), name="27_depad_csf" + ) + + msk_conform = pe.Node(niu.Function(function=_conform_mask), name="msk_conform") + merge_tpms = pe.Node(niu.Merge(in_segmentation_model[0]), name="merge_tpms") + wf.connect( + [ + (inputnode, copy_xform, [(("in_files", _pop), "hdr_file")]), + (inputnode, pad_mask, [("in_mask", "op1")]), + ( + inputnode, + atropos, + [("in_files", "intensity_images"), ("in_mask_dilated", "mask_image")], + ), + (inputnode, msk_conform, [(("in_files", _pop), "in_reference")]), + (atropos, pad_segm, [("classified_image", "op1")]), + (pad_segm, sel_labels, [("output_image", "in_segm")]), + (sel_labels, get_wm, [("out_wm", "op1")]), + (sel_labels, get_gm, [("out_gm", "op1")]), + (get_gm, fill_gm, [("output_image", "op1")]), + (get_gm, mult_gm, [("output_image", "first_input")]), + (fill_gm, mult_gm, [("output_image", "second_input")]), + (get_wm, relabel_wm, [("output_image", "first_input")]), + (sel_labels, me_csf, [("out_csf", "op1")]), + (mult_gm, add_gm, [("output_product_image", "op1")]), + (me_csf, add_gm, [("output_image", "op2")]), + (add_gm, relabel_gm, [("output_image", "first_input")]), + (relabel_wm, add_gm_wm, [("output_product_image", "op1")]), + (relabel_gm, add_gm_wm, [("output_product_image", "op2")]), + (add_gm_wm, sel_labels2, [("output_image", "in_segm")]), + (sel_labels2, add_7, [("out_wm", "op1"), ("out_gm", "op2")]), + (add_7, me_7, [("output_image", "op1")]), + (me_7, comp_7, [("output_image", "op1")]), + (comp_7, md_7, [("output_image", "op1")]), + (md_7, fill_7, [("output_image", "op1")]), + (fill_7, add_7_2, [("output_image", "op1")]), + (pad_mask, add_7_2, [("output_image", "op2")]), + (add_7_2, md_7_2, [("output_image", "op1")]), + (md_7_2, me_7_2, [("output_image", "op1")]), + (me_7_2, depad_mask, [("output_image", "op1")]), + (add_gm_wm, depad_segm, [("output_image", "op1")]), + (relabel_wm, depad_wm, [("output_product_image", "op1")]), + (relabel_gm, depad_gm, [("output_product_image", "op1")]), + (sel_labels, depad_csf, [("out_csf", "op1")]), + (depad_csf, merge_tpms, [("output_image", "in1")]), + (depad_gm, merge_tpms, [("output_image", "in2")]), + (depad_wm, merge_tpms, [("output_image", "in3")]), + (depad_mask, msk_conform, [("output_image", "in_mask")]), + (msk_conform, copy_xform, [("out", "out_mask")]), + (depad_segm, copy_xform, [("output_image", "out_segm")]), + (merge_tpms, copy_xform, [("out", "out_tpms")]), + ( + copy_xform, + outputnode, + [ + ("out_mask", "out_mask"), + ("out_segm", "out_segm"), + ("out_tpms", "out_tpms"), + ], + ), + ] + ) return wf @@ -594,35 +719,36 @@ def _pop(in_files): def _select_labels(in_segm, labels): from os import getcwd + import numpy as np - import nibabel as nb + import nibabel as nib from nipype.utils.filemanip import fname_presuffix out_files = [] cwd = getcwd() - nii = nb.load(in_segm) + nii = nib.load(in_segm) for l in labels: data = (nii.get_fdata() == l).astype(np.uint8) newnii = nii.__class__(data, nii.affine, nii.header) - newnii.set_data_dtype('uint8') - out_file = fname_presuffix(in_segm, suffix='_class-%02d' % l, - newpath=cwd) + newnii.set_data_dtype("uint8") + out_file = fname_presuffix(in_segm, suffix="_class-%02d" % l, newpath=cwd) newnii.to_filename(out_file) out_files.append(out_file) return out_files def _conform_mask(in_mask, in_reference): - """Ensures the mask headers make sense and match those of the T1w""" + """Ensures the mask headers make sense and match those of the T1w.""" from pathlib import Path - import nibabel as nb + + import nibabel as nib from nipype.utils.filemanip import fname_presuffix - ref = nb.load(in_reference) - nii = nb.load(in_mask) + ref = nib.load(in_reference) + nii = nib.load(in_mask) hdr = nii.header.copy() - hdr.set_data_dtype('int16') + hdr.set_data_dtype("int16") hdr.set_slope_inter(1, 0) qform, qcode = ref.header.get_qform(coded=True) @@ -633,13 +759,13 @@ def _conform_mask(in_mask, in_reference): if scode is not None: hdr.set_sform(sform, int(scode)) - if '_maths' in in_mask: # Cut the name at first _maths occurrence - ext = ''.join(Path(in_mask).suffixes) + if "_maths" in in_mask: # Cut the name at first _maths occurrence + ext = "".join(Path(in_mask).suffixes) basename = Path(in_mask).name - in_mask = basename.split('_maths')[0] + ext + in_mask = basename.split("_maths")[0] + ext - out_file = fname_presuffix(in_mask, suffix='_mask', - newpath=str(Path())) - nii.__class__(nii.get_fdata().astype('int16'), ref.affine, - hdr).to_filename(out_file) + out_file = fname_presuffix(in_mask, suffix="_mask", newpath=str(Path())) + nii.__class__(nii.get_fdata().astype("int16"), ref.affine, hdr).to_filename( + out_file + ) return out_file diff --git a/CPAC/anat_preproc/data/antsBrainExtractionNoLaplacian_precise.json b/CPAC/anat_preproc/data/antsBrainExtractionNoLaplacian_precise.json index 0ad52d9c32..dfd02d7c00 100644 --- a/CPAC/anat_preproc/data/antsBrainExtractionNoLaplacian_precise.json +++ b/CPAC/anat_preproc/data/antsBrainExtractionNoLaplacian_precise.json @@ -1,61 +1,127 @@ { - "collapse_output_transforms": true, - "convergence_threshold": [1E-8, 1E-8, 1E-9], - "convergence_window_size": [10, 10, 15], - "dimension": 3, - "interpolation": "LanczosWindowedSinc", - "metric": [ - "MI", - "MI", - "CC" - ], - "metric_weight": [ - 1, - 1, - 1 - ], - "number_of_iterations": [ - [1000, 500, 250, 100], - [1000, 500, 250, 100], - [50, 10, 0] - ], - "output_transform_prefix": "anat_to_template", - "output_warped_image": true, - "radius_or_number_of_bins": [ - 32, - 32, - 4 - ], - "sampling_percentage": [ - 0.25, - 0.25, - 1 - ], - "sampling_strategy": [ - "Regular", - "Regular", - "None" - ], - "shrink_factors": [ - [8, 4, 2, 1], - [8, 4, 2, 1], - [4, 2, 1] - ], - "sigma_units": ["vox", "vox", "vox"], - "smoothing_sigmas": [ - [4, 2, 1, 0], - [4, 2, 1, 0], - [2, 1, 0] - ], - "transform_parameters": [ - [0.1], - [0.1], - [0.1, 3.0, 0.0] - ], - "transforms": ["Rigid", "Affine", "SyN"], - "use_histogram_matching": true, - "verbose": true, - "winsorize_lower_quantile": 0.025, - "winsorize_upper_quantile": 0.975, - "write_composite_transform": false -} \ No newline at end of file + "collapse_output_transforms": true, + "convergence_threshold": [ + 1e-08, + 1e-08, + 1e-09 + ], + "convergence_window_size": [ + 10, + 10, + 15 + ], + "dimension": 3, + "interpolation": "LanczosWindowedSinc", + "metric": [ + "MI", + "MI", + "CC" + ], + "metric_weight": [ + 1, + 1, + 1 + ], + "number_of_iterations": [ + [ + 1000, + 500, + 250, + 100 + ], + [ + 1000, + 500, + 250, + 100 + ], + [ + 50, + 10, + 0 + ] + ], + "output_transform_prefix": "anat_to_template", + "output_warped_image": true, + "radius_or_number_of_bins": [ + 32, + 32, + 4 + ], + "sampling_percentage": [ + 0.25, + 0.25, + 1 + ], + "sampling_strategy": [ + "Regular", + "Regular", + "None" + ], + "shrink_factors": [ + [ + 8, + 4, + 2, + 1 + ], + [ + 8, + 4, + 2, + 1 + ], + [ + 4, + 2, + 1 + ] + ], + "sigma_units": [ + "vox", + "vox", + "vox" + ], + "smoothing_sigmas": [ + [ + 4, + 2, + 1, + 0 + ], + [ + 4, + 2, + 1, + 0 + ], + [ + 2, + 1, + 0 + ] + ], + "transform_parameters": [ + [ + 0.1 + ], + [ + 0.1 + ], + [ + 0.1, + 3.0, + 0.0 + ] + ], + "transforms": [ + "Rigid", + "Affine", + "SyN" + ], + "use_histogram_matching": true, + "verbose": true, + "winsorize_lower_quantile": 0.025, + "winsorize_upper_quantile": 0.975, + "write_composite_transform": false +} diff --git a/CPAC/anat_preproc/data/antsBrainExtractionNoLaplacian_testing.json b/CPAC/anat_preproc/data/antsBrainExtractionNoLaplacian_testing.json index 07e06549e1..af350d715d 100644 --- a/CPAC/anat_preproc/data/antsBrainExtractionNoLaplacian_testing.json +++ b/CPAC/anat_preproc/data/antsBrainExtractionNoLaplacian_testing.json @@ -1,61 +1,125 @@ { - "collapse_output_transforms": true, - "convergence_threshold": [1E-8, 1E-8, 1E-9], - "convergence_window_size": [10, 10, 15], - "dimension": 3, - "interpolation": "LanczosWindowedSinc", - "metric": [ - "MI", - "MI", - "CC" - ], - "metric_weight": [ - 1, - 1, - 1 - ], - "number_of_iterations": [ - [100, 500, 250, 100], - [100, 500, 250, 100], - [5, 10, 0] - ], - "output_transform_prefix": "anat_to_template", - "output_warped_image": true, - "radius_or_number_of_bins": [ - 32, - 32, - 4 - ], - "sampling_percentage": [ - 0.25, - 0.25, - 1 - ], - "sampling_strategy": [ - "Regular", - "Regular", - "None" - ], - "shrink_factors": [ - [8, 4, 2, 1], - [8, 4, 2, 1], - [2, 1] - ], - "sigma_units": ["vox", "vox", "vox"], - "smoothing_sigmas": [ - [4, 2, 1, 0], - [4, 2, 1, 0], - [1, 0] - ], - "transform_parameters": [ - [0.1], - [0.1], - [0.1, 3.0, 0.0] - ], - "transforms": ["Rigid", "Affine", "SyN"], - "use_histogram_matching": true, - "verbose": true, - "winsorize_lower_quantile": 0.025, - "winsorize_upper_quantile": 0.975, - "write_composite_transform": false -} \ No newline at end of file + "collapse_output_transforms": true, + "convergence_threshold": [ + 1e-08, + 1e-08, + 1e-09 + ], + "convergence_window_size": [ + 10, + 10, + 15 + ], + "dimension": 3, + "interpolation": "LanczosWindowedSinc", + "metric": [ + "MI", + "MI", + "CC" + ], + "metric_weight": [ + 1, + 1, + 1 + ], + "number_of_iterations": [ + [ + 100, + 500, + 250, + 100 + ], + [ + 100, + 500, + 250, + 100 + ], + [ + 5, + 10, + 0 + ] + ], + "output_transform_prefix": "anat_to_template", + "output_warped_image": true, + "radius_or_number_of_bins": [ + 32, + 32, + 4 + ], + "sampling_percentage": [ + 0.25, + 0.25, + 1 + ], + "sampling_strategy": [ + "Regular", + "Regular", + "None" + ], + "shrink_factors": [ + [ + 8, + 4, + 2, + 1 + ], + [ + 8, + 4, + 2, + 1 + ], + [ + 2, + 1 + ] + ], + "sigma_units": [ + "vox", + "vox", + "vox" + ], + "smoothing_sigmas": [ + [ + 4, + 2, + 1, + 0 + ], + [ + 4, + 2, + 1, + 0 + ], + [ + 1, + 0 + ] + ], + "transform_parameters": [ + [ + 0.1 + ], + [ + 0.1 + ], + [ + 0.1, + 3.0, + 0.0 + ] + ], + "transforms": [ + "Rigid", + "Affine", + "SyN" + ], + "use_histogram_matching": true, + "verbose": true, + "winsorize_lower_quantile": 0.025, + "winsorize_upper_quantile": 0.975, + "write_composite_transform": false +} diff --git a/CPAC/anat_preproc/data/antsBrainExtraction_precise.json b/CPAC/anat_preproc/data/antsBrainExtraction_precise.json index e0c81d7bef..7d2a3511a4 100644 --- a/CPAC/anat_preproc/data/antsBrainExtraction_precise.json +++ b/CPAC/anat_preproc/data/antsBrainExtraction_precise.json @@ -1,61 +1,142 @@ { - "collapse_output_transforms": true, - "convergence_threshold": [1E-8, 1E-8, 1E-9], - "convergence_window_size": [10, 10, 15], - "dimension": 3, - "interpolation": "LanczosWindowedSinc", - "metric": [ - "MI", - "MI", - ["CC", "CC"] - ], - "metric_weight": [ - 1, - 1, - [0.5, 0.5] - ], - "number_of_iterations": [ - [1000, 500, 250, 100], - [1000, 500, 250, 100], - [50, 10, 0] - ], - "output_transform_prefix": "anat_to_template", - "output_warped_image": true, - "radius_or_number_of_bins": [ - 32, - 32, - [4, 4] - ], - "sampling_percentage": [ - 0.25, - 0.25, - [1, 1] - ], - "sampling_strategy": [ - "Regular", - "Regular", - ["None", "None"] - ], - "shrink_factors": [ - [8, 4, 2, 1], - [8, 4, 2, 1], - [4, 2, 1] - ], - "sigma_units": ["vox", "vox", "vox"], - "smoothing_sigmas": [ - [4, 2, 1, 0], - [4, 2, 1, 0], - [2, 1, 0] - ], - "transform_parameters": [ - [0.1], - [0.1], - [0.1, 3.0, 0.0] - ], - "transforms": ["Rigid", "Affine", "SyN"], - "use_histogram_matching": true, - "verbose": true, - "winsorize_lower_quantile": 0.025, - "winsorize_upper_quantile": 0.975, - "write_composite_transform": false -} \ No newline at end of file + "collapse_output_transforms": true, + "convergence_threshold": [ + 1e-08, + 1e-08, + 1e-09 + ], + "convergence_window_size": [ + 10, + 10, + 15 + ], + "dimension": 3, + "interpolation": "LanczosWindowedSinc", + "metric": [ + "MI", + "MI", + [ + "CC", + "CC" + ] + ], + "metric_weight": [ + 1, + 1, + [ + 0.5, + 0.5 + ] + ], + "number_of_iterations": [ + [ + 1000, + 500, + 250, + 100 + ], + [ + 1000, + 500, + 250, + 100 + ], + [ + 50, + 10, + 0 + ] + ], + "output_transform_prefix": "anat_to_template", + "output_warped_image": true, + "radius_or_number_of_bins": [ + 32, + 32, + [ + 4, + 4 + ] + ], + "sampling_percentage": [ + 0.25, + 0.25, + [ + 1, + 1 + ] + ], + "sampling_strategy": [ + "Regular", + "Regular", + [ + "None", + "None" + ] + ], + "shrink_factors": [ + [ + 8, + 4, + 2, + 1 + ], + [ + 8, + 4, + 2, + 1 + ], + [ + 4, + 2, + 1 + ] + ], + "sigma_units": [ + "vox", + "vox", + "vox" + ], + "smoothing_sigmas": [ + [ + 4, + 2, + 1, + 0 + ], + [ + 4, + 2, + 1, + 0 + ], + [ + 2, + 1, + 0 + ] + ], + "transform_parameters": [ + [ + 0.1 + ], + [ + 0.1 + ], + [ + 0.1, + 3.0, + 0.0 + ] + ], + "transforms": [ + "Rigid", + "Affine", + "SyN" + ], + "use_histogram_matching": true, + "verbose": true, + "winsorize_lower_quantile": 0.025, + "winsorize_upper_quantile": 0.975, + "write_composite_transform": false +} diff --git a/CPAC/anat_preproc/data/antsBrainExtraction_testing.json b/CPAC/anat_preproc/data/antsBrainExtraction_testing.json index b1902defe1..40b62cf6f6 100644 --- a/CPAC/anat_preproc/data/antsBrainExtraction_testing.json +++ b/CPAC/anat_preproc/data/antsBrainExtraction_testing.json @@ -1,61 +1,139 @@ { - "collapse_output_transforms": true, - "convergence_threshold": [1E-8, 1E-8, 1E-9], - "convergence_window_size": [10, 10, 15], - "dimension": 3, - "interpolation": "LanczosWindowedSinc", - "metric": [ - "MI", - "MI", - ["CC", "CC"] - ], - "metric_weight": [ - 1, - 1, - [0.5, 0.5] - ], - "number_of_iterations": [ - [100, 100, 50, 10], - [100, 100, 50, 10], - [5, 0] - ], - "output_transform_prefix": "anat_to_template", - "output_warped_image": true, - "radius_or_number_of_bins": [ - 32, - 32, - [4, 4] - ], - "sampling_percentage": [ - 0.25, - 0.25, - [1, 1] - ], - "sampling_strategy": [ - "Regular", - "Regular", - ["None", "None"] - ], - "shrink_factors": [ - [8, 4, 2, 1], - [8, 4, 2, 1], - [2, 1] - ], - "sigma_units": ["vox", "vox", "vox"], - "smoothing_sigmas": [ - [4, 2, 1, 0], - [4, 2, 1, 0], - [1, 0] - ], - "transform_parameters": [ - [0.1], - [0.1], - [0.1, 3.0, 0.0] - ], - "transforms": ["Rigid", "Affine", "SyN"], - "use_histogram_matching": true, - "verbose": true, - "winsorize_lower_quantile": 0.025, - "winsorize_upper_quantile": 0.975, - "write_composite_transform": false -} \ No newline at end of file + "collapse_output_transforms": true, + "convergence_threshold": [ + 1e-08, + 1e-08, + 1e-09 + ], + "convergence_window_size": [ + 10, + 10, + 15 + ], + "dimension": 3, + "interpolation": "LanczosWindowedSinc", + "metric": [ + "MI", + "MI", + [ + "CC", + "CC" + ] + ], + "metric_weight": [ + 1, + 1, + [ + 0.5, + 0.5 + ] + ], + "number_of_iterations": [ + [ + 100, + 100, + 50, + 10 + ], + [ + 100, + 100, + 50, + 10 + ], + [ + 5, + 0 + ] + ], + "output_transform_prefix": "anat_to_template", + "output_warped_image": true, + "radius_or_number_of_bins": [ + 32, + 32, + [ + 4, + 4 + ] + ], + "sampling_percentage": [ + 0.25, + 0.25, + [ + 1, + 1 + ] + ], + "sampling_strategy": [ + "Regular", + "Regular", + [ + "None", + "None" + ] + ], + "shrink_factors": [ + [ + 8, + 4, + 2, + 1 + ], + [ + 8, + 4, + 2, + 1 + ], + [ + 2, + 1 + ] + ], + "sigma_units": [ + "vox", + "vox", + "vox" + ], + "smoothing_sigmas": [ + [ + 4, + 2, + 1, + 0 + ], + [ + 4, + 2, + 1, + 0 + ], + [ + 1, + 0 + ] + ], + "transform_parameters": [ + [ + 0.1 + ], + [ + 0.1 + ], + [ + 0.1, + 3.0, + 0.0 + ] + ], + "transforms": [ + "Rigid", + "Affine", + "SyN" + ], + "use_histogram_matching": true, + "verbose": true, + "winsorize_lower_quantile": 0.025, + "winsorize_upper_quantile": 0.975, + "write_composite_transform": false +} diff --git a/CPAC/anat_preproc/lesion_preproc.py b/CPAC/anat_preproc/lesion_preproc.py index f6f12fd983..2ef58c3d2a 100644 --- a/CPAC/anat_preproc/lesion_preproc.py +++ b/CPAC/anat_preproc/lesion_preproc.py @@ -1,9 +1,10 @@ # -*- coding: utf-8 -*- from nipype.interfaces import afni -from CPAC.pipeline import nipype_pipeline_engine as pe import nipype.interfaces.utility as util +from CPAC.pipeline import nipype_pipeline_engine as pe + def inverse_lesion(lesion_path): """ @@ -21,15 +22,16 @@ def inverse_lesion(lesion_path): path to the output file, if the lesion does not require to be inverted it returns the unchanged lesion_path input """ - import shutil - import os import ntpath + import os + import shutil - import CPAC.utils.nifti_utils as nu import nibabel as nib + import CPAC.utils.nifti_utils as nu + lesion_out = lesion_path - + if nu.more_zeros_than_ones(image=lesion_path): lesion_out = os.path.join(os.getcwd(), ntpath.basename(lesion_path)) shutil.copyfile(lesion_path, lesion_out) @@ -40,7 +42,7 @@ def inverse_lesion(lesion_path): return lesion_out -def create_lesion_preproc(wf_name='lesion_preproc'): +def create_lesion_preproc(wf_name="lesion_preproc"): """ The main purpose of this workflow is to process lesions masks. Lesion mask file is deobliqued and reoriented in the same way as the T1 in @@ -80,49 +82,45 @@ def create_lesion_preproc(wf_name='lesion_preproc'): >>> preproc.inputs.inputspec.lesion = 'sub1/anat/lesion-mask.nii.gz' >>> preproc.run() #doctest: +SKIP """ - preproc = pe.Workflow(name=wf_name) - inputnode = pe.Node(util.IdentityInterface( - fields=['lesion']), name='inputspec') + inputnode = pe.Node(util.IdentityInterface(fields=["lesion"]), name="inputspec") - outputnode = pe.Node(util.IdentityInterface(fields=['refit', - 'reorient']), - name='outputspec') + outputnode = pe.Node( + util.IdentityInterface(fields=["refit", "reorient"]), name="outputspec" + ) - lesion_deoblique = pe.Node(interface=afni.Refit(), - name='lesion_deoblique') + lesion_deoblique = pe.Node(interface=afni.Refit(), name="lesion_deoblique") lesion_deoblique.inputs.deoblique = True - lesion_inverted = pe.Node(interface=util.Function( - input_names=['lesion_path'], - output_names=['lesion_out'], - function=inverse_lesion), - name='inverse_lesion') + lesion_inverted = pe.Node( + interface=util.Function( + input_names=["lesion_path"], + output_names=["lesion_out"], + function=inverse_lesion, + ), + name="inverse_lesion", + ) # We first check and invert the lesion if needed to be used by ANTs - preproc.connect( - inputnode, 'lesion', lesion_inverted, 'lesion_path') + preproc.connect(inputnode, "lesion", lesion_inverted, "lesion_path") - preproc.connect( - lesion_inverted, 'lesion_out', lesion_deoblique, 'in_file') + preproc.connect(lesion_inverted, "lesion_out", lesion_deoblique, "in_file") - preproc.connect( - lesion_deoblique, 'out_file', outputnode, 'refit') + preproc.connect(lesion_deoblique, "out_file", outputnode, "refit") # Anatomical reorientation - lesion_reorient = pe.Node(interface=afni.Resample(), - name='lesion_reorient', - mem_gb=0, - mem_x=(0.0115, 'in_file', 't')) - - lesion_reorient.inputs.orientation = 'RPI' - lesion_reorient.inputs.outputtype = 'NIFTI_GZ' - - preproc.connect( - lesion_deoblique, 'out_file', lesion_reorient, - 'in_file') - preproc.connect( - lesion_reorient, 'out_file', outputnode, 'reorient') + lesion_reorient = pe.Node( + interface=afni.Resample(), + name="lesion_reorient", + mem_gb=0, + mem_x=(0.0115, "in_file", "t"), + ) + + lesion_reorient.inputs.orientation = "RPI" + lesion_reorient.inputs.outputtype = "NIFTI_GZ" + + preproc.connect(lesion_deoblique, "out_file", lesion_reorient, "in_file") + preproc.connect(lesion_reorient, "out_file", outputnode, "reorient") return preproc diff --git a/CPAC/anat_preproc/tests/test_anat_preproc.py b/CPAC/anat_preproc/tests/test_anat_preproc.py index 61686eeb39..517d5dc074 100755 --- a/CPAC/anat_preproc/tests/test_anat_preproc.py +++ b/CPAC/anat_preproc/tests/test_anat_preproc.py @@ -1,39 +1,38 @@ import os -import sys -from .. import anat_preproc + from nose.tools import * -import nibabel as nib import numpy as np +import nibabel as nib -class TestAnatPreproc: +from .. import anat_preproc +class TestAnatPreproc: def __init__(self): - """ - Initialize and run the the anat_preproc workflow - - Populate the node-name : node_output dictionary using the workflow object. - This dictionary serves the outputs of each of the nodes in the workflow to all the tests that need them. + Initialize and run the the anat_preproc workflow. - Parameters - ---------- + Populate the node-name : node_output dictionary using the workflow object. + This dictionary serves the outputs of each of the nodes in the workflow to all the tests that need them. + Parameters + ---------- self - Returns - ------- - + Returns + ------- None """ - - self.preproc = anat_preproc.create_anat_preproc() - self.input_anat = os.path.abspath('$FSLDIR/data/standard/MNI152_T1_2mm.nii.gz') - self.input_anat_known_brain = os.path.abspath('$FSLDIR/data/standard/MNI152_T1_2mm_brain.nii.gz') - self.base_dir = os.path.abspath('/home/data/Projects/Pipelines_testing/Dickstein_CPAC_working_dir/') + self.input_anat = os.path.abspath("$FSLDIR/data/standard/MNI152_T1_2mm.nii.gz") + self.input_anat_known_brain = os.path.abspath( + "$FSLDIR/data/standard/MNI152_T1_2mm_brain.nii.gz" + ) + self.base_dir = os.path.abspath( + "/home/data/Projects/Pipelines_testing/Dickstein_CPAC_working_dir/" + ) self.preproc.inputs.inputspec.anat = self.input_anat self.preproc.base_dir = self.base_dir self.anat_execution_graph = self.preproc.run() @@ -43,247 +42,196 @@ def __init__(self): it = self.anat_execution_graph.nodes_iter() for node in it: - name = node.fullname node_output = node.result.outputs.out_file - name = name.split('.')[1] + name = name.split(".")[1] - #print node.result.outputs.out_file, ' ', node.fullname + # print node.result.outputs.out_file, ' ', node.fullname - if not (name in self.node_names_map): + if name not in self.node_names_map: self.node_names_map[name] = node_output - -# def setUp(self): - - + # def setUp(self): def tearDown(self): - """ - Deletes the workflow object - - """ - + """Deletes the workflow object.""" del self.preproc def test_inputs(self): - """ - Checks the input anatomical image. - - Parameters - ---------- + Checks the input anatomical image. + Parameters + ---------- self - Returns - ------- - + Returns + ------- None - Notes - ----- + Notes + ----- + The tests are - The tests are - - - Check if the input image ends with .nii or .nii.gz - - Check if the input image is stored in 3D only + - Check if the input image ends with .nii or .nii.gz + - Check if the input image is stored in 3D only """ - - anat_img = self.input_anat - if not (anat_img.endswith('.nii.gz') or anat_img.endswith('.nii')): - - raise 'Input image name does not end with .nii.gz or .nii' - + if not (anat_img.endswith(".nii.gz") or anat_img.endswith(".nii")): + raise "Input image name does not end with .nii.gz or .nii" try: - img = nib.load(anat_img) dims = img.get_shape() assert len(dims) == 3 except: - - raise 'input T1 image is not 3D' - - + raise "input T1 image is not 3D" def test_anat_deoblique(self): - """ - Check the output from anat_deoblique node - - Parameters - ---------- + Check the output from anat_deoblique node. + Parameters + ---------- self - Returns - ------- - + Returns + ------- None - Notes - ----- - + Notes + ----- 3drefit command is supposed to overwrite the transformation matrix with the cardinal matrix in the header - In addition to this it also modifies a lot of other fields(aside from voxel_offset it sets other header fields - to zero) . The number of fields varies depending on how the nifti file was obtained from the source dicom files. + In addition to this it also modifies a lot of other fields(aside from voxel_offset it sets other header fields + to zero) . The number of fields varies depending on how the nifti file was obtained from the source dicom files. - So, The check tests the image Data of the output of anat_deoblique node against the input anatomical image. - Both should be the same + So, The check tests the image Data of the output of anat_deoblique node against the input anatomical image. + Both should be the same """ - - - if not ('anat_deoblique' in self.node_names_map): + if "anat_deoblique" not in self.node_names_map: assert False else: - deobliqued_anat = self.node_names_map['anat_deoblique'] + deobliqued_anat = self.node_names_map["anat_deoblique"] de_img_data = nib.load(deobliqued_anat).get_fdata() orig_image_data = nib.load(self.input_anat).get_fdata() - de_img_data = de_img_data.flatten() orig_image_data = orig_image_data.flatten() np.testing.assert_equal(de_img_data, orig_image_data) -# de_image_header = nib.load(deobliqued_anat).header -# -# de_vox_offset = de_image_header.get_data_offset() -# -# orig_image_header = nib.load(self.input_anat).header -# -# orig_vox_offset = orig_image_header.get_data_offset() -# -# -# if orig_vox_offset == de_vox_offset: -# -# print 'Its not possible for deobliqued T1 and Original T1 voxels offsets are same' -# raise -# -# else: -# not_match_count = 0 -# header_values_de = de_image_header.values() -# header_values_input = orig_image_header.values() -# -# for i in range(0, len(header_values_input)): -# -# a = (header_values_de[i] == header_values_input[i]) -# if not (a.all() == True): -# -# not_match_count += 1 -# print 'not_match_count: ', not_match_count -# #assert not_match_count == 1 - - + # de_image_header = nib.load(deobliqued_anat).header + # + # de_vox_offset = de_image_header.get_data_offset() + # + # orig_image_header = nib.load(self.input_anat).header + # + # orig_vox_offset = orig_image_header.get_data_offset() + # + # + # if orig_vox_offset == de_vox_offset: + # + # print 'Its not possible for deobliqued T1 and Original T1 voxels offsets are same' + # raise + # + # else: + # not_match_count = 0 + # header_values_de = de_image_header.values() + # header_values_input = orig_image_header.values() + # + # for i in range(0, len(header_values_input)): + # + # a = (header_values_de[i] == header_values_input[i]) + # if not (a.all() == True): + # + # not_match_count += 1 + # print 'not_match_count: ', not_match_count + # #assert not_match_count == 1 def test_anat_reorient(self): - """ - Checks the output of anat_reorient node. - - Parameters - ---------- + Checks the output of anat_reorient node. + Parameters + ---------- self - Returns - ------- - + Returns + ------- None - Notes - ----- - + Notes + ----- Checks the s-form matrix in the header of the output file. - For the output image to be in RPI , The first diagonal element must be negative - The rest of the diagonal elements must be positive. The non diagonal elements must be - zero, the offset elements can be anything. + For the output image to be in RPI , The first diagonal element must be negative + The rest of the diagonal elements must be positive. The non diagonal elements must be + zero, the offset elements can be anything. """ - - - - if not ('anat_reorient' in self.node_names_map): + if "anat_reorient" not in self.node_names_map: assert False else: - anat_reorient = self.node_names_map['anat_reorient'] + anat_reorient = self.node_names_map["anat_reorient"] anat_reorient_sform = nib.load(anat_reorient).get_sform() - #print 'sform: ', anat_reorient_sform + # print 'sform: ', anat_reorient_sform for i in range(0, 4): - for j in range(0, 4): - if i == j: if i == 0: - assert int(anat_reorient_sform[i][i]) < 0 else: assert int(anat_reorient_sform[i][i]) > 0 else: - if not (j == 3): - assert int(anat_reorient_sform[i][j]) == 0 - - def test_anat_skullstrip(self): - """ - Tests the output of anat_skullstrip node in the workflow - - Parameters - ---------- + Tests the output of anat_skullstrip node in the workflow. + Parameters + ---------- self - Returns - ------- - + Returns + ------- None - Notes - ----- - + Notes + ----- Compares the output image of anat_skullstrip node with the known skullstripped image. - Since the intensity values do not correspond to the input + Since the intensity values do not correspond to the input """ - - - if not ('anat_skullstrip' in self.node_names_map): - + if "anat_skullstrip" not in self.node_names_map: assert False else: def binarize(data): - data[np.flatnonzero(data)] = 1 return data - anat_skullstrip = self.node_names_map['anat_skullstrip'] + anat_skullstrip = self.node_names_map["anat_skullstrip"] skullstrip_data = nib.load(anat_skullstrip).get_fdata() known_brain_data = nib.load(self.input_anat_known_brain).get_fdata() @@ -293,40 +241,32 @@ def binarize(data): correlation = np.corrcoef(bin_skullstrip, bin_brain) - #print 'correlation: ', correlation + # print 'correlation: ', correlation assert correlation[0, 1] >= 0.95 - - def test_anat_brain(self): - """ - Compares the SkullStripped Anatomical Image(after its passed through step function) with the known skullstripped image available - - Parameters - ---------- + Compares the SkullStripped Anatomical Image(after its passed through step function) with the known skullstripped image available. + Parameters + ---------- self - Returns - ------- - + Returns + ------- Nothing """ - - if not ('anat_brain_only' in self.node_names_map): - + if "anat_brain_only" not in self.node_names_map: assert False else: - - anat_brain = self.node_names_map['anat_brain_only'] + anat_brain = self.node_names_map["anat_brain_only"] brain_data = nib.load(anat_brain).get_fdata() known_brain_data = nib.load(self.input_anat_known_brain).get_fdata() correlation = np.corrcoef(brain_data.flatten(), known_brain_data.flatten()) - #print 'correlation: ', correlation + # print 'correlation: ', correlation assert correlation[0, 1] >= 0.97 diff --git a/CPAC/anat_preproc/utils.py b/CPAC/anat_preproc/utils.py index 1ec23337f2..239b383f40 100644 --- a/CPAC/anat_preproc/utils.py +++ b/CPAC/anat_preproc/utils.py @@ -5,7 +5,6 @@ def fsl_aff_to_rigid(in_xfm, out_name): - out_mat = os.path.join(os.getcwd(), out_name) # Script for getting a 6 DOF approx to a 12 DOF standard transformation @@ -87,8 +86,8 @@ def fsl_aff_to_rigid(in_xfm, out_name): ainv = linalg.inv(a) # vectors v are in MNI space, vectors w are in native space - v21 = (x2 - x1) - v31 = (x3 - x1) + v21 = x2 - x1 + v31 = x3 - x1 # normalise and force orthogonality v21 = v21 / linalg.norm(v21) v31 = v31 - multiply(v31.T * v21, v21) @@ -125,7 +124,7 @@ def fsl_aff_to_rigid(in_xfm, out_name): r[0:3, 3] = trans[0:3] # Save out the result - savetxt(out_mat, r, fmt='%14.10f') + savetxt(out_mat, r, fmt="%14.10f") return out_mat @@ -148,61 +147,86 @@ def freesurfer_hemispheres(wf, reconall, pipe_num): outputs : dict """ + def split_hemi(multi_file): # pylint: disable=invalid-name lh = None rh = None for filepath in multi_file: - if 'lh.' in filepath: + if "lh." in filepath: lh = filepath - if 'rh.' in filepath: + if "rh." in filepath: rh = filepath return (lh, rh) def split_hemi_interface(): """Returns a function interface for split_hemi.""" - return util.Function(input_names=['multi_file'], - output_names=['lh', 'rh'], - function=split_hemi) + return util.Function( + input_names=["multi_file"], output_names=["lh", "rh"], function=split_hemi + ) splits = { - label: pe.Node(split_hemi_interface(), - name=f'split_{label}_{pipe_num}') for - label in ['curv', 'pial', 'smoothwm', 'sphere', 'sulc', 'thickness', - 'volume', 'white'] + label: pe.Node(split_hemi_interface(), name=f"split_{label}_{pipe_num}") + for label in [ + "curv", + "pial", + "smoothwm", + "sphere", + "sulc", + "thickness", + "volume", + "white", + ] } for label in splits: - wf.connect(reconall, label, splits[label], 'multi_file') + wf.connect(reconall, label, splits[label], "multi_file") outputs = { - 'pipeline-fs_hemi-L_desc-surface_curv': (splits['curv'], 'lh'), - 'pipeline-fs_hemi-R_desc-surface_curv': (splits['curv'], 'rh'), - 'pipeline-fs_hemi-L_desc-surfaceMesh_pial': (splits['pial'], 'lh'), - 'pipeline-fs_hemi-R_desc-surfaceMesh_pial': (splits['pial'], 'rh'), - 'pipeline-fs_hemi-L_desc-surfaceMesh_smoothwm': (splits['smoothwm'], 'lh'), - 'pipeline-fs_hemi-R_desc-surfaceMesh_smoothwm': (splits['smoothwm'], 'rh'), - 'pipeline-fs_hemi-L_desc-surfaceMesh_sphere': (splits['sphere'], 'lh'), - 'pipeline-fs_hemi-R_desc-surfaceMesh_sphere': (splits['sphere'], 'rh'), - 'pipeline-fs_hemi-L_desc-surfaceMap_sulc': (splits['sulc'], 'lh'), - 'pipeline-fs_hemi-R_desc-surfaceMap_sulc': (splits['sulc'], 'rh'), - 'pipeline-fs_hemi-L_desc-surfaceMap_thickness': (splits['thickness'], 'lh'), - 'pipeline-fs_hemi-R_desc-surfaceMap_thickness': (splits['thickness'], 'rh'), - 'pipeline-fs_hemi-L_desc-surfaceMap_volume': (splits['volume'], 'lh'), - 'pipeline-fs_hemi-R_desc-surfaceMap_volume': (splits['volume'], 'rh'), - 'pipeline-fs_hemi-L_desc-surfaceMesh_white': (splits['white'], 'lh'), - 'pipeline-fs_hemi-R_desc-surfaceMesh_white': (splits['white'], 'rh')} + "pipeline-fs_hemi-L_desc-surface_curv": (splits["curv"], "lh"), + "pipeline-fs_hemi-R_desc-surface_curv": (splits["curv"], "rh"), + "pipeline-fs_hemi-L_desc-surfaceMesh_pial": (splits["pial"], "lh"), + "pipeline-fs_hemi-R_desc-surfaceMesh_pial": (splits["pial"], "rh"), + "pipeline-fs_hemi-L_desc-surfaceMesh_smoothwm": (splits["smoothwm"], "lh"), + "pipeline-fs_hemi-R_desc-surfaceMesh_smoothwm": (splits["smoothwm"], "rh"), + "pipeline-fs_hemi-L_desc-surfaceMesh_sphere": (splits["sphere"], "lh"), + "pipeline-fs_hemi-R_desc-surfaceMesh_sphere": (splits["sphere"], "rh"), + "pipeline-fs_hemi-L_desc-surfaceMap_sulc": (splits["sulc"], "lh"), + "pipeline-fs_hemi-R_desc-surfaceMap_sulc": (splits["sulc"], "rh"), + "pipeline-fs_hemi-L_desc-surfaceMap_thickness": (splits["thickness"], "lh"), + "pipeline-fs_hemi-R_desc-surfaceMap_thickness": (splits["thickness"], "rh"), + "pipeline-fs_hemi-L_desc-surfaceMap_volume": (splits["volume"], "lh"), + "pipeline-fs_hemi-R_desc-surfaceMap_volume": (splits["volume"], "rh"), + "pipeline-fs_hemi-L_desc-surfaceMesh_white": (splits["white"], "lh"), + "pipeline-fs_hemi-R_desc-surfaceMesh_white": (splits["white"], "rh"), + } return wf, outputs -def create_3dskullstrip_arg_string(shrink_fac, var_shrink_fac, - shrink_fac_bot_lim, avoid_vent, niter, - pushout, touchup, fill_hole, avoid_eyes, - use_edge, exp_frac, NN_smooth, smooth_final, - push_to_edge, use_skull, perc_int, - max_inter_iter, blur_fwhm, fac, monkey, - mask_vol): +def create_3dskullstrip_arg_string( + shrink_fac, + var_shrink_fac, + shrink_fac_bot_lim, + avoid_vent, + niter, + pushout, + touchup, + fill_hole, + avoid_eyes, + use_edge, + exp_frac, + NN_smooth, + smooth_final, + push_to_edge, + use_skull, + perc_int, + max_inter_iter, + blur_fwhm, + fac, + monkey, + mask_vol, +): """ - Method to return option string for 3dSkullStrip + Method to return option string for 3dSkullStrip. Parameters ---------- @@ -265,7 +289,7 @@ def create_3dskullstrip_arg_string(shrink_fac, var_shrink_fac, monkey : boolean Use monkey option in SkullStripping - + mask_vol : boolean Output a mask volume instead of a skull-stripped volume. @@ -273,95 +297,94 @@ def create_3dskullstrip_arg_string(shrink_fac, var_shrink_fac, ------- opt_str : string Command args - - """ - expr = '' - defaults = dict( - fill_hole=10 if touchup else 0, - shrink_fac=0.6, - shrink_fac_bot_lim=0.4 if use_edge else 0.65, - niter=250, - exp_frac=0.1, - NN_smooth=72, - smooth_final=20, - perc_int=0, - max_inter_iter=4, - blur_fwhm=0, - fac=1.0, - monkey=False, - mask_vol=False - ) + """ + expr = "" + defaults = { + "fill_hole": 10 if touchup else 0, + "shrink_fac": 0.6, + "shrink_fac_bot_lim": 0.4 if use_edge else 0.65, + "niter": 250, + "exp_frac": 0.1, + "NN_smooth": 72, + "smooth_final": 20, + "perc_int": 0, + "max_inter_iter": 4, + "blur_fwhm": 0, + "fac": 1.0, + "monkey": False, + "mask_vol": False, + } - if float(shrink_fac) != defaults['shrink_fac']: - expr += ' -shrink_fac {0}'.format(shrink_fac) + if float(shrink_fac) != defaults["shrink_fac"]: + expr += " -shrink_fac {0}".format(shrink_fac) if not var_shrink_fac: - expr += ' -no_var_shrink_fac' + expr += " -no_var_shrink_fac" if mask_vol: - expr += ' -mask_vol' + expr += " -mask_vol" if monkey: - expr += ' -monkey' + expr += " -monkey" - if float(shrink_fac_bot_lim) != defaults['shrink_fac_bot_lim']: - expr += ' -shrink_fac_bot_lim {0}'.format(shrink_fac_bot_lim) + if float(shrink_fac_bot_lim) != defaults["shrink_fac_bot_lim"]: + expr += " -shrink_fac_bot_lim {0}".format(shrink_fac_bot_lim) if not use_edge: - expr += ' -no_use_edge' + expr += " -no_use_edge" if not avoid_vent: - expr += ' -no_avoid_vent' + expr += " -no_avoid_vent" - if int(niter) != defaults['niter']: - expr += ' -niter {0}'.format(niter) + if int(niter) != defaults["niter"]: + expr += " -niter {0}".format(niter) if not pushout: - expr += ' -no_pushout' + expr += " -no_pushout" if not touchup: - expr += ' -no_touchup' + expr += " -no_touchup" - if int(fill_hole) != defaults['fill_hole']: - expr += ' -fill_hole {0}'.format(fill_hole) + if int(fill_hole) != defaults["fill_hole"]: + expr += " -fill_hole {0}".format(fill_hole) if not avoid_eyes: - expr += ' -no_avoid_eyes' + expr += " -no_avoid_eyes" - if float(exp_frac) != defaults['exp_frac']: - expr += ' -exp_frac {0}'.format(exp_frac) + if float(exp_frac) != defaults["exp_frac"]: + expr += " -exp_frac {0}".format(exp_frac) - if int(NN_smooth) != defaults['NN_smooth']: - expr += ' -NN_smooth {0}'.format(NN_smooth) + if int(NN_smooth) != defaults["NN_smooth"]: + expr += " -NN_smooth {0}".format(NN_smooth) - if int(smooth_final) != defaults['smooth_final']: - expr += ' -smooth_final {0}'.format(smooth_final) + if int(smooth_final) != defaults["smooth_final"]: + expr += " -smooth_final {0}".format(smooth_final) if push_to_edge: - expr += ' -push_to_edge' + expr += " -push_to_edge" if use_skull: - expr += ' -use_skull' + expr += " -use_skull" - if float(perc_int) != defaults['perc_int']: - expr += ' -perc_int {0}'.format(perc_int) + if float(perc_int) != defaults["perc_int"]: + expr += " -perc_int {0}".format(perc_int) - if int(max_inter_iter) != defaults['max_inter_iter']: - expr += ' -max_inter_iter {0}'.format(max_inter_iter) + if int(max_inter_iter) != defaults["max_inter_iter"]: + expr += " -max_inter_iter {0}".format(max_inter_iter) - if float(blur_fwhm) != defaults['blur_fwhm']: - expr += ' -blur_fwhm {0}'.format(blur_fwhm) + if float(blur_fwhm) != defaults["blur_fwhm"]: + expr += " -blur_fwhm {0}".format(blur_fwhm) - if float(fac) != defaults['fac']: - expr += ' -fac {0}'.format(fac) + if float(fac) != defaults["fac"]: + expr += " -fac {0}".format(fac) return expr def mri_convert(in_file, reslice_like=None, out_file=None, args=None): """ - Method to convert files from mgz to nifti format + Method to convert files from mgz to nifti format. Parameters ---------- @@ -369,24 +392,24 @@ def mri_convert(in_file, reslice_like=None, out_file=None, args=None): A path of mgz input file args : string Arguments of mri_convert + Returns ------- out_file : string A path of nifti output file """ - import os if out_file is None: - out_file = in_file.replace('.mgz','.nii.gz') + out_file = in_file.replace(".mgz", ".nii.gz") - cmd = 'mri_convert %s %s' % (in_file, out_file) + cmd = "mri_convert %s %s" % (in_file, out_file) if reslice_like is not None: - cmd = cmd + ' -rl ' + reslice_like + cmd = cmd + " -rl " + reslice_like if args is not None: - cmd = cmd + ' ' +args + cmd = cmd + " " + args os.system(cmd) @@ -394,12 +417,11 @@ def mri_convert(in_file, reslice_like=None, out_file=None, args=None): def wb_command(in_file): - import os - out_file = in_file.replace('.nii.gz','_fill_holes.nii.gz') + out_file = in_file.replace(".nii.gz", "_fill_holes.nii.gz") - cmd = 'wb_command -volume-fill-holes %s %s' % (in_file, out_file) + cmd = "wb_command -volume-fill-holes %s %s" % (in_file, out_file) os.system(cmd) @@ -407,33 +429,43 @@ def wb_command(in_file): def fslmaths_command(in_file, number, out_file_suffix): - import os - out_filename = in_file.replace('.nii.gz', out_file_suffix+'.nii.gz') + out_filename = in_file.replace(".nii.gz", out_file_suffix + ".nii.gz") - out_file = os.path.join(os.getcwd(), out_filename[out_filename.rindex('/')+1:]) + out_file = os.path.join(os.getcwd(), out_filename[out_filename.rindex("/") + 1 :]) - cmd = 'fslmaths %s -div %f -mul 150 -abs %s' % (in_file, number, out_file) + cmd = "fslmaths %s -div %f -mul 150 -abs %s" % (in_file, number, out_file) os.system(cmd) return out_file + def normalize_wmparc(source_file, target_file, xfm, out_file): - from CPAC.utils.monitoring.custom_logging import log_subprocess import os - cmd = ['mri_vol2vol', '--mov', source_file, \ - '--targ', target_file, '--o', out_file, '--lta', xfm] + from CPAC.utils.monitoring.custom_logging import log_subprocess + + cmd = [ + "mri_vol2vol", + "--mov", + source_file, + "--targ", + target_file, + "--o", + out_file, + "--lta", + xfm, + ] log_subprocess(cmd) - output = os.path.join(os.getcwd(), out_file) - return output + return os.path.join(os.getcwd(), out_file) + """This module provides interfaces for workbench -volume-remove-islands commands""" -from nipype.interfaces.base import TraitedSpec, File, traits, CommandLineInputSpec -from nipype.interfaces.workbench.base import WBCommand from nipype import logging +from nipype.interfaces.base import CommandLineInputSpec, File, TraitedSpec +from nipype.interfaces.workbench.base import WBCommand iflogger = logging.getLogger("nipype.interface") @@ -455,7 +487,6 @@ class VolumeRemoveIslandsInputSpec(CommandLineInputSpec): ) - class VolumeRemoveIslandsOutputSpec(TraitedSpec): out_file = File(exists=True, desc="the output ROI volume") @@ -467,7 +498,7 @@ class VolumeRemoveIslands(WBCommand): REMOVE ISLANDS FROM AN ROI VOLUME wb_command -volume-remove-islands - the input ROI volume - - output - the output ROI volume + - output - the output ROI volume. Finds all face-connected parts of the ROI, and zeros out all but the largest one. @@ -476,5 +507,4 @@ class VolumeRemoveIslands(WBCommand): input_spec = VolumeRemoveIslandsInputSpec output_spec = VolumeRemoveIslandsOutputSpec - _cmd = "wb_command -volume-remove-islands" - + _cmd = "wb_command -volume-remove-islands" diff --git a/CPAC/aroma/__init__.py b/CPAC/aroma/__init__.py index bf3f9470c5..e69de29bb2 100644 --- a/CPAC/aroma/__init__.py +++ b/CPAC/aroma/__init__.py @@ -1 +0,0 @@ -from CPAC.aroma.aroma import create_aroma diff --git a/CPAC/aroma/aroma.py b/CPAC/aroma/aroma.py index 60dfc3675e..92e619848e 100644 --- a/CPAC/aroma/aroma.py +++ b/CPAC/aroma/aroma.py @@ -1,10 +1,11 @@ from nipype.interfaces import fsl -from CPAC.pipeline import nipype_pipeline_engine as pe -import nipype.interfaces.utility as util from nipype.interfaces.fsl.aroma import ICA_AROMA - +import nipype.interfaces.utility as util + +from CPAC.pipeline import nipype_pipeline_engine as pe + -def create_aroma(tr=None, wf_name='create_aroma'): +def create_aroma(tr=None, wf_name="create_aroma"): """ICA-AROMA takes in a functional file, along with the movement parameters, warp file and a mat file to denoise the artifacts using ICA-based methods and produces an output directory which contains the @@ -21,7 +22,7 @@ def create_aroma(tr=None, wf_name='create_aroma'): mask file for the ICA-AROMA interface in_file: denoise_file out_file: mask_file - Parameters: + Parameters: -f : 0.3 --ICA-AROMA : Takes in the denoise_file, par_file (FSL-McFlirt), @@ -35,48 +36,48 @@ def create_aroma(tr=None, wf_name='create_aroma'): - denoise_type : 'aggr', 'nonaggr' - TR : s - dim: - """ - + """ preproc = pe.Workflow(name=wf_name) - inputNode = pe.Node(util.IdentityInterface(fields=['denoise_file', - 'mat_file', - 'fnirt_warp_file']), - name='inputspec') + inputNode = pe.Node( + util.IdentityInterface(fields=["denoise_file", "mat_file", "fnirt_warp_file"]), + name="inputspec", + ) - inputNode_params = pe.Node(util.IdentityInterface(fields=['denoise_type', - 'dim']), - name='params') + inputNode_params = pe.Node( + util.IdentityInterface(fields=["denoise_type", "dim"]), name="params" + ) - outputNode = pe.Node(util.IdentityInterface(fields=['aggr_denoised_file', - 'nonaggr_denoised_file']), - name='outputspec') + outputNode = pe.Node( + util.IdentityInterface(fields=["aggr_denoised_file", "nonaggr_denoised_file"]), + name="outputspec", + ) - par_mcflirt = pe.Node(interface = fsl.MCFLIRT(),name='par_mcflirt') + par_mcflirt = pe.Node(interface=fsl.MCFLIRT(), name="par_mcflirt") par_mcflirt.inputs.save_plots = True - preproc.connect(inputNode,'denoise_file', par_mcflirt,'in_file') - preproc.connect(par_mcflirt,'par_file', outputNode,'par_file') + preproc.connect(inputNode, "denoise_file", par_mcflirt, "in_file") + preproc.connect(par_mcflirt, "par_file", outputNode, "par_file") - bet_aroma = pe.Node(interface=fsl.BET(),name='bet_aroma') + bet_aroma = pe.Node(interface=fsl.BET(), name="bet_aroma") bet_aroma.inputs.frac = 0.3 bet_aroma.inputs.mask = True bet_aroma.inputs.functional = True - preproc.connect(inputNode,'denoise_file', bet_aroma,'in_file') - preproc.connect(bet_aroma,'mask_file', outputNode,'mask_aroma') + preproc.connect(inputNode, "denoise_file", bet_aroma, "in_file") + preproc.connect(bet_aroma, "mask_file", outputNode, "mask_aroma") - aroma = pe.Node(ICA_AROMA(), name='aroma_wf') - aroma.inputs.out_dir = '.' + aroma = pe.Node(ICA_AROMA(), name="aroma_wf") + aroma.inputs.out_dir = "." if tr: aroma.inputs.TR = tr - preproc.connect(inputNode,'denoise_file', aroma,'in_file') - preproc.connect(inputNode,'mat_file', aroma,'mat_file') - preproc.connect(inputNode,'fnirt_warp_file', aroma,'fnirt_warp_file') - preproc.connect(par_mcflirt,'par_file', aroma,'motion_parameters') - preproc.connect(bet_aroma,'mask_file', aroma,'mask') - preproc.connect(inputNode_params,'denoise_type', aroma,'denoise_type') - preproc.connect(inputNode_params,'dim', aroma,'dim') - preproc.connect(aroma,'nonaggr_denoised_file', outputNode,'nonaggr_denoised_file') - preproc.connect(aroma,'aggr_denoised_file', outputNode,'aggr_denoised_file') - + preproc.connect(inputNode, "denoise_file", aroma, "in_file") + preproc.connect(inputNode, "mat_file", aroma, "mat_file") + preproc.connect(inputNode, "fnirt_warp_file", aroma, "fnirt_warp_file") + preproc.connect(par_mcflirt, "par_file", aroma, "motion_parameters") + preproc.connect(bet_aroma, "mask_file", aroma, "mask") + preproc.connect(inputNode_params, "denoise_type", aroma, "denoise_type") + preproc.connect(inputNode_params, "dim", aroma, "dim") + preproc.connect(aroma, "nonaggr_denoised_file", outputNode, "nonaggr_denoised_file") + preproc.connect(aroma, "aggr_denoised_file", outputNode, "aggr_denoised_file") + return preproc diff --git a/CPAC/aroma/aroma_test.py b/CPAC/aroma/aroma_test.py index 5baf795cef..d8af84175c 100644 --- a/CPAC/aroma/aroma_test.py +++ b/CPAC/aroma/aroma_test.py @@ -21,26 +21,26 @@ @author: nrajamani """ -import glob import os -import pytest +import pytest import nipype.interfaces.io as nio + from CPAC.pipeline import nipype_pipeline_engine as pe from CPAC.pipeline.nipype_pipeline_engine.plugins import MultiProcPlugin -from nipype.interfaces.fsl import ImageStats + # from nose.tools import * -@pytest.mark.skip(reason='needs refactoring') +@pytest.mark.skip(reason="needs refactoring") def run_warp_nipype(inputs, output_dir=None, run=True): import EPI_DistCorr - warp_workflow = pe.Workflow(name='preproc') + + warp_workflow = pe.Workflow(name="preproc") if output_dir is None: - output_dir = '/home/nrajamani' + output_dir = "/home/nrajamani" - workflow_dir = os.path.join(output_dir, - "workflow_output_with_aroma_with_change") + workflow_dir = os.path.join(output_dir, "workflow_output_with_aroma_with_change") warp_workflow.base_dir = workflow_dir # taken from QAP files # resource_pool = {} @@ -49,55 +49,52 @@ def run_warp_nipype(inputs, output_dir=None, run=True): # resource_pool({ # 'epireg': (warp_nipype2.warp_nipype, 'outputspec.epireg')}) t_node = EPI_DistCorr.create_EPI_DistCorr() # ### - t_node.inputs.inputspec.anat_file = \ - '/Users/nanditharajamani/Downloads/ExBox19/T1.nii.gz' - t_node.inputs.inputspec.func_file = \ - '/Users/nanditharajamani/Downloads/ExBox19/func.nii.gz' - t_node.inputs.inputspec.fmap_pha = \ - '/Users/nanditharajamani/Downloads/ExBox19/fmap_phase.nii.gz' - t_node.inputs.inputspec.fmap_mag = \ - '/Users/nanditharajamani/Downloads/ExBox19/fmap_mag.nii.gz' - t_node.inputs.inputspec.bbr_schedule = \ - '/usr/local/fsl/etc/flirtsch/bbr.sch' + t_node.inputs.inputspec.anat_file = ( + "/Users/nanditharajamani/Downloads/ExBox19/T1.nii.gz" + ) + t_node.inputs.inputspec.func_file = ( + "/Users/nanditharajamani/Downloads/ExBox19/func.nii.gz" + ) + t_node.inputs.inputspec.fmap_pha = ( + "/Users/nanditharajamani/Downloads/ExBox19/fmap_phase.nii.gz" + ) + t_node.inputs.inputspec.fmap_mag = ( + "/Users/nanditharajamani/Downloads/ExBox19/fmap_mag.nii.gz" + ) + t_node.inputs.inputspec.bbr_schedule = "/usr/local/fsl/etc/flirtsch/bbr.sch" t_node.inputs.inputspec.deltaTE = 2.46 t_node.inputs.inputspec.dwellT = 0.0005 t_node.inputs.inputspec.dwell_asym_ratio = 0.93902439 t_node.inputs.inputspec.bet_frac = 0.5 # 'home/nrajamani/FieldMap_SubjectExampleData/SubjectData/epi_run2/fMT0160-0015-00003-000003-01_BRAIN.nii.gz', # for image in inputs: -# if not(image.endswith('.nii') or image.endswith('.nii.gz')): -# raise 'The input image is not the right format' -# try: -# for image in inputs: -# size = image.get_shape() -# assert len(size) == 3 -# except: -# if len(size) < 3: -# raise 'input image is not 3D' -# intensity = ImageStats( -# in_file = t_node.inputs.inputspec.fmap_pha, op_string = '-p 90') -# if intensity < 3686: -# raise 'input phase image does not have the correct range values' - dataSink = pe.Node(nio.DataSink(), name='dataSink_file') + # if not(image.endswith('.nii') or image.endswith('.nii.gz')): + # raise 'The input image is not the right format' + # try: + # for image in inputs: + # size = image.get_shape() + # assert len(size) == 3 + # except: + # if len(size) < 3: + # raise 'input image is not 3D' + # intensity = ImageStats( + # in_file = t_node.inputs.inputspec.fmap_pha, op_string = '-p 90') + # if intensity < 3686: + # raise 'input phase image does not have the correct range values' + dataSink = pe.Node(nio.DataSink(), name="dataSink_file") dataSink.inputs.base_directory = workflow_dir # node, out_file = resource_pool["epireg"] # warp_workflow.connect(t_node,'outputspec.roi_file',dataSink,'roi_file') - warp_workflow.connect(t_node, 'outputspec.fieldmap', - dataSink, 'fieldmap_file') - warp_workflow.connect(t_node, 'outputspec.fmapmagbrain', - dataSink, 'fmapmagbrain') - warp_workflow.connect(t_node, 'outputspec.fieldmapmask', - dataSink, 'fieldmapmask') - warp_workflow.connect(t_node, 'outputspec.fmap_despiked', - dataSink, 'fmap_despiked') - warp_workflow.connect(t_node, 'outputspec.struct', - dataSink, 'epi2struct') - warp_workflow.connect(t_node, 'outputspec.anat_func', - dataSink, 'anat_func') + warp_workflow.connect(t_node, "outputspec.fieldmap", dataSink, "fieldmap_file") + warp_workflow.connect(t_node, "outputspec.fmapmagbrain", dataSink, "fmapmagbrain") + warp_workflow.connect(t_node, "outputspec.fieldmapmask", dataSink, "fieldmapmask") + warp_workflow.connect(t_node, "outputspec.fmap_despiked", dataSink, "fmap_despiked") + warp_workflow.connect(t_node, "outputspec.struct", dataSink, "epi2struct") + warp_workflow.connect(t_node, "outputspec.anat_func", dataSink, "anat_func") if run is True: - plugin_args = {'n_procs': num_of_cores} - warp_workflow.run(plugin=MultiProcPlugin(plugin_args), - plugin_args=plugin_args) + plugin_args = {"n_procs": num_of_cores} + warp_workflow.run(plugin=MultiProcPlugin(plugin_args), plugin_args=plugin_args) + return None # outpath = glob.glob( # os.path.join(workflow_dir, "EPI_DistCorr","*"))[0] # return outpath @@ -105,15 +102,19 @@ def run_warp_nipype(inputs, output_dir=None, run=True): return warp_workflow, warp_workflow.base_dir -if __name__ == '__main__': - run_warp_nipype([ - 'anat_file', - 'func_file', - 'fmap_pha', - 'fmap_mag', - 'deltaTE', - 'dwellT', - 'dwell_asym_ratio', - 'bet_frac', - 'bbr_schedule' - ], output_dir=None, run=True) +if __name__ == "__main__": + run_warp_nipype( + [ + "anat_file", + "func_file", + "fmap_pha", + "fmap_mag", + "deltaTE", + "dwellT", + "dwell_asym_ratio", + "bet_frac", + "bbr_schedule", + ], + output_dir=None, + run=True, + ) diff --git a/CPAC/connectome/connectivity_matrix.py b/CPAC/connectome/connectivity_matrix.py index 8e942b6c4c..552b9b16e9 100644 --- a/CPAC/connectome/connectivity_matrix.py +++ b/CPAC/connectome/connectivity_matrix.py @@ -19,29 +19,28 @@ """Functions for creating connectome connectivity matrices.""" import os from warnings import warn + import numpy as np from nilearn.connectome import ConnectivityMeasure from nipype import logging from nipype.interfaces import utility as util + from CPAC.pipeline import nipype_pipeline_engine as pe from CPAC.utils.interfaces.function import Function from CPAC.utils.interfaces.netcorr import NetCorr, strip_afni_output_header -logger = logging.getLogger('nipype.workflow') +logger = logging.getLogger("nipype.workflow") connectome_methods = { - 'afni': {'Pearson': '', - 'Partial': '-part_corr'}, - 'nilearn': {'Pearson': 'correlation', - 'Partial': 'partial correlation'} + "afni": {"Pearson": "", "Partial": "-part_corr"}, + "nilearn": {"Pearson": "correlation", "Partial": "partial correlation"}, } def connectome_name(atlas_name, tool, method): - """Helper function to create connectome file filename + """Helper function to create connectome file filename. Parameters ---------- - atlas_name : str atlas name @@ -55,13 +54,14 @@ def connectome_name(atlas_name, tool, method): ------- str """ - return os.path.join(os.getcwd(), '_'.join([ - f'atlas-{atlas_name}', f'desc-{tool}{method}', 'connectome.tsv' - ])) + return os.path.join( + os.getcwd(), + "_".join([f"atlas-{atlas_name}", f"desc-{tool}{method}", "connectome.tsv"]), + ) def get_connectome_method(method, tool): - """Helper function to get tool's method string + """Helper function to get tool's method string. Parameters ---------- @@ -84,8 +84,7 @@ def get_connectome_method(method, tool): """ cm_method = connectome_methods[tool.lower()].get(method, NotImplemented) if cm_method is NotImplemented: - warning_message = ( - f'{method} has not yet been implemented for {tool} in C-PAC.') + warning_message = f"{method} has not yet been implemented for {tool} in C-PAC." if logger: logger.warning(NotImplementedError(warning_message)) else: @@ -94,7 +93,7 @@ def get_connectome_method(method, tool): def compute_connectome_nilearn(in_rois, in_file, method, atlas_name): - """Function to compute a connectome matrix using Nilearn + """Function to compute a connectome matrix using Nilearn. Parameters ---------- @@ -116,106 +115,146 @@ def compute_connectome_nilearn(in_rois, in_file, method, atlas_name): """ from nilearn.input_data import NiftiLabelsMasker from nipype.utils.tmpdirs import TemporaryDirectory - tool = 'Nilearn' + + tool = "Nilearn" output = connectome_name(atlas_name, tool, method) method = get_connectome_method(method, tool) if method is NotImplemented: return NotImplemented with TemporaryDirectory() as cache_dir: - masker = NiftiLabelsMasker(labels_img=in_rois, - standardize=True, - verbose=True, - memory=cache_dir, - memory_level=3) + masker = NiftiLabelsMasker( + labels_img=in_rois, + standardize=True, + verbose=True, + memory=cache_dir, + memory_level=3, + ) timeser = masker.fit_transform(in_file) correlation_measure = ConnectivityMeasure(kind=method) corr_matrix = correlation_measure.fit_transform([timeser])[0] np.fill_diagonal(corr_matrix, 1) - np.savetxt(output, corr_matrix, delimiter='\t') + np.savetxt(output, corr_matrix, delimiter="\t") return output def create_connectome_afni(name, method, pipe_num): wf = pe.Workflow(name=name) inputspec = pe.Node( - util.IdentityInterface(fields=[ - 'in_rois', # parcellation - 'in_file', # timeseries, - 'mask', - 'method', - 'atlas_name' - ]), - name='inputspec' + util.IdentityInterface( + fields=[ + "in_rois", # parcellation + "in_file", # timeseries, + "mask", + "method", + "atlas_name", + ] + ), + name="inputspec", ) outputspec = pe.Node( - util.IdentityInterface(fields=[ - 'out_file', - ]), - name='outputspec' + util.IdentityInterface( + fields=[ + "out_file", + ] + ), + name="outputspec", ) timeseries_correlation = pe.Node(NetCorr(), name=name) if method: - timeseries_correlation.inputs.part_corr = (method == 'Partial') - - strip_header_node = pe.Node(Function( - input_names=['in_file', 'out_file'], output_names=['out_file'], - imports=['import subprocess'], - function=strip_afni_output_header), - name='netcorrStripHeader' - f'{method}_{pipe_num}') - - name_output_node = pe.Node(Function(input_names=['atlas_name', - 'tool', - 'method'], - output_names=['filename'], - imports=['import os'], - function=connectome_name), - name=f'connectomeName{method}_{pipe_num}', - as_module=True) - name_output_node.inputs.tool = 'Afni' - - wf.connect([ - (inputspec, timeseries_correlation, [('in_rois', 'in_rois'), - ('in_file', 'in_file'), - ('mask', 'mask')]), - (inputspec, name_output_node, [('atlas_name', 'atlas_name'), - ('method', 'method')]), - (timeseries_correlation, strip_header_node, [ - ('out_corr_matrix', 'in_file')]), - (name_output_node, strip_header_node, [('filename', 'out_file')]), - (strip_header_node, outputspec, [('out_file', 'out_file')])]) + timeseries_correlation.inputs.part_corr = method == "Partial" + + strip_header_node = pe.Node( + Function( + input_names=["in_file", "out_file"], + output_names=["out_file"], + imports=["import subprocess"], + function=strip_afni_output_header, + ), + name="netcorrStripHeader" f"{method}_{pipe_num}", + ) + + name_output_node = pe.Node( + Function( + input_names=["atlas_name", "tool", "method"], + output_names=["filename"], + imports=["import os"], + function=connectome_name, + ), + name=f"connectomeName{method}_{pipe_num}", + as_module=True, + ) + name_output_node.inputs.tool = "Afni" + + wf.connect( + [ + ( + inputspec, + timeseries_correlation, + [("in_rois", "in_rois"), ("in_file", "in_file"), ("mask", "mask")], + ), + ( + inputspec, + name_output_node, + [("atlas_name", "atlas_name"), ("method", "method")], + ), + ( + timeseries_correlation, + strip_header_node, + [("out_corr_matrix", "in_file")], + ), + (name_output_node, strip_header_node, [("filename", "out_file")]), + (strip_header_node, outputspec, [("out_file", "out_file")]), + ] + ) return wf -def create_connectome_nilearn(name='connectomeNilearn'): +def create_connectome_nilearn(name="connectomeNilearn"): wf = pe.Workflow(name=name) inputspec = pe.Node( - util.IdentityInterface(fields=[ - 'in_rois', # parcellation - 'in_file', # timeseries - 'method', - 'atlas_name' - ]), - name='inputspec' + util.IdentityInterface( + fields=[ + "in_rois", # parcellation + "in_file", # timeseries + "method", + "atlas_name", + ] + ), + name="inputspec", ) outputspec = pe.Node( - util.IdentityInterface(fields=[ - 'out_file', - ]), - name='outputspec' + util.IdentityInterface( + fields=[ + "out_file", + ] + ), + name="outputspec", + ) + node = pe.Node( + Function( + input_names=["in_rois", "in_file", "method", "atlas_name"], + output_names=["out_file"], + function=compute_connectome_nilearn, + as_module=True, + ), + name="connectome", + mem_gb=0.2, + mem_x=(6.7e-8, "in_file"), + ) + wf.connect( + [ + ( + inputspec, + node, + [ + ("in_rois", "in_rois"), + ("in_file", "in_file"), + ("method", "method"), + ("atlas_name", "atlas_name"), + ], + ), + (node, outputspec, [("out_file", "out_file")]), + ] ) - node = pe.Node(Function(input_names=['in_rois', 'in_file', 'method', - 'atlas_name'], - output_names=['out_file'], - function=compute_connectome_nilearn, - as_module=True), - name='connectome', mem_gb=0.2, mem_x=(6.7e-8, 'in_file')) - wf.connect([ - (inputspec, node, [('in_rois', 'in_rois'), - ('in_file', 'in_file'), - ('method', 'method'), - ('atlas_name', 'atlas_name')]), - (node, outputspec, [('out_file', 'out_file')]), - ]) return wf diff --git a/CPAC/connectome/test/test_connectome.py b/CPAC/connectome/test/test_connectome.py index 4bad1444f2..6c1b161c08 100644 --- a/CPAC/connectome/test/test_connectome.py +++ b/CPAC/connectome/test/test_connectome.py @@ -1,8 +1 @@ -"""Tests for CPAC.connectome""" -import os -import random -import re -import numpy as np -import pytest -from CPAC.pipeline.schema import valid_options - +"""Tests for CPAC.connectome.""" diff --git a/CPAC/cwas/__init__.py b/CPAC/cwas/__init__.py index e76132b61d..0e0cc1e2bd 100644 --- a/CPAC/cwas/__init__.py +++ b/CPAC/cwas/__init__.py @@ -1,7 +1,3 @@ -from .pipeline import joint_mask, \ - nifti_cwas, \ - create_cwas +from .pipeline import create_cwas, joint_mask, nifti_cwas -__all__ = ['create_cwas', - 'joint_mask', - 'nifti_cwas'] \ No newline at end of file +__all__ = ["create_cwas", "joint_mask", "nifti_cwas"] diff --git a/CPAC/cwas/cwas.py b/CPAC/cwas/cwas.py index a866773ff6..d6fe902bce 100644 --- a/CPAC/cwas/cwas.py +++ b/CPAC/cwas/cwas.py @@ -1,41 +1,41 @@ import os -import nibabel as nb import numpy as np +from numpy import inf import pandas as pd -import scipy.stats +import nibabel as nib from scipy.stats import t -from numpy import inf from CPAC.cwas.mdmr import mdmr +from CPAC.pipeline.cpac_ga_model_generator import ( + create_merge_mask, + create_merged_copefile, +) from CPAC.utils import correlation -from CPAC.pipeline.cpac_ga_model_generator import (create_merge_mask, - create_merged_copefile) - def joint_mask(subjects, mask_file=None): """ Creates a joint mask (intersection) common to all the subjects in a provided list - and a provided mask - + and a provided mask. + Parameters ---------- subjects : dict of strings A length `N` list of file paths of the nifti files of subjects mask_file : string Path to a mask file in nifti format - + Returns ------- joint_mask : string Path to joint mask file in nifti format - + """ if not mask_file: files = list(subjects.values()) - cope_file = os.path.join(os.getcwd(), 'joint_cope.nii.gz') - mask_file = os.path.join(os.getcwd(), 'joint_mask.nii.gz') + cope_file = os.path.join(os.getcwd(), "joint_cope.nii.gz") + mask_file = os.path.join(os.getcwd(), "joint_mask.nii.gz") create_merged_copefile(files, cope_file) create_merge_mask(cope_file, mask_file) return mask_file @@ -58,16 +58,17 @@ def calc_subdists(subjects_data, voxel_range): profiles = np.arctanh(np.delete(profiles, v, 1)) D[i] = correlation(profiles, profiles) - D = np.sqrt(2.0 * (1.0 - D)) - return D + return np.sqrt(2.0 * (1.0 - D)) -def calc_cwas(subjects_data, regressor, regressor_selected_cols, permutations, voxel_range): +def calc_cwas( + subjects_data, regressor, regressor_selected_cols, permutations, voxel_range +): D = calc_subdists(subjects_data, voxel_range) - F_set, p_set = calc_mdmrs( - D, regressor, regressor_selected_cols, permutations) + F_set, p_set = calc_mdmrs(D, regressor, regressor_selected_cols, permutations) return F_set, p_set + def pval_to_zval(p_set, permu): inv_pval = 1 - p_set zvals = t.ppf(inv_pval, (len(p_set) - 1)) @@ -75,11 +76,19 @@ def pval_to_zval(p_set, permu): zvals[zvals == inf] = permu / (permu + 1) return zvals -def nifti_cwas(subjects, mask_file, regressor_file, participant_column, - columns_string, permutations, voxel_range): + +def nifti_cwas( + subjects, + mask_file, + regressor_file, + participant_column, + columns_string, + permutations, + voxel_range, +): """ - Performs CWAS for a group of subjects - + Performs CWAS for a group of subjects. + Parameters ---------- subjects : dict of strings:strings @@ -95,7 +104,7 @@ def nifti_cwas(subjects, mask_file, regressor_file, participant_column, voxel_range : ndarray Indexes from range of voxels (inside the mask) to perform cwas on. Index ordering is based on the np.where(mask) command - + Returns ------- F_file : string @@ -105,26 +114,25 @@ def nifti_cwas(subjects, mask_file, regressor_file, participant_column, voxel_range : tuple Passed on by the voxel_range provided in parameters, used to make parallelization easier - + """ try: - regressor_data = pd.read_table(regressor_file, - sep=None, engine="python", - dtype={ participant_column: str }) + regressor_data = pd.read_table( + regressor_file, sep=None, engine="python", dtype={participant_column: str} + ) except: - regressor_data = pd.read_table(regressor_file, - sep=None, engine="python") - regressor_data = regressor_data.astype({ participant_column: str }) + regressor_data = pd.read_table(regressor_file, sep=None, engine="python") + regressor_data = regressor_data.astype({participant_column: str}) # drop duplicates regressor_data = regressor_data.drop_duplicates() regressor_cols = list(regressor_data.columns) - if not participant_column in regressor_cols: - raise ValueError('Participant column was not found in regressor file.') + if participant_column not in regressor_cols: + raise ValueError("Participant column was not found in regressor file.") if participant_column in columns_string: - raise ValueError('Participant column can not be a regressor.') + raise ValueError("Participant column can not be a regressor.") subject_ids = list(subjects.keys()) subject_files = list(subjects.values()) @@ -135,46 +143,47 @@ def nifti_cwas(subjects, mask_file, regressor_file, participant_column, for index, row in regressor_data.iterrows(): pheno_sub_id = str(row[participant_column]) for sub_id in subject_ids: - if str(sub_id).lstrip('0') == str(pheno_sub_id): + if str(sub_id).lstrip("0") == str(pheno_sub_id): regressor_data.at[index, participant_column] = str(sub_id) - + regressor_data.index = regressor_data[participant_column] # Keep only data from specific subjects ordered_regressor_data = regressor_data.loc[subject_ids] - columns = columns_string.split(',') - regressor_selected_cols = [ - i for i, c in enumerate(regressor_cols) if c in columns - ] + columns = columns_string.split(",") + regressor_selected_cols = [i for i, c in enumerate(regressor_cols) if c in columns] if len(regressor_selected_cols) == 0: regressor_selected_cols = [i for i, c in enumerate(regressor_cols)] regressor_selected_cols = np.array(regressor_selected_cols) # Remove participant id column from the dataframe and convert it to a numpy matrix - regressor = ordered_regressor_data \ - .drop(columns=[participant_column]) \ - .reset_index(drop=True) \ - .values \ - .astype(np.float64) + regressor = ( + ordered_regressor_data.drop(columns=[participant_column]) + .reset_index(drop=True) + .values.astype(np.float64) + ) if len(regressor.shape) == 1: regressor = regressor[:, np.newaxis] elif len(regressor.shape) != 2: - raise ValueError('Bad regressor shape: %s' % str(regressor.shape)) + raise ValueError("Bad regressor shape: %s" % str(regressor.shape)) if len(subject_files) != regressor.shape[0]: - raise ValueError('Number of subjects does not match regressor size') - mask = nb.load(mask_file).get_fdata().astype('bool') + raise ValueError("Number of subjects does not match regressor size") + mask = nib.load(mask_file).get_fdata().astype("bool") mask_indices = np.where(mask) - subjects_data = np.array([ - nb.load(subject_file).get_fdata().astype('float64')[mask_indices] - for subject_file in subject_files - ]) + subjects_data = np.array( + [ + nib.load(subject_file).get_fdata().astype("float64")[mask_indices] + for subject_file in subject_files + ] + ) - F_set, p_set = calc_cwas(subjects_data, regressor, regressor_selected_cols, - permutations, voxel_range) + F_set, p_set = calc_cwas( + subjects_data, regressor, regressor_selected_cols, permutations, voxel_range + ) cwd = os.getcwd() - F_file = os.path.join(cwd, 'pseudo_F.npy') - p_file = os.path.join(cwd, 'significance_p.npy') + F_file = os.path.join(cwd, "pseudo_F.npy") + p_file = os.path.join(cwd, "significance_p.npy") np.save(F_file, F_set) np.save(p_file, p_set) @@ -183,27 +192,23 @@ def nifti_cwas(subjects, mask_file, regressor_file, participant_column, def create_cwas_batches(mask_file, batches): - mask = nb.load(mask_file).get_fdata().astype('bool') + mask = nib.load(mask_file).get_fdata().astype("bool") voxels = mask.sum(dtype=int) return np.array_split(np.arange(voxels), batches) def volumize(mask_image, data): - mask_data = mask_image.get_fdata().astype('bool') + mask_data = mask_image.get_fdata().astype("bool") volume = np.zeros_like(mask_data, dtype=data.dtype) - volume[np.where(mask_data == True)] = data - return nb.Nifti1Image( - volume, - header=mask_image.header, - affine=mask_image.affine - ) + volume[np.where(mask_data is True)] = data + return nib.Nifti1Image(volume, header=mask_image.header, affine=mask_image.affine) def merge_cwas_batches(cwas_batches, mask_file, z_score, permutations): _, _, voxel_range = zip(*cwas_batches) voxels = np.array(np.concatenate(voxel_range)) - mask_image = nb.load(mask_file) + mask_image = nib.load(mask_file) F_set = np.zeros_like(voxels, dtype=np.float64) p_set = np.zeros_like(voxels, dtype=np.float64) @@ -220,10 +225,10 @@ def merge_cwas_batches(cwas_batches, mask_file, z_score, permutations): one_p_vol = volumize(mask_image, one_p_set) cwd = os.getcwd() - F_file = os.path.join(cwd, 'pseudo_F_volume.nii.gz') - p_file = os.path.join(cwd, 'p_significance_volume.nii.gz') - log_p_file = os.path.join(cwd, 'neglog_p_significance_volume.nii.gz') - one_p_file = os.path.join(cwd, 'one_minus_p_values.nii.gz') + F_file = os.path.join(cwd, "pseudo_F_volume.nii.gz") + p_file = os.path.join(cwd, "p_significance_volume.nii.gz") + log_p_file = os.path.join(cwd, "neglog_p_significance_volume.nii.gz") + one_p_file = os.path.join(cwd, "one_minus_p_values.nii.gz") F_vol.to_filename(F_file) p_vol.to_filename(p_file) @@ -238,13 +243,14 @@ def merge_cwas_batches(cwas_batches, mask_file, z_score, permutations): return F_file, p_file, log_p_file, one_p_file, z_file + def zstat_image(zvals, mask_file): - mask_image = nb.load(mask_file) + mask_image = nib.load(mask_file) z_vol = volumize(mask_image, zvals) cwd = os.getcwd() - z_file = os.path.join(cwd, 'zstat.nii.gz') - + z_file = os.path.join(cwd, "zstat.nii.gz") + z_vol.to_filename(z_file) return z_file diff --git a/CPAC/cwas/mdmr.py b/CPAC/cwas/mdmr.py index 7fb4854ad2..3667d490ff 100644 --- a/CPAC/cwas/mdmr.py +++ b/CPAC/cwas/mdmr.py @@ -1,61 +1,67 @@ import numpy as np + def check_rank(X): - k = X.shape[1] + k = X.shape[1] rank = np.linalg.matrix_rank(X) if rank < k: raise Exception("matrix is rank deficient (rank %i vs cols %i)" % (rank, k)) + def hat(X): Q1, _ = np.linalg.qr(X) return Q1.dot(Q1.T) + def gower(D): n = D.shape[0] - A = -0.5 * (D ** 2) + A = -0.5 * (D**2) I = np.eye(n, n) uno = np.ones((n, 1)) C = I - (1.0 / n) * uno.dot(uno.T) - G = C.dot(A).dot(C) - return G + return C.dot(A).dot(C) + def gen_h2(x, cols, indexperm): H = gen_h(x, cols, indexperm) other_cols = [i for i in range(x.shape[1]) if i not in cols] - Xj = x[:,other_cols] - H2 = H - hat(Xj) - return H2 + Xj = x[:, other_cols] + return H - hat(Xj) + def permute_design(x, cols, indexperm): Xj = x.copy() Xj[:, cols] = Xj[indexperm][:, cols] return Xj + def gen_h(x, cols, indexperm): x = permute_design(x, cols, indexperm) - H = hat(x) - return H + return hat(x) + def gen_h2_perms(x, cols, perms): nperms, nobs = perms.shape H2perms = np.zeros((nobs**2, nperms)) for i in range(nperms): - H2 = gen_h2(x, cols, perms[i,:]) - H2perms[:,i] = H2.flatten() + H2 = gen_h2(x, cols, perms[i, :]) + H2perms[:, i] = H2.flatten() return H2perms + def gen_ih_perms(x, cols, perms): nperms, nobs = perms.shape I = np.eye(nobs, nobs) - IHperms = np.zeros((nobs ** 2, nperms)) + IHperms = np.zeros((nobs**2, nperms)) for i in range(nperms): IH = I - gen_h(x, cols, perms[i, :]) IHperms[:, i] = IH.flatten() return IHperms + def calc_ssq_fast(Hs, Gs, transpose=True): if transpose: ssq = Hs.T.dot(Gs) @@ -63,35 +69,35 @@ def calc_ssq_fast(Hs, Gs, transpose=True): ssq = Hs.dot(Gs) return ssq + def ftest_fast(Hs, IHs, Gs, df_among, df_resid, **ssq_kwrds): SS_among = calc_ssq_fast(Hs, Gs, **ssq_kwrds) SS_resid = calc_ssq_fast(IHs, Gs, **ssq_kwrds) - F = (SS_among / df_among) / (SS_resid / df_resid) - return F + return (SS_among / df_among) / (SS_resid / df_resid) -def mdmr(D, X, columns, permutations): +def mdmr(D, X, columns, permutations): check_rank(X) - + subjects = X.shape[0] if subjects != D.shape[1]: raise Exception("# of subjects incompatible between X and D") - + voxels = D.shape[0] - Gs = np.zeros((subjects ** 2, voxels)) + Gs = np.zeros((subjects**2, voxels)) for di in range(voxels): Gs[:, di] = gower(D[di]).flatten() - + X1 = np.hstack((np.ones((subjects, 1)), X)) - columns = columns.copy() #removed a +1 + columns = columns.copy() # removed a +1 regressors = X1.shape[1] permutation_indexes = np.zeros((permutations, subjects), dtype=int) permutation_indexes[0, :] = range(subjects) for i in range(1, permutations): - permutation_indexes[i,:] = np.random.permutation(subjects) - + permutation_indexes[i, :] = np.random.permutation(subjects) + H2perms = gen_h2_perms(X1, columns, permutation_indexes) IHperms = gen_ih_perms(X1, columns, permutation_indexes) @@ -100,10 +106,7 @@ def mdmr(D, X, columns, permutations): F_perms = ftest_fast(H2perms, IHperms, Gs, df_among, df_resid) - p_vals = (F_perms[1:, :] >= F_perms[0, :]) \ - .sum(axis=0) \ - .astype('float') + p_vals = (F_perms[1:, :] >= F_perms[0, :]).sum(axis=0).astype("float") p_vals /= permutations return F_perms[0, :], p_vals - diff --git a/CPAC/cwas/pipeline.py b/CPAC/cwas/pipeline.py index 5dcab83340..62f1fce269 100644 --- a/CPAC/cwas/pipeline.py +++ b/CPAC/cwas/pipeline.py @@ -1,42 +1,37 @@ - import os -from CPAC.pipeline import nipype_pipeline_engine as pe + import nipype.interfaces.utility as util -from nipype import config +from CPAC.pipeline import nipype_pipeline_engine as pe from CPAC.utils.interfaces.function import Function - - from .cwas import ( - joint_mask, create_cwas_batches, + joint_mask, merge_cwas_batches, nifti_cwas, - zstat_image, ) -def create_cwas(name='cwas', working_dir=None, crash_dir=None): +def create_cwas(name="cwas", working_dir=None, crash_dir=None): """ - Connectome Wide Association Studies - + Connectome Wide Association Studies. + This workflow performs CWAS on a group of subjects. - + Parameters ---------- name : string, optional Name of the workflow. - + Returns ------- cwas : nipype.pipeline.engine.Workflow CWAS workflow. - + Notes ----- - Workflow Inputs:: - + inputspec.subjects : dict (subject id: nifti files) 4-D timeseries of a group of subjects normalized to MNI space inputspec.roi : string (nifti file) @@ -49,7 +44,7 @@ def create_cwas(name='cwas', working_dir=None, crash_dir=None): Number of permutation samples to draw from the pseudo F distribution inputspec.parallel_nodes : integer Number of nodes to create and potentially parallelize over - + Workflow Outputs:: outputspec.F_map : string (nifti file) @@ -57,16 +52,16 @@ def create_cwas(name='cwas', working_dir=None, crash_dir=None): outputspec.p_map : string (nifti file) Significance p values calculated from permutation tests outputspec.z_map : string (nifti file) - Significance p values converted to z-scores - + Significance p values converted to z-scores + CWAS Procedure: - + 1. Calculate spatial correlation of a voxel 2. Correlate spatial z-score maps for every subject pair 3. Convert matrix to distance matrix, `1-r` 4. Calculate MDMR statistics for the voxel 5. Determine significance of MDMR statistics with permutation tests - + .. exec:: from CPAC.cwas import create_cwas wf = create_cwas() @@ -76,131 +71,133 @@ def create_cwas(name='cwas', working_dir=None, crash_dir=None): ) Workflow Graph: - + .. image:: ../../images/generated/cwas.png :width: 500 - + Detailed Workflow Graph: - + .. image:: ../../images/generated/cwas_detailed.png :width: 500 - + References ---------- .. [1] Shehzad Z, Kelly C, Reiss PT, Cameron Craddock R, Emerson JW, McMahon K, Copland DA, Castellanos FX, Milham MP. A multivariate distance-based analytic framework for connectome-wide association studies. Neuroimage. 2014 Jun;93 Pt 1(0 1):74-94. doi: 10.1016/j.neuroimage.2014.02.024. Epub 2014 Feb 28. PMID: 24583255; PMCID: PMC4138049. - + """ if not working_dir: - working_dir = os.path.join(os.getcwd(), 'MDMR_work_dir') + working_dir = os.path.join(os.getcwd(), "MDMR_work_dir") if not crash_dir: - crash_dir = os.path.join(os.getcwd(), 'MDMR_crash_dir') + crash_dir = os.path.join(os.getcwd(), "MDMR_crash_dir") workflow = pe.Workflow(name=name) workflow.base_dir = working_dir - workflow.config['execution'] = {'hash_method': 'timestamp', - 'crashdump_dir': os.path.abspath(crash_dir), - 'crashfile_format': 'txt'} - - inputspec = pe.Node(util.IdentityInterface(fields=['roi', - 'subjects', - 'regressor', - 'participant_column', - 'columns', - 'permutations', - 'parallel_nodes', - 'z_score']), - name='inputspec') - - outputspec = pe.Node(util.IdentityInterface(fields=['F_map', - 'p_map', - 'neglog_p_map', - 'one_p_map', - 'z_map']), - name='outputspec') - - ccb = pe.Node(Function(input_names=['mask_file', - 'batches'], - output_names='batch_list', - function=create_cwas_batches, - as_module=True), - name='cwas_batches') - - ncwas = pe.MapNode(Function(input_names=['subjects', - 'mask_file', - 'regressor_file', - 'participant_column', - 'columns_string', - 'permutations', - 'voxel_range'], - output_names=['result_batch'], - function=nifti_cwas, - as_module=True), - name='cwas_batch', - iterfield='voxel_range') - - jmask = pe.Node(Function(input_names=['subjects', - 'mask_file'], - output_names=['joint_mask'], - function=joint_mask, - as_module=True), - name='joint_mask') - - mcwasb = pe.Node(Function(input_names=['cwas_batches', - 'mask_file', - 'z_score', - 'permutations'], - output_names=['F_file', - 'p_file', - 'neglog_p_file', - 'one_p_file', - 'z_file'], - function=merge_cwas_batches, - as_module=True), - name='cwas_volumes') - - #Compute the joint mask - workflow.connect(inputspec, 'subjects', - jmask, 'subjects') - workflow.connect(inputspec, 'roi', - jmask, 'mask_file') - - #Create batches based on the joint mask - workflow.connect(jmask, 'joint_mask', - ccb, 'mask_file') - workflow.connect(inputspec, 'parallel_nodes', - ccb, 'batches') - - #Compute CWAS over batches of voxels - workflow.connect(jmask, 'joint_mask', - ncwas, 'mask_file') - workflow.connect(inputspec, 'subjects', - ncwas, 'subjects') - workflow.connect(inputspec, 'regressor', - ncwas, 'regressor_file') - workflow.connect(inputspec, 'permutations', - ncwas, 'permutations') - workflow.connect(inputspec, 'participant_column', - ncwas, 'participant_column') - workflow.connect(inputspec, 'columns', - ncwas, 'columns_string') - - workflow.connect(ccb, 'batch_list', - ncwas, 'voxel_range') - - #Merge the computed CWAS data - workflow.connect(ncwas, 'result_batch', - mcwasb, 'cwas_batches') - workflow.connect(jmask, 'joint_mask', - mcwasb, 'mask_file') - workflow.connect(inputspec, 'z_score', - mcwasb, 'z_score') - workflow.connect(inputspec, 'permutations', - mcwasb, 'permutations') - - workflow.connect(mcwasb, 'F_file', outputspec, 'F_map') - workflow.connect(mcwasb, 'p_file', outputspec, 'p_map') - workflow.connect(mcwasb, 'neglog_p_file', outputspec, 'neglog_p_map') - workflow.connect(mcwasb, 'one_p_file', outputspec, 'one_p_map') - workflow.connect(mcwasb, 'z_file', outputspec, 'z_map') + workflow.config["execution"] = { + "hash_method": "timestamp", + "crashdump_dir": os.path.abspath(crash_dir), + "crashfile_format": "txt", + } + + inputspec = pe.Node( + util.IdentityInterface( + fields=[ + "roi", + "subjects", + "regressor", + "participant_column", + "columns", + "permutations", + "parallel_nodes", + "z_score", + ] + ), + name="inputspec", + ) + + outputspec = pe.Node( + util.IdentityInterface( + fields=["F_map", "p_map", "neglog_p_map", "one_p_map", "z_map"] + ), + name="outputspec", + ) + + ccb = pe.Node( + Function( + input_names=["mask_file", "batches"], + output_names="batch_list", + function=create_cwas_batches, + as_module=True, + ), + name="cwas_batches", + ) + + ncwas = pe.MapNode( + Function( + input_names=[ + "subjects", + "mask_file", + "regressor_file", + "participant_column", + "columns_string", + "permutations", + "voxel_range", + ], + output_names=["result_batch"], + function=nifti_cwas, + as_module=True, + ), + name="cwas_batch", + iterfield="voxel_range", + ) + + jmask = pe.Node( + Function( + input_names=["subjects", "mask_file"], + output_names=["joint_mask"], + function=joint_mask, + as_module=True, + ), + name="joint_mask", + ) + + mcwasb = pe.Node( + Function( + input_names=["cwas_batches", "mask_file", "z_score", "permutations"], + output_names=["F_file", "p_file", "neglog_p_file", "one_p_file", "z_file"], + function=merge_cwas_batches, + as_module=True, + ), + name="cwas_volumes", + ) + + # Compute the joint mask + workflow.connect(inputspec, "subjects", jmask, "subjects") + workflow.connect(inputspec, "roi", jmask, "mask_file") + + # Create batches based on the joint mask + workflow.connect(jmask, "joint_mask", ccb, "mask_file") + workflow.connect(inputspec, "parallel_nodes", ccb, "batches") + + # Compute CWAS over batches of voxels + workflow.connect(jmask, "joint_mask", ncwas, "mask_file") + workflow.connect(inputspec, "subjects", ncwas, "subjects") + workflow.connect(inputspec, "regressor", ncwas, "regressor_file") + workflow.connect(inputspec, "permutations", ncwas, "permutations") + workflow.connect(inputspec, "participant_column", ncwas, "participant_column") + workflow.connect(inputspec, "columns", ncwas, "columns_string") + + workflow.connect(ccb, "batch_list", ncwas, "voxel_range") + + # Merge the computed CWAS data + workflow.connect(ncwas, "result_batch", mcwasb, "cwas_batches") + workflow.connect(jmask, "joint_mask", mcwasb, "mask_file") + workflow.connect(inputspec, "z_score", mcwasb, "z_score") + workflow.connect(inputspec, "permutations", mcwasb, "permutations") + + workflow.connect(mcwasb, "F_file", outputspec, "F_map") + workflow.connect(mcwasb, "p_file", outputspec, "p_map") + workflow.connect(mcwasb, "neglog_p_file", outputspec, "neglog_p_map") + workflow.connect(mcwasb, "one_p_file", outputspec, "one_p_map") + workflow.connect(mcwasb, "z_file", outputspec, "z_map") return workflow diff --git a/CPAC/cwas/tests/X.csv b/CPAC/cwas/tests/X.csv index 35a03febe8..325dbaa158 100644 --- a/CPAC/cwas/tests/X.csv +++ b/CPAC/cwas/tests/X.csv @@ -497,4 +497,4 @@ -6.0016411338403,-0.0306110373950633,0.693594772092793 0.72644264287163,-0.892103149707071,-1.11329689892794 -0.502785117108356,1.06028679375368,-0.638378336988827 --0.769191795416678,0.670541523833665,-0.750911450744791 \ No newline at end of file +-0.769191795416678,0.670541523833665,-0.750911450744791 diff --git a/CPAC/cwas/tests/Y.csv b/CPAC/cwas/tests/Y.csv index e634a55f02..4ad6b4eb7c 100644 --- a/CPAC/cwas/tests/Y.csv +++ b/CPAC/cwas/tests/Y.csv @@ -497,4 +497,4 @@ -5.49743698379679,1.06476371365073,2.02929182493248,0.36101178725095,2.07158266359674,1.41066184124284,1.58472078057855,1.5680993932034,1.11069236710439,0.671411469197289 -0.673465564112587,-0.549773230876912,-0.37233368032179,-0.41387104831339,0.649072546411808,-0.0687630248171333,-0.552346312329938,-0.016128918042507,0.735792499227785,0.50277713688554 1.66191347310176,-0.121395569513754,-0.364046103819805,-1.06977422896796,-0.763443146363637,-1.58814893043643,0.328715088899028,-1.42100739994848,-1.44436319593838,-0.538179486683385 --0.382985118169075,-0.8048318876615,0.538835565795891,-1.87805520579645,-0.160051584385869,0.988250333453972,-1.04721019435067,-0.760466689366639,-1.17954042961936,-1.69802919614485 \ No newline at end of file +-0.382985118169075,-0.8048318876615,0.538835565795891,-1.87805520579645,-0.160051584385869,0.988250333453972,-1.04721019435067,-0.760466689366639,-1.17954042961936,-1.69802919614485 diff --git a/CPAC/cwas/tests/features/conn-dists_connectir.feature b/CPAC/cwas/tests/features/conn-dists_connectir.feature index 4acbb3e394..38993754f2 100755 --- a/CPAC/cwas/tests/features/conn-dists_connectir.feature +++ b/CPAC/cwas/tests/features/conn-dists_connectir.feature @@ -19,5 +19,3 @@ Scenario: Calculate distances for multiple voxels and we calculate distances for 12 seeds with numpy Then the cpac distances should be the same as numpy and the cpac distances should be like connectir - - diff --git a/CPAC/cwas/tests/features/conn_single.feature b/CPAC/cwas/tests/features/conn_single.feature index bdc2d0f4d0..27393edeb9 100755 --- a/CPAC/cwas/tests/features/conn_single.feature +++ b/CPAC/cwas/tests/features/conn_single.feature @@ -2,7 +2,7 @@ Feature: Connectivity via Pearson Correlations for a single subject @connectivity, @simulated Scenario: Correlation for simulated data - Given simulated time-series data + Given simulated time-series data When we norm the data and we compute the connectivity on normed data and we compute the connectivity with numpy diff --git a/CPAC/cwas/tests/features/steps/base_cwas.py b/CPAC/cwas/tests/features/steps/base_cwas.py index 87f91af8ed..ef5c93a3bf 100755 --- a/CPAC/cwas/tests/features/steps/base_cwas.py +++ b/CPAC/cwas/tests/features/steps/base_cwas.py @@ -4,7 +4,8 @@ def custom_corrcoef(X, Y=None): """Each of the columns in X will be correlated with each of the columns in Y. Each column represents a variable, with the rows - containing the observations.""" + containing the observations. + """ if Y is None: Y = X @@ -20,6 +21,4 @@ def custom_corrcoef(X, Y=None): xx = np.sum(X**2, axis=0) yy = np.sum(Y**2, axis=0) - r = np.dot(X.T, Y)/np.sqrt(np.multiply.outer(xx, yy)) - - return r + return np.dot(X.T, Y) / np.sqrt(np.multiply.outer(xx, yy)) diff --git a/CPAC/cwas/tests/features/steps/step_conn-dists_connectir.py b/CPAC/cwas/tests/features/steps/step_conn-dists_connectir.py index ea5d07c48f..b77d298bbb 100755 --- a/CPAC/cwas/tests/features/steps/step_conn-dists_connectir.py +++ b/CPAC/cwas/tests/features/steps/step_conn-dists_connectir.py @@ -1,78 +1,86 @@ from base_cwas import * -from pytest_bdd import scenario, given, when, then +from pytest_bdd import given, then, when + @given('a connectir-based distance folder "{sdir}"') def step(context, sdir): - bigmemory = importr('bigmemory') + importr("bigmemory") # Distances - sfile = op.join(sdir, "subdist.desc") - dmats = np.array(robjects.r("as.matrix(attach.big.matrix('%s'))" % sfile)) - context.nobs = np.sqrt(dmats.shape[0]) - context.nvoxs = dmats.shape[1] + sfile = op.join(sdir, "subdist.desc") + dmats = np.array(robjects.r("as.matrix(attach.big.matrix('%s'))" % sfile)) + context.nobs = np.sqrt(dmats.shape[0]) + context.nvoxs = dmats.shape[1] context.r_dmats = dmats.T.reshape(context.nvoxs, context.nobs, context.nobs) # Mask - mfile = op.join(sdir, "mask.nii.gz") - mask = nib.load(mfile).get_fdata().astype('bool') - mask_indices = np.where(mask) - context.mask = mask + mfile = op.join(sdir, "mask.nii.gz") + mask = nib.load(mfile).get_fdata().astype("bool") + mask_indices = np.where(mask) + context.mask = mask context.indices = mask_indices # Convert from column to row major format (aka take R data to python) - mfile = op.join(sdir, "mask_r2py.nii.gz") - inds_r2py = nib.load(mfile).get_fdata().astype('int') - inds_r2py = inds_r2py[inds_r2py.nonzero()] - 1 - context.r_dmats = context.r_dmats[inds_r2py,:,:] - + mfile = op.join(sdir, "mask_r2py.nii.gz") + inds_r2py = nib.load(mfile).get_fdata().astype("int") + inds_r2py = inds_r2py[inds_r2py.nonzero()] - 1 + context.r_dmats = context.r_dmats[inds_r2py, :, :] + + ## Done in step_connectivity_multiple_subjects.py # -#@given('the subject list "{sfile}"') -#def step(context, sfile): +# @given('the subject list "{sfile}"') +# def step(context, sfile): # context.slist = [ l.strip().strip('"') for l in open(sfile).readlines() ] # context.nsubs = len(context.slist) # -#@given('the subjects data are masked') -#def step(context): +# @given('the subjects data are masked') +# def step(context): # context.sdata = [ nib.load(sfile).get_fdata().astype('float64')[context.indices].T # for sfile in context.slist ] # context.sdata = np.array(context.sdata) # context.ntpts = context.sdata.shape[1] # context.nvoxs = context.sdata.shape[2] -@when('we calculate distances for {nseeds} seeds with cpac') + +@when("we calculate distances for {nseeds} seeds with cpac") def step(context, nseeds): - context.nseeds = int(nseeds) - context.seeds = range(context.nseeds) - context.dmats = calc_subdists(context.sdata, (0,context.nseeds)) + context.nseeds = int(nseeds) + context.seeds = range(context.nseeds) + context.dmats = calc_subdists(context.sdata, (0, context.nseeds)) + -@when('we calculate distances for {nseeds} seeds with numpy') +@when("we calculate distances for {nseeds} seeds with numpy") def step(context, nseeds): - context.nseeds = int(nseeds) - context.seeds = range(context.nseeds) - context.ref_dmats = np.zeros((context.nseeds, context.nsubs, context.nsubs)) + context.nseeds = int(nseeds) + context.seeds = range(context.nseeds) + context.ref_dmats = np.zeros((context.nseeds, context.nsubs, context.nsubs)) Smaps = np.zeros((context.nsubs, context.nseeds, context.nvoxs)) for i in range(context.nsubs): - x = custom_corrcoef(context.sdata[i][:,context.seeds].T, context.sdata[i]) - Smaps[i,:,:] = x + x = custom_corrcoef(context.sdata[i][:, context.seeds].T, context.sdata[i]) + Smaps[i, :, :] = x for j in range(context.nseeds): - S = Smaps[:,context.seeds[j],:] - S0 = np.delete(S, context.seeds[j], 1) - S0 = np.arctanh(S0) - D = 1 - np.corrcoef(S0) + S = Smaps[:, context.seeds[j], :] + S0 = np.delete(S, context.seeds[j], 1) + S0 = np.arctanh(S0) + D = 1 - np.corrcoef(S0) context.ref_dmats[j] = D -@then('the cpac distances should be the same as numpy') + +@then("the cpac distances should be the same as numpy") def step(context): comp = np.allclose(context.dmats, context.ref_dmats) assert_that(comp, "cpac vs numpy distances") -@then('the cpac distances should be like connectir') + +@then("the cpac distances should be like connectir") def step(context): import code + code.interact(local=locals()) - - dmats = context.dmats.reshape((context.nseeds, context.nsubs**2)).T - r_dmats = context.r_dmats.reshape((context.nvoxs, context.nsubs**2))[context.seeds,:].T - + + dmats = context.dmats.reshape((context.nseeds, context.nsubs**2)).T + r_dmats = context.r_dmats.reshape((context.nvoxs, context.nsubs**2))[ + context.seeds, : + ].T + corrs = np.diag(custom_corrcoef(dmats, r_dmats)) - comp = (corrs>0.99).all() + comp = (corrs > 0.99).all() assert_that(comp, "cpac vs R distances") - diff --git a/CPAC/cwas/tests/features/steps/step_conn_multiple.feature b/CPAC/cwas/tests/features/steps/step_conn_multiple.feature index 7d88bfb9f1..9a08fd04d5 100755 --- a/CPAC/cwas/tests/features/steps/step_conn_multiple.feature +++ b/CPAC/cwas/tests/features/steps/step_conn_multiple.feature @@ -45,4 +45,4 @@ def step(context): #@then('the correlation values for cpac should be like numpy') #def step(context): # comp = np.allclose(context.cmats, context.ref_cmats) -# assert_that(comp, "subject correlation") \ No newline at end of file +# assert_that(comp, "subject correlation") diff --git a/CPAC/cwas/tests/features/steps/step_conn_single.feature b/CPAC/cwas/tests/features/steps/step_conn_single.feature index 4a5888bd78..2ffd7b99d3 100755 --- a/CPAC/cwas/tests/features/steps/step_conn_single.feature +++ b/CPAC/cwas/tests/features/steps/step_conn_single.feature @@ -36,4 +36,4 @@ def step(context): @then('the correlation values for cpac should be like numpy') def step(context): comp = np.allclose(context.cmats, context.ref_cmats) - assert_that(comp, "subject correlations") \ No newline at end of file + assert_that(comp, "subject correlations") diff --git a/CPAC/cwas/tests/features/steps/step_dists.py b/CPAC/cwas/tests/features/steps/step_dists.py index e2a3f68665..fd00f38d60 100755 --- a/CPAC/cwas/tests/features/steps/step_dists.py +++ b/CPAC/cwas/tests/features/steps/step_dists.py @@ -1,27 +1,31 @@ from base_cwas import * -from pytest_bdd import scenario, given, when, then +from pytest_bdd import given, then, when -@given('simulated subjects connectivity data for {nseeds} seed voxels') + +@given("simulated subjects connectivity data for {nseeds} seed voxels") def step(context, nseeds): - context.nsubs = 10 - context.nseeds = int(nseeds) - context.nvoxs = 200 - context.cmats = np.random.random((context.nsubs, context.nseeds, context.nvoxs)) - context.cmats = 2*context.cmats - 1 + context.nsubs = 10 + context.nseeds = int(nseeds) + context.nvoxs = 200 + context.cmats = np.random.random((context.nsubs, context.nseeds, context.nvoxs)) + context.cmats = 2 * context.cmats - 1 + -@when('we compute the distances with cpac') +@when("we compute the distances with cpac") def step(context): - context.dmats = np.zeros((context.nseeds, context.nsubs, context.nsubs)) + context.dmats = np.zeros((context.nseeds, context.nsubs, context.nsubs)) for i in range(context.nseeds): - context.dmats[i] = compute_distances(context.cmats[:,i,:]) + context.dmats[i] = compute_distances(context.cmats[:, i, :]) -@when('we compute the distances with numpy') + +@when("we compute the distances with numpy") def step(context): - context.ref_dmats = np.zeros((context.nseeds, context.nsubs, context.nsubs)) + context.ref_dmats = np.zeros((context.nseeds, context.nsubs, context.nsubs)) for i in range(context.nseeds): - context.ref_dmats[i] = 1 - np.corrcoef(context.cmats[:,i,:]) + context.ref_dmats[i] = 1 - np.corrcoef(context.cmats[:, i, :]) + -@then('the cpac distances should be like numpy') +@then("the cpac distances should be like numpy") def step(context): comp = np.allclose(context.dmats, context.ref_dmats) - assert_that(comp, "distances") \ No newline at end of file + assert_that(comp, "distances") diff --git a/CPAC/cwas/tests/features/steps/step_mdmrs_connectir.py b/CPAC/cwas/tests/features/steps/step_mdmrs_connectir.py index dfe49cf785..5d915d51e4 100755 --- a/CPAC/cwas/tests/features/steps/step_mdmrs_connectir.py +++ b/CPAC/cwas/tests/features/steps/step_mdmrs_connectir.py @@ -1,76 +1,91 @@ from base_cwas import * -import code -from pytest_bdd import scenario, given, when, then +from pytest_bdd import given, then, when + @given('a connectir-based MDMR folder "{mdir}" looking at the factor "{factor}"') def step(context, mdir, factor): # Distances - sfile = op.join(op.dirname(mdir), "subdist.desc") - bigmemory = importr('bigmemory') - dmats = np.array(robjects.r("as.matrix(attach.big.matrix('%s'))" % sfile)) - context.nobs = np.sqrt(dmats.shape[0]) + sfile = op.join(op.dirname(mdir), "subdist.desc") + importr("bigmemory") + dmats = np.array(robjects.r("as.matrix(attach.big.matrix('%s'))" % sfile)) + context.nobs = np.sqrt(dmats.shape[0]) context.nvoxs = dmats.shape[1] context.dmats = dmats.T.reshape(context.nvoxs, context.nobs, context.nobs) # Fperms - ffile = op.join(mdir, "fperms_%s.desc" % factor) - context.r_F_perms = np.array(robjects.r("as.matrix(attach.big.matrix('%s'))" % ffile)) + ffile = op.join(mdir, "fperms_%s.desc" % factor) + context.r_F_perms = np.array( + robjects.r("as.matrix(attach.big.matrix('%s'))" % ffile) + ) # Fstats - context.r_Fs = context.r_F_perms[0,:] + context.r_Fs = context.r_F_perms[0, :] # Pvalues - pfile = op.join(mdir, "pvals.desc") - pvals = np.array(robjects.r("as.matrix(attach.big.matrix('%s'))" % pfile)) + pfile = op.join(mdir, "pvals.desc") + pvals = np.array(robjects.r("as.matrix(attach.big.matrix('%s'))" % pfile)) if len(pvals.shape) > 1: - mfile = op.join(mdir, "modelinfo.rda") + mfile = op.join(mdir, "modelinfo.rda") robjects.r("load(%s)" % mfile) - fnames = np.array(robjects.r("names(modelinfo$H2s)")) - fi = (fnames == factor).nonzero()[0] - pvals = pvals[:,fi] - context.r_ps = pvals + fnames = np.array(robjects.r("names(modelinfo$H2s)")) + fi = (fnames == factor).nonzero()[0] + pvals = pvals[:, fi] + context.r_ps = pvals # Permutations - pfile = op.join(mdir, "perms_%s.desc" % factor) - perms = np.array(robjects.r("as.matrix(attach.big.matrix('%s'))" % pfile)) - context.perms = (perms.T - 1)[1:,:] + pfile = op.join(mdir, "perms_%s.desc" % factor) + perms = np.array(robjects.r("as.matrix(attach.big.matrix('%s'))" % pfile)) + context.perms = (perms.T - 1)[1:, :] + -@given('{nperms} permutations') +@given("{nperms} permutations") def step(context, nperms): context.perms = int(nperms) + @given('regressors from "{rfile}" with columns of interest {cols}') def step(context, rfile, cols): context.regressors = np.loadtxt(rfile) context.cols = eval(cols) -@when('we compute many mdmrs in python') + +@when("we compute many mdmrs in python") def step(context): - context.voxs = range(12) - context.Fs, context.ps = calc_mdmrs(context.dmats[context.voxs], context.regressors, - context.cols, context.perms) + context.voxs = range(12) + context.Fs, context.ps = calc_mdmrs( + context.dmats[context.voxs], context.regressors, context.cols, context.perms + ) + -@when('we compute a single MDMR in python') +@when("we compute a single MDMR in python") def step(context): context.voxs = [0] - ps, Fs, F_perms, _ = mdmr(context.dmats[0].reshape(context.nobs**2,1), context.regressors, - context.cols, context.perms) + ps, Fs, F_perms, _ = mdmr( + context.dmats[0].reshape(context.nobs**2, 1), + context.regressors, + context.cols, + context.perms, + ) context.ps = ps context.Fs = Fs context.F_perms = F_perms -@then('the many pseudo-F values should be like R') + +@then("the many pseudo-F values should be like R") def step(context): comp = np.allclose(context.Fs, context.r_Fs[context.voxs]) assert_that(comp, "pseudo-F stats") -@then('the many F permutations should be like R') + +@then("the many F permutations should be like R") def step(context): - comp = np.allclose(context.F_perms, context.r_F_perms[:,context.voxs]) + comp = np.allclose(context.F_perms, context.r_F_perms[:, context.voxs]) assert_that(comp, "F permutations") -@then('the many p-values should be like R') + +@then("the many p-values should be like R") def step(context): comp = np.allclose(context.ps, context.r_ps[context.voxs]) assert_that(comp, "p-values") -@then('the many p-values should be similar to R') + +@then("the many p-values should be similar to R") def step(context): - comp = np.corrcoef(context.ps, context.r_ps[context.voxs])[0,1] + comp = np.corrcoef(context.ps, context.r_ps[context.voxs])[0, 1] assert_that(comp, greater_than(0.98), "p-values") diff --git a/CPAC/cwas/tests/test_cwas.py b/CPAC/cwas/tests/test_cwas.py index aab2833560..2e094bd8b1 100755 --- a/CPAC/cwas/tests/test_cwas.py +++ b/CPAC/cwas/tests/test_cwas.py @@ -1,30 +1,34 @@ import pytest + from CPAC.pipeline.nipype_pipeline_engine.plugins import MultiProcPlugin -@pytest.mark.skip(reason='requires RegressionTester') +@pytest.mark.skip(reason="requires RegressionTester") def test_adhd04(): - rt = RegressionTester('adhd04', 'diagnosis', 'diagnosis') + rt = RegressionTester("adhd04", "diagnosis", "diagnosis") rt.run() -@pytest.mark.skip(reason='requires RegressionTester') +@pytest.mark.skip(reason="requires RegressionTester") def test_adhd40(): - rt = RegressionTester('adhd40', 'diagnosis', - 'diagnosis + age + sex + meanFD') + rt = RegressionTester("adhd40", "diagnosis", "diagnosis + age + sex + meanFD") rt.run() -@pytest.mark.skip(reason='requires specific local directory') +@pytest.mark.skip(reason="requires specific local directory") class RegressionTester(object): """ tmp = RegressionTester('adhd04', 'diagnosis', 'diagnosis') - tmp.run() + tmp.run(). """ - def __init__(self, name, factor, formula, - base="/home2/data/Projects/CPAC_Regression_Test/" - "2013-05-30_cwas"): + def __init__( + self, + name, + factor, + formula, + base="/home2/data/Projects/CPAC_Regression_Test/" "2013-05-30_cwas", + ): super(RegressionTester, self).__init__() self.base = base self.name = name @@ -32,23 +36,23 @@ def __init__(self, name, factor, formula, self.formula = formula def run(self): - print("Python-Based CWAS") self.run_cwas() - print("R-Based CWAS") self.run_connectir() - print("Compare Python and R") self.compare_py_vs_r() def run_cwas(self): import os + os.chdir("%s/C-PAC" % self.base) - from CPAC.cwas import create_cwas - import numpy as np - import time from os import path as op + import time + + import numpy as np + + from CPAC.cwas import create_cwas ### # Paths and Other Inputs @@ -57,11 +61,10 @@ def run_cwas(self): ### # File with initial grey matter mask - roi_file = op.join(self.base, 'rois/grey_matter_4mm.nii.gz') + roi_file = op.join(self.base, "rois/grey_matter_4mm.nii.gz") # File with list of subject functionals - sfile = op.join(self.base, "configs", - "%s_funcpaths_4mm.txt" % self.name) + sfile = op.join(self.base, "configs", "%s_funcpaths_4mm.txt" % self.name) # File with regressors rfile = op.join(self.base, "configs", "%s_regressors.txt" % self.name) @@ -78,8 +81,8 @@ def run_cwas(self): # Read in list of subject functionals subjects_list = [ - l.strip().strip('"') for # noqa: E741 - l in open(sfile).readlines() # pylint: disable=consider-using-with + l.strip().strip('"') + for l in open(sfile).readlines() # pylint: disable=consider-using-with ] # Read in design/regressor file @@ -94,7 +97,7 @@ def run_cwas(self): c.inputs.inputspec.f_samples = nperms c.inputs.inputspec.parallel_nodes = 4 # c.base_dir = op.join(obase, 'results_fs%i_pn%i' % \ - # (c.inputs.inputspec.f_samples, c.inputs.inputspec.parallel_nodes)) # noqa: E501 # pylint: disable=line-too-long + # (c.inputs.inputspec.f_samples, c.inputs.inputspec.parallel_nodes)) # pylint: disable=line-too-long c.base_dir = op.join(self.base, "results_%s.py" % self.name) # export MKL_NUM_THREADS=X # in command line @@ -108,11 +111,10 @@ def run_cwas(self): # pass # Run it! - start = time.clock() - plugin_args = {'n_procs': 4} + time.clock() + plugin_args = {"n_procs": 4} c.run(plugin=MultiProcPlugin(plugin_args), plugin_args=plugin_args) - end = time.clock() - print("time: %.2gs" % (end-start)) + time.clock() def run_connectir(self): """ @@ -122,50 +124,52 @@ def run_connectir(self): import os import time - start = time.clock() + time.clock() - print("Subject Distances") - cmd = "connectir_subdist.R --infuncs1 %(basedir)s/configs/" \ - "%(outbase)s_funcpaths_4mm.txt --brainmask1 %(basedir)s/" \ - "results_%(outbase)s.py/cwas/joint_mask/joint_mask.nii.gz " \ - "--bg %(basedir)s/rois/standard_4mm.nii.gz --memlimit 12 " \ - "--forks 1 --threads 12 %(basedir)s/results_%(outbase)s.r" % { - 'basedir': self.base, 'outbase': self.name} - print("RUNNING: %s" % cmd) + cmd = ( + "connectir_subdist.R --infuncs1 %(basedir)s/configs/" + "%(outbase)s_funcpaths_4mm.txt --brainmask1 %(basedir)s/" + "results_%(outbase)s.py/cwas/joint_mask/joint_mask.nii.gz " + "--bg %(basedir)s/rois/standard_4mm.nii.gz --memlimit 12 " + "--forks 1 --threads 12 %(basedir)s/results_%(outbase)s.r" + % {"basedir": self.base, "outbase": self.name} + ) os.system(cmd) - print("MDMR") - cmd = "connectir_mdmr.R --indir %(basedir)s/results_%(outbase)s.r " \ - "--formula '%(formula)s' --model " \ - "%(basedir)s/configs/%(outbase)s_model.csv --permutations " \ - "%(nperms)s --factors2perm '%(factor)s' --save-perms " \ - "--memlimit 12 --forks 1 --threads 12 %(factor)s.mdmr" % { - 'basedir': self.base, 'outbase': self.name, - 'formula': self.formula, 'factor': self.factor, - 'nperms': 1000} - print("RUNNING: %s" % cmd) + cmd = ( + "connectir_mdmr.R --indir %(basedir)s/results_%(outbase)s.r " + "--formula '%(formula)s' --model " + "%(basedir)s/configs/%(outbase)s_model.csv --permutations " + "%(nperms)s --factors2perm '%(factor)s' --save-perms " + "--memlimit 12 --forks 1 --threads 12 %(factor)s.mdmr" + % { + "basedir": self.base, + "outbase": self.name, + "formula": self.formula, + "factor": self.factor, + "nperms": 1000, + } + ) os.system(cmd) - end = time.clock() - print("time: %.2gs" % (end-start)) + time.clock() @pytest.mark.skip(reason="No R installation in C-PAC image") def compare_py_vs_r(self): - """ - This will compare the output from the CPAC python vs the R connectir - """ + """This will compare the output from the CPAC python vs the R connectir.""" import os + os.chdir("%s/C-PAC" % self.base) - import numpy as np from os import path as op - import nibabel as nib - import rpy2.robjects as robjects + + import numpy as np + from rpy2 import robjects from rpy2.robjects.numpy2ri import numpy2ri from rpy2.robjects.packages import importr - robjects.conversion.py2ri = numpy2ri + import nibabel as nib - import code + robjects.conversion.py2ri = numpy2ri from hamcrest import assert_that @@ -175,11 +179,15 @@ def compare_py_vs_r(self): # however, we aren't there yet ### - mask_file = op.join(self.base, "results_%s.py" % self.name, "cwas", - "joint_mask", "joint_mask.nii.gz") + mask_file = op.join( + self.base, + "results_%s.py" % self.name, + "cwas", + "joint_mask", + "joint_mask.nii.gz", + ) - pybase = op.join(self.base, "results_%s.py" % self.name, "cwas", - "cwas_volumes") + pybase = op.join(self.base, "results_%s.py" % self.name, "cwas", "cwas_volumes") py_fs_file = op.join(pybase, "pseudo_F_volume.nii.gz") py_ps_file = op.join(pybase, "p_significance_volume.nii.gz") @@ -189,30 +197,28 @@ def compare_py_vs_r(self): r_fs_file = op.join(rbase, "fperms_%s.desc" % self.factor) r_ps_file = op.join(rbase, "pvals.desc") - ### # Get regressors and represent as hat matrices ### from pandas import read_csv + from CPAC.cwas.hats import hatify x = np.loadtxt(op.join(self.base, "configs", "%s_regressors.txt" % self.name)) py_hat = hatify(x) y = read_csv(op.join(rbase, "model_evs.txt")) - r_hat = hatify(y.ix[:,1:]) - + r_hat = hatify(y.ix[:, 1:]) ### # Create R=>Py Indices ### mfile = conv_file - inds_r2py = nib.load(mfile).get_fdata().astype('int') + inds_r2py = nib.load(mfile).get_fdata().astype("int") inds_r2py = inds_r2py[inds_r2py.nonzero()] - 1 - ### # Read in data ### @@ -222,7 +228,7 @@ def compare_py_vs_r(self): py_fs = nib.load(py_fs_file).get_fdata()[mask] py_ps = nib.load(py_ps_file).get_fdata()[mask] - bigmemory = importr('bigmemory') + importr("bigmemory") r_fs = np.array(robjects.r("attach.big.matrix('%s')[1,]" % r_fs_file)) r_ps = np.array(robjects.r("as.matrix(attach.big.matrix('%s'))" % r_ps_file)) @@ -233,10 +239,10 @@ def compare_py_vs_r(self): comp = np.allclose(py_hat, r_hat) assert_that(comp, "regressors as hat matrices") - comp = np.corrcoef(py_fs, r_fs[inds_r2py])[0,1] > 0.99 + comp = np.corrcoef(py_fs, r_fs[inds_r2py])[0, 1] > 0.99 assert_that(comp, "Fstats similarity") - comp = np.corrcoef(py_ps, r_ps[inds_r2py])[0,1] > 0.99 + comp = np.corrcoef(py_ps, r_ps[inds_r2py])[0, 1] > 0.99 assert_that(comp, "p-values similarity ") comp = abs(py_fs - r_fs[inds_r2py]).mean() < 0.01 @@ -245,8 +251,6 @@ def compare_py_vs_r(self): comp = abs(py_ps - r_ps[inds_r2py]).mean() < 0.05 assert_that(comp, "p-values difference") - print("tests were all good") - def test_cwas_connectir(): # add the code to run the same cwas with connectir @@ -255,71 +259,77 @@ def test_cwas_connectir(): @pytest.mark.skip(reason="No R installation in C-PAC image") def test_mdmr_with_connectir_distances(): - """uses the distances from output of connectir to specifically test the mdmr portion of things""" + """Uses the distances from output of connectir to specifically test the mdmr portion of things.""" import os + os.chdir("../C-PAC") - from CPAC.cwas.mdmr import mdmr - import numpy as np from os import path as op - import rpy2.robjects as robjects + + import numpy as np + from rpy2 import robjects from rpy2.robjects.numpy2ri import numpy2ri from rpy2.robjects.packages import importr + + from CPAC.cwas.mdmr import mdmr + robjects.conversion.py2ri = numpy2ri - from pandas import read_csv - bigmemory = importr('bigmemory') - base = importr('base') + importr("bigmemory") + importr("base") - sdir = '/home/data/Projects/CPAC_Regression_Test/2013-05-30_cwas/results_adhd04.r' + sdir = "/home/data/Projects/CPAC_Regression_Test/2013-05-30_cwas/results_adhd04.r" sfile = op.join(sdir, "subdist.desc") dmats = np.array(robjects.r("as.matrix(attach.big.matrix('%s'))" % sfile)) - n = np.sqrt(dmats.shape[0]) + np.sqrt(dmats.shape[0]) rfile = "/home2/data/Projects/CPAC_Regression_Test/2013-05-30_cwas/configs/adhd04_regressors.txt" regressors = np.loadtxt(rfile) - ps, Fs, _, _ = mdmr(dmats[:,:10], regressors, [1], 1000) + ps, Fs, _, _ = mdmr(dmats[:, :10], regressors, [1], 1000) @pytest.mark.skip(reason="dmats is undefined ") def test_distances(): + from os import path as op + import numpy as np from pandas import read_csv - from os import path as op n = 10 - roi_file = '/home2/data/Projects/CWAS/nki/rois/grey_matter_4mm.nii.gz' - - sdir = '/home/data/Projects/Z_CPAC_Regression_Test/2013-05-30_cwas/results_nki.r' + sdir = "/home/data/Projects/Z_CPAC_Regression_Test/2013-05-30_cwas/results_nki.r" mask_file = op.join(sdir, "mask.nii.gz") sfile = "/home2/data/Projects/CWAS/share/nki/subinfo/40_Set1_N104/short_compcor_funcpaths_4mm_smoothed.txt" - subjects_list = [ l.strip().strip('"') for l in open(sfile).readlines() ] + subjects_list = [l.strip().strip('"') for l in open(sfile).readlines()] subjects_list = subjects_list[:n] subjects_file_list = subjects_list - mask = nb.load(mask_file).get_fdata().astype('bool') + mask = nb.load(mask_file).get_fdata().astype("bool") mask_indices = np.where(mask) - subjects_data = [ nb.load(subject_file).get_fdata().astype('float64')[mask_indices].T - for subject_file in subjects_file_list ] + subjects_data = [ + nb.load(subject_file).get_fdata().astype("float64")[mask_indices].T + for subject_file in subjects_file_list + ] subjects_data = np.array(subjects_data) rfile = "/home2/data/Projects/CWAS/share/nki/subinfo/40_Set1_N104/subject_info_with_iq_and_gcors.csv" df = read_csv(rfile) - regressors = np.array([ - np.ones(n), - df.FSIQ[:n] - df.FSIQ[:n].mean(), - df.short_meanGcor[:n] - df.short_meanGcor[:n].mean(), - df.Age[:n] - df.Age[:n].mean(), - (df.Sex[:n] == "M")*1 - ]).T + np.array( + [ + np.ones(n), + df.FSIQ[:n] - df.FSIQ[:n].mean(), + df.short_meanGcor[:n] - df.short_meanGcor[:n].mean(), + df.Age[:n] - df.Age[:n].mean(), + (df.Sex[:n] == "M") * 1, + ] + ).T # run parts of the subdist code base to get... - sdir = '/home/data/Projects/Z_CPAC_Regression_Test/2013-05-30_cwas/results_nki.r' + sdir = "/home/data/Projects/Z_CPAC_Regression_Test/2013-05-30_cwas/results_nki.r" ffile = op.join(sdir, "iq_meanFD+age+sex.mdmr", "fperms_FSIQ.desc") - fperms = np.array(robjects.r("as.matrix(attach.big.matrix('%s'))" % ffile)) + np.array(robjects.r("as.matrix(attach.big.matrix('%s'))" % ffile)) n = np.sqrt(dmats.shape[0]) diff --git a/CPAC/cwas/tests/test_mdmr_cython.py b/CPAC/cwas/tests/test_mdmr_cython.py index ae382e9c6a..16415f9720 100644 --- a/CPAC/cwas/tests/test_mdmr_cython.py +++ b/CPAC/cwas/tests/test_mdmr_cython.py @@ -1,20 +1,19 @@ import os + import pytest -@pytest.mark.skip(reason='possibly deprecated') +@pytest.mark.skip(reason="possibly deprecated") def test_mdmr(): + import numpy as np - from CPAC.cwas.mdmr import mdmr from CPAC.cwas.cwas import calc_cwas - import numpy as np - X = np.genfromtxt(os.path.join(os.path.dirname(__file__), 'X.csv'), delimiter=',') - Y = np.genfromtxt(os.path.join(os.path.dirname(__file__), 'Y.csv'), delimiter=',') + X = np.genfromtxt(os.path.join(os.path.dirname(__file__), "X.csv"), delimiter=",") + Y = np.genfromtxt(os.path.join(os.path.dirname(__file__), "Y.csv"), delimiter=",") X = X.reshape((X.shape[0], X.shape[1], 1)) - F_value, p_value = calc_cwas( - X, Y, np.array([0, 1, 2], dtype=int), 1000, [0]) + F_value, p_value = calc_cwas(X, Y, np.array([0, 1, 2], dtype=int), 1000, [0]) assert np.isclose(p_value.mean(), 1.0, rtol=0.1) diff --git a/CPAC/cwas/tests/test_pipeline_cwas.py b/CPAC/cwas/tests/test_pipeline_cwas.py index f026b38318..5db2e377e2 100644 --- a/CPAC/cwas/tests/test_pipeline_cwas.py +++ b/CPAC/cwas/tests/test_pipeline_cwas.py @@ -1,57 +1,51 @@ import os from urllib.error import URLError -import nibabel as nb -import nilearn.datasets import numpy as np import pandas as pd import pytest +import nibabel as nib +import nilearn.datasets from CPAC.cwas.pipeline import create_cwas -@pytest.mark.parametrize('z_score', [[0], [1], [0, 1], []]) + +@pytest.mark.parametrize("z_score", [[0], [1], [0, 1], []]) def test_pipeline(z_score): try: # pylint: disable=invalid-name cc = nilearn.datasets.fetch_atlas_craddock_2012() except URLError: - print('Could not fetch atlas, skipping test') return try: - os.mkdir('/tmp/cwas') + os.mkdir("/tmp/cwas") except: pass abide_data = nilearn.datasets.fetch_abide_pcp(n_subjects=10) - pheno = pd.DataFrame.from_records(abide_data['phenotypic']) - images = abide_data['func_preproc'] + pheno = pd.DataFrame.from_records(abide_data["phenotypic"]) + images = abide_data["func_preproc"] - pheno = pheno[['FILE_ID', 'AGE_AT_SCAN', 'FIQ']] + pheno = pheno[["FILE_ID", "AGE_AT_SCAN", "FIQ"]] - pheno.to_csv('/tmp/cwas/pheno.csv') + pheno.to_csv("/tmp/cwas/pheno.csv") # Sanity ordering check - assert all( - FID in images[i] - for i, FID in enumerate(pheno.FILE_ID) - ) - img = nb.load(cc['scorr_mean']) + assert all(FID in images[i] for i, FID in enumerate(pheno.FILE_ID)) + img = nib.load(cc["scorr_mean"]) img_data = np.copy(img.get_fdata()[:, :, :, 10]) img_data[img_data != 2] = 0.0 - img = nb.Nifti1Image(img_data, img.affine) - nb.save(img, '/tmp/cwas/roi.nii.gz') - - workflow = create_cwas('cwas', '/tmp/cwas', '/tmp/cwas') + img = nib.Nifti1Image(img_data, img.affine) + nib.save(img, "/tmp/cwas/roi.nii.gz") + + workflow = create_cwas("cwas", "/tmp/cwas", "/tmp/cwas") - subjects = { - FID: images[i] - for i, FID in enumerate(pheno.FILE_ID) - } + subjects = {FID: images[i] for i, FID in enumerate(pheno.FILE_ID)} - roi = '/tmp/cwas/roi.nii.gz' - regressor_file = '/tmp/cwas/pheno.csv' - participant_column = 'FILE_ID' - columns = 'AGE_AT_SCAN' + roi = "/tmp/cwas/roi.nii.gz" + regressor_file = "/tmp/cwas/pheno.csv" + participant_column = "FILE_ID" + columns = "AGE_AT_SCAN" permutations = 50 parallel_nodes = 3 @@ -64,4 +58,4 @@ def test_pipeline(z_score): workflow.inputs.inputspec.parallel_nodes = parallel_nodes workflow.inputs.inputspec.z_score = z_score - workflow.run(plugin='Linear') + workflow.run(plugin="Linear") diff --git a/CPAC/distortion_correction/distortion_correction.py b/CPAC/distortion_correction/distortion_correction.py index 5221cb72d5..516fc3a9c5 100644 --- a/CPAC/distortion_correction/distortion_correction.py +++ b/CPAC/distortion_correction/distortion_correction.py @@ -18,36 +18,28 @@ # License along with C-PAC. If not, see . import os import subprocess -from CPAC.pipeline.nodeblock import nodeblock - -import nibabel as nb -from CPAC.pipeline import nipype_pipeline_engine as pe -from nipype.interfaces import afni, fsl -import nipype.interfaces.utility as util -import nipype.interfaces.ants as ants -import nipype.interfaces.afni.preprocess as preprocess +import nibabel as nib +from nipype.interfaces import afni, ants, fsl +from nipype.interfaces.afni import preprocess import nipype.interfaces.afni.utils as afni_utils +import nipype.interfaces.utility as util -from CPAC.pipeline.engine import wrap_block - +from CPAC.distortion_correction.utils import ( + choose_phase_image, + find_vnum_base, + phase_encode, + run_fsl_topup, +) +from CPAC.pipeline import nipype_pipeline_engine as pe +from CPAC.pipeline.nodeblock import nodeblock from CPAC.utils import function -from CPAC.utils.interfaces.function import Function from CPAC.utils.datasource import match_epi_fmaps - -from CPAC.func_preproc.func_preproc import bold_mask_afni, bold_masking - -from CPAC.distortion_correction.utils import run_convertwarp, \ - phase_encode, \ - run_fsl_topup, \ - z_pad, \ - choose_phase_image, \ - find_vnum_base +from CPAC.utils.interfaces.function import Function def create_afni_arg(shrink_fac): - expr = '-shrink_fac {0} '.format(shrink_fac) - return expr + return "-shrink_fac {0} ".format(shrink_fac) @nodeblock( @@ -70,7 +62,7 @@ def create_afni_arg(shrink_fac): outputs=["despiked-fieldmap", "fieldmap-mask"], ) def distcor_phasediff_fsl_fugue(wf, cfg, strat_pool, pipe_num, opt=None): - ''' + """ Fieldmap correction takes in an input magnitude image which is Skull Stripped (Tight). The magnitude images are obtained from each echo series. It also @@ -130,115 +122,125 @@ def distcor_phasediff_fsl_fugue(wf, cfg, strat_pool, pipe_num, opt=None): in_file = field map which is a 4D image (containing 2 unwarped image) - ''' - + """ # Skull-strip, outputs a masked image file - if cfg.functional_preproc['distortion_correction']['PhaseDiff'][ - 'fmap_skullstrip_option'] == 'AFNI': - - skullstrip_args = pe.Node(util.Function(input_names=['shrink_fac'], - output_names=['expr'], - function=create_afni_arg), - name=f'distcorr_skullstrip_arg_{pipe_num}') + if ( + cfg.functional_preproc["distortion_correction"]["PhaseDiff"][ + "fmap_skullstrip_option" + ] + == "AFNI" + ): + skullstrip_args = pe.Node( + util.Function( + input_names=["shrink_fac"], + output_names=["expr"], + function=create_afni_arg, + ), + name=f"distcorr_skullstrip_arg_{pipe_num}", + ) skullstrip_args.inputs.shrink_fac = cfg.functional_preproc[ - 'distortion_correction']['PhaseDiff'][ - 'fmap_skullstrip_AFNI_threshold'] - - afni = pe.Node(interface=afni.SkullStrip(), name=f'distcor_phasediff_' - f'afni_skullstrip_' - f'{pipe_num}') - afni.inputs.outputtype = 'NIFTI_GZ' - wf.connect(skullstrip_args, 'expr', afni, 'args') - - if strat_pool.check_rpool('magnitude'): - node, out = strat_pool.get_data('magnitude') - elif strat_pool.check_rpool('magnitude1'): - node, out = strat_pool.get_data('magnitude1') - - wf.connect(node, out, afni, 'in_file') - - brain_node, brain_out = (afni, 'out_file') - - elif cfg.functional_preproc['distortion_correction']['PhaseDiff'][ - 'fmap_skullstrip_option'] == 'BET': - - bet = pe.Node(interface=fsl.BET(), name='distcor_phasediff_bet_' - f'skullstrip_{pipe_num}') - bet.inputs.output_type = 'NIFTI_GZ' - bet.inputs.frac = cfg.functional_preproc['distortion_correction'][ - 'PhaseDiff']['fmap_skullstrip_BET_frac'] - - if strat_pool.check_rpool('magnitude'): - node, out = strat_pool.get_data('magnitude') - elif strat_pool.check_rpool('magnitude1'): - node, out = strat_pool.get_data('magnitude1') - wf.connect(node, out, bet, 'in_file') - - brain_node, brain_out = (bet, 'out_file') + "distortion_correction" + ]["PhaseDiff"]["fmap_skullstrip_AFNI_threshold"] + + afni = pe.Node( + interface=afni.SkullStrip(), + name=f"distcor_phasediff_" f"afni_skullstrip_" f"{pipe_num}", + ) + afni.inputs.outputtype = "NIFTI_GZ" + wf.connect(skullstrip_args, "expr", afni, "args") + + if strat_pool.check_rpool("magnitude"): + node, out = strat_pool.get_data("magnitude") + elif strat_pool.check_rpool("magnitude1"): + node, out = strat_pool.get_data("magnitude1") + + wf.connect(node, out, afni, "in_file") + + brain_node, brain_out = (afni, "out_file") + + elif ( + cfg.functional_preproc["distortion_correction"]["PhaseDiff"][ + "fmap_skullstrip_option" + ] + == "BET" + ): + bet = pe.Node( + interface=fsl.BET(), name="distcor_phasediff_bet_" f"skullstrip_{pipe_num}" + ) + bet.inputs.output_type = "NIFTI_GZ" + bet.inputs.frac = cfg.functional_preproc["distortion_correction"]["PhaseDiff"][ + "fmap_skullstrip_BET_frac" + ] + + if strat_pool.check_rpool("magnitude"): + node, out = strat_pool.get_data("magnitude") + elif strat_pool.check_rpool("magnitude1"): + node, out = strat_pool.get_data("magnitude1") + wf.connect(node, out, bet, "in_file") + + brain_node, brain_out = (bet, "out_file") # Prepare Fieldmap # prepare the field map - prepare = pe.Node(interface=fsl.epi.PrepareFieldmap(), name='prepare') + prepare = pe.Node(interface=fsl.epi.PrepareFieldmap(), name="prepare") prepare.inputs.output_type = "NIFTI_GZ" - node, out = strat_pool.get_data('deltaTE') - wf.connect(node, out, prepare, 'delta_TE') + node, out = strat_pool.get_data("deltaTE") + wf.connect(node, out, prepare, "delta_TE") + + if strat_pool.check_rpool("phase1") and strat_pool.check_rpool("phase2"): + fslmaths_sub = pe.Node( + interface=fsl.BinaryMaths(), name="fugue_phase_subtraction" + ) + fslmaths_sub.inputs.operation = "sub" + + node, out = strat_pool.get_data("phase1") + wf.connect(node, out, fslmaths_sub, "in_file") - if strat_pool.check_rpool('phase1') and strat_pool.check_rpool('phase2'): - fslmaths_sub = pe.Node(interface=fsl.BinaryMaths(), - name='fugue_phase_subtraction') - fslmaths_sub.inputs.operation = 'sub' + node, out = strat_pool.get_data("phase2") + wf.connect(node, out, fslmaths_sub, "operand_file") - node, out = strat_pool.get_data('phase1') - wf.connect(node, out, fslmaths_sub, 'in_file') - - node, out = strat_pool.get_data('phase2') - wf.connect(node, out, fslmaths_sub, 'operand_file') + node, out = (fslmaths_sub, "out_file") - node, out = (fslmaths_sub, 'out_file') + elif strat_pool.check_rpool("phasediff"): + node, out = strat_pool.get_data("phasediff") - elif strat_pool.check_rpool('phasediff'): - node, out = strat_pool.get_data('phasediff') - - wf.connect(node, out, prepare, 'in_phase') - wf.connect(brain_node, brain_out, prepare, 'in_magnitude') + wf.connect(node, out, prepare, "in_phase") + wf.connect(brain_node, brain_out, prepare, "in_magnitude") # erode the masked magnitude image - fslmath_mag = pe.Node(interface=fsl.ErodeImage(), name='fslmath_mag') - wf.connect(brain_node, brain_out, fslmath_mag, 'in_file') + fslmath_mag = pe.Node(interface=fsl.ErodeImage(), name="fslmath_mag") + wf.connect(brain_node, brain_out, fslmath_mag, "in_file") # calculate the absolute value of the eroded and masked magnitude # image - fslmath_abs = pe.Node(interface=fsl.UnaryMaths(), name='fslmath_abs') - fslmath_abs.inputs.operation = 'abs' - wf.connect(fslmath_mag, 'out_file', fslmath_abs, 'in_file') + fslmath_abs = pe.Node(interface=fsl.UnaryMaths(), name="fslmath_abs") + fslmath_abs.inputs.operation = "abs" + wf.connect(fslmath_mag, "out_file", fslmath_abs, "in_file") # binarize the absolute value of the eroded and masked magnitude # image - fslmath_bin = pe.Node(interface=fsl.UnaryMaths(), name='fslmath_bin') - fslmath_bin.inputs.operation = 'bin' - wf.connect(fslmath_abs, 'out_file', fslmath_bin, 'in_file') + fslmath_bin = pe.Node(interface=fsl.UnaryMaths(), name="fslmath_bin") + fslmath_bin.inputs.operation = "bin" + wf.connect(fslmath_abs, "out_file", fslmath_bin, "in_file") # take the absolute value of the fieldmap calculated in the prepare step - fslmath_mask_1 = pe.Node(interface=fsl.UnaryMaths(), - name='fslmath_mask_1') - fslmath_mask_1.inputs.operation = 'abs' - wf.connect(prepare, 'out_fieldmap', fslmath_mask_1, 'in_file') + fslmath_mask_1 = pe.Node(interface=fsl.UnaryMaths(), name="fslmath_mask_1") + fslmath_mask_1.inputs.operation = "abs" + wf.connect(prepare, "out_fieldmap", fslmath_mask_1, "in_file") # binarize the absolute value of the fieldmap calculated in the # prepare step - fslmath_mask_2 = pe.Node(interface=fsl.UnaryMaths(), - name='fslmath_mask_2') - fslmath_mask_2.inputs.operation = 'bin' - wf.connect(fslmath_mask_1, 'out_file', fslmath_mask_2, 'in_file') + fslmath_mask_2 = pe.Node(interface=fsl.UnaryMaths(), name="fslmath_mask_2") + fslmath_mask_2.inputs.operation = "bin" + wf.connect(fslmath_mask_1, "out_file", fslmath_mask_2, "in_file") # multiply together the binarized magnitude and fieldmap images - fslmath_mask = pe.Node(interface=fsl.BinaryMaths(), - name='fslmath_mask') - fslmath_mask.inputs.operation = 'mul' - wf.connect(fslmath_mask_2, 'out_file', fslmath_mask, 'in_file') - wf.connect(fslmath_bin, 'out_file', fslmath_mask, 'operand_file') + fslmath_mask = pe.Node(interface=fsl.BinaryMaths(), name="fslmath_mask") + fslmath_mask.inputs.operation = "mul" + wf.connect(fslmath_mask_2, "out_file", fslmath_mask, "in_file") + wf.connect(fslmath_bin, "out_file", fslmath_mask, "operand_file") # Note for the user. Ensure the phase image is within 0-4096 (upper # threshold is 90% of 4096), fsl_prepare_fieldmap will only work in the @@ -246,23 +248,23 @@ def distcor_phasediff_fsl_fugue(wf, cfg, strat_pool, pipe_num, opt=None): # option in the GUI. # fugue - fugue1 = pe.Node(interface=fsl.FUGUE(), name='fsl_fugue') + fugue1 = pe.Node(interface=fsl.FUGUE(), name="fsl_fugue") fugue1.inputs.save_fmap = True - fugue1.outputs.fmap_out_file = 'fmap_rads' + fugue1.outputs.fmap_out_file = "fmap_rads" - wf.connect(fslmath_mask, 'out_file', fugue1, 'mask_file') + wf.connect(fslmath_mask, "out_file", fugue1, "mask_file") # FSL calls EffectiveEchoSpacing "dwell_time" - node, out = strat_pool.get_data('effectiveEchoSpacing') - wf.connect(node, out, fugue1, 'dwell_time') - node, out = strat_pool.get_data('ees-asym-ratio') - wf.connect(node, out, fugue1, 'dwell_to_asym_ratio') + node, out = strat_pool.get_data("effectiveEchoSpacing") + wf.connect(node, out, fugue1, "dwell_time") + node, out = strat_pool.get_data("ees-asym-ratio") + wf.connect(node, out, fugue1, "dwell_to_asym_ratio") - wf.connect(prepare, 'out_fieldmap', fugue1, 'fmap_in_file') + wf.connect(prepare, "out_fieldmap", fugue1, "fmap_in_file") outputs = { - 'despiked-fieldmap': (fugue1, 'fmap_out_file'), - 'fieldmap-mask': (fslmath_mask, 'out_file') + "despiked-fieldmap": (fugue1, "fmap_out_file"), + "fieldmap-mask": (fslmath_mask, "out_file"), } return (wf, outputs) @@ -274,38 +276,55 @@ def same_pe_direction_prep(same_pe_epi, func_mean): This function only exists to serve as a function node in the blip_distcor_wf sub-workflow because workflow decisions cannot be made - dynamically at runtime.""" - + dynamically at runtime. + """ if not same_pe_epi: qwarp_input = func_mean elif same_pe_epi: - skullstrip_outfile = os.path.join(os.getcwd(), - "{0}_mask.nii.gz".format( - os.path.basename(same_pe_epi) - )) - skullstrip_cmd = ["3dAutomask", "-apply_prefix", - "{0}_masked.nii.gz".format(os.path.basename( - same_pe_epi - )), "-prefix", skullstrip_outfile, same_pe_epi] - retcode = subprocess.check_output(skullstrip_cmd) - - extract_brain_outfile = os.path.join(os.getcwd(), - "{0}_calc.nii.gz".format( - os.path.basename(same_pe_epi) - )) - extract_brain_cmd = ["3dcalc", "-a", same_pe_epi, "-b", - skullstrip_outfile, "-expr", "a*b", "-prefix", - extract_brain_outfile] - retcode = subprocess.check_output(extract_brain_cmd) - - align_epi_outfile = os.path.join(os.getcwd(), - "{0}_calc_flirt.nii.gz".format( - os.path.basename(same_pe_epi) - )) - align_epi_cmd = ["flirt", "-in", extract_brain_outfile, "-ref", - func_mean, "-out", align_epi_outfile, "-cost", - "corratio"] - retcode = subprocess.check_output(align_epi_cmd) + skullstrip_outfile = os.path.join( + os.getcwd(), "{0}_mask.nii.gz".format(os.path.basename(same_pe_epi)) + ) + skullstrip_cmd = [ + "3dAutomask", + "-apply_prefix", + "{0}_masked.nii.gz".format(os.path.basename(same_pe_epi)), + "-prefix", + skullstrip_outfile, + same_pe_epi, + ] + subprocess.check_output(skullstrip_cmd) + + extract_brain_outfile = os.path.join( + os.getcwd(), "{0}_calc.nii.gz".format(os.path.basename(same_pe_epi)) + ) + extract_brain_cmd = [ + "3dcalc", + "-a", + same_pe_epi, + "-b", + skullstrip_outfile, + "-expr", + "a*b", + "-prefix", + extract_brain_outfile, + ] + subprocess.check_output(extract_brain_cmd) + + align_epi_outfile = os.path.join( + os.getcwd(), "{0}_calc_flirt.nii.gz".format(os.path.basename(same_pe_epi)) + ) + align_epi_cmd = [ + "flirt", + "-in", + extract_brain_outfile, + "-ref", + func_mean, + "-out", + align_epi_outfile, + "-cost", + "corratio", + ] + subprocess.check_output(align_epi_cmd) qwarp_input = align_epi_outfile @@ -313,30 +332,37 @@ def same_pe_direction_prep(same_pe_epi, func_mean): def calculate_blip_warp(opp_pe, same_pe): - out_warp = os.path.join(os.getcwd(), "Qwarp_PLUS_WARP.nii.gz") - cmd = ["3dQwarp", "-prefix", "Qwarp.nii.gz", "-plusminus", "-base", - opp_pe, "-source", same_pe] + cmd = [ + "3dQwarp", + "-prefix", + "Qwarp.nii.gz", + "-plusminus", + "-base", + opp_pe, + "-source", + same_pe, + ] - retcode = subprocess.check_output(cmd) + subprocess.check_output(cmd) return out_warp def convert_afni_to_ants(afni_warp): afni_warp = os.path.abspath(afni_warp) - afni_warp_img = nb.load(afni_warp) + afni_warp_img = nib.load(afni_warp) afni_warp_hdr = afni_warp_img.header.copy() - afni_warp_hdr.set_data_dtype('`_ - + Workflow Inputs:: - + inputspec.z_stats : string (nifti file) z_score stats output for t or f contrast from flameo - + inputspec.merge_mask : string (nifti file) mask generated from 4D Merged derivative file - + inputspec.z_threshold : float - Z Statistic threshold value for cluster thresholding. It is used to - determine what level of activation would be statistically significant. + Z Statistic threshold value for cluster thresholding. It is used to + determine what level of activation would be statistically significant. Increasing this will result in higher estimates of required effect. - + inputspec.p_threshold : float Probability threshold for cluster thresholding. - + inputspec.paramerters : string (tuple) tuple containing which MNI and FSLDIR path information - + Workflow Outputs:: - + outputspec.cluster_threshold : string (nifti files) the thresholded Z statistic image for each t contrast - + outputspec.cluster_index : string (nifti files) - image of clusters for each t contrast; the values - in the clusters are the index numbers as used + image of clusters for each t contrast; the values + in the clusters are the index numbers as used in the cluster list. - + outputspec.overlay_threshold : string (nifti files) 3D color rendered stats overlay image for t contrast - After reloading this image, use the Statistics Color + After reloading this image, use the Statistics Color Rendering GUI to reload the color look-up-table - + outputspec.overlay_rendered_image : string (nifti files) 2D color rendered stats overlay picture for each t contrast - + outputspec.cluster_localmax_txt : string (text files) local maxima text file, defines the coordinates of maximum value in the cluster - - + + Order of commands: - + - Estimate smoothness of the image:: - + smoothest --mask= merge_mask.nii.gz --zstat=.../flameo/stats/zstat1.nii.gz - + arguments --mask : brain mask volume --zstat : filename of zstat/zfstat image - + - Create mask. For details see `fslmaths `_:: - - fslmaths ../flameo/stats/zstat1.nii.gz - -mas merge_mask.nii.gz + + fslmaths ../flameo/stats/zstat1.nii.gz + -mas merge_mask.nii.gz zstat1_mask.nii.gz - + arguments -mas : use (following image>0) to mask current image - Copy Geometry image dimensions, voxel dimensions, voxel dimensions units string, image orientation/origin or qform/sform info) from one image to another:: - + fslcpgeom MNI152_T1_2mm_brain.nii.gz zstat1_mask.nii.gz - + - Cluster based thresholding. For details see `FEAT `_:: - - cluster --dlh = 0.0023683100 - --in = zstat1_mask.nii.gz - --oindex = zstat1_cluster_index.nii.gz + + cluster --dlh = 0.0023683100 + --in = zstat1_mask.nii.gz + --oindex = zstat1_cluster_index.nii.gz --olmax = zstat1_cluster_localmax.txt - --othresh = zstat1_cluster_threshold.nii.gz - --pthresh = 0.0500000000 - --thresh = 2.3000000000 + --othresh = zstat1_cluster_threshold.nii.gz + --pthresh = 0.0500000000 + --thresh = 2.3000000000 --volume = 197071 - - arguments + + arguments --in : filename of input volume --dlh : smoothness estimate = sqrt(det(Lambda)) --oindex : filename for output of cluster index @@ -113,35 +113,35 @@ def easy_thresh(wf_name): --volume : number of voxels in the mask --pthresh : p-threshold for clusters --thresh : threshold for input volume - + Z statistic image is thresholded to show which voxels or clusters of voxels are activated at a particular significance level. - A Z statistic threshold is used to define contiguous clusters. Then each cluster's estimated significance level (from GRF-theory) + A Z statistic threshold is used to define contiguous clusters. Then each cluster's estimated significance level (from GRF-theory) is compared with the cluster probability threshold. Significant clusters are then used to mask the original Z statistic image. - - - Get the maximum intensity value of the output thresholded image. This used is while rendering the Z statistic image:: - + + - Get the maximum intensity value of the output thresholded image. This used is while rendering the Z statistic image:: + fslstats zstat1_cluster_threshold.nii.gz -R - + arguments -R : output - Rendering. For details see `FEAT `_:: - - overlay 1 0 MNI152_T1_2mm_brain.nii.gz - -a zstat1_cluster_threshold.nii.gz - 2.30 15.67 + + overlay 1 0 MNI152_T1_2mm_brain.nii.gz + -a zstat1_cluster_threshold.nii.gz + 2.30 15.67 zstat1_cluster_threshold_overlay.nii.gz - - slicer zstat1_cluster_threshold_overlay.nii.gz - -L -A 750 + + slicer zstat1_cluster_threshold_overlay.nii.gz + -L -A 750 zstat1_cluster_threshold_overlay.png - - The Z statistic range selected for rendering is automatically calculated by default, - to run from red (minimum Z statistic after thresholding) to yellow (maximum Z statistic, here + + The Z statistic range selected for rendering is automatically calculated by default, + to run from red (minimum Z statistic after thresholding) to yellow (maximum Z statistic, here maximum intensity). - + High Level Workflow Graph: - + .. exec:: from CPAC.easy_thresh import easy_thresh wf = easy_thresh('easy_thresh_wf') @@ -152,16 +152,15 @@ def easy_thresh(wf_name): .. image:: ../../images/generated/easy_thresh.png :width: 800 - - + + Detailed Workflow Graph: - + .. image:: ../../images/generated/generated.easy_thresh_detailed.png :width: 800 - + Examples -------- - >>> from CPAC.easy_thresh import easy_thresh >>> preproc = easy_thresh("new_workflow") >>> preproc.inputs.inputspec.z_stats= 'flameo/stats/zstat1.nii.gz' @@ -170,112 +169,127 @@ def easy_thresh(wf_name): >>> preproc.inputs.inputspec.p_threshold = 0.05 >>> preproc.inputs.inputspec.parameters = ('/usr/local/fsl/', 'MNI152') >>> preporc.run() # doctest: +SKIP - - """ + """ easy_thresh = pe.Workflow(name=wf_name) - inputnode = pe.Node(util.IdentityInterface(fields=['z_stats', - 'merge_mask', - 'z_threshold', - 'p_threshold', - 'parameters']), - name='inputspec') - - outputnode = pe.Node(util.IdentityInterface(fields=['cluster_threshold', - 'cluster_index', - 'cluster_localmax_txt', - 'overlay_threshold', - 'rendered_image']), - name='outputspec') + inputnode = pe.Node( + util.IdentityInterface( + fields=["z_stats", "merge_mask", "z_threshold", "p_threshold", "parameters"] + ), + name="inputspec", + ) + + outputnode = pe.Node( + util.IdentityInterface( + fields=[ + "cluster_threshold", + "cluster_index", + "cluster_localmax_txt", + "overlay_threshold", + "rendered_image", + ] + ), + name="outputspec", + ) # fsl easythresh # estimate image smoothness - smooth_estimate = pe.MapNode(interface=fsl.SmoothEstimate(), - name='smooth_estimate', - iterfield=['zstat_file']) + smooth_estimate = pe.MapNode( + interface=fsl.SmoothEstimate(), name="smooth_estimate", iterfield=["zstat_file"] + ) # run clustering after fixing stats header for talspace - zstat_mask = pe.MapNode(interface=fsl.MultiImageMaths(), - name='zstat_mask', - iterfield=['in_file']) + zstat_mask = pe.MapNode( + interface=fsl.MultiImageMaths(), name="zstat_mask", iterfield=["in_file"] + ) # operations to perform - #-mas use (following image>0) to mask current image - zstat_mask.inputs.op_string = '-mas %s' + # -mas use (following image>0) to mask current image + zstat_mask.inputs.op_string = "-mas %s" # fslcpgeom - #copy certain parts of the header information (image dimensions, - #voxel dimensions, voxel dimensions units string, image orientation/origin - #or qform/sform info) from one image to another - geo_imports = ['import subprocess'] - copy_geometry = pe.MapNode(util.Function(input_names=['infile_a', 'infile_b'], - output_names=['out_file'], - function=copy_geom, - imports=geo_imports), - name='copy_geometry', - iterfield=['infile_a', 'infile_b']) + # copy certain parts of the header information (image dimensions, + # voxel dimensions, voxel dimensions units string, image orientation/origin + # or qform/sform info) from one image to another + geo_imports = ["import subprocess"] + copy_geometry = pe.MapNode( + util.Function( + input_names=["infile_a", "infile_b"], + output_names=["out_file"], + function=copy_geom, + imports=geo_imports, + ), + name="copy_geometry", + iterfield=["infile_a", "infile_b"], + ) ##cluster-based thresholding - #After carrying out the initial statistical test, the resulting - #Z statistic image is then normally thresholded to show which voxels or - #clusters of voxels are activated at a particular significance level. - #A Z statistic threshold is used to define contiguous clusters. - #Then each cluster's estimated significance level (from GRF-theory) is - #compared with the cluster probability threshold. Significant clusters - #are then used to mask the original Z statistic image for later production - #of colour blobs.This method of thresholding is an alternative to - #Voxel-based correction, and is normally more sensitive to activation. -# cluster = pe.MapNode(interface=fsl.Cluster(), -# name='cluster', -# iterfield=['in_file', 'volume', 'dlh']) -# #output of cluster index (in size order) -# cluster.inputs.out_index_file = True -# #thresholded image -# cluster.inputs.out_threshold_file = True -# #local maxima text file -# #defines the cluster cordinates -# cluster.inputs.out_localmax_txt_file = True - - cluster_imports = ['import os', 'import re', 'import subprocess'] - cluster = pe.MapNode(util.Function(input_names=['in_file', - 'volume', - 'dlh', - 'threshold', - 'pthreshold', - 'parameters'], - output_names=['index_file', - 'threshold_file', - 'localmax_txt_file'], - function=call_cluster, - imports=cluster_imports), - name='cluster', - iterfield=['in_file', 'volume', 'dlh']) + # After carrying out the initial statistical test, the resulting + # Z statistic image is then normally thresholded to show which voxels or + # clusters of voxels are activated at a particular significance level. + # A Z statistic threshold is used to define contiguous clusters. + # Then each cluster's estimated significance level (from GRF-theory) is + # compared with the cluster probability threshold. Significant clusters + # are then used to mask the original Z statistic image for later production + # of colour blobs.This method of thresholding is an alternative to + # Voxel-based correction, and is normally more sensitive to activation. + # cluster = pe.MapNode(interface=fsl.Cluster(), + # name='cluster', + # iterfield=['in_file', 'volume', 'dlh']) + # #output of cluster index (in size order) + # cluster.inputs.out_index_file = True + # #thresholded image + # cluster.inputs.out_threshold_file = True + # #local maxima text file + # #defines the cluster cordinates + # cluster.inputs.out_localmax_txt_file = True + + cluster_imports = ["import os", "import re", "import subprocess"] + cluster = pe.MapNode( + util.Function( + input_names=[ + "in_file", + "volume", + "dlh", + "threshold", + "pthreshold", + "parameters", + ], + output_names=["index_file", "threshold_file", "localmax_txt_file"], + function=call_cluster, + imports=cluster_imports, + ), + name="cluster", + iterfield=["in_file", "volume", "dlh"], + ) # max and minimum intensity values - image_stats = pe.MapNode(interface=fsl.ImageStats(), - name='image_stats', - iterfield=['in_file']) - image_stats.inputs.op_string = '-R' + image_stats = pe.MapNode( + interface=fsl.ImageStats(), name="image_stats", iterfield=["in_file"] + ) + image_stats.inputs.op_string = "-R" # create tuple of z_threshold and max intensity value of threshold file - create_tuple = pe.MapNode(util.Function(input_names=['infile_a', - 'infile_b'], - output_names=['out_file'], - function=get_tuple), - name='create_tuple', - iterfield=['infile_b']) + create_tuple = pe.MapNode( + util.Function( + input_names=["infile_a", "infile_b"], + output_names=["out_file"], + function=get_tuple, + ), + name="create_tuple", + iterfield=["infile_b"], + ) # colour activation overlaying - overlay = pe.MapNode(interface=fsl.Overlay(), - name='overlay', - iterfield=['stat_image', 'stat_thresh']) + overlay = pe.MapNode( + interface=fsl.Overlay(), name="overlay", iterfield=["stat_image", "stat_thresh"] + ) overlay.inputs.transparency = True overlay.inputs.auto_thresh_bg = True - overlay.inputs.out_type = 'float' + overlay.inputs.out_type = "float" # colour rendering - slicer = pe.MapNode(interface=fsl.Slicer(), name='slicer', - iterfield=['in_file']) + slicer = pe.MapNode(interface=fsl.Slicer(), name="slicer", iterfield=["in_file"]) # set max picture width slicer.inputs.image_width = 750 # set output all axial slices into one picture @@ -283,108 +297,113 @@ def easy_thresh(wf_name): # function mapnode to get the standard fsl brain image based on parameters # as FSLDIR,MNI and voxel size - get_bg_imports = ['import os', 'import nibabel as nib'] - get_backgroundimage = pe.MapNode(util.Function(input_names=['in_file', - 'file_parameters'], - output_names=['out_file'], - function=get_standard_background_img, - imports=get_bg_imports), - name='get_bckgrndimg1', - iterfield=['in_file']) + get_bg_imports = ["import os", "import nibabel as nib"] + get_backgroundimage = pe.MapNode( + util.Function( + input_names=["in_file", "file_parameters"], + output_names=["out_file"], + function=get_standard_background_img, + imports=get_bg_imports, + ), + name="get_bckgrndimg1", + iterfield=["in_file"], + ) # function node to get the standard fsl brain image # outputs single file - get_backgroundimage2 = pe.Node(util.Function(input_names=['in_file', - 'file_parameters'], - output_names=['out_file'], - function=get_standard_background_img, - imports=get_bg_imports), - name='get_backgrndimg2') + get_backgroundimage2 = pe.Node( + util.Function( + input_names=["in_file", "file_parameters"], + output_names=["out_file"], + function=get_standard_background_img, + imports=get_bg_imports, + ), + name="get_backgrndimg2", + ) # connections - easy_thresh.connect(inputnode, 'z_stats', smooth_estimate, 'zstat_file') - easy_thresh.connect(inputnode, 'merge_mask', smooth_estimate, 'mask_file') + easy_thresh.connect(inputnode, "z_stats", smooth_estimate, "zstat_file") + easy_thresh.connect(inputnode, "merge_mask", smooth_estimate, "mask_file") - easy_thresh.connect(inputnode, 'z_stats', zstat_mask, 'in_file') - easy_thresh.connect(inputnode, 'merge_mask', zstat_mask, 'operand_files') + easy_thresh.connect(inputnode, "z_stats", zstat_mask, "in_file") + easy_thresh.connect(inputnode, "merge_mask", zstat_mask, "operand_files") - easy_thresh.connect(zstat_mask, 'out_file', get_backgroundimage, - 'in_file') - easy_thresh.connect(inputnode, 'parameters', get_backgroundimage, - 'file_parameters') + easy_thresh.connect(zstat_mask, "out_file", get_backgroundimage, "in_file") + easy_thresh.connect(inputnode, "parameters", get_backgroundimage, "file_parameters") - easy_thresh.connect(get_backgroundimage, 'out_file', copy_geometry, - 'infile_a') - easy_thresh.connect(zstat_mask, 'out_file', copy_geometry, 'infile_b') + easy_thresh.connect(get_backgroundimage, "out_file", copy_geometry, "infile_a") + easy_thresh.connect(zstat_mask, "out_file", copy_geometry, "infile_b") - easy_thresh.connect(copy_geometry, 'out_file', cluster, 'in_file') - easy_thresh.connect(inputnode, 'z_threshold', cluster, 'threshold') - easy_thresh.connect(inputnode, 'p_threshold', cluster, 'pthreshold') - easy_thresh.connect(smooth_estimate, 'volume', cluster, 'volume') - easy_thresh.connect(smooth_estimate, 'dlh', cluster, 'dlh') - easy_thresh.connect(inputnode, 'parameters', cluster, 'parameters') + easy_thresh.connect(copy_geometry, "out_file", cluster, "in_file") + easy_thresh.connect(inputnode, "z_threshold", cluster, "threshold") + easy_thresh.connect(inputnode, "p_threshold", cluster, "pthreshold") + easy_thresh.connect(smooth_estimate, "volume", cluster, "volume") + easy_thresh.connect(smooth_estimate, "dlh", cluster, "dlh") + easy_thresh.connect(inputnode, "parameters", cluster, "parameters") - easy_thresh.connect(cluster, 'threshold_file', image_stats, 'in_file') + easy_thresh.connect(cluster, "threshold_file", image_stats, "in_file") - easy_thresh.connect(image_stats, 'out_stat', create_tuple, 'infile_b') - easy_thresh.connect(inputnode, 'z_threshold', create_tuple, 'infile_a') + easy_thresh.connect(image_stats, "out_stat", create_tuple, "infile_b") + easy_thresh.connect(inputnode, "z_threshold", create_tuple, "infile_a") - easy_thresh.connect(cluster, 'threshold_file', overlay, 'stat_image') - easy_thresh.connect(create_tuple, 'out_file', overlay, 'stat_thresh') + easy_thresh.connect(cluster, "threshold_file", overlay, "stat_image") + easy_thresh.connect(create_tuple, "out_file", overlay, "stat_thresh") - easy_thresh.connect(inputnode, 'merge_mask', get_backgroundimage2, - 'in_file' ) - easy_thresh.connect(inputnode, 'parameters', get_backgroundimage2, - 'file_parameters') + easy_thresh.connect(inputnode, "merge_mask", get_backgroundimage2, "in_file") + easy_thresh.connect( + inputnode, "parameters", get_backgroundimage2, "file_parameters" + ) - easy_thresh.connect(get_backgroundimage2, 'out_file', overlay, - 'background_image') + easy_thresh.connect(get_backgroundimage2, "out_file", overlay, "background_image") - easy_thresh.connect(overlay, 'out_file', slicer, 'in_file') + easy_thresh.connect(overlay, "out_file", slicer, "in_file") - easy_thresh.connect(cluster, 'threshold_file', outputnode, - 'cluster_threshold') - easy_thresh.connect(cluster, 'index_file', outputnode, 'cluster_index') - easy_thresh.connect(cluster, 'localmax_txt_file', outputnode, - 'cluster_localmax_txt') - easy_thresh.connect(overlay, 'out_file', outputnode, 'overlay_threshold') - easy_thresh.connect(slicer, 'out_file', outputnode, 'rendered_image') + easy_thresh.connect(cluster, "threshold_file", outputnode, "cluster_threshold") + easy_thresh.connect(cluster, "index_file", outputnode, "cluster_index") + easy_thresh.connect( + cluster, "localmax_txt_file", outputnode, "cluster_localmax_txt" + ) + easy_thresh.connect(overlay, "out_file", outputnode, "overlay_threshold") + easy_thresh.connect(slicer, "out_file", outputnode, "rendered_image") return easy_thresh def call_cluster(in_file, volume, dlh, threshold, pthreshold, parameters): - - out_name = re.match(r'z(\w)*stat(\d)+', os.path.basename(in_file)) + out_name = re.match(r"z(\w)*stat(\d)+", os.path.basename(in_file)) filename, ext = os.path.splitext(os.path.basename(in_file)) - ext= os.path.splitext(filename)[1] + ext + ext = os.path.splitext(filename)[1] + ext filename = os.path.splitext(filename)[0] if out_name: - out_name= out_name.group(0) + out_name = out_name.group(0) else: out_name = filename FSLDIR = parameters[0] - index_file = os.path.join(os.getcwd(), 'cluster_mask_' + out_name + ext) - threshold_file = os.path.join(os.getcwd(), 'thresh_' + out_name + ext) - localmax_txt_file = os.path.join(os.getcwd(), 'cluster_' + out_name + '.txt') + index_file = os.path.join(os.getcwd(), "cluster_mask_" + out_name + ext) + threshold_file = os.path.join(os.getcwd(), "thresh_" + out_name + ext) + localmax_txt_file = os.path.join(os.getcwd(), "cluster_" + out_name + ".txt") - cmd_path = os.path.join(FSLDIR, 'bin/cluster') + cmd_path = os.path.join(FSLDIR, "bin/cluster") - f = open(localmax_txt_file,'wb') + f = open(localmax_txt_file, "wb") - cmd = subprocess.Popen([ cmd_path, - '--dlh=' + str(dlh), - '--in=' + in_file, - '--oindex=' + index_file, - '--othresh=' + threshold_file, - '--pthresh=' + str(pthreshold), - '--thresh=' + str(threshold), - '--volume=' + str(volume)], - stdout= f) + cmd = subprocess.Popen( + [ + cmd_path, + "--dlh=" + str(dlh), + "--in=" + in_file, + "--oindex=" + index_file, + "--othresh=" + threshold_file, + "--pthresh=" + str(pthreshold), + "--thresh=" + str(threshold), + "--volume=" + str(volume), + ], + stdout=f, + ) stdout_value, stderr_value = cmd.communicate() f.close() @@ -394,36 +413,35 @@ def call_cluster(in_file, volume, dlh, threshold, pthreshold, parameters): def copy_geom(infile_a, infile_b): """ - Method to call fsl fslcpgeom command to copy - certain parts of the header information (image dimensions, - voxel dimensions, voxel dimensions units string, image - orientation/origin or qform/sform info) from one image to another - + Method to call fsl fslcpgeom command to copy + certain parts of the header information (image dimensions, + voxel dimensions, voxel dimensions units string, image + orientation/origin or qform/sform info) from one image to another. + Parameters - ----------- + ---------- infile_a : nifti file input volume from which the geometry is copied from - + infile_b : nifti file input volume from which the geometry is copied to - + Returns - ------- + ------- out_file : nifti file Input volume infile_b with modified geometry information in the header. - + Raises ------ - Exception + Exception If fslcpgeom fails - - """ + """ try: out_file = infile_b - cmd = ['fslcpgeom', infile_a, out_file] - retcode = subprocess.check_output(cmd) + cmd = ["fslcpgeom", infile_a, out_file] + subprocess.check_output(cmd) return out_file except Exception: raise Exception("Error while using fslcpgeom to copy geometry") @@ -431,37 +449,35 @@ def copy_geom(infile_a, infile_b): def get_standard_background_img(in_file, file_parameters): """ - Method to get the standard brain image from FSL - standard data directory - + Method to get the standard brain image from FSL + standard data directory. + Parameters ---------- in_file : nifti file Merged 4D Zmap volume file_parameters : tuple Value FSLDIR and MNI from config file - + Returns ------- standard_path : string Standard FSL Image file path - + Raises ------ - Exception + Exception If nibabel cannot load the input nifti volume - - """ + """ try: - img = nib.load(in_file) hdr = img.header group_mm = int(hdr.get_zooms()[2]) FSLDIR, MNI = file_parameters - standard_path = \ - os.path.join(FSLDIR, 'data/standard/', - '{0}_T1_{1}mm_brain.nii.gz'.format(MNI, group_mm)) + standard_path = os.path.join( + FSLDIR, "data/standard/", "{0}_T1_{1}mm_brain.nii.gz".format(MNI, group_mm) + ) return os.path.abspath(standard_path) except Exception: @@ -473,20 +489,19 @@ def get_tuple(infile_a, infile_b): Simple method to return tuple of z_threhsold maximum intensity values of Zstatistic image for input to the overlay. - + Parameters ---------- z_theshold : float z threshold value intensity_stat : tuple of float values minimum and maximum intensity values - + Returns ------- img_min_max : tuple (float) - tuple of zthreshold and maximum intensity + tuple of zthreshold and maximum intensity value of z statistic image - + """ - out_file = (infile_a, infile_b[1]) - return out_file + return (infile_a, infile_b[1]) diff --git a/CPAC/func_preproc/__init__.py b/CPAC/func_preproc/__init__.py index b0b8d7514d..b445f8dc94 100644 --- a/CPAC/func_preproc/__init__.py +++ b/CPAC/func_preproc/__init__.py @@ -14,13 +14,24 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -"""Functional preprocessing""" -from .func_motion import calc_motion_stats, func_motion_correct, \ - func_motion_correct_only, func_motion_estimates, \ - get_motion_ref, motion_estimate_filter -from .func_preproc import slice_timing_wf, \ - get_idx +"""Functional preprocessing.""" +from .func_motion import ( + calc_motion_stats, + func_motion_correct, + func_motion_correct_only, + func_motion_estimates, + get_motion_ref, + motion_estimate_filter, +) +from .func_preproc import get_idx, slice_timing_wf -__all__ = ['calc_motion_stats', 'func_motion_correct', - 'func_motion_correct_only', 'func_motion_estimates', 'get_idx', - 'get_motion_ref', 'motion_estimate_filter', 'slice_timing_wf'] +__all__ = [ + "calc_motion_stats", + "func_motion_correct", + "func_motion_correct_only", + "func_motion_estimates", + "get_idx", + "get_motion_ref", + "motion_estimate_filter", + "slice_timing_wf", +] diff --git a/CPAC/func_preproc/func_ingress.py b/CPAC/func_preproc/func_ingress.py index d34758ff58..e335820fb0 100644 --- a/CPAC/func_preproc/func_ingress.py +++ b/CPAC/func_preproc/func_ingress.py @@ -15,44 +15,44 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . from nipype import logging -from CPAC.utils.datasource import ( - create_func_datasource, - ingress_func_metadata) -logger = logging.getLogger('nipype.workflow') +from CPAC.utils.datasource import create_func_datasource, ingress_func_metadata + +logger = logging.getLogger("nipype.workflow") -def connect_func_ingress(workflow, strat_list, c, sub_dict, subject_id, - input_creds_path, unique_id=None): - for num_strat, strat in enumerate(strat_list): - if 'func' in sub_dict: - func_paths_dict = sub_dict['func'] +def connect_func_ingress( + workflow, strat_list, c, sub_dict, subject_id, input_creds_path, unique_id=None +): + for num_strat, strat in enumerate(strat_list): + if "func" in sub_dict: + func_paths_dict = sub_dict["func"] else: - func_paths_dict = sub_dict['rest'] + func_paths_dict = sub_dict["rest"] if unique_id is None: - workflow_name = f'func_gather_{num_strat}' + workflow_name = f"func_gather_{num_strat}" else: - workflow_name = f'func_gather_{unique_id}_{num_strat}' + workflow_name = f"func_gather_{unique_id}_{num_strat}" - func_wf = create_func_datasource(func_paths_dict, - workflow_name) + func_wf = create_func_datasource(func_paths_dict, workflow_name) func_wf.inputs.inputnode.set( subject=subject_id, creds_path=input_creds_path, - dl_dir=c.pipeline_setup['working_directory']['path'] + dl_dir=c.pipeline_setup["working_directory"]["path"], ) - func_wf.get_node('inputnode').iterables = \ - ("scan", list(func_paths_dict.keys())) + func_wf.get_node("inputnode").iterables = ("scan", list(func_paths_dict.keys())) - strat.update_resource_pool({ - 'subject': (func_wf, 'outputspec.subject'), - 'scan': (func_wf, 'outputspec.scan') - }) + strat.update_resource_pool( + { + "subject": (func_wf, "outputspec.subject"), + "scan": (func_wf, "outputspec.scan"), + } + ) - (workflow, strat.rpool, diff, blip, fmap_rp_list - ) = ingress_func_metadata(workflow, c, strat.rpool, sub_dict, - subject_id, input_creds_path, unique_id) + (workflow, strat.rpool, diff, blip, fmap_rp_list) = ingress_func_metadata( + workflow, c, strat.rpool, sub_dict, subject_id, input_creds_path, unique_id + ) return (workflow, diff, blip, fmap_rp_list) diff --git a/CPAC/func_preproc/func_motion.py b/CPAC/func_preproc/func_motion.py index b60e389d0a..03ed809b2b 100644 --- a/CPAC/func_preproc/func_motion.py +++ b/CPAC/func_preproc/func_motion.py @@ -14,17 +14,22 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -"""Functions for calculating motion parameters""" +"""Functions for calculating motion parameters.""" # pylint: disable=ungrouped-imports,wrong-import-order,wrong-import-position -from CPAC.pipeline import nipype_pipeline_engine as pe -from nipype.interfaces.afni import preprocess from nipype.interfaces import afni, fsl, utility as util -from nipype.interfaces.afni import utils as afni_utils -from CPAC.func_preproc.utils import chunk_ts, oned_text_concat, \ - split_ts_chunks -from CPAC.func_preproc.utils import notch_filter_motion -from CPAC.generate_motion_statistics import affine_file_from_params_file, \ - motion_power_statistics +from nipype.interfaces.afni import preprocess, utils as afni_utils + +from CPAC.func_preproc.utils import ( + chunk_ts, + notch_filter_motion, + oned_text_concat, + split_ts_chunks, +) +from CPAC.generate_motion_statistics import ( + affine_file_from_params_file, + motion_power_statistics, +) +from CPAC.pipeline import nipype_pipeline_engine as pe from CPAC.pipeline.nodeblock import nodeblock from CPAC.pipeline.schema import valid_options from CPAC.utils.interfaces.function import Function @@ -32,71 +37,108 @@ @nodeblock( - name='calc_motion_stats', - switch=[['functional_preproc', 'run'], - ['functional_preproc', 'motion_estimates_and_correction', 'run']], - inputs=[('desc-preproc_bold', 'space-bold_desc-brain_mask', - 'desc-movementParameters_motion', 'max-displacement', - 'rels-displacement', 'filtered-coordinate-transformation', - 'coordinate-transformation'), 'subject', 'scan'], - outputs=['dvars', 'framewise-displacement-power', - 'framewise-displacement-jenkinson', 'power-params', - 'motion-params', 'motion', 'desc-summary_motion']) + name="calc_motion_stats", + switch=[ + ["functional_preproc", "run"], + ["functional_preproc", "motion_estimates_and_correction", "run"], + ], + inputs=[ + ( + "desc-preproc_bold", + "space-bold_desc-brain_mask", + "desc-movementParameters_motion", + "max-displacement", + "rels-displacement", + "filtered-coordinate-transformation", + "coordinate-transformation", + ), + "subject", + "scan", + ], + outputs=[ + "dvars", + "framewise-displacement-power", + "framewise-displacement-jenkinson", + "power-params", + "motion-params", + "motion", + "desc-summary_motion", + ], +) def calc_motion_stats(wf, cfg, strat_pool, pipe_num, opt=None): - '''Calculate motion statistics for motion parameters.''' - motion_prov = strat_pool.get_cpac_provenance( - 'desc-movementParameters_motion') + """Calculate motion statistics for motion parameters.""" + motion_prov = strat_pool.get_cpac_provenance("desc-movementParameters_motion") motion_correct_tool = check_prov_for_motion_tool(motion_prov) - coordinate_transformation = ['filtered-coordinate-transformation', - 'coordinate-transformation'] + coordinate_transformation = [ + "filtered-coordinate-transformation", + "coordinate-transformation", + ] gen_motion_stats = motion_power_statistics( - name=f'gen_motion_stats_{pipe_num}', + name=f"gen_motion_stats_{pipe_num}", motion_correct_tool=motion_correct_tool, - filtered=strat_pool.filtered_movement) + filtered=strat_pool.filtered_movement, + ) # Special case where the workflow is not getting outputs from # resource pool but is connected to functional datasource - wf.connect(*strat_pool.get_data('desc-preproc_bold'), - gen_motion_stats, 'inputspec.motion_correct') - wf.connect(*strat_pool.get_data('space-bold_desc-brain_mask'), - gen_motion_stats, 'inputspec.mask') - wf.connect(*strat_pool.get_data('desc-movementParameters_motion'), - gen_motion_stats, 'inputspec.movement_parameters') - wf.connect(*strat_pool.get_data('max-displacement'), - gen_motion_stats, 'inputspec.max_displacement') - - if strat_pool.check_rpool('rels-displacement'): - wf.connect(*strat_pool.get_data('rels-displacement'), - gen_motion_stats, 'inputspec.rels_displacement') + wf.connect( + *strat_pool.get_data("desc-preproc_bold"), + gen_motion_stats, + "inputspec.motion_correct", + ) + wf.connect( + *strat_pool.get_data("space-bold_desc-brain_mask"), + gen_motion_stats, + "inputspec.mask", + ) + wf.connect( + *strat_pool.get_data("desc-movementParameters_motion"), + gen_motion_stats, + "inputspec.movement_parameters", + ) + wf.connect( + *strat_pool.get_data("max-displacement"), + gen_motion_stats, + "inputspec.max_displacement", + ) + + if strat_pool.check_rpool("rels-displacement"): + wf.connect( + *strat_pool.get_data("rels-displacement"), + gen_motion_stats, + "inputspec.rels_displacement", + ) if strat_pool.check_rpool(coordinate_transformation): - wf.connect(*strat_pool.get_data(coordinate_transformation), - gen_motion_stats, 'inputspec.transformations') + wf.connect( + *strat_pool.get_data(coordinate_transformation), + gen_motion_stats, + "inputspec.transformations", + ) outputs = { - 'framewise-displacement-power': - (gen_motion_stats, 'outputspec.FDP_1D'), - 'framewise-displacement-jenkinson': - (gen_motion_stats, 'outputspec.FDJ_1D'), - 'dvars': (gen_motion_stats, 'outputspec.DVARS_1D'), - 'power-params': (gen_motion_stats, 'outputspec.power_params'), - 'motion-params': (gen_motion_stats, 'outputspec.motion_params'), - 'motion': (gen_motion_stats, 'outputspec.motion'), - 'desc-summary_motion': (gen_motion_stats, - 'outputspec.desc-summary_motion')} + "framewise-displacement-power": (gen_motion_stats, "outputspec.FDP_1D"), + "framewise-displacement-jenkinson": (gen_motion_stats, "outputspec.FDJ_1D"), + "dvars": (gen_motion_stats, "outputspec.DVARS_1D"), + "power-params": (gen_motion_stats, "outputspec.power_params"), + "motion-params": (gen_motion_stats, "outputspec.motion_params"), + "motion": (gen_motion_stats, "outputspec.motion"), + "desc-summary_motion": (gen_motion_stats, "outputspec.desc-summary_motion"), + } return wf, outputs def estimate_reference_image(in_file): """fMRIPrep-style BOLD reference - Ref: https://github.com/nipreps/niworkflows/blob/maint/1.3.x/niworkflows/interfaces/registration.py#L446-L549 + Ref: https://github.com/nipreps/niworkflows/blob/maint/1.3.x/niworkflows/interfaces/registration.py#L446-L549. """ import os + import numpy as np - import nibabel as nb + import nibabel as nib ref_input = [in_file] - mc_out_file = 'bold_mc.nii.gz' + mc_out_file = "bold_mc.nii.gz" # Build the nibabel spatial image we will work with ref_im = [] @@ -104,12 +146,12 @@ def estimate_reference_image(in_file): max_new_volumes = 50 - len(ref_im) if max_new_volumes <= 0: break - nib_i = nb.squeeze_image(nb.load(im_i)) + nib_i = nib.squeeze_image(nib.load(im_i)) if nib_i.dataobj.ndim == 3: ref_im.append(nib_i) elif nib_i.dataobj.ndim == 4: - ref_im += nb.four_to_three(nib_i.slicer[..., :max_new_volumes]) - ref_im = nb.squeeze_image(nb.concat_images(ref_im)) + ref_im += nib.four_to_three(nib_i.slicer[..., :max_new_volumes]) + ref_im = nib.squeeze_image(nib.concat_images(ref_im)) out_file = os.path.join(os.getcwd(), "ref_bold.nii.gz") @@ -119,28 +161,29 @@ def estimate_reference_image(in_file): ref_im.header.extensions.clear() if ref_im.shape[-1] > 40: - ref_im = nb.Nifti1Image( + ref_im = nib.Nifti1Image( ref_im.dataobj[:, :, :, 20:40], ref_im.affine, ref_im.header ) ref_name = os.path.join(os.getcwd(), "slice.nii.gz") ref_im.to_filename(ref_name) - os.system('3dvolreg -Fourier -twopass -zpad 4 ' - f'-prefix {mc_out_file} {ref_name}') + os.system("3dvolreg -Fourier -twopass -zpad 4 " f"-prefix {mc_out_file} {ref_name}") - mc_slice_nii = nb.load(mc_out_file) + mc_slice_nii = nib.load(mc_out_file) median_image_data = np.median(mc_slice_nii.get_fdata(), axis=3) - nb.Nifti1Image(median_image_data, ref_im.affine, ref_im.header - ).to_filename(out_file) + nib.Nifti1Image(median_image_data, ref_im.affine, ref_im.header).to_filename( + out_file + ) return out_file _MOTION_CORRECTED_OUTPUTS = { "desc-preproc_bold": {"Description": "Motion-corrected BOLD time-series."}, - "desc-motion_bold": {"Description": "Motion-corrected BOLD time-series."}} + "desc-motion_bold": {"Description": "Motion-corrected BOLD time-series."}, +} # the "filtered" outputs here are just for maintaining expecting # forking and connections and will not be output _MOTION_PARAM_OUTPUTS = { @@ -148,30 +191,36 @@ def estimate_reference_image(in_file): "rels-displacement": {}, "desc-movementParameters_motion": { "Description": "Each line contains for one timepoint a 6-DOF " - "rigid transform parameters in the format " - "defined by AFNI's 3dvolreg: [roll, pitch, yaw, " - "superior displacement, left displacement, " - "posterior displacement]. Rotation parameters " - "are in degrees counterclockwise, and translation " - "parameters are in millimeters." + "rigid transform parameters in the format " + "defined by AFNI's 3dvolreg: [roll, pitch, yaw, " + "superior displacement, left displacement, " + "posterior displacement]. Rotation parameters " + "are in degrees counterclockwise, and translation " + "parameters are in millimeters." }, "filtered-coordinate-transformation": {"Description": "UNFILTERED"}, "coordinate-transformation": { - "Description" : "Each row contains for one timepoint the first " - "12 values of a 4x4 affine matrix"}} + "Description": "Each row contains for one timepoint the first " + "12 values of a 4x4 affine matrix" + }, +} @nodeblock( name="motion_correction", switch=["functional_preproc", "motion_estimates_and_correction", "run"], - option_key=["functional_preproc", "motion_estimates_and_correction", - "motion_correction", "using"], + option_key=[ + "functional_preproc", + "motion_estimates_and_correction", + "motion_correction", + "using", + ], option_val=["3dvolreg", "mcflirt"], inputs=[("desc-preproc_bold", "motion-basefile")], - outputs={**_MOTION_CORRECTED_OUTPUTS, **_MOTION_PARAM_OUTPUTS}) + outputs={**_MOTION_CORRECTED_OUTPUTS, **_MOTION_PARAM_OUTPUTS}, +) def func_motion_correct(wf, cfg, strat_pool, pipe_num, opt=None): - wf, outputs = motion_correct_connections(wf, cfg, strat_pool, pipe_num, - opt) + wf, outputs = motion_correct_connections(wf, cfg, strat_pool, pipe_num, opt) return wf, outputs @@ -179,18 +228,22 @@ def func_motion_correct(wf, cfg, strat_pool, pipe_num, opt=None): @nodeblock( name="motion_correction_only", switch=["functional_preproc", "motion_estimates_and_correction", "run"], - option_key=["functional_preproc", "motion_estimates_and_correction", - "motion_correction", "using"], + option_key=[ + "functional_preproc", + "motion_estimates_and_correction", + "motion_correction", + "using", + ], option_val=["3dvolreg", "mcflirt"], inputs=[("desc-preproc_bold", "motion-basefile")], - outputs=_MOTION_CORRECTED_OUTPUTS) + outputs=_MOTION_CORRECTED_OUTPUTS, +) def func_motion_correct_only(wf, cfg, strat_pool, pipe_num, opt=None): - wf, wf_outputs = motion_correct_connections(wf, cfg, strat_pool, pipe_num, - opt) + wf, wf_outputs = motion_correct_connections(wf, cfg, strat_pool, pipe_num, opt) outputs = { - 'desc-preproc_bold': wf_outputs['desc-motion_bold'], - 'desc-motion_bold': wf_outputs['desc-motion_bold'] + "desc-preproc_bold": wf_outputs["desc-motion_bold"], + "desc-motion_bold": wf_outputs["desc-motion_bold"], } return (wf, outputs) @@ -199,499 +252,579 @@ def func_motion_correct_only(wf, cfg, strat_pool, pipe_num, opt=None): @nodeblock( name="motion_estimates", switch=["functional_preproc", "motion_estimates_and_correction", "run"], - option_key=["functional_preproc", "motion_estimates_and_correction", - "motion_correction", "using"], + option_key=[ + "functional_preproc", + "motion_estimates_and_correction", + "motion_correction", + "using", + ], option_val=["3dvolreg", "mcflirt"], inputs=[("desc-preproc_bold", "motion-basefile")], - outputs=_MOTION_PARAM_OUTPUTS) + outputs=_MOTION_PARAM_OUTPUTS, +) def func_motion_estimates(wf, cfg, strat_pool, pipe_num, opt=None): - '''Calculate motion estimates using 3dVolReg or MCFLIRT.''' + """Calculate motion estimates using 3dVolReg or MCFLIRT.""" from CPAC.pipeline.utils import present_outputs - wf, wf_outputs = motion_correct_connections(wf, cfg, strat_pool, pipe_num, - opt) - return (wf, present_outputs(wf_outputs, - ['coordinate-transformation', - 'filtered-coordinate-transformation', - 'max-displacement', - 'desc-movementParameters_motion', - 'rels-displacement'])) + + wf, wf_outputs = motion_correct_connections(wf, cfg, strat_pool, pipe_num, opt) + return ( + wf, + present_outputs( + wf_outputs, + [ + "coordinate-transformation", + "filtered-coordinate-transformation", + "max-displacement", + "desc-movementParameters_motion", + "rels-displacement", + ], + ), + ) def get_mcflirt_rms_abs(rms_files): for path in rms_files: - if 'abs.rms' in path: + if "abs.rms" in path: abs_file = path - if 'rel.rms' in path: + if "rel.rms" in path: rels_file = path return abs_file, rels_file @nodeblock( - name='get_motion_ref', - switch=['functional_preproc', 'motion_estimates_and_correction', 'run'], - option_key=['functional_preproc', 'motion_estimates_and_correction', - 'motion_correction', 'motion_correction_reference'], - option_val=['mean', 'median', 'selected_volume', 'fmriprep_reference'], - inputs=['desc-preproc_bold', 'desc-reorient_bold'], - outputs=['motion-basefile'] + name="get_motion_ref", + switch=["functional_preproc", "motion_estimates_and_correction", "run"], + option_key=[ + "functional_preproc", + "motion_estimates_and_correction", + "motion_correction", + "motion_correction_reference", + ], + option_val=["mean", "median", "selected_volume", "fmriprep_reference"], + inputs=["desc-preproc_bold", "desc-reorient_bold"], + outputs=["motion-basefile"], ) def get_motion_ref(wf, cfg, strat_pool, pipe_num, opt=None): if opt not in get_motion_ref.option_val: - raise ValueError('\n\n[!] Error: The \'motion_correction_reference\' ' - 'parameter of the \'motion_correction\' workflow ' - 'must be one of:\n\t{0}.\n\nTool input: \'{1}\'' - '\n\n'.format( - ' or '.join([f"'{val}'" for val in - get_motion_ref.option_val]), - opt)) - - if opt == 'mean': - func_get_RPI = pe.Node(interface=afni_utils.TStat(), - name=f'func_get_mean_RPI_{pipe_num}', - mem_gb=0.48, - mem_x=(1435097126797993 / - 302231454903657293676544, - 'in_file')) - - func_get_RPI.inputs.options = '-mean' - func_get_RPI.inputs.outputtype = 'NIFTI_GZ' - - node, out = strat_pool.get_data('desc-preproc_bold') - wf.connect(node, out, func_get_RPI, 'in_file') - - elif opt == 'median': - func_get_RPI = pe.Node(interface=afni_utils.TStat(), - name=f'func_get_median_RPI_{pipe_num}') - - func_get_RPI.inputs.options = '-median' - func_get_RPI.inputs.outputtype = 'NIFTI_GZ' - - node, out = strat_pool.get_data('desc-preproc_bold') - wf.connect(node, out, func_get_RPI, 'in_file') - - elif opt == 'selected_volume': - func_get_RPI = pe.Node(interface=afni.Calc(), - name=f'func_get_selected_RPI_{pipe_num}') + raise ValueError( + "\n\n[!] Error: The 'motion_correction_reference' " + "parameter of the 'motion_correction' workflow " + "must be one of:\n\t{0}.\n\nTool input: '{1}'" + "\n\n".format( + " or ".join([f"'{val}'" for val in get_motion_ref.option_val]), opt + ) + ) - func_get_RPI.inputs.set( - expr='a', - single_idx=cfg.functional_preproc[ - 'motion_estimates_and_correction'][ - 'motion_correction']['motion_correction_reference_volume'], - outputtype='NIFTI_GZ' + if opt == "mean": + func_get_RPI = pe.Node( + interface=afni_utils.TStat(), + name=f"func_get_mean_RPI_{pipe_num}", + mem_gb=0.48, + mem_x=(1435097126797993 / 302231454903657293676544, "in_file"), ) - node, out = strat_pool.get_data('desc-preproc_bold') - wf.connect(node, out, func_get_RPI, 'in_file_a') + func_get_RPI.inputs.options = "-mean" + func_get_RPI.inputs.outputtype = "NIFTI_GZ" - elif opt == 'fmriprep_reference': - func_get_RPI = pe.Node(util.Function(input_names=['in_file'], - output_names=['out_file'], - function=estimate_reference_image - ), - name=f'func_get_fmriprep_ref_{pipe_num}') + node, out = strat_pool.get_data("desc-preproc_bold") + wf.connect(node, out, func_get_RPI, "in_file") - node, out = strat_pool.get_data('desc-reorient_bold') - wf.connect(node, out, func_get_RPI, 'in_file') + elif opt == "median": + func_get_RPI = pe.Node( + interface=afni_utils.TStat(), name=f"func_get_median_RPI_{pipe_num}" + ) - outputs = { - 'motion-basefile': (func_get_RPI, 'out_file') - } + func_get_RPI.inputs.options = "-median" + func_get_RPI.inputs.outputtype = "NIFTI_GZ" + + node, out = strat_pool.get_data("desc-preproc_bold") + wf.connect(node, out, func_get_RPI, "in_file") + + elif opt == "selected_volume": + func_get_RPI = pe.Node( + interface=afni.Calc(), name=f"func_get_selected_RPI_{pipe_num}" + ) + + func_get_RPI.inputs.set( + expr="a", + single_idx=cfg.functional_preproc["motion_estimates_and_correction"][ + "motion_correction" + ]["motion_correction_reference_volume"], + outputtype="NIFTI_GZ", + ) + + node, out = strat_pool.get_data("desc-preproc_bold") + wf.connect(node, out, func_get_RPI, "in_file_a") + + elif opt == "fmriprep_reference": + func_get_RPI = pe.Node( + util.Function( + input_names=["in_file"], + output_names=["out_file"], + function=estimate_reference_image, + ), + name=f"func_get_fmriprep_ref_{pipe_num}", + ) + + node, out = strat_pool.get_data("desc-reorient_bold") + wf.connect(node, out, func_get_RPI, "in_file") + + outputs = {"motion-basefile": (func_get_RPI, "out_file")} return (wf, outputs) def motion_correct_3dvolreg(wf, cfg, strat_pool, pipe_num): - """Calculate motion parameters with 3dvolreg""" - if int(cfg.pipeline_setup['system_config']['max_cores_per_participant'] - ) > 1: - chunk_imports = ['import nibabel as nb'] - chunk = pe.Node(Function(input_names=['func_file', - 'n_chunks', - 'chunk_size'], - output_names=['TR_ranges'], - function=chunk_ts, - imports=chunk_imports), - name=f'chunk_{pipe_num}') - - #chunk.inputs.n_chunks = int(cfg.pipeline_setup['system_config'][ + """Calculate motion parameters with 3dvolreg.""" + if int(cfg.pipeline_setup["system_config"]["max_cores_per_participant"]) > 1: + chunk_imports = ["import nibabel as nb"] + chunk = pe.Node( + Function( + input_names=["func_file", "n_chunks", "chunk_size"], + output_names=["TR_ranges"], + function=chunk_ts, + imports=chunk_imports, + ), + name=f"chunk_{pipe_num}", + ) + + # chunk.inputs.n_chunks = int(cfg.pipeline_setup['system_config'][ # 'max_cores_per_participant']) # 10-TR sized chunks chunk.inputs.chunk_size = 10 node, out = strat_pool.get_data("desc-preproc_bold") - wf.connect(node, out, chunk, 'func_file') - - split_imports = ['import os', 'import subprocess'] - split = pe.Node(Function(input_names=['func_file', - 'tr_ranges'], - output_names=['split_funcs'], - function=split_ts_chunks, - imports=split_imports), - name=f'split_{pipe_num}') + wf.connect(node, out, chunk, "func_file") + + split_imports = ["import os", "import subprocess"] + split = pe.Node( + Function( + input_names=["func_file", "tr_ranges"], + output_names=["split_funcs"], + function=split_ts_chunks, + imports=split_imports, + ), + name=f"split_{pipe_num}", + ) node, out = strat_pool.get_data("desc-preproc_bold") - wf.connect(node, out, split, 'func_file') - wf.connect(chunk, 'TR_ranges', split, 'tr_ranges') + wf.connect(node, out, split, "func_file") + wf.connect(chunk, "TR_ranges", split, "tr_ranges") out_split_func = pe.Node( - interface=util.IdentityInterface(fields=['out_file']), - name=f'out_split_func_{pipe_num}') + interface=util.IdentityInterface(fields=["out_file"]), + name=f"out_split_func_{pipe_num}", + ) - wf.connect(split, 'split_funcs', out_split_func, 'out_file') + wf.connect(split, "split_funcs", out_split_func, "out_file") - func_motion_correct = pe.MapNode(interface=preprocess.Volreg(), - name='func_generate_' - f'ref_{pipe_num}', - iterfield=['in_file']) + func_motion_correct = pe.MapNode( + interface=preprocess.Volreg(), + name="func_generate_" f"ref_{pipe_num}", + iterfield=["in_file"], + ) - wf.connect(out_split_func, 'out_file', - func_motion_correct, 'in_file') + wf.connect(out_split_func, "out_file", func_motion_correct, "in_file") - func_concat = pe.Node(interface=afni_utils.TCat(), - name=f'func_concat_{pipe_num}') - func_concat.inputs.outputtype = 'NIFTI_GZ' + func_concat = pe.Node( + interface=afni_utils.TCat(), name=f"func_concat_{pipe_num}" + ) + func_concat.inputs.outputtype = "NIFTI_GZ" - wf.connect(func_motion_correct, 'out_file', - func_concat, 'in_files') + wf.connect(func_motion_correct, "out_file", func_concat, "in_files") out_motion = pe.Node( - interface=util.IdentityInterface(fields=['out_file']), - name=f'out_motion_{pipe_num}') + interface=util.IdentityInterface(fields=["out_file"]), + name=f"out_motion_{pipe_num}", + ) - wf.connect(func_concat, 'out_file', out_motion, 'out_file') + wf.connect(func_concat, "out_file", out_motion, "out_file") else: out_split_func = pe.Node( - interface=util.IdentityInterface(fields=['out_file']), - name=f'out_split_func_{pipe_num}') + interface=util.IdentityInterface(fields=["out_file"]), + name=f"out_split_func_{pipe_num}", + ) - node, out = strat_pool.get_data('desc-preproc_bold') - wf.connect(node, out, out_split_func, 'out_file') + node, out = strat_pool.get_data("desc-preproc_bold") + wf.connect(node, out, out_split_func, "out_file") - func_motion_correct = pe.Node(interface=preprocess.Volreg(), - name=f'func_generate_ref_{pipe_num}') + func_motion_correct = pe.Node( + interface=preprocess.Volreg(), name=f"func_generate_ref_{pipe_num}" + ) - wf.connect(out_split_func, 'out_file', - func_motion_correct, 'in_file') + wf.connect(out_split_func, "out_file", func_motion_correct, "in_file") out_motion = pe.Node( - interface=util.IdentityInterface(fields=['out_file']), - name=f'out_motion_{pipe_num}') + interface=util.IdentityInterface(fields=["out_file"]), + name=f"out_motion_{pipe_num}", + ) - wf.connect(func_motion_correct, 'out_file', - out_motion, 'out_file') + wf.connect(func_motion_correct, "out_file", out_motion, "out_file") func_motion_correct.inputs.zpad = 4 - func_motion_correct.inputs.outputtype = 'NIFTI_GZ' + func_motion_correct.inputs.outputtype = "NIFTI_GZ" - args = '-Fourier' - if cfg.functional_preproc['motion_estimates_and_correction'][ - 'motion_correction']['AFNI-3dvolreg']['functional_volreg_twopass']: - args = f'-twopass {args}' + args = "-Fourier" + if cfg.functional_preproc["motion_estimates_and_correction"]["motion_correction"][ + "AFNI-3dvolreg" + ]["functional_volreg_twopass"]: + args = f"-twopass {args}" func_motion_correct.inputs.args = args # Calculate motion parameters func_motion_correct_A = func_motion_correct.clone( - f'func_motion_correct_3dvolreg_{pipe_num}') - func_motion_correct_A.inputs.md1d_file = 'max_displacement.1D' + f"func_motion_correct_3dvolreg_{pipe_num}" + ) + func_motion_correct_A.inputs.md1d_file = "max_displacement.1D" func_motion_correct_A.inputs.args = args - wf.connect(out_split_func, 'out_file', - func_motion_correct_A, 'in_file') + wf.connect(out_split_func, "out_file", func_motion_correct_A, "in_file") - node, out = strat_pool.get_data('motion-basefile') - wf.connect(node, out, func_motion_correct_A, 'basefile') + node, out = strat_pool.get_data("motion-basefile") + wf.connect(node, out, func_motion_correct_A, "basefile") - if int(cfg.pipeline_setup['system_config'][ - 'max_cores_per_participant']) > 1: - motion_concat = pe.Node(interface=afni_utils.TCat(), - name=f'motion_concat_{pipe_num}') - motion_concat.inputs.outputtype = 'NIFTI_GZ' + if int(cfg.pipeline_setup["system_config"]["max_cores_per_participant"]) > 1: + motion_concat = pe.Node( + interface=afni_utils.TCat(), name=f"motion_concat_{pipe_num}" + ) + motion_concat.inputs.outputtype = "NIFTI_GZ" - wf.connect(func_motion_correct_A, 'out_file', - motion_concat, 'in_files') + wf.connect(func_motion_correct_A, "out_file", motion_concat, "in_files") out_motion_A = pe.Node( - interface=util.IdentityInterface(fields=['out_file']), - name=f'out_motion_A_{pipe_num}') - - wf.connect(motion_concat, 'out_file', - out_motion_A, 'out_file') + interface=util.IdentityInterface(fields=["out_file"]), + name=f"out_motion_A_{pipe_num}", + ) - concat_imports = ['import os'] - md1d_concat = pe.Node(Function(input_names=['in_files'], - output_names=['out_file'], - function=oned_text_concat, - imports=concat_imports), - name=f'md1d_concat_{pipe_num}') + wf.connect(motion_concat, "out_file", out_motion_A, "out_file") + + concat_imports = ["import os"] + md1d_concat = pe.Node( + Function( + input_names=["in_files"], + output_names=["out_file"], + function=oned_text_concat, + imports=concat_imports, + ), + name=f"md1d_concat_{pipe_num}", + ) - wf.connect(func_motion_correct_A, 'md1d_file', - md1d_concat, 'in_files') + wf.connect(func_motion_correct_A, "md1d_file", md1d_concat, "in_files") - oned_concat = pe.Node(Function(input_names=['in_files'], - output_names=['out_file'], - function=oned_text_concat, - imports=concat_imports), - name=f'oned_concat_{pipe_num}') + oned_concat = pe.Node( + Function( + input_names=["in_files"], + output_names=["out_file"], + function=oned_text_concat, + imports=concat_imports, + ), + name=f"oned_concat_{pipe_num}", + ) - wf.connect(func_motion_correct_A, 'oned_file', - oned_concat, 'in_files') + wf.connect(func_motion_correct_A, "oned_file", oned_concat, "in_files") oned_matrix_concat = pe.Node( - Function(input_names=['in_files'], - output_names=['out_file'], - function=oned_text_concat, - imports=concat_imports), - name=f'oned_matrix_concat_{pipe_num}') + Function( + input_names=["in_files"], + output_names=["out_file"], + function=oned_text_concat, + imports=concat_imports, + ), + name=f"oned_matrix_concat_{pipe_num}", + ) - wf.connect(func_motion_correct_A, 'oned_matrix_save', - oned_matrix_concat, 'in_files') + wf.connect( + func_motion_correct_A, "oned_matrix_save", oned_matrix_concat, "in_files" + ) out_md1d = pe.Node( - interface=util.IdentityInterface(fields=['out_file']), - name=f'out_md1d_{pipe_num}') + interface=util.IdentityInterface(fields=["out_file"]), + name=f"out_md1d_{pipe_num}", + ) - wf.connect(md1d_concat, 'out_file', - out_md1d, 'out_file') + wf.connect(md1d_concat, "out_file", out_md1d, "out_file") out_oned = pe.Node( - interface=util.IdentityInterface(fields=['out_file']), - name=f'out_oned_{pipe_num}') + interface=util.IdentityInterface(fields=["out_file"]), + name=f"out_oned_{pipe_num}", + ) - wf.connect(oned_concat, 'out_file', - out_oned, 'out_file') + wf.connect(oned_concat, "out_file", out_oned, "out_file") out_oned_matrix = pe.Node( - interface=util.IdentityInterface(fields=['out_file']), - name=f'out_oned_matrix_{pipe_num}') + interface=util.IdentityInterface(fields=["out_file"]), + name=f"out_oned_matrix_{pipe_num}", + ) - wf.connect(oned_matrix_concat, 'out_file', - out_oned_matrix, 'out_file') + wf.connect(oned_matrix_concat, "out_file", out_oned_matrix, "out_file") else: out_motion_A = pe.Node( - interface=util.IdentityInterface(fields=['out_file']), - name=f'out_motion_A_{pipe_num}') + interface=util.IdentityInterface(fields=["out_file"]), + name=f"out_motion_A_{pipe_num}", + ) - wf.connect(func_motion_correct_A, 'out_file', - out_motion_A, 'out_file') + wf.connect(func_motion_correct_A, "out_file", out_motion_A, "out_file") out_md1d = pe.Node( - interface=util.IdentityInterface(fields=['out_file']), - name=f'out_md1d_{pipe_num}') + interface=util.IdentityInterface(fields=["out_file"]), + name=f"out_md1d_{pipe_num}", + ) - wf.connect(func_motion_correct_A, 'md1d_file', - out_md1d, 'out_file') + wf.connect(func_motion_correct_A, "md1d_file", out_md1d, "out_file") out_oned = pe.Node( - interface=util.IdentityInterface(fields=['out_file']), - name=f'out_oned_{pipe_num}') + interface=util.IdentityInterface(fields=["out_file"]), + name=f"out_oned_{pipe_num}", + ) - wf.connect(func_motion_correct_A, 'oned_file', - out_oned, 'out_file') + wf.connect(func_motion_correct_A, "oned_file", out_oned, "out_file") out_oned_matrix = pe.Node( - interface=util.IdentityInterface(fields=['out_file']), - name=f'out_oned_matrix_{pipe_num}') + interface=util.IdentityInterface(fields=["out_file"]), + name=f"out_oned_matrix_{pipe_num}", + ) - wf.connect(func_motion_correct_A, 'oned_matrix_save', - out_oned_matrix, 'out_file') + wf.connect( + func_motion_correct_A, "oned_matrix_save", out_oned_matrix, "out_file" + ) outputs = { - 'desc-preproc_bold': (out_motion_A, 'out_file'), - 'desc-motion_bold': (out_motion_A, 'out_file'), - 'max-displacement': (out_md1d, 'out_file'), - 'desc-movementParameters_motion': (out_oned, 'out_file'), - 'coordinate-transformation': (out_oned_matrix, 'out_file'), - 'filtered-coordinate-transformation': (out_oned_matrix, 'out_file') + "desc-preproc_bold": (out_motion_A, "out_file"), + "desc-motion_bold": (out_motion_A, "out_file"), + "max-displacement": (out_md1d, "out_file"), + "desc-movementParameters_motion": (out_oned, "out_file"), + "coordinate-transformation": (out_oned_matrix, "out_file"), + "filtered-coordinate-transformation": (out_oned_matrix, "out_file"), } return wf, outputs def motion_correct_mcflirt(wf, cfg, strat_pool, pipe_num): - """Calculate motion parameters with MCFLIRT""" + """Calculate motion parameters with MCFLIRT.""" func_motion_correct_A = pe.Node( interface=fsl.MCFLIRT(save_mats=True, save_plots=True), - name=f'func_motion_correct_mcflirt_{pipe_num}', mem_gb=2.5) + name=f"func_motion_correct_mcflirt_{pipe_num}", + mem_gb=2.5, + ) func_motion_correct_A.inputs.save_mats = True func_motion_correct_A.inputs.save_plots = True func_motion_correct_A.inputs.save_rms = True - node, out = strat_pool.get_data('desc-preproc_bold') - wf.connect(node, out, func_motion_correct_A, 'in_file') + node, out = strat_pool.get_data("desc-preproc_bold") + wf.connect(node, out, func_motion_correct_A, "in_file") - node, out = strat_pool.get_data('motion-basefile') - wf.connect(node, out, func_motion_correct_A, 'ref_file') + node, out = strat_pool.get_data("motion-basefile") + wf.connect(node, out, func_motion_correct_A, "ref_file") - normalize_motion_params = pe.Node(Function( - input_names=['in_file'], output_names=['out_file'], - function=normalize_motion_parameters), - name=f'norm_motion_params_{pipe_num}') + normalize_motion_params = pe.Node( + Function( + input_names=["in_file"], + output_names=["out_file"], + function=normalize_motion_parameters, + ), + name=f"norm_motion_params_{pipe_num}", + ) - wf.connect(func_motion_correct_A, 'par_file', - normalize_motion_params, 'in_file') + wf.connect(func_motion_correct_A, "par_file", normalize_motion_params, "in_file") - get_rms_abs = pe.Node(Function(input_names=['rms_files'], - output_names=['abs_file', - 'rels_file'], - function=get_mcflirt_rms_abs), - name=f'get_mcflirt_rms_abs_{pipe_num}') + get_rms_abs = pe.Node( + Function( + input_names=["rms_files"], + output_names=["abs_file", "rels_file"], + function=get_mcflirt_rms_abs, + ), + name=f"get_mcflirt_rms_abs_{pipe_num}", + ) - wf.connect(func_motion_correct_A, 'rms_files', - get_rms_abs, 'rms_files') + wf.connect(func_motion_correct_A, "rms_files", get_rms_abs, "rms_files") outputs = { - 'desc-preproc_bold': (func_motion_correct_A, 'out_file'), - 'desc-motion_bold': (func_motion_correct_A, 'out_file'), - 'max-displacement': (get_rms_abs, 'abs_file'), - 'rels-displacement': (get_rms_abs, 'rels_file'), - 'desc-movementParameters_motion': (normalize_motion_params, - 'out_file'), - 'coordinate-transformation': (func_motion_correct_A, 'mat_file'), - 'filtered-coordinate-transformation': (func_motion_correct_A, - 'mat_file') + "desc-preproc_bold": (func_motion_correct_A, "out_file"), + "desc-motion_bold": (func_motion_correct_A, "out_file"), + "max-displacement": (get_rms_abs, "abs_file"), + "rels-displacement": (get_rms_abs, "rels_file"), + "desc-movementParameters_motion": (normalize_motion_params, "out_file"), + "coordinate-transformation": (func_motion_correct_A, "mat_file"), + "filtered-coordinate-transformation": (func_motion_correct_A, "mat_file"), } return wf, outputs -motion_correct = {'3dvolreg': motion_correct_3dvolreg, - 'mcflirt': motion_correct_mcflirt} +motion_correct = { + "3dvolreg": motion_correct_3dvolreg, + "mcflirt": motion_correct_mcflirt, +} def motion_correct_connections(wf, cfg, strat_pool, pipe_num, opt): """Check opt for valid option, then connect that option.""" - motion_correct_options = valid_options['motion_correction'] + motion_correct_options = valid_options["motion_correction"] if opt not in motion_correct_options: - raise KeyError("\n\n[!] Error: The 'tool' parameter of the " - "'motion_correction' workflow must be one of " - f"{str(motion_correct_options).strip('[{()}]')}" - f".\n\nTool input: {opt}\n\n") + raise KeyError( + "\n\n[!] Error: The 'tool' parameter of the " + "'motion_correction' workflow must be one of " + f"{str(motion_correct_options).strip('[{()}]')}" + f".\n\nTool input: {opt}\n\n" + ) return motion_correct[opt](wf, cfg, strat_pool, pipe_num) @nodeblock( name="motion_estimate_filter", - config=["functional_preproc", "motion_estimates_and_correction", - "motion_estimate_filter"], + config=[ + "functional_preproc", + "motion_estimates_and_correction", + "motion_estimate_filter", + ], switch=["run"], option_key="filters", option_val="USER-DEFINED", - inputs=[("desc-preproc_bold", - "space-bold_desc-brain_mask", - "max-displacement", - "rels-displacement", - "coordinate-transformation", - "desc-movementParameters_motion"), - "TR"], - outputs={"filtered-coordinate-transformation": { - "Description": "Affine matrix regenerated from" - " filtered motion parameters. Note:" - " the translation vector does not" - " account for recentering inherent" - " in rotation; this omission does" - " not seem to affect framewise" - " displacement calculation, for which" - " this matrix is used."}, - "desc-movementParameters_motion": { - "Description": "Filtered movement parameters" - " (3 rotation, 3 translation)."}, - "desc-movementParametersUnfiltered_motion": { - "Description": "Unfiltered movement parameters" - " (3 rotation, 3 translation)."}, - "motion-filter-info": {}, - "motion-filter-plot": {}}) + inputs=[ + ( + "desc-preproc_bold", + "space-bold_desc-brain_mask", + "max-displacement", + "rels-displacement", + "coordinate-transformation", + "desc-movementParameters_motion", + ), + "TR", + ], + outputs={ + "filtered-coordinate-transformation": { + "Description": "Affine matrix regenerated from" + " filtered motion parameters. Note:" + " the translation vector does not" + " account for recentering inherent" + " in rotation; this omission does" + " not seem to affect framewise" + " displacement calculation, for which" + " this matrix is used." + }, + "desc-movementParameters_motion": { + "Description": "Filtered movement parameters" + " (3 rotation, 3 translation)." + }, + "desc-movementParametersUnfiltered_motion": { + "Description": "Unfiltered movement parameters" + " (3 rotation, 3 translation)." + }, + "motion-filter-info": {}, + "motion-filter-plot": {}, + }, +) def motion_estimate_filter(wf, cfg, strat_pool, pipe_num, opt=None): - '''Filter motion parameters. + """Filter motion parameters. .. versionchanged:: 1.8.6 Beginning with version 1.8.6, C-PAC outputs both the unfiltered and the filtered motion parameters and uses the unfiltered parameters in QC. Previous versions only reported the filtered parameters and used the filtered parameters for QC. - ''' - notch_imports = ['import os', 'import numpy as np', - 'from scipy.signal import iirnotch, filtfilt, firwin, ' - 'freqz', - 'from matplotlib import pyplot as plt', - 'from CPAC.func_preproc.utils import degrees_to_mm, ' - 'mm_to_degrees'] - notch = pe.Node(Function(input_names=['motion_params', - 'filter_type', - 'TR', - 'fc_RR_min', - 'fc_RR_max', - 'center_freq', - 'freq_bw', - 'lowpass_cutoff', - 'filter_order'], - output_names=[ - 'filtered_motion_params', - 'filter_info', - 'filter_plot'], - function=notch_filter_motion, - imports=notch_imports), - name=f'filter_motion_params_{opt["Name"]}_{pipe_num}') - - notch.inputs.filter_type = opt.get('filter_type') - notch.inputs.fc_RR_min = opt.get('breathing_rate_min') - notch.inputs.fc_RR_max = opt.get('breathing_rate_max') - notch.inputs.center_freq = opt.get('center_frequency') - notch.inputs.freq_bw = opt.get('filter_bandwidth') - notch.inputs.lowpass_cutoff = opt.get('lowpass_cutoff') - notch.inputs.filter_order = opt.get('filter_order') - - movement_parameters = strat_pool.node_data( - 'desc-movementParameters_motion') - wf.connect(movement_parameters.node, movement_parameters.out, - notch, 'motion_params') - - node, out = strat_pool.get_data('TR') - wf.connect(node, out, notch, 'TR') - - affine = pe.Node(Function(input_names=['params_file'], - output_names=['affine_file'], - function=affine_file_from_params_file), - name='affine_from_filtered_params_' - f'{opt["Name"]}_{pipe_num}') - wf.connect(notch, 'filtered_motion_params', affine, 'params_file') + """ + notch_imports = [ + "import os", + "import numpy as np", + "from scipy.signal import iirnotch, filtfilt, firwin, " "freqz", + "from matplotlib import pyplot as plt", + "from CPAC.func_preproc.utils import degrees_to_mm, " "mm_to_degrees", + ] + notch = pe.Node( + Function( + input_names=[ + "motion_params", + "filter_type", + "TR", + "fc_RR_min", + "fc_RR_max", + "center_freq", + "freq_bw", + "lowpass_cutoff", + "filter_order", + ], + output_names=["filtered_motion_params", "filter_info", "filter_plot"], + function=notch_filter_motion, + imports=notch_imports, + ), + name=f'filter_motion_params_{opt["Name"]}_{pipe_num}', + ) + + notch.inputs.filter_type = opt.get("filter_type") + notch.inputs.fc_RR_min = opt.get("breathing_rate_min") + notch.inputs.fc_RR_max = opt.get("breathing_rate_max") + notch.inputs.center_freq = opt.get("center_frequency") + notch.inputs.freq_bw = opt.get("filter_bandwidth") + notch.inputs.lowpass_cutoff = opt.get("lowpass_cutoff") + notch.inputs.filter_order = opt.get("filter_order") + + movement_parameters = strat_pool.node_data("desc-movementParameters_motion") + wf.connect( + movement_parameters.node, movement_parameters.out, notch, "motion_params" + ) + + node, out = strat_pool.get_data("TR") + wf.connect(node, out, notch, "TR") + + affine = pe.Node( + Function( + input_names=["params_file"], + output_names=["affine_file"], + function=affine_file_from_params_file, + ), + name='affine_from_filtered_params_' f'{opt["Name"]}_{pipe_num}', + ) + wf.connect(notch, "filtered_motion_params", affine, "params_file") outputs = { - 'filtered-coordinate-transformation': (affine, 'affine_file'), - 'motion-filter-info': (notch, 'filter_info'), - 'motion-filter-plot': (notch, 'filter_plot'), - 'desc-movementParameters_motion': (notch, 'filtered_motion_params') + "filtered-coordinate-transformation": (affine, "affine_file"), + "motion-filter-info": (notch, "filter_info"), + "motion-filter-plot": (notch, "filter_plot"), + "desc-movementParameters_motion": (notch, "filtered_motion_params"), } - if not cfg.switch_is_off(["functional_preproc", - "motion_estimates_and_correction", - "motion_estimate_filter", "run"]): - outputs['desc-movementParametersUnfiltered_motion'] = ( - movement_parameters.node, movement_parameters.out) + if not cfg.switch_is_off( + [ + "functional_preproc", + "motion_estimates_and_correction", + "motion_estimate_filter", + "run", + ] + ): + outputs["desc-movementParametersUnfiltered_motion"] = ( + movement_parameters.node, + movement_parameters.out, + ) return (wf, outputs) def normalize_motion_parameters(in_file): - """Convert FSL mcflirt motion parameters to AFNI space""" + """Convert FSL mcflirt motion parameters to AFNI space.""" import os + import numpy as np motion_params = np.genfromtxt(in_file).T - motion_params = np.vstack((motion_params[2, :] * 180 / np.pi, - motion_params[0, :] * 180 / np.pi, - -motion_params[1, :] * 180 / np.pi, - motion_params[5, :], - motion_params[3, :], - -motion_params[4, :])) + motion_params = np.vstack( + ( + motion_params[2, :] * 180 / np.pi, + motion_params[0, :] * 180 / np.pi, + -motion_params[1, :] * 180 / np.pi, + motion_params[5, :], + motion_params[3, :], + -motion_params[4, :], + ) + ) motion_params = np.transpose(motion_params) - out_file = os.path.join(os.getcwd(), 'motion_params.tsv') + out_file = os.path.join(os.getcwd(), "motion_params.tsv") np.savetxt(out_file, motion_params) return out_file diff --git a/CPAC/func_preproc/func_preproc.py b/CPAC/func_preproc/func_preproc.py index e54d879c82..9e0509d84e 100644 --- a/CPAC/func_preproc/func_preproc.py +++ b/CPAC/func_preproc/func_preproc.py @@ -14,19 +14,22 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -"""Functional preprocessing""" +"""Functional preprocessing.""" # pylint: disable=ungrouped-imports,wrong-import-order,wrong-import-position from nipype import logging from nipype.interfaces import afni, ants, fsl, utility as util -logger = logging.getLogger('nipype.workflow') -from CPAC.pipeline import nipype_pipeline_engine as pe -from CPAC.pipeline.nodeblock import nodeblock -from nipype.interfaces.afni import preprocess -from nipype.interfaces.afni import utils as afni_utils + +logger = logging.getLogger("nipype.workflow") +from nipype.interfaces.afni import preprocess, utils as afni_utils from CPAC.func_preproc.utils import nullify -from CPAC.utils.interfaces.ants import AI # niworkflows -from CPAC.utils.interfaces.ants import PrintHeader, SetDirectionByMatrix +from CPAC.pipeline import nipype_pipeline_engine as pe +from CPAC.pipeline.nodeblock import nodeblock +from CPAC.utils.interfaces.ants import ( + AI, # niworkflows + PrintHeader, + SetDirectionByMatrix, +) from CPAC.utils.utils import add_afni_prefix @@ -35,222 +38,213 @@ def collect_arguments(*args): if args[0]: command_args += [args[1]] command_args += args[2:] - return ' '.join(command_args) + return " ".join(command_args) -def anat_refined_mask(init_bold_mask=True, wf_name='init_bold_mask'): - +def anat_refined_mask(init_bold_mask=True, wf_name="init_bold_mask"): wf = pe.Workflow(name=wf_name) - input_node = pe.Node(util.IdentityInterface(fields=['func', - 'anatomical_brain_mask', - 'anat_brain', - 'init_func_brain_mask']), - name='inputspec') + input_node = pe.Node( + util.IdentityInterface( + fields=[ + "func", + "anatomical_brain_mask", + "anat_brain", + "init_func_brain_mask", + ] + ), + name="inputspec", + ) - output_node = pe.Node(util.IdentityInterface(fields=['func_brain_mask']), - name='outputspec') + output_node = pe.Node( + util.IdentityInterface(fields=["func_brain_mask"]), name="outputspec" + ) # 1 Take single volume of func - func_single_volume = pe.Node(interface=afni.Calc(), - name='func_single_volume') + func_single_volume = pe.Node(interface=afni.Calc(), name="func_single_volume") # TODO add an option to select volume - func_single_volume.inputs.set( - expr='a', - single_idx=1, - outputtype='NIFTI_GZ' - ) + func_single_volume.inputs.set(expr="a", single_idx=1, outputtype="NIFTI_GZ") - wf.connect(input_node, 'func', - func_single_volume, 'in_file_a') + wf.connect(input_node, "func", func_single_volume, "in_file_a") # 2 get temporary func brain - func_tmp_brain = pe.Node(interface=afni_utils.Calc(), - name='func_tmp_brain') - func_tmp_brain.inputs.expr = 'a*b' - func_tmp_brain.inputs.outputtype = 'NIFTI_GZ' + func_tmp_brain = pe.Node(interface=afni_utils.Calc(), name="func_tmp_brain") + func_tmp_brain.inputs.expr = "a*b" + func_tmp_brain.inputs.outputtype = "NIFTI_GZ" - wf.connect(func_single_volume, 'out_file', - func_tmp_brain, 'in_file_a') + wf.connect(func_single_volume, "out_file", func_tmp_brain, "in_file_a") # 2.1 get a tmp func brain mask - if init_bold_mask == True: + if init_bold_mask is True: # 2.1.1 N4BiasFieldCorrection single volume of raw_func func_single_volume_n4_corrected = pe.Node( - interface=ants.N4BiasFieldCorrection(dimension=3, - copy_header=True, - bspline_fitting_distance=200), + interface=ants.N4BiasFieldCorrection( + dimension=3, copy_header=True, bspline_fitting_distance=200 + ), shrink_factor=2, - name='func_single_volume_n4_corrected') - func_single_volume_n4_corrected.inputs.args = '-r True' + name="func_single_volume_n4_corrected", + ) + func_single_volume_n4_corrected.inputs.args = "-r True" - wf.connect(func_single_volume, 'out_file', - func_single_volume_n4_corrected, 'input_image') + wf.connect( + func_single_volume, + "out_file", + func_single_volume_n4_corrected, + "input_image", + ) # 2.1.2 bet n4 corrected image - generate tmp func brain mask - func_tmp_brain_mask = pe.Node(interface=fsl.BET(), - name='func_tmp_brain_mask_pre') + func_tmp_brain_mask = pe.Node( + interface=fsl.BET(), name="func_tmp_brain_mask_pre" + ) func_tmp_brain_mask.inputs.mask = True - wf.connect(func_single_volume_n4_corrected, 'output_image', - func_tmp_brain_mask, 'in_file') + wf.connect( + func_single_volume_n4_corrected, + "output_image", + func_tmp_brain_mask, + "in_file", + ) # 2.1.3 dilate func tmp brain mask - func_tmp_brain_mask_dil = pe.Node(interface=fsl.ImageMaths(), - name='func_tmp_brain_mask_dil') - func_tmp_brain_mask_dil.inputs.op_string = '-dilM' + func_tmp_brain_mask_dil = pe.Node( + interface=fsl.ImageMaths(), name="func_tmp_brain_mask_dil" + ) + func_tmp_brain_mask_dil.inputs.op_string = "-dilM" - wf.connect(func_tmp_brain_mask, 'mask_file', - func_tmp_brain_mask_dil, 'in_file') + wf.connect(func_tmp_brain_mask, "mask_file", func_tmp_brain_mask_dil, "in_file") - wf.connect(func_tmp_brain_mask_dil, 'out_file', - func_tmp_brain, 'in_file_b') + wf.connect(func_tmp_brain_mask_dil, "out_file", func_tmp_brain, "in_file_b") else: # 2.1.1 connect dilated init func brain mask - wf.connect(input_node, 'init_func_brain_mask', - func_tmp_brain, 'in_file_b') + wf.connect(input_node, "init_func_brain_mask", func_tmp_brain, "in_file_b") # 3. get transformation of anat to func # 3.1 Register func tmp brain to anat brain to get func2anat matrix - linear_reg_func_to_anat = pe.Node(interface=fsl.FLIRT(), - name='func_to_anat_linear_reg') - linear_reg_func_to_anat.inputs.cost = 'mutualinfo' + linear_reg_func_to_anat = pe.Node( + interface=fsl.FLIRT(), name="func_to_anat_linear_reg" + ) + linear_reg_func_to_anat.inputs.cost = "mutualinfo" linear_reg_func_to_anat.inputs.dof = 6 - wf.connect(func_tmp_brain, 'out_file', - linear_reg_func_to_anat, 'in_file') + wf.connect(func_tmp_brain, "out_file", linear_reg_func_to_anat, "in_file") - wf.connect(input_node, 'anat_brain', - linear_reg_func_to_anat, 'reference') + wf.connect(input_node, "anat_brain", linear_reg_func_to_anat, "reference") # 3.2 Inverse func to anat affine - inv_func_to_anat_affine = pe.Node(interface=fsl.ConvertXFM(), - name='inv_func2anat_affine') + inv_func_to_anat_affine = pe.Node( + interface=fsl.ConvertXFM(), name="inv_func2anat_affine" + ) inv_func_to_anat_affine.inputs.invert_xfm = True - wf.connect(linear_reg_func_to_anat, 'out_matrix_file', - inv_func_to_anat_affine, 'in_file') + wf.connect( + linear_reg_func_to_anat, "out_matrix_file", inv_func_to_anat_affine, "in_file" + ) # 4. anat mask to func space # Transform anatomical mask to functional space to get BOLD mask - reg_anat_mask_to_func = pe.Node(interface=fsl.FLIRT(), - name='reg_anat_mask_to_func') + reg_anat_mask_to_func = pe.Node(interface=fsl.FLIRT(), name="reg_anat_mask_to_func") reg_anat_mask_to_func.inputs.apply_xfm = True - reg_anat_mask_to_func.inputs.cost = 'mutualinfo' + reg_anat_mask_to_func.inputs.cost = "mutualinfo" reg_anat_mask_to_func.inputs.dof = 6 - reg_anat_mask_to_func.inputs.interp = 'nearestneighbour' + reg_anat_mask_to_func.inputs.interp = "nearestneighbour" - wf.connect(input_node, 'anatomical_brain_mask', - reg_anat_mask_to_func, 'in_file') + wf.connect(input_node, "anatomical_brain_mask", reg_anat_mask_to_func, "in_file") - wf.connect(func_tmp_brain, 'out_file', - reg_anat_mask_to_func, 'reference') + wf.connect(func_tmp_brain, "out_file", reg_anat_mask_to_func, "reference") - wf.connect(inv_func_to_anat_affine, 'out_file', - reg_anat_mask_to_func, 'in_matrix_file') + wf.connect( + inv_func_to_anat_affine, "out_file", reg_anat_mask_to_func, "in_matrix_file" + ) # 5. get final func mask: refine func tmp mask with anat_mask_in_func mask - func_mask = pe.Node(interface=fsl.MultiImageMaths(), name='func_mask') + func_mask = pe.Node(interface=fsl.MultiImageMaths(), name="func_mask") func_mask.inputs.op_string = "-mul %s" - wf.connect(reg_anat_mask_to_func, 'out_file', - func_mask, 'operand_files') + wf.connect(reg_anat_mask_to_func, "out_file", func_mask, "operand_files") - if init_bold_mask == True: - wf.connect(func_tmp_brain_mask_dil, 'out_file', - func_mask, 'in_file') + if init_bold_mask is True: + wf.connect(func_tmp_brain_mask_dil, "out_file", func_mask, "in_file") else: - wf.connect(input_node, 'init_func_brain_mask', - func_mask, 'in_file') + wf.connect(input_node, "init_func_brain_mask", func_mask, "in_file") - wf.connect(func_mask, 'out_file', - output_node, 'func_brain_mask') + wf.connect(func_mask, "out_file", output_node, "func_brain_mask") return wf -def anat_based_mask(wf_name='bold_mask'): - """reference `DCAN lab BOLD mask `_ - """ +def anat_based_mask(wf_name="bold_mask"): + """Reference `DCAN lab BOLD mask `_.""" wf = pe.Workflow(name=wf_name) - input_node = pe.Node(util.IdentityInterface(fields=['func', - 'anat_brain', - 'anat_head']), - name='inputspec') + input_node = pe.Node( + util.IdentityInterface(fields=["func", "anat_brain", "anat_head"]), + name="inputspec", + ) - output_node = pe.Node(util.IdentityInterface(fields=['func_brain_mask']), - name='outputspec') + output_node = pe.Node( + util.IdentityInterface(fields=["func_brain_mask"]), name="outputspec" + ) # 0. Take single volume of func - func_single_volume = pe.Node(interface=afni.Calc(), - name='func_single_volume') + func_single_volume = pe.Node(interface=afni.Calc(), name="func_single_volume") - func_single_volume.inputs.set( - expr='a', - single_idx=1, - outputtype='NIFTI_GZ' - ) + func_single_volume.inputs.set(expr="a", single_idx=1, outputtype="NIFTI_GZ") - wf.connect(input_node, 'func', - func_single_volume, 'in_file_a') + wf.connect(input_node, "func", func_single_volume, "in_file_a") # 1. Register func head to anat head to get func2anat matrix - linear_reg_func_to_anat = pe.Node(interface=fsl.FLIRT(), - name='func_to_anat_linear_reg') + linear_reg_func_to_anat = pe.Node( + interface=fsl.FLIRT(), name="func_to_anat_linear_reg" + ) linear_reg_func_to_anat.inputs.dof = 6 - linear_reg_func_to_anat.inputs.interp = 'spline' + linear_reg_func_to_anat.inputs.interp = "spline" linear_reg_func_to_anat.inputs.searchr_x = [30, 30] linear_reg_func_to_anat.inputs.searchr_y = [30, 30] linear_reg_func_to_anat.inputs.searchr_z = [30, 30] - wf.connect(func_single_volume, 'out_file', - linear_reg_func_to_anat, 'in_file') + wf.connect(func_single_volume, "out_file", linear_reg_func_to_anat, "in_file") - wf.connect(input_node, 'anat_head', - linear_reg_func_to_anat, 'reference') + wf.connect(input_node, "anat_head", linear_reg_func_to_anat, "reference") # 2. Inverse func to anat affine, to get anat-to-func transform - inv_func_to_anat_affine = pe.Node(interface=fsl.ConvertXFM(), - name='inv_func2anat_affine') + inv_func_to_anat_affine = pe.Node( + interface=fsl.ConvertXFM(), name="inv_func2anat_affine" + ) inv_func_to_anat_affine.inputs.invert_xfm = True - wf.connect(linear_reg_func_to_anat, 'out_matrix_file', - inv_func_to_anat_affine, 'in_file') + wf.connect( + linear_reg_func_to_anat, "out_matrix_file", inv_func_to_anat_affine, "in_file" + ) # 3. get BOLD mask # 3.1 Apply anat-to-func transform to transfer anatomical brain to functional space - reg_anat_brain_to_func = pe.Node(interface=fsl.ApplyWarp(), - name='reg_anat_brain_to_func') - reg_anat_brain_to_func.inputs.interp = 'nn' + reg_anat_brain_to_func = pe.Node( + interface=fsl.ApplyWarp(), name="reg_anat_brain_to_func" + ) + reg_anat_brain_to_func.inputs.interp = "nn" reg_anat_brain_to_func.inputs.relwarp = True - wf.connect(input_node, 'anat_brain', - reg_anat_brain_to_func, 'in_file') + wf.connect(input_node, "anat_brain", reg_anat_brain_to_func, "in_file") - wf.connect(input_node, 'func', - reg_anat_brain_to_func, 'ref_file') + wf.connect(input_node, "func", reg_anat_brain_to_func, "ref_file") - wf.connect(inv_func_to_anat_affine, 'out_file', - reg_anat_brain_to_func, 'premat') + wf.connect(inv_func_to_anat_affine, "out_file", reg_anat_brain_to_func, "premat") # 3.2 Binarize transfered image and fill holes to get BOLD mask. # Binarize - func_mask_bin = pe.Node(interface=fsl.ImageMaths(), - name='func_mask') - func_mask_bin.inputs.op_string = '-bin' + func_mask_bin = pe.Node(interface=fsl.ImageMaths(), name="func_mask") + func_mask_bin.inputs.op_string = "-bin" - wf.connect(reg_anat_brain_to_func, 'out_file', - func_mask_bin, 'in_file') + wf.connect(reg_anat_brain_to_func, "out_file", func_mask_bin, "in_file") - wf.connect(func_mask_bin, 'out_file', - output_node, 'func_brain_mask') + wf.connect(func_mask_bin, "out_file", output_node, "func_brain_mask") return wf -def create_scale_func_wf(scaling_factor, wf_name='scale_func'): +def create_scale_func_wf(scaling_factor, wf_name="scale_func"): """Workflow to scale func data. Workflow Inputs:: @@ -271,31 +265,27 @@ def create_scale_func_wf(scaling_factor, wf_name='scale_func'): wf_name : str name of the workflow """ - # allocate a workflow object preproc = pe.Workflow(name=wf_name) # configure the workflow's input spec - inputNode = pe.Node(util.IdentityInterface(fields=['func']), - name='inputspec') + inputNode = pe.Node(util.IdentityInterface(fields=["func"]), name="inputspec") # configure the workflow's output spec - outputNode = pe.Node(util.IdentityInterface(fields=['scaled_func']), - name='outputspec') + outputNode = pe.Node( + util.IdentityInterface(fields=["scaled_func"]), name="outputspec" + ) # allocate a node to edit the functional file - func_scale = pe.Node(interface=afni_utils.Refit(), - name='func_scale') + func_scale = pe.Node(interface=afni_utils.Refit(), name="func_scale") func_scale.inputs.xyzscale = scaling_factor # wire in the func_get_idx node - preproc.connect(inputNode, 'func', - func_scale, 'in_file') + preproc.connect(inputNode, "func", func_scale, "in_file") # wire the output - preproc.connect(func_scale, 'out_file', - outputNode, 'scaled_func') + preproc.connect(func_scale, "out_file", outputNode, "scaled_func") return preproc @@ -333,125 +323,120 @@ def create_wf_edit_func(wf_name="edit_func"): -prefix rest_3dc.nii.gz """ - # allocate a workflow object preproc = pe.Workflow(name=wf_name) # configure the workflow's input spec - inputNode = pe.Node(util.IdentityInterface(fields=['func', - 'start_idx', - 'stop_idx']), - name='inputspec') + inputNode = pe.Node( + util.IdentityInterface(fields=["func", "start_idx", "stop_idx"]), + name="inputspec", + ) # configure the workflow's output spec - outputNode = pe.Node(util.IdentityInterface(fields=['edited_func']), - name='outputspec') + outputNode = pe.Node( + util.IdentityInterface(fields=["edited_func"]), name="outputspec" + ) # allocate a node to check that the requested edits are # reasonable given the data - func_get_idx = pe.Node(util.Function(input_names=['in_files', - 'stop_idx', - 'start_idx'], - output_names=['stopidx', - 'startidx'], - function=get_idx), - name='func_get_idx') + func_get_idx = pe.Node( + util.Function( + input_names=["in_files", "stop_idx", "start_idx"], + output_names=["stopidx", "startidx"], + function=get_idx, + ), + name="func_get_idx", + ) # wire in the func_get_idx node - preproc.connect(inputNode, 'func', - func_get_idx, 'in_files') - preproc.connect(inputNode, 'start_idx', - func_get_idx, 'start_idx') - preproc.connect(inputNode, 'stop_idx', - func_get_idx, 'stop_idx') + preproc.connect(inputNode, "func", func_get_idx, "in_files") + preproc.connect(inputNode, "start_idx", func_get_idx, "start_idx") + preproc.connect(inputNode, "stop_idx", func_get_idx, "stop_idx") # allocate a node to edit the functional file - func_drop_trs = pe.Node(interface=afni_utils.Calc(), - name='func_drop_trs', - mem_gb=0.37, - mem_x=(739971956005215 / 151115727451828646838272, - 'in_file_a')) + func_drop_trs = pe.Node( + interface=afni_utils.Calc(), + name="func_drop_trs", + mem_gb=0.37, + mem_x=(739971956005215 / 151115727451828646838272, "in_file_a"), + ) - func_drop_trs.inputs.expr = 'a' - func_drop_trs.inputs.outputtype = 'NIFTI_GZ' + func_drop_trs.inputs.expr = "a" + func_drop_trs.inputs.outputtype = "NIFTI_GZ" # wire in the inputs - preproc.connect(inputNode, 'func', - func_drop_trs, 'in_file_a') + preproc.connect(inputNode, "func", func_drop_trs, "in_file_a") - preproc.connect(func_get_idx, 'startidx', - func_drop_trs, 'start_idx') + preproc.connect(func_get_idx, "startidx", func_drop_trs, "start_idx") - preproc.connect(func_get_idx, 'stopidx', - func_drop_trs, 'stop_idx') + preproc.connect(func_get_idx, "stopidx", func_drop_trs, "stop_idx") # wire the output - preproc.connect(func_drop_trs, 'out_file', - outputNode, 'edited_func') + preproc.connect(func_drop_trs, "out_file", outputNode, "edited_func") return preproc -def slice_timing_wf(name='slice_timing', tpattern=None, tzero=None): +def slice_timing_wf(name="slice_timing", tpattern=None, tzero=None): # allocate a workflow object wf = pe.Workflow(name=name) # configure the workflow's input spec - inputNode = pe.Node(util.IdentityInterface(fields=['func_ts', - 'tr', - 'tpattern']), - name='inputspec') + inputNode = pe.Node( + util.IdentityInterface(fields=["func_ts", "tr", "tpattern"]), name="inputspec" + ) # configure the workflow's output spec outputNode = pe.Node( - util.IdentityInterface(fields=['slice_time_corrected']), - name='outputspec') + util.IdentityInterface(fields=["slice_time_corrected"]), name="outputspec" + ) # create TShift AFNI node - func_slice_timing_correction = pe.Node(interface=preprocess.TShift(), - name='slice_timing', - mem_gb=0.45, - mem_x=(5247073869855161 / - 604462909807314587353088, - 'in_file')) - func_slice_timing_correction.inputs.outputtype = 'NIFTI_GZ' + func_slice_timing_correction = pe.Node( + interface=preprocess.TShift(), + name="slice_timing", + mem_gb=0.45, + mem_x=(5247073869855161 / 604462909807314587353088, "in_file"), + ) + func_slice_timing_correction.inputs.outputtype = "NIFTI_GZ" if tzero is not None: func_slice_timing_correction.inputs.tzero = tzero - wf.connect([ - ( - inputNode, - func_slice_timing_correction, - [ - ( - 'func_ts', - 'in_file' - ), - # ( - # # add the @ prefix to the tpattern file going into - # # AFNI 3dTshift - needed this so the tpattern file - # # output from get_scan_params would be tied downstream - # # via a connection (to avoid poofing) - # ('tpattern', nullify, add_afni_prefix), - # 'tpattern' - # ), - ( - ('tr', nullify), - 'tr' - ), - ] - ), - ]) + wf.connect( + [ + ( + inputNode, + func_slice_timing_correction, + [ + ("func_ts", "in_file"), + # ( + # # add the @ prefix to the tpattern file going into + # # AFNI 3dTshift - needed this so the tpattern file + # # output from get_scan_params would be tied downstream + # # via a connection (to avoid poofing) + # ('tpattern', nullify, add_afni_prefix), + # 'tpattern' + # ), + (("tr", nullify), "tr"), + ], + ), + ] + ) if tpattern is not None: func_slice_timing_correction.inputs.tpattern = tpattern else: - wf.connect(inputNode, ('tpattern', nullify, add_afni_prefix), - func_slice_timing_correction, 'tpattern') + wf.connect( + inputNode, + ("tpattern", nullify, add_afni_prefix), + func_slice_timing_correction, + "tpattern", + ) - wf.connect(func_slice_timing_correction, 'out_file', - outputNode, 'slice_time_corrected') + wf.connect( + func_slice_timing_correction, "out_file", outputNode, "slice_time_corrected" + ) return wf @@ -461,7 +446,7 @@ def get_idx(in_files, stop_idx=None, start_idx=None): Method to get the first and the last slice for the functional run. It verifies the user specified first and last slice. If the values are not valid, it - calculates and returns the very first and the last slice + calculates and returns the very first and the last slice. Parameters ---------- @@ -485,7 +470,6 @@ def get_idx(in_files, stop_idx=None, start_idx=None): Value of last slice to consider for the functional run """ - # Import packages from nibabel import load @@ -496,12 +480,11 @@ def get_idx(in_files, stop_idx=None, start_idx=None): # Check to make sure the input file is 4-dimensional if len(shape) != 4: - raise TypeError('Input nifti file: %s is not a 4D file' % in_files) + raise TypeError("Input nifti file: %s is not a 4D file" % in_files) # Grab the number of volumes nvols = int(hdr.get_data_shape()[3]) - if (start_idx == None) or (int(start_idx) < 0) or ( - int(start_idx) > (nvols - 1)): + if (start_idx is None) or (int(start_idx) < 0) or (int(start_idx) > (nvols - 1)): startidx = 0 else: startidx = int(start_idx) @@ -515,76 +498,75 @@ def get_idx(in_files, stop_idx=None, start_idx=None): @nodeblock( - name='func_reorient', - config=['functional_preproc', 'update_header'], - switch=['run'], - inputs=['bold'], - outputs=['desc-preproc_bold', 'desc-reorient_bold'] + name="func_reorient", + config=["functional_preproc", "update_header"], + switch=["run"], + inputs=["bold"], + outputs=["desc-preproc_bold", "desc-reorient_bold"], ) def func_reorient(wf, cfg, strat_pool, pipe_num, opt=None): - - func_deoblique = pe.Node(interface=afni_utils.Refit(), - name=f'func_deoblique_{pipe_num}', - mem_gb=0.68, - mem_x=(4664065662093477 / - 1208925819614629174706176, - 'in_file')) + func_deoblique = pe.Node( + interface=afni_utils.Refit(), + name=f"func_deoblique_{pipe_num}", + mem_gb=0.68, + mem_x=(4664065662093477 / 1208925819614629174706176, "in_file"), + ) func_deoblique.inputs.deoblique = True - node, out = strat_pool.get_data('bold') - wf.connect(node, out, func_deoblique, 'in_file') + node, out = strat_pool.get_data("bold") + wf.connect(node, out, func_deoblique, "in_file") - func_reorient = pe.Node(interface=afni_utils.Resample(), - name=f'func_reorient_{pipe_num}', - mem_gb=0, - mem_x=(0.0115, 'in_file', 't')) + func_reorient = pe.Node( + interface=afni_utils.Resample(), + name=f"func_reorient_{pipe_num}", + mem_gb=0, + mem_x=(0.0115, "in_file", "t"), + ) - func_reorient.inputs.orientation = 'RPI' - func_reorient.inputs.outputtype = 'NIFTI_GZ' + func_reorient.inputs.orientation = "RPI" + func_reorient.inputs.outputtype = "NIFTI_GZ" - wf.connect(func_deoblique, 'out_file', func_reorient, 'in_file') + wf.connect(func_deoblique, "out_file", func_reorient, "in_file") outputs = { - 'desc-preproc_bold': (func_reorient, 'out_file'), - 'desc-reorient_bold': (func_reorient, 'out_file') + "desc-preproc_bold": (func_reorient, "out_file"), + "desc-reorient_bold": (func_reorient, "out_file"), } return (wf, outputs) @nodeblock( - name='func_scaling', - config=['functional_preproc', 'scaling'], - switch=['run'], - inputs=['desc-preproc_bold'], - outputs=['desc-preproc_bold'] + name="func_scaling", + config=["functional_preproc", "scaling"], + switch=["run"], + inputs=["desc-preproc_bold"], + outputs=["desc-preproc_bold"], ) def func_scaling(wf, cfg, strat_pool, pipe_num, opt=None): - scale_func_wf = create_scale_func_wf( - scaling_factor=cfg.scaling_factor, - wf_name=f"scale_func_{pipe_num}" + scaling_factor=cfg.scaling_factor, wf_name=f"scale_func_{pipe_num}" ) node, out = strat_pool.get_data("desc-preproc_bold") - wf.connect(node, out, scale_func_wf, 'inputspec.func') + wf.connect(node, out, scale_func_wf, "inputspec.func") - outputs = { - 'desc-preproc_bold': (scale_func_wf, 'outputspec.scaled_func') - } + outputs = {"desc-preproc_bold": (scale_func_wf, "outputspec.scaled_func")} return (wf, outputs) @nodeblock( - name='func_truncate', - config=['functional_preproc', 'truncation'], - inputs=['desc-preproc_bold'], - outputs={'desc-preproc_bold': { - 'Description': 'Truncated functional time-series BOLD data.'}} + name="func_truncate", + config=["functional_preproc", "truncation"], + inputs=["desc-preproc_bold"], + outputs={ + "desc-preproc_bold": { + "Description": "Truncated functional time-series BOLD data." + } + }, ) def func_truncate(wf, cfg, strat_pool, pipe_num, opt=None): - # if cfg.functional_preproc['truncation']['start_tr'] == 0 and \ # cfg.functional_preproc['truncation']['stop_tr'] == None: # data, key = strat_pool.get_data("desc-preproc_bold", @@ -592,333 +574,375 @@ def func_truncate(wf, cfg, strat_pool, pipe_num, opt=None): # outputs = {key: data} # return (wf, outputs) - trunc_wf = create_wf_edit_func( - wf_name=f"edit_func_{pipe_num}" - ) - trunc_wf.inputs.inputspec.start_idx = cfg.functional_preproc[ - 'truncation']['start_tr'] - trunc_wf.inputs.inputspec.stop_idx = cfg.functional_preproc['truncation'][ - 'stop_tr'] + trunc_wf = create_wf_edit_func(wf_name=f"edit_func_{pipe_num}") + trunc_wf.inputs.inputspec.start_idx = cfg.functional_preproc["truncation"][ + "start_tr" + ] + trunc_wf.inputs.inputspec.stop_idx = cfg.functional_preproc["truncation"]["stop_tr"] node, out = strat_pool.get_data("desc-preproc_bold") - wf.connect(node, out, trunc_wf, 'inputspec.func') + wf.connect(node, out, trunc_wf, "inputspec.func") - outputs = { - 'desc-preproc_bold': (trunc_wf, 'outputspec.edited_func') - } + outputs = {"desc-preproc_bold": (trunc_wf, "outputspec.edited_func")} return (wf, outputs) @nodeblock( - name='func_despike', - config=['functional_preproc', 'despiking'], - switch=['run'], - option_key=['space'], - option_val=['native'], - inputs=['desc-preproc_bold'], - outputs={'desc-preproc_bold': { - 'Description': 'De-spiked BOLD time-series via AFNI 3dDespike.'}} + name="func_despike", + config=["functional_preproc", "despiking"], + switch=["run"], + option_key=["space"], + option_val=["native"], + inputs=["desc-preproc_bold"], + outputs={ + "desc-preproc_bold": { + "Description": "De-spiked BOLD time-series via AFNI 3dDespike." + } + }, ) def func_despike(wf, cfg, strat_pool, pipe_num, opt=None): - - despike = pe.Node(interface=preprocess.Despike(), - name=f'func_despiked_{pipe_num}', - mem_gb=0.66, - mem_x=(8251808479088459 / 1208925819614629174706176, - 'in_file')) - despike.inputs.outputtype = 'NIFTI_GZ' + despike = pe.Node( + interface=preprocess.Despike(), + name=f"func_despiked_{pipe_num}", + mem_gb=0.66, + mem_x=(8251808479088459 / 1208925819614629174706176, "in_file"), + ) + despike.inputs.outputtype = "NIFTI_GZ" node, out = strat_pool.get_data("desc-preproc_bold") - wf.connect(node, out, despike, 'in_file') + wf.connect(node, out, despike, "in_file") - outputs = { - 'desc-preproc_bold': (despike, 'out_file') - } + outputs = {"desc-preproc_bold": (despike, "out_file")} return (wf, outputs) @nodeblock( - name='func_despike_template', - config=['functional_preproc', 'despiking'], - switch=['run'], - option_key=['space'], - option_val=['template'], - inputs=[('space-template_desc-preproc_bold', - 'space-template_res-derivative_desc-preproc_bold'), - 'T1w-template-funcreg', 'T1w-template-deriv'], - outputs={'space-template_desc-preproc_bold': { - 'Description': 'De-spiked BOLD time-series via AFNI 3dDespike.', - 'Template': 'T1w-template-funcreg'}, - 'space-template_res-derivative_desc-preproc_bold': { - 'Description': 'De-spiked BOLD time-series via AFNI 3dDespike.', - 'Template': 'T1w-template-deriv'}} + name="func_despike_template", + config=["functional_preproc", "despiking"], + switch=["run"], + option_key=["space"], + option_val=["template"], + inputs=[ + ( + "space-template_desc-preproc_bold", + "space-template_res-derivative_desc-preproc_bold", + ), + "T1w-template-funcreg", + "T1w-template-deriv", + ], + outputs={ + "space-template_desc-preproc_bold": { + "Description": "De-spiked BOLD time-series via AFNI 3dDespike.", + "Template": "T1w-template-funcreg", + }, + "space-template_res-derivative_desc-preproc_bold": { + "Description": "De-spiked BOLD time-series via AFNI 3dDespike.", + "Template": "T1w-template-deriv", + }, + }, ) def func_despike_template(wf, cfg, strat_pool, pipe_num, opt=None): - - despike = pe.Node(interface=preprocess.Despike(), - name=f'func_despiked_template_{pipe_num}', - mem_gb=0.66, - mem_x=(8251808479088459 / 1208925819614629174706176, - 'in_file')) - despike.inputs.outputtype = 'NIFTI_GZ' + despike = pe.Node( + interface=preprocess.Despike(), + name=f"func_despiked_template_{pipe_num}", + mem_gb=0.66, + mem_x=(8251808479088459 / 1208925819614629174706176, "in_file"), + ) + despike.inputs.outputtype = "NIFTI_GZ" node, out = strat_pool.get_data("space-template_desc-preproc_bold") - wf.connect(node, out, despike, 'in_file') + wf.connect(node, out, despike, "in_file") - outputs = { - 'space-template_desc-preproc_bold': (despike, 'out_file') - } - - if strat_pool.get_data("space-template_res-derivative_desc-preproc_bold"): - despike_funcderiv = pe.Node(interface=preprocess.Despike(), - name=f'func_deriv_despiked_template_{pipe_num}', - mem_gb=0.66, - mem_x=(8251808479088459 / 1208925819614629174706176, - 'in_file')) - despike_funcderiv.inputs.outputtype = 'NIFTI_GZ' + outputs = {"space-template_desc-preproc_bold": (despike, "out_file")} - node, out = strat_pool.get_data("space-template_res-derivative_desc-preproc_bold") - wf.connect(node, out, despike_funcderiv, 'in_file') - - outputs.update({ - 'space-template_res-derivative_desc-preproc_bold': - (despike_funcderiv, 'out_file')}) + if strat_pool.get_data("space-template_res-derivative_desc-preproc_bold"): + despike_funcderiv = pe.Node( + interface=preprocess.Despike(), + name=f"func_deriv_despiked_template_{pipe_num}", + mem_gb=0.66, + mem_x=(8251808479088459 / 1208925819614629174706176, "in_file"), + ) + despike_funcderiv.inputs.outputtype = "NIFTI_GZ" + + node, out = strat_pool.get_data( + "space-template_res-derivative_desc-preproc_bold" + ) + wf.connect(node, out, despike_funcderiv, "in_file") + + outputs.update( + { + "space-template_res-derivative_desc-preproc_bold": ( + despike_funcderiv, + "out_file", + ) + } + ) return (wf, outputs) @nodeblock( - name='func_slice_time', - config=['functional_preproc', 'slice_timing_correction'], - switch=['run'], - inputs=['desc-preproc_bold', 'TR', 'tpattern'], - outputs={'desc-preproc_bold': { - 'Description': 'Slice-time corrected BOLD time-series via AFNI 3dTShift.'}, - 'desc-stc_bold': { - 'Description': 'Slice-time corrected BOLD time-series via AFNI 3dTShift.'}} + name="func_slice_time", + config=["functional_preproc", "slice_timing_correction"], + switch=["run"], + inputs=["desc-preproc_bold", "TR", "tpattern"], + outputs={ + "desc-preproc_bold": { + "Description": "Slice-time corrected BOLD time-series via AFNI 3dTShift." + }, + "desc-stc_bold": { + "Description": "Slice-time corrected BOLD time-series via AFNI 3dTShift." + }, + }, ) def func_slice_time(wf, cfg, strat_pool, pipe_num, opt=None): - - slice_time = slice_timing_wf(name='func_slice_timing_correction_' - f'{pipe_num}', - tpattern=cfg.functional_preproc[ - 'slice_timing_correction']['tpattern'], - tzero=cfg.functional_preproc[ - 'slice_timing_correction']['tzero']) + slice_time = slice_timing_wf( + name="func_slice_timing_correction_" f"{pipe_num}", + tpattern=cfg.functional_preproc["slice_timing_correction"]["tpattern"], + tzero=cfg.functional_preproc["slice_timing_correction"]["tzero"], + ) node, out = strat_pool.get_data("desc-preproc_bold") - wf.connect(node, out, slice_time, 'inputspec.func_ts') + wf.connect(node, out, slice_time, "inputspec.func_ts") - node, out = strat_pool.get_data('TR') - wf.connect(node, out, slice_time, 'inputspec.tr') + node, out = strat_pool.get_data("TR") + wf.connect(node, out, slice_time, "inputspec.tr") - node, out = strat_pool.get_data('tpattern') - wf.connect(node, out, slice_time, 'inputspec.tpattern') + node, out = strat_pool.get_data("tpattern") + wf.connect(node, out, slice_time, "inputspec.tpattern") outputs = { - 'desc-preproc_bold': (slice_time, 'outputspec.slice_time_corrected'), - 'desc-stc_bold': (slice_time, 'outputspec.slice_time_corrected') + "desc-preproc_bold": (slice_time, "outputspec.slice_time_corrected"), + "desc-stc_bold": (slice_time, "outputspec.slice_time_corrected"), } return (wf, outputs) @nodeblock( - name='bold_mask_afni', - switch=[['functional_preproc', 'run'], - ['functional_preproc', 'func_masking', 'run']], - option_key=['functional_preproc', 'func_masking', 'using'], - option_val='AFNI', - inputs=['desc-preproc_bold'], - outputs={'space-bold_desc-brain_mask': - {'Description': 'Binary brain mask of the BOLD functional time-series created by AFNI 3dAutomask.'}} + name="bold_mask_afni", + switch=[ + ["functional_preproc", "run"], + ["functional_preproc", "func_masking", "run"], + ], + option_key=["functional_preproc", "func_masking", "using"], + option_val="AFNI", + inputs=["desc-preproc_bold"], + outputs={ + "space-bold_desc-brain_mask": { + "Description": "Binary brain mask of the BOLD functional time-series created by AFNI 3dAutomask." + } + }, ) def bold_mask_afni(wf, cfg, strat_pool, pipe_num, opt=None): - - func_get_brain_mask = pe.Node(interface=preprocess.Automask(), - name=f'func_get_brain_mask_AFNI_{pipe_num}') - func_get_brain_mask.inputs.outputtype = 'NIFTI_GZ' + func_get_brain_mask = pe.Node( + interface=preprocess.Automask(), name=f"func_get_brain_mask_AFNI_{pipe_num}" + ) + func_get_brain_mask.inputs.outputtype = "NIFTI_GZ" node, out = strat_pool.get_data("desc-preproc_bold") - wf.connect(node, out, func_get_brain_mask, 'in_file') + wf.connect(node, out, func_get_brain_mask, "in_file") - outputs = { - 'space-bold_desc-brain_mask': (func_get_brain_mask, 'out_file') - } + outputs = {"space-bold_desc-brain_mask": (func_get_brain_mask, "out_file")} return (wf, outputs) @nodeblock( - name='bold_mask_fsl', - switch=[['functional_preproc', 'run'], - ['functional_preproc', 'func_masking', 'run']], - option_key=['functional_preproc', 'func_masking', 'using'], - option_val='FSL', - inputs=['desc-preproc_bold'], - outputs=['space-bold_desc-brain_mask'] + name="bold_mask_fsl", + switch=[ + ["functional_preproc", "run"], + ["functional_preproc", "func_masking", "run"], + ], + option_key=["functional_preproc", "func_masking", "using"], + option_val="FSL", + inputs=["desc-preproc_bold"], + outputs=["space-bold_desc-brain_mask"], ) def bold_mask_fsl(wf, cfg, strat_pool, pipe_num, opt=None): - inputnode_bet = pe.Node( - util.IdentityInterface(fields=['frac', - 'mesh_boolean', - 'outline', - 'padding', - 'radius', - 'reduce_bias', - 'remove_eyes', - 'robust', - 'skull', - 'surfaces', - 'threshold', - 'vertical_gradient']), - name=f'BET_options_{pipe_num}') - - func_get_brain_mask = pe.Node(interface=fsl.BET(), - name=f'func_get_brain_mask_BET_{pipe_num}') - func_get_brain_mask.inputs.output_type = 'NIFTI_GZ' + util.IdentityInterface( + fields=[ + "frac", + "mesh_boolean", + "outline", + "padding", + "radius", + "reduce_bias", + "remove_eyes", + "robust", + "skull", + "surfaces", + "threshold", + "vertical_gradient", + ] + ), + name=f"BET_options_{pipe_num}", + ) + + func_get_brain_mask = pe.Node( + interface=fsl.BET(), name=f"func_get_brain_mask_BET_{pipe_num}" + ) + func_get_brain_mask.inputs.output_type = "NIFTI_GZ" func_get_brain_mask.inputs.mask = True inputnode_bet.inputs.set( - frac=cfg.functional_preproc['func_masking']['FSL-BET']['frac'], - mesh_boolean=cfg.functional_preproc['func_masking']['FSL-BET'][ - 'mesh_boolean'], - outline=cfg.functional_preproc['func_masking']['FSL-BET'][ - 'outline'], - padding=cfg.functional_preproc['func_masking']['FSL-BET'][ - 'padding'], - radius=cfg.functional_preproc['func_masking']['FSL-BET']['radius'], - reduce_bias=cfg.functional_preproc['func_masking']['FSL-BET'][ - 'reduce_bias'], - remove_eyes=cfg.functional_preproc['func_masking']['FSL-BET'][ - 'remove_eyes'], - robust=cfg.functional_preproc['func_masking']['FSL-BET']['robust'], - skull=cfg.functional_preproc['func_masking']['FSL-BET']['skull'], - surfaces=cfg.functional_preproc['func_masking']['FSL-BET'][ - 'surfaces'], - threshold=cfg.functional_preproc['func_masking']['FSL-BET'][ - 'threshold'], - vertical_gradient= - cfg.functional_preproc['func_masking']['FSL-BET'][ - 'vertical_gradient'], - ) - - wf.connect([ - (inputnode_bet, func_get_brain_mask, [ - ('frac', 'frac'), - ('mesh_boolean', 'mesh'), - ('outline', 'outline'), - ('padding', 'padding'), - ('radius', 'radius'), - ('reduce_bias', 'reduce_bias'), - ('remove_eyes', 'remove_eyes'), - ('robust', 'robust'), - ('skull', 'skull'), - ('surfaces', 'surfaces'), - ('threshold', 'threshold'), - ('vertical_gradient', 'vertical_gradient'), - ]) - ]) - - if cfg.functional_preproc['func_masking']['FSL-BET'][ - 'functional_mean_boolean']: - func_skull_mean = pe.Node(interface=afni_utils.TStat(), - name=f'func_mean_skull_{pipe_num}') - func_skull_mean.inputs.options = '-mean' - func_skull_mean.inputs.outputtype = 'NIFTI_GZ' + frac=cfg.functional_preproc["func_masking"]["FSL-BET"]["frac"], + mesh_boolean=cfg.functional_preproc["func_masking"]["FSL-BET"]["mesh_boolean"], + outline=cfg.functional_preproc["func_masking"]["FSL-BET"]["outline"], + padding=cfg.functional_preproc["func_masking"]["FSL-BET"]["padding"], + radius=cfg.functional_preproc["func_masking"]["FSL-BET"]["radius"], + reduce_bias=cfg.functional_preproc["func_masking"]["FSL-BET"]["reduce_bias"], + remove_eyes=cfg.functional_preproc["func_masking"]["FSL-BET"]["remove_eyes"], + robust=cfg.functional_preproc["func_masking"]["FSL-BET"]["robust"], + skull=cfg.functional_preproc["func_masking"]["FSL-BET"]["skull"], + surfaces=cfg.functional_preproc["func_masking"]["FSL-BET"]["surfaces"], + threshold=cfg.functional_preproc["func_masking"]["FSL-BET"]["threshold"], + vertical_gradient=cfg.functional_preproc["func_masking"]["FSL-BET"][ + "vertical_gradient" + ], + ) + + wf.connect( + [ + ( + inputnode_bet, + func_get_brain_mask, + [ + ("frac", "frac"), + ("mesh_boolean", "mesh"), + ("outline", "outline"), + ("padding", "padding"), + ("radius", "radius"), + ("reduce_bias", "reduce_bias"), + ("remove_eyes", "remove_eyes"), + ("robust", "robust"), + ("skull", "skull"), + ("surfaces", "surfaces"), + ("threshold", "threshold"), + ("vertical_gradient", "vertical_gradient"), + ], + ) + ] + ) + + if cfg.functional_preproc["func_masking"]["FSL-BET"]["functional_mean_boolean"]: + func_skull_mean = pe.Node( + interface=afni_utils.TStat(), name=f"func_mean_skull_{pipe_num}" + ) + func_skull_mean.inputs.options = "-mean" + func_skull_mean.inputs.outputtype = "NIFTI_GZ" node, out = strat_pool.get_data("desc-preproc_bold") - wf.connect(node, out, func_skull_mean, 'in_file') + wf.connect(node, out, func_skull_mean, "in_file") - out_node, out_file = (func_skull_mean, 'out_file') + out_node, out_file = (func_skull_mean, "out_file") - if cfg.functional_preproc['func_masking']['FSL-BET'][ - 'functional_mean_thr']['run']: + if cfg.functional_preproc["func_masking"]["FSL-BET"]["functional_mean_thr"][ + "run" + ]: # T=$(fslstats ${subject}_tmean.nii.gz -p 98) - threshold_T = pe.Node(interface=fsl.ImageStats(), - name=f'func_mean_skull_thr_value_{pipe_num}', - iterfield=['in_file']) - threshold_T.inputs.op_string = "-p %f " % (cfg.functional_preproc['func_masking']['FSL-BET']['functional_mean_thr']['threshold_value']) - - wf.connect(func_skull_mean, 'out_file', threshold_T, 'in_file') + threshold_T = pe.Node( + interface=fsl.ImageStats(), + name=f"func_mean_skull_thr_value_{pipe_num}", + iterfield=["in_file"], + ) + threshold_T.inputs.op_string = ( + "-p %f " + % ( + cfg.functional_preproc["func_masking"]["FSL-BET"][ + "functional_mean_thr" + ]["threshold_value"] + ) + ) + + wf.connect(func_skull_mean, "out_file", threshold_T, "in_file") # z=$(echo "$T / 10" | bc -l) def form_thr_string(thr): - threshold_z = str(float(thr/10)) - return '-thr %s' % (threshold_z) - - form_thr_string = pe.Node(util.Function(input_names=['thr'], - output_names=['out_str'], - function=form_thr_string), - name=f'form_thr_string_{pipe_num}') + threshold_z = str(float(thr / 10)) + return "-thr %s" % (threshold_z) + + form_thr_string = pe.Node( + util.Function( + input_names=["thr"], + output_names=["out_str"], + function=form_thr_string, + ), + name=f"form_thr_string_{pipe_num}", + ) - wf.connect(threshold_T, 'out_stat', form_thr_string, 'thr') + wf.connect(threshold_T, "out_stat", form_thr_string, "thr") # fslmaths ${subject}_tmean.nii.gz -thr ${z} ${subject}_tmean_thr.nii.gz - func_skull_mean_thr = pe.Node(interface=fsl.ImageMaths(), - name=f'func_mean_skull_thr_{pipe_num}') + func_skull_mean_thr = pe.Node( + interface=fsl.ImageMaths(), name=f"func_mean_skull_thr_{pipe_num}" + ) - wf.connect(func_skull_mean, 'out_file', func_skull_mean_thr, 'in_file') - wf.connect(form_thr_string, 'out_str', func_skull_mean_thr, 'op_string') + wf.connect(func_skull_mean, "out_file", func_skull_mean_thr, "in_file") + wf.connect(form_thr_string, "out_str", func_skull_mean_thr, "op_string") - out_node, out_file = (func_skull_mean_thr, 'out_file') - - if cfg.functional_preproc['func_masking']['FSL-BET'][ - 'functional_mean_bias_correction']: + out_node, out_file = (func_skull_mean_thr, "out_file") + if cfg.functional_preproc["func_masking"]["FSL-BET"][ + "functional_mean_bias_correction" + ]: # fast --nopve -B ${subject}_tmean_thr.nii.gz - func_mean_skull_fast = pe.Node(interface=fsl.FAST(), - name=f'func_mean_skull_fast_{pipe_num}') + func_mean_skull_fast = pe.Node( + interface=fsl.FAST(), name=f"func_mean_skull_fast_{pipe_num}" + ) func_mean_skull_fast.inputs.no_pve = True func_mean_skull_fast.inputs.output_biascorrected = True - wf.connect(out_node, out_file, func_mean_skull_fast, 'in_files') + wf.connect(out_node, out_file, func_mean_skull_fast, "in_files") - out_node, out_file = (func_mean_skull_fast, 'restored_image') + out_node, out_file = (func_mean_skull_fast, "restored_image") - wf.connect(out_node, out_file, func_get_brain_mask, 'in_file') + wf.connect(out_node, out_file, func_get_brain_mask, "in_file") else: func_get_brain_mask.inputs.functional = True node, out = strat_pool.get_data("desc-preproc_bold") - wf.connect(node, out, func_get_brain_mask, 'in_file') + wf.connect(node, out, func_get_brain_mask, "in_file") # erode one voxel of functional brian mask - erode_one_voxel = pe.Node(interface=fsl.ErodeImage(), - name=f'erode_one_voxel_{pipe_num}') + erode_one_voxel = pe.Node( + interface=fsl.ErodeImage(), name=f"erode_one_voxel_{pipe_num}" + ) - erode_one_voxel.inputs.kernel_shape = 'box' + erode_one_voxel.inputs.kernel_shape = "box" erode_one_voxel.inputs.kernel_size = 1.0 - wf.connect(func_get_brain_mask, 'mask_file', - erode_one_voxel, 'in_file') + wf.connect(func_get_brain_mask, "mask_file", erode_one_voxel, "in_file") - outputs = { - 'space-bold_desc-brain_mask': (erode_one_voxel, 'out_file') - } + outputs = {"space-bold_desc-brain_mask": (erode_one_voxel, "out_file")} return (wf, outputs) @nodeblock( - name='bold_mask_fsl_afni', - switch=[['functional_preproc', 'run'], - ['functional_preproc', 'func_masking', 'run']], - option_key=['functional_preproc', 'func_masking', 'using'], - option_val='FSL_AFNI', - inputs=[('motion-basefile', 'desc-preproc_bold'), 'FSL-AFNI-bold-ref', 'FSL-AFNI-brain-mask', - 'FSL-AFNI-brain-probseg'], - outputs=['space-bold_desc-brain_mask', 'desc-ref_bold'] + name="bold_mask_fsl_afni", + switch=[ + ["functional_preproc", "run"], + ["functional_preproc", "func_masking", "run"], + ], + option_key=["functional_preproc", "func_masking", "using"], + option_val="FSL_AFNI", + inputs=[ + ("motion-basefile", "desc-preproc_bold"), + "FSL-AFNI-bold-ref", + "FSL-AFNI-brain-mask", + "FSL-AFNI-brain-probseg", + ], + outputs=["space-bold_desc-brain_mask", "desc-ref_bold"], ) def bold_mask_fsl_afni(wf, cfg, strat_pool, pipe_num, opt=None): """fMRIPrep-style BOLD mask - `Ref `_ + `Ref `_. """ - # Initialize transforms with antsAI init_aff = pe.Node( AI( @@ -930,13 +954,13 @@ def bold_mask_fsl_afni(wf, cfg, strat_pool, pipe_num, opt=None): verbose=True, ), name=f"init_aff_{pipe_num}", - n_procs=cfg.pipeline_setup['system_config']['num_OMP_threads'], + n_procs=cfg.pipeline_setup["system_config"]["num_OMP_threads"], ) - node, out = strat_pool.get_data('FSL-AFNI-bold-ref') - wf.connect(node, out, init_aff, 'fixed_image') + node, out = strat_pool.get_data("FSL-AFNI-bold-ref") + wf.connect(node, out, init_aff, "fixed_image") - node, out = strat_pool.get_data('FSL-AFNI-brain-mask') - wf.connect(node, out, init_aff, 'fixed_image_mask') + node, out = strat_pool.get_data("FSL-AFNI-brain-mask") + wf.connect(node, out, init_aff, "fixed_image_mask") init_aff.inputs.search_grid = (40, (0, 40, 40)) @@ -946,27 +970,27 @@ def bold_mask_fsl_afni(wf, cfg, strat_pool, pipe_num, opt=None): winsorize_upper_quantile=0.98, winsorize_lower_quantile=0.05, float=True, - metric=['Mattes'], + metric=["Mattes"], metric_weight=[1], radius_or_number_of_bins=[64], - transforms=['Affine'], + transforms=["Affine"], transform_parameters=[[0.1]], number_of_iterations=[[200]], convergence_window_size=[10], - convergence_threshold=[1.e-9], - sampling_strategy=['Random', 'Random'], + convergence_threshold=[1.0e-9], + sampling_strategy=["Random", "Random"], smoothing_sigmas=[[2]], - sigma_units=['mm', 'mm', 'mm'], + sigma_units=["mm", "mm", "mm"], shrink_factors=[[2]], sampling_percentage=[0.2], - use_histogram_matching=[True] + use_histogram_matching=[True], ), name=f"norm_{pipe_num}", - n_procs=cfg.pipeline_setup['system_config']['num_OMP_threads'], + n_procs=cfg.pipeline_setup["system_config"]["num_OMP_threads"], ) - node, out = strat_pool.get_data('FSL-AFNI-bold-ref') - wf.connect(node, out, norm, 'fixed_image') + node, out = strat_pool.get_data("FSL-AFNI-bold-ref") + wf.connect(node, out, norm, "fixed_image") map_brainmask = pe.Node( ants.ApplyTransforms( @@ -977,12 +1001,13 @@ def bold_mask_fsl_afni(wf, cfg, strat_pool, pipe_num, opt=None): ) # Use the higher resolution and probseg for numerical stability in rounding - node, out = strat_pool.get_data('FSL-AFNI-brain-probseg') - wf.connect(node, out, map_brainmask, 'input_image') + node, out = strat_pool.get_data("FSL-AFNI-brain-probseg") + wf.connect(node, out, map_brainmask, "input_image") - binarize_mask = pe.Node(interface=fsl.maths.MathsCommand(), - name=f'binarize_mask_{pipe_num}') - binarize_mask.inputs.args = '-thr 0.85 -bin' + binarize_mask = pe.Node( + interface=fsl.maths.MathsCommand(), name=f"binarize_mask_{pipe_num}" + ) + binarize_mask.inputs.args = "-thr 0.85 -bin" # Dilate pre_mask pre_dilate = pe.Node( @@ -997,10 +1022,10 @@ def bold_mask_fsl_afni(wf, cfg, strat_pool, pipe_num, opt=None): # Fix precision errors # https://github.com/ANTsX/ANTs/wiki/Inputs-do-not-occupy-the-same-physical-space#fixing-precision-errors - print_header = pe.Node(PrintHeader(what_information=4), - name=f'print_header_{pipe_num}') - set_direction = pe.Node(SetDirectionByMatrix(), - name=f'set_direction_{pipe_num}') + print_header = pe.Node( + PrintHeader(what_information=4), name=f"print_header_{pipe_num}" + ) + set_direction = pe.Node(SetDirectionByMatrix(), name=f"set_direction_{pipe_num}") # Run N4 normally, force num_threads=1 for stability (images are # small, no need for >1) @@ -1009,600 +1034,655 @@ def bold_mask_fsl_afni(wf, cfg, strat_pool, pipe_num, opt=None): dimension=3, copy_header=True, bspline_fitting_distance=200 ), shrink_factor=2, - rescale_intensities = True, + rescale_intensities=True, name=f"n4_correct_{pipe_num}", n_procs=1, ) skullstrip_first_pass = pe.Node( fsl.BET(frac=0.2, mask=True, functional=False), - name=f'skullstrip_first_pass_{pipe_num}') + name=f"skullstrip_first_pass_{pipe_num}", + ) bet_dilate = pe.Node( - fsl.DilateImage(operation='max', kernel_shape='sphere', - kernel_size=6.0, internal_datatype='char'), - name=f'skullstrip_first_dilate_{pipe_num}') + fsl.DilateImage( + operation="max", + kernel_shape="sphere", + kernel_size=6.0, + internal_datatype="char", + ), + name=f"skullstrip_first_dilate_{pipe_num}", + ) - bet_mask = pe.Node(fsl.ApplyMask(), name=f'skullstrip_first_mask_' - f'{pipe_num}') + bet_mask = pe.Node(fsl.ApplyMask(), name=f"skullstrip_first_mask_" f"{pipe_num}") - unifize = pe.Node(afni_utils.Unifize(t2=True, outputtype='NIFTI_GZ', - args='-clfrac 0.2 -rbt 18.3 65.0 90.0', - out_file="uni.nii.gz"), - name=f'unifize_{pipe_num}') + unifize = pe.Node( + afni_utils.Unifize( + t2=True, + outputtype="NIFTI_GZ", + args="-clfrac 0.2 -rbt 18.3 65.0 90.0", + out_file="uni.nii.gz", + ), + name=f"unifize_{pipe_num}", + ) skullstrip_second_pass = pe.Node( - preprocess.Automask(dilate=1, outputtype='NIFTI_GZ'), - name=f'skullstrip_second_pass_{pipe_num}') + preprocess.Automask(dilate=1, outputtype="NIFTI_GZ"), + name=f"skullstrip_second_pass_{pipe_num}", + ) - combine_masks = pe.Node(fsl.BinaryMaths(operation='mul'), - name=f'combine_masks_{pipe_num}') + combine_masks = pe.Node( + fsl.BinaryMaths(operation="mul"), name=f"combine_masks_{pipe_num}" + ) - apply_mask = pe.Node(fsl.ApplyMask(), - name=f'extract_ref_brain_bold_{pipe_num}') + apply_mask = pe.Node(fsl.ApplyMask(), name=f"extract_ref_brain_bold_{pipe_num}") node, out = strat_pool.get_data(["motion-basefile"]) - wf.connect([(node, init_aff, [(out, "moving_image")]), - (node, map_brainmask, [(out, "reference_image")]), - (node, norm, [(out, "moving_image")]), - (init_aff, norm, [ - ("output_transform", "initial_moving_transform")]), - (norm, map_brainmask, [ + wf.connect( + [ + (node, init_aff, [(out, "moving_image")]), + (node, map_brainmask, [(out, "reference_image")]), + (node, norm, [(out, "moving_image")]), + (init_aff, norm, [("output_transform", "initial_moving_transform")]), + ( + norm, + map_brainmask, + [ ("reverse_invert_flags", "invert_transform_flags"), ("reverse_transforms", "transforms"), - ]), - (map_brainmask, binarize_mask, [("output_image", "in_file")]), - (binarize_mask, pre_dilate, [("out_file", "in_file")]), - (pre_dilate, print_header, [("out_file", "image")]), - (print_header, set_direction, [("header", "direction")]), - (node, set_direction, [(out, "infile"), (out, "outfile")]), - (set_direction, n4_correct, [("outfile", "mask_image")]), - (node, n4_correct, [(out, "input_image")]), - (n4_correct, skullstrip_first_pass, - [('output_image', 'in_file')]), - (skullstrip_first_pass, bet_dilate, - [('mask_file', 'in_file')]), - (bet_dilate, bet_mask, [('out_file', 'mask_file')]), - (skullstrip_first_pass, bet_mask, [('out_file', 'in_file')]), - (bet_mask, unifize, [('out_file', 'in_file')]), - (unifize, skullstrip_second_pass, [('out_file', 'in_file')]), - (skullstrip_first_pass, combine_masks, - [('mask_file', 'in_file')]), - (skullstrip_second_pass, combine_masks, - [('out_file', 'operand_file')]), - (unifize, apply_mask, [('out_file', 'in_file')]), - (combine_masks, apply_mask, [('out_file', 'mask_file')]), - ]) + ], + ), + (map_brainmask, binarize_mask, [("output_image", "in_file")]), + (binarize_mask, pre_dilate, [("out_file", "in_file")]), + (pre_dilate, print_header, [("out_file", "image")]), + (print_header, set_direction, [("header", "direction")]), + (node, set_direction, [(out, "infile"), (out, "outfile")]), + (set_direction, n4_correct, [("outfile", "mask_image")]), + (node, n4_correct, [(out, "input_image")]), + (n4_correct, skullstrip_first_pass, [("output_image", "in_file")]), + (skullstrip_first_pass, bet_dilate, [("mask_file", "in_file")]), + (bet_dilate, bet_mask, [("out_file", "mask_file")]), + (skullstrip_first_pass, bet_mask, [("out_file", "in_file")]), + (bet_mask, unifize, [("out_file", "in_file")]), + (unifize, skullstrip_second_pass, [("out_file", "in_file")]), + (skullstrip_first_pass, combine_masks, [("mask_file", "in_file")]), + (skullstrip_second_pass, combine_masks, [("out_file", "operand_file")]), + (unifize, apply_mask, [("out_file", "in_file")]), + (combine_masks, apply_mask, [("out_file", "mask_file")]), + ] + ) outputs = { - 'space-bold_desc-brain_mask': (combine_masks, 'out_file'), - 'desc-ref_bold': (apply_mask, 'out_file') + "space-bold_desc-brain_mask": (combine_masks, "out_file"), + "desc-ref_bold": (apply_mask, "out_file"), } return (wf, outputs) @nodeblock( - name='bold_mask_anatomical_refined', - switch=[['functional_preproc', 'run'], - ['functional_preproc', 'func_masking', 'run']], - option_key=['functional_preproc', 'func_masking', 'using'], - option_val='Anatomical_Refined', - inputs=[('bold', 'desc-preproc_bold'), - ('desc-brain_T1w', ['space-T1w_desc-brain_mask', 'space-T1w_desc-acpcbrain_mask'])], - outputs=['space-bold_desc-brain_mask'] + name="bold_mask_anatomical_refined", + switch=[ + ["functional_preproc", "run"], + ["functional_preproc", "func_masking", "run"], + ], + option_key=["functional_preproc", "func_masking", "using"], + option_val="Anatomical_Refined", + inputs=[ + ("bold", "desc-preproc_bold"), + ( + "desc-brain_T1w", + ["space-T1w_desc-brain_mask", "space-T1w_desc-acpcbrain_mask"], + ), + ], + outputs=["space-bold_desc-brain_mask"], ) def bold_mask_anatomical_refined(wf, cfg, strat_pool, pipe_num, opt=None): - # binarize anat mask, in case it is not a binary mask. - anat_brain_mask_bin = pe.Node(interface=fsl.ImageMaths(), - name=f'anat_brain_mask_bin_{pipe_num}') - anat_brain_mask_bin.inputs.op_string = '-bin' + anat_brain_mask_bin = pe.Node( + interface=fsl.ImageMaths(), name=f"anat_brain_mask_bin_{pipe_num}" + ) + anat_brain_mask_bin.inputs.op_string = "-bin" - node, out = strat_pool.get_data(['space-T1w_desc-brain_mask', - 'space-T1w_desc-acpcbrain_mask']) - wf.connect(node, out, anat_brain_mask_bin, 'in_file') + node, out = strat_pool.get_data( + ["space-T1w_desc-brain_mask", "space-T1w_desc-acpcbrain_mask"] + ) + wf.connect(node, out, anat_brain_mask_bin, "in_file") # fill holes of anat mask - anat_mask_filled = pe.Node(interface=afni.MaskTool(), - name=f'anat_brain_mask_filled_{pipe_num}') + anat_mask_filled = pe.Node( + interface=afni.MaskTool(), name=f"anat_brain_mask_filled_{pipe_num}" + ) anat_mask_filled.inputs.fill_holes = True - anat_mask_filled.inputs.outputtype = 'NIFTI_GZ' + anat_mask_filled.inputs.outputtype = "NIFTI_GZ" - wf.connect(anat_brain_mask_bin, 'out_file', - anat_mask_filled, 'in_file') + wf.connect(anat_brain_mask_bin, "out_file", anat_mask_filled, "in_file") # init_bold_mask : input raw func - init_bold_mask = anat_refined_mask(init_bold_mask=True, - wf_name=f'init_bold_mask_{pipe_num}') + init_bold_mask = anat_refined_mask( + init_bold_mask=True, wf_name=f"init_bold_mask_{pipe_num}" + ) - func_deoblique = pe.Node(interface=afni_utils.Refit(), - name=f'raw_func_deoblique_{pipe_num}') + func_deoblique = pe.Node( + interface=afni_utils.Refit(), name=f"raw_func_deoblique_{pipe_num}" + ) func_deoblique.inputs.deoblique = True - node, out = strat_pool.get_data('bold') - wf.connect(node, out, func_deoblique, 'in_file') + node, out = strat_pool.get_data("bold") + wf.connect(node, out, func_deoblique, "in_file") - func_reorient = pe.Node(interface=afni_utils.Resample(), - name=f'raw_func_reorient_{pipe_num}', - mem_gb=0, - mem_x=(0.0115, 'in_file', 't')) + func_reorient = pe.Node( + interface=afni_utils.Resample(), + name=f"raw_func_reorient_{pipe_num}", + mem_gb=0, + mem_x=(0.0115, "in_file", "t"), + ) - func_reorient.inputs.orientation = 'RPI' - func_reorient.inputs.outputtype = 'NIFTI_GZ' + func_reorient.inputs.orientation = "RPI" + func_reorient.inputs.outputtype = "NIFTI_GZ" - wf.connect(func_deoblique, 'out_file', - func_reorient, 'in_file') + wf.connect(func_deoblique, "out_file", func_reorient, "in_file") - wf.connect(func_reorient, 'out_file', - init_bold_mask, 'inputspec.func') + wf.connect(func_reorient, "out_file", init_bold_mask, "inputspec.func") - wf.connect(anat_mask_filled, 'out_file', - init_bold_mask, 'inputspec.anatomical_brain_mask') + wf.connect( + anat_mask_filled, "out_file", init_bold_mask, "inputspec.anatomical_brain_mask" + ) - node, out = strat_pool.get_data('desc-brain_T1w') - wf.connect(node, out, init_bold_mask, 'inputspec.anat_brain') + node, out = strat_pool.get_data("desc-brain_T1w") + wf.connect(node, out, init_bold_mask, "inputspec.anat_brain") # dilate init func brain mask - func_tmp_brain_mask = pe.Node(interface=fsl.ImageMaths(), - name=f'func_tmp_brain_mask_dil_{pipe_num}') - func_tmp_brain_mask.inputs.op_string = '-dilM' + func_tmp_brain_mask = pe.Node( + interface=fsl.ImageMaths(), name=f"func_tmp_brain_mask_dil_{pipe_num}" + ) + func_tmp_brain_mask.inputs.op_string = "-dilM" - wf.connect(init_bold_mask, 'outputspec.func_brain_mask', - func_tmp_brain_mask, 'in_file') + wf.connect( + init_bold_mask, "outputspec.func_brain_mask", func_tmp_brain_mask, "in_file" + ) # refined_bold_mask : input motion corrected func - refined_bold_mask = anat_refined_mask(init_bold_mask=False, - wf_name='refined_bold_mask' - f'_{pipe_num}') + refined_bold_mask = anat_refined_mask( + init_bold_mask=False, wf_name="refined_bold_mask" f"_{pipe_num}" + ) - node, out = strat_pool.get_data(["desc-preproc_bold", - "bold"]) - wf.connect(node, out, refined_bold_mask, 'inputspec.func') + node, out = strat_pool.get_data(["desc-preproc_bold", "bold"]) + wf.connect(node, out, refined_bold_mask, "inputspec.func") - node, out = strat_pool.get_data('desc-brain_T1w') - wf.connect(node, out, refined_bold_mask, 'inputspec.anat_brain') + node, out = strat_pool.get_data("desc-brain_T1w") + wf.connect(node, out, refined_bold_mask, "inputspec.anat_brain") - wf.connect(func_tmp_brain_mask, 'out_file', - refined_bold_mask, 'inputspec.init_func_brain_mask') + wf.connect( + func_tmp_brain_mask, + "out_file", + refined_bold_mask, + "inputspec.init_func_brain_mask", + ) # dilate anatomical mask - if cfg.functional_preproc['func_masking']['Anatomical_Refined'][ - 'anatomical_mask_dilation']: - anat_mask_dilate = pe.Node(interface=afni.MaskTool(), - name=f'anat_mask_dilate_{pipe_num}') - anat_mask_dilate.inputs.dilate_inputs = '1' - anat_mask_dilate.inputs.outputtype = 'NIFTI_GZ' - - wf.connect(anat_mask_filled, 'out_file', - anat_mask_dilate, 'in_file') - wf.connect(anat_mask_dilate, 'out_file', - refined_bold_mask, 'inputspec.anatomical_brain_mask') + if cfg.functional_preproc["func_masking"]["Anatomical_Refined"][ + "anatomical_mask_dilation" + ]: + anat_mask_dilate = pe.Node( + interface=afni.MaskTool(), name=f"anat_mask_dilate_{pipe_num}" + ) + anat_mask_dilate.inputs.dilate_inputs = "1" + anat_mask_dilate.inputs.outputtype = "NIFTI_GZ" + + wf.connect(anat_mask_filled, "out_file", anat_mask_dilate, "in_file") + wf.connect( + anat_mask_dilate, + "out_file", + refined_bold_mask, + "inputspec.anatomical_brain_mask", + ) else: - wf.connect(anat_mask_filled, 'out_file', - refined_bold_mask, 'inputspec.anatomical_brain_mask') + wf.connect( + anat_mask_filled, + "out_file", + refined_bold_mask, + "inputspec.anatomical_brain_mask", + ) # get final func mask - func_mask_final = pe.Node(interface=fsl.MultiImageMaths(), - name=f'func_mask_final_{pipe_num}') + func_mask_final = pe.Node( + interface=fsl.MultiImageMaths(), name=f"func_mask_final_{pipe_num}" + ) func_mask_final.inputs.op_string = "-mul %s" - wf.connect(func_tmp_brain_mask, 'out_file', - func_mask_final, 'in_file') + wf.connect(func_tmp_brain_mask, "out_file", func_mask_final, "in_file") - wf.connect(refined_bold_mask, 'outputspec.func_brain_mask', - func_mask_final, 'operand_files') + wf.connect( + refined_bold_mask, + "outputspec.func_brain_mask", + func_mask_final, + "operand_files", + ) - outputs = { - 'space-bold_desc-brain_mask': (func_mask_final, 'out_file') - } + outputs = {"space-bold_desc-brain_mask": (func_mask_final, "out_file")} return (wf, outputs) @nodeblock( - name='bold_mask_anatomical_based', - switch=[['functional_preproc', 'run'], - ['functional_preproc', 'func_masking', 'run']], - option_key=['functional_preproc', 'func_masking', 'using'], - option_val='Anatomical_Based', - inputs=['desc-preproc_bold', ('desc-brain_T1w', ['desc-preproc_T1w', 'desc-reorient_T1w', 'T1w'])], - outputs=['space-bold_desc-brain_mask'] + name="bold_mask_anatomical_based", + switch=[ + ["functional_preproc", "run"], + ["functional_preproc", "func_masking", "run"], + ], + option_key=["functional_preproc", "func_masking", "using"], + option_val="Anatomical_Based", + inputs=[ + "desc-preproc_bold", + ("desc-brain_T1w", ["desc-preproc_T1w", "desc-reorient_T1w", "T1w"]), + ], + outputs=["space-bold_desc-brain_mask"], ) def bold_mask_anatomical_based(wf, cfg, strat_pool, pipe_num, opt=None): - '''Generate the BOLD mask by basing it off of the anatomical brain mask. + """Generate the BOLD mask by basing it off of the anatomical brain mask. Adapted from `DCAN Lab's BOLD mask method from the ABCD pipeline `_. - ''' - + """ # 0. Take single volume of func - func_single_volume = pe.Node(interface=afni.Calc(), - name='func_single_volume') + func_single_volume = pe.Node(interface=afni.Calc(), name="func_single_volume") - func_single_volume.inputs.set( - expr='a', - single_idx=1, - outputtype='NIFTI_GZ' - ) + func_single_volume.inputs.set(expr="a", single_idx=1, outputtype="NIFTI_GZ") node, out = strat_pool.get_data("desc-preproc_bold") - wf.connect(node, out, func_single_volume, 'in_file_a') + wf.connect(node, out, func_single_volume, "in_file_a") # 1. Register func head to anat head to get func2anat matrix - linear_reg_func_to_anat = pe.Node(interface=fsl.FLIRT(), - name='func_to_anat_linear_reg') + linear_reg_func_to_anat = pe.Node( + interface=fsl.FLIRT(), name="func_to_anat_linear_reg" + ) linear_reg_func_to_anat.inputs.dof = 6 - linear_reg_func_to_anat.inputs.interp = 'spline' + linear_reg_func_to_anat.inputs.interp = "spline" linear_reg_func_to_anat.inputs.searchr_x = [30, 30] linear_reg_func_to_anat.inputs.searchr_y = [30, 30] linear_reg_func_to_anat.inputs.searchr_z = [30, 30] - wf.connect(func_single_volume, 'out_file', - linear_reg_func_to_anat, 'in_file') + wf.connect(func_single_volume, "out_file", linear_reg_func_to_anat, "in_file") - node, out = strat_pool.get_data(["desc-preproc_T1w", "desc-reorient_T1w", - "T1w"]) - wf.connect(node, out, linear_reg_func_to_anat, 'reference') + node, out = strat_pool.get_data(["desc-preproc_T1w", "desc-reorient_T1w", "T1w"]) + wf.connect(node, out, linear_reg_func_to_anat, "reference") # 2. Inverse func to anat affine, to get anat-to-func transform - inv_func_to_anat_affine = pe.Node(interface=fsl.ConvertXFM(), - name='inv_func2anat_affine') + inv_func_to_anat_affine = pe.Node( + interface=fsl.ConvertXFM(), name="inv_func2anat_affine" + ) inv_func_to_anat_affine.inputs.invert_xfm = True - wf.connect(linear_reg_func_to_anat, 'out_matrix_file', - inv_func_to_anat_affine, 'in_file') + wf.connect( + linear_reg_func_to_anat, "out_matrix_file", inv_func_to_anat_affine, "in_file" + ) # 3. get BOLD mask # 3.1 Apply anat-to-func transform to transfer anatomical brain to functional space - reg_anat_brain_to_func = pe.Node(interface=fsl.ApplyWarp(), - name='reg_anat_brain_to_func') - reg_anat_brain_to_func.inputs.interp = 'nn' + reg_anat_brain_to_func = pe.Node( + interface=fsl.ApplyWarp(), name="reg_anat_brain_to_func" + ) + reg_anat_brain_to_func.inputs.interp = "nn" reg_anat_brain_to_func.inputs.relwarp = True node, out = strat_pool.get_data("desc-brain_T1w") - wf.connect(node, out, reg_anat_brain_to_func, 'in_file') + wf.connect(node, out, reg_anat_brain_to_func, "in_file") node, out = strat_pool.get_data("desc-preproc_bold") - wf.connect(node, out, reg_anat_brain_to_func, 'ref_file') + wf.connect(node, out, reg_anat_brain_to_func, "ref_file") - wf.connect(inv_func_to_anat_affine, 'out_file', - reg_anat_brain_to_func, 'premat') + wf.connect(inv_func_to_anat_affine, "out_file", reg_anat_brain_to_func, "premat") # 3.2 Binarize transfered image - func_mask_bin = pe.Node(interface=fsl.ImageMaths(), - name='func_mask_bin') - func_mask_bin.inputs.op_string = '-abs -bin' + func_mask_bin = pe.Node(interface=fsl.ImageMaths(), name="func_mask_bin") + func_mask_bin.inputs.op_string = "-abs -bin" - wf.connect(reg_anat_brain_to_func, 'out_file', - func_mask_bin, 'in_file') + wf.connect(reg_anat_brain_to_func, "out_file", func_mask_bin, "in_file") # 3.3 Fill holes to get BOLD mask - func_mask_fill_holes = pe.Node(interface=afni.MaskTool(), - name='func_mask_fill_holes') + func_mask_fill_holes = pe.Node( + interface=afni.MaskTool(), name="func_mask_fill_holes" + ) func_mask_fill_holes.inputs.fill_holes = True - func_mask_fill_holes.inputs.outputtype = 'NIFTI_GZ' + func_mask_fill_holes.inputs.outputtype = "NIFTI_GZ" - wf.connect(func_mask_bin, 'out_file', - func_mask_fill_holes, 'in_file') + wf.connect(func_mask_bin, "out_file", func_mask_fill_holes, "in_file") - outputs = { - 'space-bold_desc-brain_mask': (func_mask_fill_holes, 'out_file') - } + outputs = {"space-bold_desc-brain_mask": (func_mask_fill_holes, "out_file")} return (wf, outputs) @nodeblock( - name='bold_mask_anatomical_resampled', - switch=[['functional_preproc', 'run'], - ['functional_preproc', 'func_masking', 'run']], - option_key=['functional_preproc', 'func_masking', 'using'], - option_val='Anatomical_Resampled', - inputs=['desc-preproc_bold', 'T1w-template-funcreg', 'space-template_desc-preproc_T1w', - 'space-template_desc-T1w_mask'], - outputs=['space-template_res-bold_desc-brain_T1w', 'space-template_desc-bold_mask', 'space-bold_desc-brain_mask'] + name="bold_mask_anatomical_resampled", + switch=[ + ["functional_preproc", "run"], + ["functional_preproc", "func_masking", "run"], + ], + option_key=["functional_preproc", "func_masking", "using"], + option_val="Anatomical_Resampled", + inputs=[ + "desc-preproc_bold", + "T1w-template-funcreg", + "space-template_desc-preproc_T1w", + "space-template_desc-T1w_mask", + ], + outputs=[ + "space-template_res-bold_desc-brain_T1w", + "space-template_desc-bold_mask", + "space-bold_desc-brain_mask", + ], ) def bold_mask_anatomical_resampled(wf, cfg, strat_pool, pipe_num, opt=None): - '''Resample anatomical brain mask in standard space to get BOLD brain mask in standard space + """Resample anatomical brain mask in standard space to get BOLD brain mask in standard space Adapted from `DCAN Lab's BOLD mask method from the ABCD pipeline `_. - ''' - + """ # applywarp --rel --interp=spline -i ${T1wImage} -r ${ResampRefIm} --premat=$FSLDIR/etc/flirtsch/ident.mat -o ${WD}/${T1wImageFile}.${FinalfMRIResolution} - anat_brain_to_func_res = pe.Node(interface=fsl.ApplyWarp(), - name=f'resample_anat_brain_in_standard_{pipe_num}') + anat_brain_to_func_res = pe.Node( + interface=fsl.ApplyWarp(), name=f"resample_anat_brain_in_standard_{pipe_num}" + ) - anat_brain_to_func_res.inputs.interp = 'spline' + anat_brain_to_func_res.inputs.interp = "spline" anat_brain_to_func_res.inputs.premat = cfg.registration_workflows[ - 'anatomical_registration']['registration']['FSL-FNIRT']['identity_matrix'] + "anatomical_registration" + ]["registration"]["FSL-FNIRT"]["identity_matrix"] - node, out = strat_pool.get_data('space-template_desc-preproc_T1w') - wf.connect(node, out, anat_brain_to_func_res, 'in_file') + node, out = strat_pool.get_data("space-template_desc-preproc_T1w") + wf.connect(node, out, anat_brain_to_func_res, "in_file") - node, out = strat_pool.get_data('T1w-template-funcreg') - wf.connect(node, out, anat_brain_to_func_res, 'ref_file') + node, out = strat_pool.get_data("T1w-template-funcreg") + wf.connect(node, out, anat_brain_to_func_res, "ref_file") # Create brain masks in this space from the FreeSurfer output (changing resolution) # applywarp --rel --interp=nn -i ${FreeSurferBrainMask}.nii.gz -r ${WD}/${T1wImageFile}.${FinalfMRIResolution} --premat=$FSLDIR/etc/flirtsch/ident.mat -o ${WD}/${FreeSurferBrainMaskFile}.${FinalfMRIResolution}.nii.gz - anat_brain_mask_to_func_res = pe.Node(interface=fsl.ApplyWarp(), - name=f'resample_anat_brain_mask_in_standard_{pipe_num}') + anat_brain_mask_to_func_res = pe.Node( + interface=fsl.ApplyWarp(), + name=f"resample_anat_brain_mask_in_standard_{pipe_num}", + ) - anat_brain_mask_to_func_res.inputs.interp = 'nn' + anat_brain_mask_to_func_res.inputs.interp = "nn" anat_brain_mask_to_func_res.inputs.premat = cfg.registration_workflows[ - 'anatomical_registration']['registration']['FSL-FNIRT']['identity_matrix'] + "anatomical_registration" + ]["registration"]["FSL-FNIRT"]["identity_matrix"] - node, out = strat_pool.get_data('space-template_desc-T1w_mask') - wf.connect(node, out, anat_brain_mask_to_func_res, 'in_file') + node, out = strat_pool.get_data("space-template_desc-T1w_mask") + wf.connect(node, out, anat_brain_mask_to_func_res, "in_file") - wf.connect(anat_brain_to_func_res, 'out_file', - anat_brain_mask_to_func_res, 'ref_file') + wf.connect( + anat_brain_to_func_res, "out_file", anat_brain_mask_to_func_res, "ref_file" + ) # Resample func mask in template space back to native space func_mask_template_to_native = pe.Node( interface=afni.Resample(), - name=f'resample_func_mask_to_native_{pipe_num}', + name=f"resample_func_mask_to_native_{pipe_num}", mem_gb=0, - mem_x=(0.0115, 'in_file', 't')) - func_mask_template_to_native.inputs.resample_mode = 'NN' - func_mask_template_to_native.inputs.outputtype = 'NIFTI_GZ' + mem_x=(0.0115, "in_file", "t"), + ) + func_mask_template_to_native.inputs.resample_mode = "NN" + func_mask_template_to_native.inputs.outputtype = "NIFTI_GZ" - wf.connect(anat_brain_mask_to_func_res, 'out_file', - func_mask_template_to_native, 'in_file') + wf.connect( + anat_brain_mask_to_func_res, "out_file", func_mask_template_to_native, "in_file" + ) node, out = strat_pool.get_data("desc-preproc_bold") - wf.connect(node, out, func_mask_template_to_native, 'master') + wf.connect(node, out, func_mask_template_to_native, "master") outputs = { - 'space-template_res-bold_desc-brain_T1w': (anat_brain_to_func_res, 'out_file'), - 'space-template_desc-bold_mask': (anat_brain_mask_to_func_res, 'out_file'), - 'space-bold_desc-brain_mask': (func_mask_template_to_native, 'out_file') + "space-template_res-bold_desc-brain_T1w": (anat_brain_to_func_res, "out_file"), + "space-template_desc-bold_mask": (anat_brain_mask_to_func_res, "out_file"), + "space-bold_desc-brain_mask": (func_mask_template_to_native, "out_file"), } return (wf, outputs) @nodeblock( - name='bold_mask_ccs', - switch=[['functional_preproc', 'run'], - ['functional_preproc', 'func_masking', 'run']], - option_key=['functional_preproc', 'func_masking', 'using'], - option_val='CCS_Anatomical_Refined', - inputs=[['desc-motion_bold', 'desc-preproc_bold', 'bold'], 'desc-brain_T1w', - ['desc-preproc_T1w', 'desc-reorient_T1w', 'T1w']], - outputs=['space-bold_desc-brain_mask', 'desc-ref_bold'] + name="bold_mask_ccs", + switch=[ + ["functional_preproc", "run"], + ["functional_preproc", "func_masking", "run"], + ], + option_key=["functional_preproc", "func_masking", "using"], + option_val="CCS_Anatomical_Refined", + inputs=[ + ["desc-motion_bold", "desc-preproc_bold", "bold"], + "desc-brain_T1w", + ["desc-preproc_T1w", "desc-reorient_T1w", "T1w"], + ], + outputs=["space-bold_desc-brain_mask", "desc-ref_bold"], ) def bold_mask_ccs(wf, cfg, strat_pool, pipe_num, opt=None): - '''Generate the BOLD mask by basing it off of the anatomical brain. + """Generate the BOLD mask by basing it off of the anatomical brain. Adapted from `the BOLD mask method from the CCS pipeline `_. - ''' - + """ # Run 3dAutomask to generate func initial mask - func_tmp_brain_mask = pe.Node(interface=preprocess.Automask(), - name=f'func_tmp_brain_mask_AFNI_{pipe_num}') + func_tmp_brain_mask = pe.Node( + interface=preprocess.Automask(), name=f"func_tmp_brain_mask_AFNI_{pipe_num}" + ) func_tmp_brain_mask.inputs.dilate = 1 - func_tmp_brain_mask.inputs.outputtype = 'NIFTI_GZ' + func_tmp_brain_mask.inputs.outputtype = "NIFTI_GZ" - node, out = strat_pool.get_data(["desc-motion_bold", - "desc-preproc_bold", - "bold"]) - wf.connect(node, out, func_tmp_brain_mask, 'in_file') + node, out = strat_pool.get_data(["desc-motion_bold", "desc-preproc_bold", "bold"]) + wf.connect(node, out, func_tmp_brain_mask, "in_file") # Extract 8th volume as func ROI - func_roi = pe.Node(interface=fsl.ExtractROI(), - name=f'extract_func_roi_{pipe_num}') + func_roi = pe.Node(interface=fsl.ExtractROI(), name=f"extract_func_roi_{pipe_num}") func_roi.inputs.t_min = 7 func_roi.inputs.t_size = 1 - node, out = strat_pool.get_data(["desc-motion_bold", - "desc-preproc_bold", - "bold"]) - wf.connect(node, out, func_roi, 'in_file') + node, out = strat_pool.get_data(["desc-motion_bold", "desc-preproc_bold", "bold"]) + wf.connect(node, out, func_roi, "in_file") # Apply func initial mask on func ROI volume - func_tmp_brain = pe.Node(interface=fsl.maths.ApplyMask(), - name=f'get_func_tmp_brain_{pipe_num}') + func_tmp_brain = pe.Node( + interface=fsl.maths.ApplyMask(), name=f"get_func_tmp_brain_{pipe_num}" + ) - wf.connect(func_roi, 'roi_file', - func_tmp_brain, 'in_file') + wf.connect(func_roi, "roi_file", func_tmp_brain, "in_file") - wf.connect(func_tmp_brain_mask, 'out_file', - func_tmp_brain, 'mask_file') + wf.connect(func_tmp_brain_mask, "out_file", func_tmp_brain, "mask_file") # Register func tmp brain to anat brain to get func2anat matrix - reg_func_to_anat = pe.Node(interface=fsl.FLIRT(), - name=f'func_to_anat_linear_reg_{pipe_num}') - reg_func_to_anat.inputs.interp = 'trilinear' - reg_func_to_anat.inputs.cost = 'corratio' + reg_func_to_anat = pe.Node( + interface=fsl.FLIRT(), name=f"func_to_anat_linear_reg_{pipe_num}" + ) + reg_func_to_anat.inputs.interp = "trilinear" + reg_func_to_anat.inputs.cost = "corratio" reg_func_to_anat.inputs.dof = 6 - wf.connect(func_tmp_brain, 'out_file', - reg_func_to_anat, 'in_file') + wf.connect(func_tmp_brain, "out_file", reg_func_to_anat, "in_file") node, out = strat_pool.get_data("desc-brain_T1w") - wf.connect(node, out, reg_func_to_anat, 'reference') + wf.connect(node, out, reg_func_to_anat, "reference") # Inverse func2anat matrix - inv_func_to_anat_affine = pe.Node(interface=fsl.ConvertXFM(), - name=f'inv_func2anat_affine_{pipe_num}') + inv_func_to_anat_affine = pe.Node( + interface=fsl.ConvertXFM(), name=f"inv_func2anat_affine_{pipe_num}" + ) inv_func_to_anat_affine.inputs.invert_xfm = True - wf.connect(reg_func_to_anat, 'out_matrix_file', - inv_func_to_anat_affine, 'in_file') + wf.connect(reg_func_to_anat, "out_matrix_file", inv_func_to_anat_affine, "in_file") # Transform anat brain to func space - reg_anat_brain_to_func = pe.Node(interface=fsl.FLIRT(), - name=f'reg_anat_brain_to_func_{pipe_num}') + reg_anat_brain_to_func = pe.Node( + interface=fsl.FLIRT(), name=f"reg_anat_brain_to_func_{pipe_num}" + ) reg_anat_brain_to_func.inputs.apply_xfm = True - reg_anat_brain_to_func.inputs.interp = 'trilinear' + reg_anat_brain_to_func.inputs.interp = "trilinear" node, out = strat_pool.get_data("desc-brain_T1w") - wf.connect(node, out, reg_anat_brain_to_func, 'in_file') + wf.connect(node, out, reg_anat_brain_to_func, "in_file") - wf.connect(func_roi, 'roi_file', - reg_anat_brain_to_func, 'reference') + wf.connect(func_roi, "roi_file", reg_anat_brain_to_func, "reference") - wf.connect(inv_func_to_anat_affine, 'out_file', - reg_anat_brain_to_func, 'in_matrix_file') + wf.connect( + inv_func_to_anat_affine, "out_file", reg_anat_brain_to_func, "in_matrix_file" + ) # Binarize and dilate anat brain in func space - bin_anat_brain_in_func = pe.Node(interface=fsl.ImageMaths(), - name=f'bin_anat_brain_in_func_{pipe_num}') - bin_anat_brain_in_func.inputs.op_string = '-bin -dilM' + bin_anat_brain_in_func = pe.Node( + interface=fsl.ImageMaths(), name=f"bin_anat_brain_in_func_{pipe_num}" + ) + bin_anat_brain_in_func.inputs.op_string = "-bin -dilM" - wf.connect(reg_anat_brain_to_func, 'out_file', - bin_anat_brain_in_func, 'in_file') + wf.connect(reg_anat_brain_to_func, "out_file", bin_anat_brain_in_func, "in_file") # Binarize detectable func signals - bin_func = pe.Node(interface=fsl.ImageMaths(), - name=f'bin_func_{pipe_num}') - bin_func.inputs.op_string = '-Tstd -bin' + bin_func = pe.Node(interface=fsl.ImageMaths(), name=f"bin_func_{pipe_num}") + bin_func.inputs.op_string = "-Tstd -bin" - node, out = strat_pool.get_data(["desc-motion_bold", - "desc-preproc_bold", - "bold"]) - wf.connect(node, out, bin_func, 'in_file') + node, out = strat_pool.get_data(["desc-motion_bold", "desc-preproc_bold", "bold"]) + wf.connect(node, out, bin_func, "in_file") # Take intersection of masks - merge_func_mask = pe.Node(util.Merge(2), - name=f'merge_func_mask_{pipe_num}') + merge_func_mask = pe.Node(util.Merge(2), name=f"merge_func_mask_{pipe_num}") - wf.connect(func_tmp_brain_mask, 'out_file', - merge_func_mask, 'in1') + wf.connect(func_tmp_brain_mask, "out_file", merge_func_mask, "in1") - wf.connect(bin_anat_brain_in_func, 'out_file', - merge_func_mask, 'in2') + wf.connect(bin_anat_brain_in_func, "out_file", merge_func_mask, "in2") - intersect_mask = pe.Node(interface=fsl.MultiImageMaths(), - name=f'intersect_mask_{pipe_num}') - intersect_mask.inputs.op_string = '-mul %s -mul %s' - intersect_mask.inputs.output_datatype = 'char' + intersect_mask = pe.Node( + interface=fsl.MultiImageMaths(), name=f"intersect_mask_{pipe_num}" + ) + intersect_mask.inputs.op_string = "-mul %s -mul %s" + intersect_mask.inputs.output_datatype = "char" - wf.connect(bin_func, 'out_file', - intersect_mask, 'in_file') + wf.connect(bin_func, "out_file", intersect_mask, "in_file") - wf.connect(merge_func_mask, 'out', - intersect_mask, 'operand_files') + wf.connect(merge_func_mask, "out", intersect_mask, "operand_files") # this is the func input for coreg in ccs # TODO evaluate if it's necessary to use this brain - example_func_brain = pe.Node(interface=fsl.maths.ApplyMask(), - name=f'get_example_func_brain_{pipe_num}') + example_func_brain = pe.Node( + interface=fsl.maths.ApplyMask(), name=f"get_example_func_brain_{pipe_num}" + ) - wf.connect(func_roi, 'roi_file', - example_func_brain, 'in_file') + wf.connect(func_roi, "roi_file", example_func_brain, "in_file") - wf.connect(intersect_mask, 'out_file', - example_func_brain, 'mask_file') + wf.connect(intersect_mask, "out_file", example_func_brain, "mask_file") outputs = { - 'space-bold_desc-brain_mask': (intersect_mask, 'out_file'), - 'desc-ref_bold': (example_func_brain, 'out_file') + "space-bold_desc-brain_mask": (intersect_mask, "out_file"), + "desc-ref_bold": (example_func_brain, "out_file"), } return (wf, outputs) @nodeblock( - name='bold_masking', - switch=[['functional_preproc', 'run'], - ['functional_preproc', 'func_masking', 'run'], - ['functional_preproc', 'func_masking', 'apply_func_mask_in_native_space']], - inputs=[('desc-preproc_bold', 'space-bold_desc-brain_mask')], - outputs={'desc-preproc_bold': {'Description': 'The skull-stripped BOLD time-series.', 'SkullStripped': True}, - 'desc-brain_bold': {'Description': 'The skull-stripped BOLD time-series.', 'SkullStripped': True}} + name="bold_masking", + switch=[ + ["functional_preproc", "run"], + ["functional_preproc", "func_masking", "run"], + ["functional_preproc", "func_masking", "apply_func_mask_in_native_space"], + ], + inputs=[("desc-preproc_bold", "space-bold_desc-brain_mask")], + outputs={ + "desc-preproc_bold": { + "Description": "The skull-stripped BOLD time-series.", + "SkullStripped": True, + }, + "desc-brain_bold": { + "Description": "The skull-stripped BOLD time-series.", + "SkullStripped": True, + }, + }, ) def bold_masking(wf, cfg, strat_pool, pipe_num, opt=None): - func_edge_detect = pe.Node(interface=afni_utils.Calc(), - name=f'func_extract_brain_{pipe_num}') + func_edge_detect = pe.Node( + interface=afni_utils.Calc(), name=f"func_extract_brain_{pipe_num}" + ) - func_edge_detect.inputs.expr = 'a*b' - func_edge_detect.inputs.outputtype = 'NIFTI_GZ' + func_edge_detect.inputs.expr = "a*b" + func_edge_detect.inputs.outputtype = "NIFTI_GZ" node, out = strat_pool.get_data("desc-preproc_bold") - wf.connect(node, out, func_edge_detect, 'in_file_a') + wf.connect(node, out, func_edge_detect, "in_file_a") node, out = strat_pool.get_data("space-bold_desc-brain_mask") - wf.connect(node, out, func_edge_detect, 'in_file_b') + wf.connect(node, out, func_edge_detect, "in_file_b") outputs = { - 'desc-preproc_bold': (func_edge_detect, 'out_file'), - 'desc-brain_bold': (func_edge_detect, 'out_file') + "desc-preproc_bold": (func_edge_detect, "out_file"), + "desc-brain_bold": (func_edge_detect, "out_file"), } return (wf, outputs) @nodeblock( - name='func_mean', - switch=[['functional_preproc', 'run'], ['functional_preproc', 'generate_func_mean', 'run']], - inputs=['desc-preproc_bold'], - outputs=['desc-mean_bold'] + name="func_mean", + switch=[ + ["functional_preproc", "run"], + ["functional_preproc", "generate_func_mean", "run"], + ], + inputs=["desc-preproc_bold"], + outputs=["desc-mean_bold"], ) def func_mean(wf, cfg, strat_pool, pipe_num, opt=None): + func_mean = pe.Node(interface=afni_utils.TStat(), name=f"func_mean_{pipe_num}") - func_mean = pe.Node(interface=afni_utils.TStat(), - name=f'func_mean_{pipe_num}') - - func_mean.inputs.options = '-mean' - func_mean.inputs.outputtype = 'NIFTI_GZ' + func_mean.inputs.options = "-mean" + func_mean.inputs.outputtype = "NIFTI_GZ" node, out = strat_pool.get_data("desc-preproc_bold") - wf.connect(node, out, func_mean, 'in_file') + wf.connect(node, out, func_mean, "in_file") - outputs = { - 'desc-mean_bold': (func_mean, 'out_file') - } + outputs = {"desc-mean_bold": (func_mean, "out_file")} return (wf, outputs) @nodeblock( - name='func_normalize', - switch=[['functional_preproc', 'run'], ['functional_preproc', 'normalize_func', 'run']], - inputs=['desc-preproc_bold'], - outputs=['desc-preproc_bold'] + name="func_normalize", + switch=[ + ["functional_preproc", "run"], + ["functional_preproc", "normalize_func", "run"], + ], + inputs=["desc-preproc_bold"], + outputs=["desc-preproc_bold"], ) def func_normalize(wf, cfg, strat_pool, pipe_num, opt=None): - func_normalize = pe.Node(interface=fsl.ImageMaths(), - name=f'func_normalize_{pipe_num}', - mem_gb=0.7, - mem_x=(4538494663498653 / - 604462909807314587353088, 'in_file')) - func_normalize.inputs.op_string = '-ing 10000' - func_normalize.inputs.out_data_type = 'float' + func_normalize = pe.Node( + interface=fsl.ImageMaths(), + name=f"func_normalize_{pipe_num}", + mem_gb=0.7, + mem_x=(4538494663498653 / 604462909807314587353088, "in_file"), + ) + func_normalize.inputs.op_string = "-ing 10000" + func_normalize.inputs.out_data_type = "float" node, out = strat_pool.get_data("desc-preproc_bold") - wf.connect(node, out, func_normalize, 'in_file') + wf.connect(node, out, func_normalize, "in_file") - outputs = { - 'desc-preproc_bold': (func_normalize, 'out_file') - } + outputs = {"desc-preproc_bold": (func_normalize, "out_file")} return (wf, outputs) @nodeblock( - name='func_mask_normalize', - config=['functional_preproc'], - switch=['run'], - inputs=[('desc-preproc_bold', 'space-bold_desc-brain_mask')], - outputs=['space-bold_desc-brain_mask'] + name="func_mask_normalize", + config=["functional_preproc"], + switch=["run"], + inputs=[("desc-preproc_bold", "space-bold_desc-brain_mask")], + outputs=["space-bold_desc-brain_mask"], ) def func_mask_normalize(wf, cfg, strat_pool, pipe_num, opt=None): - - func_mask_normalize = pe.Node(interface=fsl.ImageMaths(), - name=f'func_mask_normalize_{pipe_num}', - mem_gb=0.7, - mem_x=(4538494663498653 / - 604462909807314587353088, 'in_file')) - func_mask_normalize.inputs.op_string = '-Tmin -bin' - func_mask_normalize.inputs.out_data_type = 'char' + func_mask_normalize = pe.Node( + interface=fsl.ImageMaths(), + name=f"func_mask_normalize_{pipe_num}", + mem_gb=0.7, + mem_x=(4538494663498653 / 604462909807314587353088, "in_file"), + ) + func_mask_normalize.inputs.op_string = "-Tmin -bin" + func_mask_normalize.inputs.out_data_type = "char" node, out = strat_pool.get_data("desc-preproc_bold") - wf.connect(node, out, func_mask_normalize, 'in_file') + wf.connect(node, out, func_mask_normalize, "in_file") - outputs = { - 'space-bold_desc-brain_mask': (func_mask_normalize, 'out_file') - } + outputs = {"space-bold_desc-brain_mask": (func_mask_normalize, "out_file")} return (wf, outputs) diff --git a/CPAC/func_preproc/tests/test_preproc_connections.py b/CPAC/func_preproc/tests/test_preproc_connections.py index 258533d883..51ec8fe2e2 100644 --- a/CPAC/func_preproc/tests/test_preproc_connections.py +++ b/CPAC/func_preproc/tests/test_preproc_connections.py @@ -14,141 +14,188 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -"""Test graph connections for functional preprocessing""" +"""Test graph connections for functional preprocessing.""" from itertools import product from random import sample import re from typing import Union -from nipype.interfaces.utility import Function as NipypeFunction, \ - IdentityInterface -from nipype.pipeline.engine import Workflow as NipypeWorkflow import pytest from traits.trait_base import Undefined from voluptuous.error import Invalid +from nipype.interfaces.utility import Function as NipypeFunction +from nipype.pipeline.engine import Workflow as NipypeWorkflow -from CPAC.func_preproc.func_motion import calc_motion_stats, \ - func_motion_correct, func_motion_correct_only, func_motion_estimates, \ - get_motion_ref, motion_estimate_filter +from CPAC.func_preproc.func_motion import ( + calc_motion_stats, + func_motion_correct, + func_motion_correct_only, + func_motion_estimates, + get_motion_ref, + motion_estimate_filter, +) from CPAC.func_preproc.func_preproc import func_normalize from CPAC.nuisance.nuisance import choose_nuisance_blocks from CPAC.pipeline.cpac_pipeline import connect_pipeline -from CPAC.pipeline.engine import NodeBlock, ResourcePool -from CPAC.pipeline.nipype_pipeline_engine import Node, Workflow -from CPAC.registration.registration import coregistration_prep_fmriprep, \ - coregistration_prep_mean, coregistration_prep_vol +from CPAC.pipeline.engine import ResourcePool +from CPAC.pipeline.nipype_pipeline_engine import Workflow +from CPAC.registration.registration import ( + coregistration_prep_fmriprep, + coregistration_prep_mean, + coregistration_prep_vol, +) from CPAC.utils.configuration import Configuration from CPAC.utils.interfaces.function import Function as CpacFunction from CPAC.utils.test_init import create_dummy_node from CPAC.utils.typing import LIST - -_FILTERS = [{'filter_type': 'notch', 'filter_order': 4, - 'center_frequency': 0.31, 'filter_bandwidth': 0.12}, - {'filter_type': 'lowpass', 'filter_order': 4, - 'lowpass_cutoff': .0032}] -_PRE_RESOURCES = ['desc-preproc_bold', - 'label-CSF_desc-eroded_mask', - 'label-CSF_desc-preproc_mask', - 'label-CSF_mask', - 'label-GM_desc-eroded_mask', - 'label-GM_desc-preproc_mask', - 'label-GM_mask', - 'label-WM_desc-eroded_mask', - 'label-WM_desc-preproc_mask', - 'label-WM_mask', - 'lateral-ventricles-mask', - 'space-T1w_desc-brain_mask', - 'space-T1w_desc-eroded_mask', - 'space-bold_desc-brain_mask', - 'TR', - 'scan', - 'subject', - 'desc-brain_T1w', - 'from-T1w_to-template_mode-image_desc-linear_xfm', - 'from-bold_to-T1w_mode-image_desc-linear_xfm', - 'from-template_to-T1w_mode-image_desc-linear_xfm'] +_FILTERS = [ + { + "filter_type": "notch", + "filter_order": 4, + "center_frequency": 0.31, + "filter_bandwidth": 0.12, + }, + {"filter_type": "lowpass", "filter_order": 4, "lowpass_cutoff": 0.0032}, +] +_PRE_RESOURCES = [ + "desc-preproc_bold", + "label-CSF_desc-eroded_mask", + "label-CSF_desc-preproc_mask", + "label-CSF_mask", + "label-GM_desc-eroded_mask", + "label-GM_desc-preproc_mask", + "label-GM_mask", + "label-WM_desc-eroded_mask", + "label-WM_desc-preproc_mask", + "label-WM_mask", + "lateral-ventricles-mask", + "space-T1w_desc-brain_mask", + "space-T1w_desc-eroded_mask", + "space-bold_desc-brain_mask", + "TR", + "scan", + "subject", + "desc-brain_T1w", + "from-T1w_to-template_mode-image_desc-linear_xfm", + "from-bold_to-T1w_mode-image_desc-linear_xfm", + "from-template_to-T1w_mode-image_desc-linear_xfm", +] NUM_TESTS = 48 # number of parameterizations to run for many-parameter tests -def _filter_assertion_message(subwf: NipypeWorkflow, is_filtered: bool, - should_be_filtered: bool) -> str: +def _filter_assertion_message( + subwf: NipypeWorkflow, is_filtered: bool, should_be_filtered: bool +) -> str: if is_filtered and not should_be_filtered: return ( f'{subwf.name} is filtered by ' f'{" & ".join([node.name for node in is_filtered])} and should ' - 'not be') - return f'{subwf.name} is not filtered and should be' + 'not be' + ) + return f"{subwf.name} is not filtered and should be" _PARAMS = { # for test_motion_filter_connections - 'calculate_motion_first': [True, False], - 'filters': [[_FILTERS[0]], [_FILTERS[1]], _FILTERS], - 'motion_correction': [['mcflirt'], ['3dvolreg'], ['mcflirt', '3dvolreg']], - 'pre_resources': [_PRE_RESOURCES, ['desc-movementParameters_motion', - *_PRE_RESOURCES]], - 'regtool': ['ANTs', 'FSL'], - 'run': [True, False, [True, False]]} # product == 216 + "calculate_motion_first": [True, False], + "filters": [[_FILTERS[0]], [_FILTERS[1]], _FILTERS], + "motion_correction": [["mcflirt"], ["3dvolreg"], ["mcflirt", "3dvolreg"]], + "pre_resources": [ + _PRE_RESOURCES, + ["desc-movementParameters_motion", *_PRE_RESOURCES], + ], + "regtool": ["ANTs", "FSL"], + "run": [True, False, [True, False]], +} # product == 216 -@pytest.mark.parametrize(','.join(_PARAMS.keys()), # run n=NUM_TESTS subset - sample(list(product(*_PARAMS.values())), NUM_TESTS)) -def test_motion_filter_connections(run: Union[bool, LIST[bool]], - filters: LIST[dict], regtool: LIST[str], - calculate_motion_first: bool, - pre_resources: LIST[str], - motion_correction: LIST[LIST[str]]) -> None: - """Test that appropriate connections occur vis-à-vis motion filters""" +@pytest.mark.parametrize( + ",".join(_PARAMS.keys()), # run n=NUM_TESTS subset + sample(list(product(*_PARAMS.values())), NUM_TESTS), +) +def test_motion_filter_connections( + run: Union[bool, LIST[bool]], + filters: LIST[dict], + regtool: LIST[str], + calculate_motion_first: bool, + pre_resources: LIST[str], + motion_correction: LIST[LIST[str]], +) -> None: + """Test that appropriate connections occur vis-à-vis motion filters.""" if isinstance(motion_correction, list) and len(motion_correction) != 1: # Until https://github.com/FCP-INDI/C-PAC/issues/1935 is resolved with pytest.raises(Invalid) as invalid: - c = Configuration({ - 'functional_preproc': { - 'motion_estimates_and_correction': { - 'motion_correction': {'using': motion_correction}}}}) - assert 'FCP-INDI/C-PAC/issues/1935' in invalid + c = Configuration( + { + "functional_preproc": { + "motion_estimates_and_correction": { + "motion_correction": {"using": motion_correction} + } + } + } + ) + assert "FCP-INDI/C-PAC/issues/1935" in invalid return # parameterized Configuration - c = Configuration({ - 'functional_preproc': { - 'motion_estimates_and_correction': { - 'motion_correction': {'using': motion_correction}, - 'motion_estimates': { - 'calculate_motion_after': not calculate_motion_first, - 'calculate_motion_first': calculate_motion_first}, - 'motion_estimate_filter': { - 'run': run, - 'filters': filters}, - 'run': True}, - 'run': True}, - 'nuisance_corrections': { - '2-nuisance_regression': { - 'Regressors': [{ - 'Name': 'aCompCor, GSR, no censor', - 'Motion': {'include_delayed': True, - 'include_squared': True, - 'include_delayed_squared': True}, - 'aCompCor': {'summary': {'method': 'DetrendPC', - 'components': 5}, - 'tissues': ['WhiteMatter', - 'CerebrospinalFluid'], - 'extraction_resolution': 3}, - 'GlobalSignal': {'summary': 'Mean'}, - 'PolyOrt': {'degree': 2}, - 'Bandpass': {'bottom_frequency': 0.01, - 'top_frequency': 0.1}}]}}}) + c = Configuration( + { + "functional_preproc": { + "motion_estimates_and_correction": { + "motion_correction": {"using": motion_correction}, + "motion_estimates": { + "calculate_motion_after": not calculate_motion_first, + "calculate_motion_first": calculate_motion_first, + }, + "motion_estimate_filter": {"run": run, "filters": filters}, + "run": True, + }, + "run": True, + }, + "nuisance_corrections": { + "2-nuisance_regression": { + "Regressors": [ + { + "Name": "aCompCor, GSR, no censor", + "Motion": { + "include_delayed": True, + "include_squared": True, + "include_delayed_squared": True, + }, + "aCompCor": { + "summary": {"method": "DetrendPC", "components": 5}, + "tissues": ["WhiteMatter", "CerebrospinalFluid"], + "extraction_resolution": 3, + }, + "GlobalSignal": {"summary": "Mean"}, + "PolyOrt": {"degree": 2}, + "Bandpass": { + "bottom_frequency": 0.01, + "top_frequency": 0.1, + }, + } + ] + } + }, + } + ) # resource for intial inputs - before_this_test = create_dummy_node('created_before_this_test', - pre_resources) + before_this_test = create_dummy_node("created_before_this_test", pre_resources) rpool = ResourcePool(cfg=c) for resource in pre_resources: - if resource.endswith('xfm'): - rpool.set_data(resource, before_this_test, resource, {}, "", - f"created_before_this_test_{regtool}") + if resource.endswith("xfm"): + rpool.set_data( + resource, + before_this_test, + resource, + {}, + "", + f"created_before_this_test_{regtool}", + ) else: - rpool.set_data(resource, before_this_test, resource, {}, "", - "created_before_this_test") + rpool.set_data( + resource, before_this_test, resource, {}, "", "created_before_this_test" + ) # set up blocks pipeline_blocks = [] func_init_blocks = [] @@ -158,87 +205,127 @@ def test_motion_filter_connections(run: Union[bool, LIST[bool]], func_prep_blocks = [ calc_motion_stats, func_normalize, - [coregistration_prep_vol, - coregistration_prep_mean, - coregistration_prep_fmriprep] + [ + coregistration_prep_vol, + coregistration_prep_mean, + coregistration_prep_fmriprep, + ], ] # Motion Correction func_motion_blocks = [] - if c['functional_preproc', 'motion_estimates_and_correction', - 'motion_estimates', 'calculate_motion_first']: + if c[ + "functional_preproc", + "motion_estimates_and_correction", + "motion_estimates", + "calculate_motion_first", + ]: func_motion_blocks = [ get_motion_ref, func_motion_estimates, - motion_estimate_filter + motion_estimate_filter, ] else: func_motion_blocks = [ get_motion_ref, func_motion_correct, - motion_estimate_filter + motion_estimate_filter, ] - if not rpool.check_rpool('desc-movementParameters_motion'): - if c['functional_preproc', 'motion_estimates_and_correction', - 'motion_estimates', 'calculate_motion_first']: - func_blocks = func_init_blocks + func_motion_blocks + \ - func_preproc_blocks + [func_motion_correct_only] + \ - func_mask_blocks + func_prep_blocks + if not rpool.check_rpool("desc-movementParameters_motion"): + if c[ + "functional_preproc", + "motion_estimates_and_correction", + "motion_estimates", + "calculate_motion_first", + ]: + func_blocks = ( + func_init_blocks + + func_motion_blocks + + func_preproc_blocks + + [func_motion_correct_only] + + func_mask_blocks + + func_prep_blocks + ) else: - func_blocks = func_init_blocks + func_preproc_blocks + \ - func_motion_blocks + func_mask_blocks + \ - func_prep_blocks + func_blocks = ( + func_init_blocks + + func_preproc_blocks + + func_motion_blocks + + func_mask_blocks + + func_prep_blocks + ) else: - func_blocks = func_init_blocks + func_preproc_blocks + \ - func_motion_blocks + func_mask_blocks + \ - func_prep_blocks + func_blocks = ( + func_init_blocks + + func_preproc_blocks + + func_motion_blocks + + func_mask_blocks + + func_prep_blocks + ) pipeline_blocks += func_blocks # Nuisance Correction - generate_only = True not in c['nuisance_corrections', - '2-nuisance_regression', 'run'] - if not rpool.check_rpool('desc-cleaned_bold'): + generate_only = ( + True not in c["nuisance_corrections", "2-nuisance_regression", "run"] + ) + if not rpool.check_rpool("desc-cleaned_bold"): pipeline_blocks += choose_nuisance_blocks(c, generate_only) - wf = Workflow(re.sub(r'[\[\]\-\:\_ \'\",]', '', str(rpool))) + wf = Workflow(re.sub(r"[\[\]\-\:\_ \'\",]", "", str(rpool))) connect_pipeline(wf, c, rpool, pipeline_blocks) # Check that filtering is happening as expected - filter_switch_key = ['functional_preproc', - 'motion_estimates_and_correction', - 'motion_estimate_filter', 'run'] + filter_switch_key = [ + "functional_preproc", + "motion_estimates_and_correction", + "motion_estimate_filter", + "run", + ] if c.switch_is_on(filter_switch_key, exclusive=True): - assert all(strat.filtered_movement for strat in - rpool.get_strats(['desc-movementParameters_motion'] - ).values()) + assert all( + strat.filtered_movement + for strat in rpool.get_strats(["desc-movementParameters_motion"]).values() + ) elif c.switch_is_off(filter_switch_key, exclusive=True): - assert not any(strat.filtered_movement for strat in - rpool.get_strats(['desc-movementParameters_motion'] - ).values()) + assert not any( + strat.filtered_movement + for strat in rpool.get_strats(["desc-movementParameters_motion"]).values() + ) elif c.switch_is_on_off(filter_switch_key): - assert any(strat.filtered_movement for strat in - rpool.get_strats(['desc-movementParameters_motion'] - ).values()) - if 'mcflirt' in c['functional_preproc', - 'motion_estimates_and_correction', - 'motion_correction', 'using']: + assert any( + strat.filtered_movement + for strat in rpool.get_strats(["desc-movementParameters_motion"]).values() + ) + if ( + "mcflirt" + in c[ + "functional_preproc", + "motion_estimates_and_correction", + "motion_correction", + "using", + ] + ): # Only for [On, Off] + mcflirt, we should have at least one of each - assert set(wf.get_node(nodename).inputs.calc_from for nodename in - wf.list_node_names() if - nodename.endswith('.calculate_FDJ') and - wf.get_node(nodename).inputs.calc_from is - not Undefined) == {'affine', 'rms'} - regressor_subwfs = [wf.get_node(nodename[:-26]) for nodename in - wf.list_node_names() if - nodename.endswith('build_nuisance_regressors')] + assert { + wf.get_node(nodename).inputs.calc_from + for nodename in wf.list_node_names() + if nodename.endswith(".calculate_FDJ") + and wf.get_node(nodename).inputs.calc_from is not Undefined + } == {"affine", "rms"} + regressor_subwfs = [ + wf.get_node(nodename[:-26]) + for nodename in wf.list_node_names() + if nodename.endswith("build_nuisance_regressors") + ] for subwf in regressor_subwfs: # a motion filter is an input to the nuisance regressor subworkflow is_filtered = [] # a motion filter should be an input to the regressor subworkflow - should_be_filtered = ('_filt-' in subwf.name and '_filt-none' not in - subwf.name) + should_be_filtered = "_filt-" in subwf.name and "_filt-none" not in subwf.name for u, v in wf._graph.edges: # pylint: disable=invalid-name,protected-access - if (v == subwf and hasattr(u, 'interface') and - isinstance(u.interface, (NipypeFunction, CpacFunction)) and - 'notch_filter_motion' in u.interface.inputs.function_str + if ( + v == subwf + and hasattr(u, "interface") + and isinstance(u.interface, (NipypeFunction, CpacFunction)) + and "notch_filter_motion" in u.interface.inputs.function_str ): is_filtered.append(u) - assert bool(is_filtered - ) == should_be_filtered, _filter_assertion_message( - subwf, is_filtered, should_be_filtered) + assert bool(is_filtered) == should_be_filtered, _filter_assertion_message( + subwf, is_filtered, should_be_filtered + ) diff --git a/CPAC/func_preproc/utils.py b/CPAC/func_preproc/utils.py index 87e3c98d0f..34951d5458 100644 --- a/CPAC/func_preproc/utils.py +++ b/CPAC/func_preproc/utils.py @@ -1,14 +1,15 @@ +import math +import subprocess -import numpy as np -from scipy.signal import iirnotch, firwin, filtfilt, freqz from matplotlib import pyplot as plt -import nibabel as nb -import subprocess -import math +import numpy as np +import nibabel as nib +from scipy.signal import filtfilt, firwin, freqz, iirnotch def nullify(value, function=None): from traits.trait_base import Undefined + if value is None: return Undefined if function: @@ -17,40 +18,47 @@ def nullify(value, function=None): def chunk_ts(func_file, n_chunks=None, chunk_size=None): - func_img = nb.load(func_file) + func_img = nib.load(func_file) trs = func_img.shape[3] TR_ranges = [] if n_chunks: - chunk_size = trs/n_chunks + chunk_size = trs / n_chunks elif chunk_size: - n_chunks = int(trs/chunk_size) + n_chunks = int(trs / chunk_size) else: - raise Exception("\n[!] Dev error: Either 'n_chunks' or 'chunk_size' " - "arguments must be passed to 'chunk_ts' function.\n") + raise Exception( + "\n[!] Dev error: Either 'n_chunks' or 'chunk_size' " + "arguments must be passed to 'chunk_ts' function.\n" + ) for chunk_idx in range(0, n_chunks): if chunk_idx == n_chunks - 1: - TR_ranges.append((int(chunk_idx*chunk_size), int(trs - 1))) + TR_ranges.append((int(chunk_idx * chunk_size), int(trs - 1))) else: - TR_ranges.append((int(chunk_idx*chunk_size), int((chunk_idx+1)*chunk_size - 1))) + TR_ranges.append( + (int(chunk_idx * chunk_size), int((chunk_idx + 1) * chunk_size - 1)) + ) return TR_ranges def split_ts_chunks(func_file, tr_ranges): - if '.nii' in func_file: - ext = '.nii' - if '.nii.gz' in func_file: - ext = '.nii.gz' + if ".nii" in func_file: + ext = ".nii" + if ".nii.gz" in func_file: + ext = ".nii.gz" split_funcs = [] for chunk_idx, tr_range in enumerate(tr_ranges): - out_file = os.path.join(os.getcwd(), os.path.basename(func_file).replace(ext, "_{0}{1}".format(chunk_idx, ext))) + out_file = os.path.join( + os.getcwd(), + os.path.basename(func_file).replace(ext, "_{0}{1}".format(chunk_idx, ext)), + ) in_file = "{0}[{1}..{2}]".format(func_file, tr_range[0], tr_range[1]) cmd = ["3dcalc", "-a", in_file, "-expr", "a", "-prefix", out_file] - retcode = subprocess.check_output(cmd) + subprocess.check_output(cmd) split_funcs.append(out_file) @@ -58,21 +66,23 @@ def split_ts_chunks(func_file, tr_ranges): def oned_text_concat(in_files): - out_file = os.path.join(os.getcwd(), os.path.basename(in_files[0].replace("_0", ""))) + out_file = os.path.join( + os.getcwd(), os.path.basename(in_files[0].replace("_0", "")) + ) out_txt = [] for txt in in_files: - with open(txt, 'r') as f: + with open(txt, "r") as f: txt_lines = f.readlines() if not out_txt: - out_txt = [x for x in txt_lines] + out_txt = list(txt_lines) else: for line in txt_lines: if "#" in line: continue out_txt.append(line) - with open(out_file, 'wt') as f: + with open(out_file, "wt") as f: for line in out_txt: f.write(line) @@ -81,47 +91,51 @@ def oned_text_concat(in_files): def degrees_to_mm(degrees, head_radius): # function to convert degrees of motion to mm - mm = 2*math.pi*head_radius*(degrees/360) - return mm + return 2 * math.pi * head_radius * (degrees / 360) def mm_to_degrees(mm, head_radius): # function to convert mm of motion to degrees - degrees = 360*mm/(2*math.pi*head_radius) - return degrees + return 360 * mm / (2 * math.pi * head_radius) + def degrees_to_mm(degrees, head_radius): # function to convert degrees of motion to mm - mm = 2*math.pi*head_radius*(degrees/360) - return mm + return 2 * math.pi * head_radius * (degrees / 360) def mm_to_degrees(mm, head_radius): # function to convert mm of motion to degrees - degrees = 360*mm/(2*math.pi*head_radius) - return degrees + return 360 * mm / (2 * math.pi * head_radius) + def degrees_to_mm(degrees, head_radius): # function to convert degrees of motion to mm - mm = 2*math.pi*head_radius*(degrees/360) - return mm + return 2 * math.pi * head_radius * (degrees / 360) def mm_to_degrees(mm, head_radius): # function to convert mm of motion to degrees - degrees = 360*mm/(2*math.pi*head_radius) - return degrees - - -def notch_filter_motion(motion_params, filter_type, TR, fc_RR_min=None, - fc_RR_max=None, center_freq=None, freq_bw=None, - lowpass_cutoff=None, filter_order=4): + return 360 * mm / (2 * math.pi * head_radius) + + +def notch_filter_motion( + motion_params, + filter_type, + TR, + fc_RR_min=None, + fc_RR_max=None, + center_freq=None, + freq_bw=None, + lowpass_cutoff=None, + filter_order=4, +): # Adapted from DCAN Labs: # https://github.com/DCAN-Labs/dcan_bold_processing/blob/master/ # ...matlab_code/filtered_movement_regressors.m if "ms" in TR: - TR = float(TR.replace("ms", ""))/1000 + TR = float(TR.replace("ms", "")) / 1000 elif "ms" not in TR and "s" in TR: TR = float(TR.replace("s", "")) @@ -134,18 +148,16 @@ def notch_filter_motion(motion_params, filter_type, TR, fc_RR_min=None, fNy = fs / 2 if filter_type == "notch": - # Respiratory Rate if fc_RR_min and fc_RR_max: - rr = [float(fc_RR_min) / float(60), - float(fc_RR_max) / float(60)] + rr = [float(fc_RR_min) / float(60), float(fc_RR_max) / float(60)] rr_fNy = [rr[0] + fNy, rr[1] + fNy] fa = abs(rr - np.floor(np.divide(rr_fNy, fs)) * fs) elif center_freq and freq_bw: - tail = float(freq_bw)/float(2) - fa = [center_freq-tail, center_freq+tail] + tail = float(freq_bw) / float(2) + fa = [center_freq - tail, center_freq + tail] W_notch = np.divide(fa, fNy) @@ -156,17 +168,18 @@ def notch_filter_motion(motion_params, filter_type, TR, fc_RR_min=None, center_freq = Wn * fNy bandwidth = fa[1] - fa[0] - Q = Wn/bw + Q = Wn / bw [b_filt, a_filt] = iirnotch(Wn, Q) - num_f_apply = np.floor(filter_order / 2) + np.floor(filter_order / 2) - filter_info = f"Motion estimate filter information\n\nType: Notch\n" \ - f"\nCenter freq: {center_freq}\nBandwidth: {bandwidth}\n\n" \ - f"Wn: {Wn}\nQ: {Q}\n\n" \ - f"Based on:\nSampling freq: {fs}\nNyquist freq: {fNy}" + filter_info = ( + f"Motion estimate filter information\n\nType: Notch\n" + f"\nCenter freq: {center_freq}\nBandwidth: {bandwidth}\n\n" + f"Wn: {Wn}\nQ: {Q}\n\n" + f"Based on:\nSampling freq: {fs}\nNyquist freq: {fNy}" + ) elif filter_type == "lowpass": - if fc_RR_min: rr = float(fc_RR_min) / float(60) rr_fNy = rr + fNy @@ -175,36 +188,34 @@ def notch_filter_motion(motion_params, filter_type, TR, fc_RR_min=None, elif lowpass_cutoff: fa = lowpass_cutoff - Wn = fa/fNy + Wn = fa / fNy if filter_order: - b_filt = firwin(filter_order+1, Wn) + b_filt = firwin(filter_order + 1, Wn) a_filt = 1 - num_f_apply = 0 - - filter_info = f"Motion estimate filter information\n\nType: Lowpass" \ - f"\n\nCutoff freq: {fa}\nWn: {Wn}\n\n" \ - f"Based on:\nSampling freq: {fs}\nNyquist freq: {fNy}" + filter_info = ( + f"Motion estimate filter information\n\nType: Lowpass" + f"\n\nCutoff freq: {fa}\nWn: {Wn}\n\n" + f"Based on:\nSampling freq: {fs}\nNyquist freq: {fNy}" + ) - filter_design = os.path.join(os.getcwd(), - "motion_estimate_filter_design.txt") - filter_plot = os.path.join(os.getcwd(), - "motion_estimate_filter_freq-response.png") + filter_design = os.path.join(os.getcwd(), "motion_estimate_filter_design.txt") + filter_plot = os.path.join(os.getcwd(), "motion_estimate_filter_freq-response.png") # plot frequency response for user info w, h = freqz(b_filt, a_filt, fs=fs) fig, ax1 = plt.subplots() - ax1.set_title('Motion estimate filter frequency response') + ax1.set_title("Motion estimate filter frequency response") - ax1.plot(w, 20 * np.log10(abs(h)), 'b') - ax1.set_ylabel('Amplitude [dB]', color='b') - ax1.set_xlabel('Frequency [Hz]') + ax1.plot(w, 20 * np.log10(abs(h)), "b") + ax1.set_ylabel("Amplitude [dB]", color="b") + ax1.set_xlabel("Frequency [Hz]") plt.savefig(filter_plot) - with open(filter_design, 'wt') as f: + with open(filter_design, "wt") as f: f.write(filter_info) # convert rotation params from degrees to mm @@ -213,13 +224,14 @@ def notch_filter_motion(motion_params, filter_type, TR, fc_RR_min=None, filtered_params = filtfilt(b_filt, a_filt, params_data.T) # back rotation params to degrees - filtered_params[0:3,:] = mm_to_degrees(filtered_params[0:3,:], head_radius = 50) + filtered_params[0:3, :] = mm_to_degrees(filtered_params[0:3, :], head_radius=50) # back rotation params to degrees - filtered_params[0:3,:] = mm_to_degrees(filtered_params[0:3,:], head_radius = 50) + filtered_params[0:3, :] = mm_to_degrees(filtered_params[0:3, :], head_radius=50) - filtered_motion_params = os.path.join(os.getcwd(), - "{0}_filtered.1D".format(os.path.basename(motion_params))) - np.savetxt(filtered_motion_params, filtered_params.T, fmt='%f') + filtered_motion_params = os.path.join( + os.getcwd(), "{0}_filtered.1D".format(os.path.basename(motion_params)) + ) + np.savetxt(filtered_motion_params, filtered_params.T, fmt="%f") return (filtered_motion_params, filter_design, filter_plot) diff --git a/CPAC/generate_motion_statistics/__init__.py b/CPAC/generate_motion_statistics/__init__.py index 3f4e41df3b..23c8821955 100644 --- a/CPAC/generate_motion_statistics/__init__.py +++ b/CPAC/generate_motion_statistics/__init__.py @@ -14,24 +14,26 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -"""Functions for generating motion statistics""" -from .generate_motion_statistics import (motion_power_statistics, - calculate_FD_P, - calculate_FD_J, - gen_motion_parameters, - gen_power_parameters, - calculate_DVARS, - ImageTo1D) +"""Functions for generating motion statistics.""" +from .generate_motion_statistics import ( + ImageTo1D, + calculate_DVARS, + calculate_FD_J, + calculate_FD_P, + gen_motion_parameters, + gen_power_parameters, + motion_power_statistics, +) from .utils import affine_file_from_params_file, affine_from_params __all__ = [ - 'affine_file_from_params_file', - 'affine_from_params', - 'calculate_DVARS', - 'calculate_FD_P', - 'calculate_FD_J', - 'gen_motion_parameters', - 'gen_power_parameters', - 'ImageTo1D', - 'motion_power_statistics' + "affine_file_from_params_file", + "affine_from_params", + "calculate_DVARS", + "calculate_FD_P", + "calculate_FD_J", + "gen_motion_parameters", + "gen_power_parameters", + "ImageTo1D", + "motion_power_statistics", ] diff --git a/CPAC/generate_motion_statistics/generate_motion_statistics.py b/CPAC/generate_motion_statistics/generate_motion_statistics.py index f8400804ef..f2990ed39d 100644 --- a/CPAC/generate_motion_statistics/generate_motion_statistics.py +++ b/CPAC/generate_motion_statistics/generate_motion_statistics.py @@ -14,25 +14,27 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -"""Functions to calculate motion statistics""" +"""Functions to calculate motion statistics.""" import os import sys from typing import Optional -import nibabel as nb -from nipype.interfaces.afni.base import (AFNICommand, AFNICommandInputSpec) -from nipype.interfaces.base import (TraitedSpec, traits, File) -from nipype.interfaces import utility as util + import numpy as np import pandas as pd +import nibabel as nib +from nipype.interfaces import utility as util +from nipype.interfaces.afni.base import AFNICommand, AFNICommandInputSpec +from nipype.interfaces.base import File, TraitedSpec, traits + from CPAC.pipeline import nipype_pipeline_engine as pe from CPAC.utils.interfaces.function import Function from CPAC.utils.pytest import skipif from CPAC.utils.typing import LITERAL, TUPLE -def motion_power_statistics(name='motion_stats', - motion_correct_tool='3dvolreg', - filtered=False): +def motion_power_statistics( + name="motion_stats", motion_correct_tool="3dvolreg", filtered=False +): """ The main purpose of this workflow is to get various statistical measures from the movement/motion parameters obtained in functional preprocessing. @@ -45,7 +47,6 @@ def motion_power_statistics(name='motion_stats', Notes ----- - Workflow Inputs:: inputspec.motion_correct : string (func/rest file or a list of func/rest nifti file) @@ -172,7 +173,6 @@ def motion_power_statistics(name='motion_stats', References ---------- - .. [1] Power, J. D., Barnes, K. A., Snyder, A. Z., Schlaggar, B. L., & Petersen, S. E. (2012). Spurious but systematic correlations in functional connectivity MRI networks arise from subject motion. NeuroImage, 59(3), 2142-2154. doi:10.1016/j.neuroimage.2011.10.018 @@ -185,152 +185,179 @@ def motion_power_statistics(name='motion_stats', and accurate linear registration and motion correction of brain images. Neuroimage 17, 825-841. """ wf = pe.Workflow(name=name) - input_node = pe.Node(util.IdentityInterface(fields=['movement_parameters', - 'max_displacement', - 'rels_displacement', - 'motion_correct', - 'mask', - 'transformations']), - name='inputspec') - - output_node = pe.Node(util.IdentityInterface(fields=['FDP_1D', - 'FDJ_1D', - 'DVARS_1D', - 'power_params', - 'motion_params', - 'motion', - 'desc-summary_motion'] - ), - name='outputspec') - - cal_DVARS = pe.Node(ImageTo1D(method='dvars'), - name='cal_DVARS', - mem_gb=0.4, - mem_x=(739971956005215 / 151115727451828646838272, - 'in_file')) - - cal_DVARS_strip = pe.Node(Function(input_names=['file_1D'], - output_names=['out_file', 'DVARS_val'], - function=DVARS_strip_t0, - as_module=True), - name='cal_DVARS_strip') + input_node = pe.Node( + util.IdentityInterface( + fields=[ + "movement_parameters", + "max_displacement", + "rels_displacement", + "motion_correct", + "mask", + "transformations", + ] + ), + name="inputspec", + ) + + output_node = pe.Node( + util.IdentityInterface( + fields=[ + "FDP_1D", + "FDJ_1D", + "DVARS_1D", + "power_params", + "motion_params", + "motion", + "desc-summary_motion", + ] + ), + name="outputspec", + ) + + cal_DVARS = pe.Node( + ImageTo1D(method="dvars"), + name="cal_DVARS", + mem_gb=0.4, + mem_x=(739971956005215 / 151115727451828646838272, "in_file"), + ) + + cal_DVARS_strip = pe.Node( + Function( + input_names=["file_1D"], + output_names=["out_file", "DVARS_val"], + function=DVARS_strip_t0, + as_module=True, + ), + name="cal_DVARS_strip", + ) # calculate mean DVARS - wf.connect(input_node, 'motion_correct', cal_DVARS, 'in_file') - wf.connect(input_node, 'mask', cal_DVARS, 'mask') - wf.connect(cal_DVARS, 'out_file', cal_DVARS_strip, 'file_1D') - wf.connect(cal_DVARS_strip, 'out_file', output_node, 'DVARS_1D') + wf.connect(input_node, "motion_correct", cal_DVARS, "in_file") + wf.connect(input_node, "mask", cal_DVARS, "mask") + wf.connect(cal_DVARS, "out_file", cal_DVARS_strip, "file_1D") + wf.connect(cal_DVARS_strip, "out_file", output_node, "DVARS_1D") # Calculating mean Framewise Displacement as per power et al., 2012 - calculate_FDP = pe.Node(Function(input_names=['in_file'], - output_names=['out_file', 'fd'], - function=calculate_FD_P, - as_module=True), - name='calculate_FD') + calculate_FDP = pe.Node( + Function( + input_names=["in_file"], + output_names=["out_file", "fd"], + function=calculate_FD_P, + as_module=True, + ), + name="calculate_FD", + ) - wf.connect(input_node, 'movement_parameters', calculate_FDP, 'in_file') - wf.connect(calculate_FDP, 'out_file', output_node, 'FDP_1D') + wf.connect(input_node, "movement_parameters", calculate_FDP, "in_file") + wf.connect(calculate_FDP, "out_file", output_node, "FDP_1D") # Calculating mean Framewise Displacement as per jenkinson et al., 2002 - calculate_FDJ = pe.Node(Function(input_names=['in_file', - 'calc_from', - 'center'], - output_names=['out_file', 'fd'], - function=calculate_FD_J, - as_module=True), - name='calculate_FDJ') - - if filtered or motion_correct_tool == '3dvolreg': - wf.connect(input_node, 'transformations', calculate_FDJ, 'in_file') - calculate_FDJ.inputs.calc_from = 'affine' - elif motion_correct_tool == 'mcflirt': - calculate_FDJ.inputs.calc_from = 'rms' - wf.connect(input_node, 'rels_displacement', calculate_FDJ, 'in_file') - - wf.connect(calculate_FDJ, 'out_file', output_node, 'FDJ_1D') - - calc_motion_parameters = pe.Node(Function(input_names=[ - 'movement_parameters', - 'max_displacement', - 'motion_correct_tool', - 'rels_displacement'], - output_names=['out_file', - 'info', - 'maxdisp', - 'relsdisp'], - function=gen_motion_parameters, - as_module=True), - name='calc_motion_parameters') - - get_all_motion_parameters = pe.Node(Function(input_names=[ - 'fdj', - 'fdp', - 'maxdisp', - 'motion', - 'power', - 'relsdisp', - 'dvars'], - output_names=[ - 'all_motion_val', - 'summary_motion_power'], - function=get_allmotion, - as_module=True), - name='get_all_motion_parameters') + calculate_FDJ = pe.Node( + Function( + input_names=["in_file", "calc_from", "center"], + output_names=["out_file", "fd"], + function=calculate_FD_J, + as_module=True, + ), + name="calculate_FDJ", + ) + + if filtered or motion_correct_tool == "3dvolreg": + wf.connect(input_node, "transformations", calculate_FDJ, "in_file") + calculate_FDJ.inputs.calc_from = "affine" + elif motion_correct_tool == "mcflirt": + calculate_FDJ.inputs.calc_from = "rms" + wf.connect(input_node, "rels_displacement", calculate_FDJ, "in_file") + + wf.connect(calculate_FDJ, "out_file", output_node, "FDJ_1D") + + calc_motion_parameters = pe.Node( + Function( + input_names=[ + "movement_parameters", + "max_displacement", + "motion_correct_tool", + "rels_displacement", + ], + output_names=["out_file", "info", "maxdisp", "relsdisp"], + function=gen_motion_parameters, + as_module=True, + ), + name="calc_motion_parameters", + ) + + get_all_motion_parameters = pe.Node( + Function( + input_names=[ + "fdj", + "fdp", + "maxdisp", + "motion", + "power", + "relsdisp", + "dvars", + ], + output_names=["all_motion_val", "summary_motion_power"], + function=get_allmotion, + as_module=True, + ), + name="get_all_motion_parameters", + ) calc_motion_parameters.inputs.motion_correct_tool = motion_correct_tool - wf.connect(calculate_FDJ, 'fd', get_all_motion_parameters, 'fdj') - wf.connect(calculate_FDP, 'fd', get_all_motion_parameters, 'fdp') - wf.connect(calc_motion_parameters, 'maxdisp', - get_all_motion_parameters, 'maxdisp') - wf.connect(calc_motion_parameters, 'relsdisp', - get_all_motion_parameters, 'relsdisp') - wf.connect(calc_motion_parameters, 'info', - get_all_motion_parameters, 'motion') - wf.connect(cal_DVARS_strip, 'DVARS_val', - get_all_motion_parameters, 'dvars') - wf.connect(input_node, 'movement_parameters', - calc_motion_parameters, 'movement_parameters') - wf.connect(input_node, 'max_displacement', - calc_motion_parameters, 'max_displacement') - wf.connect(input_node, 'rels_displacement', - calc_motion_parameters, 'rels_displacement') - wf.connect(calc_motion_parameters, 'out_file', - output_node, 'motion_params') - wf.connect(get_all_motion_parameters, 'all_motion_val', - output_node, 'motion') - wf.connect(get_all_motion_parameters, 'summary_motion_power', - output_node, 'desc-summary_motion') - - calc_power_parameters = pe.Node(Function(input_names=['fdp', - 'fdj', - 'dvars', - 'motion_correct_tool'], - output_names=['out_file', 'info'], - function=gen_power_parameters, - as_module=True), - name='calc_power_parameters') + wf.connect(calculate_FDJ, "fd", get_all_motion_parameters, "fdj") + wf.connect(calculate_FDP, "fd", get_all_motion_parameters, "fdp") + wf.connect(calc_motion_parameters, "maxdisp", get_all_motion_parameters, "maxdisp") + wf.connect( + calc_motion_parameters, "relsdisp", get_all_motion_parameters, "relsdisp" + ) + wf.connect(calc_motion_parameters, "info", get_all_motion_parameters, "motion") + wf.connect(cal_DVARS_strip, "DVARS_val", get_all_motion_parameters, "dvars") + wf.connect( + input_node, "movement_parameters", calc_motion_parameters, "movement_parameters" + ) + wf.connect( + input_node, "max_displacement", calc_motion_parameters, "max_displacement" + ) + wf.connect( + input_node, "rels_displacement", calc_motion_parameters, "rels_displacement" + ) + wf.connect(calc_motion_parameters, "out_file", output_node, "motion_params") + wf.connect(get_all_motion_parameters, "all_motion_val", output_node, "motion") + wf.connect( + get_all_motion_parameters, + "summary_motion_power", + output_node, + "desc-summary_motion", + ) + + calc_power_parameters = pe.Node( + Function( + input_names=["fdp", "fdj", "dvars", "motion_correct_tool"], + output_names=["out_file", "info"], + function=gen_power_parameters, + as_module=True, + ), + name="calc_power_parameters", + ) calc_power_parameters.inputs.motion_correct_tool = motion_correct_tool - wf.connect(calc_power_parameters, 'info', - get_all_motion_parameters, 'power') - wf.connect(cal_DVARS, 'out_file', - calc_power_parameters, 'dvars') + wf.connect(calc_power_parameters, "info", get_all_motion_parameters, "power") + wf.connect(cal_DVARS, "out_file", calc_power_parameters, "dvars") - wf.connect(calculate_FDP, 'out_file', - calc_power_parameters, 'fdp') + wf.connect(calculate_FDP, "out_file", calc_power_parameters, "fdp") - if motion_correct_tool == '3dvolreg': - wf.connect(calculate_FDJ, 'out_file', calc_power_parameters, 'fdj') + if motion_correct_tool == "3dvolreg": + wf.connect(calculate_FDJ, "out_file", calc_power_parameters, "fdj") - wf.connect(calc_power_parameters, 'out_file', - output_node, 'power_params') + wf.connect(calc_power_parameters, "out_file", output_node, "power_params") return wf def calculate_FD_P(in_file): """ - Method to calculate Framewise Displacement (FD) as per Power et al., 2012 + Method to calculate Framewise Displacement (FD) as per Power et al., 2012. Parameters ---------- @@ -345,35 +372,39 @@ def calculate_FD_P(in_file): fd : array Frame-wise displacement mat """ - motion_params = np.genfromtxt(in_file).T rotations = np.transpose(np.abs(np.diff(motion_params[0:3, :]))) translations = np.transpose(np.abs(np.diff(motion_params[3:6, :]))) - fd = np.sum(translations, axis=1) + \ - (50 * np.pi / 180) * np.sum(rotations, axis=1) + fd = np.sum(translations, axis=1) + (50 * np.pi / 180) * np.sum(rotations, axis=1) fd = np.insert(fd, 0, 0) - out_file = os.path.join(os.getcwd(), 'FD.1D') + out_file = os.path.join(os.getcwd(), "FD.1D") np.savetxt(out_file, fd) return out_file, fd -@Function.sig_imports(['import os', 'import sys', - 'from typing import Optional', - 'import numpy as np', - 'from CPAC.utils.pytest import skipif', - 'from CPAC.utils.typing import LITERAL, TUPLE']) -@skipif(sys.version_info < (3, 10), - reason="Test requires Python 3.10 or higher") -def calculate_FD_J(in_file: str, calc_from: LITERAL['affine', 'rms'], - center: Optional[np.ndarray] = None - ) -> TUPLE[str, np.ndarray]: +@Function.sig_imports( + [ + "import os", + "import sys", + "from typing import Optional", + "import numpy as np", + "from CPAC.utils.pytest import skipif", + "from CPAC.utils.typing import LITERAL, TUPLE", + ] +) +@skipif(sys.version_info < (3, 10), reason="Test requires Python 3.10 or higher") +def calculate_FD_J( + in_file: str, + calc_from: LITERAL["affine", "rms"], + center: Optional[np.ndarray] = None, +) -> TUPLE[str, np.ndarray]: """ - Method to calculate framewise displacement as per Jenkinson et al. 2002 + Method to calculate framewise displacement as per Jenkinson et al. 2002. Parameters ---------- @@ -421,7 +452,7 @@ def calculate_FD_J(in_file: str, calc_from: LITERAL['affine', 'rms'], True >>> os.unlink(fdj_file) """ - if calc_from == 'affine': + if calc_from == "affine": if center is None: center = np.zeros((3, 1)) else: @@ -452,22 +483,22 @@ def calculate_FD_J(in_file: str, calc_from: LITERAL['affine', 'rms'], T_rb_prev = T_rb - elif calc_from == 'rms': + elif calc_from == "rms": rel_rms = np.loadtxt(in_file) fd = np.append(0, rel_rms) else: raise ValueError(f"calc_from {calc_from} not supported") - out_file = os.path.join(os.getcwd(), 'FD_J.1D') - np.savetxt(out_file, fd, fmt='%.8f') + out_file = os.path.join(os.getcwd(), "FD_J.1D") + np.savetxt(out_file, fd, fmt="%.8f") return out_file, fd -def find_volume_center(img_file : str) -> np.ndarray: +def find_volume_center(img_file: str) -> np.ndarray: """ - Find the center of mass of a Nifti image volume + Find the center of mass of a Nifti image volume. Parameters ---------- @@ -476,22 +507,22 @@ def find_volume_center(img_file : str) -> np.ndarray: Returns ------- - center : ndarray + center : ndarray volume center of mass vector """ - img = nb.load(img_file) + img = nib.load(img_file) dim = np.array(img.header["dim"][1:4]) pixdim = np.array(img.header["pixdim"][1:4]) # Calculation follows MCFLIRT # https://github.com/fithisux/FSL/blob/7aa2932949129f5c61af912ea677d4dbda843895/src/mcflirt/mcflirt.cc#L479 - center = 0.5 * (dim - 1) * pixdim - return center + return 0.5 * (dim - 1) * pixdim -def gen_motion_parameters(movement_parameters, max_displacement, - motion_correct_tool, rels_displacement=None): +def gen_motion_parameters( + movement_parameters, max_displacement, motion_correct_tool, rels_displacement=None +): """ - Method to calculate all the movement parameters + Method to calculate all the movement parameters. Parameters ---------- @@ -525,74 +556,82 @@ def gen_motion_parameters(movement_parameters, max_displacement, # remove any other information other than matrix from # max displacement file. AFNI adds information to the file - if motion_correct_tool == '3dvolreg': + if motion_correct_tool == "3dvolreg": maxdisp = np.loadtxt(max_displacement) relsdisp = [] relsdisp = pd.DataFrame(relsdisp) - elif motion_correct_tool == 'mcflirt': + elif motion_correct_tool == "mcflirt": # TODO: mcflirt outputs absdisp, instead of maxdisp maxdisp = np.loadtxt(max_displacement) # rels_disp output only for mcflirt relsdisp = np.loadtxt(rels_displacement) - abs_relative = lambda v: np.abs(np.diff(v)) - max_relative = lambda v: np.max(abs_relative(v)) - avg_relative = lambda v: np.mean(abs_relative(v)) - max_abs = lambda v: np.max(np.abs(v)) - avg_abs = lambda v: np.mean(np.abs(v)) + def abs_relative(v): + return np.abs(np.diff(v)) + + def max_relative(v): + return np.max(abs_relative(v)) + + def avg_relative(v): + return np.mean(abs_relative(v)) + + def max_abs(v): + return np.max(np.abs(v)) + + def avg_abs(v): + return np.mean(np.abs(v)) info = [ - ('Mean_Relative_RMS_Displacement', avg_relative(rms)), - ('Max_Relative_RMS_Displacement', max_relative(rms)), - ('Movements_gt_threshold', np.sum(abs_relative(rms) > 0.1)), - ('Mean_Relative_Mean_Rotation', - avg_relative(np.abs(mot[0:3]).mean(axis=0))), - ('Mean_Relative_Maxdisp', avg_relative(maxdisp)), # to be updated - ('Max_Relative_Maxdisp', max_relative(maxdisp)), # to be updated - ('Max_Abs_Maxdisp', max_abs(maxdisp)), # to be updated - ('Max Relative_Roll', max_relative(mot[0])), - ('Max_Relative_Pitch', max_relative(mot[1])), - ('Max_Relative_Yaw', max_relative(mot[2])), - ('Max_Relative_dS-I', max_relative(mot[3])), - ('Max_Relative_dL-R', max_relative(mot[4])), - ('Max_Relative_dP-A', max_relative(mot[5])), - ('Mean_Relative_Roll', avg_relative(mot[0])), - ('Mean_Relative_Pitch', avg_relative(mot[1])), - ('Mean_Relative_Yaw', avg_relative(mot[2])), - ('Mean_Relative_dS-I', avg_relative(mot[3])), - ('Mean_Relative_dL-R', avg_relative(mot[4])), - ('Mean_Relative_dP-A', avg_relative(mot[5])), - ('Max_Abs_Roll', max_abs(mot[0])), - ('Max_Abs_Pitch', max_abs(mot[1])), - ('Max_Abs_Yaw', max_abs(mot[2])), - ('Max_Abs_dS-I', max_abs(mot[3])), - ('Max_Abs_dL-R', max_abs(mot[4])), - ('Max_Abs_dP-A', max_abs(mot[5])), - ('Mean_Abs_Roll', avg_abs(mot[0])), - ('Mean_Abs_Pitch', avg_abs(mot[1])), - ('Mean_Abs_Yaw', avg_abs(mot[2])), - ('Mean_Abs_dS-I', avg_abs(mot[3])), - ('Mean_Abs_dL-R', avg_abs(mot[4])), - ('Mean_Abs_dP-A', avg_abs(mot[5])), + ("Mean_Relative_RMS_Displacement", avg_relative(rms)), + ("Max_Relative_RMS_Displacement", max_relative(rms)), + ("Movements_gt_threshold", np.sum(abs_relative(rms) > 0.1)), + ("Mean_Relative_Mean_Rotation", avg_relative(np.abs(mot[0:3]).mean(axis=0))), + ("Mean_Relative_Maxdisp", avg_relative(maxdisp)), # to be updated + ("Max_Relative_Maxdisp", max_relative(maxdisp)), # to be updated + ("Max_Abs_Maxdisp", max_abs(maxdisp)), # to be updated + ("Max Relative_Roll", max_relative(mot[0])), + ("Max_Relative_Pitch", max_relative(mot[1])), + ("Max_Relative_Yaw", max_relative(mot[2])), + ("Max_Relative_dS-I", max_relative(mot[3])), + ("Max_Relative_dL-R", max_relative(mot[4])), + ("Max_Relative_dP-A", max_relative(mot[5])), + ("Mean_Relative_Roll", avg_relative(mot[0])), + ("Mean_Relative_Pitch", avg_relative(mot[1])), + ("Mean_Relative_Yaw", avg_relative(mot[2])), + ("Mean_Relative_dS-I", avg_relative(mot[3])), + ("Mean_Relative_dL-R", avg_relative(mot[4])), + ("Mean_Relative_dP-A", avg_relative(mot[5])), + ("Max_Abs_Roll", max_abs(mot[0])), + ("Max_Abs_Pitch", max_abs(mot[1])), + ("Max_Abs_Yaw", max_abs(mot[2])), + ("Max_Abs_dS-I", max_abs(mot[3])), + ("Max_Abs_dL-R", max_abs(mot[4])), + ("Max_Abs_dP-A", max_abs(mot[5])), + ("Mean_Abs_Roll", avg_abs(mot[0])), + ("Mean_Abs_Pitch", avg_abs(mot[1])), + ("Mean_Abs_Yaw", avg_abs(mot[2])), + ("Mean_Abs_dS-I", avg_abs(mot[3])), + ("Mean_Abs_dL-R", avg_abs(mot[4])), + ("Mean_Abs_dP-A", avg_abs(mot[5])), ] - out_file = os.path.join(os.getcwd(), 'motion_parameters.txt') - with open(out_file, 'w') as f: - f.write(','.join(t for t, v in info)) - f.write('\n') - f.write(','.join( - v if type(v) == str else '{0:.6f}'.format(v) for t, v in info)) - f.write('\n') + out_file = os.path.join(os.getcwd(), "motion_parameters.txt") + with open(out_file, "w") as f: + f.write(",".join(t for t, v in info)) + f.write("\n") + f.write(",".join(v if type(v) == str else "{0:.6f}".format(v) for t, v in info)) + f.write("\n") return out_file, info, maxdisp, relsdisp -def gen_power_parameters(fdp=None, fdj=None, dvars=None, - motion_correct_tool='3dvolreg'): +def gen_power_parameters( + fdp=None, fdj=None, dvars=None, motion_correct_tool="3dvolreg" +): """ - Method to generate Power parameters for scrubbing + Method to generate Power parameters for scrubbing. Parameters ---------- @@ -629,7 +668,7 @@ def gen_power_parameters(fdp=None, fdj=None, dvars=None, # Mean DVARS meanDVARS = np.mean(dvars_data) - if motion_correct_tool == '3dvolreg': + if motion_correct_tool == "3dvolreg": if fdj: fdj_data = np.loadtxt(fdj) @@ -645,24 +684,22 @@ def gen_power_parameters(fdp=None, fdj=None, dvars=None, FDJquartile = np.mean(np.sort(fdj_data)[::-1][:quat]) info = [ - ('MeanFD_Power', meanFD_Power), - ('MeanFD_Jenkinson', meanFD_Jenkinson), - ('rootMeanSquareFD', rmsFDJ), - ('FDquartile(top1/4thFD)', FDJquartile), - ('MeanDVARS', meanDVARS)] - - elif motion_correct_tool == 'mcflirt': - info = [ - ('MeanFD_Power', meanFD_Power), - ('MeanDVARS', meanDVARS)] - - out_file = os.path.join(os.getcwd(), 'pow_params.txt') - with open(out_file, 'a') as f: - f.write(','.join(t for t, v in info)) - f.write('\n') - f.write(','.join( - v if type(v) == str else '{0:.4f}'.format(v) for t, v in info)) - f.write('\n') + ("MeanFD_Power", meanFD_Power), + ("MeanFD_Jenkinson", meanFD_Jenkinson), + ("rootMeanSquareFD", rmsFDJ), + ("FDquartile(top1/4thFD)", FDJquartile), + ("MeanDVARS", meanDVARS), + ] + + elif motion_correct_tool == "mcflirt": + info = [("MeanFD_Power", meanFD_Power), ("MeanDVARS", meanDVARS)] + + out_file = os.path.join(os.getcwd(), "pow_params.txt") + with open(out_file, "a") as f: + f.write(",".join(t for t, v in info)) + f.write("\n") + f.write(",".join(v if type(v) == str else "{0:.4f}".format(v) for t, v in info)) + f.write("\n") return out_file, info @@ -671,52 +708,64 @@ def DVARS_strip_t0(file_1D): x = np.loadtxt(file_1D) x = x[1:] x = np.insert(x, 0, 0) - np.savetxt('dvars_strip.1D', x) - return os.path.abspath('dvars_strip.1D'), x + np.savetxt("dvars_strip.1D", x) + return os.path.abspath("dvars_strip.1D"), x class ImageTo1DInputSpec(AFNICommandInputSpec): - in_file = File(desc='input file to 3dTto1D', - argstr='-input %s', - position=1, - mandatory=True, - exists=True, - copyfile=False) + in_file = File( + desc="input file to 3dTto1D", + argstr="-input %s", + position=1, + mandatory=True, + exists=True, + copyfile=False, + ) - mask = File(desc='-mask dset = use dset as mask to include/exclude voxels', - argstr='-mask %s', - position=2, - exists=True) + mask = File( + desc="-mask dset = use dset as mask to include/exclude voxels", + argstr="-mask %s", + position=2, + exists=True, + ) - out_file = File(name_template="%s_3DtoT1.1D", desc='output 1D file name', - argstr='-prefix %s', name_source="in_file", keep_extension=True) + out_file = File( + name_template="%s_3DtoT1.1D", + desc="output 1D file name", + argstr="-prefix %s", + name_source="in_file", + keep_extension=True, + ) _methods = [ - 'enorm', 'dvars', - 'rms', 'srms', 's_srms', - 'mdiff', 'smdiff', - '4095_count', '4095_frac', '4095_warn', + "enorm", + "dvars", + "rms", + "srms", + "s_srms", + "mdiff", + "smdiff", + "4095_count", + "4095_frac", + "4095_warn", ] - method = traits.Enum( - *_methods, - argstr='-method %s' - ) + method = traits.Enum(*_methods, argstr="-method %s") class ImageTo1DOutputSpec(TraitedSpec): - out_file = File(desc='output 1D file name') + out_file = File(desc="output 1D file name") class ImageTo1D(AFNICommand): - _cmd = '3dTto1D' + _cmd = "3dTto1D" input_spec = ImageTo1DInputSpec output_spec = ImageTo1DOutputSpec def calculate_DVARS(func_brain, mask): """ - Method to calculate DVARS as per power's method + Method to calculate DVARS as per power's method. Parameters ---------- @@ -733,9 +782,10 @@ def calculate_DVARS(func_brain, mask): file containing array of DVARS calculation for each voxel """ import numpy as np - import nibabel as nb - rest_data = nb.load(func_brain).get_fdata().astype(np.float32) - mask_data = nb.load(mask).get_fdata().astype('bool') + import nibabel as nib + + rest_data = nib.load(func_brain).get_fdata().astype(np.float32) + mask_data = nib.load(mask).get_fdata().astype("bool") # square of relative intensity value for each voxel across every timepoint data = np.square(np.diff(rest_data, axis=3)) @@ -746,7 +796,7 @@ def calculate_DVARS(func_brain, mask): # square root and mean across all timepoints inside mask dvars = np.sqrt(np.mean(data, axis=0)) - out_file = os.path.join(os.getcwd(), 'DVARS.txt') + out_file = os.path.join(os.getcwd(), "DVARS.txt") np.savetxt(out_file, dvars) dvars = np.insert(dvars, 0, 0) @@ -756,7 +806,7 @@ def calculate_DVARS(func_brain, mask): def get_allmotion(fdj, fdp, maxdisp, motion, power, relsdisp=None, dvars=None): """ - Method to append all the motion and power parameters into 2 files + Method to append all the motion and power parameters into 2 files. Parameters ---------- @@ -783,32 +833,41 @@ def get_allmotion(fdj, fdp, maxdisp, motion, power, relsdisp=None, dvars=None): summary_motion_power : str path to file containing all motion parameters appended """ - all_motion_val = os.path.join(os.getcwd(), 'motion.tsv') - summary_motion_power = os.path.join(os.getcwd(), 'desc-summary_motion.tsv') + all_motion_val = os.path.join(os.getcwd(), "motion.tsv") + summary_motion_power = os.path.join(os.getcwd(), "desc-summary_motion.tsv") df_fdj = pd.DataFrame(fdj) - df_fdj.columns = ['Framewise displacement Jenkinson'] + df_fdj.columns = ["Framewise displacement Jenkinson"] df_fdp = pd.DataFrame(fdp) - df_fdp.columns = ['Framewise displacement Power'] + df_fdp.columns = ["Framewise displacement Power"] df_dvars = pd.DataFrame(dvars) - df_dvars.columns = ['DVARS'] + df_dvars.columns = ["DVARS"] df_maxdisp = pd.DataFrame(maxdisp) - df_maxdisp.columns = ['Max Displacement'] + df_maxdisp.columns = ["Max Displacement"] df_relsdisp = pd.DataFrame(relsdisp) - df_maxdisp.columns = ['Rels Displacement'] + df_maxdisp.columns = ["Rels Displacement"] data_frames = [df_fdj, df_fdp, df_dvars, df_maxdisp, df_relsdisp] all_motion_val_df = pd.concat(data_frames, axis=1) if len(all_motion_val_df.columns) == 5: - np.savetxt(all_motion_val, all_motion_val_df, delimiter="\t", - header="Framewise displacement Jenkinson\tFramewise " - "displacement power\tDVARS\tMax Displacement" - "\tRels Displacement", comments='') + np.savetxt( + all_motion_val, + all_motion_val_df, + delimiter="\t", + header="Framewise displacement Jenkinson\tFramewise " + "displacement power\tDVARS\tMax Displacement" + "\tRels Displacement", + comments="", + ) if len(all_motion_val_df.columns) == 4: - np.savetxt(all_motion_val, all_motion_val_df, delimiter="\t", - header="Framewise displacement Jenkinson\tFramewise " - "displacement power\tDVARS\tMax Displacement", - comments='') + np.savetxt( + all_motion_val, + all_motion_val_df, + delimiter="\t", + header="Framewise displacement Jenkinson\tFramewise " + "displacement power\tDVARS\tMax Displacement", + comments="", + ) df_motion = pd.DataFrame(motion) df_power = pd.DataFrame(power) @@ -816,7 +875,8 @@ def get_allmotion(fdj, fdp, maxdisp, motion, power, relsdisp=None, dvars=None): summary_motion_pow_df = pd.concat(data_frames_motionpower).T summary_motion_pow_df.columns = summary_motion_pow_df.iloc[0] summary_motion_pow_df.drop(summary_motion_pow_df.index[0], inplace=True) - summary_motion_pow_df.to_csv(summary_motion_power, sep='\t', header=True, - index=False) + summary_motion_pow_df.to_csv( + summary_motion_power, sep="\t", header=True, index=False + ) return all_motion_val, summary_motion_power diff --git a/CPAC/generate_motion_statistics/test/test_dvars.py b/CPAC/generate_motion_statistics/test/test_dvars.py index e7b0b0219d..9c26ca20e9 100644 --- a/CPAC/generate_motion_statistics/test/test_dvars.py +++ b/CPAC/generate_motion_statistics/test/test_dvars.py @@ -1,31 +1,32 @@ import os -import nibabel as nb + import numpy as np +import nibabel as nib + from CPAC.generate_motion_statistics import ImageTo1D, calculate_DVARS np.random.seed(10) def test_dvars(): - - os.chdir('/tmp') + os.chdir("/tmp") img_data = np.random.uniform(-3000, 3000, (30, 35, 30, 120)) - img = nb.Nifti1Image(img_data, np.eye(4)) - nb.save(img, 'dvars_data.nii.gz') + img = nib.Nifti1Image(img_data, np.eye(4)) + nib.save(img, "dvars_data.nii.gz") mask_data = np.ones((30, 35, 30)) - mask = nb.Nifti1Image(mask_data, np.eye(4)) - nb.save(mask, 'dvars_mask.nii.gz') + mask = nib.Nifti1Image(mask_data, np.eye(4)) + nib.save(mask, "dvars_mask.nii.gz") - node = ImageTo1D(method='dvars') - node.inputs.in_file = 'dvars_data.nii.gz' - node.inputs.mask = 'dvars_mask.nii.gz' + node = ImageTo1D(method="dvars") + node.inputs.in_file = "dvars_data.nii.gz" + node.inputs.mask = "dvars_mask.nii.gz" node.run() - calculate_DVARS('dvars_data.nii.gz', 'dvars_mask.nii.gz') + calculate_DVARS("dvars_data.nii.gz", "dvars_mask.nii.gz") - afni_result = np.loadtxt('dvars_data_3DtoT1.1D')[1:] - python_result = np.loadtxt('DVARS.txt') + afni_result = np.loadtxt("dvars_data_3DtoT1.1D")[1:] + python_result = np.loadtxt("DVARS.txt") assert all(np.isclose(afni_result, python_result, 1e-4)) diff --git a/CPAC/generate_motion_statistics/utils.py b/CPAC/generate_motion_statistics/utils.py index ae74d81984..aa3a4f0c9f 100644 --- a/CPAC/generate_motion_statistics/utils.py +++ b/CPAC/generate_motion_statistics/utils.py @@ -14,12 +14,15 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -"""Utilities for motion parameters""" +"""Utilities for motion parameters.""" +from typing import Optional + import numpy as np -def affine_file_from_params_file(params_file: str, affine_file: str = None - ) -> str: +def affine_file_from_params_file( + params_file: str, affine_file: Optional[str] = None +) -> str: """Convert a 6-DOF motion parameters array into a 4x4 affine matrix. Parameters @@ -39,30 +42,40 @@ def affine_file_from_params_file(params_file: str, affine_file: str = None one timepoint per line) """ import os + import numpy as np + from CPAC.generate_motion_statistics.utils import affine_from_params ## load parameters file into array affine = affine_from_params(np.genfromtxt(params_file)) - header = '' + header = "" if affine_file: # grab comment(s), if any - with open(affine_file, 'r', encoding='utf-8') as _f: - header = '\n'.join([line for line in _f.readlines() - if line.lstrip().startswith('#')]) - basename = (os.path.basename(affine_file) if affine_file - else f'affine_{os.path.basename(params_file)}') - affine_file = f'{os.getcwd()}/filtered_{basename}' + with open(affine_file, "r", encoding="utf-8") as _f: + header = "\n".join( + [line for line in _f.readlines() if line.lstrip().startswith("#")] + ) + basename = ( + os.path.basename(affine_file) + if affine_file + else f"affine_{os.path.basename(params_file)}" + ) + affine_file = f"{os.getcwd()}/filtered_{basename}" # drop bottom [0, 0, 0, 1] row from each matrix # insert original comments, if any, as header - np.savetxt(affine_file, affine[:, :3].reshape(affine.shape[0], 12), - header=header, comments='') + np.savetxt( + affine_file, + affine[:, :3].reshape(affine.shape[0], 12), + header=header, + comments="", + ) return affine_file def affine_from_params(params: np.ndarray) -> np.ndarray: - """Convert a 6-DOF motion parameters array into a 4x4 affine matrix + """Convert a 6-DOF motion parameters array into a 4x4 affine matrix. Parameters ---------- @@ -87,9 +100,10 @@ def affine_from_params(params: np.ndarray) -> np.ndarray: out = [] for i in range(params.shape[0]): - affine = _get_affine_matrix(params=params[i], source='AFNI') - affine[:3, :3] = Rotation.from_euler("ZXY", -params[i, :3], - degrees=True).as_matrix() + affine = _get_affine_matrix(params=params[i], source="AFNI") + affine[:3, :3] = Rotation.from_euler( + "ZXY", -params[i, :3], degrees=True + ).as_matrix() out.append(affine) return np.array(out) @@ -97,7 +111,7 @@ def affine_from_params(params: np.ndarray) -> np.ndarray: def load_mats(mat_dir: str) -> np.ndarray: """ Given a directory of affince matrices as output by MCFLIRT, - return an array of these matrices + return an array of these matrices. Parameters ---------- @@ -110,11 +124,12 @@ def load_mats(mat_dir: str) -> np.ndarray: t x 4 x 4 affine matrix """ from pathlib import Path + import numpy as np + mats = [] - mat_paths = sorted(list(Path(mat_dir).glob("MAT_*"))) + mat_paths = sorted(Path(mat_dir).glob("MAT_*")) for path in mat_paths: mat = np.loadtxt(path) mats.append(mat) - mats = np.stack(mats) - return mats + return np.stack(mats) diff --git a/CPAC/group_analysis/__init__.py b/CPAC/group_analysis/__init__.py index 7e3ee5815f..168e1cbbae 100644 --- a/CPAC/group_analysis/__init__.py +++ b/CPAC/group_analysis/__init__.py @@ -1,6 +1,3 @@ -from .group_analysis import create_fsl_flame_wf, \ - get_operation +from .group_analysis import create_fsl_flame_wf, get_operation - -__all__ = ['create_fsl_flame_wf', \ - 'get_operation'] +__all__ = ["create_fsl_flame_wf", "get_operation"] diff --git a/CPAC/group_analysis/group_analysis.py b/CPAC/group_analysis/group_analysis.py index db04bca928..867e4ec4bc 100644 --- a/CPAC/group_analysis/group_analysis.py +++ b/CPAC/group_analysis/group_analysis.py @@ -1,13 +1,14 @@ -from CPAC.pipeline import nipype_pipeline_engine as pe -import nipype.interfaces.fsl as fsl +from nipype.interfaces import fsl import nipype.interfaces.utility as util + from CPAC.easy_thresh import easy_thresh +from CPAC.pipeline import nipype_pipeline_engine as pe def get_operation(in_file): """ Method to create operation string - for fslmaths + for fslmaths. Parameters ---------- @@ -27,19 +28,19 @@ def get_operation(in_file): """ try: from nibabel import load + img = load(in_file) hdr = img.header n_vol = int(hdr.get_data_shape()[3]) - op_string = '-abs -bin -Tmean -mul %d' % (n_vol) - return op_string + return "-abs -bin -Tmean -mul %d" % (n_vol) except: raise IOError("Unable to load the input nifti image") def label_zstat_files(zstat_list, con_file): """Take in the z-stat file outputs of FSL FLAME and rename them after the - contrast labels of the contrasts provided.""" - + contrast labels of the contrasts provided. + """ cons = [] new_zstat_list = [] @@ -61,142 +62,142 @@ def label_zstat_files(zstat_list, con_file): return new_zstat_list -def create_fsl_flame_wf(ftest=False, wf_name='groupAnalysis'): +def create_fsl_flame_wf(ftest=False, wf_name="groupAnalysis"): """ FSL `FEAT `_ - BASED Group Analysis + BASED Group Analysis. Parameters ---------- ftest : boolean, optional(default=False) Ftest help investigate several contrasts at the same time - for example to see whether any of them (or any combination of them) is - significantly non-zero. Also, the F-test allows you to compare the - contribution of each contrast to the model and decide on significant + for example to see whether any of them (or any combination of them) is + significantly non-zero. Also, the F-test allows you to compare the + contribution of each contrast to the model and decide on significant and non-significant ones - - wf_name : string + + wf_name : string Workflow name - - Returns + + Returns ------- grp_analysis : workflow object Group Analysis workflow object - + Notes ----- `Source `_ - + Workflow Inputs:: - + inputspec.mat_file : string (existing file) - Mat file containing matrix for design - + Mat file containing matrix for design + inputspec.con_file : string (existing file) - Contrast file containing contrast vectors - + Contrast file containing contrast vectors + inputspec.grp_file : string (existing file) file containing matrix specifying the groups the covariance is split into - + inputspec.zmap_files : string (existing nifti file) derivative or the zmap file for which the group analysis is to be run - + inputspec.z_threshold : float - Z Statistic threshold value for cluster thresholding. It is used to - determine what level of activation would be statistically significant. + Z Statistic threshold value for cluster thresholding. It is used to + determine what level of activation would be statistically significant. Increasing this will result in higher estimates of required effect. - + inputspec.p_threshold : float Probability threshold for cluster thresholding. - + inputspec.fts_file : string (existing file) file containing matrix specifying f-contrasts - + inputspec.paramerters : string (tuple) tuple containing which MNI and FSLDIR path information - + Workflow Outputs:: - + outputspec.merged : string (nifti file) - 4D volume file after merging all the derivative + 4D volume file after merging all the derivative files from each specified subject. - + outputspec.zstats : list (nifti files) Z statistic image for each t contrast - + outputspec.zfstats : list (nifti files) Z statistic image for each f contrast - + outputspec.fstats : list (nifti files) - F statistic for each contrast - + F statistic for each contrast + outputspec.cluster_threshold : list (nifti files) the thresholded Z statistic image for each t contrast - + outputspec.cluster_index : list (nifti files) - image of clusters for each t contrast; the values - in the clusters are the index numbers as used + image of clusters for each t contrast; the values + in the clusters are the index numbers as used in the cluster list. - + outputspec.cluster_localmax_txt : list (text files) - local maxima text file for each t contrast, + local maxima text file for each t contrast, defines the coordinates of maximum value in the cluster - + outputspec.overlay_threshold : list (nifti files) 3D color rendered stats overlay image for t contrast - After reloading this image, use the Statistics Color + After reloading this image, use the Statistics Color Rendering GUI to reload the color look-up-table - + outputspec.overlay_rendered_image : list (nifti files) 2D color rendered stats overlay picture for each t contrast - + outputspec.cluster_threshold_zf : list (nifti files) the thresholded Z statistic image for each f contrast - + outputspec.cluster_index_zf : list (nifti files) - image of clusters for each f contrast; the values - in the clusters are the index numbers as used + image of clusters for each f contrast; the values + in the clusters are the index numbers as used in the cluster list. - + outputspec.cluster_localmax_txt_zf : list (text files) - local maxima text file for each f contrast, + local maxima text file for each f contrast, defines the coordinates of maximum value in the cluster - + outputspec.overlay_threshold_zf : list (nifti files) 3D color rendered stats overlay image for f contrast - After reloading this image, use the Statistics Color + After reloading this image, use the Statistics Color Rendering GUI to reload the color look-up-table - + outputspec.overlay_rendered_image_zf : list (nifti files) 2D color rendered stats overlay picture for each f contrast - + Order of commands: - Merge all the Z-map 3D images into 4D image file. For details see `fslmerge `_:: - - fslmerge -t sub01/sca/seed1/sca_Z_FWHM_merged.nii - sub02/sca/seed1/sca_Z_FWHM.nii.gz .... + + fslmerge -t sub01/sca/seed1/sca_Z_FWHM_merged.nii + sub02/sca/seed1/sca_Z_FWHM.nii.gz .... merge.nii.gz - - arguments + + arguments -t : concatenate images in time - + - Create mask specific for analysis. For details see `fslmaths `_:: - - fslmaths merged.nii.gz + + fslmaths merged.nii.gz -abs -Tmin -bin mean_mask.nii.gz - - arguments + + arguments -Tmin : min across time -abs : absolute value -bin : use (current image>0) to binarise - + - FSL FLAMEO to perform higher level analysis. For details see `flameo `_:: - - flameo --copefile = merged.nii.gz --covsplitfile = anova_with_meanFD.grp --designfile = anova_with_meanFD.mat - --fcontrastsfile = anova_with_meanFD.fts --ld=stats --maskfile = mean_mask.nii.gz --runmode=ols + + flameo --copefile = merged.nii.gz --covsplitfile = anova_with_meanFD.grp --designfile = anova_with_meanFD.mat + --fcontrastsfile = anova_with_meanFD.fts --ld=stats --maskfile = mean_mask.nii.gz --runmode=ols --tcontrastsfile = anova_with_meanFD.con - + arguments --copefile : cope regressor data file --designfile : design matrix file @@ -204,15 +205,15 @@ def create_fsl_flame_wf(ftest=False, wf_name='groupAnalysis'): --tcontrastsfile : file containing an ASCII matrix specifying the t contrasts --fcontrastsfile : file containing an ASCII matrix specifying the f contrasts --runmode : Interference to perform (mixed effects - OLS) - - - Run FSL Easy thresh - + + - Run FSL Easy thresh + Easy thresh is a simple script for carrying out cluster-based thresholding and colour activation overlaying:: - + easythresh [--mm] - + A seperate workflow called easythresh is called to run easythresh steps. - + .. exec:: from CPAC.group_analysis import create_fsl_flame_wf wf = create_fsl_flame_wf() @@ -222,64 +223,75 @@ def create_fsl_flame_wf(ftest=False, wf_name='groupAnalysis'): ) High Level Workflow Graph: - + .. image:: ../../images/generated/group_analysis.png :width: 800 - - + + Detailed Workflow Graph: - + .. image:: ../../images/generated/group_analysis_detailed.png :width: 800 Examples -------- - >>> from CPAC.group_analysis import create_fsl_flame_wf >>> preproc = create_fsl_flame_wf() >>> preproc.inputs.inputspec.mat_file = '../group_models/anova_with_meanFD/anova_with_meanFD.mat' # doctest: +SKIP >>> preproc.inputs.inputspec.con_file = '../group_models/anova_with_meanFD/anova_with_meanFD.con' # doctest: +SKIP >>> preproc.inputs.inputspec.grp_file = '../group_models/anova_with_meanFD/anova_with_meanFD.grp' # doctest: +SKIP >>> preproc.inputs.inputspec.zmap_files = [ - ... 'subjects/sub01/seeds_rest_Dickstein_DLPFC/sca_Z_FWHM.nii.gz', + ... 'subjects/sub01/seeds_rest_Dickstein_DLPFC/sca_Z_FWHM.nii.gz', ... 'subjects/sub02/seeds_rest_Dickstein_DLPFC/sca_Z_FWHM.nii.gz' ... ] # doctest: +SKIP >>> preproc.inputs.inputspec.z_threshold = 2.3 >>> preproc.inputs.inputspec.p_threshold = 0.05 >>> preproc.inputs.inputspec.parameters = ('/usr/local/fsl/', 'MNI152') >>> preproc.run() # doctest: +SKIP - + """ grp_analysis = pe.Workflow(name=wf_name) - inputnode = pe.Node(util.IdentityInterface(fields=['merged_file', - 'merge_mask', - 'mat_file', - 'con_file', - 'grp_file', - 'fts_file', - 'z_threshold', - 'p_threshold', - 'parameters']), - name='inputspec') - - outputnode = pe.Node(util.IdentityInterface(fields=['merged', - 'zstats', - 'zfstats', - 'fstats', - 'cluster_threshold', - 'cluster_index', - 'cluster_localmax_txt', - 'overlay_threshold', - 'rendered_image', - 'cluster_localmax_txt_zf', - 'cluster_threshold_zf', - 'cluster_index_zf', - 'overlay_threshold_zf', - 'rendered_image_zf']), - name='outputspec') - - ''' + inputnode = pe.Node( + util.IdentityInterface( + fields=[ + "merged_file", + "merge_mask", + "mat_file", + "con_file", + "grp_file", + "fts_file", + "z_threshold", + "p_threshold", + "parameters", + ] + ), + name="inputspec", + ) + + outputnode = pe.Node( + util.IdentityInterface( + fields=[ + "merged", + "zstats", + "zfstats", + "fstats", + "cluster_threshold", + "cluster_index", + "cluster_localmax_txt", + "overlay_threshold", + "rendered_image", + "cluster_localmax_txt_zf", + "cluster_threshold_zf", + "cluster_index_zf", + "overlay_threshold_zf", + "rendered_image_zf", + ] + ), + name="outputspec", + ) + + """ merge_to_4d = pe.Node(interface=fslMerge(), name='merge_to_4d') merge_to_4d.inputs.dimension = 't' @@ -291,26 +303,29 @@ def create_fsl_flame_wf(ftest=False, wf_name='groupAnalysis'): merge_mask = pe.Node(interface=fsl.ImageMaths(), name='merge_mask') merge_mask.inputs.op_string = '-abs -Tmin -bin' - ''' + """ - fsl_flameo = pe.Node(interface=fsl.FLAMEO(), - name='fsl_flameo') - fsl_flameo.inputs.run_mode = 'ols' + fsl_flameo = pe.Node(interface=fsl.FLAMEO(), name="fsl_flameo") + fsl_flameo.inputs.run_mode = "ols" # rename the FLAME zstat outputs after the contrast string labels for # easier interpretation label_zstat_imports = ["import os"] - label_zstat = pe.Node(util.Function(input_names=['zstat_list', - 'con_file'], - output_names=['new_zstat_list'], - function=label_zstat_files, - imports=label_zstat_imports), - name='label_zstat') - - rename_zstats = pe.MapNode(interface=util.Rename(), - name='rename_zstats', - iterfield=['in_file', - 'format_string']) + label_zstat = pe.Node( + util.Function( + input_names=["zstat_list", "con_file"], + output_names=["new_zstat_list"], + function=label_zstat_files, + imports=label_zstat_imports, + ), + name="label_zstat", + ) + + rename_zstats = pe.MapNode( + interface=util.Rename(), + name="rename_zstats", + iterfield=["in_file", "format_string"], + ) rename_zstats.inputs.keep_ext = True # create analysis specific mask @@ -321,123 +336,146 @@ def create_fsl_flame_wf(ftest=False, wf_name='groupAnalysis'): # in our analysis overlay with each other and the MNI brain. # e.g., maybe there is one subject with limited coverage. # not attached to sink currently - merge_mean_mask = pe.Node(interface=fsl.ImageMaths(), - name='merge_mean_mask') + merge_mean_mask = pe.Node(interface=fsl.ImageMaths(), name="merge_mean_mask") # function node to get the operation string for fslmaths command - get_opstring = pe.Node(util.Function(input_names=['in_file'], - output_names=['out_file'], - function=get_operation), - name='get_opstring') + get_opstring = pe.Node( + util.Function( + input_names=["in_file"], output_names=["out_file"], function=get_operation + ), + name="get_opstring", + ) # connections - ''' + """ grp_analysis.connect(inputnode, 'zmap_files', merge_to_4d, 'in_files') grp_analysis.connect(merge_to_4d, 'merged_file', merge_mask, 'in_file') - ''' - grp_analysis.connect(inputnode, 'merged_file', - fsl_flameo, 'cope_file') - grp_analysis.connect(inputnode, 'merge_mask', - fsl_flameo, 'mask_file') - grp_analysis.connect(inputnode, 'mat_file', - fsl_flameo, 'design_file') - grp_analysis.connect(inputnode, 'con_file', - fsl_flameo, 't_con_file') - grp_analysis.connect(inputnode, 'grp_file', - fsl_flameo, 'cov_split_file') - - grp_analysis.connect(fsl_flameo, 'zstats', label_zstat, 'zstat_list') - grp_analysis.connect(inputnode, 'con_file', label_zstat, 'con_file') - - grp_analysis.connect(fsl_flameo, 'zstats', rename_zstats, 'in_file') - - grp_analysis.connect(label_zstat, 'new_zstat_list', - rename_zstats, 'format_string') + """ + grp_analysis.connect(inputnode, "merged_file", fsl_flameo, "cope_file") + grp_analysis.connect(inputnode, "merge_mask", fsl_flameo, "mask_file") + grp_analysis.connect(inputnode, "mat_file", fsl_flameo, "design_file") + grp_analysis.connect(inputnode, "con_file", fsl_flameo, "t_con_file") + grp_analysis.connect(inputnode, "grp_file", fsl_flameo, "cov_split_file") + + grp_analysis.connect(fsl_flameo, "zstats", label_zstat, "zstat_list") + grp_analysis.connect(inputnode, "con_file", label_zstat, "con_file") + + grp_analysis.connect(fsl_flameo, "zstats", rename_zstats, "in_file") + + grp_analysis.connect(label_zstat, "new_zstat_list", rename_zstats, "format_string") if ftest: - grp_analysis.connect(inputnode, 'fts_file', fsl_flameo, 'f_con_file') - - easy_thresh_zf = easy_thresh('easy_thresh_zf') - - grp_analysis.connect(fsl_flameo, 'zfstats', - easy_thresh_zf, 'inputspec.z_stats') - grp_analysis.connect(inputnode, 'merge_mask', - easy_thresh_zf, 'inputspec.merge_mask') - grp_analysis.connect(inputnode, 'z_threshold', - easy_thresh_zf, 'inputspec.z_threshold') - grp_analysis.connect(inputnode, 'p_threshold', - easy_thresh_zf, 'inputspec.p_threshold') - grp_analysis.connect(inputnode, 'parameters', - easy_thresh_zf, 'inputspec.parameters') - grp_analysis.connect(easy_thresh_zf, 'outputspec.cluster_threshold', - outputnode, 'cluster_threshold_zf') - grp_analysis.connect(easy_thresh_zf, 'outputspec.cluster_index', - outputnode, 'cluster_index_zf') - grp_analysis.connect(easy_thresh_zf, 'outputspec.cluster_localmax_txt', - outputnode, 'cluster_localmax_txt_zf') - grp_analysis.connect(easy_thresh_zf, 'outputspec.overlay_threshold', - outputnode, 'overlay_threshold_zf') - grp_analysis.connect(easy_thresh_zf, 'outputspec.rendered_image', - outputnode, 'rendered_image_zf') + grp_analysis.connect(inputnode, "fts_file", fsl_flameo, "f_con_file") + + easy_thresh_zf = easy_thresh("easy_thresh_zf") + + grp_analysis.connect(fsl_flameo, "zfstats", easy_thresh_zf, "inputspec.z_stats") + grp_analysis.connect( + inputnode, "merge_mask", easy_thresh_zf, "inputspec.merge_mask" + ) + grp_analysis.connect( + inputnode, "z_threshold", easy_thresh_zf, "inputspec.z_threshold" + ) + grp_analysis.connect( + inputnode, "p_threshold", easy_thresh_zf, "inputspec.p_threshold" + ) + grp_analysis.connect( + inputnode, "parameters", easy_thresh_zf, "inputspec.parameters" + ) + grp_analysis.connect( + easy_thresh_zf, + "outputspec.cluster_threshold", + outputnode, + "cluster_threshold_zf", + ) + grp_analysis.connect( + easy_thresh_zf, "outputspec.cluster_index", outputnode, "cluster_index_zf" + ) + grp_analysis.connect( + easy_thresh_zf, + "outputspec.cluster_localmax_txt", + outputnode, + "cluster_localmax_txt_zf", + ) + grp_analysis.connect( + easy_thresh_zf, + "outputspec.overlay_threshold", + outputnode, + "overlay_threshold_zf", + ) + grp_analysis.connect( + easy_thresh_zf, "outputspec.rendered_image", outputnode, "rendered_image_zf" + ) # calling easythresh for zstats files - easy_thresh_z = easy_thresh('easy_thresh_z') - grp_analysis.connect(rename_zstats, 'out_file', - easy_thresh_z, 'inputspec.z_stats') - grp_analysis.connect(inputnode, 'merge_mask', - easy_thresh_z, 'inputspec.merge_mask') - grp_analysis.connect(inputnode, 'z_threshold', - easy_thresh_z, 'inputspec.z_threshold') - grp_analysis.connect(inputnode, 'p_threshold', - easy_thresh_z, 'inputspec.p_threshold') - grp_analysis.connect(inputnode, 'parameters', - easy_thresh_z, 'inputspec.parameters') - - grp_analysis.connect(inputnode, 'merged_file', - get_opstring, 'in_file') - grp_analysis.connect(inputnode, 'merged_file', - merge_mean_mask, 'in_file') - grp_analysis.connect(get_opstring, 'out_file', - merge_mean_mask, 'op_string') - - grp_analysis.connect(fsl_flameo, 'zfstats', - outputnode, 'zfstats') - grp_analysis.connect(fsl_flameo, 'fstats', - outputnode, 'fstats') - grp_analysis.connect(inputnode, 'merged_file', - outputnode, 'merged') - - grp_analysis.connect(rename_zstats, 'out_file', outputnode, 'zstats') - - grp_analysis.connect(easy_thresh_z, 'outputspec.cluster_threshold', - outputnode, 'cluster_threshold') - grp_analysis.connect(easy_thresh_z, 'outputspec.cluster_index', - outputnode, 'cluster_index') - grp_analysis.connect(easy_thresh_z, 'outputspec.cluster_localmax_txt', - outputnode, 'cluster_localmax_txt') - grp_analysis.connect(easy_thresh_z, 'outputspec.overlay_threshold', - outputnode, 'overlay_threshold') - grp_analysis.connect(easy_thresh_z, 'outputspec.rendered_image', - outputnode, 'rendered_image') + easy_thresh_z = easy_thresh("easy_thresh_z") + grp_analysis.connect(rename_zstats, "out_file", easy_thresh_z, "inputspec.z_stats") + grp_analysis.connect(inputnode, "merge_mask", easy_thresh_z, "inputspec.merge_mask") + grp_analysis.connect( + inputnode, "z_threshold", easy_thresh_z, "inputspec.z_threshold" + ) + grp_analysis.connect( + inputnode, "p_threshold", easy_thresh_z, "inputspec.p_threshold" + ) + grp_analysis.connect(inputnode, "parameters", easy_thresh_z, "inputspec.parameters") + + grp_analysis.connect(inputnode, "merged_file", get_opstring, "in_file") + grp_analysis.connect(inputnode, "merged_file", merge_mean_mask, "in_file") + grp_analysis.connect(get_opstring, "out_file", merge_mean_mask, "op_string") + + grp_analysis.connect(fsl_flameo, "zfstats", outputnode, "zfstats") + grp_analysis.connect(fsl_flameo, "fstats", outputnode, "fstats") + grp_analysis.connect(inputnode, "merged_file", outputnode, "merged") + + grp_analysis.connect(rename_zstats, "out_file", outputnode, "zstats") + + grp_analysis.connect( + easy_thresh_z, "outputspec.cluster_threshold", outputnode, "cluster_threshold" + ) + grp_analysis.connect( + easy_thresh_z, "outputspec.cluster_index", outputnode, "cluster_index" + ) + grp_analysis.connect( + easy_thresh_z, + "outputspec.cluster_localmax_txt", + outputnode, + "cluster_localmax_txt", + ) + grp_analysis.connect( + easy_thresh_z, "outputspec.overlay_threshold", outputnode, "overlay_threshold" + ) + grp_analysis.connect( + easy_thresh_z, "outputspec.rendered_image", outputnode, "rendered_image" + ) return grp_analysis -def run_feat_pipeline(group_config, merge_file, merge_mask, f_test, - mat_file, con_file, grp_file, out_dir, work_dir, log_dir, - model_name, fts_file=None): - ''' +def run_feat_pipeline( + group_config, + merge_file, + merge_mask, + f_test, + mat_file, + con_file, + grp_file, + out_dir, + work_dir, + log_dir, + model_name, + fts_file=None, +): + """ needed: - z thresh, p thresh - out dir - work, crash, log etc. - - + - - ''' - + """ import nipype.interfaces.io as nio # get thresholds @@ -445,13 +483,12 @@ def run_feat_pipeline(group_config, merge_file, merge_mask, f_test, p_threshold = float(group_config.p_threshold[0]) # workflow time - wf_name = "fsl-feat_".format(model_name) + wf_name = "fsl-feat_".format() wf = pe.Workflow(name=wf_name) wf.base_dir = work_dir - wf.config['execution'] = {'hash_method': 'timestamp', - 'crashdump_dir': log_dir} + wf.config["execution"] = {"hash_method": "timestamp", "crashdump_dir": log_dir} gpa_wf = create_fsl_flame_wf(f_test, "fsl-flame") @@ -460,7 +497,7 @@ def run_feat_pipeline(group_config, merge_file, merge_mask, f_test, gpa_wf.inputs.inputspec.z_threshold = z_threshold gpa_wf.inputs.inputspec.p_threshold = p_threshold - gpa_wf.inputs.inputspec.parameters = (group_config.FSLDIR, 'MNI152') + gpa_wf.inputs.inputspec.parameters = (group_config.FSLDIR, "MNI152") gpa_wf.inputs.inputspec.mat_file = mat_file gpa_wf.inputs.inputspec.con_file = con_file @@ -469,38 +506,37 @@ def run_feat_pipeline(group_config, merge_file, merge_mask, f_test, if f_test: gpa_wf.inputs.inputspec.fts_file = fts_file - ds = pe.Node(nio.DataSink(), name='gpa_sink') + ds = pe.Node(nio.DataSink(), name="gpa_sink") ds.inputs.base_directory = str(out_dir) - ds.inputs.container = '' - - ds.inputs.regexp_substitutions = [(r'(?<=rendered)(.)*[/]', '/'), - (r'(?<=model_files)(.)*[/]', '/'), - (r'(?<=merged)(.)*[/]', '/'), - (r'(?<=stats/clusterMap)(.)*[/]', '/'), - (r'(?<=stats/unthreshold)(.)*[/]', '/'), - (r'(?<=stats/threshold)(.)*[/]', '/'), - (r'_cluster(.)*[/]', ''), - (r'_slicer(.)*[/]', ''), - (r'_overlay(.)*[/]', '')] - - wf.connect(gpa_wf, 'outputspec.merged', ds, 'merged') - wf.connect(gpa_wf, 'outputspec.zstats', ds, 'stats.unthreshold') - wf.connect(gpa_wf, 'outputspec.zfstats', ds, 'stats.unthreshold.@01') - wf.connect(gpa_wf, 'outputspec.fstats', ds, 'stats.unthreshold.@02') - wf.connect(gpa_wf, 'outputspec.cluster_threshold_zf', ds, 'stats.threshold') - wf.connect(gpa_wf, 'outputspec.cluster_index_zf', ds, 'stats.clusterMap') - wf.connect(gpa_wf, 'outputspec.cluster_localmax_txt_zf', - ds, 'stats.clusterMap.@01') - wf.connect(gpa_wf, 'outputspec.overlay_threshold_zf', ds, 'rendered') - wf.connect(gpa_wf, 'outputspec.rendered_image_zf', ds, 'rendered.@01') - wf.connect(gpa_wf, 'outputspec.cluster_threshold', - ds, 'stats.threshold.@01') - wf.connect(gpa_wf, 'outputspec.cluster_index', ds, 'stats.clusterMap.@02') - wf.connect(gpa_wf, 'outputspec.cluster_localmax_txt', - ds, 'stats.clusterMap.@03') - wf.connect(gpa_wf, 'outputspec.overlay_threshold', ds, 'rendered.@02') - wf.connect(gpa_wf, 'outputspec.rendered_image', ds, 'rendered.@03') + ds.inputs.container = "" + + ds.inputs.regexp_substitutions = [ + (r"(?<=rendered)(.)*[/]", "/"), + (r"(?<=model_files)(.)*[/]", "/"), + (r"(?<=merged)(.)*[/]", "/"), + (r"(?<=stats/clusterMap)(.)*[/]", "/"), + (r"(?<=stats/unthreshold)(.)*[/]", "/"), + (r"(?<=stats/threshold)(.)*[/]", "/"), + (r"_cluster(.)*[/]", ""), + (r"_slicer(.)*[/]", ""), + (r"_overlay(.)*[/]", ""), + ] + + wf.connect(gpa_wf, "outputspec.merged", ds, "merged") + wf.connect(gpa_wf, "outputspec.zstats", ds, "stats.unthreshold") + wf.connect(gpa_wf, "outputspec.zfstats", ds, "stats.unthreshold.@01") + wf.connect(gpa_wf, "outputspec.fstats", ds, "stats.unthreshold.@02") + wf.connect(gpa_wf, "outputspec.cluster_threshold_zf", ds, "stats.threshold") + wf.connect(gpa_wf, "outputspec.cluster_index_zf", ds, "stats.clusterMap") + wf.connect(gpa_wf, "outputspec.cluster_localmax_txt_zf", ds, "stats.clusterMap.@01") + wf.connect(gpa_wf, "outputspec.overlay_threshold_zf", ds, "rendered") + wf.connect(gpa_wf, "outputspec.rendered_image_zf", ds, "rendered.@01") + wf.connect(gpa_wf, "outputspec.cluster_threshold", ds, "stats.threshold.@01") + wf.connect(gpa_wf, "outputspec.cluster_index", ds, "stats.clusterMap.@02") + wf.connect(gpa_wf, "outputspec.cluster_localmax_txt", ds, "stats.clusterMap.@03") + wf.connect(gpa_wf, "outputspec.overlay_threshold", ds, "rendered.@02") + wf.connect(gpa_wf, "outputspec.rendered_image", ds, "rendered.@03") # Run the actual group analysis workflow wf.run() diff --git a/CPAC/image_utils/__init__.py b/CPAC/image_utils/__init__.py index 6d79f91bd8..2296e7d6fa 100644 --- a/CPAC/image_utils/__init__.py +++ b/CPAC/image_utils/__init__.py @@ -15,8 +15,16 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . from .spatial_smoothing import set_gauss, spatial_smoothing -from .statistical_transforms import calc_avg, fisher_z_score_standardize, \ - z_score_standardize +from .statistical_transforms import ( + calc_avg, + fisher_z_score_standardize, + z_score_standardize, +) -__all__ = ['calc_avg', 'fisher_z_score_standardize', 'set_gauss', - 'spatial_smoothing', 'z_score_standardize'] +__all__ = [ + "calc_avg", + "fisher_z_score_standardize", + "set_gauss", + "spatial_smoothing", + "z_score_standardize", +] diff --git a/CPAC/image_utils/spatial_smoothing.py b/CPAC/image_utils/spatial_smoothing.py index f14dff8d87..facebaf923 100644 --- a/CPAC/image_utils/spatial_smoothing.py +++ b/CPAC/image_utils/spatial_smoothing.py @@ -16,94 +16,91 @@ # License along with C-PAC. If not, see . from nipype.interfaces import fsl, utility as util from nipype.interfaces.afni import preprocess as afni + from CPAC.pipeline import nipype_pipeline_engine as pe def set_gauss(fwhm): """ Compute the sigma value, given Full Width Half Max. - Returns an operand string + Returns an operand string. Parameters ---------- - fwhm : float Returns ------- - op_string : string """ - sigma = float(fwhm) / 2.3548 - op_string = "-kernel gauss %f -fmean -mas " % sigma + "%s" - - return op_string - + return "-kernel gauss %f -fmean -mas " % sigma + "%s" -def spatial_smoothing(wf_name, fwhm, input_image_type='func_derivative', - opt=None): +def spatial_smoothing(wf_name, fwhm, input_image_type="func_derivative", opt=None): wf = pe.Workflow(name=wf_name) - inputnode = pe.Node(util.IdentityInterface(fields=['in_file', - 'mask']), - name='inputspec') + inputnode = pe.Node( + util.IdentityInterface(fields=["in_file", "mask"]), name="inputspec" + ) - inputnode_fwhm = pe.Node(util.IdentityInterface(fields=['fwhm']), - name='fwhm_input') + inputnode_fwhm = pe.Node(util.IdentityInterface(fields=["fwhm"]), name="fwhm_input") inputnode_fwhm.iterables = ("fwhm", fwhm) - image_types = ['func_derivative', 'func_derivative_multi', 'func_4d', - 'func_mask'] + image_types = ["func_derivative", "func_derivative_multi", "func_4d", "func_mask"] if input_image_type not in image_types: - raise ValueError('Input image type {0} should be one of ' - '{1}'.format(input_image_type, - ', '.join(image_types))) + raise ValueError( + "Input image type {0} should be one of " "{1}".format( + input_image_type, ", ".join(image_types) + ) + ) - if opt == 'FSL': + if opt == "FSL": output_smooth_mem_gb = 4.0 - if input_image_type == 'func_derivative_multi': - output_smooth = pe.MapNode(interface=fsl.MultiImageMaths(), - name='smooth_multi', - iterfield=['in_file'], - mem_gb=output_smooth_mem_gb) + if input_image_type == "func_derivative_multi": + output_smooth = pe.MapNode( + interface=fsl.MultiImageMaths(), + name="smooth_multi", + iterfield=["in_file"], + mem_gb=output_smooth_mem_gb, + ) else: - output_smooth = pe.Node(interface=fsl.MultiImageMaths(), - name='smooth', - mem_gb=output_smooth_mem_gb) - - elif opt == 'AFNI': - if input_image_type == 'func_derivative_multi': - output_smooth = pe.MapNode(interface=afni.BlurToFWHM(), - name='smooth_multi', - iterfield=['in_file']) + output_smooth = pe.Node( + interface=fsl.MultiImageMaths(), + name="smooth", + mem_gb=output_smooth_mem_gb, + ) + + elif opt == "AFNI": + if input_image_type == "func_derivative_multi": + output_smooth = pe.MapNode( + interface=afni.BlurToFWHM(), name="smooth_multi", iterfield=["in_file"] + ) else: - output_smooth = pe.Node(interface=afni.BlurToFWHM(), - name='smooth', - iterfield=['in_file']) - output_smooth.inputs.outputtype = 'NIFTI_GZ' + output_smooth = pe.Node( + interface=afni.BlurToFWHM(), name="smooth", iterfield=["in_file"] + ) + output_smooth.inputs.outputtype = "NIFTI_GZ" - if opt == 'FSL': + if opt == "FSL": # wire in the resource to be smoothed - wf.connect(inputnode, 'in_file', output_smooth, 'in_file') + wf.connect(inputnode, "in_file", output_smooth, "in_file") # get the parameters for fwhm - wf.connect(inputnode_fwhm, ('fwhm', set_gauss), - output_smooth, 'op_string') - wf.connect(inputnode, 'mask', output_smooth, 'operand_files') - elif opt =='AFNI': - wf.connect(inputnode, 'in_file', output_smooth, 'in_file') - wf.connect(inputnode_fwhm, 'fwhm', output_smooth, 'fwhm') - wf.connect(inputnode, 'mask', output_smooth, 'mask') - - outputnode = pe.Node(util.IdentityInterface(fields=['out_file', - 'fwhm']), - name='outputspec') - - wf.connect(output_smooth, 'out_file', outputnode, 'out_file') - wf.connect(inputnode_fwhm, 'fwhm', outputnode, 'fwhm') + wf.connect(inputnode_fwhm, ("fwhm", set_gauss), output_smooth, "op_string") + wf.connect(inputnode, "mask", output_smooth, "operand_files") + elif opt == "AFNI": + wf.connect(inputnode, "in_file", output_smooth, "in_file") + wf.connect(inputnode_fwhm, "fwhm", output_smooth, "fwhm") + wf.connect(inputnode, "mask", output_smooth, "mask") + + outputnode = pe.Node( + util.IdentityInterface(fields=["out_file", "fwhm"]), name="outputspec" + ) + + wf.connect(output_smooth, "out_file", outputnode, "out_file") + wf.connect(inputnode_fwhm, "fwhm", outputnode, "fwhm") return wf diff --git a/CPAC/image_utils/statistical_transforms.py b/CPAC/image_utils/statistical_transforms.py index c3b989931c..83a01036d9 100644 --- a/CPAC/image_utils/statistical_transforms.py +++ b/CPAC/image_utils/statistical_transforms.py @@ -16,111 +16,111 @@ # License along with C-PAC. If not, see . from nipype.interfaces import utility as util from nipype.interfaces.afni import preprocess + from CPAC.pipeline import nipype_pipeline_engine as pe from CPAC.utils import function -from CPAC.utils.utils import ( - extract_output_mean, - get_zscore, - get_fisher_zscore -) - +from CPAC.utils.utils import extract_output_mean, get_fisher_zscore, get_zscore -def z_score_standardize(wf_name, input_image_type='func_derivative', - opt=None): +def z_score_standardize(wf_name, input_image_type="func_derivative", opt=None): wf = pe.Workflow(name=wf_name) map_node = False - if input_image_type == 'func_derivative_multi': + if input_image_type == "func_derivative_multi": map_node = True - inputnode = pe.Node(util.IdentityInterface(fields=['in_file', - 'mask']), - name='inputspec') + inputnode = pe.Node( + util.IdentityInterface(fields=["in_file", "mask"]), name="inputspec" + ) - z_score_std = get_zscore(map_node, 'z_score_std') + z_score_std = get_zscore(map_node, "z_score_std") - wf.connect(inputnode, 'in_file', z_score_std, 'inputspec.input_file') - wf.connect(inputnode, 'mask', z_score_std, 'inputspec.mask_file') + wf.connect(inputnode, "in_file", z_score_std, "inputspec.input_file") + wf.connect(inputnode, "mask", z_score_std, "inputspec.mask_file") - outputnode = pe.Node(util.IdentityInterface(fields=['out_file']), - name='outputspec') + outputnode = pe.Node(util.IdentityInterface(fields=["out_file"]), name="outputspec") - wf.connect(z_score_std, 'outputspec.z_score_img', outputnode, 'out_file') + wf.connect(z_score_std, "outputspec.z_score_img", outputnode, "out_file") return wf -def fisher_z_score_standardize(wf_name, label, - input_image_type='func_derivative', opt=None): - +def fisher_z_score_standardize( + wf_name, label, input_image_type="func_derivative", opt=None +): wf = pe.Workflow(name=wf_name) map_node = False - if input_image_type == 'func_derivative_multi': + if input_image_type == "func_derivative_multi": map_node = True - inputnode = pe.Node(util.IdentityInterface(fields=['correlation_file', - 'timeseries_oned']), - name='inputspec') + inputnode = pe.Node( + util.IdentityInterface(fields=["correlation_file", "timeseries_oned"]), + name="inputspec", + ) - fisher_z_score_std = get_fisher_zscore(label, map_node, - 'fisher_z_score_std') - wf.connect(inputnode, 'correlation_file', - fisher_z_score_std, 'inputspec.correlation_file') + fisher_z_score_std = get_fisher_zscore(label, map_node, "fisher_z_score_std") + wf.connect( + inputnode, "correlation_file", fisher_z_score_std, "inputspec.correlation_file" + ) - wf.connect(inputnode, 'timeseries_oned', - fisher_z_score_std, 'inputspec.timeseries_one_d') + wf.connect( + inputnode, "timeseries_oned", fisher_z_score_std, "inputspec.timeseries_one_d" + ) - outputnode = pe.Node(util.IdentityInterface(fields=['out_file']), - name='outputspec') + outputnode = pe.Node(util.IdentityInterface(fields=["out_file"]), name="outputspec") - wf.connect(fisher_z_score_std, 'outputspec.fisher_z_score_img', - outputnode, 'out_file') + wf.connect( + fisher_z_score_std, "outputspec.fisher_z_score_img", outputnode, "out_file" + ) return wf def calc_avg(workflow, output_name, strat, num_strat, map_node=False): """Calculate the average of an output using AFNI 3dmaskave.""" - if map_node: - calc_average = pe.MapNode(interface=preprocess.Maskave(), - name='{0}_mean_{1}'.format(output_name, - num_strat), - iterfield=['in_file']) - - mean_to_csv = pe.MapNode(function.Function(input_names=['in_file', - 'output_name'], - output_names=[ - 'output_mean'], - function=extract_output_mean, - as_module=True), - name='{0}_mean_to_txt_{1}'.format(output_name, - num_strat), - iterfield=['in_file']) + calc_average = pe.MapNode( + interface=preprocess.Maskave(), + name="{0}_mean_{1}".format(output_name, num_strat), + iterfield=["in_file"], + ) + + mean_to_csv = pe.MapNode( + function.Function( + input_names=["in_file", "output_name"], + output_names=["output_mean"], + function=extract_output_mean, + as_module=True, + ), + name="{0}_mean_to_txt_{1}".format(output_name, num_strat), + iterfield=["in_file"], + ) else: - calc_average = pe.Node(interface=preprocess.Maskave(), - name='{0}_mean_{1}'.format(output_name, - num_strat)) - - mean_to_csv = pe.Node(function.Function(input_names=['in_file', - 'output_name'], - output_names=['output_mean'], - function=extract_output_mean, - as_module=True), - name='{0}_mean_to_txt_{1}'.format(output_name, - num_strat)) + calc_average = pe.Node( + interface=preprocess.Maskave(), + name="{0}_mean_{1}".format(output_name, num_strat), + ) + + mean_to_csv = pe.Node( + function.Function( + input_names=["in_file", "output_name"], + output_names=["output_mean"], + function=extract_output_mean, + as_module=True, + ), + name="{0}_mean_to_txt_{1}".format(output_name, num_strat), + ) mean_to_csv.inputs.output_name = output_name node, out_file = strat[output_name] - workflow.connect(node, out_file, calc_average, 'in_file') - workflow.connect(calc_average, 'out_file', mean_to_csv, 'in_file') + workflow.connect(node, out_file, calc_average, "in_file") + workflow.connect(calc_average, "out_file", mean_to_csv, "in_file") strat.append_name(calc_average.name) - strat.update_resource_pool({ - 'output_means.@{0}_average'.format(output_name): (mean_to_csv, 'output_mean') - }) + strat.update_resource_pool( + {"output_means.@{0}_average".format(output_name): (mean_to_csv, "output_mean")} + ) return strat diff --git a/CPAC/image_utils/tests/test_smooth.py b/CPAC/image_utils/tests/test_smooth.py index 19db4f94f9..538f9cc8aa 100644 --- a/CPAC/image_utils/tests/test_smooth.py +++ b/CPAC/image_utils/tests/test_smooth.py @@ -1,104 +1,149 @@ import os + import pytest +from CPAC.image_utils import spatial_smoothing from CPAC.pipeline import nipype_pipeline_engine as pe -import nipype.interfaces.utility as util - +import CPAC.utils.test_init as test_utils from CPAC.utils.test_mocks import configuration_strategy_mock -from CPAC.image_utils import spatial_smoothing -import CPAC.utils.test_init as test_utils -@pytest.mark.skip(reason='needs refactoring') +@pytest.mark.skip(reason="needs refactoring") def test_smooth(): + test_name = "test_smooth_nodes" - test_name = 'test_smooth_nodes' - - c, strat = configuration_strategy_mock(method='FSL') + c, strat = configuration_strategy_mock(method="FSL") num_strat = 0 # build the workflow workflow = pe.Workflow(name=test_name) workflow.base_dir = c.workingDirectory - workflow.config['execution'] = { - 'hash_method': 'timestamp', - 'crashdump_dir': os.path.abspath(c.crashLogDirectory) + workflow.config["execution"] = { + "hash_method": "timestamp", + "crashdump_dir": os.path.abspath(c.crashLogDirectory), } - spatial_smoothing(workflow, 'mean_functional', 'functional_brain_mask', - 'mean_functional_smooth'.format(num_strat), strat, num_strat, c) - - func_node, func_output = strat['mean_functional'] - mask_node, mask_output = strat['functional_brain_mask'] + spatial_smoothing( + workflow, + "mean_functional", + "functional_brain_mask", + "mean_functional_smooth".format(), + strat, + num_strat, + c, + ) + + func_node, func_output = strat["mean_functional"] + mask_node, mask_output = strat["functional_brain_mask"] + + spatial_smoothing( + workflow, + (func_node, func_output), + (mask_node, mask_output), + "mean_functional_smooth_nodes".format(), + strat, + num_strat, + c, + ) - spatial_smoothing(workflow, (func_node, func_output), (mask_node, mask_output), - 'mean_functional_smooth_nodes'.format(num_strat), strat, num_strat, c) - - print(workflow.list_node_names()) workflow.run() correlations = [] for fwhm in c.fwhm: - - out_name1 = os.path.join(c.workingDirectory, test_name, - '_fwhm_{0}/mean_functional_smooth_0/'.format(fwhm), - 'sub-M10978008_ses-NFB3_task-test_bold_calc_tshift_resample_volreg_calc_tstat_maths.nii.gz') - - out_name2 = os.path.join(c.workingDirectory, test_name, - '_fwhm_{0}/mean_functional_smooth_nodes_0/'.format(fwhm), - 'sub-M10978008_ses-NFB3_task-test_bold_calc_tshift_resample_volreg_calc_tstat_maths.nii.gz') + out_name1 = os.path.join( + c.workingDirectory, + test_name, + "_fwhm_{0}/mean_functional_smooth_0/".format(fwhm), + "sub-M10978008_ses-NFB3_task-test_bold_calc_tshift_resample_volreg_calc_tstat_maths.nii.gz", + ) + + out_name2 = os.path.join( + c.workingDirectory, + test_name, + "_fwhm_{0}/mean_functional_smooth_nodes_0/".format(fwhm), + "sub-M10978008_ses-NFB3_task-test_bold_calc_tshift_resample_volreg_calc_tstat_maths.nii.gz", + ) correlations.append(test_utils.pearson_correlation(out_name1, out_name2) > 0.99) assert all(correlations) -@pytest.mark.skip(reason='needs refactoring') +@pytest.mark.skip(reason="needs refactoring") def test_smooth_mapnode(): + test_name = "test_smooth_mapnode" - test_name = 'test_smooth_mapnode' - - c, strat = configuration_strategy_mock(method='FSL') + c, strat = configuration_strategy_mock(method="FSL") num_strat = 0 # build the workflow workflow = pe.Workflow(name=test_name) workflow.base_dir = c.workingDirectory - workflow.config['execution'] = { - 'hash_method': 'timestamp', - 'crashdump_dir': os.path.abspath(c.crashLogDirectory) + workflow.config["execution"] = { + "hash_method": "timestamp", + "crashdump_dir": os.path.abspath(c.crashLogDirectory), } - spatial_smoothing(workflow, 'dr_tempreg_maps_files', 'functional_brain_mask', - 'dr_tempreg_maps_smooth'.format(num_strat), strat, num_strat, c, - input_image_type='func_derivative_multi') + spatial_smoothing( + workflow, + "dr_tempreg_maps_files", + "functional_brain_mask", + "dr_tempreg_maps_smooth".format(), + strat, + num_strat, + c, + input_image_type="func_derivative_multi", + ) + + func_node, func_output = strat["dr_tempreg_maps_files"] + mask_node, mask_output = strat["functional_brain_mask"] + + spatial_smoothing( + workflow, + (func_node, func_output), + (mask_node, mask_output), + "dr_tempreg_maps_smooth_nodes".format(), + strat, + num_strat, + c, + input_image_type="func_derivative_multi", + ) - func_node, func_output = strat['dr_tempreg_maps_files'] - mask_node, mask_output = strat['functional_brain_mask'] - - spatial_smoothing(workflow, (func_node, func_output), (mask_node, mask_output), - 'dr_tempreg_maps_smooth_nodes'.format(num_strat), strat, num_strat, c, - input_image_type='func_derivative_multi') - - print(workflow.list_node_names()) workflow.run() correlations = [] for fwhm in c.fwhm: - - dr_spatmaps_after_smooth1=[os.path.join(c.workingDirectory, test_name, - '_fwhm_{0}/dr_tempreg_maps_smooth_multi_0/mapflow'.format(fwhm), - '_dr_tempreg_maps_smooth_multi_0{0}/temp_reg_map_000{0}_maths.nii.gz'.format(n)) - for n in range(0,10)] - - dr_spatmaps_after_smooth2=[os.path.join(c.workingDirectory, test_name, - '_fwhm_{0}/dr_tempreg_maps_smooth_nodes_multi_0/mapflow'.format(fwhm), - '_dr_tempreg_maps_smooth_nodes_multi_0{0}/temp_reg_map_000{0}_maths.nii.gz'.format(n)) - for n in range(0,10)] - - correlations += [test_utils.pearson_correlation(file1, file2) > 0.99 \ - for file1, file2 in zip(dr_spatmaps_after_smooth1, dr_spatmaps_after_smooth2)] + dr_spatmaps_after_smooth1 = [ + os.path.join( + c.workingDirectory, + test_name, + "_fwhm_{0}/dr_tempreg_maps_smooth_multi_0/mapflow".format(fwhm), + "_dr_tempreg_maps_smooth_multi_0{0}/temp_reg_map_000{0}_maths.nii.gz".format( + n + ), + ) + for n in range(0, 10) + ] + + dr_spatmaps_after_smooth2 = [ + os.path.join( + c.workingDirectory, + test_name, + "_fwhm_{0}/dr_tempreg_maps_smooth_nodes_multi_0/mapflow".format(fwhm), + "_dr_tempreg_maps_smooth_nodes_multi_0{0}/temp_reg_map_000{0}_maths.nii.gz".format( + n + ), + ) + for n in range(0, 10) + ] + + correlations += [ + test_utils.pearson_correlation(file1, file2) > 0.99 + for file1, file2 in zip( + dr_spatmaps_after_smooth1, dr_spatmaps_after_smooth2 + ) + ] assert all(correlations) diff --git a/CPAC/info.py b/CPAC/info.py index 7002a07b14..ba5eace889 100644 --- a/CPAC/info.py +++ b/CPAC/info.py @@ -36,18 +36,19 @@ settings in setup.py, the CPAC top-level docstring, and for building the docs. In setup.py in particular, we exec this file, so it cannot import CPAC. This script was borrowed from and inspired by nipype's info.py file -(https://github.com/nipy/nipype/blob/08391871/nipype/info.py).""" +(https://github.com/nipy/nipype/blob/08391871/nipype/info.py). +""" # CPAC version information. An empty _version_extra corresponds to a # full release. 'dev' as a _version_extra string means this is a development # version _version_major = 1 _version_minor = 8 _version_micro = 7 -_version_extra = 'dev1' +_version_extra = "dev1" def get_cpac_gitversion(): - """CPAC version as reported by the last commit in git + """CPAC version as reported by the last commit in git. Returns ------- @@ -58,51 +59,50 @@ def get_cpac_gitversion(): import os import subprocess - gitpath = os.path.realpath(os.path.join(os.path.dirname(__file__), - os.path.pardir)) + gitpath = os.path.realpath(os.path.join(os.path.dirname(__file__), os.path.pardir)) - gitpathgit = os.path.join(gitpath, '.git') + gitpathgit = os.path.join(gitpath, ".git") if not os.path.exists(gitpathgit): return None ver = None try: - o, _ = subprocess.Popen('git describe --always', shell=True, - cwd=gitpath, stdout=subprocess.PIPE - ).communicate() + o, _ = subprocess.Popen( + "git describe --always", shell=True, cwd=gitpath, stdout=subprocess.PIPE + ).communicate() except Exception: pass else: - ver = o.decode().strip().split('-')[-1] + ver = o.decode().strip().split("-")[-1] return ver -if 'dev' in _version_extra: +if "dev" in _version_extra: gitversion = get_cpac_gitversion() if gitversion: - _version_extra = f'{_version_extra}+{gitversion}' + _version_extra = f"{_version_extra}+{gitversion}" -__version__ = "%s.%s.%s" % (_version_major, - _version_minor, - _version_micro) +__version__ = "%s.%s.%s" % (_version_major, _version_minor, _version_micro) if _version_extra: __version__ += ".%s" % _version_extra -ga_tracker = 'UA-19224662-10' +ga_tracker = "UA-19224662-10" -CLASSIFIERS = ["Development Status :: 4 - Beta", - "Environment :: Console", - "Intended Audience :: Science/Research", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Topic :: Scientific/Engineering"] +CLASSIFIERS = [ + "Development Status :: 4 - Beta", + "Environment :: Console", + "Intended Audience :: Science/Research", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Topic :: Scientific/Engineering", +] # pylint: disable=invalid-name -description = 'Configurable Pipeline for the Analysis of Connectomes' +description = "Configurable Pipeline for the Analysis of Connectomes" # Note: this long_description is actually a copy/paste from the top-level # README.md, so that it shows up nicely on PyPI. So please remember to edit @@ -149,9 +149,9 @@ def get_cpac_gitversion(): find a bug, have a question that is not answered in the User Guide, or would like to suggest a new feature, please create an issue on CPAC github issue page: https://github.com/FCP-INDI/C-PAC/issues?state=open -""" # noqa: E501 -CYTHON_MIN_VERSION = '0.12.1' -NAME = 'CPAC' +""" +CYTHON_MIN_VERSION = "0.12.1" +NAME = "CPAC" MAINTAINER = "C-PAC developers" MAINTAINER_EMAIL = "CNL@childmind.org" DESCRIPTION = description @@ -165,9 +165,9 @@ def get_cpac_gitversion(): MAJOR = _version_major MINOR = _version_minor MICRO = _version_micro -ISRELEASE = _version_extra == '' +ISRELEASE = _version_extra == "" VERSION = __version__ -STATUS = 'stable' +STATUS = "stable" REQUIREMENTS = [ "boto3", "ciftify", @@ -204,9 +204,6 @@ def get_cpac_gitversion(): "semver", "traits", "voluptuous>=0.12.0", - "xvfbwrapper" -] -UNET_REQUIREMENTS = [ - "torch==1.13.1", - "torchvision==0.14.1" + "xvfbwrapper", ] +UNET_REQUIREMENTS = ["torch==1.13.1", "torchvision==0.14.1"] diff --git a/CPAC/isc/__init__.py b/CPAC/isc/__init__.py index a5682fb093..faa18be5bb 100644 --- a/CPAC/isc/__init__.py +++ b/CPAC/isc/__init__.py @@ -1,2 +1,2 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- \ No newline at end of file +# -*- coding: utf-8 -*- diff --git a/CPAC/isc/isc.py b/CPAC/isc/isc.py index be43f17f34..058ef13f0a 100644 --- a/CPAC/isc/isc.py +++ b/CPAC/isc/isc.py @@ -1,12 +1,10 @@ import numpy as np from CPAC.utils import correlation - from .utils import p_from_null, phase_randomize def isc(D, std=None, collapse_subj=True): - assert D.ndim == 3 n_vox, _, n_subj = D.shape @@ -19,9 +17,7 @@ def isc(D, std=None, collapse_subj=True): for loo_subj in range(n_subj): loo_subj_ts = D[:, :, loo_subj] ISC += correlation( - loo_subj_ts, - (group_sum - loo_subj_ts) / n_subj_loo, - match_rows=True + loo_subj_ts, (group_sum - loo_subj_ts) / n_subj_loo, match_rows=True ) ISC /= n_subj @@ -37,28 +33,19 @@ def isc(D, std=None, collapse_subj=True): for loo_subj in range(n_subj): loo_subj_ts = D[:, :, loo_subj] ISC[loo_subj] = correlation( - loo_subj_ts, - (group_sum - loo_subj_ts) / n_subj_loo, - match_rows=True + loo_subj_ts, (group_sum - loo_subj_ts) / n_subj_loo, match_rows=True ) - + masked = np.array([True] * n_vox) return ISC, masked def isc_significance(ISC, min_null, max_null, two_sided=False): - p = p_from_null(ISC, - max_null=max_null, - min_null=min_null, - two_sided=two_sided) - return p + return p_from_null(ISC, max_null=max_null, min_null=min_null, two_sided=two_sided) def isc_permutation(permutation, D, masked, collapse_subj=True, random_state=0): - - print("Permutation", permutation) - min_null = 1 max_null = -1 @@ -75,12 +62,9 @@ def isc_permutation(permutation, D, masked, collapse_subj=True, random_state=0): for loo_subj in range(n_subj): loo_subj_ts = D[:, :, loo_subj] - ISC_subj = \ - correlation( - loo_subj_ts, - (group_sum - loo_subj_ts) / n_subj_loo, - match_rows=True - ) + ISC_subj = correlation( + loo_subj_ts, (group_sum - loo_subj_ts) / n_subj_loo, match_rows=True + ) if collapse_subj: ISC_null += ISC_subj diff --git a/CPAC/isc/isfc.py b/CPAC/isc/isfc.py index 29d2cdb1a0..21ba03cc23 100644 --- a/CPAC/isc/isfc.py +++ b/CPAC/isc/isfc.py @@ -1,11 +1,10 @@ import numpy as np -from CPAC.utils import correlation +from CPAC.utils import correlation from .utils import p_from_null, phase_randomize def isfc(D, std=None, collapse_subj=True): - assert D.ndim == 3 n_vox, _, n_subj = D.shape @@ -19,9 +18,7 @@ def isfc(D, std=None, collapse_subj=True): for loo_subj in range(n_subj): loo_subj_ts = D[:, :, loo_subj] ISFC += correlation( - loo_subj_ts, - (group_sum - loo_subj_ts) / n_subj_loo, - symmetric=True + loo_subj_ts, (group_sum - loo_subj_ts) / n_subj_loo, symmetric=True ) ISFC /= n_subj @@ -35,9 +32,7 @@ def isfc(D, std=None, collapse_subj=True): for loo_subj in range(n_subj): loo_subj_ts = D[:, :, loo_subj] ISFC[:, :, loo_subj] = correlation( - loo_subj_ts, - (group_sum - loo_subj_ts) / n_subj_loo, - symmetric=True + loo_subj_ts, (group_sum - loo_subj_ts) / n_subj_loo, symmetric=True ) if masked is not None: @@ -49,17 +44,10 @@ def isfc(D, std=None, collapse_subj=True): def isfc_significance(ISFC, min_null, max_null, two_sided=False): - p = p_from_null(ISFC, - max_null=max_null, - min_null=min_null, - two_sided=two_sided) - return p + return p_from_null(ISFC, max_null=max_null, min_null=min_null, two_sided=two_sided) def isfc_permutation(permutation, D, masked, collapse_subj=True, random_state=0): - - print("Permutation", permutation) - min_null = 1 max_null = -1 @@ -77,19 +65,16 @@ def isfc_permutation(permutation, D, masked, collapse_subj=True, random_state=0) for loo_subj in range(n_subj): loo_subj_ts = D[:, :, loo_subj] - ISFC_subj = \ - correlation( - loo_subj_ts, - (group_sum - loo_subj_ts) / n_subj_loo, - symmetric=True - ) + ISFC_subj = correlation( + loo_subj_ts, (group_sum - loo_subj_ts) / n_subj_loo, symmetric=True + ) if collapse_subj: ISFC_null += ISFC_subj else: max_null = max(np.max(ISFC_subj), max_null) min_null = min(np.min(ISFC_subj), min_null) - + if collapse_subj: ISFC_null /= n_subj max_null = np.max(ISFC_null) diff --git a/CPAC/isc/pipeline.py b/CPAC/isc/pipeline.py index ae120b1f89..56e7e537cd 100644 --- a/CPAC/isc/pipeline.py +++ b/CPAC/isc/pipeline.py @@ -1,33 +1,25 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from CPAC.pipeline import nipype_pipeline_engine as pe -import nipype.interfaces.utility as util -from nipype.interfaces.base import BaseInterface, \ - BaseInterfaceInputSpec, traits, File, TraitedSpec - -from nilearn import input_data, masking, image, datasets -from nilearn.image import resample_to_img, concat_imgs -from nilearn.input_data import NiftiMasker, NiftiLabelsMasker - -from CPAC.utils.interfaces.function import Function - import os -import copy + import numpy as np -import nibabel as nb +import nibabel as nib +from nilearn.input_data import NiftiMasker +import nipype.interfaces.utility as util from CPAC.isc.isc import ( isc, - isc_significance, isc_permutation, + isc_significance, ) - from CPAC.isc.isfc import ( isfc, - isfc_significance, isfc_permutation, + isfc_significance, ) +from CPAC.pipeline import nipype_pipeline_engine as pe +from CPAC.utils.interfaces.function import Function def _permutations(perm): @@ -36,58 +28,63 @@ def _permutations(perm): def load_data(subjects): subject_ids = list(subjects.keys()) - subject_files = list(subjects[i] for i in subject_ids) + subject_files = [subjects[i] for i in subject_ids] - if subject_files[0].endswith('.csv'): - data = np.array([ - np.genfromtxt(img).T - for img in subject_files - ]) + if subject_files[0].endswith(".csv"): + data = np.array([np.genfromtxt(img).T for img in subject_files]) voxel_masker = None else: - images = [nb.load(img) for img in subject_files] + images = [nib.load(img) for img in subject_files] voxel_masker = NiftiMasker() voxel_masker.fit(images) - data = np.array([ - voxel_masker.transform(img).T - for img in images - ]) + data = np.array([voxel_masker.transform(img).T for img in images]) # Reshape to voxel x time x subject - data = np.moveaxis(data, 0, -1).copy(order='C') + data = np.moveaxis(data, 0, -1).copy(order="C") - data_file = os.path.abspath('./data.npy') + data_file = os.path.abspath("./data.npy") np.save(data_file, data) return subject_ids, data_file, voxel_masker def save_data_isc(subject_ids, ISC, p, out_dir, collapse_subj=True, voxel_masker=None): - ISC = np.load(ISC) p = np.load(p) - subject_ids_file = os.path.abspath('./subject_ids.txt') + subject_ids_file = os.path.abspath("./subject_ids.txt") np.savetxt(subject_ids_file, np.array(subject_ids), fmt="%s") if voxel_masker: - corr_file, p_file = os.path.abspath('./correlations.nii.gz'), os.path.abspath('./significance.nii.gz') - corr_out, p_out = os.path.join(out_dir, 'correlations.nii.gz'), os.path.join(out_dir, 'significance.nii.gz') + corr_file, p_file = ( + os.path.abspath("./correlations.nii.gz"), + os.path.abspath("./significance.nii.gz"), + ) + corr_out, p_out = ( + os.path.join(out_dir, "correlations.nii.gz"), + os.path.join(out_dir, "significance.nii.gz"), + ) ISC_image = voxel_masker.inverse_transform(ISC.T) p_image = voxel_masker.inverse_transform(p.T) - nb.save(ISC_image, corr_file) - nb.save(p_image, p_file) + nib.save(ISC_image, corr_file) + nib.save(p_image, p_file) os.makedirs(out_dir) - nb.save(ISC_image, corr_out) - nb.save(p_image, p_out) + nib.save(ISC_image, corr_out) + nib.save(p_image, p_out) else: - corr_file, p_file = os.path.abspath('./correlations.csv'), os.path.abspath('./significance.csv') - corr_out, p_out = os.path.join(out_dir, 'correlations.csv'), os.path.join(out_dir, 'significance.csv') + corr_file, p_file = ( + os.path.abspath("./correlations.csv"), + os.path.abspath("./significance.csv"), + ) + corr_out, p_out = ( + os.path.join(out_dir, "correlations.csv"), + os.path.join(out_dir, "significance.csv"), + ) if ISC.ndim == 1: ISC = np.expand_dims(ISC, axis=0) @@ -95,32 +92,31 @@ def save_data_isc(subject_ids, ISC, p, out_dir, collapse_subj=True, voxel_masker if p.ndim == 1: p = np.expand_dims(p, axis=0) - np.savetxt(corr_file, ISC, delimiter=',', fmt='%1.10f') - np.savetxt(p_file, p, delimiter=',', fmt='%1.10f') + np.savetxt(corr_file, ISC, delimiter=",", fmt="%1.10f") + np.savetxt(p_file, p, delimiter=",", fmt="%1.10f") os.makedirs(out_dir) - np.savetxt(corr_out, ISC, delimiter=',', fmt='%1.10f') - np.savetxt(p_out, p, delimiter=',', fmt='%1.10f') + np.savetxt(corr_out, ISC, delimiter=",", fmt="%1.10f") + np.savetxt(p_out, p, delimiter=",", fmt="%1.10f") return subject_ids_file, corr_file, p_file def save_data_isfc(subject_ids, ISFC, p, out_dir, collapse_subj=True): - - subject_ids_file = os.path.abspath('./subject_ids.txt') + subject_ids_file = os.path.abspath("./subject_ids.txt") np.savetxt(subject_ids_file, np.array(subject_ids), fmt="%s") os.makedirs(out_dir) - corr_file = os.path.abspath('./correlations.npy') - corr_out = os.path.join(out_dir, 'correlations.npy') + corr_file = os.path.abspath("./correlations.npy") + corr_out = os.path.join(out_dir, "correlations.npy") corr = np.load(ISFC) np.save(corr_file, corr if collapse_subj else np.moveaxis(corr, -1, 0)) - np.save(corr_out, corr if collapse_subj else np.moveaxis(corr, -1, 0)) + np.save(corr_out, corr if collapse_subj else np.moveaxis(corr, -1, 0)) - p_file = os.path.abspath('./significance.npy') - p_out = os.path.join(out_dir, 'significance.npy') + p_file = os.path.abspath("./significance.npy") + p_out = os.path.join(out_dir, "significance.npy") p = np.load(p) np.save(p_file, p if collapse_subj else np.moveaxis(p, -1, 0)) @@ -133,20 +129,20 @@ def node_isc(D, std=None, collapse_subj=True): D = np.load(D) ISC, ISC_mask = isc(D, std, collapse_subj) - - f = os.path.abspath('./isc.npy') + + f = os.path.abspath("./isc.npy") np.save(f, ISC) - f_mask = os.path.abspath('./isc_mask.npy') + f_mask = os.path.abspath("./isc_mask.npy") np.save(f_mask, ISC_mask) - + return f, f_mask def node_isc_significance(ISC, min_null, max_null, two_sided=False): ISC = np.load(ISC) p = isc_significance(ISC, min_null, max_null, two_sided) - f = os.path.abspath('./isc-p.npy') + f = os.path.abspath("./isc-p.npy") np.save(f, p) return f @@ -154,11 +150,9 @@ def node_isc_significance(ISC, min_null, max_null, two_sided=False): def node_isc_permutation(permutation, D, masked, collapse_subj=True, random_state=0): D = np.load(D) masked = np.load(masked) - permutation, min_null, max_null = isc_permutation(permutation, - D, - masked, - collapse_subj, - random_state) + permutation, min_null, max_null = isc_permutation( + permutation, D, masked, collapse_subj, random_state + ) return permutation, min_null, max_null @@ -167,19 +161,19 @@ def node_isfc(D, std=None, collapse_subj=True): ISFC, ISFC_mask = isfc(D, std, collapse_subj) - f = os.path.abspath('./isfc.npy') + f = os.path.abspath("./isfc.npy") np.save(f, ISFC) - f_mask = os.path.abspath('./isfc_mask.npy') + f_mask = os.path.abspath("./isfc_mask.npy") np.save(f_mask, ISFC_mask) - + return f, f_mask def node_isfc_significance(ISFC, min_null, max_null, two_sided=False): ISFC = np.load(ISFC) p = isfc_significance(ISFC, min_null, max_null, two_sided) - f = os.path.abspath('./isfc-p.npy') + f = os.path.abspath("./isfc-p.npy") np.save(f, p) return f @@ -187,37 +181,34 @@ def node_isfc_significance(ISFC, min_null, max_null, two_sided=False): def node_isfc_permutation(permutation, D, masked, collapse_subj=True, random_state=0): D = np.load(D) masked = np.load(masked) - permutation, min_null, max_null = isfc_permutation(permutation, - D, - masked, - collapse_subj, - random_state) + permutation, min_null, max_null = isfc_permutation( + permutation, D, masked, collapse_subj, random_state + ) return permutation, min_null, max_null -def create_isc(name='isc', output_dir=None, working_dir=None, crash_dir=None): +def create_isc(name="isc", output_dir=None, working_dir=None, crash_dir=None): """ - Inter-Subject Correlation - + Inter-Subject Correlation. + Parameters ---------- name : string, optional Name of the workflow. - + Returns ------- workflow : nipype.pipeline.engine.Workflow ISFC workflow. - + Notes ----- - Workflow Inputs:: - - + + Workflow Outputs:: - + References ---------- .. [1] Simony, E., Honey, C. J., Chen, J., Lositsky, O., Yeshurun, @@ -225,139 +216,155 @@ def create_isc(name='isc', output_dir=None, working_dir=None, crash_dir=None): default mode network during narrative comprehension. Nature Communications, 7(May 2015), 1-13. https://doi.org/10.1038/ncomms12141 - - """ + """ if not output_dir: - output_dir = os.path.join(os.getcwd(), 'ISC_output_dir') + output_dir = os.path.join(os.getcwd(), "ISC_output_dir") if not working_dir: - working_dir = os.path.join(os.getcwd(), 'ISC_work_dir') + working_dir = os.path.join(os.getcwd(), "ISC_work_dir") if not crash_dir: - crash_dir = os.path.join(os.getcwd(), 'ISC_crash_dir') + crash_dir = os.path.join(os.getcwd(), "ISC_crash_dir") wf = pe.Workflow(name=name) wf.base_dir = working_dir - wf.config['execution'] = {'hash_method': 'timestamp', - 'crashdump_dir': os.path.abspath(crash_dir)} + wf.config["execution"] = { + "hash_method": "timestamp", + "crashdump_dir": os.path.abspath(crash_dir), + } inputspec = pe.Node( - util.IdentityInterface(fields=[ - 'subjects', - 'permutations', - 'collapse_subj', - 'std', - 'two_sided', - 'random_state' - ]), - name='inputspec' + util.IdentityInterface( + fields=[ + "subjects", + "permutations", + "collapse_subj", + "std", + "two_sided", + "random_state", + ] + ), + name="inputspec", ) outputspec = pe.Node( - util.IdentityInterface(fields=[ - 'correlations', - 'significance' - ]), - name='outputspec' + util.IdentityInterface(fields=["correlations", "significance"]), + name="outputspec", ) - data_node = pe.Node(Function(input_names=['subjects'], - output_names=['subject_ids', 'D', 'voxel_masker'], - function=load_data, - as_module=True), - name='data') - - save_node = pe.Node(Function(input_names=['subject_ids', 'ISC', 'p', 'out_dir', 'collapse_subj', 'voxel_masker'], - output_names=['subject_ids', 'correlations', 'significance'], - function=save_data_isc, - as_module=True), - name='save') + data_node = pe.Node( + Function( + input_names=["subjects"], + output_names=["subject_ids", "D", "voxel_masker"], + function=load_data, + as_module=True, + ), + name="data", + ) + + save_node = pe.Node( + Function( + input_names=[ + "subject_ids", + "ISC", + "p", + "out_dir", + "collapse_subj", + "voxel_masker", + ], + output_names=["subject_ids", "correlations", "significance"], + function=save_data_isc, + as_module=True, + ), + name="save", + ) save_node.inputs.out_dir = output_dir - isc_node = pe.Node(Function(input_names=['D', - 'std', - 'collapse_subj'], - output_names=['ISC', 'masked'], - function=node_isc, - as_module=True), - name='ISC') - - permutations_node = pe.MapNode(Function(input_names=['permutation', - 'D', - 'masked', - 'collapse_subj', - 'random_state'], - output_names=['permutation', - 'min_null', - 'max_null'], - function=node_isc_permutation, - as_module=True), - name='ISC_permutation', iterfield='permutation') - - significance_node = pe.Node(Function(input_names=['ISC', - 'min_null', - 'max_null', - 'two_sided'], - output_names=['p'], - function=node_isc_significance, - as_module=True), - name='ISC_p') - - wf.connect([ - (inputspec, data_node, [('subjects', 'subjects')]), - (inputspec, isc_node, [('collapse_subj', 'collapse_subj')]), - (inputspec, isc_node, [('std', 'std')]), - (data_node, isc_node, [('D', 'D')]), - - (isc_node, significance_node, [('ISC', 'ISC')]), - - (data_node, permutations_node, [('D', 'D')]), - (isc_node, permutations_node, [('masked', 'masked')]), - (inputspec, permutations_node, [('collapse_subj', 'collapse_subj')]), - (inputspec, permutations_node, [(('permutations', _permutations), 'permutation')]), - (inputspec, permutations_node, [('random_state', 'random_state')]), - - (permutations_node, significance_node, [('min_null', 'min_null')]), - (permutations_node, significance_node, [('max_null', 'max_null')]), - (inputspec, significance_node, [('two_sided', 'two_sided')]), - - (isc_node, save_node, [('ISC', 'ISC')]), - (inputspec, save_node, [('collapse_subj', 'collapse_subj')]), - (significance_node, save_node, [('p', 'p')]), - (data_node, save_node, [('subject_ids', 'subject_ids')]), - (data_node, save_node, [('voxel_masker', 'voxel_masker')]), - - (save_node, outputspec, [('subject_ids', 'subject_ids')]), - (save_node, outputspec, [('correlations', 'correlations')]), - (save_node, outputspec, [('significance', 'significance')]), - ]) + isc_node = pe.Node( + Function( + input_names=["D", "std", "collapse_subj"], + output_names=["ISC", "masked"], + function=node_isc, + as_module=True, + ), + name="ISC", + ) + + permutations_node = pe.MapNode( + Function( + input_names=["permutation", "D", "masked", "collapse_subj", "random_state"], + output_names=["permutation", "min_null", "max_null"], + function=node_isc_permutation, + as_module=True, + ), + name="ISC_permutation", + iterfield="permutation", + ) + + significance_node = pe.Node( + Function( + input_names=["ISC", "min_null", "max_null", "two_sided"], + output_names=["p"], + function=node_isc_significance, + as_module=True, + ), + name="ISC_p", + ) + + wf.connect( + [ + (inputspec, data_node, [("subjects", "subjects")]), + (inputspec, isc_node, [("collapse_subj", "collapse_subj")]), + (inputspec, isc_node, [("std", "std")]), + (data_node, isc_node, [("D", "D")]), + (isc_node, significance_node, [("ISC", "ISC")]), + (data_node, permutations_node, [("D", "D")]), + (isc_node, permutations_node, [("masked", "masked")]), + (inputspec, permutations_node, [("collapse_subj", "collapse_subj")]), + ( + inputspec, + permutations_node, + [(("permutations", _permutations), "permutation")], + ), + (inputspec, permutations_node, [("random_state", "random_state")]), + (permutations_node, significance_node, [("min_null", "min_null")]), + (permutations_node, significance_node, [("max_null", "max_null")]), + (inputspec, significance_node, [("two_sided", "two_sided")]), + (isc_node, save_node, [("ISC", "ISC")]), + (inputspec, save_node, [("collapse_subj", "collapse_subj")]), + (significance_node, save_node, [("p", "p")]), + (data_node, save_node, [("subject_ids", "subject_ids")]), + (data_node, save_node, [("voxel_masker", "voxel_masker")]), + (save_node, outputspec, [("subject_ids", "subject_ids")]), + (save_node, outputspec, [("correlations", "correlations")]), + (save_node, outputspec, [("significance", "significance")]), + ] + ) return wf -def create_isfc(name='isfc', output_dir=None, working_dir=None, - crash_dir=None): +def create_isfc(name="isfc", output_dir=None, working_dir=None, crash_dir=None): """ - Inter-Subject Functional Correlation - + Inter-Subject Functional Correlation. + Parameters ---------- name : string, optional Name of the workflow. - + Returns ------- workflow : nipype.pipeline.engine.Workflow ISFC workflow. - + Notes ----- - Workflow Inputs:: - - + + Workflow Outputs:: - + References ---------- .. [1] Simony, E., Honey, C. J., Chen, J., Lositsky, O., Yeshurun, @@ -365,107 +372,120 @@ def create_isfc(name='isfc', output_dir=None, working_dir=None, default mode network during narrative comprehension. Nature Communications, 7(May 2015), 1-13. https://doi.org/10.1038/ncomms12141 - - """ - + """ if not output_dir: - output_dir = os.path.join(os.getcwd(), 'ISC_output_dir') + output_dir = os.path.join(os.getcwd(), "ISC_output_dir") if not working_dir: - working_dir = os.path.join(os.getcwd(), 'ISC_work_dir') + working_dir = os.path.join(os.getcwd(), "ISC_work_dir") if not crash_dir: - crash_dir = os.path.join(os.getcwd(), 'ISC_crash_dir') + crash_dir = os.path.join(os.getcwd(), "ISC_crash_dir") wf = pe.Workflow(name=name) wf.base_dir = working_dir - wf.config['execution'] = {'hash_method': 'timestamp', - 'crashdump_dir': os.path.abspath(crash_dir)} + wf.config["execution"] = { + "hash_method": "timestamp", + "crashdump_dir": os.path.abspath(crash_dir), + } inputspec = pe.Node( - util.IdentityInterface(fields=[ - 'subjects', - 'permutations', - 'collapse_subj', - 'std', - 'two_sided', - 'random_state' - ]), - name='inputspec' + util.IdentityInterface( + fields=[ + "subjects", + "permutations", + "collapse_subj", + "std", + "two_sided", + "random_state", + ] + ), + name="inputspec", ) - data_node = pe.Node(Function(input_names=['subjects'], - output_names=['subject_ids', 'D', 'voxel_masker'], - function=load_data, - as_module=True), - name='data') - - save_node = pe.Node(Function(input_names=['subject_ids', 'ISFC', 'p', 'out_dir', 'collapse_subj'], - output_names=['subject_ids', 'correlations', 'significance'], - function=save_data_isfc, - as_module=True), - name='save') + data_node = pe.Node( + Function( + input_names=["subjects"], + output_names=["subject_ids", "D", "voxel_masker"], + function=load_data, + as_module=True, + ), + name="data", + ) + + save_node = pe.Node( + Function( + input_names=["subject_ids", "ISFC", "p", "out_dir", "collapse_subj"], + output_names=["subject_ids", "correlations", "significance"], + function=save_data_isfc, + as_module=True, + ), + name="save", + ) save_node.inputs.out_dir = output_dir outputspec = pe.Node( - util.IdentityInterface(fields=['correlations', 'significance']), - name='outputspec' + util.IdentityInterface(fields=["correlations", "significance"]), + name="outputspec", ) - isfc_node = pe.Node(Function(input_names=['D', - 'std', - 'collapse_subj'], - output_names=['ISFC', 'masked'], - function=node_isfc, - as_module=True), - name='ISFC') - - permutations_node = pe.MapNode(Function(input_names=['permutation', - 'D', - 'masked', - 'collapse_subj', - 'random_state'], - output_names=['permutation', - 'min_null', - 'max_null'], - function=node_isfc_permutation, - as_module=True), - name='ISFC_permutation', iterfield='permutation') - - significance_node = pe.Node(Function(input_names=['ISFC', - 'min_null', - 'max_null', - 'two_sided'], - output_names=['p'], - function=node_isfc_significance, - as_module=True), - name='ISFC_p') - - wf.connect([ - (inputspec, data_node, [('subjects', 'subjects')]), - (inputspec, isfc_node, [('collapse_subj', 'collapse_subj')]), - (inputspec, isfc_node, [('std', 'std')]), - (data_node, isfc_node, [('D', 'D')]), - - (isfc_node, significance_node, [('ISFC', 'ISFC')]), - - (data_node, permutations_node, [('D', 'D')]), - (isfc_node, permutations_node, [('masked', 'masked')]), - (inputspec, permutations_node, [('collapse_subj', 'collapse_subj')]), - (inputspec, permutations_node, [(('permutations', _permutations), 'permutation')]), - (inputspec, permutations_node, [('random_state', 'random_state')]), - - (permutations_node, significance_node, [('min_null', 'min_null')]), - (permutations_node, significance_node, [('max_null', 'max_null')]), - (inputspec, significance_node, [('two_sided', 'two_sided')]), - - (data_node, save_node, [('subject_ids', 'subject_ids')]), - (inputspec, save_node, [('collapse_subj', 'collapse_subj')]), - (isfc_node, save_node, [('ISFC', 'ISFC')]), - (significance_node, save_node, [('p', 'p')]), - - (save_node, outputspec, [('subject_ids', 'subject_ids')]), - (save_node, outputspec, [('correlations', 'correlations')]), - (save_node, outputspec, [('significance', 'significance')]), - ]) + isfc_node = pe.Node( + Function( + input_names=["D", "std", "collapse_subj"], + output_names=["ISFC", "masked"], + function=node_isfc, + as_module=True, + ), + name="ISFC", + ) + + permutations_node = pe.MapNode( + Function( + input_names=["permutation", "D", "masked", "collapse_subj", "random_state"], + output_names=["permutation", "min_null", "max_null"], + function=node_isfc_permutation, + as_module=True, + ), + name="ISFC_permutation", + iterfield="permutation", + ) + + significance_node = pe.Node( + Function( + input_names=["ISFC", "min_null", "max_null", "two_sided"], + output_names=["p"], + function=node_isfc_significance, + as_module=True, + ), + name="ISFC_p", + ) + + wf.connect( + [ + (inputspec, data_node, [("subjects", "subjects")]), + (inputspec, isfc_node, [("collapse_subj", "collapse_subj")]), + (inputspec, isfc_node, [("std", "std")]), + (data_node, isfc_node, [("D", "D")]), + (isfc_node, significance_node, [("ISFC", "ISFC")]), + (data_node, permutations_node, [("D", "D")]), + (isfc_node, permutations_node, [("masked", "masked")]), + (inputspec, permutations_node, [("collapse_subj", "collapse_subj")]), + ( + inputspec, + permutations_node, + [(("permutations", _permutations), "permutation")], + ), + (inputspec, permutations_node, [("random_state", "random_state")]), + (permutations_node, significance_node, [("min_null", "min_null")]), + (permutations_node, significance_node, [("max_null", "max_null")]), + (inputspec, significance_node, [("two_sided", "two_sided")]), + (data_node, save_node, [("subject_ids", "subject_ids")]), + (inputspec, save_node, [("collapse_subj", "collapse_subj")]), + (isfc_node, save_node, [("ISFC", "ISFC")]), + (significance_node, save_node, [("p", "p")]), + (save_node, outputspec, [("subject_ids", "subject_ids")]), + (save_node, outputspec, [("correlations", "correlations")]), + (save_node, outputspec, [("significance", "significance")]), + ] + ) return wf diff --git a/CPAC/isc/tests/test_pipeline_isc.py b/CPAC/isc/tests/test_pipeline_isc.py index 9ef6efdc61..80e8e978bb 100644 --- a/CPAC/isc/tests/test_pipeline_isc.py +++ b/CPAC/isc/tests/test_pipeline_isc.py @@ -1,27 +1,26 @@ import numpy as np import pytest + from CPAC.isc.pipeline import create_isc, create_isfc -@pytest.mark.skip(reason='needs refactored') +@pytest.mark.skip(reason="needs refactored") def test_pipeline_isc(): - wf = create_isc() wf.inputs.inputspec.D = np.random.uniform(size=(3, 100, 20)) wf.inputs.inputspec.permutations = 1 wf.inputs.inputspec.collapse_subj = True wf.inputs.inputspec.random_state = 42 - wf.run(plugin='Linear') + wf.run(plugin="Linear") -@pytest.mark.skip(reason='needs refactored') +@pytest.mark.skip(reason="needs refactored") def test_pipeline_isfc(): - wf = create_isfc() wf.inputs.inputspec.D = np.random.uniform(size=(3, 100, 20)) wf.inputs.inputspec.permutations = 10 wf.inputs.inputspec.collapse_subj = True wf.inputs.inputspec.random_state = 42 - wf.run(plugin='Linear') \ No newline at end of file + wf.run(plugin="Linear") diff --git a/CPAC/isc/utils.py b/CPAC/isc/utils.py index deb6d8aa40..1dd65322f1 100644 --- a/CPAC/isc/utils.py +++ b/CPAC/isc/utils.py @@ -21,8 +21,7 @@ def phase_randomize(D, random_state=0): pos_freq = np.arange(1, (D.shape[1] - 1) // 2 + 1) neg_freq = np.arange(D.shape[1] - 1, (D.shape[1] - 1) // 2, -1) - shift = random_state.rand(D.shape[0], len(pos_freq), - D.shape[2]) * 2 * np.pi + shift = random_state.rand(D.shape[0], len(pos_freq), D.shape[2]) * 2 * np.pi F[:, pos_freq, :] *= np.exp(1j * shift) F[:, neg_freq, :] *= np.exp(-1j * shift) @@ -30,9 +29,7 @@ def phase_randomize(D, random_state=0): return np.real(ifft(F, axis=1)) -def p_from_null(X, - max_null, min_null, - two_sided=False): +def p_from_null(X, max_null, min_null, two_sided=False): max_null_ecdf = ecdf(max_null) if two_sided: min_null_ecdf = ecdf(min_null) diff --git a/CPAC/longitudinal_pipeline/longitudinal_preproc.py b/CPAC/longitudinal_pipeline/longitudinal_preproc.py index 44f88675b4..701ccd8d34 100644 --- a/CPAC/longitudinal_pipeline/longitudinal_preproc.py +++ b/CPAC/longitudinal_pipeline/longitudinal_preproc.py @@ -1,15 +1,17 @@ # -*- coding: utf-8 -*- +from collections import Counter +from multiprocessing.dummy import Pool as ThreadPool import os -import six + import numpy as np +import six import nibabel as nib -from CPAC.pipeline import nipype_pipeline_engine as pe +from nipype.interfaces import fsl import nipype.interfaces.utility as util -import nipype.interfaces.fsl as fsl -from nipype.interfaces.fsl import ConvertXFM + +from CPAC.pipeline import nipype_pipeline_engine as pe from CPAC.utils.nifti_utils import nifti_image_input -from multiprocessing.dummy import Pool as ThreadPool -from collections import Counter + def read_ants_mat(ants_mat_file): if not os.path.exists(ants_mat_file): @@ -17,12 +19,13 @@ def read_ants_mat(ants_mat_file): with open(ants_mat_file) as f: for line in f: - tmp = line.split(':') - if tmp[0] == 'Parameters': + tmp = line.split(":") + if tmp[0] == "Parameters": oth_transform = np.reshape( - np.fromstring(tmp[1], float, sep=' '), (-1, 3)) - if tmp[0] == 'FixedParameters': - translation = np.fromstring(tmp[1], float, sep=' ') + np.fromstring(tmp[1], float, sep=" "), (-1, 3) + ) + if tmp[0] == "FixedParameters": + translation = np.fromstring(tmp[1], float, sep=" ") return translation, oth_transform @@ -33,11 +36,14 @@ def read_mat(input_mat): if os.path.exists(input_mat): mat = np.loadtxt(input_mat) else: - raise IOError("ERROR norm_transformation: " + input_mat + - " file does not exist") + raise IOError( + "ERROR norm_transformation: " + input_mat + " file does not exist" + ) else: - raise TypeError("ERROR norm_transformation: input_mat should be" + - " either a str or a numpy.ndarray matrix") + raise TypeError( + "ERROR norm_transformation: input_mat should be" + + " either a str or a numpy.ndarray matrix" + ) if mat.shape != (4, 4): raise ValueError("ERROR norm_transformation: the matrix should be 4x4") @@ -52,7 +58,7 @@ def read_mat(input_mat): def norm_transformations(translation, oth_transform): tr_norm = np.linalg.norm(translation) - affine_norm = np.linalg.norm(oth_transform - np.identity(3), 'fro') + affine_norm = np.linalg.norm(oth_transform - np.identity(3), "fro") return pow(tr_norm, 2) + pow(affine_norm, 2) @@ -60,8 +66,8 @@ def norm_transformation(input_mat): """ Calculate the squared norm of the translation + squared Frobenium norm of the difference between other affine transformations and the identity - from an fsl FLIRT transformation matrix - + from an fsl FLIRT transformation matrix. + Parameters ---------- input_mat : str or numpy.ndarray @@ -79,11 +85,14 @@ def norm_transformation(input_mat): if os.path.exists(input_mat): mat = np.loadtxt(input_mat) else: - raise IOError("ERROR norm_transformation: " + input_mat + - " file does not exist") + raise IOError( + "ERROR norm_transformation: " + input_mat + " file does not exist" + ) else: - raise TypeError("ERROR norm_transformation: input_mat should be" + - " either a str (file_path) or a numpy.ndarray matrix") + raise TypeError( + "ERROR norm_transformation: input_mat should be" + + " either a str (file_path) or a numpy.ndarray matrix" + ) if mat.shape != (4, 4): raise ValueError("ERROR norm_transformation: the matrix should be 4x4") @@ -93,15 +102,16 @@ def norm_transformation(input_mat): # 3x3 matrice of rotation, scaling and skewing oth_affine_transform = mat[0:3, 0:3] tr_norm = np.linalg.norm(translation) - affine_norm = np.linalg.norm(oth_affine_transform - np.identity(3), 'fro') + affine_norm = np.linalg.norm(oth_affine_transform - np.identity(3), "fro") return pow(tr_norm, 2) + pow(affine_norm, 2) -def template_convergence(mat_file, mat_type='matrix', - convergence_threshold=np.finfo(np.float64).eps): +def template_convergence( + mat_file, mat_type="matrix", convergence_threshold=np.finfo(np.float64).eps +): """ Calculate the distance between transformation matrix with a matrix of no - transformation + transformation. Parameters ---------- @@ -119,25 +129,30 @@ def template_convergence(mat_file, mat_type='matrix', ------- """ - if mat_type == 'matrix': + if mat_type == "matrix": translation, oth_transform = read_mat(mat_file) - elif mat_type == 'ITK': + elif mat_type == "ITK": translation, oth_transform = read_ants_mat(mat_file) else: - raise ValueError("ERROR template_convergence: this matrix type does " + - "not exist") + raise ValueError( + "ERROR template_convergence: this matrix type does " + "not exist" + ) distance = norm_transformations(translation, oth_transform) - print("distance = " + str(abs(distance))) return abs(distance) <= convergence_threshold -def create_temporary_template(input_brain_list, input_skull_list, - output_brain_path, output_skull_path, avg_method='median'): +def create_temporary_template( + input_brain_list, + input_skull_list, + output_brain_path, + output_skull_path, + avg_method="median", +): """ Average all the 3D images of the list into one 3D image WARNING---the function assumes that all the images have the same header, - the output image will have the same header as the first image of the list + the output image will have the same header as the first image of the list. Parameters ---------- @@ -159,22 +174,27 @@ def create_temporary_template(input_brain_list, input_skull_list, output_skull_path : Nifti1Image temporary longitudinal skull template """ - if not input_brain_list or not input_skull_list: - raise ValueError('ERROR create_temporary_template: image list is empty') + raise ValueError("ERROR create_temporary_template: image list is empty") if len(input_brain_list) == 1 and len(input_skull_list) == 1: - return input_brain_list[0], input_skull_list[0] + return input_brain_list[0], input_skull_list[0] # ALIGN CENTERS avg_brain_data = getattr(np, avg_method)( - np.asarray([nifti_image_input(img).get_fdata() for img in input_brain_list]), 0) + np.asarray([nifti_image_input(img).get_fdata() for img in input_brain_list]), 0 + ) avg_skull_data = getattr(np, avg_method)( - np.asarray([nifti_image_input(img).get_fdata() for img in input_skull_list]), 0) + np.asarray([nifti_image_input(img).get_fdata() for img in input_skull_list]), 0 + ) - nii_brain = nib.Nifti1Image(avg_brain_data, nifti_image_input(input_brain_list[0]).affine) - nii_skull = nib.Nifti1Image(avg_skull_data, nifti_image_input(input_skull_list[0]).affine) + nii_brain = nib.Nifti1Image( + avg_brain_data, nifti_image_input(input_brain_list[0]).affine + ) + nii_skull = nib.Nifti1Image( + avg_skull_data, nifti_image_input(input_skull_list[0]).affine + ) nib.save(nii_brain, output_brain_path) nib.save(nii_skull, output_skull_path) @@ -182,8 +202,15 @@ def create_temporary_template(input_brain_list, input_skull_list, return output_brain_path, output_skull_path -def register_img_list(input_brain_list, ref_img, dof=12, interp='trilinear', cost='corratio', - thread_pool=2, unique_id_list=None): +def register_img_list( + input_brain_list, + ref_img, + dof=12, + interp="trilinear", + cost="corratio", + thread_pool=2, + unique_id_list=None, +): """ Register a list of images to the reference image. @@ -209,6 +236,7 @@ def register_img_list(input_brain_list, ref_img, dof=12, interp='trilinear', cos whether there exists duplicated basename which may happen in non-BIDS dataset unique_id_list : list a list of unique IDs in data + Returns ------- node_list : list of Node @@ -217,29 +245,46 @@ def register_img_list(input_brain_list, ref_img, dof=12, interp='trilinear', cos node.inputs.out_matrix_file contains the path to the transformation matrix """ - if not input_brain_list: - raise ValueError('ERROR register_img_list: image list is empty') + raise ValueError("ERROR register_img_list: image list is empty") - basename_list = [str(os.path.basename(img).split('.')[0]) for img in input_brain_list] + basename_list = [ + str(os.path.basename(img).split(".")[0]) for img in input_brain_list + ] counter = Counter(basename_list) duplicated_basename_list = [i for i, j in counter.items() if j > 1] - - if not duplicated_basename_list: - output_img_list = [os.path.join(os.getcwd(), os.path.basename(img)) - for img in input_brain_list] - output_mat_list = [os.path.join(os.getcwd(), - str(os.path.basename(img).split('.')[0]) + '.mat') - for img in input_brain_list] + if not duplicated_basename_list: + output_img_list = [ + os.path.join(os.getcwd(), os.path.basename(img)) for img in input_brain_list + ] + + output_mat_list = [ + os.path.join(os.getcwd(), str(os.path.basename(img).split(".")[0]) + ".mat") + for img in input_brain_list + ] else: - output_img_list = [os.path.join(os.getcwd(), - str(os.path.basename(img).split('.')[0]) + '_' + unique_id_list[i] + '.nii.gz') - for i, img in enumerate(input_brain_list)] - - output_mat_list = [os.path.join(os.getcwd(), - str(os.path.basename(img).split('.')[0]) + '_' + unique_id_list[i] + '.mat') - for i, img in enumerate(input_brain_list)] + output_img_list = [ + os.path.join( + os.getcwd(), + str(os.path.basename(img).split(".")[0]) + + "_" + + unique_id_list[i] + + ".nii.gz", + ) + for i, img in enumerate(input_brain_list) + ] + + output_mat_list = [ + os.path.join( + os.getcwd(), + str(os.path.basename(img).split(".")[0]) + + "_" + + unique_id_list[i] + + ".mat", + ) + for i, img in enumerate(input_brain_list) + ] def flirt_node(in_img, output_img, output_mat): linear_reg = fsl.FLIRT() @@ -258,9 +303,12 @@ def flirt_node(in_img, output_img, output_mat): else: pool = thread_pool - node_list = [flirt_node(img, out_img, out_mat) - for (img, out_img, out_mat) in zip( - input_brain_list, output_img_list, output_mat_list)] + node_list = [ + flirt_node(img, out_img, out_mat) + for (img, out_img, out_mat) in zip( + input_brain_list, output_img_list, output_mat_list + ) + ] pool.map(lambda node: node.run(), node_list) if isinstance(thread_pool, int): @@ -270,9 +318,19 @@ def flirt_node(in_img, output_img, output_mat): return node_list -def template_creation_flirt(input_brain_list, input_skull_list, init_reg=None, avg_method='median', dof=12, - interp='trilinear', cost='corratio', mat_type='matrix', - convergence_threshold=-1, thread_pool=2, unique_id_list=None): +def template_creation_flirt( + input_brain_list, + input_skull_list, + init_reg=None, + avg_method="median", + dof=12, + interp="trilinear", + cost="corratio", + mat_type="matrix", + convergence_threshold=-1, + thread_pool=2, + unique_id_list=None, +): """ Parameters ---------- @@ -309,6 +367,7 @@ def template_creation_flirt(input_brain_list, input_skull_list, init_reg=None, a node will be added to it to be run. unique_id_list : list of str list of unique IDs in data config + Returns ------- template : str @@ -317,26 +376,41 @@ def template_creation_flirt(input_brain_list, input_skull_list, init_reg=None, a """ # DEBUG to skip the longitudinal template generation which takes a lot of time. # return 'CECI_EST_UN_TEST' - + if not input_brain_list or not input_skull_list: - raise ValueError('ERROR create_temporary_template: image list is empty') - + raise ValueError("ERROR create_temporary_template: image list is empty") + warp_list = [] # check if image basename_list are the same - basename_list = [str(os.path.basename(img).split('.')[0]) for img in input_brain_list] + basename_list = [ + str(os.path.basename(img).split(".")[0]) for img in input_brain_list + ] counter = Counter(basename_list) duplicated_basename_list = [i for i, j in counter.items() if j > 1] - duplicated_basename = False - if not duplicated_basename_list: # if duplicated_basename_list is empty, no duplicated basenames - warp_list_filenames = [os.path.join(os.getcwd(), - str(os.path.basename(img).split('.')[0]) + '_anat_to_template.mat') for img in input_brain_list] + if ( + not duplicated_basename_list + ): # if duplicated_basename_list is empty, no duplicated basenames + warp_list_filenames = [ + os.path.join( + os.getcwd(), + str(os.path.basename(img).split(".")[0]) + "_anat_to_template.mat", + ) + for img in input_brain_list + ] else: if len(unique_id_list) == len(input_brain_list): - duplicated_basename = True - warp_list_filenames = [os.path.join(os.getcwd(), - str(os.path.basename(img).split('.')[0]) + '_' + unique_id_list[i] + '_anat_to_template.mat') for i, img in enumerate(input_brain_list)] + warp_list_filenames = [ + os.path.join( + os.getcwd(), + str(os.path.basename(img).split(".")[0]) + + "_" + + unique_id_list[i] + + "_anat_to_template.mat", + ) + for i, img in enumerate(input_brain_list) + ] if isinstance(thread_pool, int): pool = ThreadPool(thread_pool) @@ -347,9 +421,17 @@ def template_creation_flirt(input_brain_list, input_skull_list, init_reg=None, a convergence_threshold = np.finfo(np.float64).eps if len(input_brain_list) == 1 or len(input_skull_list) == 1: - warnings.warn("input_brain_list or input_skull_list contains only 1 image, no need to calculate template") - warp_list.append(np.identity(4, dtype = float)) # return an identity matrix - return input_brain_list[0], input_skull_list[0], input_brain_list, input_skull_list, warp_list + warnings.warn( + "input_brain_list or input_skull_list contains only 1 image, no need to calculate template" + ) + warp_list.append(np.identity(4, dtype=float)) # return an identity matrix + return ( + input_brain_list[0], + input_skull_list[0], + input_brain_list, + input_skull_list, + warp_list, + ) # Chris: I added this part because it is mentioned in the paper but I actually never used it # You could run a first register_img_list() with a selected image as starting point and @@ -360,8 +442,10 @@ def template_creation_flirt(input_brain_list, input_skull_list, init_reg=None, a mat_list = [node.inputs.out_matrix_file for node in init_reg] warp_list = mat_list # test if every transformation matrix has reached the convergence - convergence_list = [template_convergence( - mat, mat_type, convergence_threshold) for mat in mat_list] + convergence_list = [ + template_convergence(mat, mat_type, convergence_threshold) + for mat in mat_list + ] converged = all(convergence_list) else: raise ValueError("init_reg must be a list of FLIRT nipype nodes files") @@ -370,8 +454,12 @@ def template_creation_flirt(input_brain_list, input_skull_list, init_reg=None, a output_skull_list = input_skull_list converged = False - temporary_brain_template = os.path.join(os.getcwd(), 'temporary_brain_template.nii.gz') - temporary_skull_template = os.path.join(os.getcwd(), 'temporary_skull_template.nii.gz') + temporary_brain_template = os.path.join( + os.getcwd(), "temporary_brain_template.nii.gz" + ) + temporary_skull_template = os.path.join( + os.getcwd(), "temporary_skull_template.nii.gz" + ) """ First is calculated an average image of the dataset to be the temporary template and the loop stops when this temporary template is close enough (with a transformation @@ -379,35 +467,54 @@ def template_creation_flirt(input_brain_list, input_skull_list, init_reg=None, a """ while not converged: temporary_brain_template, temporary_skull_template = create_temporary_template( - input_brain_list=output_brain_list, - input_skull_list=output_skull_list, - output_brain_path=temporary_brain_template, - output_skull_path=temporary_skull_template, - avg_method=avg_method) - - reg_list_node = register_img_list(input_brain_list=output_brain_list, - ref_img=temporary_brain_template, - dof=dof, - interp=interp, - cost=cost, - unique_id_list=unique_id_list) + input_brain_list=output_brain_list, + input_skull_list=output_skull_list, + output_brain_path=temporary_brain_template, + output_skull_path=temporary_skull_template, + avg_method=avg_method, + ) + + reg_list_node = register_img_list( + input_brain_list=output_brain_list, + ref_img=temporary_brain_template, + dof=dof, + interp=interp, + cost=cost, + unique_id_list=unique_id_list, + ) mat_list = [node.inputs.out_matrix_file for node in reg_list_node] - # TODO clean code, refactor variables + # TODO clean code, refactor variables if len(warp_list) == 0: warp_list = mat_list for index, mat in enumerate(mat_list): - cmd = "flirt -in %s -ref %s -applyxfm -init %s -dof %s -interp %s -cost %s -out %s" % (output_skull_list[index], - temporary_skull_template, mat, dof, interp, cost, - os.path.join(os.getcwd(), os.path.basename(output_skull_list[index]))) + cmd = ( + "flirt -in %s -ref %s -applyxfm -init %s -dof %s -interp %s -cost %s -out %s" + % ( + output_skull_list[index], + temporary_skull_template, + mat, + dof, + interp, + cost, + os.path.join( + os.getcwd(), os.path.basename(output_skull_list[index]) + ), + ) + ) os.system(cmd) - output_skull_list[index] = os.path.join(os.getcwd(), os.path.basename(output_skull_list[index])) - + output_skull_list[index] = os.path.join( + os.getcwd(), os.path.basename(output_skull_list[index]) + ) + # why inverse? - cmd = "convert_xfm -omat %s -inverse %s" % (warp_list_filenames[index], warp_list[index]) + cmd = "convert_xfm -omat %s -inverse %s" % ( + warp_list_filenames[index], + warp_list[index], + ) os.system(cmd) warp_list[index] = warp_list_filenames[index] @@ -415,8 +522,10 @@ def template_creation_flirt(input_brain_list, input_skull_list, init_reg=None, a output_brain_list = [node.inputs.out_file for node in reg_list_node] # test if every transformation matrix has reached the convergence - convergence_list = [template_convergence( - mat, mat_type, convergence_threshold) for mat in mat_list] + convergence_list = [ + template_convergence(mat, mat_type, convergence_threshold) + for mat in mat_list + ] converged = all(convergence_list) if isinstance(thread_pool, int): @@ -427,20 +536,29 @@ def template_creation_flirt(input_brain_list, input_skull_list, init_reg=None, a skull_template = temporary_skull_template # register T1 to longitudinal template space - reg_list_node = register_img_list(input_brain_list, - ref_img=temporary_brain_template, - dof=dof, - interp=interp, - cost=cost, - unique_id_list=unique_id_list) + reg_list_node = register_img_list( + input_brain_list, + ref_img=temporary_brain_template, + dof=dof, + interp=interp, + cost=cost, + unique_id_list=unique_id_list, + ) warp_list = [node.inputs.out_matrix_file for node in reg_list_node] - - return brain_template, skull_template, output_brain_list, output_skull_list, warp_list + + return ( + brain_template, + skull_template, + output_brain_list, + output_skull_list, + warp_list, + ) -def subject_specific_template(workflow_name='subject_specific_template', - method='flirt'): +def subject_specific_template( + workflow_name="subject_specific_template", method="flirt" +): """ Parameters ---------- @@ -451,45 +569,47 @@ def subject_specific_template(workflow_name='subject_specific_template', ------- """ imports = [ - 'import os', - 'import warnings', - 'import numpy as np', - 'from collections import Counter', - 'from multiprocessing.dummy import Pool as ThreadPool', - 'from nipype.interfaces.fsl import ConvertXFM', - 'from CPAC.longitudinal_pipeline.longitudinal_preproc import (' - ' create_temporary_template,' - ' register_img_list,' - ' template_convergence' - ')' + "import os", + "import warnings", + "import numpy as np", + "from collections import Counter", + "from multiprocessing.dummy import Pool as ThreadPool", + "from nipype.interfaces.fsl import ConvertXFM", + "from CPAC.longitudinal_pipeline.longitudinal_preproc import (" + " create_temporary_template," + " register_img_list," + " template_convergence" + ")", ] - if method == 'flirt': + if method == "flirt": template_gen_node = pe.Node( util.Function( input_names=[ - 'input_brain_list', - 'input_skull_list', - 'init_reg', - 'avg_method', - 'dof', - 'interp', - 'cost', - 'mat_type', - 'convergence_threshold', - 'thread_pool', - 'unique_id_list'], - output_names=['brain_template', - 'skull_template', - 'output_brain_list', - 'output_skull_list', - 'warp_list'], + "input_brain_list", + "input_skull_list", + "init_reg", + "avg_method", + "dof", + "interp", + "cost", + "mat_type", + "convergence_threshold", + "thread_pool", + "unique_id_list", + ], + output_names=[ + "brain_template", + "skull_template", + "output_brain_list", + "output_skull_list", + "warp_list", + ], imports=imports, - function=template_creation_flirt + function=template_creation_flirt, ), - name=workflow_name + name=workflow_name, ) else: - raise ValueError(str(method) - + 'this method has not yet been implemented') + raise ValueError(str(method) + "this method has not yet been implemented") - return template_gen_node \ No newline at end of file + return template_gen_node diff --git a/CPAC/longitudinal_pipeline/longitudinal_workflow.py b/CPAC/longitudinal_pipeline/longitudinal_workflow.py index fb12d49ab7..1ab66ec97b 100644 --- a/CPAC/longitudinal_pipeline/longitudinal_workflow.py +++ b/CPAC/longitudinal_pipeline/longitudinal_workflow.py @@ -16,60 +16,38 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . import os -import copy -import time -import shutil -from CPAC.pipeline.nodeblock import nodeblock -from nipype import config from nipype import logging -from CPAC.pipeline import nipype_pipeline_engine as pe -import nipype.interfaces.afni as afni -import nipype.interfaces.fsl as fsl +from nipype.interfaces import fsl import nipype.interfaces.io as nio -from nipype.interfaces.utility import Merge, IdentityInterface -import nipype.interfaces.utility as util - from indi_aws import aws_utils -from CPAC.utils.utils import concat_list -from CPAC.utils.interfaces.datasink import DataSink -from CPAC.utils.interfaces.function import Function - -import CPAC - -from CPAC.pipeline.cpac_pipeline import initialize_nipype_wf, \ - connect_pipeline, build_anat_preproc_stack, build_T1w_registration_stack,\ - build_segmentation_stack -from CPAC.pipeline.engine import initiate_rpool, ingress_output_dir - +from CPAC.longitudinal_pipeline.longitudinal_preproc import subject_specific_template +from CPAC.pipeline import nipype_pipeline_engine as pe +from CPAC.pipeline.cpac_pipeline import ( + build_anat_preproc_stack, + build_segmentation_stack, + build_T1w_registration_stack, + connect_pipeline, + initialize_nipype_wf, +) +from CPAC.pipeline.engine import ingress_output_dir, initiate_rpool +from CPAC.pipeline.nodeblock import nodeblock from CPAC.registration import ( create_fsl_flirt_linear_reg, create_fsl_fnirt_nonlinear_reg, - create_wf_calculate_ants_warp + create_wf_calculate_ants_warp, ) - from CPAC.registration.registration import apply_transform - from CPAC.utils.datasource import ( resolve_resolution, - create_anat_datasource, - create_check_for_s3_node -) - -from CPAC.longitudinal_pipeline.longitudinal_preproc import ( - subject_specific_template ) - -from CPAC.utils import find_files, function -from CPAC.utils.outputs import Outputs +from CPAC.utils.interfaces.datasink import DataSink +from CPAC.utils.interfaces.function import Function from CPAC.utils.strategy import Strategy -from CPAC.utils.utils import ( - check_config_resources, - check_prov_for_regtool -) +from CPAC.utils.utils import check_config_resources, check_prov_for_regtool -logger = logging.getLogger('nipype.workflow') +logger = logging.getLogger("nipype.workflow") @nodeblock( @@ -80,24 +58,28 @@ outputs=["space-T1w_desc-brain_mask"], ) def mask_T1w_longitudinal_template(wf, cfg, strat_pool, pipe_num, opt=None): + brain_mask = pe.Node( + interface=fsl.maths.MathsCommand(), + name=f"longitudinal_anatomical_brain_mask_" f"{pipe_num}", + ) + brain_mask.inputs.args = "-bin" - brain_mask = pe.Node(interface=fsl.maths.MathsCommand(), - name=f'longitudinal_anatomical_brain_mask_' - f'{pipe_num}') - brain_mask.inputs.args = '-bin' - - node, out = strat_pool.get_data('desc-brain_T1w') - wf.connect(node, out, brain_mask, 'in_file') + node, out = strat_pool.get_data("desc-brain_T1w") + wf.connect(node, out, brain_mask, "in_file") - outputs = { - 'space-T1w_desc-brain_mask': (brain_mask, 'out_file') - } + outputs = {"space-T1w_desc-brain_mask": (brain_mask, "out_file")} return (wf, outputs) -def create_datasink(datasink_name, config, subject_id, session_id='', - strat_name='', map_node_iterfield=None): +def create_datasink( + datasink_name, + config, + subject_id, + session_id="", + strat_name="", + map_node_iterfield=None, +): """ Parameters @@ -114,8 +96,7 @@ def create_datasink(datasink_name, config, subject_id, session_id='', """ try: - encrypt_data = bool( - config.pipeline_setup['Amazon-AWS']['s3_encryption']) + encrypt_data = bool(config.pipeline_setup["Amazon-AWS"]["s3_encryption"]) except: encrypt_data = False @@ -123,58 +104,62 @@ def create_datasink(datasink_name, config, subject_id, session_id='', # Extract credentials path for output if it exists try: # Get path to creds file - creds_path = '' - if config.pipeline_setup['Amazon-AWS'][ - 'aws_output_bucket_credentials']: - creds_path = str(config.pipeline_setup['Amazon-AWS'][ - 'aws_output_bucket_credentials']) + creds_path = "" + if config.pipeline_setup["Amazon-AWS"]["aws_output_bucket_credentials"]: + creds_path = str( + config.pipeline_setup["Amazon-AWS"]["aws_output_bucket_credentials"] + ) creds_path = os.path.abspath(creds_path) - if config.pipeline_setup['output_directory'][ - 'path'].lower().startswith('s3://'): + if ( + config.pipeline_setup["output_directory"]["path"] + .lower() + .startswith("s3://") + ): # Test for s3 write access - s3_write_access = \ - aws_utils.test_bucket_access(creds_path, - config.pipeline_setup[ - 'output_directory']['path']) + s3_write_access = aws_utils.test_bucket_access( + creds_path, config.pipeline_setup["output_directory"]["path"] + ) if not s3_write_access: - raise Exception('Not able to write to bucket!') + raise Exception("Not able to write to bucket!") except Exception as e: - if config.pipeline_setup['output_directory'][ - 'path'].lower().startswith('s3://'): - err_msg = 'There was an error processing credentials or ' \ - 'accessing the S3 bucket. Check and try again.\n' \ - 'Error: %s' % e + if ( + config.pipeline_setup["output_directory"]["path"] + .lower() + .startswith("s3://") + ): + err_msg = ( + "There was an error processing credentials or " + "accessing the S3 bucket. Check and try again.\n" + "Error: %s" % e + ) raise Exception(err_msg) if map_node_iterfield is not None: ds = pe.MapNode( DataSink(infields=map_node_iterfield), - name='sinker_{}'.format(datasink_name), - iterfield=map_node_iterfield + name="sinker_{}".format(datasink_name), + iterfield=map_node_iterfield, ) else: - ds = pe.Node( - DataSink(), - name='sinker_{}'.format(datasink_name) - ) + ds = pe.Node(DataSink(), name="sinker_{}".format(datasink_name)) - ds.inputs.base_directory = config.pipeline_setup['output_directory'][ - 'path'] + ds.inputs.base_directory = config.pipeline_setup["output_directory"]["path"] ds.inputs.creds_path = creds_path ds.inputs.encrypt_bucket_keys = encrypt_data ds.inputs.container = os.path.join( - 'pipeline_%s_%s' % ( - config.pipeline_setup['pipeline_name'], strat_name), - subject_id, session_id + "pipeline_%s_%s" % (config.pipeline_setup["pipeline_name"], strat_name), + subject_id, + session_id, ) return ds -def connect_anat_preproc_inputs(strat, anat_preproc, strat_name, - strat_nodes_list_list, workflow): +def connect_anat_preproc_inputs( + strat, anat_preproc, strat_name, strat_nodes_list_list, workflow +): """ Parameters ---------- @@ -196,26 +181,23 @@ def connect_anat_preproc_inputs(strat, anat_preproc, strat_name, strat_nodes_list_list : list a list of strat_nodes_list """ - new_strat = strat.fork() - tmp_node, out_key = new_strat['anatomical'] - workflow.connect(tmp_node, out_key, anat_preproc, 'inputspec.anat') + tmp_node, out_key = new_strat["anatomical"] + workflow.connect(tmp_node, out_key, anat_preproc, "inputspec.anat") - tmp_node, out_key = new_strat['template_cmass'] - workflow.connect(tmp_node, out_key, anat_preproc, - 'inputspec.template_cmass') + tmp_node, out_key = new_strat["template_cmass"] + workflow.connect(tmp_node, out_key, anat_preproc, "inputspec.template_cmass") new_strat.append_name(anat_preproc.name) - new_strat.update_resource_pool({ - 'anatomical_brain': ( - anat_preproc, 'outputspec.brain'), - 'anatomical_skull_leaf': ( - anat_preproc, 'outputspec.reorient'), - 'anatomical_brain_mask': ( - anat_preproc, 'outputspec.brain_mask'), - }) + new_strat.update_resource_pool( + { + "anatomical_brain": (anat_preproc, "outputspec.brain"), + "anatomical_skull_leaf": (anat_preproc, "outputspec.reorient"), + "anatomical_brain_mask": (anat_preproc, "outputspec.brain_mask"), + } + ) try: strat_nodes_list_list[strat_name].append(new_strat) @@ -239,10 +221,10 @@ def select_session(session, output_brains, warps): brain_path = None warp_path = None for brain_path in output_brains: - if f'{session}_' in brain_path: + if f"{session}_" in brain_path: break for warp_path in warps: - if f'{session}_' in warp_path: + if f"{session}_" in warp_path: break return (brain_path, warp_path) @@ -255,17 +237,16 @@ def select_session(session, output_brains, warps): outputs=["space-longitudinal_desc-brain_mask"], ) def mask_longitudinal_T1w_brain(wf, cfg, strat_pool, pipe_num, opt=None): - - brain_mask = pe.Node(interface=fsl.maths.MathsCommand(), - name=f'longitudinal_T1w_brain_mask_{pipe_num}') - brain_mask.inputs.args = '-bin' + brain_mask = pe.Node( + interface=fsl.maths.MathsCommand(), + name=f"longitudinal_T1w_brain_mask_{pipe_num}", + ) + brain_mask.inputs.args = "-bin" node, out = strat_pool.get_data("space-longitudinal_desc-brain_T1w") - wf.connect(node, out, brain_mask, 'in_file') + wf.connect(node, out, brain_mask, "in_file") - outputs = { - 'space-longitudinal_desc-brain_mask': (brain_mask, 'out_file') - } + outputs = {"space-longitudinal_desc-brain_mask": (brain_mask, "out_file")} return (wf, outputs) @@ -282,46 +263,43 @@ def mask_longitudinal_T1w_brain(wf, cfg, strat_pool, pipe_num, opt=None): ], outputs=["space-template_desc-brain_T1w"], ) -def warp_longitudinal_T1w_to_template(wf, cfg, strat_pool, pipe_num, - opt=None): - +def warp_longitudinal_T1w_to_template(wf, cfg, strat_pool, pipe_num, opt=None): xfm_prov = strat_pool.get_cpac_provenance( - 'from-longitudinal_to-template_mode-image_xfm') + "from-longitudinal_to-template_mode-image_xfm" + ) reg_tool = check_prov_for_regtool(xfm_prov) - num_cpus = cfg.pipeline_setup['system_config'][ - 'max_cores_per_participant'] + num_cpus = cfg.pipeline_setup["system_config"]["max_cores_per_participant"] - num_ants_cores = cfg.pipeline_setup['system_config']['num_ants_threads'] + num_ants_cores = cfg.pipeline_setup["system_config"]["num_ants_threads"] - apply_xfm = apply_transform(f'warp_longitudinal_to_T1template_{pipe_num}', - reg_tool, time_series=False, - num_cpus=num_cpus, - num_ants_cores=num_ants_cores) + apply_xfm = apply_transform( + f"warp_longitudinal_to_T1template_{pipe_num}", + reg_tool, + time_series=False, + num_cpus=num_cpus, + num_ants_cores=num_ants_cores, + ) - if reg_tool == 'ants': + if reg_tool == "ants": apply_xfm.inputs.inputspec.interpolation = cfg.registration_workflows[ - 'anatomical_registration']['registration']['ANTs'][ - 'interpolation'] - elif reg_tool == 'fsl': + "anatomical_registration" + ]["registration"]["ANTs"]["interpolation"] + elif reg_tool == "fsl": apply_xfm.inputs.inputspec.interpolation = cfg.registration_workflows[ - 'anatomical_registration']['registration']['FSL-FNIRT'][ - 'interpolation'] + "anatomical_registration" + ]["registration"]["FSL-FNIRT"]["interpolation"] node, out = strat_pool.get_data("space-longitudinal_desc-brain_T1w") - wf.connect(node, out, apply_xfm, 'inputspec.input_image') + wf.connect(node, out, apply_xfm, "inputspec.input_image") node, out = strat_pool.get_data("T1w_brain_template") - wf.connect(node, out, apply_xfm, 'inputspec.reference') + wf.connect(node, out, apply_xfm, "inputspec.reference") - node, out = \ - strat_pool.get_data("from-longitudinal_to-template_mode-image_xfm") - wf.connect(node, out, apply_xfm, 'inputspec.transform') + node, out = strat_pool.get_data("from-longitudinal_to-template_mode-image_xfm") + wf.connect(node, out, apply_xfm, "inputspec.transform") - outputs = { - 'space-template_desc-brain_T1w': - (apply_xfm, 'outputspec.output_image') - } + outputs = {"space-template_desc-brain_T1w": (apply_xfm, "outputspec.output_image")} return (wf, outputs) @@ -357,50 +335,57 @@ def warp_longitudinal_T1w_to_template(wf, cfg, strat_pool, pipe_num, ], ) def warp_longitudinal_seg_to_T1w(wf, cfg, strat_pool, pipe_num, opt=None): - xfm_prov = strat_pool.get_cpac_provenance( - 'from-longitudinal_to-T1w_mode-image_desc-linear_xfm') + "from-longitudinal_to-T1w_mode-image_desc-linear_xfm" + ) reg_tool = check_prov_for_regtool(xfm_prov) - num_cpus = cfg.pipeline_setup['system_config'][ - 'max_cores_per_participant'] + num_cpus = cfg.pipeline_setup["system_config"]["max_cores_per_participant"] - num_ants_cores = cfg.pipeline_setup['system_config']['num_ants_threads'] + num_ants_cores = cfg.pipeline_setup["system_config"]["num_ants_threads"] outputs = {} labels = [ - 'CSF_mask', 'CSF_desc-preproc_mask', 'CSF_probseg', - 'GM_mask', 'GM_desc-preproc_mask', 'GM_probseg', - 'WM_mask', 'WM_desc-preproc_mask', 'WM_probseg', + "CSF_mask", + "CSF_desc-preproc_mask", + "CSF_probseg", + "GM_mask", + "GM_desc-preproc_mask", + "GM_probseg", + "WM_mask", + "WM_desc-preproc_mask", + "WM_probseg", ] for label in labels: - apply_xfm = apply_transform(f'warp_longitudinal_seg_to_T1w_{label}_' - f'{pipe_num}', reg_tool, - time_series=False, num_cpus=num_cpus, - num_ants_cores=num_ants_cores) - - if reg_tool == 'ants': - apply_xfm.inputs.inputspec.interpolation = \ - cfg.registration_workflows['anatomical_registration'][ - 'registration']['ANTs']['interpolation'] - elif reg_tool == 'fsl': - apply_xfm.inputs.inputspec.interpolation = \ - cfg.registration_workflows['anatomical_registration'][ - 'registration']['FSL-FNIRT']['interpolation'] + apply_xfm = apply_transform( + f"warp_longitudinal_seg_to_T1w_{label}_" f"{pipe_num}", + reg_tool, + time_series=False, + num_cpus=num_cpus, + num_ants_cores=num_ants_cores, + ) + + if reg_tool == "ants": + apply_xfm.inputs.inputspec.interpolation = cfg.registration_workflows[ + "anatomical_registration" + ]["registration"]["ANTs"]["interpolation"] + elif reg_tool == "fsl": + apply_xfm.inputs.inputspec.interpolation = cfg.registration_workflows[ + "anatomical_registration" + ]["registration"]["FSL-FNIRT"]["interpolation"] node, out = strat_pool.get_data("space-longitudinal_desc-brain_T1w") - wf.connect(node, out, apply_xfm, 'inputspec.input_image') + wf.connect(node, out, apply_xfm, "inputspec.input_image") node, out = strat_pool.get_data("T1w_brain_template") - wf.connect(node, out, apply_xfm, 'inputspec.reference') + wf.connect(node, out, apply_xfm, "inputspec.reference") - node, out = \ - strat_pool.get_data("from-longitudinal_to-template_mode-image_xfm") - wf.connect(node, out, apply_xfm, 'inputspec.transform') + node, out = strat_pool.get_data("from-longitudinal_to-template_mode-image_xfm") + wf.connect(node, out, apply_xfm, "inputspec.transform") - outputs[f'label-{label}'] = (apply_xfm, 'outputspec.output_image') + outputs[f"label-{label}"] = (apply_xfm, "outputspec.output_image") return (wf, outputs) @@ -421,42 +406,44 @@ def anat_longitudinal_wf(subject_id, sub_list, config): ------- None """ - # list of lists for every strategy session_id_list = [] session_wfs = {} cpac_dirs = [] - out_dir = config.pipeline_setup['output_directory']['path'] + out_dir = config.pipeline_setup["output_directory"]["path"] - orig_pipe_name = config.pipeline_setup['pipeline_name'] + orig_pipe_name = config.pipeline_setup["pipeline_name"] # Loop over the sessions to create the input for the longitudinal # algorithm for session in sub_list: - - unique_id = session['unique_id'] + unique_id = session["unique_id"] session_id_list.append(unique_id) try: - creds_path = session['creds_path'] - if creds_path and 'none' not in creds_path.lower(): + creds_path = session["creds_path"] + if creds_path and "none" not in creds_path.lower(): if os.path.exists(creds_path): input_creds_path = os.path.abspath(creds_path) else: - err_msg = 'Credentials path: "%s" for subject "%s" ' \ - 'session "%s" was not found. Check this path ' \ - 'and try again.' % (creds_path, subject_id, - unique_id) + err_msg = ( + 'Credentials path: "%s" for subject "%s" ' + 'session "%s" was not found. Check this path ' + "and try again." % (creds_path, subject_id, unique_id) + ) raise Exception(err_msg) else: input_creds_path = None except KeyError: input_creds_path = None - workflow = initialize_nipype_wf(config, sub_list[0], - # just grab the first one for the name - name="anat_longitudinal_pre-preproc") + workflow = initialize_nipype_wf( + config, + sub_list[0], + # just grab the first one for the name + name="anat_longitudinal_pre-preproc", + ) workflow, rpool = initiate_rpool(workflow, config, session) pipeline_blocks = build_anat_preproc_stack(rpool, config) @@ -468,9 +455,10 @@ def anat_longitudinal_wf(subject_id, sub_list, config): workflow.run() - cpac_dir = os.path.join(out_dir, f'pipeline_{orig_pipe_name}', - f'{subject_id}_{unique_id}') - cpac_dirs.append(os.path.join(cpac_dir, 'anat')) + cpac_dir = os.path.join( + out_dir, f"pipeline_{orig_pipe_name}", f"{subject_id}_{unique_id}" + ) + cpac_dirs.append(os.path.join(cpac_dir, "anat")) # Now we have all the anat_preproc set up for every session # loop over the different anat preproc strategies @@ -479,88 +467,108 @@ def anat_longitudinal_wf(subject_id, sub_list, config): for cpac_dir in cpac_dirs: if os.path.isdir(cpac_dir): for filename in os.listdir(cpac_dir): - if 'T1w.nii' in filename: - for tag in filename.split('_'): - if 'desc-' in tag and 'brain' in tag: + if "T1w.nii" in filename: + for tag in filename.split("_"): + if "desc-" in tag and "brain" in tag: if tag not in strats_brain_dct: strats_brain_dct[tag] = [] - strats_brain_dct[tag].append(os.path.join(cpac_dir, - filename)) + strats_brain_dct[tag].append( + os.path.join(cpac_dir, filename) + ) if tag not in strats_head_dct: strats_head_dct[tag] = [] - head_file = filename.replace(tag, 'desc-reorient') - strats_head_dct[tag].append(os.path.join(cpac_dir, - head_file)) + head_file = filename.replace(tag, "desc-reorient") + strats_head_dct[tag].append( + os.path.join(cpac_dir, head_file) + ) for strat in strats_brain_dct.keys(): + wf = initialize_nipype_wf( + config, + sub_list[0], + # just grab the first one for the name + name=f"template_node_{strat}", + ) - wf = initialize_nipype_wf(config, sub_list[0], - # just grab the first one for the name - name=f"template_node_{strat}") - - config.pipeline_setup[ - 'pipeline_name'] = f'longitudinal_{orig_pipe_name}' + config.pipeline_setup["pipeline_name"] = f"longitudinal_{orig_pipe_name}" - template_node_name = f'longitudinal_anat_template_{strat}' + template_node_name = f"longitudinal_anat_template_{strat}" # This node will generate the longitudinal template (the functions are # in longitudinal_preproc) # Later other algorithms could be added to calculate it, like the # multivariate template from ANTS # It would just require to change it here. - template_node = subject_specific_template( - workflow_name=template_node_name - ) + template_node = subject_specific_template(workflow_name=template_node_name) template_node.inputs.set( - avg_method=config.longitudinal_template_generation[ - 'average_method'], - dof=config.longitudinal_template_generation['dof'], - interp=config.longitudinal_template_generation['interp'], - cost=config.longitudinal_template_generation['cost'], + avg_method=config.longitudinal_template_generation["average_method"], + dof=config.longitudinal_template_generation["dof"], + interp=config.longitudinal_template_generation["interp"], + cost=config.longitudinal_template_generation["cost"], convergence_threshold=config.longitudinal_template_generation[ - 'convergence_threshold'], - thread_pool=config.longitudinal_template_generation[ - 'thread_pool'], - unique_id_list=list(session_wfs.keys()) + "convergence_threshold" + ], + thread_pool=config.longitudinal_template_generation["thread_pool"], + unique_id_list=list(session_wfs.keys()), ) template_node.inputs.input_brain_list = strats_brain_dct[strat] template_node.inputs.input_skull_list = strats_head_dct[strat] - long_id = f'longitudinal_{subject_id}_strat-{strat}' + long_id = f"longitudinal_{subject_id}_strat-{strat}" wf, rpool = initiate_rpool(wf, config, part_id=long_id) - rpool.set_data("space-longitudinal_desc-brain_T1w", - template_node, 'brain_template', {}, - "", template_node_name) + rpool.set_data( + "space-longitudinal_desc-brain_T1w", + template_node, + "brain_template", + {}, + "", + template_node_name, + ) - rpool.set_data("space-longitudinal_desc-brain_T1w-template", - template_node, 'brain_template', {}, - "", template_node_name) + rpool.set_data( + "space-longitudinal_desc-brain_T1w-template", + template_node, + "brain_template", + {}, + "", + template_node_name, + ) - rpool.set_data("space-longitudinal_desc-reorient_T1w", - template_node, 'skull_template', {}, - "", template_node_name) + rpool.set_data( + "space-longitudinal_desc-reorient_T1w", + template_node, + "skull_template", + {}, + "", + template_node_name, + ) - rpool.set_data("space-longitudinal_desc-reorient_T1w-template", - template_node, 'skull_template', {}, - "", template_node_name) + rpool.set_data( + "space-longitudinal_desc-reorient_T1w-template", + template_node, + "skull_template", + {}, + "", + template_node_name, + ) pipeline_blocks = [mask_longitudinal_T1w_brain] - pipeline_blocks = build_T1w_registration_stack(rpool, config, - pipeline_blocks) + pipeline_blocks = build_T1w_registration_stack(rpool, config, pipeline_blocks) - pipeline_blocks = build_segmentation_stack(rpool, config, - pipeline_blocks) + pipeline_blocks = build_segmentation_stack(rpool, config, pipeline_blocks) wf = connect_pipeline(wf, config, rpool, pipeline_blocks) - excl = ['space-longitudinal_desc-brain_T1w', - 'space-longitudinal_desc-reorient_T1w', - 'space-longitudinal_desc-brain_mask'] + excl = [ + "space-longitudinal_desc-brain_T1w", + "space-longitudinal_desc-reorient_T1w", + "space-longitudinal_desc-brain_mask", + ] rpool.gather_pipes(wf, config, add_excl=excl) # this is going to run multiple times! @@ -568,21 +576,21 @@ def anat_longitudinal_wf(subject_id, sub_list, config): wf.run() # now, just write out a copy of the above to each session - config.pipeline_setup['pipeline_name'] = orig_pipe_name + config.pipeline_setup["pipeline_name"] = orig_pipe_name for session in sub_list: - - unique_id = session['unique_id'] + unique_id = session["unique_id"] try: - creds_path = session['creds_path'] - if creds_path and 'none' not in creds_path.lower(): + creds_path = session["creds_path"] + if creds_path and "none" not in creds_path.lower(): if os.path.exists(creds_path): input_creds_path = os.path.abspath(creds_path) else: - err_msg = 'Credentials path: "%s" for subject "%s" ' \ - 'session "%s" was not found. Check this path ' \ - 'and try again.' % (creds_path, subject_id, - unique_id) + err_msg = ( + 'Credentials path: "%s" for subject "%s" ' + 'session "%s" was not found. Check this path ' + "and try again." % (creds_path, subject_id, unique_id) + ) raise Exception(err_msg) else: input_creds_path = None @@ -593,56 +601,64 @@ def anat_longitudinal_wf(subject_id, sub_list, config): wf, rpool = initiate_rpool(wf, config, session) - config.pipeline_setup[ - 'pipeline_name'] = f'longitudinal_{orig_pipe_name}' - rpool = ingress_output_dir(config, rpool, long_id, - creds_path=input_creds_path) - - select_node_name = f'select_{unique_id}' - select_sess = pe.Node(Function(input_names=['session', - 'output_brains', - 'warps'], - output_names=['brain_path', - 'warp_path'], - function=select_session), - name=select_node_name) + config.pipeline_setup["pipeline_name"] = f"longitudinal_{orig_pipe_name}" + rpool = ingress_output_dir( + config, rpool, long_id, creds_path=input_creds_path + ) + + select_node_name = f"select_{unique_id}" + select_sess = pe.Node( + Function( + input_names=["session", "output_brains", "warps"], + output_names=["brain_path", "warp_path"], + function=select_session, + ), + name=select_node_name, + ) select_sess.inputs.session = unique_id - wf.connect(template_node, 'output_brain_list', select_sess, - 'output_brains') - wf.connect(template_node, 'warp_list', select_sess, 'warps') + wf.connect(template_node, "output_brain_list", select_sess, "output_brains") + wf.connect(template_node, "warp_list", select_sess, "warps") - rpool.set_data("space-longitudinal_desc-brain_T1w", - select_sess, 'brain_path', {}, "", - select_node_name) + rpool.set_data( + "space-longitudinal_desc-brain_T1w", + select_sess, + "brain_path", + {}, + "", + select_node_name, + ) - rpool.set_data("from-T1w_to-longitudinal_mode-image_" - "desc-linear_xfm", - select_sess, 'warp_path', {}, "", - select_node_name) + rpool.set_data( + "from-T1w_to-longitudinal_mode-image_" "desc-linear_xfm", + select_sess, + "warp_path", + {}, + "", + select_node_name, + ) - config.pipeline_setup['pipeline_name'] = orig_pipe_name - excl = ['space-template_desc-brain_T1w', - 'space-T1w_desc-brain_mask'] + config.pipeline_setup["pipeline_name"] = orig_pipe_name + excl = ["space-template_desc-brain_T1w", "space-T1w_desc-brain_mask"] rpool.gather_pipes(wf, config, add_excl=excl) wf.run() # begin single-session stuff again for session in sub_list: - - unique_id = session['unique_id'] + unique_id = session["unique_id"] try: - creds_path = session['creds_path'] - if creds_path and 'none' not in creds_path.lower(): + creds_path = session["creds_path"] + if creds_path and "none" not in creds_path.lower(): if os.path.exists(creds_path): input_creds_path = os.path.abspath(creds_path) else: - err_msg = 'Credentials path: "%s" for subject "%s" ' \ - 'session "%s" was not found. Check this path ' \ - 'and try again.' % (creds_path, subject_id, - unique_id) + err_msg = ( + 'Credentials path: "%s" for subject "%s" ' + 'session "%s" was not found. Check this path ' + "and try again." % (creds_path, subject_id, unique_id) + ) raise Exception(err_msg) else: input_creds_path = None @@ -653,8 +669,10 @@ def anat_longitudinal_wf(subject_id, sub_list, config): wf, rpool = initiate_rpool(wf, config, session) - pipeline_blocks = [warp_longitudinal_T1w_to_template, - warp_longitudinal_seg_to_T1w] + pipeline_blocks = [ + warp_longitudinal_T1w_to_template, + warp_longitudinal_seg_to_T1w, + ] wf = connect_pipeline(wf, config, rpool, pipeline_blocks) @@ -665,8 +683,6 @@ def anat_longitudinal_wf(subject_id, sub_list, config): wf.run() - - # TODO check: # 1 func alone works # 2 anat + func works, pass anat strategy list? @@ -687,42 +703,43 @@ def func_preproc_longitudinal_wf(subject_id, sub_list, config): strat_list_ses_list : list of list a list of strategies; within each strategy, a list of sessions """ - - datasink = pe.Node(nio.DataSink(), name='sinker') - datasink.inputs.base_directory = \ - config.pipeline_setup['working_directory']['path'] + datasink = pe.Node(nio.DataSink(), name="sinker") + datasink.inputs.base_directory = config.pipeline_setup["working_directory"]["path"] session_id_list = [] ses_list_strat_list = {} - workflow_name = 'func_preproc_longitudinal_' + str(subject_id) + workflow_name = "func_preproc_longitudinal_" + str(subject_id) workflow = pe.Workflow(name=workflow_name) - workflow.base_dir = config.pipeline_setup['working_directory']['path'] - workflow.config['execution'] = { - 'hash_method': 'timestamp', - 'crashdump_dir': os.path.abspath( - config.pipeline_setup['crash_directory']['path']) + workflow.base_dir = config.pipeline_setup["working_directory"]["path"] + workflow.config["execution"] = { + "hash_method": "timestamp", + "crashdump_dir": os.path.abspath( + config.pipeline_setup["crash_directory"]["path"] + ), } for sub_dict in sub_list: - if 'func' in sub_dict or 'rest' in sub_dict: - if 'func' in sub_dict: - func_paths_dict = sub_dict['func'] + if "func" in sub_dict or "rest" in sub_dict: + if "func" in sub_dict: + sub_dict["func"] else: - func_paths_dict = sub_dict['rest'] + sub_dict["rest"] - unique_id = sub_dict['unique_id'] + unique_id = sub_dict["unique_id"] session_id_list.append(unique_id) try: - creds_path = sub_dict['creds_path'] - if creds_path and 'none' not in creds_path.lower(): + creds_path = sub_dict["creds_path"] + if creds_path and "none" not in creds_path.lower(): if os.path.exists(creds_path): input_creds_path = os.path.abspath(creds_path) else: - err_msg = 'Credentials path: "%s" for subject "%s" was not ' \ - 'found. Check this path and try again.' % ( - creds_path, subject_id) + err_msg = ( + 'Credentials path: "%s" for subject "%s" was not ' + "found. Check this path and try again." + % (creds_path, subject_id) + ) raise Exception(err_msg) else: input_creds_path = None @@ -731,7 +748,7 @@ def func_preproc_longitudinal_wf(subject_id, sub_list, config): strat = Strategy() strat_list = [strat] - node_suffix = '_'.join([subject_id, unique_id]) + node_suffix = "_".join([subject_id, unique_id]) # Functional Ingress Workflow # add optional flag @@ -742,24 +759,23 @@ def func_preproc_longitudinal_wf(subject_id, sub_list, config): sub_dict, subject_id, input_creds_path, - node_suffix) + node_suffix, + ) # Functional Initial Prep Workflow - workflow, strat_list = connect_func_init(workflow, strat_list, - config, node_suffix) + workflow, strat_list = connect_func_init( + workflow, strat_list, config, node_suffix + ) # Functional Image Preprocessing Workflow - workflow, strat_list = connect_func_preproc(workflow, strat_list, - config, node_suffix) + workflow, strat_list = connect_func_preproc( + workflow, strat_list, config, node_suffix + ) # Distortion Correction - workflow, strat_list = connect_distortion_correction(workflow, - strat_list, - config, - diff, - blip, - fmap_rp_list, - node_suffix) + workflow, strat_list = connect_distortion_correction( + workflow, strat_list, config, diff, blip, fmap_rp_list, node_suffix + ) ses_list_strat_list[node_suffix] = strat_list @@ -771,10 +787,10 @@ def func_preproc_longitudinal_wf(subject_id, sub_list, config): # TODO rename and reorganize dict # TODO update strat name strat_list_ses_list = {} - strat_list_ses_list['func_default'] = [] + strat_list_ses_list["func_default"] = [] for sub_ses_id, strat_nodes_list in ses_list_strat_list.items(): - strat_list_ses_list['func_default'].append(strat_nodes_list[0]) + strat_list_ses_list["func_default"].append(strat_nodes_list[0]) workflow.run() @@ -795,16 +811,15 @@ def merge_func_preproc(working_directory): skull_list : list a list of func preprocessed skull """ - brain_list = [] skull_list = [] for dirpath, dirnames, filenames in os.walk(working_directory): for f in filenames: - if 'func_get_preprocessed_median' in dirpath and '.nii.gz' in f: + if "func_get_preprocessed_median" in dirpath and ".nii.gz" in f: filepath = os.path.join(dirpath, f) brain_list.append(filepath) - if 'func_get_motion_correct_median' in dirpath and '.nii.gz' in f: + if "func_get_motion_correct_median" in dirpath and ".nii.gz" in f: filepath = os.path.join(dirpath, f) skull_list.append(filepath) @@ -815,70 +830,84 @@ def merge_func_preproc(working_directory): def register_func_longitudinal_template_to_standard( - longitudinal_template_node, c, workflow, strat_init, strat_name): - sub_mem_gb, num_cores_per_sub, num_ants_cores, num_omp_cores = \ - check_config_resources(c) + longitudinal_template_node, c, workflow, strat_init, strat_name +): + ( + sub_mem_gb, + num_cores_per_sub, + num_ants_cores, + num_omp_cores, + ) = check_config_resources(c) strat_init_new = strat_init.fork() - strat_init_new.update_resource_pool({ - 'functional_preprocessed_median': ( - longitudinal_template_node, 'brain_template'), - 'motion_correct_median': ( - longitudinal_template_node, 'skull_template') - }) + strat_init_new.update_resource_pool( + { + "functional_preprocessed_median": ( + longitudinal_template_node, + "brain_template", + ), + "motion_correct_median": (longitudinal_template_node, "skull_template"), + } + ) strat_list = [strat_init_new] new_strat_list = [] - regOption = c.anatomical_preproc[ - 'registration_workflow' - ]['registration']['using'] - - if 'FSL' in regOption: + regOption = c.anatomical_preproc["registration_workflow"]["registration"]["using"] + if "FSL" in regOption: for num_strat, strat in enumerate(strat_list): - flirt_reg_func_mni = create_fsl_flirt_linear_reg( - 'func_mni_flirt_register_%s_%d' % (strat_name, num_strat) + "func_mni_flirt_register_%s_%d" % (strat_name, num_strat) ) - if c.functional_registration['2-func_registration_to_template'][ - 'FNIRT_pipelines']['interpolation'] not in ["trilinear", - "sinc", "spline"]: + if c.functional_registration["2-func_registration_to_template"][ + "FNIRT_pipelines" + ]["interpolation"] not in ["trilinear", "sinc", "spline"]: err_msg = 'The selected FSL interpolation method may be in the list of values: "trilinear", "sinc", "spline"' raise Exception(err_msg) # Input registration parameters - flirt_reg_func_mni.inputs.inputspec.interp = \ - c.functional_registration['2-func_registration_to_template'][ - 'FNIRT_pipelines']['interpolation'] + flirt_reg_func_mni.inputs.inputspec.interp = c.functional_registration[ + "2-func_registration_to_template" + ]["FNIRT_pipelines"]["interpolation"] - node, out_file = strat['functional_preprocessed_median'] - workflow.connect(node, out_file, - flirt_reg_func_mni, 'inputspec.input_brain') + node, out_file = strat["functional_preprocessed_median"] + workflow.connect( + node, out_file, flirt_reg_func_mni, "inputspec.input_brain" + ) # pass the reference files - node, out_file = strat['template_brain_for_func_preproc'] - workflow.connect(node, out_file, flirt_reg_func_mni, - 'inputspec.reference_brain') + node, out_file = strat["template_brain_for_func_preproc"] + workflow.connect( + node, out_file, flirt_reg_func_mni, "inputspec.reference_brain" + ) - if 'ANTS' in regOption: + if "ANTS" in regOption: strat = strat.fork() new_strat_list.append(strat) strat.append_name(flirt_reg_func_mni.name) - strat.update_resource_pool({ - 'registration_method': 'FSL', - 'func_longitudinal_to_mni_linear_xfm': ( - flirt_reg_func_mni, 'outputspec.linear_xfm'), - 'mni_to_func_longitudinal_linear_xfm': ( - flirt_reg_func_mni, 'outputspec.invlinear_xfm'), - 'func_longitudinal_template_to_standard': ( - flirt_reg_func_mni, 'outputspec.output_brain') - }) + strat.update_resource_pool( + { + "registration_method": "FSL", + "func_longitudinal_to_mni_linear_xfm": ( + flirt_reg_func_mni, + "outputspec.linear_xfm", + ), + "mni_to_func_longitudinal_linear_xfm": ( + flirt_reg_func_mni, + "outputspec.invlinear_xfm", + ), + "func_longitudinal_template_to_standard": ( + flirt_reg_func_mni, + "outputspec.output_brain", + ), + } + ) strat_list += new_strat_list @@ -889,51 +918,52 @@ def register_func_longitudinal_template_to_standard( except AttributeError: fsl_linear_reg_only = [0] - if 'FSL' in regOption and 0 in fsl_linear_reg_only: - + if "FSL" in regOption and 0 in fsl_linear_reg_only: for num_strat, strat in enumerate(strat_list): - - if strat.get('registration_method') == 'FSL': - + if strat.get("registration_method") == "FSL": fnirt_reg_func_mni = create_fsl_fnirt_nonlinear_reg( - 'func_mni_fnirt_register_%s_%d' % (strat_name, num_strat) + "func_mni_fnirt_register_%s_%d" % (strat_name, num_strat) ) # brain input - node, out_file = strat['functional_preprocessed_median'] - workflow.connect(node, out_file, - fnirt_reg_func_mni, 'inputspec.input_brain') + node, out_file = strat["functional_preprocessed_median"] + workflow.connect( + node, out_file, fnirt_reg_func_mni, "inputspec.input_brain" + ) # brain reference - node, out_file = strat['template_brain_for_func_preproc'] - workflow.connect(node, out_file, - fnirt_reg_func_mni, - 'inputspec.reference_brain') + node, out_file = strat["template_brain_for_func_preproc"] + workflow.connect( + node, out_file, fnirt_reg_func_mni, "inputspec.reference_brain" + ) # skull input - node, out_file = strat['motion_correct_median'] - workflow.connect(node, out_file, - fnirt_reg_func_mni, 'inputspec.input_skull') + node, out_file = strat["motion_correct_median"] + workflow.connect( + node, out_file, fnirt_reg_func_mni, "inputspec.input_skull" + ) # skull reference - node, out_file = strat['template_skull_for_func_preproc'] - workflow.connect(node, out_file, - fnirt_reg_func_mni, - 'inputspec.reference_skull') + node, out_file = strat["template_skull_for_func_preproc"] + workflow.connect( + node, out_file, fnirt_reg_func_mni, "inputspec.reference_skull" + ) - node, out_file = strat['func_longitudinal_to_mni_linear_xfm'] - workflow.connect(node, out_file, - fnirt_reg_func_mni, 'inputspec.linear_aff') + node, out_file = strat["func_longitudinal_to_mni_linear_xfm"] + workflow.connect( + node, out_file, fnirt_reg_func_mni, "inputspec.linear_aff" + ) - node, out_file = strat['template_ref_mask'] - workflow.connect(node, out_file, - fnirt_reg_func_mni, 'inputspec.ref_mask') + node, out_file = strat["template_ref_mask"] + workflow.connect( + node, out_file, fnirt_reg_func_mni, "inputspec.ref_mask" + ) # assign the FSL FNIRT config file specified in pipeline # config.yml - fnirt_reg_func_mni.inputs.inputspec.fnirt_config = \ - c.anatomical_preproc['registration_workflow']['registration'][ - 'FSL-FNIRT']['fnirt_config'] + fnirt_reg_func_mni.inputs.inputspec.fnirt_config = c.anatomical_preproc[ + "registration_workflow" + ]["registration"]["FSL-FNIRT"]["fnirt_config"] if 1 in fsl_linear_reg_only: strat = strat.fork() @@ -941,132 +971,143 @@ def register_func_longitudinal_template_to_standard( strat.append_name(fnirt_reg_func_mni.name) - strat.update_resource_pool({ - 'func_longitudinal_to_mni_nonlinear_xfm': ( - fnirt_reg_func_mni, 'outputspec.nonlinear_xfm'), - 'func_longitudinal_template_to_standard': ( - fnirt_reg_func_mni, 'outputspec.output_brain') - }, override=True) + strat.update_resource_pool( + { + "func_longitudinal_to_mni_nonlinear_xfm": ( + fnirt_reg_func_mni, + "outputspec.nonlinear_xfm", + ), + "func_longitudinal_template_to_standard": ( + fnirt_reg_func_mni, + "outputspec.output_brain", + ), + }, + override=True, + ) strat_list += new_strat_list new_strat_list = [] for num_strat, strat in enumerate(strat_list): - # or run ANTS anatomical-to-MNI registration instead - if 'ANTS' in regOption and \ - strat.get('registration_method') != 'FSL': - - ants_reg_func_mni = \ - create_wf_calculate_ants_warp( - 'func_mni_ants_register_%s_%d' % (strat_name, num_strat), - num_threads=num_ants_cores, - reg_ants_skull= - c.anatomical_preproc['registration_workflow'][ - 'reg_with_skull'] - ) + if "ANTS" in regOption and strat.get("registration_method") != "FSL": + ants_reg_func_mni = create_wf_calculate_ants_warp( + "func_mni_ants_register_%s_%d" % (strat_name, num_strat), + num_threads=num_ants_cores, + reg_ants_skull=c.anatomical_preproc["registration_workflow"][ + "reg_with_skull" + ], + ) - if c.functional_registration['2-func_registration_to_template'][ - 'ANTs_pipelines']['interpolation'] not in ['Linear', - 'BSpline', - 'LanczosWindowedSinc']: + if c.functional_registration["2-func_registration_to_template"][ + "ANTs_pipelines" + ]["interpolation"] not in ["Linear", "BSpline", "LanczosWindowedSinc"]: err_msg = 'The selected ANTS interpolation method may be in the list of values: "Linear", "BSpline", "LanczosWindowedSinc"' raise Exception(err_msg) # Input registration parameters - ants_reg_func_mni.inputs.inputspec.interp = \ - c.functional_registration['2-func_registration_to_template'][ - 'ANTs_pipelines']['interpolation'] + ants_reg_func_mni.inputs.inputspec.interp = c.functional_registration[ + "2-func_registration_to_template" + ]["ANTs_pipelines"]["interpolation"] # calculating the transform with the skullstripped is # reported to be better, but it requires very high # quality skullstripping. If skullstripping is imprecise # registration with skull is preferred - if c.anatomical_preproc['registration_workflow'][ - 'reg_with_skull']: - + if c.anatomical_preproc["registration_workflow"]["reg_with_skull"]: # get the skull-stripped anatomical from resource pool - node, out_file = strat['functional_preprocessed_median'] + node, out_file = strat["functional_preprocessed_median"] # pass the anatomical to the workflow - workflow.connect(node, out_file, - ants_reg_func_mni, 'inputspec.moving_brain') + workflow.connect( + node, out_file, ants_reg_func_mni, "inputspec.moving_brain" + ) # get the reorient skull-on anatomical from resource pool - node, out_file = strat['motion_correct_median'] + node, out_file = strat["motion_correct_median"] # pass the anatomical to the workflow - workflow.connect(node, out_file, - ants_reg_func_mni, 'inputspec.moving_skull') + workflow.connect( + node, out_file, ants_reg_func_mni, "inputspec.moving_skull" + ) # pass the reference file - node, out_file = strat['template_brain_for_func_preproc'] - workflow.connect(node, out_file, - ants_reg_func_mni, - 'inputspec.reference_brain') + node, out_file = strat["template_brain_for_func_preproc"] + workflow.connect( + node, out_file, ants_reg_func_mni, "inputspec.reference_brain" + ) # pass the reference file - node, out_file = strat['template_skull_for_func_preproc'] - workflow.connect(node, out_file, - ants_reg_func_mni, - 'inputspec.reference_skull') + node, out_file = strat["template_skull_for_func_preproc"] + workflow.connect( + node, out_file, ants_reg_func_mni, "inputspec.reference_skull" + ) else: + node, out_file = strat["functional_preprocessed_median"] - node, out_file = strat['functional_preprocessed_median'] - - workflow.connect(node, out_file, - ants_reg_func_mni, 'inputspec.moving_brain') + workflow.connect( + node, out_file, ants_reg_func_mni, "inputspec.moving_brain" + ) # pass the reference file - node, out_file = strat['template_brain_for_func_preproc'] - workflow.connect(node, out_file, - ants_reg_func_mni, - 'inputspec.reference_brain') + node, out_file = strat["template_brain_for_func_preproc"] + workflow.connect( + node, out_file, ants_reg_func_mni, "inputspec.reference_brain" + ) # pass the reference mask file - node, out_file = strat['template_brain_mask_for_func_preproc'] + node, out_file = strat["template_brain_mask_for_func_preproc"] workflow.connect( - node, out_file, - ants_reg_func_mni, 'inputspec.reference_mask' + node, out_file, ants_reg_func_mni, "inputspec.reference_mask" ) # pass the reference mask file - node, out_file = strat['functional_brain_mask'] - workflow.connect( - node, out_file, - ants_reg_func_mni, 'inputspec.moving_mask' - ) + node, out_file = strat["functional_brain_mask"] + workflow.connect(node, out_file, ants_reg_func_mni, "inputspec.moving_mask") - ants_reg_func_mni.inputs.inputspec.ants_para = \ - c.anatomical_preproc['registration_workflow']['registration'][ - 'ANTs']['T1_registration'] + ants_reg_func_mni.inputs.inputspec.ants_para = c.anatomical_preproc[ + "registration_workflow" + ]["registration"]["ANTs"]["T1_registration"] ants_reg_func_mni.inputs.inputspec.fixed_image_mask = None strat.append_name(ants_reg_func_mni.name) - strat.update_resource_pool({ - 'registration_method': 'ANTS', - 'ants_initial_xfm': ( - ants_reg_func_mni, 'outputspec.ants_initial_xfm'), - 'ants_rigid_xfm': ( - ants_reg_func_mni, 'outputspec.ants_rigid_xfm'), - 'ants_affine_xfm': ( - ants_reg_func_mni, 'outputspec.ants_affine_xfm'), - 'func_longitudinal_to_mni_nonlinear_xfm': ( - ants_reg_func_mni, 'outputspec.warp_field'), - 'mni_to_func_longitudinal_nonlinear_xfm': ( - ants_reg_func_mni, 'outputspec.inverse_warp_field'), - 'func_longitudinal_to_mni_ants_composite_xfm': ( - ants_reg_func_mni, 'outputspec.composite_transform'), - 'func_longitudinal_template_to_standard': ( - ants_reg_func_mni, 'outputspec.normalized_output_brain') - }) + strat.update_resource_pool( + { + "registration_method": "ANTS", + "ants_initial_xfm": ( + ants_reg_func_mni, + "outputspec.ants_initial_xfm", + ), + "ants_rigid_xfm": (ants_reg_func_mni, "outputspec.ants_rigid_xfm"), + "ants_affine_xfm": ( + ants_reg_func_mni, + "outputspec.ants_affine_xfm", + ), + "func_longitudinal_to_mni_nonlinear_xfm": ( + ants_reg_func_mni, + "outputspec.warp_field", + ), + "mni_to_func_longitudinal_nonlinear_xfm": ( + ants_reg_func_mni, + "outputspec.inverse_warp_field", + ), + "func_longitudinal_to_mni_ants_composite_xfm": ( + ants_reg_func_mni, + "outputspec.composite_transform", + ), + "func_longitudinal_template_to_standard": ( + ants_reg_func_mni, + "outputspec.normalized_output_brain", + ), + } + ) strat_list += new_strat_list - ''' + """ # Func -> T1 Registration (Initial Linear Reg) workflow, strat_list, diff_complete = connect_func_to_anat_init_reg(workflow, strat_list, c) @@ -1075,13 +1116,13 @@ def register_func_longitudinal_template_to_standard( # Func -> T1/EPI Template workflow, strat_list = connect_func_to_template_reg(workflow, strat_list, c) - ''' + """ return workflow, strat_list def func_longitudinal_template_wf(subject_id, strat_list, config): - ''' + """ Parameters ---------- subject_id : string @@ -1094,74 +1135,106 @@ def func_longitudinal_template_wf(subject_id, strat_list, config): Returns ------- None - ''' - - workflow_name = 'func_longitudinal_template_' + str(subject_id) + """ + workflow_name = "func_longitudinal_template_" + str(subject_id) workflow = pe.Workflow(name=workflow_name) - workflow.base_dir = config.pipeline_setup['working_directory']['path'] - workflow.config['execution'] = { - 'hash_method': 'timestamp', - 'crashdump_dir': os.path.abspath( - config.pipeline_setup['crash_directory']['path']) + workflow.base_dir = config.pipeline_setup["working_directory"]["path"] + workflow.config["execution"] = { + "hash_method": "timestamp", + "crashdump_dir": os.path.abspath( + config.pipeline_setup["crash_directory"]["path"] + ), } # strat_nodes_list = strat_list['func_default'] strat_init = Strategy() templates_for_resampling = [ - (config.resolution_for_func_preproc, - config.template_brain_only_for_func, - 'template_brain_for_func_preproc', 'resolution_for_func_preproc'), - (config.resolution_for_func_preproc, config.template_skull_for_func, - 'template_skull_for_func_preproc', 'resolution_for_func_preproc'), - (config.resolution_for_func_preproc, config.ref_mask_for_func, - 'template_ref_mask', 'resolution_for_func_preproc'), + ( + config.resolution_for_func_preproc, + config.template_brain_only_for_func, + "template_brain_for_func_preproc", + "resolution_for_func_preproc", + ), + ( + config.resolution_for_func_preproc, + config.template_skull_for_func, + "template_skull_for_func_preproc", + "resolution_for_func_preproc", + ), + ( + config.resolution_for_func_preproc, + config.ref_mask_for_func, + "template_ref_mask", + "resolution_for_func_preproc", + ), # TODO check float resolution - (config.resolution_for_func_preproc, - config.functional_registration['2-func_registration_to_template'][ - 'target_template']['EPI_template']['template_epi'], - 'template_epi', 'resolution_for_func_preproc'), - (config.resolution_for_func_derivative, - config.functional_registration['2-func_registration_to_template'][ - 'target_template']['EPI_template']['template_epi'], - 'template_epi_derivative', 'resolution_for_func_derivative'), - (config.resolution_for_func_derivative, - config.template_brain_only_for_func, - 'template_brain_for_func_derivative', 'resolution_for_func_preproc'), ( - config.resolution_for_func_derivative, config.template_skull_for_func, - 'template_skull_for_func_derivative', 'resolution_for_func_preproc'), + config.resolution_for_func_preproc, + config.functional_registration["2-func_registration_to_template"][ + "target_template" + ]["EPI_template"]["template_epi"], + "template_epi", + "resolution_for_func_preproc", + ), + ( + config.resolution_for_func_derivative, + config.functional_registration["2-func_registration_to_template"][ + "target_template" + ]["EPI_template"]["template_epi"], + "template_epi_derivative", + "resolution_for_func_derivative", + ), + ( + config.resolution_for_func_derivative, + config.template_brain_only_for_func, + "template_brain_for_func_derivative", + "resolution_for_func_preproc", + ), + ( + config.resolution_for_func_derivative, + config.template_skull_for_func, + "template_skull_for_func_derivative", + "resolution_for_func_preproc", + ), ] for resolution, template, template_name, tag in templates_for_resampling: - resampled_template = pe.Node(Function( - input_names=['resolution', 'template', 'template_name', 'tag'], - output_names=['resampled_template'], - function=resolve_resolution, - as_module=True), - name='resampled_' + template_name) + resampled_template = pe.Node( + Function( + input_names=["resolution", "template", "template_name", "tag"], + output_names=["resampled_template"], + function=resolve_resolution, + as_module=True, + ), + name="resampled_" + template_name, + ) resampled_template.inputs.resolution = resolution resampled_template.inputs.template = template resampled_template.inputs.template_name = template_name resampled_template.inputs.tag = tag - strat_init.update_resource_pool({ - template_name: (resampled_template, 'resampled_template') - }) + strat_init.update_resource_pool( + {template_name: (resampled_template, "resampled_template")} + ) merge_func_preproc_node = pe.Node( - Function(input_names=['working_directory'], - output_names=['brain_list', 'skull_list'], - function=merge_func_preproc, - as_module=True), - name='merge_func_preproc') + Function( + input_names=["working_directory"], + output_names=["brain_list", "skull_list"], + function=merge_func_preproc, + as_module=True, + ), + name="merge_func_preproc", + ) - merge_func_preproc_node.inputs.working_directory = \ - config.pipeline_setup['working_directory']['path'] + merge_func_preproc_node.inputs.working_directory = config.pipeline_setup[ + "working_directory" + ]["path"] template_node = subject_specific_template( - workflow_name='subject_specific_func_template_' + subject_id + workflow_name="subject_specific_func_template_" + subject_id ) template_node.inputs.set( @@ -1173,20 +1246,16 @@ def func_longitudinal_template_wf(subject_id, strat_list, config): thread_pool=config.longitudinal_template_thread_pool, ) - workflow.connect(merge_func_preproc_node, 'brain_list', - template_node, 'input_brain_list') + workflow.connect( + merge_func_preproc_node, "brain_list", template_node, "input_brain_list" + ) - workflow.connect(merge_func_preproc_node, 'skull_list', - template_node, 'input_skull_list') + workflow.connect( + merge_func_preproc_node, "skull_list", template_node, "input_skull_list" + ) workflow, strat_list = register_func_longitudinal_template_to_standard( - template_node, - config, - workflow, - strat_init, - 'default' + template_node, config, workflow, strat_init, "default" ) workflow.run() - - return diff --git a/CPAC/median_angle/median_angle.py b/CPAC/median_angle/median_angle.py index 2e6426501e..450696fcb7 100644 --- a/CPAC/median_angle/median_angle.py +++ b/CPAC/median_angle/median_angle.py @@ -1,36 +1,37 @@ -from CPAC.pipeline import nipype_pipeline_engine as pe import nipype.interfaces.utility as util +from CPAC.pipeline import nipype_pipeline_engine as pe def median_angle_correct(target_angle_deg, realigned_file): """ Performs median angle correction on fMRI data. Median angle correction algorithm based on [1]_. - + Parameters ---------- target_angle_deg : float Target median angle to adjust the time-series data. realigned_file : string Path of a realigned nifti file. - + Returns ------- corrected_file : string Path of corrected file (nifti file). angles_file : string - Path of numpy file (.npy file) containing the angles (in radians) of all voxels with + Path of numpy file (.npy file) containing the angles (in radians) of all voxels with the 5 largest principal components. - + References ---------- .. [1] H. He and T. T. Liu, "A geometric view of global signal confounds in resting-state functional MRI," NeuroImage, Sep. 2011. - + """ - import numpy as np - import nibabel as nb import os + + import numpy as np + import nibabel as nib from scipy.stats.stats import pearsonr def shiftCols(pc, A, dtheta): @@ -41,17 +42,15 @@ def shiftCols(pc, A, dtheta): theta_new = theta + dtheta x /= np.tile(np.sqrt((x * x).sum(0)), (x.shape[0], 1)) - v_new = np.dot(pc[:, np.newaxis],\ - np.cos(theta_new)[np.newaxis, :]) + (np.sin(theta_new) * x) - - return v_new + return np.dot(pc[:, np.newaxis], np.cos(theta_new)[np.newaxis, :]) + ( + np.sin(theta_new) * x + ) def writeToFile(data, nii, fname): - img_whole_y = nb.Nifti1Image(data,\ - header=nii.header, affine=nii.affine) + img_whole_y = nib.Nifti1Image(data, header=nii.header, affine=nii.affine) img_whole_y.to_filename(fname) - nii = nb.load(realigned_file) + nii = nib.load(realigned_file) data = nii.get_fdata().astype(np.float64) mask = (data != 0).sum(-1) != 0 @@ -63,21 +62,21 @@ def writeToFile(data, nii, fname): U, S, Vh = np.linalg.svd(Yn, full_matrices=False) G = Yc.mean(1) - #Correlation of Global and U + # Correlation of Global and U corr_gu = pearsonr(G, U[:, 0]) PC1 = U[:, 0] if corr_gu[0] >= 0 else -U[:, 0] median_angle = np.median(np.arccos(np.dot(PC1.T, Yn))) angle_shift = (np.pi / 180) * target_angle_deg - median_angle - if(angle_shift > 0): - #Shifting all vectors + if angle_shift > 0: + # Shifting all vectors Ynf = shiftCols(PC1, Yn, angle_shift) else: #'Median Angle >= Target Angle, skipping correction' Ynf = Yn - corrected_file = os.path.join(os.getcwd(), 'median_angle_corrected.nii.gz') - angles_file = os.path.join(os.getcwd(), 'angles_U5_Yn.npy') + corrected_file = os.path.join(os.getcwd(), "median_angle_corrected.nii.gz") + angles_file = os.path.join(os.getcwd(), "angles_U5_Yn.npy") angles_U5_Yn = np.arccos(np.dot(U[:, 0:5].T, Yn)) np.save(angles_file, angles_U5_Yn) @@ -88,115 +87,115 @@ def writeToFile(data, nii, fname): return corrected_file, angles_file + def calc_median_angle_params(subject): """ - Calculates median angle parameters of a subject - + Calculates median angle parameters of a subject. + Parameters ---------- subject : string Path of a subject's nifti file. - + Returns ------- mean_bold : float - Mean bold amplitude of a subject. + Mean bold amplitude of a subject. median_angle : float Median angle of a subject. """ import numpy as np - import nibabel as nb - - data = nb.load(subject).get_fdata().astype('float64') + import nibabel as nib + + data = nib.load(subject).get_fdata().astype("float64") mask = (data != 0).sum(-1) != 0 - + Y = data[mask].T - + Yc = Y - np.tile(Y.mean(0), (Y.shape[0], 1)) - Yn = Yc/np.tile(np.sqrt((Yc*Yc).sum(0)), (Yc.shape[0], 1)) - U,S,Vh = np.linalg.svd(Yn, full_matrices=False) - - glb = (Yn/np.tile(Yn.std(0), (Y.shape[0], 1))).mean(1) + Yn = Yc / np.tile(np.sqrt((Yc * Yc).sum(0)), (Yc.shape[0], 1)) + U, S, Vh = np.linalg.svd(Yn, full_matrices=False) + + glb = (Yn / np.tile(Yn.std(0), (Y.shape[0], 1))).mean(1) from scipy.stats.stats import pearsonr - corr = pearsonr(U[:,0],glb) - PC1 = U[:,0] if corr[0] >= 0 else -U[:,0] + corr = pearsonr(U[:, 0], glb) + + PC1 = U[:, 0] if corr[0] >= 0 else -U[:, 0] median_angle = np.median(np.arccos(np.dot(PC1.T, Yn))) - median_angle *= 180.0/np.pi + median_angle *= 180.0 / np.pi Yp = Yc mean_bold = Yp.std(0).mean() - return mean_bold, median_angle + def calc_target_angle(mean_bolds, median_angles): """ Calculates a target angle based on median angle parameters of the group. - + Parameters ---------- mean_bolds : list (floats) List of mean bold amplitudes of the group median_angles : list (floats) List of median angles of the group - + Returns ------- target_angle : float Calculated target angle of the given group """ import numpy as np - - if(len(mean_bolds) != len(median_angles)): - raise ValueError('Length of parameters do not match') + + if len(mean_bolds) != len(median_angles): + raise ValueError("Length of parameters do not match") X = np.ones((len(mean_bolds), 2)) - X[:,1] = np.array(mean_bolds) - + X[:, 1] = np.array(mean_bolds) + Y = np.array(median_angles) - + B = np.linalg.inv(X.T.dot(X)).dot(X.T).dot(Y) - target_bold = X[:,1].min() - target_angle = target_bold*B[1] + B[0] - - return target_angle + target_bold = X[:, 1].min() + return target_bold * B[1] + B[0] -def create_median_angle_correction(name='median_angle_correction'): + +def create_median_angle_correction(name="median_angle_correction"): """ - Median Angle Correction - + Median Angle Correction. + Parameters ---------- name : string, optional Name of the workflow. - + Returns ------- median_angle_correction : nipype.pipeline.engine.Workflow Median Angle Correction workflow. - + Notes ----- - Workflow Inputs:: - + inputspec.subject : string (nifti file) Realigned nifti file of a subject inputspec.target_angle : integer Target angle in degrees to correct the median angle to - + Workflow Outputs:: - + outputspec.subject : string (nifti file) Median angle corrected nifti file of the given subject outputspec.pc_angles : string (.npy file) - Numpy file (.npy file) containing the angles (in radians) of all voxels with + Numpy file (.npy file) containing the angles (in radians) of all voxels with the 5 largest principal components. Median Angle Correction Procedure: - + 1. Compute the median angle with respect to the first principal component of the subject 2. Shift the angle of every voxel so that the new median angle equals the target angle @@ -209,77 +208,75 @@ def create_median_angle_correction(name='median_angle_correction'): ) Workflow Graph: - + .. image:: ../../images/generated/median_angle_correction.png :width: 500 - + Detailed Workflow Graph: - + .. image:: ../../images/generated/median_angle_correction_detailed.png - :width: 500 + :width: 500 """ median_angle_correction = pe.Workflow(name=name) - - inputspec = pe.Node(util.IdentityInterface(fields=['subject', - 'target_angle']), - name='inputspec') - outputspec = pe.Node(util.IdentityInterface(fields=['subject', - 'pc_angles']), - name='outputspec') - - mac = pe.Node(util.Function(input_names=['target_angle_deg', - 'realigned_file'], - output_names=['corrected_file', - 'angles_file'], - function=median_angle_correct), - name='median_angle_correct') - - median_angle_correction.connect(inputspec, 'subject', - mac, 'realigned_file') - median_angle_correction.connect(inputspec, 'target_angle', - mac, 'target_angle_deg') - median_angle_correction.connect(mac, 'corrected_file', - outputspec, 'subject') - median_angle_correction.connect(mac, 'angles_file', - outputspec, 'pc_angles') - + + inputspec = pe.Node( + util.IdentityInterface(fields=["subject", "target_angle"]), name="inputspec" + ) + outputspec = pe.Node( + util.IdentityInterface(fields=["subject", "pc_angles"]), name="outputspec" + ) + + mac = pe.Node( + util.Function( + input_names=["target_angle_deg", "realigned_file"], + output_names=["corrected_file", "angles_file"], + function=median_angle_correct, + ), + name="median_angle_correct", + ) + + median_angle_correction.connect(inputspec, "subject", mac, "realigned_file") + median_angle_correction.connect(inputspec, "target_angle", mac, "target_angle_deg") + median_angle_correction.connect(mac, "corrected_file", outputspec, "subject") + median_angle_correction.connect(mac, "angles_file", outputspec, "pc_angles") + return median_angle_correction -def create_target_angle(name='target_angle'): + +def create_target_angle(name="target_angle"): """ - Target Angle Calculation - + Target Angle Calculation. + Parameters ---------- name : string, optional Name of the workflow. - + Returns ------- target_angle : nipype.pipeline.engine.Workflow Target angle workflow. - + Notes ----- - Workflow Inputs:: - + inputspec.subjects : list (nifti files) List of subject paths. - + Workflow Outputs:: - + outputspec.target_angle : float Target angle over the provided group of subjects. - + Target Angle procedure: - + 1. Compute the median angle and mean bold amplitude of each subject in the group. 2. Fit a linear model with median angle as the dependent variable. - 3. Calculate the corresponding median_angle on the fitted model for the subject + 3. Calculate the corresponding median_angle on the fitted model for the subject with the smallest mean bold amplitude of the group. - + .. exec:: from CPAC.median_angle import create_target_angle wf = create_target_angle() @@ -289,43 +286,45 @@ def create_target_angle(name='target_angle'): ) Workflow Graph: - + .. image:: ../../images/generated/target_angle.png :width: 500 - + Detailed Workflow Graph: - + .. image:: ../../images/generated/target_angle_detailed.png :width: 500 - + """ target_angle = pe.Workflow(name=name) - - inputspec = pe.Node(util.IdentityInterface(fields=['subjects']), - name='inputspec') - outputspec = pe.Node(util.IdentityInterface(fields=['target_angle']), - name='outputspec') - - cmap = pe.MapNode(util.Function(input_names=['subject'], - output_names=['mean_bold', - 'median_angle'], - function=calc_median_angle_params), - name='median_angle_params', - iterfield=['subject']) - - cta = pe.Node(util.Function(input_names=['mean_bolds', - 'median_angles'], - output_names=['target_angle'], - function=calc_target_angle), - name='target_angle') - - target_angle.connect(inputspec, 'subjects', - cmap, 'subject') - target_angle.connect(cmap, 'mean_bold', - cta, 'mean_bolds') - target_angle.connect(cmap, 'median_angle', - cta, 'median_angles') - target_angle.connect(cta, 'target_angle', - outputspec, 'target_angle') - + + inputspec = pe.Node(util.IdentityInterface(fields=["subjects"]), name="inputspec") + outputspec = pe.Node( + util.IdentityInterface(fields=["target_angle"]), name="outputspec" + ) + + cmap = pe.MapNode( + util.Function( + input_names=["subject"], + output_names=["mean_bold", "median_angle"], + function=calc_median_angle_params, + ), + name="median_angle_params", + iterfield=["subject"], + ) + + cta = pe.Node( + util.Function( + input_names=["mean_bolds", "median_angles"], + output_names=["target_angle"], + function=calc_target_angle, + ), + name="target_angle", + ) + + target_angle.connect(inputspec, "subjects", cmap, "subject") + target_angle.connect(cmap, "mean_bold", cta, "mean_bolds") + target_angle.connect(cmap, "median_angle", cta, "median_angles") + target_angle.connect(cta, "target_angle", outputspec, "target_angle") + return target_angle diff --git a/CPAC/network_centrality/__init__.py b/CPAC/network_centrality/__init__.py index 5507407e4b..35b3486b4b 100644 --- a/CPAC/network_centrality/__init__.py +++ b/CPAC/network_centrality/__init__.py @@ -16,4 +16,4 @@ # License along with C-PAC. If not, see . from .utils import convert_pvalue_to_r, create_merge_node -__all__ = ['convert_pvalue_to_r', 'create_merge_node'] +__all__ = ["convert_pvalue_to_r", "create_merge_node"] diff --git a/CPAC/network_centrality/network_centrality.py b/CPAC/network_centrality/network_centrality.py index ae6b6778af..6fdbab18c4 100644 --- a/CPAC/network_centrality/network_centrality.py +++ b/CPAC/network_centrality/network_centrality.py @@ -16,8 +16,10 @@ # License along with C-PAC. If not, see . from pathlib import Path from typing import Optional, Union -from nipype.interfaces.afni.preprocess import DegreeCentrality, LFCD + +from nipype.interfaces.afni.preprocess import LFCD, DegreeCentrality from nipype.pipeline.engine import Workflow + from CPAC.network_centrality.utils import ThresholdOptionError from CPAC.pipeline.schema import valid_options from CPAC.utils.docs import docstring_parameter @@ -25,16 +27,21 @@ from CPAC.utils.typing import LIST -@docstring_parameter(m_options=valid_options['centrality']['method_options'], - t_options=valid_options['centrality'][ - 'threshold_options'], - w_options=valid_options['centrality']['weight_options']) -def create_centrality_wf(wf_name: str, method_option: str, - weight_options: LIST[str], threshold_option: str, - threshold: float, num_threads: Optional[int] = 1, - memory_gb: Optional[float] = 1.0, - base_dir: Optional[Union[Path, str]] = None - ) -> Workflow: +@docstring_parameter( + m_options=valid_options["centrality"]["method_options"], + t_options=valid_options["centrality"]["threshold_options"], + w_options=valid_options["centrality"]["weight_options"], +) +def create_centrality_wf( + wf_name: str, + method_option: str, + weight_options: LIST[str], + threshold_option: str, + threshold: float, + num_threads: Optional[int] = 1, + memory_gb: Optional[float] = 1.0, + base_dir: Optional[Union[Path, str]] = None, +) -> Workflow: """ Function to create the afni-based centrality workflow. @@ -87,123 +94,154 @@ def create_centrality_wf(wf_name: str, method_option: str, outputspec.outfile_list : list of strings list of paths to output files (binarized and weighted) """ # pylint: disable=line-too-long - from CPAC.pipeline import nipype_pipeline_engine as pe from nipype.interfaces import utility as util + from CPAC.network_centrality import utils + from CPAC.pipeline import nipype_pipeline_engine as pe from CPAC.utils.interfaces.function import Function test_thresh = threshold - if threshold_option == 'Sparsity threshold': + if threshold_option == "Sparsity threshold": test_thresh = threshold / 100.0 - method_option, threshold_option = \ - utils.check_centrality_params(method_option, threshold_option, - test_thresh) + method_option, threshold_option = utils.check_centrality_params( + method_option, threshold_option, test_thresh + ) # Eigenvector centrality and AFNI ≥ 21.1.1? - ecm_gte_21_1_01 = ((method_option == 'eigenvector_centrality') and - AFNI_GTE_21_1_1) - out_names = tuple(f'{method_option}_{x}' for x in weight_options) + ecm_gte_21_1_01 = (method_option == "eigenvector_centrality") and AFNI_GTE_21_1_1 + out_names = tuple(f"{method_option}_{x}" for x in weight_options) if base_dir is None: centrality_wf = pe.Workflow(name=wf_name) else: centrality_wf = pe.Workflow(name=wf_name, base_dir=base_dir) - input_node = pe.Node(util.IdentityInterface(fields=['in_file', - 'template', - 'threshold']), - name='inputspec') + input_node = pe.Node( + util.IdentityInterface(fields=["in_file", "template", "threshold"]), + name="inputspec", + ) input_node.inputs.threshold = threshold - output_node = pe.Node(util.IdentityInterface(fields=['outfile_list']), - name='outputspec') + output_node = pe.Node( + util.IdentityInterface(fields=["outfile_list"]), name="outputspec" + ) # Degree centrality - if method_option == 'degree_centrality': - afni_centrality_node = pe.Node(DegreeCentrality(environ={ - 'OMP_NUM_THREADS': str(num_threads) - }), name='afni_centrality', mem_gb=memory_gb) - afni_centrality_node.inputs.out_file = \ - 'degree_centrality_merged.nii.gz' + if method_option == "degree_centrality": + afni_centrality_node = pe.Node( + DegreeCentrality(environ={"OMP_NUM_THREADS": str(num_threads)}), + name="afni_centrality", + mem_gb=memory_gb, + ) + afni_centrality_node.inputs.out_file = "degree_centrality_merged.nii.gz" # Eigenvector centrality - elif method_option == 'eigenvector_centrality': + elif method_option == "eigenvector_centrality": if ecm_gte_21_1_01: - afni_centrality_node = pe.MapNode(ECM(environ={ - 'OMP_NUM_THREADS': str(num_threads) - }), name='afni_centrality', mem_gb=memory_gb, - iterfield=['do_binary', 'out_file']) + afni_centrality_node = pe.MapNode( + ECM(environ={"OMP_NUM_THREADS": str(num_threads)}), + name="afni_centrality", + mem_gb=memory_gb, + iterfield=["do_binary", "out_file"], + ) afni_centrality_node.inputs.out_file = [ - f'eigenvector_centrality_{w_option}.nii.gz' for - w_option in weight_options] + f"eigenvector_centrality_{w_option}.nii.gz" + for w_option in weight_options + ] afni_centrality_node.inputs.do_binary = [ - w_option == 'Binarized' for w_option in weight_options] - centrality_wf.connect(afni_centrality_node, 'out_file', - output_node, 'outfile_list') + w_option == "Binarized" for w_option in weight_options + ] + centrality_wf.connect( + afni_centrality_node, "out_file", output_node, "outfile_list" + ) else: - afni_centrality_node = pe.Node(ECM(environ={ - 'OMP_NUM_THREADS': str(num_threads) - }), name='afni_centrality', mem_gb=memory_gb) - afni_centrality_node.inputs.out_file = \ - 'eigenvector_centrality_merged.nii.gz' + afni_centrality_node = pe.Node( + ECM(environ={"OMP_NUM_THREADS": str(num_threads)}), + name="afni_centrality", + mem_gb=memory_gb, + ) + afni_centrality_node.inputs.out_file = ( + "eigenvector_centrality_merged.nii.gz" + ) afni_centrality_node.inputs.memory = memory_gb # 3dECM input only # lFCD - elif method_option == 'local_functional_connectivity_density': - afni_centrality_node = pe.Node(LFCD(environ={ - 'OMP_NUM_THREADS': str(num_threads) - }), name='afni_centrality', mem_gb=memory_gb) - afni_centrality_node.inputs.out_file = 'lfcd_merged.nii.gz' + elif method_option == "local_functional_connectivity_density": + afni_centrality_node = pe.Node( + LFCD(environ={"OMP_NUM_THREADS": str(num_threads)}), + name="afni_centrality", + mem_gb=memory_gb, + ) + afni_centrality_node.inputs.out_file = "lfcd_merged.nii.gz" if not ecm_gte_21_1_01: # Need to separate sub-briks except for 3dECM if AFNI > 21.1.01 - sep_subbriks_node = \ - pe.Node(Function(input_names=['nifti_file', 'out_names'], - output_names=['output_niftis'], - function=utils.sep_nifti_subbriks), - name='sep_nifti_subbriks') + sep_subbriks_node = pe.Node( + Function( + input_names=["nifti_file", "out_names"], + output_names=["output_niftis"], + function=utils.sep_nifti_subbriks, + ), + name="sep_nifti_subbriks", + ) sep_subbriks_node.inputs.out_names = out_names - centrality_wf.connect([(afni_centrality_node, sep_subbriks_node, - [('out_file', 'nifti_file')]), - (sep_subbriks_node, output_node, - [('output_niftis', 'outfile_list')])]) + centrality_wf.connect( + [ + (afni_centrality_node, sep_subbriks_node, [("out_file", "nifti_file")]), + (sep_subbriks_node, output_node, [("output_niftis", "outfile_list")]), + ] + ) afni_centrality_node.interface.num_threads = num_threads # Connect input image and mask template - centrality_wf.connect([(input_node, afni_centrality_node, - [('in_file', 'in_file'), - ('template', 'mask')])]) + centrality_wf.connect( + [ + ( + input_node, + afni_centrality_node, + [("in_file", "in_file"), ("template", "mask")], + ) + ] + ) # If we're doing significance thresholding, convert to correlation - if threshold_option == 'Significance threshold': + if threshold_option == "Significance threshold": # Check and (possibly) conver threshold convert_thr_node = pe.Node( - Function(input_names=['datafile', - 'p_value', - 'two_tailed'], - output_names=['rvalue_threshold'], - function=utils.convert_pvalue_to_r), - name='convert_threshold') + Function( + input_names=["datafile", "p_value", "two_tailed"], + output_names=["rvalue_threshold"], + function=utils.convert_pvalue_to_r, + ), + name="convert_threshold", + ) # Wire workflow to connect in conversion node - centrality_wf.connect([(input_node, convert_thr_node, - [('in_file', 'datafile'), - ('threshold', 'p_value')]), - (convert_thr_node, afni_centrality_node, - [('rvalue_threshold', 'thresh')])]) + centrality_wf.connect( + [ + ( + input_node, + convert_thr_node, + [("in_file", "datafile"), ("threshold", "p_value")], + ), + ( + convert_thr_node, + afni_centrality_node, + [("rvalue_threshold", "thresh")], + ), + ] + ) # Sparsity thresholding - elif threshold_option == 'Sparsity threshold': + elif threshold_option == "Sparsity threshold": # Check to make sure it's not lFCD - if method_option == 'local_functional_connectivity_density': + if method_option == "local_functional_connectivity_density": raise ThresholdOptionError(threshold_option, method_option) # Otherwise, connect threshold to sparsity input - centrality_wf.connect(input_node, 'threshold', - afni_centrality_node, 'sparsity') + centrality_wf.connect(input_node, "threshold", afni_centrality_node, "sparsity") # Correlation thresholding - elif threshold_option == 'Correlation threshold': - centrality_wf.connect(input_node, 'threshold', - afni_centrality_node, 'thresh') + elif threshold_option == "Correlation threshold": + centrality_wf.connect(input_node, "threshold", afni_centrality_node, "thresh") return centrality_wf diff --git a/CPAC/network_centrality/pipeline.py b/CPAC/network_centrality/pipeline.py index 554b4b882c..4c356f25ea 100644 --- a/CPAC/network_centrality/pipeline.py +++ b/CPAC/network_centrality/pipeline.py @@ -14,23 +14,28 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -from CPAC.pipeline import nipype_pipeline_engine as pe -from nipype.interfaces import fsl - from nipype import logging +from nipype.interfaces import fsl -from CPAC.pipeline.nodeblock import nodeblock from CPAC.network_centrality.network_centrality import create_centrality_wf -from CPAC.network_centrality.utils import check_centrality_params, \ - create_merge_node +from CPAC.network_centrality.utils import check_centrality_params, create_merge_node +from CPAC.pipeline import nipype_pipeline_engine as pe +from CPAC.pipeline.nodeblock import nodeblock from CPAC.pipeline.schema import valid_options -logger = logging.getLogger('nipype.workflow') +logger = logging.getLogger("nipype.workflow") -def connect_centrality_workflow(workflow, c, resample_functional_to_template, - template_node, template_out, merge_node, - method_option, pipe_num): +def connect_centrality_workflow( + workflow, + c, + resample_functional_to_template, + template_node, + template_out, + merge_node, + method_option, + pipe_num, +): """ .. exec:: @@ -84,56 +89,64 @@ def connect_centrality_workflow(workflow, c, resample_functional_to_template, :width: 500 """ # Set method_options variables - if method_option == 'degree_centrality': - out_list = 'deg_list' - elif method_option == 'eigenvector_centrality': - out_list = 'eig_list' - elif method_option == 'local_functional_connectivity_density': - out_list = 'lfcd_list' + if method_option == "degree_centrality": + out_list = "deg_list" + elif method_option == "eigenvector_centrality": + out_list = "eig_list" + elif method_option == "local_functional_connectivity_density": + out_list = "lfcd_list" threshold_option = c.network_centrality[method_option][ - 'correlation_threshold_option' + "correlation_threshold_option" ] - threshold = c.network_centrality[method_option]['correlation_threshold'] + threshold = c.network_centrality[method_option]["correlation_threshold"] # Init workflow name and resource limits - wf_name = f'afni_centrality_{method_option}_{pipe_num}' - num_threads = c.pipeline_setup['system_config'][ - 'max_cores_per_participant' - ] - memory = c.network_centrality['memory_allocation'] + wf_name = f"afni_centrality_{method_option}_{pipe_num}" + num_threads = c.pipeline_setup["system_config"]["max_cores_per_participant"] + memory = c.network_centrality["memory_allocation"] # Format method and threshold options properly and check for # errors - method_option, threshold_option = check_centrality_params(method_option, - threshold_option, - threshold) + method_option, threshold_option = check_centrality_params( + method_option, threshold_option, threshold + ) # Change sparsity thresholding to % to work with afni - if threshold_option == 'Sparsity threshold': + if threshold_option == "Sparsity threshold": threshold = threshold * 100 - afni_centrality_wf = \ - create_centrality_wf(wf_name, method_option, - c.network_centrality[method_option][ - 'weight_options'], threshold_option, - threshold, num_threads, memory) + afni_centrality_wf = create_centrality_wf( + wf_name, + method_option, + c.network_centrality[method_option]["weight_options"], + threshold_option, + threshold, + num_threads, + memory, + ) - workflow.connect(resample_functional_to_template, 'out_file', - afni_centrality_wf, 'inputspec.in_file') + workflow.connect( + resample_functional_to_template, + "out_file", + afni_centrality_wf, + "inputspec.in_file", + ) - workflow.connect(template_node, template_out, - afni_centrality_wf, 'inputspec.template') + workflow.connect( + template_node, template_out, afni_centrality_wf, "inputspec.template" + ) - if 'degree' in method_option: - out_list = 'deg_list' - elif 'eigen' in method_option: - out_list = 'eig_list' - elif 'lfcd' in method_option: - out_list = 'lfcd_list' + if "degree" in method_option: + out_list = "deg_list" + elif "eigen" in method_option: + out_list = "eig_list" + elif "lfcd" in method_option: + out_list = "lfcd_list" - workflow.connect(afni_centrality_wf, 'outputspec.outfile_list', - merge_node, out_list) + workflow.connect( + afni_centrality_wf, "outputspec.outfile_list", merge_node, out_list + ) @nodeblock( @@ -154,61 +167,60 @@ def connect_centrality_workflow(workflow, c, resample_functional_to_template, }, ) def network_centrality(wf, cfg, strat_pool, pipe_num, opt=None): - '''Run Network Centrality. - ''' - + """Run Network Centrality.""" # Resample the functional mni to the centrality mask resolution resample_functional_to_template = pe.Node( interface=fsl.FLIRT(), - name=f'resample_functional_to_template_{pipe_num}', - mem_gb=4.0) + name=f"resample_functional_to_template_{pipe_num}", + mem_gb=4.0, + ) resample_functional_to_template.inputs.set( - interp='trilinear', - in_matrix_file=cfg.registration_workflows['functional_registration'][ - 'func_registration_to_template']['FNIRT_pipelines'][ - 'identity_matrix'], - apply_xfm=True + interp="trilinear", + in_matrix_file=cfg.registration_workflows["functional_registration"][ + "func_registration_to_template" + ]["FNIRT_pipelines"]["identity_matrix"], + apply_xfm=True, ) node, out = strat_pool.get_data("space-template_desc-preproc_bold") - wf.connect(node, out, resample_functional_to_template, 'in_file') + wf.connect(node, out, resample_functional_to_template, "in_file") node, out = strat_pool.get_data("template-specification-file") - wf.connect(node, out, resample_functional_to_template, 'reference') + wf.connect(node, out, resample_functional_to_template, "reference") merge_node = create_merge_node(pipe_num) outputs = {} - for option in valid_options['centrality']['method_options']: - if cfg.network_centrality[option]['weight_options']: - connect_centrality_workflow(wf, cfg, - resample_functional_to_template, - node, out, merge_node, - option, pipe_num) - for weight in cfg.network_centrality[option]['weight_options']: + for option in valid_options["centrality"]["method_options"]: + if cfg.network_centrality[option]["weight_options"]: + connect_centrality_workflow( + wf, + cfg, + resample_functional_to_template, + node, + out, + merge_node, + option, + pipe_num, + ) + for weight in cfg.network_centrality[option]["weight_options"]: _option = option.lower() _weight = weight.lower() - if 'degree' in _option: - if 'weight' in _weight: - outputs['space-template_dcw'] = (merge_node, - 'degree_weighted') - elif 'binarize' in _weight: - outputs['space-template_dcb'] = (merge_node, - 'degree_binarized') - elif 'eigen' in _option: - if 'weight' in _weight: - outputs['space-template_ecw'] = (merge_node, - 'eigen_weighted') - elif 'binarize' in _weight: - outputs['space-template_ecb'] = (merge_node, - 'eigen_binarized') - elif 'lfcd' in _option or 'local_functional' in _option: - if 'weight' in _weight: - outputs['space-template_lfcdw'] = (merge_node, - 'lfcd_weighted') - elif 'binarize' in _weight: - outputs['space-template_lfcdb'] = (merge_node, - 'lfcd_binarized') + if "degree" in _option: + if "weight" in _weight: + outputs["space-template_dcw"] = (merge_node, "degree_weighted") + elif "binarize" in _weight: + outputs["space-template_dcb"] = (merge_node, "degree_binarized") + elif "eigen" in _option: + if "weight" in _weight: + outputs["space-template_ecw"] = (merge_node, "eigen_weighted") + elif "binarize" in _weight: + outputs["space-template_ecb"] = (merge_node, "eigen_binarized") + elif "lfcd" in _option or "local_functional" in _option: + if "weight" in _weight: + outputs["space-template_lfcdw"] = (merge_node, "lfcd_weighted") + elif "binarize" in _weight: + outputs["space-template_lfcdb"] = (merge_node, "lfcd_binarized") return (wf, outputs) diff --git a/CPAC/network_centrality/tests/test_network_centrality.py b/CPAC/network_centrality/tests/test_network_centrality.py index 5a8c01d759..2e9bec4b54 100644 --- a/CPAC/network_centrality/tests/test_network_centrality.py +++ b/CPAC/network_centrality/tests/test_network_centrality.py @@ -16,49 +16,79 @@ # License along with C-PAC. If not, see . from itertools import combinations from pathlib import Path + import pytest + from CPAC.network_centrality.network_centrality import create_centrality_wf from CPAC.pipeline.schema import valid_options from CPAC.utils.interfaces.afni import AFNI_SEMVER from CPAC.utils.typing import LIST -_DATA_DIR = Path(__file__).parent / 'data' +_DATA_DIR = Path(__file__).parent / "data" """Path to test data directory""" -@pytest.mark.parametrize('method_option', - valid_options['centrality']['method_options']) -@pytest.mark.parametrize('weight_options', - [*combinations(valid_options['centrality']['weight_options'], 1), - *combinations(valid_options['centrality']['weight_options'], 2)]) -@pytest.mark.parametrize('threshold_option', - valid_options['centrality']['threshold_options']) -@pytest.mark.parametrize('threshold', [0.001, 0.6]) -@pytest.mark.skipif(AFNI_SEMVER == '0.0.0', reason='AFNI not installed') -def test_create_centrality_wf(method_option: str, weight_options: LIST[str], - threshold_option: str, - threshold: float, tmpdir: Path) -> None: - '''Integration test of - ~CPAC.network_centrality.network_centrality.create_centrality_wf''' - wf_name = (f'test_{method_option[0]}' - f'{"".join([_[0] for _ in weight_options])}' - f'{threshold_option[0]}{threshold}'.replace('.', 'p')) - if (method_option == 'local_functional_connectivity_density' and - threshold_option == 'Sparsity threshold' +@pytest.mark.parametrize("method_option", valid_options["centrality"]["method_options"]) +@pytest.mark.parametrize( + "weight_options", + [ + *combinations(valid_options["centrality"]["weight_options"], 1), + *combinations(valid_options["centrality"]["weight_options"], 2), + ], +) +@pytest.mark.parametrize( + "threshold_option", valid_options["centrality"]["threshold_options"] +) +@pytest.mark.parametrize("threshold", [0.001, 0.6]) +@pytest.mark.skipif(AFNI_SEMVER == "0.0.0", reason="AFNI not installed") +def test_create_centrality_wf( + method_option: str, + weight_options: LIST[str], + threshold_option: str, + threshold: float, + tmpdir: Path, +) -> None: + """Integration test of + ~CPAC.network_centrality.network_centrality.create_centrality_wf. + """ + wf_name = ( + f'test_{method_option[0]}' + f'{"".join([_[0] for _ in weight_options])}' + f'{threshold_option[0]}{threshold}'.replace(".", "p") + ) + if ( + method_option == "local_functional_connectivity_density" + and threshold_option == "Sparsity threshold" ): with pytest.raises(ValueError): - centrality_wf = create_centrality_wf(wf_name, method_option, - weight_options, threshold_option, threshold, base_dir=tmpdir) + centrality_wf = create_centrality_wf( + wf_name, + method_option, + weight_options, + threshold_option, + threshold, + base_dir=tmpdir, + ) return - if method_option == 'eigenvector_centrality': - centrality_wf = create_centrality_wf(wf_name, method_option, - weight_options, threshold_option, threshold, memory_gb=3.0, - base_dir=tmpdir) + if method_option == "eigenvector_centrality": + centrality_wf = create_centrality_wf( + wf_name, + method_option, + weight_options, + threshold_option, + threshold, + memory_gb=3.0, + base_dir=tmpdir, + ) else: - centrality_wf = create_centrality_wf(wf_name, method_option, - weight_options, threshold_option, threshold, base_dir=tmpdir) - centrality_wf.inputs.inputspec.in_file = (_DATA_DIR / - 'in_file.nii.gz').absolute() - centrality_wf.inputs.inputspec.template = (_DATA_DIR / - 'template.nii.gz').absolute() + centrality_wf = create_centrality_wf( + wf_name, + method_option, + weight_options, + threshold_option, + threshold, + base_dir=tmpdir, + ) + centrality_wf.inputs.inputspec.in_file = (_DATA_DIR / "in_file.nii.gz").absolute() + centrality_wf.inputs.inputspec.template = (_DATA_DIR / "template.nii.gz").absolute() centrality_wf.run() diff --git a/CPAC/network_centrality/utils.py b/CPAC/network_centrality/utils.py index 2c2facafbc..baef80bb84 100644 --- a/CPAC/network_centrality/utils.py +++ b/CPAC/network_centrality/utils.py @@ -17,7 +17,9 @@ import os from pathlib import Path from typing import Optional, Union + import nibabel as nib + from CPAC.pipeline.nipype_pipeline_engine import Node from CPAC.pipeline.schema import valid_options from CPAC.utils.docs import docstring_parameter @@ -26,8 +28,8 @@ def convert_pvalue_to_r(datafile, p_value, two_tailed=False): - ''' - Method to calculate correlation threshold from p_value + """ + Method to calculate correlation threshold from p_value. Parameters ---------- @@ -43,10 +45,9 @@ def convert_pvalue_to_r(datafile, p_value, two_tailed=False): ------- r_value : float correlation threshold value - ''' - - import nibabel as nb + """ import numpy as np + import nibabel as nb import scipy.stats # Get two-tailed distribution @@ -58,7 +59,7 @@ def convert_pvalue_to_r(datafile, p_value, two_tailed=False): t_pts = img.shape[-1] # N-2 degrees of freedom with Pearson correlation (two sample means) - deg_freedom = t_pts-2 + deg_freedom = t_pts - 2 # Inverse Survival Function (Inverse of SF) # Note: survival function (SF) is also known as the complementary @@ -67,16 +68,17 @@ def convert_pvalue_to_r(datafile, p_value, two_tailed=False): # where x is a value under the distribution of the random variable X # such that the probability of getting greater than x, is p t_value = scipy.stats.t.isf(p_value, deg_freedom) - r_value = np.sqrt(t_value ** 2 / (deg_freedom + t_value ** 2)) + return np.sqrt(t_value**2 / (deg_freedom + t_value**2)) # Return correlation coefficient - return r_value -def merge_lists(deg_list: Optional[LIST[str]] = None, - eig_list: Optional[LIST[str]] = None, - lfcd_list: Optional[LIST[str]] = None): - '''Function to actually do the list merging. +def merge_lists( + deg_list: Optional[LIST[str]] = None, + eig_list: Optional[LIST[str]] = None, + lfcd_list: Optional[LIST[str]] = None, +): + """Function to actually do the list merging. Parameters ---------- @@ -108,7 +110,7 @@ def merge_lists(deg_list: Optional[LIST[str]] = None, lfcd_binarized : str path to binarized local functional connectivity density output - ''' + """ if deg_list is None: deg_list = [] if eig_list is None: @@ -128,25 +130,31 @@ def merge_lists(deg_list: Optional[LIST[str]] = None, lfcd_weighted = None lfcd_binarized = None for path in merged_list: - if 'degree' in path and 'Weighted' in path: + if "degree" in path and "Weighted" in path: degree_weighted = path - elif 'degree' in path and 'Binarize' in path: + elif "degree" in path and "Binarize" in path: degree_binarized = path - elif 'eigen' in path and 'Weighted' in path: + elif "eigen" in path and "Weighted" in path: eigen_weighted = path - elif 'eigen' in path and 'Binarize' in path: + elif "eigen" in path and "Binarize" in path: eigen_binarized = path - elif 'local_functional' in path and 'Weighted' in path: + elif "local_functional" in path and "Weighted" in path: lfcd_weighted = path - elif 'local_functional' in path and 'Binarize' in path: + elif "local_functional" in path and "Binarize" in path: lfcd_binarized = path - return (degree_weighted, degree_binarized, eigen_weighted, - eigen_binarized, lfcd_weighted, lfcd_binarized) + return ( + degree_weighted, + degree_binarized, + eigen_weighted, + eigen_binarized, + lfcd_weighted, + lfcd_binarized, + ) def create_merge_node(pipe_num: int) -> Node: - '''Create a Function Node to merge lists for the centrality workflow + """Create a Function Node to merge lists for the centrality workflow. Parameters ---------- @@ -189,28 +197,43 @@ def create_merge_node(pipe_num: int) -> Node: lfcd_binarized : string path to binarized local functional connectivity density output - ''' - return Node(Function(input_names=['deg_list', 'eig_list', 'lfcd_list'], - output_names=['degree_weighted', - 'degree_binarized', - 'eigen_weighted', - 'eigen_binarized', - 'lfcd_weighted', - 'lfcd_binarized'], - function=merge_lists, as_module=True), - name=f'centrality_merge_node_{pipe_num}') - - -@Function.sig_imports(['from typing import Union', 'import os', - 'from pathlib import Path', 'import nibabel as nib', - 'from CPAC.pipeline.schema import valid_options', - 'from CPAC.utils.docs import docstring_parameter', - 'from CPAC.utils.typing import ITERABLE, LIST']) + """ + return Node( + Function( + input_names=["deg_list", "eig_list", "lfcd_list"], + output_names=[ + "degree_weighted", + "degree_binarized", + "eigen_weighted", + "eigen_binarized", + "lfcd_weighted", + "lfcd_binarized", + ], + function=merge_lists, + as_module=True, + ), + name=f"centrality_merge_node_{pipe_num}", + ) + + +@Function.sig_imports( + [ + "from typing import Union", + "import os", + "from pathlib import Path", + "import nibabel as nib", + "from CPAC.pipeline.schema import valid_options", + "from CPAC.utils.docs import docstring_parameter", + "from CPAC.utils.typing import ITERABLE, LIST", + ] +) @docstring_parameter( - weight_options=tuple(valid_options['centrality']['weight_options'])) -def sep_nifti_subbriks(nifti_file: Union[Path, str], out_names: ITERABLE[str] - ) -> LIST[str]: - '''Separate sub-briks of niftis and save specified out + weight_options=tuple(valid_options["centrality"]["weight_options"]) +) +def sep_nifti_subbriks( + nifti_file: Union[Path, str], out_names: ITERABLE[str] +) -> LIST[str]: + """Separate sub-briks of niftis and save specified out Parameters ---------- @@ -224,10 +247,10 @@ def sep_nifti_subbriks(nifti_file: Union[Path, str], out_names: ITERABLE[str] ------- list of str paths to each of the specified outputs as its own file - ''' + """ output_niftis = [] - weight_options = valid_options['centrality']['weight_options'] - selected_options = {_[::-1].split('_', 1)[0][::-1]: _ for _ in out_names} + weight_options = valid_options["centrality"]["weight_options"] + selected_options = {_[::-1].split("_", 1)[0][::-1]: _ for _ in out_names} nii_img = nib.load(nifti_file) nii_arr = nii_img.get_fdata() @@ -238,8 +261,9 @@ def sep_nifti_subbriks(nifti_file: Union[Path, str], out_names: ITERABLE[str] if len(nii_dims) == 3 and len(out_names) == 1: pass else: - err_msg = 'out_names must have same number of elements as '\ - 'nifti sub-briks' + err_msg = ( + "out_names must have same number of elements as " "nifti sub-briks" + ) raise Exception(err_msg) for brik, option in enumerate(weight_options): @@ -248,8 +272,7 @@ def sep_nifti_subbriks(nifti_file: Union[Path, str], out_names: ITERABLE[str] brik_arr = nii_arr elif len(nii_dims) > 3: brik_arr = nii_arr[:, :, :, 0, brik] - out_file = os.path.join(os.getcwd(), - selected_options[option] + '.nii.gz') + out_file = os.path.join(os.getcwd(), selected_options[option] + ".nii.gz") out_img = nib.Nifti1Image(brik_arr, nii_affine) out_img.to_filename(out_file) output_niftis.append(out_file) @@ -257,11 +280,12 @@ def sep_nifti_subbriks(nifti_file: Union[Path, str], out_names: ITERABLE[str] return output_niftis -@docstring_parameter(m_options=valid_options['centrality']['method_options'], - t_options=valid_options['centrality'][ - 'threshold_options']) +@docstring_parameter( + m_options=valid_options["centrality"]["method_options"], + t_options=valid_options["centrality"]["threshold_options"], +) def check_centrality_params(method_option, threshold_option, threshold): - ''' + """ Function to check the centrality parameters. Parameters @@ -281,72 +305,69 @@ def check_centrality_params(method_option, threshold_option, threshold): threshold_option : str one of {t_options} - ''' + """ # Check method option if isinstance(method_option, int): - if method_option < len(valid_options['centrality']['method_options']): - method_option = valid_options[ - 'centrality']['method_options'][method_option] + if method_option < len(valid_options["centrality"]["method_options"]): + method_option = valid_options["centrality"]["method_options"][method_option] else: raise MethodOptionError(method_option) elif not isinstance(method_option, str): - raise TypeError('Method option must be a string, but type \'%s\' ' - 'provided' % type(method_option).__name__) + raise TypeError( + "Method option must be a string, but type '%s' " + "provided" % type(method_option).__name__ + ) # Check threshold option if type(threshold_option) is list: threshold_option = threshold_option[0] if type(threshold_option) is int: - if threshold_option < len( - valid_options['centrality']['threshold_options'] - ): - threshold_option = valid_options[ - 'centrality']['threshold_options'][threshold_option] + if threshold_option < len(valid_options["centrality"]["threshold_options"]): + threshold_option = valid_options["centrality"]["threshold_options"][ + threshold_option + ] else: raise ThresholdOptionError(threshold_option, method_option) elif type(threshold_option) is not str: - raise TypeError('Threshold option must be a string, but type \'%s\' ' - 'provided' % type(threshold_option).__name__) + raise TypeError( + "Threshold option must be a string, but type '%s' " + "provided" % type(threshold_option).__name__ + ) # Format input strings - method_option = method_option.lower().rstrip(' ') - method_options_v1 = ['degree', 'eigenvector', 'lfcd'] + method_option = method_option.lower().rstrip(" ") + method_options_v1 = ["degree", "eigenvector", "lfcd"] if method_option in method_options_v1: - method_option = valid_options['centrality']['method_options'][ + method_option = valid_options["centrality"]["method_options"][ method_options_v1.index(method_option) ] - if ' ' not in threshold_option: - threshold_option = ' '.join([threshold_option, 'threshold']) - threshold_option = threshold_option.capitalize().rstrip(' ') + if " " not in threshold_option: + threshold_option = " ".join([threshold_option, "threshold"]) + threshold_option = threshold_option.capitalize().rstrip(" ") # Check for strings properly formatted - if method_option not in valid_options['centrality']['method_options']: + if method_option not in valid_options["centrality"]["method_options"]: raise MethodOptionError(method_option) # Check for strings properly formatted - if threshold_option not in valid_options['centrality'][ - 'threshold_options' - ]: + if threshold_option not in valid_options["centrality"]["threshold_options"]: raise ThresholdOptionError(threshold_option, method_option) # Check for invalid combinations of method_option + threshold_option if ( - method_option == 'local_functional_connectivity_density' and - threshold_option == 'Sparsity threshold' + method_option == "local_functional_connectivity_density" + and threshold_option == "Sparsity threshold" ): raise ThresholdOptionError(threshold_option, method_option) # If it's significance/sparsity thresholding, check for (0,1] - if ( - threshold_option == 'Significance threshold' or - threshold_option == 'Sparsity threshold' - ): + if threshold_option in ("Significance threshold", "Sparsity threshold"): if threshold <= 0 or threshold > 1: raise ThresholdError(threshold_option, threshold) # If it's correlation, check for [-1,1] - elif threshold_option == 'Correlation threshold': + elif threshold_option == "Correlation threshold": if threshold < -1 or threshold > 1: raise ThresholdError(threshold_option, threshold) else: @@ -357,11 +378,11 @@ def check_centrality_params(method_option, threshold_option, threshold): class MethodOptionError(ValueError): - """Raised when a selected centrality method option is not supported. - """ + """Raised when a selected centrality method option is not supported.""" + def __init__(self, method_option): self.method_option = method_option - self.message = 'Method option \'%s\' not supported' % method_option + self.message = "Method option '%s' not supported" % method_option super().__init__(self.message) @@ -369,22 +390,20 @@ class ThresholdError(ValueError): """Raised when a selected threshold value is not supported for a selected threshold option. """ + def __init__(self, threshold_option, threshold): self.threshold_option = threshold_option self.threshold = threshold - print(type(threshold)) - self.message = f'For \'{threshold_option}\', threshold value must be ' - if ( - threshold_option == 'Significance threshold' or - threshold_option == 'Sparsity threshold' - ): - self.message += 'a positive number greater than 0 ' - elif threshold_option == 'Correlation threshold': - self.message += 'greater than or equal to -1 ' + self.message = f"For '{threshold_option}', threshold value must be " + if threshold_option in ("Significance threshold", "Sparsity threshold"): + self.message += "a positive number greater than 0 " + elif threshold_option == "Correlation threshold": + self.message += "greater than or equal to -1 " else: raise ThresholdOptionError(threshold_option) - self.message += 'and less than or equal to 1.\n Currently it is set ' \ - f'at {threshold}' + self.message += ( + "and less than or equal to 1.\n Currently it is set " f"at {threshold}" + ) super().__init__(self.message) @@ -392,22 +411,24 @@ class ThresholdOptionError(ValueError): """Raised when a selected threshold option is not supported for a selected centrality measure. """ + def __init__(self, threshold_option, method_option=None): self.method_option = method_option self.threshold_option = threshold_option - self.message = f'Threshold option \'{threshold_option}\' not supported' + self.message = f"Threshold option '{threshold_option}' not supported" if self.method_option: - self.message += ' for network centrality measure ' \ - f'\'{method_option}\'' - self.message += '; fix this in the pipeline config' + self.message += " for network centrality measure " f"'{method_option}'" + self.message += "; fix this in the pipeline config" if ( - method_option == 'local_functional_connectivity_density' and - threshold_option == 'Sparsity threshold' + method_option == "local_functional_connectivity_density" + and threshold_option == "Sparsity threshold" ): - _valid_options = ' or '.join([ - f"'{t}'" for t in valid_options[ - 'centrality' - ]['threshold_options'] if t != threshold_option - ]) - self.message += f'. \'{method_option}\' must use {_valid_options}.' + _valid_options = " or ".join( + [ + f"'{t}'" + for t in valid_options["centrality"]["threshold_options"] + if t != threshold_option + ] + ) + self.message += f". '{method_option}' must use {_valid_options}." super().__init__(self.message) diff --git a/CPAC/nuisance/__init__.py b/CPAC/nuisance/__init__.py index 934dfa4315..bb6bc06a66 100644 --- a/CPAC/nuisance/__init__.py +++ b/CPAC/nuisance/__init__.py @@ -1,31 +1,23 @@ -from .utils import ( - find_offending_time_points, - temporal_variance_mask, - generate_summarize_tissue_mask, - NuisanceRegressor -) - +from .bandpass import bandpass_voxels from .nuisance import ( - create_regressor_workflow, create_nuisance_regression_workflow, - filtering_bold_and_regressors -) - -from .bandpass import ( - bandpass_voxels + create_regressor_workflow, + filtering_bold_and_regressors, ) - -from .utils.compcor import ( - cosine_filter +from .utils import ( + find_offending_time_points, + generate_summarize_tissue_mask, + temporal_variance_mask, ) +from .utils.compcor import cosine_filter __all__ = [ - 'create_regressor_workflow', - 'create_nuisance_regression_workflow', - 'filtering_bold_and_regressors', - 'find_offending_time_points', - 'temporal_variance_mask', - 'generate_summarize_tissue_mask', - 'bandpass_voxels', - 'cosine_filter' -] \ No newline at end of file + "create_regressor_workflow", + "create_nuisance_regression_workflow", + "filtering_bold_and_regressors", + "find_offending_time_points", + "temporal_variance_mask", + "generate_summarize_tissue_mask", + "bandpass_voxels", + "cosine_filter", +] diff --git a/CPAC/nuisance/bandpass.py b/CPAC/nuisance/bandpass.py index 91c72124c8..6a5073a88c 100644 --- a/CPAC/nuisance/bandpass.py +++ b/CPAC/nuisance/bandpass.py @@ -1,53 +1,52 @@ import os -import numpy as np -import nibabel as nb +import numpy as np +import nibabel as nib from scipy.fftpack import fft, ifft def ideal_bandpass(data, sample_period, bandpass_freqs): - # Derived from YAN Chao-Gan 120504 based on REST. - sample_freq = 1. / sample_period + # Derived from YAN Chao-Gan 120504 based on REST. + sample_freq = 1.0 / sample_period sample_length = data.shape[0] - data_p = np.zeros(int(2**np.ceil(np.log2(sample_length)))) + data_p = np.zeros(int(2 ** np.ceil(np.log2(sample_length)))) data_p[:sample_length] = data LowCutoff, HighCutoff = bandpass_freqs - if (LowCutoff is None): # No lower cutoff (low-pass filter) + if LowCutoff is None: # No lower cutoff (low-pass filter) low_cutoff_i = 0 - elif (LowCutoff > sample_freq / 2.): - # Cutoff beyond fs/2 (all-stop filter) + elif LowCutoff > sample_freq / 2.0: + # Cutoff beyond fs/2 (all-stop filter) low_cutoff_i = int(data_p.shape[0] / 2) else: - low_cutoff_i = np.ceil( - LowCutoff * data_p.shape[0] * sample_period).astype('int') + low_cutoff_i = np.ceil(LowCutoff * data_p.shape[0] * sample_period).astype( + "int" + ) - if (HighCutoff > sample_freq / 2. or HighCutoff is None): - # Cutoff beyond fs/2 or unspecified (become a highpass filter) + if HighCutoff > sample_freq / 2.0 or HighCutoff is None: + # Cutoff beyond fs/2 or unspecified (become a highpass filter) high_cutoff_i = int(data_p.shape[0] / 2) else: - high_cutoff_i = np.fix( - HighCutoff * data_p.shape[0] * sample_period).astype('int') + high_cutoff_i = np.fix(HighCutoff * data_p.shape[0] * sample_period).astype( + "int" + ) - freq_mask = np.zeros_like(data_p, dtype='bool') - freq_mask[low_cutoff_i:high_cutoff_i + 1] = True + freq_mask = np.zeros_like(data_p, dtype="bool") + freq_mask[low_cutoff_i : high_cutoff_i + 1] = True freq_mask[ - data_p.shape[0] - - high_cutoff_i:data_p.shape[0] + 1 - low_cutoff_i - ] = True + data_p.shape[0] - high_cutoff_i : data_p.shape[0] + 1 - low_cutoff_i + ] = True f_data = fft(data_p) - f_data[freq_mask != True] = 0. - data_bp = np.real_if_close(ifft(f_data)[:sample_length]) - return data_bp + f_data[freq_mask is not True] = 0.0 + return np.real_if_close(ifft(f_data)[:sample_length]) -def bandpass_voxels(realigned_file, regressor_file, bandpass_freqs, - sample_period=None): +def bandpass_voxels(realigned_file, regressor_file, bandpass_freqs, sample_period=None): """Performs ideal bandpass filtering on each voxel time-series. - + Parameters ---------- realigned_file : string @@ -57,15 +56,15 @@ def bandpass_voxels(realigned_file, regressor_file, bandpass_freqs, sample_period : float, optional Length of sampling period in seconds. If not specified, this value is read from the nifti file provided. - + Returns ------- bandpassed_file : string Path of filtered output (nifti file). - + """ - nii = nb.load(realigned_file) - data = nii.get_fdata().astype('float64') + nii = nib.load(realigned_file) + data = nii.get_fdata().astype("float64") mask = (data != 0).sum(-1) != 0 Y = data[mask].T Yc = Y - np.tile(Y.mean(0), (Y.shape[0], 1)) @@ -82,19 +81,16 @@ def bandpass_voxels(realigned_file, regressor_file, bandpass_freqs, Y_bp[:, j] = ideal_bandpass(Yc[:, j], sample_period, bandpass_freqs) data[mask] = Y_bp.T - img = nb.Nifti1Image(data, header=nii.header, - affine=nii.affine) - bandpassed_file = os.path.join(os.getcwd(), - 'bandpassed_demeaned_filtered.nii.gz') + img = nib.Nifti1Image(data, header=nii.header, affine=nii.affine) + bandpassed_file = os.path.join(os.getcwd(), "bandpassed_demeaned_filtered.nii.gz") img.to_filename(bandpassed_file) regressor_bandpassed_file = None if regressor_file is not None: - - if regressor_file.endswith('.nii.gz') or regressor_file.endswith('.nii'): - nii = nb.load(regressor_file) - data = nii.get_fdata().astype('float64') + if regressor_file.endswith(".nii.gz") or regressor_file.endswith(".nii"): + nii = nib.load(regressor_file) + data = nii.get_fdata().astype("float64") mask = (data != 0).sum(-1) != 0 Y = data[mask].T Yc = Y - np.tile(Y.mean(0), (Y.shape[0], 1)) @@ -102,52 +98,54 @@ def bandpass_voxels(realigned_file, regressor_file, bandpass_freqs, for j in range(Y.shape[1]): Y_bp[:, j] = ideal_bandpass(Yc[:, j], sample_period, bandpass_freqs) data[mask] = Y_bp.T - - img = nb.Nifti1Image(data, header=nii.header, - affine=nii.affine) - regressor_bandpassed_file = os.path.join(os.getcwd(), - 'regressor_bandpassed_demeaned_filtered.nii.gz') + + img = nib.Nifti1Image(data, header=nii.header, affine=nii.affine) + regressor_bandpassed_file = os.path.join( + os.getcwd(), "regressor_bandpassed_demeaned_filtered.nii.gz" + ) img.to_filename(regressor_bandpassed_file) - + else: - with open(regressor_file, 'r') as f: + with open(regressor_file, "r") as f: header = [] - # header wouldn't be longer than 5, right? I don't want to + # header wouldn't be longer than 5, right? I don't want to # loop over the whole file for i in range(5): line = f.readline() - if line.startswith('#') or isinstance(line[0], str): + if line.startswith("#") or isinstance(line[0], str): header.append(line) - + # usecols=[list] regressor = np.loadtxt(regressor_file, skiprows=len(header)) Yc = regressor - np.tile(regressor.mean(0), (regressor.shape[0], 1)) Y_bp = np.zeros_like(Yc) # Modify to allow just 1 regressor column - shape = regressor.shape[0] if len(regressor.shape) < 1 else regressor.shape[1] + shape = ( + regressor.shape[0] if len(regressor.shape) < 1 else regressor.shape[1] + ) for j in range(shape): - Y_bp[:, j] = ideal_bandpass(Yc[:, j], sample_period, - bandpass_freqs) + Y_bp[:, j] = ideal_bandpass(Yc[:, j], sample_period, bandpass_freqs) - regressor_bandpassed_file = os.path.join(os.getcwd(), - 'regressor_bandpassed_demeaned_filtered.1D') + regressor_bandpassed_file = os.path.join( + os.getcwd(), "regressor_bandpassed_demeaned_filtered.1D" + ) with open(regressor_bandpassed_file, "w") as ofd: # write out the header information for line in header: ofd.write(line) nuisance_regressors = np.array(Y_bp) - np.savetxt(ofd, nuisance_regressors, fmt='%.18f', - delimiter='\t') + np.savetxt(ofd, nuisance_regressors, fmt="%.18f", delimiter="\t") return bandpassed_file, regressor_bandpassed_file def afni_1dBandpass(in_file, highpass, lowpass, tr=1): - ''' - Perform AFNI 1dBandpass - Parameters + """ + Perform AFNI 1dBandpass. + + Parameters. ---------- in_file : string Path of an input 1D file @@ -160,16 +158,14 @@ def afni_1dBandpass(in_file, highpass, lowpass, tr=1): ------- out_file : string Path of an output 1D file - ''' - + """ import os basename = os.path.basename(in_file) filename, file_extension = os.path.splitext(basename) - out_file = os.path.join(os.getcwd(), filename + '_bp' + file_extension) + out_file = os.path.join(os.getcwd(), filename + "_bp" + file_extension) - cmd = '1dBandpass -dt %f %f %f %s > %s' % ( - tr, highpass, lowpass, in_file, out_file) + cmd = "1dBandpass -dt %f %f %f %s > %s" % (tr, highpass, lowpass, in_file, out_file) os.system(cmd) return out_file diff --git a/CPAC/nuisance/nuisance.py b/CPAC/nuisance/nuisance.py index 97b5186f1a..2a905ef15b 100644 --- a/CPAC/nuisance/nuisance.py +++ b/CPAC/nuisance/nuisance.py @@ -14,56 +14,55 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -import re import os -from CPAC.pipeline.nodeblock import nodeblock -import numpy as np -import nibabel as nb -# pylint: disable=wrong-import-order -from nipype.pipeline.engine.workflows import Workflow -from CPAC.pipeline import nipype_pipeline_engine as pe -import nipype.interfaces.utility as util -import CPAC +import numpy as np +import nibabel as nib from nipype import logging -from nipype.interfaces import fsl -from nipype.interfaces import ants -from nipype.interfaces import c3 -from nipype.interfaces import afni +from nipype.interfaces import afni, fsl from nipype.interfaces.afni import utils as afni_utils -from scipy.fftpack import fft, ifft -from CPAC.pipeline.engine import ResourcePool -from CPAC.utils.configuration import Configuration -from CPAC.utils.interfaces.function import Function -from CPAC.utils.interfaces.masktool import MaskTool -from CPAC.utils.interfaces.pc import PC -from CPAC.utils.typing import LITERAL, TUPLE +import nipype.interfaces.utility as util -from CPAC.registration.registration import warp_timeseries_to_T1template, \ - warp_timeseries_to_EPItemplate, apply_transform -from CPAC.aroma.aroma import create_aroma +# pylint: disable=wrong-import-order +from nipype.pipeline.engine.workflows import Workflow +import CPAC +from CPAC.aroma.aroma import create_aroma from CPAC.nuisance.utils import ( find_offending_time_points, generate_summarize_tissue_mask, - temporal_variance_mask) + temporal_variance_mask, +) from CPAC.nuisance.utils.compcor import ( + TR_string_to_float, calc_compcor_components, cosine_filter, - TR_string_to_float) - +) +from CPAC.pipeline import nipype_pipeline_engine as pe +from CPAC.pipeline.engine import ResourcePool +from CPAC.pipeline.nodeblock import nodeblock +from CPAC.registration.registration import ( + apply_transform, + warp_timeseries_to_EPItemplate, + warp_timeseries_to_T1template, +) from CPAC.seg_preproc.utils import erosion, mask_erosion - +from CPAC.utils.configuration import Configuration from CPAC.utils.datasource import check_for_s3 +from CPAC.utils.interfaces.function import Function +from CPAC.utils.interfaces.masktool import MaskTool +from CPAC.utils.interfaces.pc import PC +from CPAC.utils.typing import LITERAL, TUPLE from CPAC.utils.utils import check_prov_for_regtool -from .bandpass import (bandpass_voxels, afni_1dBandpass) -logger = logging.getLogger('nipype.workflow') +from .bandpass import afni_1dBandpass, bandpass_voxels + +logger = logging.getLogger("nipype.workflow") def choose_nuisance_blocks(cfg, rpool, generate_only=False): - ''' + """ Function to handle selecting appropriate blocks based on - existing config and resource pool + existing config and resource pool. Parameters ---------- @@ -75,113 +74,130 @@ def choose_nuisance_blocks(cfg, rpool, generate_only=False): Returns ------- nuisance : list - ''' + """ nuisance = [] - to_template_cfg = cfg.registration_workflows['functional_registration'][ - 'func_registration_to_template'] - apply_transform_using = to_template_cfg['apply_transform']['using'] + to_template_cfg = cfg.registration_workflows["functional_registration"][ + "func_registration_to_template" + ] + apply_transform_using = to_template_cfg["apply_transform"]["using"] input_interface = { - 'default': ('desc-preproc_bold', ['desc-preproc_bold', 'bold']), - 'abcd': ('desc-preproc_bold', 'bold'), - 'single_step_resampling_from_stc': ("desc-preproc_bold", - "desc-stc_bold") + "default": ("desc-preproc_bold", ["desc-preproc_bold", "bold"]), + "abcd": ("desc-preproc_bold", "bold"), + "single_step_resampling_from_stc": ("desc-preproc_bold", "desc-stc_bold"), }.get(apply_transform_using) if input_interface is not None: - if 'T1_template' in to_template_cfg['target_template']['using']: - nuisance.append((nuisance_regressors_generation_T1w, - input_interface)) - if 'EPI_template' in to_template_cfg['target_template']['using']: - nuisance.append((nuisance_regressors_generation_EPItemplate, - input_interface)) - - if not generate_only and cfg['nuisance_corrections', - '2-nuisance_regression', - 'space'] == 'native': + if "T1_template" in to_template_cfg["target_template"]["using"]: + nuisance.append((nuisance_regressors_generation_T1w, input_interface)) + if "EPI_template" in to_template_cfg["target_template"]["using"]: + nuisance.append( + (nuisance_regressors_generation_EPItemplate, input_interface) + ) + + if ( + not generate_only + and cfg["nuisance_corrections", "2-nuisance_regression", "space"] + == "native" + ): nuisance.append((nuisance_regression_native, input_interface)) return nuisance def erode_mask(name, segmentmap=True): - wf = pe.Workflow(name=name) - inputspec = pe.Node(util.IdentityInterface(fields=['mask', - 'erode_mm', - 'erode_prop', - 'brain_mask', - 'mask_erode_mm']), - name='inputspec') + inputspec = pe.Node( + util.IdentityInterface( + fields=["mask", "erode_mm", "erode_prop", "brain_mask", "mask_erode_mm"] + ), + name="inputspec", + ) - outputspec = pe.Node(util.IdentityInterface(fields=['eroded_mask']), - name='outputspec') + outputspec = pe.Node( + util.IdentityInterface(fields=["eroded_mask"]), name="outputspec" + ) def form_mask_erosion_prop(erosion_prop): if not isinstance(erosion_prop, (int, float)): erosion_prop = 0 - return erosion_prop ** 3 - - ero_imports = ['import scipy.ndimage as nd', 'import numpy as np', - 'import nibabel as nb', 'import os', - 'from CPAC.seg_preproc.utils import _erode'] - - eroded_mask = pe.Node(util.Function( - input_names=['roi_mask', 'skullstrip_mask', 'mask_erosion_mm', - 'mask_erosion_prop'], - output_names=['output_roi_mask', 'eroded_skullstrip_mask'], - function=mask_erosion, - imports=ero_imports), - name='erode_skullstrip_mask', - mem_gb=2.3, - mem_x=(4664065662093477 / 2417851639229258349412352, - 'roi_mask')) - - wf.connect(inputspec, 'brain_mask', eroded_mask, 'skullstrip_mask') - wf.connect(inputspec, 'mask', eroded_mask, 'roi_mask') - - wf.connect(inputspec, ('erode_prop', form_mask_erosion_prop), eroded_mask, - 'mask_erosion_prop') - wf.connect(inputspec, 'mask_erode_mm', eroded_mask, 'mask_erosion_mm') + return erosion_prop**3 + + ero_imports = [ + "import scipy.ndimage as nd", + "import numpy as np", + "import nibabel as nb", + "import os", + "from CPAC.seg_preproc.utils import _erode", + ] + + eroded_mask = pe.Node( + util.Function( + input_names=[ + "roi_mask", + "skullstrip_mask", + "mask_erosion_mm", + "mask_erosion_prop", + ], + output_names=["output_roi_mask", "eroded_skullstrip_mask"], + function=mask_erosion, + imports=ero_imports, + ), + name="erode_skullstrip_mask", + mem_gb=2.3, + mem_x=(4664065662093477 / 2417851639229258349412352, "roi_mask"), + ) + + wf.connect(inputspec, "brain_mask", eroded_mask, "skullstrip_mask") + wf.connect(inputspec, "mask", eroded_mask, "roi_mask") + + wf.connect( + inputspec, + ("erode_prop", form_mask_erosion_prop), + eroded_mask, + "mask_erosion_prop", + ) + wf.connect(inputspec, "mask_erode_mm", eroded_mask, "mask_erosion_mm") if not segmentmap: - wf.connect(eroded_mask, 'output_roi_mask', outputspec, 'eroded_mask') + wf.connect(eroded_mask, "output_roi_mask", outputspec, "eroded_mask") if segmentmap: - erosion_segmentmap = pe.Node(util.Function(input_names=['roi_mask', - 'erosion_mm', - 'erosion_prop' - ], - output_names=[ - 'eroded_roi_mask'], - function=erosion, - imports=ero_imports), - name='erode_mask') + erosion_segmentmap = pe.Node( + util.Function( + input_names=["roi_mask", "erosion_mm", "erosion_prop"], + output_names=["eroded_roi_mask"], + function=erosion, + imports=ero_imports, + ), + name="erode_mask", + ) - wf.connect(eroded_mask, 'output_roi_mask', erosion_segmentmap, 'roi_mask') + wf.connect(eroded_mask, "output_roi_mask", erosion_segmentmap, "roi_mask") - wf.connect(inputspec, 'erode_prop', erosion_segmentmap, 'erosion_prop') - wf.connect(inputspec, 'erode_mm', erosion_segmentmap, 'erosion_mm') + wf.connect(inputspec, "erode_prop", erosion_segmentmap, "erosion_prop") + wf.connect(inputspec, "erode_mm", erosion_segmentmap, "erosion_mm") - wf.connect(erosion_segmentmap, 'eroded_roi_mask', - outputspec, 'eroded_mask') + wf.connect(erosion_segmentmap, "eroded_roi_mask", outputspec, "eroded_mask") return wf -def gather_nuisance(functional_file_path, - selector, - grey_matter_summary_file_path=None, - white_matter_summary_file_path=None, - csf_summary_file_path=None, - acompcor_file_path=None, - tcompcor_file_path=None, - global_summary_file_path=None, - motion_parameters_file_path=None, - custom_file_paths=None, - censor_file_path=None): +def gather_nuisance( + functional_file_path, + selector, + grey_matter_summary_file_path=None, + white_matter_summary_file_path=None, + csf_summary_file_path=None, + acompcor_file_path=None, + tcompcor_file_path=None, + global_summary_file_path=None, + motion_parameters_file_path=None, + custom_file_paths=None, + censor_file_path=None, +): """ Gathers the various nuisance regressors together into a single tab- separated values file that is an appropriate for input into - 3dTproject + 3dTproject. :param functional_file_path: path to file that the regressors are being calculated for, is used to calculate the length of the @@ -211,60 +227,64 @@ def gather_nuisance(functional_file_path, should be censored :return: out_file (str), censor_indices (list) """ - # Basic checks for the functional image - if not functional_file_path or \ - (not functional_file_path.endswith(".nii") and - not functional_file_path.endswith(".nii.gz")): - - raise ValueError("Invalid value for input_file ({}). Should be a nifti " - "file and should exist".format(functional_file_path)) + if not functional_file_path or ( + not functional_file_path.endswith(".nii") + and not functional_file_path.endswith(".nii.gz") + ): + raise ValueError( + "Invalid value for input_file ({}). Should be a nifti " + "file and should exist".format(functional_file_path) + ) try: - functional_image = nb.load(functional_file_path) + functional_image = nib.load(functional_file_path) except: - raise ValueError("Invalid value for input_file ({}). Should be a nifti " - "file and should exist".format(functional_file_path)) + raise ValueError( + "Invalid value for input_file ({}). Should be a nifti " + "file and should exist".format(functional_file_path) + ) if len(functional_image.shape) < 4 or functional_image.shape[3] < 2: - raise ValueError("Invalid input_file ({}). Expected 4D file." - .format(functional_file_path)) + raise ValueError( + "Invalid input_file ({}). Expected 4D file.".format(functional_file_path) + ) regressor_length = functional_image.shape[3] - #selector = selector.selector + # selector = selector.selector if not isinstance(selector, dict): - raise ValueError("Invalid type for selectors {0}, expecting dict" - .format(type(selector))) + raise ValueError( + "Invalid type for selectors {0}, expecting dict".format(type(selector)) + ) regressor_files = { - 'aCompCor': acompcor_file_path, - 'tCompCor': tcompcor_file_path, - 'GlobalSignal': global_summary_file_path, - 'GreyMatter': grey_matter_summary_file_path, - 'WhiteMatter': white_matter_summary_file_path, - 'CerebrospinalFluid': csf_summary_file_path, - 'Motion': motion_parameters_file_path, + "aCompCor": acompcor_file_path, + "tCompCor": tcompcor_file_path, + "GlobalSignal": global_summary_file_path, + "GreyMatter": grey_matter_summary_file_path, + "WhiteMatter": white_matter_summary_file_path, + "CerebrospinalFluid": csf_summary_file_path, + "Motion": motion_parameters_file_path, } regressors_order = [ - 'Motion', - 'GlobalSignal', - 'aCompCor', - 'tCompCor', - 'CerebrospinalFluid', - 'WhiteMatter', - 'GreyMatter', + "Motion", + "GlobalSignal", + "aCompCor", + "tCompCor", + "CerebrospinalFluid", + "WhiteMatter", + "GreyMatter", ] - motion_labels = ['RotY', 'RotX', 'RotZ', 'Y', 'X', 'Z'] + motion_labels = ["RotY", "RotX", "RotZ", "Y", "X", "Z"] # Compile regressors into a matrix column_names = [] nuisance_regressors = [] for regressor_type in regressors_order: - if regressor_type not in selector: continue @@ -272,39 +292,41 @@ def gather_nuisance(functional_file_path, regressor_selector = selector.get(regressor_type) or {} - if 'summary' in regressor_selector: - if type(regressor_selector['summary']) is str: - regressor_selector['summary'] = { - 'method': regressor_selector['summary'], + if "summary" in regressor_selector: + if type(regressor_selector["summary"]) is str: + regressor_selector["summary"] = { + "method": regressor_selector["summary"], } if not regressor_file or not os.path.isfile(regressor_file): - raise ValueError("Regressor type {0} specified in selectors " - "but the corresponding file was not found!" - .format(regressor_type)) + raise ValueError( + "Regressor type {0} specified in selectors " + "but the corresponding file was not found!".format(regressor_type) + ) try: regressors = np.loadtxt(regressor_file) except: - print("Could not read regressor {0} from {1}." - .format(regressor_type, regressor_file)) raise if regressors.shape[0] != regressor_length: - raise ValueError("Number of time points in {0} ({1}) is " - "inconsistent with length of functional " - "file {2} ({3})" - .format(regressor_file, - regressors.shape[0], - functional_file_path, - regressor_length)) + raise ValueError( + "Number of time points in {0} ({1}) is " + "inconsistent with length of functional " + "file {2} ({3})".format( + regressor_file, + regressors.shape[0], + functional_file_path, + regressor_length, + ) + ) if regressor_type == "Motion": num_regressors = 6 - elif not regressor_selector.get('summary', {}).get('components'): + elif not regressor_selector.get("summary", {}).get("components"): num_regressors = 1 else: - num_regressors = regressor_selector['summary']['components'] + num_regressors = regressor_selector["summary"]["components"] if len(regressors.shape) == 1: regressors = np.expand_dims(regressors, axis=1) @@ -312,56 +334,52 @@ def gather_nuisance(functional_file_path, regressors = regressors[:, 0:num_regressors] if regressors.shape[1] != num_regressors: - raise ValueError("Expecting {0} regressors for {1}, but " - "found {2} in file {3}." - .format(num_regressors, - regressor_type, - regressors.shape[1], - regressor_file)) + raise ValueError( + "Expecting {0} regressors for {1}, but " + "found {2} in file {3}.".format( + num_regressors, regressor_type, regressors.shape[1], regressor_file + ) + ) # Add in the regressors, making sure to also add in the column name for regressor_index in range(regressors.shape[1]): if regressor_type == "Motion": regressor_name = motion_labels[regressor_index] else: - summary_method = regressor_selector['summary'] + summary_method = regressor_selector["summary"] if type(summary_method) is dict: - summary_method = summary_method['method'] + summary_method = summary_method["method"] - regressor_name = "{0}{1}{2}".format(regressor_type, - summary_method, - regressor_index) + regressor_name = "{0}{1}{2}".format( + regressor_type, summary_method, regressor_index + ) column_names.append(regressor_name) nuisance_regressors.append(regressors[:, regressor_index]) - if regressor_selector.get('include_delayed', False): + if regressor_selector.get("include_delayed", False): column_names.append("{0}Delay".format(regressor_name)) nuisance_regressors.append( np.append([0.0], regressors[0:-1, regressor_index]) ) - if regressor_selector.get('include_backdiff', False): + if regressor_selector.get("include_backdiff", False): column_names.append("{0}BackDiff".format(regressor_name)) nuisance_regressors.append( np.append([0.0], np.diff(regressors[:, regressor_index], n=1)) ) - if regressor_selector.get('include_squared', False): + if regressor_selector.get("include_squared", False): column_names.append("{0}Sq".format(regressor_name)) - nuisance_regressors.append( - np.square(regressors[:, regressor_index]) - ) + nuisance_regressors.append(np.square(regressors[:, regressor_index])) - if regressor_selector.get('include_delayed_squared', False): + if regressor_selector.get("include_delayed_squared", False): column_names.append("{0}DelaySq".format(regressor_name)) nuisance_regressors.append( - np.square( - np.append([0.0], regressors[0:-1, regressor_index]) - ) + np.square(np.append([0.0], regressors[0:-1, regressor_index])) ) - if regressor_selector.get('include_backdiff_squared', False): + if regressor_selector.get("include_backdiff_squared", False): column_names.append("{0}BackDiffSq".format(regressor_name)) nuisance_regressors.append( np.square( @@ -372,14 +390,16 @@ def gather_nuisance(functional_file_path, # Add custom regressors if custom_file_paths: for custom_file_path in custom_file_paths: - try: custom_regressor = np.loadtxt(custom_file_path) except: - raise ValueError("Could not read regressor {0} from {1}." - .format('Custom', custom_file_path)) + raise ValueError( + "Could not read regressor {0} from {1}.".format( + "Custom", custom_file_path + ) + ) - if (len(custom_regressor.shape) > 1 and custom_regressor.shape[1] > 1): + if len(custom_regressor.shape) > 1 and custom_regressor.shape[1] > 1: raise ValueError( "Invalid format for censor file {0}, should be a single " "column containing 1s for volumes to keep and 0s for volumes " @@ -391,24 +411,15 @@ def gather_nuisance(functional_file_path, censor_indices = [] # Add spike regressors - if selector.get('Censor', {}).get('method') == 'SpikeRegression': - - selector = selector['Censor'] + if selector.get("Censor", {}).get("method") == "SpikeRegression": + selector = selector["Censor"] regressor_file = censor_file_path if not regressor_file: # ↓ This section is gross and temporary ↓ - num_thresh = len(selector['thresholds']) - plural_s = '' if num_thresh == 1 else 's' - thresh_list = [ - thresh.get('value') for thresh in selector['thresholds'] - ] - print(f"{selector['method']} Censor " - "specified with " - f"{'no ' if num_thresh == 0 else ''}threshold" - f"{plural_s} {str(thresh_list)} in selectors but " - f" threshold was not reached.") + len(selector["thresholds"]) + [thresh.get("value") for thresh in selector["thresholds"]] # ↑ This section is gross and temporary ↑ # All good to pass through if nothing to censor censor_volumes = np.ones((regressor_length,), dtype=int) @@ -416,12 +427,15 @@ def gather_nuisance(functional_file_path, try: censor_volumes = np.loadtxt(regressor_file) except: - raise ValueError("Could not read regressor {0} from {1}." - .format(regressor_type, regressor_file)) - - if (len(censor_volumes.shape) > 1 and censor_volumes.shape[1] > 1) or \ - not np.all(np.isin(censor_volumes, [0, 1])): + raise ValueError( + "Could not read regressor {0} from {1}.".format( + regressor_type, regressor_file + ) + ) + if ( + len(censor_volumes.shape) > 1 and censor_volumes.shape[1] > 1 + ) or not np.all(np.isin(censor_volumes, [0, 1])): raise ValueError( "Invalid format for censor file {0}, should be a single " "column containing 1s for volumes to keep and 0s for volumes " @@ -439,24 +453,22 @@ def gather_nuisance(functional_file_path, "regressor length is {2}".format( censor_indices[out_of_range_censors], regressor_file, - regressor_length + regressor_length, ) ) if len(censor_indices) > 0: - # if number_of_previous_trs_to_censor and number_of_subsequent_trs_to_censor # are not set, assume they should be zero - previous_trs_to_censor = \ - selector.get('number_of_previous_trs_to_censor', 0) + previous_trs_to_censor = selector.get("number_of_previous_trs_to_censor", 0) - subsequent_trs_to_censor = \ - selector.get('number_of_subsequent_trs_to_censor', 0) + subsequent_trs_to_censor = selector.get( + "number_of_subsequent_trs_to_censor", 0 + ) spike_regressors = np.zeros(regressor_length) for censor_index in censor_indices: - censor_begin_index = censor_index - previous_trs_to_censor if censor_begin_index < 0: censor_begin_index = 0 @@ -465,10 +477,9 @@ def gather_nuisance(functional_file_path, if censor_end_index >= regressor_length: censor_end_index = regressor_length - 1 - spike_regressors[censor_begin_index:censor_end_index + 1] = 1 + spike_regressors[censor_begin_index : censor_end_index + 1] = 1 for censor_index in np.where(spike_regressors == 1)[0]: - column_names.append("SpikeRegression{0}".format(censor_index)) spike_regressor_index = np.zeros(regressor_length) spike_regressor_index[censor_index] = 1 @@ -481,24 +492,25 @@ def gather_nuisance(functional_file_path, output_file_path = os.path.join(os.getcwd(), "nuisance_regressors.1D") with open(output_file_path, "w") as ofd: - # write out the header information ofd.write("# C-PAC {0}\n".format(CPAC.__version__)) ofd.write("# Nuisance regressors:\n") ofd.write("# " + "\t".join(column_names) + "\n") nuisance_regressors = np.array(nuisance_regressors) - np.savetxt(ofd, nuisance_regressors.T, fmt='%.18f', delimiter='\t') + np.savetxt(ofd, nuisance_regressors.T, fmt="%.18f", delimiter="\t") return output_file_path, censor_indices -def create_regressor_workflow(nuisance_selectors, - use_ants, - ventricle_mask_exist, - csf_mask_exist, - all_bold=False, - name='nuisance_regressors'): +def create_regressor_workflow( + nuisance_selectors, + use_ants, + ventricle_mask_exist, + csf_mask_exist, + all_bold=False, + name="nuisance_regressors", +): """ Workflow for the removal of various signals considered to be noise from resting state fMRI data. The residual signals for linear regression denoising is performed in a single @@ -514,7 +526,6 @@ def create_regressor_workflow(nuisance_selectors, Notes ----- - Workflow Inputs --------------- Workflow Inputs:: @@ -752,132 +763,125 @@ def create_regressor_workflow(nuisance_selectors, :width: 1000 """ - nuisance_wf = pe.Workflow(name=name) - inputspec = pe.Node(util.IdentityInterface(fields=[ - 'selector', - 'functional_file_path', - 'anatomical_file_path', - 'anatomical_eroded_brain_mask_file_path', - 'gm_mask_file_path', - 'wm_mask_file_path', - 'csf_mask_file_path', - 'lat_ventricles_mask_file_path', - 'functional_brain_mask_file_path', - 'func_to_anat_linear_xfm_file_path', - 'anat_to_func_linear_xfm_file_path', - 'mni_to_anat_linear_xfm_file_path', - 'anat_to_mni_linear_xfm_file_path', - 'motion_parameters_file_path', - 'fd_j_file_path', - 'fd_p_file_path', - 'dvars_file_path', - 'creds_path', - 'dl_dir', - 'tr', - ]), name='inputspec') - - outputspec = pe.Node(util.IdentityInterface( - fields=['regressors_file_path', 'censor_indices']), name='outputspec') - - functional_mean = pe.Node(interface=afni_utils.TStat(), - name='functional_mean') - - functional_mean.inputs.options = '-mean' - functional_mean.inputs.outputtype = 'NIFTI_GZ' - - nuisance_wf.connect(inputspec, 'functional_file_path', - functional_mean, 'in_file') + inputspec = pe.Node( + util.IdentityInterface( + fields=[ + "selector", + "functional_file_path", + "anatomical_file_path", + "anatomical_eroded_brain_mask_file_path", + "gm_mask_file_path", + "wm_mask_file_path", + "csf_mask_file_path", + "lat_ventricles_mask_file_path", + "functional_brain_mask_file_path", + "func_to_anat_linear_xfm_file_path", + "anat_to_func_linear_xfm_file_path", + "mni_to_anat_linear_xfm_file_path", + "anat_to_mni_linear_xfm_file_path", + "motion_parameters_file_path", + "fd_j_file_path", + "fd_p_file_path", + "dvars_file_path", + "creds_path", + "dl_dir", + "tr", + ] + ), + name="inputspec", + ) + + outputspec = pe.Node( + util.IdentityInterface(fields=["regressors_file_path", "censor_indices"]), + name="outputspec", + ) + + functional_mean = pe.Node(interface=afni_utils.TStat(), name="functional_mean") + + functional_mean.inputs.options = "-mean" + functional_mean.inputs.outputtype = "NIFTI_GZ" + + nuisance_wf.connect(inputspec, "functional_file_path", functional_mean, "in_file") # Resources to create regressors pipeline_resource_pool = { - "Anatomical": (inputspec, 'anatomical_file_path'), - "AnatomicalErodedMask": (inputspec, 'anatomical_eroded_brain_mask_file_path'), - "Functional": (inputspec, 'functional_file_path'), - "Functional_mean" : (functional_mean, 'out_file'), - "GlobalSignal": (inputspec, 'functional_brain_mask_file_path'), - "WhiteMatter": (inputspec, 'wm_mask_file_path'), - "CerebrospinalFluid": (inputspec, 'csf_mask_file_path'), - "GreyMatter": (inputspec, 'gm_mask_file_path'), - "Ventricles": (inputspec, 'lat_ventricles_mask_file_path'), - + "Anatomical": (inputspec, "anatomical_file_path"), + "AnatomicalErodedMask": (inputspec, "anatomical_eroded_brain_mask_file_path"), + "Functional": (inputspec, "functional_file_path"), + "Functional_mean": (functional_mean, "out_file"), + "GlobalSignal": (inputspec, "functional_brain_mask_file_path"), + "WhiteMatter": (inputspec, "wm_mask_file_path"), + "CerebrospinalFluid": (inputspec, "csf_mask_file_path"), + "GreyMatter": (inputspec, "gm_mask_file_path"), + "Ventricles": (inputspec, "lat_ventricles_mask_file_path"), "Transformations": { "func_to_anat_linear_xfm": (inputspec, "func_to_anat_linear_xfm_file_path"), "anat_to_func_linear_xfm": (inputspec, "anat_to_func_linear_xfm_file_path"), "mni_to_anat_linear_xfm": (inputspec, "mni_to_anat_linear_xfm_file_path"), - "anat_to_mni_linear_xfm": (inputspec, "anat_to_mni_linear_xfm_file_path") + "anat_to_mni_linear_xfm": (inputspec, "anat_to_mni_linear_xfm_file_path"), }, - - "DVARS": (inputspec, 'dvars_file_path'), - "FD_J": (inputspec, 'framewise_displacement_j_file_path'), - "FD_P": (inputspec, 'framewise_displacement_p_file_path'), - "Motion": (inputspec, 'motion_parameters_file_path'), + "DVARS": (inputspec, "dvars_file_path"), + "FD_J": (inputspec, "framewise_displacement_j_file_path"), + "FD_P": (inputspec, "framewise_displacement_p_file_path"), + "Motion": (inputspec, "motion_parameters_file_path"), } # Regressor map to simplify construction of the needed regressors regressors = { - 'GreyMatter': ['grey_matter_summary_file_path', (), 'ort'], - 'WhiteMatter': ['white_matter_summary_file_path', (), 'ort'], - 'CerebrospinalFluid': ['csf_summary_file_path', (), 'ort'], - 'aCompCor': ['acompcor_file_path', (), 'ort'], - 'tCompCor': ['tcompcor_file_path', (), 'ort'], - 'GlobalSignal': ['global_summary_file_path', (), 'ort'], - 'Custom': ['custom_file_paths', (), 'ort'], - 'VoxelCustom': ['custom_file_paths', (), 'dsort'], - 'DVARS': ['dvars_file_path', (), 'ort'], - 'FD_J': ['framewise_displacement_j_file_path', (), 'ort'], - 'FD_P': ['framewise_displacement_p_file_path', (), 'ort'], - 'Motion': ['motion_parameters_file_path', (), 'ort'] + "GreyMatter": ["grey_matter_summary_file_path", (), "ort"], + "WhiteMatter": ["white_matter_summary_file_path", (), "ort"], + "CerebrospinalFluid": ["csf_summary_file_path", (), "ort"], + "aCompCor": ["acompcor_file_path", (), "ort"], + "tCompCor": ["tcompcor_file_path", (), "ort"], + "GlobalSignal": ["global_summary_file_path", (), "ort"], + "Custom": ["custom_file_paths", (), "ort"], + "VoxelCustom": ["custom_file_paths", (), "dsort"], + "DVARS": ["dvars_file_path", (), "ort"], + "FD_J": ["framewise_displacement_j_file_path", (), "ort"], + "FD_P": ["framewise_displacement_p_file_path", (), "ort"], + "Motion": ["motion_parameters_file_path", (), "ort"], } - motion = ['DVARS', 'FD_J', 'FD_P', 'Motion'] - derived = ['tCompCor', 'aCompCor'] - tissues = ['GreyMatter', 'WhiteMatter', 'CerebrospinalFluid'] + motion = ["DVARS", "FD_J", "FD_P", "Motion"] + derived = ["tCompCor", "aCompCor"] + tissues = ["GreyMatter", "WhiteMatter", "CerebrospinalFluid"] for regressor_type, regressor_resource in regressors.items(): - if regressor_type not in nuisance_selectors: continue regressor_selector = nuisance_selectors[regressor_type] - if regressor_type == 'Custom': - + if regressor_type == "Custom": custom_ort_check_s3_nodes = [] custom_dsort_check_s3_nodes = [] custom_dsort_convolve_nodes = [] - for file_num, custom_regressor in enumerate(sorted( - regressor_selector, key=lambda c: c['file'] - )): - custom_regressor_file = custom_regressor['file'] - - custom_check_s3_node = pe.Node(Function( - input_names=[ - 'file_path', - 'creds_path', - 'dl_dir', - 'img_type' - ], - output_names=[ - 'local_path' - ], - function=check_for_s3, - as_module=True), - name=f'custom_check_for_s3_{name}_{file_num}') + for file_num, custom_regressor in enumerate( + sorted(regressor_selector, key=lambda c: c["file"]) + ): + custom_regressor_file = custom_regressor["file"] + + custom_check_s3_node = pe.Node( + Function( + input_names=["file_path", "creds_path", "dl_dir", "img_type"], + output_names=["local_path"], + function=check_for_s3, + as_module=True, + ), + name=f"custom_check_for_s3_{name}_{file_num}", + ) custom_check_s3_node.inputs.set( - file_path=custom_regressor_file, - img_type='func' + file_path=custom_regressor_file, img_type="func" ) - if ( - custom_regressor_file.endswith('.nii.gz') or - custom_regressor_file.endswith('.nii') - ): - - if custom_regressor.get('convolve'): + if custom_regressor_file.endswith( + ".nii.gz" + ) or custom_regressor_file.endswith(".nii"): + if custom_regressor.get("convolve"): custom_dsort_convolve_nodes += [custom_check_s3_node] else: custom_dsort_check_s3_nodes += [custom_check_s3_node] @@ -887,200 +891,261 @@ def create_regressor_workflow(nuisance_selectors, if len(custom_ort_check_s3_nodes) > 0: custom_ort_merge = pe.Node( - util.Merge(len(custom_ort_check_s3_nodes)), - name='custom_ort_merge' + util.Merge(len(custom_ort_check_s3_nodes)), name="custom_ort_merge" ) for i, custom_check_s3_node in enumerate(custom_ort_check_s3_nodes): nuisance_wf.connect( - custom_check_s3_node, 'local_path', - custom_ort_merge, "in{}".format(i + 1) + custom_check_s3_node, + "local_path", + custom_ort_merge, + "in{}".format(i + 1), ) - pipeline_resource_pool['custom_ort_file_paths'] = \ - (custom_ort_merge, 'out') + pipeline_resource_pool["custom_ort_file_paths"] = ( + custom_ort_merge, + "out", + ) - regressors['Custom'][1] = \ - pipeline_resource_pool['custom_ort_file_paths'] + regressors["Custom"][1] = pipeline_resource_pool[ + "custom_ort_file_paths" + ] if len(custom_dsort_convolve_nodes) > 0: custom_dsort_convolve_merge = pe.Node( util.Merge(len(custom_dsort_convolve_nodes)), - name='custom_dsort_convolve_merge' + name="custom_dsort_convolve_merge", ) for i, custom_check_s3_node in enumerate(custom_dsort_convolve_nodes): nuisance_wf.connect( - custom_check_s3_node, 'local_path', - custom_dsort_convolve_merge, "in{}".format(i + 1) + custom_check_s3_node, + "local_path", + custom_dsort_convolve_merge, + "in{}".format(i + 1), ) if len(custom_dsort_check_s3_nodes) > 0: - images_to_merge = len(custom_dsort_check_s3_nodes) if len(custom_dsort_convolve_nodes) > 0: images_to_merge += 1 custom_dsort_merge = pe.Node( - util.Merge(images_to_merge), - name='custom_dsort_merge' + util.Merge(images_to_merge), name="custom_dsort_merge" ) for i, custom_check_s3_node in enumerate(custom_dsort_check_s3_nodes): nuisance_wf.connect( - custom_check_s3_node, 'local_path', - custom_dsort_merge, "in{}".format(i + 1) + custom_check_s3_node, + "local_path", + custom_dsort_merge, + "in{}".format(i + 1), ) if len(custom_dsort_convolve_nodes) > 0: nuisance_wf.connect( - custom_dsort_convolve_merge, 'out', - custom_dsort_merge, "in{}".format(i + 1) + custom_dsort_convolve_merge, + "out", + custom_dsort_merge, + "in{}".format(i + 1), ) - pipeline_resource_pool['custom_dsort_file_paths'] = \ - (custom_dsort_merge, 'out') + pipeline_resource_pool["custom_dsort_file_paths"] = ( + custom_dsort_merge, + "out", + ) - regressors['VoxelCustom'][1] = \ - pipeline_resource_pool['custom_dsort_file_paths'] + regressors["VoxelCustom"][1] = pipeline_resource_pool[ + "custom_dsort_file_paths" + ] continue if regressor_type in motion: - regressor_resource[1] = \ - pipeline_resource_pool[regressor_type] + regressor_resource[1] = pipeline_resource_pool[regressor_type] continue # Set summary method for tCompCor and aCompCor if regressor_type in derived: + if "summary" not in regressor_selector: + regressor_selector["summary"] = {} - if 'summary' not in regressor_selector: - regressor_selector['summary'] = {} - - if type(regressor_selector['summary']) is not dict: - raise ValueError("Regressor {0} requires PC summary method, " - "but {1} specified" - .format(regressor_type, - regressor_selector['summary'])) + if type(regressor_selector["summary"]) is not dict: + raise ValueError( + "Regressor {0} requires PC summary method, " + "but {1} specified".format( + regressor_type, regressor_selector["summary"] + ) + ) - if not regressor_selector['summary'].get('components'): - regressor_selector['summary']['components'] = 1 + if not regressor_selector["summary"].get("components"): + regressor_selector["summary"]["components"] = 1 # If regressor is not present, build up the regressor if not regressor_resource[1]: - # We don't have the regressor, look for it in the resource pool, # build a corresponding key, this is seperated in to a mask key # and an extraction key, which when concatenated provide the # resource key for the regressor - regressor_descriptor = {'tissue': regressor_type} + regressor_descriptor = {"tissue": regressor_type} - if regressor_type == 'aCompCor': - if not regressor_selector.get('tissues'): - raise ValueError("Tissue type required for aCompCor, " - "but none specified") + if regressor_type == "aCompCor": + if not regressor_selector.get("tissues"): + raise ValueError( + "Tissue type required for aCompCor, " "but none specified" + ) - regressor_descriptor = { - 'tissue': regressor_selector['tissues'] - } + regressor_descriptor = {"tissue": regressor_selector["tissues"]} - if regressor_type == 'tCompCor': - if not regressor_selector.get('threshold'): - raise ValueError("Threshold required for tCompCor, " - "but none specified.") + if regressor_type == "tCompCor": + if not regressor_selector.get("threshold"): + raise ValueError( + "Threshold required for tCompCor, " "but none specified." + ) regressor_descriptor = { - 'tissue': 'FunctionalVariance-{}' - .format(regressor_selector['threshold']) + "tissue": "FunctionalVariance-{}".format( + regressor_selector["threshold"] + ) } - if regressor_selector.get('by_slice'): - regressor_descriptor['tissue'] += '-BySlice' + if regressor_selector.get("by_slice"): + regressor_descriptor["tissue"] += "-BySlice" else: - regressor_selector['by_slice'] = False + regressor_selector["by_slice"] = False - if regressor_selector.get('erode_mask_mm'): - erosion_mm = regressor_selector['erode_mask_mm'] + if regressor_selector.get("erode_mask_mm"): + erosion_mm = regressor_selector["erode_mask_mm"] else: erosion_mm = False - if regressor_selector.get('degree'): - degree = regressor_selector['degree'] + if regressor_selector.get("degree"): + degree = regressor_selector["degree"] else: degree = 1 - temporal_wf = temporal_variance_mask(regressor_selector['threshold'], - by_slice=regressor_selector['by_slice'], - erosion=erosion_mm, - degree=degree) + temporal_wf = temporal_variance_mask( + regressor_selector["threshold"], + by_slice=regressor_selector["by_slice"], + erosion=erosion_mm, + degree=degree, + ) - nuisance_wf.connect(*(pipeline_resource_pool['Functional'] + (temporal_wf, 'inputspec.functional_file_path'))) + nuisance_wf.connect( + *( + pipeline_resource_pool["Functional"] + + (temporal_wf, "inputspec.functional_file_path") + ) + ) - if erosion_mm: # TODO: in func/anat space + if erosion_mm: # TODO: in func/anat space # transform eroded anat brain mask to functional space # convert_xfm - anat_to_func_linear_xfm = pe.Node(interface=fsl.ConvertXFM(), name='anat_to_func_linear_xfm') + anat_to_func_linear_xfm = pe.Node( + interface=fsl.ConvertXFM(), name="anat_to_func_linear_xfm" + ) anat_to_func_linear_xfm.inputs.invert_xfm = True - nuisance_wf.connect(*(pipeline_resource_pool['Transformations']['func_to_anat_linear_xfm'] + (anat_to_func_linear_xfm, 'in_file'))) + nuisance_wf.connect( + *( + pipeline_resource_pool["Transformations"][ + "func_to_anat_linear_xfm" + ] + + (anat_to_func_linear_xfm, "in_file") + ) + ) # flirt - anat_to_func_mask = pe.Node(interface=fsl.FLIRT(), name='Functional_eroded_mask') - anat_to_func_mask.inputs.output_type = 'NIFTI_GZ' + anat_to_func_mask = pe.Node( + interface=fsl.FLIRT(), name="Functional_eroded_mask" + ) + anat_to_func_mask.inputs.output_type = "NIFTI_GZ" anat_to_func_mask.inputs.apply_xfm = True - anat_to_func_mask.inputs.interp = 'nearestneighbour' - nuisance_wf.connect(anat_to_func_linear_xfm, 'out_file', anat_to_func_mask, 'in_matrix_file') - nuisance_wf.connect(*(pipeline_resource_pool['AnatomicalErodedMask'] + (anat_to_func_mask, 'in_file'))) - nuisance_wf.connect(*(pipeline_resource_pool['GlobalSignal'] + (anat_to_func_mask, 'reference'))) + anat_to_func_mask.inputs.interp = "nearestneighbour" + nuisance_wf.connect( + anat_to_func_linear_xfm, + "out_file", + anat_to_func_mask, + "in_matrix_file", + ) + nuisance_wf.connect( + *( + pipeline_resource_pool["AnatomicalErodedMask"] + + (anat_to_func_mask, "in_file") + ) + ) + nuisance_wf.connect( + *( + pipeline_resource_pool["GlobalSignal"] + + (anat_to_func_mask, "reference") + ) + ) # connect workflow - nuisance_wf.connect(anat_to_func_mask, 'out_file', temporal_wf, 'inputspec.mask_file_path') + nuisance_wf.connect( + anat_to_func_mask, + "out_file", + temporal_wf, + "inputspec.mask_file_path", + ) else: - nuisance_wf.connect(*(pipeline_resource_pool['GlobalSignal'] + (temporal_wf, 'inputspec.mask_file_path'))) + nuisance_wf.connect( + *( + pipeline_resource_pool["GlobalSignal"] + + (temporal_wf, "inputspec.mask_file_path") + ) + ) - pipeline_resource_pool[regressor_descriptor['tissue']] = \ - (temporal_wf, 'outputspec.mask') + pipeline_resource_pool[regressor_descriptor["tissue"]] = ( + temporal_wf, + "outputspec.mask", + ) - if type(regressor_selector['summary']) is not dict: - regressor_selector['summary'] = { - "filter": regressor_selector['summary'], - "method": regressor_selector['summary'] + if type(regressor_selector["summary"]) is not dict: + regressor_selector["summary"] = { + "filter": regressor_selector["summary"], + "method": regressor_selector["summary"], } # Add selector into regressor description - if regressor_selector.get('extraction_resolution'): - regressor_descriptor['resolution'] = \ - str(regressor_selector['extraction_resolution']) + "mm" + if regressor_selector.get("extraction_resolution"): + regressor_descriptor["resolution"] = ( + str(regressor_selector["extraction_resolution"]) + "mm" + ) elif regressor_type in tissues: - regressor_selector['extraction_resolution'] = "Functional" - regressor_descriptor['resolution'] = "Functional" - - if regressor_selector.get('erode_mask'): - regressor_descriptor['erosion'] = 'Eroded' + regressor_selector["extraction_resolution"] = "Functional" + regressor_descriptor["resolution"] = "Functional" - if not regressor_selector.get('summary'): - raise ValueError("Summary method required for {0}, " - "but none specified".format(regressor_type)) + if regressor_selector.get("erode_mask"): + regressor_descriptor["erosion"] = "Eroded" - regressor_descriptor['extraction'] = \ - regressor_selector['summary']['method'] + if not regressor_selector.get("summary"): + raise ValueError( + "Summary method required for {0}, " "but none specified".format( + regressor_type + ) + ) - if regressor_descriptor['extraction'] in ['DetrendPC', 'PC']: - if not regressor_selector['summary'].get('components'): - raise ValueError("Summary method PC requires components, " - "but received none.") + regressor_descriptor["extraction"] = regressor_selector["summary"]["method"] - regressor_descriptor['extraction'] += \ - '_{0}'.format(regressor_selector['summary']['components']) + if regressor_descriptor["extraction"] in ["DetrendPC", "PC"]: + if not regressor_selector["summary"].get("components"): + raise ValueError( + "Summary method PC requires components, " "but received none." + ) - if type(regressor_descriptor['tissue']) is not list: - regressor_descriptor['tissue'] = \ - [regressor_descriptor['tissue']] + regressor_descriptor["extraction"] += "_{0}".format( + regressor_selector["summary"]["components"] + ) - if regressor_selector.get('extraction_resolution') and \ - regressor_selector["extraction_resolution"] != "Functional": + if type(regressor_descriptor["tissue"]) is not list: + regressor_descriptor["tissue"] = [regressor_descriptor["tissue"]] + if ( + regressor_selector.get("extraction_resolution") + and regressor_selector["extraction_resolution"] != "Functional" + ): functional_at_resolution_key = "Functional_{0}mm".format( regressor_selector["extraction_resolution"] ) @@ -1090,362 +1155,403 @@ def create_regressor_workflow(nuisance_selectors, ) if anatomical_at_resolution_key not in pipeline_resource_pool: - anat_resample = pe.Node( interface=fsl.FLIRT(), - name='{}_flirt' - .format(anatomical_at_resolution_key), + name="{}_flirt".format(anatomical_at_resolution_key), mem_gb=3.63, - mem_x=(3767129957844731 / 1208925819614629174706176, - 'in_file') + mem_x=(3767129957844731 / 1208925819614629174706176, "in_file"), ) - anat_resample.inputs.apply_isoxfm = regressor_selector["extraction_resolution"] + anat_resample.inputs.apply_isoxfm = regressor_selector[ + "extraction_resolution" + ] - nuisance_wf.connect(*( - pipeline_resource_pool['Anatomical'] + - (anat_resample, 'in_file') - )) + nuisance_wf.connect( + *( + pipeline_resource_pool["Anatomical"] + + (anat_resample, "in_file") + ) + ) - nuisance_wf.connect(*( - pipeline_resource_pool['Anatomical'] + - (anat_resample, 'reference') - )) + nuisance_wf.connect( + *( + pipeline_resource_pool["Anatomical"] + + (anat_resample, "reference") + ) + ) - pipeline_resource_pool[anatomical_at_resolution_key] = \ - (anat_resample, 'out_file') + pipeline_resource_pool[anatomical_at_resolution_key] = ( + anat_resample, + "out_file", + ) if functional_at_resolution_key not in pipeline_resource_pool: - func_resample = pe.Node( interface=fsl.FLIRT(), - name='{}_flirt' - .format(functional_at_resolution_key), + name="{}_flirt".format(functional_at_resolution_key), mem_gb=0.521, - mem_x=(4394984950818853 / 302231454903657293676544, - 'in_file') + mem_x=(4394984950818853 / 302231454903657293676544, "in_file"), ) func_resample.inputs.apply_xfm = True - nuisance_wf.connect(*( - pipeline_resource_pool['Transformations']['func_to_anat_linear_xfm'] + - (func_resample, 'in_matrix_file') - )) + nuisance_wf.connect( + *( + pipeline_resource_pool["Transformations"][ + "func_to_anat_linear_xfm" + ] + + (func_resample, "in_matrix_file") + ) + ) - nuisance_wf.connect(*( - pipeline_resource_pool['Functional'] + - (func_resample, 'in_file') - )) + nuisance_wf.connect( + *( + pipeline_resource_pool["Functional"] + + (func_resample, "in_file") + ) + ) - nuisance_wf.connect(*( - pipeline_resource_pool[anatomical_at_resolution_key] + - (func_resample, 'reference') - )) + nuisance_wf.connect( + *( + pipeline_resource_pool[anatomical_at_resolution_key] + + (func_resample, "reference") + ) + ) - pipeline_resource_pool[functional_at_resolution_key] = \ - (func_resample, 'out_file') + pipeline_resource_pool[functional_at_resolution_key] = ( + func_resample, + "out_file", + ) # Create merger to summarize the functional timeseries regressor_mask_file_resource_keys = [] - for tissue in regressor_descriptor['tissue']: - + for tissue in regressor_descriptor["tissue"]: # Ignore non tissue masks - if tissue not in tissues and \ - not tissue.startswith('FunctionalVariance'): + if tissue not in tissues and not tissue.startswith( + "FunctionalVariance" + ): regressor_mask_file_resource_keys += [tissue] continue tissue_regressor_descriptor = regressor_descriptor.copy() - tissue_regressor_descriptor['tissue'] = tissue + tissue_regressor_descriptor["tissue"] = tissue # Generate resource masks - (pipeline_resource_pool, - regressor_mask_file_resource_key) = \ - generate_summarize_tissue_mask( - nuisance_wf, - pipeline_resource_pool, - tissue_regressor_descriptor, - regressor_selector, - csf_mask_exist, - use_ants=use_ants, - ventricle_mask_exist=ventricle_mask_exist, - all_bold=all_bold - ) + ( + pipeline_resource_pool, + regressor_mask_file_resource_key, + ) = generate_summarize_tissue_mask( + nuisance_wf, + pipeline_resource_pool, + tissue_regressor_descriptor, + regressor_selector, + csf_mask_exist, + use_ants=use_ants, + ventricle_mask_exist=ventricle_mask_exist, + all_bold=all_bold, + ) - regressor_mask_file_resource_keys += \ - [regressor_mask_file_resource_key] + regressor_mask_file_resource_keys += [regressor_mask_file_resource_key] # Keep tissues ordered, to avoid duplicates - regressor_mask_file_resource_keys = \ - list(sorted(regressor_mask_file_resource_keys)) + regressor_mask_file_resource_keys = sorted( + regressor_mask_file_resource_keys + ) # Create key for the final regressors - regressor_file_resource_key = "_".join([ - "-".join(regressor_descriptor[key]) - if type(regressor_descriptor[key]) == list - else regressor_descriptor[key] - - for key in ['tissue', 'resolution', 'erosion', 'extraction'] - if key in regressor_descriptor - ]) + regressor_file_resource_key = "_".join( + [ + "-".join(regressor_descriptor[key]) + if type(regressor_descriptor[key]) == list + else regressor_descriptor[key] + for key in ["tissue", "resolution", "erosion", "extraction"] + if key in regressor_descriptor + ] + ) if regressor_file_resource_key not in pipeline_resource_pool: - # Merge mask paths to extract voxel timeseries merge_masks_paths = pe.Node( util.Merge(len(regressor_mask_file_resource_keys)), - name='{}_merge_masks'.format(regressor_type) + name="{}_merge_masks".format(regressor_type), ) - for i, regressor_mask_file_resource_key in \ - enumerate(regressor_mask_file_resource_keys): - - node, node_output = \ - pipeline_resource_pool[regressor_mask_file_resource_key] + for i, regressor_mask_file_resource_key in enumerate( + regressor_mask_file_resource_keys + ): + node, node_output = pipeline_resource_pool[ + regressor_mask_file_resource_key + ] nuisance_wf.connect( - node, node_output, - merge_masks_paths, "in{}".format(i + 1) + node, node_output, merge_masks_paths, "in{}".format(i + 1) ) union_masks_paths = pe.Node( - MaskTool(outputtype='NIFTI_GZ'), - name='{}_union_masks'.format(regressor_type), + MaskTool(outputtype="NIFTI_GZ"), + name="{}_union_masks".format(regressor_type), mem_gb=2.1, - mem_x=(1708448960473801 / 1208925819614629174706176, - 'in_files') + mem_x=(1708448960473801 / 1208925819614629174706176, "in_files"), ) nuisance_wf.connect( - merge_masks_paths, 'out', - union_masks_paths, 'in_files' + merge_masks_paths, "out", union_masks_paths, "in_files" ) - functional_key = 'Functional' - if regressor_selector.get('extraction_resolution') and \ - regressor_selector["extraction_resolution"] != "Functional": - - functional_key = 'Functional_{}mm'.format( - regressor_selector['extraction_resolution'] + functional_key = "Functional" + if ( + regressor_selector.get("extraction_resolution") + and regressor_selector["extraction_resolution"] != "Functional" + ): + functional_key = "Functional_{}mm".format( + regressor_selector["extraction_resolution"] ) - summary_filter = regressor_selector['summary'].get('filter', '') + summary_filter = regressor_selector["summary"].get("filter", "") summary_filter_input = pipeline_resource_pool[functional_key] - summary_method = regressor_selector['summary']['method'] + summary_method = regressor_selector["summary"]["method"] summary_method_input = pipeline_resource_pool[functional_key] - if 'DetrendPC' in summary_method: - - compcor_imports = ['import os', - 'import scipy.signal as signal', - 'import nibabel as nb', - 'import numpy as np', - 'from CPAC.utils import safe_shape'] - - compcor_node = pe.Node(Function(input_names=['data_filename', - 'num_components', - 'mask_filename'], - output_names=[ - 'compcor_file'], - function=calc_compcor_components, - imports=compcor_imports), - name='{}_DetrendPC'.format(regressor_type), - mem_gb=0.4, - mem_x=(3811976743057169 / - 151115727451828646838272, - 'data_filename')) - - compcor_node.inputs.num_components = regressor_selector['summary']['components'] + if "DetrendPC" in summary_method: + compcor_imports = [ + "import os", + "import scipy.signal as signal", + "import nibabel as nb", + "import numpy as np", + "from CPAC.utils import safe_shape", + ] + + compcor_node = pe.Node( + Function( + input_names=[ + "data_filename", + "num_components", + "mask_filename", + ], + output_names=["compcor_file"], + function=calc_compcor_components, + imports=compcor_imports, + ), + name="{}_DetrendPC".format(regressor_type), + mem_gb=0.4, + mem_x=( + 3811976743057169 / 151115727451828646838272, + "data_filename", + ), + ) + + compcor_node.inputs.num_components = regressor_selector["summary"][ + "components" + ] nuisance_wf.connect( - summary_method_input[0], summary_method_input[1], - compcor_node, 'data_filename' + summary_method_input[0], + summary_method_input[1], + compcor_node, + "data_filename", ) nuisance_wf.connect( - union_masks_paths, 'out_file', - compcor_node, 'mask_filename' + union_masks_paths, "out_file", compcor_node, "mask_filename" ) - summary_method_input = (compcor_node, 'compcor_file') + summary_method_input = (compcor_node, "compcor_file") else: - if 'cosine' in summary_filter: - cosfilter_imports = ['import os', - 'import numpy as np', - 'import nibabel as nb', - 'from nipype import logging'] + if "cosine" in summary_filter: + cosfilter_imports = [ + "import os", + "import numpy as np", + "import nibabel as nb", + "from nipype import logging", + ] cosfilter_node = pe.Node( - util.Function(input_names=['input_image_path', - 'timestep'], - output_names=['cosfiltered_img'], - function=cosine_filter, - imports=cosfilter_imports), - name='{}_cosine_filter'.format(regressor_type), - mem_gb=8.0) - nuisance_wf.connect( - summary_filter_input[0], summary_filter_input[1], - cosfilter_node, 'input_image_path' + util.Function( + input_names=["input_image_path", "timestep"], + output_names=["cosfiltered_img"], + function=cosine_filter, + imports=cosfilter_imports, + ), + name="{}_cosine_filter".format(regressor_type), + mem_gb=8.0, ) - tr_string2float_node = pe.Node(util.Function(input_names=['tr'], - output_names=[ - 'tr_float'], - function=TR_string_to_float), - name='{}_tr_string2float'.format(regressor_type)) - nuisance_wf.connect( - inputspec, 'tr', - tr_string2float_node, 'tr' + summary_filter_input[0], + summary_filter_input[1], + cosfilter_node, + "input_image_path", + ) + tr_string2float_node = pe.Node( + util.Function( + input_names=["tr"], + output_names=["tr_float"], + function=TR_string_to_float, + ), + name="{}_tr_string2float".format(regressor_type), ) + nuisance_wf.connect(inputspec, "tr", tr_string2float_node, "tr") + nuisance_wf.connect( - tr_string2float_node, 'tr_float', - cosfilter_node, 'timestep' + tr_string2float_node, "tr_float", cosfilter_node, "timestep" ) - summary_method_input = ( - cosfilter_node, 'cosfiltered_img') - - if 'Detrend' in summary_method: + summary_method_input = (cosfilter_node, "cosfiltered_img") + if "Detrend" in summary_method: detrend_node = pe.Node( - afni.Detrend(args='-polort 1', outputtype='NIFTI'), - name='{}_detrend'.format(regressor_type) + afni.Detrend(args="-polort 1", outputtype="NIFTI"), + name="{}_detrend".format(regressor_type), ) nuisance_wf.connect( - summary_method_input[0], summary_method_input[1], - detrend_node, 'in_file' + summary_method_input[0], + summary_method_input[1], + detrend_node, + "in_file", ) - summary_method_input = (detrend_node, 'out_file') - - if 'Norm' in summary_method: + summary_method_input = (detrend_node, "out_file") + if "Norm" in summary_method: l2norm_node = pe.Node( - afni.TStat(args='-l2norm', outputtype='NIFTI'), - name='{}_l2norm'.format(regressor_type) + afni.TStat(args="-l2norm", outputtype="NIFTI"), + name="{}_l2norm".format(regressor_type), ) nuisance_wf.connect( - summary_method_input[0], summary_method_input[1], - l2norm_node, 'in_file' + summary_method_input[0], + summary_method_input[1], + l2norm_node, + "in_file", ) nuisance_wf.connect( - union_masks_paths, 'out_file', - l2norm_node, 'mask' + union_masks_paths, "out_file", l2norm_node, "mask" ) norm_node = pe.Node( - afni.Calc(expr='a/b', outputtype='NIFTI'), - name='{}_norm'.format(regressor_type), + afni.Calc(expr="a/b", outputtype="NIFTI"), + name="{}_norm".format(regressor_type), mem_gb=1.7, - mem_x=(1233286593342025 / - 151115727451828646838272, - 'in_file_a') + mem_x=( + 1233286593342025 / 151115727451828646838272, + "in_file_a", + ), ) nuisance_wf.connect( - summary_method_input[0], summary_method_input[1], - norm_node, 'in_file_a' + summary_method_input[0], + summary_method_input[1], + norm_node, + "in_file_a", ) nuisance_wf.connect( - l2norm_node, 'out_file', - norm_node, 'in_file_b' + l2norm_node, "out_file", norm_node, "in_file_b" ) - summary_method_input = (norm_node, 'out_file') - - if 'Mean' in summary_method: + summary_method_input = (norm_node, "out_file") + if "Mean" in summary_method: mean_node = pe.Node( - afni.ROIStats(quiet=False, args='-1Dformat'), - name='{}_mean'.format(regressor_type), - mem_gb=5.0 + afni.ROIStats(quiet=False, args="-1Dformat"), + name="{}_mean".format(regressor_type), + mem_gb=5.0, ) nuisance_wf.connect( - summary_method_input[0], summary_method_input[1], - mean_node, 'in_file' + summary_method_input[0], + summary_method_input[1], + mean_node, + "in_file", ) nuisance_wf.connect( - union_masks_paths, 'out_file', - mean_node, 'mask_file' + union_masks_paths, "out_file", mean_node, "mask_file" ) - summary_method_input = (mean_node, 'out_file') - - if 'PC' in summary_method: + summary_method_input = (mean_node, "out_file") + if "PC" in summary_method: std_node = pe.Node( - afni.TStat(args='-nzstdev', outputtype='NIFTI'), - name='{}_std'.format(regressor_type) + afni.TStat(args="-nzstdev", outputtype="NIFTI"), + name="{}_std".format(regressor_type), ) nuisance_wf.connect( - summary_method_input[0], summary_method_input[1], - std_node, 'in_file' + summary_method_input[0], + summary_method_input[1], + std_node, + "in_file", ) nuisance_wf.connect( - union_masks_paths, 'out_file', - std_node, 'mask' + union_masks_paths, "out_file", std_node, "mask" ) standardized_node = pe.Node( - afni.Calc(expr='a/b', outputtype='NIFTI'), - name='{}_standardized'.format(regressor_type) + afni.Calc(expr="a/b", outputtype="NIFTI"), + name="{}_standardized".format(regressor_type), ) nuisance_wf.connect( - summary_method_input[0], summary_method_input[1], - standardized_node, 'in_file_a' + summary_method_input[0], + summary_method_input[1], + standardized_node, + "in_file_a", ) nuisance_wf.connect( - std_node, 'out_file', - standardized_node, 'in_file_b' + std_node, "out_file", standardized_node, "in_file_b" ) pc_node = pe.Node( - PC(args='-vmean -nscale', pcs=regressor_selector['summary']['components'], outputtype='NIFTI_GZ'), - name='{}_pc'.format(regressor_type) + PC( + args="-vmean -nscale", + pcs=regressor_selector["summary"]["components"], + outputtype="NIFTI_GZ", + ), + name="{}_pc".format(regressor_type), ) nuisance_wf.connect( - standardized_node, 'out_file', - pc_node, 'in_file' + standardized_node, "out_file", pc_node, "in_file" ) nuisance_wf.connect( - union_masks_paths, 'out_file', - pc_node, 'mask' + union_masks_paths, "out_file", pc_node, "mask" ) - summary_method_input = (pc_node, 'pcs_file') + summary_method_input = (pc_node, "pcs_file") - pipeline_resource_pool[regressor_file_resource_key] = \ - summary_method_input + pipeline_resource_pool[ + regressor_file_resource_key + ] = summary_method_input # Add it to internal resource pool - regressor_resource[1] = \ - pipeline_resource_pool[regressor_file_resource_key] + regressor_resource[1] = pipeline_resource_pool[ + regressor_file_resource_key + ] # Build regressors and combine them into a single file - build_nuisance_regressors = pe.Node(Function( - input_names=['functional_file_path', - 'selector', - 'grey_matter_summary_file_path', - 'white_matter_summary_file_path', - 'csf_summary_file_path', - 'acompcor_file_path', - 'tcompcor_file_path', - 'global_summary_file_path', - 'motion_parameters_file_path', - 'custom_file_paths', - 'censor_file_path'], - output_names=['out_file', 'censor_indices'], - function=gather_nuisance, - as_module=True - ), name="build_nuisance_regressors") + build_nuisance_regressors = pe.Node( + Function( + input_names=[ + "functional_file_path", + "selector", + "grey_matter_summary_file_path", + "white_matter_summary_file_path", + "csf_summary_file_path", + "acompcor_file_path", + "tcompcor_file_path", + "global_summary_file_path", + "motion_parameters_file_path", + "custom_file_paths", + "censor_file_path", + ], + output_names=["out_file", "censor_indices"], + function=gather_nuisance, + as_module=True, + ), + name="build_nuisance_regressors", + ) nuisance_wf.connect( - inputspec, 'functional_file_path', - build_nuisance_regressors, 'functional_file_path' + inputspec, + "functional_file_path", + build_nuisance_regressors, + "functional_file_path", ) build_nuisance_regressors.inputs.selector = nuisance_selectors @@ -1453,332 +1559,385 @@ def create_regressor_workflow(nuisance_selectors, # Check for any regressors to combine into files has_nuisance_regressors = any( regressor_node - for regressor_key, (regressor_arg, regressor_node, regressor_target) - in regressors.items() - if regressor_target == 'ort' + for regressor_key, ( + regressor_arg, + regressor_node, + regressor_target, + ) in regressors.items() + if regressor_target == "ort" ) if has_nuisance_regressors: for regressor_key, ( - regressor_arg, regressor_node, regressor_target + regressor_arg, + regressor_node, + regressor_target, ) in regressors.items(): - if regressor_target != 'ort': + if regressor_target != "ort": continue if regressor_key in nuisance_selectors: nuisance_wf.connect( - regressor_node[0], regressor_node[1], - build_nuisance_regressors, regressor_arg + regressor_node[0], + regressor_node[1], + build_nuisance_regressors, + regressor_arg, ) # Check for any regressors to combine into files has_voxel_nuisance_regressors = any( regressor_node for regressor_key, ( - regressor_arg, regressor_node, regressor_target + regressor_arg, + regressor_node, + regressor_target, ) in regressors.items() - if regressor_target == 'dsort' + if regressor_target == "dsort" ) if has_voxel_nuisance_regressors: - voxel_nuisance_regressors = [ (regressor_key, (regressor_arg, regressor_node, regressor_target)) for regressor_key, ( - regressor_arg, regressor_node, regressor_target + regressor_arg, + regressor_node, + regressor_target, ) in regressors.items() - if regressor_target == 'dsort' + if regressor_target == "dsort" ] voxel_nuisance_regressors_merge = pe.Node( util.Merge(len(voxel_nuisance_regressors)), - name='voxel_nuisance_regressors_merge' + name="voxel_nuisance_regressors_merge", ) - for i, (regressor_key, ( - regressor_arg, regressor_node, regressor_target) + for i, ( + regressor_key, + (regressor_arg, regressor_node, regressor_target), ) in enumerate(voxel_nuisance_regressors): - - if regressor_target != 'dsort': + if regressor_target != "dsort": continue node, node_output = regressor_node nuisance_wf.connect( - node, node_output, - voxel_nuisance_regressors_merge, "in{}".format(i + 1) + node, node_output, voxel_nuisance_regressors_merge, "in{}".format(i + 1) ) - nuisance_wf.connect([(build_nuisance_regressors, outputspec, [ - ('out_file', 'regressors_file_path'), - ('censor_indices', 'censor_indices')])]) + nuisance_wf.connect( + [ + ( + build_nuisance_regressors, + outputspec, + [ + ("out_file", "regressors_file_path"), + ("censor_indices", "censor_indices"), + ], + ) + ] + ) return nuisance_wf -def create_nuisance_regression_workflow(nuisance_selectors, - name='nuisance_regression'): - inputspec = pe.Node(util.IdentityInterface(fields=[ - 'selector', - 'functional_file_path', - 'functional_brain_mask_file_path', - 'regressor_file', - 'fd_j_file_path', - 'fd_p_file_path', - 'dvars_file_path' - ]), name='inputspec') +def create_nuisance_regression_workflow(nuisance_selectors, name="nuisance_regression"): + inputspec = pe.Node( + util.IdentityInterface( + fields=[ + "selector", + "functional_file_path", + "functional_brain_mask_file_path", + "regressor_file", + "fd_j_file_path", + "fd_p_file_path", + "dvars_file_path", + ] + ), + name="inputspec", + ) - outputspec = pe.Node(util.IdentityInterface(fields=['residual_file_path']), - name='outputspec') + outputspec = pe.Node( + util.IdentityInterface(fields=["residual_file_path"]), name="outputspec" + ) nuisance_wf = pe.Workflow(name=name) - if nuisance_selectors.get('Censor'): - - censor_methods = ['Kill', 'Zero', 'Interpolate', 'SpikeRegression'] - - censor_selector = nuisance_selectors.get('Censor') - if censor_selector.get('method') not in censor_methods: - raise ValueError("Improper censoring method specified ({0}), " - "should be one of {1}." - .format(censor_selector.get('method'), - censor_methods)) - - find_censors = pe.Node(Function( - input_names=['fd_j_file_path', - 'fd_j_threshold', - 'fd_p_file_path', - 'fd_p_threshold', - 'dvars_file_path', - 'dvars_threshold', - 'number_of_previous_trs_to_censor', - 'number_of_subsequent_trs_to_censor'], - output_names=['out_file'], - function=find_offending_time_points, - as_module=True - ), name="find_offending_time_points") - - if not censor_selector.get('thresholds'): + if nuisance_selectors.get("Censor"): + censor_methods = ["Kill", "Zero", "Interpolate", "SpikeRegression"] + + censor_selector = nuisance_selectors.get("Censor") + if censor_selector.get("method") not in censor_methods: raise ValueError( - 'Censoring requested, but thresh_metric not provided.' + "Improper censoring method specified ({0}), " + "should be one of {1}.".format( + censor_selector.get("method"), censor_methods + ) ) - for threshold in censor_selector['thresholds']: + find_censors = pe.Node( + Function( + input_names=[ + "fd_j_file_path", + "fd_j_threshold", + "fd_p_file_path", + "fd_p_threshold", + "dvars_file_path", + "dvars_threshold", + "number_of_previous_trs_to_censor", + "number_of_subsequent_trs_to_censor", + ], + output_names=["out_file"], + function=find_offending_time_points, + as_module=True, + ), + name="find_offending_time_points", + ) - if 'type' not in threshold or threshold['type'] not in ['DVARS', 'FD_J', 'FD_P']: - raise ValueError( - 'Censoring requested, but with invalid threshold type.' - ) + if not censor_selector.get("thresholds"): + raise ValueError("Censoring requested, but thresh_metric not provided.") - if 'value' not in threshold: + for threshold in censor_selector["thresholds"]: + if "type" not in threshold or threshold["type"] not in [ + "DVARS", + "FD_J", + "FD_P", + ]: raise ValueError( - 'Censoring requested, but threshold not provided.' + "Censoring requested, but with invalid threshold type." ) - if threshold['type'] == 'FD_J': - find_censors.inputs.fd_j_threshold = threshold['value'] - nuisance_wf.connect(inputspec, "fd_j_file_path", - find_censors, "fd_j_file_path") + if "value" not in threshold: + raise ValueError("Censoring requested, but threshold not provided.") - if threshold['type'] == 'FD_P': - find_censors.inputs.fd_p_threshold = threshold['value'] - nuisance_wf.connect(inputspec, "fd_p_file_path", - find_censors, "fd_p_file_path") + if threshold["type"] == "FD_J": + find_censors.inputs.fd_j_threshold = threshold["value"] + nuisance_wf.connect( + inputspec, "fd_j_file_path", find_censors, "fd_j_file_path" + ) - if threshold['type'] == 'DVARS': - find_censors.inputs.dvars_threshold = threshold['value'] - nuisance_wf.connect(inputspec, "dvars_file_path", - find_censors, "dvars_file_path") + if threshold["type"] == "FD_P": + find_censors.inputs.fd_p_threshold = threshold["value"] + nuisance_wf.connect( + inputspec, "fd_p_file_path", find_censors, "fd_p_file_path" + ) - if censor_selector.get('number_of_previous_trs_to_censor') and \ - censor_selector['method'] != 'SpikeRegression': + if threshold["type"] == "DVARS": + find_censors.inputs.dvars_threshold = threshold["value"] + nuisance_wf.connect( + inputspec, "dvars_file_path", find_censors, "dvars_file_path" + ) - find_censors.inputs.number_of_previous_trs_to_censor = \ - censor_selector['number_of_previous_trs_to_censor'] + if ( + censor_selector.get("number_of_previous_trs_to_censor") + and censor_selector["method"] != "SpikeRegression" + ): + find_censors.inputs.number_of_previous_trs_to_censor = censor_selector[ + "number_of_previous_trs_to_censor" + ] else: find_censors.inputs.number_of_previous_trs_to_censor = 0 - if censor_selector.get('number_of_subsequent_trs_to_censor') and \ - censor_selector['method'] != 'SpikeRegression': - - find_censors.inputs.number_of_subsequent_trs_to_censor = \ - censor_selector['number_of_subsequent_trs_to_censor'] + if ( + censor_selector.get("number_of_subsequent_trs_to_censor") + and censor_selector["method"] != "SpikeRegression" + ): + find_censors.inputs.number_of_subsequent_trs_to_censor = censor_selector[ + "number_of_subsequent_trs_to_censor" + ] else: find_censors.inputs.number_of_subsequent_trs_to_censor = 0 # Use 3dTproject to perform nuisance variable regression - nuisance_regression = pe.Node(interface=afni.TProject(), - name='nuisance_regression', - mem_gb=1.716, - mem_x=(6278549929741219 / - 604462909807314587353088, 'in_file')) - - nuisance_regression.inputs.out_file = 'residuals.nii.gz' - nuisance_regression.inputs.outputtype = 'NIFTI_GZ' + nuisance_regression = pe.Node( + interface=afni.TProject(), + name="nuisance_regression", + mem_gb=1.716, + mem_x=(6278549929741219 / 604462909807314587353088, "in_file"), + ) + + nuisance_regression.inputs.out_file = "residuals.nii.gz" + nuisance_regression.inputs.outputtype = "NIFTI_GZ" nuisance_regression.inputs.norm = False - if nuisance_selectors.get('Censor'): - if nuisance_selectors['Censor']['method'] == 'SpikeRegression': - nuisance_wf.connect(find_censors, 'out_file', - nuisance_regression, 'censor') + if nuisance_selectors.get("Censor"): + if nuisance_selectors["Censor"]["method"] == "SpikeRegression": + nuisance_wf.connect(find_censors, "out_file", nuisance_regression, "censor") else: - if nuisance_selectors['Censor']['method'] == 'Interpolate': - nuisance_regression.inputs.cenmode = 'NTRP' + if nuisance_selectors["Censor"]["method"] == "Interpolate": + nuisance_regression.inputs.cenmode = "NTRP" else: - nuisance_regression.inputs.cenmode = \ - nuisance_selectors['Censor']['method'].upper() + nuisance_regression.inputs.cenmode = nuisance_selectors["Censor"][ + "method" + ].upper() - nuisance_wf.connect(find_censors, 'out_file', - nuisance_regression, 'censor') + nuisance_wf.connect(find_censors, "out_file", nuisance_regression, "censor") - if nuisance_selectors.get('PolyOrt'): - if not nuisance_selectors['PolyOrt'].get('degree'): - raise ValueError("Polynomial orthogonalization requested, " - "but degree not provided.") + if nuisance_selectors.get("PolyOrt"): + if not nuisance_selectors["PolyOrt"].get("degree"): + raise ValueError( + "Polynomial orthogonalization requested, " "but degree not provided." + ) - nuisance_regression.inputs.polort = \ - nuisance_selectors['PolyOrt']['degree'] + nuisance_regression.inputs.polort = nuisance_selectors["PolyOrt"]["degree"] else: nuisance_regression.inputs.polort = 0 - nuisance_wf.connect(inputspec, 'functional_file_path', - nuisance_regression, 'in_file') + nuisance_wf.connect( + inputspec, "functional_file_path", nuisance_regression, "in_file" + ) - nuisance_wf.connect(inputspec, 'functional_brain_mask_file_path', - nuisance_regression, 'mask') + nuisance_wf.connect( + inputspec, "functional_brain_mask_file_path", nuisance_regression, "mask" + ) - if nuisance_selectors.get('Custom'): - if nuisance_selectors['Custom'][0].get('file'): - if nuisance_selectors['Custom'][0]['file'].endswith('.nii') or \ - nuisance_selectors['Custom'][0]['file'].endswith('.nii.gz'): - nuisance_wf.connect(inputspec, 'regressor_file', - nuisance_regression, 'dsort') + if nuisance_selectors.get("Custom"): + if nuisance_selectors["Custom"][0].get("file"): + if nuisance_selectors["Custom"][0]["file"].endswith( + ".nii" + ) or nuisance_selectors["Custom"][0]["file"].endswith(".nii.gz"): + nuisance_wf.connect( + inputspec, "regressor_file", nuisance_regression, "dsort" + ) else: - nuisance_wf.connect(inputspec, 'regressor_file', - nuisance_regression, 'ort') + nuisance_wf.connect( + inputspec, "regressor_file", nuisance_regression, "ort" + ) else: - nuisance_wf.connect(inputspec, 'regressor_file', - nuisance_regression, 'ort') + nuisance_wf.connect(inputspec, "regressor_file", nuisance_regression, "ort") else: # there's no regressor file generated if only Bandpass in nuisance_selectors - if not ('Bandpass' in nuisance_selectors and len(nuisance_selectors.keys()) == 1): - nuisance_wf.connect(inputspec, 'regressor_file', - nuisance_regression, 'ort') + if not ( + "Bandpass" in nuisance_selectors and len(nuisance_selectors.keys()) == 1 + ): + nuisance_wf.connect(inputspec, "regressor_file", nuisance_regression, "ort") - nuisance_wf.connect(nuisance_regression, 'out_file', - outputspec, 'residual_file_path') + nuisance_wf.connect( + nuisance_regression, "out_file", outputspec, "residual_file_path" + ) return nuisance_wf -def filtering_bold_and_regressors(nuisance_selectors, - name='filtering_bold_and_regressors'): - - inputspec = pe.Node(util.IdentityInterface(fields=[ - 'functional_file_path', - 'regressors_file_path', - 'functional_brain_mask_file_path', - 'nuisance_selectors', - 'tr' - ]), name='inputspec') +def filtering_bold_and_regressors( + nuisance_selectors, name="filtering_bold_and_regressors" +): + inputspec = pe.Node( + util.IdentityInterface( + fields=[ + "functional_file_path", + "regressors_file_path", + "functional_brain_mask_file_path", + "nuisance_selectors", + "tr", + ] + ), + name="inputspec", + ) - outputspec = pe.Node(util.IdentityInterface(fields=['residual_file_path', - 'residual_regressor']), - name='outputspec') + outputspec = pe.Node( + util.IdentityInterface(fields=["residual_file_path", "residual_regressor"]), + name="outputspec", + ) filtering_wf = pe.Workflow(name=name) - bandpass_selector = nuisance_selectors.get('Bandpass') + bandpass_selector = nuisance_selectors.get("Bandpass") - if bandpass_selector.get('method'): - bandpass_method = bandpass_selector.get('method') + if bandpass_selector.get("method"): + bandpass_method = bandpass_selector.get("method") else: - bandpass_method = 'default' - - if bandpass_method == 'default': + bandpass_method = "default" + if bandpass_method == "default": frequency_filter = pe.Node( - Function(input_names=['realigned_file', - 'regressor_file', - 'bandpass_freqs', - 'sample_period'], - output_names=['bandpassed_file', - 'regressor_file'], - function=bandpass_voxels, - as_module=True), - name='frequency_filter', - mem_gb=0.5, - mem_x=(3811976743057169 / 151115727451828646838272, - 'realigned_file') - ) + Function( + input_names=[ + "realigned_file", + "regressor_file", + "bandpass_freqs", + "sample_period", + ], + output_names=["bandpassed_file", "regressor_file"], + function=bandpass_voxels, + as_module=True, + ), + name="frequency_filter", + mem_gb=0.5, + mem_x=(3811976743057169 / 151115727451828646838272, "realigned_file"), + ) frequency_filter.inputs.bandpass_freqs = [ - bandpass_selector.get('bottom_frequency'), - bandpass_selector.get('top_frequency') - ] - - filtering_wf.connect(inputspec, 'functional_file_path', - frequency_filter, 'realigned_file') + bandpass_selector.get("bottom_frequency"), + bandpass_selector.get("top_frequency"), + ] - filtering_wf.connect(inputspec, 'regressors_file_path', - frequency_filter, 'regressor_file') + filtering_wf.connect( + inputspec, "functional_file_path", frequency_filter, "realigned_file" + ) - filtering_wf.connect(frequency_filter, 'bandpassed_file', - outputspec, 'residual_file_path') + filtering_wf.connect( + inputspec, "regressors_file_path", frequency_filter, "regressor_file" + ) - filtering_wf.connect(frequency_filter, 'regressor_file', - outputspec, 'residual_regressor') + filtering_wf.connect( + frequency_filter, "bandpassed_file", outputspec, "residual_file_path" + ) - elif bandpass_method == 'AFNI': + filtering_wf.connect( + frequency_filter, "regressor_file", outputspec, "residual_regressor" + ) - bandpass_ts = pe.Node(interface=afni.Bandpass(), - name='bandpass_ts') + elif bandpass_method == "AFNI": + bandpass_ts = pe.Node(interface=afni.Bandpass(), name="bandpass_ts") - bandpass_ts.inputs.highpass = bandpass_selector.get('bottom_frequency') - bandpass_ts.inputs.lowpass = bandpass_selector.get('top_frequency') - bandpass_ts.inputs.outputtype = 'NIFTI_GZ' + bandpass_ts.inputs.highpass = bandpass_selector.get("bottom_frequency") + bandpass_ts.inputs.lowpass = bandpass_selector.get("top_frequency") + bandpass_ts.inputs.outputtype = "NIFTI_GZ" - tr_string2float_node = pe.Node(util.Function(input_names=['tr'], - output_names=['tr_float'], - function=TR_string_to_float), - name='tr_string2float') + tr_string2float_node = pe.Node( + util.Function( + input_names=["tr"], + output_names=["tr_float"], + function=TR_string_to_float, + ), + name="tr_string2float", + ) - filtering_wf.connect(inputspec, 'tr', - tr_string2float_node, 'tr') + filtering_wf.connect(inputspec, "tr", tr_string2float_node, "tr") - filtering_wf.connect(tr_string2float_node, 'tr_float', - bandpass_ts, 'tr') + filtering_wf.connect(tr_string2float_node, "tr_float", bandpass_ts, "tr") - filtering_wf.connect(inputspec, 'functional_file_path', - bandpass_ts, 'in_file') + filtering_wf.connect(inputspec, "functional_file_path", bandpass_ts, "in_file") - filtering_wf.connect(inputspec, 'functional_brain_mask_file_path', - bandpass_ts, 'mask') + filtering_wf.connect( + inputspec, "functional_brain_mask_file_path", bandpass_ts, "mask" + ) - filtering_wf.connect(bandpass_ts, 'out_file', - outputspec, 'residual_file_path') + filtering_wf.connect(bandpass_ts, "out_file", outputspec, "residual_file_path") - bandpass_regressor = pe.Node(Function(input_names=['in_file', - 'highpass', - 'lowpass', - 'tr'], - output_names=['out_file'], - function=afni_1dBandpass), - name='bandpass_regressor') + bandpass_regressor = pe.Node( + Function( + input_names=["in_file", "highpass", "lowpass", "tr"], + output_names=["out_file"], + function=afni_1dBandpass, + ), + name="bandpass_regressor", + ) - bandpass_regressor.inputs.highpass = bandpass_selector.get('bottom_frequency') - bandpass_regressor.inputs.lowpass = bandpass_selector.get('top_frequency') + bandpass_regressor.inputs.highpass = bandpass_selector.get("bottom_frequency") + bandpass_regressor.inputs.lowpass = bandpass_selector.get("top_frequency") - filtering_wf.connect(inputspec, 'regressors_file_path', - bandpass_regressor, 'in_file') + filtering_wf.connect( + inputspec, "regressors_file_path", bandpass_regressor, "in_file" + ) - filtering_wf.connect(tr_string2float_node, 'tr_float', - bandpass_regressor, 'tr') + filtering_wf.connect(tr_string2float_node, "tr_float", bandpass_regressor, "tr") - filtering_wf.connect(bandpass_regressor, 'out_file', - outputspec, 'residual_regressor') + filtering_wf.connect( + bandpass_regressor, "out_file", outputspec, "residual_regressor" + ) return filtering_wf @@ -1795,37 +1954,33 @@ def filtering_bold_and_regressors(nuisance_selectors, outputs=["desc-preproc_bold", "desc-cleaned_bold"], ) def ICA_AROMA_FSLreg(wf, cfg, strat_pool, pipe_num, opt=None): - - xfm_prov = strat_pool.get_cpac_provenance('from-T1w_to-template_mode-image_xfm') + xfm_prov = strat_pool.get_cpac_provenance("from-T1w_to-template_mode-image_xfm") reg_tool = check_prov_for_regtool(xfm_prov) - if reg_tool != 'fsl': + if reg_tool != "fsl": return (wf, None) - aroma_preproc = create_aroma(tr=None, wf_name=f'create_aroma_{pipe_num}') + aroma_preproc = create_aroma(tr=None, wf_name=f"create_aroma_{pipe_num}") - aroma_preproc.inputs.params.denoise_type = \ - cfg.nuisance_corrections['1-ICA-AROMA']['denoising_type'] + aroma_preproc.inputs.params.denoise_type = cfg.nuisance_corrections["1-ICA-AROMA"][ + "denoising_type" + ] node, out = strat_pool.get_data("desc-preproc_bold") - wf.connect(node, out, aroma_preproc, 'inputspec.denoise_file') + wf.connect(node, out, aroma_preproc, "inputspec.denoise_file") - node, out = strat_pool.get_data( - "from-bold_to-T1w_mode-image_desc-linear_xfm") - wf.connect(node, out, aroma_preproc, 'inputspec.mat_file') + node, out = strat_pool.get_data("from-bold_to-T1w_mode-image_desc-linear_xfm") + wf.connect(node, out, aroma_preproc, "inputspec.mat_file") node, out = strat_pool.get_data("from-T1w_to-template_mode-image_xfm") - wf.connect(node, out, aroma_preproc, 'inputspec.fnirt_warp_file') + wf.connect(node, out, aroma_preproc, "inputspec.fnirt_warp_file") - if cfg.nuisance_corrections['1-ICA-AROMA']['denoising_type'] == 'nonaggr': - node, out = (aroma_preproc, 'outputspec.nonaggr_denoised_file') - elif cfg.nuisance_corrections['1-ICA-AROMA']['denoising_type'] == 'aggr': - node, out = (aroma_preproc, 'outputspec.aggr_denoised_file') + if cfg.nuisance_corrections["1-ICA-AROMA"]["denoising_type"] == "nonaggr": + node, out = (aroma_preproc, "outputspec.nonaggr_denoised_file") + elif cfg.nuisance_corrections["1-ICA-AROMA"]["denoising_type"] == "aggr": + node, out = (aroma_preproc, "outputspec.aggr_denoised_file") - outputs = { - 'desc-preproc_bold': (node, out), - 'desc-cleaned_bold': (node, out) - } + outputs = {"desc-preproc_bold": (node, out), "desc-cleaned_bold": (node, out)} return (wf, outputs) @@ -1846,53 +2001,54 @@ def ICA_AROMA_FSLreg(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["desc-preproc_bold", "desc-cleaned_bold"], ) def ICA_AROMA_ANTsreg(wf, cfg, strat_pool, pipe_num, opt=None): - - xfm_prov = strat_pool.get_cpac_provenance( - 'from-bold_to-template_mode-image_xfm') + xfm_prov = strat_pool.get_cpac_provenance("from-bold_to-template_mode-image_xfm") reg_tool = check_prov_for_regtool(xfm_prov) - if reg_tool != 'ants': + if reg_tool != "ants": return (wf, None) - num_cpus = cfg.pipeline_setup['system_config'][ - 'max_cores_per_participant'] + num_cpus = cfg.pipeline_setup["system_config"]["max_cores_per_participant"] - num_ants_cores = cfg.pipeline_setup['system_config']['num_ants_threads'] + num_ants_cores = cfg.pipeline_setup["system_config"]["num_ants_threads"] - aroma_preproc = create_aroma(tr=None, wf_name=f'create_aroma_{pipe_num}') - aroma_preproc.inputs.params.denoise_type = \ - cfg.nuisance_corrections['1-ICA-AROMA']['denoising_type'] + aroma_preproc = create_aroma(tr=None, wf_name=f"create_aroma_{pipe_num}") + aroma_preproc.inputs.params.denoise_type = cfg.nuisance_corrections["1-ICA-AROMA"][ + "denoising_type" + ] wf, outputs = warp_timeseries_to_T1template(wf, cfg, strat_pool, pipe_num) for key, val in outputs.items(): node, out = val - wf.connect(node, out, aroma_preproc, 'inputspec.denoise_file') + wf.connect(node, out, aroma_preproc, "inputspec.denoise_file") - apply_xfm = apply_transform(f'ICA-AROMA_ANTs_template_to_bold_{pipe_num}', - reg_tool=reg_tool, time_series=True, - num_cpus=num_cpus, - num_ants_cores=num_ants_cores) + apply_xfm = apply_transform( + f"ICA-AROMA_ANTs_template_to_bold_{pipe_num}", + reg_tool=reg_tool, + time_series=True, + num_cpus=num_cpus, + num_ants_cores=num_ants_cores, + ) apply_xfm.inputs.inputspec.interpolation = cfg.registration_workflows[ - 'functional_registration']['func_registration_to_template'][ - 'ANTs_pipelines']['interpolation'] + "functional_registration" + ]["func_registration_to_template"]["ANTs_pipelines"]["interpolation"] - if cfg.nuisance_corrections['1-ICA-AROMA']['denoising_type'] == 'nonaggr': - node, out = (aroma_preproc, 'outputspec.nonaggr_denoised_file') - elif cfg.nuisance_corrections['1-ICA-AROMA']['denoising_type'] == 'aggr': - node, out = (aroma_preproc, 'outputspec.aggr_denoised_file') + if cfg.nuisance_corrections["1-ICA-AROMA"]["denoising_type"] == "nonaggr": + node, out = (aroma_preproc, "outputspec.nonaggr_denoised_file") + elif cfg.nuisance_corrections["1-ICA-AROMA"]["denoising_type"] == "aggr": + node, out = (aroma_preproc, "outputspec.aggr_denoised_file") - wf.connect(node, out, apply_xfm, 'inputspec.input_image') + wf.connect(node, out, apply_xfm, "inputspec.input_image") node, out = strat_pool.get_data("sbref") - wf.connect(node, out, apply_xfm, 'inputspec.reference') + wf.connect(node, out, apply_xfm, "inputspec.reference") - node, out = strat_pool.get_data('from-template_to-bold_mode-image_xfm') - wf.connect(node, out, apply_xfm, 'inputspec.transform') + node, out = strat_pool.get_data("from-template_to-bold_mode-image_xfm") + wf.connect(node, out, apply_xfm, "inputspec.transform") outputs = { - 'desc-preproc_bold': (apply_xfm, 'outputspec.output_image'), - 'desc-cleaned_bold': (apply_xfm, 'outputspec.output_image') + "desc-preproc_bold": (apply_xfm, "outputspec.output_image"), + "desc-cleaned_bold": (apply_xfm, "outputspec.output_image"), } return (wf, outputs) @@ -1916,34 +2072,32 @@ def ICA_AROMA_ANTsreg(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["desc-preproc_bold", "desc-cleaned_bold"], ) def ICA_AROMA_FSLEPIreg(wf, cfg, strat_pool, pipe_num, opt=None): - - xfm_prov = strat_pool.get_cpac_provenance('from-bold_to-EPItemplate_mode-image_xfm') + xfm_prov = strat_pool.get_cpac_provenance("from-bold_to-EPItemplate_mode-image_xfm") reg_tool = check_prov_for_regtool(xfm_prov) - if reg_tool != 'fsl': + if reg_tool != "fsl": return (wf, None) - aroma_preproc = create_aroma(tr=None, wf_name=f'create_aroma_{pipe_num}') + aroma_preproc = create_aroma(tr=None, wf_name=f"create_aroma_{pipe_num}") - aroma_preproc.inputs.params.denoise_type = \ - cfg.nuisance_corrections['1-ICA-AROMA']['denoising_type'] + aroma_preproc.inputs.params.denoise_type = cfg.nuisance_corrections["1-ICA-AROMA"][ + "denoising_type" + ] - node, out = strat_pool.get_data(["desc-brain_bold", "desc-motion_bold", - "desc-preproc_bold", "bold"]) - wf.connect(node, out, aroma_preproc, 'inputspec.denoise_file') + node, out = strat_pool.get_data( + ["desc-brain_bold", "desc-motion_bold", "desc-preproc_bold", "bold"] + ) + wf.connect(node, out, aroma_preproc, "inputspec.denoise_file") node, out = strat_pool.get_data("from-bold_to-EPItemplate_mode-image_xfm") - wf.connect(node, out, aroma_preproc, 'inputspec.fnirt_warp_file') + wf.connect(node, out, aroma_preproc, "inputspec.fnirt_warp_file") - if cfg.nuisance_corrections['1-ICA-AROMA']['denoising_type'] == 'nonaggr': - node, out = (aroma_preproc, 'outputspec.nonaggr_denoised_file') - elif cfg.nuisance_corrections['1-ICA-AROMA']['denoising_type'] == 'aggr': - node, out = (aroma_preproc, 'outputspec.aggr_denoised_file') + if cfg.nuisance_corrections["1-ICA-AROMA"]["denoising_type"] == "nonaggr": + node, out = (aroma_preproc, "outputspec.nonaggr_denoised_file") + elif cfg.nuisance_corrections["1-ICA-AROMA"]["denoising_type"] == "aggr": + node, out = (aroma_preproc, "outputspec.aggr_denoised_file") - outputs = { - 'desc-preproc_bold': (node, out), - 'desc-cleaned_bold': (node, out) - } + outputs = {"desc-preproc_bold": (node, out), "desc-cleaned_bold": (node, out)} return (wf, outputs) @@ -1971,54 +2125,54 @@ def ICA_AROMA_FSLEPIreg(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["desc-preproc_bold", "desc-cleaned_bold"], ) def ICA_AROMA_ANTsEPIreg(wf, cfg, strat_pool, pipe_num, opt=None): - - xfm_prov = strat_pool.get_cpac_provenance( - 'from-bold_to-EPItemplate_mode-image_xfm') + xfm_prov = strat_pool.get_cpac_provenance("from-bold_to-EPItemplate_mode-image_xfm") reg_tool = check_prov_for_regtool(xfm_prov) - if reg_tool != 'ants': + if reg_tool != "ants": return (wf, None) - num_cpus = cfg.pipeline_setup['system_config'][ - 'max_cores_per_participant'] + num_cpus = cfg.pipeline_setup["system_config"]["max_cores_per_participant"] - num_ants_cores = cfg.pipeline_setup['system_config']['num_ants_threads'] + num_ants_cores = cfg.pipeline_setup["system_config"]["num_ants_threads"] - aroma_preproc = create_aroma(tr=None, wf_name=f'create_aroma_{pipe_num}') - aroma_preproc.inputs.params.denoise_type = \ - cfg.nuisance_corrections['1-ICA-AROMA']['denoising_type'] + aroma_preproc = create_aroma(tr=None, wf_name=f"create_aroma_{pipe_num}") + aroma_preproc.inputs.params.denoise_type = cfg.nuisance_corrections["1-ICA-AROMA"][ + "denoising_type" + ] - wf, outputs = warp_timeseries_to_EPItemplate(wf, cfg, strat_pool, - pipe_num) + wf, outputs = warp_timeseries_to_EPItemplate(wf, cfg, strat_pool, pipe_num) for key, val in outputs.items(): node, out = val - wf.connect(node, out, aroma_preproc, 'inputspec.denoise_file') + wf.connect(node, out, aroma_preproc, "inputspec.denoise_file") - apply_xfm = apply_transform(f'ICA-AROMA_ANTs_EPItemplate_to_bold_{pipe_num}', - reg_tool=reg_tool, time_series=True, - num_cpus=num_cpus, - num_ants_cores=num_ants_cores) + apply_xfm = apply_transform( + f"ICA-AROMA_ANTs_EPItemplate_to_bold_{pipe_num}", + reg_tool=reg_tool, + time_series=True, + num_cpus=num_cpus, + num_ants_cores=num_ants_cores, + ) apply_xfm.inputs.inputspec.interpolation = cfg.registration_workflows[ - 'functional_registration']['func_registration_to_template'][ - 'ANTs_pipelines']['interpolation'] + "functional_registration" + ]["func_registration_to_template"]["ANTs_pipelines"]["interpolation"] - if cfg.nuisance_corrections['1-ICA-AROMA']['denoising_type'] == 'nonaggr': - node, out = (aroma_preproc, 'outputspec.nonaggr_denoised_file') - elif cfg.nuisance_corrections['1-ICA-AROMA']['denoising_type'] == 'aggr': - node, out = (aroma_preproc, 'outputspec.aggr_denoised_file') + if cfg.nuisance_corrections["1-ICA-AROMA"]["denoising_type"] == "nonaggr": + node, out = (aroma_preproc, "outputspec.nonaggr_denoised_file") + elif cfg.nuisance_corrections["1-ICA-AROMA"]["denoising_type"] == "aggr": + node, out = (aroma_preproc, "outputspec.aggr_denoised_file") - wf.connect(node, out, apply_xfm, 'inputspec.input_image') + wf.connect(node, out, apply_xfm, "inputspec.input_image") node, out = strat_pool.get_data("sbref") - wf.connect(node, out, apply_xfm, 'inputspec.reference') + wf.connect(node, out, apply_xfm, "inputspec.reference") - node, out = strat_pool.get_data('from-EPItemplate_to-bold_mode-image_xfm') - wf.connect(node, out, apply_xfm, 'inputspec.transform') + node, out = strat_pool.get_data("from-EPItemplate_to-bold_mode-image_xfm") + wf.connect(node, out, apply_xfm, "inputspec.transform") outputs = { - 'desc-preproc_bold': (apply_xfm, 'outputspec.output_image'), - 'desc-cleaned_bold': (apply_xfm, 'outputspec.output_image') + "desc-preproc_bold": (apply_xfm, "outputspec.output_image"), + "desc-cleaned_bold": (apply_xfm, "outputspec.output_image"), } return (wf, outputs) @@ -2042,25 +2196,21 @@ def ICA_AROMA_ANTsEPIreg(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["space-T1w_desc-eroded_mask"], ) def erode_mask_T1w(wf, cfg, strat_pool, pipe_num, opt=None): - - erode = erode_mask(f'erode_T1w_mask_{pipe_num}', segmentmap=False) + erode = erode_mask(f"erode_T1w_mask_{pipe_num}", segmentmap=False) erode.inputs.inputspec.mask_erode_mm = cfg.nuisance_corrections[ - '2-nuisance_regression']['regressor_masks'][ - 'erode_anatomical_brain_mask']['brain_mask_erosion_mm'] + "2-nuisance_regression" + ]["regressor_masks"]["erode_anatomical_brain_mask"]["brain_mask_erosion_mm"] erode.inputs.inputspec.erode_prop = cfg.nuisance_corrections[ - '2-nuisance_regression']['regressor_masks'][ - 'erode_anatomical_brain_mask']['brain_mask_erosion_prop'] + "2-nuisance_regression" + ]["regressor_masks"]["erode_anatomical_brain_mask"]["brain_mask_erosion_prop"] - node, out = strat_pool.get_data('space-T1w_desc-brain_mask') - wf.connect(node, out, erode, 'inputspec.brain_mask') + node, out = strat_pool.get_data("space-T1w_desc-brain_mask") + wf.connect(node, out, erode, "inputspec.brain_mask") - node, out = strat_pool.get_data(['label-CSF_desc-preproc_mask', - 'label-CSF_mask']) - wf.connect(node, out, erode, 'inputspec.mask') + node, out = strat_pool.get_data(["label-CSF_desc-preproc_mask", "label-CSF_mask"]) + wf.connect(node, out, erode, "inputspec.mask") - outputs = { - 'space-T1w_desc-eroded_mask': (erode, 'outputspec.eroded_mask') - } + outputs = {"space-T1w_desc-eroded_mask": (erode, "outputspec.eroded_mask")} return (wf, outputs) @@ -2083,29 +2233,25 @@ def erode_mask_T1w(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["label-CSF_desc-eroded_mask"], ) def erode_mask_CSF(wf, cfg, strat_pool, pipe_num, opt=None): - - erode = erode_mask(f'erode_CSF_mask_{pipe_num}') - erode.inputs.inputspec.erode_mm = cfg.nuisance_corrections[ - '2-nuisance_regression']['regressor_masks']['erode_csf'][ - 'csf_erosion_mm'] + erode = erode_mask(f"erode_CSF_mask_{pipe_num}") + erode.inputs.inputspec.erode_mm = cfg.nuisance_corrections["2-nuisance_regression"][ + "regressor_masks" + ]["erode_csf"]["csf_erosion_mm"] erode.inputs.inputspec.erode_prop = cfg.nuisance_corrections[ - '2-nuisance_regression']['regressor_masks']['erode_csf'][ - 'csf_erosion_prop'] + "2-nuisance_regression" + ]["regressor_masks"]["erode_csf"]["csf_erosion_prop"] erode.inputs.inputspec.mask_erode_mm = cfg.nuisance_corrections[ - '2-nuisance_regression']['regressor_masks']['erode_csf'][ - 'csf_mask_erosion_mm'] + "2-nuisance_regression" + ]["regressor_masks"]["erode_csf"]["csf_mask_erosion_mm"] - node, out = strat_pool.get_data(['label-CSF_desc-preproc_mask', - 'label-CSF_mask']) - wf.connect(node, out, erode, 'inputspec.mask') + node, out = strat_pool.get_data(["label-CSF_desc-preproc_mask", "label-CSF_mask"]) + wf.connect(node, out, erode, "inputspec.mask") - node, out = strat_pool.get_data('space-T1w_desc-brain_mask') - wf.connect(node, out, erode, 'inputspec.brain_mask') + node, out = strat_pool.get_data("space-T1w_desc-brain_mask") + wf.connect(node, out, erode, "inputspec.brain_mask") - outputs = { - 'label-CSF_desc-eroded_mask': (erode, 'outputspec.eroded_mask') - } + outputs = {"label-CSF_desc-eroded_mask": (erode, "outputspec.eroded_mask")} return (wf, outputs) @@ -2126,26 +2272,22 @@ def erode_mask_CSF(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["label-GM_desc-eroded_mask"], ) def erode_mask_GM(wf, cfg, strat_pool, pipe_num, opt=None): - - erode = erode_mask(f'erode_GM_mask_{pipe_num}') - erode.inputs.inputspec.erode_mm = cfg.nuisance_corrections[ - '2-nuisance_regression']['regressor_masks']['erode_gm'][ - 'gm_erosion_mm'] + erode = erode_mask(f"erode_GM_mask_{pipe_num}") + erode.inputs.inputspec.erode_mm = cfg.nuisance_corrections["2-nuisance_regression"][ + "regressor_masks" + ]["erode_gm"]["gm_erosion_mm"] erode.inputs.inputspec.erode_prop = cfg.nuisance_corrections[ - '2-nuisance_regression']['regressor_masks']['erode_gm'][ - 'gm_erosion_prop'] + "2-nuisance_regression" + ]["regressor_masks"]["erode_gm"]["gm_erosion_prop"] erode.inputs.inputspec.mask_erode_mm = cfg.nuisance_corrections[ - '2-nuisance_regression']['regressor_masks']['erode_gm'][ - 'gm_mask_erosion_mm'] + "2-nuisance_regression" + ]["regressor_masks"]["erode_gm"]["gm_mask_erosion_mm"] - node, out = strat_pool.get_data(['label-GM_desc-preproc_mask', - 'label-GM_mask']) - wf.connect(node, out, erode, 'inputspec.mask') + node, out = strat_pool.get_data(["label-GM_desc-preproc_mask", "label-GM_mask"]) + wf.connect(node, out, erode, "inputspec.mask") - outputs = { - 'label-GM_desc-eroded_mask': (erode, 'outputspec.eroded_mask') - } + outputs = {"label-GM_desc-eroded_mask": (erode, "outputspec.eroded_mask")} return (wf, outputs) @@ -2168,29 +2310,25 @@ def erode_mask_GM(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["label-WM_desc-eroded_mask"], ) def erode_mask_WM(wf, cfg, strat_pool, pipe_num, opt=None): - - erode = erode_mask(f'erode_WM_mask_{pipe_num}') - erode.inputs.inputspec.erode_mm = cfg.nuisance_corrections[ - '2-nuisance_regression']['regressor_masks']['erode_wm'][ - 'wm_erosion_mm'] + erode = erode_mask(f"erode_WM_mask_{pipe_num}") + erode.inputs.inputspec.erode_mm = cfg.nuisance_corrections["2-nuisance_regression"][ + "regressor_masks" + ]["erode_wm"]["wm_erosion_mm"] erode.inputs.inputspec.erode_prop = cfg.nuisance_corrections[ - '2-nuisance_regression']['regressor_masks']['erode_wm'][ - 'wm_erosion_prop'] + "2-nuisance_regression" + ]["regressor_masks"]["erode_wm"]["wm_erosion_prop"] erode.inputs.inputspec.mask_erode_mm = cfg.nuisance_corrections[ - '2-nuisance_regression']['regressor_masks']['erode_wm'][ - 'wm_mask_erosion_mm'] + "2-nuisance_regression" + ]["regressor_masks"]["erode_wm"]["wm_mask_erosion_mm"] - node, out = strat_pool.get_data(['label-WM_desc-preproc_mask', - 'label-WM_mask']) - wf.connect(node, out, erode, 'inputspec.mask') + node, out = strat_pool.get_data(["label-WM_desc-preproc_mask", "label-WM_mask"]) + wf.connect(node, out, erode, "inputspec.mask") - node, out = strat_pool.get_data('space-T1w_desc-brain_mask') - wf.connect(node, out, erode, 'inputspec.brain_mask') + node, out = strat_pool.get_data("space-T1w_desc-brain_mask") + wf.connect(node, out, erode, "inputspec.brain_mask") - outputs = { - 'label-WM_desc-eroded_mask': (erode, 'outputspec.eroded_mask') - } + outputs = {"label-WM_desc-eroded_mask": (erode, "outputspec.eroded_mask")} return (wf, outputs) @@ -2234,11 +2372,9 @@ def erode_mask_WM(wf, cfg, strat_pool, pipe_num, opt=None): ], outputs=["desc-confounds_timeseries", "censor-indices"], ) +def nuisance_regressors_generation_EPItemplate(wf, cfg, strat_pool, pipe_num, opt=None): + return nuisance_regressors_generation(wf, cfg, strat_pool, pipe_num, opt, "bold") -def nuisance_regressors_generation_EPItemplate(wf, cfg, strat_pool, pipe_num, - opt=None): - return nuisance_regressors_generation(wf, cfg, strat_pool, pipe_num, opt, - 'bold') @nodeblock( name="nuisance_regressors_generation_T1w", @@ -2280,18 +2416,19 @@ def nuisance_regressors_generation_EPItemplate(wf, cfg, strat_pool, pipe_num, ], outputs=["desc-confounds_timeseries", "censor-indices"], ) -def nuisance_regressors_generation_T1w(wf, cfg, strat_pool, pipe_num, opt=None - ): - return nuisance_regressors_generation(wf, cfg, strat_pool, pipe_num, opt, - 'T1w') - - -def nuisance_regressors_generation(wf: Workflow, cfg: Configuration, - strat_pool: ResourcePool, - pipe_num: int, opt: dict, - space: LITERAL['T1w', 'bold'] - ) -> TUPLE[Workflow, dict]: - ''' +def nuisance_regressors_generation_T1w(wf, cfg, strat_pool, pipe_num, opt=None): + return nuisance_regressors_generation(wf, cfg, strat_pool, pipe_num, opt, "T1w") + + +def nuisance_regressors_generation( + wf: Workflow, + cfg: Configuration, + strat_pool: ResourcePool, + pipe_num: int, + opt: dict, + space: LITERAL["T1w", "bold"], +) -> TUPLE[Workflow, dict]: + """ Parameters ---------- wf : ~nipype.pipeline.engine.workflows.Workflow @@ -2312,193 +2449,232 @@ def nuisance_regressors_generation(wf: Workflow, cfg: Configuration, wf : nipype.pipeline.engine.workflows.Workflow outputs : dict - ''' - prefixes = [f'space-{space}_'] * 2 + """ + prefixes = [f"space-{space}_"] * 2 reg_tool = None - if space == 'T1w': - prefixes[0] = '' - if strat_pool.check_rpool( - 'from-template_to-T1w_mode-image_desc-linear_xfm'): + if space == "T1w": + prefixes[0] = "" + if strat_pool.check_rpool("from-template_to-T1w_mode-image_desc-linear_xfm"): xfm_prov = strat_pool.get_cpac_provenance( - 'from-template_to-T1w_mode-image_desc-linear_xfm') + "from-template_to-T1w_mode-image_desc-linear_xfm" + ) reg_tool = check_prov_for_regtool(xfm_prov) - elif space == 'bold': + elif space == "bold": xfm_prov = strat_pool.get_cpac_provenance( - 'from-EPItemplate_to-bold_mode-image_desc-linear_xfm') + "from-EPItemplate_to-bold_mode-image_desc-linear_xfm" + ) reg_tool = check_prov_for_regtool(xfm_prov) if reg_tool is not None: - use_ants = reg_tool == 'ants' - if cfg.switch_is_on(['functional_preproc', - 'motion_estimates_and_correction', - 'motion_estimate_filter', 'run']): - wf_name = (f'nuisance_regressors_{opt["Name"]}_filt-' - f'{strat_pool.filter_name(cfg)}_{pipe_num}') + use_ants = reg_tool == "ants" + if cfg.switch_is_on( + [ + "functional_preproc", + "motion_estimates_and_correction", + "motion_estimate_filter", + "run", + ] + ): + wf_name = ( + f'nuisance_regressors_{opt["Name"]}_filt-' + f'{strat_pool.filter_name(cfg)}_{pipe_num}' + ) else: wf_name = f'nuisance_regressors_{opt["Name"]}_{pipe_num}' - ventricle = strat_pool.check_rpool('lateral-ventricles-mask') - csf_mask = strat_pool.check_rpool([f'{prefixes[0]}label-CSF_' - 'desc-eroded_mask', - f'{prefixes[0]}label-CSF_' - 'desc-preproc_mask', - f'{prefixes[0]}label-CSF_mask']) + ventricle = strat_pool.check_rpool("lateral-ventricles-mask") + csf_mask = strat_pool.check_rpool( + [ + f"{prefixes[0]}label-CSF_" "desc-eroded_mask", + f"{prefixes[0]}label-CSF_" "desc-preproc_mask", + f"{prefixes[0]}label-CSF_mask", + ] + ) - regressors = create_regressor_workflow(opt, use_ants, - ventricle_mask_exist=ventricle, - all_bold=space == 'bold', - csf_mask_exist=csf_mask, - name=wf_name) + regressors = create_regressor_workflow( + opt, + use_ants, + ventricle_mask_exist=ventricle, + all_bold=space == "bold", + csf_mask_exist=csf_mask, + name=wf_name, + ) node, out = strat_pool.get_data("desc-preproc_bold") - wf.connect(node, out, regressors, 'inputspec.functional_file_path') - - node, out = strat_pool.get_data('space-bold_desc-brain_mask') - wf.connect(node, out, - regressors, 'inputspec.functional_brain_mask_file_path') - - if strat_pool.check_rpool(f'desc-brain_{space}'): - node, out = strat_pool.get_data(f'desc-brain_{space}') - wf.connect(node, out, regressors, 'inputspec.anatomical_file_path') - - if strat_pool.check_rpool([f'{prefixes[1]}desc-eroded_mask', - f'{prefixes[1]}desc-brain_mask']): - node, out = strat_pool.get_data([f'{prefixes[1]}desc-eroded_mask', - f'{prefixes[1]}desc-brain_mask']) - wf.connect(node, out, regressors, - 'inputspec.anatomical_eroded_brain_mask_file_path') + wf.connect(node, out, regressors, "inputspec.functional_file_path") + + node, out = strat_pool.get_data("space-bold_desc-brain_mask") + wf.connect(node, out, regressors, "inputspec.functional_brain_mask_file_path") + + if strat_pool.check_rpool(f"desc-brain_{space}"): + node, out = strat_pool.get_data(f"desc-brain_{space}") + wf.connect(node, out, regressors, "inputspec.anatomical_file_path") + + if strat_pool.check_rpool( + [f"{prefixes[1]}desc-eroded_mask", f"{prefixes[1]}desc-brain_mask"] + ): + node, out = strat_pool.get_data( + [f"{prefixes[1]}desc-eroded_mask", f"{prefixes[1]}desc-brain_mask"] + ) + wf.connect( + node, out, regressors, "inputspec.anatomical_eroded_brain_mask_file_path" + ) else: - logger.warning('No %s-space brain mask found in resource pool.', space) - - if strat_pool.check_rpool([f'{prefixes[0]}label-CSF_desc-eroded_mask', - f'{prefixes[0]}label-CSF_desc-preproc_mask', - f'{prefixes[0]}label-CSF_mask']): - node, out = strat_pool.get_data([f'{prefixes[0]}label-CSF_' - 'desc-eroded_mask', - f'{prefixes[0]}label-CSF_' - 'desc-preproc_mask', - f'{prefixes[0]}label-CSF_mask']) - wf.connect(node, out, regressors, 'inputspec.csf_mask_file_path') + logger.warning("No %s-space brain mask found in resource pool.", space) + + if strat_pool.check_rpool( + [ + f"{prefixes[0]}label-CSF_desc-eroded_mask", + f"{prefixes[0]}label-CSF_desc-preproc_mask", + f"{prefixes[0]}label-CSF_mask", + ] + ): + node, out = strat_pool.get_data( + [ + f"{prefixes[0]}label-CSF_" "desc-eroded_mask", + f"{prefixes[0]}label-CSF_" "desc-preproc_mask", + f"{prefixes[0]}label-CSF_mask", + ] + ) + wf.connect(node, out, regressors, "inputspec.csf_mask_file_path") else: - logger.warning('No %s-space CSF mask found in resource pool.', space) - - if strat_pool.check_rpool([f'{prefixes[0]}label-WM_desc-eroded_mask', - f'{prefixes[0]}label-WM_desc-preproc_mask', - f'{prefixes[0]}label-WM_mask']): - node, out = strat_pool.get_data([f'{prefixes[0]}label-WM_' - 'desc-eroded_mask', - f'{prefixes[0]}label-WM_' - 'desc-preproc_mask', - f'{prefixes[0]}label-WM_mask']) - wf.connect(node, out, regressors, 'inputspec.wm_mask_file_path') + logger.warning("No %s-space CSF mask found in resource pool.", space) + + if strat_pool.check_rpool( + [ + f"{prefixes[0]}label-WM_desc-eroded_mask", + f"{prefixes[0]}label-WM_desc-preproc_mask", + f"{prefixes[0]}label-WM_mask", + ] + ): + node, out = strat_pool.get_data( + [ + f"{prefixes[0]}label-WM_" "desc-eroded_mask", + f"{prefixes[0]}label-WM_" "desc-preproc_mask", + f"{prefixes[0]}label-WM_mask", + ] + ) + wf.connect(node, out, regressors, "inputspec.wm_mask_file_path") else: - logger.warning('No %s-space WM mask found in resource pool.', space) - - if strat_pool.check_rpool([f'{prefixes[0]}label-GM_desc-eroded_mask', - f'{prefixes[0]}label-GM_desc-preproc_mask', - f'{prefixes[0]}label-GM_mask']): - node, out = strat_pool.get_data([f'{prefixes[0]}label-GM_' - 'desc-eroded_mask', - f'{prefixes[0]}label-GM_' - 'desc-preproc_mask', - f'{prefixes[0]}label-GM_mask']) - wf.connect(node, out, regressors, 'inputspec.gm_mask_file_path') + logger.warning("No %s-space WM mask found in resource pool.", space) + + if strat_pool.check_rpool( + [ + f"{prefixes[0]}label-GM_desc-eroded_mask", + f"{prefixes[0]}label-GM_desc-preproc_mask", + f"{prefixes[0]}label-GM_mask", + ] + ): + node, out = strat_pool.get_data( + [ + f"{prefixes[0]}label-GM_" "desc-eroded_mask", + f"{prefixes[0]}label-GM_" "desc-preproc_mask", + f"{prefixes[0]}label-GM_mask", + ] + ) + wf.connect(node, out, regressors, "inputspec.gm_mask_file_path") else: - logger.warning('No %s-space GM mask found in resource pool.', space) + logger.warning("No %s-space GM mask found in resource pool.", space) if ventricle: - node, out = strat_pool.get_data('lateral-ventricles-mask') - wf.connect(node, out, - regressors, 'inputspec.lat_ventricles_mask_file_path') + node, out = strat_pool.get_data("lateral-ventricles-mask") + wf.connect(node, out, regressors, "inputspec.lat_ventricles_mask_file_path") - if space == 'T1w': - if strat_pool.check_rpool( - 'from-bold_to-T1w_mode-image_desc-linear_xfm'): + if space == "T1w": + if strat_pool.check_rpool("from-bold_to-T1w_mode-image_desc-linear_xfm"): node, out = strat_pool.get_data( - 'from-bold_to-T1w_mode-image_desc-linear_xfm') - wf.connect(node, out, - regressors, - 'inputspec.func_to_anat_linear_xfm_file_path') + "from-bold_to-T1w_mode-image_desc-linear_xfm" + ) + wf.connect( + node, out, regressors, "inputspec.func_to_anat_linear_xfm_file_path" + ) # invert func2anat matrix to get anat2func_linear_xfm - anat2func_linear_xfm = pe.Node(interface=fsl.ConvertXFM(), - name=f'anat_to_func_linear_xfm_' - f'{opt["Name"]}_{pipe_num}') + anat2func_linear_xfm = pe.Node( + interface=fsl.ConvertXFM(), + name=f'anat_to_func_linear_xfm_' f'{opt["Name"]}_{pipe_num}', + ) anat2func_linear_xfm.inputs.invert_xfm = True - wf.connect(node, out, anat2func_linear_xfm, 'in_file') + wf.connect(node, out, anat2func_linear_xfm, "in_file") - wf.connect(anat2func_linear_xfm, 'out_file', - regressors, - 'inputspec.anat_to_func_linear_xfm_file_path') + wf.connect( + anat2func_linear_xfm, + "out_file", + regressors, + "inputspec.anat_to_func_linear_xfm_file_path", + ) - if strat_pool.check_rpool( - 'from-template_to-T1w_mode-image_desc-linear_xfm'): + if strat_pool.check_rpool("from-template_to-T1w_mode-image_desc-linear_xfm"): node, out = strat_pool.get_data( - 'from-template_to-T1w_mode-image_desc-linear_xfm') - wf.connect(node, out, - regressors, - 'inputspec.mni_to_anat_linear_xfm_file_path') + "from-template_to-T1w_mode-image_desc-linear_xfm" + ) + wf.connect( + node, out, regressors, "inputspec.mni_to_anat_linear_xfm_file_path" + ) - if strat_pool.check_rpool( - 'from-T1w_to-template_mode-image_desc-linear_xfm'): + if strat_pool.check_rpool("from-T1w_to-template_mode-image_desc-linear_xfm"): node, out = strat_pool.get_data( - 'from-T1w_to-template_mode-image_desc-linear_xfm') - wf.connect(node, out, - regressors, - 'inputspec.anat_to_mni_linear_xfm_file_path') + "from-T1w_to-template_mode-image_desc-linear_xfm" + ) + wf.connect( + node, out, regressors, "inputspec.anat_to_mni_linear_xfm_file_path" + ) - elif space == 'bold': + elif space == "bold": if strat_pool.check_rpool( - 'from-EPItemplate_to-bold_mode-image_desc-linear_xfm'): + "from-EPItemplate_to-bold_mode-image_desc-linear_xfm" + ): node, out = strat_pool.get_data( - 'from-EPItemplate_to-bold_mode-image_desc-linear_xfm') - wf.connect(node, out, - regressors, - 'inputspec.mni_to_anat_linear_xfm_file_path') - wf.connect(node, out, - regressors, - 'inputspec.anat_to_func_linear_xfm_file_path') + "from-EPItemplate_to-bold_mode-image_desc-linear_xfm" + ) + wf.connect( + node, out, regressors, "inputspec.mni_to_anat_linear_xfm_file_path" + ) + wf.connect( + node, out, regressors, "inputspec.anat_to_func_linear_xfm_file_path" + ) if strat_pool.check_rpool( - 'from-bold_to-EPItemplate_mode-image_desc-linear_xfm'): + "from-bold_to-EPItemplate_mode-image_desc-linear_xfm" + ): node, out = strat_pool.get_data( - 'from-bold_to-EPItemplate_mode-image_desc-linear_xfm') - wf.connect(node, out, - regressors, - 'inputspec.anat_to_mni_linear_xfm_file_path') - wf.connect(node, out, - regressors, - 'inputspec.func_to_anat_linear_xfm_file_path') - - if strat_pool.check_rpool('desc-movementParameters_motion'): - node, out = strat_pool.get_data('desc-movementParameters_motion') - wf.connect(node, out, - regressors, 'inputspec.motion_parameters_file_path') - - if strat_pool.check_rpool('framewise-displacement-jenkinson'): - node, out = strat_pool.get_data('framewise-displacement-jenkinson') - wf.connect(node, out, regressors, 'inputspec.fd_j_file_path') - - if strat_pool.check_rpool('framewise-displacement-power'): - node, out = strat_pool.get_data('framewise-displacement-power') - wf.connect(node, out, regressors, 'inputspec.fd_p_file_path') - - if strat_pool.check_rpool('dvars'): - node, out = strat_pool.get_data('dvars') - wf.connect(node, out, regressors, 'inputspec.dvars_file_path') - - node, out = strat_pool.get_data('TR') - wf.connect(node, out, regressors, 'inputspec.tr') + "from-bold_to-EPItemplate_mode-image_desc-linear_xfm" + ) + wf.connect( + node, out, regressors, "inputspec.anat_to_mni_linear_xfm_file_path" + ) + wf.connect( + node, out, regressors, "inputspec.func_to_anat_linear_xfm_file_path" + ) + + if strat_pool.check_rpool("desc-movementParameters_motion"): + node, out = strat_pool.get_data("desc-movementParameters_motion") + wf.connect(node, out, regressors, "inputspec.motion_parameters_file_path") + + if strat_pool.check_rpool("framewise-displacement-jenkinson"): + node, out = strat_pool.get_data("framewise-displacement-jenkinson") + wf.connect(node, out, regressors, "inputspec.fd_j_file_path") + + if strat_pool.check_rpool("framewise-displacement-power"): + node, out = strat_pool.get_data("framewise-displacement-power") + wf.connect(node, out, regressors, "inputspec.fd_p_file_path") + + if strat_pool.check_rpool("dvars"): + node, out = strat_pool.get_data("dvars") + wf.connect(node, out, regressors, "inputspec.dvars_file_path") + + node, out = strat_pool.get_data("TR") + wf.connect(node, out, regressors, "inputspec.tr") outputs = { - 'desc-confounds_timeseries': (regressors, 'outputspec.regressors_file_path'), - 'censor-indices': (regressors, 'outputspec.censor_indices') + "desc-confounds_timeseries": (regressors, "outputspec.regressors_file_path"), + "censor-indices": (regressors, "outputspec.censor_indices"), } return (wf, outputs) def nuisance_regression(wf, cfg, strat_pool, pipe_num, opt, space, res=None): - '''Nuisance regression in native (BOLD) or template space + """Nuisance regression in native (BOLD) or template space. Parameters ---------- @@ -2513,190 +2689,223 @@ def nuisance_regression(wf, cfg, strat_pool, pipe_num, opt, space, res=None): wf : nipype.pipeline.engine.workflows.Workflow outputs : dict - ''' + """ opt = strat_pool.regressor_dct(cfg) - bandpass = 'Bandpass' in opt - bandpass_before = bandpass and cfg['nuisance_corrections', - '2-nuisance_regression', - 'bandpass_filtering_order'] == 'Before' + bandpass = "Bandpass" in opt + bandpass_before = ( + bandpass + and cfg[ + "nuisance_corrections", "2-nuisance_regression", "bandpass_filtering_order" + ] + == "Before" + ) - name_suff = (f'space-{space}_reg-{opt["Name"]}_{pipe_num}' - if res is None else - f'space-{space}_res-{res}_reg-{opt["Name"]}_{pipe_num}') - nuis_name = f'nuisance_regression_{name_suff}' + name_suff = ( + f'space-{space}_reg-{opt["Name"]}_{pipe_num}' + if res is None + else f'space-{space}_res-{res}_reg-{opt["Name"]}_{pipe_num}' + ) + nuis_name = f"nuisance_regression_{name_suff}" nuis = create_nuisance_regression_workflow(opt, name=nuis_name) if bandpass_before: - nofilter_nuis = nuis.clone(name=f'{nuis.name}-noFilter') + nofilter_nuis = nuis.clone(name=f"{nuis.name}-noFilter") - desc_keys = ('desc-preproc_bold', 'desc-cleaned_bold', - 'desc-denoisedNofilt_bold') - if space != 'native': - new_label = f'space-{space}' + desc_keys = ("desc-preproc_bold", "desc-cleaned_bold", "desc-denoisedNofilt_bold") + if space != "native": + new_label = f"space-{space}" if res: - new_label = f'{new_label}_res-{res}' - desc_keys = tuple(f'{new_label}_{key}' for key in desc_keys) + new_label = f"{new_label}_res-{res}" + desc_keys = tuple(f"{new_label}_{key}" for key in desc_keys) - if space == 'template': + if space == "template": # sometimes mm dimensions match but the voxel dimensions don't # so here we align the mask to the resampled data before applying - match_grid = pe.Node(afni.Resample(), - name='align_template_mask_to_template_data_' - f'{name_suff}') - match_grid.inputs.outputtype = 'NIFTI_GZ' - match_grid.inputs.resample_mode = 'Cu' - node, out = strat_pool.get_data('FSL-AFNI-brain-mask') - wf.connect(node, out, match_grid, 'in_file') + match_grid = pe.Node( + afni.Resample(), name="align_template_mask_to_template_data_" f"{name_suff}" + ) + match_grid.inputs.outputtype = "NIFTI_GZ" + match_grid.inputs.resample_mode = "Cu" + node, out = strat_pool.get_data("FSL-AFNI-brain-mask") + wf.connect(node, out, match_grid, "in_file") node, out = strat_pool.get_data(desc_keys[0]) - wf.connect(node, out, match_grid, 'master') - wf.connect(match_grid, 'out_file', - nuis, 'inputspec.functional_brain_mask_file_path') + wf.connect(node, out, match_grid, "master") + wf.connect( + match_grid, "out_file", nuis, "inputspec.functional_brain_mask_file_path" + ) if bandpass_before: - wf.connect(match_grid, 'out_file', - nofilter_nuis, - 'inputspec.functional_brain_mask_file_path') + wf.connect( + match_grid, + "out_file", + nofilter_nuis, + "inputspec.functional_brain_mask_file_path", + ) else: - node, out = strat_pool.get_data('space-bold_desc-brain_mask') - wf.connect(node, out, - nuis, 'inputspec.functional_brain_mask_file_path') + node, out = strat_pool.get_data("space-bold_desc-brain_mask") + wf.connect(node, out, nuis, "inputspec.functional_brain_mask_file_path") if bandpass_before: - wf.connect(node, out, - nofilter_nuis, - 'inputspec.functional_brain_mask_file_path') + wf.connect( + node, out, nofilter_nuis, "inputspec.functional_brain_mask_file_path" + ) - node, out = strat_pool.get_data(['desc-confounds_timeseries', 'parsed_regressors']) - wf.connect(node, out, nuis, 'inputspec.regressor_file') + node, out = strat_pool.get_data(["desc-confounds_timeseries", "parsed_regressors"]) + wf.connect(node, out, nuis, "inputspec.regressor_file") if bandpass_before: - wf.connect(node, out, nofilter_nuis, 'inputspec.regressor_file') + wf.connect(node, out, nofilter_nuis, "inputspec.regressor_file") - if strat_pool.check_rpool('framewise-displacement-jenkinson'): - node, out = strat_pool.get_data('framewise-displacement-jenkinson') - wf.connect(node, out, nuis, 'inputspec.fd_j_file_path') + if strat_pool.check_rpool("framewise-displacement-jenkinson"): + node, out = strat_pool.get_data("framewise-displacement-jenkinson") + wf.connect(node, out, nuis, "inputspec.fd_j_file_path") if bandpass_before: - wf.connect(node, out, nofilter_nuis, 'inputspec.fd_j_file_path') + wf.connect(node, out, nofilter_nuis, "inputspec.fd_j_file_path") - if strat_pool.check_rpool('framewise-displacement-power'): - node, out = strat_pool.get_data('framewise-displacement-power') - wf.connect(node, out, nuis, 'inputspec.fd_p_file_path') + if strat_pool.check_rpool("framewise-displacement-power"): + node, out = strat_pool.get_data("framewise-displacement-power") + wf.connect(node, out, nuis, "inputspec.fd_p_file_path") if bandpass_before: - wf.connect(node, out, nofilter_nuis, 'inputspec.fd_p_file_path') + wf.connect(node, out, nofilter_nuis, "inputspec.fd_p_file_path") - if strat_pool.check_rpool('dvars'): - node, out = strat_pool.get_data('dvars') - wf.connect(node, out, nuis, 'inputspec.dvars_file_path') + if strat_pool.check_rpool("dvars"): + node, out = strat_pool.get_data("dvars") + wf.connect(node, out, nuis, "inputspec.dvars_file_path") if bandpass_before: - wf.connect(node, out, nofilter_nuis, 'inputspec.dvars_file_path') + wf.connect(node, out, nofilter_nuis, "inputspec.dvars_file_path") if bandpass: - filt = filtering_bold_and_regressors(opt, name=f'filtering_bold_and_' - f'regressors_{name_suff}') + filt = filtering_bold_and_regressors( + opt, name=f"filtering_bold_and_" f"regressors_{name_suff}" + ) filt.inputs.inputspec.nuisance_selectors = opt - node, out = strat_pool.get_data(['desc-confounds_timeseries', 'parsed_regressors']) - wf.connect(node, out, filt, 'inputspec.regressors_file_path') - - if space == 'template': - wf.connect(match_grid, 'out_file', - filt, 'inputspec.functional_brain_mask_file_path') + node, out = strat_pool.get_data( + ["desc-confounds_timeseries", "parsed_regressors"] + ) + wf.connect(node, out, filt, "inputspec.regressors_file_path") + + if space == "template": + wf.connect( + match_grid, + "out_file", + filt, + "inputspec.functional_brain_mask_file_path", + ) else: - node, out = strat_pool.get_data('space-bold_desc-brain_mask') - wf.connect(node, out, - filt, 'inputspec.functional_brain_mask_file_path') - - node, out = strat_pool.get_data('TR') - wf.connect(node, out, filt, 'inputspec.tr') + node, out = strat_pool.get_data("space-bold_desc-brain_mask") + wf.connect(node, out, filt, "inputspec.functional_brain_mask_file_path") - if cfg['nuisance_corrections', '2-nuisance_regression', - 'bandpass_filtering_order'] == 'After': + node, out = strat_pool.get_data("TR") + wf.connect(node, out, filt, "inputspec.tr") + if ( + cfg[ + "nuisance_corrections", + "2-nuisance_regression", + "bandpass_filtering_order", + ] + == "After" + ): node, out = strat_pool.get_data(desc_keys[0]) - wf.connect(node, out, nuis, 'inputspec.functional_file_path') + wf.connect(node, out, nuis, "inputspec.functional_file_path") - wf.connect(nuis, 'outputspec.residual_file_path', - filt, 'inputspec.functional_file_path') + wf.connect( + nuis, + "outputspec.residual_file_path", + filt, + "inputspec.functional_file_path", + ) outputs = { - desc_keys[0]: (filt, 'outputspec.residual_file_path'), - desc_keys[1]: (filt, 'outputspec.residual_file_path'), - desc_keys[2]: (nuis, 'outputspec.residual_file_path'), - 'desc-confounds_timeseries': (filt, 'outputspec.residual_regressor') + desc_keys[0]: (filt, "outputspec.residual_file_path"), + desc_keys[1]: (filt, "outputspec.residual_file_path"), + desc_keys[2]: (nuis, "outputspec.residual_file_path"), + "desc-confounds_timeseries": (filt, "outputspec.residual_regressor"), } elif bandpass_before: - node, out = strat_pool.get_data(desc_keys[0]) - wf.connect(node, out, filt, 'inputspec.functional_file_path') - wf.connect(node, out, - nofilter_nuis, 'inputspec.functional_file_path') - - wf.connect(filt, 'outputspec.residual_file_path', - nuis, 'inputspec.functional_file_path') - + wf.connect(node, out, filt, "inputspec.functional_file_path") + wf.connect(node, out, nofilter_nuis, "inputspec.functional_file_path") + + wf.connect( + filt, + "outputspec.residual_file_path", + nuis, + "inputspec.functional_file_path", + ) outputs = { - desc_keys[0]: (nuis, 'outputspec.residual_file_path'), - desc_keys[1]: (nuis, 'outputspec.residual_file_path'), - desc_keys[2]: (nofilter_nuis, 'outputspec.residual_file_path'), - 'desc-confounds_timeseries': (filt, 'outputspec.residual_regressor')} + desc_keys[0]: (nuis, "outputspec.residual_file_path"), + desc_keys[1]: (nuis, "outputspec.residual_file_path"), + desc_keys[2]: (nofilter_nuis, "outputspec.residual_file_path"), + "desc-confounds_timeseries": (filt, "outputspec.residual_regressor"), + } else: node, out = strat_pool.get_data(desc_keys[0]) - wf.connect(node, out, nuis, 'inputspec.functional_file_path') + wf.connect(node, out, nuis, "inputspec.functional_file_path") - outputs = {desc_key: (nuis, 'outputspec.residual_file_path') for - desc_key in desc_keys} + outputs = { + desc_key: (nuis, "outputspec.residual_file_path") for desc_key in desc_keys + } return (wf, outputs) + @nodeblock( name="ingress_regressors", - switch=[["nuisance_corrections", "2-nuisance_regression", "run"], - ["nuisance_corrections", "2-nuisance_regression", "ingress_regressors", "run"]], + switch=[ + ["nuisance_corrections", "2-nuisance_regression", "run"], + ["nuisance_corrections", "2-nuisance_regression", "ingress_regressors", "run"], + ], inputs=["pipeline-ingress_desc-confounds_timeseries"], - outputs=["parsed_regressors"] + outputs=["parsed_regressors"], ) def ingress_regressors(wf, cfg, strat_pool, pipe_num, opt=None): + regressors_list = cfg.nuisance_corrections["2-nuisance_regression"][ + "ingress_regressors" + ]["Regressors"]["Columns"] - regressors_list = cfg.nuisance_corrections['2-nuisance_regression']['ingress_regressors'][ - 'Regressors']['Columns'] - # Will need to generalize the name - node, out = strat_pool.get_data('pipeline-ingress_desc-confounds_timeseries') + node, out = strat_pool.get_data("pipeline-ingress_desc-confounds_timeseries") if not regressors_list: - logger.warning("\n[!] Ingress regressors is on, but no regressors provided. " - "The whole regressors file will be applied, but it may be" - "too large for the timeseries data!") - outputs = { - 'parsed_regressors': (node, out) - } + logger.warning( + "\n[!] Ingress regressors is on, but no regressors provided. " + "The whole regressors file will be applied, but it may be" + "too large for the timeseries data!" + ) + outputs = {"parsed_regressors": (node, out)} else: - ingress_imports = ['import numpy as np', - 'import numpy as np', 'import os', - 'import CPAC', 'from nipype import logging', - 'logger = logging.getLogger("nipype.workflow")'] - ingress_regressors = pe.Node(Function( - input_names=['regressors_file', - 'regressors_list'], - output_names=['parsed_regressors'], + ingress_imports = [ + "import numpy as np", + "import numpy as np", + "import os", + "import CPAC", + "from nipype import logging", + 'logger = logging.getLogger("nipype.workflow")', + ] + ingress_regressors = pe.Node( + Function( + input_names=["regressors_file", "regressors_list"], + output_names=["parsed_regressors"], function=parse_regressors, - imports=ingress_imports - ), name="parse_regressors_file") + imports=ingress_imports, + ), + name="parse_regressors_file", + ) - wf.connect(node, out, ingress_regressors, 'regressors_file') + wf.connect(node, out, ingress_regressors, "regressors_file") ingress_regressors.inputs.regressors_list = regressors_list - outputs = { - 'parsed_regressors': (ingress_regressors, 'parsed_regressors') - } - + outputs = {"parsed_regressors": (ingress_regressors, "parsed_regressors")} + return wf, outputs -def parse_regressors(regressors_file, regressors_list): +def parse_regressors(regressors_file, regressors_list): """ - + Parses regressors file from outdir ingress. - + Parameters ---------- confounds / regressors file : string @@ -2704,16 +2913,16 @@ def parse_regressors(regressors_file, regressors_list): regressors list : list, can be empty List containing names of regressors to select - + Returns ------- parsed_regressors: dataframe - Regressors - + Regressors + """ import pandas as pd - with open(regressors_file, 'r') as f: + with open(regressors_file, "r"): full_file = pd.read_table(regressors_file) parsed_regressors = pd.DataFrame() header = [] @@ -2721,9 +2930,11 @@ def parse_regressors(regressors_file, regressors_list): # Look through first 3 rows in case the header is nonstandard if regressor in full_file.iloc[:3]: header.append(regressor) - parsed_regressors[regressor] = full_file.loc[:,regressor] + parsed_regressors[regressor] = full_file.loc[:, regressor] else: - logger.warning(f"\n[!] Regressor {regressor} not found in {regressors_file}") + logger.warning( + f"\n[!] Regressor {regressor} not found in {regressors_file}" + ) if parsed_regressors.empty: raise Exception(f"Regressors not found in {regressors_file}") @@ -2731,18 +2942,20 @@ def parse_regressors(regressors_file, regressors_list): parsed_regressors = parsed_regressors.to_numpy() check_vals = np.any(np.isnan(parsed_regressors)) if check_vals: - raise Exception('\n[!] This regressors file contains "N/A" values.\n' - '[!] Please choose a different dataset or ' - 'remove regressors with those values.') + raise Exception( + '\n[!] This regressors file contains "N/A" values.\n' + "[!] Please choose a different dataset or " + "remove regressors with those values." + ) with open(regressors_path, "w") as ofd: - # write out the header information ofd.write("# C-PAC {0}\n".format(CPAC.__version__)) ofd.write("# Ingressed nuisance regressors:\n") - np.savetxt(ofd, parsed_regressors, fmt='%.18f', delimiter='\t') + np.savetxt(ofd, parsed_regressors, fmt="%.18f", delimiter="\t") return regressors_path + @nodeblock( name="nuisance_regression_native", config=["nuisance_corrections", "2-nuisance_regression"], @@ -2770,12 +2983,13 @@ def parse_regressors(regressors_file, regressors_list): "desc-denoisedNofilt_bold": { "Description": "Preprocessed BOLD image that was nuisance-regressed in native space, but without frequency filtering." }, - "desc-confounds_timeseries": {"Description": "Regressors that were applied in native space"}, + "desc-confounds_timeseries": { + "Description": "Regressors that were applied in native space" + }, }, ) def nuisance_regression_native(wf, cfg, strat_pool, pipe_num, opt=None): - - return nuisance_regression(wf, cfg, strat_pool, pipe_num, opt, 'native') + return nuisance_regression(wf, cfg, strat_pool, pipe_num, opt, "native") @nodeblock( @@ -2817,18 +3031,18 @@ def nuisance_regression_native(wf, cfg, strat_pool, pipe_num, opt=None): "space-template_res-derivative_desc-denoisedNofilt_bold": { "Description": "Preprocessed BOLD image that was nuisance-regressed in template space, but without frequency filtering." }, - "desc-confounds_timeseries": {"Description": "Regressors that were applied in template space"}, + "desc-confounds_timeseries": { + "Description": "Regressors that were applied in template space" + }, }, ) def nuisance_regression_template(wf, cfg, strat_pool, pipe_num, opt=None): - '''Apply nuisance regression to template-space image''' - wf, outputs = nuisance_regression(wf, cfg, strat_pool, pipe_num, opt, - 'template') - if strat_pool.check_rpool( - 'space-template_res-derivative_desc-preproc_bold' - ): - wf, res_outputs = nuisance_regression(wf, cfg, strat_pool, pipe_num, - opt, 'template', 'derivative') + """Apply nuisance regression to template-space image.""" + wf, outputs = nuisance_regression(wf, cfg, strat_pool, pipe_num, opt, "template") + if strat_pool.check_rpool("space-template_res-derivative_desc-preproc_bold"): + wf, res_outputs = nuisance_regression( + wf, cfg, strat_pool, pipe_num, opt, "template", "derivative" + ) outputs.update(res_outputs) return (wf, outputs) @@ -2859,25 +3073,23 @@ def nuisance_regression_template(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["space-bold_desc-eroded_mask"], ) def erode_mask_bold(wf, cfg, strat_pool, pipe_num, opt=None): - - erode = erode_mask(f'erode_T1w_mask_{pipe_num}', segmentmap=False) + erode = erode_mask(f"erode_T1w_mask_{pipe_num}", segmentmap=False) erode.inputs.inputspec.mask_erode_mm = cfg.nuisance_corrections[ - '2-nuisance_regression']['regressor_masks'][ - 'erode_anatomical_brain_mask']['brain_mask_erosion_mm'] + "2-nuisance_regression" + ]["regressor_masks"]["erode_anatomical_brain_mask"]["brain_mask_erosion_mm"] erode.inputs.inputspec.erode_prop = cfg.nuisance_corrections[ - '2-nuisance_regression']['regressor_masks'][ - 'erode_anatomical_brain_mask']['brain_mask_erosion_prop'] + "2-nuisance_regression" + ]["regressor_masks"]["erode_anatomical_brain_mask"]["brain_mask_erosion_prop"] - node, out = strat_pool.get_data('space-bold_desc-brain_mask') - wf.connect(node, out, erode, 'inputspec.brain_mask') + node, out = strat_pool.get_data("space-bold_desc-brain_mask") + wf.connect(node, out, erode, "inputspec.brain_mask") - node, out = strat_pool.get_data(['space-bold_label-CSF_desc-preproc_mask', - 'space-bold_label-CSF_mask']) - wf.connect(node, out, erode, 'inputspec.mask') + node, out = strat_pool.get_data( + ["space-bold_label-CSF_desc-preproc_mask", "space-bold_label-CSF_mask"] + ) + wf.connect(node, out, erode, "inputspec.mask") - outputs = { - 'space-bold_desc-eroded_mask': (erode, 'outputspec.eroded_mask') - } + outputs = {"space-bold_desc-eroded_mask": (erode, "outputspec.eroded_mask")} return (wf, outputs) @@ -2908,28 +3120,28 @@ def erode_mask_bold(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["space-bold_label-CSF_desc-eroded_mask"], ) def erode_mask_boldCSF(wf, cfg, strat_pool, pipe_num, opt=None): - - erode = erode_mask(f'erode_CSF_mask_{pipe_num}') - erode.inputs.inputspec.erode_mm = cfg.nuisance_corrections[ - '2-nuisance_regression']['regressor_masks']['erode_csf'][ - 'csf_erosion_mm'] + erode = erode_mask(f"erode_CSF_mask_{pipe_num}") + erode.inputs.inputspec.erode_mm = cfg.nuisance_corrections["2-nuisance_regression"][ + "regressor_masks" + ]["erode_csf"]["csf_erosion_mm"] erode.inputs.inputspec.erode_prop = cfg.nuisance_corrections[ - '2-nuisance_regression']['regressor_masks']['erode_csf'][ - 'csf_erosion_prop'] + "2-nuisance_regression" + ]["regressor_masks"]["erode_csf"]["csf_erosion_prop"] erode.inputs.inputspec.mask_erode_mm = cfg.nuisance_corrections[ - '2-nuisance_regression']['regressor_masks']['erode_csf'][ - 'csf_mask_erosion_mm'] + "2-nuisance_regression" + ]["regressor_masks"]["erode_csf"]["csf_mask_erosion_mm"] - node, out = strat_pool.get_data(['space-bold_label-CSF_desc-preproc_mask', - 'space-bold_label-CSF_mask']) - wf.connect(node, out, erode, 'inputspec.mask') + node, out = strat_pool.get_data( + ["space-bold_label-CSF_desc-preproc_mask", "space-bold_label-CSF_mask"] + ) + wf.connect(node, out, erode, "inputspec.mask") - node, out = strat_pool.get_data('space-bold_desc-brain_mask') - wf.connect(node, out, erode, 'inputspec.brain_mask') + node, out = strat_pool.get_data("space-bold_desc-brain_mask") + wf.connect(node, out, erode, "inputspec.brain_mask") outputs = { - 'space-bold_label-CSF_desc-eroded_mask': (erode, 'outputspec.eroded_mask') + "space-bold_label-CSF_desc-eroded_mask": (erode, "outputspec.eroded_mask") } return (wf, outputs) @@ -2956,25 +3168,25 @@ def erode_mask_boldCSF(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["space-bold_label-GM_desc-eroded_mask"], ) def erode_mask_boldGM(wf, cfg, strat_pool, pipe_num, opt=None): - - erode = erode_mask(f'erode_GM_mask_{pipe_num}') - erode.inputs.inputspec.erode_mm = cfg.nuisance_corrections[ - '2-nuisance_regression']['regressor_masks']['erode_gm'][ - 'gm_erosion_mm'] + erode = erode_mask(f"erode_GM_mask_{pipe_num}") + erode.inputs.inputspec.erode_mm = cfg.nuisance_corrections["2-nuisance_regression"][ + "regressor_masks" + ]["erode_gm"]["gm_erosion_mm"] erode.inputs.inputspec.erode_prop = cfg.nuisance_corrections[ - '2-nuisance_regression']['regressor_masks']['erode_gm'][ - 'gm_erosion_prop'] + "2-nuisance_regression" + ]["regressor_masks"]["erode_gm"]["gm_erosion_prop"] erode.inputs.inputspec.mask_erode_mm = cfg.nuisance_corrections[ - '2-nuisance_regression']['regressor_masks']['erode_gm'][ - 'gm_mask_erosion_mm'] + "2-nuisance_regression" + ]["regressor_masks"]["erode_gm"]["gm_mask_erosion_mm"] - node, out = strat_pool.get_data(['space-bold_label-GM_desc-preproc_mask', - 'space-bold_label-GM_mask']) - wf.connect(node, out, erode, 'inputspec.mask') + node, out = strat_pool.get_data( + ["space-bold_label-GM_desc-preproc_mask", "space-bold_label-GM_mask"] + ) + wf.connect(node, out, erode, "inputspec.mask") outputs = { - 'space-bold_label-GM_desc-eroded_mask': (erode, 'outputspec.eroded_mask') + "space-bold_label-GM_desc-eroded_mask": (erode, "outputspec.eroded_mask") } return (wf, outputs) @@ -3006,29 +3218,28 @@ def erode_mask_boldGM(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["space-bold_label-WM_desc-eroded_mask"], ) def erode_mask_boldWM(wf, cfg, strat_pool, pipe_num, opt=None): - - erode = erode_mask(f'erode_WM_mask_{pipe_num}') - erode.inputs.inputspec.erode_mm = cfg.nuisance_corrections[ - '2-nuisance_regression']['regressor_masks']['erode_wm'][ - 'wm_erosion_mm'] + erode = erode_mask(f"erode_WM_mask_{pipe_num}") + erode.inputs.inputspec.erode_mm = cfg.nuisance_corrections["2-nuisance_regression"][ + "regressor_masks" + ]["erode_wm"]["wm_erosion_mm"] erode.inputs.inputspec.erode_prop = cfg.nuisance_corrections[ - '2-nuisance_regression']['regressor_masks']['erode_wm'][ - 'wm_erosion_prop'] + "2-nuisance_regression" + ]["regressor_masks"]["erode_wm"]["wm_erosion_prop"] erode.inputs.inputspec.mask_erode_mm = cfg.nuisance_corrections[ - '2-nuisance_regression']['regressor_masks']['erode_wm'][ - 'wm_mask_erosion_mm'] + "2-nuisance_regression" + ]["regressor_masks"]["erode_wm"]["wm_mask_erosion_mm"] - node, out = strat_pool.get_data(['space-bold_label-WM_desc-preproc_mask', - 'space-bold_label-WM_mask']) - wf.connect(node, out, erode, 'inputspec.mask') + node, out = strat_pool.get_data( + ["space-bold_label-WM_desc-preproc_mask", "space-bold_label-WM_mask"] + ) + wf.connect(node, out, erode, "inputspec.mask") - node, out = strat_pool.get_data('space-bold_desc-brain_mask') - wf.connect(node, out, erode, 'inputspec.brain_mask') + node, out = strat_pool.get_data("space-bold_desc-brain_mask") + wf.connect(node, out, erode, "inputspec.brain_mask") outputs = { - 'space-bold_label-WM_desc-eroded_mask': (erode, - 'outputspec.eroded_mask') + "space-bold_label-WM_desc-eroded_mask": (erode, "outputspec.eroded_mask") } return (wf, outputs) diff --git a/CPAC/nuisance/tests/motion_statistics/DVARS.1D b/CPAC/nuisance/tests/motion_statistics/DVARS.1D index 3094515951..43fdf14ac1 100644 --- a/CPAC/nuisance/tests/motion_statistics/DVARS.1D +++ b/CPAC/nuisance/tests/motion_statistics/DVARS.1D @@ -6,4 +6,4 @@ 2.0 2.0 2.0 -2.0 \ No newline at end of file +2.0 diff --git a/CPAC/nuisance/tests/motion_statistics/FD_J.1D b/CPAC/nuisance/tests/motion_statistics/FD_J.1D index d7af04face..171509bb09 100644 --- a/CPAC/nuisance/tests/motion_statistics/FD_J.1D +++ b/CPAC/nuisance/tests/motion_statistics/FD_J.1D @@ -7,4 +7,4 @@ 2.0 2.0 2.0 -2.0 \ No newline at end of file +2.0 diff --git a/CPAC/nuisance/tests/motion_statistics/FD_P.1D b/CPAC/nuisance/tests/motion_statistics/FD_P.1D index 49308eed37..18813e608d 100644 --- a/CPAC/nuisance/tests/motion_statistics/FD_P.1D +++ b/CPAC/nuisance/tests/motion_statistics/FD_P.1D @@ -7,4 +7,4 @@ 2.0 4.0 2.0 -2.0 \ No newline at end of file +2.0 diff --git a/CPAC/nuisance/tests/test_nuisance.py b/CPAC/nuisance/tests/test_nuisance.py index d286a6cbc2..d484621a0b 100755 --- a/CPAC/nuisance/tests/test_nuisance.py +++ b/CPAC/nuisance/tests/test_nuisance.py @@ -1,27 +1,27 @@ import glob from os import environ + import pytest import yaml - -from CPAC.pipeline import nipype_pipeline_engine as pe import nipype.interfaces.utility as util from CPAC.nuisance import create_nuisance_regression_workflow from CPAC.nuisance.utils import NuisanceRegressor +from CPAC.pipeline import nipype_pipeline_engine as pe selector_config = """ Regressors: - PolyOrt: degree: 2 - + tCompCor: summary: method: PC components: 5 threshold: 1.5SD by_slice: true - + aCompCor: summary: method: PC @@ -30,7 +30,7 @@ - WhiteMatter - CerebrospinalFluid extraction_resolution: 2 - + WhiteMatter: summary: method: PC @@ -43,7 +43,7 @@ components: 5 extraction_resolution: 2 erode_mask: true - + GlobalSignal: summary: Mean include_delayed: True @@ -64,67 +64,98 @@ value: 0.5 - type: DVARS value: 17 - + """ -def identity_input(name, field, val): - pool_input = pe.Node(util.IdentityInterface( - fields=[field]), - name=name - ) +def identity_input(name, field, val): + pool_input = pe.Node(util.IdentityInterface(fields=[field]), name=name) - pool_input.inputs.set({ field: val }) + pool_input.inputs.set({field: val}) return pool_input, field -@pytest.mark.skip(reason='needs refactoring') +@pytest.mark.skip(reason="needs refactoring") def test_nuisance_workflow_type1(): - - base_dir = '/tmp/nuisance_working_dir' + base_dir = "/tmp/nuisance_working_dir" try: import shutil + shutil.rmtree(base_dir) except: pass - selector_test = yaml.safe_load(selector_config)['Regressors'] + selector_test = yaml.safe_load(selector_config)["Regressors"] for selector in selector_test: - - nuisance_regression_workflow = \ - create_nuisance_regression_workflow( - nuisance_selectors=NuisanceRegressor(selector), - use_ants=True, - name='nuisance' - ) - - nuisance_regression_workflow.write_graph(graph2use='orig', simple_form=False) - - preprocessed = '/cc_dev/cpac_working/old_compcor' - - nuisance_regression_workflow.inputs.inputspec.anatomical_file_path = glob.glob(preprocessed + '/anat_preproc_already_0/anat_reorient/*_resample.nii.gz')[0] - nuisance_regression_workflow.inputs.inputspec.functional_file_path = glob.glob(preprocessed + '/func_preproc_automask_0/_scan_*/func_normalize/*_calc_tshift_resample_volreg_calc_maths.nii.gz')[0] - nuisance_regression_workflow.inputs.inputspec.functional_brain_mask_file_path = glob.glob(preprocessed + '/func_preproc_automask_0/_scan_*/func_mask_normalize/*_calc_tshift_resample_volreg_calc_maths_maths.nii.gz')[0] - nuisance_regression_workflow.inputs.inputspec.wm_mask_file_path = glob.glob(preprocessed + '/seg_preproc_0/WM/WM_mask/segment_seg_2_maths.nii.gz')[0] - nuisance_regression_workflow.inputs.inputspec.csf_mask_file_path = glob.glob(preprocessed + '/seg_preproc_0/GM/GM_mask/segment_seg_1_maths.nii.gz')[0] - nuisance_regression_workflow.inputs.inputspec.gm_mask_file_path = glob.glob(preprocessed + '/seg_preproc_0/GM/GM_mask/segment_seg_1_maths.nii.gz')[0] - nuisance_regression_workflow.inputs.inputspec.lat_ventricles_mask_file_path = glob.glob( - f'{environ.get("FSLDIR")}/data/standard/' - 'MNI152_T1_2mm_VentricleMask.nii.gz')[0] + nuisance_regression_workflow = create_nuisance_regression_workflow( + nuisance_selectors=NuisanceRegressor(selector), + use_ants=True, + name="nuisance", + ) + + nuisance_regression_workflow.write_graph(graph2use="orig", simple_form=False) + + preprocessed = "/cc_dev/cpac_working/old_compcor" + + nuisance_regression_workflow.inputs.inputspec.anatomical_file_path = glob.glob( + preprocessed + "/anat_preproc_already_0/anat_reorient/*_resample.nii.gz" + )[0] + nuisance_regression_workflow.inputs.inputspec.functional_file_path = glob.glob( + preprocessed + + "/func_preproc_automask_0/_scan_*/func_normalize/*_calc_tshift_resample_volreg_calc_maths.nii.gz" + )[0] + nuisance_regression_workflow.inputs.inputspec.functional_brain_mask_file_path = glob.glob( + preprocessed + + "/func_preproc_automask_0/_scan_*/func_mask_normalize/*_calc_tshift_resample_volreg_calc_maths_maths.nii.gz" + )[0] + nuisance_regression_workflow.inputs.inputspec.wm_mask_file_path = glob.glob( + preprocessed + "/seg_preproc_0/WM/WM_mask/segment_seg_2_maths.nii.gz" + )[0] + nuisance_regression_workflow.inputs.inputspec.csf_mask_file_path = glob.glob( + preprocessed + "/seg_preproc_0/GM/GM_mask/segment_seg_1_maths.nii.gz" + )[0] + nuisance_regression_workflow.inputs.inputspec.gm_mask_file_path = glob.glob( + preprocessed + "/seg_preproc_0/GM/GM_mask/segment_seg_1_maths.nii.gz" + )[0] + nuisance_regression_workflow.inputs.inputspec.lat_ventricles_mask_file_path = ( + glob.glob( + f'{environ.get("FSLDIR")}/data/standard/' + 'MNI152_T1_2mm_VentricleMask.nii.gz' + )[0] + ) # nuisance_regression_workflow.inputs.inputspec.mni_to_anat_linear_xfm_file_path = glob.glob('')[0] - nuisance_regression_workflow.inputs.inputspec.func_to_anat_linear_xfm_file_path = glob.glob(preprocessed + '/func_to_anat_bbreg_0/_scan_*/bbreg_func_to_anat/*_calc_tshift_resample_volreg_calc_tstat_flirt.mat')[0] - nuisance_regression_workflow.inputs.inputspec.anat_to_mni_initial_xfm_file_path = glob.glob(preprocessed + '/anat_mni_ants_register_0/calc_ants_warp/transform0DerivedInitialMovingTranslation.mat')[0] - nuisance_regression_workflow.inputs.inputspec.anat_to_mni_rigid_xfm_file_path = glob.glob(preprocessed + '/anat_mni_ants_register_0/calc_ants_warp/transform1Rigid.mat')[0] - nuisance_regression_workflow.inputs.inputspec.anat_to_mni_affine_xfm_file_path = glob.glob(preprocessed + '/anat_mni_ants_register_0/calc_ants_warp/transform2Affine.mat')[0] - nuisance_regression_workflow.inputs.inputspec.motion_parameters_file_path = glob.glob(preprocessed + '/func_preproc_automask_0/_scan_*/func_motion_correct_A/*_calc_tshift_resample.1D')[0] - nuisance_regression_workflow.inputs.inputspec.dvars_file_path = glob.glob(preprocessed + '/gen_motion_stats_0/_scan_*/cal_DVARS/DVARS.*')[0] - nuisance_regression_workflow.inputs.inputspec.fd_j_file_path = glob.glob(preprocessed + '/gen_motion_stats_0/_scan_*/calculate_FDJ/FD_J.1D')[0] - nuisance_regression_workflow.get_node('inputspec').iterables = ([ - ('selector', [NuisanceRegressor(selector)]), - ]) + nuisance_regression_workflow.inputs.inputspec.func_to_anat_linear_xfm_file_path = glob.glob( + preprocessed + + "/func_to_anat_bbreg_0/_scan_*/bbreg_func_to_anat/*_calc_tshift_resample_volreg_calc_tstat_flirt.mat" + )[0] + nuisance_regression_workflow.inputs.inputspec.anat_to_mni_initial_xfm_file_path = glob.glob( + preprocessed + + "/anat_mni_ants_register_0/calc_ants_warp/transform0DerivedInitialMovingTranslation.mat" + )[0] + nuisance_regression_workflow.inputs.inputspec.anat_to_mni_rigid_xfm_file_path = glob.glob( + preprocessed + + "/anat_mni_ants_register_0/calc_ants_warp/transform1Rigid.mat" + )[0] + nuisance_regression_workflow.inputs.inputspec.anat_to_mni_affine_xfm_file_path = glob.glob( + preprocessed + + "/anat_mni_ants_register_0/calc_ants_warp/transform2Affine.mat" + )[0] + nuisance_regression_workflow.inputs.inputspec.motion_parameters_file_path = glob.glob( + preprocessed + + "/func_preproc_automask_0/_scan_*/func_motion_correct_A/*_calc_tshift_resample.1D" + )[0] + nuisance_regression_workflow.inputs.inputspec.dvars_file_path = glob.glob( + preprocessed + "/gen_motion_stats_0/_scan_*/cal_DVARS/DVARS.*" + )[0] + nuisance_regression_workflow.inputs.inputspec.fd_j_file_path = glob.glob( + preprocessed + "/gen_motion_stats_0/_scan_*/calculate_FDJ/FD_J.1D" + )[0] + nuisance_regression_workflow.get_node("inputspec").iterables = [ + ("selector", [NuisanceRegressor(selector)]), + ] nuisance_regression_workflow.base_dir = base_dir diff --git a/CPAC/nuisance/tests/test_nuisance_representations.py b/CPAC/nuisance/tests/test_nuisance_representations.py index 8037a8220d..be574b4057 100644 --- a/CPAC/nuisance/tests/test_nuisance_representations.py +++ b/CPAC/nuisance/tests/test_nuisance_representations.py @@ -1,21 +1,27 @@ import yaml + from CPAC.nuisance.utils import NuisanceRegressor def test_representations(): - - assert NuisanceRegressor.encode({ - 'CerebrospinalFluid': { - 'erode_mask': True, - 'extraction_resolution': 2, - 'summary': { - 'method': 'PC', - 'components': 5, + assert ( + NuisanceRegressor.encode( + { + "CerebrospinalFluid": { + "erode_mask": True, + "extraction_resolution": 2, + "summary": { + "method": "PC", + "components": 5, + }, + } } - } - }) == 'CSF-2mmE-PC5' + ) + == "CSF-2mmE-PC5" + ) - selector_test = yaml.safe_load(""" + selector_test = yaml.safe_load( + """ tCompCor: summary: @@ -23,7 +29,7 @@ def test_representations(): components: 5 threshold: 1.5SD by_slice: true - + aCompCor: summary: method: PC @@ -32,14 +38,16 @@ def test_representations(): - WhiteMatter - CerebrospinalFluid extraction_resolution: 2 - - """) - assert NuisanceRegressor.encode({ - 'tCompCor': selector_test['tCompCor'] - }) == 'tC-S1.5SD-PC5' + """ + ) - assert NuisanceRegressor.encode({ - 'aCompCor': selector_test['aCompCor'] - }) == 'aC-CSF+WM-2mm-PC5' + assert ( + NuisanceRegressor.encode({"tCompCor": selector_test["tCompCor"]}) + == "tC-S1.5SD-PC5" + ) + assert ( + NuisanceRegressor.encode({"aCompCor": selector_test["aCompCor"]}) + == "aC-CSF+WM-2mm-PC5" + ) diff --git a/CPAC/nuisance/tests/test_utils.py b/CPAC/nuisance/tests/test_utils.py index c7ef6121a7..3c0be4f48d 100644 --- a/CPAC/nuisance/tests/test_utils.py +++ b/CPAC/nuisance/tests/test_utils.py @@ -1,50 +1,41 @@ -import numpy as np import os +import tempfile + +import numpy as np import pkg_resources as p import pytest -import tempfile -from CPAC.nuisance.utils import find_offending_time_points -from CPAC.nuisance.utils import calc_compcor_components - -mocked_outputs = \ - p.resource_filename( - "CPAC", - os.path.join( - 'nuisance', - 'tests', - 'motion_statistics' - ) - ) +from CPAC.nuisance.utils import calc_compcor_components, find_offending_time_points + +mocked_outputs = p.resource_filename( + "CPAC", os.path.join("nuisance", "tests", "motion_statistics") +) -@pytest.mark.skip(reason='needs refactoring') -def test_find_offending_time_points(): +@pytest.mark.skip(reason="needs refactoring") +def test_find_offending_time_points(): dl_dir = tempfile.mkdtemp() os.chdir(dl_dir) censored = find_offending_time_points( - os.path.join(mocked_outputs, 'FD_J.1D'), - os.path.join(mocked_outputs, 'FD_P.1D'), - os.path.join(mocked_outputs, 'DVARS.1D'), + os.path.join(mocked_outputs, "FD_J.1D"), + os.path.join(mocked_outputs, "FD_P.1D"), + os.path.join(mocked_outputs, "DVARS.1D"), 2.0, 2.0, - '1.5SD' + "1.5SD", ) censored = np.loadtxt(censored).astype(bool) - assert set(np.where(np.logical_not(censored))[0].tolist()) == set([1, 3, 7]) + assert set(np.where(np.logical_not(censored))[0].tolist()) == {1, 3, 7} -@pytest.mark.skip(reason='needs local files not included in package') +@pytest.mark.skip(reason="needs local files not included in package") def test_calc_compcor_components(): - data_filename = "/cc_dev/cpac_working/old_compcor/nuisance_0_0/_scan_test/_selector_CSF-2mmE-M_aC-WM-2mm-DPC5_G-M_M-SDB_P-2_BP-B0.01-T0.1/Functional_2mm_flirt/sub-M10978008_ses-NFB3_task-test_bold_calc_tshift_resample_volreg_calc_maths_flirt.nii.gz" mask_filename = "/cc_dev/cpac_working/old_compcor/nuisance_0_0/_scan_test/_selector_CSF-2mmE-M_aC-WM-2mm-DPC5_G-M_M-SDB_P-2_BP-B0.01-T0.1/aCompCor_union_masks/segment_seg_2_maths_flirt_mask.nii.gz" + calc_compcor_components(data_filename, 5, mask_filename) - compcor_filename = calc_compcor_components(data_filename, 5, mask_filename) - - print('compcor components written to {0}'.format(compcor_filename)) assert 0 == 1 diff --git a/CPAC/nuisance/tests/utils/test_compcor.py b/CPAC/nuisance/tests/utils/test_compcor.py index e5b0498540..39fe9c4b5b 100644 --- a/CPAC/nuisance/tests/utils/test_compcor.py +++ b/CPAC/nuisance/tests/utils/test_compcor.py @@ -4,22 +4,22 @@ def test_TR_string_to_float(): - assert abs(compcor.TR_string_to_float('123') - 123.) < 1e-08 - assert abs(compcor.TR_string_to_float('123s') - 123.) < 1e-08 - assert abs(compcor.TR_string_to_float('123ms') - 123. / 1000) < 1e-08 + assert abs(compcor.TR_string_to_float("123") - 123.0) < 1e-08 + assert abs(compcor.TR_string_to_float("123s") - 123.0) < 1e-08 + assert abs(compcor.TR_string_to_float("123ms") - 123.0 / 1000) < 1e-08 - assert abs(compcor.TR_string_to_float('1.23') - 1.23) < 1e-08 - assert abs(compcor.TR_string_to_float('1.23s') - 1.23) < 1e-08 - assert abs(compcor.TR_string_to_float('1.23ms') - 1.23 / 1000) < 1e-08 + assert abs(compcor.TR_string_to_float("1.23") - 1.23) < 1e-08 + assert abs(compcor.TR_string_to_float("1.23s") - 1.23) < 1e-08 + assert abs(compcor.TR_string_to_float("1.23ms") - 1.23 / 1000) < 1e-08 with pytest.raises(Exception): compcor.TR_string_to_float(None) with pytest.raises(Exception): - compcor.TR_string_to_float(['123']) + compcor.TR_string_to_float(["123"]) with pytest.raises(Exception): compcor.TR_string_to_float(123) with pytest.raises(Exception): - compcor.TR_string_to_float('ms') + compcor.TR_string_to_float("ms") diff --git a/CPAC/nuisance/utils/__init__.py b/CPAC/nuisance/utils/__init__.py index eba4ff528a..4bb9d55505 100644 --- a/CPAC/nuisance/utils/__init__.py +++ b/CPAC/nuisance/utils/__init__.py @@ -1,27 +1,30 @@ +from collections import OrderedDict import os import re -from collections import OrderedDict -import nipype.interfaces.ants as ants -import nipype.interfaces.fsl as fsl -import nipype.interfaces.utility as util -from CPAC.pipeline import nipype_pipeline_engine as pe -from nipype.interfaces import afni from nipype import logging +from nipype.interfaces import afni, ants, fsl +import nipype.interfaces.utility as util -from CPAC.nuisance.utils.compcor import calc_compcor_components from CPAC.nuisance.utils.crc import encode as crc_encode +from CPAC.pipeline import nipype_pipeline_engine as pe +from CPAC.registration.utils import generate_inverse_transform_flags from CPAC.utils.interfaces.fsl import Merge as fslMerge from CPAC.utils.interfaces.function import Function -from CPAC.registration.utils import check_transforms, generate_inverse_transform_flags -logger = logging.getLogger('nipype.workflow') +logger = logging.getLogger("nipype.workflow") -def find_offending_time_points(fd_j_file_path=None, fd_p_file_path=None, dvars_file_path=None, - fd_j_threshold=None, fd_p_threshold=None, dvars_threshold=None, - number_of_previous_trs_to_censor=0, - number_of_subsequent_trs_to_censor=0): +def find_offending_time_points( + fd_j_file_path=None, + fd_p_file_path=None, + dvars_file_path=None, + fd_j_threshold=None, + fd_p_threshold=None, + dvars_threshold=None, + number_of_previous_trs_to_censor=0, + number_of_subsequent_trs_to_censor=0, +): """ Applies criterion in method to find time points whose FD or DVARS (or both) are above threshold. @@ -48,65 +51,69 @@ def find_offending_time_points(fd_j_file_path=None, fd_p_file_path=None, dvars_f :return: File path to TSV file containing the volumes to be censored. """ - import numpy as np import os import re + import numpy as np + offending_time_points = set() time_course_len = 0 - types = ['FDJ', 'FDP', 'DVARS'] + types = ["FDJ", "FDP", "DVARS"] file_paths = [fd_j_file_path, fd_p_file_path, dvars_file_path] thresholds = [fd_j_threshold, fd_p_threshold, dvars_threshold] - #types = ['FDP', 'DVARS'] - #file_paths = [fd_p_file_path, dvars_file_path] - #thresholds = [fd_p_threshold, dvars_threshold] + # types = ['FDP', 'DVARS'] + # file_paths = [fd_p_file_path, dvars_file_path] + # thresholds = [fd_p_threshold, dvars_threshold] for type, file_path, threshold in zip(types, file_paths, thresholds): - if not file_path: continue if not os.path.isfile(file_path): - raise ValueError( - "File {0} could not be found." - .format(file_path) - ) + raise ValueError("File {0} could not be found.".format(file_path)) if not threshold: - raise ValueError("Method requires the specification of a threshold, none received") + raise ValueError( + "Method requires the specification of a threshold, none received" + ) metric = np.loadtxt(file_path) - if type == 'DVARS': - metric = np.array([0.0] + metric.tolist()) + if type == "DVARS": + metric = np.array([0.0, *metric.tolist()]) if not time_course_len: time_course_len = metric.shape[0] else: - assert time_course_len == metric.shape[0], "Threshold metric files does not have same size." + assert ( + time_course_len == metric.shape[0] + ), "Threshold metric files does not have same size." try: - threshold_sd = \ - re.match(r"([0-9]*\.*[0-9]*)\s*SD", str(threshold)) + threshold_sd = re.match(r"([0-9]*\.*[0-9]*)\s*SD", str(threshold)) if threshold_sd: threshold_sd = float(threshold_sd.groups()[0]) - threshold = metric.mean() + \ - threshold_sd * metric.std() + threshold = metric.mean() + threshold_sd * metric.std() else: threshold = float(threshold) except: - raise ValueError("Could not translate threshold {0} into a " - "meaningful value".format(threshold)) + raise ValueError( + "Could not translate threshold {0} into a " "meaningful value".format( + threshold + ) + ) - offending_time_points |= \ - set(np.where(metric > threshold)[0].tolist()) + offending_time_points |= set(np.where(metric > threshold)[0].tolist()) extended_censors = [] for censor in offending_time_points: - extended_censors += list(range( - (censor - number_of_previous_trs_to_censor), - (censor + number_of_subsequent_trs_to_censor + 1))) + extended_censors += list( + range( + (censor - number_of_previous_trs_to_censor), + (censor + number_of_subsequent_trs_to_censor + 1), + ) + ) extended_censors = [ censor @@ -118,9 +125,7 @@ def find_offending_time_points(fd_j_file_path=None, fd_p_file_path=None, dvars_f censor_vector[extended_censors] = 0 out_file_path = os.path.join(os.getcwd(), "censors.tsv") - np.savetxt( - out_file_path, censor_vector, fmt='%d', header='censor', comments='' - ) + np.savetxt(out_file_path, censor_vector, fmt="%d", header="censor", comments="") return out_file_path @@ -130,30 +135,28 @@ def compute_threshold(in_file, mask, threshold): def compute_pct_threshold(in_file, mask, threshold_pct): - import nibabel as nb import numpy as np - m = nb.load(mask).get_fdata().astype(bool) + import nibabel as nib + + m = nib.load(mask).get_fdata().astype(bool) if not np.any(m): return 0.0 - d = nb.load(in_file).get_fdata()[m] - return np.percentile( - d, - 100.0 - threshold_pct - ) + d = nib.load(in_file).get_fdata()[m] + return np.percentile(d, 100.0 - threshold_pct) def compute_sd_threshold(in_file, mask, threshold_sd): - import nibabel as nb import numpy as np - m = nb.load(mask).get_fdata().astype(bool) + import nibabel as nib + + m = nib.load(mask).get_fdata().astype(bool) if not np.any(m): return 0.0 - d = nb.load(in_file).get_fdata()[m] + d = nib.load(in_file).get_fdata()[m] return d.mean() + threshold_sd * d.std() def temporal_variance_mask(threshold, by_slice=False, erosion=False, degree=1): - threshold_method = "VAR" if isinstance(threshold, str): @@ -171,113 +174,152 @@ def temporal_variance_mask(threshold, by_slice=False, erosion=False, degree=1): try: threshold_value = float(threshold_value) except: - raise ValueError("Error converting threshold value {0} from {1} to a " - "floating point number. The threshold value can " - "contain SD or PCT for selecting a threshold based on " - "the variance distribution, otherwise it should be a " - "floating point number.".format(threshold_value, - threshold)) + raise ValueError( + "Error converting threshold value {0} from {1} to a " + "floating point number. The threshold value can " + "contain SD or PCT for selecting a threshold based on " + "the variance distribution, otherwise it should be a " + "floating point number.".format(threshold_value, threshold) + ) if threshold_value < 0: - raise ValueError("Threshold value should be positive, instead of {0}." - .format(threshold_value)) + raise ValueError( + "Threshold value should be positive, instead of {0}.".format( + threshold_value + ) + ) if threshold_method == "PCT" and threshold_value >= 100.0: - raise ValueError("Percentile should be less than 100, received {0}." - .format(threshold_value)) + raise ValueError( + "Percentile should be less than 100, received {0}.".format(threshold_value) + ) threshold = threshold_value - wf = pe.Workflow(name='tcompcor') + wf = pe.Workflow(name="tcompcor") - input_node = pe.Node(util.IdentityInterface(fields=['functional_file_path', 'mask_file_path']), name='inputspec') - output_node = pe.Node(util.IdentityInterface(fields=['mask']), name='outputspec') + input_node = pe.Node( + util.IdentityInterface(fields=["functional_file_path", "mask_file_path"]), + name="inputspec", + ) + output_node = pe.Node(util.IdentityInterface(fields=["mask"]), name="outputspec") # C-PAC default performs linear regression while nipype performs quadratic regression - detrend = pe.Node(afni.Detrend(args='-polort {0}'.format(degree), outputtype='NIFTI'), name='detrend') - wf.connect(input_node, 'functional_file_path', detrend, 'in_file') + detrend = pe.Node( + afni.Detrend(args="-polort {0}".format(degree), outputtype="NIFTI"), + name="detrend", + ) + wf.connect(input_node, "functional_file_path", detrend, "in_file") - std = pe.Node(afni.TStat(args='-nzstdev', outputtype='NIFTI'), name='std') - wf.connect(input_node, 'mask_file_path', std, 'mask') - wf.connect(detrend, 'out_file', std, 'in_file') + std = pe.Node(afni.TStat(args="-nzstdev", outputtype="NIFTI"), name="std") + wf.connect(input_node, "mask_file_path", std, "mask") + wf.connect(detrend, "out_file", std, "in_file") - var = pe.Node(afni.Calc(expr='a*a', outputtype='NIFTI'), name='var') - wf.connect(std, 'out_file', var, 'in_file_a') + var = pe.Node(afni.Calc(expr="a*a", outputtype="NIFTI"), name="var") + wf.connect(std, "out_file", var, "in_file_a") if by_slice: - slices = pe.Node(fsl.Slice(), name='slicer') - wf.connect(var, 'out_file', slices, 'in_file') + slices = pe.Node(fsl.Slice(), name="slicer") + wf.connect(var, "out_file", slices, "in_file") - mask_slices = pe.Node(fsl.Slice(), name='mask_slicer') - wf.connect(input_node, 'mask_file_path', mask_slices, 'in_file') + mask_slices = pe.Node(fsl.Slice(), name="mask_slicer") + wf.connect(input_node, "mask_file_path", mask_slices, "in_file") - mapper = pe.MapNode(util.IdentityInterface(fields=['out_file', 'mask_file']), name='slice_mapper', iterfield=['out_file', 'mask_file']) - wf.connect(slices, 'out_files', mapper, 'out_file') - wf.connect(mask_slices, 'out_files', mapper, 'mask_file') + mapper = pe.MapNode( + util.IdentityInterface(fields=["out_file", "mask_file"]), + name="slice_mapper", + iterfield=["out_file", "mask_file"], + ) + wf.connect(slices, "out_files", mapper, "out_file") + wf.connect(mask_slices, "out_files", mapper, "mask_file") else: - mapper_list = pe.Node(util.Merge(1), name='slice_mapper_list') - wf.connect(var, 'out_file', mapper_list, 'in1') + mapper_list = pe.Node(util.Merge(1), name="slice_mapper_list") + wf.connect(var, "out_file", mapper_list, "in1") - mask_mapper_list = pe.Node(util.Merge(1), name='slice_mask_mapper_list') - wf.connect(input_node, 'mask_file_path', mask_mapper_list, 'in1') + mask_mapper_list = pe.Node(util.Merge(1), name="slice_mask_mapper_list") + wf.connect(input_node, "mask_file_path", mask_mapper_list, "in1") - mapper = pe.Node(util.IdentityInterface(fields=['out_file', 'mask_file']), name='slice_mapper') - wf.connect(mapper_list, 'out', mapper, 'out_file') - wf.connect(mask_mapper_list, 'out', mapper, 'mask_file') + mapper = pe.Node( + util.IdentityInterface(fields=["out_file", "mask_file"]), + name="slice_mapper", + ) + wf.connect(mapper_list, "out", mapper, "out_file") + wf.connect(mask_mapper_list, "out", mapper, "mask_file") if threshold_method == "PCT": - threshold_node = pe.MapNode(Function(input_names=['in_file', 'mask', 'threshold_pct'], - output_names=['threshold'], - function=compute_pct_threshold, as_module=True), - name='threshold_value', iterfield=['in_file', 'mask']) + threshold_node = pe.MapNode( + Function( + input_names=["in_file", "mask", "threshold_pct"], + output_names=["threshold"], + function=compute_pct_threshold, + as_module=True, + ), + name="threshold_value", + iterfield=["in_file", "mask"], + ) threshold_node.inputs.threshold_pct = threshold_value - wf.connect(mapper, 'out_file', threshold_node, 'in_file') - wf.connect(mapper, 'mask_file', threshold_node, 'mask') + wf.connect(mapper, "out_file", threshold_node, "in_file") + wf.connect(mapper, "mask_file", threshold_node, "mask") elif threshold_method == "SD": - threshold_node = pe.MapNode(Function(input_names=['in_file', 'mask', 'threshold_sd'], - output_names=['threshold'], - function=compute_sd_threshold, as_module=True), - name='threshold_value', iterfield=['in_file', 'mask']) + threshold_node = pe.MapNode( + Function( + input_names=["in_file", "mask", "threshold_sd"], + output_names=["threshold"], + function=compute_sd_threshold, + as_module=True, + ), + name="threshold_value", + iterfield=["in_file", "mask"], + ) threshold_node.inputs.threshold_sd = threshold_value - wf.connect(mapper, 'out_file', threshold_node, 'in_file') - wf.connect(mapper, 'mask_file', threshold_node, 'mask') + wf.connect(mapper, "out_file", threshold_node, "in_file") + wf.connect(mapper, "mask_file", threshold_node, "mask") else: - threshold_node = pe.MapNode(Function(input_names=['in_file', 'mask', 'threshold'], - output_names=['threshold'], - function=compute_threshold, as_module=True), - name='threshold_value', iterfield=['in_file', 'mask']) + threshold_node = pe.MapNode( + Function( + input_names=["in_file", "mask", "threshold"], + output_names=["threshold"], + function=compute_threshold, + as_module=True, + ), + name="threshold_value", + iterfield=["in_file", "mask"], + ) threshold_node.inputs.threshold = threshold_value - wf.connect(mapper, 'out_file', threshold_node, 'in_file') - wf.connect(mapper, 'mask_file', threshold_node, 'mask') - - threshold_mask = pe.MapNode(interface=fsl.maths.Threshold(), name='threshold', iterfield=['in_file', 'thresh']) - threshold_mask.inputs.args = '-bin' - wf.connect(mapper, 'out_file', threshold_mask, 'in_file') - wf.connect(threshold_node, 'threshold', threshold_mask, 'thresh') - - merge_slice_masks = pe.Node(interface=fslMerge(), name='merge_slice_masks') - merge_slice_masks.inputs.dimension = 'z' - wf.connect( - threshold_mask, 'out_file', - merge_slice_masks, 'in_files' + wf.connect(mapper, "out_file", threshold_node, "in_file") + wf.connect(mapper, "mask_file", threshold_node, "mask") + + threshold_mask = pe.MapNode( + interface=fsl.maths.Threshold(), + name="threshold", + iterfield=["in_file", "thresh"], ) + threshold_mask.inputs.args = "-bin" + wf.connect(mapper, "out_file", threshold_mask, "in_file") + wf.connect(threshold_node, "threshold", threshold_mask, "thresh") - wf.connect(merge_slice_masks, 'merged_file', output_node, 'mask') + merge_slice_masks = pe.Node(interface=fslMerge(), name="merge_slice_masks") + merge_slice_masks.inputs.dimension = "z" + wf.connect(threshold_mask, "out_file", merge_slice_masks, "in_files") + + wf.connect(merge_slice_masks, "merged_file", output_node, "mask") return wf -def generate_summarize_tissue_mask(nuisance_wf, - pipeline_resource_pool, - regressor_descriptor, - regressor_selector, - csf_mask_exist, - use_ants=True, - ventricle_mask_exist=True, - all_bold=False): +def generate_summarize_tissue_mask( + nuisance_wf, + pipeline_resource_pool, + regressor_descriptor, + regressor_selector, + csf_mask_exist, + use_ants=True, + ventricle_mask_exist=True, + all_bold=False, +): """ Add tissue mask generation into pipeline according to the selector. @@ -290,227 +332,309 @@ def generate_summarize_tissue_mask(nuisance_wf, :return: the full path of the 3D nifti file containing the mask created by this operation. """ - steps = [ key - for key in ['tissue', 'resolution', 'erosion'] + for key in ["tissue", "resolution", "erosion"] if key in regressor_descriptor ] - full_mask_key = "_".join( - regressor_descriptor[s] - for s in steps - ) + full_mask_key = "_".join(regressor_descriptor[s] for s in steps) for step_i, step in enumerate(steps): - - mask_key = "_".join( - regressor_descriptor[s] - for s in steps[:step_i+1] - ) + mask_key = "_".join(regressor_descriptor[s] for s in steps[: step_i + 1]) if mask_key in pipeline_resource_pool: continue node_mask_key = re.sub(r"[^\w]", "_", mask_key) - prev_mask_key = "_".join( - regressor_descriptor[s] - for s in steps[:step_i] - ) + prev_mask_key = "_".join(regressor_descriptor[s] for s in steps[:step_i]) - if step == 'tissue': + if step == "tissue": pass - elif step == 'resolution': - + elif step == "resolution": if all_bold: pass if csf_mask_exist: - mask_to_epi = pe.Node(interface=fsl.FLIRT(), - name='{}_flirt'.format(node_mask_key), - mem_gb=3.63, - mem_x=(3767129957844731 / 1208925819614629174706176, - 'in_file')) + mask_to_epi = pe.Node( + interface=fsl.FLIRT(), + name="{}_flirt".format(node_mask_key), + mem_gb=3.63, + mem_x=(3767129957844731 / 1208925819614629174706176, "in_file"), + ) - mask_to_epi.inputs.interp = 'nearestneighbour' + mask_to_epi.inputs.interp = "nearestneighbour" - if regressor_selector['extraction_resolution'] == "Functional": + if regressor_selector["extraction_resolution"] == "Functional": # apply anat2func matrix mask_to_epi.inputs.apply_xfm = True - mask_to_epi.inputs.output_type = 'NIFTI_GZ' - nuisance_wf.connect(*( - pipeline_resource_pool['Functional_mean'] + - (mask_to_epi, 'reference') - )) - nuisance_wf.connect(*( - pipeline_resource_pool['Transformations']['anat_to_func_linear_xfm'] + - (mask_to_epi, 'in_matrix_file') - )) + mask_to_epi.inputs.output_type = "NIFTI_GZ" + nuisance_wf.connect( + *( + pipeline_resource_pool["Functional_mean"] + + (mask_to_epi, "reference") + ) + ) + nuisance_wf.connect( + *( + pipeline_resource_pool["Transformations"][ + "anat_to_func_linear_xfm" + ] + + (mask_to_epi, "in_matrix_file") + ) + ) else: - resolution = regressor_selector['extraction_resolution'] + resolution = regressor_selector["extraction_resolution"] mask_to_epi.inputs.apply_isoxfm = resolution - nuisance_wf.connect(*( - pipeline_resource_pool['Anatomical_{}mm' - .format(resolution)] + - (mask_to_epi, 'reference') - )) - - nuisance_wf.connect(*( - pipeline_resource_pool[prev_mask_key] + - (mask_to_epi, 'in_file') - )) - - pipeline_resource_pool[mask_key] = \ - (mask_to_epi, 'out_file') - - if full_mask_key.startswith('CerebrospinalFluid'): - pipeline_resource_pool = generate_summarize_tissue_mask_ventricles_masking( - nuisance_wf, - pipeline_resource_pool, - regressor_descriptor, - regressor_selector, - node_mask_key, - csf_mask_exist, - use_ants, - ventricle_mask_exist + nuisance_wf.connect( + *( + pipeline_resource_pool["Anatomical_{}mm".format(resolution)] + + (mask_to_epi, "reference") + ) + ) + + nuisance_wf.connect( + *(pipeline_resource_pool[prev_mask_key] + (mask_to_epi, "in_file")) ) - elif step == 'erosion': + pipeline_resource_pool[mask_key] = (mask_to_epi, "out_file") + + if full_mask_key.startswith("CerebrospinalFluid"): + pipeline_resource_pool = ( + generate_summarize_tissue_mask_ventricles_masking( + nuisance_wf, + pipeline_resource_pool, + regressor_descriptor, + regressor_selector, + node_mask_key, + csf_mask_exist, + use_ants, + ventricle_mask_exist, + ) + ) + elif step == "erosion": erode_mask_node = pe.Node( - afni.Calc(args='-b a+i -c a-i -d a+j -e a-j -f a+k -g a-k', expr='a*(1-amongst(0,b,c,d,e,f,g))', outputtype='NIFTI_GZ'), - name='{}'.format(node_mask_key) + afni.Calc( + args="-b a+i -c a-i -d a+j -e a-j -f a+k -g a-k", + expr="a*(1-amongst(0,b,c,d,e,f,g))", + outputtype="NIFTI_GZ", + ), + name="{}".format(node_mask_key), ) - nuisance_wf.connect(*( - pipeline_resource_pool[prev_mask_key] + - (erode_mask_node, 'in_file_a') - )) - - pipeline_resource_pool[mask_key] = \ - (erode_mask_node, 'out_file') - - return pipeline_resource_pool, full_mask_key - - -def generate_summarize_tissue_mask_ventricles_masking(nuisance_wf, - pipeline_resource_pool, - regressor_descriptor, - regressor_selector, - mask_key, - csf_mask_exist, - use_ants=True, - ventricle_mask_exist=True): + nuisance_wf.connect( + *( + pipeline_resource_pool[prev_mask_key] + + (erode_mask_node, "in_file_a") + ) + ) - if csf_mask_exist == False: - logger.warning('Segmentation is Off, - therefore will be using ' - 'lateral_ventricle_mask as CerebrospinalFluid_mask.') + pipeline_resource_pool[mask_key] = (erode_mask_node, "out_file") + + return pipeline_resource_pool, full_mask_key + + +def generate_summarize_tissue_mask_ventricles_masking( + nuisance_wf, + pipeline_resource_pool, + regressor_descriptor, + regressor_selector, + mask_key, + csf_mask_exist, + use_ants=True, + ventricle_mask_exist=True, +): + if csf_mask_exist is False: + logger.warning( + "Segmentation is Off, - therefore will be using " + "lateral_ventricle_mask as CerebrospinalFluid_mask." + ) # Mask CSF with Ventricles - if '{}_Unmasked'.format(mask_key) not in pipeline_resource_pool: - + if "{}_Unmasked".format(mask_key) not in pipeline_resource_pool: if ventricle_mask_exist: - ventricles_key = 'VentriclesToAnat' + ventricles_key = "VentriclesToAnat" - if 'resolution' in regressor_descriptor: - ventricles_key += '_{}'.format(regressor_descriptor['resolution']) + if "resolution" in regressor_descriptor: + ventricles_key += "_{}".format(regressor_descriptor["resolution"]) if ventricles_key not in pipeline_resource_pool: - - transforms = pipeline_resource_pool['Transformations'] + transforms = pipeline_resource_pool["Transformations"] if use_ants is True: - # perform the transform using ANTS - collect_linear_transforms = pe.Node(util.Merge(3), name='{}_ants_transforms'.format(ventricles_key)) + collect_linear_transforms = pe.Node( + util.Merge(3), name="{}_ants_transforms".format(ventricles_key) + ) - nuisance_wf.connect(*(transforms['mni_to_anat_linear_xfm'] + (collect_linear_transforms, 'in1'))) + nuisance_wf.connect( + *( + transforms["mni_to_anat_linear_xfm"] + + (collect_linear_transforms, "in1") + ) + ) # generate inverse transform flags, which depends on the number of transforms - inverse_transform_flags = pe.Node(util.Function(input_names=['transform_list'], - output_names=['inverse_transform_flags'], - function=generate_inverse_transform_flags), - name='{0}_inverse_transform_flags'.format(ventricles_key)) - nuisance_wf.connect(collect_linear_transforms, 'out', inverse_transform_flags, 'transform_list') + inverse_transform_flags = pe.Node( + util.Function( + input_names=["transform_list"], + output_names=["inverse_transform_flags"], + function=generate_inverse_transform_flags, + ), + name="{0}_inverse_transform_flags".format(ventricles_key), + ) + nuisance_wf.connect( + collect_linear_transforms, + "out", + inverse_transform_flags, + "transform_list", + ) lat_ven_mni_to_anat = pe.Node( interface=ants.ApplyTransforms(), - name='{}_ants'.format(ventricles_key), + name="{}_ants".format(ventricles_key), mem_gb=0.683, - mem_x=(3811976743057169 / 302231454903657293676544, - 'input_image')) - lat_ven_mni_to_anat.inputs.interpolation = 'NearestNeighbor' + mem_x=( + 3811976743057169 / 302231454903657293676544, + "input_image", + ), + ) + lat_ven_mni_to_anat.inputs.interpolation = "NearestNeighbor" lat_ven_mni_to_anat.inputs.dimension = 3 - nuisance_wf.connect(inverse_transform_flags, 'inverse_transform_flags', lat_ven_mni_to_anat, 'invert_transform_flags') - nuisance_wf.connect(collect_linear_transforms, 'out', lat_ven_mni_to_anat, 'transforms') - - nuisance_wf.connect(*(pipeline_resource_pool['Ventricles'] + (lat_ven_mni_to_anat, 'input_image'))) - resolution = regressor_selector['extraction_resolution'] + nuisance_wf.connect( + inverse_transform_flags, + "inverse_transform_flags", + lat_ven_mni_to_anat, + "invert_transform_flags", + ) + nuisance_wf.connect( + collect_linear_transforms, + "out", + lat_ven_mni_to_anat, + "transforms", + ) + + nuisance_wf.connect( + *( + pipeline_resource_pool["Ventricles"] + + (lat_ven_mni_to_anat, "input_image") + ) + ) + resolution = regressor_selector["extraction_resolution"] if csf_mask_exist: - nuisance_wf.connect(*( - pipeline_resource_pool[mask_key] + - (lat_ven_mni_to_anat, 'reference_image'))) - elif resolution == 'Functional': - nuisance_wf.connect(*( - pipeline_resource_pool['Functional_mean'] + - (lat_ven_mni_to_anat, 'reference_image'))) + nuisance_wf.connect( + *( + pipeline_resource_pool[mask_key] + + (lat_ven_mni_to_anat, "reference_image") + ) + ) + elif resolution == "Functional": + nuisance_wf.connect( + *( + pipeline_resource_pool["Functional_mean"] + + (lat_ven_mni_to_anat, "reference_image") + ) + ) else: - nuisance_wf.connect(*( - pipeline_resource_pool['Anatomical_{}mm'.format(resolution)] + - (lat_ven_mni_to_anat, 'reference_image'))) - - pipeline_resource_pool[ventricles_key] = (lat_ven_mni_to_anat, 'output_image') + nuisance_wf.connect( + *( + pipeline_resource_pool[ + "Anatomical_{}mm".format(resolution) + ] + + (lat_ven_mni_to_anat, "reference_image") + ) + ) + + pipeline_resource_pool[ventricles_key] = ( + lat_ven_mni_to_anat, + "output_image", + ) else: # perform the transform using FLIRT - lat_ven_mni_to_anat = pe.Node(interface=fsl.ApplyWarp(), - name='{}_fsl_applywarp'.format(ventricles_key)) - lat_ven_mni_to_anat.inputs.interp = 'nn' - - nuisance_wf.connect(*(transforms['mni_to_anat_linear_xfm'] + (lat_ven_mni_to_anat, 'field_file'))) - nuisance_wf.connect(*(pipeline_resource_pool['Ventricles'] + (lat_ven_mni_to_anat, 'in_file'))) - nuisance_wf.connect(*(pipeline_resource_pool[mask_key] + (lat_ven_mni_to_anat, 'ref_file'))) - - pipeline_resource_pool[ventricles_key] = (lat_ven_mni_to_anat, 'out_file') + lat_ven_mni_to_anat = pe.Node( + interface=fsl.ApplyWarp(), + name="{}_fsl_applywarp".format(ventricles_key), + ) + lat_ven_mni_to_anat.inputs.interp = "nn" + + nuisance_wf.connect( + *( + transforms["mni_to_anat_linear_xfm"] + + (lat_ven_mni_to_anat, "field_file") + ) + ) + nuisance_wf.connect( + *( + pipeline_resource_pool["Ventricles"] + + (lat_ven_mni_to_anat, "in_file") + ) + ) + nuisance_wf.connect( + *( + pipeline_resource_pool[mask_key] + + (lat_ven_mni_to_anat, "ref_file") + ) + ) + + pipeline_resource_pool[ventricles_key] = ( + lat_ven_mni_to_anat, + "out_file", + ) if csf_mask_exist: # reduce CSF mask to the lateral ventricles - mask_csf_with_lat_ven = pe.Node(interface=afni.Calc(outputtype='NIFTI_GZ'), - name='{}_Ventricles'.format(mask_key)) - mask_csf_with_lat_ven.inputs.expr = 'a*b' - mask_csf_with_lat_ven.inputs.out_file = 'csf_lat_ven_mask.nii.gz' - - nuisance_wf.connect(*(pipeline_resource_pool[ventricles_key] + (mask_csf_with_lat_ven, 'in_file_a'))) - nuisance_wf.connect(*(pipeline_resource_pool[mask_key] + (mask_csf_with_lat_ven, 'in_file_b'))) + mask_csf_with_lat_ven = pe.Node( + interface=afni.Calc(outputtype="NIFTI_GZ"), + name="{}_Ventricles".format(mask_key), + ) + mask_csf_with_lat_ven.inputs.expr = "a*b" + mask_csf_with_lat_ven.inputs.out_file = "csf_lat_ven_mask.nii.gz" + + nuisance_wf.connect( + *( + pipeline_resource_pool[ventricles_key] + + (mask_csf_with_lat_ven, "in_file_a") + ) + ) + nuisance_wf.connect( + *( + pipeline_resource_pool[mask_key] + + (mask_csf_with_lat_ven, "in_file_b") + ) + ) - pipeline_resource_pool['{}_Unmasked'.format(mask_key)] = pipeline_resource_pool[mask_key] - pipeline_resource_pool[mask_key] = (mask_csf_with_lat_ven, 'out_file') + pipeline_resource_pool[ + "{}_Unmasked".format(mask_key) + ] = pipeline_resource_pool[mask_key] + pipeline_resource_pool[mask_key] = (mask_csf_with_lat_ven, "out_file") else: - pipeline_resource_pool[mask_key] = pipeline_resource_pool[ventricles_key] - - return pipeline_resource_pool + pipeline_resource_pool[mask_key] = pipeline_resource_pool[ + ventricles_key + ] + return pipeline_resource_pool + return None class NuisanceRegressor(object): - def __init__(self, selector): self.selector = selector - if 'Bandpass' in self.selector: - s = self.selector['Bandpass'] - if type(s) is not dict or \ - (not s.get('bottom_frequency') and - not s.get('top_frequency')): - - del self.selector['Bandpass'] + if "Bandpass" in self.selector: + s = self.selector["Bandpass"] + if type(s) is not dict or ( + not s.get("bottom_frequency") and not s.get("top_frequency") + ): + del self.selector["Bandpass"] def get(self, key, default=None): return self.selector.get(key, default) @@ -523,39 +647,39 @@ def __getitem__(self, key): @staticmethod def _derivative_params(selector): - nr_repr = '' + nr_repr = "" if not selector: return nr_repr - if selector.get('include_squared'): - nr_repr += 'S' - if selector.get('include_delayed'): - nr_repr += 'D' - if selector.get('include_delayed_squared'): - nr_repr += 'B' - if selector.get('include_backdiff'): - nr_repr += 'V' - if selector.get('include_backdiff_squared'): - nr_repr += 'C' + if selector.get("include_squared"): + nr_repr += "S" + if selector.get("include_delayed"): + nr_repr += "D" + if selector.get("include_delayed_squared"): + nr_repr += "B" + if selector.get("include_backdiff"): + nr_repr += "V" + if selector.get("include_backdiff_squared"): + nr_repr += "C" return nr_repr @staticmethod def _summary_params(selector): - summ = selector['summary'] + summ = selector["summary"] methods = { - 'PC': 'PC', - 'DetrendPC': 'DPC', - 'Mean': 'M', - 'NormMean': 'NM', - 'DetrendMean': 'DM', - 'DetrendNormMean': 'DNM', + "PC": "PC", + "DetrendPC": "DPC", + "Mean": "M", + "NormMean": "NM", + "DetrendMean": "DM", + "DetrendNormMean": "DNM", } if type(summ) == dict: - method = summ['method'] + method = summ["method"] rep = methods[method] - if method in ['DetrendPC', 'PC']: - rep += "%d" % summ['components'] + if method in ["DetrendPC", "PC"]: + rep += "%d" % summ["components"] else: rep = methods[summ] @@ -563,21 +687,23 @@ def _summary_params(selector): @staticmethod def encode(selector): - regs = OrderedDict([ - ('GreyMatter', 'GM'), - ('WhiteMatter', 'WM'), - ('CerebrospinalFluid', 'CSF'), - ('tCompCor', 'tC'), - ('aCompCor', 'aC'), - ('GlobalSignal', 'G'), - ('Motion', 'M'), - ('Custom', 'T'), - ('PolyOrt', 'P'), - ('Bandpass', 'BP'), - ('Censor', 'C') - ]) - - tissues = ['GreyMatter', 'WhiteMatter', 'CerebrospinalFluid'] + regs = OrderedDict( + [ + ("GreyMatter", "GM"), + ("WhiteMatter", "WM"), + ("CerebrospinalFluid", "CSF"), + ("tCompCor", "tC"), + ("aCompCor", "aC"), + ("GlobalSignal", "G"), + ("Motion", "M"), + ("Custom", "T"), + ("PolyOrt", "P"), + ("Bandpass", "BP"), + ("Censor", "C"), + ] + ) + + tissues = ["GreyMatter", "WhiteMatter", "CerebrospinalFluid"] selectors_representations = [] @@ -603,106 +729,107 @@ def encode(selector): pieces = [regs[r]] if r in tissues: - if s.get('extraction_resolution') and s['extraction_resolution'] != 'Functional': - res = "%.2gmm" % s['extraction_resolution'] - if s.get('erode_mask'): - res += 'E' - pieces += [res] + if ( + s.get("extraction_resolution") + and s["extraction_resolution"] != "Functional" + ): + res = "%.2gmm" % s["extraction_resolution"] + if s.get("erode_mask"): + res += "E" + pieces += [res] pieces += [NuisanceRegressor._summary_params(s)] pieces += [NuisanceRegressor._derivative_params(s)] - elif r == 'tCompCor': - + elif r == "tCompCor": threshold = "" - if s.get('by_slice'): - threshold += 'S' - t = s.get('threshold') + if s.get("by_slice"): + threshold += "S" + t = s.get("threshold") if t: if type(t) != str: t = "%.2f" % t threshold += t - if s.get('erode_mask'): - threshold += 'E' - if s.get('degree'): - d = s.get('degree') + if s.get("erode_mask"): + threshold += "E" + if s.get("degree"): + d = s.get("degree") threshold += str(d) pieces += [threshold] pieces += [NuisanceRegressor._summary_params(s)] pieces += [NuisanceRegressor._derivative_params(s)] - elif r == 'aCompCor': - if s.get('tissues'): - pieces += ["+".join([regs[t] for t in sorted(s['tissues'])])] + elif r == "aCompCor": + if s.get("tissues"): + pieces += ["+".join([regs[t] for t in sorted(s["tissues"])])] - if s.get('extraction_resolution'): - res = "%.2gmm" % s['extraction_resolution'] - if s.get('erode_mask'): - res += 'E' + if s.get("extraction_resolution"): + res = "%.2gmm" % s["extraction_resolution"] + if s.get("erode_mask"): + res += "E" pieces += [res] pieces += [NuisanceRegressor._summary_params(s)] pieces += [NuisanceRegressor._derivative_params(s)] - elif r == 'Custom': + elif r == "Custom": for ss in s: pieces += [ - os.path.basename(ss['file'])[0:5] + - crc_encode(ss['file']) + os.path.basename(ss["file"])[0:5] + crc_encode(ss["file"]) ] - elif r == 'GlobalSignal': + elif r == "GlobalSignal": pieces += [NuisanceRegressor._summary_params(s)] pieces += [NuisanceRegressor._derivative_params(s)] - elif r == 'Motion': + elif r == "Motion": pieces += [NuisanceRegressor._derivative_params(s)] - elif r == 'PolyOrt': - pieces += ['%d' % s['degree']] + elif r == "PolyOrt": + pieces += ["%d" % s["degree"]] - elif r == 'Bandpass': - if s.get('bottom_frequency'): - pieces += ['B%.2g' % s['bottom_frequency']] - if s.get('top_frequency'): - pieces += ['T%.2g' % s['top_frequency']] + elif r == "Bandpass": + if s.get("bottom_frequency"): + pieces += ["B%.2g" % s["bottom_frequency"]] + if s.get("top_frequency"): + pieces += ["T%.2g" % s["top_frequency"]] - elif r == 'Censor': + elif r == "Censor": censoring = { - 'Kill': 'K', - 'Zero': 'Z', - 'Interpolate': 'I', - 'SpikeRegression': 'S', + "Kill": "K", + "Zero": "Z", + "Interpolate": "I", + "SpikeRegression": "S", } thresholds = { - 'FD_J': 'FD-J', - 'FD_P': 'FD-P', - 'DVARS': 'DV', + "FD_J": "FD-J", + "FD_P": "FD-P", + "DVARS": "DV", } - pieces += [censoring[s['method']]] + pieces += [censoring[s["method"]]] - trs_range = ['0', '0'] - if s.get('number_of_previous_trs_to_censor'): - trs_range[0] = '%d' % s['number_of_previous_trs_to_censor'] - if s.get('number_of_subsequent_trs_to_censor'): - trs_range[1] = '%d' % s['number_of_subsequent_trs_to_censor'] + trs_range = ["0", "0"] + if s.get("number_of_previous_trs_to_censor"): + trs_range[0] = "%d" % s["number_of_previous_trs_to_censor"] + if s.get("number_of_subsequent_trs_to_censor"): + trs_range[1] = "%d" % s["number_of_subsequent_trs_to_censor"] - pieces += ['+'.join(trs_range)] + pieces += ["+".join(trs_range)] - threshs = sorted(s['thresholds'], reverse=True, key=lambda d: d['type']) + threshs = sorted(s["thresholds"], reverse=True, key=lambda d: d["type"]) for st in threshs: - thresh = thresholds[st['type']] - if type(st['value']) == str: - thresh += st['value'] + thresh = thresholds[st["type"]] + if type(st["value"]) == str: + thresh += st["value"] else: - thresh += "%.2g" % st['value'] + thresh += "%.2g" % st["value"] pieces += [thresh] - selectors_representations += ['-'.join([_f for _f in pieces if _f])] + selectors_representations += ["-".join([_f for _f in pieces if _f])] return "_".join(selectors_representations) diff --git a/CPAC/nuisance/utils/compcor.py b/CPAC/nuisance/utils/compcor.py index 00320933a0..c6093c49cc 100644 --- a/CPAC/nuisance/utils/compcor.py +++ b/CPAC/nuisance/utils/compcor.py @@ -1,92 +1,104 @@ import os -import scipy.signal as signal -import nibabel as nb + import numpy as np -from CPAC.utils import safe_shape +import nibabel as nib from nipype import logging +from scipy import signal from scipy.linalg import svd -iflogger = logging.getLogger('nipype.interface') +from CPAC.utils import safe_shape +iflogger = logging.getLogger("nipype.interface") -def calc_compcor_components(data_filename, num_components, mask_filename): +def calc_compcor_components(data_filename, num_components, mask_filename): if num_components < 1: - raise ValueError('Improper value for num_components ({0}), should be >= 1.'.format(num_components)) + raise ValueError( + "Improper value for num_components ({0}), should be >= 1.".format( + num_components + ) + ) try: - image_data = nb.load(data_filename).get_fdata().astype(np.float64) + image_data = nib.load(data_filename).get_fdata().astype(np.float64) except: - print('Unable to load data from {0}'.format(data_filename)) raise try: - binary_mask = nb.load(mask_filename).get_fdata().astype(np.int16) + binary_mask = nib.load(mask_filename).get_fdata().astype(np.int16) except: - print('Unable to load data from {0}'.format(mask_filename)) + pass if not safe_shape(image_data, binary_mask): - raise ValueError('The data in {0} and {1} do not have a consistent shape'.format(data_filename, mask_filename)) + raise ValueError( + "The data in {0} and {1} do not have a consistent shape".format( + data_filename, mask_filename + ) + ) # make sure that the values in binary_mask are binary binary_mask[binary_mask > 0] = 1 binary_mask[binary_mask != 1] = 0 # reduce the image data to only the voxels in the binary mask - image_data = image_data[binary_mask==1, :] + image_data = image_data[binary_mask == 1, :] # filter out any voxels whose variance equals 0 - print('Removing zero variance components') - image_data = image_data[image_data.std(1)!=0,:] + image_data = image_data[image_data.std(1) != 0, :] if image_data.shape.count(0): - err = "\n\n[!] No wm or csf signals left after removing those " \ - "with zero variance.\n\n" + err = ( + "\n\n[!] No wm or csf signals left after removing those " + "with zero variance.\n\n" + ) raise Exception(err) - print('Detrending and centering data') - Y = signal.detrend(image_data, axis=1, type='linear').T + Y = signal.detrend(image_data, axis=1, type="linear").T Yc = Y - np.tile(Y.mean(0), (Y.shape[0], 1)) - Yc = Yc / np.tile(np.array(Yc.std(0)).reshape(1,Yc.shape[1]), (Yc.shape[0],1)) + Yc = Yc / np.tile(np.array(Yc.std(0)).reshape(1, Yc.shape[1]), (Yc.shape[0], 1)) - print('Calculating SVD decomposition of Y*Y\'') U, S, Vh = np.linalg.svd(Yc, full_matrices=False) # write out the resulting regressor file - regressor_file = os.path.join(os.getcwd(), 'compcor_regressors.1D') - np.savetxt(regressor_file, U[:, :num_components], delimiter='\t', fmt='%16g') + regressor_file = os.path.join(os.getcwd(), "compcor_regressors.1D") + np.savetxt(regressor_file, U[:, :num_components], delimiter="\t", fmt="%16g") return regressor_file # cosine_filter adapted from nipype 'https://github.com/nipy/nipype/blob/d353f0d879826031334b09d33e9443b8c9b3e7fe/nipype/algorithms/confounds.py' -def cosine_filter(input_image_path, timestep, period_cut=128, remove_mean=True, axis=-1, failure_mode='error'): +def cosine_filter( + input_image_path, + timestep, + period_cut=128, + remove_mean=True, + axis=-1, + failure_mode="error", +): """ input_image_path: string Bold image to be filtered. timestep: float 'Repetition time (TR) of series (in sec) - derived from image header if unspecified' period_cut: float - Minimum period (in sec) for DCT high-pass filter, nipype default value: 128 + Minimum period (in sec) for DCT high-pass filter, nipype default value: 128. """ + from CPAC.nuisance.utils.compcor import _cosine_drift, _full_rank - from CPAC.nuisance.utils.compcor import _full_rank - from CPAC.nuisance.utils.compcor import _cosine_drift - - input_img = nb.load(input_image_path) + input_img = nib.load(input_image_path) input_data = input_img.get_fdata() datashape = input_data.shape timepoints = datashape[axis] - if datashape[0] == 0 and failure_mode != 'error': + if datashape[0] == 0 and failure_mode != "error": return input_data, np.array([]) input_data = input_data.reshape((-1, timepoints)) frametimes = timestep * np.arange(timepoints) X = _full_rank(_cosine_drift(period_cut, frametimes))[0] - non_constant_regressors = X[:, :-1] if X.shape[1] > 1 else np.array([]) + X[:, :-1] if X.shape[1] > 1 else np.array([]) betas = np.linalg.lstsq(X, input_data.T)[0] @@ -99,10 +111,9 @@ def cosine_filter(input_image_path, timestep, period_cut=128, remove_mean=True, output_data = residuals.reshape(datashape) hdr = input_img.header - output_img = nb.Nifti1Image(output_data, header=hdr, - affine=input_img.affine) + output_img = nib.Nifti1Image(output_data, header=hdr, affine=input_img.affine) - file_name = input_image_path[input_image_path.rindex('/')+1:] + file_name = input_image_path[input_image_path.rindex("/") + 1 :] cosfiltered_img = os.path.join(os.getcwd(), file_name) @@ -114,13 +125,15 @@ def cosine_filter(input_image_path, timestep, period_cut=128, remove_mean=True, # _cosine_drift and _full_rank copied from nipype 'https://github.com/nipy/nipype/blob/d353f0d879826031334b09d33e9443b8c9b3e7fe/nipype/algorithms/confounds.py' def _cosine_drift(period_cut, frametimes): """ - Create a cosine drift matrix with periods greater or equals to period_cut - Parameters + Create a cosine drift matrix with periods greater or equals to period_cut. + + Parameters. ---------- period_cut : float Cut period of the low-pass filter (in sec) frametimes : array of shape(nscans) The sampling times (in sec) + Returns ------- cdrift : array of shape(n_scans, n_drifts) @@ -129,7 +142,7 @@ def _cosine_drift(period_cut, frametimes): """ len_tim = len(frametimes) n_times = np.arange(len_tim) - hfcut = 1. / period_cut # input parameter is the period + hfcut = 1.0 / period_cut # input parameter is the period # frametimes.max() should be (len_tim-1)*dt dt = frametimes[1] - frametimes[0] @@ -140,20 +153,22 @@ def _cosine_drift(period_cut, frametimes): nfct = np.sqrt(2.0 / len_tim) for k in range(1, order): - cdrift[:, k - 1] = nfct * np.cos( - (np.pi / len_tim) * (n_times + .5) * k) + cdrift[:, k - 1] = nfct * np.cos((np.pi / len_tim) * (n_times + 0.5) * k) - cdrift[:, order - 1] = 1. # or 1./sqrt(len_tim) to normalize + cdrift[:, order - 1] = 1.0 # or 1./sqrt(len_tim) to normalize return cdrift + def _full_rank(X, cmax=1e15): """ This function possibly adds a scalar matrix to X to guarantee that the condition number is smaller than a given threshold. - Parameters + + Parameters. ---------- X : array of shape(nrows, ncols) cmax=1.e-15, float tolerance for condition number + Returns ------- X : array of shape(nrows, ncols) after regularization @@ -164,7 +179,7 @@ def _full_rank(X, cmax=1e15): c = smax / smin if c < cmax: return X, c - iflogger.warning('Matrix is singular at working precision, regularizing...') + iflogger.warning("Matrix is singular at working precision, regularizing...") lda = (smax - cmax * smin) / (cmax - 1) s = s + lda X = np.dot(U, np.dot(np.diag(s), V)) @@ -177,7 +192,9 @@ def fallback_svd(a, full_matrices=True, compute_uv=True): except np.linalg.LinAlgError: pass - return svd(a, full_matrices=full_matrices, compute_uv=compute_uv, lapack_driver='gesvd') + return svd( + a, full_matrices=full_matrices, compute_uv=compute_uv, lapack_driver="gesvd" + ) def TR_string_to_float(tr): @@ -195,14 +212,14 @@ def TR_string_to_float(tr): tr in seconds (float) """ if not isinstance(tr, str): - raise TypeError(f'Improper type for TR_string_to_float ({tr}).') + raise TypeError(f"Improper type for TR_string_to_float ({tr}).") - tr_str = tr.replace(' ', '') + tr_str = tr.replace(" ", "") try: - if tr_str.endswith('ms'): + if tr_str.endswith("ms"): tr_numeric = float(tr_str[:-2]) * 0.001 - elif tr.endswith('s'): + elif tr.endswith("s"): tr_numeric = float(tr_str[:-1]) else: tr_numeric = float(tr_str) diff --git a/CPAC/nuisance/utils/convolve.py b/CPAC/nuisance/utils/convolve.py index 9b30bd6e4d..a89b8ea130 100644 --- a/CPAC/nuisance/utils/convolve.py +++ b/CPAC/nuisance/utils/convolve.py @@ -1,17 +1,19 @@ +import nibabel as nib import scipy.signal -import nibabel as nb -def convolve(functional_file_path, function_file_paths): - func_img = nb.load(functional_file_path) +def convolve(functional_file_path, function_file_paths): + func_img = nib.load(functional_file_path) func_data = func_img.get_fdata() files = [] for i, function_file_path in enumerate(function_file_paths): - function = nb.load(function_file_path).get_fdata() - convolved = scipy.signal.fftconvolve(func_data, function, axes=[-1], mode='same') - function_img = nb.Nifti1Image(convolved, affine=func_img.affine) - function_img.to_filename('./convolved_%d.nii.gz' % i) - files += ['./convolved_%d.nii.gz' % i] - - return files \ No newline at end of file + function = nib.load(function_file_path).get_fdata() + convolved = scipy.signal.fftconvolve( + func_data, function, axes=[-1], mode="same" + ) + function_img = nib.Nifti1Image(convolved, affine=func_img.affine) + function_img.to_filename("./convolved_%d.nii.gz" % i) + files += ["./convolved_%d.nii.gz" % i] + + return files diff --git a/CPAC/nuisance/utils/crc.py b/CPAC/nuisance/utils/crc.py index a19e351439..43b3939023 100644 --- a/CPAC/nuisance/utils/crc.py +++ b/CPAC/nuisance/utils/crc.py @@ -15,17 +15,16 @@ def _initial(c): c = c << 1 return crc + _tab = [_initial(i) for i in range(256)] def _update_crc(crc, c): - cc = 0xff & c + cc = 0xFF & c tmp = (crc >> 8) ^ cc - crc = (crc << 8) ^ _tab[tmp & 0xff] - crc = crc & 0xffff - - return crc + crc = (crc << 8) ^ _tab[tmp & 0xFF] + return crc & 0xFFFF def crc(str): @@ -37,6 +36,6 @@ def crc(str): def encode(string): scrc = str(crc(string)) - bcrc = scrc.encode('ascii') - base64crc = base64.urlsafe_b64encode(bcrc).strip(b'=') + bcrc = scrc.encode("ascii") + base64crc = base64.urlsafe_b64encode(bcrc).strip(b"=") return base64crc.decode() diff --git a/CPAC/pipeline/__init__.py b/CPAC/pipeline/__init__.py index a1c596e002..3f2986c7d1 100644 --- a/CPAC/pipeline/__init__.py +++ b/CPAC/pipeline/__init__.py @@ -1,4 +1,4 @@ -"""The C-PAC pipeline and its underlying infrastructure +"""The C-PAC pipeline and its underlying infrastructure. Copyright (C) 2022 C-PAC Developers @@ -15,8 +15,10 @@ License for more details. You should have received a copy of the GNU Lesser General Public -License along with C-PAC. If not, see .""" +License along with C-PAC. If not, see . +""" import os + import pkg_resources as p from CPAC.pipeline.nipype_pipeline_engine.monkeypatch import patch_base_interface @@ -24,13 +26,19 @@ patch_base_interface() # Monkeypatch Nipypes BaseInterface class ALL_PIPELINE_CONFIGS = os.listdir( - p.resource_filename("CPAC", os.path.join("resources", "configs"))) -ALL_PIPELINE_CONFIGS = [x.split('_')[2].replace('.yml', '') for - x in ALL_PIPELINE_CONFIGS if 'pipeline_config' in x] + p.resource_filename("CPAC", os.path.join("resources", "configs")) +) +ALL_PIPELINE_CONFIGS = [ + x.split("_")[2].replace(".yml", "") + for x in ALL_PIPELINE_CONFIGS + if "pipeline_config" in x +] ALL_PIPELINE_CONFIGS.sort() -AVAILABLE_PIPELINE_CONFIGS = [preconfig for preconfig in ALL_PIPELINE_CONFIGS - if preconfig not in - ['benchmark-ANTS', 'monkey-ABCD'] and - not preconfig.startswith('regtest-')] - -__all__ = ['ALL_PIPELINE_CONFIGS', 'AVAILABLE_PIPELINE_CONFIGS'] +AVAILABLE_PIPELINE_CONFIGS = [ + preconfig + for preconfig in ALL_PIPELINE_CONFIGS + if preconfig not in ["benchmark-ANTS", "monkey-ABCD"] + and not preconfig.startswith("regtest-") +] + +__all__ = ["ALL_PIPELINE_CONFIGS", "AVAILABLE_PIPELINE_CONFIGS"] diff --git a/CPAC/pipeline/check_outputs.py b/CPAC/pipeline/check_outputs.py index be46459799..f7d47c1424 100644 --- a/CPAC/pipeline/check_outputs.py +++ b/CPAC/pipeline/check_outputs.py @@ -14,22 +14,22 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -"""Test to check if all expected outputs were generated""" +"""Test to check if all expected outputs were generated.""" from itertools import chain from logging import Logger import os from pathlib import Path import re + import yaml + from CPAC.utils.bids_utils import with_key, without_key from CPAC.utils.datasource import bidsier_prefix -from CPAC.utils.monitoring.custom_logging import getLogger, MockLogger, \ - set_up_logger +from CPAC.utils.monitoring.custom_logging import MockLogger, getLogger, set_up_logger -def check_outputs(output_dir: str, log_dir: str, pipe_name: str, - unique_id: str) -> str: - """Check if all expected outputs were generated +def check_outputs(output_dir: str, log_dir: str, pipe_name: str, unique_id: str) -> str: + """Check if all expected outputs were generated. Parameters ---------- @@ -46,68 +46,90 @@ def check_outputs(output_dir: str, log_dir: str, pipe_name: str, message : str """ output_dir = Path(output_dir) - outputs_logger = getLogger(f'{unique_id}_expectedOutputs') + outputs_logger = getLogger(f"{unique_id}_expectedOutputs") missing_outputs = ExpectedOutputs() - subject, session = unique_id.split('_', 1) + subject, session = unique_id.split("_", 1) # allow any combination of keyed/unkeyed subject and session directories - containers = [os.path.join(f'pipeline_{pipe_name}', - '/'.join((sub, ses))) for sub in [ - fxn(subject, 'sub') for fxn in (with_key, without_key) - ] for ses in [fxn(session, 'ses') for fxn in - (with_key, without_key)]] - if (isinstance(outputs_logger, (Logger, MockLogger)) and - len(outputs_logger.handlers)): - outputs_log = getattr(outputs_logger.handlers[0], 'baseFilename', None) + containers = [ + os.path.join(f"pipeline_{pipe_name}", "/".join((sub, ses))) + for sub in [fxn(subject, "sub") for fxn in (with_key, without_key)] + for ses in [fxn(session, "ses") for fxn in (with_key, without_key)] + ] + if isinstance(outputs_logger, (Logger, MockLogger)) and len( + outputs_logger.handlers + ): + outputs_log = getattr(outputs_logger.handlers[0], "baseFilename", None) else: outputs_log = None if outputs_log is None: - message = 'Could not find expected outputs log file' + message = "Could not find expected outputs log file" else: - with open(outputs_log, 'r', encoding='utf-8') as expected_outputs_file: + with open(outputs_log, "r", encoding="utf-8") as expected_outputs_file: expected_outputs = yaml.safe_load(expected_outputs_file.read()) for subdir, filenames in expected_outputs.items(): - observed_outputs = list(chain.from_iterable([ - output_dir.glob(f'{container}/{subdir}') for - container in containers])) + observed_outputs = list( + chain.from_iterable( + [ + output_dir.glob(f"{container}/{subdir}") + for container in containers + ] + ) + ) for filename in filenames: try: - if not (observed_outputs and list( - observed_outputs[0].glob( - re.sub(r'\*\**', r'*', f'*{filename}*')))): + if not ( + observed_outputs + and list( + observed_outputs[0].glob( + re.sub(r"\*\**", r"*", f"*{filename}*") + ) + ) + ): missing_outputs += (subdir, filename) except Exception as exception: # pylint: disable=broad-except - logger = getLogger('nipype.workflow') + logger = getLogger("nipype.workflow") logger.error(str(exception)) if missing_outputs: - missing_log = set_up_logger(f'missingOutputs_{unique_id}', - filename='_'.join([ - bidsier_prefix(unique_id), - 'missingOutputs.yml']), - level='info', log_dir=log_dir, - mock=True) + missing_log = set_up_logger( + f"missingOutputs_{unique_id}", + filename="_".join([bidsier_prefix(unique_id), "missingOutputs.yml"]), + level="info", + log_dir=log_dir, + mock=True, + ) missing_log.info(missing_outputs) try: - log_note = 'Missing outputs have been logged in ' \ - f'{missing_log.handlers[0].baseFilename}' + log_note = ( + "Missing outputs have been logged in " + f"{missing_log.handlers[0].baseFilename}" + ) except (AttributeError, IndexError): - log_note = '' - message = '\n'.join([string for string in [ - 'Missing expected outputs:', str(missing_outputs), log_note, - '┌──────────────────────────────────────────────────────┐\n' - '│Tip: Look for "crash-*.txt" files in the log directory│\n' - '│for more information. Usually the chronological first │\n' - '│crash file is the most informative. │\n' - '└──────────────────────────────────────────────────────┘' - ] if string]) + log_note = "" + message = "\n".join( + [ + string + for string in [ + "Missing expected outputs:", + str(missing_outputs), + log_note, + "┌──────────────────────────────────────────────────────┐\n" + '│Tip: Look for "crash-*.txt" files in the log directory│\n' + "│for more information. Usually the chronological first │\n" + "│crash file is the most informative. │\n" + "└──────────────────────────────────────────────────────┘", + ] + if string + ] + ) missing_log.delete() else: - message = 'All expected outputs were generated' + message = "All expected outputs were generated" outputs_logger.delete() return message class ExpectedOutputs: - r'''Class to hold expected outputs for a pipeline + r"""Class to hold expected outputs for a pipeline. Attributes ---------- @@ -142,7 +164,8 @@ class ExpectedOutputs: - task-rest*_bold.nii.gz* >>> len(expected_outputs) 4 - ''' # noqa: E501 # pylint: disable=line-too-long + """ # pylint: disable=line-too-long + def __init__(self, expected=None): self.expected_outputs = {} if expected is None else expected if not isinstance(self.expected_outputs, dict): @@ -152,20 +175,28 @@ def __bool__(self): return bool(len(self)) def __iter__(self): - yield from {subdir: sorted(list(filename)) for - subdir, filename in self.expected_outputs.items()}.items() + yield from { + subdir: sorted(filename) + for subdir, filename in self.expected_outputs.items() + }.items() def __iadd__(self, other): if not isinstance(other, tuple) or len(other) != 2: raise TypeError( - f'{self.__module__}.{self.__class__.__name__} requires a ' - "tuple of ('subdir', 'output') for addition") + f"{self.__module__}.{self.__class__.__name__} requires a " + "tuple of ('subdir', 'output') for addition" + ) self.add(*other) return self def __len__(self): - return len([filepath for subdir, filepaths in - self.expected_outputs.items() for filepath in filepaths]) + return len( + [ + filepath + for subdir, filepaths in self.expected_outputs.items() + for filepath in filepaths + ] + ) def __repr__(self): return str(self).rstrip() @@ -174,7 +205,7 @@ def __str__(self): return yaml.dump(dict(self)) def add(self, subdir, output): - '''Add an expected output to the expected outputs dictionary + """Add an expected output to the expected outputs dictionary. Parameters ---------- @@ -183,17 +214,17 @@ def add(self, subdir, output): output : str filename of expected output - ''' + """ # add wildcard to the end of each BIDS entity before the last # also add wildcard before dashes after the first in an entity # TODO: revisit once we only have one dash per BIDS entity new_output = [] - for entity in output.split('_'): - if entity.count('-') > 1: - key, value = entity.split('-', 1) - entity = '-'.join([key, value.replace('-', '*-')]) + for entity in output.split("_"): + if entity.count("-") > 1: + key, value = entity.split("-", 1) + entity = "-".join([key, value.replace("-", "*-")]) new_output.append(entity) - output = f"{'*_'.join(new_output)}*".replace('**', '*') + output = f"{'*_'.join(new_output)}*".replace("**", "*") del new_output if subdir in self.expected_outputs: self.expected_outputs[subdir].add(output) diff --git a/CPAC/pipeline/cpac_basc_pipeline.py b/CPAC/pipeline/cpac_basc_pipeline.py index e82b571862..3a14b09168 100644 --- a/CPAC/pipeline/cpac_basc_pipeline.py +++ b/CPAC/pipeline/cpac_basc_pipeline.py @@ -14,71 +14,62 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -import re import os -import sys -import glob -import nipype.interfaces.utility as util + import nipype.interfaces.io as nio + from CPAC.pipeline import nipype_pipeline_engine as pe from CPAC.pipeline.nipype_pipeline_engine.plugins import MultiProcPlugin from CPAC.utils import Configuration def prep_basc_workflow(c, subject_infos): - print('Preparing BASC workflow') p_id, s_ids, scan_ids, s_paths = (list(tup) for tup in zip(*subject_infos)) - print('Subjects', s_ids) - - wf = pe.Workflow(name='basc_workflow') - wf.base_dir = c.pipeline_setup['working_directory']['path'] - + + wf = pe.Workflow(name="basc_workflow") + wf.base_dir = c.pipeline_setup["working_directory"]["path"] + from CPAC.basc import create_basc - + b = create_basc() b.inputs.inputspec.roi = c.bascROIFile b.inputs.inputspec.subjects = s_paths b.inputs.inputspec.k_clusters = c.bascClusters b.inputs.inputspec.dataset_bootstraps = c.bascDatasetBootstraps b.inputs.inputspec.timeseries_bootstraps = c.bascTimeseriesBootstraps - - aff_list = open(c.bascAffinityThresholdFile, 'r').readlines() - aff_list = [ float(aff.rstrip('\r\n')) for aff in aff_list] - + + aff_list = open(c.bascAffinityThresholdFile, "r").readlines() + aff_list = [float(aff.rstrip("\r\n")) for aff in aff_list] + b.inputs.inputspec.affinity_threshold = aff_list - - ds = pe.Node(nio.DataSink(), name='basc_sink') - out_dir = os.path.dirname(s_paths[0]).replace(s_ids[0], 'basc_results') + + ds = pe.Node(nio.DataSink(), name="basc_sink") + out_dir = os.path.dirname(s_paths[0]).replace(s_ids[0], "basc_results") ds.inputs.base_directory = out_dir - ds.inputs.container = '' - -# wf.connect(b, 'outputspec.gsm', -# ds, 'gsm') -# wf.connect(b, 'outputspec.gsclusters', -# ds, 'gsclusters') -# wf.connect(b, 'outputspec.gsmap', -# ds, 'gsmap') - wf.connect(b, 'outputspec.gsclusters_img', - ds, 'gsclusters_img') - wf.connect(b, 'outputspec.ismap_imgs', - ds, 'ismap_imgs') - - plugin_args = {'n_procs': c.numCoresPerSubject} - wf.run(plugin=MultiProcPlugin(plugin_args), - plugin_args=plugin_args) + ds.inputs.container = "" + + # wf.connect(b, 'outputspec.gsm', + # ds, 'gsm') + # wf.connect(b, 'outputspec.gsclusters', + # ds, 'gsclusters') + # wf.connect(b, 'outputspec.gsmap', + # ds, 'gsmap') + wf.connect(b, "outputspec.gsclusters_img", ds, "gsclusters_img") + wf.connect(b, "outputspec.ismap_imgs", ds, "ismap_imgs") + + plugin_args = {"n_procs": c.numCoresPerSubject} + wf.run(plugin=MultiProcPlugin(plugin_args), plugin_args=plugin_args) def run(config, subject_infos): - import re import subprocess - subprocess.getoutput('source ~/.bashrc') + + subprocess.getoutput("source ~/.bashrc") import os - import sys import pickle - import yaml - - c = Configuration(yaml.safe_load(open(os.path.realpath(config), 'r'))) + import yaml - prep_basc_workflow(c, pickle.load(open(subject_infos, 'r') )) + c = Configuration(yaml.safe_load(open(os.path.realpath(config), "r"))) + prep_basc_workflow(c, pickle.load(open(subject_infos, "r"))) diff --git a/CPAC/pipeline/cpac_cwas_pipeline.py b/CPAC/pipeline/cpac_cwas_pipeline.py index d628f17e05..368e53576e 100644 --- a/CPAC/pipeline/cpac_cwas_pipeline.py +++ b/CPAC/pipeline/cpac_cwas_pipeline.py @@ -17,54 +17,54 @@ import os import nipype.interfaces.io as nio + from CPAC.pipeline import nipype_pipeline_engine as pe from CPAC.pipeline.nipype_pipeline_engine.plugins import MultiProcPlugin from CPAC.utils.configuration import Configuration def prep_cwas_workflow(c, subject_infos): - print('Preparing CWAS workflow') p_id, s_ids, scan_ids, s_paths = (list(tup) for tup in zip(*subject_infos)) - print('Subjects', s_ids) - wf = pe.Workflow(name='cwas_workflow') - wf.base_dir = c.pipeline_setup['working_directory']['path'] + wf = pe.Workflow(name="cwas_workflow") + wf.base_dir = c.pipeline_setup["working_directory"]["path"] - from CPAC.cwas import create_cwas import numpy as np + + from CPAC.cwas import create_cwas + regressor = np.loadtxt(c.cwasRegressorFile) cw = create_cwas() - cw.inputs.inputspec.roi = c.cwasROIFile - cw.inputs.inputspec.subjects = s_paths - cw.inputs.inputspec.regressor = regressor - cw.inputs.inputspec.cols = c.cwasRegressorCols - cw.inputs.inputspec.f_samples = c.cwasFSamples - cw.inputs.inputspec.strata = c.cwasRegressorStrata # will stay None? + cw.inputs.inputspec.roi = c.cwasROIFile + cw.inputs.inputspec.subjects = s_paths + cw.inputs.inputspec.regressor = regressor + cw.inputs.inputspec.cols = c.cwasRegressorCols + cw.inputs.inputspec.f_samples = c.cwasFSamples + cw.inputs.inputspec.strata = c.cwasRegressorStrata # will stay None? cw.inputs.inputspec.parallel_nodes = c.cwasParallelNodes - ds = pe.Node(nio.DataSink(), name='cwas_sink') - out_dir = os.path.dirname(s_paths[0]).replace(s_ids[0], 'cwas_results') + ds = pe.Node(nio.DataSink(), name="cwas_sink") + out_dir = os.path.dirname(s_paths[0]).replace(s_ids[0], "cwas_results") ds.inputs.base_directory = out_dir - ds.inputs.container = '' + ds.inputs.container = "" - wf.connect(cw, 'outputspec.F_map', - ds, 'F_map') - wf.connect(cw, 'outputspec.p_map', - ds, 'p_map') + wf.connect(cw, "outputspec.F_map", ds, "F_map") + wf.connect(cw, "outputspec.p_map", ds, "p_map") - plugin_args = {'n_procs': c.numCoresPerSubject} - wf.run(plugin=MultiProcPlugin(plugin_args), - plugin_args=plugin_args) + plugin_args = {"n_procs": c.numCoresPerSubject} + wf.run(plugin=MultiProcPlugin(plugin_args), plugin_args=plugin_args) def run(config, subject_infos): import subprocess - subprocess.getoutput('source ~/.bashrc') + + subprocess.getoutput("source ~/.bashrc") import os import pickle + import yaml - c = Configuration(yaml.safe_load(open(os.path.realpath(config), 'r'))) + c = Configuration(yaml.safe_load(open(os.path.realpath(config), "r"))) - prep_cwas_workflow(c, pickle.load(open(subject_infos, 'r') )) + prep_cwas_workflow(c, pickle.load(open(subject_infos, "r"))) diff --git a/CPAC/pipeline/cpac_ga_model_generator.py b/CPAC/pipeline/cpac_ga_model_generator.py index 3d2803c449..fb9ace888f 100755 --- a/CPAC/pipeline/cpac_ga_model_generator.py +++ b/CPAC/pipeline/cpac_ga_model_generator.py @@ -1,4 +1,4 @@ -"""Copyright (C) 2022 C-PAC Developers +"""Copyright (C) 2022 C-PAC Developers. This file is part of C-PAC. @@ -13,87 +13,86 @@ License for more details. You should have received a copy of the GNU Lesser General Public -License along with C-PAC. If not, see .""" -import re +License along with C-PAC. If not, see . +""" import os -import sys -import glob - -from CPAC.utils.datasource import create_grp_analysis_dataflow -from CPAC.utils import Configuration +import re def write_new_sub_file(current_mod_path, subject_list, new_participant_list): # write the new participant list - new_sub_file = os.path.join(current_mod_path,os.path.basename(subject_list)) + new_sub_file = os.path.join(current_mod_path, os.path.basename(subject_list)) try: with open(new_sub_file, "w") as f: for part_ID in new_participant_list: print(part_ID, file=f) except Exception as e: - err = "\n\n[!] CPAC says: Could not write new participant list for " \ - "current model and derivative in group-level analysis. Ensure "\ - "you have write access to the directory:\n%s\n\nError " \ - "details: %s\n\n" % (current_mod_path, e) + err = ( + "\n\n[!] CPAC says: Could not write new participant list for " + "current model and derivative in group-level analysis. Ensure " + "you have write access to the directory:\n%s\n\nError " + "details: %s\n\n" % (current_mod_path, e) + ) raise Exception(err) return new_sub_file def create_dir(dir_path, description): - if not os.path.isdir(dir_path): try: os.makedirs(dir_path) except Exception as e: - err = "\n\n[!] Could not create the %s directory.\n\n" \ - "Attempted directory creation: %s\n\n" \ - "Error details: %s\n\n" % (description, dir_path, e) + err = ( + "\n\n[!] Could not create the %s directory.\n\n" + "Attempted directory creation: %s\n\n" + "Error details: %s\n\n" % (description, dir_path, e) + ) raise Exception(err) def create_merged_copefile(list_of_output_files, merged_outfile): - import subprocess merge_string = ["fslmerge", "-t", merged_outfile] merge_string = merge_string + list_of_output_files try: - retcode = subprocess.check_output(merge_string) + subprocess.check_output(merge_string) except Exception as e: - err = "\n\n[!] Something went wrong with FSL's fslmerge during the " \ - "creation of the 4D merged file for group analysis.\n\n" \ - "Attempted to create file: %s\n\nLength of list of files to " \ - "merge: %d\n\nError details: %s\n\n" \ - % (merged_outfile, len(list_of_output_files), e) + err = ( + "\n\n[!] Something went wrong with FSL's fslmerge during the " + "creation of the 4D merged file for group analysis.\n\n" + "Attempted to create file: %s\n\nLength of list of files to " + "merge: %d\n\nError details: %s\n\n" + % (merged_outfile, len(list_of_output_files), e) + ) raise Exception(err) return merged_outfile def create_merge_mask(merged_file, mask_outfile): - import subprocess - mask_string = ["fslmaths", merged_file, "-abs", "-Tmin", "-bin", - mask_outfile] + mask_string = ["fslmaths", merged_file, "-abs", "-Tmin", "-bin", mask_outfile] try: - retcode = subprocess.check_output(mask_string) + subprocess.check_output(mask_string) except Exception as e: - err = "\n\n[!] Something went wrong with FSL's fslmaths during the " \ - "creation of the merged copefile group mask.\n\nAttempted to " \ - "create file: %s\n\nMerged file: %s\n\nError details: %s\n\n" \ - % (mask_outfile, merged_file, e) + err = ( + "\n\n[!] Something went wrong with FSL's fslmaths during the " + "creation of the merged copefile group mask.\n\nAttempted to " + "create file: %s\n\nMerged file: %s\n\nError details: %s\n\n" + % (mask_outfile, merged_file, e) + ) raise Exception(err) return mask_outfile def check_merged_file(list_of_output_files, merged_outfile): - import subprocess # make sure the order is correct @@ -101,61 +100,70 @@ def check_merged_file(list_of_output_files, merged_outfile): # with the output file it should correspond to i = 0 for output_file in list_of_output_files: - test_string = ["3ddot", "-demean", - "{0}[{1}]".format(merged_outfile, str(i)), output_file] + test_string = [ + "3ddot", + "-demean", + "{0}[{1}]".format(merged_outfile, str(i)), + output_file, + ] try: retcode = subprocess.check_output(test_string) except Exception as e: - err = "\n\n[!] Something went wrong while trying to run AFNI's " \ - "3ddot for the purpose of testing the merge file output." \ - "\n\nError details: %s\n\n" % e + err = ( + "\n\n[!] Something went wrong while trying to run AFNI's " + "3ddot for the purpose of testing the merge file output." + "\n\nError details: %s\n\n" % e + ) raise Exception(err) retcode = retcode.rstrip("\n").rstrip("\t") if retcode != "1": - err = "\n\n[!] The volumes of the merged file do not correspond "\ - "to the correct order of output files as described in the "\ - "phenotype matrix. If you are seeing this error, " \ - "something possibly went wrong with FSL's fslmerge.\n\n" \ - "Merged file: %s\n\nMismatch between merged file volume " \ - "%d and derivative file %s\n\nEach volume should " \ - "correspond to the derivative output file for each " \ - "participant in the model.\n\n" \ - % (merged_outfile, i, output_file) + err = ( + "\n\n[!] The volumes of the merged file do not correspond " + "to the correct order of output files as described in the " + "phenotype matrix. If you are seeing this error, " + "something possibly went wrong with FSL's fslmerge.\n\n" + "Merged file: %s\n\nMismatch between merged file volume " + "%d and derivative file %s\n\nEach volume should " + "correspond to the derivative output file for each " + "participant in the model.\n\n" % (merged_outfile, i, output_file) + ) raise Exception(err) i += 1 def calculate_measure_mean_in_df(model_df, merge_mask): - import subprocess + import pandas as pd mm_dict_list = [] for raw_file in model_df["Raw_Filepath"]: - mask_string = ["3dmaskave", "-mask", merge_mask, raw_file] # calculate try: retcode = subprocess.check_output(mask_string) except Exception as e: - err = "\n\n[!] AFNI's 3dMaskAve failed for raw output: %s\n" \ - "Error details: %s\n\n" % (raw_file, e) + err = ( + "\n\n[!] AFNI's 3dMaskAve failed for raw output: %s\n" + "Error details: %s\n\n" % (raw_file, e) + ) raise Exception(err) # if this breaks, 3dmaskave output to STDOUT has changed try: mean = retcode.split(" ")[0] except Exception as e: - err = "\n\n[!] Something went wrong with parsing the output of " \ - "AFNI's 3dMaskAve - this is used in calculating the " \ - "Measure Mean in group analysis.\n\nError details: %s\n\n" \ - % e + err = ( + "\n\n[!] Something went wrong with parsing the output of " + "AFNI's 3dMaskAve - this is used in calculating the " + "Measure Mean in group analysis.\n\nError details: %s\n\n" % e + ) raise Exception(err) mm_dict = {} @@ -167,51 +175,56 @@ def calculate_measure_mean_in_df(model_df, merge_mask): # demean! mm_df["Measure_Mean"] = mm_df["Measure_Mean"].astype(float) - mm_df["Measure_Mean"] = \ - mm_df["Measure_Mean"].sub(mm_df["Measure_Mean"].mean()) - - model_df = pd.merge(model_df, mm_df, how="inner", on=["Raw_Filepath"]) - - return model_df + mm_df["Measure_Mean"] = mm_df["Measure_Mean"].sub(mm_df["Measure_Mean"].mean()) + return pd.merge(model_df, mm_df, how="inner", on=["Raw_Filepath"]) -def check_mask_file_resolution(data_file, roi_mask, group_mask, out_dir, \ - output_id=None): +def check_mask_file_resolution( + data_file, roi_mask, group_mask, out_dir, output_id=None +): import os import subprocess - import nibabel as nb + + import nibabel as nib # let's check if we need to resample the custom ROI mask - raw_file_img = nb.load(data_file) + raw_file_img = nib.load(data_file) raw_file_hdr = raw_file_img.header - roi_mask_img = nb.load(roi_mask) + roi_mask_img = nib.load(roi_mask) roi_mask_hdr = roi_mask_img.header raw_file_dims = raw_file_hdr.get_zooms() roi_mask_dims = roi_mask_hdr.get_zooms() if raw_file_dims != roi_mask_dims: - print("\n\nWARNING: The custom ROI mask file is a different " \ - "resolution than the output data! Resampling the ROI mask " \ - "file to match the original output data!\n\nCustom ROI mask " \ - "file: %s\n\nOutput measure: %s\n\n" % (roi_mask, output_id)) - - resampled_outfile = os.path.join(out_dir, \ - "resampled_%s" \ - % os.path.basename(roi_mask)) - - resample_str = ["flirt", "-in", roi_mask, "-ref", group_mask, \ - "-applyisoxfm", str(raw_file_dims[0]), "-interp", \ - "nearestneighbour", "-out", resampled_outfile] + resampled_outfile = os.path.join( + out_dir, "resampled_%s" % os.path.basename(roi_mask) + ) + + resample_str = [ + "flirt", + "-in", + roi_mask, + "-ref", + group_mask, + "-applyisoxfm", + str(raw_file_dims[0]), + "-interp", + "nearestneighbour", + "-out", + resampled_outfile, + ] try: - retcode = subprocess.check_output(resample_str) + subprocess.check_output(resample_str) except Exception as e: - err = "\n\n[!] Something went wrong with running FSL FLIRT for " \ - "the purpose of resampling the custom ROI mask to match " \ - "the original output file's resolution.\n\nCustom ROI " \ - "mask file: %s\n\nError details: %s\n\n" % (roi_mask, e) + err = ( + "\n\n[!] Something went wrong with running FSL FLIRT for " + "the purpose of resampling the custom ROI mask to match " + "the original output file's resolution.\n\nCustom ROI " + "mask file: %s\n\nError details: %s\n\n" % (roi_mask, e) + ) raise Exception(err) roi_mask = resampled_outfile @@ -220,46 +233,45 @@ def check_mask_file_resolution(data_file, roi_mask, group_mask, out_dir, \ def trim_mask(input_mask, ref_mask, output_mask_path): - - import os import subprocess # mask the mask mask_string = ["fslmaths", input_mask, "-mul", ref_mask, output_mask_path] try: - retcode = subprocess.check_output(mask_string) + subprocess.check_output(mask_string) except Exception as e: - err = "\n\n[!] Something went wrong with running FSL's fslmaths for "\ - "the purpose of trimming the custom ROI masks to fit within " \ - "the merged group mask.\n\nCustom ROI mask file: %s\n\nMerged "\ - "group mask file: %s\n\nError details: %s\n\n" \ - % (input_mask, ref_mask, e) + err = ( + "\n\n[!] Something went wrong with running FSL's fslmaths for " + "the purpose of trimming the custom ROI masks to fit within " + "the merged group mask.\n\nCustom ROI mask file: %s\n\nMerged " + "group mask file: %s\n\nError details: %s\n\n" % (input_mask, ref_mask, e) + ) raise Exception(err) return output_mask_path def calculate_custom_roi_mean_in_df(model_df, roi_mask): - - import os import subprocess + import pandas as pd # calculate the ROI means roi_dict_list = [] for raw_file in model_df["Raw_Filepath"]: - roi_string = ["3dROIstats", "-mask", roi_mask, raw_file] try: retcode = subprocess.check_output(roi_string) except Exception as e: - err = "\n\n[!] Something went wrong with running AFNI's " \ - "3dROIstats while calculating the custom ROI means.\n\n" \ - "Custom ROI mask file: %s\n\nRaw output filepath: %s\n\n" \ - "Error details: %s\n\n" % (roi_mask, raw_file, e) + err = ( + "\n\n[!] Something went wrong with running AFNI's " + "3dROIstats while calculating the custom ROI means.\n\n" + "Custom ROI mask file: %s\n\nRaw output filepath: %s\n\n" + "Error details: %s\n\n" % (roi_mask, raw_file, e) + ) raise Exception(err) # process the output string @@ -278,11 +290,13 @@ def calculate_custom_roi_mean_in_df(model_df, roi_mask): roi_means_list.append(roi_mean) if len(roi_means_list) != retcode.count("Mean_"): - err = "\n\n[!] Something went wrong with parsing the output "\ - "of AFNI's 3dROIstats during the calculation of the " \ - "custom ROI means.\n\nOutput file: %s\n\nCustom ROI " \ - "mask file: %s\n\n3dROIstats output: %s\n\n" \ - % (raw_file, roi_mask, retcode) + err = ( + "\n\n[!] Something went wrong with parsing the output " + "of AFNI's 3dROIstats during the calculation of the " + "custom ROI means.\n\nOutput file: %s\n\nCustom ROI " + "mask file: %s\n\n3dROIstats output: %s\n\n" + % (raw_file, roi_mask, retcode) + ) raise Exception(err) # add in the custom ROI means! @@ -297,7 +311,6 @@ def calculate_custom_roi_mean_in_df(model_df, roi_mask): roi_dict_list.append(roi_dict) - roi_df = pd.DataFrame(roi_dict_list) # demean! @@ -308,13 +321,10 @@ def calculate_custom_roi_mean_in_df(model_df, roi_mask): roi_df[roi_label] = roi_df[roi_label].astype(float) roi_df[roi_label] = roi_df[roi_label].sub(roi_df[roi_label].mean()) - model_df = pd.merge(model_df, roi_df, how="inner", on=["Raw_Filepath"]) - - return model_df + return pd.merge(model_df, roi_df, how="inner", on=["Raw_Filepath"]) def parse_out_covariates(design_formula): - patsy_ops = ["~", "+", "-", "*", "/", ":", "**", ")", "("] for op in patsy_ops: @@ -323,20 +333,19 @@ def parse_out_covariates(design_formula): words = design_formula.split(" ") - covariates = [x for x in words if x != ""] - - return covariates + return [x for x in words if x != ""] def split_groups(pheno_df, group_ev, ev_list, cat_list): - import pandas as pd new_ev_list = [] new_cat_list = [] if group_ev not in cat_list: - err = "\n\n[!] The grouping variable must be one of the categorical "\ - "covariates!\n\n" + err = ( + "\n\n[!] The grouping variable must be one of the categorical " + "covariates!\n\n" + ) raise Exception(err) # map for the .grp file for FLAME @@ -364,7 +373,6 @@ def split_groups(pheno_df, group_ev, ev_list, cat_list): level_df_list = [] for level in group_levels: - level_df = pheno_df[pheno_df[group_ev] == level] rename = {} @@ -379,7 +387,11 @@ def split_groups(pheno_df, group_ev, ev_list, cat_list): for other_lev in group_levels: if other_lev != level: for col in level_df.columns: - if (col != group_ev) and (col not in join_column) and (col in ev_list): + if ( + (col != group_ev) + and (col not in join_column) + and (col in ev_list) + ): newcol = col + "__FOR_%s_%s" % (group_ev, other_lev) level_df[newcol] = 0 if newcol not in new_ev_list: @@ -391,17 +403,16 @@ def split_groups(pheno_df, group_ev, ev_list, cat_list): level_df_list.append(level_df) # the grouping variable has to be in the categorical list too - #new_cat_list.append(group_ev) + # new_cat_list.append(group_ev) # get it back into order! pheno_df = pheno_df[join_column].merge(pd.concat(level_df_list), on=join_column) - pheno_df = pheno_df.drop(join_column,1) + pheno_df = pheno_df.drop(join_column, 1) return pheno_df, grp_vector, new_ev_list, new_cat_list def patsify_design_formula(formula, categorical_list, encoding="Treatment"): - closer = ")" if encoding == "Treatment": closer = ")" @@ -418,7 +429,7 @@ def patsify_design_formula(formula, categorical_list, encoding="Treatment"): # pad end with single space so the formula.replace below won't miss the last # covariate when relevant - formula = '{0} '.format(formula) + formula = "{0} ".format(formula) for ev in categorical_list: if ev in formula: @@ -442,106 +453,104 @@ def patsify_design_formula(formula, categorical_list, encoding="Treatment"): def check_multicollinearity(matrix): - import numpy as np - print("\nChecking for multicollinearity in the model..") - U, s, V = np.linalg.svd(matrix) max_singular = np.max(s) min_singular = np.min(s) - print("Max singular: ", max_singular) - print("Min singular: ", min_singular) - print("Rank: ", np.linalg.matrix_rank(matrix), "\n") - if min_singular == 0: - print('[!] CPAC warns: Detected multicollinearity in the ' \ - 'computed group-level analysis model. Please double-' \ - 'check your model design.\n\n') + pass else: - condition_number = float(max_singular)/float(min_singular) - print("Condition number: %f" % condition_number) + condition_number = float(max_singular) / float(min_singular) if condition_number > 30: - print('[!] CPAC warns: Detected multicollinearity in the ' \ - 'computed group-level analysis model. Please double-' \ - 'check your model design.\n\n') + pass else: - print('Looks good..\n') + pass def create_contrasts_dict(dmatrix_obj, contrasts_list, output_measure): - contrasts_vectors = [] for con_equation in contrasts_list: - try: lincon = dmatrix_obj.design_info.linear_constraint(str(con_equation)) except Exception as e: - err = "\n\n[!] Could not process contrast equation:\n%s\n\n" \ - "Design matrix EVs/covariates:\n%s\n\nError details:\n%s" \ - "\n\nNote: If the design matrix EVs are different than " \ - "what was shown in the model design creator, this may be " \ - "because missing participants, sessions, or series for " \ - "this measure (%s) may have altered the group design.\n\n" \ - % (str(con_equation), dmatrix_obj.design_info.column_names,\ - e, output_measure) + err = ( + "\n\n[!] Could not process contrast equation:\n%s\n\n" + "Design matrix EVs/covariates:\n%s\n\nError details:\n%s" + "\n\nNote: If the design matrix EVs are different than " + "what was shown in the model design creator, this may be " + "because missing participants, sessions, or series for " + "this measure (%s) may have altered the group design.\n\n" + % ( + str(con_equation), + dmatrix_obj.design_info.column_names, + e, + output_measure, + ) + ) raise Exception(err) con_vec = lincon.coefs[0] contrasts_vectors.append(con_vec) - return contrasts_vectors -def build_feat_model(model_df, model_name, group_config_file, resource_id, - preproc_strat, session_id, series_or_repeated_label): - +def build_feat_model( + model_df, + model_name, + group_config_file, + resource_id, + preproc_strat, + session_id, + series_or_repeated_label, +): # # this function runs once per derivative type and preproc strat combo # during group analysis # import os - import patsy - import pandas as pd + import numpy as np + import pandas as pd + import patsy - from CPAC.pipeline import nipype_pipeline_engine as pe - import nipype.interfaces.utility as util - import nipype.interfaces.io as nio from CPAC.pipeline.cpac_group_runner import load_config_yml - - from CPAC.utils.create_group_analysis_info_files import write_design_matrix_csv, \ - write_blank_contrast_csv + from CPAC.utils.create_group_analysis_info_files import ( + write_design_matrix_csv, + ) group_config_obj = load_config_yml(group_config_file) - pipeline_ID = group_config_obj.pipeline_dir.rstrip('/').split('/')[-1] - #sublist_txt = group_config_obj.participant_list + pipeline_ID = group_config_obj.pipeline_dir.rstrip("/").split("/")[-1] + # sublist_txt = group_config_obj.participant_list - #if sublist_txt == None: + # if sublist_txt == None: # print ("Warning! You have not provided a subject list. CPAC will use all the subjects in pipeline directory") # sublist_txt = group_config_obj.participant_list - #else: + # else: # sublist_txt = group_config_obj.particpant_list # remove file names from preproc_strat filename = preproc_strat.split("/")[-1] - preproc_strat = preproc_strat.replace('.nii', '').replace('.gz', '') + preproc_strat = preproc_strat.replace(".nii", "").replace(".gz", "") preproc_strat = preproc_strat.lstrip("/").rstrip("/") - ftest_list = [] readme_flags = [] # determine if f-tests are included or not custom_confile = group_config_obj.custom_contrasts - if ((custom_confile is None) or (custom_confile == '') or - ("None" in custom_confile) or ("none" in custom_confile)): + if ( + (custom_confile is None) + or (custom_confile == "") + or ("None" in custom_confile) + or ("none" in custom_confile) + ): custom_confile = None # if (len(group_config_obj.f_tests) == 0) or \ @@ -551,7 +560,7 @@ def build_feat_model(model_df, model_name, group_config_file, resource_id, # fTest = True # ftest_list = group_config_obj.f_tests - #else: + # else: # if not os.path.exists(custom_confile): # errmsg = "\n[!] CPAC says: You've specified a custom contrasts " \ # ".CSV file for your group model, but this file cannot " \ @@ -572,54 +581,72 @@ def build_feat_model(model_df, model_name, group_config_file, resource_id, # count_ftests += 1 # create path for output directory - model_dir = os.path.join(group_config_obj.output_dir, - 'cpac_group_analysis', - 'FSL_FEAT', - '{0}'.format(pipeline_ID), - 'group_model_{0}'.format(model_name)) - - out_dir = os.path.join(model_dir, - resource_id, - session_id, - series_or_repeated_label, - preproc_strat) + model_dir = os.path.join( + group_config_obj.output_dir, + "cpac_group_analysis", + "FSL_FEAT", + "{0}".format(pipeline_ID), + "group_model_{0}".format(model_name), + ) + + out_dir = os.path.join( + model_dir, resource_id, session_id, series_or_repeated_label, preproc_strat + ) try: - preset_contrast = group_config_obj.preset preset = True except AttributeError: preset = False - if 'sca_roi' in resource_id: - out_dir = os.path.join(out_dir, - re.search('sca_ROI_(\d)+', os.path.splitext(\ - os.path.splitext(os.path.basename(\ - model_df["Filepath"][0]))[0])[0]).group(0)) - - if 'dr_tempreg_maps_zstat_files_to_standard_smooth' in resource_id: - out_dir = os.path.join(out_dir, - re.search('temp_reg_map_z_(\d)+', os.path.splitext(\ - os.path.splitext(os.path.basename(\ - model_df["Filepath"][0]))[0])[0]).group(0)) - - if 'centrality' in resource_id: - names = ['degree_centrality_binarize', - 'degree_centrality_weighted', - 'eigenvector_centrality_binarize', - 'eigenvector_centrality_weighted', - 'lfcd_binarize', 'lfcd_weighted'] + if "sca_roi" in resource_id: + out_dir = os.path.join( + out_dir, + re.search( + r"sca_ROI_(\d)+", + os.path.splitext( + os.path.splitext(os.path.basename(model_df["Filepath"][0]))[0] + )[0], + ).group(0), + ) + + if "dr_tempreg_maps_zstat_files_to_standard_smooth" in resource_id: + out_dir = os.path.join( + out_dir, + re.search( + r"temp_reg_map_z_(\d)+", + os.path.splitext( + os.path.splitext(os.path.basename(model_df["Filepath"][0]))[0] + )[0], + ).group(0), + ) + + if "centrality" in resource_id: + names = [ + "degree_centrality_binarize", + "degree_centrality_weighted", + "eigenvector_centrality_binarize", + "eigenvector_centrality_weighted", + "lfcd_binarize", + "lfcd_weighted", + ] for name in names: if name in filename: out_dir = os.path.join(out_dir, name) break - if 'tempreg_maps' in resource_id: - out_dir = os.path.join(out_dir, re.search('\w*[#]*\d+', - os.path.splitext(os.path.splitext(os.path.basename(\ - model_df["Filepath"][0]))[0])[0]).group(0)) + if "tempreg_maps" in resource_id: + out_dir = os.path.join( + out_dir, + re.search( + r"\w*[#]*\d+", + os.path.splitext( + os.path.splitext(os.path.basename(model_df["Filepath"][0]))[0] + )[0], + ).group(0), + ) - model_path = os.path.join(out_dir, 'model_files') + model_path = os.path.join(out_dir, "model_files") # create the actual directories create_dir(model_path, "group analysis output") @@ -634,17 +661,17 @@ def build_feat_model(model_df, model_name, group_config_file, resource_id, if part not in new_participant_list: new_participant_list.append(part) - if group_config_obj.participant_list == None: - #participant_list = os.listdir(group_config_obj.pipeline_dir) - new_sub_file = write_new_sub_file(model_path, - group_config_obj.pipeline_dir, - new_participant_list) + if group_config_obj.participant_list is None: + # participant_list = os.listdir(group_config_obj.pipeline_dir) + new_sub_file = write_new_sub_file( + model_path, group_config_obj.pipeline_dir, new_participant_list + ) else: - new_sub_file = write_new_sub_file(model_path, - group_config_obj.participant_list, - new_participant_list) + new_sub_file = write_new_sub_file( + model_path, group_config_obj.participant_list, new_participant_list + ) - group_config_obj.update('participant_list', new_sub_file) + group_config_obj.update("participant_list", new_sub_file) num_subjects = len(list(model_df["participant_id"])) @@ -652,7 +679,7 @@ def build_feat_model(model_df, model_name, group_config_file, resource_id, design_formula = group_config_obj.design_formula # demean EVs set for demeaning - for demean_EV in group_config_obj.ev_selections.get("demean",[]): + for demean_EV in group_config_obj.ev_selections.get("demean", []): model_df[demean_EV] = model_df[demean_EV].astype(float) model_df[demean_EV] = model_df[demean_EV].sub(model_df[demean_EV].mean()) @@ -667,30 +694,26 @@ def build_feat_model(model_df, model_name, group_config_file, resource_id, # matrix merge_outfile = model_name + "_" + resource_id + "_merged.nii.gz" merge_outfile = os.path.join(model_path, merge_outfile) - merge_file = create_merged_copefile(model_df["Filepath"].tolist(), - merge_outfile) + merge_file = create_merged_copefile(model_df["Filepath"].tolist(), merge_outfile) # create merged group mask - merge_mask_outfile = '_'.join([model_name, resource_id, - "merged_mask.nii.gz"]) + merge_mask_outfile = "_".join([model_name, resource_id, "merged_mask.nii.gz"]) merge_mask_outfile = os.path.join(model_path, merge_mask_outfile) merge_mask = create_merge_mask(merge_file, merge_mask_outfile) if "Group Mask" in group_config_obj.mean_mask: mask_for_means = merge_mask else: - individual_masks_dir = os.path.join(model_path, - "individual_masks") + individual_masks_dir = os.path.join(model_path, "individual_masks") create_dir(individual_masks_dir, "individual masks") for unique_id, series_id, raw_filepath in zip( - model_df["participant_id"], - model_df["Series"], model_df["Raw_Filepath"]): - mask_for_means_path = os.path.join(individual_masks_dir, - "%s_%s_%s_mask.nii.gz" % ( - unique_id, series_id, - resource_id)) - mask_for_means = create_merge_mask(raw_filepath, - mask_for_means_path) + model_df["participant_id"], model_df["Series"], model_df["Raw_Filepath"] + ): + mask_for_means_path = os.path.join( + individual_masks_dir, + "%s_%s_%s_mask.nii.gz" % (unique_id, series_id, resource_id), + ) + mask_for_means = create_merge_mask(raw_filepath, mask_for_means_path) readme_flags.append("individual_masks") # calculate measure means, and demean @@ -699,26 +722,28 @@ def build_feat_model(model_df, model_name, group_config_file, resource_id, # calculate custom ROIs, and demean (in workflow?) if "Custom_ROI_Mean" in design_formula: - custom_roi_mask = group_config_obj.custom_roi_mask - if (custom_roi_mask == None) or (custom_roi_mask == "None") or \ - (custom_roi_mask == "none") or (custom_roi_mask == ""): - err = "\n\n[!] You included 'Custom_ROI_Mean' in your design " \ - "formula, but you didn't supply a custom ROI mask file." \ - "\n\nDesign formula: %s\n\n" % design_formula + if custom_roi_mask in (None, "None", "none", ""): + err = ( + "\n\n[!] You included 'Custom_ROI_Mean' in your design " + "formula, but you didn't supply a custom ROI mask file." + "\n\nDesign formula: %s\n\n" % design_formula + ) raise Exception(err) # make sure the custom ROI mask file is the same resolution as the # output files - if not, resample and warn the user - roi_mask = check_mask_file_resolution(list(model_df["Raw_Filepath"])[0], - custom_roi_mask, mask_for_means, - model_path, resource_id) - + roi_mask = check_mask_file_resolution( + next(iter(model_df["Raw_Filepath"])), + custom_roi_mask, + mask_for_means, + model_path, + resource_id, + ) # trim the custom ROI mask to be within mask constraints - output_mask = os.path.join(model_path, "masked_%s" \ - % os.path.basename(roi_mask)) + output_mask = os.path.join(model_path, "masked_%s" % os.path.basename(roi_mask)) roi_mask = trim_mask(roi_mask, mask_for_means, output_mask) readme_flags.append("custom_roi_mask_trimmed") @@ -733,9 +758,8 @@ def build_feat_model(model_df, model_name, group_config_file, resource_id, if str(col) == "Custom_ROI_Mean_1": new_design_substring = new_design_substring + " %s" % col else: - new_design_substring = new_design_substring +" + %s" % col - design_formula = design_formula.replace("Custom_ROI_Mean", - new_design_substring) + new_design_substring = new_design_substring + " + %s" % col + design_formula = design_formula.replace("Custom_ROI_Mean", new_design_substring) cat_list = [] if "categorical" in group_config_obj.ev_selections.keys(): @@ -772,7 +796,6 @@ def build_feat_model(model_df, model_name, group_config_file, resource_id, grp_vector = [1] * num_subjects if group_config_obj.group_sep: - # check if the group_ev parameter is a list instead of a string: # this was added to handle the new group-level analysis presets. this # is the only modification that was required to the group analysis @@ -800,17 +823,20 @@ def build_feat_model(model_df, model_name, group_config_file, resource_id, elif y == 1: grp_vector.append(2) else: - err = "\n\n[!] The two categorical covariates you " \ - "provided as the two separate groups (in order " \ - "to model each group's variances separately) " \ - "either have more than 2 levels (1/0), or are " \ - "not encoded as 1's and 0's.\n\nCovariates:\n" \ - "{0}\n{1}\n\n".format(group_ev[0], group_ev[1]) + err = ( + "\n\n[!] The two categorical covariates you " + "provided as the two separate groups (in order " + "to model each group's variances separately) " + "either have more than 2 levels (1/0), or are " + "not encoded as 1's and 0's.\n\nCovariates:\n" + "{0}\n{1}\n\n".format(group_ev[0], group_ev[1]) + ) raise Exception(err) elif len(group_ev) == 3: - for x, y, z in zip(model_df[group_ev[0]], model_df[group_ev[1]], - model_df[group_ev[2]]): + for x, y, z in zip( + model_df[group_ev[0]], model_df[group_ev[1]], model_df[group_ev[2]] + ): if x == 1: grp_vector.append(1) elif y == 1: @@ -818,44 +844,50 @@ def build_feat_model(model_df, model_name, group_config_file, resource_id, elif z == 1: grp_vector.append(3) else: - err = "\n\n[!] The three categorical covariates you " \ - "provided as the three separate groups (in order " \ - "to model each group's variances separately) " \ - "either have more than 2 levels (1/0), or are " \ - "not encoded as 1's and 0's.\n\nCovariates:\n" \ - "{0}\n{1}\n{2}\n\n".format(group_ev[0], - group_ev[1], - group_ev[2]) + err = ( + "\n\n[!] The three categorical covariates you " + "provided as the three separate groups (in order " + "to model each group's variances separately) " + "either have more than 2 levels (1/0), or are " + "not encoded as 1's and 0's.\n\nCovariates:\n" + "{0}\n{1}\n{2}\n\n".format( + group_ev[0], group_ev[1], group_ev[2] + ) + ) raise Exception(err) else: # we're only going to see this if someone plays around with # their preset or config file manually - err = "\n\n[!] If you are seeing this message, it's because:\n" \ - "1. You are using the group-level analysis presets\n" \ - "2. You are running a model with multiple groups having " \ - "their variances modeled separately (i.e. multiple " \ - "values in the FSL FLAME .grp input file), and\n" \ - "3. For some reason, the configuration has been set up " \ - "in a way where CPAC currently thinks you're including " \ - "only one group, or more than three, neither of which " \ - "are supported.\n\nGroups provided:\n{0}" \ - "\n\n".format(str(group_ev)) + err = ( + "\n\n[!] If you are seeing this message, it's because:\n" + "1. You are using the group-level analysis presets\n" + "2. You are running a model with multiple groups having " + "their variances modeled separately (i.e. multiple " + "values in the FSL FLAME .grp input file), and\n" + "3. For some reason, the configuration has been set up " + "in a way where CPAC currently thinks you're including " + "only one group, or more than three, neither of which " + "are supported.\n\nGroups provided:\n{0}" + "\n\n".format(str(group_ev)) + ) raise Exception(err) else: # model group variances separately old_ev_list = ev_list - model_df, grp_vector, ev_list, cat_list = split_groups(model_df, - group_config_obj.grouping_var, - ev_list, cat_list) + model_df, grp_vector, ev_list, cat_list = split_groups( + model_df, group_config_obj.grouping_var, ev_list, cat_list + ) # make the grouping variable categorical for Patsy (if we try to # do this automatically below, it will categorical-ize all of # the substrings too) - design_formula = design_formula.replace(group_config_obj.grouping_var, - "C(" + group_config_obj.grouping_var + ")") + design_formula = design_formula.replace( + group_config_obj.grouping_var, + "C(" + group_config_obj.grouping_var + ")", + ) if group_config_obj.coding_scheme == "Sum": design_formula = design_formula.replace(")", ", Sum)") @@ -869,12 +901,14 @@ def build_feat_model(model_df, model_name, group_config_file, resource_id, rename[old_ev].append(new_ev) for old_ev in rename.keys(): - design_formula = design_formula.replace(old_ev, - " + ".join(rename[old_ev])) + design_formula = design_formula.replace( + old_ev, " + ".join(rename[old_ev]) + ) # prep design formula for Patsy - design_formula = patsify_design_formula(design_formula, cat_list, - group_config_obj.coding_scheme[0]) + design_formula = patsify_design_formula( + design_formula, cat_list, group_config_obj.coding_scheme[0] + ) if not preset: # send to Patsy @@ -883,33 +917,39 @@ def build_feat_model(model_df, model_name, group_config_file, resource_id, dmatrix.design_info.column_names.append(model_df["Filepath"]) dmatrix_column_names = dmatrix.design_info.column_names except Exception as e: - err = "\n\n[!] Something went wrong with processing the group model "\ - "design matrix using the Python Patsy package. Patsy might " \ - "not be properly installed, or there may be an issue with the "\ - "formatting of the design matrix.\n\nDesign matrix columns: " \ - "%s\n\nPatsy-formatted design formula: %s\n\nError details: " \ - "%s\n\n" % (model_df.columns, design_formula, e) + err = ( + "\n\n[!] Something went wrong with processing the group model " + "design matrix using the Python Patsy package. Patsy might " + "not be properly installed, or there may be an issue with the " + "formatting of the design matrix.\n\nDesign matrix columns: " + "%s\n\nPatsy-formatted design formula: %s\n\nError details: " + "%s\n\n" % (model_df.columns, design_formula, e) + ) raise Exception(err) else: - if 'Sessions' in model_df: - sess_levels = list(set(list(model_df['Sessions'].values))) + if "Sessions" in model_df: + sess_levels = list(set(model_df["Sessions"].values)) if len(sess_levels) > 1: - sess_map = {sess_levels[0]: '1', sess_levels[1]: '-1'} + sess_map = {sess_levels[0]: "1", sess_levels[1]: "-1"} if len(sess_levels) == 3: - sess_map.update({sess_levels[2]: '0'}) - new_sess = [s.replace(s, sess_map[s]) for s in list(model_df['Sessions'].values)] - model_df['Sessions'] = new_sess - if 'Series' in model_df: - sess_levels = list(set(list(model_df['Series'].values))) + sess_map.update({sess_levels[2]: "0"}) + new_sess = [ + s.replace(s, sess_map[s]) for s in list(model_df["Sessions"].values) + ] + model_df["Sessions"] = new_sess + if "Series" in model_df: + sess_levels = list(set(model_df["Series"].values)) if len(sess_levels) > 1: - sess_map = {sess_levels[0]: '1', sess_levels[1]: '-1'} + sess_map = {sess_levels[0]: "1", sess_levels[1]: "-1"} if len(sess_levels) == 3: - sess_map.update({sess_levels[2]: '0'}) - new_sess = [s.replace(s, sess_map[s]) for s in list(model_df['Series'].values)] - model_df['Series'] = new_sess + sess_map.update({sess_levels[2]: "0"}) + new_sess = [ + s.replace(s, sess_map[s]) for s in list(model_df["Series"].values) + ] + model_df["Series"] = new_sess keep_cols = [x for x in model_df.columns if x in design_formula] - dmatrix = model_df[keep_cols].astype('float') + dmatrix = model_df[keep_cols].astype("float") dmatrix_column_names = list(dmatrix.columns) # check the model for multicollinearity - Patsy takes care of this, but @@ -925,7 +965,7 @@ def build_feat_model(model_df, model_name, group_config_file, resource_id, dmat_cols = [] dmat_id_cols = [] for dmat_col in dmatrix_column_names: - if 'participant_' in dmat_col: + if "participant_" in dmat_col: dmat_id_cols.append(dmat_col) else: dmat_cols.append(dmat_col) @@ -935,27 +975,36 @@ def build_feat_model(model_df, model_name, group_config_file, resource_id, # check to make sure there are more time points than EVs! if len(column_names) >= num_subjects: - err = "\n\n################## MODEL NOT GENERATED ##################" \ - "\n\n[!] CPAC says: There are more EVs than there are " \ - "participants currently included in the model for:\n\n" \ - "Derivative: {0}\nSession: {1}\nScan: {2}\nPreproc strategy:" \ - "\n {3}\n\n" \ - "There must be more participants than EVs in the design.\n\n" \ - "Number of participants: {4}\nNumber of EVs: {5}\n\nEV/" \ - "covariate list: {6}\n\nNote: If you specified to model group " \ - "variances separately, the amount of EVs can nearly double " \ - "once they are split along the grouping variable.\n\nIf the " \ - "number of participants is lower than the number of " \ - "participants in your group analysis inclusion list, this " \ - "may be because not every participant originally included has " \ - "an output for {7} for this scan and preprocessing strategy in " \ - "the individual-level analysis output directory.\n\nDesign " \ - "formula going in: {8}" \ - "\n\n#########################################################" \ - "\n\n".format(resource_id, session_id, series_or_repeated_label, - preproc_strat, num_subjects, len(column_names), - column_names, resource_id, design_formula) - print(err) + err = ( + "\n\n################## MODEL NOT GENERATED ##################" + "\n\n[!] CPAC says: There are more EVs than there are " + "participants currently included in the model for:\n\n" + "Derivative: {0}\nSession: {1}\nScan: {2}\nPreproc strategy:" + "\n {3}\n\n" + "There must be more participants than EVs in the design.\n\n" + "Number of participants: {4}\nNumber of EVs: {5}\n\nEV/" + "covariate list: {6}\n\nNote: If you specified to model group " + "variances separately, the amount of EVs can nearly double " + "once they are split along the grouping variable.\n\nIf the " + "number of participants is lower than the number of " + "participants in your group analysis inclusion list, this " + "may be because not every participant originally included has " + "an output for {7} for this scan and preprocessing strategy in " + "the individual-level analysis output directory.\n\nDesign " + "formula going in: {8}" + "\n\n#########################################################" + "\n\n".format( + resource_id, + session_id, + series_or_repeated_label, + preproc_strat, + num_subjects, + len(column_names), + column_names, + resource_id, + design_formula, + ) + ) # check the merged file's order check_merged_file(model_df["Filepath"], merge_file) @@ -983,9 +1032,11 @@ def build_feat_model(model_df, model_name, group_config_file, resource_id, dmatrix = dmat_T.transpose() readme_flags.append("cat_demeaned") - dmatrix_df = pd.DataFrame(np.array(dmatrix), - index=model_df["participant_id"], - columns=dmatrix_column_names) + dmatrix_df = pd.DataFrame( + np.array(dmatrix), + index=model_df["participant_id"], + columns=dmatrix_column_names, + ) cols = dmatrix_df.columns.tolist() # make sure "column_names" is in the same order as the original EV column @@ -994,7 +1045,7 @@ def build_feat_model(model_df, model_name, group_config_file, resource_id, dmat_cols = [] dmat_id_cols = [] for dmat_col in cols: - if 'participant_' in dmat_col: + if "participant_" in dmat_col: dmat_id_cols.append(dmat_col) else: dmat_cols.append(dmat_col) @@ -1006,17 +1057,20 @@ def build_feat_model(model_df, model_name, group_config_file, resource_id, dmat_csv_path = os.path.join(model_path, "design_matrix.csv") - write_design_matrix_csv(dmatrix_df, model_df["participant_id"], - column_names, dmat_csv_path) + write_design_matrix_csv( + dmatrix_df, model_df["participant_id"], column_names, dmat_csv_path + ) # time for contrasts - if (group_config_obj.custom_contrasts == None) or (group_config_obj.contrasts == None): + if (group_config_obj.custom_contrasts is None) or ( + group_config_obj.contrasts is None + ): # if no custom contrasts matrix CSV provided (i.e. the user # specified contrasts in the GUI) contrasts_columns = column_names if group_config_obj.f_tests: - for i in group_config_obj.f_tests[1:len(group_config_obj.f_tests)-1]: - contrasts_columns.append('f_test_{0}'.format(i)) + for i in group_config_obj.f_tests[1 : len(group_config_obj.f_tests) - 1]: + contrasts_columns.append("f_test_{0}".format(i)) else: pass @@ -1029,39 +1083,49 @@ def build_feat_model(model_df, model_name, group_config_file, resource_id, else: if os.path.isfile(contrast_out_path): contrasts_df = pd.read_csv(contrast_out_path) - if contrasts_df.shape[0] > 1 or np.count_nonzero(contrasts_df.values[0][1:]) > 0: - msg = "\n\n[!] C-PAC says: It appears you have modified your " \ - "contrasts CSV file already- back up this file before " \ - "building your model again to avoid overwriting your " \ - "changes.\n\nContrasts file:\n{0}" \ - "\n\n".format(contrast_out_path) + if ( + contrasts_df.shape[0] > 1 + or np.count_nonzero(contrasts_df.values[0][1:]) > 0 + ): + msg = ( + "\n\n[!] C-PAC says: It appears you have modified your " + "contrasts CSV file already- back up this file before " + "building your model again to avoid overwriting your " + "changes.\n\nContrasts file:\n{0}" + "\n\n".format(contrast_out_path) + ) raise Exception(msg) with open(contrast_out_path, "w") as f: - f.write('Contrasts') + f.write("Contrasts") for col in contrasts_columns: - f.write(',{0}'.format(col)) - f.write('\ncontrast_1') + f.write(",{0}".format(col)) + f.write("\ncontrast_1") for col in contrasts_columns: - f.write(',0') + f.write(",0") - groups_out_path = os.path.join(model_path, 'groups.txt') - with open(groups_out_path, 'w') as f: + groups_out_path = os.path.join(model_path, "groups.txt") + with open(groups_out_path, "w") as f: for val in grp_vector: - f.write('{0}\n'.format(val)) - - msg = 'Model successfully generated for..\nDerivative: {0}\nSession: {1}' \ - '\nScan: {2}\nPreprocessing strategy:\n {3}\n\nModel directory:' \ - '\n{4}\n\nGroup configuration file:\n{5}\n\nContrasts template CSV:' \ - '\n{6}\n\nDefine your contrasts in this contrasts template CSV and ' \ - 'save your changes, then run FSL-FEAT ' \ - 'through the command-line like so:\n\n cpac group ' \ - 'feat run ' \ - '\n'.format(resource_id, session_id, series_or_repeated_label, - preproc_strat, model_path, group_config_file, - contrast_out_path) - print('-------------------------------------------------------------------') - print(msg) - print('-------------------------------------------------------------------') + f.write("{0}\n".format(val)) + + msg = ( + "Model successfully generated for..\nDerivative: {0}\nSession: {1}" + "\nScan: {2}\nPreprocessing strategy:\n {3}\n\nModel directory:" + "\n{4}\n\nGroup configuration file:\n{5}\n\nContrasts template CSV:" + "\n{6}\n\nDefine your contrasts in this contrasts template CSV and " + "save your changes, then run FSL-FEAT " + "through the command-line like so:\n\n cpac group " + "feat run " + "\n".format( + resource_id, + session_id, + series_or_repeated_label, + preproc_strat, + model_path, + group_config_file, + contrast_out_path, + ) + ) return dmat_csv_path, new_sub_file, contrast_out_path diff --git a/CPAC/pipeline/cpac_group_runner.py b/CPAC/pipeline/cpac_group_runner.py index fd9e542723..7b44a833ee 100644 --- a/CPAC/pipeline/cpac_group_runner.py +++ b/CPAC/pipeline/cpac_group_runner.py @@ -14,14 +14,11 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -import os import fnmatch -from CPAC.pipeline.nipype_pipeline_engine.plugins import MultiProcPlugin -from CPAC.utils.monitoring import log_nodes_cb +import os def load_config_yml(config_file, individual=False): - # loads a configuration YAML file # # input @@ -31,31 +28,41 @@ def load_config_yml(config_file, individual=False): # config: Configuration object import os + import yaml try: config_path = os.path.realpath(config_file) - config_dict = yaml.safe_load(open(config_path, 'r')) + config_dict = yaml.safe_load(open(config_path, "r")) config = config_dict except Exception as e: - err = "\n\n[!] CPAC says: Could not load or read the configuration " \ - "YAML file:\n%s\nDetails: %s\n\n" % (config_file, e) + err = ( + "\n\n[!] CPAC says: Could not load or read the configuration " + "YAML file:\n%s\nDetails: %s\n\n" % (config_file, e) + ) raise Exception(err) if individual: - config.logDirectory = os.path.abspath(config["pipeline_setup"]["log_directory"]["path"]) - config.workingDirectory = os.path.abspath(config["pipeline_setup"]["working_directory"]["path"]) - config.outputDirectory = os.path.abspath(config["pipeline_setup"]["output_directory"]["output_path"]) - config.crashLogDirectory = os.path.abspath(config["pipeline_setup"]["crash_log_directory"]["path"]) + config.logDirectory = os.path.abspath( + config["pipeline_setup"]["log_directory"]["path"] + ) + config.workingDirectory = os.path.abspath( + config["pipeline_setup"]["working_directory"]["path"] + ) + config.outputDirectory = os.path.abspath( + config["pipeline_setup"]["output_directory"]["output_path"] + ) + config.crashLogDirectory = os.path.abspath( + config["pipeline_setup"]["crash_log_directory"]["path"] + ) return config def load_text_file(filepath, label="file"): - # loads a text file and returns the lines in a list # # input @@ -65,51 +72,60 @@ def load_text_file(filepath, label="file"): # lines_list: list of lines from text file if not filepath.endswith(".txt"): - err = "\n\n[!] CPAC says: The %s should be a text file (.txt).\n" \ - "Path provided: %s\n\n" % (label, filepath) + err = ( + "\n\n[!] CPAC says: The %s should be a text file (.txt).\n" + "Path provided: %s\n\n" % (label, filepath) + ) raise Exception(err) try: - with open(filepath,"r") as f: + with open(filepath, "r") as f: lines_list = f.readlines() except Exception as e: - err = "\n\n[!] CPAC says: Could not load or read the %s:\n%s\n" \ - "Details: %s\n\n" % (label, filepath, e) + err = ( + "\n\n[!] CPAC says: Could not load or read the %s:\n%s\n" + "Details: %s\n\n" % (label, filepath, e) + ) raise Exception(err) # get rid of those \n's that love to show up everywhere - lines_list = [i.rstrip("\n") for i in lines_list] - - return lines_list + return [i.rstrip("\n") for i in lines_list] def grab_pipeline_dir_subs(pipeline_dir, ses=False): import os + inclusion_list = [] if ses: - pipeline_list = [x for x in os.listdir(pipeline_dir) if os.path.isdir(os.path.join(pipeline_dir, x))] + pipeline_list = [ + x + for x in os.listdir(pipeline_dir) + if os.path.isdir(os.path.join(pipeline_dir, x)) + ] else: - pipeline_list = [x.split('_')[0] for x in os.listdir(pipeline_dir) if os.path.isdir(os.path.join(pipeline_dir, x))] + pipeline_list = [ + x.split("_")[0] + for x in os.listdir(pipeline_dir) + if os.path.isdir(os.path.join(pipeline_dir, x)) + ] for sub_id in pipeline_list: if sub_id not in inclusion_list: inclusion_list.append(sub_id) - inclusion_list = sorted(inclusion_list) - return inclusion_list + return sorted(inclusion_list) def read_pheno_csv_into_df(pheno_csv, id_label=None): """Read the phenotypic file CSV or TSV into a Pandas DataFrame.""" - import pandas as pd with open(pheno_csv, "r") as f: if id_label: - if '.tsv' in pheno_csv or '.TSV' in pheno_csv: + if ".tsv" in pheno_csv or ".TSV" in pheno_csv: pheno_df = pd.read_table(f, dtype={id_label: object}) else: pheno_df = pd.read_csv(f, dtype={id_label: object}) else: - if '.tsv' in pheno_csv or '.TSV' in pheno_csv: + if ".tsv" in pheno_csv or ".TSV" in pheno_csv: pheno_df = pd.read_table(f) else: pheno_df = pd.read_csv(f) @@ -117,8 +133,7 @@ def read_pheno_csv_into_df(pheno_csv, id_label=None): return pheno_df -def gather_nifti_globs(pipeline_output_folder, resource_list, - pull_func=False): +def gather_nifti_globs(pipeline_output_folder, resource_list, pull_func=False): # the number of directory levels under each participant's output folder # can vary depending on what preprocessing strategies were chosen, and # there may be several output filepaths with varying numbers of directory @@ -126,30 +141,34 @@ def gather_nifti_globs(pipeline_output_folder, resource_list, # this parses them quickly while also catching each preprocessing strategy - import os import glob + import os + import pandas as pd import pkg_resources as p exts = ".nii" nifti_globs = [] - keys_tsv = p.resource_filename('CPAC', 'resources/cpac_outputs.tsv') + keys_tsv = p.resource_filename("CPAC", "resources/cpac_outputs.tsv") try: - keys = pd.read_csv(keys_tsv, delimiter='\t') + keys = pd.read_csv(keys_tsv, delimiter="\t") except Exception as e: - err = "\n[!] Could not access or read the cpac_outputs.tsv " \ - "resource file:\n{0}\n\nError details {1}\n".format(keys_tsv, e) + err = ( + "\n[!] Could not access or read the cpac_outputs.tsv " + "resource file:\n{0}\n\nError details {1}\n".format(keys_tsv, e) + ) raise Exception(err) - derivative_list = list( - keys[keys['Sub-Directory'] == 'func']['Resource']) + derivative_list = list(keys[keys["Sub-Directory"] == "func"]["Resource"]) derivative_list = derivative_list + list( - keys[keys['Sub-Directory'] == 'anat']['Resource']) + keys[keys["Sub-Directory"] == "anat"]["Resource"] + ) if pull_func: derivative_list = derivative_list + list( - keys[keys['Space'] == 'functional']['Resource']) + keys[keys["Space"] == "functional"]["Resource"] + ) if len(resource_list) == 0: err = "\n\n[!] No derivatives selected!\n\n" @@ -158,9 +177,6 @@ def gather_nifti_globs(pipeline_output_folder, resource_list, # remove any extra /'s pipeline_output_folder = pipeline_output_folder.rstrip("/") - print("\n\nGathering the output file paths from " - "{0}...".format(pipeline_output_folder)) - # this is just to keep the fsl feat config file derivative_list entries # nice and lean dirs_to_grab = [] @@ -173,79 +189,83 @@ def gather_nifti_globs(pipeline_output_folder, resource_list, dirs_to_grab.append("framewise-displacement-jenkinson") for resource_name in dirs_to_grab: - glob_string = os.path.join(pipeline_output_folder, "*", "*", - f"*{resource_name}*") + glob_string = os.path.join( + pipeline_output_folder, "*", "*", f"*{resource_name}*" + ) # get all glob strings that result in a list of paths where every path # ends with a NIFTI file prog_string = ".." while len(glob.glob(glob_string)) != 0: - - if any(exts in x for x in glob.glob(glob_string)) == True: + if any(exts in x for x in glob.glob(glob_string)) is True: nifti_globs.append(glob_string) - + glob_string = os.path.join(glob_string, "*") prog_string = prog_string + "." if len(nifti_globs) == 0: - err = "\n\n[!] No output filepaths found in the pipeline output " \ - "directory provided for the derivatives selected!\n\nPipeline " \ - "output directory provided: %s\nDerivatives selected:%s\n\n" \ - % (pipeline_output_folder, resource_list) + err = ( + "\n\n[!] No output filepaths found in the pipeline output " + "directory provided for the derivatives selected!\n\nPipeline " + "output directory provided: %s\nDerivatives selected:%s\n\n" + % (pipeline_output_folder, resource_list) + ) raise Exception(err) return nifti_globs def grab_raw_score_filepath(filepath, resource_id): - # this lives in the output path collector - import os import glob + import os if "vmhc" in resource_id: - raw_score_path = filepath.replace(resource_id,"vmhc_raw_score") - raw_score_path = raw_score_path.replace(raw_score_path.split("/")[-1],"") - raw_score_path = glob.glob(os.path.join(raw_score_path,"*"))[0] + raw_score_path = filepath.replace(resource_id, "vmhc_raw_score") + raw_score_path = raw_score_path.replace(raw_score_path.split("/")[-1], "") + raw_score_path = glob.glob(os.path.join(raw_score_path, "*"))[0] else: - raw_score_path = filepath.replace("_zstd","") - raw_score_path = raw_score_path.replace("_fisher","") - raw_score_path = raw_score_path.replace("_zstat","") + raw_score_path = filepath.replace("_zstd", "") + raw_score_path = raw_score_path.replace("_fisher", "") + raw_score_path = raw_score_path.replace("_zstat", "") if "sca_roi_files_to_standard" in resource_id: sub_folder = raw_score_path.split("/")[-2] + "/" if "z_score" in sub_folder: - raw_score_path = raw_score_path.replace(sub_folder,"") + raw_score_path = raw_score_path.replace(sub_folder, "") elif "sca_tempreg_maps_zstat" in resource_id: sca_filename = raw_score_path.split("/")[-1] globpath = raw_score_path.replace(sca_filename, "*") globpath = os.path.join(globpath, sca_filename) raw_score_path = glob.glob(globpath)[0] elif "dr_tempreg_maps" in resource_id: - raw_score_path = raw_score_path.replace("map_z_","map_") + raw_score_path = raw_score_path.replace("map_z_", "map_") raw_filename = raw_score_path.split("/")[-1] - raw_score_path = raw_score_path.replace(raw_filename,"") - raw_score_path = glob.glob(os.path.join(raw_score_path,"*",raw_filename))[0] + raw_score_path = raw_score_path.replace(raw_filename, "") + raw_score_path = glob.glob(os.path.join(raw_score_path, "*", raw_filename))[ + 0 + ] else: # in case filenames are different between z-standardized and raw - raw_score_path = raw_score_path.replace(raw_score_path.split("/")[-1],"") + raw_score_path = raw_score_path.replace(raw_score_path.split("/")[-1], "") try: - raw_score_path = glob.glob(os.path.join(raw_score_path,"*"))[0] + raw_score_path = glob.glob(os.path.join(raw_score_path, "*"))[0] except: - raw_score_path = os.path.join(raw_score_path,"*") + raw_score_path = os.path.join(raw_score_path, "*") if (raw_score_path is None) or (not os.path.exists(raw_score_path)): - err = "\n\n[!] The filepath for the raw score of " \ - "%s can not be found.\nFilepath: %s\n\nThis " \ - "is needed for the Measure Mean calculation." \ - "\n\n" % (resource_id, raw_score_path) + err = ( + "\n\n[!] The filepath for the raw score of " + "%s can not be found.\nFilepath: %s\n\nThis " + "is needed for the Measure Mean calculation." + "\n\n" % (resource_id, raw_score_path) + ) raise Exception(err) return raw_score_path def find_power_params_file(filepath, resource_id, series_id): - import os try: @@ -255,9 +275,11 @@ def find_power_params_file(filepath, resource_id, series_id): power_first_half = os.path.join(power_first_half, series_id_string) participant_id = power_first_half.split("/")[-3] except Exception as e: - err = "\n\n[!] Something went wrong with finding the power " \ - "parameters file for at least one of the participants.\n\n" \ - "Error details: %s\n\n" % e + err = ( + "\n\n[!] Something went wrong with finding the power " + "parameters file for at least one of the participants.\n\n" + "Error details: %s\n\n" % e + ) raise Exception(err) power_params_file = None @@ -268,41 +290,49 @@ def find_power_params_file(filepath, resource_id, series_id): power_params_file = filepath if not power_params_file: - err = "\n\n[!] Could not find the power parameters file for the " \ - "following participant and series..\nParticipant: %s\n" \ - "Series: %s\n\nIt should be available here: %s\n\n" \ - % (participant_id, series_id, power_first_half) + err = ( + "\n\n[!] Could not find the power parameters file for the " + "following participant and series..\nParticipant: %s\n" + "Series: %s\n\nIt should be available here: %s\n\n" + % (participant_id, series_id, power_first_half) + ) raise Exception(err) return power_params_file def extract_power_params(power_params_lines, power_params_filepath): - # check formatting if len(power_params_lines) != 2: - err = "\n\n[!] There is something wrong with the formatting of the " \ - "power parameters file.\nFilepath: %s\n\n" \ - % power_params_filepath + err = ( + "\n\n[!] There is something wrong with the formatting of the " + "power parameters file.\nFilepath: %s\n\n" % power_params_filepath + ) raise Exception(err) names_list = power_params_lines[0].split(",") values_list = power_params_lines[1].split(",") # let's make extra sure - if (values_list[0].replace(" ", "") not in power_params_filepath) or \ - (values_list[1].replace(" ", "") not in power_params_filepath): - err = "\n\n[!] There is a mismatch between the contents of the " \ - "power parameters file and where it is located!\n" \ - "Filepath: %s\n\n" % power_params_filepath + if (values_list[0].replace(" ", "") not in power_params_filepath) or ( + values_list[1].replace(" ", "") not in power_params_filepath + ): + err = ( + "\n\n[!] There is a mismatch between the contents of the " + "power parameters file and where it is located!\n" + "Filepath: %s\n\n" % power_params_filepath + ) raise Exception(err) - if (names_list[2].replace(" ", "") != "MeanFD_Power") or \ - (names_list[3].replace(" ", "") != "MeanFD_Jenkinson") or \ - (names_list[-1].replace(" ", "") != "MeanDVARS"): - err = "\n\n[!] There is a mismatch between the power parameters " \ - "format and what is expected!!\nFilepath: %s\n\n" \ - % power_params_filepath + if ( + (names_list[2].replace(" ", "") != "MeanFD_Power") + or (names_list[3].replace(" ", "") != "MeanFD_Jenkinson") + or (names_list[-1].replace(" ", "") != "MeanDVARS") + ): + err = ( + "\n\n[!] There is a mismatch between the power parameters " + "format and what is expected!!\nFilepath: %s\n\n" % power_params_filepath + ) raise Exception(err) meanfd_power = values_list[2] @@ -312,14 +342,18 @@ def extract_power_params(power_params_lines, power_params_filepath): return meanfd_power, meanfd_jenk, meandvars -def create_output_dict_list(nifti_globs, pipeline_output_folder, - resource_list, get_motion=False, - get_raw_score=False, pull_func=False, - derivatives=None, exts=['nii', 'nii.gz']): - +def create_output_dict_list( + nifti_globs, + pipeline_output_folder, + resource_list, + get_motion=False, + get_raw_score=False, + pull_func=False, + derivatives=None, + exts=["nii", "nii.gz"], +): import os - import glob - import itertools + import pandas as pd import pkg_resources as p @@ -328,30 +362,29 @@ def create_output_dict_list(nifti_globs, pipeline_output_folder, raise Exception(err) if derivatives is None: - - keys_tsv = p.resource_filename('CPAC', 'resources/cpac_outputs.tsv') + keys_tsv = p.resource_filename("CPAC", "resources/cpac_outputs.tsv") try: - keys = pd.read_csv(keys_tsv,delimiter='\t') + keys = pd.read_csv(keys_tsv, delimiter="\t") except Exception as e: - err = "\n[!] Could not access or read the cpac_outputs.csv " \ + err = ( + "\n[!] Could not access or read the cpac_outputs.csv " "resource file:\n{0}\n\nError details {1}\n".format(keys_tsv, e) + ) raise Exception(err) - derivatives = list( - keys[keys['Sub-Directory'] == 'func']['Resource']) + derivatives = list(keys[keys["Sub-Directory"] == "func"]["Resource"]) derivatives = derivatives + list( - keys[keys['Sub-Directory'] == 'anat']['Resource']) + keys[keys["Sub-Directory"] == "anat"]["Resource"] + ) if pull_func: derivatives = derivatives + list( - keys[keys['Space'] == 'functional']['Resource']) + keys[keys["Space"] == "functional"]["Resource"] + ) # remove any extra /'s pipeline_output_folder = pipeline_output_folder.rstrip("/") - print("\n\nGathering the output file paths from " - "{0}...".format(pipeline_output_folder)) - # this is just to keep the fsl feat config file derivatives entries # nice and lean search_dirs = [] @@ -360,26 +393,25 @@ def create_output_dict_list(nifti_globs, pipeline_output_folder, if resource_name in derivative_name: search_dirs.append(derivative_name) - ''' + """ search_dirs = [ resource_name for resource_name in resource_list if any([resource_name in derivative_name for derivative_name in derivatives]) ] - ''' + """ # grab MeanFD_Jenkinson just in case search_dirs += ["framewise-displacement-jenkinson"] - exts = ['.' + ext.lstrip('.') for ext in exts] + exts = ["." + ext.lstrip(".") for ext in exts] # parse each result of each "valid" glob string output_dict_list = {} for root, _, files in os.walk(pipeline_output_folder): for filename in files: - filepath = os.path.join(root, filename) if not any(fnmatch.fnmatch(filepath, pattern) for pattern in nifti_globs): continue @@ -389,7 +421,7 @@ def create_output_dict_list(nifti_globs, pipeline_output_folder, relative_filepath = filepath.split(pipeline_output_folder)[1] filepath_pieces = [_f for _f in relative_filepath.split("/") if _f] - resource_id = '_'.join(filepath_pieces[2].split(".")[0].split("_")[3:]) + resource_id = "_".join(filepath_pieces[2].split(".")[0].split("_")[3:]) if resource_id not in search_dirs: continue @@ -409,35 +441,31 @@ def create_output_dict_list(nifti_globs, pipeline_output_folder, new_row_dict = {} new_row_dict["participant_session_id"] = unique_id - new_row_dict["participant_id"], new_row_dict["Sessions"] = \ - unique_id.split('_') + new_row_dict["participant_id"], new_row_dict["Sessions"] = unique_id.split( + "_" + ) new_row_dict["Series"] = series_id new_row_dict["Filepath"] = filepath - print('{0} - {1} - {2}'.format( - unique_id.split("_")[0], - series_id, - resource_id - )) - if get_motion: # if we're including motion measures - power_params_file = find_power_params_file(filepath, - resource_id, series_id) - power_params_lines = load_text_file(power_params_file, - "power parameters file") - meanfd_p, meanfd_j, meandvars = \ - extract_power_params(power_params_lines, - power_params_file) + power_params_file = find_power_params_file( + filepath, resource_id, series_id + ) + power_params_lines = load_text_file( + power_params_file, "power parameters file" + ) + meanfd_p, meanfd_j, meandvars = extract_power_params( + power_params_lines, power_params_file + ) new_row_dict["MeanFD_Power"] = meanfd_p new_row_dict["MeanFD_Jenkinson"] = meanfd_j new_row_dict["MeanDVARS"] = meandvars if get_raw_score: # grab raw score for measure mean just in case - raw_score_path = grab_raw_score_filepath(filepath, - resource_id) + raw_score_path = grab_raw_score_filepath(filepath, resource_id) new_row_dict["Raw_Filepath"] = raw_score_path # unique_resource_id is tuple (resource_id,strat_info) @@ -447,14 +475,12 @@ def create_output_dict_list(nifti_globs, pipeline_output_folder, def create_output_df_dict(output_dict_list, inclusion_list=None): - import pandas as pd output_df_dict = {} # unique_resource_id is tuple (resource_id,strat_info) for unique_resource_id in output_dict_list.keys(): - # NOTE: this dataframe reflects what was found in the C-PAC output # directory for individual-level analysis outputs, # NOT what is in the pheno file @@ -465,10 +491,6 @@ def create_output_df_dict(output_dict_list, inclusion_list=None): new_df = new_df[new_df.participant_id.isin(inclusion_list)] if new_df.empty: - print("No outputs found for {0} for the participants " - "listed in the the group analysis participant list you " - "used. Skipping generating a model for this " - "output.".format(unique_resource_id)) continue # unique_resource_id is tuple (resource_id,strat_info) @@ -478,15 +500,16 @@ def create_output_df_dict(output_dict_list, inclusion_list=None): return output_df_dict -def gather_outputs(pipeline_folder, resource_list, inclusion_list, - get_motion, get_raw_score, get_func=False, - derivatives=None): - - nifti_globs = gather_nifti_globs( - pipeline_folder, - resource_list, - get_func - ) +def gather_outputs( + pipeline_folder, + resource_list, + inclusion_list, + get_motion, + get_raw_score, + get_func=False, + derivatives=None, +): + nifti_globs = gather_nifti_globs(pipeline_folder, resource_list, get_func) output_dict_list = create_output_dict_list( nifti_globs, @@ -495,16 +518,15 @@ def gather_outputs(pipeline_folder, resource_list, inclusion_list, get_motion, get_raw_score, get_func, - derivatives + derivatives, ) - output_df_dict = create_output_df_dict(output_dict_list, inclusion_list) - - return output_df_dict + return create_output_df_dict(output_dict_list, inclusion_list) def pheno_sessions_to_repeated_measures(pheno_df, sessions_list): import pandas as pd + """Take in the selected session names, and match them to the unique participant-session IDs appropriately for an FSL FEAT repeated measures analysis. @@ -534,7 +556,9 @@ def pheno_sessions_to_repeated_measures(pheno_df, sessions_list): if "participant_" in col_names: num_partic_cols += 1 - if num_partic_cols > 1 and ("Sessions" in pheno_df.columns or "Sessions_column_one" in pheno_df.columns): + if num_partic_cols > 1 and ( + "Sessions" in pheno_df.columns or "Sessions_column_one" in pheno_df.columns + ): for part_id in pheno_df["participant_id"]: if "participant_{0}".format(part_id) in pheno_df.columns: continue @@ -549,12 +573,14 @@ def pheno_sessions_to_repeated_measures(pheno_df, sessions_list): another_new_row = [] # grab the ordered sublist before we double the rows - sublist = pheno_df['participant_id'] + sublist = pheno_df["participant_id"] for session in sessions_list: sub_pheno_df = pheno_df.copy() sub_pheno_df["Sessions"] = session - sub_pheno_df["participant_session_id"] = pheno_df.participant_id+'_ses-%s' % session + sub_pheno_df["participant_session_id"] = ( + pheno_df.participant_id + "_ses-%s" % session + ) new_rows.append(sub_pheno_df) another_new_row.append(sub_pheno_df) pheno_df = pd.concat(new_rows) @@ -572,7 +598,7 @@ def pheno_sessions_to_repeated_measures(pheno_df, sessions_list): for session in sessions_list: if session in participant_unique_id.split("_")[1]: - #print(participant_unique_id)# generate/update sessions categorical column + # print(participant_unique_id)# generate/update sessions categorical column part_id = participant_unique_id.split("_")[0] part_ids_col.append(str(part_id)) @@ -593,16 +619,13 @@ def pheno_sessions_to_repeated_measures(pheno_df, sessions_list): # add new participant ID columns for sub_id in sublist: - new_col = 'participant_{0}'.format(sub_id) + new_col = "participant_{0}".format(sub_id) pheno_df[new_col] = participant_id_cols[new_col] - pheno_df = pheno_df.astype('object') - - return pheno_df + return pheno_df.astype("object") -def pheno_series_to_repeated_measures(pheno_df, series_list, - repeated_sessions=False): +def pheno_series_to_repeated_measures(pheno_df, series_list, repeated_sessions=False): import pandas as pd # take in the selected series/scans, and create all of the permutations @@ -641,7 +664,6 @@ def pheno_series_to_repeated_measures(pheno_df, series_list, i = 0 for participant_unique_id in pheno_df["participant_id"]: - part_col = [0] * len(pheno_df["participant_id"]) header_title = "participant_%s" % participant_unique_id @@ -656,13 +678,10 @@ def pheno_series_to_repeated_measures(pheno_df, series_list, for new_col in participant_id_cols.keys(): pheno_df[new_col] = participant_id_cols[new_col] - pheno_df = pheno_df.astype('object') - - return pheno_df + return pheno_df.astype("object") def balance_repeated_measures(pheno_df, sessions_list, series_list=None): - # this is for repeated measures only. # if the user selects a participant list like this: # sub001_session_1 @@ -700,19 +719,30 @@ def prep_feat_inputs(group_config_file): # config_file: filepath to the C-PAC group-level config file import os + import pandas as pd import pkg_resources as p - keys_tsv = p.resource_filename('CPAC', 'resources/cpac_outputs.tsv') + keys_tsv = p.resource_filename("CPAC", "resources/cpac_outputs.tsv") try: - keys = pd.read_csv(keys_tsv, delimiter='\t') + keys = pd.read_csv(keys_tsv, delimiter="\t") except Exception as e: - err = "\n[!] Could not access or read the cpac_outputs.tsv " \ - "resource file:\n{0}\n\nError details {1}\n".format(keys_tsv, e) + err = ( + "\n[!] Could not access or read the cpac_outputs.tsv " + "resource file:\n{0}\n\nError details {1}\n".format(keys_tsv, e) + ) raise Exception(err) - derivatives = list(keys[keys['Derivative'] == 'yes'][keys['Space'] == 'template'][keys['Values'] == 'z-score']['Resource']) - derivatives = derivatives + list(keys[keys['Derivative'] == 'yes'][keys['Space'] == 'template'][keys['Values'] == 'z-stat']['Resource']) + derivatives = list( + keys[keys["Derivative"] == "yes"][keys["Space"] == "template"][ + keys["Values"] == "z-score" + ]["Resource"] + ) + derivatives = derivatives + list( + keys[keys["Derivative"] == "yes"][keys["Space"] == "template"][ + keys["Values"] == "z-stat" + ]["Resource"] + ) group_model = load_config_yml(group_config_file) pipeline_dir = group_model.pipeline_dir @@ -726,34 +756,36 @@ def prep_feat_inputs(group_config_file): if not group_model.participant_list: inclusion_list = grab_pipeline_dir_subs(pipeline_dir) - elif '.' in group_model.participant_list: + elif "." in group_model.participant_list: if not os.path.isfile(group_model.participant_list): - raise Exception('\n[!] C-PAC says: Your participant ' - 'inclusion list is not a valid file!\n\n' - 'File path: {0}' - '\n'.format(group_model.participant_list)) + raise Exception( + "\n[!] C-PAC says: Your participant " + "inclusion list is not a valid file!\n\n" + "File path: {0}" + "\n".format(group_model.participant_list) + ) else: - inclusion_list = load_text_file(group_model.participant_list, - "group-level analysis participant " - "list") + inclusion_list = load_text_file( + group_model.participant_list, "group-level analysis participant " "list" + ) else: inclusion_list = grab_pipeline_dir_subs(pipeline_dir) output_measure_list = group_model.derivative_list # if any of the models will require motion parameters - if ("MeanFD" in group_model.design_formula) or ("MeanDVARS" in group_model.design_formula): + if ("MeanFD" in group_model.design_formula) or ( + "MeanDVARS" in group_model.design_formula + ): get_motion = True # make sure "None" gets processed properly here... - if (group_model.custom_roi_mask == "None") or \ - (group_model.custom_roi_mask == "none"): + if group_model.custom_roi_mask in ("None", "none"): custom_roi_mask = None else: custom_roi_mask = group_model.custom_roi_mask - if ("Measure_Mean" in group_model.design_formula) or \ - (custom_roi_mask != None): + if ("Measure_Mean" in group_model.design_formula) or (custom_roi_mask is not None): get_raw_score = True # sammin sammin mmmm samin in gray v @@ -766,11 +798,9 @@ def prep_feat_inputs(group_config_file): # - each dataframe will contain output filepaths and their associated # information, and each dataframe will include ALL SERIES/SCANS # - the dataframes will be pruned for each model LATER - output_df_dict = gather_outputs(pipeline_dir, - output_measure_list, - inclusion_list, - get_motion, - get_raw_score) + output_df_dict = gather_outputs( + pipeline_dir, output_measure_list, inclusion_list, get_motion, get_raw_score + ) # alright, group model processing time # going to merge the phenotype DFs with the output file DF @@ -779,19 +809,23 @@ def prep_feat_inputs(group_config_file): model_name = group_model.model_name if len(group_model.derivative_list) == 0: - err = "\n\n[!] There are no derivatives listed in the " \ - "derivative_list field of your group analysis " \ - "configuration file.\n\nConfiguration file: " \ - "{0}\n".format(group_config_file) + err = ( + "\n\n[!] There are no derivatives listed in the " + "derivative_list field of your group analysis " + "configuration file.\n\nConfiguration file: " + "{0}\n".format(group_config_file) + ) raise Exception(err) # load original phenotype CSV into a dataframe - pheno_df = read_pheno_csv_into_df(group_model.pheno_file, - group_model.participant_id_label) + pheno_df = read_pheno_csv_into_df( + group_model.pheno_file, group_model.participant_id_label + ) # enforce the sub ID label to "Participant" - pheno_df.rename(columns={group_model.participant_id_label:"participant_id"}, - inplace=True) + pheno_df.rename( + columns={group_model.participant_id_label: "participant_id"}, inplace=True + ) pheno_df["participant_id"] = pheno_df["participant_id"].astype(str) # unique_resource = (output_measure_type, preprocessing strategy) @@ -830,38 +864,39 @@ def prep_feat_inputs(group_config_file): inclusion_list = grab_pipeline_dir_subs(pipeline_dir) output_df = output_df[output_df["participant_id"].isin(inclusion_list)] elif os.path.isfile(group_model.participant_list): - inclusion_list = load_text_file(group_model.participant_list, - "group-level analysis " - "participant list") + inclusion_list = load_text_file( + group_model.participant_list, "group-level analysis " "participant list" + ) output_df = output_df[output_df["participant_id"].isin(inclusion_list)] else: - raise Exception('\nCannot read group-level analysis participant ' \ - 'list.\n') + raise Exception("\nCannot read group-level analysis participant " "list.\n") new_pheno_df = pheno_df.copy() # check for inconsistency with leading zeroes # (sometimes, the sub_ids from individual will be something like # '0002601' and the phenotype will have '2601') - sublist_subs = output_df['participant_id'] - pheno_subs = list(new_pheno_df['participant_id']) + sublist_subs = output_df["participant_id"] + pheno_subs = list(new_pheno_df["participant_id"]) for sub in sublist_subs: if sub in pheno_subs: # okay, there's at least one match break else: - new_sublist_subs = [str(x).lstrip('0') for x in sublist_subs] + new_sublist_subs = [str(x).lstrip("0") for x in sublist_subs] for sub in new_sublist_subs: if sub in pheno_subs: # that's better - output_df['participant_id'] = new_sublist_subs + output_df["participant_id"] = new_sublist_subs break else: - raise Exception('the participant IDs in your group ' - 'analysis participant list and the ' - 'participant IDs in your phenotype file ' - 'do not match') + raise Exception( + "the participant IDs in your group " + "analysis participant list and the " + "participant IDs in your phenotype file " + "do not match" + ) repeated_measures = False repeated_sessions = False @@ -880,19 +915,18 @@ def prep_feat_inputs(group_config_file): if repeated_sessions: # IF USING FSL PRESETS: new_pheno_df will get passed # through unchanged - new_pheno_df = \ - pheno_sessions_to_repeated_measures(new_pheno_df, - group_model.sessions_list) + new_pheno_df = pheno_sessions_to_repeated_measures( + new_pheno_df, group_model.sessions_list + ) # create new rows for all of the series, if applicable # ex. if 10 subjects and two sessions, 10 rows -> 20 rows if repeated_series: # IF USING FSL PRESETS: new_pheno_df will get passed # through unchanged - new_pheno_df = \ - pheno_series_to_repeated_measures(new_pheno_df, - group_model.series_list, - repeated_sessions) + new_pheno_df = pheno_series_to_repeated_measures( + new_pheno_df, group_model.series_list, repeated_sessions + ) # drop the pheno rows - if there are participants missing in # the output files (ex. if ReHo did not complete for 2 of the @@ -900,13 +934,16 @@ def prep_feat_inputs(group_config_file): # we are dropping all instances of this participant, all # sessions and all series, because in repeated measures/ # within-subject, if one goes, they all have to go - new_pheno_df = \ - new_pheno_df[pheno_df["participant_id"].isin(output_df["participant_id"])] + new_pheno_df = new_pheno_df[ + pheno_df["participant_id"].isin(output_df["participant_id"]) + ] if len(new_pheno_df) == 0: - err = "\n\n[!] There is a mis-match between the "\ - "participant IDs in the output directory/particip" \ - "ant list and the phenotype file.\n\n" + err = ( + "\n\n[!] There is a mis-match between the " + "participant IDs in the output directory/particip" + "ant list and the phenotype file.\n\n" + ) raise Exception(err) join_columns = ["participant_id"] @@ -918,44 +955,62 @@ def prep_feat_inputs(group_config_file): # check in case the pheno has series IDs that doesn't # exist in the output directory, first - new_pheno_df = \ - new_pheno_df[new_pheno_df["Series"].isin(output_df["Series"])] + new_pheno_df = new_pheno_df[ + new_pheno_df["Series"].isin(output_df["Series"]) + ] # okay, now check against the user-specified series list - new_pheno_df = \ - new_pheno_df[new_pheno_df["Series"].isin(group_model.series_list)] + new_pheno_df = new_pheno_df[ + new_pheno_df["Series"].isin(group_model.series_list) + ] join_columns.append("Series") # pull together the pheno DF and the output files DF! - new_pheno_df = pd.merge(new_pheno_df, output_df, - how="inner", on=join_columns) + new_pheno_df = pd.merge( + new_pheno_df, output_df, how="inner", on=join_columns + ) if repeated_sessions: # this can be removed/modified once sessions are no # longer integrated in the full unique participant IDs - new_pheno_df, dropped_parts = \ - balance_repeated_measures(new_pheno_df, - group_model.sessions_list, - group_model.series_list) + new_pheno_df, dropped_parts = balance_repeated_measures( + new_pheno_df, group_model.sessions_list, group_model.series_list + ) - session = 'repeated_measures_multiple_sessions' - series = 'repeated_measures_multiple_series' + session = "repeated_measures_multiple_sessions" + series = "repeated_measures_multiple_series" else: series = "repeated_measures_multiple_series" - if 'session' in output_df: - for ses_df_tuple in new_pheno_df.groupby('Sessions'): - session = 'ses-{0}'.format(ses_df_tuple[0]) + if "session" in output_df: + for ses_df_tuple in new_pheno_df.groupby("Sessions"): + session = "ses-{0}".format(ses_df_tuple[0]) ses_df = ses_df_tuple[1] # send it in - analysis_dict[(model_name, group_config_file, resource_id, - strat_info, session, series)] = new_pheno_df + analysis_dict[ + ( + model_name, + group_config_file, + resource_id, + strat_info, + session, + series, + ) + ] = new_pheno_df else: # default a session - session = 'ses-1' + session = "ses-1" # send it in - analysis_dict[(model_name, group_config_file, resource_id, - strat_info, session, series)] = new_pheno_df + analysis_dict[ + ( + model_name, + group_config_file, + resource_id, + strat_info, + session, + series, + ) + ] = new_pheno_df else: # this runs if there are repeated sessions but not @@ -963,39 +1018,57 @@ def prep_feat_inputs(group_config_file): # split up the series here # iterate over the Series/Scans for series_df_tuple in output_df.groupby("Series"): - - session = 'repeated_measures_multiple_sessions' + session = "repeated_measures_multiple_sessions" series = series_df_tuple[0] # series_df is output_df but with only one of the # Series series_df = series_df_tuple[1] - series_df = series_df.astype('str') + series_df = series_df.astype("str") # trim down the pheno DF to match the output DF, remove any # extra participant_ identity columns, and merge - newer_pheno_df = new_pheno_df[pheno_df["participant_id"].isin(series_df["participant_id"])] + newer_pheno_df = new_pheno_df[ + pheno_df["participant_id"].isin(series_df["participant_id"]) + ] for col in newer_pheno_df.columns: - if 'participant_' in col and 'id' not in col and 'session' not in col: - if col.replace('participant_', '') not in list(series_df['participant_id']): + if ( + "participant_" in col + and "id" not in col + and "session" not in col + ): + if col.replace("participant_", "") not in list( + series_df["participant_id"] + ): newer_pheno_df = newer_pheno_df.drop(labels=col, axis=1) - newer_pheno_df = newer_pheno_df.astype('str') - newer_pheno_df = pd.merge(newer_pheno_df, series_df, how="inner", on=["participant_id", "Sessions"]) + newer_pheno_df = newer_pheno_df.astype("str") + newer_pheno_df = pd.merge( + newer_pheno_df, + series_df, + how="inner", + on=["participant_id", "Sessions"], + ) # this can be removed/modified once sessions are no # longer integrated in the full unique participant IDs if "Sessions" in newer_pheno_df.columns: - newer_pheno_df, dropped_parts = \ - balance_repeated_measures(newer_pheno_df, - group_model.sessions_list, - None) + newer_pheno_df, dropped_parts = balance_repeated_measures( + newer_pheno_df, group_model.sessions_list, None + ) # unique_resource = # (output_measure_type, preprocessing strategy) - analysis_dict[(model_name, group_config_file, - resource_id, strat_info, session, - series)] = newer_pheno_df + analysis_dict[ + ( + model_name, + group_config_file, + resource_id, + strat_info, + session, + series, + ) + ] = newer_pheno_df else: # no repeated measures @@ -1014,43 +1087,62 @@ def prep_feat_inputs(group_config_file): series_df = series_df_tuple[1] # trim down the pheno DF to match the output DF and merge - newer_pheno_df = \ - new_pheno_df[pheno_df["participant_id"].isin(series_df["participant_id"])] + newer_pheno_df = new_pheno_df[ + pheno_df["participant_id"].isin(series_df["participant_id"]) + ] # multiple sessions? - if 'Sessions' in series_df: - for ses_df_tuple in series_df.groupby('Sessions'): - session = 'ses-{0}'.format(ses_df_tuple[0]) + if "Sessions" in series_df: + for ses_df_tuple in series_df.groupby("Sessions"): + session = "ses-{0}".format(ses_df_tuple[0]) ses_df = ses_df_tuple[1] - newer_ses_pheno_df = \ - pd.merge(newer_pheno_df, ses_df, how="inner", - on=["participant_id"]) + newer_ses_pheno_df = pd.merge( + newer_pheno_df, ses_df, how="inner", on=["participant_id"] + ) # send it in - analysis_dict[(model_name, group_config_file, resource_id, - strat_info, session, series)] = newer_ses_pheno_df + analysis_dict[ + ( + model_name, + group_config_file, + resource_id, + strat_info, + session, + series, + ) + ] = newer_ses_pheno_df else: # default a session - session = 'ses-1' - newer_pheno_df = pd.merge(newer_pheno_df, series_df, - how="inner", - on=["participant_id"]) + session = "ses-1" + newer_pheno_df = pd.merge( + newer_pheno_df, series_df, how="inner", on=["participant_id"] + ) # send it in - analysis_dict[(model_name, group_config_file, resource_id, - strat_info, session, series)] = newer_pheno_df + analysis_dict[ + ( + model_name, + group_config_file, + resource_id, + strat_info, + session, + series, + ) + ] = newer_pheno_df if len(analysis_dict) == 0: - err = '\n\n[!] C-PAC says: Could not find a match between the ' \ - 'participants in your pipeline output directory that were ' \ - 'included in your analysis, and the participants in the ' \ - 'phenotype file provided.\n\n' + err = ( + "\n\n[!] C-PAC says: Could not find a match between the " + "participants in your pipeline output directory that were " + "included in your analysis, and the participants in the " + "phenotype file provided.\n\n" + ) raise Exception(err) return analysis_dict def build_feat_models(group_config_file): - import os + from CPAC.pipeline.cpac_ga_model_generator import build_feat_model analysis_dict = prep_feat_inputs(group_config_file) @@ -1069,13 +1161,15 @@ def build_feat_models(group_config_file): series = unique_resource_id[5] model_df = analysis_dict[unique_resource_id] - dmat_csv_path, new_sub_file, empty_csv = build_feat_model(model_df, - model_name, - group_config_file, - resource_id, - preproc_strat, - session, - series) + dmat_csv_path, new_sub_file, empty_csv = build_feat_model( + model_df, + model_name, + group_config_file, + resource_id, + preproc_strat, + session, + series, + ) if os.path.isfile(empty_csv): return 0 @@ -1084,11 +1178,12 @@ def build_feat_models(group_config_file): def run_feat(group_config_file, feat=True): - + from multiprocessing import Process import os + import numpy as np import pandas as pd - from multiprocessing import Process + from CPAC.pipeline.cpac_ga_model_generator import create_dir from CPAC.utils.create_flame_model_files import create_flame_model_files @@ -1102,34 +1197,42 @@ def run_feat(group_config_file, feat=True): model_name = c["fsl_feat"]["model_name"] out_dir = c["pipeline_setup"]["output_directory"]["output_path"] - pipeline_name = pipeline_dir.rstrip('/').split('/')[-1] + pipeline_name = pipeline_dir.rstrip("/").split("/")[-1] - model_dir = os.path.join(out_dir, - 'cpac_group_analysis', - 'FSL_FEAT', - '{0}'.format(pipeline_name), - 'group_model_{0}'.format(model_name)) + model_dir = os.path.join( + out_dir, + "cpac_group_analysis", + "FSL_FEAT", + "{0}".format(pipeline_name), + "group_model_{0}".format(model_name), + ) - custom_contrasts_csv = os.path.join(model_dir, 'contrasts.csv') + custom_contrasts_csv = os.path.join(model_dir, "contrasts.csv") contrasts_df = pd.read_csv(custom_contrasts_csv) if contrasts_df.shape[0] == 1 and np.count_nonzero(contrasts_df.values[0][1:]) == 0: - err = "\n\n[!] C-PAC says: It appears you haven't defined any " \ - "contrasts in your contrasts CSV file.\n\nContrasts file:\n{0}" \ - "\n\nDefine your contrasts in this file and run again." \ - "\n\n".format(custom_contrasts_csv) + err = ( + "\n\n[!] C-PAC says: It appears you haven't defined any " + "contrasts in your contrasts CSV file.\n\nContrasts file:\n{0}" + "\n\nDefine your contrasts in this file and run again." + "\n\n".format(custom_contrasts_csv) + ) raise Exception(err) models = {} for root, dirs, files in os.walk(model_dir): for filename in files: filepath = os.path.join(root, filename) - second_half = filepath.split(model_dir)[1].split('/') - second_half.remove('') + second_half = filepath.split(model_dir)[1].split("/") + second_half.remove("") try: - id_tuple = (second_half[0], second_half[1], second_half[2], - second_half[3]) + id_tuple = ( + second_half[0], + second_half[1], + second_half[2], + second_half[3], + ) except IndexError: # not a file we are interested in continue @@ -1137,133 +1240,183 @@ def run_feat(group_config_file, feat=True): if id_tuple not in models.keys(): models[id_tuple] = {} - if 'group_sublist' in filepath: - models[id_tuple]['group_sublist'] = filepath - elif 'design_matrix' in filepath: - models[id_tuple]['design_matrix'] = filepath - models[id_tuple]['dir_path'] = filepath.replace('model_files/design_matrix.csv', '') - elif 'groups' in filepath: - models[id_tuple]['group_vector'] = filepath - elif 'merged_mask' in filepath: - models[id_tuple]['merged_mask'] = filepath - elif 'merged' in filepath: - models[id_tuple]['merged'] = filepath + if "group_sublist" in filepath: + models[id_tuple]["group_sublist"] = filepath + elif "design_matrix" in filepath: + models[id_tuple]["design_matrix"] = filepath + models[id_tuple]["dir_path"] = filepath.replace( + "model_files/design_matrix.csv", "" + ) + elif "groups" in filepath: + models[id_tuple]["group_vector"] = filepath + elif "merged_mask" in filepath: + models[id_tuple]["merged_mask"] = filepath + elif "merged" in filepath: + models[id_tuple]["merged"] = filepath if len(models) == 0: - err = '\n\n[!] C-PAC says: Cannot find the FSL-FEAT/Randomise model ' \ - 'files.\n\nI am looking here:\n{0}\n\nIf that doesn\'t sound ' \ - 'right, double-check your group configuration file.\n\nDid you ' \ - 'build the model beforehand?\n\n'.format(model_dir) + err = ( + "\n\n[!] C-PAC says: Cannot find the FSL-FEAT/Randomise model " + "files.\n\nI am looking here:\n{0}\n\nIf that doesn't sound " + "right, double-check your group configuration file.\n\nDid you " + "build the model beforehand?\n\n".format(model_dir) + ) raise Exception(err) for id_tuple in models.keys(): - # generate working/log directory for this sub-model's group analysis run - work_dir = os.path.join(c.work_dir, - models[id_tuple]['dir_path'].replace(out_dir, '').lstrip('/')) - work_dir = work_dir.replace('cpac_group_analysis', 'cpac_group_analysis_workdir') - work_dir = work_dir.replace('model_files/', '') - log_dir = os.path.join(c["pipeline_setup"]["log_directory"]["path"], - models[id_tuple]['dir_path'].replace(out_dir, '').lstrip('/')) - log_dir = log_dir.replace('cpac_group_analysis', 'cpac_group_analysis_logdir') - log_dir = log_dir.replace('model_files/', '') - - model_out_dir = os.path.join(models[id_tuple]['dir_path'], - 'fsl-feat_results') - model_out_dir = model_out_dir.replace('model_files/', '') - input_files_dir = os.path.join(models[id_tuple]['dir_path'], - 'flame_input_files') + work_dir = os.path.join( + c.work_dir, models[id_tuple]["dir_path"].replace(out_dir, "").lstrip("/") + ) + work_dir = work_dir.replace( + "cpac_group_analysis", "cpac_group_analysis_workdir" + ) + work_dir = work_dir.replace("model_files/", "") + log_dir = os.path.join( + c["pipeline_setup"]["log_directory"]["path"], + models[id_tuple]["dir_path"].replace(out_dir, "").lstrip("/"), + ) + log_dir = log_dir.replace("cpac_group_analysis", "cpac_group_analysis_logdir") + log_dir = log_dir.replace("model_files/", "") + + model_out_dir = os.path.join(models[id_tuple]["dir_path"], "fsl-feat_results") + model_out_dir = model_out_dir.replace("model_files/", "") + input_files_dir = os.path.join( + models[id_tuple]["dir_path"], "flame_input_files" + ) create_dir(work_dir, "group analysis working") create_dir(log_dir, "group analysis logfile") create_dir(input_files_dir, "FSL-FEAT FLAME tool input files") create_dir(model_out_dir, "FSL-FEAT output files") - design_matrix = pd.read_csv(models[id_tuple]['design_matrix']) - design_matrix = design_matrix.drop(labels='participant_id', axis=1) - - grp_vector = load_text_file(models[id_tuple]['group_vector'], - 'group vector file') - - mat, grp, con, fts = create_flame_model_files(design_matrix, - design_matrix.columns, - None, None, - custom_contrasts_csv, - None, c["fsl_feat"]["group_sep"], - grp_vector, - c["fsl_feat"]["coding_scheme"], - model_name, - id_tuple[0], - input_files_dir) + design_matrix = pd.read_csv(models[id_tuple]["design_matrix"]) + design_matrix = design_matrix.drop(labels="participant_id", axis=1) + + grp_vector = load_text_file( + models[id_tuple]["group_vector"], "group vector file" + ) + + mat, grp, con, fts = create_flame_model_files( + design_matrix, + design_matrix.columns, + None, + None, + custom_contrasts_csv, + None, + c["fsl_feat"]["group_sep"], + grp_vector, + c["fsl_feat"]["coding_scheme"], + model_name, + id_tuple[0], + input_files_dir, + ) if fts: f_test = True else: f_test = False if not con: - msg = '\n\n################## MODEL NOT BEING INCLUDED ###########'\ - '#######' \ - '\n\n[!] C-PAC says: There is a mismatch between the design '\ - 'matrix and contrasts matrix for this model:\n\n' \ - 'Derivative: {0}\nSession: {1}\nScan: {2}\nPreprocessing ' \ - 'strategy:\n {3}\n\nThe model is not proceeding into the '\ - 'FSL-FEAT FLAME run.\n\n'\ - '#########################################################' \ - '\n'.format(id_tuple[0], id_tuple[1], id_tuple[2], id_tuple[3]) - print(msg) + "\n\n################## MODEL NOT BEING INCLUDED ###########" "#######" "\n\n[!] C-PAC says: There is a mismatch between the design " "matrix and contrasts matrix for this model:\n\n" "Derivative: {0}\nSession: {1}\nScan: {2}\nPreprocessing " "strategy:\n {3}\n\nThe model is not proceeding into the " "FSL-FEAT FLAME run.\n\n" "#########################################################" "\n".format( + id_tuple[0], id_tuple[1], id_tuple[2], id_tuple[3] + ) continue if feat: from CPAC.group_analysis.group_analysis import run_feat_pipeline - procss.append(Process(target=run_feat_pipeline, - args=(c, models[id_tuple]['merged'], - models[id_tuple]['merged_mask'], - f_test, mat, con, grp, model_out_dir, - work_dir, log_dir, model_name, fts))) + + procss.append( + Process( + target=run_feat_pipeline, + args=( + c, + models[id_tuple]["merged"], + models[id_tuple]["merged_mask"], + f_test, + mat, + con, + grp, + model_out_dir, + work_dir, + log_dir, + model_name, + fts, + ), + ) + ) else: from CPAC.randomise.randomise import prep_randomise_workflow - model_out_dir = model_out_dir.replace('feat_results', - 'randomise_results') - procss.append(Process(target=prep_randomise_workflow, - args=(c, models[id_tuple]['merged'], - models[id_tuple]['merged_mask'], - f_test, mat, con, grp, model_out_dir, - work_dir, log_dir, model_name, fts))) - manage_processes(procss, out_dir, c["fsl_feat"]["num_models_at_once"]) + model_out_dir = model_out_dir.replace("feat_results", "randomise_results") + procss.append( + Process( + target=prep_randomise_workflow, + args=( + c, + models[id_tuple]["merged"], + models[id_tuple]["merged_mask"], + f_test, + mat, + con, + grp, + model_out_dir, + work_dir, + log_dir, + model_name, + fts, + ), + ) + ) + manage_processes(procss, out_dir, c["fsl_feat"]["num_models_at_once"]) -def run_cwas_group(pipeline_dir, out_dir, working_dir, crash_dir, roi_file, - regressor_file, participant_column, columns, - permutations, parallel_nodes, plugin_args, z_score, inclusion=None): +def run_cwas_group( + pipeline_dir, + out_dir, + working_dir, + crash_dir, + roi_file, + regressor_file, + participant_column, + columns, + permutations, + parallel_nodes, + plugin_args, + z_score, + inclusion=None, +): import os - import numpy as np - from multiprocessing import pool + from CPAC.cwas.pipeline import create_cwas - from nipype import config pipeline_dir = os.path.abspath(pipeline_dir) - out_dir = os.path.join(out_dir, 'cpac_group_analysis', 'MDMR', - os.path.basename(pipeline_dir)) + out_dir = os.path.join( + out_dir, "cpac_group_analysis", "MDMR", os.path.basename(pipeline_dir) + ) - working_dir = os.path.join(working_dir, 'cpac_group_analysis', 'MDMR', - os.path.basename(pipeline_dir)) + working_dir = os.path.join( + working_dir, "cpac_group_analysis", "MDMR", os.path.basename(pipeline_dir) + ) - crash_dir = os.path.join(crash_dir, 'cpac_group_analysis', 'MDMR', - os.path.basename(pipeline_dir)) + crash_dir = os.path.join( + crash_dir, "cpac_group_analysis", "MDMR", os.path.basename(pipeline_dir) + ) inclusion_list = None - + if inclusion: - inclusion_list = load_text_file(inclusion, "MDMR participant " - "inclusion list") + inclusion_list = load_text_file(inclusion, "MDMR participant " "inclusion list") - output_df_dct = gather_outputs(pipeline_dir, - ['space-template_desc-preproc_bold'], - inclusion_list, False, False, - get_func=True) + output_df_dct = gather_outputs( + pipeline_dir, + ["space-template_desc-preproc_bold"], + inclusion_list, + False, + False, + get_func=True, + ) for preproc_strat in output_df_dct.keys(): # go over each preprocessing strategy @@ -1271,7 +1424,6 @@ def run_cwas_group(pipeline_dir, out_dir, working_dir, crash_dir, roi_file, df_dct = {} strat_df = output_df_dct[preproc_strat] - if len(set(strat_df["Series"])) > 1: # more than one scan/series ID for strat_scan in list(set(strat_df["Series"])): @@ -1279,25 +1431,25 @@ def run_cwas_group(pipeline_dir, out_dir, working_dir, crash_dir, roi_file, # from one scan ID each df_dct[strat_scan] = strat_df[strat_df["Series"] == strat_scan] else: - df_dct[list(set(strat_df["Series"]))[0]] = strat_df + df_dct[next(iter(set(strat_df["Series"])))] = strat_df for df_scan in df_dct.keys(): func_paths = { p.split("_")[0]: f - for p, f in - zip( - df_dct[df_scan].participant_id, - df_dct[df_scan].Filepath + for p, f in zip( + df_dct[df_scan].participant_id, df_dct[df_scan].Filepath ) } - - if plugin_args['n_procs'] == 1: - plugin = 'Linear' + + if plugin_args["n_procs"] == 1: + plugin = "Linear" else: - plugin = 'MultiProc' - - cwas_wf = create_cwas(name="MDMR_{0}".format(df_scan), - working_dir=working_dir, - crash_dir=crash_dir) + plugin = "MultiProc" + + cwas_wf = create_cwas( + name="MDMR_{0}".format(df_scan), + working_dir=working_dir, + crash_dir=crash_dir, + ) cwas_wf.inputs.inputspec.subjects = func_paths cwas_wf.inputs.inputspec.roi = roi_file cwas_wf.inputs.inputspec.regressor = regressor_file @@ -1310,20 +1462,22 @@ def run_cwas_group(pipeline_dir, out_dir, working_dir, crash_dir, roi_file, def run_cwas(pipeline_config): - import os + import yaml pipeline_config = os.path.abspath(pipeline_config) - pipeconfig_dct = yaml.safe_load(open(pipeline_config, 'r')) - + pipeconfig_dct = yaml.safe_load(open(pipeline_config, "r")) + num_cpus = pipeconfig_dct["pipeline_setup"]["system_config"]["num_cpus"] mem_gb = pipeconfig_dct["pipeline_setup"]["system_config"]["num_memory"] - - plugin_args = {'n_procs' : num_cpus, 'memory_gb' : mem_gb} - pipeline = pipeconfig_dct["pipeline_setup"]["output_directory"]["source_outputs_path"] + plugin_args = {"n_procs": num_cpus, "memory_gb": mem_gb} + + pipeline = pipeconfig_dct["pipeline_setup"]["output_directory"][ + "source_outputs_path" + ] output_dir = pipeconfig_dct["pipeline_setup"]["output_directory"]["output_path"] working_dir = pipeconfig_dct["pipeline_setup"]["working_directory"]["path"] crash_dir = pipeconfig_dct["pipeline_setup"]["log_directory"]["path"] @@ -1340,54 +1494,65 @@ def run_cwas(pipeline_config): if not inclusion or "None" in inclusion or "none" in inclusion: inclusion = None - run_cwas_group(pipeline, output_dir, working_dir, crash_dir, roi_file, - regressor_file, participant_column, columns, - permutations, parallel_nodes, plugin_args, z_score, - inclusion=inclusion) + run_cwas_group( + pipeline, + output_dir, + working_dir, + crash_dir, + roi_file, + regressor_file, + participant_column, + columns, + permutations, + parallel_nodes, + plugin_args, + z_score, + inclusion=inclusion, + ) def find_other_res_template(template_path, new_resolution): """ Find the same template/standard file in another resolution, if it exists. - template_path: file path to the template NIfTI file - + template_path: file path to the template NIfTI file. + new_resolution: (int) the resolution of the template file you need NOTE: Makes an assumption regarding the filename format of the files. - - """ + """ # TODO: this is assuming there is a mm resolution in the file path - not # TODO: robust to varying templates - look into alternatives ref_file = None if "mm" in template_path: - template_parts = template_path.rsplit('mm', 1) + template_parts = template_path.rsplit("mm", 1) if len(template_parts) < 2: # TODO: better message - raise Exception('no resolution in the file path!') + raise Exception("no resolution in the file path!") - template_parts[0] = str(new_resolution).join(template_parts[0].rsplit(template_parts[0][-1], 1)) + template_parts[0] = str(new_resolution).join( + template_parts[0].rsplit(template_parts[0][-1], 1) + ) ref_file = "{0}{1}".format(template_parts[0], template_parts[1]) elif "${resolution_for_func_preproc}" in template_path: - ref_file = template_path.replace("${resolution_for_func_preproc}", - "{0}mm".format(new_resolution)) + ref_file = template_path.replace( + "${resolution_for_func_preproc}", "{0}mm".format(new_resolution) + ) if ref_file: - print("\nAttempting to find {0}mm version of the template:\n{1}" - "\n\n".format(new_resolution, ref_file)) + pass return ref_file -def check_cpac_output_image(image_path, reference_path, out_dir=None, - roi_file=False): - +def check_cpac_output_image(image_path, reference_path, out_dir=None, roi_file=False): import os - import nibabel as nb + + import nibabel as nib if not out_dir: out_dir = os.getcwd() @@ -1396,13 +1561,13 @@ def check_cpac_output_image(image_path, reference_path, out_dir=None, # but place that sub-tree into the BASC working directory (in this case, # 'out_dir') try: - orig_dir = "pipeline_{0}".format(image_path.split('pipeline_')[1]) + orig_dir = "pipeline_{0}".format(image_path.split("pipeline_")[1]) except IndexError: if roi_file: orig_dir = os.path.join("ROI_files", os.path.basename(image_path)) else: raise IndexError(image_path) - out_path = os.path.join(out_dir, 'resampled_input_images', orig_dir) + out_path = os.path.join(out_dir, "resampled_input_images", orig_dir) # if this was already done if os.path.isfile(out_path): @@ -1410,72 +1575,55 @@ def check_cpac_output_image(image_path, reference_path, out_dir=None, resample = False - image_nb = nb.load(image_path) - ref_nb = nb.load(reference_path) + image_nb = nib.load(image_path) + ref_nb = nib.load(reference_path) # check: do we even need to resample? if int(image_nb.header.get_zooms()[0]) != int(ref_nb.header.get_zooms()[0]): - print("Input image resolution is {0}mm\nTemplate image resolution " - "is {1}mm\n".format(image_nb.header.get_zooms()[0], - ref_nb.header.get_zooms()[0])) resample = True if image_nb.shape != ref_nb.shape: - print("Input image shape is {0}\nTemplate image shape is " - "{1}\n".format(image_nb.shape, ref_nb.shape)) resample = True if resample: - if not os.path.isdir( - out_path.replace(os.path.basename(out_path), "")): + if not os.path.isdir(out_path.replace(os.path.basename(out_path), "")): try: os.makedirs(out_path.replace(os.path.basename(out_path), "")) except: # TODO: better message raise Exception("couldn't make the dirs!") - print("Resampling input image:\n{0}\n\n..to this reference:\n{1}" - "\n\n..and writing this file here:\n{2}" - "\n".format(image_path, reference_path, out_path)) - cmd = ['flirt', '-in', image_path, '-ref', reference_path, '-out', - out_path] + cmd = ["flirt", "-in", image_path, "-ref", reference_path, "-out", out_path] if roi_file: - cmd.append('-interp') - cmd.append('nearestneighbour') + cmd.append("-interp") + cmd.append("nearestneighbour") return cmd else: return resample def resample_cpac_output_image(cmd_args): - import subprocess - print("Running:\n{0}\n\n".format(" ".join(cmd_args))) - retcode = subprocess.check_output(cmd_args) + subprocess.check_output(cmd_args) for arg in cmd_args: - if 'resampled_input_images' in arg: + if "resampled_input_images" in arg: out_file = arg return out_file def launch_PyBASC(pybasc_config): - import subprocess - print('Running PyBASC with configuration file:\n' - '{0}'.format(pybasc_config)) - cmd_args = ['PyBASC', pybasc_config] - retcode = subprocess.check_output(cmd_args) - - return retcode + cmd_args = ["PyBASC", pybasc_config] + return subprocess.check_output(cmd_args) def run_basc(pipeline_config): """ Run the PyBASC module. - + PyBASC is a separate Python package built and maintained by Aki Nikolaidis which implements the BASC analysis via Python. PyBASC is based off of the following work: @@ -1514,50 +1662,62 @@ def run_basc(pipeline_config): running. 10. Launch PyBASC for each configuration generated. """ - + from multiprocessing import pool import os + import yaml + from CPAC.utils.datasource import check_for_s3 - from multiprocessing import pool pipeline_config = os.path.abspath(pipeline_config) - pipeconfig_dct = yaml.safe_load(open(pipeline_config, 'r')) + pipeconfig_dct = yaml.safe_load(open(pipeline_config, "r")) - output_dir = os.path.abspath(pipeconfig_dct["pipeline_setup"]["output_directory"]["output_path"]) - working_dir = os.path.abspath(pipeconfig_dct["pipeline_setup"]["working_directory"]["path"]) - if pipeconfig_dct["pipeline_setup"]["Amazon-AWS"]['aws_output_bucket_credentials']: + output_dir = os.path.abspath( + pipeconfig_dct["pipeline_setup"]["output_directory"]["output_path"] + ) + working_dir = os.path.abspath( + pipeconfig_dct["pipeline_setup"]["working_directory"]["path"] + ) + if pipeconfig_dct["pipeline_setup"]["Amazon-AWS"]["aws_output_bucket_credentials"]: creds_path = os.path.abspath( - pipeconfig_dct["pipeline_setup"]["Amazon-AWS"]['aws_output_bucket_credentials']) + pipeconfig_dct["pipeline_setup"]["Amazon-AWS"][ + "aws_output_bucket_credentials" + ] + ) func_template = pipeconfig_dct["basc"]["template_brain_only_for_func"] - if '$FSLDIR' in func_template: - if os.environ.get('FSLDIR'): - func_template = func_template.replace('$FSLDIR', - os.environ['FSLDIR']) + if "$FSLDIR" in func_template: + if os.environ.get("FSLDIR"): + func_template = func_template.replace("$FSLDIR", os.environ["FSLDIR"]) - basc_inclusion = pipeconfig_dct["pipeline_setup"]["output_directory"]["participant_list"] + basc_inclusion = pipeconfig_dct["pipeline_setup"]["output_directory"][ + "participant_list" + ] basc_scan_inclusion = pipeconfig_dct["basc"]["scan_inclusion"] basc_resolution = pipeconfig_dct["basc"]["resolution"] - basc_config_dct = {'run': True, - 'reruns': 1} + basc_config_dct = {"run": True, "reruns": 1} for key in pipeconfig_dct.keys(): - if 'basc' in key: + if "basc" in key: basc_config_dct = pipeconfig_dct[key] - iterables = ['dataset_bootstrap_list', 'timeseries_bootstrap_list', - 'blocklength_list', 'n_clusters_list', 'output_sizes'] + iterables = [ + "dataset_bootstrap_list", + "timeseries_bootstrap_list", + "blocklength_list", + "n_clusters_list", + "output_sizes", + ] for iterable in iterables: basc_config_dct[iterable] = [ - int(x) for x in str(basc_config_dct[iterable]).split(',') + int(x) for x in str(basc_config_dct[iterable]).split(",") ] - basc_config_dct['proc_mem'] = [basc_config_dct['proc'], - basc_config_dct['memory']] - del basc_config_dct['proc'] - del basc_config_dct['memory'] + basc_config_dct["proc_mem"] = [basc_config_dct["proc"], basc_config_dct["memory"]] + del basc_config_dct["proc"] + del basc_config_dct["memory"] if "None" in basc_inclusion or "none" in basc_inclusion: basc_inclusion = None @@ -1566,7 +1726,7 @@ def run_basc(pipeline_config): # sure everything is the same resolution and shape (as what the user has # selected) if "mm" in basc_resolution: - basc_resolution = basc_resolution.replace('mm', '') + basc_resolution = basc_resolution.replace("mm", "") # get the functional template, but in the specified resolution for BASC ref_file = find_other_res_template(func_template, basc_resolution) @@ -1574,57 +1734,70 @@ def run_basc(pipeline_config): # did that actually work? if not os.path.isfile(ref_file): # TODO: better message - raise Exception('\n[!] The reference file could not be found.\nPath: ' - '{0}\n'.format(ref_file)) - - working_dir = os.path.join(working_dir, 'cpac_group_analysis', 'PyBASC', - '{0}mm_resolution'.format(basc_resolution), - 'working_dir') + raise Exception( + "\n[!] The reference file could not be found.\nPath: " "{0}\n".format( + ref_file + ) + ) + + working_dir = os.path.join( + working_dir, + "cpac_group_analysis", + "PyBASC", + "{0}mm_resolution".format(basc_resolution), + "working_dir", + ) # check for S3 of ROI files here! - for key in ['roi_mask_file', 'cross_cluster_mask_file']: - if 's3://' in basc_config_dct[key]: - basc_config_dct[key] = check_for_s3(basc_config_dct[key], - creds_path=creds_path, - dl_dir=working_dir) + for key in ["roi_mask_file", "cross_cluster_mask_file"]: + if "s3://" in basc_config_dct[key]: + basc_config_dct[key] = check_for_s3( + basc_config_dct[key], creds_path=creds_path, dl_dir=working_dir + ) # resample ROI files if necessary - roi_cmd_args = check_cpac_output_image(basc_config_dct['roi_mask_file'], - ref_file, - out_dir=working_dir, - roi_file=True) - roi_two_cmd_args = check_cpac_output_image(basc_config_dct['cross_cluster_mask_file'], - ref_file, - out_dir=working_dir, - roi_file=True) + roi_cmd_args = check_cpac_output_image( + basc_config_dct["roi_mask_file"], ref_file, out_dir=working_dir, roi_file=True + ) + roi_two_cmd_args = check_cpac_output_image( + basc_config_dct["cross_cluster_mask_file"], + ref_file, + out_dir=working_dir, + roi_file=True, + ) if roi_cmd_args: roi_file = resample_cpac_output_image(roi_cmd_args) - basc_config_dct['roi_mask_file'] = roi_file + basc_config_dct["roi_mask_file"] = roi_file if roi_two_cmd_args: roi_file_two = resample_cpac_output_image(roi_two_cmd_args) - basc_config_dct['cross_cluster_mask_file'] = roi_file_two + basc_config_dct["cross_cluster_mask_file"] = roi_file_two - pipeline_dir = os.path.abspath(pipeconfig_dct["pipeline_setup"] - ["output_directory"]["source_outputs_path"]) + pipeline_dir = os.path.abspath( + pipeconfig_dct["pipeline_setup"]["output_directory"]["source_outputs_path"] + ) - out_dir = os.path.join(output_dir, 'cpac_group_analysis', 'PyBASC', - '{0}mm_resolution'.format(basc_resolution), - os.path.basename(pipeline_dir)) - working_dir = os.path.join(working_dir, - os.path.basename(pipeline_dir)) + out_dir = os.path.join( + output_dir, + "cpac_group_analysis", + "PyBASC", + "{0}mm_resolution".format(basc_resolution), + os.path.basename(pipeline_dir), + ) + working_dir = os.path.join(working_dir, os.path.basename(pipeline_dir)) inclusion_list = None scan_inclusion = None if basc_inclusion: - inclusion_list = load_text_file(basc_inclusion, "BASC participant" - " inclusion list") + inclusion_list = load_text_file( + basc_inclusion, "BASC participant" " inclusion list" + ) - if 'none' in basc_scan_inclusion.lower(): + if "none" in basc_scan_inclusion.lower(): basc_scan_inclusion = None if basc_scan_inclusion: - scan_inclusion = basc_scan_inclusion.split(',') + scan_inclusion = basc_scan_inclusion.split(",") # create encompassing output dataframe dictionary # note, it is still limited to the lowest common denominator of all @@ -1633,10 +1806,14 @@ def run_basc(pipeline_config): # type and preprocessing strategy # - each dataframe will contain output filepaths and their associated # information, and each dataframe will include ALL SERIES/SCANS - output_df_dct = gather_outputs(pipeline_dir, - ["space-template_bold"], - inclusion_list, False, False, - get_func=True) + output_df_dct = gather_outputs( + pipeline_dir, + ["space-template_bold"], + inclusion_list, + False, + False, + get_func=True, + ) for preproc_strat in output_df_dct.keys(): # go over each preprocessing strategy @@ -1644,19 +1821,18 @@ def run_basc(pipeline_config): df_dct = {} strat_df = output_df_dct[preproc_strat] - nuisance_string = \ - preproc_strat[1].replace(os.path.basename(preproc_strat[1]), - '') + nuisance_string = preproc_strat[1].replace( + os.path.basename(preproc_strat[1]), "" + ) if len(set(strat_df["Series"])) > 1: # more than one scan/series ID for strat_scan in list(set(strat_df["Series"])): # make a list of sub-dataframes, each one with only file paths # from one scan ID each - df_dct[strat_scan] = strat_df[ - strat_df["Series"] == strat_scan] + df_dct[strat_scan] = strat_df[strat_df["Series"] == strat_scan] else: - df_dct[list(set(strat_df["Series"]))[0]] = strat_df + df_dct[next(iter(set(strat_df["Series"])))] = strat_df # TODO: need a catch for if none of the df_scans below are in scan_inclusion @@ -1666,24 +1842,25 @@ def run_basc(pipeline_config): if df_scan not in scan_inclusion: continue - basc_config_dct['analysis_ID'] = '{0}_{1}'.format( - os.path.basename(pipeline_dir), df_scan) + basc_config_dct["analysis_ID"] = "{0}_{1}".format( + os.path.basename(pipeline_dir), df_scan + ) # add scan label and nuisance regression strategy label to the # output directory path - scan_out_dir = os.path.join(out_dir, df_scan, - nuisance_string.lstrip('/')) - scan_working_dir = os.path.join(working_dir, df_scan, - nuisance_string.lstrip('/')) + scan_out_dir = os.path.join(out_dir, df_scan, nuisance_string.lstrip("/")) + scan_working_dir = os.path.join( + working_dir, df_scan, nuisance_string.lstrip("/") + ) - basc_config_dct['home'] = scan_out_dir - basc_config_dct['cluster_methods'] = ['ward'] + basc_config_dct["home"] = scan_out_dir + basc_config_dct["cluster_methods"] = ["ward"] func_paths = list(df_dct[df_scan]["Filepath"]) # affinity threshold is an iterable, and must match the number of # functional file paths for the MapNodes - affinity_thresh = pipeconfig_dct["basc"]["affinity_thresh"] * len(func_paths) + pipeconfig_dct["basc"]["affinity_thresh"] * len(func_paths) # resampling if necessary # each run should take the file, resample it and write it @@ -1692,64 +1869,78 @@ def run_basc(pipeline_config): # these file paths in it ref_file_iterable = [ref_file] * len(func_paths) working_dir_iterable = [scan_working_dir] * len(func_paths) - func_cmd_args_list = map(check_cpac_output_image, func_paths, - ref_file_iterable, - working_dir_iterable) + func_cmd_args_list = map( + check_cpac_output_image, + func_paths, + ref_file_iterable, + working_dir_iterable, + ) # resample them now if func_cmd_args_list[0]: - p = pool.Pool(int(basc_config_dct['proc_mem'][0])) - func_paths = p.map(resample_cpac_output_image, - func_cmd_args_list) + p = pool.Pool(int(basc_config_dct["proc_mem"][0])) + func_paths = p.map(resample_cpac_output_image, func_cmd_args_list) # TODO: add list into basc_config here - basc_config_dct['subject_file_list'] = func_paths + basc_config_dct["subject_file_list"] = func_paths - basc_config_outfile = os.path.join(scan_working_dir, - 'PyBASC_config.yml') - print('\nWriting PyBASC configuration file for {0} scan in\n' - '{1}'.format(df_scan, basc_config_outfile)) - with open(basc_config_outfile, 'wt') as f: + basc_config_outfile = os.path.join(scan_working_dir, "PyBASC_config.yml") + with open(basc_config_outfile, "wt") as f: noalias_dumper = yaml.dumper.SafeDumper noalias_dumper.ignore_aliases = lambda self, data: True - f.write(yaml.dump(basc_config_dct, - default_flow_style=False, - Dumper=noalias_dumper)) + f.write( + yaml.dump( + basc_config_dct, default_flow_style=False, Dumper=noalias_dumper + ) + ) # go! launch_PyBASC(basc_config_outfile) -def run_isc_group(pipeline_dir, out_dir, working_dir, crash_dir, - isc, isfc, levels=[], permutations=1000, - std_filter=None, scan_inclusion=None, - roi_inclusion=None, num_cpus=1): - +def run_isc_group( + pipeline_dir, + out_dir, + working_dir, + crash_dir, + isc, + isfc, + levels=[], + permutations=1000, + std_filter=None, + scan_inclusion=None, + roi_inclusion=None, + num_cpus=1, +): import os + from CPAC.isc.pipeline import create_isc, create_isfc pipeline_dir = os.path.abspath(pipeline_dir) - out_dir = os.path.join(out_dir, 'cpac_group_analysis', 'ISC', - os.path.basename(pipeline_dir)) + out_dir = os.path.join( + out_dir, "cpac_group_analysis", "ISC", os.path.basename(pipeline_dir) + ) - working_dir = os.path.join(working_dir, 'cpac_group_analysis', 'ISC', - os.path.basename(pipeline_dir)) + working_dir = os.path.join( + working_dir, "cpac_group_analysis", "ISC", os.path.basename(pipeline_dir) + ) - crash_dir = os.path.join(crash_dir, 'cpac_group_analysis', 'ISC', - os.path.basename(pipeline_dir)) + crash_dir = os.path.join( + crash_dir, "cpac_group_analysis", "ISC", os.path.basename(pipeline_dir) + ) output_df_dct = gather_outputs( pipeline_dir, ["space-template_bold", "space-template_desc-Mean_timeseries"], inclusion_list=None, - get_motion=False, get_raw_score=False, get_func=True, - derivatives=["space-template_bold", - "space-template_desc-Mean_timeseries"], + get_motion=False, + get_raw_score=False, + get_func=True, + derivatives=["space-template_bold", "space-template_desc-Mean_timeseries"], # exts=['nii', 'nii.gz', 'csv'] ) - iteration_ids = [] for preproc_strat in output_df_dct.keys(): # go over each preprocessing strategy @@ -1758,8 +1949,7 @@ def run_isc_group(pipeline_dir, out_dir, working_dir, crash_dir, if "voxel" not in levels and derivative == "space-template_bold": continue - if ("roi" not in levels and - derivative == "space-template_desc-Mean_timeseries"): + if "roi" not in levels and derivative == "space-template_desc-Mean_timeseries": continue if derivative == "space-template_desc-Mean_timeseries": @@ -1769,8 +1959,6 @@ def run_isc_group(pipeline_dir, out_dir, working_dir, crash_dir, if roi_label in _: break else: - print("ROI label '{0}' not found in\n" - "{1}/{2}\n".format(roi_label, derivative, _)) continue df_dct = {} @@ -1783,7 +1971,7 @@ def run_isc_group(pipeline_dir, out_dir, working_dir, crash_dir, # from one scan ID each df_dct[strat_scan] = strat_df[strat_df["Series"] == strat_scan] else: - df_dct[list(set(strat_df["Series"]))[0]] = strat_df + df_dct[next(iter(set(strat_df["Series"])))] = strat_df if isc: for df_scan in df_dct.keys(): @@ -1792,30 +1980,28 @@ def run_isc_group(pipeline_dir, out_dir, working_dir, crash_dir, continue func_paths = { p.split("_")[0]: f - for p, f in - zip( - df_dct[df_scan].participant_id, - df_dct[df_scan].Filepath + for p, f in zip( + df_dct[df_scan].participant_id, df_dct[df_scan].Filepath ) } - unique_out_dir = os.path.join(out_dir, "ISC", derivative, _, - df_scan) + unique_out_dir = os.path.join(out_dir, "ISC", derivative, _, df_scan) - it_id = "ISC_{0}_{1}_{2}".format(df_scan, derivative, - _.replace('.', '').replace( - '+', '')) + it_id = "ISC_{0}_{1}_{2}".format( + df_scan, derivative, _.replace(".", "").replace("+", "") + ) - isc_wf = create_isc(name=it_id, - output_dir=unique_out_dir, - working_dir=working_dir, - crash_dir=crash_dir) + isc_wf = create_isc( + name=it_id, + output_dir=unique_out_dir, + working_dir=working_dir, + crash_dir=crash_dir, + ) isc_wf.inputs.inputspec.subjects = func_paths isc_wf.inputs.inputspec.permutations = permutations isc_wf.inputs.inputspec.std = std_filter isc_wf.inputs.inputspec.collapse_subj = False - isc_wf.run(plugin='MultiProc', - plugin_args={'n_procs': num_cpus}) + isc_wf.run(plugin="MultiProc", plugin_args={"n_procs": num_cpus}) if isfc: for df_scan in df_dct.keys(): @@ -1824,42 +2010,42 @@ def run_isc_group(pipeline_dir, out_dir, working_dir, crash_dir, continue func_paths = { p.split("_")[0]: f - for p, f in - zip( - df_dct[df_scan].participant_id, - df_dct[df_scan].Filepath + for p, f in zip( + df_dct[df_scan].participant_id, df_dct[df_scan].Filepath ) } - unique_out_dir = os.path.join(out_dir, "ISFC", derivative, _, - df_scan) + unique_out_dir = os.path.join(out_dir, "ISFC", derivative, _, df_scan) - it_id = "ISFC_{0}_{1}_{2}".format(df_scan, derivative, - _.replace('.', '').replace( - '+', '')) + it_id = "ISFC_{0}_{1}_{2}".format( + df_scan, derivative, _.replace(".", "").replace("+", "") + ) - isfc_wf = create_isfc(name=it_id, - output_dir=unique_out_dir, - working_dir=working_dir, - crash_dir=crash_dir) + isfc_wf = create_isfc( + name=it_id, + output_dir=unique_out_dir, + working_dir=working_dir, + crash_dir=crash_dir, + ) isfc_wf.inputs.inputspec.subjects = func_paths isfc_wf.inputs.inputspec.permutations = permutations isfc_wf.inputs.inputspec.std = std_filter isfc_wf.inputs.inputspec.collapse_subj = False - isfc_wf.run(plugin='MultiProc', - plugin_args={'n_procs': num_cpus}) + isfc_wf.run(plugin="MultiProc", plugin_args={"n_procs": num_cpus}) def run_isc(pipeline_config): - import os + import yaml pipeline_config = os.path.abspath(pipeline_config) - pipeconfig_dct = yaml.safe_load(open(pipeline_config, 'r')) + pipeconfig_dct = yaml.safe_load(open(pipeline_config, "r")) - pipeline_dir = pipeconfig_dct["pipeline_setup"]["output_directory"]["source_outputs_path"] + pipeline_dir = pipeconfig_dct["pipeline_setup"]["output_directory"][ + "source_outputs_path" + ] output_dir = pipeconfig_dct["pipeline_setup"]["output_directory"]["output_path"] working_dir = pipeconfig_dct["pipeline_setup"]["working_directory"]["path"] @@ -1867,7 +2053,9 @@ def run_isc(pipeline_config): scan_inclusion = None if "scan_inclusion" in pipeconfig_dct.keys(): - scan_inclusion = pipeconfig_dct["pipeline_setup"]["system_config"]["scan_inclusion"] + scan_inclusion = pipeconfig_dct["pipeline_setup"]["system_config"][ + "scan_inclusion" + ] roi_inclusion = None if "isc_roi_inclusion" in pipeconfig_dct.keys(): @@ -1896,8 +2084,6 @@ def run_isc(pipeline_config): return if not isc and not isfc: - print("\nISC and ISFC are not enabled to run in the group-level " - "analysis configuration YAML file, and will not run.\n") return pipeline_dirs = [] @@ -1908,34 +2094,51 @@ def run_isc(pipeline_config): pipeline_dirs.append(os.path.join(pipeline_dir, dirname)) if not pipeline_dirs: - print("\nNo pipeline output directories found- make sure your " - "'pipeline_dir' field in the group configuration YAML " - "file is pointing to a C-PAC pipeline output directory " - "populated with a folder or folders that begin with the " - "'pipeline_' prefix.\n\nPipeline directory " - "provided:\n{0}\n".format(pipeline_dir)) + pass for pipeline in pipeline_dirs: - run_isc_group(pipeline, output_dir, working_dir, crash_dir, - isc=isc, isfc=isfc, levels=levels, - permutations=permutations, std_filter=std_filter, - scan_inclusion=scan_inclusion, - roi_inclusion=roi_inclusion, num_cpus=num_cpus) + run_isc_group( + pipeline, + output_dir, + working_dir, + crash_dir, + isc=isc, + isfc=isfc, + levels=levels, + permutations=permutations, + std_filter=std_filter, + scan_inclusion=scan_inclusion, + roi_inclusion=roi_inclusion, + num_cpus=num_cpus, + ) def run_qpp(group_config_file): - from CPAC.qpp.pipeline import create_qpp c = load_config_yml(group_config_file) - pipeline_dir = os.path.abspath(c["pipeline_setup"]["output_directory"]["source_outputs_path"]) - out_dir = os.path.join(c["pipeline_setup"]["output_directory"]["output_path"], 'cpac_group_analysis', 'QPP', - os.path.basename(pipeline_dir)) - working_dir = os.path.join(c["pipeline_setup"]["working_directory"]["path"], 'cpac_group_analysis', 'QPP', - os.path.basename(pipeline_dir)) - crash_dir = os.path.join(c["pipeline_setup"]["crash_log_directory"]["path"], 'cpac_group_analysis', 'QPP', - os.path.basename(pipeline_dir)) + pipeline_dir = os.path.abspath( + c["pipeline_setup"]["output_directory"]["source_outputs_path"] + ) + out_dir = os.path.join( + c["pipeline_setup"]["output_directory"]["output_path"], + "cpac_group_analysis", + "QPP", + os.path.basename(pipeline_dir), + ) + working_dir = os.path.join( + c["pipeline_setup"]["working_directory"]["path"], + "cpac_group_analysis", + "QPP", + os.path.basename(pipeline_dir), + ) + crash_dir = os.path.join( + c["pipeline_setup"]["crash_log_directory"]["path"], + "cpac_group_analysis", + "QPP", + os.path.basename(pipeline_dir), + ) try: os.makedirs(out_dir) @@ -1947,26 +2150,28 @@ def run_qpp(group_config_file): outputs = gather_outputs( pipeline_dir, ["space-template_bold"], - inclusion_list=c["pipeline_setup"]["output_directory"] - ["participant_list"], - get_motion=False, get_raw_score=False, get_func=True, + inclusion_list=c["pipeline_setup"]["output_directory"]["participant_list"], + get_motion=False, + get_raw_score=False, + get_func=True, derivatives=["space-template_bold"], - #exts=['nii', 'nii.gz'] + # exts=['nii', 'nii.gz'] ) - if c["qpp"]["stratification"] == 'Scan': - qpp_stratification = ['Series'] - elif c["qpp"]["stratification"] == 'Session': - qpp_stratification = ['Sessions'] - elif c["qpp"]["stratification"] in ['Session and Scan', 'Scan and Session']: - qpp_stratification = ['Sessions', 'Series'] + if c["qpp"]["stratification"] == "Scan": + qpp_stratification = ["Series"] + elif c["qpp"]["stratification"] == "Session": + qpp_stratification = ["Sessions"] + elif c["qpp"]["stratification"] in ["Session and Scan", "Scan and Session"]: + qpp_stratification = ["Sessions", "Series"] else: qpp_stratification = [] for (resource_id, strat_info), output_df in outputs.items(): - if c["qpp"]["session_inclusion"]: - output_df = output_df[output_df["Sessions"].isin(c["qpp"]["session_inclusion"])] + output_df = output_df[ + output_df["Sessions"].isin(c["qpp"]["session_inclusion"]) + ] if c["qpp"]["scan_inclusion"]: output_df = output_df[output_df["Series"].isin(c["qpp"]["scan_inclusion"])] @@ -1976,13 +2181,21 @@ def run_qpp(group_config_file): output_df_groups = [([], output_df)] for group_id, output_df_group in output_df_groups: - group = list(zip(qpp_stratification, group_id)) - group_id = "_".join(["%s-%s" % ({ - "Sessions": "ses", - "Series": "scan", - }[k], v) for k, v in group]) + group_id = "_".join( + [ + "%s-%s" + % ( + { + "Sessions": "ses", + "Series": "scan", + }[k], + v, + ) + for k, v in group + ] + ) group_working_dir = os.path.join(working_dir, group_id) group_crash_dir = os.path.join(crash_dir, group_id) @@ -1990,19 +2203,27 @@ def run_qpp(group_config_file): output_df_group, _ = balance_repeated_measures( output_df_group, output_df_group.Sessions.unique(), - output_df_group.Series.unique() + output_df_group.Series.unique(), ) - output_df_group = output_df_group.sort_values(by='participant_session_id') + output_df_group = output_df_group.sort_values(by="participant_session_id") - wf = create_qpp(name="QPP", working_dir=group_working_dir, crash_dir=group_crash_dir) + wf = create_qpp( + name="QPP", working_dir=group_working_dir, crash_dir=group_crash_dir + ) wf.inputs.inputspec.window_length = c["qpp"]["window"] wf.inputs.inputspec.permutations = c["qpp"]["permutations"] - wf.inputs.inputspec.lower_correlation_threshold = c["qpp"]["initial_threshold"] - wf.inputs.inputspec.higher_correlation_threshold = c["qpp"]["final_threshold"] + wf.inputs.inputspec.lower_correlation_threshold = c["qpp"][ + "initial_threshold" + ] + wf.inputs.inputspec.higher_correlation_threshold = c["qpp"][ + "final_threshold" + ] wf.inputs.inputspec.iterations = c["qpp"]["iterations"] - wf.inputs.inputspec.correlation_threshold_iteration = c["qpp"]["initial_threshold_iterations"] + wf.inputs.inputspec.correlation_threshold_iteration = c["qpp"][ + "initial_threshold_iterations" + ] wf.inputs.inputspec.convergence_iterations = 1 wf.inputs.inputspec.datasets = output_df_group.Filepath.tolist() @@ -2011,11 +2232,10 @@ def run_qpp(group_config_file): def manage_processes(procss, output_dir, num_parallel=1): - import os # start kicking it off - pid = open(os.path.join(output_dir, 'pid_group.txt'), 'w') + pid = open(os.path.join(output_dir, "pid_group.txt"), "w") jobQueue = [] if len(procss) <= num_parallel: @@ -2039,7 +2259,7 @@ def manage_processes(procss, output_dir, num_parallel=1): while idx < len(procss): if len(jobQueue) == 0 and idx == 0: idc = idx - for p in procss[idc: idc + num_parallel]: + for p in procss[idc : idc + num_parallel]: p.start() print(p.pid, file=pid) jobQueue.append(p) @@ -2047,7 +2267,6 @@ def manage_processes(procss, output_dir, num_parallel=1): else: for job in jobQueue: if not job.is_alive(): - print('found dead job {0}'.format(job)) loc = jobQueue.index(job) del jobQueue[loc] procss[idx].start() @@ -2058,7 +2277,6 @@ def manage_processes(procss, output_dir, num_parallel=1): def run(config_file): - # this runs all group analyses, and this function only really exists for # the "Run Group-Level Analysis" command on the GUI @@ -2085,6 +2303,6 @@ def run(config_file): if 1 in c["fsl_randomise"]["run"]: run_feat(config_file, feat=False) - #Run QPP, if selected + # Run QPP, if selected if 1 in c["qpp"]["run"]: - run_qpp(config_file) \ No newline at end of file + run_qpp(config_file) diff --git a/CPAC/pipeline/cpac_pipeline.py b/CPAC/pipeline/cpac_pipeline.py index 4e4e7f9a71..ceb8b222e0 100644 --- a/CPAC/pipeline/cpac_pipeline.py +++ b/CPAC/pipeline/cpac_pipeline.py @@ -14,226 +14,215 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . +import copy +import csv +import faulthandler import os +import pickle +import shutil import sys import time -import csv -import shutil -import pickle -import copy -import faulthandler - from time import strftime -import nipype import yaml -# pylint: disable=wrong-import-order -from CPAC.pipeline import nipype_pipeline_engine as pe -from CPAC.pipeline.nipype_pipeline_engine.plugins import \ - LegacyMultiProcPlugin, MultiProcPlugin +import nipype from nipype import config, logging - +from flowdump import WorkflowJSONMeta, save_workflow_json from indi_aws import aws_utils, fetch_creds import CPAC - -from CPAC.pipeline.check_outputs import check_outputs -from CPAC.pipeline.engine import NodeBlock, initiate_rpool +from CPAC.alff.alff import alff_falff, alff_falff_space_template from CPAC.anat_preproc.anat_preproc import ( - freesurfer_reconall, - freesurfer_abcd_preproc, - anatomical_init, - acpc_align_head, - acpc_align_head_with_mask, acpc_align_brain, + acpc_align_brain_T2, acpc_align_brain_with_mask, - registration_T2w_to_T1w, - non_local_means, - n4_bias_correction, - t1t2_bias_correction, - brain_mask_afni, - brain_mask_fsl, - brain_mask_niworkflows_ants, - brain_mask_unet, - brain_mask_freesurfer, - brain_mask_freesurfer_abcd, - brain_mask_freesurfer_fsl_tight, - brain_mask_freesurfer_fsl_loose, + acpc_align_brain_with_mask_T2, + acpc_align_head, + acpc_align_head_T2, + acpc_align_head_with_mask, + acpc_align_head_with_mask_T2, + anatomical_init, + anatomical_init_T2, + brain_extraction, + brain_extraction_T2, + brain_extraction_temp, + brain_extraction_temp_T2, brain_mask_acpc_afni, + brain_mask_acpc_freesurfer, + brain_mask_acpc_freesurfer_abcd, + brain_mask_acpc_freesurfer_fsl_loose, + brain_mask_acpc_freesurfer_fsl_tight, brain_mask_acpc_fsl, brain_mask_acpc_niworkflows_ants, + brain_mask_acpc_T2, brain_mask_acpc_unet, - brain_mask_acpc_freesurfer, - brain_mask_acpc_freesurfer_abcd, + brain_mask_afni, + brain_mask_freesurfer, + brain_mask_freesurfer_abcd, + brain_mask_freesurfer_fsl_loose, + brain_mask_freesurfer_fsl_tight, + brain_mask_fsl, + brain_mask_niworkflows_ants, + brain_mask_T2, + brain_mask_unet, correct_restore_brain_intensity_abcd, - brain_mask_acpc_freesurfer_fsl_tight, - brain_mask_acpc_freesurfer_fsl_loose, - brain_extraction_temp, - brain_extraction, - anatomical_init_T2, - acpc_align_head_T2, - acpc_align_head_with_mask_T2, - acpc_align_brain_T2, - acpc_align_brain_with_mask_T2, - non_local_means_T2, + freesurfer_abcd_preproc, + freesurfer_reconall, + n4_bias_correction, n4_bias_correction_T2, - brain_mask_T2, - brain_mask_acpc_T2, - brain_extraction_temp_T2, - brain_extraction_T2, -) - -from CPAC.registration.registration import ( - register_ANTs_anat_to_template, - overwrite_transform_anat_to_template, - register_FSL_anat_to_template, - register_symmetric_ANTs_anat_to_template, - register_symmetric_FSL_anat_to_template, - register_ANTs_EPI_to_template, - register_FSL_EPI_to_template, - coregistration_prep_vol, - coregistration_prep_mean, - coregistration_prep_fmriprep, - coregistration, - create_func_to_T1template_xfm, - create_func_to_T1template_symmetric_xfm, - warp_wholeheadT1_to_template, - warp_T1mask_to_template, - apply_phasediff_to_timeseries_separately, - apply_blip_to_timeseries_separately, - warp_timeseries_to_T1template, - warp_timeseries_to_T1template_deriv, - warp_sbref_to_T1template, - warp_bold_mask_to_T1template, - warp_deriv_mask_to_T1template, - warp_timeseries_to_EPItemplate, - warp_bold_mean_to_EPItemplate, - warp_bold_mask_to_EPItemplate, - warp_deriv_mask_to_EPItemplate, - warp_timeseries_to_T1template_abcd, - single_step_resample_timeseries_to_T1template, - warp_timeseries_to_T1template_dcan_nhp, - warp_tissuemask_to_T1template, - warp_tissuemask_to_EPItemplate + non_local_means, + non_local_means_T2, + registration_T2w_to_T1w, + t1t2_bias_correction, ) - -from CPAC.seg_preproc.seg_preproc import ( - tissue_seg_fsl_fast, - tissue_seg_T1_template_based, - tissue_seg_EPI_template_based, - tissue_seg_ants_prior, - tissue_seg_freesurfer +from CPAC.distortion_correction.distortion_correction import ( + distcor_blip_afni_qwarp, + distcor_blip_fsl_topup, + distcor_phasediff_fsl_fugue, ) - from CPAC.func_preproc import ( calc_motion_stats, func_motion_correct, func_motion_correct_only, func_motion_estimates, get_motion_ref, - motion_estimate_filter + motion_estimate_filter, ) - from CPAC.func_preproc.func_preproc import ( - func_scaling, - func_truncate, - func_despike, - func_despike_template, - func_slice_time, - func_reorient, bold_mask_afni, - bold_mask_fsl, - bold_mask_fsl_afni, - bold_mask_anatomical_refined, bold_mask_anatomical_based, + bold_mask_anatomical_refined, bold_mask_anatomical_resampled, bold_mask_ccs, + bold_mask_fsl, + bold_mask_fsl_afni, bold_masking, + func_despike, + func_despike_template, func_mean, - func_normalize -) - -from CPAC.distortion_correction.distortion_correction import ( - distcor_phasediff_fsl_fugue, - distcor_blip_afni_qwarp, - distcor_blip_fsl_topup + func_normalize, + func_reorient, + func_scaling, + func_slice_time, + func_truncate, ) - +from CPAC.network_centrality.pipeline import network_centrality from CPAC.nuisance.nuisance import ( - choose_nuisance_blocks, - ICA_AROMA_ANTsreg, - ICA_AROMA_FSLreg, ICA_AROMA_ANTsEPIreg, + ICA_AROMA_ANTsreg, ICA_AROMA_FSLEPIreg, - erode_mask_T1w, - erode_mask_CSF, - erode_mask_GM, - erode_mask_WM, + ICA_AROMA_FSLreg, + choose_nuisance_blocks, erode_mask_bold, erode_mask_boldCSF, erode_mask_boldGM, erode_mask_boldWM, + erode_mask_CSF, + erode_mask_GM, + erode_mask_T1w, + erode_mask_WM, + ingress_regressors, nuisance_regression_template, - ingress_regressors ) -from CPAC.surface.surf_preproc import surface_postproc - -from CPAC.timeseries.timeseries_analysis import ( - timeseries_extraction_AVG, - timeseries_extraction_Voxel, - spatial_regression +# pylint: disable=wrong-import-order +from CPAC.pipeline import nipype_pipeline_engine as pe +from CPAC.pipeline.check_outputs import check_outputs +from CPAC.pipeline.engine import NodeBlock, initiate_rpool +from CPAC.pipeline.nipype_pipeline_engine.plugins import ( + LegacyMultiProcPlugin, + MultiProcPlugin, ) - -from CPAC.sca.sca import ( - SCA_AVG, - dual_regression, - multiple_regression +from CPAC.pipeline.random_state import set_up_random_state_logger +from CPAC.pipeline.schema import valid_options +from CPAC.qc.pipeline import create_qc_workflow +from CPAC.qc.xcp import qc_xcp +from CPAC.registration.registration import ( + apply_blip_to_timeseries_separately, + apply_phasediff_to_timeseries_separately, + coregistration, + coregistration_prep_fmriprep, + coregistration_prep_mean, + coregistration_prep_vol, + create_func_to_T1template_symmetric_xfm, + create_func_to_T1template_xfm, + overwrite_transform_anat_to_template, + register_ANTs_anat_to_template, + register_ANTs_EPI_to_template, + register_FSL_anat_to_template, + register_FSL_EPI_to_template, + register_symmetric_ANTs_anat_to_template, + register_symmetric_FSL_anat_to_template, + single_step_resample_timeseries_to_T1template, + warp_bold_mask_to_EPItemplate, + warp_bold_mask_to_T1template, + warp_bold_mean_to_EPItemplate, + warp_deriv_mask_to_EPItemplate, + warp_deriv_mask_to_T1template, + warp_sbref_to_T1template, + warp_T1mask_to_template, + warp_timeseries_to_EPItemplate, + warp_timeseries_to_T1template, + warp_timeseries_to_T1template_abcd, + warp_timeseries_to_T1template_dcan_nhp, + warp_timeseries_to_T1template_deriv, + warp_tissuemask_to_EPItemplate, + warp_tissuemask_to_T1template, + warp_wholeheadT1_to_template, ) - -from CPAC.alff.alff import alff_falff, alff_falff_space_template from CPAC.reho.reho import reho, reho_space_template -from flowdump import save_workflow_json, WorkflowJSONMeta - -from CPAC.utils.workflow_serialization import cpac_flowdump_serializer -from CPAC.vmhc.vmhc import ( - smooth_func_vmhc, - warp_timeseries_to_sym_template, - vmhc +from CPAC.sca.sca import SCA_AVG, dual_regression, multiple_regression +from CPAC.seg_preproc.seg_preproc import ( + tissue_seg_ants_prior, + tissue_seg_EPI_template_based, + tissue_seg_freesurfer, + tissue_seg_fsl_fast, + tissue_seg_T1_template_based, ) - -from CPAC.network_centrality.pipeline import ( - network_centrality +from CPAC.surface.surf_preproc import surface_postproc +from CPAC.timeseries.timeseries_analysis import ( + spatial_regression, + timeseries_extraction_AVG, + timeseries_extraction_Voxel, ) - -from CPAC.pipeline.random_state import set_up_random_state_logger -from CPAC.pipeline.schema import valid_options -from CPAC.utils.trimmer import the_trimmer from CPAC.utils import Configuration, set_subject from CPAC.utils.docs import version_report -from CPAC.utils.versioning import REQUIREMENTS -from CPAC.qc.pipeline import create_qc_workflow -from CPAC.qc.xcp import qc_xcp - -from CPAC.utils.monitoring import getLogger, log_nodes_cb, log_nodes_initial, \ - LOGTAIL, set_up_logger, \ - WARNING_FREESURFER_OFF_WITH_DATA +from CPAC.utils.monitoring import ( + LOGTAIL, + WARNING_FREESURFER_OFF_WITH_DATA, + getLogger, + log_nodes_cb, + log_nodes_initial, + set_up_logger, +) from CPAC.utils.monitoring.draw_gantt_chart import resource_report +from CPAC.utils.trimmer import the_trimmer from CPAC.utils.utils import ( check_config_resources, check_system_deps, ) +from CPAC.utils.versioning import REQUIREMENTS +from CPAC.utils.workflow_serialization import cpac_flowdump_serializer +from CPAC.vmhc.vmhc import smooth_func_vmhc, vmhc, warp_timeseries_to_sym_template -logger = getLogger('nipype.workflow') +logger = getLogger("nipype.workflow") faulthandler.enable() # config.enable_debug_mode() -def run_workflow(sub_dict, c, run, pipeline_timing_info=None, p_name=None, - plugin='MultiProc', plugin_args=None, test_config=False): - ''' - Function to prepare and, optionally, run the C-PAC workflow +def run_workflow( + sub_dict, + c, + run, + pipeline_timing_info=None, + p_name=None, + plugin="MultiProc", + plugin_args=None, + test_config=False, +): + """ + Function to prepare and, optionally, run the C-PAC workflow. Parameters ---------- @@ -258,7 +247,7 @@ def run_workflow(sub_dict, c, run, pipeline_timing_info=None, p_name=None, workflow : nipype workflow the prepared nipype workflow object containing the parameters specified in the config - ''' + """ from CPAC.utils.datasource import bidsier_prefix if plugin is not None and not isinstance(plugin, str): @@ -266,33 +255,43 @@ def run_workflow(sub_dict, c, run, pipeline_timing_info=None, p_name=None, 'CPAC.pipeline.cpac_pipeline.run_workflow requires a ' 'string for the optional "plugin" argument, but a ' f'{getattr(type(plugin), "__name__", str(type(plugin)))} ' - 'was provided.') + 'was provided.' + ) # Assure that changes on config will not affect other parts c = copy.copy(c) subject_id, p_name, log_dir = set_subject(sub_dict, c) - c['subject_id'] = subject_id - - set_up_logger(f'{subject_id}_expectedOutputs', - filename=f'{bidsier_prefix(c["subject_id"])}_' - 'expectedOutputs.yml', - level='info', log_dir=log_dir, mock=True, - overwrite_existing=True) - if c.pipeline_setup['Debugging']['verbose']: - set_up_logger('engine', level='debug', log_dir=log_dir, mock=True) - - config.update_config({ - 'logging': { - 'log_directory': log_dir, - 'log_to_file': bool(getattr(c.pipeline_setup['log_directory'], - 'run_logging', True)) - }, - 'execution': { - 'crashfile_format': 'txt', - 'resource_monitor_frequency': 0.2, - 'stop_on_first_crash': c['pipeline_setup', 'system_config', - 'fail_fast']}}) + c["subject_id"] = subject_id + + set_up_logger( + f"{subject_id}_expectedOutputs", + filename=f'{bidsier_prefix(c["subject_id"])}_' 'expectedOutputs.yml', + level="info", + log_dir=log_dir, + mock=True, + overwrite_existing=True, + ) + if c.pipeline_setup["Debugging"]["verbose"]: + set_up_logger("engine", level="debug", log_dir=log_dir, mock=True) + + config.update_config( + { + "logging": { + "log_directory": log_dir, + "log_to_file": bool( + getattr(c.pipeline_setup["log_directory"], "run_logging", True) + ), + }, + "execution": { + "crashfile_format": "txt", + "resource_monitor_frequency": 0.2, + "stop_on_first_crash": c[ + "pipeline_setup", "system_config", "fail_fast" + ], + }, + } + ) config.enable_resource_monitor() logging.update_logging(config) @@ -304,50 +303,56 @@ def run_workflow(sub_dict, c, run, pipeline_timing_info=None, p_name=None, # Check pipeline config resources ( - sub_mem_gb, num_cores_per_sub, num_ants_cores, num_omp_cores + sub_mem_gb, + num_cores_per_sub, + num_ants_cores, + num_omp_cores, ) = check_config_resources(c) if not plugin_args: plugin_args = {} - plugin_args['memory_gb'] = sub_mem_gb - plugin_args['n_procs'] = num_cores_per_sub - plugin_args['raise_insufficient'] = c['pipeline_setup', 'system_config', - 'raise_insufficient'] - plugin_args['status_callback'] = log_nodes_cb + plugin_args["memory_gb"] = sub_mem_gb + plugin_args["n_procs"] = num_cores_per_sub + plugin_args["raise_insufficient"] = c[ + "pipeline_setup", "system_config", "raise_insufficient" + ] + plugin_args["status_callback"] = log_nodes_cb # perhaps in future allow user to set threads maximum # this is for centrality mostly # import mkl - os.environ['OMP_NUM_THREADS'] = str(num_omp_cores) - os.environ['MKL_NUM_THREADS'] = '1' # str(num_cores_per_sub) - os.environ['ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS'] = str(num_ants_cores) + os.environ["OMP_NUM_THREADS"] = str(num_omp_cores) + os.environ["MKL_NUM_THREADS"] = "1" # str(num_cores_per_sub) + os.environ["ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS"] = str(num_ants_cores) # TODO: TEMPORARY # TODO: solve the UNet model hanging issue during MultiProc - if "UNet" in c.anatomical_preproc['brain_extraction']['using']: - c.pipeline_setup['system_config']['max_cores_per_participant'] = 1 - logger.info("\n\n[!] LOCKING CPUs PER PARTICIPANT TO 1 FOR U-NET " - "MODEL.\n\nThis is a temporary measure due to a known " - "issue preventing Nipype's parallelization from running " - "U-Net properly.\n\n") + if "UNet" in c.anatomical_preproc["brain_extraction"]["using"]: + c.pipeline_setup["system_config"]["max_cores_per_participant"] = 1 + logger.info( + "\n\n[!] LOCKING CPUs PER PARTICIPANT TO 1 FOR U-NET " + "MODEL.\n\nThis is a temporary measure due to a known " + "issue preventing Nipype's parallelization from running " + "U-Net properly.\n\n" + ) # calculate maximum potential use of cores according to current pipeline # configuration max_core_usage = int( - c.pipeline_setup['system_config']['max_cores_per_participant']) * \ - int(c.pipeline_setup['system_config'][ - 'num_participants_at_once']) + c.pipeline_setup["system_config"]["max_cores_per_participant"] + ) * int(c.pipeline_setup["system_config"]["num_participants_at_once"]) try: - creds_path = sub_dict['creds_path'] - if creds_path and 'none' not in creds_path.lower(): + creds_path = sub_dict["creds_path"] + if creds_path and "none" not in creds_path.lower(): if os.path.exists(creds_path): input_creds_path = os.path.abspath(creds_path) else: - err_msg = 'Credentials path: "%s" for subject "%s" was not ' \ - 'found. Check this path and try again.' % ( - creds_path, subject_id) + err_msg = ( + 'Credentials path: "%s" for subject "%s" was not ' + "found. Check this path and try again." % (creds_path, subject_id) + ) raise Exception(err_msg) else: input_creds_path = None @@ -356,8 +361,7 @@ def run_workflow(sub_dict, c, run, pipeline_timing_info=None, p_name=None, # TODO enforce value with schema validation try: - encrypt_data = bool( - config.pipeline_setup['Amazon-AWS']['s3_encryption']) + encrypt_data = bool(config.pipeline_setup["Amazon-AWS"]["s3_encryption"]) except: encrypt_data = False @@ -379,7 +383,7 @@ def run_workflow(sub_dict, c, run, pipeline_timing_info=None, p_name=None, Setting ANTS/ITK thread usage to {ants_threads} Maximum potential number of cores that might be used during this run: {max_cores} {random_seed} -""" # noqa: E501 +""" execution_info = """ @@ -394,101 +398,115 @@ def run_workflow(sub_dict, c, run, pipeline_timing_info=None, p_name=None, System time of start: {run_start} System time of completion: {run_finish} {output_check} -""" # noqa: E501 - - logger.info('%s', information.format( - run_command=' '.join(['run', *sys.argv[1:]]), - cpac_version=CPAC.__version__, - dependency_versions=version_report().replace('\n', '\n '), - cores=c.pipeline_setup['system_config']['max_cores_per_participant'], - participants=c.pipeline_setup['system_config'][ - 'num_participants_at_once'], - omp_threads=c.pipeline_setup['system_config']['num_OMP_threads'], - ants_threads=c.pipeline_setup['system_config']['num_ants_threads'], - max_cores=max_core_usage, - random_seed=( - ' Random seed: %s' % - c.pipeline_setup['system_config']['random_seed']) if - c.pipeline_setup['system_config']['random_seed'] is not None else '', - license_notice=CPAC.license_notice.replace('\n', '\n '))) +""" + + logger.info( + "%s", + information.format( + run_command=" ".join(["run", *sys.argv[1:]]), + cpac_version=CPAC.__version__, + dependency_versions=version_report().replace("\n", "\n "), + cores=c.pipeline_setup["system_config"]["max_cores_per_participant"], + participants=c.pipeline_setup["system_config"]["num_participants_at_once"], + omp_threads=c.pipeline_setup["system_config"]["num_OMP_threads"], + ants_threads=c.pipeline_setup["system_config"]["num_ants_threads"], + max_cores=max_core_usage, + random_seed=( + " Random seed: %s" % c.pipeline_setup["system_config"]["random_seed"] + ) + if c.pipeline_setup["system_config"]["random_seed"] is not None + else "", + license_notice=CPAC.license_notice.replace("\n", "\n "), + ), + ) subject_info = {} - subject_info['subject_id'] = subject_id - subject_info['start_time'] = pipeline_start_time + subject_info["subject_id"] = subject_id + subject_info["start_time"] = pipeline_start_time - check_centrality_degree = c.network_centrality['run'] and \ - (len(c.network_centrality['degree_centrality'][ - 'weight_options']) != 0 or \ - len(c.network_centrality[ - 'eigenvector_centrality'][ - 'weight_options']) != 0) + check_centrality_degree = c.network_centrality["run"] and ( + len(c.network_centrality["degree_centrality"]["weight_options"]) != 0 + or len(c.network_centrality["eigenvector_centrality"]["weight_options"]) != 0 + ) - check_centrality_lfcd = c.network_centrality['run'] and \ - len(c.network_centrality[ - 'local_functional_connectivity_density'][ - 'weight_options']) != 0 + check_centrality_lfcd = ( + c.network_centrality["run"] + and len( + c.network_centrality["local_functional_connectivity_density"][ + "weight_options" + ] + ) + != 0 + ) if not test_config: # Check system dependencies - check_ica_aroma = c.nuisance_corrections['1-ICA-AROMA']['run'] + check_ica_aroma = c.nuisance_corrections["1-ICA-AROMA"]["run"] if isinstance(check_ica_aroma, list): check_ica_aroma = True in check_ica_aroma - check_system_deps(check_ants='ANTS' in c.registration_workflows[ - 'anatomical_registration']['registration']['using'], - check_ica_aroma=check_ica_aroma, - check_centrality_degree=check_centrality_degree, - check_centrality_lfcd=check_centrality_lfcd) + check_system_deps( + check_ants="ANTS" + in c.registration_workflows["anatomical_registration"]["registration"][ + "using" + ], + check_ica_aroma=check_ica_aroma, + check_centrality_degree=check_centrality_degree, + check_centrality_lfcd=check_centrality_lfcd, + ) # absolute paths of the dirs - c.pipeline_setup['working_directory']['path'] = os.path.join( - os.path.abspath(c.pipeline_setup['working_directory']['path']), - p_name) - if 's3://' not in c.pipeline_setup['output_directory']['path']: - c.pipeline_setup['output_directory']['path'] = os.path.abspath( - c.pipeline_setup['output_directory']['path']) - - if c.pipeline_setup['system_config']['random_seed'] is not None: + c.pipeline_setup["working_directory"]["path"] = os.path.join( + os.path.abspath(c.pipeline_setup["working_directory"]["path"]), p_name + ) + if "s3://" not in c.pipeline_setup["output_directory"]["path"]: + c.pipeline_setup["output_directory"]["path"] = os.path.abspath( + c.pipeline_setup["output_directory"]["path"] + ) + + if c.pipeline_setup["system_config"]["random_seed"] is not None: set_up_random_state_logger(log_dir) try: - workflow = build_workflow( - subject_id, sub_dict, c, p_name, num_ants_cores - ) + workflow = build_workflow(subject_id, sub_dict, c, p_name, num_ants_cores) except Exception as exception: - logger.exception('Building workflow failed') + logger.exception("Building workflow failed") raise exception - wf_graph = c['pipeline_setup', 'log_directory', 'graphviz', - 'entire_workflow'] - if wf_graph.get('generate'): - for graph2use in wf_graph.get('graph2use'): - dotfilename = os.path.join(log_dir, f'{p_name}_{graph2use}.dot') - for graph_format in wf_graph.get('format'): + wf_graph = c["pipeline_setup", "log_directory", "graphviz", "entire_workflow"] + if wf_graph.get("generate"): + for graph2use in wf_graph.get("graph2use"): + dotfilename = os.path.join(log_dir, f"{p_name}_{graph2use}.dot") + for graph_format in wf_graph.get("format"): try: - workflow.write_graph(dotfilename=dotfilename, - graph2use=graph2use, - format=graph_format, - simple_form=wf_graph.get( - 'simple_form', True)) + workflow.write_graph( + dotfilename=dotfilename, + graph2use=graph2use, + format=graph_format, + simple_form=wf_graph.get("simple_form", True), + ) except Exception as exception: - raise RuntimeError(f'Failed to visualize {p_name} (' - f'{graph2use}, {graph_format})' - ) from exception + raise RuntimeError( + f"Failed to visualize {p_name} (" + f"{graph2use}, {graph_format})" + ) from exception - workflow_meta = WorkflowJSONMeta(pipeline_name=p_name, stage='pre') + workflow_meta = WorkflowJSONMeta(pipeline_name=p_name, stage="pre") save_workflow_json( filename=os.path.join(log_dir, workflow_meta.filename()), workflow=workflow, meta=workflow_meta, - custom_serializer=cpac_flowdump_serializer + custom_serializer=cpac_flowdump_serializer, ) if test_config: - logger.info('This has been a test of the pipeline configuration ' - 'file, the pipeline was built successfully, but was ' - 'not run') + logger.info( + "This has been a test of the pipeline configuration " + "file, the pipeline was built successfully, but was " + "not run" + ) else: working_dir = os.path.join( - c.pipeline_setup['working_directory']['path'], workflow.name) + c.pipeline_setup["working_directory"]["path"], workflow.name + ) # if c.write_debugging_outputs: # with open(os.path.join(working_dir, 'resource_pool.pkl'), 'wb') as f: @@ -506,8 +524,9 @@ def run_workflow(sub_dict, c, run, pipeline_timing_info=None, p_name=None, # else: # shutil.rmtree(f) - if hasattr(c, 'trim') and c.trim: - logger.warning(""" + if hasattr(c, "trim") and c.trim: + logger.warning( + """ Trimming is an experimental feature, and if used wrongly, it can lead to unreproducible results. It is useful for performance optimization, but only if used correctly. @@ -518,11 +537,12 @@ def run_workflow(sub_dict, c, run, pipeline_timing_info=None, p_name=None, - Your softwares versions has not changed; - Your C-PAC version has not changed; - You do not have access to the working directory. -""") +""" + ) workflow, _ = the_trimmer( workflow, - output_dir=c.pipeline_setup['output_directory']['path'], + output_dir=c.pipeline_setup["output_directory"]["path"], s3_creds_path=input_creds_path, ) @@ -530,17 +550,17 @@ def run_workflow(sub_dict, c, run, pipeline_timing_info=None, p_name=None, workflow_result = None try: - subject_info['resource_pool'] = [] + subject_info["resource_pool"] = [] # for strat_no, strat in enumerate(strat_list): # strat_label = 'strat_%d' % strat_no # subject_info[strat_label] = strat.get_name() # subject_info['resource_pool'].append(strat.get_resource_pool()) - subject_info['status'] = 'Running' + subject_info["status"] = "Running" # Create callback logger - cb_log_filename = os.path.join(log_dir, 'callback.log') + cb_log_filename = os.path.join(log_dir, "callback.log") try: if not os.path.exists(os.path.dirname(cb_log_filename)): @@ -549,30 +569,30 @@ def run_workflow(sub_dict, c, run, pipeline_timing_info=None, p_name=None, pass # Add handler to callback log file - set_up_logger('callback', cb_log_filename, 'debug', log_dir, - mock=True) + set_up_logger("callback", cb_log_filename, "debug", log_dir, mock=True) # Log initial information from all the nodes log_nodes_initial(workflow) # Add status callback function that writes in callback log - nipype_version = REQUIREMENTS['nipype'] + nipype_version = REQUIREMENTS["nipype"] if nipype.__version__ != nipype_version: - logger.warning('This version of Nipype may not be compatible ' - f'with CPAC v{CPAC.__version__}, please ' - f'install Nipype version {nipype_version}\n') + logger.warning( + "This version of Nipype may not be compatible " + f"with CPAC v{CPAC.__version__}, please " + f"install Nipype version {nipype_version}\n" + ) - if plugin_args['n_procs'] == 1: - plugin = 'Linear' - if not plugin or plugin == 'LegacyMultiProc': + if plugin_args["n_procs"] == 1: + plugin = "Linear" + if not plugin or plugin == "LegacyMultiProc": plugin = LegacyMultiProcPlugin(plugin_args) - elif plugin == 'MultiProc': + elif plugin == "MultiProc": plugin = MultiProcPlugin(plugin_args) try: # Actually run the pipeline now, for the current subject - workflow_result = workflow.run(plugin=plugin, - plugin_args=plugin_args) + workflow_result = workflow.run(plugin=plugin, plugin_args=plugin_args) except UnicodeDecodeError: raise EnvironmentError( "C-PAC migrated from Python 2 to Python 3 in v1.6.2 (see " @@ -599,18 +619,17 @@ def run_workflow(sub_dict, c, run, pipeline_timing_info=None, p_name=None, # c.PyPEER['minimal_nuisance_correction']['peer_scrub'], c.PyPEER['minimal_nuisance_correction']['scrub_thresh']) # Dump subject info pickle file to subject log dir - subject_info['status'] = 'Completed' + subject_info["status"] = "Completed" subject_info_file = os.path.join( - log_dir, 'subject_info_%s.pkl' % subject_id + log_dir, "subject_info_%s.pkl" % subject_id ) - with open(subject_info_file, 'wb') as info: + with open(subject_info_file, "wb") as info: pickle.dump(list(subject_info), info) # have this check in case the user runs cpac_runner from terminal and # the timing parameter list is not supplied as usual by the GUI if pipeline_timing_info is not None: - # pipeline_timing_info list: # [0] - unique pipeline ID # [1] - pipeline start time stamp (first click of 'run' from GUI) @@ -624,7 +643,8 @@ def run_workflow(sub_dict, c, run, pipeline_timing_info=None, p_name=None, elapsed_time_data = [] elapsed_time_data.append( - int(((time.time() - pipeline_start_time) / 60))) + int(((time.time() - pipeline_start_time) / 60)) + ) # elapsedTimeBin list: # [0] - cumulative elapsed time (minutes) across all subjects @@ -637,72 +657,76 @@ def run_workflow(sub_dict, c, run, pipeline_timing_info=None, p_name=None, # code to delete .tmp file timing_temp_file_path = os.path.join( - c.pipeline_setup['log_directory']['path'], - '%s_pipeline_timing.tmp' % unique_pipeline_id) + c.pipeline_setup["log_directory"]["path"], + "%s_pipeline_timing.tmp" % unique_pipeline_id, + ) if not os.path.isfile(timing_temp_file_path): elapsedTimeBin = [] elapsedTimeBin.append(0) elapsedTimeBin.append(0) - with open(timing_temp_file_path, 'wb') as handle: + with open(timing_temp_file_path, "wb") as handle: pickle.dump(elapsedTimeBin, handle) - with open(timing_temp_file_path, 'rb') as handle: + with open(timing_temp_file_path, "rb") as handle: elapsedTimeBin = pickle.loads(handle.read()) elapsedTimeBin[0] = elapsedTimeBin[0] + elapsed_time_data[0] elapsedTimeBin[1] = elapsedTimeBin[1] + 1 - with open(timing_temp_file_path, 'wb') as handle: + with open(timing_temp_file_path, "wb") as handle: pickle.dump(elapsedTimeBin, handle) # this happens once the last subject has finished running! if elapsedTimeBin[1] == num_subjects: - pipelineTimeDict = {} - pipelineTimeDict['Pipeline'] = c.pipeline_setup[ - 'pipeline_name'] - pipelineTimeDict['Cores_Per_Subject'] = \ - c.pipeline_setup['system_config'][ - 'max_cores_per_participant'] - pipelineTimeDict['Simultaneous_Subjects'] = \ - c.pipeline_setup['system_config'][ - 'num_participants_at_once'] - pipelineTimeDict['Number_of_Subjects'] = num_subjects - pipelineTimeDict['Start_Time'] = pipeline_start_stamp - pipelineTimeDict['End_Time'] = strftime( - "%Y-%m-%d_%H:%M:%S") - pipelineTimeDict['Elapsed_Time_(minutes)'] = \ - elapsedTimeBin[0] - pipelineTimeDict['Status'] = 'Complete' + pipelineTimeDict["Pipeline"] = c.pipeline_setup["pipeline_name"] + pipelineTimeDict["Cores_Per_Subject"] = c.pipeline_setup[ + "system_config" + ]["max_cores_per_participant"] + pipelineTimeDict["Simultaneous_Subjects"] = c.pipeline_setup[ + "system_config" + ]["num_participants_at_once"] + pipelineTimeDict["Number_of_Subjects"] = num_subjects + pipelineTimeDict["Start_Time"] = pipeline_start_stamp + pipelineTimeDict["End_Time"] = strftime("%Y-%m-%d_%H:%M:%S") + pipelineTimeDict["Elapsed_Time_(minutes)"] = elapsedTimeBin[0] + pipelineTimeDict["Status"] = "Complete" gpaTimeFields = [ - 'Pipeline', 'Cores_Per_Subject', - 'Simultaneous_Subjects', - 'Number_of_Subjects', 'Start_Time', - 'End_Time', 'Elapsed_Time_(minutes)', - 'Status' + "Pipeline", + "Cores_Per_Subject", + "Simultaneous_Subjects", + "Number_of_Subjects", + "Start_Time", + "End_Time", + "Elapsed_Time_(minutes)", + "Status", ] timeHeader = dict(zip(gpaTimeFields, gpaTimeFields)) - with open(os.path.join( - c.pipeline_setup['log_directory']['path'], - 'cpac_individual_timing_%s.csv' % - c.pipeline_setup['pipeline_name'] - ), 'a') as timeCSV, open(os.path.join( - c.pipeline_setup['log_directory']['path'], - 'cpac_individual_timing_%s.csv' % - c.pipeline_setup['pipeline_name'] - ), 'r') as readTimeCSV: - - timeWriter = csv.DictWriter(timeCSV, - fieldnames=gpaTimeFields) + with open( + os.path.join( + c.pipeline_setup["log_directory"]["path"], + "cpac_individual_timing_%s.csv" + % c.pipeline_setup["pipeline_name"], + ), + "a", + ) as timeCSV, open( + os.path.join( + c.pipeline_setup["log_directory"]["path"], + "cpac_individual_timing_%s.csv" + % c.pipeline_setup["pipeline_name"], + ), + "r", + ) as readTimeCSV: + timeWriter = csv.DictWriter(timeCSV, fieldnames=gpaTimeFields) timeReader = csv.DictReader(readTimeCSV) headerExists = False for line in timeReader: - if 'Start_Time' in line: + if "Start_Time" in line: headerExists = True if headerExists is False: @@ -714,44 +738,44 @@ def run_workflow(sub_dict, c, run, pipeline_timing_info=None, p_name=None, os.remove(timing_temp_file_path) # Upload logs to s3 if s3_str in output directory - if c.pipeline_setup['output_directory'][ - 'path'].lower().startswith('s3://'): - + if c.pipeline_setup["output_directory"]["path"].lower().startswith("s3://"): try: # Store logs in s3 output director/logs/... s3_log_dir = os.path.join( - c.pipeline_setup['output_directory']['path'], - 'logs', - os.path.basename(log_dir) + c.pipeline_setup["output_directory"]["path"], + "logs", + os.path.basename(log_dir), ) - bucket_name = \ - c.pipeline_setup['output_directory']['path'].split('/')[2] - bucket = fetch_creds.return_bucket(creds_path, - bucket_name) + bucket_name = c.pipeline_setup["output_directory"]["path"].split( + "/" + )[2] + bucket = fetch_creds.return_bucket(creds_path, bucket_name) # Collect local log files local_log_files = [] for root, _, files in os.walk(log_dir): - local_log_files.extend([os.path.join(root, fil) - for fil in files]) + local_log_files.extend( + [os.path.join(root, fil) for fil in files] + ) # Form destination keys - s3_log_files = [loc.replace(log_dir, s3_log_dir) - for loc in local_log_files] + s3_log_files = [ + loc.replace(log_dir, s3_log_dir) for loc in local_log_files + ] # Upload logs - aws_utils.s3_upload(bucket, - (local_log_files, s3_log_files), - encrypt=encrypt_data) + aws_utils.s3_upload( + bucket, (local_log_files, s3_log_files), encrypt=encrypt_data + ) # Delete local log files for log_f in local_log_files: os.remove(log_f) except Exception as exc: - err_msg = ( - 'Unable to upload CPAC log files in: %s.\nError: %s') + err_msg = "Unable to upload CPAC log files in: %s.\nError: %s" logger.error(err_msg, log_dir, exc) except Exception: import traceback + traceback.print_exc() execution_info = """ @@ -768,47 +792,47 @@ def run_workflow(sub_dict, c, run, pipeline_timing_info=None, p_name=None, """ finally: - if workflow: if os.path.exists(cb_log_filename): - resource_report(cb_log_filename, - num_cores_per_sub, logger) - - logger.info('%s', execution_info.format( - workflow=workflow.name, - pipeline=c.pipeline_setup['pipeline_name'], - log_dir=c.pipeline_setup['log_directory']['path'], - elapsed=(time.time() - pipeline_start_time) / 60, - run_start=pipeline_start_datetime, - run_finish=strftime("%Y-%m-%d %H:%M:%S"), - output_check=check_outputs( - c.pipeline_setup['output_directory']['path'], - log_dir, c.pipeline_setup['pipeline_name'], - c['subject_id']) - )) + resource_report(cb_log_filename, num_cores_per_sub, logger) + + logger.info( + "%s", + execution_info.format( + workflow=workflow.name, + pipeline=c.pipeline_setup["pipeline_name"], + log_dir=c.pipeline_setup["log_directory"]["path"], + elapsed=(time.time() - pipeline_start_time) / 60, + run_start=pipeline_start_datetime, + run_finish=strftime("%Y-%m-%d %H:%M:%S"), + output_check=check_outputs( + c.pipeline_setup["output_directory"]["path"], + log_dir, + c.pipeline_setup["pipeline_name"], + c["subject_id"], + ), + ), + ) if workflow_result is not None: workflow_meta.stage = "post" save_workflow_json( - filename=os.path.join(log_dir, - workflow_meta.filename()), + filename=os.path.join(log_dir, workflow_meta.filename()), workflow=workflow_result, meta=workflow_meta, - custom_serializer=cpac_flowdump_serializer + custom_serializer=cpac_flowdump_serializer, ) # Remove working directory when done - if c.pipeline_setup['working_directory'][ - 'remove_working_dir']: + if c.pipeline_setup["working_directory"]["remove_working_dir"]: remove_workdir(working_dir) # Remove just .local from working directory else: - remove_workdir(os.path.join(os.environ["CPAC_WORKDIR"], - '.local')) + remove_workdir(os.path.join(os.environ["CPAC_WORKDIR"], ".local")) def remove_workdir(wdpath: str) -> None: - """Remove a given working directory if possible, warn if impossible + """Remove a given working directory if possible, warn if impossible. Parameters ---------- @@ -820,22 +844,21 @@ def remove_workdir(wdpath: str) -> None: logger.info("Removing working dir: %s", wdpath) shutil.rmtree(wdpath) except (FileNotFoundError, PermissionError): - logger.warning( - 'Could not remove working directory %s', wdpath) + logger.warning("Could not remove working directory %s", wdpath) def initialize_nipype_wf(cfg, sub_data_dct, name=""): - if name: - name = f'_{name}' + name = f"_{name}" - workflow_name = f'cpac{name}_{sub_data_dct["subject_id"]}_{sub_data_dct["unique_id"]}' + workflow_name = ( + f'cpac{name}_{sub_data_dct["subject_id"]}_{sub_data_dct["unique_id"]}' + ) wf = pe.Workflow(name=workflow_name) - wf.base_dir = cfg.pipeline_setup['working_directory']['path'] - wf.config['execution'] = { - 'hash_method': 'timestamp', - 'crashdump_dir': os.path.abspath(cfg.pipeline_setup['log_directory'][ - 'path']) + wf.base_dir = cfg.pipeline_setup["working_directory"]["path"] + wf.config["execution"] = { + "hash_method": "timestamp", + "crashdump_dir": os.path.abspath(cfg.pipeline_setup["log_directory"]["path"]), } return wf @@ -848,15 +871,11 @@ def load_cpac_pipe_config(pipe_config): if not os.path.exists(config_file): raise IOError else: - cfg = Configuration(yaml.safe_load(open(config_file, 'r'))) + cfg = Configuration(yaml.safe_load(open(config_file, "r"))) except IOError: - print("config file %s doesn't exist" % config_file) raise except yaml.parser.ParserError as e: - error_detail = "\"%s\" at line %d" % ( - e.problem, - e.problem_mark.line - ) + error_detail = '"%s" at line %d' % (e.problem, e.problem_mark.line) raise Exception( "Error parsing config file: {0}\n\n" "Error details:\n" @@ -874,43 +893,47 @@ def load_cpac_pipe_config(pipe_config): def build_anat_preproc_stack(rpool, cfg, pipeline_blocks=None): - if not pipeline_blocks: pipeline_blocks = [] # T1w Anatomical Preprocessing - if not rpool.check_rpool('desc-reorient_T1w') and \ - not rpool.check_rpool('desc-preproc_T1w'): - anat_init_blocks = [ - anatomical_init - ] + if not rpool.check_rpool("desc-reorient_T1w") and not rpool.check_rpool( + "desc-preproc_T1w" + ): + anat_init_blocks = [anatomical_init] pipeline_blocks += anat_init_blocks - - using_brain_extraction = cfg.anatomical_preproc['brain_extraction']['using'] - if not rpool.check_rpool('freesurfer-subject-dir') and 'FreeSurfer-ABCD' not in using_brain_extraction: - pipeline_blocks += [freesurfer_reconall] # includes postproc + using_brain_extraction = cfg.anatomical_preproc["brain_extraction"]["using"] - if not rpool.check_rpool('desc-preproc_T1w'): + if ( + not rpool.check_rpool("freesurfer-subject-dir") + and "FreeSurfer-ABCD" not in using_brain_extraction + ): + pipeline_blocks += [freesurfer_reconall] # includes postproc + if not rpool.check_rpool("desc-preproc_T1w"): # brain masking for ACPC alignment - if cfg.anatomical_preproc['acpc_alignment']['acpc_target'] == 'brain': - acpc_blocks = [ - [brain_mask_acpc_afni, - brain_mask_acpc_fsl, - brain_mask_acpc_niworkflows_ants, - brain_mask_acpc_unet, - brain_mask_acpc_freesurfer_abcd, - brain_mask_acpc_freesurfer, - brain_mask_acpc_freesurfer_fsl_tight, - brain_mask_acpc_freesurfer_fsl_loose], - acpc_align_brain_with_mask, - brain_extraction_temp, - acpc_align_brain - ] - elif cfg.anatomical_preproc['acpc_alignment']['acpc_target'] == 'whole-head': - if (rpool.check_rpool('space-T1w_desc-brain_mask') and \ - cfg.anatomical_preproc['acpc_alignment']['align_brain_mask']): + if cfg.anatomical_preproc["acpc_alignment"]["acpc_target"] == "brain": + acpc_blocks = [ + [ + brain_mask_acpc_afni, + brain_mask_acpc_fsl, + brain_mask_acpc_niworkflows_ants, + brain_mask_acpc_unet, + brain_mask_acpc_freesurfer_abcd, + brain_mask_acpc_freesurfer, + brain_mask_acpc_freesurfer_fsl_tight, + brain_mask_acpc_freesurfer_fsl_loose, + ], + acpc_align_brain_with_mask, + brain_extraction_temp, + acpc_align_brain, + ] + elif cfg.anatomical_preproc["acpc_alignment"]["acpc_target"] == "whole-head": + if ( + rpool.check_rpool("space-T1w_desc-brain_mask") + and cfg.anatomical_preproc["acpc_alignment"]["align_brain_mask"] + ): acpc_blocks = [ acpc_align_head_with_mask # outputs space-T1w_desc-brain_mask for later - keep the mask (the user provided) @@ -919,15 +942,15 @@ def build_anat_preproc_stack(rpool, cfg, pipeline_blocks=None): acpc_blocks = [ acpc_align_head # does not output nor generate a mask ] - anat_preproc_blocks = [ - (non_local_means, ('T1w', ['desc-preproc_T1w', - 'desc-reorient_T1w', - 'T1w'])), - n4_bias_correction + ( + non_local_means, + ("T1w", ["desc-preproc_T1w", "desc-reorient_T1w", "T1w"]), + ), + n4_bias_correction, ] - if cfg.anatomical_preproc['acpc_alignment']['run_before_preproc']: + if cfg.anatomical_preproc["acpc_alignment"]["run_before_preproc"]: anat_blocks = acpc_blocks + anat_preproc_blocks else: anat_blocks = anat_preproc_blocks + acpc_blocks @@ -936,42 +959,44 @@ def build_anat_preproc_stack(rpool, cfg, pipeline_blocks=None): pipeline_blocks += [freesurfer_abcd_preproc] - if not rpool.check_rpool('freesurfer-subject-dir') and 'FreeSurfer-ABCD' in using_brain_extraction: + if ( + not rpool.check_rpool("freesurfer-subject-dir") + and "FreeSurfer-ABCD" in using_brain_extraction + ): pipeline_blocks += [freesurfer_reconall] # includes postproc # Anatomical T1 brain masking anat_brain_mask_blocks = [ - [brain_mask_afni, - brain_mask_fsl, - brain_mask_niworkflows_ants, - brain_mask_unet, - brain_mask_freesurfer_abcd, - brain_mask_freesurfer, - brain_mask_freesurfer_fsl_tight, - brain_mask_freesurfer_fsl_loose] + [ + brain_mask_afni, + brain_mask_fsl, + brain_mask_niworkflows_ants, + brain_mask_unet, + brain_mask_freesurfer_abcd, + brain_mask_freesurfer, + brain_mask_freesurfer_fsl_tight, + brain_mask_freesurfer_fsl_loose, + ] ] pipeline_blocks += anat_brain_mask_blocks # T2w Anatomical Preprocessing - if rpool.check_rpool('T2w'): - if not rpool.check_rpool('desc-reorient_T2w'): - anat_init_blocks_T2 = [ - anatomical_init_T2 - ] + if rpool.check_rpool("T2w"): + if not rpool.check_rpool("desc-reorient_T2w"): + anat_init_blocks_T2 = [anatomical_init_T2] pipeline_blocks += anat_init_blocks_T2 # TODO: T2 freesurfer_preproc? # pipeline_blocks += [freesurfer_preproc] - if not rpool.check_rpool('desc-preproc_T2w'): - + if not rpool.check_rpool("desc-preproc_T2w"): # brain masking for ACPC alignment - if cfg.anatomical_preproc['acpc_alignment']['acpc_target'] == 'brain': - if rpool.check_rpool('space-T2w_desc-brain_mask'): + if cfg.anatomical_preproc["acpc_alignment"]["acpc_target"] == "brain": + if rpool.check_rpool("space-T2w_desc-brain_mask"): acpc_blocks_T2 = [ brain_extraction_temp_T2, - acpc_align_brain_with_mask_T2 + acpc_align_brain_with_mask_T2, # outputs space-T2w_desc-brain_mask for later - keep the mask (the user provided) ] else: @@ -979,11 +1004,12 @@ def build_anat_preproc_stack(rpool, cfg, pipeline_blocks=None): brain_mask_acpc_T2, # we don't want these masks to be used later, only used in brain_extraction_temp_T2 brain_extraction_temp_T2, - acpc_align_brain_T2 + acpc_align_brain_T2, ] - elif cfg.anatomical_preproc['acpc_alignment'][ - 'acpc_target'] == 'whole-head': - if rpool.check_rpool('space-T2w_desc-brain_mask'): + elif ( + cfg.anatomical_preproc["acpc_alignment"]["acpc_target"] == "whole-head" + ): + if rpool.check_rpool("space-T2w_desc-brain_mask"): acpc_blocks_T2 = [ acpc_align_head_with_mask_T2 # outputs space-T2w_desc-brain_mask for later - keep the mask (the user provided) @@ -997,9 +1023,9 @@ def build_anat_preproc_stack(rpool, cfg, pipeline_blocks=None): registration_T2w_to_T1w, non_local_means_T2, n4_bias_correction_T2, - t1t2_bias_correction + t1t2_bias_correction, ] - if cfg.anatomical_preproc['acpc_alignment']['run_before_preproc']: + if cfg.anatomical_preproc["acpc_alignment"]["run_before_preproc"]: anat_blocks_T2 = acpc_blocks_T2 + anat_preproc_blocks_T2 else: anat_blocks_T2 = anat_preproc_blocks_T2 + acpc_blocks_T2 @@ -1007,85 +1033,92 @@ def build_anat_preproc_stack(rpool, cfg, pipeline_blocks=None): pipeline_blocks += anat_blocks_T2 # Anatomical T1 brain extraction - if not rpool.check_rpool('desc-brain_T1w'): - anat_brain_blocks = [ - brain_extraction - ] + if not rpool.check_rpool("desc-brain_T1w"): + anat_brain_blocks = [brain_extraction] pipeline_blocks += anat_brain_blocks # T2 brain masking - if not rpool.check_rpool('space-T2w_desc-brain_mask'): - anat_brain_mask_blocks_T2 = [ - brain_mask_T2 - ] + if not rpool.check_rpool("space-T2w_desc-brain_mask"): + anat_brain_mask_blocks_T2 = [brain_mask_T2] pipeline_blocks += anat_brain_mask_blocks_T2 - if not rpool.check_rpool('desc-brain_T2w'): - anat_brain_blocks_T2 = [ - brain_extraction_T2 - ] + if not rpool.check_rpool("desc-brain_T2w"): + anat_brain_blocks_T2 = [brain_extraction_T2] pipeline_blocks += anat_brain_blocks_T2 return pipeline_blocks def build_T1w_registration_stack(rpool, cfg, pipeline_blocks=None): - if not pipeline_blocks: pipeline_blocks = [] reg_blocks = [] - if not rpool.check_rpool('from-T1w_to-template_mode-image_xfm'): + if not rpool.check_rpool("from-T1w_to-template_mode-image_xfm"): reg_blocks = [ [register_ANTs_anat_to_template, register_FSL_anat_to_template], overwrite_transform_anat_to_template, warp_wholeheadT1_to_template, - warp_T1mask_to_template + warp_T1mask_to_template, ] - if not rpool.check_rpool('desc-restore-brain_T1w'): + if not rpool.check_rpool("desc-restore-brain_T1w"): reg_blocks.append(correct_restore_brain_intensity_abcd) - if cfg.voxel_mirrored_homotopic_connectivity['run']: - if not rpool.check_rpool('from-T1w_to-symtemplate_mode-image_xfm'): - reg_blocks.append([register_symmetric_ANTs_anat_to_template, - register_symmetric_FSL_anat_to_template]) + if cfg.voxel_mirrored_homotopic_connectivity["run"]: + if not rpool.check_rpool("from-T1w_to-symtemplate_mode-image_xfm"): + reg_blocks.append( + [ + register_symmetric_ANTs_anat_to_template, + register_symmetric_FSL_anat_to_template, + ] + ) pipeline_blocks += reg_blocks return pipeline_blocks def build_segmentation_stack(rpool, cfg, pipeline_blocks=None): - if not pipeline_blocks: pipeline_blocks = [] - if not rpool.check_rpool('label-CSF_mask') or \ - not rpool.check_rpool('label-WM_mask'): + if not rpool.check_rpool("label-CSF_mask") or not rpool.check_rpool( + "label-WM_mask" + ): seg_blocks = [ - [tissue_seg_fsl_fast, - tissue_seg_ants_prior, - tissue_seg_freesurfer] + [tissue_seg_fsl_fast, tissue_seg_ants_prior, tissue_seg_freesurfer] ] - if 'T1_Template' in cfg.segmentation['tissue_segmentation'][ - 'Template_Based']['template_for_segmentation']: + if ( + "T1_Template" + in cfg.segmentation["tissue_segmentation"]["Template_Based"][ + "template_for_segmentation" + ] + ): seg_blocks.append(tissue_seg_T1_template_based) - if 'EPI_Template' in cfg.segmentation['tissue_segmentation'][ - 'Template_Based']['template_for_segmentation']: + if ( + "EPI_Template" + in cfg.segmentation["tissue_segmentation"]["Template_Based"][ + "template_for_segmentation" + ] + ): seg_blocks.append(tissue_seg_EPI_template_based) pipeline_blocks += seg_blocks - if cfg.registration_workflows['anatomical_registration']['run'] and 'T1_Template' in cfg.segmentation[ - 'tissue_segmentation']['Template_Based']['template_for_segmentation']: + if ( + cfg.registration_workflows["anatomical_registration"]["run"] + and "T1_Template" + in cfg.segmentation["tissue_segmentation"]["Template_Based"][ + "template_for_segmentation" + ] + ): pipeline_blocks.append(warp_tissuemask_to_T1template) - return pipeline_blocks def list_blocks(pipeline_blocks, indent=None): - """Function to list node blocks line by line + """Function to list node blocks line by line. Parameters ---------- @@ -1098,54 +1131,71 @@ def list_blocks(pipeline_blocks, indent=None): ------- str """ - blockstring = yaml.dump([ - getattr(block, '__name__', getattr(block, 'name', yaml.safe_load( - list_blocks(list(block))) if - isinstance(block, (tuple, list, set)) else str(block)) - ) for block in pipeline_blocks]) + blockstring = yaml.dump( + [ + getattr( + block, + "__name__", + getattr( + block, + "name", + yaml.safe_load(list_blocks(list(block))) + if isinstance(block, (tuple, list, set)) + else str(block), + ), + ) + for block in pipeline_blocks + ] + ) if isinstance(indent, int): - blockstring = '\n'.join([ - '\t' + ' ' * indent + line.replace('- - ', '- ') for - line in blockstring.split('\n')]) + blockstring = "\n".join( + [ + "\t" + " " * indent + line.replace("- - ", "- ") + for line in blockstring.split("\n") + ] + ) return blockstring def connect_pipeline(wf, cfg, rpool, pipeline_blocks): - logger.info('\n'.join([ - 'Connecting pipeline blocks:', - list_blocks(pipeline_blocks, indent=1)])) + logger.info( + "\n".join( + ["Connecting pipeline blocks:", list_blocks(pipeline_blocks, indent=1)] + ) + ) previous_nb = None for block in pipeline_blocks: try: - nb = NodeBlock(block, debug=cfg['pipeline_setup', 'Debugging', - 'verbose']) + nb = NodeBlock(block, debug=cfg["pipeline_setup", "Debugging", "verbose"]) wf = nb.connect_block(wf, cfg, rpool) except LookupError as e: - if nb.name == 'freesurfer_postproc': + if nb.name == "freesurfer_postproc": logger.warning(WARNING_FREESURFER_OFF_WITH_DATA) - LOGTAIL['warnings'].append(WARNING_FREESURFER_OFF_WITH_DATA) + LOGTAIL["warnings"].append(WARNING_FREESURFER_OFF_WITH_DATA) continue previous_nb_str = ( - f"after node block '{previous_nb.get_name()}':" - ) if previous_nb else 'at beginning:' + (f"after node block '{previous_nb.get_name()}':") + if previous_nb + else "at beginning:" + ) # Alert user to block that raises error if isinstance(block, list): node_block_names = str([NodeBlock(b).get_name() for b in block]) e.args = ( - f'When trying to connect one of the node blocks ' + f"When trying to connect one of the node blocks " f"{node_block_names} " f"to workflow '{wf}' {previous_nb_str} {e.args[0]}", ) else: node_block_names = NodeBlock(block).get_name() e.args = ( - f'When trying to connect node block ' + f"When trying to connect node block " f"'{node_block_names}' " f"to workflow '{wf}' {previous_nb_str} {e.args[0]}", ) - if cfg.pipeline_setup['Debugging']['verbose']: - verbose_logger = getLogger('engine') + if cfg.pipeline_setup["Debugging"]["verbose"]: + verbose_logger = getLogger("engine") verbose_logger.debug(e.args[0]) verbose_logger.debug(rpool) raise @@ -1154,8 +1204,7 @@ def connect_pipeline(wf, cfg, rpool, pipeline_blocks): return wf -def build_workflow(subject_id, sub_dict, cfg, pipeline_name=None, - num_ants_cores=1): +def build_workflow(subject_id, sub_dict, cfg, pipeline_name=None, num_ants_cores=1): from CPAC.utils.datasource import gather_extraction_maps # Workflow setup @@ -1163,81 +1212,83 @@ def build_workflow(subject_id, sub_dict, cfg, pipeline_name=None, # Extract credentials path if it exists try: - creds_path = sub_dict['creds_path'] - if creds_path and 'none' not in creds_path.lower(): + creds_path = sub_dict["creds_path"] + if creds_path and "none" not in creds_path.lower(): if os.path.exists(creds_path): input_creds_path = os.path.abspath(creds_path) else: - err_msg = 'Credentials path: "%s" for subject "%s" was not ' \ - 'found. Check this path and try again.' % ( - creds_path, subject_id) + err_msg = ( + 'Credentials path: "%s" for subject "%s" was not ' + "found. Check this path and try again." % (creds_path, subject_id) + ) raise Exception(err_msg) else: input_creds_path = None except KeyError: input_creds_path = None - cfg.pipeline_setup['input_creds_path'] = input_creds_path + cfg.pipeline_setup["input_creds_path"] = input_creds_path - """"""""""""""""""""""""""""""""""""""""""""""""""" + """""" """""" """""" """""" """""" """""" """""" """""" """ PREPROCESSING - """"""""""""""""""""""""""""""""""""""""""""""""""" + """ """""" """""" """""" """""" """""" """""" """""" """""" wf, rpool = initiate_rpool(wf, cfg, sub_dict) pipeline_blocks = build_anat_preproc_stack(rpool, cfg) # Anatomical to T1 template registration - pipeline_blocks = build_T1w_registration_stack(rpool, cfg, - pipeline_blocks) + pipeline_blocks = build_T1w_registration_stack(rpool, cfg, pipeline_blocks) # Anatomical tissue segmentation pipeline_blocks = build_segmentation_stack(rpool, cfg, pipeline_blocks) # Functional Preprocessing, including motion correction and BOLD masking - if cfg.functional_preproc['run']: - func_init_blocks = [ - func_reorient, - func_scaling, - func_truncate - ] - func_preproc_blocks = [ - func_despike, - func_slice_time - ] + if cfg.functional_preproc["run"]: + func_init_blocks = [func_reorient, func_scaling, func_truncate] + func_preproc_blocks = [func_despike, func_slice_time] - if not rpool.check_rpool('desc-mean_bold'): + if not rpool.check_rpool("desc-mean_bold"): func_preproc_blocks.append(func_mean) func_mask_blocks = [] - if not rpool.check_rpool('space-bold_desc-brain_mask'): + if not rpool.check_rpool("space-bold_desc-brain_mask"): func_mask_blocks = [ - [bold_mask_afni, bold_mask_fsl, bold_mask_fsl_afni, - bold_mask_anatomical_refined, bold_mask_anatomical_based, - bold_mask_anatomical_resampled, bold_mask_ccs], - bold_masking] + [ + bold_mask_afni, + bold_mask_fsl, + bold_mask_fsl_afni, + bold_mask_anatomical_refined, + bold_mask_anatomical_based, + bold_mask_anatomical_resampled, + bold_mask_ccs, + ], + bold_masking, + ] func_prep_blocks = [ calc_motion_stats, func_normalize, - [coregistration_prep_vol, - coregistration_prep_mean, - coregistration_prep_fmriprep] + [ + coregistration_prep_vol, + coregistration_prep_mean, + coregistration_prep_fmriprep, + ], ] # Distortion/Susceptibility Correction distcor_blocks = [] - if 'fmap' in sub_dict: - fmap_keys = sub_dict['fmap'] - if 'phasediff' in fmap_keys or 'phase1' in fmap_keys: - if 'magnitude' in fmap_keys or 'magnitude1' in fmap_keys: + if "fmap" in sub_dict: + fmap_keys = sub_dict["fmap"] + if "phasediff" in fmap_keys or "phase1" in fmap_keys: + if "magnitude" in fmap_keys or "magnitude1" in fmap_keys: distcor_blocks.append(distcor_phasediff_fsl_fugue) if len(fmap_keys) == 2: for key in fmap_keys: - if 'epi_' not in key: + if "epi_" not in key: break else: - distcor_blocks.append(distcor_blip_afni_qwarp) + distcor_blocks.append(distcor_blip_afni_qwarp) distcor_blocks.append(distcor_blip_fsl_topup) if distcor_blocks: @@ -1246,250 +1297,294 @@ def build_workflow(subject_id, sub_dict, cfg, pipeline_name=None, func_prep_blocks += distcor_blocks func_motion_blocks = [] - if not rpool.check_rpool('desc-movementParameters_motion'): - if cfg['functional_preproc']['motion_estimates_and_correction'][ - 'motion_estimates']['calculate_motion_first']: + if not rpool.check_rpool("desc-movementParameters_motion"): + if cfg["functional_preproc"]["motion_estimates_and_correction"][ + "motion_estimates" + ]["calculate_motion_first"]: func_motion_blocks = [ get_motion_ref, func_motion_estimates, - motion_estimate_filter + motion_estimate_filter, ] - func_blocks = func_init_blocks + func_motion_blocks + \ - func_preproc_blocks + [func_motion_correct_only] + \ - func_mask_blocks + func_prep_blocks + func_blocks = ( + func_init_blocks + + func_motion_blocks + + func_preproc_blocks + + [func_motion_correct_only] + + func_mask_blocks + + func_prep_blocks + ) else: func_motion_blocks = [ get_motion_ref, func_motion_correct, - motion_estimate_filter + motion_estimate_filter, ] - func_blocks = func_init_blocks + func_preproc_blocks + \ - func_motion_blocks + func_mask_blocks + \ - func_prep_blocks + func_blocks = ( + func_init_blocks + + func_preproc_blocks + + func_motion_blocks + + func_mask_blocks + + func_prep_blocks + ) else: - func_blocks = func_init_blocks + func_preproc_blocks + \ - func_motion_blocks + func_mask_blocks + \ - func_prep_blocks + func_blocks = ( + func_init_blocks + + func_preproc_blocks + + func_motion_blocks + + func_mask_blocks + + func_prep_blocks + ) pipeline_blocks += func_blocks # BOLD to T1 coregistration - if cfg.registration_workflows['functional_registration'][ - 'coregistration']['run'] and \ - (not rpool.check_rpool('space-T1w_sbref') or - not rpool.check_rpool('from-bold_to-T1w_mode-image_desc-linear_xfm')): - coreg_blocks = [ - coregistration - ] + if cfg.registration_workflows["functional_registration"]["coregistration"][ + "run" + ] and ( + not rpool.check_rpool("space-T1w_sbref") + or not rpool.check_rpool("from-bold_to-T1w_mode-image_desc-linear_xfm") + ): + coreg_blocks = [coregistration] pipeline_blocks += coreg_blocks # BOLD to EPI-template registration (no T1w involved) - if not rpool.check_rpool('space-EPItemplate_desc-brain_bold'): + if not rpool.check_rpool("space-EPItemplate_desc-brain_bold"): if coregistration not in pipeline_blocks: pipeline_blocks += [coregistration_prep_vol, coregistration_prep_mean] - EPI_reg_blocks = [ - [register_ANTs_EPI_to_template, register_FSL_EPI_to_template] - ] + EPI_reg_blocks = [[register_ANTs_EPI_to_template, register_FSL_EPI_to_template]] pipeline_blocks += EPI_reg_blocks - if (cfg['registration_workflows', 'functional_registration', - 'EPI_registration', 'run'] and - 'EPI_Template' in cfg['segmentation', 'tissue_segmentation', - 'Template_Based', 'template_for_segmentation']): + if ( + cfg[ + "registration_workflows", + "functional_registration", + "EPI_registration", + "run", + ] + and "EPI_Template" + in cfg[ + "segmentation", + "tissue_segmentation", + "Template_Based", + "template_for_segmentation", + ] + ): pipeline_blocks.append(warp_tissuemask_to_EPItemplate) # Generate the composite transform for BOLD-to-template for the T1 # anatomical template (the BOLD-to- EPI template is already created above) - if cfg.registration_workflows['functional_registration'][ - 'coregistration']['run' - ] and 'T1_template' in cfg.registration_workflows[ - 'functional_registration']['func_registration_to_template'][ - 'target_template']['using']: + if ( + cfg.registration_workflows["functional_registration"]["coregistration"]["run"] + and "T1_template" + in cfg.registration_workflows["functional_registration"][ + "func_registration_to_template" + ]["target_template"]["using"] + ): pipeline_blocks += [create_func_to_T1template_xfm] - if cfg.voxel_mirrored_homotopic_connectivity['run']: + if cfg.voxel_mirrored_homotopic_connectivity["run"]: pipeline_blocks += [create_func_to_T1template_symmetric_xfm] # Nuisance Correction - generate_only = True not in cfg['nuisance_corrections', - '2-nuisance_regression', 'run'] - if not rpool.check_rpool('desc-cleaned_bold'): - nuisance = [ICA_AROMA_ANTsreg, ICA_AROMA_FSLreg, - ICA_AROMA_ANTsEPIreg, ICA_AROMA_FSLEPIreg] - - nuisance_masks = [erode_mask_T1w, - erode_mask_CSF, - erode_mask_GM, - erode_mask_WM, - erode_mask_bold, - erode_mask_boldCSF, - erode_mask_boldGM, - erode_mask_boldWM] - nuisance += nuisance_masks + choose_nuisance_blocks(cfg, rpool, \ - generate_only) + generate_only = ( + True not in cfg["nuisance_corrections", "2-nuisance_regression", "run"] + ) + if not rpool.check_rpool("desc-cleaned_bold"): + nuisance = [ + ICA_AROMA_ANTsreg, + ICA_AROMA_FSLreg, + ICA_AROMA_ANTsEPIreg, + ICA_AROMA_FSLEPIreg, + ] + + nuisance_masks = [ + erode_mask_T1w, + erode_mask_CSF, + erode_mask_GM, + erode_mask_WM, + erode_mask_bold, + erode_mask_boldCSF, + erode_mask_boldGM, + erode_mask_boldWM, + ] + nuisance += nuisance_masks + choose_nuisance_blocks(cfg, rpool, generate_only) pipeline_blocks += nuisance pipeline_blocks.append(ingress_regressors) apply_func_warp = {} - _r_w_f_r = cfg.registration_workflows['functional_registration'] + _r_w_f_r = cfg.registration_workflows["functional_registration"] # Warp the functional time series to template space - apply_func_warp['T1'] = ( - _r_w_f_r['coregistration']['run'] and - _r_w_f_r['func_registration_to_template']['run']) - template_funcs = [ - 'space-template_desc-preproc_bold', - 'space-template_bold' - ] + apply_func_warp["T1"] = ( + _r_w_f_r["coregistration"]["run"] + and _r_w_f_r["func_registration_to_template"]["run"] + ) + template_funcs = ["space-template_desc-preproc_bold", "space-template_bold"] for func in template_funcs: if rpool.check_rpool(func): - apply_func_warp['T1'] = False - - target_space_nuis = cfg.nuisance_corrections['2-nuisance_regression'][ - 'space'] - target_space_alff = cfg.amplitude_low_frequency_fluctuation['target_space'] - target_space_reho = cfg.regional_homogeneity['target_space'] - - if apply_func_warp['T1']: - - ts_to_T1template_block = [apply_phasediff_to_timeseries_separately, - apply_blip_to_timeseries_separately, - warp_timeseries_to_T1template, - warp_timeseries_to_T1template_dcan_nhp] + apply_func_warp["T1"] = False + + target_space_nuis = cfg.nuisance_corrections["2-nuisance_regression"]["space"] + target_space_alff = cfg.amplitude_low_frequency_fluctuation["target_space"] + target_space_reho = cfg.regional_homogeneity["target_space"] + + if apply_func_warp["T1"]: + ts_to_T1template_block = [ + apply_phasediff_to_timeseries_separately, + apply_blip_to_timeseries_separately, + warp_timeseries_to_T1template, + warp_timeseries_to_T1template_dcan_nhp, + ] - if 'Template' in target_space_alff or 'Template' in target_space_reho: + if "Template" in target_space_alff or "Template" in target_space_reho: ts_to_T1template_block += [warp_timeseries_to_T1template_deriv] - if cfg.nuisance_corrections['2-nuisance_regression']['create_regressors']: - ts_to_T1template_block += [(warp_timeseries_to_T1template_abcd, ('desc-preproc_bold', 'bold'))] + if cfg.nuisance_corrections["2-nuisance_regression"]["create_regressors"]: + ts_to_T1template_block += [ + (warp_timeseries_to_T1template_abcd, ("desc-preproc_bold", "bold")) + ] ts_to_T1template_block.append(single_step_resample_timeseries_to_T1template) else: ts_to_T1template_block.append(warp_timeseries_to_T1template_abcd) ts_to_T1template_block.append(single_step_resample_timeseries_to_T1template) - pipeline_blocks += [ts_to_T1template_block, - warp_sbref_to_T1template] + pipeline_blocks += [ts_to_T1template_block, warp_sbref_to_T1template] - if not rpool.check_rpool('space-template_desc-bold_mask'): - pipeline_blocks += [warp_bold_mask_to_T1template, - warp_deriv_mask_to_T1template] + if not rpool.check_rpool("space-template_desc-bold_mask"): + pipeline_blocks += [warp_bold_mask_to_T1template, warp_deriv_mask_to_T1template] pipeline_blocks += [func_despike_template] - if 'Template' in target_space_alff and target_space_nuis == 'native': + if "Template" in target_space_alff and target_space_nuis == "native": pipeline_blocks += [warp_denoiseNofilt_to_T1template] - template = cfg.registration_workflows['functional_registration'][ - 'func_registration_to_template']['target_template']['using'] + template = cfg.registration_workflows["functional_registration"][ + "func_registration_to_template" + ]["target_template"]["using"] - if 'T1_template' in template: - apply_func_warp['EPI'] = (_r_w_f_r['coregistration']['run'] and - _r_w_f_r['func_registration_to_template' - ]['run_EPI']) + if "T1_template" in template: + apply_func_warp["EPI"] = ( + _r_w_f_r["coregistration"]["run"] + and _r_w_f_r["func_registration_to_template"]["run_EPI"] + ) else: - apply_func_warp['EPI'] = (_r_w_f_r['func_registration_to_template' - ]['run_EPI']) + apply_func_warp["EPI"] = _r_w_f_r["func_registration_to_template"]["run_EPI"] del _r_w_f_r template_funcs = [ - 'space-EPItemplate_desc-cleaned_bold', - 'space-EPItemplate_desc-brain_bold', - 'space-EPItemplate_desc-motion_bold', - 'space-EPItemplate_desc-preproc_bold', - 'space-EPItemplate_bold' + "space-EPItemplate_desc-cleaned_bold", + "space-EPItemplate_desc-brain_bold", + "space-EPItemplate_desc-motion_bold", + "space-EPItemplate_desc-preproc_bold", + "space-EPItemplate_bold", ] for func in template_funcs: if rpool.check_rpool(func): - apply_func_warp['EPI'] = False + apply_func_warp["EPI"] = False - if apply_func_warp['EPI']: - pipeline_blocks += [warp_timeseries_to_EPItemplate, - warp_bold_mean_to_EPItemplate] + if apply_func_warp["EPI"]: + pipeline_blocks += [ + warp_timeseries_to_EPItemplate, + warp_bold_mean_to_EPItemplate, + ] - if not rpool.check_rpool('space-EPItemplate_desc-bold_mask'): - pipeline_blocks += [warp_bold_mask_to_EPItemplate, - warp_deriv_mask_to_EPItemplate] + if not rpool.check_rpool("space-EPItemplate_desc-bold_mask"): + pipeline_blocks += [ + warp_bold_mask_to_EPItemplate, + warp_deriv_mask_to_EPItemplate, + ] # Template-space nuisance regression - nuisance_template = (cfg['nuisance_corrections', '2-nuisance_regression', - 'space'] == 'template') and (not generate_only) + nuisance_template = ( + cfg["nuisance_corrections", "2-nuisance_regression", "space"] == "template" + ) and (not generate_only) if nuisance_template: pipeline_blocks += [nuisance_regression_template] # pipeline_blocks += [(nuisance_regression_template, # ("desc-preproc_bold", "desc-stc_bold"))] # PostFreeSurfer and fMRISurface - if not rpool.check_rpool('space-fsLR_den-32k_bold.dtseries'): - + if not rpool.check_rpool("space-fsLR_den-32k_bold.dtseries"): pipeline_blocks += [surface_postproc] # Extractions and Derivatives tse_atlases, sca_atlases = gather_extraction_maps(cfg) - cfg.timeseries_extraction['tse_atlases'] = tse_atlases - cfg.seed_based_correlation_analysis['sca_atlases'] = sca_atlases + cfg.timeseries_extraction["tse_atlases"] = tse_atlases + cfg.seed_based_correlation_analysis["sca_atlases"] = sca_atlases - if not rpool.check_rpool('space-template_desc-Mean_timeseries') and \ - 'Avg' in tse_atlases: + if ( + not rpool.check_rpool("space-template_desc-Mean_timeseries") + and "Avg" in tse_atlases + ): pipeline_blocks += [timeseries_extraction_AVG] - if not rpool.check_rpool('desc-Voxel_timeseries') and \ - 'Voxel' in tse_atlases: + if not rpool.check_rpool("desc-Voxel_timeseries") and "Voxel" in tse_atlases: pipeline_blocks += [timeseries_extraction_Voxel] - if not rpool.check_rpool('desc-SpatReg_timeseries') and \ - 'SpatialReg' in tse_atlases: + if not rpool.check_rpool("desc-SpatReg_timeseries") and "SpatialReg" in tse_atlases: pipeline_blocks += [spatial_regression] - if not rpool.check_rpool('space-template_desc-MeanSCA_correlations') and \ - 'Avg' in sca_atlases: + if ( + not rpool.check_rpool("space-template_desc-MeanSCA_correlations") + and "Avg" in sca_atlases + ): pipeline_blocks += [SCA_AVG] - if not rpool.check_rpool('space-template_desc-DualReg_correlations') and \ - 'DualReg' in sca_atlases: + if ( + not rpool.check_rpool("space-template_desc-DualReg_correlations") + and "DualReg" in sca_atlases + ): pipeline_blocks += [dual_regression] - if not rpool.check_rpool('space-template_desc-MultReg_correlations') and \ - 'MultReg' in sca_atlases: + if ( + not rpool.check_rpool("space-template_desc-MultReg_correlations") + and "MultReg" in sca_atlases + ): pipeline_blocks += [multiple_regression] - if 'Native' in target_space_alff: - if not rpool.check_rpool('alff'): + if "Native" in target_space_alff: + if not rpool.check_rpool("alff"): pipeline_blocks += [alff_falff] - if 'Template' in target_space_alff: - if not rpool.check_rpool('space-template_alff'): + if "Template" in target_space_alff: + if not rpool.check_rpool("space-template_alff"): pipeline_blocks += [alff_falff_space_template] - if 'Native' in target_space_reho: - if not rpool.check_rpool('reho'): + if "Native" in target_space_reho: + if not rpool.check_rpool("reho"): pipeline_blocks += [reho] - if 'Template' in target_space_reho: - if not rpool.check_rpool('space-template_reho'): + if "Template" in target_space_reho: + if not rpool.check_rpool("space-template_reho"): pipeline_blocks += [reho_space_template] - if not rpool.check_rpool('vmhc'): - pipeline_blocks += [smooth_func_vmhc, - warp_timeseries_to_sym_template, - vmhc] + if not rpool.check_rpool("vmhc"): + pipeline_blocks += [smooth_func_vmhc, warp_timeseries_to_sym_template, vmhc] - if not rpool.check_rpool('centrality') and \ - any(cfg.network_centrality[option]['weight_options'] for - option in valid_options['centrality']['method_options']): + if not rpool.check_rpool("centrality") and any( + cfg.network_centrality[option]["weight_options"] + for option in valid_options["centrality"]["method_options"] + ): pipeline_blocks += [network_centrality] - if cfg.pipeline_setup['output_directory']['quality_control'][ - 'generate_xcpqc_files' + if cfg.pipeline_setup["output_directory"]["quality_control"][ + "generate_xcpqc_files" ]: pipeline_blocks += [qc_xcp] - if cfg.pipeline_setup['output_directory']['quality_control'][ - 'generate_quality_control_images' + if cfg.pipeline_setup["output_directory"]["quality_control"][ + "generate_quality_control_images" ]: - qc_stack, qc_montage_id_a, qc_montage_id_s, qc_hist_id, qc_plot_id = \ - create_qc_workflow(cfg) + ( + qc_stack, + qc_montage_id_a, + qc_montage_id_s, + qc_hist_id, + qc_plot_id, + ) = create_qc_workflow(cfg) pipeline_blocks += qc_stack # Connect the entire pipeline! @@ -1497,63 +1592,67 @@ def build_workflow(subject_id, sub_dict, cfg, pipeline_name=None, wf = connect_pipeline(wf, cfg, rpool, pipeline_blocks) except LookupError as lookup_error: missing_key = None - errorstrings = [arg for arg in lookup_error.args[0].split('\n') if - arg.strip()] - if lookup_error.args[0].startswith('When trying to connect node b'): + errorstrings = [arg for arg in lookup_error.args[0].split("\n") if arg.strip()] + if lookup_error.args[0].startswith("When trying to connect node b"): missing_key = lookup_error.args[0].split("': ")[-1] for errorstring in [ - '[!] C-PAC says: The listed resource is not in the resource pool:', - '[!] C-PAC says: None of the listed resources are in the resource ' - 'pool:', - '[!] C-PAC says: None of the listed resources in the node block ' - 'being connected exist in the resource pool.\n\nResources:' + "[!] C-PAC says: The listed resource is not in the resource pool:", + "[!] C-PAC says: None of the listed resources are in the resource " "pool:", + "[!] C-PAC says: None of the listed resources in the node block " + "being connected exist in the resource pool.\n\nResources:", ]: if errorstring in lookup_error.args[0]: missing_key = errorstrings[errorstrings.index(errorstring) + 1] - if missing_key and missing_key.endswith('_bold' - ) and 'func' not in sub_dict: + if missing_key and missing_key.endswith("_bold") and "func" not in sub_dict: raise FileNotFoundError( - 'The provided pipeline configuration requires functional ' - 'data but no functional data were found for ' + - '/'.join([sub_dict[key] for key in ['site', 'subject_id', - 'unique_id'] if key in sub_dict]) + '. Please check ' - 'your data and pipeline configurations.') from lookup_error + "The provided pipeline configuration requires functional " + "data but no functional data were found for " + + "/".join( + [ + sub_dict[key] + for key in ["site", "subject_id", "unique_id"] + if key in sub_dict + ] + ) + + ". Please check " + "your data and pipeline configurations." + ) from lookup_error raise lookup_error # Write out the data # TODO enforce value with schema validation try: - encrypt_data = bool(cfg.pipeline_setup['Amazon-AWS']['s3_encryption']) + bool(cfg.pipeline_setup["Amazon-AWS"]["s3_encryption"]) except: - encrypt_data = False + pass # TODO enforce value with schema validation # Extract credentials path for output if it exists try: # Get path to creds file - creds_path = '' - if cfg.pipeline_setup['Amazon-AWS']['aws_output_bucket_credentials']: - creds_path = str(cfg.pipeline_setup['Amazon-AWS'][ - 'aws_output_bucket_credentials']) + creds_path = "" + if cfg.pipeline_setup["Amazon-AWS"]["aws_output_bucket_credentials"]: + creds_path = str( + cfg.pipeline_setup["Amazon-AWS"]["aws_output_bucket_credentials"] + ) creds_path = os.path.abspath(creds_path) - if cfg.pipeline_setup['output_directory'][ - 'path'].lower().startswith('s3://'): + if cfg.pipeline_setup["output_directory"]["path"].lower().startswith("s3://"): # Test for s3 write access - s3_write_access = \ - aws_utils.test_bucket_access(creds_path, - cfg.pipeline_setup[ - 'output_directory']['path']) + s3_write_access = aws_utils.test_bucket_access( + creds_path, cfg.pipeline_setup["output_directory"]["path"] + ) if not s3_write_access: - raise Exception('Not able to write to bucket!') + raise Exception("Not able to write to bucket!") except Exception as e: - if cfg.pipeline_setup['output_directory'][ - 'path'].lower().startswith('s3://'): - err_msg = 'There was an error processing credentials or ' \ - 'accessing the S3 bucket. Check and try again.\n' \ - 'Error: %s' % e + if cfg.pipeline_setup["output_directory"]["path"].lower().startswith("s3://"): + err_msg = ( + "There was an error processing credentials or " + "accessing the S3 bucket. Check and try again.\n" + "Error: %s" % e + ) raise Exception(err_msg) # Collect all pipeline variants and write to output directory diff --git a/CPAC/pipeline/cpac_randomise_pipeline.py b/CPAC/pipeline/cpac_randomise_pipeline.py index 6599cf0eea..5eb562c4d6 100644 --- a/CPAC/pipeline/cpac_randomise_pipeline.py +++ b/CPAC/pipeline/cpac_randomise_pipeline.py @@ -17,6 +17,7 @@ import os import nipype.interfaces.io as nio + from CPAC.pipeline import nipype_pipeline_engine as pe from CPAC.pipeline.cpac_group_runner import load_config_yml from CPAC.pipeline.nipype_pipeline_engine.plugins import MultiProcPlugin @@ -27,75 +28,72 @@ def load_subject_file(group_config_path): group_config_obj = load_config_yml(group_config_path) pipeline_output_folder = group_config_obj.pipeline_dir - - if not group_config_obj.participant_list == None: - s_paths = group_config_obj.participant_list + + if group_config_obj.participant_list is not None: + pass else: - s_paths = [x for x in os.listdir(pipeline_output_folder) if os.path.isdir(x)] + [x for x in os.listdir(pipeline_output_folder) if os.path.isdir(x)] + def randomise_merged_file(s_paths): - - merge = pe.Node(interface=fslMerge(), name='fsl_merge') - merge.inputs.dimension = 't' + merge = pe.Node(interface=fslMerge(), name="fsl_merge") + merge.inputs.dimension = "t" merge.inputs.merged_file = "randomise_merged.nii.gz" - merge.inputs.in_files = s_paths + merge.inputs.in_files = s_paths -def randomise_merged_mask(s_paths): - mask = pe.Node(interface=fsl.maths.MathsCommand(), name='fsl_maths') - mask.inputs.args = '-abs -Tmin -bin' +def randomise_merged_mask(s_paths): + mask = pe.Node(interface=fsl.maths.MathsCommand(), name="fsl_maths") + mask.inputs.args = "-abs -Tmin -bin" mask.inputs.out_file = "randomise_mask.nii.gz" mask.inputs.in_file = s_paths + def prep_randomise_workflow(c, subject_infos): - print('Preparing Randomise workflow') p_id, s_ids, scan_ids, s_paths = (list(tup) for tup in zip(*subject_infos)) - print('Subjects', s_ids) - wf = pe.Workflow(name='randomise_workflow') - wf.base_dir = c.pipeline_setup['working_directory']['path'] + wf = pe.Workflow(name="randomise_workflow") + wf.base_dir = c.pipeline_setup["working_directory"]["path"] from CPAC.randomise import create_randomise rw = create_randomise() rw.inputs.inputspec.permutations = c.randopermutations - rw.inputs.inputspec.subjects = s_paths - #rw.inputs.inputspec.pipeline_ouput_folder = c.os.path.join(c.outputDirectory, - # 'pipeline_{0}'.format(c.pipelineName)) - rw.inputs.inputspec.mask_boolean = c.mask_boolean #TODO pipe from output dir, not the user input - rw.inputs.inputspec.tfce = c.tfce # will stay None? - rw.inputs.inputspec.demean = c.demean - rw.inputs.inputspec.c_thresh = c.c_thresh - - ds = pe.Node(nio.DataSink(), name='randomise_sink') - out_dir = os.path.dirname(s_paths[0]).replace(s_ids[0], 'randomise_results') + rw.inputs.inputspec.subjects = s_paths + # rw.inputs.inputspec.pipeline_ouput_folder = c.os.path.join(c.outputDirectory, + # 'pipeline_{0}'.format(c.pipelineName)) + rw.inputs.inputspec.mask_boolean = ( + c.mask_boolean + ) # TODO pipe from output dir, not the user input + rw.inputs.inputspec.tfce = c.tfce # will stay None? + rw.inputs.inputspec.demean = c.demean + rw.inputs.inputspec.c_thresh = c.c_thresh + + ds = pe.Node(nio.DataSink(), name="randomise_sink") + out_dir = os.path.dirname(s_paths[0]).replace(s_ids[0], "randomise_results") ds.inputs.base_directory = out_dir - ds.inputs.container = '' -#'tstat_files' ,'t_corrected_p_files','index_file','threshold_file','localmax_txt_file','localmax_vol_file','max_file','mean_file','pval_file','size_file' - wf.connect(rw, 'outputspec.tstat_files', - ds, 'tstat_files') - wf.connect(rw, 'outputspec.t_corrected_p_files', - ds, 't_corrected_p_files') - wf.connect(rw, 'outputspec.index_file', ds, 'index_file') - wf.connect(rw, 'outputspec.threshold_file', ds, 'threshold_file') - wf.connect(rw, 'outputspec.localmax_vol_file', ds, 'localmax_vol_file') - wf.connect(rw, 'outputspec.localmax_txt_file', ds, 'localmax_txt_file') - wf.connect(rw, 'outputspec.max_file', ds, 'max_file') - wf.connect(rw, 'outputspec.mean_file', ds, 'mean_file') - wf.connect(rw, 'outputspec.max_file', ds, 'max_file') - wf.connect(rw, 'outputspec.pval_file', ds, 'pval_file') - wf.connect(rw, 'outputspec.size_file', ds, 'size_file') - - plugin_args = {'n_procs': c.numCoresPerSubject, - 'status_callback': log_nodes_cb} - wf.run(plugin=MultiProcPlugin(plugin_args), - plugin_args=plugin_args) + ds.inputs.container = "" + #'tstat_files' ,'t_corrected_p_files','index_file','threshold_file','localmax_txt_file','localmax_vol_file','max_file','mean_file','pval_file','size_file' + wf.connect(rw, "outputspec.tstat_files", ds, "tstat_files") + wf.connect(rw, "outputspec.t_corrected_p_files", ds, "t_corrected_p_files") + wf.connect(rw, "outputspec.index_file", ds, "index_file") + wf.connect(rw, "outputspec.threshold_file", ds, "threshold_file") + wf.connect(rw, "outputspec.localmax_vol_file", ds, "localmax_vol_file") + wf.connect(rw, "outputspec.localmax_txt_file", ds, "localmax_txt_file") + wf.connect(rw, "outputspec.max_file", ds, "max_file") + wf.connect(rw, "outputspec.mean_file", ds, "mean_file") + wf.connect(rw, "outputspec.max_file", ds, "max_file") + wf.connect(rw, "outputspec.pval_file", ds, "pval_file") + wf.connect(rw, "outputspec.size_file", ds, "size_file") + + plugin_args = {"n_procs": c.numCoresPerSubject, "status_callback": log_nodes_cb} + wf.run(plugin=MultiProcPlugin(plugin_args), plugin_args=plugin_args) return wf - -#def run(config, subject_infos): + +# def run(config, subject_infos): # import re # import commands # commands.getoutput('source ~/.bashrc') @@ -108,4 +106,3 @@ def prep_randomise_workflow(c, subject_infos): # prep_randomise_workflow(c, pickle.load(open(subject_infos, 'r') )) - diff --git a/CPAC/pipeline/cpac_runner.py b/CPAC/pipeline/cpac_runner.py index 30d6b0fc18..8ba4f60e28 100644 --- a/CPAC/pipeline/cpac_runner.py +++ b/CPAC/pipeline/cpac_runner.py @@ -1,4 +1,4 @@ -"""Copyright (C) 2022 C-PAC Developers +"""Copyright (C) 2022 C-PAC Developers. This file is part of C-PAC. @@ -13,181 +13,229 @@ License for more details. You should have received a copy of the GNU Lesser General Public -License along with C-PAC. If not, see .""" -import os -import sys -import warnings +License along with C-PAC. If not, see . +""" from multiprocessing import Process +import os from time import strftime -import yaml +import warnings + from voluptuous.error import Invalid -from CPAC.utils.configuration import check_pname, Configuration, set_subject +import yaml + +from CPAC.longitudinal_pipeline.longitudinal_workflow import anat_longitudinal_wf +from CPAC.utils.configuration import Configuration, check_pname, set_subject +from CPAC.utils.configuration.yaml_template import upgrade_pipeline_to_1_8 from CPAC.utils.ga import track_run from CPAC.utils.monitoring import failed_to_start, log_nodes_cb -from CPAC.longitudinal_pipeline.longitudinal_workflow import \ - anat_longitudinal_wf -from CPAC.utils.configuration.yaml_template import upgrade_pipeline_to_1_8 # Run condor jobs def run_condor_jobs(c, config_file, subject_list_file, p_name): - # Import packages - import subprocess from time import strftime try: - sublist = yaml.safe_load(open(os.path.realpath(subject_list_file), 'r')) + sublist = yaml.safe_load(open(os.path.realpath(subject_list_file), "r")) except: - raise Exception("Subject list is not in proper YAML format. Please check your file") + raise Exception( + "Subject list is not in proper YAML format. Please check your file" + ) - cluster_files_dir = os.path.join(os.getcwd(), 'cluster_files') - subject_bash_file = os.path.join(cluster_files_dir, 'submit_%s.condor' % str(strftime("%Y_%m_%d_%H_%M_%S"))) - f = open(subject_bash_file, 'w') + cluster_files_dir = os.path.join(os.getcwd(), "cluster_files") + subject_bash_file = os.path.join( + cluster_files_dir, "submit_%s.condor" % str(strftime("%Y_%m_%d_%H_%M_%S")) + ) + f = open(subject_bash_file, "w") print("Executable = /usr/bin/python", file=f) print("Universe = vanilla", file=f) print("transfer_executable = False", file=f) print("getenv = True", file=f) - print("log = %s" % os.path.join(cluster_files_dir, 'c-pac_%s.log' % str(strftime("%Y_%m_%d_%H_%M_%S"))), file=f) - - for sidx in range(1,len(sublist)+1): - print("error = %s" % os.path.join(cluster_files_dir, 'c-pac_%s.%s.err' % (str(strftime("%Y_%m_%d_%H_%M_%S")), str(sidx))), file=f) - print("output = %s" % os.path.join(cluster_files_dir, 'c-pac_%s.%s.out' % (str(strftime("%Y_%m_%d_%H_%M_%S")), str(sidx))), file=f) + print( + "log = %s" + % os.path.join( + cluster_files_dir, "c-pac_%s.log" % str(strftime("%Y_%m_%d_%H_%M_%S")) + ), + file=f, + ) + + for sidx in range(1, len(sublist) + 1): + print( + "error = %s" + % os.path.join( + cluster_files_dir, + "c-pac_%s.%s.err" % (str(strftime("%Y_%m_%d_%H_%M_%S")), str(sidx)), + ), + file=f, + ) + print( + "output = %s" + % os.path.join( + cluster_files_dir, + "c-pac_%s.%s.out" % (str(strftime("%Y_%m_%d_%H_%M_%S")), str(sidx)), + ), + file=f, + ) - print("arguments = \"-c 'import CPAC; CPAC.pipeline.cpac_pipeline.run( ''%s'',''%s'',''%s'',''%s'',''%s'',''%s'',''%s'')\'\"" % (str(config_file), subject_list_file, str(sidx), c.maskSpecificationFile, c.roiSpecificationFile, c.templateSpecificationFile, p_name), file=f) + print( + "arguments = \"-c 'import CPAC; CPAC.pipeline.cpac_pipeline.run( ''%s'',''%s'',''%s'',''%s'',''%s'',''%s'',''%s'')'\"" + % ( + str(config_file), + subject_list_file, + str(sidx), + c.maskSpecificationFile, + c.roiSpecificationFile, + c.templateSpecificationFile, + p_name, + ), + file=f, + ) print("queue", file=f) f.close() - #commands.getoutput('chmod +x %s' % subject_bash_file ) - print(subprocess.getoutput("condor_submit %s " % (subject_bash_file))) + # commands.getoutput('chmod +x %s' % subject_bash_file ) # Create and run script for CPAC to run on cluster -def run_cpac_on_cluster(config_file, subject_list_file, - cluster_files_dir): - ''' +def run_cpac_on_cluster(config_file, subject_list_file, cluster_files_dir): + """ Function to build a SLURM batch job submission script and - submit it to the scheduler via 'sbatch' - ''' - + submit it to the scheduler via 'sbatch'. + """ # Import packages - import subprocess import getpass import re + import subprocess from time import strftime + from indi_schedulers import cluster_templates # Load in pipeline config try: - pipeline_dict = yaml.safe_load(open(os.path.realpath(config_file), 'r')) + pipeline_dict = yaml.safe_load(open(os.path.realpath(config_file), "r")) pipeline_config = Configuration(pipeline_dict) except: - raise Exception('Pipeline config is not in proper YAML format. '\ - 'Please check your file') + raise Exception( + "Pipeline config is not in proper YAML format. " "Please check your file" + ) # Load in the subject list try: - sublist = yaml.safe_load(open(os.path.realpath(subject_list_file), 'r')) + sublist = yaml.safe_load(open(os.path.realpath(subject_list_file), "r")) except: - raise Exception('Subject list is not in proper YAML format. '\ - 'Please check your file') + raise Exception( + "Subject list is not in proper YAML format. " "Please check your file" + ) # Init variables timestamp = str(strftime("%Y_%m_%d_%H_%M_%S")) - job_scheduler = pipeline_config.pipeline_setup['system_config']['on_grid']['resource_manager'].lower() + job_scheduler = pipeline_config.pipeline_setup["system_config"]["on_grid"][ + "resource_manager" + ].lower() # For SLURM time limit constraints only, hh:mm:ss hrs_limit = 8 * len(sublist) - time_limit = '%d:00:00' % hrs_limit + time_limit = "%d:00:00" % hrs_limit # Batch file variables - shell = subprocess.getoutput('echo $SHELL') + shell = subprocess.getoutput("echo $SHELL") user_account = getpass.getuser() num_subs = len(sublist) # Run CPAC via python -c command - python_cpac_str = 'python -c "from CPAC.pipeline.cpac_pipeline import run; '\ - 'run(\'%(config_file)s\', \'%(subject_list_file)s\', '\ - '%(env_arr_idx)s, \'%(pipeline_name)s\', '\ - 'plugin=\'MultiProc\', plugin_args=%(plugin_args)s)"' + python_cpac_str = ( + 'python -c "from CPAC.pipeline.cpac_pipeline import run; ' + "run('%(config_file)s', '%(subject_list_file)s', " + "%(env_arr_idx)s, '%(pipeline_name)s', " + "plugin='MultiProc', plugin_args=%(plugin_args)s)\"" + ) # Init plugin arguments plugin_args = { - 'n_procs': pipeline_config.pipeline_setup['system_config'][ - 'max_cores_per_participant'], - 'memory_gb': pipeline_config.pipeline_setup['system_config'][ - 'maximum_memory_per_participant'], - 'raise_insufficient': pipeline_config.pipeline_setup['system_config'][ - 'raise_insufficient'], - 'status_callback': log_nodes_cb} + "n_procs": pipeline_config.pipeline_setup["system_config"][ + "max_cores_per_participant" + ], + "memory_gb": pipeline_config.pipeline_setup["system_config"][ + "maximum_memory_per_participant" + ], + "raise_insufficient": pipeline_config.pipeline_setup["system_config"][ + "raise_insufficient" + ], + "status_callback": log_nodes_cb, + } # Set up run command dictionary - run_cmd_dict = {'config_file': config_file, - 'subject_list_file': subject_list_file, - 'pipeline_name': pipeline_config.pipeline_setup[ - 'pipeline_name'], - 'plugin_args': plugin_args} + run_cmd_dict = { + "config_file": config_file, + "subject_list_file": subject_list_file, + "pipeline_name": pipeline_config.pipeline_setup["pipeline_name"], + "plugin_args": plugin_args, + } # Set up config dictionary - config_dict = {'timestamp': timestamp, - 'shell': shell, - 'job_name': 'CPAC_' + pipeline_config.pipeline_setup[ - 'pipeline_name'], - 'num_tasks': num_subs, - 'queue': pipeline_config.pipeline_setup['system_config'][ - 'on_grid']['SGE']['queue'], - 'par_env': pipeline_config.pipeline_setup['system_config'][ - 'on_grid']['SGE']['parallel_environment'], - 'cores_per_task': pipeline_config.pipeline_setup[ - 'system_config']['max_cores_per_participant'], - 'user': user_account, - 'work_dir': cluster_files_dir, - 'time_limit': time_limit} + config_dict = { + "timestamp": timestamp, + "shell": shell, + "job_name": "CPAC_" + pipeline_config.pipeline_setup["pipeline_name"], + "num_tasks": num_subs, + "queue": pipeline_config.pipeline_setup["system_config"]["on_grid"]["SGE"][ + "queue" + ], + "par_env": pipeline_config.pipeline_setup["system_config"]["on_grid"]["SGE"][ + "parallel_environment" + ], + "cores_per_task": pipeline_config.pipeline_setup["system_config"][ + "max_cores_per_participant" + ], + "user": user_account, + "work_dir": cluster_files_dir, + "time_limit": time_limit, + } # Get string template for job scheduler - if job_scheduler == 'pbs': - env_arr_idx = '$PBS_ARRAYID' + if job_scheduler == "pbs": + env_arr_idx = "$PBS_ARRAYID" batch_file_contents = cluster_templates.pbs_template - confirm_str = '(?<=Your job-array )\d+' - exec_cmd = 'qsub' - elif job_scheduler == 'sge': - env_arr_idx = '$SGE_TASK_ID' + confirm_str = r"(?<=Your job-array )\d+" + exec_cmd = "qsub" + elif job_scheduler == "sge": + env_arr_idx = "$SGE_TASK_ID" batch_file_contents = cluster_templates.sge_template - confirm_str = '(?<=Your job-array )\d+' - exec_cmd = 'qsub' - elif job_scheduler == 'slurm': - env_arr_idx = '$SLURM_ARRAY_TASK_ID' + confirm_str = r"(?<=Your job-array )\d+" + exec_cmd = "qsub" + elif job_scheduler == "slurm": + env_arr_idx = "$SLURM_ARRAY_TASK_ID" batch_file_contents = cluster_templates.slurm_template - confirm_str = '(?<=Submitted batch job )\d+' - exec_cmd = 'sbatch' + confirm_str = r"(?<=Submitted batch job )\d+" + exec_cmd = "sbatch" # Populate rest of dictionary - config_dict['env_arr_idx'] = env_arr_idx - run_cmd_dict['env_arr_idx'] = env_arr_idx - config_dict['run_cmd'] = python_cpac_str % run_cmd_dict + config_dict["env_arr_idx"] = env_arr_idx + run_cmd_dict["env_arr_idx"] = env_arr_idx + config_dict["run_cmd"] = python_cpac_str % run_cmd_dict # Populate string from config dict values batch_file_contents = batch_file_contents % config_dict # Write file - batch_filepath = os.path.join(cluster_files_dir, 'cpac_submit_%s.%s' \ - % (timestamp, job_scheduler)) - with open(batch_filepath, 'w') as f: + batch_filepath = os.path.join( + cluster_files_dir, "cpac_submit_%s.%s" % (timestamp, job_scheduler) + ) + with open(batch_filepath, "w") as f: f.write(batch_file_contents) # Get output response from job submission - out = subprocess.getoutput('%s %s' % (exec_cmd, batch_filepath)) + out = subprocess.getoutput("%s %s" % (exec_cmd, batch_filepath)) # Check for successful qsub submission - if re.search(confirm_str, out) == None: - err_msg = 'Error submitting C-PAC pipeline run to %s queue' \ - % job_scheduler + if re.search(confirm_str, out) is None: + err_msg = "Error submitting C-PAC pipeline run to %s queue" % job_scheduler raise Exception(err_msg) # Get pid and send to pid file pid = re.search(confirm_str, out).group(0) - pid_file = os.path.join(cluster_files_dir, 'pid.txt') - with open(pid_file, 'w') as f: + pid_file = os.path.join(cluster_files_dir, "pid.txt") + with open(pid_file, "w") as f: f.write(pid) @@ -195,29 +243,37 @@ def run_T1w_longitudinal(sublist, cfg): subject_id_dict = {} for sub in sublist: - if sub['subject_id'] in subject_id_dict: - subject_id_dict[sub['subject_id']].append(sub) + if sub["subject_id"] in subject_id_dict: + subject_id_dict[sub["subject_id"]].append(sub) else: - subject_id_dict[sub['subject_id']] = [sub] + subject_id_dict[sub["subject_id"]] = [sub] # subject_id_dict has the subject_id as keys and a list of # sessions for each participant as value - valid_longitudinal_data = False for subject_id, sub_list in subject_id_dict.items(): if len(sub_list) > 1: - valid_longitudinal_data = True anat_longitudinal_wf(subject_id, sub_list, cfg) elif len(sub_list) == 1: - warnings.warn("\n\nThere is only one anatomical session " - "for sub-%s. Longitudinal preprocessing " - "will be skipped for this subject." - "\n\n" % subject_id) + warnings.warn( + "\n\nThere is only one anatomical session " + "for sub-%s. Longitudinal preprocessing " + "will be skipped for this subject." + "\n\n" % subject_id + ) # Run C-PAC subjects via job queue -def run(subject_list_file, config_file=None, p_name=None, plugin=None, - plugin_args=None, tracking=True, num_subs_at_once=None, debug=False, - test_config=False) -> int: +def run( + subject_list_file, + config_file=None, + p_name=None, + plugin=None, + plugin_args=None, + tracking=True, + num_subs_at_once=None, + debug=False, + test_config=False, +) -> int: """ Returns ------- @@ -232,32 +288,29 @@ def run(subject_list_file, config_file=None, p_name=None, plugin=None, from CPAC.pipeline.cpac_pipeline import run_workflow - print('Run called with config file {0}'.format(config_file)) - if plugin_args is None: - plugin_args = {'status_callback': log_nodes_cb} + plugin_args = {"status_callback": log_nodes_cb} if not config_file: import pkg_resources as p - config_file = \ - p.resource_filename("CPAC", - os.path.join("resources", - "configs", - "pipeline_config_template.yml")) + + config_file = p.resource_filename( + "CPAC", os.path.join("resources", "configs", "pipeline_config_template.yml") + ) # Init variables sublist = None - if '.yaml' in subject_list_file or '.yml' in subject_list_file: + if ".yaml" in subject_list_file or ".yml" in subject_list_file: subject_list_file = os.path.realpath(subject_list_file) else: - from CPAC.utils.bids_utils import collect_bids_files_configs, \ - bids_gen_cpac_sublist - (file_paths, config) = collect_bids_files_configs(subject_list_file, - None) - sublist = bids_gen_cpac_sublist(subject_list_file, file_paths, - config, None) + from CPAC.utils.bids_utils import ( + bids_gen_cpac_sublist, + collect_bids_files_configs, + ) + + (file_paths, config) = collect_bids_files_configs(subject_list_file, None) + sublist = bids_gen_cpac_sublist(subject_list_file, file_paths, config, None) if not sublist: - print(f"Did not find data in {subject_list_file}") return 1 # take date+time stamp for run identification purposes @@ -271,28 +324,19 @@ def run(subject_list_file, config_file=None, p_name=None, plugin=None, raise IOError else: try: - c = Configuration(yaml.safe_load(open(config_file, 'r'))) + c = Configuration(yaml.safe_load(open(config_file, "r"))) except Invalid: try: upgrade_pipeline_to_1_8(config_file) - c = Configuration(yaml.safe_load(open(config_file, 'r'))) + c = Configuration(yaml.safe_load(open(config_file, "r"))) except Exception as e: - print( - 'C-PAC could not upgrade pipeline configuration file ' - f'{config_file} to v1.8 syntax', - file=sys.stderr - ) raise e except Exception as e: raise e except IOError: - print("config file %s doesn't exist" % config_file) raise except yaml.parser.ParserError as e: - error_detail = "\"%s\" at line %d" % ( - e.problem, - e.problem_mark.line - ) + error_detail = '"%s" at line %d' % (e.problem, e.problem_mark.line) raise Exception( "Error parsing config file: {0}\n\n" "Error details:\n" @@ -307,28 +351,40 @@ def run(subject_list_file, config_file=None, p_name=None, plugin=None, "\n\n".format(config_file, e) ) - c.pipeline_setup['log_directory']['path'] = os.path.abspath(c.pipeline_setup['log_directory']['path']) - c.pipeline_setup['working_directory']['path'] = os.path.abspath(c.pipeline_setup['working_directory']['path']) - if 's3://' not in c.pipeline_setup['output_directory']['path']: - c.pipeline_setup['output_directory']['path'] = os.path.abspath(c.pipeline_setup['output_directory']['path']) + c.pipeline_setup["log_directory"]["path"] = os.path.abspath( + c.pipeline_setup["log_directory"]["path"] + ) + c.pipeline_setup["working_directory"]["path"] = os.path.abspath( + c.pipeline_setup["working_directory"]["path"] + ) + if "s3://" not in c.pipeline_setup["output_directory"]["path"]: + c.pipeline_setup["output_directory"]["path"] = os.path.abspath( + c.pipeline_setup["output_directory"]["path"] + ) if debug: - c.pipeline_setup['output_directory']['path']['write_debugging_outputs'] = "[1]" + c.pipeline_setup["output_directory"]["path"]["write_debugging_outputs"] = "[1]" if num_subs_at_once: if not str(num_subs_at_once).isdigit(): - raise Exception('[!] Value entered for --num_cores not a digit.') - c.pipeline_setup['system_config']['num_participants_at_once'] = int(num_subs_at_once) + raise Exception("[!] Value entered for --num_cores not a digit.") + c.pipeline_setup["system_config"]["num_participants_at_once"] = int( + num_subs_at_once + ) # Do some validation - if not c.pipeline_setup['working_directory']['path']: - raise Exception('Working directory not specified') - - if len(c.pipeline_setup['working_directory']['path']) > 70: - warnings.warn("We recommend that the working directory full path " - "should have less then 70 characters. " - "Long paths might not work in your operating system.") - warnings.warn("Current working directory: " - f"{c.pipeline_setup['working_directory']['path']}") + if not c.pipeline_setup["working_directory"]["path"]: + raise Exception("Working directory not specified") + + if len(c.pipeline_setup["working_directory"]["path"]) > 70: + warnings.warn( + "We recommend that the working directory full path " + "should have less then 70 characters. " + "Long paths might not work in your operating system." + ) + warnings.warn( + "Current working directory: " + f"{c.pipeline_setup['working_directory']['path']}" + ) # Get the pipeline name p_name = check_pname(p_name, c) @@ -336,35 +392,30 @@ def run(subject_list_file, config_file=None, p_name=None, plugin=None, # Load in subject list try: if not sublist: - sublist = yaml.safe_load(open(subject_list_file, 'r')) + sublist = yaml.safe_load(open(subject_list_file, "r")) except: - print("Subject list is not in proper YAML format. Please check " \ - "your file") raise Exception # Populate subject scan map sub_scan_map = {} try: for sub in sublist: - if sub['unique_id']: - s = sub['subject_id'] + "_" + sub["unique_id"] + if sub["unique_id"]: + s = sub["subject_id"] + "_" + sub["unique_id"] else: - s = sub['subject_id'] - scan_ids = ['scan_anat'] + s = sub["subject_id"] + scan_ids = ["scan_anat"] - if 'func' in sub: - for id in sub['func']: - scan_ids.append('scan_'+ str(id)) + if "func" in sub: + for id in sub["func"]: + scan_ids.append("scan_" + str(id)) - if 'rest' in sub: - for id in sub['rest']: - scan_ids.append('scan_'+ str(id)) + if "rest" in sub: + for id in sub["rest"]: + scan_ids.append("scan_" + str(id)) sub_scan_map[s] = scan_ids except: - print("\n\n" + "ERROR: Subject list file not in proper format - " \ - "check if you loaded the correct file?" + "\n" + \ - "Error name: cpac_runner_0001" + "\n\n") raise Exception pipeline_timing_info = [] @@ -375,22 +426,26 @@ def run(subject_list_file, config_file=None, p_name=None, plugin=None, if tracking: try: track_run( - level='participant' if not test_config else 'test', - participants=len(sublist) + level="participant" if not test_config else "test", + participants=len(sublist), ) except: - print("Usage tracking failed for this run.") + pass # If we're running on cluster, execute job scheduler - if c.pipeline_setup['system_config']['on_grid']['run']: - + if c.pipeline_setup["system_config"]["on_grid"]["run"]: # Create cluster log dir - cluster_files_dir = os.path.join(c.pipeline_setup['log_directory']['path'], 'cluster_files') + cluster_files_dir = os.path.join( + c.pipeline_setup["log_directory"]["path"], "cluster_files" + ) if not os.path.exists(cluster_files_dir): os.makedirs(cluster_files_dir) # Check if its a condor job, and run that - if 'condor' in c.pipeline_setup['system_config']['on_grid']['resource_manager'].lower(): + if ( + "condor" + in c.pipeline_setup["system_config"]["on_grid"]["resource_manager"].lower() + ): run_condor_jobs(c, config_file, subject_list_file, p_name) # All other schedulers are supported else: @@ -399,15 +454,18 @@ def run(subject_list_file, config_file=None, p_name=None, plugin=None, # Run on one computer else: # Create working dir - if not os.path.exists(c.pipeline_setup['working_directory']['path']): + if not os.path.exists(c.pipeline_setup["working_directory"]["path"]): try: - os.makedirs(c.pipeline_setup['working_directory']['path']) + os.makedirs(c.pipeline_setup["working_directory"]["path"]) except: - err = "\n\n[!] CPAC says: Could not create the working " \ - "directory: %s\n\nMake sure you have permissions " \ - "to write to this directory.\n\n" % c.pipeline_setup['working_directory']['path'] + err = ( + "\n\n[!] CPAC says: Could not create the working " + "directory: %s\n\nMake sure you have permissions " + "to write to this directory.\n\n" + % c.pipeline_setup["working_directory"]["path"] + ) raise Exception(err) - ''' + """ if not os.path.exists(c.pipeline_setup['log_directory']['path']): try: os.makedirs(c.pipeline_setup['log_directory']['path']) @@ -416,16 +474,17 @@ def run(subject_list_file, config_file=None, p_name=None, plugin=None, "directory: %s\n\nMake sure you have permissions " \ "to write to this directory.\n\n" % c.pipeline_setup['log_directory']['path'] raise Exception(err) - ''' + """ # BEGIN LONGITUDINAL TEMPLATE PIPELINE - if hasattr(c, 'longitudinal_template_generation') and \ - c.longitudinal_template_generation['run']: - + if ( + hasattr(c, "longitudinal_template_generation") + and c.longitudinal_template_generation["run"] + ): run_T1w_longitudinal(sublist, c) # TODO functional longitudinal pipeline - ''' + """ if valid_longitudinal_data: rsc_file_list = [] for dirpath, dirnames, filenames in os.walk(c.pipeline_setup[ @@ -445,8 +504,8 @@ def run(subject_list_file, config_file=None, p_name=None, plugin=None, subj = [s for s in subject_specific_dict.keys() if s in rsc_path] if subj: subject_specific_dict[subj[0]].append(rsc_path) - - # update individual-specific outputs: + + # update individual-specific outputs: # anatomical_brain, anatomical_brain_mask and anatomical_reorient for key in session_specific_dict.keys(): for f in session_specific_dict[key]: @@ -549,7 +608,7 @@ def replace_index(target1, target2, file_path): ses_list = [subj for subj in sublist if key in subj['anat']] for ses in ses_list: for reg_strat in strat_list: - try: + try: ss_strat_list = list(ses['resource_pool']) for strat_key in ss_strat_list: try: @@ -570,15 +629,23 @@ def replace_index(target1, target2, file_path): not c.functional_preproc['run'] ): sys.exit() - ''' + """ # END LONGITUDINAL TEMPLATE PIPELINE # If it only allows one, run it linearly - if c.pipeline_setup['system_config']['num_participants_at_once'] == 1: + if c.pipeline_setup["system_config"]["num_participants_at_once"] == 1: for sub in sublist: try: - run_workflow(sub, c, True, pipeline_timing_info, - p_name, plugin, plugin_args, test_config) + run_workflow( + sub, + c, + True, + pipeline_timing_info, + p_name, + plugin, + plugin_args, + test_config, + ) except Exception as exception: # pylint: disable=broad-except exitcode = 1 failed_to_start(set_subject(sub, c)[2], exception) @@ -588,22 +655,36 @@ def replace_index(target1, target2, file_path): job_queue = [] # Allocate processes - processes = [Process(target=run_workflow, - args=(sub, c, True, pipeline_timing_info, p_name, - plugin, plugin_args, test_config)) for - sub in sublist] - working_dir = os.path.join(c['pipeline_setup', 'working_directory', - 'path'], p_name) + processes = [ + Process( + target=run_workflow, + args=( + sub, + c, + True, + pipeline_timing_info, + p_name, + plugin, + plugin_args, + test_config, + ), + ) + for sub in sublist + ] + working_dir = os.path.join( + c["pipeline_setup", "working_directory", "path"], p_name + ) # Create pipeline-specific working dir if not exists if not os.path.exists(working_dir): os.makedirs(working_dir) # Set PID context to pipeline-specific file - with open(os.path.join(working_dir, 'pid.txt'), 'w', encoding='utf-8' - ) as pid: + with open(os.path.join(working_dir, "pid.txt"), "w", encoding="utf-8") as pid: # If we're allocating more processes than are subjects, run # them all - if len(sublist) <= c.pipeline_setup['system_config'][ - 'num_participants_at_once']: + if ( + len(sublist) + <= c.pipeline_setup["system_config"]["num_participants_at_once"] + ): for i, _p in enumerate(processes): try: _p.start() @@ -611,8 +692,7 @@ def replace_index(target1, target2, file_path): # pylint: disable=broad-except except Exception as exception: exitcode = 1 - failed_to_start(set_subject(sublist[i], c)[2], - exception) + failed_to_start(set_subject(sublist[i], c)[2], exception) # Otherwise manage resources to run processes incrementally else: idx = 0 @@ -622,8 +702,12 @@ def replace_index(target1, target2, file_path): # Init subject process index idc = idx # Launch processes (one for each subject) - for _p in processes[idc: idc + c.pipeline_setup[ - 'system_config']['num_participants_at_once']]: + for _p in processes[ + idc : idc + + c.pipeline_setup["system_config"][ + "num_participants_at_once" + ] + ]: try: _p.start() print(_p.pid, file=pid) @@ -632,8 +716,9 @@ def replace_index(target1, target2, file_path): # pylint: disable=broad-except except Exception as exception: exitcode = 1 - failed_to_start(set_subject(sublist[idx], - c)[2], exception) + failed_to_start( + set_subject(sublist[idx], c)[2], exception + ) # Otherwise, jobs are running - check them else: # Check every job in the queue's status @@ -641,7 +726,6 @@ def replace_index(target1, target2, file_path): # If the job is not alive if not job.is_alive(): # Find job and delete it from queue - print('found dead job ', job) loc = job_queue.index(job) del job_queue[loc] # ...and start the next available @@ -656,13 +740,13 @@ def replace_index(target1, target2, file_path): # pylint: disable=broad-except except Exception as exception: exitcode = 1 - failed_to_start(set_subject(sublist[idx], - c)[2], - exception) + failed_to_start( + set_subject(sublist[idx], c)[2], exception + ) # Add sleep so while loop isn't consuming 100% of CPU time.sleep(2) # set exitcode to 1 if any exception - if hasattr(pid, 'exitcode'): + if hasattr(pid, "exitcode"): exitcode = exitcode or pid.exitcode # Close PID txt file to indicate finish pid.close() diff --git a/CPAC/pipeline/engine.py b/CPAC/pipeline/engine.py index fa1980238b..8a1eb79454 100644 --- a/CPAC/pipeline/engine.py +++ b/CPAC/pipeline/engine.py @@ -17,51 +17,58 @@ import ast import copy import hashlib -import json from itertools import chain +import json import logging import os import re -from typing import Any, Optional, Union +from typing import Optional, Union import warnings -from CPAC.pipeline import \ - nipype_pipeline_engine as pe # pylint: disable=ungrouped-imports from nipype import config, logging # pylint: disable=wrong-import-order -from CPAC.pipeline.nodeblock import NodeBlockFunction # pylint: disable=ungrouped-imports -from nipype.interfaces.utility import \ - Rename # pylint: disable=wrong-import-order +from nipype.interfaces.utility import Rename # pylint: disable=wrong-import-order + from CPAC.image_utils.spatial_smoothing import spatial_smoothing -from CPAC.image_utils.statistical_transforms import z_score_standardize, \ - fisher_z_score_standardize +from CPAC.image_utils.statistical_transforms import ( + fisher_z_score_standardize, + z_score_standardize, +) +from CPAC.pipeline import ( + nipype_pipeline_engine as pe, # pylint: disable=ungrouped-imports +) from CPAC.pipeline.check_outputs import ExpectedOutputs +from CPAC.pipeline.nodeblock import ( + NodeBlockFunction, # pylint: disable=ungrouped-imports +) from CPAC.pipeline.utils import MOVEMENT_FILTER_KEYS, name_fork, source_set from CPAC.registration.registration import transform_derivative +from CPAC.resources.templates.lookup_table import lookup_identifier from CPAC.utils.bids_utils import res_in_filename from CPAC.utils.datasource import ( create_anat_datasource, create_func_datasource, - ingress_func_metadata, create_general_datasource, - resolve_resolution + ingress_func_metadata, + resolve_resolution, ) -from CPAC.utils.interfaces.function import Function from CPAC.utils.interfaces.datasink import DataSink -from CPAC.utils.monitoring import getLogger, LOGTAIL, \ - WARNING_FREESURFER_OFF_WITH_DATA +from CPAC.utils.interfaces.function import Function +from CPAC.utils.monitoring import LOGTAIL, WARNING_FREESURFER_OFF_WITH_DATA, getLogger from CPAC.utils.outputs import Outputs from CPAC.utils.typing import LIST_OR_STR, TUPLE -from CPAC.utils.utils import check_prov_for_regtool, \ - create_id_string, get_last_prov_entry, read_json, write_output_json - -from CPAC.resources.templates.lookup_table import lookup_identifier +from CPAC.utils.utils import ( + check_prov_for_regtool, + create_id_string, + get_last_prov_entry, + read_json, + write_output_json, +) -logger = getLogger('nipype.workflow') +logger = getLogger("nipype.workflow") class ResourcePool: def __init__(self, rpool=None, name=None, cfg=None, pipe_list=None): - if not rpool: self.rpool = {} else: @@ -77,58 +84,72 @@ def __init__(self, rpool=None, name=None, cfg=None, pipe_list=None): if cfg: self.cfg = cfg - self.logdir = cfg.pipeline_setup['log_directory']['path'] - - self.num_cpus = cfg.pipeline_setup['system_config'][ - 'max_cores_per_participant'] - self.num_ants_cores = cfg.pipeline_setup['system_config'][ - 'num_ants_threads'] - - self.ants_interp = cfg.registration_workflows[ - 'functional_registration']['func_registration_to_template'][ - 'ANTs_pipelines']['interpolation'] - self.fsl_interp = cfg.registration_workflows[ - 'functional_registration']['func_registration_to_template'][ - 'FNIRT_pipelines']['interpolation'] - - self.func_reg = cfg.registration_workflows[ - 'functional_registration']['func_registration_to_template'][ - 'run'] - - self.run_smoothing = 'smoothed' in cfg.post_processing[ - 'spatial_smoothing']['output'] - self.smoothing_bool = cfg.post_processing['spatial_smoothing']['run'] - self.run_zscoring = 'z-scored' in cfg.post_processing[ - 'z-scoring']['output'] - self.zscoring_bool = cfg.post_processing['z-scoring']['run'] - self.fwhm = cfg.post_processing['spatial_smoothing']['fwhm'] - self.smooth_opts = cfg.post_processing['spatial_smoothing'][ - 'smoothing_method'] - - self.xfm = ['alff', 'desc-sm_alff', 'desc-zstd_alff', - 'desc-sm-zstd_alff', - 'falff', 'desc-sm_falff', 'desc-zstd_falff', - 'desc-sm-zstd_falff', - 'reho', 'desc-sm_reho', 'desc-zstd_reho', - 'desc-sm-zstd_reho'] + self.logdir = cfg.pipeline_setup["log_directory"]["path"] + + self.num_cpus = cfg.pipeline_setup["system_config"][ + "max_cores_per_participant" + ] + self.num_ants_cores = cfg.pipeline_setup["system_config"][ + "num_ants_threads" + ] + + self.ants_interp = cfg.registration_workflows["functional_registration"][ + "func_registration_to_template" + ]["ANTs_pipelines"]["interpolation"] + self.fsl_interp = cfg.registration_workflows["functional_registration"][ + "func_registration_to_template" + ]["FNIRT_pipelines"]["interpolation"] + + self.func_reg = cfg.registration_workflows["functional_registration"][ + "func_registration_to_template" + ]["run"] + + self.run_smoothing = ( + "smoothed" in cfg.post_processing["spatial_smoothing"]["output"] + ) + self.smoothing_bool = cfg.post_processing["spatial_smoothing"]["run"] + self.run_zscoring = "z-scored" in cfg.post_processing["z-scoring"]["output"] + self.zscoring_bool = cfg.post_processing["z-scoring"]["run"] + self.fwhm = cfg.post_processing["spatial_smoothing"]["fwhm"] + self.smooth_opts = cfg.post_processing["spatial_smoothing"][ + "smoothing_method" + ] + + self.xfm = [ + "alff", + "desc-sm_alff", + "desc-zstd_alff", + "desc-sm-zstd_alff", + "falff", + "desc-sm_falff", + "desc-zstd_falff", + "desc-sm-zstd_falff", + "reho", + "desc-sm_reho", + "desc-zstd_reho", + "desc-sm-zstd_reho", + ] def __repr__(self) -> str: - params = [f"{param}={getattr(self, param)}" for param in - ["rpool", "name", "cfg", "pipe_list"] if - getattr(self, param, None) is not None] + params = [ + f"{param}={getattr(self, param)}" + for param in ["rpool", "name", "cfg", "pipe_list"] + if getattr(self, param, None) is not None + ] return f'ResourcePool({", ".join(params)})' def __str__(self) -> str: if self.name: - return f'ResourcePool({self.name}): {list(self.rpool)}' - return f'ResourcePool: {list(self.rpool)}' + return f"ResourcePool({self.name}): {list(self.rpool)}" + return f"ResourcePool: {list(self.rpool)}" def append_name(self, name): self.name.append(name) - def back_propogate_template_name(self, wf, resource_idx: str, json_info: dict, - id_string: 'pe.Node') -> None: - """Find and apply the template name from a resource's provenance + def back_propogate_template_name( + self, wf, resource_idx: str, json_info: dict, id_string: "pe.Node" + ) -> None: + """Find and apply the template name from a resource's provenance. Parameters ---------- @@ -142,27 +163,29 @@ def back_propogate_template_name(self, wf, resource_idx: str, json_info: dict, ------- None """ - if ('template' in resource_idx and self.check_rpool('derivatives-dir')): - if self.check_rpool('template'): - node, out = self.get_data('template') - wf.connect(node, out, id_string, 'template_desc') - elif 'Template' in json_info: - id_string.inputs.template_desc = json_info['Template'] - elif ('template' in resource_idx and - len(json_info.get('CpacProvenance', [])) > 1): - for resource in source_set(json_info['CpacProvenance']): - source, value = resource.split(':', 1) - if value.startswith('template_' - ) and source != 'FSL-AFNI-bold-ref': + if "template" in resource_idx and self.check_rpool("derivatives-dir"): + if self.check_rpool("template"): + node, out = self.get_data("template") + wf.connect(node, out, id_string, "template_desc") + elif "Template" in json_info: + id_string.inputs.template_desc = json_info["Template"] + elif ( + "template" in resource_idx and len(json_info.get("CpacProvenance", [])) > 1 + ): + for resource in source_set(json_info["CpacProvenance"]): + source, value = resource.split(":", 1) + if value.startswith("template_") and source != "FSL-AFNI-bold-ref": # 'FSL-AFNI-bold-ref' is currently allowed to be in # a different space, so don't use it as the space for # descendents try: - anscestor_json = list(self.rpool.get(source).items() - )[0][1].get('json', {}) - if 'Description' in anscestor_json: + anscestor_json = next(iter(self.rpool.get(source).items()))[ + 1 + ].get("json", {}) + if "Description" in anscestor_json: id_string.inputs.template_desc = anscestor_json[ - 'Description'] + "Description" + ] return except (IndexError, KeyError): pass @@ -195,17 +218,19 @@ def get_resources(self): return self.rpool.keys() def copy_rpool(self): - return ResourcePool(rpool=copy.deepcopy(self.get_entire_rpool()), - name=self.name, - cfg=self.cfg, - pipe_list=copy.deepcopy(self.pipe_list)) + return ResourcePool( + rpool=copy.deepcopy(self.get_entire_rpool()), + name=self.name, + cfg=self.cfg, + pipe_list=copy.deepcopy(self.pipe_list), + ) @staticmethod def get_raw_label(resource: str) -> str: - """Removes ``desc-*`` label""" - for tag in resource.split('_'): - if 'desc-' in tag: - resource = resource.replace(f'{tag}_', '') + """Removes ``desc-*`` label.""" + for tag in resource.split("_"): + if "desc-" in tag: + resource = resource.replace(f"{tag}_", "") break return resource @@ -213,32 +238,34 @@ def get_strat_info(self, prov, label=None, logdir=None): strat_info = {} for entry in prov: if isinstance(entry, list): - strat_info[entry[-1].split(':')[0]] = entry + strat_info[entry[-1].split(":")[0]] = entry elif isinstance(entry, str): - strat_info[entry.split(':')[0]] = entry.split(':')[1] + strat_info[entry.split(":")[0]] = entry.split(":")[1] if label: if not logdir: logdir = self.logdir - print(f'\n\nPrinting out strategy info for {label} in {logdir}\n') - write_output_json(strat_info, f'{label}_strat_info', - indent=4, basedir=logdir) + write_output_json( + strat_info, f"{label}_strat_info", indent=4, basedir=logdir + ) def set_json_info(self, resource, pipe_idx, key, val): - #TODO: actually should probably be able to inititialize resource/pipe_idx + # TODO: actually should probably be able to inititialize resource/pipe_idx if pipe_idx not in self.rpool[resource]: - raise Exception('\n[!] DEV: The pipeline/strat ID does not exist ' - f'in the resource pool.\nResource: {resource}' - f'Pipe idx: {pipe_idx}\nKey: {key}\nVal: {val}\n') + raise Exception( + "\n[!] DEV: The pipeline/strat ID does not exist " + f"in the resource pool.\nResource: {resource}" + f"Pipe idx: {pipe_idx}\nKey: {key}\nVal: {val}\n" + ) else: - if 'json' not in self.rpool[resource][pipe_idx]: - self.rpool[resource][pipe_idx]['json'] = {} - self.rpool[resource][pipe_idx]['json'][key] = val + if "json" not in self.rpool[resource][pipe_idx]: + self.rpool[resource][pipe_idx]["json"] = {} + self.rpool[resource][pipe_idx]["json"][key] = val def get_json_info(self, resource, pipe_idx, key): - #TODO: key checks + # TODO: key checks if not pipe_idx: - for pipe_idx, val in self.rpool[resource].items(): - return val['json'][key] + for pipe_idx, val in self.rpool[resource].items(): + return val["json"][key] return self.rpool[resource][pipe_idx][key] @staticmethod @@ -250,66 +277,85 @@ def get_resource_from_prov(prov): if not len(prov): return None if isinstance(prov[-1], list): - return prov[-1][-1].split(':')[0] + return prov[-1][-1].split(":")[0] elif isinstance(prov[-1], str): - return prov[-1].split(':')[0] + return prov[-1].split(":")[0] + return None def regressor_dct(self, cfg) -> dict: """Returns the regressor dictionary for the current strategy if - one exists. Raises KeyError otherwise.""" + one exists. Raises KeyError otherwise. + """ # pylint: disable=attribute-defined-outside-init - if hasattr(self, '_regressor_dct'): # memoized + if hasattr(self, "_regressor_dct"): # memoized # pylint: disable=access-member-before-definition return self._regressor_dct - key_error = KeyError("[!] No regressors in resource pool. \n\n" - "Try turning on create_regressors or " - "ingress_regressors.") - _nr = cfg['nuisance_corrections', '2-nuisance_regression'] - if not hasattr(self, 'timeseries'): - self.regressors = {reg["Name"]: reg for reg in _nr['Regressors']} - if self.check_rpool('parsed_regressors'): # ingressed regressor + key_error = KeyError( + "[!] No regressors in resource pool. \n\n" + "Try turning on create_regressors or " + "ingress_regressors." + ) + _nr = cfg["nuisance_corrections", "2-nuisance_regression"] + if not hasattr(self, "timeseries"): + self.regressors = {reg["Name"]: reg for reg in _nr["Regressors"]} + if self.check_rpool("parsed_regressors"): # ingressed regressor # name regressor workflow without regressor_prov - strat_name = _nr['ingress_regressors']['Regressors']['Name'] + strat_name = _nr["ingress_regressors"]["Regressors"]["Name"] if strat_name in self.regressors: self._regressor_dct = self.regressors[strat_name] return self._regressor_dct raise key_error - prov = self.get_cpac_provenance('desc-confounds_timeseries') - strat_name_components = prov[-1].split('_') - for _ in list(range(prov[-1].count('_'))): - reg_name = '_'.join(strat_name_components[-_:]) + prov = self.get_cpac_provenance("desc-confounds_timeseries") + strat_name_components = prov[-1].split("_") + for _ in list(range(prov[-1].count("_"))): + reg_name = "_".join(strat_name_components[-_:]) if reg_name in self.regressors: self._regressor_dct = self.regressors[reg_name] return self._regressor_dct raise key_error - def set_data(self, resource, node, output, json_info, pipe_idx, node_name, - fork=False, inject=False): + def set_data( + self, + resource, + node, + output, + json_info, + pipe_idx, + node_name, + fork=False, + inject=False, + ): json_info = json_info.copy() cpac_prov = [] - if 'CpacProvenance' in json_info: - cpac_prov = json_info['CpacProvenance'] + if "CpacProvenance" in json_info: + cpac_prov = json_info["CpacProvenance"] current_prov_list = list(cpac_prov) - new_prov_list = list(cpac_prov) # <---- making a copy, it was already a list + new_prov_list = list(cpac_prov) # <---- making a copy, it was already a list if not inject: - new_prov_list.append(f'{resource}:{node_name}') + new_prov_list.append(f"{resource}:{node_name}") try: res, new_pipe_idx = self.generate_prov_string(new_prov_list) except IndexError: - raise IndexError(f'\n\nThe set_data() call for {resource} has no ' - 'provenance information and should not be an ' - 'injection.') + raise IndexError( + f"\n\nThe set_data() call for {resource} has no " + "provenance information and should not be an " + "injection." + ) if not json_info: - json_info = {'RawSources': [resource]} # <---- this will be repopulated to the full file path at the end of the pipeline building, in gather_pipes() - json_info['CpacProvenance'] = new_prov_list + json_info = { + "RawSources": [resource] + } # <---- this will be repopulated to the full file path at the end of the pipeline building, in gather_pipes() + json_info["CpacProvenance"] = new_prov_list if resource not in self.rpool.keys(): self.rpool[resource] = {} else: - if not fork: # <--- in the event of multiple strategies/options, this will run for every option; just keep in mind + if not fork: # <--- in the event of multiple strategies/options, this will run for every option; just keep in mind search = False if self.get_resource_from_prov(current_prov_list) == resource: - pipe_idx = self.generate_prov_string(current_prov_list)[1] # CHANGING PIPE_IDX, BE CAREFUL DOWNSTREAM IN THIS FUNCTION + pipe_idx = self.generate_prov_string(current_prov_list)[ + 1 + ] # CHANGING PIPE_IDX, BE CAREFUL DOWNSTREAM IN THIS FUNCTION if pipe_idx not in self.rpool[resource].keys(): search = True else: @@ -318,24 +364,37 @@ def set_data(self, resource, node, output, json_info, pipe_idx, node_name, for idx in current_prov_list: if self.get_resource_from_prov(idx) == resource: if isinstance(idx, list): - pipe_idx = self.generate_prov_string(idx)[1] # CHANGING PIPE_IDX, BE CAREFUL DOWNSTREAM IN THIS FUNCTION + pipe_idx = self.generate_prov_string( + idx + )[ + 1 + ] # CHANGING PIPE_IDX, BE CAREFUL DOWNSTREAM IN THIS FUNCTION elif isinstance(idx, str): pipe_idx = idx break - if pipe_idx in self.rpool[resource].keys(): # <--- in case the resource name is now new, and not the original - del self.rpool[resource][pipe_idx] # <--- remove old keys so we don't end up with a new strat for every new node unit (unless we fork) + if ( + pipe_idx in self.rpool[resource].keys() + ): # <--- in case the resource name is now new, and not the original + del self.rpool[ + resource + ][ + pipe_idx + ] # <--- remove old keys so we don't end up with a new strat for every new node unit (unless we fork) if new_pipe_idx not in self.rpool[resource]: self.rpool[resource][new_pipe_idx] = {} if new_pipe_idx not in self.pipe_list: self.pipe_list.append(new_pipe_idx) - self.rpool[resource][new_pipe_idx]['data'] = (node, output) - self.rpool[resource][new_pipe_idx]['json'] = json_info + self.rpool[resource][new_pipe_idx]["data"] = (node, output) + self.rpool[resource][new_pipe_idx]["json"] = json_info - def get(self, resource: LIST_OR_STR, pipe_idx: Optional[str] = None, - report_fetched: Optional[bool] = False, - optional: Optional[bool] = False) -> Union[ - TUPLE[Optional[dict], Optional[str]], Optional[dict]]: + def get( + self, + resource: LIST_OR_STR, + pipe_idx: Optional[str] = None, + report_fetched: Optional[bool] = False, + optional: Optional[bool] = False, + ) -> Union[TUPLE[Optional[dict], Optional[str]], Optional[dict]]: # NOTE!!! # if this is the main rpool, this will return a dictionary of strats, and inside those, are dictionaries like {'data': (node, out), 'json': info} # BUT, if this is a sub rpool (i.e. a strat_pool), this will return a one-level dictionary of {'data': (node, out), 'json': info} WITHOUT THE LEVEL OF STRAT KEYS ABOVE IT @@ -367,24 +426,26 @@ def get(self, resource: LIST_OR_STR, pipe_idx: Optional[str] = None, "your C-PAC output directory.\n- If you have done these, " "and you still get this message, please let us know " "through any of our support channels at: " - "https://fcp-indi.github.io/\n") + "https://fcp-indi.github.io/\n" + ) - def get_data(self, resource, pipe_idx=None, report_fetched=False, - quick_single=False): + def get_data( + self, resource, pipe_idx=None, report_fetched=False, quick_single=False + ): if report_fetched: if pipe_idx: - connect, fetched = self.get(resource, pipe_idx=pipe_idx, - report_fetched=report_fetched) - return (connect['data'], fetched) - connect, fetched =self.get(resource, - report_fetched=report_fetched) - return (connect['data'], fetched) + connect, fetched = self.get( + resource, pipe_idx=pipe_idx, report_fetched=report_fetched + ) + return (connect["data"], fetched) + connect, fetched = self.get(resource, report_fetched=report_fetched) + return (connect["data"], fetched) elif pipe_idx: - return self.get(resource, pipe_idx=pipe_idx)['data'] + return self.get(resource, pipe_idx=pipe_idx)["data"] elif quick_single or len(self.get(resource)) == 1: for key, val in self.get(resource).items(): - return val['data'] - return self.get(resource)['data'] + return val["data"] + return self.get(resource)["data"] def copy_resource(self, resource, new_name): try: @@ -410,12 +471,14 @@ def get_json(self, resource, strat=None): # TODO: the below hits the exception if you use get_cpac_provenance on # TODO: the main rpool (i.e. if strat=None) - if 'json' in resource_strat_dct: - strat_json = resource_strat_dct['json'] + if "json" in resource_strat_dct: + strat_json = resource_strat_dct["json"] else: - raise Exception('\n[!] Developer info: the JSON ' - f'information for {resource} and {strat} ' - f'is incomplete.\n') + raise Exception( + "\n[!] Developer info: the JSON " + f"information for {resource} and {strat} " + f"is incomplete.\n" + ) return strat_json def get_cpac_provenance(self, resource, strat=None): @@ -428,7 +491,7 @@ def get_cpac_provenance(self, resource, strat=None): except KeyError: continue json_data = self.get_json(resource, strat) - return json_data['CpacProvenance'] + return json_data["CpacProvenance"] @staticmethod def generate_prov_string(prov): @@ -436,17 +499,21 @@ def generate_prov_string(prov): # MULTIPLE PRECEDING RESOURCES (or single, if just one) # NOTE: this DOES NOT merge multiple resources!!! (i.e. for merging-strat pipe_idx generation) if not isinstance(prov, list): - raise Exception('\n[!] Developer info: the CpacProvenance ' - f'entry for {prov} has to be a list.\n') + raise Exception( + "\n[!] Developer info: the CpacProvenance " + f"entry for {prov} has to be a list.\n" + ) last_entry = get_last_prov_entry(prov) - resource = last_entry.split(':')[0] + resource = last_entry.split(":")[0] return (resource, str(prov)) @staticmethod def generate_prov_list(prov_str): if not isinstance(prov_str, str): - raise Exception('\n[!] Developer info: the CpacProvenance ' - f'entry for {str(prov_str)} has to be a string.\n') + raise Exception( + "\n[!] Developer info: the CpacProvenance " + f"entry for {prov_str!s} has to be a string.\n" + ) return ast.literal_eval(prov_str) @staticmethod @@ -458,15 +525,15 @@ def get_resource_strats_from_prov(prov): # {rpool entry}: {that entry's provenance} resource_strat_dct = {} if isinstance(prov, str): - resource = prov.split(':')[0] + resource = prov.split(":")[0] resource_strat_dct[resource] = prov else: for spot, entry in enumerate(prov): if isinstance(entry, list): - resource = entry[-1].split(':')[0] + resource = entry[-1].split(":")[0] resource_strat_dct[resource] = entry elif isinstance(entry, str): - resource = entry.split(':')[0] + resource = entry.split(":")[0] resource_strat_dct[resource] = entry return resource_strat_dct @@ -481,9 +548,9 @@ def flatten_prov(self, prov): else: flat_prov.append(entry) return flat_prov + return None def get_strats(self, resources, debug=False): - # TODO: NOTE: NOT COMPATIBLE WITH SUB-RPOOL/STRAT_POOLS # TODO: (and it doesn't have to be) @@ -492,16 +559,16 @@ def get_strats(self, resources, debug=False): linked_resources = [] resource_list = [] if debug: - verbose_logger = getLogger('engine') - verbose_logger.debug('\nresources: %s', resources) + verbose_logger = getLogger("engine") + verbose_logger.debug("\nresources: %s", resources) for resource in resources: # grab the linked-input tuples if isinstance(resource, tuple): linked = [] for label in list(resource): - rp_dct, fetched_resource = self.get(label, - report_fetched=True, - optional=True) + rp_dct, fetched_resource = self.get( + label, report_fetched=True, optional=True + ) if not rp_dct: continue linked.append(fetched_resource) @@ -516,43 +583,45 @@ def get_strats(self, resources, debug=False): variant_pool = {} len_inputs = len(resource_list) if debug: - verbose_logger = getLogger('engine') - verbose_logger.debug('linked_resources: %s', - linked_resources) - verbose_logger.debug('resource_list: %s', resource_list) + verbose_logger = getLogger("engine") + verbose_logger.debug("linked_resources: %s", linked_resources) + verbose_logger.debug("resource_list: %s", resource_list) for resource in resource_list: - rp_dct, fetched_resource = self.get(resource, - report_fetched=True, # <---- rp_dct has the strats/pipe_idxs as the keys on first level, then 'data' and 'json' on each strat level underneath - optional=True) # oh, and we make the resource fetching in get_strats optional so we can have optional inputs, but they won't be optional in the node block unless we want them to be + rp_dct, fetched_resource = self.get( + resource, + report_fetched=True, # <---- rp_dct has the strats/pipe_idxs as the keys on first level, then 'data' and 'json' on each strat level underneath + optional=True, + ) # oh, and we make the resource fetching in get_strats optional so we can have optional inputs, but they won't be optional in the node block unless we want them to be if not rp_dct: len_inputs -= 1 continue sub_pool = [] if debug: - verbose_logger.debug('len(rp_dct): %s\n', len(rp_dct)) + verbose_logger.debug("len(rp_dct): %s\n", len(rp_dct)) for strat in rp_dct.keys(): json_info = self.get_json(fetched_resource, strat) - cpac_prov = json_info['CpacProvenance'] + cpac_prov = json_info["CpacProvenance"] sub_pool.append(cpac_prov) if fetched_resource not in variant_pool: variant_pool[fetched_resource] = [] - if 'CpacVariant' in json_info: - for key, val in json_info['CpacVariant'].items(): + if "CpacVariant" in json_info: + for key, val in json_info["CpacVariant"].items(): if val not in variant_pool[fetched_resource]: variant_pool[fetched_resource] += val - variant_pool[fetched_resource].append( - f'NO-{val[0]}') + variant_pool[fetched_resource].append(f"NO-{val[0]}") if debug: - verbose_logger = getLogger('engine') - verbose_logger.debug('%s sub_pool: %s\n', resource, sub_pool) + verbose_logger = getLogger("engine") + verbose_logger.debug("%s sub_pool: %s\n", resource, sub_pool) total_pool.append(sub_pool) if not total_pool: - raise LookupError('\n\n[!] C-PAC says: None of the listed ' - 'resources in the node block being connected ' - 'exist in the resource pool.\n\nResources:\n' - '%s\n\n' % resource_list) + raise LookupError( + "\n\n[!] C-PAC says: None of the listed " + "resources in the node block being connected " + "exist in the resource pool.\n\nResources:\n" + "%s\n\n" % resource_list + ) # TODO: right now total_pool is: # TODO: [[[T1w:anat_ingress, desc-preproc_T1w:anatomical_init, desc-preproc_T1w:acpc_alignment], [T1w:anat_ingress,desc-preproc_T1w:anatomical_init]], @@ -570,7 +639,7 @@ def get_strats(self, resources, debug=False): new_strats = {} # get rid of duplicates - TODO: refactor .product - strat_str_list = [] + strat_str_list = [] strat_list_list = [] for strat_tuple in strats: strat_list = list(copy.deepcopy(strat_tuple)) @@ -580,18 +649,14 @@ def get_strats(self, resources, debug=False): strat_list_list.append(strat_list) if debug: - verbose_logger = getLogger('engine') - verbose_logger.debug('len(strat_list_list): %s\n', - len(strat_list_list)) + verbose_logger = getLogger("engine") + verbose_logger.debug("len(strat_list_list): %s\n", len(strat_list_list)) for strat_list in strat_list_list: - json_dct = {} for strat in strat_list: # strat is a prov list for a single resource/input - strat_resource, strat_idx = \ - self.generate_prov_string(strat) - strat_json = self.get_json(strat_resource, - strat=strat_idx) + strat_resource, strat_idx = self.generate_prov_string(strat) + strat_json = self.get_json(strat_resource, strat=strat_idx) json_dct[strat_resource] = strat_json drop = False @@ -607,38 +672,38 @@ def get_strats(self, resources, debug=False): if xlabel == ylabel: continue yjson = copy.deepcopy(json_dct[ylabel]) - - if 'CpacVariant' not in xjson: - xjson['CpacVariant'] = {} - if 'CpacVariant' not in yjson: - yjson['CpacVariant'] = {} - + + if "CpacVariant" not in xjson: + xjson["CpacVariant"] = {} + if "CpacVariant" not in yjson: + yjson["CpacVariant"] = {} + current_strat = [] - for key, val in xjson['CpacVariant'].items(): + for key, val in xjson["CpacVariant"].items(): if isinstance(val, list): current_strat.append(val[0]) else: current_strat.append(val) current_spread = list(set(variant_pool[xlabel])) for spread_label in current_spread: - if 'NO-' in spread_label: + if "NO-" in spread_label: continue if spread_label not in current_strat: - current_strat.append(f'NO-{spread_label}') - + current_strat.append(f"NO-{spread_label}") + other_strat = [] - for key, val in yjson['CpacVariant'].items(): + for key, val in yjson["CpacVariant"].items(): if isinstance(val, list): other_strat.append(val[0]) else: other_strat.append(val) other_spread = list(set(variant_pool[ylabel])) for spread_label in other_spread: - if 'NO-' in spread_label: + if "NO-" in spread_label: continue if spread_label not in other_strat: - other_strat.append(f'NO-{spread_label}') - + other_strat.append(f"NO-{spread_label}") + for variant in current_spread: in_current_strat = False in_other_strat = False @@ -665,7 +730,7 @@ def get_strats(self, resources, debug=False): if in_other_spread: if not in_current_strat: drop = True - break + break if drop: break if drop: @@ -674,61 +739,85 @@ def get_strats(self, resources, debug=False): # make the merged strat label from the multiple inputs # strat_list is actually the merged CpacProvenance lists pipe_idx = str(strat_list) - new_strats[pipe_idx] = ResourcePool() # <----- new_strats is A DICTIONARY OF RESOURCEPOOL OBJECTS! + new_strats[ + pipe_idx + ] = ( + ResourcePool() + ) # <----- new_strats is A DICTIONARY OF RESOURCEPOOL OBJECTS! # placing JSON info at one level higher only for copy convenience - new_strats[pipe_idx].rpool['json'] = {} - new_strats[pipe_idx].rpool['json']['subjson'] = {} - new_strats[pipe_idx].rpool['json']['CpacProvenance'] = strat_list + new_strats[pipe_idx].rpool["json"] = {} + new_strats[pipe_idx].rpool["json"]["subjson"] = {} + new_strats[pipe_idx].rpool["json"]["CpacProvenance"] = strat_list # now just invert resource:strat to strat:resource for each resource:strat for cpac_prov in strat_list: resource, strat = self.generate_prov_string(cpac_prov) - resource_strat_dct = self.rpool[resource][strat] # <----- remember, this is the dct of 'data' and 'json'. - new_strats[pipe_idx].rpool[resource] = resource_strat_dct # <----- new_strats is A DICTIONARY OF RESOURCEPOOL OBJECTS! each one is a new slice of the resource pool combined together. + resource_strat_dct = self.rpool[resource][ + strat + ] # <----- remember, this is the dct of 'data' and 'json'. + new_strats[pipe_idx].rpool[ + resource + ] = resource_strat_dct # <----- new_strats is A DICTIONARY OF RESOURCEPOOL OBJECTS! each one is a new slice of the resource pool combined together. self.pipe_list.append(pipe_idx) - if 'CpacVariant' in resource_strat_dct['json']: - if 'CpacVariant' not in new_strats[pipe_idx].rpool['json']: - new_strats[pipe_idx].rpool['json']['CpacVariant'] = {} - for younger_resource, variant_list in resource_strat_dct['json']['CpacVariant'].items(): - if younger_resource not in new_strats[pipe_idx].rpool['json']['CpacVariant']: - new_strats[pipe_idx].rpool['json']['CpacVariant'][younger_resource] = variant_list + if "CpacVariant" in resource_strat_dct["json"]: + if "CpacVariant" not in new_strats[pipe_idx].rpool["json"]: + new_strats[pipe_idx].rpool["json"]["CpacVariant"] = {} + for younger_resource, variant_list in resource_strat_dct[ + "json" + ]["CpacVariant"].items(): + if ( + younger_resource + not in new_strats[pipe_idx].rpool["json"]["CpacVariant"] + ): + new_strats[pipe_idx].rpool["json"]["CpacVariant"][ + younger_resource + ] = variant_list # preserve each input's JSON info also - data_type = resource.split('_')[-1] - if data_type not in new_strats[pipe_idx].rpool['json']['subjson']: - new_strats[pipe_idx].rpool['json']['subjson'][data_type] = {} - new_strats[pipe_idx].rpool['json']['subjson'][data_type].update(copy.deepcopy(resource_strat_dct['json'])) + data_type = resource.split("_")[-1] + if data_type not in new_strats[pipe_idx].rpool["json"]["subjson"]: + new_strats[pipe_idx].rpool["json"]["subjson"][data_type] = {} + new_strats[pipe_idx].rpool["json"]["subjson"][data_type].update( + copy.deepcopy(resource_strat_dct["json"]) + ) else: new_strats = {} - for resource_strat_list in total_pool: # total_pool will have only one list of strats, for the one input - for cpac_prov in resource_strat_list: # <------- cpac_prov here doesn't need to be modified, because it's not merging with other inputs + for resource_strat_list in ( + total_pool + ): # total_pool will have only one list of strats, for the one input + for cpac_prov in resource_strat_list: # <------- cpac_prov here doesn't need to be modified, because it's not merging with other inputs resource, pipe_idx = self.generate_prov_string(cpac_prov) - resource_strat_dct = self.rpool[resource][pipe_idx] # <----- remember, this is the dct of 'data' and 'json'. - new_strats[pipe_idx] = ResourcePool(rpool={resource: resource_strat_dct}) # <----- again, new_strats is A DICTIONARY OF RESOURCEPOOL OBJECTS! + resource_strat_dct = self.rpool[resource][ + pipe_idx + ] # <----- remember, this is the dct of 'data' and 'json'. + new_strats[pipe_idx] = ResourcePool( + rpool={resource: resource_strat_dct} + ) # <----- again, new_strats is A DICTIONARY OF RESOURCEPOOL OBJECTS! # placing JSON info at one level higher only for copy convenience - new_strats[pipe_idx].rpool['json'] = resource_strat_dct['json'] # TODO: WARNING- THIS IS A LEVEL HIGHER THAN THE ORIGINAL 'JSON' FOR EASE OF ACCESS IN CONNECT_BLOCK WITH THE .GET(JSON) - new_strats[pipe_idx].rpool['json']['subjson'] = {} - new_strats[pipe_idx].rpool['json']['CpacProvenance'] = cpac_prov + new_strats[pipe_idx].rpool["json"] = resource_strat_dct[ + "json" + ] # TODO: WARNING- THIS IS A LEVEL HIGHER THAN THE ORIGINAL 'JSON' FOR EASE OF ACCESS IN CONNECT_BLOCK WITH THE .GET(JSON) + new_strats[pipe_idx].rpool["json"]["subjson"] = {} + new_strats[pipe_idx].rpool["json"]["CpacProvenance"] = cpac_prov # preserve each input's JSON info also - data_type = resource.split('_')[-1] - if data_type not in new_strats[pipe_idx].rpool['json']['subjson']: - new_strats[pipe_idx].rpool['json']['subjson'][data_type] = {} - new_strats[pipe_idx].rpool['json']['subjson'][data_type].update(copy.deepcopy(resource_strat_dct['json'])) + data_type = resource.split("_")[-1] + if data_type not in new_strats[pipe_idx].rpool["json"]["subjson"]: + new_strats[pipe_idx].rpool["json"]["subjson"][data_type] = {} + new_strats[pipe_idx].rpool["json"]["subjson"][data_type].update( + copy.deepcopy(resource_strat_dct["json"]) + ) return new_strats - def derivative_xfm(self, wf, label, connection, json_info, pipe_idx, - pipe_x): - + def derivative_xfm(self, wf, label, connection, json_info, pipe_idx, pipe_x): if label in self.xfm: - json_info = dict(json_info) # get the bold-to-template transform from the current strat_pool # info xfm_idx = None - xfm_label = 'from-bold_to-template_mode-image_xfm' - for entry in json_info['CpacProvenance']: + xfm_label = "from-bold_to-template_mode-image_xfm" + for entry in json_info["CpacProvenance"]: if isinstance(entry, list): - if entry[-1].split(':')[0] == xfm_label: + if entry[-1].split(":")[0] == xfm_label: xfm_prov = entry xfm_idx = self.generate_prov_string(xfm_prov)[1] break @@ -739,55 +828,66 @@ def derivative_xfm(self, wf, label, connection, json_info, pipe_idx, if not xfm_idx: xfm_info = [] for pipe_idx, entry in self.get(xfm_label).items(): - xfm_info.append((pipe_idx, entry['json']['CpacProvenance'])) + xfm_info.append((pipe_idx, entry["json"]["CpacProvenance"])) else: xfm_info = [(xfm_idx, xfm_prov)] for num, xfm_entry in enumerate(xfm_info): - xfm_idx, xfm_prov = xfm_entry reg_tool = check_prov_for_regtool(xfm_prov) - xfm = transform_derivative(f'{label}_xfm_{pipe_x}_{num}', - label, reg_tool, self.num_cpus, - self.num_ants_cores, - ants_interp=self.ants_interp, - fsl_interp=self.fsl_interp, - opt=None) - wf.connect(connection[0], connection[1], - xfm, 'inputspec.in_file') - - node, out = self.get_data("T1w-brain-template-deriv", - quick_single=True) - wf.connect(node, out, xfm, 'inputspec.reference') - - node, out = self.get_data('from-bold_to-template_mode-image_xfm', - pipe_idx=xfm_idx) - wf.connect(node, out, xfm, 'inputspec.transform') - - label = f'space-template_{label}' - json_info['Template'] = self.get_json_info('T1w-brain-template-deriv', - None, 'Description') - new_prov = json_info['CpacProvenance'] + xfm_prov - json_info['CpacProvenance'] = new_prov + xfm = transform_derivative( + f"{label}_xfm_{pipe_x}_{num}", + label, + reg_tool, + self.num_cpus, + self.num_ants_cores, + ants_interp=self.ants_interp, + fsl_interp=self.fsl_interp, + opt=None, + ) + wf.connect(connection[0], connection[1], xfm, "inputspec.in_file") + + node, out = self.get_data("T1w-brain-template-deriv", quick_single=True) + wf.connect(node, out, xfm, "inputspec.reference") + + node, out = self.get_data( + "from-bold_to-template_mode-image_xfm", pipe_idx=xfm_idx + ) + wf.connect(node, out, xfm, "inputspec.transform") + + label = f"space-template_{label}" + json_info["Template"] = self.get_json_info( + "T1w-brain-template-deriv", None, "Description" + ) + new_prov = json_info["CpacProvenance"] + xfm_prov + json_info["CpacProvenance"] = new_prov new_pipe_idx = self.generate_prov_string(new_prov) - self.set_data(label, xfm, 'outputspec.out_file', json_info, - new_pipe_idx, f'{label}_xfm_{num}', fork=True) + self.set_data( + label, + xfm, + "outputspec.out_file", + json_info, + new_pipe_idx, + f"{label}_xfm_{num}", + fork=True, + ) return wf @property def filtered_movement(self) -> bool: """ - Check if the movement parameters have been filtered in this strat_pool + Check if the movement parameters have been filtered in this strat_pool. Returns ------- bool """ try: - return 'motion_estimate_filter' in str(self.get_cpac_provenance( - 'desc-movementParameters_motion')) + return "motion_estimate_filter" in str( + self.get_cpac_provenance("desc-movementParameters_motion") + ) except KeyError: # not a strat_pool or no movement parameters in strat_pool return False @@ -795,52 +895,56 @@ def filtered_movement(self) -> bool: def filter_name(self, cfg) -> str: """ In a strat_pool with filtered movement parameters, return the - name of the filter for this strategy + name of the filter for this strategy. Returns ------- str """ - motion_filters = cfg['functional_preproc', - 'motion_estimates_and_correction', - 'motion_estimate_filter', 'filters'] - if len(motion_filters) == 1 and cfg.switch_is_on([ - 'functional_preproc', 'motion_estimates_and_correction', - 'motion_estimate_filter', 'run'], exclusive=True + motion_filters = cfg[ + "functional_preproc", + "motion_estimates_and_correction", + "motion_estimate_filter", + "filters", + ] + if len(motion_filters) == 1 and cfg.switch_is_on( + [ + "functional_preproc", + "motion_estimates_and_correction", + "motion_estimate_filter", + "run", + ], + exclusive=True, ): - return motion_filters[0]['Name'] + return motion_filters[0]["Name"] try: - key = 'motion' - sidecar = self.get_json('desc-movementParameters_motion') + key = "motion" + sidecar = self.get_json("desc-movementParameters_motion") except KeyError: sidecar = None - if sidecar is not None and 'CpacVariant' in sidecar: - if sidecar['CpacVariant'][key]: - return sidecar['CpacVariant'][key][0][::-1].split('_', - 1)[0][::-1] - return 'none' - - def post_process(self, wf, label, connection, json_info, pipe_idx, pipe_x, - outs): + if sidecar is not None and "CpacVariant" in sidecar: + if sidecar["CpacVariant"][key]: + return sidecar["CpacVariant"][key][0][::-1].split("_", 1)[0][::-1] + return "none" - input_type = 'func_derivative' + def post_process(self, wf, label, connection, json_info, pipe_idx, pipe_x, outs): + input_type = "func_derivative" post_labels = [(label, connection[0], connection[1])] - if re.match(r'(.*_)?[ed]c[bw]$', label) or re.match(r'(.*_)?lfcd[bw]$', - label): + if re.match(r"(.*_)?[ed]c[bw]$", label) or re.match(r"(.*_)?lfcd[bw]$", label): # suffix: [eigenvector or degree] centrality [binarized or weighted] # or lfcd [binarized or weighted] - mask = 'template-specification-file' - elif 'space-template' in label: - mask = 'space-template_res-derivative_desc-bold_mask' + mask = "template-specification-file" + elif "space-template" in label: + mask = "space-template_res-derivative_desc-bold_mask" else: - mask = 'space-bold_desc-brain_mask' + mask = "space-bold_desc-brain_mask" mask_idx = None - for entry in json_info['CpacProvenance']: + for entry in json_info["CpacProvenance"]: if isinstance(entry, list): - if entry[-1].split(':')[0] == mask: + if entry[-1].split(":")[0] == mask: mask_prov = entry mask_idx = self.generate_prov_string(mask_prov)[1] break @@ -848,96 +952,119 @@ def post_process(self, wf, label, connection, json_info, pipe_idx, pipe_x, if self.smoothing_bool: if label in Outputs.to_smooth: for smooth_opt in self.smooth_opts: - - sm = spatial_smoothing(f'{label}_smooth_{smooth_opt}_' - f'{pipe_x}', - self.fwhm, input_type, smooth_opt) - wf.connect(connection[0], connection[1], - sm, 'inputspec.in_file') - node, out = self.get_data(mask, pipe_idx=mask_idx, - quick_single=mask_idx is None) - wf.connect(node, out, sm, 'inputspec.mask') - - if 'desc-' not in label: - if 'space-' in label: - for tag in label.split('_'): - if 'space-' in tag: - smlabel = label.replace(tag, - f'{tag}_desc-sm') + sm = spatial_smoothing( + f"{label}_smooth_{smooth_opt}_" f"{pipe_x}", + self.fwhm, + input_type, + smooth_opt, + ) + wf.connect(connection[0], connection[1], sm, "inputspec.in_file") + node, out = self.get_data( + mask, pipe_idx=mask_idx, quick_single=mask_idx is None + ) + wf.connect(node, out, sm, "inputspec.mask") + + if "desc-" not in label: + if "space-" in label: + for tag in label.split("_"): + if "space-" in tag: + smlabel = label.replace(tag, f"{tag}_desc-sm") break else: - smlabel = f'desc-sm_{label}' + smlabel = f"desc-sm_{label}" else: - for tag in label.split('_'): - if 'desc-' in tag: - newtag = f'{tag}-sm' + for tag in label.split("_"): + if "desc-" in tag: + newtag = f"{tag}-sm" smlabel = label.replace(tag, newtag) break - post_labels.append((smlabel, sm, 'outputspec.out_file')) - - self.set_data(smlabel, sm, 'outputspec.out_file', - json_info, pipe_idx, - f'spatial_smoothing_{smooth_opt}', - fork=True) - self.set_data('fwhm', sm, 'outputspec.fwhm', json_info, - pipe_idx, f'spatial_smoothing_{smooth_opt}', - fork=True) - - if self.zscoring_bool: + post_labels.append((smlabel, sm, "outputspec.out_file")) + + self.set_data( + smlabel, + sm, + "outputspec.out_file", + json_info, + pipe_idx, + f"spatial_smoothing_{smooth_opt}", + fork=True, + ) + self.set_data( + "fwhm", + sm, + "outputspec.fwhm", + json_info, + pipe_idx, + f"spatial_smoothing_{smooth_opt}", + fork=True, + ) + + if self.zscoring_bool: for label_con_tpl in post_labels: label = label_con_tpl[0] connection = (label_con_tpl[1], label_con_tpl[2]) if label in Outputs.to_zstd: - zstd = z_score_standardize(f'{label}_zstd_{pipe_x}', - input_type) + zstd = z_score_standardize(f"{label}_zstd_{pipe_x}", input_type) - wf.connect(connection[0], connection[1], - zstd, 'inputspec.in_file') + wf.connect(connection[0], connection[1], zstd, "inputspec.in_file") node, out = self.get_data(mask, pipe_idx=mask_idx) - wf.connect(node, out, zstd, 'inputspec.mask') + wf.connect(node, out, zstd, "inputspec.mask") - if 'desc-' not in label: - if 'space-template' in label: - new_label = label.replace('space-template', - 'space-template_desc-zstd') + if "desc-" not in label: + if "space-template" in label: + new_label = label.replace( + "space-template", "space-template_desc-zstd" + ) else: - new_label = f'desc-zstd_{label}' + new_label = f"desc-zstd_{label}" else: - for tag in label.split('_'): - if 'desc-' in tag: - newtag = f'{tag}-zstd' + for tag in label.split("_"): + if "desc-" in tag: + newtag = f"{tag}-zstd" new_label = label.replace(tag, newtag) break - post_labels.append((new_label, zstd, 'outputspec.out_file')) + post_labels.append((new_label, zstd, "outputspec.out_file")) - self.set_data(new_label, zstd, 'outputspec.out_file', - json_info, pipe_idx, f'zscore_standardize', - fork=True) + self.set_data( + new_label, + zstd, + "outputspec.out_file", + json_info, + pipe_idx, + "zscore_standardize", + fork=True, + ) elif label in Outputs.to_fisherz: + zstd = fisher_z_score_standardize( + f"{label}_zstd_{pipe_x}", label, input_type + ) - zstd = fisher_z_score_standardize(f'{label}_zstd_{pipe_x}', - label, input_type) - - wf.connect(connection[0], connection[1], - zstd, 'inputspec.correlation_file') + wf.connect( + connection[0], connection[1], zstd, "inputspec.correlation_file" + ) # if the output is 'space-template_desc-MeanSCA_correlations', we want # 'desc-MeanSCA_timeseries' - oned = label.replace('correlations', 'timeseries') + oned = label.replace("correlations", "timeseries") node, out = outs[oned] - wf.connect(node, out, zstd, 'inputspec.timeseries_oned') + wf.connect(node, out, zstd, "inputspec.timeseries_oned") - post_labels.append((new_label, zstd, 'outputspec.out_file')) + post_labels.append((new_label, zstd, "outputspec.out_file")) - self.set_data(new_label, zstd, 'outputspec.out_file', - json_info, pipe_idx, - 'fisher_zscore_standardize', - fork=True) + self.set_data( + new_label, + zstd, + "outputspec.out_file", + json_info, + pipe_idx, + "fisher_zscore_standardize", + fork=True, + ) return (wf, post_labels) @@ -950,16 +1077,15 @@ def gather_pipes(self, wf, cfg, all=False, add_incl=None, add_excl=None): if add_excl: excl += add_excl - if 'nonsmoothed' not in cfg.post_processing['spatial_smoothing'][ - 'output']: + if "nonsmoothed" not in cfg.post_processing["spatial_smoothing"]["output"]: excl += Outputs.native_nonsmooth excl += Outputs.template_nonsmooth - if 'raw' not in cfg.post_processing['z-scoring']['output']: + if "raw" not in cfg.post_processing["z-scoring"]["output"]: excl += Outputs.native_raw excl += Outputs.template_raw - if not cfg.pipeline_setup['output_directory']['write_debugging_outputs']: + if not cfg.pipeline_setup["output_directory"]["write_debugging_outputs"]: # substring_excl.append(['bold']) excl += Outputs.debugging @@ -988,45 +1114,43 @@ def gather_pipes(self, wf, cfg, all=False, add_incl=None, add_excl=None): if drop: continue - subdir = 'other' + subdir = "other" if resource in Outputs.anat: - subdir = 'anat' - #TODO: get acq- etc. + subdir = "anat" + # TODO: get acq- etc. elif resource in Outputs.func: - subdir = 'func' - #TODO: other stuff like acq- etc. + subdir = "func" + # TODO: other stuff like acq- etc. for pipe_idx in self.rpool[resource]: unique_id = self.get_name() - part_id = unique_id.split('_')[0] - ses_id = unique_id.split('_')[1] + part_id = unique_id.split("_")[0] + ses_id = unique_id.split("_")[1] - if 'ses-' not in ses_id: + if "ses-" not in ses_id: ses_id = f"ses-{ses_id}" - out_dir = cfg.pipeline_setup['output_directory']['path'] - pipe_name = cfg.pipeline_setup['pipeline_name'] - container = os.path.join(f'pipeline_{pipe_name}', part_id, - ses_id) - filename = f'{unique_id}_{res_in_filename(self.cfg, resource)}' + out_dir = cfg.pipeline_setup["output_directory"]["path"] + pipe_name = cfg.pipeline_setup["pipeline_name"] + container = os.path.join(f"pipeline_{pipe_name}", part_id, ses_id) + filename = f"{unique_id}_{res_in_filename(self.cfg, resource)}" out_path = os.path.join(out_dir, container, subdir, filename) out_dct = { - 'unique_id': unique_id, - 'out_dir': out_dir, - 'container': container, - 'subdir': subdir, - 'filename': filename, - 'out_path': out_path + "unique_id": unique_id, + "out_dir": out_dir, + "container": container, + "subdir": subdir, + "filename": filename, + "out_path": out_path, } - self.rpool[resource][pipe_idx]['out'] = out_dct + self.rpool[resource][pipe_idx]["out"] = out_dct # TODO: have to link the pipe_idx's here. and call up 'desc-preproc_T1w' from a Sources in a json and replace. here. # TODO: can do the pipeline_description.json variants here too! for resource in self.rpool.keys(): - if resource not in Outputs.any: continue @@ -1054,30 +1178,46 @@ def gather_pipes(self, wf, cfg, all=False, add_incl=None, add_excl=None): num_variant = 0 if len(self.rpool[resource]) == 1: num_variant = "" - all_jsons = [self.rpool[resource][pipe_idx]['json'] for pipe_idx in - self.rpool[resource]] - unlabelled = set(key for json_info in all_jsons for key in - json_info.get('CpacVariant', {}).keys() if - key not in (*MOVEMENT_FILTER_KEYS, 'regressors')) - if 'bold' in unlabelled: + all_jsons = [ + self.rpool[resource][pipe_idx]["json"] + for pipe_idx in self.rpool[resource] + ] + unlabelled = { + key + for json_info in all_jsons + for key in json_info.get("CpacVariant", {}).keys() + if key not in (*MOVEMENT_FILTER_KEYS, "regressors") + } + if "bold" in unlabelled: all_bolds = list( - chain.from_iterable(json_info['CpacVariant']['bold'] for - json_info in all_jsons if - 'CpacVariant' in json_info and - 'bold' in json_info['CpacVariant'])) + chain.from_iterable( + json_info["CpacVariant"]["bold"] + for json_info in all_jsons + if "CpacVariant" in json_info + and "bold" in json_info["CpacVariant"] + ) + ) # not any(not) because all is overloaded as a parameter here - if not any(not re.match(r'apply_(phasediff|blip)_to_' - r'timeseries_separately_.*', _bold) - for _bold in all_bolds): + if not any( + not re.match( + r"apply_(phasediff|blip)_to_" r"timeseries_separately_.*", _bold + ) + for _bold in all_bolds + ): # this fork point should only result in 0 or 1 forks - unlabelled.remove('bold') + unlabelled.remove("bold") del all_bolds - all_forks = {key: set( - chain.from_iterable(json_info['CpacVariant'][key] for - json_info in all_jsons if - 'CpacVariant' in json_info and - key in json_info['CpacVariant'])) for - key in unlabelled} + all_forks = { + key: set( + chain.from_iterable( + json_info["CpacVariant"][key] + for json_info in all_jsons + if "CpacVariant" in json_info + and key in json_info["CpacVariant"] + ) + ) + for key in unlabelled + } # del all_jsons for key, forks in all_forks.items(): if len(forks) < 2: # no int suffix needed if only one fork @@ -1085,8 +1225,8 @@ def gather_pipes(self, wf, cfg, all=False, add_incl=None, add_excl=None): # del all_forks for pipe_idx in self.rpool[resource]: pipe_x = self.get_pipe_number(pipe_idx) - json_info = self.rpool[resource][pipe_idx]['json'] - out_dct = self.rpool[resource][pipe_idx]['out'] + json_info = self.rpool[resource][pipe_idx]["json"] + out_dct = self.rpool[resource][pipe_idx]["out"] try: if unlabelled: @@ -1095,140 +1235,158 @@ def gather_pipes(self, wf, cfg, all=False, add_incl=None, add_excl=None): pass try: - del json_info['subjson'] + del json_info["subjson"] except KeyError: pass - if out_dct['subdir'] == 'other' and not all: + if out_dct["subdir"] == "other" and not all: continue - unique_id = out_dct['unique_id'] + unique_id = out_dct["unique_id"] resource_idx = resource if isinstance(num_variant, int): - resource_idx, out_dct = name_fork(resource_idx, cfg, - json_info, out_dct) + resource_idx, out_dct = name_fork( + resource_idx, cfg, json_info, out_dct + ) if unlabelled: - if 'desc-' in out_dct['filename']: - for key in out_dct['filename'].split('_')[::-1]: + if "desc-" in out_dct["filename"]: + for key in out_dct["filename"].split("_")[::-1]: # final `desc` entity - if key.startswith('desc-'): - out_dct['filename'] = out_dct['filename' - ].replace( - key, f'{key}-{num_variant}') + if key.startswith("desc-"): + out_dct["filename"] = out_dct["filename"].replace( + key, f"{key}-{num_variant}" + ) resource_idx = resource_idx.replace( - key, f'{key}-{num_variant}') + key, f"{key}-{num_variant}" + ) break else: - suff = resource.split('_')[-1] - newdesc_suff = f'desc-{num_variant}_{suff}' - resource_idx = resource_idx.replace(suff, - newdesc_suff) - id_string = pe.Node(Function(input_names=['cfg', 'unique_id', - 'resource', - 'scan_id', - 'template_desc', - 'atlas_id', - 'fwhm', - 'subdir'], - output_names=['out_filename'], - function=create_id_string), - name=f'id_string_{resource_idx}_{pipe_x}') + suff = resource.split("_")[-1] + newdesc_suff = f"desc-{num_variant}_{suff}" + resource_idx = resource_idx.replace(suff, newdesc_suff) + id_string = pe.Node( + Function( + input_names=[ + "cfg", + "unique_id", + "resource", + "scan_id", + "template_desc", + "atlas_id", + "fwhm", + "subdir", + ], + output_names=["out_filename"], + function=create_id_string, + ), + name=f"id_string_{resource_idx}_{pipe_x}", + ) id_string.inputs.cfg = self.cfg id_string.inputs.unique_id = unique_id id_string.inputs.resource = resource_idx - id_string.inputs.subdir = out_dct['subdir'] + id_string.inputs.subdir = out_dct["subdir"] # grab the iterable scan ID - if out_dct['subdir'] == 'func': - node, out = self.rpool['scan']["['scan:func_ingress']"][ - 'data'] - wf.connect(node, out, id_string, 'scan_id') - - self.back_propogate_template_name(wf, resource_idx, json_info, - id_string) + if out_dct["subdir"] == "func": + node, out = self.rpool["scan"]["['scan:func_ingress']"]["data"] + wf.connect(node, out, id_string, "scan_id") + + self.back_propogate_template_name( + wf, resource_idx, json_info, id_string + ) # grab the FWHM if smoothed - for tag in resource.split('_'): - if 'desc-' in tag and '-sm' in tag: - fwhm_idx = pipe_idx.replace(f'{resource}:', 'fwhm:') + for tag in resource.split("_"): + if "desc-" in tag and "-sm" in tag: + fwhm_idx = pipe_idx.replace(f"{resource}:", "fwhm:") try: - node, out = self.rpool['fwhm'][fwhm_idx]['data'] - wf.connect(node, out, id_string, 'fwhm') + node, out = self.rpool["fwhm"][fwhm_idx]["data"] + wf.connect(node, out, id_string, "fwhm") except KeyError: # smoothing was not done for this resource in the # engine.py smoothing pass break - atlas_suffixes = ['timeseries', 'correlations', 'statmap'] + atlas_suffixes = ["timeseries", "correlations", "statmap"] # grab the iterable atlas ID atlas_id = None - if not resource.endswith('desc-confounds_timeseries'): - if resource.split('_')[-1] in atlas_suffixes: - atlas_idx = pipe_idx.replace(resource, 'atlas_name') + if not resource.endswith("desc-confounds_timeseries"): + if resource.split("_")[-1] in atlas_suffixes: + atlas_idx = pipe_idx.replace(resource, "atlas_name") # need the single quote and the colon inside the double # quotes - it's the encoded pipe_idx - #atlas_idx = new_idx.replace(f"'{temp_rsc}:", + # atlas_idx = new_idx.replace(f"'{temp_rsc}:", # "'atlas_name:") - if atlas_idx in self.rpool['atlas_name']: - node, out = self.rpool['atlas_name'][atlas_idx][ - 'data'] - wf.connect(node, out, id_string, 'atlas_id') - elif 'atlas-' in resource: - for tag in resource.split('_'): - if 'atlas-' in tag: - atlas_id = tag.replace('atlas-', '') + if atlas_idx in self.rpool["atlas_name"]: + node, out = self.rpool["atlas_name"][atlas_idx]["data"] + wf.connect(node, out, id_string, "atlas_id") + elif "atlas-" in resource: + for tag in resource.split("_"): + if "atlas-" in tag: + atlas_id = tag.replace("atlas-", "") id_string.inputs.atlas_id = atlas_id else: - warnings.warn(str( - LookupError("\n[!] No atlas ID found for " - f"{out_dct['filename']}.\n"))) - nii_name = pe.Node(Rename(), name=f'nii_{resource_idx}_' - f'{pipe_x}') + warnings.warn( + str( + LookupError( + "\n[!] No atlas ID found for " + f"{out_dct['filename']}.\n" + ) + ) + ) + nii_name = pe.Node(Rename(), name=f"nii_{resource_idx}_" f"{pipe_x}") nii_name.inputs.keep_ext = True - wf.connect(id_string, 'out_filename', - nii_name, 'format_string') - - node, out = self.rpool[resource][pipe_idx]['data'] + wf.connect(id_string, "out_filename", nii_name, "format_string") + + node, out = self.rpool[resource][pipe_idx]["data"] try: - wf.connect(node, out, nii_name, 'in_file') + wf.connect(node, out, nii_name, "in_file") except OSError as os_error: logger.warning(os_error) continue - write_json_imports = ['import os', 'import json'] - write_json = pe.Node(Function(input_names=['json_data', - 'filename'], - output_names=['json_file'], - function=write_output_json, - imports=write_json_imports), - name=f'json_{resource_idx}_{pipe_x}') + write_json_imports = ["import os", "import json"] + write_json = pe.Node( + Function( + input_names=["json_data", "filename"], + output_names=["json_file"], + function=write_output_json, + imports=write_json_imports, + ), + name=f"json_{resource_idx}_{pipe_x}", + ) write_json.inputs.json_data = json_info - wf.connect(id_string, 'out_filename', write_json, 'filename') - ds = pe.Node(DataSink(), name=f'sinker_{resource_idx}_' - f'{pipe_x}') + wf.connect(id_string, "out_filename", write_json, "filename") + ds = pe.Node(DataSink(), name=f"sinker_{resource_idx}_" f"{pipe_x}") ds.inputs.parameterization = False - ds.inputs.base_directory = out_dct['out_dir'] - ds.inputs.encrypt_bucket_keys = cfg.pipeline_setup[ - 'Amazon-AWS']['s3_encryption'] - ds.inputs.container = out_dct['container'] - - if cfg.pipeline_setup['Amazon-AWS'][ - 'aws_output_bucket_credentials']: - ds.inputs.creds_path = cfg.pipeline_setup['Amazon-AWS'][ - 'aws_output_bucket_credentials'] - expected_outputs += (out_dct['subdir'], create_id_string( - self.cfg, unique_id, resource_idx, - template_desc=id_string.inputs.template_desc, - atlas_id=atlas_id, subdir=out_dct['subdir'])) - wf.connect(nii_name, 'out_file', - ds, f'{out_dct["subdir"]}.@data') - wf.connect(write_json, 'json_file', - ds, f'{out_dct["subdir"]}.@json') + ds.inputs.base_directory = out_dct["out_dir"] + ds.inputs.encrypt_bucket_keys = cfg.pipeline_setup["Amazon-AWS"][ + "s3_encryption" + ] + ds.inputs.container = out_dct["container"] + + if cfg.pipeline_setup["Amazon-AWS"]["aws_output_bucket_credentials"]: + ds.inputs.creds_path = cfg.pipeline_setup["Amazon-AWS"][ + "aws_output_bucket_credentials" + ] + expected_outputs += ( + out_dct["subdir"], + create_id_string( + self.cfg, + unique_id, + resource_idx, + template_desc=id_string.inputs.template_desc, + atlas_id=atlas_id, + subdir=out_dct["subdir"], + ), + ) + wf.connect(nii_name, "out_file", ds, f'{out_dct["subdir"]}.@data') + wf.connect(write_json, "json_file", ds, f'{out_dct["subdir"]}.@json') outputs_logger.info(expected_outputs) def node_data(self, resource, **kwargs): - '''Factory function to create NodeData objects + """Factory function to create NodeData objects. Parameters ---------- @@ -1237,7 +1395,7 @@ def node_data(self, resource, **kwargs): Returns ------- NodeData - ''' + """ return NodeData(self, resource, **kwargs) @@ -1248,8 +1406,7 @@ def __init__(self, node_block_functions, debug=False): self.node_blocks = {} - for node_block_function in node_block_functions: # <---- sets up the NodeBlock object in case you gave it a list of node blocks instead of a single one - for option forking. - + for node_block_function in node_block_functions: # <---- sets up the NodeBlock object in case you gave it a list of node blocks instead of a single one - for option forking. self.input_interface = [] if isinstance(node_block_function, tuple): self.input_interface = node_block_function[1] @@ -1259,9 +1416,11 @@ def __init__(self, node_block_functions, debug=False): if not isinstance(node_block_function, NodeBlockFunction): # If the object is a plain function `__name__` will be more useful then `str()` - obj_str = node_block_function.__name__ \ - if hasattr(node_block_function, '__name__') else \ - str(node_block_function) + obj_str = ( + node_block_function.__name__ + if hasattr(node_block_function, "__name__") + else str(node_block_function) + ) raise TypeError(f'Object is not a nodeblock: "{obj_str}"') name = node_block_function.name @@ -1286,29 +1445,30 @@ def __init__(self, node_block_functions, debug=False): for key, val in node_block_function.legacy_nodeblock_dict().items(): self.node_blocks[name][key] = val - self.node_blocks[name]['block_function'] = node_block_function + self.node_blocks[name]["block_function"] = node_block_function - #TODO: fix/replace below + # TODO: fix/replace below self.outputs = {} for out in node_block_function.outputs: self.outputs[out] = None - self.options = ['base'] + self.options = ["base"] if node_block_function.outputs is not None: self.options = node_block_function.outputs - logger.info('Connecting %s...', name) + logger.info("Connecting %s...", name) if debug: - config.update_config( - {'logging': {'workflow_level': 'DEBUG'}}) + config.update_config({"logging": {"workflow_level": "DEBUG"}}) logging.update_logging(config) - logger.debug('"inputs": %s\n\t "outputs": %s%s', - node_block_function.inputs, - list(self.outputs.keys()), - f'\n\t"options": {self.options}' - if self.options != ['base'] else '') - config.update_config( - {'logging': {'workflow_level': 'INFO'}}) + logger.debug( + '"inputs": %s\n\t "outputs": %s%s', + node_block_function.inputs, + list(self.outputs.keys()), + f'\n\t"options": {self.options}' + if self.options != ["base"] + else "", + ) + config.update_config({"logging": {"workflow_level": "INFO"}}) logging.update_logging(config) def get_name(self): @@ -1316,14 +1476,16 @@ def get_name(self): def check_null(self, val): if isinstance(val, str): - val = None if val.lower() == 'none' else val + val = None if val.lower() == "none" else val return val def check_output(self, outputs, label, name): if label not in outputs: - raise NameError(f'\n[!] Output name "{label}" in the block ' - 'function does not match the outputs list ' - f'{outputs} in Node Block "{name}"\n') + raise NameError( + f'\n[!] Output name "{label}" in the block ' + "function does not match the outputs list " + f'{outputs} in Node Block "{name}"\n' + ) def grab_tiered_dct(self, cfg, key_list): cfg_dct = cfg @@ -1332,13 +1494,13 @@ def grab_tiered_dct(self, cfg, key_list): return cfg_dct def connect_block(self, wf, cfg, rpool): - debug = cfg.pipeline_setup['Debugging']['verbose'] + debug = cfg.pipeline_setup["Debugging"]["verbose"] all_opts = [] for name, block_dct in self.node_blocks.items(): opts = [] - config = self.check_null(block_dct['config']) - option_key = self.check_null(block_dct['option_key']) - option_val = self.check_null(block_dct['option_val']) + config = self.check_null(block_dct["config"]) + option_key = self.check_null(block_dct["option_key"]) + option_val = self.check_null(block_dct["option_val"]) if option_key and option_val: if not isinstance(option_key, list): option_key = [option_key] @@ -1348,13 +1510,15 @@ def connect_block(self, wf, cfg, rpool): key_list = config + option_key else: key_list = option_key - if 'USER-DEFINED' in option_val: + if "USER-DEFINED" in option_val: # load custom config data into each 'opt' opts = self.grab_tiered_dct(cfg, key_list) else: for option in option_val: try: - if option in self.grab_tiered_dct(cfg, key_list): # <---- goes over the option_vals in the node block docstring, and checks if the user's pipeline config included it in the forking list + if ( + option in self.grab_tiered_dct(cfg, key_list) + ): # <---- goes over the option_vals in the node block docstring, and checks if the user's pipeline config included it in the forking list opts.append(option) except AttributeError as err: raise Exception(f"{err}\nNode Block: {name}") @@ -1365,12 +1529,14 @@ def connect_block(self, wf, cfg, rpool): elif option_key and not option_val: # enables multiple config forking entries if not isinstance(option_key[0], list): - raise Exception(f'[!] The option_key field ({option_key}) ' - f'for {name} exists but there is no ' - 'option_val.\n\nIf you are trying to ' - 'populate multiple option keys, the ' - 'option_val field must contain a list of ' - 'a list.\n') + raise Exception( + f"[!] The option_key field ({option_key}) " + f"for {name} exists but there is no " + "option_val.\n\nIf you are trying to " + "populate multiple option keys, the " + "option_val field must contain a list of " + "a list.\n" + ) for option_config in option_key: # option_config is a list of pipe config levels down to the option if config: @@ -1379,29 +1545,35 @@ def connect_block(self, wf, cfg, rpool): key_list = option_config option_val = option_config[-1] if option_val in self.grab_tiered_dct(cfg, key_list[:-1]): - opts.append(option_val) - else: # AND, if there are multiple option-val's (in a list) in the docstring, it gets iterated below in 'for opt in option' etc. AND THAT'S WHEN YOU HAVE TO DELINEATE WITHIN THE NODE BLOCK CODE!!! + opts.append(option_val) + else: # AND, if there are multiple option-val's (in a list) in the docstring, it gets iterated below in 'for opt in option' etc. AND THAT'S WHEN YOU HAVE TO DELINEATE WITHIN THE NODE BLOCK CODE!!! opts = [None] all_opts += opts sidecar_additions = { - 'CpacConfigHash': hashlib.sha1(json.dumps(cfg.dict(), sort_keys=True).encode('utf-8')).hexdigest(), - 'CpacConfig': cfg.dict() + "CpacConfigHash": hashlib.sha1( + json.dumps(cfg.dict(), sort_keys=True).encode("utf-8") + ).hexdigest(), + "CpacConfig": cfg.dict(), } - if cfg['pipeline_setup']['output_directory'].get('user_defined'): - sidecar_additions['UserDefined'] = cfg['pipeline_setup']['output_directory']['user_defined'] - - for name, block_dct in self.node_blocks.items(): # <--- iterates over either the single node block in the sequence, or a list of node blocks within the list of node blocks, i.e. for option forking. + if cfg["pipeline_setup"]["output_directory"].get("user_defined"): + sidecar_additions["UserDefined"] = cfg["pipeline_setup"][ + "output_directory" + ]["user_defined"] - switch = self.check_null(block_dct['switch']) - config = self.check_null(block_dct['config']) - option_key = self.check_null(block_dct['option_key']) - option_val = self.check_null(block_dct['option_val']) - inputs = self.check_null(block_dct['inputs']) - outputs = self.check_null(block_dct['outputs']) + for ( + name, + block_dct, + ) in self.node_blocks.items(): # <--- iterates over either the single node block in the sequence, or a list of node blocks within the list of node blocks, i.e. for option forking. + switch = self.check_null(block_dct["switch"]) + config = self.check_null(block_dct["config"]) + option_key = self.check_null(block_dct["option_key"]) + option_val = self.check_null(block_dct["option_val"]) + inputs = self.check_null(block_dct["inputs"]) + outputs = self.check_null(block_dct["outputs"]) - block_function = block_dct['block_function'] + block_function = block_dct["block_function"] opts = [] if option_key and option_val: @@ -1413,15 +1585,19 @@ def connect_block(self, wf, cfg, rpool): key_list = config + option_key else: key_list = option_key - if 'USER-DEFINED' in option_val: + if "USER-DEFINED" in option_val: # load custom config data into each 'opt' opts = self.grab_tiered_dct(cfg, key_list) else: for option in option_val: - if option in self.grab_tiered_dct(cfg, key_list): # <---- goes over the option_vals in the node block docstring, and checks if the user's pipeline config included it in the forking list + if ( + option in self.grab_tiered_dct(cfg, key_list) + ): # <---- goes over the option_vals in the node block docstring, and checks if the user's pipeline config included it in the forking list opts.append(option) - else: # AND, if there are multiple option-val's (in a list) in the docstring, it gets iterated below in 'for opt in option' etc. AND THAT'S WHEN YOU HAVE TO DELINEATE WITHIN THE NODE BLOCK CODE!!! - opts = [None] # THIS ALSO MEANS the multiple option-val's in docstring node blocks can be entered once in the entire node-block sequence, not in a list of multiples + else: # AND, if there are multiple option-val's (in a list) in the docstring, it gets iterated below in 'for opt in option' etc. AND THAT'S WHEN YOU HAVE TO DELINEATE WITHIN THE NODE BLOCK CODE!!! + opts = [ + None + ] # THIS ALSO MEANS the multiple option-val's in docstring node blocks can be entered once in the entire node-block sequence, not in a list of multiples if not opts: # for node blocks where the options are split into different # block functions - opts will be empty for non-selected @@ -1435,9 +1611,11 @@ def connect_block(self, wf, cfg, rpool): try: key_list = config + switch except TypeError: - raise Exception("\n\n[!] Developer info: Docstring error " - f"for {name}, make sure the 'config' or " - "'switch' fields are lists.\n\n") + raise Exception( + "\n\n[!] Developer info: Docstring error " + f"for {name}, make sure the 'config' or " + "'switch' fields are lists.\n\n" + ) switch = self.grab_tiered_dct(cfg, key_list) else: if isinstance(switch[0], list): @@ -1466,9 +1644,12 @@ def connect_block(self, wf, cfg, rpool): switch = [switch] if True in switch: for pipe_idx, strat_pool in rpool.get_strats( - inputs, debug).items(): # strat_pool is a ResourcePool like {'desc-preproc_T1w': { 'json': info, 'data': (node, out) }, 'desc-brain_mask': etc.} - fork = False in switch # keep in mind rpool.get_strats(inputs) = {pipe_idx1: {'desc-preproc_T1w': etc.}, pipe_idx2: {..} } - for opt in opts: # it's a dictionary of ResourcePools called strat_pools, except those sub-ResourcePools only have one level! no pipe_idx strat keys. + inputs, debug + ).items(): # strat_pool is a ResourcePool like {'desc-preproc_T1w': { 'json': info, 'data': (node, out) }, 'desc-brain_mask': etc.} + fork = ( + False in switch + ) # keep in mind rpool.get_strats(inputs) = {pipe_idx1: {'desc-preproc_T1w': etc.}, pipe_idx2: {..} } + for opt in opts: # it's a dictionary of ResourcePools called strat_pools, except those sub-ResourcePools only have one level! no pipe_idx strat keys. # remember, you can get 'data' or 'json' from strat_pool with member functions # strat_pool has all of the JSON information of all the inputs! # so when we set_data below for the TOP-LEVEL MAIN RPOOL (not the strat_pool), we can generate new merged JSON information for each output. @@ -1487,130 +1668,155 @@ def connect_block(self, wf, cfg, rpool): strat_pool.copy_resource(input_name, interface[0]) replaced_inputs.append(interface[0]) try: - wf, outs = block_function(wf, cfg, strat_pool, - pipe_x, opt) + wf, outs = block_function(wf, cfg, strat_pool, pipe_x, opt) except IOError as e: # duplicate node logger.warning(e) continue if not outs: - if (block_function.__name__ == 'freesurfer_' - 'postproc'): - logger.warning( - WARNING_FREESURFER_OFF_WITH_DATA) - LOGTAIL['warnings'].append( - WARNING_FREESURFER_OFF_WITH_DATA) + if block_function.__name__ == "freesurfer_" "postproc": + logger.warning(WARNING_FREESURFER_OFF_WITH_DATA) + LOGTAIL["warnings"].append( + WARNING_FREESURFER_OFF_WITH_DATA + ) continue if opt and len(option_val) > 1: - node_name = f'{node_name}_{opt}' - elif opt and 'USER-DEFINED' in option_val: + node_name = f"{node_name}_{opt}" + elif opt and "USER-DEFINED" in option_val: node_name = f'{node_name}_{opt["Name"]}' if debug: - verbose_logger = getLogger('engine') - verbose_logger.debug('\n=======================') - verbose_logger.debug('Node name: %s', node_name) - prov_dct = \ - rpool.get_resource_strats_from_prov( - ast.literal_eval(pipe_idx)) + verbose_logger = getLogger("engine") + verbose_logger.debug("\n=======================") + verbose_logger.debug("Node name: %s", node_name) + prov_dct = rpool.get_resource_strats_from_prov( + ast.literal_eval(pipe_idx) + ) for key, val in prov_dct.items(): - verbose_logger.debug('-------------------') - verbose_logger.debug('Input - %s:', key) - sub_prov_dct = \ - rpool.get_resource_strats_from_prov(val) + verbose_logger.debug("-------------------") + verbose_logger.debug("Input - %s:", key) + sub_prov_dct = rpool.get_resource_strats_from_prov(val) for sub_key, sub_val in sub_prov_dct.items(): - sub_sub_dct = \ - rpool.get_resource_strats_from_prov( - sub_val) - verbose_logger.debug(' sub-input - %s:', - sub_key) - verbose_logger.debug(' prov = %s', - sub_val) + sub_sub_dct = rpool.get_resource_strats_from_prov( + sub_val + ) + verbose_logger.debug(" sub-input - %s:", sub_key) + verbose_logger.debug(" prov = %s", sub_val) verbose_logger.debug( - ' sub_sub_inputs = %s', - sub_sub_dct.keys()) + " sub_sub_inputs = %s", sub_sub_dct.keys() + ) for label, connection in outs.items(): self.check_output(outputs, label, name) - new_json_info = copy.deepcopy(strat_pool.get('json')) + new_json_info = copy.deepcopy(strat_pool.get("json")) # transfer over data-specific json info # for example, if the input data json is _bold and the output is also _bold - data_type = label.split('_')[-1] - if data_type in new_json_info['subjson']: - if 'SkullStripped' in new_json_info['subjson'][data_type]: - new_json_info['SkullStripped'] = new_json_info['subjson'][data_type]['SkullStripped'] - - # determine sources for the outputs, i.e. all input data into the node block - new_json_info['Sources'] = [x for x in strat_pool.get_entire_rpool() if x != 'json' and x not in replaced_inputs] - + data_type = label.split("_")[-1] + if data_type in new_json_info["subjson"]: + if ( + "SkullStripped" + in new_json_info["subjson"][data_type] + ): + new_json_info["SkullStripped"] = new_json_info[ + "subjson" + ][data_type]["SkullStripped"] + + # determine sources for the outputs, i.e. all input data into the node block + new_json_info["Sources"] = [ + x + for x in strat_pool.get_entire_rpool() + if x != "json" and x not in replaced_inputs + ] + if isinstance(outputs, dict): new_json_info.update(outputs[label]) - if 'Description' not in outputs[label]: + if "Description" not in outputs[label]: # don't propagate old Description try: - del new_json_info['Description'] + del new_json_info["Description"] except KeyError: pass - if 'Template' in outputs[label]: - template_key = outputs[label]['Template'] - if template_key in new_json_info['Sources']: + if "Template" in outputs[label]: + template_key = outputs[label]["Template"] + if template_key in new_json_info["Sources"]: # only if the pipeline config template key is entered as the 'Template' field # otherwise, skip this and take in the literal 'Template' string try: - new_json_info['Template'] = new_json_info['subjson'][template_key]['Description'] + new_json_info["Template"] = new_json_info[ + "subjson" + ][template_key]["Description"] except KeyError: pass try: - new_json_info['Resolution'] = new_json_info['subjson'][template_key]['Resolution'] + new_json_info["Resolution"] = new_json_info[ + "subjson" + ][template_key]["Resolution"] except KeyError: pass else: # don't propagate old Description try: - del new_json_info['Description'] + del new_json_info["Description"] except KeyError: pass - if 'Description' in new_json_info: - new_json_info['Description'] = ' '.join(new_json_info['Description'].split()) + if "Description" in new_json_info: + new_json_info["Description"] = " ".join( + new_json_info["Description"].split() + ) for sidecar_key, sidecar_value in sidecar_additions.items(): if sidecar_key not in new_json_info: new_json_info[sidecar_key] = sidecar_value try: - del new_json_info['subjson'] + del new_json_info["subjson"] except KeyError: pass if fork or len(opts) > 1 or len(all_opts) > 1: - if 'CpacVariant' not in new_json_info: - new_json_info['CpacVariant'] = {} + if "CpacVariant" not in new_json_info: + new_json_info["CpacVariant"] = {} raw_label = rpool.get_raw_label(label) - if raw_label not in new_json_info['CpacVariant']: - new_json_info['CpacVariant'][raw_label] = [] - new_json_info['CpacVariant'][raw_label].append(node_name) - - rpool.set_data(label, - connection[0], - connection[1], - new_json_info, - pipe_idx, node_name, fork) + if raw_label not in new_json_info["CpacVariant"]: + new_json_info["CpacVariant"][raw_label] = [] + new_json_info["CpacVariant"][raw_label].append( + node_name + ) + + rpool.set_data( + label, + connection[0], + connection[1], + new_json_info, + pipe_idx, + node_name, + fork, + ) wf, post_labels = rpool.post_process( - wf, label, connection, new_json_info, pipe_idx, - pipe_x, outs) + wf, + label, + connection, + new_json_info, + pipe_idx, + pipe_x, + outs, + ) if rpool.func_reg: for postlabel in post_labels: connection = (postlabel[1], postlabel[2]) - wf = rpool.derivative_xfm(wf, postlabel[0], - connection, - new_json_info, - pipe_idx, - pipe_x) + wf = rpool.derivative_xfm( + wf, + postlabel[0], + connection, + new_json_info, + pipe_idx, + pipe_x, + ) return wf @@ -1658,192 +1864,196 @@ def wrap_block(node_blocks, interface, wf, cfg, strat_pool, pipe_num, opt): """ for block in node_blocks: - #new_pool = copy.deepcopy(strat_pool) + # new_pool = copy.deepcopy(strat_pool) for in_resource, val in interface.items(): if isinstance(val, tuple): - strat_pool.set_data(in_resource, val[0], val[1], {}, "", "", - fork=True)# - if 'sub_num' not in strat_pool.get_pool_info(): - strat_pool.set_pool_info({'sub_num': 0}) - sub_num = strat_pool.get_pool_info()['sub_num'] - - wf, outputs = block(wf, cfg, strat_pool, f'{pipe_num}-{sub_num}', opt)# + strat_pool.set_data( + in_resource, val[0], val[1], {}, "", "", fork=True + ) # + if "sub_num" not in strat_pool.get_pool_info(): + strat_pool.set_pool_info({"sub_num": 0}) + sub_num = strat_pool.get_pool_info()["sub_num"] + + wf, outputs = block(wf, cfg, strat_pool, f"{pipe_num}-{sub_num}", opt) # for out, val in outputs.items(): if out in interface and isinstance(interface[out], str): - strat_pool.set_data(interface[out], outputs[out][0], outputs[out][1], - {}, "", "") + strat_pool.set_data( + interface[out], outputs[out][0], outputs[out][1], {}, "", "" + ) else: - strat_pool.set_data(out, outputs[out][0], outputs[out][1], - {}, "", "") + strat_pool.set_data(out, outputs[out][0], outputs[out][1], {}, "", "") sub_num += 1 - strat_pool.set_pool_info({'sub_num': sub_num}) + strat_pool.set_pool_info({"sub_num": sub_num}) return (wf, strat_pool) -def ingress_raw_anat_data(wf, rpool, cfg, data_paths, unique_id, part_id, - ses_id): - if 'anat' not in data_paths: - print('No anatomical data present.') +def ingress_raw_anat_data(wf, rpool, cfg, data_paths, unique_id, part_id, ses_id): + if "anat" not in data_paths: return rpool - if 'creds_path' not in data_paths: - data_paths['creds_path'] = None + if "creds_path" not in data_paths: + data_paths["creds_path"] = None - anat_flow = create_anat_datasource(f'anat_T1w_gather_{part_id}_{ses_id}') + anat_flow = create_anat_datasource(f"anat_T1w_gather_{part_id}_{ses_id}") anat = {} - if type(data_paths['anat']) is str: - anat['T1']=data_paths['anat'] - elif 'T1w' in data_paths['anat']: - anat['T1']=data_paths['anat']['T1w'] + if type(data_paths["anat"]) is str: + anat["T1"] = data_paths["anat"] + elif "T1w" in data_paths["anat"]: + anat["T1"] = data_paths["anat"]["T1w"] - if 'T1' in anat: + if "T1" in anat: anat_flow.inputs.inputnode.set( subject=part_id, - anat=anat['T1'], - creds_path=data_paths['creds_path'], - dl_dir=cfg.pipeline_setup['working_directory']['path'], - img_type='anat' + anat=anat["T1"], + creds_path=data_paths["creds_path"], + dl_dir=cfg.pipeline_setup["working_directory"]["path"], + img_type="anat", ) - rpool.set_data('T1w', anat_flow, 'outputspec.anat', {}, - "", "anat_ingress") - - if 'T2w' in data_paths['anat']: - anat_flow_T2 = create_anat_datasource(f'anat_T2w_gather_{part_id}_{ses_id}') + rpool.set_data("T1w", anat_flow, "outputspec.anat", {}, "", "anat_ingress") + + if "T2w" in data_paths["anat"]: + anat_flow_T2 = create_anat_datasource(f"anat_T2w_gather_{part_id}_{ses_id}") anat_flow_T2.inputs.inputnode.set( subject=part_id, - anat=data_paths['anat']['T2w'], - creds_path=data_paths['creds_path'], - dl_dir=cfg.pipeline_setup['working_directory']['path'], - img_type='anat' + anat=data_paths["anat"]["T2w"], + creds_path=data_paths["creds_path"], + dl_dir=cfg.pipeline_setup["working_directory"]["path"], + img_type="anat", + ) + rpool.set_data("T2w", anat_flow_T2, "outputspec.anat", {}, "", "anat_ingress") + + if cfg.surface_analysis["freesurfer"]["ingress_reconall"]: + rpool = ingress_freesurfer( + wf, rpool, cfg, data_paths, unique_id, part_id, ses_id ) - rpool.set_data('T2w', anat_flow_T2, 'outputspec.anat', {}, - "", "anat_ingress") - if cfg.surface_analysis['freesurfer']['ingress_reconall']: - rpool = ingress_freesurfer(wf, rpool, cfg, data_paths, unique_id, part_id, - ses_id) - return rpool -def ingress_freesurfer(wf, rpool, cfg, data_paths, unique_id, part_id, - ses_id): - - if 'anat' not in data_paths: - print('No FreeSurfer data present.') + +def ingress_freesurfer(wf, rpool, cfg, data_paths, unique_id, part_id, ses_id): + if "anat" not in data_paths: return rpool - - if 'freesurfer_dir' in data_paths['anat']: - fs_ingress = create_general_datasource('gather_freesurfer_dir') + + if "freesurfer_dir" in data_paths["anat"]: + fs_ingress = create_general_datasource("gather_freesurfer_dir") fs_ingress.inputs.inputnode.set( unique_id=unique_id, - data=data_paths['anat']['freesurfer_dir'], - creds_path=data_paths['creds_path'], - dl_dir=cfg.pipeline_setup['working_directory']['path']) - rpool.set_data("freesurfer-subject-dir", fs_ingress, 'outputspec.data', - {}, "", "freesurfer_config_ingress") + data=data_paths["anat"]["freesurfer_dir"], + creds_path=data_paths["creds_path"], + dl_dir=cfg.pipeline_setup["working_directory"]["path"], + ) + rpool.set_data( + "freesurfer-subject-dir", + fs_ingress, + "outputspec.data", + {}, + "", + "freesurfer_config_ingress", + ) recon_outs = { - 'pipeline-fs_raw-average': 'mri/rawavg.mgz', - 'pipeline-fs_subcortical-seg': 'mri/aseg.mgz', - 'pipeline-fs_brainmask': 'mri/brainmask.mgz', - 'pipeline-fs_wmparc': 'mri/wmparc.mgz', - 'pipeline-fs_T1': 'mri/T1.mgz', - 'pipeline-fs_hemi-L_desc-surface_curv': 'surf/lh.curv', - 'pipeline-fs_hemi-R_desc-surface_curv': 'surf/rh.curv', - 'pipeline-fs_hemi-L_desc-surfaceMesh_pial': 'surf/lh.pial', - 'pipeline-fs_hemi-R_desc-surfaceMesh_pial': 'surf/rh.pial', - 'pipeline-fs_hemi-L_desc-surfaceMesh_smoothwm': 'surf/lh.smoothwm', - 'pipeline-fs_hemi-R_desc-surfaceMesh_smoothwm': 'surf/rh.smoothwm', - 'pipeline-fs_hemi-L_desc-surfaceMesh_sphere': 'surf/lh.sphere', - 'pipeline-fs_hemi-R_desc-surfaceMesh_sphere': 'surf/rh.sphere', - 'pipeline-fs_hemi-L_desc-surfaceMap_sulc': 'surf/lh.sulc', - 'pipeline-fs_hemi-R_desc-surfaceMap_sulc': 'surf/rh.sulc', - 'pipeline-fs_hemi-L_desc-surfaceMap_thickness': 'surf/lh.thickness', - 'pipeline-fs_hemi-R_desc-surfaceMap_thickness': 'surf/rh.thickness', - 'pipeline-fs_hemi-L_desc-surfaceMap_volume': 'surf/lh.volume', - 'pipeline-fs_hemi-R_desc-surfaceMap_volume': 'surf/rh.volume', - 'pipeline-fs_hemi-L_desc-surfaceMesh_white': 'surf/lh.white', - 'pipeline-fs_hemi-R_desc-surfaceMesh_white': 'surf/rh.white', - 'pipeline-fs_xfm': 'mri/transforms/talairach.lta' + "pipeline-fs_raw-average": "mri/rawavg.mgz", + "pipeline-fs_subcortical-seg": "mri/aseg.mgz", + "pipeline-fs_brainmask": "mri/brainmask.mgz", + "pipeline-fs_wmparc": "mri/wmparc.mgz", + "pipeline-fs_T1": "mri/T1.mgz", + "pipeline-fs_hemi-L_desc-surface_curv": "surf/lh.curv", + "pipeline-fs_hemi-R_desc-surface_curv": "surf/rh.curv", + "pipeline-fs_hemi-L_desc-surfaceMesh_pial": "surf/lh.pial", + "pipeline-fs_hemi-R_desc-surfaceMesh_pial": "surf/rh.pial", + "pipeline-fs_hemi-L_desc-surfaceMesh_smoothwm": "surf/lh.smoothwm", + "pipeline-fs_hemi-R_desc-surfaceMesh_smoothwm": "surf/rh.smoothwm", + "pipeline-fs_hemi-L_desc-surfaceMesh_sphere": "surf/lh.sphere", + "pipeline-fs_hemi-R_desc-surfaceMesh_sphere": "surf/rh.sphere", + "pipeline-fs_hemi-L_desc-surfaceMap_sulc": "surf/lh.sulc", + "pipeline-fs_hemi-R_desc-surfaceMap_sulc": "surf/rh.sulc", + "pipeline-fs_hemi-L_desc-surfaceMap_thickness": "surf/lh.thickness", + "pipeline-fs_hemi-R_desc-surfaceMap_thickness": "surf/rh.thickness", + "pipeline-fs_hemi-L_desc-surfaceMap_volume": "surf/lh.volume", + "pipeline-fs_hemi-R_desc-surfaceMap_volume": "surf/rh.volume", + "pipeline-fs_hemi-L_desc-surfaceMesh_white": "surf/lh.white", + "pipeline-fs_hemi-R_desc-surfaceMesh_white": "surf/rh.white", + "pipeline-fs_xfm": "mri/transforms/talairach.lta", } - + for key, outfile in recon_outs.items(): - fullpath = os.path.join(data_paths['anat']['freesurfer_dir'], - outfile) + fullpath = os.path.join(data_paths["anat"]["freesurfer_dir"], outfile) if os.path.exists(fullpath): - fs_ingress = create_general_datasource(f'gather_fs_{key}_dir') + fs_ingress = create_general_datasource(f"gather_fs_{key}_dir") fs_ingress.inputs.inputnode.set( unique_id=unique_id, data=fullpath, - creds_path=data_paths['creds_path'], - dl_dir=cfg.pipeline_setup['working_directory']['path']) - rpool.set_data(key, fs_ingress, 'outputspec.data', - {}, "", f"fs_{key}_ingress") + creds_path=data_paths["creds_path"], + dl_dir=cfg.pipeline_setup["working_directory"]["path"], + ) + rpool.set_data( + key, fs_ingress, "outputspec.data", {}, "", f"fs_{key}_ingress" + ) else: - warnings.warn(str( - LookupError("\n[!] Path does not exist for " - f"{fullpath}.\n"))) - + warnings.warn( + str(LookupError("\n[!] Path does not exist for " f"{fullpath}.\n")) + ) + return rpool -def ingress_raw_func_data(wf, rpool, cfg, data_paths, unique_id, part_id, - ses_id): - func_paths_dct = data_paths['func'] +def ingress_raw_func_data(wf, rpool, cfg, data_paths, unique_id, part_id, ses_id): + func_paths_dct = data_paths["func"] - func_wf = create_func_datasource(func_paths_dct, rpool, - f'func_ingress_{part_id}_{ses_id}') + func_wf = create_func_datasource( + func_paths_dct, rpool, f"func_ingress_{part_id}_{ses_id}" + ) func_wf.inputs.inputnode.set( subject=part_id, - creds_path=data_paths['creds_path'], - dl_dir=cfg.pipeline_setup['working_directory']['path'] + creds_path=data_paths["creds_path"], + dl_dir=cfg.pipeline_setup["working_directory"]["path"], ) - func_wf.get_node('inputnode').iterables = \ - ("scan", list(func_paths_dct.keys())) - - rpool.set_data('subject', func_wf, 'outputspec.subject', {}, "", - "func_ingress") - rpool.set_data('bold', func_wf, 'outputspec.rest', {}, "", "func_ingress") - rpool.set_data('scan', func_wf, 'outputspec.scan', {}, "", "func_ingress") - rpool.set_data('scan-params', func_wf, 'outputspec.scan_params', {}, "", - "scan_params_ingress") - + func_wf.get_node("inputnode").iterables = ("scan", list(func_paths_dct.keys())) + + rpool.set_data("subject", func_wf, "outputspec.subject", {}, "", "func_ingress") + rpool.set_data("bold", func_wf, "outputspec.rest", {}, "", "func_ingress") + rpool.set_data("scan", func_wf, "outputspec.scan", {}, "", "func_ingress") + rpool.set_data( + "scan-params", func_wf, "outputspec.scan_params", {}, "", "scan_params_ingress" + ) + # TODO: CHECK FOR PARAMETERS - wf, rpool, diff, blip, fmap_rp_list = \ - ingress_func_metadata(wf, cfg, rpool, data_paths, part_id, - data_paths['creds_path'], ses_id) + wf, rpool, diff, blip, fmap_rp_list = ingress_func_metadata( + wf, cfg, rpool, data_paths, part_id, data_paths["creds_path"], ses_id + ) # Memoize list of local functional scans # TODO: handle S3 files # Skip S3 files for now local_func_scans = [ - func_paths_dct[scan]['scan'] for scan in func_paths_dct.keys() if not - func_paths_dct[scan]['scan'].startswith('s3://')] + func_paths_dct[scan]["scan"] + for scan in func_paths_dct.keys() + if not func_paths_dct[scan]["scan"].startswith("s3://") + ] if local_func_scans: # pylint: disable=protected-access wf._local_func_scans = local_func_scans - if cfg.pipeline_setup['Debugging']['verbose']: - verbose_logger = getLogger('engine') - verbose_logger.debug('local_func_scans: %s', local_func_scans) + if cfg.pipeline_setup["Debugging"]["verbose"]: + verbose_logger = getLogger("engine") + verbose_logger.debug("local_func_scans: %s", local_func_scans) del local_func_scans return (wf, rpool, diff, blip, fmap_rp_list) -def ingress_output_dir(wf, cfg, rpool, unique_id, data_paths, part_id, ses_id, creds_path=None): - - dir_path = data_paths['derivatives_dir'] - - print(f"\nPulling outputs from {dir_path}.\n") +def ingress_output_dir( + wf, cfg, rpool, unique_id, data_paths, part_id, ses_id, creds_path=None +): + dir_path = data_paths["derivatives_dir"] - anat = os.path.join(dir_path, 'anat') - func = os.path.join(dir_path, 'func') + anat = os.path.join(dir_path, "anat") + func = os.path.join(dir_path, "func") - exts = ['.nii', '.gz', '.mat', '.1D', '.txt', '.csv', '.rms', '.tsv'] + exts = [".nii", ".gz", ".mat", ".1D", ".txt", ".csv", ".rms", ".tsv"] outdir_anat = [] outdir_func = [] @@ -1856,89 +2066,103 @@ def ingress_output_dir(wf, cfg, rpool, unique_id, data_paths, part_id, ses_id, c for ext in exts: if ext in filename: if subdir == anat: - outdir_anat.append(os.path.join(subdir, - filename)) + outdir_anat.append(os.path.join(subdir, filename)) else: - outdir_func.append(os.path.join(subdir, - filename)) + outdir_func.append(os.path.join(subdir, filename)) - # Add derivatives directory to rpool - ingress = create_general_datasource(f'gather_derivatives_dir') + # Add derivatives directory to rpool + ingress = create_general_datasource("gather_derivatives_dir") ingress.inputs.inputnode.set( - unique_id=unique_id, - data=dir_path, - creds_path=creds_path, - dl_dir=cfg.pipeline_setup['working_directory']['path'] - ) - rpool.set_data("derivatives-dir", ingress, 'outputspec.data', - {}, "", "outdir_config_ingress") + unique_id=unique_id, + data=dir_path, + creds_path=creds_path, + dl_dir=cfg.pipeline_setup["working_directory"]["path"], + ) + rpool.set_data( + "derivatives-dir", ingress, "outputspec.data", {}, "", "outdir_config_ingress" + ) for subdir in [outdir_anat, outdir_func]: for filepath in subdir: filename = str(filepath) for ext in exts: - filename = filename.split("/")[-1].replace(ext, '') + filename = filename.split("/")[-1].replace(ext, "") - data_label = filename.split(unique_id)[1].lstrip('_') + data_label = filename.split(unique_id)[1].lstrip("_") if len(filename) == len(data_label): - raise Exception('\n\n[!] Possibly wrong participant or ' - 'session in this directory?\n\n' - f'Filepath: {filepath}\n\n') + raise Exception( + "\n\n[!] Possibly wrong participant or " + "session in this directory?\n\n" + f"Filepath: {filepath}\n\n" + ) - bidstag = '' - for tag in data_label.split('_'): - for prefix in ['task-', 'run-', 'acq-', 'rec']: + bidstag = "" + for tag in data_label.split("_"): + for prefix in ["task-", "run-", "acq-", "rec"]: if tag.startswith(prefix): - bidstag += f'{tag}_' - data_label = data_label.replace(f'{tag}_', '') + bidstag += f"{tag}_" + data_label = data_label.replace(f"{tag}_", "") data_label, json = strip_template(data_label, dir_path, filename) - rpool, json_info, pipe_idx, node_name, data_label = \ - json_outdir_ingress(rpool, filepath, \ - exts, data_label, json) + rpool, json_info, pipe_idx, node_name, data_label = json_outdir_ingress( + rpool, filepath, exts, data_label, json + ) - if ('template' in data_label and not json_info['Template'] == \ - cfg.pipeline_setup['outdir_ingress']['Template']): + if ( + "template" in data_label + and not json_info["Template"] + == cfg.pipeline_setup["outdir_ingress"]["Template"] + ): continue # Rename confounds to avoid confusion in nuisance regression - if data_label.endswith('desc-confounds_timeseries'): - data_label = 'pipeline-ingress_desc-confounds_timeseries' + if data_label.endswith("desc-confounds_timeseries"): + data_label = "pipeline-ingress_desc-confounds_timeseries" if len(bidstag) > 1: # Remove tail symbol bidstag = bidstag[:-1] - if bidstag.startswith('task-'): - bidstag = bidstag.replace('task-', '') + if bidstag.startswith("task-"): + bidstag = bidstag.replace("task-", "") # Rename bold mask for CPAC naming convention # and to avoid collision with anat brain mask - if data_label.endswith('desc-brain_mask') and filepath in outdir_func: - data_label = data_label.replace('brain_mask', 'bold_mask') + if data_label.endswith("desc-brain_mask") and filepath in outdir_func: + data_label = data_label.replace("brain_mask", "bold_mask") try: pipe_x = rpool.get_pipe_number(pipe_idx) except ValueError: pipe_x = len(rpool.pipe_list) if filepath in outdir_anat: - ingress = create_general_datasource(f'gather_anat_outdir_{str(data_label)}_{pipe_x}') + ingress = create_general_datasource( + f"gather_anat_outdir_{data_label!s}_{pipe_x}" + ) ingress.inputs.inputnode.set( unique_id=unique_id, data=filepath, creds_path=creds_path, - dl_dir=cfg.pipeline_setup['working_directory']['path'] + dl_dir=cfg.pipeline_setup["working_directory"]["path"], + ) + rpool.set_data( + data_label, + ingress, + "outputspec.data", + json_info, + pipe_idx, + node_name, + f"outdir_{data_label}_ingress", + inject=True, ) - rpool.set_data(data_label, ingress, 'outputspec.data', json_info, - pipe_idx, node_name, f"outdir_{data_label}_ingress", inject=True) else: - if data_label.endswith('desc-preproc_bold'): + if data_label.endswith("desc-preproc_bold"): func_key = data_label func_dict[bidstag] = {} - func_dict[bidstag]['scan'] = str(filepath) - func_dict[bidstag]['scan_parameters'] = json_info - func_dict[bidstag]['pipe_idx'] = pipe_idx - if data_label.endswith('desc-brain_mask'): - data_label = data_label.replace('brain_mask', 'bold_mask') + func_dict[bidstag]["scan"] = str(filepath) + func_dict[bidstag]["scan_parameters"] = json_info + func_dict[bidstag]["pipe_idx"] = pipe_idx + if data_label.endswith("desc-brain_mask"): + data_label = data_label.replace("brain_mask", "bold_mask") try: func_paths[data_label].append(filepath) except: @@ -1946,166 +2170,189 @@ def ingress_output_dir(wf, cfg, rpool, unique_id, data_paths, part_id, ses_id, c func_paths[data_label].append(filepath) if func_dict: - wf, rpool = func_outdir_ingress(wf, cfg, func_dict, rpool, unique_id, \ - creds_path, part_id, func_key, func_paths) + wf, rpool = func_outdir_ingress( + wf, + cfg, + func_dict, + rpool, + unique_id, + creds_path, + part_id, + func_key, + func_paths, + ) - if cfg.surface_analysis['freesurfer']['ingress_reconall']: - rpool = ingress_freesurfer(wf, rpool, cfg, data_paths, unique_id, part_id, - ses_id) + if cfg.surface_analysis["freesurfer"]["ingress_reconall"]: + rpool = ingress_freesurfer( + wf, rpool, cfg, data_paths, unique_id, part_id, ses_id + ) return wf, rpool + def json_outdir_ingress(rpool, filepath, exts, data_label, json): - desc_val = None - for tag in data_label.split('_'): - if 'desc-' in tag: + for tag in data_label.split("_"): + if "desc-" in tag: desc_val = tag break jsonpath = str(filepath) for ext in exts: - jsonpath = jsonpath.replace(ext, '') + jsonpath = jsonpath.replace(ext, "") jsonpath = f"{jsonpath}.json" if not os.path.exists(jsonpath): - print(f'\n\n[!] No JSON found for file {filepath}.\nCreating ' - f'{jsonpath}..\n\n') json_info = { - 'Description': 'This data was generated elsewhere and ' - 'supplied by the user into this C-PAC run\'s ' - 'output directory. This JSON file was ' - 'automatically generated by C-PAC because a ' - 'JSON file was not supplied with the data.' + "Description": "This data was generated elsewhere and " + "supplied by the user into this C-PAC run's " + "output directory. This JSON file was " + "automatically generated by C-PAC because a " + "JSON file was not supplied with the data." } json_info = {**json_info, **json} write_output_json(json_info, jsonpath) else: json_info = read_json(jsonpath) json_info = {**json_info, **json} - if 'CpacProvenance' in json_info: + if "CpacProvenance" in json_info: if desc_val: # it's a C-PAC output, let's check for pipe_idx/strat integer # suffixes in the desc- entries. only_desc = str(desc_val) - + if only_desc[-1].isdigit(): for idx in range(0, 3): # let's stop at 3, please don't run >999 strategies okay? if only_desc[-1].isdigit(): only_desc = only_desc[:-1] - - if only_desc[-1] == '-': - only_desc = only_desc.rstrip('-') - else: - raise Exception('\n[!] Something went wrong with either ' - 'reading in the output directory or when ' - 'it was written out previously.\n\nGive ' - 'this to your friendly local C-PAC ' - f'developer:\n\n{str(data_label)}\n') - # remove the integer at the end of the desc-* variant, we will + if only_desc[-1] == "-": + only_desc = only_desc.rstrip("-") + else: + raise Exception( + "\n[!] Something went wrong with either " + "reading in the output directory or when " + "it was written out previously.\n\nGive " + "this to your friendly local C-PAC " + f"developer:\n\n{data_label!s}\n" + ) + + # remove the integer at the end of the desc-* variant, we will # get the unique pipe_idx from the CpacProvenance below data_label = data_label.replace(desc_val, only_desc) # preserve cpac provenance/pipe_idx - pipe_idx = rpool.generate_prov_string(json_info['CpacProvenance']) + pipe_idx = rpool.generate_prov_string(json_info["CpacProvenance"]) node_name = "" - + else: - json_info['CpacProvenance'] = [f'{data_label}:Non-C-PAC Origin: {filepath}'] - if not 'Description' in json_info: - json_info['Description'] = 'This data was generated elsewhere and ' \ - 'supplied by the user into this C-PAC run\'s '\ - 'output directory. This JSON file was '\ - 'automatically generated by C-PAC because a '\ - 'JSON file was not supplied with the data.' - pipe_idx = rpool.generate_prov_string(json_info['CpacProvenance']) + json_info["CpacProvenance"] = [f"{data_label}:Non-C-PAC Origin: {filepath}"] + if "Description" not in json_info: + json_info["Description"] = ( + "This data was generated elsewhere and " + "supplied by the user into this C-PAC run's " + "output directory. This JSON file was " + "automatically generated by C-PAC because a " + "JSON file was not supplied with the data." + ) + pipe_idx = rpool.generate_prov_string(json_info["CpacProvenance"]) node_name = f"{data_label}_ingress" return rpool, json_info, pipe_idx, node_name, data_label -def func_outdir_ingress(wf, cfg, func_dict, rpool, unique_id, creds_path, part_id, key, \ - func_paths): + +def func_outdir_ingress( + wf, cfg, func_dict, rpool, unique_id, creds_path, part_id, key, func_paths +): pipe_x = len(rpool.pipe_list) - exts = ['.nii', '.gz', '.mat', '.1D', '.txt', '.csv', '.rms', '.tsv'] - ingress = create_func_datasource(func_dict, rpool, f'gather_func_outdir_{key}_{pipe_x}') + ingress = create_func_datasource( + func_dict, rpool, f"gather_func_outdir_{key}_{pipe_x}" + ) ingress.inputs.inputnode.set( subject=unique_id, creds_path=creds_path, - dl_dir=cfg.pipeline_setup['working_directory']['path'] + dl_dir=cfg.pipeline_setup["working_directory"]["path"], + ) + rpool.set_data("subject", ingress, "outputspec.subject", {}, "", "func_ingress") + ingress.get_node("inputnode").iterables = ("scan", list(func_dict.keys())) + rpool.set_data(key, ingress, "outputspec.rest", {}, "", "func_ingress") + + rpool.set_data("scan", ingress, "outputspec.scan", {}, "", "func_ingress") + rpool.set_data( + "scan-params", ingress, "outputspec.scan_params", {}, "", "scan_params_ingress" ) - rpool.set_data('subject', ingress, 'outputspec.subject', {}, "", - "func_ingress") - ingress.get_node('inputnode').iterables = \ - ("scan", list(func_dict.keys())) - rpool.set_data(key, ingress, 'outputspec.rest', {}, "", - "func_ingress") - - rpool.set_data('scan', ingress, 'outputspec.scan', {}, "", 'func_ingress') - rpool.set_data('scan-params', ingress, 'outputspec.scan_params', {}, "", - "scan_params_ingress") - wf, rpool, diff, blip, fmap_rp_list = ingress_func_metadata(wf, cfg, \ - rpool, func_dict, part_id, creds_path, key) - + wf, rpool, diff, blip, fmap_rp_list = ingress_func_metadata( + wf, cfg, rpool, func_dict, part_id, creds_path, key + ) + # Have to do it this weird way to save the parsed BIDS tag & filepath - mask_paths_key = 'desc-bold_mask' if 'desc-bold_mask' in func_paths else \ - 'space-template_desc-bold_mask' - ts_paths_key = 'pipeline-ingress_desc-confounds_timeseries' + mask_paths_key = ( + "desc-bold_mask" + if "desc-bold_mask" in func_paths + else "space-template_desc-bold_mask" + ) + ts_paths_key = "pipeline-ingress_desc-confounds_timeseries" # Connect func data with approproate scan name - iterables = pe.Node(Function(input_names=['scan', - 'mask_paths', - 'ts_paths'], - output_names=['out_scan', - 'mask', - 'confounds'], - function=set_iterables), - name=f'set_iterables_{pipe_x}') + iterables = pe.Node( + Function( + input_names=["scan", "mask_paths", "ts_paths"], + output_names=["out_scan", "mask", "confounds"], + function=set_iterables, + ), + name=f"set_iterables_{pipe_x}", + ) iterables.inputs.mask_paths = func_paths[mask_paths_key] iterables.inputs.ts_paths = func_paths[ts_paths_key] - wf.connect(ingress, 'outputspec.scan', iterables, 'scan') + wf.connect(ingress, "outputspec.scan", iterables, "scan") for key in func_paths: - if key == mask_paths_key or key == ts_paths_key: - ingress_func = create_general_datasource(f'ingress_func_data_{key}') + if key in (mask_paths_key, ts_paths_key): + ingress_func = create_general_datasource(f"ingress_func_data_{key}") ingress_func.inputs.inputnode.set( unique_id=unique_id, creds_path=creds_path, - dl_dir=cfg.pipeline_setup['working_directory']['path']) - wf.connect(iterables, 'out_scan', ingress_func, 'inputnode.scan') + dl_dir=cfg.pipeline_setup["working_directory"]["path"], + ) + wf.connect(iterables, "out_scan", ingress_func, "inputnode.scan") if key == mask_paths_key: - wf.connect(iterables, 'mask', ingress_func, 'inputnode.data') - rpool.set_data(key, ingress_func, 'inputnode.data', {}, "", f"outdir_{key}_ingress") + wf.connect(iterables, "mask", ingress_func, "inputnode.data") + rpool.set_data( + key, ingress_func, "inputnode.data", {}, "", f"outdir_{key}_ingress" + ) elif key == ts_paths_key: - wf.connect(iterables, 'confounds', ingress_func, 'inputnode.data') - rpool.set_data(key, ingress_func, 'inputnode.data', {}, "", f"outdir_{key}_ingress") + wf.connect(iterables, "confounds", ingress_func, "inputnode.data") + rpool.set_data( + key, ingress_func, "inputnode.data", {}, "", f"outdir_{key}_ingress" + ) return wf, rpool + def set_iterables(scan, mask_paths=None, ts_paths=None): - # match scan with filepath to get filepath mask_path = [path for path in mask_paths if scan in path] ts_path = [path for path in ts_paths if scan in path] - return (scan, mask_path[0], ts_path[0]) + return (scan, mask_path[0], ts_path[0]) + def strip_template(data_label, dir_path, filename): - json = {} - # rename to template - for prefix in ['space-', 'from-', 'to-']: - for bidstag in data_label.split('_'): + # rename to template + for prefix in ["space-", "from-", "to-"]: + for bidstag in data_label.split("_"): if bidstag.startswith(prefix): - template_key, template_val = bidstag.split('-') + template_key, template_val = bidstag.split("-") template_name, _template_desc = lookup_identifier(template_val) if template_name: - json['Template'] = template_val - data_label = data_label.replace(template_val, 'template') - elif bidstag.startswith('res-'): - res_key, res_val = bidstag.split('-') - json['Resolution'] = res_val - data_label = data_label.replace(bidstag, '') - if data_label.find('__'): data_label = data_label.replace('__', '_') + json["Template"] = template_val + data_label = data_label.replace(template_val, "template") + elif bidstag.startswith("res-"): + res_key, res_val = bidstag.split("-") + json["Resolution"] = res_val + data_label = data_label.replace(bidstag, "") + if data_label.find("__"): + data_label = data_label.replace("__", "_") return data_label, json @@ -2113,18 +2360,16 @@ def ingress_pipeconfig_paths(cfg, rpool, unique_id, creds_path=None): # ingress config file paths # TODO: may want to change the resource keys for each to include one level up in the YAML as well - import pkg_resources as p import pandas as pd - import ast + import pkg_resources as p - template_csv = p.resource_filename('CPAC', 'resources/cpac_templates.csv') + template_csv = p.resource_filename("CPAC", "resources/cpac_templates.csv") template_df = pd.read_csv(template_csv, keep_default_na=False) - + for row in template_df.itertuples(): - key = row.Key val = row.Pipeline_Config_Entry - val = cfg.get_nested(cfg, [x.lstrip() for x in val.split(',')]) + val = cfg.get_nested(cfg, [x.lstrip() for x in val.split(",")]) resolution = row.Intended_Resolution_Config_Entry desc = row.Description @@ -2132,72 +2377,96 @@ def ingress_pipeconfig_paths(cfg, rpool, unique_id, creds_path=None): continue if resolution: - res_keys = [x.lstrip() for x in resolution.split(',')] + res_keys = [x.lstrip() for x in resolution.split(",")] tag = res_keys[-1] - json_info = {} - - if '$FSLDIR' in val: - val = val.replace('$FSLDIR', cfg.pipeline_setup[ - 'system_config']['FSLDIR']) - if '$priors_path' in val: - priors_path = cfg.segmentation['tissue_segmentation']['FSL-FAST']['use_priors']['priors_path'] or '' - if '$FSLDIR' in priors_path: - priors_path = priors_path.replace('$FSLDIR', cfg.pipeline_setup['system_config']['FSLDIR']) - val = val.replace('$priors_path', priors_path) - if '${resolution_for_anat}' in val: - val = val.replace('${resolution_for_anat}', cfg.registration_workflows['anatomical_registration']['resolution_for_anat']) - if '${func_resolution}' in val: - val = val.replace('${func_resolution}', cfg.registration_workflows[ - 'functional_registration']['func_registration_to_template'][ - 'output_resolution'][tag]) + json_info = {} + + if "$FSLDIR" in val: + val = val.replace("$FSLDIR", cfg.pipeline_setup["system_config"]["FSLDIR"]) + if "$priors_path" in val: + priors_path = ( + cfg.segmentation["tissue_segmentation"]["FSL-FAST"]["use_priors"][ + "priors_path" + ] + or "" + ) + if "$FSLDIR" in priors_path: + priors_path = priors_path.replace( + "$FSLDIR", cfg.pipeline_setup["system_config"]["FSLDIR"] + ) + val = val.replace("$priors_path", priors_path) + if "${resolution_for_anat}" in val: + val = val.replace( + "${resolution_for_anat}", + cfg.registration_workflows["anatomical_registration"][ + "resolution_for_anat" + ], + ) + if "${func_resolution}" in val: + val = val.replace( + "${func_resolution}", + cfg.registration_workflows["functional_registration"][ + "func_registration_to_template" + ]["output_resolution"][tag], + ) if desc: template_name, _template_desc = lookup_identifier(val) if template_name: desc = f"{template_name} - {desc}" - json_info['Description'] = f"{desc} - {val}" + json_info["Description"] = f"{desc} - {val}" if resolution: resolution = cfg.get_nested(cfg, res_keys) - json_info['Resolution'] = resolution - - resampled_template = pe.Node(Function(input_names=['resolution', - 'template', - 'template_name', - 'tag'], - output_names=['resampled_template'], - function=resolve_resolution, - as_module=True), - name='resampled_' + key) + json_info["Resolution"] = resolution + + resampled_template = pe.Node( + Function( + input_names=["resolution", "template", "template_name", "tag"], + output_names=["resampled_template"], + function=resolve_resolution, + as_module=True, + ), + name="resampled_" + key, + ) resampled_template.inputs.resolution = resolution resampled_template.inputs.template = val resampled_template.inputs.template_name = key resampled_template.inputs.tag = tag - + # the set_data below is set up a little differently, because we are # injecting and also over-writing already-existing entries # other alternative would have been to ingress into the # resampled_template node from the already existing entries, but we # didn't do that here - rpool.set_data(key, - resampled_template, - 'resampled_template', - json_info, "", - "template_resample") #, inject=True) # pipe_idx (after the blank json {}) should be the previous strat that you want deleted! because you're not connecting this the regular way, you have to do it manually + rpool.set_data( + key, + resampled_template, + "resampled_template", + json_info, + "", + "template_resample", + ) # , inject=True) # pipe_idx (after the blank json {}) should be the previous strat that you want deleted! because you're not connecting this the regular way, you have to do it manually else: if val: - config_ingress = create_general_datasource(f'gather_{key}') + config_ingress = create_general_datasource(f"gather_{key}") config_ingress.inputs.inputnode.set( unique_id=unique_id, data=val, creds_path=creds_path, - dl_dir=cfg.pipeline_setup['working_directory']['path'] + dl_dir=cfg.pipeline_setup["working_directory"]["path"], + ) + rpool.set_data( + key, + config_ingress, + "outputspec.data", + json_info, + "", + f"{key}_config_ingress", ) - rpool.set_data(key, config_ingress, 'outputspec.data', - json_info, "", f"{key}_config_ingress") # templates, resampling from config - ''' + """ template_keys = [ ("anat", ["network_centrality", "template_specification_file"]), ("anat", ["nuisance_corrections", "2-nuisance_regression", @@ -2280,13 +2549,13 @@ def _set_nested(attr, keys): map_node=True ) cfg.set_nested(cfg, key, node) - ''' + """ return rpool def initiate_rpool(wf, cfg, data_paths=None, part_id=None): - ''' + """ data_paths format: {'anat': { @@ -2304,22 +2573,21 @@ def initiate_rpool(wf, cfg, data_paths=None, part_id=None): 'site_id': 'site-ID', 'subject_id': 'sub-01', 'unique_id': 'ses-1', - 'derivatives_dir': '{derivatives_dir path}'} - ''' - + 'derivatives_dir': '{derivatives_dir path}'}. + """ # TODO: refactor further, integrate with the ingress_data functionality # TODO: used for BIDS-Derivatives (below), and possible refactoring of # TODO: the raw data config to use 'T1w' label instead of 'anat' etc. if data_paths: - part_id = data_paths['subject_id'] - ses_id = data_paths['unique_id'] - if 'creds_path' not in data_paths: + part_id = data_paths["subject_id"] + ses_id = data_paths["unique_id"] + if "creds_path" not in data_paths: creds_path = None else: - creds_path = data_paths['creds_path'] - unique_id = f'{part_id}_{ses_id}' - + creds_path = data_paths["creds_path"] + unique_id = f"{part_id}_{ses_id}" + elif part_id: unique_id = part_id creds_path = None @@ -2328,18 +2596,29 @@ def initiate_rpool(wf, cfg, data_paths=None, part_id=None): if data_paths: # ingress outdir - try: - if data_paths['derivatives_dir'] and cfg.pipeline_setup['outdir_ingress']['run']: - wf, rpool = \ - ingress_output_dir(wf, cfg, rpool, unique_id, data_paths, part_id, \ - ses_id, creds_path=None) + try: + if ( + data_paths["derivatives_dir"] + and cfg.pipeline_setup["outdir_ingress"]["run"] + ): + wf, rpool = ingress_output_dir( + wf, + cfg, + rpool, + unique_id, + data_paths, + part_id, + ses_id, + creds_path=None, + ) except: - rpool = ingress_raw_anat_data(wf, rpool, cfg, data_paths, unique_id, - part_id, ses_id) - if 'func' in data_paths: - wf, rpool, diff, blip, fmap_rp_list = \ - ingress_raw_func_data(wf, rpool, cfg, data_paths, unique_id, - part_id, ses_id) + rpool = ingress_raw_anat_data( + wf, rpool, cfg, data_paths, unique_id, part_id, ses_id + ) + if "func" in data_paths: + wf, rpool, diff, blip, fmap_rp_list = ingress_raw_func_data( + wf, rpool, cfg, data_paths, unique_id, part_id, ses_id + ) # grab any file paths from the pipeline config YAML rpool = ingress_pipeconfig_paths(cfg, rpool, unique_id, creds_path) @@ -2351,45 +2630,42 @@ def initiate_rpool(wf, cfg, data_paths=None, part_id=None): def run_node_blocks(blocks, data_paths, cfg=None): import os + from CPAC.pipeline import nipype_pipeline_engine as pe from CPAC.pipeline.engine import NodeBlock if not cfg: cfg = { - 'pipeline_setup': { - 'working_directory': { - 'path': os.getcwd() - }, - 'log_directory': { - 'path': os.getcwd() - } + "pipeline_setup": { + "working_directory": {"path": os.getcwd()}, + "log_directory": {"path": os.getcwd()}, } } # TODO: WE HAVE TO PARSE OVER UNIQUE ID'S!!! _, rpool = initiate_rpool(cfg, data_paths) - wf = pe.Workflow(name='node_blocks') - wf.base_dir = cfg.pipeline_setup['working_directory']['path'] - wf.config['execution'] = { - 'hash_method': 'timestamp', - 'crashdump_dir': cfg.pipeline_setup['log_directory']['path'] + wf = pe.Workflow(name="node_blocks") + wf.base_dir = cfg.pipeline_setup["working_directory"]["path"] + wf.config["execution"] = { + "hash_method": "timestamp", + "crashdump_dir": cfg.pipeline_setup["log_directory"]["path"], } run_blocks = [] - if rpool.check_rpool('desc-preproc_T1w'): - print("Preprocessed T1w found, skipping anatomical preprocessing.") + if rpool.check_rpool("desc-preproc_T1w"): + pass else: run_blocks += blocks[0] - if rpool.check_rpool('desc-preproc_bold'): - print("Preprocessed BOLD found, skipping functional preprocessing.") + if rpool.check_rpool("desc-preproc_bold"): + pass else: run_blocks += blocks[1] for block in run_blocks: - wf = NodeBlock(block, debug=cfg['pipeline_setup', 'Debugging', - 'verbose']).connect_block( - wf, cfg, rpool) + wf = NodeBlock( + block, debug=cfg["pipeline_setup", "Debugging", "verbose"] + ).connect_block(wf, cfg, rpool) rpool.gather_pipes(wf, cfg) wf.run() @@ -2397,7 +2673,7 @@ def run_node_blocks(blocks, data_paths, cfg=None): class NodeData: r"""Class to hold outputs of - CPAC.pipeline.engine.ResourcePool().get_data(), so one can do + CPAC.pipeline.engine.ResourcePool().get_data(), so one can do. ``node_data = strat_pool.node_data(resource)`` and have ``node_data.node`` and ``node_data.out`` instead of doing @@ -2427,6 +2703,7 @@ class NodeData: ... print(str(lookup_error).strip().split('\n')[0].strip()) [!] C-PAC says: None of the listed resources are in the resource pool: """ + # pylint: disable=too-few-public-methods def __init__(self, strat_pool=None, resource=None, **kwargs): self.node = NotImplemented diff --git a/CPAC/pipeline/nipype_pipeline_engine/__init__.py b/CPAC/pipeline/nipype_pipeline_engine/__init__.py index fc346b5068..fef097b47b 100644 --- a/CPAC/pipeline/nipype_pipeline_engine/__init__.py +++ b/CPAC/pipeline/nipype_pipeline_engine/__init__.py @@ -14,21 +14,36 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -'''Module to import Nipype Pipeline engine and override some Classes. +"""Module to import Nipype Pipeline engine and override some Classes. See https://fcp-indi.github.io/docs/developer/nodes for C-PAC-specific documentation. See https://nipype.readthedocs.io/en/latest/api/generated/nipype.pipeline.engine.html -for Nipype's documentation.''' # noqa: E501 # pylint: disable=line-too-long +for Nipype's documentation. +""" # pylint: disable=line-too-long from nipype.pipeline import engine as pe + # import everything in nipype.pipeline.engine.__all__ -from nipype.pipeline.engine import * # noqa: F401,F403 +from nipype.pipeline.engine import * # noqa: F403 + # import our DEFAULT_MEM_GB and override Node, MapNode -from .engine import DEFAULT_MEM_GB, export_graph, get_data_size, Node, \ - MapNode, UNDEFINED_SIZE, Workflow +from .engine import ( + DEFAULT_MEM_GB, + UNDEFINED_SIZE, + MapNode, + Node, + Workflow, + export_graph, + get_data_size, +) -__all__ = [ - interface for interface in dir(pe) if not interface.startswith('_') -] + ['DEFAULT_MEM_GB', 'export_graph', 'get_data_size', 'Node', 'MapNode', - 'UNDEFINED_SIZE', 'Workflow'] +__all__ = [interface for interface in dir(pe) if not interface.startswith("_")] + [ + "DEFAULT_MEM_GB", + "export_graph", + "get_data_size", + "Node", + "MapNode", + "UNDEFINED_SIZE", + "Workflow", +] del pe diff --git a/CPAC/pipeline/nipype_pipeline_engine/engine.py b/CPAC/pipeline/nipype_pipeline_engine/engine.py index 899f69bbcf..e93aacd7a3 100644 --- a/CPAC/pipeline/nipype_pipeline_engine/engine.py +++ b/CPAC/pipeline/nipype_pipeline_engine/engine.py @@ -42,32 +42,35 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -'''Module to import Nipype Pipeline engine and override some Classes. +"""Module to import Nipype Pipeline engine and override some Classes. See https://fcp-indi.github.io/docs/developer/nodes for C-PAC-specific documentation. See https://nipype.readthedocs.io/en/latest/api/generated/nipype.pipeline.engine.html -for Nipype's documentation.''' # noqa: E501 # pylint: disable=line-too-long -import os -import re +for Nipype's documentation. +""" # pylint: disable=line-too-long from copy import deepcopy from inspect import Parameter, Signature, signature +import os +import re + +from numpy import prod +from traits.trait_base import Undefined +from traits.trait_handlers import TraitListObject from nibabel import load from nipype.interfaces.utility import Function from nipype.pipeline import engine as pe from nipype.pipeline.engine.utils import ( _create_dot_graph, + _replacefunk, + _run_dot, format_dot, generate_expanded_graph, get_print_name, load_resultfile as _load_resultfile, - _replacefunk, - _run_dot ) from nipype.utils.filemanip import fname_presuffix from nipype.utils.functions import getsource -from numpy import prod -from traits.trait_base import Undefined -from traits.trait_handlers import TraitListObject + from CPAC.utils.monitoring.custom_logging import getLogger # set global default mem_gb @@ -78,7 +81,7 @@ def _check_mem_x_path(mem_x_path): - '''Function to check if a supplied multiplier path exists. + """Function to check if a supplied multiplier path exists. Parameters ---------- @@ -87,17 +90,16 @@ def _check_mem_x_path(mem_x_path): Returns ------- bool - ''' + """ mem_x_path = _grab_first_path(mem_x_path) try: - return mem_x_path is not Undefined and os.path.exists( - mem_x_path) + return mem_x_path is not Undefined and os.path.exists(mem_x_path) except (TypeError, ValueError): return False def _doctest_skiplines(docstring, lines_to_skip): - ''' + """ Function to add ' # doctest: +SKIP' to the end of docstring lines to skip in imported docstrings. @@ -115,23 +117,21 @@ def _doctest_skiplines(docstring, lines_to_skip): -------- >>> _doctest_skiplines('skip this line', {'skip this line'}) 'skip this line # doctest: +SKIP' - ''' - if ( - not isinstance(lines_to_skip, set) and - not isinstance(lines_to_skip, list) - ): - raise TypeError( - '_doctest_skiplines: `lines_to_skip` must be a set or list.') - - return '\n'.join([ - f'{line} # doctest: +SKIP' if line in lines_to_skip else line - for line in docstring.split('\n') - ]) + """ + if not isinstance(lines_to_skip, set) and not isinstance(lines_to_skip, list): + raise TypeError("_doctest_skiplines: `lines_to_skip` must be a set or list.") + + return "\n".join( + [ + f"{line} # doctest: +SKIP" if line in lines_to_skip else line + for line in docstring.split("\n") + ] + ) def _grab_first_path(mem_x_path): - '''Function to grab the first path if multiple paths for given - multiplier input + """Function to grab the first path if multiple paths for given + multiplier input. Parameters ---------- @@ -140,7 +140,7 @@ def _grab_first_path(mem_x_path): Returns ------- str, Undefined or None - ''' + """ if isinstance(mem_x_path, (list, TraitListObject, tuple)): mem_x_path = mem_x_path[0] if len(mem_x_path) else Undefined return mem_x_path @@ -149,13 +149,13 @@ def _grab_first_path(mem_x_path): class Node(pe.Node): # pylint: disable=empty-docstring,too-many-instance-attributes __doc__ = _doctest_skiplines( - pe.Node.__doc__, - {" >>> realign.inputs.in_files = 'functional.nii'"} + pe.Node.__doc__, {" >>> realign.inputs.in_files = 'functional.nii'"} ) def __init__(self, *args, mem_gb=DEFAULT_MEM_GB, **kwargs): # pylint: disable=import-outside-toplevel from CPAC.pipeline.random_state import random_seed + super().__init__(*args, mem_gb=mem_gb, **kwargs) self.logger = getLogger("nipype.workflow") self.seed = random_seed() @@ -164,44 +164,48 @@ def __init__(self, *args, mem_gb=DEFAULT_MEM_GB, **kwargs): self._debug = False self.verbose_logger = None self._mem_x = {} - if 'mem_x' in kwargs and isinstance( - kwargs['mem_x'], (tuple, list) - ): - if len(kwargs['mem_x']) == 3: + if "mem_x" in kwargs and isinstance(kwargs["mem_x"], (tuple, list)): + if len(kwargs["mem_x"]) == 3: ( - self._mem_x['multiplier'], - self._mem_x['file'], - self._mem_x['mode'] - ) = kwargs['mem_x'] + self._mem_x["multiplier"], + self._mem_x["file"], + self._mem_x["mode"], + ) = kwargs["mem_x"] else: - self._mem_x['mode'] = 'xyzt' - if len(kwargs['mem_x']) == 2: - ( - self._mem_x['multiplier'], - self._mem_x['file'] - ) = kwargs['mem_x'] + self._mem_x["mode"] = "xyzt" + if len(kwargs["mem_x"]) == 2: + (self._mem_x["multiplier"], self._mem_x["file"]) = kwargs["mem_x"] else: - self._mem_x['multiplier'] = kwargs['mem_x'] - self._mem_x['file'] = None + self._mem_x["multiplier"] = kwargs["mem_x"] + self._mem_x["file"] = None else: - delattr(self, '_mem_x') - setattr(self, 'skip_timeout', False) + delattr(self, "_mem_x") + setattr(self, "skip_timeout", False) orig_sig_params = list(signature(pe.Node).parameters.items()) - __init__.__signature__ = Signature(parameters=[ - p[1] if p[0] != 'mem_gb' else ( - 'mem_gb', - Parameter('mem_gb', Parameter.POSITIONAL_OR_KEYWORD, - default=DEFAULT_MEM_GB) - )[1] for p in orig_sig_params[:-1]] + [ - Parameter('mem_x', Parameter.KEYWORD_ONLY), - orig_sig_params[-1][1] - ]) - - __init__.__doc__ = re.sub(r'(? 1000: self.logger.warning( - '%s is estimated to use %.3f GB (%s).', + "%s is estimated to use %.3f GB (%s).", self.name, self._mem_gb, - getattr(self, '_mem_x') + getattr(self, "_mem_x"), ) except FileNotFoundError: pass del self._mem_x if self._debug: - self.verbose_logger.debug('%s._mem_gb: %s', self.name, - self._mem_gb) + self.verbose_logger.debug("%s._mem_gb: %s", self.name, self._mem_gb) return self._mem_gb def _apply_random_seed(self): - '''Apply flags for the first matched interface''' + """Apply flags for the first matched interface.""" # pylint: disable=import-outside-toplevel from CPAC.pipeline.random_state import random_seed_flags + if isinstance(self.interface, Function): - for rsf, flags in random_seed_flags()['functions'].items(): + for rsf, flags in random_seed_flags()["functions"].items(): if self.interface.inputs.function_str == getsource(rsf): self.interface.inputs.function_str = flags( - self.interface.inputs.function_str) + self.interface.inputs.function_str + ) self.seed_applied = True return - for rsf, flags in random_seed_flags()['interfaces'].items(): + for rsf, flags in random_seed_flags()["interfaces"].items(): if isinstance(self.interface, rsf): self._add_flags(flags) self.seed_applied = True @@ -369,40 +387,45 @@ def _apply_random_seed(self): @property def mem_gb(self): - """Get estimated memory (GB)""" + """Get estimated memory (GB).""" if hasattr(self._interface, "estimated_memory_gb"): self._mem_gb = self._interface.estimated_memory_gb self.logger.warning( 'Setting "estimated_memory_gb" on Interfaces has been ' "deprecated as of nipype 1.0, please use Node.mem_gb." ) - if hasattr(self, '_mem_x'): - if self._mem_x['file'] is None: + if hasattr(self, "_mem_x"): + if self._mem_x["file"] is None: return self._apply_mem_x() try: - mem_x_path = getattr(self.inputs, self._mem_x['file']) + mem_x_path = getattr(self.inputs, self._mem_x["file"]) except AttributeError as attribute_error: raise AttributeError( - f'{attribute_error.args[0]} in Node \'{self.name}\'' + f"{attribute_error.args[0]} in Node '{self.name}'" ) from attribute_error if _check_mem_x_path(mem_x_path): # constant + mem_x[0] * t return self._apply_mem_x() - raise FileNotFoundError(2, 'The memory estimate for Node ' - f"'{self.name}' depends on the input " - f"'{self._mem_x['file']}' but " - 'no such file or directory', mem_x_path) + raise FileNotFoundError( + 2, + "The memory estimate for Node " + f"'{self.name}' depends on the input " + f"'{self._mem_x['file']}' but " + "no such file or directory", + mem_x_path, + ) return self._mem_gb @property def mem_x(self): """Get dict of 'multiplier' (memory multiplier), 'file' (input file) and multiplier mode (spatial * temporal, spatial only or - temporal only). Returns ``None`` if already consumed or not set.""" - return getattr(self, '_mem_x', None) + temporal only). Returns ``None`` if already consumed or not set. + """ + return getattr(self, "_mem_x", None) def _mem_x_file(self): - return getattr(self.inputs, getattr(self, '_mem_x', {}).get('file')) + return getattr(self.inputs, getattr(self, "_mem_x", {}).get("file")) def override_mem_gb(self, new_mem_gb): """Override the Node's memory estimate with a new value. @@ -412,40 +435,50 @@ def override_mem_gb(self, new_mem_gb): new_mem_gb : int or float new memory estimate in GB """ - if hasattr(self, '_mem_x'): - delattr(self, '_mem_x') - setattr(self, '_mem_gb', new_mem_gb) + if hasattr(self, "_mem_x"): + delattr(self, "_mem_x") + setattr(self, "_mem_gb", new_mem_gb) def run(self, updatehash=False): - self.__doc__ = getattr(super(), '__doc__', '') + self.__doc__ = getattr(super(), "__doc__", "") if self.seed is not None: self._apply_random_seed() if self.seed_applied: - random_state_logger = getLogger('random') - random_state_logger.info('%s\t%s', '# (Atropos constant)' if - 'atropos' in self.name else - str(self.seed), self.name) + random_state_logger = getLogger("random") + random_state_logger.info( + "%s\t%s", + "# (Atropos constant)" + if "atropos" in self.name + else str(self.seed), + self.name, + ) return super().run(updatehash) class MapNode(Node, pe.MapNode): # pylint: disable=empty-docstring __doc__ = _doctest_skiplines( - pe.MapNode.__doc__, - {" ... 'functional3.nii']"} + pe.MapNode.__doc__, {" ... 'functional3.nii']"} ) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - if not self.name.endswith('_'): - self.name = f'{self.name}_' - - __init__.__signature__ = Signature(parameters=[ - p[1] if p[0] != 'mem_gb' else ( - 'mem_gb', - Parameter('mem_gb', Parameter.POSITIONAL_OR_KEYWORD, - default=DEFAULT_MEM_GB) - )[1] for p in signature(pe.Node).parameters.items()]) + if not self.name.endswith("_"): + self.name = f"{self.name}_" + + __init__.__signature__ = Signature( + parameters=[ + p[1] + if p[0] != "mem_gb" + else ( + "mem_gb", + Parameter( + "mem_gb", Parameter.POSITIONAL_OR_KEYWORD, default=DEFAULT_MEM_GB + ), + )[1] + for p in signature(pe.Node).parameters.items() + ] + ) class Workflow(pe.Workflow): @@ -453,7 +486,8 @@ class Workflow(pe.Workflow): def __init__(self, name, base_dir=None, debug=False): """Create a workflow object. - Parameters + + Parameters. ---------- name : alphanumeric string unique identifier for the workflow @@ -466,14 +500,14 @@ def __init__(self, name, base_dir=None, debug=False): super().__init__(name, base_dir) self._debug = debug - self.verbose_logger = getLogger('engine') if debug else None + self.verbose_logger = getLogger("engine") if debug else None self._graph = nx.DiGraph() self._nodes_cache = set() self._nested_workflows_cache = set() def _configure_exec_nodes(self, graph): - """Ensure that each node knows where to get inputs from""" + """Ensure that each node knows where to get inputs from.""" for node in graph.nodes(): node._debug = self._debug # pylint: disable=protected-access node.verbose_logger = self.verbose_logger @@ -482,16 +516,20 @@ def _configure_exec_nodes(self, graph): data = graph.get_edge_data(*edge) for sourceinfo, field in data["connect"]: node.input_source[field] = ( - os.path.join(edge[0].output_dir(), - "result_%s.pklz" % edge[0].name), + os.path.join( + edge[0].output_dir(), "result_%s.pklz" % edge[0].name + ), sourceinfo, ) - if node and hasattr(node, '_mem_x'): - if isinstance( - node._mem_x, # pylint: disable=protected-access - dict - ) and node._mem_x[ # pylint: disable=protected-access - 'file'] == field: + if node and hasattr(node, "_mem_x"): + if ( + isinstance( + node._mem_x, # pylint: disable=protected-access + dict, + ) + and node._mem_x["file"] # pylint: disable=protected-access + == field + ): input_resultfile = node.input_source.get(field) if input_resultfile: # pylint: disable=protected-access @@ -500,18 +538,16 @@ def _configure_exec_nodes(self, graph): try: # memoize node._mem_gb if path # already exists - node._apply_mem_x(_load_resultfile( - input_resultfile - ).inputs[field]) - except (FileNotFoundError, KeyError, - TypeError): + node._apply_mem_x( + _load_resultfile(input_resultfile).inputs[field] + ) + except (FileNotFoundError, KeyError, TypeError): self._handle_just_in_time_exception(node) def _get_dot( - self, prefix=None, hierarchy=None, colored=False, simple_form=True, - level=0 + self, prefix=None, hierarchy=None, colored=False, simple_form=True, level=0 ): - """Create a dot file with connection info""" + """Create a dot file with connection info.""" # pylint: disable=invalid-name,protected-access import networkx as nx @@ -538,40 +574,45 @@ def _get_dot( quoted_prefix = f'"{prefix}"' if len(prefix.strip()) else prefix dotlist = [f'{quoted_prefix}label="{self.name}";'] for node in nx.topological_sort(self._graph): - fullname = ".".join(hierarchy + [node.fullname]) + fullname = ".".join([*hierarchy, node.fullname]) nodename = fullname.replace(".", "_") if not isinstance(node, Workflow): node_class_name = get_print_name(node, simple_form=simple_form) if not simple_form: node_class_name = ".".join(node_class_name.split(".")[1:]) if hasattr(node, "iterables") and node.iterables: - dotlist.append(f'"{nodename}"[label="{node_class_name}", ' - "shape=box3d, style=filled, color=black, " - "colorscheme=greys7 fillcolor=2];") + dotlist.append( + f'"{nodename}"[label="{node_class_name}", ' + "shape=box3d, style=filled, color=black, " + "colorscheme=greys7 fillcolor=2];" + ) else: if colored: - dotlist.append(f'"{nodename}"[label="' - f'{node_class_name}", style=filled,' - f' fillcolor="{colorset[level]}"];') + dotlist.append( + f'"{nodename}"[label="' + f'{node_class_name}", style=filled,' + f' fillcolor="{colorset[level]}"];' + ) else: - dotlist.append(f'"{nodename}"[label="' - f'{node_class_name}"];') + dotlist.append(f'"{nodename}"[label="' f'{node_class_name}"];') for node in nx.topological_sort(self._graph): if isinstance(node, Workflow): - fullname = ".".join(hierarchy + [node.fullname]) + fullname = ".".join([*hierarchy, node.fullname]) nodename = fullname.replace(".", "_") - dotlist.append(f"subgraph \"cluster_{nodename}\" {{") + dotlist.append(f'subgraph "cluster_{nodename}" {{') if colored: - dotlist.append(f'{prefix}{prefix}edge [color="' - f'{colorset[level + 1]}"];') + dotlist.append( + f'{prefix}{prefix}edge [color="' f'{colorset[level + 1]}"];' + ) dotlist.append(f"{prefix}{prefix}style=filled;") - dotlist.append(f'{prefix}{prefix}fillcolor=' - f'"{colorset[level + 2]}";') + dotlist.append( + f"{prefix}{prefix}fillcolor=" f'"{colorset[level + 2]}";' + ) dotlist.append( node._get_dot( prefix=prefix + prefix, - hierarchy=hierarchy + [self.name], + hierarchy=[*hierarchy, self.name], colored=colored, simple_form=simple_form, level=level + 3, @@ -583,20 +624,17 @@ def _get_dot( if node._hierarchy != subnode._hierarchy: continue if not isinstance(subnode, Workflow): - nodefullname = ".".join(hierarchy + [node.fullname]) - subnodefullname = ".".join( - hierarchy + [subnode.fullname]) + nodefullname = ".".join([*hierarchy, node.fullname]) + subnodefullname = ".".join([*hierarchy, subnode.fullname]) nodename = nodefullname.replace(".", "_") subnodename = subnodefullname.replace(".", "_") - for _ in self._graph.get_edge_data( - node, subnode - )["connect"]: + for _ in self._graph.get_edge_data(node, subnode)["connect"]: dotlist.append(f'"{nodename}" -> "{subnodename}";') logger.debug("connection: %s", dotlist[-1]) # add between workflow connections for u, v, d in self._graph.edges(data=True): - uname = ".".join(hierarchy + [u.fullname]) - vname = ".".join(hierarchy + [v.fullname]) + uname = ".".join([*hierarchy, u.fullname]) + vname = ".".join([*hierarchy, v.fullname]) for src, dest in d["connect"]: uname1 = uname vname1 = vname @@ -608,26 +646,25 @@ def _get_dot( uname1 += "." + ".".join(srcname.split(".")[:-1]) if "." in dest and "@" not in dest: if not isinstance(v, Workflow): - if "datasink" not in str( - v._interface.__class__ - ).lower(): + if "datasink" not in str(v._interface.__class__).lower(): vname1 += "." + ".".join(dest.split(".")[:-1]) else: vname1 += "." + ".".join(dest.split(".")[:-1]) if uname1.split(".")[:-1] != vname1.split(".")[:-1]: - dotlist.append(f'"{uname1.replace(".", "_")}" -> ' - f'"{vname1.replace(".", "_")}";') + dotlist.append( + f'"{uname1.replace(".", "_")}" -> ' + f'"{vname1.replace(".", "_")}";' + ) logger.debug("cross connection: %s", dotlist[-1]) return ("\n" + prefix).join(dotlist) def _handle_just_in_time_exception(self, node): # pylint: disable=protected-access - if hasattr(self, '_local_func_scans'): - node._apply_mem_x( - self._local_func_scans) # pylint: disable=no-member + if hasattr(self, "_local_func_scans"): + node._apply_mem_x(self._local_func_scans) # pylint: disable=no-member else: # TODO: handle S3 files - node._apply_mem_x(UNDEFINED_SIZE) # noqa: W0212 + node._apply_mem_x(UNDEFINED_SIZE) def write_graph( self, @@ -652,8 +689,10 @@ def write_graph( os.makedirs(base_dir, exist_ok=True) if graph2use in ["hierarchical", "colored"]: if self.name[:1].isdigit(): # these graphs break if int - raise ValueError(f"{graph2use} graph failed, workflow name " - "cannot begin with a number") + raise ValueError( + f"{graph2use} graph failed, workflow name " + "cannot begin with a number" + ) dotfilename = os.path.join(base_dir, dotfilename) self.write_hierarchical_dotfile( dotfilename=dotfilename, @@ -675,9 +714,12 @@ def write_graph( simple_form=simple_form, ) - logger.info("Generated workflow graph: %s " - "(graph2use=%s, simple_form=%s).", - outfname, graph2use, simple_form) + logger.info( + "Generated workflow graph: %s " "(graph2use=%s, simple_form=%s).", + outfname, + graph2use, + simple_form, + ) return outfname write_graph.__doc__ = pe.Workflow.write_graph.__doc__ @@ -686,9 +728,10 @@ def write_hierarchical_dotfile( self, dotfilename=None, colored=False, simple_form=True ): # pylint: disable=invalid-name - dotlist = [f"digraph \"{self.name}\"{{"] - dotlist.append(self._get_dot(prefix=" ", colored=colored, - simple_form=simple_form)) + dotlist = [f'digraph "{self.name}"{{'] + dotlist.append( + self._get_dot(prefix=" ", colored=colored, simple_form=simple_form) + ) dotlist.append("}") dotstr = "\n".join(dotlist) if dotfilename: @@ -699,8 +742,8 @@ def write_hierarchical_dotfile( logger.info(dotstr) -def get_data_size(filepath, mode='xyzt'): - """Function to return the size of a functional image (x * y * z * t) +def get_data_size(filepath, mode="xyzt"): + """Function to return the size of a functional image (x * y * z * t). Parameters ---------- @@ -724,13 +767,13 @@ def get_data_size(filepath, mode='xyzt'): data_shape = load(filepath).shape elif isinstance(filepath, tuple) and len(filepath) == 4: data_shape = filepath - if mode == 't': + if mode == "t": # if the data has muptiple TRs, return that number if len(data_shape) > 3: return data_shape[3] # otherwise return 1 return 1 - if mode == 'xyz': + if mode == "xyz": return prod(data_shape[0:3]).item() return prod(data_shape).item() @@ -748,7 +791,8 @@ def export_graph( """Displays the graph layout of the pipeline This function requires that pygraphviz and matplotlib are available on the system. - Parameters + + Parameters. ---------- show : boolean Indicate whether to generate pygraphviz output fromn @@ -772,8 +816,9 @@ def export_graph( base_dir = os.getcwd() os.makedirs(base_dir, exist_ok=True) - out_dot = fname_presuffix(dotfilename, suffix="_detailed.dot", - use_ext=False, newpath=base_dir) + out_dot = fname_presuffix( + dotfilename, suffix="_detailed.dot", use_ext=False, newpath=base_dir + ) _write_detailed_dot(graph, out_dot) # Convert .dot if format != 'dot' @@ -782,8 +827,9 @@ def export_graph( logger.warning("dot2png: %s", res.runtime.stderr) pklgraph = _create_dot_graph(graph, show_connectinfo, simple_form) - simple_dot = fname_presuffix(dotfilename, suffix=".dot", use_ext=False, - newpath=base_dir) + simple_dot = fname_presuffix( + dotfilename, suffix=".dot", use_ext=False, newpath=base_dir + ) nx.drawing.nx_pydot.write_dot(pklgraph, simple_dot) # Convert .dot if format != 'dot' @@ -811,7 +857,7 @@ def _write_detailed_dot(graph, dotfilename): struct1:f1 -> struct2:f0; struct1:f0 -> struct2:f1; struct1:f2 -> struct3:here; - } + }. """ # pylint: disable=invalid-name import networkx as nx @@ -831,15 +877,19 @@ def _write_detailed_dot(graph, dotfilename): inport = cd[1] ipstrip = f"in{_replacefunk(inport)}" opstrip = f"out{_replacefunk(outport)}" - edges.append(f'"{u.itername.replace(".", "")}":' - f'"{opstrip}":e -> ' - f'"{v.itername.replace(".", "")}":' - f'"{ipstrip}":w;') + edges.append( + f'"{u.itername.replace(".", "")}":' + f'"{opstrip}":e -> ' + f'"{v.itername.replace(".", "")}":' + f'"{ipstrip}":w;' + ) if inport not in inports: inports.append(inport) - inputstr = (["{IN"] - + [f"| {ip}" for - ip in sorted(inports)] + ["}"]) + inputstr = ( + ["{IN"] + + [f"| {ip}" for ip in sorted(inports)] + + ["}"] + ) outports = [] for u, v, d in graph.out_edges(nbunch=n, data=True): for cd in d["connect"]: @@ -851,18 +901,22 @@ def _write_detailed_dot(graph, dotfilename): outports.append(outport) outputstr = ( ["{OUT"] - + [f"| {oport}" for - oport in sorted(outports)] + ["}"]) + + [f"| {oport}" for oport in sorted(outports)] + + ["}"] + ) srcpackage = "" if hasattr(n, "_interface"): pkglist = n.interface.__class__.__module__.split(".") if len(pkglist) > 2: srcpackage = pkglist[2] srchierarchy = ".".join(nodename.split(".")[1:-1]) - nodenamestr = (f"{{ {nodename.split('.')[-1]} | {srcpackage} | " - f"{srchierarchy} }}") - text += [f'"{nodename.replace(".", "")}" [label=' - f'"{"".join(inputstr)}|{nodenamestr}|{"".join(outputstr)}"];'] + nodenamestr = ( + f"{{ {nodename.split('.')[-1]} | {srcpackage} | " f"{srchierarchy} }}" + ) + text += [ + f'"{nodename.replace(".", "")}" [label=' + f'"{"".join(inputstr)}|{nodenamestr}|{"".join(outputstr)}"];' + ] # write edges for edge in sorted(edges): text.append(edge) diff --git a/CPAC/pipeline/nipype_pipeline_engine/monkeypatch.py b/CPAC/pipeline/nipype_pipeline_engine/monkeypatch.py index 5c0ff44632..cd0402ca46 100644 --- a/CPAC/pipeline/nipype_pipeline_engine/monkeypatch.py +++ b/CPAC/pipeline/nipype_pipeline_engine/monkeypatch.py @@ -7,11 +7,11 @@ def patch_base_interface(): """ from nipype.interfaces.base.core import ( BaseInterface, + InterfaceResult, + RuntimeContext, config, indirectory, - InterfaceResult, os, - RuntimeContext, str2bool, write_provenance, ) @@ -61,5 +61,5 @@ def run(self, cwd=None, ignore_exception=None, **inputs): # Apply patch import nipype.interfaces.base.core as base_core - base_core.BaseInterface.run = PatchedBaseInterface.run + base_core.BaseInterface.run = PatchedBaseInterface.run diff --git a/CPAC/pipeline/nipype_pipeline_engine/plugins/__init__.py b/CPAC/pipeline/nipype_pipeline_engine/plugins/__init__.py index da6f4339d0..943825ad50 100644 --- a/CPAC/pipeline/nipype_pipeline_engine/plugins/__init__.py +++ b/CPAC/pipeline/nipype_pipeline_engine/plugins/__init__.py @@ -1,4 +1,4 @@ -"""Import Nipype's pipeline plugins and selectively override +"""Import Nipype's pipeline plugins and selectively override. Copyright (C) 2022 C-PAC Developers @@ -15,9 +15,12 @@ License for more details. You should have received a copy of the GNU Lesser General Public -License along with C-PAC. If not, see .""" -from nipype.pipeline.plugins import * # noqa: F401,F403 +License along with C-PAC. If not, see . +""" +from nipype.pipeline.plugins import * # noqa: F403 + # Override LegacyMultiProc from .legacymultiproc import LegacyMultiProcPlugin # noqa: F401 + # Override MultiProc from .multiproc import MultiProcPlugin # noqa: F401 diff --git a/CPAC/pipeline/nipype_pipeline_engine/plugins/cpac_nipype_custom.py b/CPAC/pipeline/nipype_pipeline_engine/plugins/cpac_nipype_custom.py index a843931783..2ff9fe9b1c 100644 --- a/CPAC/pipeline/nipype_pipeline_engine/plugins/cpac_nipype_custom.py +++ b/CPAC/pipeline/nipype_pipeline_engine/plugins/cpac_nipype_custom.py @@ -32,18 +32,20 @@ This file is part of C-PAC. """ +from copy import deepcopy import gc import json +from logging import INFO import platform import resource import sys -from copy import deepcopy -from logging import INFO from textwrap import indent from traceback import format_exception -from nipype.pipeline.plugins.multiproc import logger + from numpy import flatnonzero -from CPAC.pipeline.nipype_pipeline_engine import MapNode, UNDEFINED_SIZE +from nipype.pipeline.plugins.multiproc import logger + +from CPAC.pipeline.nipype_pipeline_engine import UNDEFINED_SIZE, MapNode from CPAC.utils.monitoring import log_nodes_cb @@ -62,10 +64,10 @@ def get_peak_usage(): proc_peak = self_usage.ru_maxrss # getrusage.ru_maxrss in bytes on Macs - if platform.system() == 'Darwin': - proc_peak /= 1024. + if platform.system() == "Darwin": + proc_peak /= 1024.0 - return proc_peak / 1024. / 1024. + return proc_peak / 1024.0 / 1024.0 def parse_previously_observed_mem_gb(callback_log_path): @@ -81,32 +83,34 @@ def parse_previously_observed_mem_gb(callback_log_path): dict Dictionary of per-node memory usage. """ - with open(callback_log_path, 'r') as cbl: - return {line['id'].split('.', 1)[-1]: line['runtime_memory_gb'] for - line in [json.loads(line) for line in cbl.readlines()] if - 'runtime_memory_gb' in line} + with open(callback_log_path, "r") as cbl: + return { + line["id"].split(".", 1)[-1]: line["runtime_memory_gb"] + for line in [json.loads(line) for line in cbl.readlines()] + if "runtime_memory_gb" in line + } # pylint: disable=too-few-public-methods, missing-class-docstring -class CpacNipypeCustomPluginMixin(): +class CpacNipypeCustomPluginMixin: def __init__(self, plugin_args=None): if not isinstance(plugin_args, dict): plugin_args = {} - if 'status_callback' not in plugin_args: - plugin_args['status_callback'] = log_nodes_cb - if 'runtime' in plugin_args: - self.runtime = {node_key: observation * ( - 1 + plugin_args['runtime']['buffer'] / 100 - ) for node_key, observation in parse_previously_observed_mem_gb( - plugin_args['runtime']['usage']).items()} + if "status_callback" not in plugin_args: + plugin_args["status_callback"] = log_nodes_cb + if "runtime" in plugin_args: + self.runtime = { + node_key: observation * (1 + plugin_args["runtime"]["buffer"] / 100) + for node_key, observation in parse_previously_observed_mem_gb( + plugin_args["runtime"]["usage"] + ).items() + } super().__init__(plugin_args=plugin_args) self.peak = 0 self._stats = None def _check_resources_(self, running_tasks): - """ - Make sure there are resources available - """ + """Make sure there are resources available.""" free_memory_gb = self.memory_gb free_processors = self.processors for _, jobid in running_tasks: @@ -117,7 +121,8 @@ def _check_resources_(self, running_tasks): def _check_resources(self, running_tasks): """Make sure there are resources available, accounting for - Nipype memory usage""" + Nipype memory usage. + """ free_memory_gb, free_processors = self._check_resources_(running_tasks) # Nipype memory usage @@ -128,15 +133,12 @@ def _check_resources(self, running_tasks): def _clean_exception(self, jobid, graph): traceback = format_exception(*sys.exc_info()) - self._clean_queue( - jobid, graph, result={"result": None, - "traceback": traceback} - ) + self._clean_queue(jobid, graph, result={"result": None, "traceback": traceback}) def _override_memory_estimate(self, node): """ Override node memory estimate with provided runtime memory - usage, buffered + usage, buffered. Parameters ---------- @@ -146,21 +148,21 @@ def _override_memory_estimate(self, node): ------- None """ - if hasattr(node, 'list_node_names'): + if hasattr(node, "list_node_names"): for node_id in node.list_node_names(): # drop top-level node name - node_id = node_id.split('.', 1)[-1] + node_id = node_id.split(".", 1)[-1] else: - node_id = node.fullname.split('.', 1)[-1] + node_id = node.fullname.split(".", 1)[-1] if self._match_for_overrides(node, node_id): return - while '.' in node_id: # iterate through levels of specificity - node_id = node_id.rsplit('.', 1)[0] + while "." in node_id: # iterate through levels of specificity + node_id = node_id.rsplit(".", 1)[0] if self._match_for_overrides(node, node_id): return def _match_for_overrides(self, node, node_id): - """Match node memory estimate with provided runtime memory usage key + """Match node memory estimate with provided runtime memory usage key. Parameters ---------- @@ -177,14 +179,14 @@ def _match_for_overrides(self, node, node_id): return True partial_matches = [nid for nid in self.runtime if node_id in nid] if any(partial_matches): - node.override_mem_gb(max( - self.runtime[partial_match] for - partial_match in partial_matches)) + node.override_mem_gb( + max(self.runtime[partial_match] for partial_match in partial_matches) + ) return True return False def _prerun_check(self, graph): - """Check if any node exeeds the available resources""" + """Check if any node exeeds the available resources.""" tasks_mem_gb = [] tasks_num_th = [] overrun_message_mem = None @@ -192,7 +194,7 @@ def _prerun_check(self, graph): # estimate of C-PAC + Nipype overhead (GB): overhead_memory_estimate = 1 for node in graph.nodes(): - if hasattr(self, 'runtime'): + if hasattr(self, "runtime"): self._override_memory_estimate(node) try: node_memory_estimate = node.mem_gb @@ -206,9 +208,9 @@ def _prerun_check(self, graph): tasks_num_th.append((node.name, node.n_procs)) if tasks_mem_gb: - overrun_message_mem = '\n'.join([ - f'\t{overrun[0]}: {overrun[1]} GB' for overrun in tasks_mem_gb - ]) + overrun_message_mem = "\n".join( + [f"\t{overrun[0]}: {overrun[1]} GB" for overrun in tasks_mem_gb] + ) logger.warning( "The following nodes are estimated to exceed the total amount " f"of memory available (%0.2fGB): \n{overrun_message_mem}", @@ -216,9 +218,9 @@ def _prerun_check(self, graph): ) if tasks_num_th: - overrun_message_th = '\n'.join([ - f'\t{overrun[0]}: {overrun[1]} threads' for overrun in - tasks_num_th]) + overrun_message_th = "\n".join( + [f"\t{overrun[0]}: {overrun[1]} threads" for overrun in tasks_num_th] + ) logger.warning( "Some nodes demand for more threads than available (%d): " f"\n{overrun_message_th}", @@ -226,17 +228,26 @@ def _prerun_check(self, graph): ) if self.raise_insufficient and (tasks_mem_gb or tasks_num_th): - raise RuntimeError("\n".join([msg for msg in [ - "Insufficient resources available for job:", - overrun_message_mem, overrun_message_th - ] if msg is not None])) + raise RuntimeError( + "\n".join( + [ + msg + for msg in [ + "Insufficient resources available for job:", + overrun_message_mem, + overrun_message_th, + ] + if msg is not None + ] + ) + ) def _send_procs_to_workers(self, updatehash=False, graph=None): """ Sends jobs to workers when system resources are available. Customized from https://github.com/nipy/nipype/blob/79e2fdfc/nipype/pipeline/plugins/legacymultiproc.py#L311-L462 - to catch overhead deadlocks - """ # noqa: E501 # pylint: disable=line-too-long + to catch overhead deadlocks. + """ # pylint: disable=line-too-long # pylint: disable=too-many-branches, too-many-statements # Check to see if a job is available (jobs with all dependencies run) # See https://github.com/nipy/nipype/pull/2200#discussion_r141605722 @@ -246,8 +257,7 @@ def _send_procs_to_workers(self, updatehash=False, graph=None): ) # Check available resources by summing all threads and memory used - free_memory_gb, free_processors = self._check_resources( - self.pending_tasks) + free_memory_gb, free_processors = self._check_resources(self.pending_tasks) num_pending = len(self.pending_tasks) num_ready = len(jobids) @@ -274,7 +284,7 @@ def _send_procs_to_workers(self, updatehash=False, graph=None): logger.info( "[%s] Running %d tasks, and %d jobs ready. Free " "memory (GB): %0.2f/%0.2f, Free processors: %d/%d.%s", - type(self).__name__[:-len('Plugin')], + type(self).__name__[: -len("Plugin")], num_pending, num_ready, free_memory_gb, @@ -297,8 +307,7 @@ def _send_procs_to_workers(self, updatehash=False, graph=None): ) return - jobids = self._sort_jobs(jobids, - scheduler=self.plugin_args.get("scheduler")) + jobids = self._sort_jobs(jobids, scheduler=self.plugin_args.get("scheduler")) # Run garbage collector before potentially submitting jobs gc.collect() @@ -324,9 +333,7 @@ def _send_procs_to_workers(self, updatehash=False, graph=None): next_job_th = min(self.procs[jobid].n_procs, self.processors) # If node does not fit, skip at this moment - if not self.raise_insufficient and ( - num_pending == 0 and num_ready > 0 - ): + if not self.raise_insufficient and (num_pending == 0 and num_ready > 0): force_allocate_job = True free_processors -= 1 if not force_allocate_job and ( @@ -364,8 +371,7 @@ def _send_procs_to_workers(self, updatehash=False, graph=None): # updatehash and run_without_submitting are also run locally if updatehash or self.procs[jobid].run_without_submitting: - logger.debug("Running node %s on master thread", - self.procs[jobid]) + logger.debug("Running node %s on master thread", self.procs[jobid]) try: self.procs[jobid].run(updatehash=updatehash) except Exception: # pylint: disable=broad-except @@ -387,8 +393,7 @@ def _send_procs_to_workers(self, updatehash=False, graph=None): # Send job to task manager and add to pending tasks if self._status_callback: self._status_callback(self.procs[jobid], "start") - tid = self._submit_job(deepcopy(self.procs[jobid]), - updatehash=updatehash) + tid = self._submit_job(deepcopy(self.procs[jobid]), updatehash=updatehash) if tid is None: self.proc_done[jobid] = False self.proc_pending[jobid] = False diff --git a/CPAC/pipeline/nipype_pipeline_engine/plugins/legacymultiproc.py b/CPAC/pipeline/nipype_pipeline_engine/plugins/legacymultiproc.py index 57679f9df8..2d64e2c288 100644 --- a/CPAC/pipeline/nipype_pipeline_engine/plugins/legacymultiproc.py +++ b/CPAC/pipeline/nipype_pipeline_engine/plugins/legacymultiproc.py @@ -20,8 +20,10 @@ You should have received a copy of the GNU Lesser General Public License along with C-PAC. If not, see . """ -from nipype.pipeline.plugins.legacymultiproc import \ - LegacyMultiProcPlugin as LegacyMultiProc +from nipype.pipeline.plugins.legacymultiproc import ( + LegacyMultiProcPlugin as LegacyMultiProc, +) + from .cpac_nipype_custom import CpacNipypeCustomPluginMixin diff --git a/CPAC/pipeline/nipype_pipeline_engine/plugins/multiproc.py b/CPAC/pipeline/nipype_pipeline_engine/plugins/multiproc.py index fc3b55187c..1100089600 100644 --- a/CPAC/pipeline/nipype_pipeline_engine/plugins/multiproc.py +++ b/CPAC/pipeline/nipype_pipeline_engine/plugins/multiproc.py @@ -21,6 +21,7 @@ License along with C-PAC. If not, see . """ from nipype.pipeline.plugins.multiproc import MultiProcPlugin as MultiProc + from .cpac_nipype_custom import CpacNipypeCustomPluginMixin diff --git a/CPAC/pipeline/nodeblock.py b/CPAC/pipeline/nodeblock.py index 83a465ff75..b9d18d9223 100644 --- a/CPAC/pipeline/nodeblock.py +++ b/CPAC/pipeline/nodeblock.py @@ -1,22 +1,20 @@ -"""Class and decorator for NodeBlock functions""" -from typing import Callable, List, Union, Optional, Dict, Any +"""Class and decorator for NodeBlock functions.""" +from typing import Any, Callable, Dict, List, Optional, Union class NodeBlockFunction: - """ - Stores a reference to the nodeblock function and all of its meta-data. - """ + """Stores a reference to the nodeblock function and all of its meta-data.""" def __init__( - self, - func: Callable, - name: Optional[str] = None, - config: Optional[List[str]] = None, - switch: Optional[Union[List[str], List[List[str]]]] = None, - option_key: Optional[Union[str, List[str]]] = None, - option_val: Optional[Union[str, List[str]]] = None, - inputs: Optional[List[Union[str, list, tuple]]] = None, - outputs: Optional[Union[List[str], Dict[str, Any]]] = None + self, + func: Callable, + name: Optional[str] = None, + config: Optional[List[str]] = None, + switch: Optional[Union[List[str], List[List[str]]]] = None, + option_key: Optional[Union[str, List[str]]] = None, + option_val: Optional[Union[str, List[str]]] = None, + inputs: Optional[List[Union[str, list, tuple]]] = None, + outputs: Optional[Union[List[str], Dict[str, Any]]] = None, ) -> None: self.func = func """Nodeblock function reference.""" @@ -54,9 +52,13 @@ def __init__( self.__name__ = func.__name__ self.__qualname__ = func.__qualname__ self.__annotations__ = func.__annotations__ - self.__doc__ = ''.join([_.replace(' ', '') for _ in [ - func.__doc__, '', '', NodeBlockFunction.__call__.__doc__ - ] if _ is not None]).rstrip() + self.__doc__ = "".join( + [ + _.replace(" ", "") + for _ in [func.__doc__, "", "", NodeBlockFunction.__call__.__doc__] + if _ is not None + ] + ).rstrip() # all node block functions have this signature def __call__(self, wf, cfg, strat_pool, pipe_num, opt=None): @@ -83,39 +85,39 @@ def __call__(self, wf, cfg, strat_pool, pipe_num, opt=None): return self.func(wf, cfg, strat_pool, pipe_num, opt) def legacy_nodeblock_dict(self): - """ - Returns nodeblock metadata as a dictionary. Helper for compatibility reasons. - """ + """Returns nodeblock metadata as a dictionary. Helper for compatibility reasons.""" return { - 'name': self.name, - 'config': self.config, - 'switch': self.switch, - 'option_key': self.option_key, - 'option_val': self.option_val, - 'inputs': self.inputs, - 'outputs': self.outputs + "name": self.name, + "config": self.config, + "switch": self.switch, + "option_key": self.option_key, + "option_val": self.option_val, + "inputs": self.inputs, + "outputs": self.outputs, } def __repr__(self) -> str: - return (f'NodeBlockFunction({self.func.__module__}.' - f'{self.func.__name__}, "{self.name}", ' - f'config={self.config}, switch={self.switch}, ' - f'option_key={self.option_key}, option_val=' - f'{self.option_val}, inputs={self.inputs}, ' - f'outputs={self.outputs})') + return ( + f"NodeBlockFunction({self.func.__module__}." + f'{self.func.__name__}, "{self.name}", ' + f"config={self.config}, switch={self.switch}, " + f"option_key={self.option_key}, option_val=" + f"{self.option_val}, inputs={self.inputs}, " + f"outputs={self.outputs})" + ) def __str__(self) -> str: - return f'NodeBlockFunction({self.name})' + return f"NodeBlockFunction({self.name})" def nodeblock( - name: Optional[str] = None, - config: Optional[List[str]] = None, - switch: Optional[Union[List[str], List[List[str]]]] = None, - option_key: Optional[Union[str, List[str]]] = None, - option_val: Optional[Union[str, List[str]]] = None, - inputs: Optional[List[Union[str, list, tuple]]] = None, - outputs: Optional[Union[List[str], Dict[str, Any]]] = None + name: Optional[str] = None, + config: Optional[List[str]] = None, + switch: Optional[Union[List[str], List[List[str]]]] = None, + option_key: Optional[Union[str, List[str]]] = None, + option_val: Optional[Union[str, List[str]]] = None, + inputs: Optional[List[Union[str, list, tuple]]] = None, + outputs: Optional[Union[List[str], Dict[str, Any]]] = None, ): """ Define a node block: Connections to the pipeline configuration and to other node blocks. @@ -154,5 +156,5 @@ def nodeblock( option_key, option_val, inputs, - outputs + outputs, ) diff --git a/CPAC/pipeline/random_state/__init__.py b/CPAC/pipeline/random_state/__init__.py index 5956f33416..8f5620b840 100644 --- a/CPAC/pipeline/random_state/__init__.py +++ b/CPAC/pipeline/random_state/__init__.py @@ -1,6 +1,14 @@ -'''Random state for C-PAC''' -from .seed import random_seed, random_seed_flags, set_up_random_state, \ - set_up_random_state_logger +"""Random state for C-PAC.""" +from .seed import ( + random_seed, + random_seed_flags, + set_up_random_state, + set_up_random_state_logger, +) -__all__ = ['random_seed', 'random_seed_flags', 'set_up_random_state', - 'set_up_random_state_logger'] +__all__ = [ + "random_seed", + "random_seed_flags", + "set_up_random_state", + "set_up_random_state_logger", +] diff --git a/CPAC/pipeline/random_state/seed.py b/CPAC/pipeline/random_state/seed.py index 8a2591b7d7..07329aa715 100644 --- a/CPAC/pipeline/random_state/seed.py +++ b/CPAC/pipeline/random_state/seed.py @@ -14,8 +14,7 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -'''Functions to set, check, and log random seed''' -import os +"""Functions to set, check, and log random seed.""" import random import numpy as np @@ -28,11 +27,11 @@ from CPAC.utils.interfaces.ants import AI from CPAC.utils.monitoring.custom_logging import getLogger, set_up_logger -_seed = {'seed': None} +_seed = {"seed": None} def random_random_seed(): - '''Returns a random postive integer up to 2147483647 + """Returns a random postive integer up to 2147483647. Parameters ---------- @@ -46,12 +45,12 @@ def random_random_seed(): -------- >>> 0 < random_random_seed() <= np.iinfo(np.int32).max True - ''' + """ return random.randint(1, np.iinfo(np.int32).max) def random_seed(): - '''Function to access current random seed + """Function to access current random seed. Parameters ---------- @@ -60,14 +59,14 @@ def random_seed(): Returns ------- seed : int or None - ''' - if _seed['seed'] == 'random': - _seed['seed'] = random_random_seed() - return _seed['seed'] + """ + if _seed["seed"] == "random": + _seed["seed"] = random_random_seed() + return _seed["seed"] def random_seed_flags(): - '''Function to return dictionary of flags with current random seed. + """Function to return dictionary of flags with current random seed. Parameters ---------- @@ -105,20 +104,21 @@ def random_seed_flags(): >>> all([isinstance(random_seed_flags()[key], dict) for key in [ ... 'functions', 'interfaces']]) True - ''' + """ from CPAC.registration.utils import hardcoded_reg + seed = random_seed() if seed is None: - return {'functions': {}, 'interfaces': {}} + return {"functions": {}, "interfaces": {}} return { - 'functions': { + "functions": { # function: lambda function to apply to function source hardcoded_reg: lambda fn_string: fn_string.replace( 'regcmd = ["antsRegistration"]', - f'regcmd = ["antsRegistration", "--random-seed", \"{seed}\"]' + f'regcmd = ["antsRegistration", "--random-seed", "{seed}"]', ) }, - 'interfaces': { + "interfaces": { # interface: [flags to apply] # OR # interface: ([flags to apply], [flags to remove]) @@ -129,31 +129,33 @@ def random_seed_flags(): # constant seed number," so for Atropos nodes, the built-in # Atropos constant seed is used if a seed is specified for # C-PAC - AI: _reusable_flags()['ANTs'], - Registration: _reusable_flags()['ANTs'], - Atropos: (['--use-random-seed 0'], - [flag for one in ['', ' 1'] for flag in - [f'--use-random-seed{one}', f'-r{one}']]), + AI: _reusable_flags()["ANTs"], + Registration: _reusable_flags()["ANTs"], + Atropos: ( + ["--use-random-seed 0"], + [ + flag + for one in ["", " 1"] + for flag in [f"--use-random-seed{one}", f"-r{one}"] + ], + ), # FreeSurfer - ReconAll: ['-norandomness', f'-rng-seed {seed}'], - ApplyVolTransform: [f'--seed {seed}'], + ReconAll: ["-norandomness", f"-rng-seed {seed}"], + ApplyVolTransform: [f"--seed {seed}"], # FSL - ImageMaths: _reusable_flags()['FSL'], - MathsCommand: _reusable_flags()['FSL'] - } + ImageMaths: _reusable_flags()["FSL"], + MathsCommand: _reusable_flags()["FSL"], + }, } def _reusable_flags(): seed = random_seed() - return { - 'ANTs': [f'--random-seed {seed}'], - 'FSL': [f'-seed {seed}'] - } + return {"ANTs": [f"--random-seed {seed}"], "FSL": [f"-seed {seed}"]} def set_up_random_state(seed): - '''Set global random seed + """Set global random seed. Parameters ---------- @@ -177,28 +179,30 @@ def set_up_random_state(seed): ValueError: Valid random seeds are positive integers up to 2147483647, "random", or None, not 0 >>> set_up_random_state(None) - ''' # noqa: E501 # pylint: disable=line-too-long + """ # pylint: disable=line-too-long if seed is not None: - if seed == 'random': + if seed == "random": seed = random_random_seed() else: try: seed = int(seed) assert 0 < seed <= np.iinfo(np.int32).max - except(ValueError, TypeError, AssertionError): - raise ValueError('Valid random seeds are positive integers up to ' - f'2147483647, "random", or None, not {seed}') - - _seed['seed'] = seed + except (ValueError, TypeError, AssertionError): + raise ValueError( + "Valid random seeds are positive integers up to " + f'2147483647, "random", or None, not {seed}' + ) + + _seed["seed"] = seed return random_seed() def set_up_random_state_logger(log_dir): - '''Prepare C-PAC for logging random seed use. + """Prepare C-PAC for logging random seed use. Parameters ---------- log_dir : str - ''' - set_up_logger('random', level='info', log_dir=log_dir, mock=True) - getLogger('random').info('seed: %s', random_seed()) + """ + set_up_logger("random", level="info", log_dir=log_dir, mock=True) + getLogger("random").info("seed: %s", random_seed()) diff --git a/CPAC/pipeline/schema.py b/CPAC/pipeline/schema.py index f24598dde2..93deed34fd 100644 --- a/CPAC/pipeline/schema.py +++ b/CPAC/pipeline/schema.py @@ -15,35 +15,54 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -"""Validation schema for C-PAC pipeline configurations""" +"""Validation schema for C-PAC pipeline configurations.""" # pylint: disable=too-many-lines -import re from itertools import chain, permutations +import re + import numpy as np from pathvalidate import sanitize_filename -from voluptuous import All, ALLOW_EXTRA, Any, BooleanInvalid, Capitalize, \ - Coerce, CoerceInvalid, ExclusiveInvalid, In, Length, \ - LengthInvalid, Lower, Match, Maybe, MultipleInvalid, \ - Optional, Range, Required, Schema, Title +from voluptuous import ( + ALLOW_EXTRA, + All, + Any, + BooleanInvalid, + Capitalize, + Coerce, + CoerceInvalid, + ExclusiveInvalid, + In, + Length, + LengthInvalid, + Lower, + Match, + Maybe, + MultipleInvalid, + Optional, + Range, + Required, + Schema, + Title, +) + from CPAC.utils.datatypes import ItemFromList, ListFromItem from CPAC.utils.docs import DOCS_URL_PREFIX from CPAC.utils.utils import YAML_BOOLS # 1 or more digits, optional decimal, 'e', optional '-', 1 or more digits -SCIENTIFIC_NOTATION_STR_REGEX = r'^([0-9]+(\.[0-9]*)*(e)-{0,1}[0-9]+)*$' +SCIENTIFIC_NOTATION_STR_REGEX = r"^([0-9]+(\.[0-9]*)*(e)-{0,1}[0-9]+)*$" # (1 or more digits, optional decimal, 0 or more lowercase characters (units)) # ('x', # 1 or more digits, optional decimal, 0 or more lowercase characters (units) # ) 0 or more times -RESOLUTION_REGEX = r'^[0-9]+(\.[0-9]*){0,1}[a-z]*' \ - r'(x[0-9]+(\.[0-9]*){0,1}[a-z]*)*$' +RESOLUTION_REGEX = r"^[0-9]+(\.[0-9]*){0,1}[a-z]*" r"(x[0-9]+(\.[0-9]*){0,1}[a-z]*)*$" Number = Any(float, int, All(str, Match(SCIENTIFIC_NOTATION_STR_REGEX))) def str_to_bool1_1(x): # pylint: disable=invalid-name - '''Convert strings to Booleans for YAML1.1 syntax + """Convert strings to Booleans for YAML1.1 syntax. Ref https://yaml.org/type/bool.html @@ -54,7 +73,7 @@ def str_to_bool1_1(x): # pylint: disable=invalid-name Returns ------- bool - ''' + """ if isinstance(x, str): try: x = float(x) @@ -62,201 +81,240 @@ def str_to_bool1_1(x): # pylint: disable=invalid-name return False except ValueError: pass - x = (True if str(x).lower() in YAML_BOOLS[True] else - False if str(x).lower() in YAML_BOOLS[False] else x) + x = ( + True + if str(x).lower() in YAML_BOOLS[True] + else False + if str(x).lower() in YAML_BOOLS[False] + else x + ) if not isinstance(x, (bool, int)): - raise BooleanInvalid('Type boolean value was expected, type ' - f'{getattr(type(x), "__name__", str(type(x)))} ' - f'value\n\n{x}\n\nwas provided') + raise BooleanInvalid( + 'Type boolean value was expected, type ' + f'{getattr(type(x), "__name__", str(type(x)))} ' + f'value\n\n{x}\n\nwas provided' + ) return bool(x) bool1_1 = All(str_to_bool1_1, bool) forkable = All(Coerce(ListFromItem), [bool1_1], Length(max=2)) valid_options = { - 'acpc': { - 'target': ['brain', 'whole-head'] - }, - 'brain_extraction': { - 'using': ['3dSkullStrip', 'BET', 'UNet', 'niworkflows-ants', - 'FreeSurfer-BET-Tight', 'FreeSurfer-BET-Loose', - 'FreeSurfer-ABCD', 'FreeSurfer-Brainmask'] + "acpc": {"target": ["brain", "whole-head"]}, + "brain_extraction": { + "using": [ + "3dSkullStrip", + "BET", + "UNet", + "niworkflows-ants", + "FreeSurfer-BET-Tight", + "FreeSurfer-BET-Loose", + "FreeSurfer-ABCD", + "FreeSurfer-Brainmask", + ] }, - 'centrality': { - 'method_options': ['degree_centrality', 'eigenvector_centrality', - 'local_functional_connectivity_density'], - 'threshold_options': ['Significance threshold', 'Sparsity threshold', - 'Correlation threshold'], - 'weight_options': ['Binarized', 'Weighted'] + "centrality": { + "method_options": [ + "degree_centrality", + "eigenvector_centrality", + "local_functional_connectivity_density", + ], + "threshold_options": [ + "Significance threshold", + "Sparsity threshold", + "Correlation threshold", + ], + "weight_options": ["Binarized", "Weighted"], }, - 'motion_correction': ['3dvolreg', 'mcflirt'], - 'sca': { - 'roi_paths': ['Avg', 'DualReg', 'MultReg'], + "motion_correction": ["3dvolreg", "mcflirt"], + "sca": { + "roi_paths": ["Avg", "DualReg", "MultReg"], }, - 'segmentation': { - 'using': ['FSL-FAST', 'ANTs_Prior_Based', 'Template_Based'], - 'template': ['EPI_Template', 'T1_Template'], + "segmentation": { + "using": ["FSL-FAST", "ANTs_Prior_Based", "Template_Based"], + "template": ["EPI_Template", "T1_Template"], }, - 'timeseries': { - 'roi_paths': ['Avg', 'Voxel', 'SpatialReg'], + "timeseries": { + "roi_paths": ["Avg", "Voxel", "SpatialReg"], }, - 'connectivity_matrix': { - 'using': ['AFNI', 'Nilearn', 'ndmg'], - 'measure': ['Pearson', 'Partial', 'Spearman', 'MGC', - # 'TangentEmbed' # "Skip tangent embedding for now" + "connectivity_matrix": { + "using": ["AFNI", "Nilearn", "ndmg"], + "measure": [ + "Pearson", + "Partial", + "Spearman", + "MGC", + # 'TangentEmbed' # "Skip tangent embedding for now" ], }, - 'Regressors': { - 'CompCor': { - 'degree': int, - 'erode_mask_mm': bool1_1, - 'summary': { - 'method': str, - 'components': int, - 'filter': str, + "Regressors": { + "CompCor": { + "degree": int, + "erode_mask_mm": bool1_1, + "summary": { + "method": str, + "components": int, + "filter": str, }, - 'threshold': str, - 'tissues': [str], - 'extraction_resolution': int + "threshold": str, + "tissues": [str], + "extraction_resolution": int, }, - 'segmentation': { - 'erode_mask': bool1_1, - 'extraction_resolution': Any( - int, float, 'Functional', All(str, Match(RESOLUTION_REGEX)) - ), - 'include_delayed': bool1_1, - 'include_delayed_squared': bool1_1, - 'include_squared': bool1_1, - 'summary': Any( - str, {'components': int, 'method': str} + "segmentation": { + "erode_mask": bool1_1, + "extraction_resolution": Any( + int, float, "Functional", All(str, Match(RESOLUTION_REGEX)) ), + "include_delayed": bool1_1, + "include_delayed_squared": bool1_1, + "include_squared": bool1_1, + "summary": Any(str, {"components": int, "method": str}), }, }, - 'target_space': ['Native', 'Template'] + "target_space": ["Native", "Template"], } -valid_options['space'] = list({option.lower() for option in - valid_options['target_space']}) +valid_options["space"] = list( + {option.lower() for option in valid_options["target_space"]} +) mutex = { # mutually exclusive booleans - 'FSL-BET': { + "FSL-BET": { # exactly zero or one of each of the following can be True for FSL-BET - 'mutex': ['reduce_bias', 'robust', 'padding', 'remove_eyes', - 'surfaces'], + "mutex": ["reduce_bias", "robust", "padding", "remove_eyes", "surfaces"], # the remaining keys: validators for FSL-BET - 'rem': { - 'frac': float, - 'mesh_boolean': bool1_1, - 'outline': bool1_1, - 'radius': int, - 'skull': bool1_1, - 'threshold': bool1_1, - 'vertical_gradient': Range(min=-1, max=1, min_included=False, - max_included=False), - 'functional_mean_thr': { - 'run': bool1_1, - 'threshold_value': Maybe(int), + "rem": { + "frac": float, + "mesh_boolean": bool1_1, + "outline": bool1_1, + "radius": int, + "skull": bool1_1, + "threshold": bool1_1, + "vertical_gradient": Range( + min=-1, max=1, min_included=False, max_included=False + ), + "functional_mean_thr": { + "run": bool1_1, + "threshold_value": Maybe(int), }, - 'functional_mean_bias_correction': bool1_1, - } + "functional_mean_bias_correction": bool1_1, + }, } } ANTs_parameter_transforms = { - 'gradientStep': Number, - 'metric': { - 'type': str, - 'metricWeight': int, - 'numberOfBins': int, - 'samplingStrategy': str, - 'samplingPercentage': Number, - 'radius': Number, + "gradientStep": Number, + "metric": { + "type": str, + "metricWeight": int, + "numberOfBins": int, + "samplingStrategy": str, + "samplingPercentage": Number, + "radius": Number, }, - 'convergence': { - 'iteration': All(str, Match(RESOLUTION_REGEX)), - 'convergenceThreshold': Number, - 'convergenceWindowSize': int, + "convergence": { + "iteration": All(str, Match(RESOLUTION_REGEX)), + "convergenceThreshold": Number, + "convergenceWindowSize": int, }, - 'smoothing-sigmas': All(str, Match(RESOLUTION_REGEX)), - 'shrink-factors': All(str, Match(RESOLUTION_REGEX)), - 'use-histogram-matching': bool1_1, - 'updateFieldVarianceInVoxelSpace': Number, - 'totalFieldVarianceInVoxelSpace': Number, - 'winsorize-image-intensities': { - 'lowerQuantile': float, - 'upperQuantile': float, + "smoothing-sigmas": All(str, Match(RESOLUTION_REGEX)), + "shrink-factors": All(str, Match(RESOLUTION_REGEX)), + "use-histogram-matching": bool1_1, + "updateFieldVarianceInVoxelSpace": Number, + "totalFieldVarianceInVoxelSpace": Number, + "winsorize-image-intensities": { + "lowerQuantile": float, + "upperQuantile": float, }, } -ANTs_parameters = [Any( - { - 'collapse-output-transforms': int - }, { - 'dimensionality': int - }, { - 'initial-moving-transform': { - 'initializationFeature': int, +ANTs_parameters = [ + Any( + {"collapse-output-transforms": int}, + {"dimensionality": int}, + { + "initial-moving-transform": { + "initializationFeature": int, + }, + }, + { + "transforms": [ + Any( + { + "Rigid": ANTs_parameter_transforms, + }, + { + "Affine": ANTs_parameter_transforms, + }, + { + "SyN": ANTs_parameter_transforms, + }, + ) + ], }, - }, { - 'transforms': [Any({ - 'Rigid': ANTs_parameter_transforms, - }, { - 'Affine': ANTs_parameter_transforms, - }, { - 'SyN': ANTs_parameter_transforms, - })], - }, { - 'verbose': Any(Coerce(int), In({0, 1})), - }, { - 'float': Any(Coerce(int), In({0, 1})), - }, { - 'masks': { - 'fixed_image_mask': bool1_1, - 'moving_image_mask': bool1_1, + { + "verbose": Any(Coerce(int), In({0, 1})), }, - }, dict # TODO: specify other valid ANTs parameters -)] -motion_estimate_filter = Any({ # notch filter with breathing_rate_* set - Required('filter_type'): 'notch', - Required('filter_order'): int, - Required('breathing_rate_min'): Number, - 'breathing_rate_max': Number, - 'center_frequency': Maybe(Number), - 'filter_bandwidth': Maybe(Number), - 'lowpass_cutoff': Maybe(Number), - 'Name': Maybe(str) - }, { # notch filter with manual parameters set - Required('filter_type'): 'notch', - Required('filter_order'): int, - 'breathing_rate_min': None, - 'breathing_rate_max': None, - Required('center_frequency'): Number, - Required('filter_bandwidth'): Number, - 'lowpass_cutoff': Maybe(Number), - 'Name': Maybe(str) - }, { # lowpass filter with breathing_rate_min - Required('filter_type'): 'lowpass', - Required('filter_order'): int, - Required('breathing_rate_min'): Number, - 'breathing_rate_max': Maybe(Number), - 'center_frequency': Maybe(Number), - 'filter_bandwidth': Maybe(Number), - 'lowpass_cutoff': Maybe(Number), - 'Name': Maybe(str) - }, { # lowpass filter with lowpass_cutoff - Required('filter_type'): 'lowpass', - Required('filter_order'): int, - Required('breathing_rate_min', default=None): None, - 'breathing_rate_max': Maybe(Number), - 'center_frequency': Maybe(Number), - 'filter_bandwidth': Maybe(Number), - Required('lowpass_cutoff'): Number, - 'Name': Maybe(str)}, - msg='`motion_estimate_filter` configuration is invalid.\nSee ' - f'{DOCS_URL_PREFIX}/user/' - 'func#motion-estimate-filter-valid-options for details.\n') -target_space = All(Coerce(ListFromItem), - [All(Title, In(valid_options['target_space']))]) + { + "float": Any(Coerce(int), In({0, 1})), + }, + { + "masks": { + "fixed_image_mask": bool1_1, + "moving_image_mask": bool1_1, + }, + }, + dict, # TODO: specify other valid ANTs parameters + ) +] +motion_estimate_filter = Any( + { # notch filter with breathing_rate_* set + Required("filter_type"): "notch", + Required("filter_order"): int, + Required("breathing_rate_min"): Number, + "breathing_rate_max": Number, + "center_frequency": Maybe(Number), + "filter_bandwidth": Maybe(Number), + "lowpass_cutoff": Maybe(Number), + "Name": Maybe(str), + }, + { # notch filter with manual parameters set + Required("filter_type"): "notch", + Required("filter_order"): int, + "breathing_rate_min": None, + "breathing_rate_max": None, + Required("center_frequency"): Number, + Required("filter_bandwidth"): Number, + "lowpass_cutoff": Maybe(Number), + "Name": Maybe(str), + }, + { # lowpass filter with breathing_rate_min + Required("filter_type"): "lowpass", + Required("filter_order"): int, + Required("breathing_rate_min"): Number, + "breathing_rate_max": Maybe(Number), + "center_frequency": Maybe(Number), + "filter_bandwidth": Maybe(Number), + "lowpass_cutoff": Maybe(Number), + "Name": Maybe(str), + }, + { # lowpass filter with lowpass_cutoff + Required("filter_type"): "lowpass", + Required("filter_order"): int, + Required("breathing_rate_min", default=None): None, + "breathing_rate_max": Maybe(Number), + "center_frequency": Maybe(Number), + "filter_bandwidth": Maybe(Number), + Required("lowpass_cutoff"): Number, + "Name": Maybe(str), + }, + msg="`motion_estimate_filter` configuration is invalid.\nSee " + f"{DOCS_URL_PREFIX}/user/" + "func#motion-estimate-filter-valid-options for details.\n", +) +target_space = All( + Coerce(ListFromItem), [All(Title, In(valid_options["target_space"]))] +) def name_motion_filter(mfilter, mfilters=None): - '''Given a motion filter, create a short string for the filename + """Given a motion filter, create a short string for the filename. Parameters ---------- @@ -289,41 +347,45 @@ def name_motion_filter(mfilter, mfilters=None): ... 'breathing_rate_min': 0.19}, [{'Name': 'lowpass2fl0p19'}, ... {'Name': 'lowpass2fl0p19dup1'}]) 'lowpass2fl0p19dup2' - ''' + """ if mfilters is None: mfilters = [] - if 'Name' in mfilter: - name = mfilter['Name'] + if "Name" in mfilter: + name = mfilter["Name"] else: - if mfilter['filter_type'] == 'notch': - if mfilter.get('breathing_rate_min'): - range_str = (f'fl{mfilter["breathing_rate_min"]}' - f'fu{mfilter["breathing_rate_max"]}') + if mfilter["filter_type"] == "notch": + if mfilter.get("breathing_rate_min"): + range_str = ( + f'fl{mfilter["breathing_rate_min"]}' + f'fu{mfilter["breathing_rate_max"]}' + ) else: - range_str = (f'fc{mfilter["center_frequency"]}' - f'bw{mfilter["filter_bandwidth"]}') + range_str = ( + f'fc{mfilter["center_frequency"]}' + f'bw{mfilter["filter_bandwidth"]}' + ) else: - if mfilter.get('breathing_rate_min'): + if mfilter.get("breathing_rate_min"): range_str = f'fl{mfilter["breathing_rate_min"]}' else: range_str = f'fc{mfilter["lowpass_cutoff"]}' - range_str = range_str.replace('.', 'p') + range_str = range_str.replace(".", "p") name = f'{mfilter["filter_type"]}{mfilter["filter_order"]}{range_str}' - dupes = 'Name' not in mfilter and len([_ for _ in (_.get('Name', '') for - _ in mfilters) if - _.startswith(name)]) + dupes = "Name" not in mfilter and len( + [_ for _ in (_.get("Name", "") for _ in mfilters) if _.startswith(name)] + ) if dupes: - dup = re.search('(?=[A-Za-z0-9]*)(dup[0-9]*)', name) + dup = re.search("(?=[A-Za-z0-9]*)(dup[0-9]*)", name) if dup: # Don't chain 'dup' suffixes - name = name.replace(dup.group(), f'dup{dupes}') + name = name.replace(dup.group(), f"dup{dupes}") else: - name = f'{name}dup{dupes}' + name = f"{name}dup{dupes}" return name def permutation_message(key, options): - '''Function to give a clean, human-readable error message for keys - that accept permutation values + """Function to give a clean, human-readable error message for keys + that accept permutation values. Parameters ---------- @@ -333,8 +395,9 @@ def permutation_message(key, options): Returns ------- - msg: str''' # noqa: E501 - return f''' + msg: str + """ + return f""" \'{key}\' takes a dictionary with paths to region-of-interest (ROI) NIFTI files (.nii or .nii.gz) as keys and a comma separated string @@ -345,746 +408,841 @@ def permutation_message(key, options): Available analyses for \'{key}\' are {options} -''' +""" def sanitize(filename): - '''Sanitize a filename and replace whitespaces with underscores''' - return re.sub(r'\s+', '_', sanitize_filename(filename)) + """Sanitize a filename and replace whitespaces with underscores.""" + return re.sub(r"\s+", "_", sanitize_filename(filename)) -latest_schema = Schema({ - 'FROM': Maybe(str), - 'pipeline_setup': { - 'pipeline_name': All(str, Length(min=1), sanitize), - 'output_directory': { - 'path': str, - 'source_outputs_dir': Maybe(str), - 'pull_source_once': bool1_1, - 'write_func_outputs': bool1_1, - 'write_debugging_outputs': bool1_1, - 'output_tree': str, - 'quality_control': { - 'generate_quality_control_images': bool1_1, - 'generate_xcpqc_files': bool1_1, - }, - 'user_defined': Maybe(str), - }, - 'working_directory': { - 'path': str, - 'remove_working_dir': bool1_1, - }, - 'log_directory': { - 'run_logging': bool1_1, - 'path': str, - 'graphviz': { - 'entire_workflow': { - 'generate': bool, - 'graph2use': Maybe(All(Coerce(ListFromItem), - [All(Lower, - In(('orig', 'hierarchical', 'flat', - 'exec', 'colored')))])), - 'format': Maybe(All(Coerce(ListFromItem), - [All(Lower, In(('png', 'svg')))])), - 'simple_form': Maybe(bool)}}, - }, - 'crash_log_directory': { - 'path': Maybe(str), - }, - 'system_config': { - 'fail_fast': bool1_1, - 'FSLDIR': Maybe(str), - 'on_grid': { - 'run': bool1_1, - 'resource_manager': Maybe(str), - 'SGE': { - 'parallel_environment': Maybe(str), - 'queue': Maybe(str), +latest_schema = Schema( + { + "FROM": Maybe(str), + "pipeline_setup": { + "pipeline_name": All(str, Length(min=1), sanitize), + "output_directory": { + "path": str, + "source_outputs_dir": Maybe(str), + "pull_source_once": bool1_1, + "write_func_outputs": bool1_1, + "write_debugging_outputs": bool1_1, + "output_tree": str, + "quality_control": { + "generate_quality_control_images": bool1_1, + "generate_xcpqc_files": bool1_1, }, + "user_defined": Maybe(str), }, - 'maximum_memory_per_participant': Number, - 'raise_insufficient': bool1_1, - 'max_cores_per_participant': int, - 'num_ants_threads': int, - 'num_OMP_threads': int, - 'num_participants_at_once': int, - 'random_seed': Maybe(Any( - 'random', - All(int, Range(min=1, max=np.iinfo(np.int32).max)))), - 'observed_usage': { - 'callback_log': Maybe(str), - 'buffer': Number, + "working_directory": { + "path": str, + "remove_working_dir": bool1_1, }, - }, - 'Amazon-AWS': { - 'aws_output_bucket_credentials': Maybe(str), - 's3_encryption': bool1_1, - }, - 'Debugging': { - 'verbose': bool1_1, - }, - 'outdir_ingress': { - 'run': bool1_1, - 'Template': Maybe(str), - }, - }, - 'anatomical_preproc': { - 'run': bool1_1, - 'run_t2': bool1_1, - 'non_local_means_filtering': { - 'run': forkable, - 'noise_model': Maybe(str), - }, - 'n4_bias_field_correction': { - 'run': forkable, - 'shrink_factor': int, - }, - 't1t2_bias_field_correction': Required( - # require 'T1w_brain_ACPC_template' if 'acpc_target' is 'brain' - Any({ - 'run': False, - 'BiasFieldSmoothingSigma': Maybe(int), - }, { - 'run': True, - 'BiasFieldSmoothingSigma': Maybe(int), - },), - ), - - 'acpc_alignment': Required( - # require 'T1w_brain_ACPC_template' and - # 'T2w_brain_ACPC_template' if 'acpc_target' is 'brain' - Any({ - 'run': False, - 'run_before_preproc': Maybe(bool1_1), - 'brain_size': Maybe(int), - 'FOV_crop': Maybe(In({'robustfov', 'flirt'})), - 'acpc_target': Maybe(In(valid_options['acpc']['target'])), - 'align_brain_mask': Maybe(bool1_1), - 'T1w_ACPC_template': Maybe(str), - 'T1w_brain_ACPC_template': Maybe(str), - 'T2w_ACPC_template': Maybe(str), - 'T2w_brain_ACPC_template': Maybe(str), - }, { - 'run': True, - 'run_before_preproc': bool1_1, - 'brain_size': int, - 'FOV_crop': In({'robustfov', 'flirt'}), - 'acpc_target': valid_options['acpc']['target'][1], - 'align_brain_mask': Maybe(bool1_1), - 'T1w_ACPC_template': str, - 'T1w_brain_ACPC_template': Maybe(str), - 'T2w_ACPC_template': Maybe(str), - 'T2w_brain_ACPC_template': Maybe(str), - }, { - 'run': True, - 'run_before_preproc': bool1_1, - 'brain_size': int, - 'FOV_crop': In({'robustfov', 'flirt'}), - 'acpc_target': valid_options['acpc']['target'][0], - 'align_brain_mask': Maybe(bool1_1), - 'T1w_ACPC_template': str, - 'T1w_brain_ACPC_template': str, - 'T2w_ACPC_template': Maybe(str), - 'T2w_brain_ACPC_template': Maybe(str), - },), - msg='\'brain\' requires \'T1w_brain_ACPC_template\' and ' - '\'T2w_brain_ACPC_template\' to ' - 'be populated if \'run\' is not set to Off', - ), - 'brain_extraction': { - 'run': bool1_1, - 'using': [In(valid_options['brain_extraction']['using'])], - 'AFNI-3dSkullStrip': { - 'mask_vol': bool1_1, - 'shrink_factor': Number, - 'var_shrink_fac': bool1_1, - 'shrink_factor_bot_lim': Number, - 'avoid_vent': bool1_1, - 'n_iterations': int, - 'pushout': bool1_1, - 'touchup': bool1_1, - 'fill_hole': int, - 'NN_smooth': int, - 'smooth_final': int, - 'avoid_eyes': bool1_1, - 'use_edge': bool1_1, - 'exp_frac': Number, - 'push_to_edge': bool1_1, - 'use_skull': bool1_1, - 'perc_int': Number, - 'max_inter_iter': int, - 'fac': Number, - 'blur_fwhm': Number, - 'monkey': bool1_1, - }, - 'FSL-FNIRT': { - 'interpolation': In({ - 'trilinear', 'sinc', 'spline' - }), - }, - 'FSL-BET': { - 'frac': Number, - 'mask_boolean': bool1_1, - 'mesh_boolean': bool1_1, - 'outline': bool1_1, - 'padding': bool1_1, - 'radius': int, - 'reduce_bias': bool1_1, - 'remove_eyes': bool1_1, - 'robust': bool1_1, - 'skull': bool1_1, - 'surfaces': bool1_1, - 'threshold': bool1_1, - 'vertical_gradient': Range(min=-1, max=1) - }, - 'UNet': { - 'unet_model': Maybe(str), - }, - 'niworkflows-ants': { - 'template_path': Maybe(str), - 'mask_path': Maybe(str), - 'regmask_path': Maybe(str), + "log_directory": { + "run_logging": bool1_1, + "path": str, + "graphviz": { + "entire_workflow": { + "generate": bool, + "graph2use": Maybe( + All( + Coerce(ListFromItem), + [ + All( + Lower, + In( + ( + "orig", + "hierarchical", + "flat", + "exec", + "colored", + ) + ), + ) + ], + ) + ), + "format": Maybe( + All(Coerce(ListFromItem), [All(Lower, In(("png", "svg")))]) + ), + "simple_form": Maybe(bool), + } + }, }, - 'FreeSurfer-BET': { - 'T1w_brain_template_mask_ccs': Maybe(str) + "crash_log_directory": { + "path": Maybe(str), }, - }, - }, - 'segmentation': { - 'run': bool1_1, - 'tissue_segmentation': { - 'using': [In( - {'FSL-FAST', 'FreeSurfer', 'ANTs_Prior_Based', - 'Template_Based'} - )], - 'FSL-FAST': { - 'thresholding': { - 'use': In({'Auto', 'Custom'}), - 'Custom': { - 'CSF_threshold_value': float, - 'WM_threshold_value': float, - 'GM_threshold_value': float, + "system_config": { + "fail_fast": bool1_1, + "FSLDIR": Maybe(str), + "on_grid": { + "run": bool1_1, + "resource_manager": Maybe(str), + "SGE": { + "parallel_environment": Maybe(str), + "queue": Maybe(str), }, }, - 'use_priors': { - 'run': bool1_1, - 'priors_path': Maybe(str), - 'WM_path': Maybe(str), - 'GM_path': Maybe(str), - 'CSF_path': Maybe(str) + "maximum_memory_per_participant": Number, + "raise_insufficient": bool1_1, + "max_cores_per_participant": int, + "num_ants_threads": int, + "num_OMP_threads": int, + "num_participants_at_once": int, + "random_seed": Maybe( + Any("random", All(int, Range(min=1, max=np.iinfo(np.int32).max))) + ), + "observed_usage": { + "callback_log": Maybe(str), + "buffer": Number, }, }, - 'FreeSurfer': { - 'erode': Maybe(int), - 'CSF_label': Maybe([int]), - 'GM_label': Maybe([int]), - 'WM_label': Maybe([int]), + "Amazon-AWS": { + "aws_output_bucket_credentials": Maybe(str), + "s3_encryption": bool1_1, }, - 'ANTs_Prior_Based': { - 'run': forkable, - 'template_brain_list': Maybe(Any([str], [])), - 'template_segmentation_list': Maybe(Any([str], [])), - 'CSF_label': [int], - 'GM_label': [int], - 'WM_label': [int], + "Debugging": { + "verbose": bool1_1, }, - 'Template_Based': { - 'run': forkable, - 'template_for_segmentation': [In( - valid_options['segmentation']['template'] - )], - 'WHITE': Maybe(str), - 'GRAY': Maybe(str), - 'CSF': Maybe(str), + "outdir_ingress": { + "run": bool1_1, + "Template": Maybe(str), }, }, - }, - 'registration_workflows': { - 'anatomical_registration': { - 'run': bool1_1, - 'resolution_for_anat': All(str, Match(RESOLUTION_REGEX)), - 'T1w_brain_template': Maybe(str), - 'T1w_template': Maybe(str), - 'T1w_brain_template_mask': Maybe(str), - 'reg_with_skull': bool1_1, - 'registration': { - 'using': [In({'ANTS', 'FSL', 'FSL-linear'})], - 'ANTs': { - 'use_lesion_mask': bool1_1, - 'T1_registration': Maybe(ANTs_parameters), - 'interpolation': In({ - 'Linear', 'BSpline', 'LanczosWindowedSinc' - }), - }, - 'FSL-FNIRT': { - 'fnirt_config': Maybe(str), - 'ref_resolution': All(str, Match(RESOLUTION_REGEX)), - 'FNIRT_T1w_brain_template': Maybe(str), - 'FNIRT_T1w_template': Maybe(str), - 'interpolation': In({ - 'trilinear', 'sinc', 'spline' - }), - 'identity_matrix': Maybe(str), - 'ref_mask': Maybe(str), - 'ref_mask_res-2': Maybe(str), - 'T1w_template_res-2': Maybe(str), - }, + "anatomical_preproc": { + "run": bool1_1, + "run_t2": bool1_1, + "non_local_means_filtering": { + "run": forkable, + "noise_model": Maybe(str), }, - 'overwrite_transform': { - 'run': bool1_1, - 'using': In({'FSL'}), + "n4_bias_field_correction": { + "run": forkable, + "shrink_factor": int, }, - }, - 'functional_registration': { - 'coregistration': { - 'run': bool1_1, - 'reference': In({'brain', 'restore-brain'}), - 'interpolation': In({'trilinear', 'sinc', 'spline'}), - 'using': str, - 'input': str, - 'cost': str, - 'dof': int, - 'arguments': Maybe(str), - 'func_input_prep': { - 'reg_with_skull': bool1_1, - 'input': [In({ - 'Mean_Functional', 'Selected_Functional_Volume', - 'fmriprep_reference' - })], - 'Mean Functional': { - 'n4_correct_func': bool1_1 + "t1t2_bias_field_correction": Required( + # require 'T1w_brain_ACPC_template' if 'acpc_target' is 'brain' + Any( + { + "run": False, + "BiasFieldSmoothingSigma": Maybe(int), + }, + { + "run": True, + "BiasFieldSmoothingSigma": Maybe(int), + }, + ), + ), + "acpc_alignment": Required( + # require 'T1w_brain_ACPC_template' and + # 'T2w_brain_ACPC_template' if 'acpc_target' is 'brain' + Any( + { + "run": False, + "run_before_preproc": Maybe(bool1_1), + "brain_size": Maybe(int), + "FOV_crop": Maybe(In({"robustfov", "flirt"})), + "acpc_target": Maybe(In(valid_options["acpc"]["target"])), + "align_brain_mask": Maybe(bool1_1), + "T1w_ACPC_template": Maybe(str), + "T1w_brain_ACPC_template": Maybe(str), + "T2w_ACPC_template": Maybe(str), + "T2w_brain_ACPC_template": Maybe(str), }, - 'Selected Functional Volume': { - 'func_reg_input_volume': int + { + "run": True, + "run_before_preproc": bool1_1, + "brain_size": int, + "FOV_crop": In({"robustfov", "flirt"}), + "acpc_target": valid_options["acpc"]["target"][1], + "align_brain_mask": Maybe(bool1_1), + "T1w_ACPC_template": str, + "T1w_brain_ACPC_template": Maybe(str), + "T2w_ACPC_template": Maybe(str), + "T2w_brain_ACPC_template": Maybe(str), + }, + { + "run": True, + "run_before_preproc": bool1_1, + "brain_size": int, + "FOV_crop": In({"robustfov", "flirt"}), + "acpc_target": valid_options["acpc"]["target"][0], + "align_brain_mask": Maybe(bool1_1), + "T1w_ACPC_template": str, + "T1w_brain_ACPC_template": str, + "T2w_ACPC_template": Maybe(str), + "T2w_brain_ACPC_template": Maybe(str), }, + ), + msg="'brain' requires 'T1w_brain_ACPC_template' and " + "'T2w_brain_ACPC_template' to " + "be populated if 'run' is not set to Off", + ), + "brain_extraction": { + "run": bool1_1, + "using": [In(valid_options["brain_extraction"]["using"])], + "AFNI-3dSkullStrip": { + "mask_vol": bool1_1, + "shrink_factor": Number, + "var_shrink_fac": bool1_1, + "shrink_factor_bot_lim": Number, + "avoid_vent": bool1_1, + "n_iterations": int, + "pushout": bool1_1, + "touchup": bool1_1, + "fill_hole": int, + "NN_smooth": int, + "smooth_final": int, + "avoid_eyes": bool1_1, + "use_edge": bool1_1, + "exp_frac": Number, + "push_to_edge": bool1_1, + "use_skull": bool1_1, + "perc_int": Number, + "max_inter_iter": int, + "fac": Number, + "blur_fwhm": Number, + "monkey": bool1_1, }, - 'boundary_based_registration': { - 'run': forkable, - 'bbr_schedule': str, - 'bbr_wm_map': In({'probability_map', 'partial_volume_map'}), - 'bbr_wm_mask_args': str, - 'reference': In({'whole-head', 'brain'}) + "FSL-FNIRT": { + "interpolation": In({"trilinear", "sinc", "spline"}), }, - }, - 'EPI_registration': { - 'run': bool1_1, - 'using': [In({'ANTS', 'FSL', 'FSL-linear'})], - 'EPI_template': Maybe(str), - 'EPI_template_mask': Maybe(str), - 'ANTs': { - 'parameters': Maybe(ANTs_parameters), - 'interpolation': In({ - 'Linear', 'BSpline', 'LanczosWindowedSinc' - }), + "FSL-BET": { + "frac": Number, + "mask_boolean": bool1_1, + "mesh_boolean": bool1_1, + "outline": bool1_1, + "padding": bool1_1, + "radius": int, + "reduce_bias": bool1_1, + "remove_eyes": bool1_1, + "robust": bool1_1, + "skull": bool1_1, + "surfaces": bool1_1, + "threshold": bool1_1, + "vertical_gradient": Range(min=-1, max=1), }, - 'FSL-FNIRT': { - 'fnirt_config': Maybe(str), - 'interpolation': In({'trilinear', 'sinc', 'spline'}), - 'identity_matrix': Maybe(str), + "UNet": { + "unet_model": Maybe(str), }, - }, - 'func_registration_to_template': { - 'run': bool1_1, - 'run_EPI': bool1_1, - 'output_resolution': { - 'func_preproc_outputs': All( - str, Match(RESOLUTION_REGEX)), - 'func_derivative_outputs': All( - str, Match(RESOLUTION_REGEX) - ), + "niworkflows-ants": { + "template_path": Maybe(str), + "mask_path": Maybe(str), + "regmask_path": Maybe(str), }, - 'target_template': { - 'using': [In({'T1_template', 'EPI_template'})], - 'T1_template': { - 'T1w_brain_template_funcreg': Maybe(str), - 'T1w_template_funcreg': Maybe(str), - 'T1w_brain_template_mask_funcreg': Maybe(str), - 'T1w_template_for_resample': Maybe(str), + "FreeSurfer-BET": {"T1w_brain_template_mask_ccs": Maybe(str)}, + }, + }, + "segmentation": { + "run": bool1_1, + "tissue_segmentation": { + "using": [ + In({"FSL-FAST", "FreeSurfer", "ANTs_Prior_Based", "Template_Based"}) + ], + "FSL-FAST": { + "thresholding": { + "use": In({"Auto", "Custom"}), + "Custom": { + "CSF_threshold_value": float, + "WM_threshold_value": float, + "GM_threshold_value": float, + }, }, - 'EPI_template': { - 'EPI_template_funcreg': Maybe(str), - 'EPI_template_mask_funcreg': Maybe(str), - 'EPI_template_for_resample': Maybe(str) + "use_priors": { + "run": bool1_1, + "priors_path": Maybe(str), + "WM_path": Maybe(str), + "GM_path": Maybe(str), + "CSF_path": Maybe(str), }, }, - 'ANTs_pipelines': { - 'interpolation': In({ - 'Linear', 'BSpline', 'LanczosWindowedSinc'}) + "FreeSurfer": { + "erode": Maybe(int), + "CSF_label": Maybe([int]), + "GM_label": Maybe([int]), + "WM_label": Maybe([int]), }, - 'FNIRT_pipelines': { - 'interpolation': In({'trilinear', 'sinc', 'spline'}), - 'identity_matrix': Maybe(str), + "ANTs_Prior_Based": { + "run": forkable, + "template_brain_list": Maybe(Any([str], [])), + "template_segmentation_list": Maybe(Any([str], [])), + "CSF_label": [int], + "GM_label": [int], + "WM_label": [int], }, - 'apply_transform': { - 'using': In({'default', 'abcd', 'dcan_nhp', - 'single_step_resampling_from_stc'}), + "Template_Based": { + "run": forkable, + "template_for_segmentation": [ + In(valid_options["segmentation"]["template"]) + ], + "WHITE": Maybe(str), + "GRAY": Maybe(str), + "CSF": Maybe(str), }, }, }, - }, - 'surface_analysis': { - 'abcd_prefreesurfer_prep':{ - 'run': bool1_1, - }, - 'freesurfer': { - 'run_reconall': bool1_1, - 'reconall_args': Maybe(str), - # 'generate_masks': bool1_1, - 'ingress_reconall': bool1_1, - }, - 'post_freesurfer': { - 'run': bool1_1, - 'surf_atlas_dir': Maybe(str), - 'gray_ordinates_dir': Maybe(str), - 'gray_ordinates_res': Maybe(int), - 'high_res_mesh': Maybe(int), - 'low_res_mesh': Maybe(int), - 'subcortical_gray_labels': Maybe(str), - 'freesurfer_labels': Maybe(str), - 'fmri_res': Maybe(int), - 'smooth_fwhm': Maybe(int), - }, - }, - 'longitudinal_template_generation': { - 'run': bool1_1, - 'average_method': In({'median', 'mean', 'std'}), - 'dof': In({12, 9, 7, 6}), - 'interp': In({'trilinear', 'nearestneighbour', 'sinc', 'spline'}), - 'cost': In({ - 'corratio', 'mutualinfo', 'normmi', 'normcorr', 'leastsq', - 'labeldiff', 'bbr'}), - 'thread_pool': int, - 'convergence_threshold': Number, - }, - 'functional_preproc': { - 'run': bool1_1, - 'truncation': { - 'start_tr': int, - 'stop_tr': Maybe(Any(int, All(Capitalize, 'End'))) - }, - 'update_header': { - 'run': bool1_1, - }, - 'scaling': { - 'run': bool1_1, - 'scaling_factor': Number - }, - 'despiking': { - 'run': forkable, - 'space': In({'native', 'template'}) - }, - 'slice_timing_correction': { - 'run': forkable, - 'tpattern': Maybe(str), - 'tzero': Maybe(int), - }, - 'motion_estimates_and_correction': { - 'run': bool1_1, - 'motion_estimates': { - 'calculate_motion_first': bool1_1, - 'calculate_motion_after': bool1_1, + "registration_workflows": { + "anatomical_registration": { + "run": bool1_1, + "resolution_for_anat": All(str, Match(RESOLUTION_REGEX)), + "T1w_brain_template": Maybe(str), + "T1w_template": Maybe(str), + "T1w_brain_template_mask": Maybe(str), + "reg_with_skull": bool1_1, + "registration": { + "using": [In({"ANTS", "FSL", "FSL-linear"})], + "ANTs": { + "use_lesion_mask": bool1_1, + "T1_registration": Maybe(ANTs_parameters), + "interpolation": In( + {"Linear", "BSpline", "LanczosWindowedSinc"} + ), + }, + "FSL-FNIRT": { + "fnirt_config": Maybe(str), + "ref_resolution": All(str, Match(RESOLUTION_REGEX)), + "FNIRT_T1w_brain_template": Maybe(str), + "FNIRT_T1w_template": Maybe(str), + "interpolation": In({"trilinear", "sinc", "spline"}), + "identity_matrix": Maybe(str), + "ref_mask": Maybe(str), + "ref_mask_res-2": Maybe(str), + "T1w_template_res-2": Maybe(str), + }, + }, + "overwrite_transform": { + "run": bool1_1, + "using": In({"FSL"}), + }, }, - 'motion_correction': { - 'using': Optional(All(Coerce(ListFromItem), - Length(min=0, max=1, - msg='Forking is currently broken for this option. ' - 'Please use separate configs if you want to ' - 'use each of 3dvolreg and mcflirt. Follow ' - 'https://github.com/FCP-INDI/C-PAC/issues/1935 ' - 'to see when this issue is resolved.'), - [In(valid_options['motion_correction'])])), - 'AFNI-3dvolreg': { - 'functional_volreg_twopass': bool1_1, + "functional_registration": { + "coregistration": { + "run": bool1_1, + "reference": In({"brain", "restore-brain"}), + "interpolation": In({"trilinear", "sinc", "spline"}), + "using": str, + "input": str, + "cost": str, + "dof": int, + "arguments": Maybe(str), + "func_input_prep": { + "reg_with_skull": bool1_1, + "input": [ + In( + { + "Mean_Functional", + "Selected_Functional_Volume", + "fmriprep_reference", + } + ) + ], + "Mean Functional": {"n4_correct_func": bool1_1}, + "Selected Functional Volume": {"func_reg_input_volume": int}, + }, + "boundary_based_registration": { + "run": forkable, + "bbr_schedule": str, + "bbr_wm_map": In({"probability_map", "partial_volume_map"}), + "bbr_wm_mask_args": str, + "reference": In({"whole-head", "brain"}), + }, + }, + "EPI_registration": { + "run": bool1_1, + "using": [In({"ANTS", "FSL", "FSL-linear"})], + "EPI_template": Maybe(str), + "EPI_template_mask": Maybe(str), + "ANTs": { + "parameters": Maybe(ANTs_parameters), + "interpolation": In( + {"Linear", "BSpline", "LanczosWindowedSinc"} + ), + }, + "FSL-FNIRT": { + "fnirt_config": Maybe(str), + "interpolation": In({"trilinear", "sinc", "spline"}), + "identity_matrix": Maybe(str), + }, + }, + "func_registration_to_template": { + "run": bool1_1, + "run_EPI": bool1_1, + "output_resolution": { + "func_preproc_outputs": All(str, Match(RESOLUTION_REGEX)), + "func_derivative_outputs": All(str, Match(RESOLUTION_REGEX)), + }, + "target_template": { + "using": [In({"T1_template", "EPI_template"})], + "T1_template": { + "T1w_brain_template_funcreg": Maybe(str), + "T1w_template_funcreg": Maybe(str), + "T1w_brain_template_mask_funcreg": Maybe(str), + "T1w_template_for_resample": Maybe(str), + }, + "EPI_template": { + "EPI_template_funcreg": Maybe(str), + "EPI_template_mask_funcreg": Maybe(str), + "EPI_template_for_resample": Maybe(str), + }, + }, + "ANTs_pipelines": { + "interpolation": In( + {"Linear", "BSpline", "LanczosWindowedSinc"} + ) + }, + "FNIRT_pipelines": { + "interpolation": In({"trilinear", "sinc", "spline"}), + "identity_matrix": Maybe(str), + }, + "apply_transform": { + "using": In( + { + "default", + "abcd", + "dcan_nhp", + "single_step_resampling_from_stc", + } + ), + }, }, - 'motion_correction_reference': [In({ - 'mean', 'median', 'selected_volume', - 'fmriprep_reference'})], - 'motion_correction_reference_volume': int, }, - 'motion_estimate_filter': Required( - Any({'run': forkable, - 'filters': [motion_estimate_filter]}, - {'run': All(forkable, [In([False], [])]), - 'filters': Maybe(list)}) - ), }, - 'distortion_correction': { - 'run': forkable, - 'using': [In(['PhaseDiff', 'Blip', 'Blip-FSL-TOPUP'])], - 'PhaseDiff': { - 'fmap_skullstrip_option': In(['BET', 'AFNI']), - 'fmap_skullstrip_BET_frac': float, - 'fmap_skullstrip_AFNI_threshold': float, + "surface_analysis": { + "abcd_prefreesurfer_prep": { + "run": bool1_1, }, - 'Blip-FSL-TOPUP': { - 'warpres': int, - 'subsamp': int, - 'fwhm': int, - 'miter': int, - 'lambda': int, - 'ssqlambda': int, - 'regmod': In({'bending_energy', 'membrane_energy'}), - 'estmov': int, - 'minmet': int, - 'splineorder': int, - 'numprec': str, - 'interp': In({'spline', 'linear'}), - 'scale': int, - 'regrid': int - } - }, - 'func_masking': { - 'run': bool1_1, - 'using': [In( - ['AFNI', 'FSL', 'FSL_AFNI', 'Anatomical_Refined', - 'Anatomical_Based', 'Anatomical_Resampled', - 'CCS_Anatomical_Refined'] - )], - # handle validating mutually-exclusive booleans for FSL-BET - # functional_mean_boolean must be True if one of the mutually- - # exclusive options are - # see mutex definition for more definition - 'FSL-BET': Maybe(Any(*( - # exactly one mutually exclusive option on - [{k: d[k] for d in r for k in d} for r in [[ - { - **mutex['FSL-BET']['rem'], - 'functional_mean_boolean': True, - k1: True, - k2: False - } for k2 in mutex['FSL-BET']['mutex'] if k2 != k1 - ] for k1 in mutex['FSL-BET']['mutex']]] + - # no mutually-exclusive options on - [{ - **mutex['FSL-BET']['rem'], - 'functional_mean_boolean': bool1_1, - **{k: False for k in mutex['FSL-BET']['mutex']} - }])) - ), - 'FSL_AFNI': { - 'bold_ref': Maybe(str), - 'brain_mask': Maybe(str), - 'brain_probseg': Maybe(str), + "freesurfer": { + "run_reconall": bool1_1, + "reconall_args": Maybe(str), + # 'generate_masks': bool1_1, + "ingress_reconall": bool1_1, }, - 'Anatomical_Refined': { - 'anatomical_mask_dilation': Maybe(bool1_1), + "post_freesurfer": { + "run": bool1_1, + "surf_atlas_dir": Maybe(str), + "gray_ordinates_dir": Maybe(str), + "gray_ordinates_res": Maybe(int), + "high_res_mesh": Maybe(int), + "low_res_mesh": Maybe(int), + "subcortical_gray_labels": Maybe(str), + "freesurfer_labels": Maybe(str), + "fmri_res": Maybe(int), + "smooth_fwhm": Maybe(int), }, - 'apply_func_mask_in_native_space': bool1_1, - }, - 'generate_func_mean': { - 'run': bool1_1, }, - 'normalize_func': { - 'run': bool1_1, - }, - 'coreg_prep': { - 'run': bool1_1, - }, - }, - 'nuisance_corrections': { - '1-ICA-AROMA': { - 'run': forkable, - 'denoising_type': In({'aggr', 'nonaggr'}), + "longitudinal_template_generation": { + "run": bool1_1, + "average_method": In({"median", "mean", "std"}), + "dof": In({12, 9, 7, 6}), + "interp": In({"trilinear", "nearestneighbour", "sinc", "spline"}), + "cost": In( + { + "corratio", + "mutualinfo", + "normmi", + "normcorr", + "leastsq", + "labeldiff", + "bbr", + } + ), + "thread_pool": int, + "convergence_threshold": Number, }, - '2-nuisance_regression': { - 'run': forkable, - 'space': All(Coerce(ItemFromList), - Lower, In({'native', 'template'})), - 'create_regressors': bool1_1, - 'ingress_regressors': { - 'run': bool1_1, - 'Regressors': { - 'Name': Maybe(str), - 'Columns': [str]}, + "functional_preproc": { + "run": bool1_1, + "truncation": { + "start_tr": int, + "stop_tr": Maybe(Any(int, All(Capitalize, "End"))), + }, + "update_header": { + "run": bool1_1, + }, + "scaling": {"run": bool1_1, "scaling_factor": Number}, + "despiking": {"run": forkable, "space": In({"native", "template"})}, + "slice_timing_correction": { + "run": forkable, + "tpattern": Maybe(str), + "tzero": Maybe(int), }, - 'Regressors': Maybe([Schema({ - 'Name': Required(str), - 'Censor': { - 'method': str, - 'thresholds': [{ - 'type': str, - 'value': float, - }], - 'number_of_previous_trs_to_censor': Maybe(int), - 'number_of_subsequent_trs_to_censor': Maybe(int), + "motion_estimates_and_correction": { + "run": bool1_1, + "motion_estimates": { + "calculate_motion_first": bool1_1, + "calculate_motion_after": bool1_1, }, - 'Motion': { - 'include_delayed': bool1_1, - 'include_squared': bool1_1, - 'include_delayed_squared': bool1_1 + "motion_correction": { + "using": Optional( + All( + Coerce(ListFromItem), + Length( + min=0, + max=1, + msg="Forking is currently broken for this option. " + "Please use separate configs if you want to " + "use each of 3dvolreg and mcflirt. Follow " + "https://github.com/FCP-INDI/C-PAC/issues/1935 " + "to see when this issue is resolved.", + ), + [In(valid_options["motion_correction"])], + ) + ), + "AFNI-3dvolreg": { + "functional_volreg_twopass": bool1_1, + }, + "motion_correction_reference": [ + In({"mean", "median", "selected_volume", "fmriprep_reference"}) + ], + "motion_correction_reference_volume": int, }, - 'aCompCor': valid_options['Regressors']['CompCor'], - 'tCompCor': valid_options['Regressors']['CompCor'], - 'CerebrospinalFluid': valid_options[ - 'Regressors' - ]['segmentation'], - 'WhiteMatter': valid_options[ - 'Regressors' - ]['segmentation'], - 'GreyMatter': valid_options[ - 'Regressors' - ]['segmentation'], - 'GlobalSignal': {'summary': str}, - 'PolyOrt': {'degree': int}, - 'Bandpass': { - 'bottom_frequency': float, - 'top_frequency': float, - 'method': str, - } # how to check if [0] is > than [1]? - }, extra=ALLOW_EXTRA)]), - 'lateral_ventricles_mask': Maybe(str), - 'bandpass_filtering_order': Maybe( - In({'After', 'Before'})), - 'regressor_masks': { - 'erode_anatomical_brain_mask': { - 'run': bool1_1, - 'brain_mask_erosion_prop': Maybe(Number), - 'brain_mask_erosion_mm': Maybe(Number), - 'brain_erosion_mm': Maybe(Number) + "motion_estimate_filter": Required( + Any( + {"run": forkable, "filters": [motion_estimate_filter]}, + { + "run": All(forkable, [In([False], [])]), + "filters": Maybe(list), + }, + ) + ), + }, + "distortion_correction": { + "run": forkable, + "using": [In(["PhaseDiff", "Blip", "Blip-FSL-TOPUP"])], + "PhaseDiff": { + "fmap_skullstrip_option": In(["BET", "AFNI"]), + "fmap_skullstrip_BET_frac": float, + "fmap_skullstrip_AFNI_threshold": float, + }, + "Blip-FSL-TOPUP": { + "warpres": int, + "subsamp": int, + "fwhm": int, + "miter": int, + "lambda": int, + "ssqlambda": int, + "regmod": In({"bending_energy", "membrane_energy"}), + "estmov": int, + "minmet": int, + "splineorder": int, + "numprec": str, + "interp": In({"spline", "linear"}), + "scale": int, + "regrid": int, }, - 'erode_csf': { - 'run': bool1_1, - 'csf_erosion_prop': Maybe(Number), - 'csf_mask_erosion_mm': Maybe(Number), - 'csf_erosion_mm': Maybe(Number), + }, + "func_masking": { + "run": bool1_1, + "using": [ + In( + [ + "AFNI", + "FSL", + "FSL_AFNI", + "Anatomical_Refined", + "Anatomical_Based", + "Anatomical_Resampled", + "CCS_Anatomical_Refined", + ] + ) + ], + # handle validating mutually-exclusive booleans for FSL-BET + # functional_mean_boolean must be True if one of the mutually- + # exclusive options are + # see mutex definition for more definition + "FSL-BET": Maybe( + Any( + *( + # exactly one mutually exclusive option on + [ + {k: d[k] for d in r for k in d} + for r in [ + [ + { + **mutex["FSL-BET"]["rem"], + "functional_mean_boolean": True, + k1: True, + k2: False, + } + for k2 in mutex["FSL-BET"]["mutex"] + if k2 != k1 + ] + for k1 in mutex["FSL-BET"]["mutex"] + ] + ] + + + # no mutually-exclusive options on + [ + { + **mutex["FSL-BET"]["rem"], + "functional_mean_boolean": bool1_1, + **{k: False for k in mutex["FSL-BET"]["mutex"]}, + } + ] + ) + ) + ), + "FSL_AFNI": { + "bold_ref": Maybe(str), + "brain_mask": Maybe(str), + "brain_probseg": Maybe(str), }, - 'erode_wm': { - 'run': bool1_1, - 'wm_erosion_prop': Maybe(Number), - 'wm_mask_erosion_mm': Maybe(Number), - 'wm_erosion_mm': Maybe(Number), + "Anatomical_Refined": { + "anatomical_mask_dilation": Maybe(bool1_1), }, - 'erode_gm': { - 'run': bool1_1, - 'gm_erosion_prop': Maybe(Number), - 'gm_mask_erosion_mm': Maybe(Number), - 'gm_erosion_mm': Maybe(Number), - } + "apply_func_mask_in_native_space": bool1_1, + }, + "generate_func_mean": { + "run": bool1_1, + }, + "normalize_func": { + "run": bool1_1, + }, + "coreg_prep": { + "run": bool1_1, }, }, - }, - 'amplitude_low_frequency_fluctuation': { - 'run': bool1_1, - 'target_space': target_space, - 'highpass_cutoff': [float], - 'lowpass_cutoff': [float], - }, - 'voxel_mirrored_homotopic_connectivity': { - 'run': bool1_1, - 'symmetric_registration': { - 'T1w_brain_template_symmetric': Maybe(str), - 'T1w_brain_template_symmetric_funcreg': Maybe(str), - 'T1w_brain_template_symmetric_for_resample': Maybe(str), - 'T1w_template_symmetric': Maybe(str), - 'T1w_template_symmetric_funcreg': Maybe(str), - 'T1w_template_symmetric_for_resample': Maybe(str), - 'dilated_symmetric_brain_mask': Maybe(str), - 'dilated_symmetric_brain_mask_for_resample': Maybe(str), + "nuisance_corrections": { + "1-ICA-AROMA": { + "run": forkable, + "denoising_type": In({"aggr", "nonaggr"}), + }, + "2-nuisance_regression": { + "run": forkable, + "space": All(Coerce(ItemFromList), Lower, In({"native", "template"})), + "create_regressors": bool1_1, + "ingress_regressors": { + "run": bool1_1, + "Regressors": {"Name": Maybe(str), "Columns": [str]}, + }, + "Regressors": Maybe( + [ + Schema( + { + "Name": Required(str), + "Censor": { + "method": str, + "thresholds": [ + { + "type": str, + "value": float, + } + ], + "number_of_previous_trs_to_censor": Maybe(int), + "number_of_subsequent_trs_to_censor": Maybe(int), + }, + "Motion": { + "include_delayed": bool1_1, + "include_squared": bool1_1, + "include_delayed_squared": bool1_1, + }, + "aCompCor": valid_options["Regressors"]["CompCor"], + "tCompCor": valid_options["Regressors"]["CompCor"], + "CerebrospinalFluid": valid_options["Regressors"][ + "segmentation" + ], + "WhiteMatter": valid_options["Regressors"][ + "segmentation" + ], + "GreyMatter": valid_options["Regressors"][ + "segmentation" + ], + "GlobalSignal": {"summary": str}, + "PolyOrt": {"degree": int}, + "Bandpass": { + "bottom_frequency": float, + "top_frequency": float, + "method": str, + }, # how to check if [0] is > than [1]? + }, + extra=ALLOW_EXTRA, + ) + ] + ), + "lateral_ventricles_mask": Maybe(str), + "bandpass_filtering_order": Maybe(In({"After", "Before"})), + "regressor_masks": { + "erode_anatomical_brain_mask": { + "run": bool1_1, + "brain_mask_erosion_prop": Maybe(Number), + "brain_mask_erosion_mm": Maybe(Number), + "brain_erosion_mm": Maybe(Number), + }, + "erode_csf": { + "run": bool1_1, + "csf_erosion_prop": Maybe(Number), + "csf_mask_erosion_mm": Maybe(Number), + "csf_erosion_mm": Maybe(Number), + }, + "erode_wm": { + "run": bool1_1, + "wm_erosion_prop": Maybe(Number), + "wm_mask_erosion_mm": Maybe(Number), + "wm_erosion_mm": Maybe(Number), + }, + "erode_gm": { + "run": bool1_1, + "gm_erosion_prop": Maybe(Number), + "gm_mask_erosion_mm": Maybe(Number), + "gm_erosion_mm": Maybe(Number), + }, + }, + }, }, - }, - 'regional_homogeneity': { - 'run': bool1_1, - 'target_space': target_space, - 'cluster_size': In({7, 19, 27}), - }, - 'post_processing': { - 'spatial_smoothing': { - 'run': bool1_1, - 'output': [In({'smoothed', 'nonsmoothed'})], - 'smoothing_method': [In({'FSL', 'AFNI'})], - 'fwhm': [int] + "amplitude_low_frequency_fluctuation": { + "run": bool1_1, + "target_space": target_space, + "highpass_cutoff": [float], + "lowpass_cutoff": [float], }, - 'z-scoring': { - 'run': bool1_1, - 'output': [In({'z-scored', 'raw'})], + "voxel_mirrored_homotopic_connectivity": { + "run": bool1_1, + "symmetric_registration": { + "T1w_brain_template_symmetric": Maybe(str), + "T1w_brain_template_symmetric_funcreg": Maybe(str), + "T1w_brain_template_symmetric_for_resample": Maybe(str), + "T1w_template_symmetric": Maybe(str), + "T1w_template_symmetric_funcreg": Maybe(str), + "T1w_template_symmetric_for_resample": Maybe(str), + "dilated_symmetric_brain_mask": Maybe(str), + "dilated_symmetric_brain_mask_for_resample": Maybe(str), + }, }, - }, - 'timeseries_extraction': { - 'run': bool1_1, - Optional('roi_paths_fully_specified'): bool1_1, - 'tse_roi_paths': Optional( - Maybe({ - str: In({', '.join( - list(options) - ) for options in list(chain.from_iterable([list( - permutations(valid_options['timeseries']['roi_paths'], - number_of) - ) for number_of in range(1, 6)]))}), - }), - msg=permutation_message( - 'tse_roi_paths', valid_options['timeseries']['roi_paths']) - ), - 'realignment': In({'ROI_to_func', 'func_to_ROI'}), - 'connectivity_matrix': { - option: Maybe([In(valid_options['connectivity_matrix'][option])]) - for option in ['using', 'measure'] + "regional_homogeneity": { + "run": bool1_1, + "target_space": target_space, + "cluster_size": In({7, 19, 27}), }, - }, - 'seed_based_correlation_analysis': { - 'run': bool1_1, - Optional('roi_paths_fully_specified'): bool1_1, - 'sca_roi_paths': Optional( - Maybe({ - str: In({', '.join(list( - options - )) for options in list(chain.from_iterable([list( - permutations(valid_options['sca']['roi_paths'], number_of) - ) for number_of in range(1, 4)]))}) - }), - msg=permutation_message( - 'sca_roi_paths', valid_options['sca']['roi_paths']) - ), - 'norm_timeseries_for_DR': bool1_1, - }, - 'network_centrality': { - 'run': bool1_1, - 'memory_allocation': Number, - 'template_specification_file': Maybe(str), - 'degree_centrality': { - 'weight_options': [In( - valid_options['centrality']['weight_options'] - )], - 'correlation_threshold_option': In( - valid_options['centrality']['threshold_options']), - 'correlation_threshold': Range(min=-1, max=1) + "post_processing": { + "spatial_smoothing": { + "run": bool1_1, + "output": [In({"smoothed", "nonsmoothed"})], + "smoothing_method": [In({"FSL", "AFNI"})], + "fwhm": [int], + }, + "z-scoring": { + "run": bool1_1, + "output": [In({"z-scored", "raw"})], + }, }, - 'eigenvector_centrality': { - 'weight_options': [In( - valid_options['centrality']['weight_options'] - )], - 'correlation_threshold_option': In( - valid_options['centrality']['threshold_options'] + "timeseries_extraction": { + "run": bool1_1, + Optional("roi_paths_fully_specified"): bool1_1, + "tse_roi_paths": Optional( + Maybe( + { + str: In( + { + ", ".join(list(options)) + for options in list( + chain.from_iterable( + [ + list( + permutations( + valid_options["timeseries"][ + "roi_paths" + ], + number_of, + ) + ) + for number_of in range(1, 6) + ] + ) + ) + } + ), + } + ), + msg=permutation_message( + "tse_roi_paths", valid_options["timeseries"]["roi_paths"] + ), ), - 'correlation_threshold': Range(min=-1, max=1) + "realignment": In({"ROI_to_func", "func_to_ROI"}), + "connectivity_matrix": { + option: Maybe([In(valid_options["connectivity_matrix"][option])]) + for option in ["using", "measure"] + }, }, - 'local_functional_connectivity_density': { - 'weight_options': [In( - valid_options['centrality']['weight_options'] - )], - 'correlation_threshold_option': In([ - o for o in valid_options['centrality']['threshold_options'] if - o != 'Sparsity threshold' - ]), - 'correlation_threshold': Range(min=-1, max=1) + "seed_based_correlation_analysis": { + "run": bool1_1, + Optional("roi_paths_fully_specified"): bool1_1, + "sca_roi_paths": Optional( + Maybe( + { + str: In( + { + ", ".join(list(options)) + for options in list( + chain.from_iterable( + [ + list( + permutations( + valid_options["sca"]["roi_paths"], + number_of, + ) + ) + for number_of in range(1, 4) + ] + ) + ) + } + ) + } + ), + msg=permutation_message( + "sca_roi_paths", valid_options["sca"]["roi_paths"] + ), + ), + "norm_timeseries_for_DR": bool1_1, }, - }, - 'PyPEER': { - 'run': bool1_1, - 'eye_scan_names': Maybe(Any([str], [])), - 'data_scan_names': Maybe(Any([str], [])), - 'eye_mask_path': Maybe(str), - 'stimulus_path': Maybe(str), - 'minimal_nuisance_correction': { - 'peer_gsr': bool1_1, - 'peer_scrub': bool1_1, - 'scrub_thresh': float, + "network_centrality": { + "run": bool1_1, + "memory_allocation": Number, + "template_specification_file": Maybe(str), + "degree_centrality": { + "weight_options": [In(valid_options["centrality"]["weight_options"])], + "correlation_threshold_option": In( + valid_options["centrality"]["threshold_options"] + ), + "correlation_threshold": Range(min=-1, max=1), + }, + "eigenvector_centrality": { + "weight_options": [In(valid_options["centrality"]["weight_options"])], + "correlation_threshold_option": In( + valid_options["centrality"]["threshold_options"] + ), + "correlation_threshold": Range(min=-1, max=1), + }, + "local_functional_connectivity_density": { + "weight_options": [In(valid_options["centrality"]["weight_options"])], + "correlation_threshold_option": In( + [ + o + for o in valid_options["centrality"]["threshold_options"] + if o != "Sparsity threshold" + ] + ), + "correlation_threshold": Range(min=-1, max=1), + }, }, - }, -}) + "PyPEER": { + "run": bool1_1, + "eye_scan_names": Maybe(Any([str], [])), + "data_scan_names": Maybe(Any([str], [])), + "eye_mask_path": Maybe(str), + "stimulus_path": Maybe(str), + "minimal_nuisance_correction": { + "peer_gsr": bool1_1, + "peer_scrub": bool1_1, + "scrub_thresh": float, + }, + }, + } +) def schema(config_dict): - '''Validate a pipeline configuration against the latest validation schema + """Validate a pipeline configuration against the latest validation schema by first applying backwards-compatibility patches, then applying Voluptuous validation, then handling complex configuration interaction checks before returning validated config_dict. @@ -1096,105 +1254,135 @@ def schema(config_dict): Returns ------- dict - ''' + """ from CPAC.utils.utils import _changes_1_8_0_to_1_8_1 + try: - partially_validated = latest_schema( - _changes_1_8_0_to_1_8_1(config_dict)) + partially_validated = latest_schema(_changes_1_8_0_to_1_8_1(config_dict)) except MultipleInvalid as multiple_invalid: - if (multiple_invalid.path == ['nuisance_corrections', - '2-nuisance_regression', 'space'] and - isinstance(multiple_invalid.errors[0], CoerceInvalid)): + if multiple_invalid.path == [ + "nuisance_corrections", + "2-nuisance_regression", + "space", + ] and isinstance(multiple_invalid.errors[0], CoerceInvalid): raise CoerceInvalid( 'Nusiance regression space is not forkable. Please choose ' f'only one of {valid_options["space"]}', - path=multiple_invalid.path) from multiple_invalid + path=multiple_invalid.path, + ) from multiple_invalid raise multiple_invalid try: - if (partially_validated['registration_workflows'][ - 'functional_registration' - ]['func_registration_to_template']['apply_transform'][ - 'using' - ] == 'single_step_resampling_from_stc'): - or_else = ('or choose a different option for ' - '``registration_workflows: functional_registration: ' - 'func_registration_to_template: apply_transform: ' - 'using``') - if True in partially_validated['nuisance_corrections'][ - '2-nuisance_regression']['run'] and partially_validated[ - 'nuisance_corrections' - ]['2-nuisance_regression']['space'] != 'template': + if ( + partially_validated["registration_workflows"]["functional_registration"][ + "func_registration_to_template" + ]["apply_transform"]["using"] + == "single_step_resampling_from_stc" + ): + or_else = ( + "or choose a different option for " + "``registration_workflows: functional_registration: " + "func_registration_to_template: apply_transform: " + "using``" + ) + if ( + True + in partially_validated["nuisance_corrections"]["2-nuisance_regression"][ + "run" + ] + and partially_validated["nuisance_corrections"][ + "2-nuisance_regression" + ]["space"] + != "template" + ): raise ExclusiveInvalid( - '``single_step_resampling_from_stc`` requires ' - 'template-space nuisance regression. Either set ' - '``nuisance_corrections: 2-nuisance_regression: space`` ' - f'to ``template`` {or_else}') - if any(registration != 'ANTS' for registration in - partially_validated['registration_workflows'][ - 'anatomical_registration']['registration']['using']): + "``single_step_resampling_from_stc`` requires " + "template-space nuisance regression. Either set " + "``nuisance_corrections: 2-nuisance_regression: space`` " + f"to ``template`` {or_else}" + ) + if any( + registration != "ANTS" + for registration in partially_validated["registration_workflows"][ + "anatomical_registration" + ]["registration"]["using"] + ): raise ExclusiveInvalid( - '``single_step_resampling_from_stc`` requires ' - 'ANTS registration. Either set ' - '``registration_workflows: anatomical_registration: ' - f'registration: using`` to ``ANTS`` {or_else}') + "``single_step_resampling_from_stc`` requires " + "ANTS registration. Either set " + "``registration_workflows: anatomical_registration: " + f"registration: using`` to ``ANTS`` {or_else}" + ) except KeyError: pass try: - motion_filters = partially_validated['functional_preproc'][ - 'motion_estimates_and_correction']['motion_estimate_filter'] - if True in motion_filters['run']: - for motion_filter in motion_filters['filters']: - motion_filter['Name'] = name_motion_filter( - motion_filter, motion_filters['filters']) + motion_filters = partially_validated["functional_preproc"][ + "motion_estimates_and_correction" + ]["motion_estimate_filter"] + if True in motion_filters["run"]: + for motion_filter in motion_filters["filters"]: + motion_filter["Name"] = name_motion_filter( + motion_filter, motion_filters["filters"] + ) else: - motion_filters['filters'] = [] + motion_filters["filters"] = [] except KeyError: pass try: # 'motion_correction.using' is only optional if 'run' is Off - mec = partially_validated['functional_preproc'][ - 'motion_estimates_and_correction'] - if mec['run']: + mec = partially_validated["functional_preproc"][ + "motion_estimates_and_correction" + ] + if mec["run"]: try: # max should be len(valid_options['motion_correction']) # once #1935 is resolved - Length(min=1, max=1)(mec['motion_correction']['using']) + Length(min=1, max=1)(mec["motion_correction"]["using"]) except LengthInvalid: - mec_path = ['functional_preproc', - 'motion_estimates_and_correction'] + mec_path = ["functional_preproc", "motion_estimates_and_correction"] raise LengthInvalid( # pylint: disable=raise-missing-from f'If data[{"][".join(map(repr, mec_path))}][\'run\'] is ' # length must be between 1 and # len(valid_options['motion_correction']) once #1935 is # resolved 'True, length of list must be exactly 1', - path=[*mec_path, 'motion_correction', 'using']) + path=[*mec_path, "motion_correction", "using"], + ) except KeyError: pass try: # Check for mutually exclusive options - if (partially_validated['nuisance_corrections'][ - '2-nuisance_regression']['ingress_regressors']['run'] and - partially_validated['nuisance_corrections'][ - '2-nuisance_regression']['create_regressors']): + if ( + partially_validated["nuisance_corrections"]["2-nuisance_regression"][ + "ingress_regressors" + ]["run"] + and partially_validated["nuisance_corrections"]["2-nuisance_regression"][ + "create_regressors" + ] + ): raise ExclusiveInvalid( "[!] Ingress_regressors and create_regressors can't both run! " - " Try turning one option off.\n ") + " Try turning one option off.\n " + ) except KeyError: pass try: - if 'unet' in [using.lower() for using in - partially_validated['anatomical_preproc'][ - 'brain_extraction']['using']]: + if "unet" in [ + using.lower() + for using in partially_validated["anatomical_preproc"]["brain_extraction"][ + "using" + ] + ]: try: from importlib import import_module - import_module('CPAC.unet') + + import_module("CPAC.unet") except (ImportError, ModuleNotFoundError, OSError) as error: import site + raise OSError( - 'U-Net brain extraction requires torch to be installed, ' - 'but the installation path in this container is ' - 'read-only. Please bind a local writable path to ' + "U-Net brain extraction requires torch to be installed, " + "but the installation path in this container is " + "read-only. Please bind a local writable path to " f'"{site.USER_BASE}" in the container to use U-Net.' ) from error except KeyError: diff --git a/CPAC/pipeline/test/sample_data.py b/CPAC/pipeline/test/sample_data.py index fab34ae17d..e5e1097bb5 100644 --- a/CPAC/pipeline/test/sample_data.py +++ b/CPAC/pipeline/test/sample_data.py @@ -1,32 +1,37 @@ -sub_list = [{ - 'anat': '/fake/data/sub-0001/ses-NFB3/anat/' - 'sub-0001_ses-NFB3_T1w.nii.gz', - 'func': {'MSIT': { - 'fmap_mag': '/fake/data/sub-0001/ses-NFB3/fmap/' - 'sub-0001_ses-NFB3_magnitude1.nii.gz', - 'fmap_phase': '/fake/data/sub-0001/ses-NFB3/fmap/' - 'sub-0001_ses-NFB3_phasediff.nii.gz', - 'scan': '/fake/data/sub-0001/ses-NFB3/func/' - 'sub-0001_ses-NFB3_task-MSIT_bold.nii.gz', - 'scan_parameters': '/fake/data/task-MSIT_bold.json' - }, 'PEER1': { - 'fmap_mag': '/fake/data/sub-0001/ses-NFB3/fmap/' - 'sub-0001_ses-NFB3_magnitude1.nii.gz', - 'fmap_phase': '/fake/data/sub-0001/ses-NFB3/fmap/' - 'sub-0001_ses-NFB3_phasediff.nii.gz', - 'scan': '/fake/data/sub-0001/ses-NFB3/func/' - 'sub-0001_ses-NFB3_task-PEER1_bold.nii.gz', - 'scan_parameters': '/fake/data/task-PEER1_bold.json' - }, 'PEER2': { - 'fmap_mag': '/fake/data/sub-0001/ses-NFB3/fmap/' - 'sub-0001_ses-NFB3_magnitude1.nii.gz', - 'fmap_phase': '/fake/data/sub-0001/ses-NFB3/fmap/' - 'sub-0001_ses-NFB3_phasediff.nii.gz', - 'scan': '/fake/data/sub-0001/ses-NFB3/func/' - 'sub-0001_ses-NFB3_task-PEER2_bold.nii.gz', - 'scan_parameters': '/fake/data/task-PEER2_bold.json' - }}, - 'site': 'site-1', - 'subject_id': '0001', - 'unique_id': 'NFB3' -}] \ No newline at end of file +sub_list = [ + { + "anat": "/fake/data/sub-0001/ses-NFB3/anat/" "sub-0001_ses-NFB3_T1w.nii.gz", + "func": { + "MSIT": { + "fmap_mag": "/fake/data/sub-0001/ses-NFB3/fmap/" + "sub-0001_ses-NFB3_magnitude1.nii.gz", + "fmap_phase": "/fake/data/sub-0001/ses-NFB3/fmap/" + "sub-0001_ses-NFB3_phasediff.nii.gz", + "scan": "/fake/data/sub-0001/ses-NFB3/func/" + "sub-0001_ses-NFB3_task-MSIT_bold.nii.gz", + "scan_parameters": "/fake/data/task-MSIT_bold.json", + }, + "PEER1": { + "fmap_mag": "/fake/data/sub-0001/ses-NFB3/fmap/" + "sub-0001_ses-NFB3_magnitude1.nii.gz", + "fmap_phase": "/fake/data/sub-0001/ses-NFB3/fmap/" + "sub-0001_ses-NFB3_phasediff.nii.gz", + "scan": "/fake/data/sub-0001/ses-NFB3/func/" + "sub-0001_ses-NFB3_task-PEER1_bold.nii.gz", + "scan_parameters": "/fake/data/task-PEER1_bold.json", + }, + "PEER2": { + "fmap_mag": "/fake/data/sub-0001/ses-NFB3/fmap/" + "sub-0001_ses-NFB3_magnitude1.nii.gz", + "fmap_phase": "/fake/data/sub-0001/ses-NFB3/fmap/" + "sub-0001_ses-NFB3_phasediff.nii.gz", + "scan": "/fake/data/sub-0001/ses-NFB3/func/" + "sub-0001_ses-NFB3_task-PEER2_bold.nii.gz", + "scan_parameters": "/fake/data/task-PEER2_bold.json", + }, + }, + "site": "site-1", + "subject_id": "0001", + "unique_id": "NFB3", + } +] diff --git a/CPAC/pipeline/test/test_cpac_group_runner.py b/CPAC/pipeline/test/test_cpac_group_runner.py index 1c5d7bf6e7..c64307565a 100644 --- a/CPAC/pipeline/test/test_cpac_group_runner.py +++ b/CPAC/pipeline/test/test_cpac_group_runner.py @@ -1,8 +1,6 @@ - - def run_gather_outputs_func(pipeline_out_dir): from CPAC.pipeline import cpac_group_runner as cgr - df_dct = cgr.gather_outputs(pipeline_out_dir, ["functional_to_standard"], - None, False, False, get_func=True) - print(df_dct) + cgr.gather_outputs( + pipeline_out_dir, ["functional_to_standard"], None, False, False, get_func=True + ) diff --git a/CPAC/pipeline/test/test_cpac_pipeline.py b/CPAC/pipeline/test/test_cpac_pipeline.py index 08cb35891c..b4157ac975 100644 --- a/CPAC/pipeline/test/test_cpac_pipeline.py +++ b/CPAC/pipeline/test/test_cpac_pipeline.py @@ -1,25 +1,25 @@ -"""Tests for cpac_pipeline.py""" +"""Tests for cpac_pipeline.py.""" import pytest + from CPAC.pipeline.cpac_pipeline import run_workflow from CPAC.pipeline.nipype_pipeline_engine.plugins import MultiProcPlugin from CPAC.utils.configuration import Configuration -@pytest.mark.parametrize('plugin', [ - MultiProcPlugin(), False, 'MultiProc', None]) +@pytest.mark.parametrize("plugin", [MultiProcPlugin(), False, "MultiProc", None]) def test_plugin_param(plugin): """If we pass a non-string to run_workflow, a TypeError should be raised. Otherwise, we should get an KeyError from our empty - `sub_dict` + `sub_dict`. """ cfg = Configuration() with pytest.raises((TypeError, KeyError)) as e: run_workflow({}, cfg, False, plugin=plugin) if isinstance(plugin, str) or plugin is None: - assert e.typename == 'KeyError' + assert e.typename == "KeyError" else: - assert e.typename == 'TypeError' + assert e.typename == "TypeError" if isinstance(plugin, MultiProcPlugin): assert "MultiProcPlugin" in str(e.value) elif isinstance(plugin, bool): diff --git a/CPAC/pipeline/test/test_cpac_runner.py b/CPAC/pipeline/test/test_cpac_runner.py index 79510d824a..7ee91f5125 100644 --- a/CPAC/pipeline/test/test_cpac_runner.py +++ b/CPAC/pipeline/test/test_cpac_runner.py @@ -1,33 +1,32 @@ - import os -import pytest + import pkg_resources as p -from CPAC.pipeline.cpac_runner import run_T1w_longitudinal +import pytest + from CPAC.pipeline.cpac_pipeline import load_cpac_pipe_config +from CPAC.pipeline.cpac_runner import run_T1w_longitudinal from CPAC.utils.bids_utils import create_cpac_data_config -@pytest.mark.skip(reason='not a pytest test') +@pytest.mark.skip(reason="not a pytest test") def test_run_T1w_longitudinal(bids_dir, cfg, test_dir, part_id): - - sub_data_list = create_cpac_data_config(bids_dir, - participant_label=part_id, - skip_bids_validator=True) + sub_data_list = create_cpac_data_config( + bids_dir, participant_label=part_id, skip_bids_validator=True + ) cfg = load_cpac_pipe_config(cfg) - cfg.pipeline_setup['output_directory']['path'] = \ - os.path.join(test_dir, 'out') - cfg.pipeline_setup['working_directory']['path'] = \ - os.path.join(test_dir, 'work') + cfg.pipeline_setup["output_directory"]["path"] = os.path.join(test_dir, "out") + cfg.pipeline_setup["working_directory"]["path"] = os.path.join(test_dir, "work") run_T1w_longitudinal(sub_data_list, cfg) -cfg = p.resource_filename("CPAC", os.path.join( - "resources", "configs", "pipeline_config_default.yml")) +cfg = p.resource_filename( + "CPAC", os.path.join("resources", "configs", "pipeline_config_default.yml") +) bids_dir = "/Users/steven.giavasis/data/neurodata_hnu" test_dir = "/test_dir" part_id = "0025427" -if __name__ == '__main__': +if __name__ == "__main__": test_run_T1w_longitudinal(bids_dir, cfg, test_dir, part_id) diff --git a/CPAC/pipeline/test/test_engine.py b/CPAC/pipeline/test/test_engine.py index 3988a61f95..316ffe1a06 100644 --- a/CPAC/pipeline/test/test_engine.py +++ b/CPAC/pipeline/test/test_engine.py @@ -1,98 +1,92 @@ import os + import pytest -from CPAC.pipeline.cpac_pipeline import initialize_nipype_wf, \ - load_cpac_pipe_config, \ - connect_pipeline, \ - build_anat_preproc_stack, \ - build_workflow -from CPAC.pipeline.engine import ResourcePool, ingress_raw_anat_data, \ - ingress_raw_func_data, \ - ingress_pipeconfig_paths, initiate_rpool + +from CPAC.pipeline.cpac_pipeline import ( + build_anat_preproc_stack, + build_workflow, + connect_pipeline, + initialize_nipype_wf, + load_cpac_pipe_config, +) +from CPAC.pipeline.engine import ( + ResourcePool, + ingress_pipeconfig_paths, + ingress_raw_anat_data, + ingress_raw_func_data, + initiate_rpool, +) from CPAC.utils.bids_utils import create_cpac_data_config -@pytest.mark.skip(reason='not a pytest test') +@pytest.mark.skip(reason="not a pytest test") def test_ingress_func_raw_data(pipe_config, bids_dir, test_dir): - - sub_data_dct = create_cpac_data_config(bids_dir, - skip_bids_validator=True)[0] + sub_data_dct = create_cpac_data_config(bids_dir, skip_bids_validator=True)[0] cfg = load_cpac_pipe_config(pipe_config) - cfg.pipeline_setup['output_directory']['path'] = \ - os.path.join(test_dir, 'out') - cfg.pipeline_setup['working_directory']['path'] = \ - os.path.join(test_dir, 'work') + cfg.pipeline_setup["output_directory"]["path"] = os.path.join(test_dir, "out") + cfg.pipeline_setup["working_directory"]["path"] = os.path.join(test_dir, "work") wf = initialize_nipype_wf(cfg, sub_data_dct) - part_id = sub_data_dct['subject_id'] - ses_id = sub_data_dct['unique_id'] + part_id = sub_data_dct["subject_id"] + ses_id = sub_data_dct["unique_id"] - unique_id = f'{part_id}_{ses_id}' + unique_id = f"{part_id}_{ses_id}" rpool = ResourcePool(name=unique_id, cfg=cfg) - if 'func' in sub_data_dct: - wf, rpool, diff, blip, fmap_rp_list = \ - ingress_raw_func_data(wf, rpool, cfg, sub_data_dct, unique_id, - part_id, ses_id) + if "func" in sub_data_dct: + wf, rpool, diff, blip, fmap_rp_list = ingress_raw_func_data( + wf, rpool, cfg, sub_data_dct, unique_id, part_id, ses_id + ) rpool.gather_pipes(wf, cfg, all=True) wf.run() -@pytest.mark.skip(reason='not a pytest test') +@pytest.mark.skip(reason="not a pytest test") def test_ingress_anat_raw_data(pipe_config, bids_dir, test_dir): - - sub_data_dct = create_cpac_data_config(bids_dir, - skip_bids_validator=True)[0] + sub_data_dct = create_cpac_data_config(bids_dir, skip_bids_validator=True)[0] cfg = load_cpac_pipe_config(pipe_config) - cfg.pipeline_setup['output_directory']['path'] = \ - os.path.join(test_dir, 'out') - cfg.pipeline_setup['working_directory']['path'] = \ - os.path.join(test_dir, 'work') + cfg.pipeline_setup["output_directory"]["path"] = os.path.join(test_dir, "out") + cfg.pipeline_setup["working_directory"]["path"] = os.path.join(test_dir, "work") wf = initialize_nipype_wf(cfg, sub_data_dct) - part_id = sub_data_dct['subject_id'] - ses_id = sub_data_dct['unique_id'] + part_id = sub_data_dct["subject_id"] + ses_id = sub_data_dct["unique_id"] - unique_id = f'{part_id}_{ses_id}' + unique_id = f"{part_id}_{ses_id}" rpool = ResourcePool(name=unique_id, cfg=cfg) - rpool = ingress_raw_anat_data(wf, rpool, cfg, - sub_data_dct, - unique_id, - part_id, ses_id) + rpool = ingress_raw_anat_data( + wf, rpool, cfg, sub_data_dct, unique_id, part_id, ses_id + ) rpool.gather_pipes(wf, cfg, all=True) wf.run() -@pytest.mark.skip(reason='not a pytest test') +@pytest.mark.skip(reason="not a pytest test") def test_ingress_pipeconfig_data(pipe_config, bids_dir, test_dir): - - sub_data_dct = create_cpac_data_config(bids_dir, - skip_bids_validator=True)[0] + sub_data_dct = create_cpac_data_config(bids_dir, skip_bids_validator=True)[0] cfg = load_cpac_pipe_config(pipe_config) - cfg.pipeline_setup['output_directory']['path'] = \ - os.path.join(test_dir, 'out') - cfg.pipeline_setup['working_directory']['path'] = \ - os.path.join(test_dir, 'work') - cfg.pipeline_setup['log_directory']['path'] = \ - os.path.join(test_dir, 'logs') + cfg.pipeline_setup["output_directory"]["path"] = os.path.join(test_dir, "out") + cfg.pipeline_setup["working_directory"]["path"] = os.path.join(test_dir, "work") + cfg.pipeline_setup["log_directory"]["path"] = os.path.join(test_dir, "logs") wf = initialize_nipype_wf(cfg, sub_data_dct) - part_id = sub_data_dct['subject_id'] - ses_id = sub_data_dct['unique_id'] + part_id = sub_data_dct["subject_id"] + ses_id = sub_data_dct["unique_id"] - unique_id = f'{part_id}_{ses_id}' + unique_id = f"{part_id}_{ses_id}" rpool = ResourcePool(name=unique_id, cfg=cfg) @@ -103,19 +97,14 @@ def test_ingress_pipeconfig_data(pipe_config, bids_dir, test_dir): wf.run() -@pytest.mark.skip(reason='not a pytest test') +@pytest.mark.skip(reason="not a pytest test") def test_build_anat_preproc_stack(pipe_config, bids_dir, test_dir): - - sub_data_dct = create_cpac_data_config(bids_dir, - skip_bids_validator=True)[0] + sub_data_dct = create_cpac_data_config(bids_dir, skip_bids_validator=True)[0] cfg = load_cpac_pipe_config(pipe_config) - cfg.pipeline_setup['output_directory']['path'] = \ - os.path.join(test_dir, 'out') - cfg.pipeline_setup['working_directory']['path'] = \ - os.path.join(test_dir, 'work') - cfg.pipeline_setup['log_directory']['path'] = \ - os.path.join(test_dir, 'logs') + cfg.pipeline_setup["output_directory"]["path"] = os.path.join(test_dir, "out") + cfg.pipeline_setup["working_directory"]["path"] = os.path.join(test_dir, "work") + cfg.pipeline_setup["log_directory"]["path"] = os.path.join(test_dir, "logs") wf = initialize_nipype_wf(cfg, sub_data_dct) @@ -127,44 +116,39 @@ def test_build_anat_preproc_stack(pipe_config, bids_dir, test_dir): rpool.gather_pipes(wf, cfg) wf.run() - -@pytest.mark.skip(reason='not a pytest test') + +@pytest.mark.skip(reason="not a pytest test") def test_build_workflow(pipe_config, bids_dir, test_dir): - - sub_data_dct = create_cpac_data_config(bids_dir, - skip_bids_validator=True)[0] + sub_data_dct = create_cpac_data_config(bids_dir, skip_bids_validator=True)[0] cfg = load_cpac_pipe_config(pipe_config) - cfg.pipeline_setup['output_directory']['path'] = \ - os.path.join(test_dir, 'out') - cfg.pipeline_setup['working_directory']['path'] = \ - os.path.join(test_dir, 'work') - cfg.pipeline_setup['log_directory']['path'] = \ - os.path.join(test_dir, 'logs') + cfg.pipeline_setup["output_directory"]["path"] = os.path.join(test_dir, "out") + cfg.pipeline_setup["working_directory"]["path"] = os.path.join(test_dir, "work") + cfg.pipeline_setup["log_directory"]["path"] = os.path.join(test_dir, "logs") wf = initialize_nipype_wf(cfg, sub_data_dct) wf, rpool = initiate_rpool(wf, cfg, sub_data_dct) - wf, _, _ = build_workflow( - sub_data_dct['subject_id'], sub_data_dct, cfg) + wf, _, _ = build_workflow(sub_data_dct["subject_id"], sub_data_dct, cfg) rpool.gather_pipes(wf, cfg) wf.run() + # bids_dir = "/Users/steven.giavasis/data/HBN-SI_dataset/rawdata" # test_dir = "/test_dir" # cfg = "/Users/hecheng.jin/GitHub/DevBranch/CPAC/resources/configs/pipeline_config_monkey-ABCD.yml" cfg = "/Users/hecheng.jin/GitHub/pipeline_config_monkey-ABCDlocal.yml" -bids_dir = '/Users/hecheng.jin/Monkey/monkey_data_oxford/site-ucdavis' +bids_dir = "/Users/hecheng.jin/Monkey/monkey_data_oxford/site-ucdavis" test_dir = "/Users/hecheng.jin/GitHub/Test/T2preproc" # test_ingress_func_raw_data(cfg, bids_dir, test_dir) # test_ingress_anat_raw_data(cfg, bids_dir, test_dir) # test_ingress_pipeconfig_data(cfg, bids_dir, test_dir) # test_build_anat_preproc_stack(cfg, bids_dir, test_dir) -if __name__ == '__main__': +if __name__ == "__main__": test_build_workflow(cfg, bids_dir, test_dir) diff --git a/CPAC/pipeline/test/test_nipype_pipeline_engine.py b/CPAC/pipeline/test/test_nipype_pipeline_engine.py index f722a839cb..40cdf01b67 100644 --- a/CPAC/pipeline/test/test_nipype_pipeline_engine.py +++ b/CPAC/pipeline/test/test_nipype_pipeline_engine.py @@ -1,11 +1,18 @@ import os + import pytest +from traits.trait_base import Undefined from nibabel.testing import data_path from nipype import Function from nipype.interfaces.utility import IdentityInterface -from traits.trait_base import Undefined + from CPAC.pipeline.nipype_pipeline_engine import ( - DEFAULT_MEM_GB, get_data_size, Node, MapNode, Workflow) + DEFAULT_MEM_GB, + MapNode, + Node, + Workflow, + get_data_size, +) def get_sample_data(filepath): @@ -15,7 +22,7 @@ def get_sample_data(filepath): def square_func(x): # pylint: disable=invalid-name # pylint: disable=missing-function-docstring - return x ** 2 + return x**2 square = Function(["x"], ["f_x"], square_func) @@ -36,34 +43,40 @@ def test_Node(): # pylint: disable=invalid-name def test_Workflow(tmpdir): # pylint: disable=invalid-name - example_filepath = os.path.join(data_path, 'example4d.nii.gz') - pass_in_filepath = IdentityInterface(fields=['x']) + example_filepath = os.path.join(data_path, "example4d.nii.gz") + pass_in_filepath = IdentityInterface(fields=["x"]) pass_in_filepath.inputs.x = example_filepath - node_1 = Node(pass_in_filepath, 'pass_in_filepath') + node_1 = Node(pass_in_filepath, "pass_in_filepath") # This node just returns the filepath given. We're testing the # just-in-time memory allocation - node_2 = Node(Function(['filepath'], ['filepath'], get_sample_data), - name='get_sample_data', - inputs=['filepath'], - outputs=['filepath']) + node_2 = Node( + Function(["filepath"], ["filepath"], get_sample_data), + name="get_sample_data", + inputs=["filepath"], + outputs=["filepath"], + ) assert node_2.mem_gb == DEFAULT_MEM_GB - node_2 = Node(Function(['filepath'], ['filepath'], get_sample_data), - name='get_sample_data', - inputs=['filepath'], - outputs=['filepath'], - mem_x=(0.1, 'filepath')) + node_2 = Node( + Function(["filepath"], ["filepath"], get_sample_data), + name="get_sample_data", + inputs=["filepath"], + outputs=["filepath"], + mem_x=(0.1, "filepath"), + ) with pytest.raises(FileNotFoundError): assert node_2.mem_gb - wf = Workflow('example_workflow', base_dir=tmpdir) - wf.connect(node_1, 'x', node_2, 'filepath') + wf = Workflow("example_workflow", base_dir=tmpdir) + wf.connect(node_1, "x", node_2, "filepath") - assert wf.get_node('get_sample_data').inputs.filepath is Undefined + assert wf.get_node("get_sample_data").inputs.filepath is Undefined out = wf.run() - out_node = list(out.nodes)[0] - assert out_node.mem_gb == DEFAULT_MEM_GB + get_data_size( - example_filepath, 'xyzt') * 0.1 + out_node = next(iter(out.nodes)) + assert ( + out_node.mem_gb + == DEFAULT_MEM_GB + get_data_size(example_filepath, "xyzt") * 0.1 + ) diff --git a/CPAC/pipeline/test/test_schema_validation.py b/CPAC/pipeline/test/test_schema_validation.py index 7f24ba057d..02456915af 100644 --- a/CPAC/pipeline/test/test_schema_validation.py +++ b/CPAC/pipeline/test/test_schema_validation.py @@ -1,60 +1,106 @@ -'''Tests for schema.py''' +"""Tests for schema.py.""" from itertools import combinations + import pytest from voluptuous.error import ExclusiveInvalid, Invalid + from CPAC.utils.configuration import Configuration -@pytest.mark.parametrize('run_value', [ - True, False, [True], [False], [True, False], [False, True]]) +@pytest.mark.parametrize( + "run_value", [True, False, [True], [False], [True, False], [False, True]] +) def test_motion_estimates_and_correction(run_value): - '''Test that any truthy forkable option for 'run' throws the custom + """Test that any truthy forkable option for 'run' throws the custom human-readable exception for an invalid motion_estimate_filter. - ''' + """ # pylint: disable=invalid-name - d = {'FROM': 'default', - 'functional_preproc': {'motion_estimates_and_correction': { - 'motion_estimate_filter': {'run': run_value, - 'filters': [{ - 'filter_type': 'notch', - 'filter_order': 0, - 'breathing_rate_min': None, - 'breathing_rate_max': 101.5}]}}}} + d = { + "FROM": "default", + "functional_preproc": { + "motion_estimates_and_correction": { + "motion_estimate_filter": { + "run": run_value, + "filters": [ + { + "filter_type": "notch", + "filter_order": 0, + "breathing_rate_min": None, + "breathing_rate_max": 101.5, + } + ], + } + } + }, + } if bool(run_value) and run_value not in [[False], []]: with pytest.raises(Invalid) as e: Configuration(d) assert "func#motion-estimate-filter-valid-options" in str(e.value) else: Configuration(d) - d = {'FROM': 'default', - 'functional_preproc': {'motion_estimates_and_correction': { - 'motion_estimate_filter': {'run': run_value, - 'filters': [{ - 'filter_type': 'notch', - 'filter_order': 4, - 'center_frequency': .31, - 'filter_bandwidth': .12}]}}}} + d = { + "FROM": "default", + "functional_preproc": { + "motion_estimates_and_correction": { + "motion_estimate_filter": { + "run": run_value, + "filters": [ + { + "filter_type": "notch", + "filter_order": 4, + "center_frequency": 0.31, + "filter_bandwidth": 0.12, + } + ], + } + } + }, + } c = Configuration(d) - if c['functional_preproc', 'motion_estimates_and_correction', - 'motion_estimate_filter', 'filters']: - assert c['functional_preproc', 'motion_estimates_and_correction', - 'motion_estimate_filter', 'filters', 0, 'Name' - ] == 'notch4fc0p31bw0p12' + if c[ + "functional_preproc", + "motion_estimates_and_correction", + "motion_estimate_filter", + "filters", + ]: + assert ( + c[ + "functional_preproc", + "motion_estimates_and_correction", + "motion_estimate_filter", + "filters", + 0, + "Name", + ] + == "notch4fc0p31bw0p12" + ) -@pytest.mark.parametrize('registration_using', - [list(combo) for _ in [list(combinations( - ['ANTS', 'FSL', 'FSL-linear'], i)) for i in - range(1, 4)] for combo in _]) +@pytest.mark.parametrize( + "registration_using", + [ + list(combo) + for _ in [ + list(combinations(["ANTS", "FSL", "FSL-linear"], i)) for i in range(1, 4) + ] + for combo in _ + ], +) def test_single_step_vs_registration(registration_using): - '''Test that single-step resampling requires ANTS registration''' + """Test that single-step resampling requires ANTS registration.""" # pylint: disable=invalid-name - d = {'registration_workflows': {'anatomical_registration': { - 'registration': {'using': registration_using}}, - 'functional_registration': { - 'func_registration_to_template': {'apply_transform': { - 'using': 'single_step_resampling_from_stc'}}}}} - if registration_using == ['ANTS']: + d = { + "registration_workflows": { + "anatomical_registration": {"registration": {"using": registration_using}}, + "functional_registration": { + "func_registration_to_template": { + "apply_transform": {"using": "single_step_resampling_from_stc"} + } + }, + } + } + if registration_using == ["ANTS"]: Configuration(d) # validates without exception else: with pytest.raises(ExclusiveInvalid) as e: @@ -63,6 +109,6 @@ def test_single_step_vs_registration(registration_using): def test_pipeline_name(): - '''Test that pipeline_name sucessfully sanitizes''' - c = Configuration({'pipeline_setup': {'pipeline_name': ':va:lid name'}}) - assert c['pipeline_setup', 'pipeline_name'] == 'valid_name' + """Test that pipeline_name sucessfully sanitizes.""" + c = Configuration({"pipeline_setup": {"pipeline_name": ":va:lid name"}}) + assert c["pipeline_setup", "pipeline_name"] == "valid_name" diff --git a/CPAC/pipeline/utils.py b/CPAC/pipeline/utils.py index d39a111ba7..1b3069c18c 100644 --- a/CPAC/pipeline/utils.py +++ b/CPAC/pipeline/utils.py @@ -14,9 +14,10 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -"""C-PAC pipeline engine utilities""" -from typing import Union +"""C-PAC pipeline engine utilities.""" from itertools import chain +from typing import Union + from CPAC.func_preproc.func_motion import motion_estimate_filter from CPAC.utils.bids_utils import insert_entity @@ -24,7 +25,7 @@ def name_fork(resource_idx, cfg, json_info, out_dct): - """Create and insert entities for forkpoints + """Create and insert entities for forkpoints. Parameters ---------- @@ -42,39 +43,51 @@ def name_fork(resource_idx, cfg, json_info, out_dct): out_dct : dict """ - if cfg.switch_is_on(['functional_preproc', - 'motion_estimates_and_correction', - 'motion_estimate_filter', 'run']): + if cfg.switch_is_on( + [ + "functional_preproc", + "motion_estimates_and_correction", + "motion_estimate_filter", + "run", + ] + ): filt_value = None _motion_variant = { - _key: json_info['CpacVariant'][_key] + _key: json_info["CpacVariant"][_key] for _key in MOVEMENT_FILTER_KEYS - if _key in json_info.get('CpacVariant', {})} - if 'unfiltered-' in resource_idx: - resource_idx = resource_idx.replace('unfiltered-', '') - filt_value = 'none' + if _key in json_info.get("CpacVariant", {}) + } + if "unfiltered-" in resource_idx: + resource_idx = resource_idx.replace("unfiltered-", "") + filt_value = "none" else: try: - filt_value = [ - json_info['CpacVariant'][_k][0].replace( - 'motion_estimate_filter_', '' - ) for _k, _v in _motion_variant.items() - if _v][0] + filt_value = next( + json_info["CpacVariant"][_k][0].replace( + "motion_estimate_filter_", "" + ) + for _k, _v in _motion_variant.items() + if _v + ) except (IndexError, KeyError): - filt_value = 'none' - resource_idx, out_dct = _update_resource_idx(resource_idx, out_dct, - 'filt', filt_value) - if cfg.switch_is_on(['nuisance_corrections', - '2-nuisance_regression', 'run']): - variants = [variant.split('_')[-1] for variant in - chain.from_iterable(json_info.get('CpacVariant', {}).values()) if - variant.startswith('nuisance_regressors_generation')] - if cfg.switch_is_off(['nuisance_corrections', '2-nuisance_regression', - 'run']): - variants.append('Off') + filt_value = "none" + resource_idx, out_dct = _update_resource_idx( + resource_idx, out_dct, "filt", filt_value + ) + if cfg.switch_is_on(["nuisance_corrections", "2-nuisance_regression", "run"]): + variants = [ + variant.split("_")[-1] + for variant in chain.from_iterable( + json_info.get("CpacVariant", {}).values() + ) + if variant.startswith("nuisance_regressors_generation") + ] + if cfg.switch_is_off(["nuisance_corrections", "2-nuisance_regression", "run"]): + variants.append("Off") reg_value = variants[0] if variants else None - resource_idx, out_dct = _update_resource_idx(resource_idx, out_dct, - 'reg', reg_value) + resource_idx, out_dct = _update_resource_idx( + resource_idx, out_dct, "reg", reg_value + ) return resource_idx, out_dct @@ -116,7 +129,7 @@ def present_outputs(outputs: dict, keys: list) -> dict: def source_set(sources: Union[str, list, set]) -> set: - """Given a CpacProvenance, return a set of {resource}:{source} strings + """Given a CpacProvenance, return a set of {resource}:{source} strings. Parameters ---------- @@ -190,7 +203,7 @@ def source_set(sources: Union[str, list, set]) -> set: def _update_resource_idx(resource_idx, out_dct, key, value): """ Given a resource_idx and an out_dct, insert fork-based keys as - appropriate + appropriate. Parameters ---------- @@ -210,6 +223,5 @@ def _update_resource_idx(resource_idx, out_dct, key, value): """ if value is not None: resource_idx = insert_entity(resource_idx, key, value) - out_dct['filename'] = insert_entity(out_dct['filename'], key, - value) + out_dct["filename"] = insert_entity(out_dct["filename"], key, value) return resource_idx, out_dct diff --git a/CPAC/pypeer/__init__.py b/CPAC/pypeer/__init__.py index 819bbc8ef9..75de0166c1 100644 --- a/CPAC/pypeer/__init__.py +++ b/CPAC/pypeer/__init__.py @@ -1,15 +1,15 @@ from .peer import ( + motion_scrub, + prep_for_pypeer, pypeer_eye_masking, - pypeer_zscore, pypeer_ravel_data, - motion_scrub, - prep_for_pypeer + pypeer_zscore, ) __all__ = [ - 'pypeer_eye_masking', - 'pypeer_zscore', - 'pypeer_ravel_data', - 'motion_scrub', - 'prep_for_pypeer' -] \ No newline at end of file + "pypeer_eye_masking", + "pypeer_zscore", + "pypeer_ravel_data", + "motion_scrub", + "prep_for_pypeer", +] diff --git a/CPAC/pypeer/peer.py b/CPAC/pypeer/peer.py index cf85a1aa93..97644a8282 100644 --- a/CPAC/pypeer/peer.py +++ b/CPAC/pypeer/peer.py @@ -1,26 +1,30 @@ -import os import csv import glob -import nibabel as nb +import os + import numpy as np +import nibabel as nib # check if they have PyPEER installed try: import PyPEER - from PyPEER.peer_func import \ - global_signal_regression, \ - prepare_data_for_svr, \ - train_model, \ - save_model, \ - load_model, \ - predict_fixations, \ - save_fixations, \ - estimate_em, \ - load_data + from PyPEER.peer_func import ( + estimate_em, + global_signal_regression, + load_data, + load_model, + predict_fixations, + prepare_data_for_svr, + save_fixations, + save_model, + train_model, + ) except ImportError: - raise ImportError("\n\n[!] PyPEER is not installed. Please double-" - "check your Python environment and ensure that the " - "PyPEER package is available.") + raise ImportError( + "\n\n[!] PyPEER is not installed. Please double-" + "check your Python environment and ensure that the " + "PyPEER package is available." + ) def make_pypeer_dir(dirpath): @@ -28,13 +32,15 @@ def make_pypeer_dir(dirpath): if not os.path.isdir(dirpath): os.makedirs(dirpath) except: - raise Exception("\n\n[!] Could not create the output directory for " - "PyPEER. Double-check your permissions?\n\nAttempted " - "directory path:\n{0}\n".format(dirpath)) + raise Exception( + "\n\n[!] Could not create the output directory for " + "PyPEER. Double-check your permissions?\n\nAttempted " + "directory path:\n{0}\n".format(dirpath) + ) def pypeer_eye_masking(data_path, eye_mask_path): - eye_mask = nb.load(eye_mask_path).get_fdata() + eye_mask = nib.load(eye_mask_path).get_fdata() data = load_data(data_path) @@ -52,20 +58,20 @@ def pypeer_zscore(data): def pypeer_ravel_data(data): - raveled_data = [data[:, :, :, vol].ravel() for vol in - np.arange(data.shape[3])] - return raveled_data + return [data[:, :, :, vol].ravel() for vol in np.arange(data.shape[3])] def motion_scrub(_ms_filename, _motion_threshold): """ - Determines volumes with high motion artifact - Parameters + Determines volumes with high motion artifact. + + Parameters. ---------- _ms_filename : string Pathname of the CSV file containing the framewise displacement per time point for a given fMRI scan _motion_threshold : float Threshold for high motion (framewise displacement, defined by Power et al. 2012) + Returns ------- _removed_indices : int @@ -77,54 +83,51 @@ def motion_scrub(_ms_filename, _motion_threshold): Adapted to accept a direct file path to the mean FD 1D file. """ - - with open(_ms_filename, 'r') as f: + with open(_ms_filename, "r") as f: reader = csv.reader(f) censor_pre = [x[0] for x in reader] nuissance_vector = [float(x) for x in censor_pre] - _removed_indices = [i for i, x in enumerate(nuissance_vector) if x >= float(_motion_threshold)] - - return _removed_indices - - -def prep_for_pypeer(peer_scan_names, data_scan_names, eye_mask_path, - output_dir, sub_id, pipeline_ids, stim_path, gsr=False, - scrub=False, scrub_thresh=None): - - print("\n\n=== C-PAC now executing PyPEER for {0}. ===\n\n".format(sub_id)) - print("PEER scans for training model:\n{0}" - "\n\n".format(str(peer_scan_names))) - print("Data scans to estimate eye movements for:\n{0}" - "\n\n".format(str(data_scan_names))) - + return [i for i, x in enumerate(nuissance_vector) if x >= float(_motion_threshold)] + + +def prep_for_pypeer( + peer_scan_names, + data_scan_names, + eye_mask_path, + output_dir, + sub_id, + pipeline_ids, + stim_path, + gsr=False, + scrub=False, + scrub_thresh=None, +): # note these are non-nuisance-regression strategy paths - cpac_func_standard_paths = os.path.join(output_dir, - "pipeline_*", - sub_id, - "functional_to_standard", - "_scan_*", "*.nii*") + cpac_func_standard_paths = os.path.join( + output_dir, "pipeline_*", sub_id, "functional_to_standard", "_scan_*", "*.nii*" + ) func_standard_paths = glob.glob(cpac_func_standard_paths) if not func_standard_paths: - raise Exception("\n\n[!] Could not find any 'functional_to_standard' " - "file paths in your output directory - did your " - "C-PAC run complete successfully?\n\n") - - eye_mask_glob = os.path.join(output_dir, - "pipeline_*", - sub_id, - "template_eye_mask", - "*") + raise Exception( + "\n\n[!] Could not find any 'functional_to_standard' " + "file paths in your output directory - did your " + "C-PAC run complete successfully?\n\n" + ) + + eye_mask_glob = os.path.join( + output_dir, "pipeline_*", sub_id, "template_eye_mask", "*" + ) eye_mask_path = glob.glob(eye_mask_glob)[0] if not os.path.isfile(eye_mask_path): - raise Exception("\n\n[!] Could not find the template eye mask " - "file path in your output directory - did your " - "C-PAC run complete successfully?\n\n") - - print("Found input files:\n{0}\n".format(func_standard_paths)) + raise Exception( + "\n\n[!] Could not find the template eye mask " + "file path in your output directory - did your " + "C-PAC run complete successfully?\n\n" + ) pypeer_outdir = func_standard_paths[0].split("functional_to_standard")[0] pypeer_outdir = os.path.join(pypeer_outdir, "PyPEER") @@ -138,27 +141,25 @@ def prep_for_pypeer(peer_scan_names, data_scan_names, eye_mask_path, scan_label = func_path.split("/")[-2].replace("_scan_", "") if scan_label in peer_scan_names or scan_label in data_scan_names: - print("Eye-masking and z-score standardizing " - "{0}..".format(scan_label)) masked_data = pypeer_eye_masking(func_path, eye_mask_path) data = pypeer_zscore(masked_data) if gsr: - print("Global signal regression for {0}..".format(scan_label)) data = global_signal_regression(data, eye_mask_path) removed_indices = None if scrub and scan_label in peer_scan_names: - print("Motion scrubbing (Power 2012) for " - "{0}..".format(scan_label)) - fd_path = func_path.replace("functional_to_standard", - "frame_wise_displacement_power") + fd_path = func_path.replace( + "functional_to_standard", "frame_wise_displacement_power" + ) fd_path = fd_path.replace(fd_path.split("/")[-1], "FD.1D") if not os.path.isfile(fd_path): - raise Exception("\n\n[!] Could not find the mean framewise " - "displacement 1D file in your C-PAC output " - "directory.") + raise Exception( + "\n\n[!] Could not find the mean framewise " + "displacement 1D file in your C-PAC output " + "directory." + ) removed_indices = motion_scrub(fd_path, scrub_thresh) @@ -169,37 +170,41 @@ def prep_for_pypeer(peer_scan_names, data_scan_names, eye_mask_path, data_scans[func_path] = [raveled_data, scan_label] for peer_scan_path in peer_scans.keys(): - print("Training the eye estimation model using:\n{0}" - "\n\n".format(peer_scan_path)) data = peer_scans[peer_scan_path][0] peername = peer_scans[peer_scan_path][1] removed_indices = peer_scans[peer_scan_path][2] - data_for_training, calibration_points_removed = prepare_data_for_svr(data, - removed_indices, - eye_mask_path) - xmodel, ymodel = train_model(data_for_training, - calibration_points_removed, - stim_path) + data_for_training, calibration_points_removed = prepare_data_for_svr( + data, removed_indices, eye_mask_path + ) + xmodel, ymodel = train_model( + data_for_training, calibration_points_removed, stim_path + ) model_dir = os.path.join(pypeer_outdir, "peer_model-{0}".format(peername)) make_pypeer_dir(model_dir) - save_model(xmodel, ymodel, os.path.basename(peer_scan_path), str(scrub), - str(gsr), model_dir) + save_model( + xmodel, + ymodel, + os.path.basename(peer_scan_path), + str(scrub), + str(gsr), + model_dir, + ) for data_scan_path in data_scans.keys(): - print("Estimating eye movements for:\n{0}" - "\n\n".format(data_scan_path)) data = data_scans[data_scan_path][0] name = data_scans[data_scan_path][1] xmodel, ymodel, xname, yname = load_model(model_dir) xfix, yfix = predict_fixations(xmodel, ymodel, data) - estimate_dir = os.path.join(pypeer_outdir, - "estimations-{0}_model-{1}".format(name, peername)) + estimate_dir = os.path.join( + pypeer_outdir, "estimations-{0}_model-{1}".format(name, peername) + ) make_pypeer_dir(estimate_dir) - fix_xname, fix_yname = save_fixations(xfix, yfix, xname, yname, - estimate_dir) + fix_xname, fix_yname = save_fixations( + xfix, yfix, xname, yname, estimate_dir + ) estimate_em(xfix, yfix, fix_xname, fix_yname, estimate_dir) diff --git a/CPAC/qc/__init__.py b/CPAC/qc/__init__.py index 75ee654fec..4191369247 100644 --- a/CPAC/qc/__init__.py +++ b/CPAC/qc/__init__.py @@ -1,2 +1,2 @@ -from .utils import * from .qc import * +from .utils import * diff --git a/CPAC/qc/data/index.html b/CPAC/qc/data/index.html index a0580f3ad4..b84a6b45ac 100644 --- a/CPAC/qc/data/index.html +++ b/CPAC/qc/data/index.html @@ -86,7 +86,7 @@ 'centrality_zstd_smooth': 'Network Centrality (z-score standardized, smoothed)', 'centrality_zstd': 'Network Centrality (z-score standardized)', 'centrality': 'Network Centrality', - + 'csf_gm_wm': 'Grey Matter, White Matter & CSF', 'falff_smooth_hist': 'Histogram of Fractional Amplitude of Low-Frequency Fluctuation (smoothed)', @@ -137,14 +137,14 @@ 'sca_tempreg_maps_zstat_files_smooth': 'Seed-based Correlation Analysis', 'sca_tempreg_maps_zstat_files_smooth_hist': 'Seed-based Correlation Analysis', - + 'skullstrip_vis': 'Visual Result of Skull Strip', 'snr_hist': 'Histogram of Signal to Noise Ratio', 'snr': 'Signal to Noise Ratio', 'temporal_dual_regression_smooth_hist': 'Histogram of Temporal Dual Regression', 'temporal_dual_regression_smooth': 'Temporal Dual Regression', - + 'vmhc_smooth': 'Voxel-Mirrored Homotopic Connectivity (smoothed)', 'vmhc_smooth_hist': 'Histogram of Voxel-Mirrored Homotopic Connectivity (smoothed)', 'vmhc_fisher_zstd': 'Fisher-Z transform map of Voxel-Mirrored Homotopic Connectivity (z-score standardized)', @@ -171,7 +171,7 @@ const uris_string = /*CPAC*/``/*CPAC*/ .split(/(\r\n|[\n\v\f\r\x85\u2028\u2029])/) - + var uris = [] for (var uri_i in uris_string) { var uri = uris_string[uri_i].trim() @@ -236,7 +236,7 @@ filteredImages.push(image.image) } } - + $('#images img').remove() for (var image_i in filteredImages) { @@ -314,7 +314,7 @@ } $('#filter').append($('
').attr('data-name', field).append(sel)) } - + $('#filter div:not(.fixed) select').off('change').on('change', function () { $('#loading').show() options.filters = filters() @@ -352,4 +352,4 @@ }) - \ No newline at end of file + diff --git a/CPAC/qc/pipeline.py b/CPAC/qc/pipeline.py index 97d47e2dbd..15d6b35e09 100644 --- a/CPAC/qc/pipeline.py +++ b/CPAC/qc/pipeline.py @@ -1,35 +1,26 @@ -import os import pkg_resources as p + from CPAC.pipeline import nipype_pipeline_engine as pe from CPAC.pipeline.nodeblock import nodeblock -from nipype.interfaces import afni -from nipype.interfaces import fsl -from CPAC.utils.interfaces.function import Function - from CPAC.qc.qc import ( + afni_Edge3, create_montage, create_montage_gm_wm_csf, - qa_montages, - create_qc_snr, - create_qc_motion, + create_qc_carpet, create_qc_fd, + create_qc_motion, create_qc_skullstrip, - create_qc_carpet, - afni_Edge3 + create_qc_snr, ) - from CPAC.qc.utils import ( register_pallete, - generate_qc_pages, ) +from CPAC.utils.interfaces.function import Function # register color palettes -palletes = ['red', 'green', 'blue', 'red_to_blue', 'cyan_to_yellow'] +palletes = ["red", "green", "blue", "red_to_blue", "cyan_to_yellow"] for pallete in palletes: - register_pallete( - p.resource_filename('CPAC', 'qc/colors/%s.txt' % pallete), - pallete - ) + register_pallete(p.resource_filename("CPAC", "qc/colors/%s.txt" % pallete), pallete) @nodeblock( @@ -50,35 +41,29 @@ ], ) def qc_snr_plot(wf, cfg, strat_pool, pipe_num, opt=None): - # make SNR plot - qc_workflow = create_qc_snr(f'qc_snr_{pipe_num}') + qc_workflow = create_qc_snr(f"qc_snr_{pipe_num}") node, out = strat_pool.get_data("desc-preproc_bold") - wf.connect(node, out, qc_workflow, 'inputspec.functional_preprocessed') + wf.connect(node, out, qc_workflow, "inputspec.functional_preprocessed") node, out = strat_pool.get_data("space-bold_desc-brain_mask") - wf.connect(node, out, qc_workflow, 'inputspec.functional_brain_mask') + wf.connect(node, out, qc_workflow, "inputspec.functional_brain_mask") - node, out = \ - strat_pool.get_data('from-bold_to-T1w_mode-image_desc-linear_xfm') - wf.connect(node, out, - qc_workflow, 'inputspec.functional_to_anat_linear_xfm') + node, out = strat_pool.get_data("from-bold_to-T1w_mode-image_desc-linear_xfm") + wf.connect(node, out, qc_workflow, "inputspec.functional_to_anat_linear_xfm") - node, out = strat_pool.get_data('desc-preproc_T1w') - wf.connect(node, out, qc_workflow, 'inputspec.anatomical_brain') + node, out = strat_pool.get_data("desc-preproc_T1w") + wf.connect(node, out, qc_workflow, "inputspec.anatomical_brain") - node, out = strat_pool.get_data('space-T1w_sbref') - wf.connect(node, out, qc_workflow, 'inputspec.mean_functional_in_anat') + node, out = strat_pool.get_data("space-T1w_sbref") + wf.connect(node, out, qc_workflow, "inputspec.mean_functional_in_anat") outputs = { - 'desc-boldSnrAxial_quality': (qc_workflow, - 'outputspec.snr_axial_image'), - 'desc-boldSnrSagittal_quality': - (qc_workflow, 'outputspec.snr_sagittal_image'), - 'desc-boldSnrHist_quality': ( - qc_workflow, 'outputspec.snr_histogram_image'), - 'desc-boldSnr_quality': (qc_workflow, 'outputspec.snr_mean') + "desc-boldSnrAxial_quality": (qc_workflow, "outputspec.snr_axial_image"), + "desc-boldSnrSagittal_quality": (qc_workflow, "outputspec.snr_sagittal_image"), + "desc-boldSnrHist_quality": (qc_workflow, "outputspec.snr_histogram_image"), + "desc-boldSnr_quality": (qc_workflow, "outputspec.snr_mean"), } return (wf, outputs) @@ -88,28 +73,32 @@ def qc_snr_plot(wf, cfg, strat_pool, pipe_num, opt=None): name="qc_motion_plot", config=["pipeline_setup", "output_directory", "quality_control"], switch=["generate_quality_control_images"], - inputs=[["desc-movementParametersUnfiltered_motion", - "desc-movementParameters_motion"]], + inputs=[ + ["desc-movementParametersUnfiltered_motion", "desc-movementParameters_motion"] + ], outputs=[ "desc-movementParametersTrans_quality", "desc-movementParametersRot_quality", ], ) def qc_motion_plot(wf, cfg, strat_pool, pipe_num, opt=None): - # make motion parameters plot - qc_workflow = create_qc_motion(f'qc_motion_{pipe_num}') + qc_workflow = create_qc_motion(f"qc_motion_{pipe_num}") - node, out = strat_pool.get_data([ - "desc-movementParametersUnfiltered_motion", - "desc-movementParameters_motion"]) - wf.connect(node, out, qc_workflow, 'inputspec.motion_parameters') + node, out = strat_pool.get_data( + ["desc-movementParametersUnfiltered_motion", "desc-movementParameters_motion"] + ) + wf.connect(node, out, qc_workflow, "inputspec.motion_parameters") outputs = { - 'desc-movementParametersTrans_quality': ( - qc_workflow, 'outputspec.motion_translation_plot'), - 'desc-movementParametersRot_quality': ( - qc_workflow, 'outputspec.motion_rotation_plot') + "desc-movementParametersTrans_quality": ( + qc_workflow, + "outputspec.motion_translation_plot", + ), + "desc-movementParametersRot_quality": ( + qc_workflow, + "outputspec.motion_rotation_plot", + ), } return (wf, outputs) @@ -123,15 +112,16 @@ def qc_motion_plot(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["desc-framewiseDisplacementJenkinsonPlot_quality"], ) def qc_fd_plot(wf, cfg, strat_pool, pipe_num, opt=None): + qc_workflow = create_qc_fd(f"qc_fd_{pipe_num}") - qc_workflow = create_qc_fd(f'qc_fd_{pipe_num}') - - node, out = strat_pool.get_data('framewise-displacement-jenkinson') - wf.connect(node, out, qc_workflow, 'inputspec.fd') + node, out = strat_pool.get_data("framewise-displacement-jenkinson") + wf.connect(node, out, qc_workflow, "inputspec.fd") outputs = { - 'desc-framewiseDisplacementJenkinsonPlot_quality': - (qc_workflow, 'outputspec.fd_histogram_plot') + "desc-framewiseDisplacementJenkinsonPlot_quality": ( + qc_workflow, + "outputspec.fd_histogram_plot", + ) } return (wf, outputs) @@ -145,23 +135,21 @@ def qc_fd_plot(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["desc-brain_desc-T1wAxial_quality", "desc-brain_desc-T1wSagittal_quality"], ) def qc_brain_extraction(wf, cfg, strat_pool, pipe_num, opt=None): - # make QC montages for Skull Stripping Visualization - qc_workflow = create_qc_skullstrip( - f'qc_skullstrip_{pipe_num}' - ) + qc_workflow = create_qc_skullstrip(f"qc_skullstrip_{pipe_num}") - node, out = strat_pool.get_data('desc-preproc_T1w') - wf.connect(node, out, qc_workflow, 'inputspec.anatomical_brain') + node, out = strat_pool.get_data("desc-preproc_T1w") + wf.connect(node, out, qc_workflow, "inputspec.anatomical_brain") - node, out = strat_pool.get_data('desc-head_T1w') - wf.connect(node, out, qc_workflow, 'inputspec.anatomical_reorient') + node, out = strat_pool.get_data("desc-head_T1w") + wf.connect(node, out, qc_workflow, "inputspec.anatomical_reorient") outputs = { - 'desc-brain_desc-T1wAxial_quality': (qc_workflow, - 'outputspec.axial_image'), - 'desc-brain_desc-T1wSagittal_quality': - (qc_workflow, 'outputspec.sagittal_image') + "desc-brain_desc-T1wAxial_quality": (qc_workflow, "outputspec.axial_image"), + "desc-brain_desc-T1wSagittal_quality": ( + qc_workflow, + "outputspec.sagittal_image", + ), } return (wf, outputs) @@ -178,32 +166,38 @@ def qc_brain_extraction(wf, cfg, strat_pool, pipe_num, opt=None): ], ) def qc_T1w_standard(wf, cfg, strat_pool, pipe_num, opt=None): - # make QC montages for mni normalized anatomical image - montage_mni_anat = create_montage(f'montage_mni_anat_{pipe_num}', - 'red', 'mni_anat', - mapnode=False) - - node, out = strat_pool.get_data('space-template_desc-preproc_T1w') - wf.connect(node, out, montage_mni_anat, 'inputspec.underlay') + montage_mni_anat = create_montage( + f"montage_mni_anat_{pipe_num}", "red", "mni_anat", mapnode=False + ) - anat_template_edge = pe.Node(Function(input_names=['in_file'], - output_names=['out_file'], - function=afni_Edge3, - as_module=True), - name=f'anat_template_edge_{pipe_num}') + node, out = strat_pool.get_data("space-template_desc-preproc_T1w") + wf.connect(node, out, montage_mni_anat, "inputspec.underlay") + + anat_template_edge = pe.Node( + Function( + input_names=["in_file"], + output_names=["out_file"], + function=afni_Edge3, + as_module=True, + ), + name=f"anat_template_edge_{pipe_num}", + ) - node, out = strat_pool.get_data('T1w-brain-template') - wf.connect(node, out, anat_template_edge, 'in_file') + node, out = strat_pool.get_data("T1w-brain-template") + wf.connect(node, out, anat_template_edge, "in_file") - wf.connect(anat_template_edge, 'out_file', - montage_mni_anat, 'inputspec.overlay') + wf.connect(anat_template_edge, "out_file", montage_mni_anat, "inputspec.overlay") outputs = { - 'space-template_desc-brain_desc-T1wAxial_quality': - (montage_mni_anat, 'outputspec.axial_png'), - 'space-template_desc-brain_desc-T1wSagittal_quality': - (montage_mni_anat, 'outputspec.sagittal_png') + "space-template_desc-brain_desc-T1wAxial_quality": ( + montage_mni_anat, + "outputspec.axial_png", + ), + "space-template_desc-brain_desc-T1wSagittal_quality": ( + montage_mni_anat, + "outputspec.sagittal_png", + ), } return (wf, outputs) @@ -224,30 +218,26 @@ def qc_T1w_standard(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["desc-dsegAxial_quality", "desc-dsegSagittal_quality"], ) def qc_segmentation(wf, cfg, strat_pool, pipe_num, opt=None): - # make QC montages for CSF WM GM montage_csf_gm_wm = create_montage_gm_wm_csf( - f'montage_csf_gm_wm_{pipe_num}', 'montage_csf_gm_wm') + f"montage_csf_gm_wm_{pipe_num}", "montage_csf_gm_wm" + ) - node, out = strat_pool.get_data('desc-preproc_T1w') - wf.connect(node, out, montage_csf_gm_wm, 'inputspec.underlay') + node, out = strat_pool.get_data("desc-preproc_T1w") + wf.connect(node, out, montage_csf_gm_wm, "inputspec.underlay") - node, out = strat_pool.get_data(['label-CSF_desc-preproc_mask', - 'label-CSF_mask']) - wf.connect(node, out, montage_csf_gm_wm, 'inputspec.overlay_csf') + node, out = strat_pool.get_data(["label-CSF_desc-preproc_mask", "label-CSF_mask"]) + wf.connect(node, out, montage_csf_gm_wm, "inputspec.overlay_csf") - node, out = strat_pool.get_data(['label-WM_desc-preproc_mask', - 'label-WM_mask']) - wf.connect(node, out, montage_csf_gm_wm, 'inputspec.overlay_wm') + node, out = strat_pool.get_data(["label-WM_desc-preproc_mask", "label-WM_mask"]) + wf.connect(node, out, montage_csf_gm_wm, "inputspec.overlay_wm") - node, out = strat_pool.get_data(['label-GM_desc-preproc_mask', - 'label-GM_mask']) - wf.connect(node, out, montage_csf_gm_wm, 'inputspec.overlay_gm') + node, out = strat_pool.get_data(["label-GM_desc-preproc_mask", "label-GM_mask"]) + wf.connect(node, out, montage_csf_gm_wm, "inputspec.overlay_gm") outputs = { - 'desc-dsegAxial_quality': (montage_csf_gm_wm, 'outputspec.axial_png'), - 'desc-dsegSagittal_quality': (montage_csf_gm_wm, - 'outputspec.sagittal_png') + "desc-dsegAxial_quality": (montage_csf_gm_wm, "outputspec.axial_png"), + "desc-dsegSagittal_quality": (montage_csf_gm_wm, "outputspec.sagittal_png"), } return (wf, outputs) @@ -268,31 +258,32 @@ def qc_segmentation(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["epi-desc-dsegAxial_quality", "epi-desc-dsegSagittal_quality"], ) def qc_epi_segmentation(wf, cfg, strat_pool, pipe_num, opt=None): - # make QC montages for CSF WM GM montage_csf_gm_wm = create_montage_gm_wm_csf( - f'montage_csf_gm_wm_{pipe_num}', 'montage_csf_gm_wm') + f"montage_csf_gm_wm_{pipe_num}", "montage_csf_gm_wm" + ) - node, out = strat_pool.get_data('desc-preproc_bold') - wf.connect(node, out, montage_csf_gm_wm, 'inputspec.underlay') + node, out = strat_pool.get_data("desc-preproc_bold") + wf.connect(node, out, montage_csf_gm_wm, "inputspec.underlay") - node, out = strat_pool.get_data(['space-bold_label-CSF_desc-preproc_mask', - 'space-bold_label-CSF_mask']) - wf.connect(node, out, montage_csf_gm_wm, 'inputspec.overlay_csf') + node, out = strat_pool.get_data( + ["space-bold_label-CSF_desc-preproc_mask", "space-bold_label-CSF_mask"] + ) + wf.connect(node, out, montage_csf_gm_wm, "inputspec.overlay_csf") - node, out = strat_pool.get_data(['space-bold_label-WM_desc-preproc_mask', - 'space-bold_label-WM_mask']) - wf.connect(node, out, montage_csf_gm_wm, 'inputspec.overlay_wm') + node, out = strat_pool.get_data( + ["space-bold_label-WM_desc-preproc_mask", "space-bold_label-WM_mask"] + ) + wf.connect(node, out, montage_csf_gm_wm, "inputspec.overlay_wm") - node, out = strat_pool.get_data(['space-bold_label-GM_desc-preproc_mask', - 'space-bold_label-GM_mask']) - wf.connect(node, out, montage_csf_gm_wm, 'inputspec.overlay_gm') + node, out = strat_pool.get_data( + ["space-bold_label-GM_desc-preproc_mask", "space-bold_label-GM_mask"] + ) + wf.connect(node, out, montage_csf_gm_wm, "inputspec.overlay_gm") outputs = { - 'epi-desc-dsegAxial_quality': (montage_csf_gm_wm, - 'outputspec.axial_png'), - 'epi-desc-dsegSagittal_quality': (montage_csf_gm_wm, - 'outputspec.sagittal_png') + "epi-desc-dsegAxial_quality": (montage_csf_gm_wm, "outputspec.axial_png"), + "epi-desc-dsegSagittal_quality": (montage_csf_gm_wm, "outputspec.sagittal_png"), } return (wf, outputs) @@ -311,30 +302,33 @@ def qc_epi_segmentation(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["space-template_desc-preprocBoldCarpet_quality"], ) def qc_carpet_plot(wf, cfg, strat_pool, pipe_num, opt=None): - # make QC Carpet plot - carpet_seg = create_qc_carpet(f'carpet_seg_{pipe_num}', 'carpet_seg') + carpet_seg = create_qc_carpet(f"carpet_seg_{pipe_num}", "carpet_seg") - connection, resource = \ - strat_pool.get_data(["space-template_desc-preproc_bold"], - report_fetched=True) + connection, resource = strat_pool.get_data( + ["space-template_desc-preproc_bold"], report_fetched=True + ) node, out = connection - wf.connect(node, out, carpet_seg, 'inputspec.functional_to_standard') + wf.connect(node, out, carpet_seg, "inputspec.functional_to_standard") node, out = strat_pool.get_data("space-template_sbref") - wf.connect(node, out, carpet_seg, 'inputspec.mean_functional_to_standard') + wf.connect(node, out, carpet_seg, "inputspec.mean_functional_to_standard") node, out = strat_pool.get_data("GM-path") - wf.connect(node, out, carpet_seg, 'inputspec.anatomical_gm_mask') + wf.connect(node, out, carpet_seg, "inputspec.anatomical_gm_mask") node, out = strat_pool.get_data("WM-path") - wf.connect(node, out, carpet_seg, 'inputspec.anatomical_wm_mask') + wf.connect(node, out, carpet_seg, "inputspec.anatomical_wm_mask") node, out = strat_pool.get_data("CSF-path") - wf.connect(node, out, carpet_seg, 'inputspec.anatomical_csf_mask') + wf.connect(node, out, carpet_seg, "inputspec.anatomical_csf_mask") - outputs = {'space-template_desc-preprocBoldCarpet_quality': ( - carpet_seg, 'outputspec.carpet_plot')} + outputs = { + "space-template_desc-preprocBoldCarpet_quality": ( + carpet_seg, + "outputspec.carpet_plot", + ) + } return (wf, outputs) @@ -347,31 +341,35 @@ def qc_carpet_plot(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["space-T1w_desc-boldAxial_quality", "space-T1w_desc-boldSagittal_quality"], ) def qc_coregistration(wf, cfg, strat_pool, pipe_num, opt=None): - # make QC montage for Mean Functional in T1 with T1 edge - anat_edge = pe.Node(Function(input_names=['in_file'], - output_names=['out_file'], - function=afni_Edge3, - as_module=True), - name=f'anat_edge_{pipe_num}') + anat_edge = pe.Node( + Function( + input_names=["in_file"], + output_names=["out_file"], + function=afni_Edge3, + as_module=True, + ), + name=f"anat_edge_{pipe_num}", + ) - node, out = strat_pool.get_data('desc-preproc_T1w') - wf.connect(node, out, anat_edge, 'in_file') + node, out = strat_pool.get_data("desc-preproc_T1w") + wf.connect(node, out, anat_edge, "in_file") - montage_anat = create_montage(f'montage_anat_{pipe_num}', 'red', - 't1_edge_on_mean_func_in_t1', - mapnode=False) + montage_anat = create_montage( + f"montage_anat_{pipe_num}", "red", "t1_edge_on_mean_func_in_t1", mapnode=False + ) - wf.connect(anat_edge, 'out_file', montage_anat, 'inputspec.overlay') + wf.connect(anat_edge, "out_file", montage_anat, "inputspec.overlay") - node, out = strat_pool.get_data('space-T1w_sbref') - wf.connect(node, out, montage_anat, 'inputspec.underlay') + node, out = strat_pool.get_data("space-T1w_sbref") + wf.connect(node, out, montage_anat, "inputspec.underlay") outputs = { - 'space-T1w_desc-boldAxial_quality': - (montage_anat, 'outputspec.axial_png'), - 'space-T1w_desc-boldSagittal_quality': - (montage_anat, 'outputspec.sagittal_png') + "space-T1w_desc-boldAxial_quality": (montage_anat, "outputspec.axial_png"), + "space-T1w_desc-boldSagittal_quality": ( + montage_anat, + "outputspec.sagittal_png", + ), } return (wf, outputs) @@ -395,32 +393,35 @@ def qc_coregistration(wf, cfg, strat_pool, pipe_num, opt=None): ], ) def qc_bold_registration(wf, cfg, strat_pool, pipe_num, opt=None): - # make QC montage for Mean Functional in MNI with MNI edge - montage_mfi = create_montage(f'montage_mfi_{pipe_num}', 'red', - 'MNI_edge_on_mean_func_mni', - mapnode=False) - - node, out = strat_pool.get_data('space-template_sbref') - wf.connect(node, out, montage_mfi, 'inputspec.underlay') + montage_mfi = create_montage( + f"montage_mfi_{pipe_num}", "red", "MNI_edge_on_mean_func_mni", mapnode=False + ) - func_template_edge = pe.Node(Function(input_names=['in_file'], - output_names=['out_file'], - function=afni_Edge3, - as_module=True), - name=f'func_template_edge_{pipe_num}') + node, out = strat_pool.get_data("space-template_sbref") + wf.connect(node, out, montage_mfi, "inputspec.underlay") + + func_template_edge = pe.Node( + Function( + input_names=["in_file"], + output_names=["out_file"], + function=afni_Edge3, + as_module=True, + ), + name=f"func_template_edge_{pipe_num}", + ) node, out = strat_pool.get_data("T1w-brain-template-funcreg") - wf.connect(node, out, func_template_edge, 'in_file') + wf.connect(node, out, func_template_edge, "in_file") - wf.connect(func_template_edge, 'out_file', - montage_mfi, 'inputspec.overlay') + wf.connect(func_template_edge, "out_file", montage_mfi, "inputspec.overlay") outputs = { - 'space-template_desc-boldAxial_quality': - (montage_mfi, 'outputspec.axial_png'), - 'space-template_desc-boldSagittal_quality': - (montage_mfi, 'outputspec.sagittal_png') + "space-template_desc-boldAxial_quality": (montage_mfi, "outputspec.axial_png"), + "space-template_desc-boldSagittal_quality": ( + montage_mfi, + "outputspec.sagittal_png", + ), } return (wf, outputs) @@ -452,38 +453,40 @@ def qc_bold_registration(wf, cfg, strat_pool, pipe_num, opt=None): ], ) def qc_bold_EPI_registration(wf, cfg, strat_pool, pipe_num, opt=None): - # make QC montage for Mean Functional in MNI with MNI edge - montage_mfi = create_montage(f'montage_mfi_{pipe_num}', 'red', - 'EPI_MNI_edge_on_mean_func_mni', - mapnode=False) - - node, out = strat_pool.get_data('space-template_sbref') - wf.connect(node, out, montage_mfi, 'inputspec.underlay') + montage_mfi = create_montage( + f"montage_mfi_{pipe_num}", "red", "EPI_MNI_edge_on_mean_func_mni", mapnode=False + ) - func_template_edge = pe.Node(Function(input_names=['in_file'], - output_names=['out_file'], - function=afni_Edge3, - as_module=True), - name=f'EPI_func_template_edge_{pipe_num}') + node, out = strat_pool.get_data("space-template_sbref") + wf.connect(node, out, montage_mfi, "inputspec.underlay") + + func_template_edge = pe.Node( + Function( + input_names=["in_file"], + output_names=["out_file"], + function=afni_Edge3, + as_module=True, + ), + name=f"EPI_func_template_edge_{pipe_num}", + ) node, out = strat_pool.get_data("EPI-template-funcreg") - wf.connect(node, out, func_template_edge, 'in_file') + wf.connect(node, out, func_template_edge, "in_file") - wf.connect(func_template_edge, 'out_file', - montage_mfi, 'inputspec.overlay') + wf.connect(func_template_edge, "out_file", montage_mfi, "inputspec.overlay") outputs = { - 'space-template_desc-boldAxial_quality': - (montage_mfi, 'outputspec.axial_png'), - 'space-template_desc-boldSagittal_quality': - (montage_mfi, 'outputspec.sagittal_png') + "space-template_desc-boldAxial_quality": (montage_mfi, "outputspec.axial_png"), + "space-template_desc-boldSagittal_quality": ( + montage_mfi, + "outputspec.sagittal_png", + ), } return (wf, outputs) - def create_qc_workflow(cfg): qc_montage_id_a = {} qc_montage_id_s = {} @@ -492,76 +495,83 @@ def create_qc_workflow(cfg): qc_stack = [] - if cfg.functional_preproc['run'] and cfg.registration_workflows[ - 'functional_registration']['coregistration']['run']: - + if ( + cfg.functional_preproc["run"] + and cfg.registration_workflows["functional_registration"]["coregistration"][ + "run" + ] + ): qc_stack += [qc_snr_plot, qc_coregistration] - if not 0 in qc_montage_id_a: - qc_montage_id_a[0] = 'snr_a' - qc_montage_id_s[0] = 'snr_s' - qc_hist_id[0] = 'snr_hist' + if 0 not in qc_montage_id_a: + qc_montage_id_a[0] = "snr_a" + qc_montage_id_s[0] = "snr_s" + qc_hist_id[0] = "snr_hist" - if not 9 in qc_montage_id_a: - qc_montage_id_a[9] = 'mean_func_with_t1_edge_a' - qc_montage_id_s[9] = 'mean_func_with_t1_edge_s' - - if cfg.functional_preproc['run']: + if 9 not in qc_montage_id_a: + qc_montage_id_a[9] = "mean_func_with_t1_edge_a" + qc_montage_id_s[9] = "mean_func_with_t1_edge_s" + if cfg.functional_preproc["run"]: qc_stack += [qc_motion_plot, qc_fd_plot] - if not 1 in qc_plot_id: - qc_plot_id[1] = 'movement_trans_plot' - if not 2 in qc_plot_id: - qc_plot_id[2] = 'movement_rot_plot' - if not 3 in qc_plot_id: - qc_plot_id[3] = 'fd_plot' - - if cfg.anatomical_preproc['run']: + if 1 not in qc_plot_id: + qc_plot_id[1] = "movement_trans_plot" + if 2 not in qc_plot_id: + qc_plot_id[2] = "movement_rot_plot" + if 3 not in qc_plot_id: + qc_plot_id[3] = "fd_plot" + if cfg.anatomical_preproc["run"]: qc_stack.append(qc_brain_extraction) - if not 4 in qc_montage_id_a: - qc_montage_id_a[4] = 'skullstrip_vis_a' - qc_montage_id_s[4] = 'skullstrip_vis_s' - - if cfg.registration_workflows['anatomical_registration']['run']: + if 4 not in qc_montage_id_a: + qc_montage_id_a[4] = "skullstrip_vis_a" + qc_montage_id_s[4] = "skullstrip_vis_s" + if cfg.registration_workflows["anatomical_registration"]["run"]: qc_stack.append(qc_T1w_standard) - if not 5 in qc_montage_id_a: - qc_montage_id_a[5] = 'mni_normalized_anatomical_a' - qc_montage_id_s[5] = 'mni_normalized_anatomical_s' - - if cfg.anatomical_preproc['run'] and cfg.segmentation['run']: - - if 'T1_Template' in cfg.segmentation['tissue_segmentation'][ - 'Template_Based']['template_for_segmentation']: + if 5 not in qc_montage_id_a: + qc_montage_id_a[5] = "mni_normalized_anatomical_a" + qc_montage_id_s[5] = "mni_normalized_anatomical_s" + + if cfg.anatomical_preproc["run"] and cfg.segmentation["run"]: + if ( + "T1_Template" + in cfg.segmentation["tissue_segmentation"]["Template_Based"][ + "template_for_segmentation" + ] + ): qc_stack.append(qc_segmentation) - if 'EPI_Template' in cfg.segmentation['tissue_segmentation'][ - 'Template_Based']['template_for_segmentation']: + if ( + "EPI_Template" + in cfg.segmentation["tissue_segmentation"]["Template_Based"][ + "template_for_segmentation" + ] + ): qc_stack.append(qc_epi_segmentation) - if not 7 in qc_montage_id_a: - qc_montage_id_a[7] = 'csf_gm_wm_a' - qc_montage_id_s[7] = 'csf_gm_wm_s' - - if cfg.registration_workflows['functional_registration'][ - 'func_registration_to_template']['run']: + if 7 not in qc_montage_id_a: + qc_montage_id_a[7] = "csf_gm_wm_a" + qc_montage_id_s[7] = "csf_gm_wm_s" + if cfg.registration_workflows["functional_registration"][ + "func_registration_to_template" + ]["run"]: qc_stack.append(qc_carpet_plot) - if not 8 in qc_plot_id: - qc_plot_id[8] = 'carpet' - - if cfg.registration_workflows['functional_registration'][ - 'func_registration_to_template']['run']: + if 8 not in qc_plot_id: + qc_plot_id[8] = "carpet" + if cfg.registration_workflows["functional_registration"][ + "func_registration_to_template" + ]["run"]: qc_stack.append(qc_bold_registration) - if not 10 in qc_montage_id_a: - qc_montage_id_a[10] = 'mean_func_with_mni_edge_a' - qc_montage_id_s[10] = 'mean_func_with_mni_edge_s' + if 10 not in qc_montage_id_a: + qc_montage_id_a[10] = "mean_func_with_mni_edge_a" + qc_montage_id_s[10] = "mean_func_with_mni_edge_s" return qc_stack, qc_montage_id_a, qc_montage_id_s, qc_hist_id, qc_plot_id diff --git a/CPAC/qc/qc.py b/CPAC/qc/qc.py index 8dae271822..2307147fc6 100644 --- a/CPAC/qc/qc.py +++ b/CPAC/qc/qc.py @@ -1,474 +1,565 @@ -import matplotlib -matplotlib.use('Agg', force=True) +import matplotlib as mpl -from nipype.interfaces import afni -from CPAC.utils.interfaces.function import Function +mpl.use("Agg", force=True) + +from nipype.interfaces import afni, fsl +import nipype.interfaces.utility as util + +from CPAC.pipeline import nipype_pipeline_engine as pe from CPAC.qc.utils import ( - resample_1mm, montage_axial, montage_sagittal, - montage_gm_wm_csf_axial, montage_gm_wm_csf_sagittal, - cal_snr_val, gen_histogram, drop_percent, gen_motion_plt, + cal_snr_val, + drop_percent, + gen_carpet_plt, + gen_histogram, + gen_motion_plt, gen_plot_png, - gen_carpet_plt + montage_axial, + montage_gm_wm_csf_axial, + montage_gm_wm_csf_sagittal, + montage_sagittal, + resample_1mm, ) - -from CPAC.pipeline import nipype_pipeline_engine as pe -import nipype.interfaces.utility as util -from nipype.interfaces import afni -import nipype.interfaces.fsl as fsl +from CPAC.utils.interfaces.function import Function def create_montage(wf_name, cbar_name, png_name, mapnode=True): - wf = pe.Workflow(name=wf_name) - inputnode = pe.Node(util.IdentityInterface(fields=['underlay', - 'overlay']), - name='inputspec') + inputnode = pe.Node( + util.IdentityInterface(fields=["underlay", "overlay"]), name="inputspec" + ) - outputnode = pe.Node(util.IdentityInterface(fields=['axial_png', - 'sagittal_png', - 'resampled_underlay', - 'resampled_overlay']), - name='outputspec') + outputnode = pe.Node( + util.IdentityInterface( + fields=[ + "axial_png", + "sagittal_png", + "resampled_underlay", + "resampled_overlay", + ] + ), + name="outputspec", + ) # node for resampling create_montage images to 1mm for QC pages - resample_u = pe.Node(Function(input_names=['file_'], - output_names=['new_fname'], - function=resample_1mm, - as_module=True), - name='resample_u', - mem_gb=0, - mem_x=(0.0115, 'file_', 't')) + resample_u = pe.Node( + Function( + input_names=["file_"], + output_names=["new_fname"], + function=resample_1mm, + as_module=True, + ), + name="resample_u", + mem_gb=0, + mem_x=(0.0115, "file_", "t"), + ) - wf.connect(inputnode, 'underlay', resample_u, 'file_') - wf.connect(resample_u, 'new_fname', outputnode, 'resampled_underlay') + wf.connect(inputnode, "underlay", resample_u, "file_") + wf.connect(resample_u, "new_fname", outputnode, "resampled_underlay") # same for overlays (resampling to 1mm) - resample_o = pe.Node(Function(input_names=['file_'], - output_names=['new_fname'], - function=resample_1mm, - as_module=True), - name='resample_o', - mem_gb=0, - mem_x=(0.0115, 'file_', 't')) + resample_o = pe.Node( + Function( + input_names=["file_"], + output_names=["new_fname"], + function=resample_1mm, + as_module=True, + ), + name="resample_o", + mem_gb=0, + mem_x=(0.0115, "file_", "t"), + ) - wf.connect(inputnode, 'overlay', resample_o, 'file_') - wf.connect(resample_o, 'new_fname', outputnode, 'resampled_overlay') + wf.connect(inputnode, "overlay", resample_o, "file_") + wf.connect(resample_o, "new_fname", outputnode, "resampled_overlay") # node for axial montages if mapnode: - montage_a = pe.MapNode(Function(input_names=['overlay', - 'underlay', - 'png_name', - 'cbar_name'], - output_names=['png_name'], - function=montage_axial, - as_module=True), - name='montage_a', - iterfield=['overlay']) + montage_a = pe.MapNode( + Function( + input_names=["overlay", "underlay", "png_name", "cbar_name"], + output_names=["png_name"], + function=montage_axial, + as_module=True, + ), + name="montage_a", + iterfield=["overlay"], + ) else: - montage_a = pe.Node(Function(input_names=['overlay', - 'underlay', - 'png_name', - 'cbar_name'], - output_names=['png_name'], - function=montage_axial, - as_module=True), - name='montage_a') + montage_a = pe.Node( + Function( + input_names=["overlay", "underlay", "png_name", "cbar_name"], + output_names=["png_name"], + function=montage_axial, + as_module=True, + ), + name="montage_a", + ) montage_a.inputs.cbar_name = cbar_name - montage_a.inputs.png_name = png_name + '_a.png' + montage_a.inputs.png_name = png_name + "_a.png" - wf.connect(resample_u, 'new_fname', montage_a, 'underlay') - wf.connect(resample_o, 'new_fname', montage_a, 'overlay') + wf.connect(resample_u, "new_fname", montage_a, "underlay") + wf.connect(resample_o, "new_fname", montage_a, "overlay") # node for sagittal montages if mapnode: - montage_s = pe.MapNode(Function(input_names=['overlay', - 'underlay', - 'png_name', - 'cbar_name'], - output_names=['png_name'], - function=montage_sagittal, - as_module=True), - name='montage_s', - iterfield=['overlay']) + montage_s = pe.MapNode( + Function( + input_names=["overlay", "underlay", "png_name", "cbar_name"], + output_names=["png_name"], + function=montage_sagittal, + as_module=True, + ), + name="montage_s", + iterfield=["overlay"], + ) else: - montage_s = pe.Node(Function(input_names=['overlay', - 'underlay', - 'png_name', - 'cbar_name'], - output_names=['png_name'], - function=montage_sagittal, - as_module=True), - name='montage_s') + montage_s = pe.Node( + Function( + input_names=["overlay", "underlay", "png_name", "cbar_name"], + output_names=["png_name"], + function=montage_sagittal, + as_module=True, + ), + name="montage_s", + ) montage_s.inputs.cbar_name = cbar_name - montage_s.inputs.png_name = png_name + '_s.png' + montage_s.inputs.png_name = png_name + "_s.png" - wf.connect(resample_u, 'new_fname', montage_s, 'underlay') - wf.connect(resample_o, 'new_fname', montage_s, 'overlay') + wf.connect(resample_u, "new_fname", montage_s, "underlay") + wf.connect(resample_o, "new_fname", montage_s, "overlay") - wf.connect(montage_a, 'png_name', outputnode, 'axial_png') - wf.connect(montage_s, 'png_name', outputnode, 'sagittal_png') + wf.connect(montage_a, "png_name", outputnode, "axial_png") + wf.connect(montage_s, "png_name", outputnode, "sagittal_png") return wf def create_montage_gm_wm_csf(wf_name, png_name): - wf = pe.Workflow(name=wf_name) - inputNode = pe.Node(util.IdentityInterface(fields=['underlay', - 'overlay_csf', - 'overlay_wm', - 'overlay_gm']), - name='inputspec') - - outputNode = pe.Node(util.IdentityInterface(fields=['axial_png', - 'sagittal_png', - 'resampled_underlay', - 'resampled_overlay_csf', - 'resampled_overlay_wm', - 'resampled_overlay_gm']), - name='outputspec') - - resample_u = pe.Node(Function(input_names=['file_'], - output_names=['new_fname'], - function=resample_1mm, - as_module=True), - name='resample_u', - mem_gb=0, - mem_x=(0.0115, 'file_', 't')) - - resample_o_csf = resample_u.clone('resample_o_csf') - resample_o_wm = resample_u.clone('resample_o_wm') - resample_o_gm = resample_u.clone('resample_o_gm') - - wf.connect(inputNode, 'underlay', resample_u, 'file_') - wf.connect(inputNode, 'overlay_csf', resample_o_csf, 'file_') - wf.connect(inputNode, 'overlay_gm', resample_o_gm, 'file_') - wf.connect(inputNode, 'overlay_wm', resample_o_wm, 'file_') - - montage_a = pe.Node(Function(input_names=['overlay_csf', - 'overlay_wm', - 'overlay_gm', - 'underlay', - 'png_name'], - output_names=['png_name'], - function=montage_gm_wm_csf_axial, - as_module=True), - name='montage_a') - - - wf.connect(resample_u, 'new_fname', montage_a, 'underlay') - wf.connect(resample_o_csf, 'new_fname', montage_a, 'overlay_csf') - wf.connect(resample_o_gm, 'new_fname', montage_a, 'overlay_gm') - wf.connect(resample_o_wm, 'new_fname', montage_a, 'overlay_wm') - montage_a.inputs.png_name = png_name + '_a.png' - - montage_s = pe.Node(Function(input_names=['overlay_csf', - 'overlay_wm', - 'overlay_gm', - 'underlay', - 'png_name'], - output_names=['png_name'], - function=montage_gm_wm_csf_sagittal, - as_module=True), - name='montage_s') - montage_s.inputs.png_name = png_name + '_s.png' - - wf.connect(resample_u, 'new_fname', montage_s, 'underlay') - wf.connect(resample_o_csf, 'new_fname', montage_s, 'overlay_csf') - wf.connect(resample_o_gm, 'new_fname', montage_s, 'overlay_gm') - wf.connect(resample_o_wm, 'new_fname', montage_s, 'overlay_wm') - - wf.connect(resample_u, 'new_fname', outputNode, 'resampled_underlay') - wf.connect(resample_o_csf, 'new_fname', outputNode, 'resampled_overlay_csf') - wf.connect(resample_o_wm, 'new_fname', outputNode, 'resampled_overlay_wm') - wf.connect(resample_o_gm, 'new_fname', outputNode, 'resampled_overlay_gm') - wf.connect(montage_a, 'png_name', outputNode, 'axial_png') - wf.connect(montage_s, 'png_name', outputNode, 'sagittal_png') + inputNode = pe.Node( + util.IdentityInterface( + fields=["underlay", "overlay_csf", "overlay_wm", "overlay_gm"] + ), + name="inputspec", + ) - return wf + outputNode = pe.Node( + util.IdentityInterface( + fields=[ + "axial_png", + "sagittal_png", + "resampled_underlay", + "resampled_overlay_csf", + "resampled_overlay_wm", + "resampled_overlay_gm", + ] + ), + name="outputspec", + ) + resample_u = pe.Node( + Function( + input_names=["file_"], + output_names=["new_fname"], + function=resample_1mm, + as_module=True, + ), + name="resample_u", + mem_gb=0, + mem_x=(0.0115, "file_", "t"), + ) + + resample_o_csf = resample_u.clone("resample_o_csf") + resample_o_wm = resample_u.clone("resample_o_wm") + resample_o_gm = resample_u.clone("resample_o_gm") + + wf.connect(inputNode, "underlay", resample_u, "file_") + wf.connect(inputNode, "overlay_csf", resample_o_csf, "file_") + wf.connect(inputNode, "overlay_gm", resample_o_gm, "file_") + wf.connect(inputNode, "overlay_wm", resample_o_wm, "file_") + + montage_a = pe.Node( + Function( + input_names=[ + "overlay_csf", + "overlay_wm", + "overlay_gm", + "underlay", + "png_name", + ], + output_names=["png_name"], + function=montage_gm_wm_csf_axial, + as_module=True, + ), + name="montage_a", + ) + + wf.connect(resample_u, "new_fname", montage_a, "underlay") + wf.connect(resample_o_csf, "new_fname", montage_a, "overlay_csf") + wf.connect(resample_o_gm, "new_fname", montage_a, "overlay_gm") + wf.connect(resample_o_wm, "new_fname", montage_a, "overlay_wm") + montage_a.inputs.png_name = png_name + "_a.png" + + montage_s = pe.Node( + Function( + input_names=[ + "overlay_csf", + "overlay_wm", + "overlay_gm", + "underlay", + "png_name", + ], + output_names=["png_name"], + function=montage_gm_wm_csf_sagittal, + as_module=True, + ), + name="montage_s", + ) + montage_s.inputs.png_name = png_name + "_s.png" + + wf.connect(resample_u, "new_fname", montage_s, "underlay") + wf.connect(resample_o_csf, "new_fname", montage_s, "overlay_csf") + wf.connect(resample_o_gm, "new_fname", montage_s, "overlay_gm") + wf.connect(resample_o_wm, "new_fname", montage_s, "overlay_wm") + + wf.connect(resample_u, "new_fname", outputNode, "resampled_underlay") + wf.connect(resample_o_csf, "new_fname", outputNode, "resampled_overlay_csf") + wf.connect(resample_o_wm, "new_fname", outputNode, "resampled_overlay_wm") + wf.connect(resample_o_gm, "new_fname", outputNode, "resampled_overlay_gm") + wf.connect(montage_a, "png_name", outputNode, "axial_png") + wf.connect(montage_s, "png_name", outputNode, "sagittal_png") + + return wf -def qa_montages(workflow, c, strat, num_strat, - qc_montage_id_a, qc_montage_id_s, qc_hist_id, - measure, idx): +def qa_montages( + workflow, + c, + strat, + num_strat, + qc_montage_id_a, + qc_montage_id_s, + qc_hist_id, + measure, + idx, +): try: overlay, out_file = strat[measure] overlay_drop_percent = pe.MapNode( - Function(input_names=['measure_file', - 'percent'], - output_names=['modified_measure_file'], - function=drop_percent, - as_module=True), - name='dp_%s_%d' % ( - measure, num_strat), - iterfield=['measure_file']) + Function( + input_names=["measure_file", "percent"], + output_names=["modified_measure_file"], + function=drop_percent, + as_module=True, + ), + name="dp_%s_%d" % (measure, num_strat), + iterfield=["measure_file"], + ) overlay_drop_percent.inputs.percent = 99.999 - workflow.connect(overlay, out_file, - overlay_drop_percent, 'measure_file') + workflow.connect(overlay, out_file, overlay_drop_percent, "measure_file") - montage = create_montage('montage_%s_%d' % (measure, num_strat), - 'cyan_to_yellow', measure) - - node, out_file_template = strat['template-brain-for-func-derivative'] - workflow.connect(node, out_file_template, - montage, 'inputspec.underlay') + montage = create_montage( + "montage_%s_%d" % (measure, num_strat), "cyan_to_yellow", measure + ) - workflow.connect(overlay_drop_percent, 'modified_measure_file', - montage, 'inputspec.overlay') + node, out_file_template = strat["template-brain-for-func-derivative"] + workflow.connect(node, out_file_template, montage, "inputspec.underlay") - if 'centrality' in measure: + workflow.connect( + overlay_drop_percent, "modified_measure_file", montage, "inputspec.overlay" + ) + + if "centrality" in measure: histogram = pe.MapNode( - Function(input_names=['measure_file', - 'measure'], - output_names=['hist_path'], - function=gen_histogram, - as_module=True), - name='hist_{0}_{1}'.format(measure, num_strat), - iterfield=['measure_file']) + Function( + input_names=["measure_file", "measure"], + output_names=["hist_path"], + function=gen_histogram, + as_module=True, + ), + name="hist_{0}_{1}".format(measure, num_strat), + iterfield=["measure_file"], + ) else: histogram = pe.Node( - Function(input_names=['measure_file', - 'measure'], - output_names=['hist_path'], - function=gen_histogram, - as_module=True), - name='hist_{0}_{1}'.format(measure, num_strat)) + Function( + input_names=["measure_file", "measure"], + output_names=["hist_path"], + function=gen_histogram, + as_module=True, + ), + name="hist_{0}_{1}".format(measure, num_strat), + ) histogram.inputs.measure = measure - workflow.connect(overlay, out_file, - histogram, 'measure_file') + workflow.connect(overlay, out_file, histogram, "measure_file") - strat.update_resource_pool({'qc___%s_a' % measure: (montage, 'outputspec.axial_png'), - 'qc___%s_s' % measure: (montage, 'outputspec.sagittal_png'), - 'qc___%s_hist' % measure: (histogram, 'hist_path')}) + strat.update_resource_pool( + { + "qc___%s_a" % measure: (montage, "outputspec.axial_png"), + "qc___%s_s" % measure: (montage, "outputspec.sagittal_png"), + "qc___%s_hist" % measure: (histogram, "hist_path"), + } + ) - if not idx in qc_montage_id_a: - qc_montage_id_a[idx] = '%s_a' % measure - qc_montage_id_s[idx] = '%s_s' % measure - qc_hist_id[idx] = '%s_hist' % measure + if idx not in qc_montage_id_a: + qc_montage_id_a[idx] = "%s_a" % measure + qc_montage_id_s[idx] = "%s_s" % measure + qc_hist_id[idx] = "%s_hist" % measure - except Exception as e: - print("[!] Connection of QA montages workflow for %s " \ - "has failed.\n" % measure) - print("Error: %s" % e) + except Exception: pass -def create_qc_snr(wf_name='qc_snr'): - +def create_qc_snr(wf_name="qc_snr"): wf = pe.Workflow(name=wf_name) - input_node = pe.Node(util.IdentityInterface(fields=['functional_preprocessed', - 'functional_brain_mask', - 'functional_to_anat_linear_xfm', - 'anatomical_brain', - 'mean_functional_in_anat']), - name='inputspec') - - output_node = pe.Node(util.IdentityInterface(fields=['snr_axial_image', - 'snr_sagittal_image', - 'snr_histogram_image', - 'snr_mean']), - name='outputspec') - - std_dev = pe.Node(afni.TStat(args='-stdev'), - name='std_dev') - - std_dev.inputs.outputtype = 'NIFTI_GZ' - wf.connect(input_node, 'functional_preprocessed', std_dev, 'in_file') - wf.connect(input_node, 'functional_brain_mask', std_dev, 'mask') - - - std_dev_anat = pe.Node(fsl.ApplyWarp(interp='trilinear'), - name='std_dev_anat') - wf.connect(input_node, 'functional_to_anat_linear_xfm', std_dev_anat, 'premat') - wf.connect(std_dev, 'out_file', std_dev_anat, 'in_file') - wf.connect(input_node, 'anatomical_brain', std_dev_anat, 'ref_file') + input_node = pe.Node( + util.IdentityInterface( + fields=[ + "functional_preprocessed", + "functional_brain_mask", + "functional_to_anat_linear_xfm", + "anatomical_brain", + "mean_functional_in_anat", + ] + ), + name="inputspec", + ) - snr = pe.Node(afni.Calc(expr='b/a'), name='snr') - snr.inputs.outputtype = 'NIFTI_GZ' - wf.connect(input_node, 'mean_functional_in_anat', snr, 'in_file_b') - wf.connect(std_dev_anat, 'out_file', snr, 'in_file_a') + output_node = pe.Node( + util.IdentityInterface( + fields=[ + "snr_axial_image", + "snr_sagittal_image", + "snr_histogram_image", + "snr_mean", + ] + ), + name="outputspec", + ) - snr_val = pe.Node(Function(input_names=['measure_file'], - output_names=['snr_storefl'], - function=cal_snr_val, - as_module=True), - name='snr_val') + std_dev = pe.Node(afni.TStat(args="-stdev"), name="std_dev") + + std_dev.inputs.outputtype = "NIFTI_GZ" + wf.connect(input_node, "functional_preprocessed", std_dev, "in_file") + wf.connect(input_node, "functional_brain_mask", std_dev, "mask") + + std_dev_anat = pe.Node(fsl.ApplyWarp(interp="trilinear"), name="std_dev_anat") + wf.connect(input_node, "functional_to_anat_linear_xfm", std_dev_anat, "premat") + wf.connect(std_dev, "out_file", std_dev_anat, "in_file") + wf.connect(input_node, "anatomical_brain", std_dev_anat, "ref_file") + + snr = pe.Node(afni.Calc(expr="b/a"), name="snr") + snr.inputs.outputtype = "NIFTI_GZ" + wf.connect(input_node, "mean_functional_in_anat", snr, "in_file_b") + wf.connect(std_dev_anat, "out_file", snr, "in_file_a") + + snr_val = pe.Node( + Function( + input_names=["measure_file"], + output_names=["snr_storefl"], + function=cal_snr_val, + as_module=True, + ), + name="snr_val", + ) - wf.connect(snr, 'out_file', snr_val, 'measure_file') + wf.connect(snr, "out_file", snr_val, "measure_file") - hist_snr = pe.Node(Function(input_names=['measure_file', 'measure'], - output_names=['hist_path'], - function=gen_histogram, - as_module=True), - name='hist_snr') + hist_snr = pe.Node( + Function( + input_names=["measure_file", "measure"], + output_names=["hist_path"], + function=gen_histogram, + as_module=True, + ), + name="hist_snr", + ) - hist_snr.inputs.measure = 'snr' + hist_snr.inputs.measure = "snr" - wf.connect(snr, 'out_file', hist_snr, 'measure_file') + wf.connect(snr, "out_file", hist_snr, "measure_file") snr_drop_percent = pe.Node( - Function(input_names=['measure_file', - 'percent'], - output_names=['modified_measure_file'], - function=drop_percent, - as_module=True), - name='dp_snr' + Function( + input_names=["measure_file", "percent"], + output_names=["modified_measure_file"], + function=drop_percent, + as_module=True, + ), + name="dp_snr", ) snr_drop_percent.inputs.percent = 99 - wf.connect(snr, 'out_file', snr_drop_percent, 'measure_file') + wf.connect(snr, "out_file", snr_drop_percent, "measure_file") - montage_snr = create_montage('montage_snr', - 'red_to_blue', - 'snr', mapnode=False) + montage_snr = create_montage("montage_snr", "red_to_blue", "snr", mapnode=False) - wf.connect(snr_drop_percent, 'modified_measure_file', montage_snr, 'inputspec.overlay') - wf.connect(input_node, 'anatomical_brain', montage_snr, 'inputspec.underlay') + wf.connect( + snr_drop_percent, "modified_measure_file", montage_snr, "inputspec.overlay" + ) + wf.connect(input_node, "anatomical_brain", montage_snr, "inputspec.underlay") - wf.connect(montage_snr, 'outputspec.axial_png', output_node, 'snr_axial_image') - wf.connect(montage_snr, 'outputspec.sagittal_png', output_node, 'snr_sagittal_image') - wf.connect(hist_snr, 'hist_path', output_node, 'snr_histogram_image') - wf.connect(snr_val, 'snr_storefl', output_node, 'snr_mean') + wf.connect(montage_snr, "outputspec.axial_png", output_node, "snr_axial_image") + wf.connect( + montage_snr, "outputspec.sagittal_png", output_node, "snr_sagittal_image" + ) + wf.connect(hist_snr, "hist_path", output_node, "snr_histogram_image") + wf.connect(snr_val, "snr_storefl", output_node, "snr_mean") return wf -def create_qc_motion(wf_name='qc_motion'): - +def create_qc_motion(wf_name="qc_motion"): wf = pe.Workflow(name=wf_name) - input_node = pe.Node(util.IdentityInterface(fields=['motion_parameters']), - name='inputspec') + input_node = pe.Node( + util.IdentityInterface(fields=["motion_parameters"]), name="inputspec" + ) - output_node = pe.Node(util.IdentityInterface(fields=['motion_translation_plot', - 'motion_rotation_plot']), - name='outputspec') + output_node = pe.Node( + util.IdentityInterface( + fields=["motion_translation_plot", "motion_rotation_plot"] + ), + name="outputspec", + ) - mov_plot = pe.Node(Function(input_names=['motion_parameters'], - output_names=['translation_plot', - 'rotation_plot'], - function=gen_motion_plt, - as_module=True), - name='motion_plot') + mov_plot = pe.Node( + Function( + input_names=["motion_parameters"], + output_names=["translation_plot", "rotation_plot"], + function=gen_motion_plt, + as_module=True, + ), + name="motion_plot", + ) - wf.connect(input_node, 'motion_parameters', mov_plot, 'motion_parameters') - wf.connect(mov_plot, 'translation_plot', output_node, 'motion_translation_plot') - wf.connect(mov_plot, 'rotation_plot', output_node, 'motion_rotation_plot') + wf.connect(input_node, "motion_parameters", mov_plot, "motion_parameters") + wf.connect(mov_plot, "translation_plot", output_node, "motion_translation_plot") + wf.connect(mov_plot, "rotation_plot", output_node, "motion_rotation_plot") return wf -def create_qc_fd(wf_name='qc_fd'): - +def create_qc_fd(wf_name="qc_fd"): wf = pe.Workflow(name=wf_name) - input_node = pe.Node(util.IdentityInterface(fields=['fd', 'excluded_volumes']), - name='inputspec') + input_node = pe.Node( + util.IdentityInterface(fields=["fd", "excluded_volumes"]), name="inputspec" + ) - output_node = pe.Node(util.IdentityInterface(fields=['fd_histogram_plot']), - name='outputspec') + output_node = pe.Node( + util.IdentityInterface(fields=["fd_histogram_plot"]), name="outputspec" + ) - fd_plot = pe.Node(Function(input_names=['arr', - 'measure', - 'ex_vol'], - output_names=['hist_path'], - function=gen_plot_png, - as_module=True), - name='fd_plot') + fd_plot = pe.Node( + Function( + input_names=["arr", "measure", "ex_vol"], + output_names=["hist_path"], + function=gen_plot_png, + as_module=True, + ), + name="fd_plot", + ) - fd_plot.inputs.measure = 'FD' + fd_plot.inputs.measure = "FD" - wf.connect(input_node, 'fd', fd_plot, 'arr') - wf.connect(input_node, 'excluded_volumes', fd_plot, 'ex_vol') - wf.connect(fd_plot, 'hist_path', output_node, 'fd_histogram_plot') + wf.connect(input_node, "fd", fd_plot, "arr") + wf.connect(input_node, "excluded_volumes", fd_plot, "ex_vol") + wf.connect(fd_plot, "hist_path", output_node, "fd_histogram_plot") return wf -def create_qc_carpet(wf_name='qc_carpet', output_image='qc_carpet'): +def create_qc_carpet(wf_name="qc_carpet", output_image="qc_carpet"): wf = pe.Workflow(name=wf_name) - input_node = pe.Node(util.IdentityInterface(fields=['functional_to_standard', - 'mean_functional_to_standard', - 'anatomical_gm_mask', - 'anatomical_wm_mask', - 'anatomical_csf_mask']), - name='inputspec') - - output_node = pe.Node(util.IdentityInterface(fields=['carpet_plot']), - name='outputspec') - - gm_resample = pe.Node(afni.Resample(), - name='gm_resample', - mem_gb=0, - mem_x=(0.0115, 'in_file', 't')) - gm_resample.inputs.outputtype = 'NIFTI' - wf.connect(input_node, 'anatomical_gm_mask', gm_resample, 'in_file') - wf.connect(input_node, 'mean_functional_to_standard', - gm_resample, 'master') + input_node = pe.Node( + util.IdentityInterface( + fields=[ + "functional_to_standard", + "mean_functional_to_standard", + "anatomical_gm_mask", + "anatomical_wm_mask", + "anatomical_csf_mask", + ] + ), + name="inputspec", + ) - gm_mask = pe.Node(afni.Calc(), name="gm_mask") - gm_mask.inputs.expr = 'astep(a, 0.5)' - gm_mask.inputs.outputtype = 'NIFTI' - wf.connect(gm_resample, 'out_file', gm_mask, 'in_file_a') - - wm_resample = pe.Node(afni.Resample(), - name='wm_resample', - mem_gb=0, - mem_x=(0.0115, 'in_file', 't')) - wm_resample.inputs.outputtype = 'NIFTI' - wf.connect(input_node, 'anatomical_wm_mask', wm_resample, 'in_file') - wf.connect(input_node, 'mean_functional_to_standard', - wm_resample, 'master') + output_node = pe.Node( + util.IdentityInterface(fields=["carpet_plot"]), name="outputspec" + ) - wm_mask = pe.Node(afni.Calc(), name="wm_mask") - wm_mask.inputs.expr = 'astep(a, 0.5)' - wm_mask.inputs.outputtype = 'NIFTI' - wf.connect(wm_resample, 'out_file', wm_mask, 'in_file_a') - - csf_resample = pe.Node(afni.Resample(), - name='csf_resample', - mem_gb=0, - mem_x=(0.0115, 'in_file', 't')) - csf_resample.inputs.outputtype = 'NIFTI' - wf.connect(input_node, 'anatomical_csf_mask', csf_resample, 'in_file') - wf.connect(input_node, 'mean_functional_to_standard', - csf_resample, 'master') + gm_resample = pe.Node( + afni.Resample(), name="gm_resample", mem_gb=0, mem_x=(0.0115, "in_file", "t") + ) + gm_resample.inputs.outputtype = "NIFTI" + wf.connect(input_node, "anatomical_gm_mask", gm_resample, "in_file") + wf.connect(input_node, "mean_functional_to_standard", gm_resample, "master") - csf_mask = pe.Node(afni.Calc(), name="csf_mask") - csf_mask.inputs.expr = 'astep(a, 0.5)' - csf_mask.inputs.outputtype = 'NIFTI' - wf.connect(csf_resample, 'out_file', csf_mask, 'in_file_a') + gm_mask = pe.Node(afni.Calc(), name="gm_mask") + gm_mask.inputs.expr = "astep(a, 0.5)" + gm_mask.inputs.outputtype = "NIFTI" + wf.connect(gm_resample, "out_file", gm_mask, "in_file_a") + wm_resample = pe.Node( + afni.Resample(), name="wm_resample", mem_gb=0, mem_x=(0.0115, "in_file", "t") + ) + wm_resample.inputs.outputtype = "NIFTI" + wf.connect(input_node, "anatomical_wm_mask", wm_resample, "in_file") + wf.connect(input_node, "mean_functional_to_standard", wm_resample, "master") + wm_mask = pe.Node(afni.Calc(), name="wm_mask") + wm_mask.inputs.expr = "astep(a, 0.5)" + wm_mask.inputs.outputtype = "NIFTI" + wf.connect(wm_resample, "out_file", wm_mask, "in_file_a") - carpet_plot = pe.Node(Function(input_names=['gm_mask', - 'wm_mask', - 'csf_mask', - 'functional_to_standard', - 'output'], - output_names=['carpet_plot'], - function=gen_carpet_plt, - as_module=True), - name='carpet_plot', mem_gb=4.0) + csf_resample = pe.Node( + afni.Resample(), name="csf_resample", mem_gb=0, mem_x=(0.0115, "in_file", "t") + ) + csf_resample.inputs.outputtype = "NIFTI" + wf.connect(input_node, "anatomical_csf_mask", csf_resample, "in_file") + wf.connect(input_node, "mean_functional_to_standard", csf_resample, "master") + + csf_mask = pe.Node(afni.Calc(), name="csf_mask") + csf_mask.inputs.expr = "astep(a, 0.5)" + csf_mask.inputs.outputtype = "NIFTI" + wf.connect(csf_resample, "out_file", csf_mask, "in_file_a") + + carpet_plot = pe.Node( + Function( + input_names=[ + "gm_mask", + "wm_mask", + "csf_mask", + "functional_to_standard", + "output", + ], + output_names=["carpet_plot"], + function=gen_carpet_plt, + as_module=True, + ), + name="carpet_plot", + mem_gb=4.0, + ) carpet_plot.inputs.output = output_image - wf.connect(gm_mask, 'out_file', carpet_plot, 'gm_mask') - wf.connect(wm_mask, 'out_file', carpet_plot, 'wm_mask') - wf.connect(csf_mask, 'out_file', carpet_plot, 'csf_mask') - wf.connect(input_node, 'functional_to_standard', carpet_plot, 'functional_to_standard') - wf.connect(carpet_plot, 'carpet_plot', output_node, 'carpet_plot') + wf.connect(gm_mask, "out_file", carpet_plot, "gm_mask") + wf.connect(wm_mask, "out_file", carpet_plot, "wm_mask") + wf.connect(csf_mask, "out_file", carpet_plot, "csf_mask") + wf.connect( + input_node, "functional_to_standard", carpet_plot, "functional_to_standard" + ) + wf.connect(carpet_plot, "carpet_plot", output_node, "carpet_plot") return wf @@ -480,35 +571,39 @@ def afni_Edge3(in_file): comm = "3dedge3 -input %s -prefix skull_edge.nii.gz" % in_file subprocess.getoutput(comm) - return os.path.abspath('./skull_edge.nii.gz') - + return os.path.abspath("./skull_edge.nii.gz") -def create_qc_skullstrip(wf_name='qc_skullstrip'): +def create_qc_skullstrip(wf_name="qc_skullstrip"): wf = pe.Workflow(name=wf_name) - input_node = pe.Node(util.IdentityInterface(fields=['anatomical_brain', - 'anatomical_reorient']), - name='inputspec') - - output_node = pe.Node(util.IdentityInterface(fields=['axial_image', - 'sagittal_image']), - name='outputspec') - - skull_edge = pe.Node(Function(input_names=['in_file'], - output_names=['out_file'], - function=afni_Edge3, - as_module=True), - name='skull_edge') - - montage_skull = create_montage('montage_skull', 'red', 'skull_vis', - mapnode=False) - - wf.connect(input_node, 'anatomical_reorient', skull_edge, 'in_file') - wf.connect(input_node, 'anatomical_brain', montage_skull, 'inputspec.underlay') - wf.connect(skull_edge, 'out_file', montage_skull, 'inputspec.overlay') - - wf.connect(montage_skull, 'outputspec.axial_png', output_node, 'axial_image') - wf.connect(montage_skull, 'outputspec.sagittal_png', output_node, 'sagittal_image') + input_node = pe.Node( + util.IdentityInterface(fields=["anatomical_brain", "anatomical_reorient"]), + name="inputspec", + ) + + output_node = pe.Node( + util.IdentityInterface(fields=["axial_image", "sagittal_image"]), + name="outputspec", + ) + + skull_edge = pe.Node( + Function( + input_names=["in_file"], + output_names=["out_file"], + function=afni_Edge3, + as_module=True, + ), + name="skull_edge", + ) + + montage_skull = create_montage("montage_skull", "red", "skull_vis", mapnode=False) + + wf.connect(input_node, "anatomical_reorient", skull_edge, "in_file") + wf.connect(input_node, "anatomical_brain", montage_skull, "inputspec.underlay") + wf.connect(skull_edge, "out_file", montage_skull, "inputspec.overlay") + + wf.connect(montage_skull, "outputspec.axial_png", output_node, "axial_image") + wf.connect(montage_skull, "outputspec.sagittal_png", output_node, "sagittal_image") return wf diff --git a/CPAC/qc/qcmetrics.py b/CPAC/qc/qcmetrics.py index f1c68d2f67..cdbc4a8875 100644 --- a/CPAC/qc/qcmetrics.py +++ b/CPAC/qc/qcmetrics.py @@ -1,22 +1,23 @@ -"""QC metrics from XCP-D v0.0.9 +"""QC metrics from XCP-D v0.0.9. Ref: https://github.com/PennLINC/xcp_d/tree/0.0.9 """ # pylint: disable=invalid-name, redefined-outer-name -import nibabel as nb import numpy as np +import nibabel as nib def regisQ(bold2t1w_mask, t1w_mask, bold2template_mask, template_mask): - reg_qc = {'coregDice': [dc(bold2t1w_mask, t1w_mask)], - 'coregJaccard': [jc(bold2t1w_mask, t1w_mask)], - 'coregCrossCorr': [crosscorr(bold2t1w_mask, t1w_mask)], - 'coregCoverage': [coverage(bold2t1w_mask, t1w_mask)], - 'normDice': [dc(bold2template_mask, template_mask)], - 'normJaccard': [jc(bold2template_mask, template_mask)], - 'normCrossCorr': [crosscorr(bold2template_mask, template_mask)], - 'normCoverage': [coverage(bold2template_mask, template_mask)]} - return reg_qc + return { + "coregDice": [dc(bold2t1w_mask, t1w_mask)], + "coregJaccard": [jc(bold2t1w_mask, t1w_mask)], + "coregCrossCorr": [crosscorr(bold2t1w_mask, t1w_mask)], + "coregCoverage": [coverage(bold2t1w_mask, t1w_mask)], + "normDice": [dc(bold2template_mask, template_mask)], + "normJaccard": [jc(bold2template_mask, template_mask)], + "normCrossCorr": [crosscorr(bold2template_mask, template_mask)], + "normCoverage": [coverage(bold2template_mask, template_mask)], + } def dc(input1, input2): @@ -24,7 +25,7 @@ def dc(input1, input2): Dice coefficient Computes the Dice coefficient (also known as Sorensen index) between the binary objects in two images. - The metric is defined as + The metric is defined as. .. math:: DC=\frac{2|A\cap B|}{|A|+|B|} @@ -53,8 +54,8 @@ def dc(input1, input2): ----- This is a real metric. """ - input1 = nb.load(input1).get_fdata() - input2 = nb.load(input2).get_fdata() + input1 = nib.load(input1).get_fdata() + input2 = nib.load(input2).get_fdata() input1 = np.atleast_1d(input1.astype(bool)) input2 = np.atleast_1d(input2.astype(bool)) @@ -64,7 +65,7 @@ def dc(input1, input2): size_i2 = np.count_nonzero(input2) try: - dc = 2. * intersection / float(size_i1 + size_i2) + dc = 2.0 * intersection / float(size_i1 + size_i2) except ZeroDivisionError: dc = 0.0 @@ -75,7 +76,8 @@ def jc(input1, input2): r""" Jaccard coefficient Computes the Jaccard coefficient between the binary objects in two images. - Parameters + + Parameters. ---------- input1: array_like Input data containing objects. Can be any type but will be @@ -83,43 +85,42 @@ def jc(input1, input2): input2: array_like Input data containing objects. Can be any type but will be converted into binary: background where 0, object everywhere else. + Returns ------- jc: float The Jaccard coefficient between the object(s) in `input1` and the object(s) in `input2`. It ranges from 0 (no overlap) to 1 (perfect overlap). + Notes ----- This is a real metric. """ - input1 = nb.load(input1).get_fdata() - input2 = nb.load(input2).get_fdata() + input1 = nib.load(input1).get_fdata() + input2 = nib.load(input2).get_fdata() input1 = np.atleast_1d(input1.astype(bool)) input2 = np.atleast_1d(input2.astype(bool)) intersection = np.count_nonzero(input1 & input2) union = np.count_nonzero(input1 | input2) - jc = float(intersection) / float(union) - - return jc + return float(intersection) / float(union) def crosscorr(input1, input2): - r"""cross correlation: compute cross correction bewteen input masks""" - input1 = nb.load(input1).get_fdata() - input2 = nb.load(input2).get_fdata() + r"""Cross correlation: compute cross correction bewteen input masks.""" + input1 = nib.load(input1).get_fdata() + input2 = nib.load(input2).get_fdata() input1 = np.atleast_1d(input1.astype(bool)).flatten() input2 = np.atleast_1d(input2.astype(bool)).flatten() - cc = np.corrcoef(input1, input2)[0][1] - return cc + return np.corrcoef(input1, input2)[0][1] def coverage(input1, input2): """Estimate the coverage between two masks.""" - input1 = nb.load(input1).get_fdata() - input2 = nb.load(input2).get_fdata() + input1 = nib.load(input1).get_fdata() + input2 = nib.load(input2).get_fdata() input1 = np.atleast_1d(input1.astype(bool)) input2 = np.atleast_1d(input2.astype(bool)) intsec = np.count_nonzero(input1 & input2) @@ -127,5 +128,4 @@ def coverage(input1, input2): smallv = np.sum(input2) else: smallv = np.sum(input1) - cov = float(intsec)/float(smallv) - return cov + return float(intsec) / float(smallv) diff --git a/CPAC/qc/tests/test_qc.py b/CPAC/qc/tests/test_qc.py index 0abf56132d..7382c75142 100644 --- a/CPAC/qc/tests/test_qc.py +++ b/CPAC/qc/tests/test_qc.py @@ -1,39 +1,35 @@ import os + import pytest from nipype.interfaces import utility as util + from CPAC.pipeline import nipype_pipeline_engine as pe from CPAC.pipeline.cpac_group_runner import gather_outputs from CPAC.qc.pipeline import create_qc_workflow -from CPAC.qc.utils import generate_qc_pages from CPAC.utils.configuration import Configuration from CPAC.utils.outputs import Outputs from CPAC.utils.strategy import Strategy def file_node(path): - input_node = pe.Node( - util.IdentityInterface(fields=['file']), name='inputspec' - ) + input_node = pe.Node(util.IdentityInterface(fields=["file"]), name="inputspec") input_node.inputs.file = path - return input_node, 'file' + return input_node, "file" -@pytest.mark.skip(reason='needs refactoring') +@pytest.mark.skip(reason="needs refactoring") def test_qc(): - outputs = Outputs() - c = Configuration({ - "workingDirectory": "", - "crashLogDirectory": "", - "outputDirectory":"" - }) + c = Configuration( + {"workingDirectory": "", "crashLogDirectory": "", "outputDirectory": ""} + ) - workflow = pe.Workflow(name='workflow_name') + workflow = pe.Workflow(name="workflow_name") workflow.base_dir = c.workingDirectory - workflow.config['execution'] = { - 'hash_method': 'timestamp', - 'crashdump_dir': os.path.abspath(c.crashLogDirectory) + workflow.config["execution"] = { + "hash_method": "timestamp", + "crashdump_dir": os.path.abspath(c.crashLogDirectory), } strat_initial = Strategy() @@ -65,13 +61,12 @@ def test_qc(): "frame_wise_displacement_power", "frame_wise_displacement_jenkinson", ], - exts=['nii', 'nii.gz', '1D', 'mat', 'txt'] + exts=["nii", "nii.gz", "1D", "mat", "txt"], ) for (resource, _), df in output_df_dct.items(): - strat_initial.update_resource_pool({ - resource: file_node(df.Filepath[0]) - }) + strat_initial.update_resource_pool({resource: file_node(df.Filepath[0])}) - qc_montage_id_a, qc_montage_id_s, qc_hist_id, qc_plot_id = \ - create_qc_workflow(workflow, c, strat_list, outputs.qc) + qc_montage_id_a, qc_montage_id_s, qc_hist_id, qc_plot_id = create_qc_workflow( + workflow, c, strat_list, outputs.qc + ) diff --git a/CPAC/qc/utils.py b/CPAC/qc/utils.py index 8d3682ff72..818f6d283e 100644 --- a/CPAC/qc/utils.py +++ b/CPAC/qc/utils.py @@ -1,24 +1,17 @@ import os import subprocess -import pkg_resources as p -import numpy as np -import nibabel as nb -import numpy.ma as ma +import matplotlib as mpl import numpy +import numpy as np +from numpy import ma +import pkg_resources as p +import nibabel as nib -import matplotlib -matplotlib.use('Agg') -from matplotlib import pyplot as plt - -import matplotlib.cm as cm -from matplotlib import gridspec as mgs +mpl.use("Agg") +from matplotlib import cm, gridspec as mgs, pyplot as plt from matplotlib.colors import ListedColormap -from nipype.interfaces import afni -from CPAC.pipeline import nipype_pipeline_engine as pe -import nipype.interfaces.utility as util - def generate_qc_pages(qc_dir): """Generates the QC HTML files populated with the QC images that were @@ -36,30 +29,25 @@ def generate_qc_pages(qc_dir): None """ - qc_dir = os.path.abspath(qc_dir) try: if not os.path.exists(qc_dir): os.makedirs(qc_dir) except IOError: - print("\n\n[!] Could not create a directory for the QC dashboard. " - "Please check write permissions.\n\nDirectory attempted:\n " - "{0}".format(qc_dir)) raise IOError files = [] for root, _, fs in os.walk(qc_dir): - root = root[len(qc_dir) + 1:] + root = root[len(qc_dir) + 1 :] files += [os.path.join(root, f) for f in fs] - with open(p.resource_filename('CPAC.qc', 'data/index.html'), 'rb') as f: + with open(p.resource_filename("CPAC.qc", "data/index.html"), "rb") as f: qc_content = f.read() qc_content = qc_content.replace( - b'/*CPAC*/``/*CPAC*/', - ('`' + '\n'.join(files) + '`').encode() + b"/*CPAC*/``/*CPAC*/", ("`" + "\n".join(files) + "`").encode() ) - with open(os.path.join(qc_dir, 'index.html'), 'wb') as f: + with open(os.path.join(qc_dir, "index.html"), "wb") as f: f.write(qc_content) @@ -77,15 +65,14 @@ def cal_snr_val(measure_file): a text file store average snr value """ - - data = nb.load(measure_file).get_fdata() + data = nib.load(measure_file).get_fdata() data_flat = data.flatten() data_no0 = data_flat[data_flat > 0] snr_val = ma.mean(data_no0) - avg_snr_file = os.path.join(os.getcwd(), 'average_snr_file.txt') - with open(avg_snr_file, 'w') as f: - f.write(str(snr_val) + '\n') + avg_snr_file = os.path.join(os.getcwd(), "average_snr_file.txt") + with open(avg_snr_file, "w") as f: + f.write(str(snr_val) + "\n") return avg_snr_file @@ -109,14 +96,13 @@ def gen_plot_png(arr, measure, ex_vol=None): png_name : string path to the generated plot png """ - - matplotlib.rcParams.update({'font.size': 8}) + mpl.rcParams.update({"font.size": 8}) arr = np.loadtxt(arr) if ex_vol: try: - ex_vol = np.genfromtxt(ex_vol, delimiter=',', dtype=int) + ex_vol = np.genfromtxt(ex_vol, delimiter=",", dtype=int) ex_vol = ex_vol[ex_vol > 0] except: ex_vol = [] @@ -129,44 +115,46 @@ def gen_plot_png(arr, measure, ex_vol=None): ex_vol = np.array(del_el) fig = plt.figure(figsize=(10, 6)) - plt.plot([i for i in range(len(arr))], arr, '-') - fig.suptitle('%s plot with Mean %s = %0.4f' % (measure, measure, - arr.mean())) - if measure == 'FD' and len(ex_vol) > 0: - + plt.plot(list(range(len(arr))), arr, "-") + fig.suptitle("%s plot with Mean %s = %0.4f" % (measure, measure, arr.mean())) + if measure == "FD" and len(ex_vol) > 0: plt.scatter(ex_vol, arr[ex_vol], c="red", zorder=2) for x in ex_vol: - plt.annotate('( %d , %0.3f)' % (x, arr[x]), xy=(x, arr[x]), - arrowprops=dict(facecolor='black', shrink=0.0)) - - plt.xlabel('Volumes') - plt.ylabel('%s' % measure) - png_name = os.path.join(os.getcwd(), '%s_plot.png' % measure) + plt.annotate( + "( %d , %0.3f)" % (x, arr[x]), + xy=(x, arr[x]), + arrowprops={"facecolor": "black", "shrink": 0.0}, + ) + + plt.xlabel("Volumes") + plt.ylabel("%s" % measure) + png_name = os.path.join(os.getcwd(), "%s_plot.png" % measure) fig.savefig(os.path.join(os.getcwd(), png_name)) plt.close() - matplotlib.rcdefaults() + mpl.rcdefaults() return png_name def gen_carpet_plt(gm_mask, wm_mask, csf_mask, functional_to_standard, output): - size = (950, 800) - carpet_plot_path = os.path.join(os.getcwd(), output + '.png') + carpet_plot_path = os.path.join(os.getcwd(), output + ".png") - func = nb.load(functional_to_standard).get_fdata() - gm_voxels = func[nb.load(gm_mask).get_fdata().astype(bool)] - wm_voxels = func[nb.load(wm_mask).get_fdata().astype(bool)] - csf_voxels = func[nb.load(csf_mask).get_fdata().astype(bool)] + func = nib.load(functional_to_standard).get_fdata() + gm_voxels = func[nib.load(gm_mask).get_fdata().astype(bool)] + wm_voxels = func[nib.load(wm_mask).get_fdata().astype(bool)] + csf_voxels = func[nib.load(csf_mask).get_fdata().astype(bool)] del func data = np.concatenate((gm_voxels, wm_voxels, csf_voxels)) - seg = np.concatenate(( - np.ones(gm_voxels.shape[0]) * 1, - np.ones(wm_voxels.shape[0]) * 2, - np.ones(csf_voxels.shape[0]) * 3 - )) + seg = np.concatenate( + ( + np.ones(gm_voxels.shape[0]) * 1, + np.ones(wm_voxels.shape[0]) * 2, + np.ones(csf_voxels.shape[0]) * 3, + ) + ) p_dec = 1 + data.shape[0] // size[0] if p_dec: @@ -180,53 +168,56 @@ def gen_carpet_plt(gm_mask, wm_mask, csf_mask, functional_to_standard, output): interval = max((int(data.shape[-1] + 1) // 10, int(data.shape[-1] + 1) // 5, 1)) xticks = list(range(0, data.shape[-1])[::interval]) - mycolors = ListedColormap(cm.get_cmap('tab10').colors[:4][::-1]) + mycolors = ListedColormap(cm.get_cmap("tab10").colors[:4][::-1]) - gs = mgs.GridSpecFromSubplotSpec(1, 2, subplot_spec=mgs.GridSpec(1, 1)[0], - width_ratios=[1, 100], - wspace=0.0) + gs = mgs.GridSpecFromSubplotSpec( + 1, 2, subplot_spec=mgs.GridSpec(1, 1)[0], width_ratios=[1, 100], wspace=0.0 + ) ax0 = plt.subplot(gs[0]) ax0.set_yticks([]) ax0.set_xticks([]) - ax0.imshow(seg[:, np.newaxis], interpolation='none', aspect='auto', - cmap=mycolors, vmin=1, vmax=4) + ax0.imshow( + seg[:, np.newaxis], + interpolation="none", + aspect="auto", + cmap=mycolors, + vmin=1, + vmax=4, + ) ax0.grid(False) ax0.spines["left"].set_visible(False) ax0.spines["top"].set_visible(False) ax1 = plt.subplot(gs[1]) - ax1.imshow(data, interpolation='nearest', aspect='auto', cmap='gray') + ax1.imshow(data, interpolation="nearest", aspect="auto", cmap="gray") ax1.grid(False) ax1.set_yticks([]) ax1.set_yticklabels([]) ax1.set_xticks(xticks) - ax1.set_xlabel('time (frames)') + ax1.set_xlabel("time (frames)") ax1.spines["top"].set_visible(False) ax1.spines["right"].set_visible(False) - ax1.yaxis.set_ticks_position('left') - ax1.xaxis.set_ticks_position('bottom') + ax1.yaxis.set_ticks_position("left") + ax1.xaxis.set_ticks_position("bottom") - plt.savefig(carpet_plot_path, dpi=200, bbox_inches='tight') + plt.savefig(carpet_plot_path, dpi=200, bbox_inches="tight") plt.close() return carpet_plot_path def gen_motion_plt(motion_parameters): - """ Function to Generate Matplotlib plot for motion. Separate plots for Translation and Rotation are generated. Parameters ---------- - motion_parameters : string Motion Parameters file Returns ------- - translation_plot : string path to translation plot @@ -234,29 +225,28 @@ def gen_motion_plt(motion_parameters): path to rotation plot """ - - rotation_plot = os.path.join(os.getcwd(), 'motion_rot_plot.png') - translation_plot = os.path.join(os.getcwd(), 'motion_trans_plot.png') + rotation_plot = os.path.join(os.getcwd(), "motion_rot_plot.png") + translation_plot = os.path.join(os.getcwd(), "motion_trans_plot.png") data = np.loadtxt(motion_parameters).T - plt.gca().set_prop_cycle(color=['red', 'green', 'blue']) + plt.gca().set_prop_cycle(color=["red", "green", "blue"]) plt.plot(data[0]) plt.plot(data[1]) plt.plot(data[2]) - plt.legend(['roll', 'pitch', 'yaw'], loc='upper right') - plt.ylabel('Rotation (degrees)') - plt.xlabel('Volume') + plt.legend(["roll", "pitch", "yaw"], loc="upper right") + plt.ylabel("Rotation (degrees)") + plt.xlabel("Volume") plt.savefig(rotation_plot) plt.close() - plt.gca().set_prop_cycle(color=['red', 'green', 'blue']) + plt.gca().set_prop_cycle(color=["red", "green", "blue"]) plt.plot(data[3]) plt.plot(data[4]) plt.plot(data[5]) - plt.legend(['x', 'y', 'z'], loc='upper right') - plt.ylabel('Translation (mm)') - plt.xlabel('Volume') + plt.legend(["x", "y", "z"], loc="upper right") + plt.ylabel("Translation (mm)") + plt.xlabel("Volume") plt.savefig(translation_plot) plt.close() @@ -282,41 +272,56 @@ def gen_histogram(measure_file, measure): """ hist_path = None - from CPAC.qc.utils import make_histogram import os + + from CPAC.qc.utils import make_histogram + m_ = measure if isinstance(measure_file, list): hist_path = [] for file_ in measure_file: measure = m_ - if 'sca_roi' in measure.lower(): - fname = os.path.basename(os.path.splitext(os.path.splitext(file_)[0])[0]) - if 'ROI_' in fname: - fname = fname.rsplit('ROI_')[1] - elif 'roi_' in fname: - fname = fname.rsplit('roi_')[1] - fname = 'sca_roi_' + fname.split('_')[0] + if "sca_roi" in measure.lower(): + fname = os.path.basename( + os.path.splitext(os.path.splitext(file_)[0])[0] + ) + if "ROI_" in fname: + fname = fname.rsplit("ROI_")[1] + elif "roi_" in fname: + fname = fname.rsplit("roi_")[1] + fname = "sca_roi_" + fname.split("_")[0] measure = fname - if 'sca_tempreg' in measure.lower(): - fname = os.path.basename(os.path.splitext(os.path.splitext(file_)[0])[0]) - fname = fname.split('z_maps_roi_')[1] - fname = 'sca_mult_regression_maps_roi_' + fname.split('_')[0] + if "sca_tempreg" in measure.lower(): + fname = os.path.basename( + os.path.splitext(os.path.splitext(file_)[0])[0] + ) + fname = fname.split("z_maps_roi_")[1] + fname = "sca_mult_regression_maps_roi_" + fname.split("_")[0] measure = fname - if 'dr_tempreg' in measure.lower(): - fname = os.path.basename(os.path.splitext(os.path.splitext(file_)[0])[0]) - for i in ['temp_reg_map_', 'tempreg_map_', 'tempreg_maps_', 'temp_reg_maps_']: + if "dr_tempreg" in measure.lower(): + fname = os.path.basename( + os.path.splitext(os.path.splitext(file_)[0])[0] + ) + for i in [ + "temp_reg_map_", + "tempreg_map_", + "tempreg_maps_", + "temp_reg_maps_", + ]: if i in fname: try: fname = fname.rsplit(i)[1] break except IndexError: continue - fname = 'dual_regression_map_'+ fname.split('_')[0] + fname = "dual_regression_map_" + fname.split("_")[0] measure = fname - if 'centrality' in measure.lower(): - fname = os.path.basename(os.path.splitext(os.path.splitext(file_)[0])[0]) - type_, fname = fname.split('centrality_') - fname = type_ + 'centrality_' + fname.split('_')[0] + if "centrality" in measure.lower(): + fname = os.path.basename( + os.path.splitext(os.path.splitext(file_)[0])[0] + ) + type_, fname = fname.split("centrality_") + fname = type_ + "centrality_" + fname.split("_")[0] measure = fname hist_path.append(make_histogram(file_, measure)) @@ -328,14 +333,12 @@ def gen_histogram(measure_file, measure): def make_histogram(measure_file, measure): - """ Generates Histogram Image of intensities for a given input nifti file. Parameters ---------- - measure_file : string path to input nifti file @@ -347,36 +350,31 @@ def make_histogram(measure_file, measure): Returns ------- - hist_path : string Path to the generated histogram png """ + import os - import matplotlib from matplotlib import pyplot as plt import numpy as np - import nibabel as nb - import os + import nibabel as nib - data = nb.load(measure_file).get_fdata() - data_flat = data.flatten(order='F') + data = nib.load(measure_file).get_fdata() + data_flat = data.flatten(order="F") y, binEdges = np.histogram( - data_flat[ - (np.isfinite(data_flat)) & (data_flat != 0) - ], - bins=100 + data_flat[(np.isfinite(data_flat)) & (data_flat != 0)], bins=100 ) - bincenters = 0.5*(binEdges[1:]+binEdges[:-1]) + bincenters = 0.5 * (binEdges[1:] + binEdges[:-1]) fig = plt.figure() - fig.suptitle('%s intensity plot' % measure) - plt.plot(bincenters, y, '-') - plt.xlabel('intensity') - plt.ylabel('# of voxels') + fig.suptitle("%s intensity plot" % measure) + plt.plot(bincenters, y, "-") + plt.xlabel("intensity") + plt.ylabel("# of voxels") - png_name = os.path.join(os.getcwd(), '%s_hist_plot.png' % measure) + png_name = os.path.join(os.getcwd(), "%s_hist_plot.png" % measure) fig.savefig(os.path.join(os.getcwd(), png_name)) plt.close() @@ -398,11 +396,10 @@ def make_histogram(measure_file, measure): def drop_percent(measure_file, percent): """ Zeros out voxels in measure files whose intensity doesnt fall in percent - of voxel intensities + of voxel intensities. Parameters ---------- - measure_file : string Input nifti file @@ -411,47 +408,41 @@ def drop_percent(measure_file, percent): Returns ------- - modified_measure_file : string measure_file with 1 - percent voxels zeroed out """ - import os - import nibabel as nb + import numpy as np + import nibabel as nib - img = nb.load(measure_file) + img = nib.load(measure_file) data = img.get_fdata() max_val = np.percentile(data[data != 0.0], percent) data[data >= max_val] = 0.0 - save_img = nb.Nifti1Image(data, header=img.header, affine=img.affine) + save_img = nib.Nifti1Image(data, header=img.header, affine=img.affine) - if '.nii.gz' in measure_file: - ext = '.nii.gz' + if ".nii.gz" in measure_file: + ext = ".nii.gz" else: - ext = '.nii' + ext = ".nii" f_name = os.path.basename(os.path.splitext(os.path.splitext(measure_file)[0])[0]) - saved_name = '%s_%d_%s' % (f_name, percent, ext) + saved_name = "%s_%d_%s" % (f_name, percent, ext) save_img.to_filename(saved_name) - modified_measure_file = os.path.join(os.getcwd(), - saved_name) - - return modified_measure_file + return os.path.join(os.getcwd(), saved_name) def get_spacing(across, down, dimension): - """ Get Spacing in slices to be selected for montage - display varying in given dimension + display varying in given dimension. Parameters ---------- - across : integer # images placed horizontally in montage @@ -461,28 +452,25 @@ def get_spacing(across, down, dimension): Returns ------- - space : integer # of images to skip before displaying next one """ - space = 10 - prod = (across*down*space) + prod = across * down * space if prod > dimension: - while(across*down*space) > dimension: + while (across * down * space) > dimension: space -= 1 else: - while(across*down*space) < dimension: + while (across * down * space) < dimension: space += 1 return space def determine_start_and_end(data, direction, percent): - """ Determine start slice and end slice in data file in given direction with at least threshold percent of voxels @@ -490,7 +478,6 @@ def determine_start_and_end(data, direction, percent): Parameters ---------- - data : string input nifti file @@ -503,7 +490,6 @@ def determine_start_and_end(data, direction, percent): Returns ------- - start : integer Index of starting slice @@ -511,7 +497,6 @@ def determine_start_and_end(data, direction, percent): Index of the last slice """ - x, y, z = data.shape xx1 = 0 @@ -523,37 +508,33 @@ def determine_start_and_end(data, direction, percent): start = None end = None - if 'axial' in direction: - - while(zz2 > 0): - + if "axial" in direction: + while zz2 > 0: d = len(np.nonzero(data[:, :, zz2].flatten())[0]) if float(d) > thresh: break zz2 -= 1 - while(zz1 < zz2): - + while zz1 < zz2: d = len(np.nonzero(data[:, :, zz1].flatten())[0]) if float(d) > thresh: break zz1 += 1 - start = zz1 + start = zz1 end = zz2 else: - while(xx2 > 0): + while xx2 > 0: d = len(np.nonzero(data[xx2, :, :].flatten())[0]) if float(d) > thresh: break xx2 -= 1 - while(xx1 < xx2): - + while xx1 < xx2: d = len(np.nonzero(data[xx1, :, :].flatten())[0]) if float(d) > thresh: break @@ -572,7 +553,6 @@ def montage_axial(overlay, underlay, png_name, cbar_name): Parameters ---------- - overlay : string Nifi file @@ -587,33 +567,29 @@ def montage_axial(overlay, underlay, png_name, cbar_name): Returns ------- - png_name : Path to generated PNG """ - pngs = None if isinstance(overlay, list): pngs = [] for ov in overlay: fname = os.path.basename(os.path.splitext(os.path.splitext(ov)[0])[0]) - pngs.append(make_montage_axial(ov, underlay, - fname + '_' + png_name, cbar_name)) + pngs.append( + make_montage_axial(ov, underlay, fname + "_" + png_name, cbar_name) + ) else: pngs = make_montage_axial(overlay, underlay, png_name, cbar_name) - png_name = pngs - - return png_name + return pngs def make_montage_axial(overlay, underlay, png_name, cbar_name): """ - Draws Montage using overlay on Anatomical brain in Axial Direction + Draws Montage using overlay on Anatomical brain in Axial Direction. Parameters ---------- - overlay : string Nifi file @@ -628,61 +604,73 @@ def make_montage_axial(overlay, underlay, png_name, cbar_name): Returns ------- - png_name : Path to generated PNG """ import os - import matplotlib - matplotlib.rcParams.update({'font.size': 5}) - import matplotlib.cm as cm - from mpl_toolkits.axes_grid1 import ImageGrid + + import matplotlib as mpl + + mpl.rcParams.update({"font.size": 5}) + from matplotlib import cm import matplotlib.pyplot as plt - import nibabel as nb + from mpl_toolkits.axes_grid1 import ImageGrid import numpy as np + import nibabel as nib - Y = nb.load(underlay).get_fdata() - X = nb.load(overlay).get_fdata() + Y = nib.load(underlay).get_fdata() + X = nib.load(overlay).get_fdata() X = X.astype(np.float32) Y = Y.astype(np.float32) - if 'skull_vis' in png_name: + if "skull_vis" in png_name: X[X < 20.0] = 0.0 - if 'skull_vis' in png_name or \ - 't1_edge_on_mean_func_in_t1' in png_name or \ - 'MNI_edge_on_mean_func_mni' in png_name: + if ( + "skull_vis" in png_name + or "t1_edge_on_mean_func_in_t1" in png_name + or "MNI_edge_on_mean_func_mni" in png_name + ): max_ = np.nanmax(np.abs(X.flatten())) X[X != 0.0] = max_ - z1, z2 = determine_start_and_end(Y, 'axial', 0.0001) + z1, z2 = determine_start_and_end(Y, "axial", 0.0001) spacing = get_spacing(6, 3, z2 - z1) x, y, z = Y.shape fig = plt.figure(1) max_ = np.max(np.abs(Y)) - if ('snr' in png_name) or ('reho' in png_name) or \ - ('vmhc' in png_name) or ('sca_' in png_name) or \ - ('alff' in png_name) or ('centrality' in png_name) or \ - ('dr_tempreg' in png_name): - grid = ImageGrid(fig, 111, nrows_ncols=(3, 6), share_all=True, - aspect=True, cbar_mode="single", cbar_pad=0.2, - direction="row") + if ( + ("snr" in png_name) + or ("reho" in png_name) + or ("vmhc" in png_name) + or ("sca_" in png_name) + or ("alff" in png_name) + or ("centrality" in png_name) + or ("dr_tempreg" in png_name) + ): + grid = ImageGrid( + fig, + 111, + nrows_ncols=(3, 6), + share_all=True, + aspect=True, + cbar_mode="single", + cbar_pad=0.2, + direction="row", + ) else: - grid = ImageGrid(fig, 111, nrows_ncols=(3, 6), share_all=True, - aspect=True, direction="row") + grid = ImageGrid( + fig, 111, nrows_ncols=(3, 6), share_all=True, aspect=True, direction="row" + ) zz = z1 - for i in range(6*3): + for i in range(6 * 3): if zz >= z2: break try: im = grid[i].imshow(np.rot90(Y[:, :, zz]), cmap=cm.Greys_r) - except IndexError as e: + except IndexError: # TODO: send this to the logger instead - print("\n[!] QC Interface: Had a problem with creating the " - "axial montage for {0}\n\nDetails:{1}. This error might occur because of a registration error encountered while using ANTs.\ - Please refer to the png image located in your working directory for more insight." - "\n".format(png_name, e)) pass zz += spacing @@ -692,29 +680,37 @@ def make_montage_axial(overlay, underlay, png_name, cbar_name): zz = z1 im = None - for i in range(6*3): + for i in range(6 * 3): if zz >= z2: break try: - if cbar_name == 'red_to_blue': - im = grid[i].imshow(np.rot90(X[:, :, zz]), - cmap=cm.get_cmap(cbar_name), alpha=0.82, - vmin=0, vmax=max_) - elif cbar_name == 'green': - im = grid[i].imshow(np.rot90(X[:, :, zz]), - cmap=cm.get_cmap(cbar_name), alpha=0.82, - vmin=0, vmax=max_) + if cbar_name == "red_to_blue": + im = grid[i].imshow( + np.rot90(X[:, :, zz]), + cmap=cm.get_cmap(cbar_name), + alpha=0.82, + vmin=0, + vmax=max_, + ) + elif cbar_name == "green": + im = grid[i].imshow( + np.rot90(X[:, :, zz]), + cmap=cm.get_cmap(cbar_name), + alpha=0.82, + vmin=0, + vmax=max_, + ) else: - im = grid[i].imshow(np.rot90(X[:, :, zz]), - cmap=cm.get_cmap(cbar_name), alpha=0.82, - vmin=- max_, vmax=max_) - except IndexError as e: + im = grid[i].imshow( + np.rot90(X[:, :, zz]), + cmap=cm.get_cmap(cbar_name), + alpha=0.82, + vmin=-max_, + vmax=max_, + ) + except IndexError: # TODO: send this to the logger instead - print("\n[!] QC Interface: Had a problem with creating the " - "axial montage for {0}\n\nDetails:{1}.This error might occur because of a registration error encountered while using ANTs.\ - Please refer to the image located in your working directory for more insight" - "\n".format(png_name, e)) pass grid[i].axes.get_xaxis().set_visible(False) @@ -723,20 +719,25 @@ def make_montage_axial(overlay, underlay, png_name, cbar_name): cbar = grid.cbar_axes[0].colorbar(im) - if 'snr' in png_name: + if "snr" in png_name: cbar.ax.set_yticks(np.linspace(0, max_, 8)) - elif ('reho' in png_name) or ('vmhc' in png_name) or \ - ('sca_' in png_name) or ('alff' in png_name) or \ - ('centrality' in png_name) or ('dr_tempreg' in png_name): + elif ( + ("reho" in png_name) + or ("vmhc" in png_name) + or ("sca_" in png_name) + or ("alff" in png_name) + or ("centrality" in png_name) + or ("dr_tempreg" in png_name) + ): cbar.ax.set_yticks(np.linspace(-max_, max_, 8)) plt.axis("off") png_name = os.path.join(os.getcwd(), png_name) - plt.savefig(png_name, dpi=200, bbox_inches='tight') + plt.savefig(png_name, dpi=200, bbox_inches="tight") plt.close() - matplotlib.rcdefaults() + mpl.rcdefaults() return png_name @@ -744,11 +745,10 @@ def make_montage_axial(overlay, underlay, png_name, cbar_name): def montage_sagittal(overlay, underlay, png_name, cbar_name): """ Draws Montage using overlay on Anatomical brain in Sagittal Direction - calls make_montage_sagittal + calls make_montage_sagittal. Parameters ---------- - overlay : string Nifi file @@ -763,32 +763,29 @@ def montage_sagittal(overlay, underlay, png_name, cbar_name): Returns ------- - png_name : Path to generated PNG """ - pngs = None if isinstance(overlay, list): pngs = [] for ov in overlay: fname = os.path.basename(os.path.splitext(os.path.splitext(ov)[0])[0]) - pngs.append(make_montage_sagittal(ov, underlay, fname + '_' + png_name, cbar_name)) + pngs.append( + make_montage_sagittal(ov, underlay, fname + "_" + png_name, cbar_name) + ) else: pngs = make_montage_sagittal(overlay, underlay, png_name, cbar_name) - png_name = pngs - - return png_name + return pngs def make_montage_sagittal(overlay, underlay, png_name, cbar_name): """ - Draws Montage using overlay on Anatomical brain in Sagittal Direction + Draws Montage using overlay on Anatomical brain in Sagittal Direction. Parameters ---------- - overlay : string Nifi file @@ -803,71 +800,87 @@ def make_montage_sagittal(overlay, underlay, png_name, cbar_name): Returns ------- - png_name : Path to generated PNG """ - import matplotlib import os + + import matplotlib as mpl import numpy as np - matplotlib.rcParams.update({'font.size': 5}) + mpl.rcParams.update({"font.size": 5}) try: from mpl_toolkits.axes_grid1 import ImageGrid except: from mpl_toolkits.axes_grid import ImageGrid - import matplotlib.cm as cm + from matplotlib import cm import matplotlib.pyplot as plt - import nibabel as nb - from mpl_toolkits.axes_grid1 import ImageGrid + import nibabel as nib from CPAC.qc.utils import determine_start_and_end, get_spacing - Y = nb.load(underlay).get_fdata() - X = nb.load(overlay).get_fdata() + Y = nib.load(underlay).get_fdata() + X = nib.load(overlay).get_fdata() X = X.astype(np.float32) Y = Y.astype(np.float32) - if 'skull_vis' in png_name: + if "skull_vis" in png_name: X[X < 20.0] = 0.0 - if 'skull_vis' in png_name or \ - 't1_edge_on_mean_func_in_t1' in png_name or \ - 'MNI_edge_on_mean_func_mni' in png_name: + if ( + "skull_vis" in png_name + or "t1_edge_on_mean_func_in_t1" in png_name + or "MNI_edge_on_mean_func_mni" in png_name + ): max_ = np.nanmax(np.abs(X.flatten())) X[X != 0.0] = max_ - x1, x2 = determine_start_and_end(Y, 'sagittal', 0.0001) + x1, x2 = determine_start_and_end(Y, "sagittal", 0.0001) spacing = get_spacing(6, 3, x2 - x1) x, y, z = Y.shape fig = plt.figure(1) max_ = np.max(np.abs(Y)) - if ('snr' in png_name) or ('reho' in png_name) or \ - ('vmhc' in png_name) or ('sca_' in png_name) or \ - ('alff' in png_name) or ('centrality' in png_name) or \ - ('dr_tempreg' in png_name): - grid = ImageGrid(fig, 111, nrows_ncols=(3, 6), share_all=True, - aspect=True, cbar_mode="single", cbar_pad=0.5, - direction="row") + if ( + ("snr" in png_name) + or ("reho" in png_name) + or ("vmhc" in png_name) + or ("sca_" in png_name) + or ("alff" in png_name) + or ("centrality" in png_name) + or ("dr_tempreg" in png_name) + ): + grid = ImageGrid( + fig, + 111, + nrows_ncols=(3, 6), + share_all=True, + aspect=True, + cbar_mode="single", + cbar_pad=0.5, + direction="row", + ) else: - grid = ImageGrid(fig, 111, nrows_ncols=(3, 6), share_all=True, - aspect=True, cbar_mode=None, direction="row") + grid = ImageGrid( + fig, + 111, + nrows_ncols=(3, 6), + share_all=True, + aspect=True, + cbar_mode=None, + direction="row", + ) xx = x1 - for i in range(6*3): + for i in range(6 * 3): if xx >= x2: break try: im = grid[i].imshow(np.rot90(Y[xx, :, :]), cmap=cm.Greys_r) - except IndexError as e: + except IndexError: # TODO: send this to the logger instead - print("\n[!] QC Interface: Had a problem with creating the " - "sagittal montage for {0}\n\nDetails:{1}.This error might occur because of a registration error encountered while using ANTs\ - Please refer to the image located in your working directory for more insight" - "\n".format(png_name, e)) pass grid[i].get_xaxis().set_visible(False) @@ -878,30 +891,38 @@ def make_montage_sagittal(overlay, underlay, png_name, cbar_name): X[X == 0.0] = np.nan max_ = np.nanmax(np.abs(X.flatten())) xx = x1 - for i in range(6*3): + for i in range(6 * 3): if xx >= x2: break im = None try: - if cbar_name == 'red_to_blue': - im = grid[i].imshow(np.rot90(X[xx, :, :]), - cmap=cm.get_cmap(cbar_name), alpha=0.82, - vmin=0, vmax=max_) - elif cbar_name == 'green': - im = grid[i].imshow(np.rot90(X[xx, :, :]), - cmap=cm.get_cmap(cbar_name), alpha=0.82, - vmin=0, vmax=max_) + if cbar_name == "red_to_blue": + im = grid[i].imshow( + np.rot90(X[xx, :, :]), + cmap=cm.get_cmap(cbar_name), + alpha=0.82, + vmin=0, + vmax=max_, + ) + elif cbar_name == "green": + im = grid[i].imshow( + np.rot90(X[xx, :, :]), + cmap=cm.get_cmap(cbar_name), + alpha=0.82, + vmin=0, + vmax=max_, + ) else: - im = grid[i].imshow(np.rot90(X[xx, :, :]), - cmap=cm.get_cmap(cbar_name), alpha=0.82, - vmin=- max_, vmax=max_) - except IndexError as e: + im = grid[i].imshow( + np.rot90(X[xx, :, :]), + cmap=cm.get_cmap(cbar_name), + alpha=0.82, + vmin=-max_, + vmax=max_, + ) + except IndexError: # TODO: send this to the logger instead - print("\n[!] QC Interface: Had a problem with creating the " - "sagittal montage for {0}\n\nDetails:{1}.This error might occur because of a registration error encountered while using ANTs.\ - Please refer to the image located in your working directory for more insight" - "\n".format(png_name, e)) pass xx += spacing @@ -909,37 +930,37 @@ def make_montage_sagittal(overlay, underlay, png_name, cbar_name): try: cbar = grid.cbar_axes[0].colorbar(im) - if 'snr' in png_name: + if "snr" in png_name: cbar.ax.set_yticks(np.linspace(0, max_, 8)) - elif ('reho' in png_name) or ('vmhc' in png_name) or \ - ('sca_' in png_name) or ('alff' in png_name) or \ - ('centrality' in png_name) or ('dr_tempreg' in png_name): + elif ( + ("reho" in png_name) + or ("vmhc" in png_name) + or ("sca_" in png_name) + or ("alff" in png_name) + or ("centrality" in png_name) + or ("dr_tempreg" in png_name) + ): cbar.ax.set_yticks(np.linspace(-max_, max_, 8)) - except AttributeError as e: + except AttributeError: # TODO: send this to the logger instead - print("\n[!] QC Interface: Had a problem with creating the " - "sagittal montage for {0}\n\nDetails:{1}" - "\n".format(png_name, e)) pass plt.axis("off") png_name = os.path.join(os.getcwd(), png_name) - plt.savefig(png_name, dpi=200, bbox_inches='tight') + plt.savefig(png_name, dpi=200, bbox_inches="tight") plt.close() - matplotlib.rcdefaults() + mpl.rcdefaults() return png_name def montage_gm_wm_csf_axial(overlay_csf, overlay_wm, overlay_gm, underlay, png_name): - """ - Draws Montage using GM WM and CSF overlays on Anatomical brain in Sagittal Direction + Draws Montage using GM WM and CSF overlays on Anatomical brain in Sagittal Direction. Parameters ---------- - overlay_csf : string Nifi file CSF MAP @@ -957,22 +978,21 @@ def montage_gm_wm_csf_axial(overlay_csf, overlay_wm, overlay_gm, underlay, png_n Returns ------- - png_name : Path to generated PNG """ - import numpy as np - from mpl_toolkits.axes_grid1 import ImageGrid as ImageGrid + from matplotlib import cm import matplotlib.pyplot as plt - import nibabel as nb - import matplotlib.cm as cm + from mpl_toolkits.axes_grid1 import ImageGrid + import numpy as np + import nibabel as nib - Y = nb.load(underlay).get_fdata() - z1, z2 = determine_start_and_end(Y, 'axial', 0.0001) + Y = nib.load(underlay).get_fdata() + z1, z2 = determine_start_and_end(Y, "axial", 0.0001) spacing = get_spacing(6, 3, z2 - z1) - X_csf = nb.load(overlay_csf).get_fdata() - X_wm = nb.load(overlay_wm).get_fdata() - X_gm = nb.load(overlay_gm).get_fdata() + X_csf = nib.load(overlay_csf).get_fdata() + X_wm = nib.load(overlay_wm).get_fdata() + X_gm = nib.load(overlay_gm).get_fdata() X_csf = X_csf.astype(np.float32) X_wm = X_wm.astype(np.float32) X_gm = X_gm.astype(np.float32) @@ -987,14 +1007,28 @@ def montage_gm_wm_csf_axial(overlay_csf, overlay_wm, overlay_gm, underlay, png_n fig = plt.figure(1) try: - grid = ImageGrid(fig, 111, nrows_ncols=(3, 6), share_all=True, - aspect=True, cbar_mode=None, direction="row") + grid = ImageGrid( + fig, + 111, + nrows_ncols=(3, 6), + share_all=True, + aspect=True, + cbar_mode=None, + direction="row", + ) except: - grid = ImageGrid(fig, 111, nrows_ncols=(3, 6), share_all=True, - aspect=True, cbar_mode=None, direction="row") + grid = ImageGrid( + fig, + 111, + nrows_ncols=(3, 6), + share_all=True, + aspect=True, + cbar_mode=None, + direction="row", + ) zz = z1 - for i in range(6*3): + for i in range(6 * 3): if zz >= z2: break im = grid[i].imshow(np.rot90(Y[:, :, zz]), cmap=cm.Greys_r) @@ -1007,22 +1041,40 @@ def montage_gm_wm_csf_axial(overlay_csf, overlay_wm, overlay_gm, underlay, png_n zz = z1 im = None - for i in range(6*3): + for i in range(6 * 3): if zz >= z2: break - im = grid[i].imshow(np.rot90(X_csf[:, :, zz]), cmap=cm.get_cmap('green'), alpha=0.82, vmin=0, vmax=max_csf) - im = grid[i].imshow(np.rot90(X_wm[:, :, zz]), cmap=cm.get_cmap('blue'), alpha=0.82, vmin=0, vmax=max_wm) - im = grid[i].imshow(np.rot90(X_gm[:, :, zz]), cmap=cm.get_cmap('red'), alpha=0.82, vmin=0, vmax=max_gm) + im = grid[i].imshow( + np.rot90(X_csf[:, :, zz]), + cmap=cm.get_cmap("green"), + alpha=0.82, + vmin=0, + vmax=max_csf, + ) + im = grid[i].imshow( + np.rot90(X_wm[:, :, zz]), + cmap=cm.get_cmap("blue"), + alpha=0.82, + vmin=0, + vmax=max_wm, + ) + im = grid[i].imshow( + np.rot90(X_gm[:, :, zz]), + cmap=cm.get_cmap("red"), + alpha=0.82, + vmin=0, + vmax=max_gm, + ) grid[i].axes.get_xaxis().set_visible(False) grid[i].axes.get_yaxis().set_visible(False) zz += spacing - cbar = grid.cbar_axes[0].colorbar(im) + grid.cbar_axes[0].colorbar(im) plt.axis("off") png_name = os.path.join(os.getcwd(), png_name) - plt.savefig(png_name, dpi=200, bbox_inches='tight') + plt.savefig(png_name, dpi=200, bbox_inches="tight") plt.close() return png_name @@ -1030,11 +1082,10 @@ def montage_gm_wm_csf_axial(overlay_csf, overlay_wm, overlay_gm, underlay, png_n def montage_gm_wm_csf_sagittal(overlay_csf, overlay_wm, overlay_gm, underlay, png_name): """ - Draws Montage using GM WM and CSF overlays on Anatomical brain in Sagittal Direction + Draws Montage using GM WM and CSF overlays on Anatomical brain in Sagittal Direction. Parameters ---------- - overlay_csf : string Nifi file CSF MAP @@ -1052,23 +1103,21 @@ def montage_gm_wm_csf_sagittal(overlay_csf, overlay_wm, overlay_gm, underlay, pn Returns ------- - png_name : Path to generated PNG """ - - import numpy as np - from mpl_toolkits.axes_grid1 import ImageGrid as ImageGrid + from matplotlib import cm import matplotlib.pyplot as plt - import matplotlib.cm as cm - import nibabel as nb + from mpl_toolkits.axes_grid1 import ImageGrid + import numpy as np + import nibabel as nib - Y = nb.load(underlay).get_fdata() - x1, x2 = determine_start_and_end(Y, 'sagittal', 0.0001) + Y = nib.load(underlay).get_fdata() + x1, x2 = determine_start_and_end(Y, "sagittal", 0.0001) spacing = get_spacing(6, 3, x2 - x1) - X_csf = nb.load(overlay_csf).get_fdata() - X_wm = nb.load(overlay_wm).get_fdata() - X_gm = nb.load(overlay_gm).get_fdata() + X_csf = nib.load(overlay_csf).get_fdata() + X_wm = nib.load(overlay_wm).get_fdata() + X_gm = nib.load(overlay_gm).get_fdata() X_csf = X_csf.astype(np.float32) X_wm = X_wm.astype(np.float32) X_gm = X_gm.astype(np.float32) @@ -1082,17 +1131,31 @@ def montage_gm_wm_csf_sagittal(overlay_csf, overlay_wm, overlay_gm, underlay, pn X_gm[X_gm != 0.0] = max_gm x, y, z = Y.shape fig = plt.figure(1) - max_ = np.max(np.abs(Y)) + np.max(np.abs(Y)) try: - grid = ImageGrid(fig, 111, nrows_ncols=(3, 6), share_all=True, - aspect=True, cbar_mode=None, direction="row") + grid = ImageGrid( + fig, + 111, + nrows_ncols=(3, 6), + share_all=True, + aspect=True, + cbar_mode=None, + direction="row", + ) except: - grid = ImageGrid(fig, 111, nrows_ncols=(3, 6), share_all=True, - aspect=True, cbar_mode=None, direction="row") + grid = ImageGrid( + fig, + 111, + nrows_ncols=(3, 6), + share_all=True, + aspect=True, + cbar_mode=None, + direction="row", + ) zz = x1 - for i in range(6*3): + for i in range(6 * 3): if zz >= x2: break im = grid[i].imshow(np.rot90(Y[zz, :, :]), cmap=cm.Greys_r) @@ -1105,42 +1168,52 @@ def montage_gm_wm_csf_sagittal(overlay_csf, overlay_wm, overlay_gm, underlay, pn zz = x1 im = None - for i in range(6*3): + for i in range(6 * 3): if zz >= x2: break - im = grid[i].imshow(np.rot90(X_csf[zz, :, :]), - cmap=cm.get_cmap('green'), alpha=0.82, vmin=0, - vmax=max_csf) - im = grid[i].imshow(np.rot90(X_wm[zz, :, :]), - cmap=cm.get_cmap('blue'), alpha=0.82, vmin=0, - vmax=max_wm) - im = grid[i].imshow(np.rot90(X_gm[zz, :, :]), - cmap=cm.get_cmap('red'), alpha=0.82, vmin=0, - vmax=max_gm) + im = grid[i].imshow( + np.rot90(X_csf[zz, :, :]), + cmap=cm.get_cmap("green"), + alpha=0.82, + vmin=0, + vmax=max_csf, + ) + im = grid[i].imshow( + np.rot90(X_wm[zz, :, :]), + cmap=cm.get_cmap("blue"), + alpha=0.82, + vmin=0, + vmax=max_wm, + ) + im = grid[i].imshow( + np.rot90(X_gm[zz, :, :]), + cmap=cm.get_cmap("red"), + alpha=0.82, + vmin=0, + vmax=max_gm, + ) grid[i].axes.get_xaxis().set_visible(False) grid[i].axes.get_yaxis().set_visible(False) zz += spacing - cbar = grid.cbar_axes[0].colorbar(im) + grid.cbar_axes[0].colorbar(im) plt.axis("off") png_name = os.path.join(os.getcwd(), png_name) - plt.savefig(png_name, dpi=200, bbox_inches='tight') + plt.savefig(png_name, dpi=200, bbox_inches="tight") plt.close() return png_name def register_pallete(colors_file, cbar_name): - """ - Registers color pallete to matplotlib + Registers color pallete to matplotlib. Parameters ---------- - colors_file : string file containing colors in hexadecimal formats in each line @@ -1150,34 +1223,29 @@ def register_pallete(colors_file, cbar_name): Returns ------- - None """ - + from matplotlib import cm import matplotlib.colors as col - import matplotlib.cm as cm - with open(colors_file, 'r') as f: - colors = [c.rstrip('\r\n') for c in reversed(f.readlines())] + with open(colors_file, "r") as f: + colors = [c.rstrip("\r\n") for c in reversed(f.readlines())] cmap3 = col.ListedColormap(colors, cbar_name) cm.register_cmap(cmap=cmap3) def resample_1mm(file_): - """ - Calls make_resample_1mm which resamples file to 1mm space + Calls make_resample_1mm which resamples file to 1mm space. Parameters ---------- - file_ : string path to the scan Returns ------- - new_fname : string path to 1mm resampled nifti file @@ -1195,41 +1263,39 @@ def resample_1mm(file_): def make_resample_1mm(file_): - """ - Resamples input nifti file to 1mm space + Resamples input nifti file to 1mm space. Parameters ---------- - file_ : string Input Nifti File Returns ------- - new_fname : string Input Nifti resampled to 1mm space """ remainder, ext_ = os.path.splitext(file_) remainder, ext1_ = os.path.splitext(remainder) - ext = ''.join([ext1_, ext_]) + ext = "".join([ext1_, ext_]) - new_fname = ''.join([remainder, '_1mm', ext]) + new_fname = "".join([remainder, "_1mm", ext]) new_fname = os.path.join(os.getcwd(), os.path.basename(new_fname)) - cmd = " 3dresample -dxyz 1.0 1.0 1.0 -prefix %s " \ - "-inset %s " % (new_fname, file_) + cmd = " 3dresample -dxyz 1.0 1.0 1.0 -prefix %s " "-inset %s " % (new_fname, file_) subprocess.getoutput(cmd) return new_fname + # own modules + # code def dc(input1, input2): - """ - Dice coefficient + r""" + Dice coefficient. Computes the Dice coefficient (also known as Sorensen index) between the binary objects in two images. @@ -1271,7 +1337,7 @@ def dc(input1, input2): size_i2 = numpy.count_nonzero(input2) try: - dc = 2. * intersection / float(size_i1 + size_i2) + dc = 2.0 * intersection / float(size_i1 + size_i2) except ZeroDivisionError: dc = 0.0 @@ -1280,7 +1346,7 @@ def dc(input1, input2): def jc(input1, input2): """ - Jaccard coefficient + Jaccard coefficient. Computes the Jaccard coefficient between the binary objects in two images. @@ -1309,34 +1375,30 @@ def jc(input1, input2): intersection = numpy.count_nonzero(input1 & input2) union = numpy.count_nonzero(input1 | input2) - jc = float(intersection) / float(union) + return float(intersection) / float(union) - return jc -def crosscorr(input1,input2): +def crosscorr(input1, input2): """ cross correlation - computer compute cross correction bewteen input mask + computer compute cross correction bewteen input mask. """ - input1 = numpy.atleast_1d(input1.astype(bool)) input2 = numpy.atleast_1d(input2.astype(bool)) from scipy.stats.stats import pearsonr - cc=pearsonr(input1,input2) - return cc -def coverage(input1,input2): - """ - estimate the coverage between two mask - """ + return pearsonr(input1, input2) + + +def coverage(input1, input2): + """Estimate the coverage between two mask.""" input1 = numpy.atleast_1d(input1.astype(bool)) input2 = numpy.atleast_1d(input2.astype(bool)) - intsec=numpy.count_nonzero(input1 & input2) - if numpy.sum(input1)> numpy.sum(input2): - smallv=numpy.sum(input2) + intsec = numpy.count_nonzero(input1 & input2) + if numpy.sum(input1) > numpy.sum(input2): + smallv = numpy.sum(input2) else: - smallv=numpy.sum(input1) - cov=float(intsec)/float(smallv) - return cov + smallv = numpy.sum(input1) + return float(intsec) / float(smallv) diff --git a/CPAC/qc/xcp.py b/CPAC/qc/xcp.py index 3104256c02..ebcaff6020 100644 --- a/CPAC/qc/xcp.py +++ b/CPAC/qc/xcp.py @@ -54,26 +54,32 @@ "Normalization of T1w/Functional to Template:[…] cross correlation" :footcite:`xcp_22,Ciri19` normCoverage : float "Normalization of T1w/Functional to Template:[…] Coverage index" :footcite:`xcp_22,Ciri19` -""" # noqa: E501 # pylint: disable=line-too-long +""" # pylint: disable=line-too-long +from io import BufferedReader import os import re -from io import BufferedReader -import nibabel as nb +from bids.layout import parse_file_entities import numpy as np import pandas as pd -from bids.layout import parse_file_entities +import nibabel as nib from nipype.interfaces import afni, fsl -from CPAC.generate_motion_statistics.generate_motion_statistics import \ - DVARS_strip_t0, ImageTo1D +from CPAC.generate_motion_statistics.generate_motion_statistics import ( + DVARS_strip_t0, + ImageTo1D, +) from CPAC.pipeline import nipype_pipeline_engine as pe from CPAC.pipeline.nodeblock import nodeblock from CPAC.qc.qcmetrics import regisQ from CPAC.utils.interfaces.function import Function -motion_params = ['dvars', 'framewise-displacement-jenkinson', - 'desc-movementParametersUnfiltered_motion', 'desc-movementParameters_motion'] +motion_params = [ + "dvars", + "framewise-displacement-jenkinson", + "desc-movementParametersUnfiltered_motion", + "desc-movementParameters_motion", +] def _connect_motion(wf, nodes, strat_pool, qc_file, pipe_num): @@ -102,53 +108,88 @@ def _connect_motion(wf, nodes, strat_pool, qc_file, pipe_num): """ # pylint: disable=invalid-name, too-many-arguments try: - nodes = {**nodes, - 'censor-indices': strat_pool.node_data('censor-indices')} - wf.connect(nodes['censor-indices'].node, nodes['censor-indices'].out, - qc_file, 'censor_indices') + nodes = {**nodes, "censor-indices": strat_pool.node_data("censor-indices")} + wf.connect( + nodes["censor-indices"].node, + nodes["censor-indices"].out, + qc_file, + "censor_indices", + ) except LookupError: qc_file.inputs.censor_indices = [] - cal_DVARS = pe.Node(ImageTo1D(method='dvars'), - name=f'cal_DVARS_{pipe_num}', - mem_gb=0.4, - mem_x=(739971956005215 / 151115727451828646838272, - 'in_file')) - cal_DVARS_strip = pe.Node(Function(input_names=['file_1D'], - output_names=['out_file'], - function=DVARS_strip_t0, - as_module=True), - name=f'cal_DVARS_strip_{pipe_num}') - wf.connect([ - (nodes['desc-preproc_bold'].node, cal_DVARS, [ - (nodes['desc-preproc_bold'].out, 'in_file')]), - (nodes['space-bold_desc-brain_mask'].node, cal_DVARS, [ - (nodes['space-bold_desc-brain_mask'].out, 'mask')]), - (cal_DVARS, cal_DVARS_strip, [('out_file', 'file_1D')]), - (cal_DVARS_strip, qc_file, [('out_file', 'dvars_after')]), - *[(nodes[node].node, qc_file, [ - (nodes[node].out, node.replace('-', '_')) - ]) for node in motion_params if node in nodes]]) + cal_DVARS = pe.Node( + ImageTo1D(method="dvars"), + name=f"cal_DVARS_{pipe_num}", + mem_gb=0.4, + mem_x=(739971956005215 / 151115727451828646838272, "in_file"), + ) + cal_DVARS_strip = pe.Node( + Function( + input_names=["file_1D"], + output_names=["out_file"], + function=DVARS_strip_t0, + as_module=True, + ), + name=f"cal_DVARS_strip_{pipe_num}", + ) + wf.connect( + [ + ( + nodes["desc-preproc_bold"].node, + cal_DVARS, + [(nodes["desc-preproc_bold"].out, "in_file")], + ), + ( + nodes["space-bold_desc-brain_mask"].node, + cal_DVARS, + [(nodes["space-bold_desc-brain_mask"].out, "mask")], + ), + (cal_DVARS, cal_DVARS_strip, [("out_file", "file_1D")]), + (cal_DVARS_strip, qc_file, [("out_file", "dvars_after")]), + *[ + (nodes[node].node, qc_file, [(nodes[node].out, node.replace("-", "_"))]) + for node in motion_params + if node in nodes + ], + ] + ) return wf def dvcorr(dvars, fdj): - """Function to correlate DVARS and FD-J""" + """Function to correlate DVARS and FD-J.""" dvars = np.loadtxt(dvars) fdj = np.loadtxt(fdj) if len(dvars) != len(fdj) - 1: raise ValueError( - 'len(DVARS) should be 1 less than len(FDJ), but their respective ' - f'lengths are {len(dvars)} and {len(fdj)}.' + "len(DVARS) should be 1 less than len(FDJ), but their respective " + f"lengths are {len(dvars)} and {len(fdj)}." ) return np.corrcoef(dvars, fdj[1:])[0, 1] -def generate_xcp_qc(sub, ses, task, run, desc, regressors, bold2t1w_mask, - t1w_mask, bold2template_mask, template_mask, original_func, - final_func, movement_parameters, dvars, censor_indices, - framewise_displacement_jenkinson, dvars_after, template): +def generate_xcp_qc( + sub, + ses, + task, + run, + desc, + regressors, + bold2t1w_mask, + t1w_mask, + bold2template_mask, + template_mask, + original_func, + final_func, + movement_parameters, + dvars, + censor_indices, + framewise_displacement_jenkinson, + dvars_after, + template, +): # pylint: disable=too-many-arguments, too-many-locals, invalid-name - """Function to generate an RBC-style QC CSV + """Function to generate an RBC-style QC CSV. Parameters ---------- @@ -213,78 +254,79 @@ def generate_xcp_qc(sub, ses, task, run, desc, regressors, bold2t1w_mask, path to space-template_desc-xcp_quality TSV """ columns = ( - 'sub,ses,task,run,desc,regressors,space,meanFD,relMeansRMSMotion,' - 'relMaxRMSMotion,meanDVInit,meanDVFinal,nVolCensored,nVolsRemoved,' - 'motionDVCorrInit,motionDVCorrFinal,coregDice,coregJaccard,' - 'coregCrossCorr,coregCoverage,normDice,normJaccard,normCrossCorr,' - 'normCoverage'.split(',') + "sub,ses,task,run,desc,regressors,space,meanFD,relMeansRMSMotion," + "relMaxRMSMotion,meanDVInit,meanDVFinal,nVolCensored,nVolsRemoved," + "motionDVCorrInit,motionDVCorrFinal,coregDice,coregJaccard," + "coregCrossCorr,coregCoverage,normDice,normJaccard,normCrossCorr," + "normCoverage".split(",") ) images = { - 'original_func': nb.load(original_func), - 'final_func': nb.load(final_func), + "original_func": nib.load(original_func), + "final_func": nib.load(final_func), } # `sub` through `space` from_bids = { - 'sub': sub, 'ses': ses, 'task': task, 'run': run, 'desc': desc, - 'regressors': regressors, - 'space': os.path.basename(template).split('.', 1)[0].split('_', 1)[0] + "sub": sub, + "ses": ses, + "task": task, + "run": run, + "desc": desc, + "regressors": regressors, + "space": os.path.basename(template).split(".", 1)[0].split("_", 1)[0], } - if from_bids['space'].startswith('tpl-'): - from_bids['space'] = from_bids['space'][4:] + if from_bids["space"].startswith("tpl-"): + from_bids["space"] = from_bids["space"][4:] # `nVolCensored` & `nVolsRemoved` - n_vols_censored = len( - censor_indices) if censor_indices is not None else 'unknown' - shape_params = {'nVolCensored': n_vols_censored, - 'nVolsRemoved': images['original_func'].shape[3] - - images['final_func'].shape[3]} + n_vols_censored = len(censor_indices) if censor_indices is not None else "unknown" + shape_params = { + "nVolCensored": n_vols_censored, + "nVolsRemoved": images["original_func"].shape[3] + - images["final_func"].shape[3], + } if isinstance(final_func, BufferedReader): final_func = final_func.name - qc_filepath = os.path.join(os.getcwd(), 'xcpqc.tsv') + qc_filepath = os.path.join(os.getcwd(), "xcpqc.tsv") - desc_span = re.search(r'_desc-.*_', final_func) + desc_span = re.search(r"_desc-.*_", final_func) if desc_span: desc_span = desc_span.span() - final_func = '_'.join([ - final_func[:desc_span[0]], - final_func[desc_span[1]:] - ]) + final_func = "_".join([final_func[: desc_span[0]], final_func[desc_span[1] :]]) del desc_span # `meanFD (Jenkinson)` - power_params = {'meanFD': np.mean(np.loadtxt( - framewise_displacement_jenkinson))} + power_params = {"meanFD": np.mean(np.loadtxt(framewise_displacement_jenkinson))} # `relMeansRMSMotion` & `relMaxRMSMotion` mot = np.genfromtxt(movement_parameters).T # Relative RMS of translation rms = np.sqrt(mot[3] ** 2 + mot[4] ** 2 + mot[5] ** 2) - rms_params = { - 'relMeansRMSMotion': [np.mean(rms)], - 'relMaxRMSMotion': [np.max(rms)] - } + rms_params = {"relMeansRMSMotion": [np.mean(rms)], "relMaxRMSMotion": [np.max(rms)]} # `meanDVInit` & `meanDVFinal` - meanDV = {'meanDVInit': np.mean(np.loadtxt(dvars))} + meanDV = {"meanDVInit": np.mean(np.loadtxt(dvars))} try: - meanDV['motionDVCorrInit'] = dvcorr( - dvars, framewise_displacement_jenkinson) + meanDV["motionDVCorrInit"] = dvcorr(dvars, framewise_displacement_jenkinson) except ValueError as value_error: - meanDV['motionDVCorrInit'] = f'ValueError({str(value_error)})' - meanDV['meanDVFinal'] = np.mean(np.loadtxt(dvars_after)) + meanDV["motionDVCorrInit"] = f"ValueError({value_error!s})" + meanDV["meanDVFinal"] = np.mean(np.loadtxt(dvars_after)) try: - meanDV['motionDVCorrFinal'] = dvcorr(dvars_after, - framewise_displacement_jenkinson) + meanDV["motionDVCorrFinal"] = dvcorr( + dvars_after, framewise_displacement_jenkinson + ) except ValueError as value_error: - meanDV['motionDVCorrFinal'] = f'ValueError({str(value_error)})' + meanDV["motionDVCorrFinal"] = f"ValueError({value_error!s})" # Overlap - overlap_params = regisQ(bold2t1w_mask=bold2t1w_mask, t1w_mask=t1w_mask, - bold2template_mask=bold2template_mask, - template_mask=template_mask) + overlap_params = regisQ( + bold2t1w_mask=bold2t1w_mask, + t1w_mask=t1w_mask, + bold2template_mask=bold2template_mask, + template_mask=template_mask, + ) qc_dict = { **from_bids, @@ -292,16 +334,16 @@ def generate_xcp_qc(sub, ses, task, run, desc, regressors, bold2t1w_mask, **rms_params, **shape_params, **overlap_params, - **meanDV + **meanDV, } df = pd.DataFrame(qc_dict, columns=columns) - df.to_csv(qc_filepath, sep='\t', index=False) + df.to_csv(qc_filepath, sep="\t", index=False) return qc_filepath def get_bids_info(subject, scan, wf_name): """ - Function to gather BIDS information from a strat_pool + Function to gather BIDS information from a strat_pool. Parameters ---------- @@ -345,35 +387,32 @@ def get_bids_info(subject, scan, wf_name): ... wf_name='cpac_sub-colornest035_ses-1') ('colornest035', '1', 'rest', '01') """ - returns = ('subject', 'session', 'task', 'run') - ses = wf_name.split('_')[-1] - if not ses.startswith('ses-'): - ses = f'ses-{ses}' - if not subject.startswith('sub-'): - subject = f'sub-{subject}' - resource = '_'.join([subject, ses, - scan if 'task' in scan else f'task-{scan}']) + returns = ("subject", "session", "task", "run") + ses = wf_name.split("_")[-1] + if not ses.startswith("ses-"): + ses = f"ses-{ses}" + if not subject.startswith("sub-"): + subject = f"sub-{subject}" + resource = "_".join([subject, ses, scan if "task" in scan else f"task-{scan}"]) entities = parse_file_entities(resource) returns = {key: entities.get(key) for key in returns} if any(value is None for value in returns.values()): - entity_parts = resource.split('_') + entity_parts = resource.split("_") def get_entity_part(key): - key = f'{key}-' - matching_parts = [part for part in entity_parts if - part.startswith(key)] + key = f"{key}-" + matching_parts = [part for part in entity_parts if part.startswith(key)] if matching_parts: - return matching_parts[0].replace(key, '') + return matching_parts[0].replace(key, "") return None for key, value in returns.items(): if value is None: - if key == 'task': + if key == "task": returns[key] = get_entity_part(key) else: returns[key] = get_entity_part(key[:3]) - return tuple(str(returns.get(key)) for key in [ - 'subject', 'session', 'task', 'run']) + return tuple(str(returns.get(key)) for key in ["subject", "session", "task", "run"]) @nodeblock( @@ -395,7 +434,10 @@ def get_entity_part(key): ["space-template_desc-bold_mask", "space-EPItemplate_desc-bold_mask"], "regressors", ["T1w-brain-template-funcreg", "EPI-brain-template-funcreg"], - ["desc-movementParametersUnfiltered_motion", "desc-movementParameters_motion"], + [ + "desc-movementParametersUnfiltered_motion", + "desc-movementParameters_motion", + ], "dvars", "framewise-displacement-jenkinson", ) @@ -406,79 +448,146 @@ def get_entity_part(key): ) def qc_xcp(wf, cfg, strat_pool, pipe_num, opt=None): # pylint: disable=invalid-name, unused-argument - if cfg['nuisance_corrections', '2-nuisance_regression', 'run' - ] and not strat_pool.check_rpool('regressors'): + if cfg[ + "nuisance_corrections", "2-nuisance_regression", "run" + ] and not strat_pool.check_rpool("regressors"): return wf, {} - bids_info = pe.Node(Function(input_names=['subject', 'scan', 'wf_name'], - output_names=['subject', 'session', 'task', - 'run'], - imports=['from bids.layout import ' - 'parse_file_entities'], - function=get_bids_info, as_module=True), - name=f'bids_info_{pipe_num}') + bids_info = pe.Node( + Function( + input_names=["subject", "scan", "wf_name"], + output_names=["subject", "session", "task", "run"], + imports=["from bids.layout import " "parse_file_entities"], + function=get_bids_info, + as_module=True, + ), + name=f"bids_info_{pipe_num}", + ) bids_info.inputs.wf_name = wf.name - qc_file = pe.Node(Function(input_names=['sub', 'ses', 'task', 'run', - 'desc', 'bold2t1w_mask', - 't1w_mask', 'bold2template_mask', - 'template_mask', 'original_func', - 'final_func', 'template', - 'movement_parameters', 'dvars', - 'censor_indices', 'regressors', - 'framewise_displacement_jenkinson', - 'dvars_after'], - output_names=['qc_file'], - function=generate_xcp_qc, - as_module=True), - name=f'qcxcp_{pipe_num}') - qc_file.inputs.desc = 'preproc' - qc_file.inputs.regressors = strat_pool.node_data( - 'regressors').node.name.split('regressors_' - )[-1][::-1].split('_', 1)[-1][::-1] - bold_to_T1w_mask = pe.Node(interface=fsl.ImageMaths(), - name=f'binarize_bold_to_T1w_mask_{pipe_num}', - op_string='-bin ') - nodes = {key: strat_pool.node_data(key) for key in [ - 'bold', 'desc-preproc_bold', 'max-displacement', - 'scan', 'space-bold_desc-brain_mask', 'space-T1w_desc-brain_mask', - 'space-T1w_sbref', 'space-template_desc-preproc_bold', - 'subject', *motion_params] if strat_pool.check_rpool(key)} - nodes['bold2template_mask'] = strat_pool.node_data([ - 'space-template_desc-bold_mask', 'space-EPItemplate_desc-bold_mask']) - nodes['template_mask'] = strat_pool.node_data( - ['T1w-brain-template-mask', 'EPI-template-mask']) - nodes['template'] = strat_pool.node_data(['T1w-brain-template-funcreg', - 'EPI-brain-template-funcreg']) + qc_file = pe.Node( + Function( + input_names=[ + "sub", + "ses", + "task", + "run", + "desc", + "bold2t1w_mask", + "t1w_mask", + "bold2template_mask", + "template_mask", + "original_func", + "final_func", + "template", + "movement_parameters", + "dvars", + "censor_indices", + "regressors", + "framewise_displacement_jenkinson", + "dvars_after", + ], + output_names=["qc_file"], + function=generate_xcp_qc, + as_module=True, + ), + name=f"qcxcp_{pipe_num}", + ) + qc_file.inputs.desc = "preproc" + qc_file.inputs.regressors = ( + strat_pool.node_data("regressors") + .node.name.split("regressors_")[-1][::-1] + .split("_", 1)[-1][::-1] + ) + bold_to_T1w_mask = pe.Node( + interface=fsl.ImageMaths(), + name=f"binarize_bold_to_T1w_mask_{pipe_num}", + op_string="-bin ", + ) + nodes = { + key: strat_pool.node_data(key) + for key in [ + "bold", + "desc-preproc_bold", + "max-displacement", + "scan", + "space-bold_desc-brain_mask", + "space-T1w_desc-brain_mask", + "space-T1w_sbref", + "space-template_desc-preproc_bold", + "subject", + *motion_params, + ] + if strat_pool.check_rpool(key) + } + nodes["bold2template_mask"] = strat_pool.node_data( + ["space-template_desc-bold_mask", "space-EPItemplate_desc-bold_mask"] + ) + nodes["template_mask"] = strat_pool.node_data( + ["T1w-brain-template-mask", "EPI-template-mask"] + ) + nodes["template"] = strat_pool.node_data( + ["T1w-brain-template-funcreg", "EPI-brain-template-funcreg"] + ) resample_bold_mask_to_template = pe.Node( - afni.Resample(), name=f'resample_bold_mask_to_anat_res_{pipe_num}', - mem_gb=0, mem_x=(0.0115, 'in_file', 't')) - resample_bold_mask_to_template.inputs.outputtype = 'NIFTI_GZ' + afni.Resample(), + name=f"resample_bold_mask_to_anat_res_{pipe_num}", + mem_gb=0, + mem_x=(0.0115, "in_file", "t"), + ) + resample_bold_mask_to_template.inputs.outputtype = "NIFTI_GZ" wf = _connect_motion(wf, nodes, strat_pool, qc_file, pipe_num=pipe_num) - wf.connect([ - (nodes['subject'].node, bids_info, [ - (nodes['subject'].out, 'subject')]), - (nodes['scan'].node, bids_info, [(nodes['scan'].out, 'scan')]), - (nodes['space-T1w_sbref'].node, bold_to_T1w_mask, [ - (nodes['space-T1w_sbref'].out, 'in_file')]), - (nodes['space-T1w_desc-brain_mask'].node, qc_file, [ - (nodes['space-T1w_desc-brain_mask'].out, 't1w_mask')]), - (bold_to_T1w_mask, qc_file, [('out_file', 'bold2t1w_mask')]), - (nodes['template_mask'].node, qc_file, [ - (nodes['template_mask'].out, 'template_mask')]), - (nodes['bold'].node, qc_file, [(nodes['bold'].out, 'original_func')]), - (nodes['space-template_desc-preproc_bold'].node, qc_file, [ - (nodes['space-template_desc-preproc_bold'].out, 'final_func')]), - (nodes['template'].node, qc_file, [ - (nodes['template'].out, 'template')]), - (nodes['template_mask'].node, resample_bold_mask_to_template, [ - (nodes['template_mask'].out, 'master')]), - (nodes['bold2template_mask'].node, resample_bold_mask_to_template, - [(nodes['bold2template_mask'].out, 'in_file')]), - (resample_bold_mask_to_template, qc_file, [ - ('out_file', 'bold2template_mask')]), - (bids_info, qc_file, [ - ('subject', 'sub'), - ('session', 'ses'), - ('task', 'task'), - ('run', 'run')])]) - - return wf, {'space-template_desc-xcp_quality': (qc_file, 'qc_file')} + wf.connect( + [ + (nodes["subject"].node, bids_info, [(nodes["subject"].out, "subject")]), + (nodes["scan"].node, bids_info, [(nodes["scan"].out, "scan")]), + ( + nodes["space-T1w_sbref"].node, + bold_to_T1w_mask, + [(nodes["space-T1w_sbref"].out, "in_file")], + ), + ( + nodes["space-T1w_desc-brain_mask"].node, + qc_file, + [(nodes["space-T1w_desc-brain_mask"].out, "t1w_mask")], + ), + (bold_to_T1w_mask, qc_file, [("out_file", "bold2t1w_mask")]), + ( + nodes["template_mask"].node, + qc_file, + [(nodes["template_mask"].out, "template_mask")], + ), + (nodes["bold"].node, qc_file, [(nodes["bold"].out, "original_func")]), + ( + nodes["space-template_desc-preproc_bold"].node, + qc_file, + [(nodes["space-template_desc-preproc_bold"].out, "final_func")], + ), + (nodes["template"].node, qc_file, [(nodes["template"].out, "template")]), + ( + nodes["template_mask"].node, + resample_bold_mask_to_template, + [(nodes["template_mask"].out, "master")], + ), + ( + nodes["bold2template_mask"].node, + resample_bold_mask_to_template, + [(nodes["bold2template_mask"].out, "in_file")], + ), + ( + resample_bold_mask_to_template, + qc_file, + [("out_file", "bold2template_mask")], + ), + ( + bids_info, + qc_file, + [ + ("subject", "sub"), + ("session", "ses"), + ("task", "task"), + ("run", "run"), + ], + ), + ] + ) + + return wf, {"space-template_desc-xcp_quality": (qc_file, "qc_file")} diff --git a/CPAC/qpp/pipeline.py b/CPAC/qpp/pipeline.py index 8845e916c1..1f64abd00c 100644 --- a/CPAC/qpp/pipeline.py +++ b/CPAC/qpp/pipeline.py @@ -1,37 +1,44 @@ - import os -import nibabel as nb import numpy as np +import nibabel as nib +from nipype.interfaces import fsl +import nipype.interfaces.utility as util from CPAC.pipeline import nipype_pipeline_engine as pe -import nipype.interfaces.utility as util -import nipype.interfaces.fsl as fsl from CPAC.utils.interfaces.fsl import Merge as fslMerge - from CPAC.utils.interfaces.function import Function + def length(it): return len(it) -def detect_qpp(datasets, - joint_datasets, joint_mask, - window_length, permutations, - lower_correlation_threshold, higher_correlation_threshold, - correlation_threshold_iteration, - iterations, convergence_iterations): - + +def detect_qpp( + datasets, + joint_datasets, + joint_mask, + window_length, + permutations, + lower_correlation_threshold, + higher_correlation_threshold, + correlation_threshold_iteration, + iterations, + convergence_iterations, +): from CPAC.qpp.qpp import detect_qpp - joint_mask_img = nb.load(joint_mask) + joint_mask_img = nib.load(joint_mask) joint_mask = joint_mask_img.get_fdata().astype(bool) - joint_datasets_img = nb.load(joint_datasets) + joint_datasets_img = nib.load(joint_datasets) joint_datasets = joint_datasets_img.get_fdata()[joint_mask] - correlation_threshold = lambda i: \ - higher_correlation_threshold \ - if i > correlation_threshold_iteration else \ - lower_correlation_threshold + def correlation_threshold(i): + return ( + higher_correlation_threshold + if i > correlation_threshold_iteration + else lower_correlation_threshold + ) best_template_segment, _, _ = detect_qpp( joint_datasets, @@ -40,82 +47,102 @@ def detect_qpp(datasets, permutations, correlation_threshold, iterations, - convergence_iterations + convergence_iterations, ) qpp = np.zeros(joint_datasets_img.shape[0:3] + (window_length,)) qpp[joint_mask] = best_template_segment - qpp_img = nb.Nifti1Image(qpp, joint_mask_img.affine) - qpp_img.to_filename('./qpp.nii.gz') + qpp_img = nib.Nifti1Image(qpp, joint_mask_img.affine) + qpp_img.to_filename("./qpp.nii.gz") - return os.path.abspath('./qpp.nii.gz') + return os.path.abspath("./qpp.nii.gz") -def create_qpp(name='qpp', working_dir=None, crash_dir=None): - +def create_qpp(name="qpp", working_dir=None, crash_dir=None): if not working_dir: - working_dir = os.path.join(os.getcwd(), 'QPP_work_dir') + working_dir = os.path.join(os.getcwd(), "QPP_work_dir") if not crash_dir: - crash_dir = os.path.join(os.getcwd(), 'QPP_crash_dir') + crash_dir = os.path.join(os.getcwd(), "QPP_crash_dir") workflow = pe.Workflow(name=name) workflow.base_dir = working_dir - workflow.config['execution'] = {'hash_method': 'timestamp', - 'crashdump_dir': os.path.abspath(crash_dir)} - - inputspec = pe.Node(util.IdentityInterface(fields=[ - 'datasets', - 'window_length', - 'permutations', - 'lower_correlation_threshold', - 'higher_correlation_threshold', - 'correlation_threshold_iteration', - 'iterations', - 'convergence_iterations', - ]), name='inputspec') - - outputspec = pe.Node(util.IdentityInterface(fields=['qpp']), - name='outputspec') - - merge = pe.Node(fslMerge(), name='joint_datasets') - merge.inputs.dimension = 't' - merge.inputs.output_type = 'NIFTI_GZ' - - mask = pe.Node(interface=fsl.ImageMaths(), name='joint_mask') - mask.inputs.op_string = '-abs -Tmin -bin' - - detect = pe.Node(Function(input_names=['datasets', - 'joint_datasets', - 'joint_mask', - 'window_length', - 'permutations', - 'lower_correlation_threshold', - 'higher_correlation_threshold', - 'correlation_threshold_iteration', - 'iterations', - 'convergence_iterations'], - output_names=['qpp'], - function=detect_qpp, - as_module=True), - name='detect_qpp') - - workflow.connect([ - (inputspec, merge, [('datasets', 'in_files')]), - (merge, mask, [('merged_file', 'in_file')]), - (merge, detect, [('merged_file', 'joint_datasets')]), - (mask, detect, [('out_file', 'joint_mask')]), - (inputspec, detect, [ - (('datasets', length), 'datasets'), - ('window_length' ,'window_length'), - ('permutations' ,'permutations'), - ('lower_correlation_threshold' ,'lower_correlation_threshold'), - ('higher_correlation_threshold' ,'higher_correlation_threshold'), - ('correlation_threshold_iteration' ,'correlation_threshold_iteration'), - ('iterations' ,'iterations'), - ('convergence_iterations' ,'convergence_iterations'), - ]), - (detect, outputspec, [('qpp', 'qpp')]), - ]) + workflow.config["execution"] = { + "hash_method": "timestamp", + "crashdump_dir": os.path.abspath(crash_dir), + } + + inputspec = pe.Node( + util.IdentityInterface( + fields=[ + "datasets", + "window_length", + "permutations", + "lower_correlation_threshold", + "higher_correlation_threshold", + "correlation_threshold_iteration", + "iterations", + "convergence_iterations", + ] + ), + name="inputspec", + ) + + outputspec = pe.Node(util.IdentityInterface(fields=["qpp"]), name="outputspec") + + merge = pe.Node(fslMerge(), name="joint_datasets") + merge.inputs.dimension = "t" + merge.inputs.output_type = "NIFTI_GZ" + + mask = pe.Node(interface=fsl.ImageMaths(), name="joint_mask") + mask.inputs.op_string = "-abs -Tmin -bin" + + detect = pe.Node( + Function( + input_names=[ + "datasets", + "joint_datasets", + "joint_mask", + "window_length", + "permutations", + "lower_correlation_threshold", + "higher_correlation_threshold", + "correlation_threshold_iteration", + "iterations", + "convergence_iterations", + ], + output_names=["qpp"], + function=detect_qpp, + as_module=True, + ), + name="detect_qpp", + ) + + workflow.connect( + [ + (inputspec, merge, [("datasets", "in_files")]), + (merge, mask, [("merged_file", "in_file")]), + (merge, detect, [("merged_file", "joint_datasets")]), + (mask, detect, [("out_file", "joint_mask")]), + ( + inputspec, + detect, + [ + (("datasets", length), "datasets"), + ("window_length", "window_length"), + ("permutations", "permutations"), + ("lower_correlation_threshold", "lower_correlation_threshold"), + ("higher_correlation_threshold", "higher_correlation_threshold"), + ( + "correlation_threshold_iteration", + "correlation_threshold_iteration", + ), + ("iterations", "iterations"), + ("convergence_iterations", "convergence_iterations"), + ], + ), + (detect, outputspec, [("qpp", "qpp")]), + ] + ) return workflow diff --git a/CPAC/qpp/qpp.py b/CPAC/qpp/qpp.py index 9d6d7d66c0..0846b33865 100644 --- a/CPAC/qpp/qpp.py +++ b/CPAC/qpp/qpp.py @@ -1,49 +1,47 @@ import numpy as np -import nibabel as nib -import os -from numpy import ndarray -import matplotlib.pyplot as plt -from scipy.signal import find_peaks -from nilearn.masking import apply_mask -from nilearn.masking import compute_epi_mask import numpy.matlib +from scipy.signal import find_peaks from CPAC.utils import check_random_state, correlation def smooth(x): - """ - Temporary moving average - """ + """Temporary moving average.""" return np.array( - [x[0]] + - [np.mean(x[0:3])] + - (np.convolve(x, np.ones(5), 'valid') / 5).tolist() + - [np.mean(x[-3:])] + - [x[-1]] + [ + x[0], + np.mean(x[0:3]), + *(np.convolve(x, np.ones(5), "valid") / 5).tolist(), + np.mean(x[-3:]), + x[-1], + ] ) def flattened_segment(data, window_length, pos): - return data[:, pos:pos + window_length].flatten(order='F') + return data[:, pos : pos + window_length].flatten(order="F") def normalize_segment(segment, df): segment -= np.sum(segment) / df - segment = segment / np.sqrt(np.dot(segment, segment)) - return segment - - -def detect_qpp(data, num_scans, window_length, - permutations, correlation_threshold, - iterations, convergence_iterations=1, - random_state=None): + return segment / np.sqrt(np.dot(segment, segment)) + + +def detect_qpp( + data, + num_scans, + window_length, + permutations, + correlation_threshold, + iterations, + convergence_iterations=1, + random_state=None, +): """ This code is adapted from the paper "Quasi-periodic patterns (QP): Large- scale dynamics in resting state fMRI that correlate with local infraslow electrical activity", Shella Keilholz et al. NeuroImage, 2014. """ - random_state = check_random_state(random_state) voxels, trs = data.shape @@ -66,20 +64,24 @@ def detect_qpp(data, num_scans, window_length, permutation_result = [{} for _ in range(permutations)] for perm in range(permutations): - template_holder = np.zeros(trs) - random_initial_window = normalize_segment(flattened_segment(data, window_length, initial_trs[perm]), df) + random_initial_window = normalize_segment( + flattened_segment(data, window_length, initial_trs[perm]), df + ) for tr in inpectable_trs: - scan_window = normalize_segment(flattened_segment(data, window_length, tr), df) + scan_window = normalize_segment( + flattened_segment(data, window_length, tr), df + ) template_holder[tr] = np.dot(random_initial_window, scan_window) template_holder_convergence = np.zeros((convergence_iterations, trs)) for iteration in range(iterations): - peak_threshold = correlation_thresholds[iteration] - peaks, _ = find_peaks(template_holder, height=peak_threshold, distance=window_length) + peaks, _ = find_peaks( + template_holder, height=peak_threshold, distance=window_length + ) peaks = np.delete(peaks, np.where(~np.isin(peaks, inpectable_trs))[0]) template_holder = smooth(template_holder) @@ -90,16 +92,22 @@ def detect_qpp(data, num_scans, window_length, peaks_segments = flattened_segment(data, window_length, peaks[0]) for peak in peaks[1:]: - peaks_segments = peaks_segments + flattened_segment(data, window_length, peak) + peaks_segments = peaks_segments + flattened_segment( + data, window_length, peak + ) peaks_segments = peaks_segments / found_peaks peaks_segments = normalize_segment(peaks_segments, df) for tr in inpectable_trs: - scan_window = normalize_segment(flattened_segment(data, window_length, tr), df) + scan_window = normalize_segment( + flattened_segment(data, window_length, tr), df + ) template_holder[tr] = np.dot(peaks_segments, scan_window) - if np.all(correlation(template_holder, template_holder_convergence) > 0.9999): + if np.all( + correlation(template_holder, template_holder_convergence) > 0.9999 + ): break if convergence_iterations > 1: @@ -108,23 +116,25 @@ def detect_qpp(data, num_scans, window_length, if found_peaks > 1: permutation_result[perm] = { - 'template': template_holder, - 'peaks': peaks, - 'final_iteration': iteration, - 'correlation_score': np.sum(template_holder[peaks]), + "template": template_holder, + "peaks": peaks, + "final_iteration": iteration, + "correlation_score": np.sum(template_holder[peaks]), } # Retrieve max correlation of template from permutations - correlation_scores = np.array([ - r['correlation_score'] if r else 0.0 for r in permutation_result - ]) + correlation_scores = np.array( + [r["correlation_score"] if r else 0.0 for r in permutation_result] + ) if not np.any(correlation_scores): - raise Exception("C-PAC could not find QPP in your data. " - "Please lower your correlation threshold and try again.") + raise Exception( + "C-PAC could not find QPP in your data. " + "Please lower your correlation threshold and try again." + ) max_correlation = np.argsort(correlation_scores)[-1] - best_template = permutation_result[max_correlation]['template'] - best_selected_peaks = permutation_result[max_correlation]['peaks'] + best_template = permutation_result[max_correlation]["template"] + best_selected_peaks = permutation_result[max_correlation]["peaks"] best_template_metrics = [ np.median(best_template[best_selected_peaks]), @@ -133,13 +143,13 @@ def detect_qpp(data, num_scans, window_length, ] window_length_start = round(window_length / 2) - window_length_end = window_length_start - window_length % 2 + window_length_start - window_length % 2 best_template_segment = np.zeros((voxels, window_length)) for best_peak in best_selected_peaks: - start_tr = int(best_peak - np.ceil(window_length / 2.)) - end_tr = int(best_peak + np.floor(window_length / 2.)) + start_tr = int(best_peak - np.ceil(window_length / 2.0)) + end_tr = int(best_peak + np.floor(window_length / 2.0)) start_segment = np.zeros((voxels, 0)) if start_tr <= 0: @@ -153,11 +163,14 @@ def detect_qpp(data, num_scans, window_length, data_segment = data[:, start_tr:end_tr] - best_template_segment += np.concatenate([ - start_segment, - data_segment, - end_segment, - ], axis=1) + best_template_segment += np.concatenate( + [ + start_segment, + data_segment, + end_segment, + ], + axis=1, + ) best_template_segment /= len(best_selected_peaks) diff --git a/CPAC/qpp/tests/test_qpp.py b/CPAC/qpp/tests/test_qpp.py index 79ac7863c4..35545b1dba 100644 --- a/CPAC/qpp/tests/test_qpp.py +++ b/CPAC/qpp/tests/test_qpp.py @@ -2,15 +2,14 @@ import numpy as np import pytest import scipy.io + from CPAC.qpp.qpp import detect_qpp np.random.seed(10) -@pytest.mark.skip( - reason="No such file or directory: 'CPAC/qpp/tests/matlab/qpp.mat'") +@pytest.mark.skip(reason="No such file or directory: 'CPAC/qpp/tests/matlab/qpp.mat'") def test_qpp(): - voxels, trs = 600, 200 window_length = 15 @@ -19,10 +18,9 @@ def test_qpp(): x -= x.mean() x /= x.std() - scipy.io.savemat('CPAC/qpp/tests/matlab/qpp.mat', mdict={ - 'B': x, - 'msk': np.ones((voxels)) - }) + scipy.io.savemat( + "CPAC/qpp/tests/matlab/qpp.mat", mdict={"B": x, "msk": np.ones((voxels))} + ) best_template_segment, best_selected_peaks, best_template_metrics = detect_qpp( data=x, @@ -31,30 +29,56 @@ def test_qpp(): permutations=1, correlation_threshold=0.3, iterations=1, - convergence_iterations=1 + convergence_iterations=1, ) plt.figure() - plt.fill_between(range(trs), x.mean(axis=0) - x.std(axis=0), x.mean(axis=0) + x.std(axis=0), facecolor='blue', alpha=0.2) - plt.plot(range(trs), x.mean(axis=0), label="Data", color='blue') + plt.fill_between( + range(trs), + x.mean(axis=0) - x.std(axis=0), + x.mean(axis=0) + x.std(axis=0), + facecolor="blue", + alpha=0.2, + ) + plt.plot(range(trs), x.mean(axis=0), label="Data", color="blue") convolved = np.zeros((voxels, trs)) for i in range(trs): - convolved[i] = (np.convolve(x[i], best_template_segment[i], mode='full') / window_length)[:trs] + convolved[i] = ( + np.convolve(x[i], best_template_segment[i], mode="full") / window_length + )[:trs] - plt.fill_between(range(trs), convolved.mean(axis=0) - convolved.std(axis=0), convolved.mean(axis=0) + convolved.std(axis=0), facecolor='red', alpha=0.2) - plt.plot(range(trs), convolved.mean(axis=0), label="Convolved QPP", color='red') + plt.fill_between( + range(trs), + convolved.mean(axis=0) - convolved.std(axis=0), + convolved.mean(axis=0) + convolved.std(axis=0), + facecolor="red", + alpha=0.2, + ) + plt.plot(range(trs), convolved.mean(axis=0), label="Convolved QPP", color="red") - plt.plot(range(-window_length, 0), best_template_segment.mean(axis=0), '--', label="QPP", color='green') - plt.axvline(x=0, color='black') + plt.plot( + range(-window_length, 0), + best_template_segment.mean(axis=0), + "--", + label="QPP", + color="green", + ) + plt.axvline(x=0, color="black") plt.legend() plt.show() plt.figure() - plt.fill_between(range(trs), x.mean(axis=0) - x.std(axis=0), x.mean(axis=0) + x.std(axis=0), facecolor='blue', alpha=0.2) - plt.plot(range(trs), x.mean(axis=0), label="Data", color='blue') + plt.fill_between( + range(trs), + x.mean(axis=0) - x.std(axis=0), + x.mean(axis=0) + x.std(axis=0), + facecolor="blue", + alpha=0.2, + ) + plt.plot(range(trs), x.mean(axis=0), label="Data", color="blue") for xc in best_selected_peaks: - plt.axvline(x=xc, color='r') + plt.axvline(x=xc, color="r") plt.legend() - plt.show() \ No newline at end of file + plt.show() diff --git a/CPAC/randomise/pipeline.py b/CPAC/randomise/pipeline.py index cb606bd002..35aca2f6c0 100644 --- a/CPAC/randomise/pipeline.py +++ b/CPAC/randomise/pipeline.py @@ -1,113 +1,147 @@ -from CPAC.pipeline import nipype_pipeline_engine as pe -import nipype.interfaces.utility as util -from nipype.interfaces.base import BaseInterface, BaseInterfaceInputSpec, traits, File, TraitedSpec -from nipype.interfaces import fsl - -from nilearn import input_data, masking, image, datasets -from nilearn.image import resample_to_img, concat_imgs -from nilearn.input_data import NiftiMasker, NiftiLabelsMasker - -from CPAC.utils.interfaces.function import Function - import os -import copy -import numpy as np -import nibabel as nb +from nipype.interfaces import fsl +import nipype.interfaces.utility as util +from CPAC.pipeline import nipype_pipeline_engine as pe +from CPAC.utils.interfaces.function import Function -def create_randomise(name='randomise',working_dir=None,crash_dir=None): +def create_randomise(name="randomise", working_dir=None, crash_dir=None): """ Parameters ---------- - + Returns ------- workflow : nipype.pipeline.engine.Workflow Randomise workflow. - + Notes ----- - Workflow Inputs:: - - + + Workflow Outputs:: - + References ---------- - - """ + """ if not working_dir: - working_dir = os.path.join(os.getcwd(), 'Randomise_work_dir') + working_dir = os.path.join(os.getcwd(), "Randomise_work_dir") if not crash_dir: - crash_dir = os.path.join(os.getcwd(), 'Randomise_crash_dir') + crash_dir = os.path.join(os.getcwd(), "Randomise_crash_dir") wf = pe.Workflow(name=name) wf.base_dir = working_dir - wf.config['execution'] = {'hash_method': 'timestamp', - 'crashdump_dir': os.path.abspath(crash_dir)} - - inputspec = pe.Node(util.IdentityInterface(fields=['subjects_list','pipeline_output_folder','permutations','mask_boolean','demean','c_thresh']),name='inputspec') - - outputspec = pe.Node(util.IdentityInterface(fields=['tstat_files' ,'t_corrected_p_files','index_file','threshold_file','localmax_txt_file','localmax_vol_file','max_file','mean_file','pval_file','size_file']), name='outputspec') - - - #merge = pe.Node(interface=fslMerge(), name='fsl_merge') - #merge.inputs.dimension = 't' - #merge.inputs.merged_file = "randomise_merged.nii.gz" - - #wf.connect(inputspec, 'subjects', merge, 'in_files') - - #mask = pe.Node(interface=fsl.maths.MathsCommand(), name='fsl_maths') - #mask.inputs.args = '-abs -Tmin -bin' - #mask.inputs.out_file = "randomise_mask.nii.gz" - #wf.connect(inputspec, 'subjects', mask, 'in_file') - - randomise = pe.Node(interface=fsl.Randomise(), name='randomise') + wf.config["execution"] = { + "hash_method": "timestamp", + "crashdump_dir": os.path.abspath(crash_dir), + } + + inputspec = pe.Node( + util.IdentityInterface( + fields=[ + "subjects_list", + "pipeline_output_folder", + "permutations", + "mask_boolean", + "demean", + "c_thresh", + ] + ), + name="inputspec", + ) + + outputspec = pe.Node( + util.IdentityInterface( + fields=[ + "tstat_files", + "t_corrected_p_files", + "index_file", + "threshold_file", + "localmax_txt_file", + "localmax_vol_file", + "max_file", + "mean_file", + "pval_file", + "size_file", + ] + ), + name="outputspec", + ) + + # merge = pe.Node(interface=fslMerge(), name='fsl_merge') + # merge.inputs.dimension = 't' + # merge.inputs.merged_file = "randomise_merged.nii.gz" + + # wf.connect(inputspec, 'subjects', merge, 'in_files') + + # mask = pe.Node(interface=fsl.maths.MathsCommand(), name='fsl_maths') + # mask.inputs.args = '-abs -Tmin -bin' + # mask.inputs.out_file = "randomise_mask.nii.gz" + # wf.connect(inputspec, 'subjects', mask, 'in_file') + + randomise = pe.Node(interface=fsl.Randomise(), name="randomise") randomise.inputs.base_name = "randomise" randomise.inputs.demean = True randomise.inputs.tfce = True - wf.connect([(inputspec, randomise, [('subjects', 'in_file'), - ('design_matrix_file', 'design_mat'), - ('constrast_file', 'tcon'), - ('permutations', 'num_perm'), - ])]) - wf.connect(randomise,'tstat_files',outputspec,'tstat_files') - wf.connect(randomise,'t_corrected_p_files',outputspec,'t_corrected_p_files') -#------------- issue here arises while using tfce. By not using tfce, you don't get t_corrected_p files. R V in a conundrum? --------------------# - - - select_tcorrp_files = pe.Node(Function(input_names=['input_list'],output_names=['out_file'],function=select),name='select_t_corrp') - - wf.connect(randomise, 't_corrected_p_files',select_tcorrp_files, 'input_list') - wf.connect(select_tcorrp_files,'out_file',outputspec,'out_tcorr_corrected') - - select_tstat_files = pe.Node(Function(input_names=['input_list'],output_names=['out_file'],function=select),name='select_t_stat') - - wf.connect(randomise, 'tstat_files',select_tstat_files, 'input_list') - wf.connect(select_tstat_files,'out_file',outputspec,'out_tstat_corrected') - - thresh = pe.Node(interface=fsl.Threshold(),name='fsl_threshold_contrast') + wf.connect( + [ + ( + inputspec, + randomise, + [ + ("subjects", "in_file"), + ("design_matrix_file", "design_mat"), + ("constrast_file", "tcon"), + ("permutations", "num_perm"), + ], + ) + ] + ) + wf.connect(randomise, "tstat_files", outputspec, "tstat_files") + wf.connect(randomise, "t_corrected_p_files", outputspec, "t_corrected_p_files") + # ------------- issue here arises while using tfce. By not using tfce, you don't get t_corrected_p files. R V in a conundrum? --------------------# + + select_tcorrp_files = pe.Node( + Function( + input_names=["input_list"], output_names=["out_file"], function=select + ), + name="select_t_corrp", + ) + + wf.connect(randomise, "t_corrected_p_files", select_tcorrp_files, "input_list") + wf.connect(select_tcorrp_files, "out_file", outputspec, "out_tcorr_corrected") + + select_tstat_files = pe.Node( + Function( + input_names=["input_list"], output_names=["out_file"], function=select + ), + name="select_t_stat", + ) + + wf.connect(randomise, "tstat_files", select_tstat_files, "input_list") + wf.connect(select_tstat_files, "out_file", outputspec, "out_tstat_corrected") + + thresh = pe.Node(interface=fsl.Threshold(), name="fsl_threshold_contrast") thresh.inputs.thresh = 0.95 - thresh.inputs.out_file = 'rando_pipe_thresh_tstat.nii.gz' - wf.connect(select_tstat_files, 'out_file', thresh, 'in_file') - wf.connect(thresh,'out_file',outputspec,'rando_pipe_thresh_tstat.nii.gz') + thresh.inputs.out_file = "rando_pipe_thresh_tstat.nii.gz" + wf.connect(select_tstat_files, "out_file", thresh, "in_file") + wf.connect(thresh, "out_file", outputspec, "rando_pipe_thresh_tstat.nii.gz") - thresh_bin = pe.Node(interface=fsl.UnaryMaths(),name='fsl_threshold_bin_contrast') - thresh_bin.inputs.operation = 'bin' - wf.connect(thresh, 'out_file', thresh_bin, 'in_file') - wf.connect(thresh_bin,'out_file',outputspec,'thresh_bin_out') + thresh_bin = pe.Node(interface=fsl.UnaryMaths(), name="fsl_threshold_bin_contrast") + thresh_bin.inputs.operation = "bin" + wf.connect(thresh, "out_file", thresh_bin, "in_file") + wf.connect(thresh_bin, "out_file", outputspec, "thresh_bin_out") - apply_mask = pe.Node(interface=fsl.ApplyMask(),name='fsl_applymask_contrast') - wf.connect(select_tstat_files, 'out_file', apply_mask, 'in_file') - wf.connect(thresh_bin, 'out_file', apply_mask, 'mask_file') + apply_mask = pe.Node(interface=fsl.ApplyMask(), name="fsl_applymask_contrast") + wf.connect(select_tstat_files, "out_file", apply_mask, "in_file") + wf.connect(thresh_bin, "out_file", apply_mask, "mask_file") - - cluster = pe.Node(interface=fsl.Cluster(),name='cluster_contrast') + cluster = pe.Node(interface=fsl.Cluster(), name="cluster_contrast") cluster.inputs.threshold = 0.0001 cluster.inputs.out_index_file = "index_file" cluster.inputs.out_localmax_txt_file = "lmax_contrast.txt" @@ -117,19 +151,16 @@ def create_randomise(name='randomise',working_dir=None,crash_dir=None): cluster.inputs.out_mean_file = True cluster.inputs.out_pval_file = True cluster.inputs.out_size_file = True - - - wf.connect(apply_mask, 'out_file', cluster, 'in_file') - - wf.connect(cluster,'index_file',outputspec,'index_file') - wf.connect(cluster,'threshold_file',outputspec,'threshold_file') - wf.connect(cluster,'localmax_txt_file',outputspec,'localmax_txt_file') - wf.connect(cluster,'localmax_vol_file',outputspec,'localmax_vol_file') - wf.connect(cluster,'max_file',outputspec,'max_file') - wf.connect(cluster,'mean_file',outputspec,'meal_file') - wf.connect(cluster,'pval_file',outputspec,'pval_file') - wf.connect(cluster,'size_file',outputspec,'size_file') - - + + wf.connect(apply_mask, "out_file", cluster, "in_file") + + wf.connect(cluster, "index_file", outputspec, "index_file") + wf.connect(cluster, "threshold_file", outputspec, "threshold_file") + wf.connect(cluster, "localmax_txt_file", outputspec, "localmax_txt_file") + wf.connect(cluster, "localmax_vol_file", outputspec, "localmax_vol_file") + wf.connect(cluster, "max_file", outputspec, "max_file") + wf.connect(cluster, "mean_file", outputspec, "meal_file") + wf.connect(cluster, "pval_file", outputspec, "pval_file") + wf.connect(cluster, "size_file", outputspec, "size_file") return wf diff --git a/CPAC/randomise/randomise.py b/CPAC/randomise/randomise.py index 188d0fd436..46a51756f2 100644 --- a/CPAC/randomise/randomise.py +++ b/CPAC/randomise/randomise.py @@ -1,35 +1,44 @@ - from CPAC.pipeline import nipype_pipeline_engine as pe def select(input_list): - import nibabel as nb + import nibabel as nib + for i in input_list: - img = nb.load(i) + img = nib.load(i) hdr = img.header - if hdr['cal_min'] == 0 and hdr['cal_max'] == 0: - print("Warning! {} is an empty image because of no positive " - "values in the unpermuted statistic image, and it could " - "not be processed with tfce.".format('i')) - if not hdr['cal_max'] == 0 and hdr['cal_min'] == 0: - selected_file = i + if hdr["cal_min"] == 0 and hdr["cal_max"] == 0: + pass + if not hdr["cal_max"] == 0 and hdr["cal_min"] == 0: + pass return i -def prep_randomise_workflow(c, merged_file, mask_file, f_test, mat_file, - con_file, grp_file, output_dir, working_dir, - log_dir, model_name, fts_file=None): - - import nipype.interfaces.utility as util - import nipype.interfaces.fsl as fsl +def prep_randomise_workflow( + c, + merged_file, + mask_file, + f_test, + mat_file, + con_file, + grp_file, + output_dir, + working_dir, + log_dir, + model_name, + fts_file=None, +): + from nipype.interfaces import fsl import nipype.interfaces.io as nio + import nipype.interfaces.utility as util - wf = pe.Workflow(name='randomise_workflow') + wf = pe.Workflow(name="randomise_workflow") wf.base_dir = c.work_dir - randomise = pe.Node(interface=fsl.Randomise(), - name='fsl-randomise_{0}'.format(model_name)) + randomise = pe.Node( + interface=fsl.Randomise(), name="fsl-randomise_{0}".format(model_name) + ) randomise.inputs.base_name = model_name randomise.inputs.in_file = merged_file randomise.inputs.mask = mask_file @@ -44,39 +53,38 @@ def prep_randomise_workflow(c, merged_file, mask_file, f_test, mat_file, if fts_file: randomise.inputs.fcon = fts_file - select_tcorrp_files = pe.Node(util.Function(input_names=['input_list'], - output_names=['out_file'], - function=select), - name='select_t_corrp') + select_tcorrp_files = pe.Node( + util.Function( + input_names=["input_list"], output_names=["out_file"], function=select + ), + name="select_t_corrp", + ) - wf.connect(randomise, 't_corrected_p_files', select_tcorrp_files, 'input_list') + wf.connect(randomise, "t_corrected_p_files", select_tcorrp_files, "input_list") + select_tstat_files = pe.Node( + util.Function( + input_names=["input_list"], output_names=["out_file"], function=select + ), + name="select_t_stat", + ) - select_tstat_files = pe.Node(util.Function(input_names=['input_list'], - output_names=['out_file'], - function=select), - name='select_t_stat') + wf.connect(randomise, "tstat_files", select_tstat_files, "input_list") - wf.connect(randomise, 'tstat_files', select_tstat_files, 'input_list') - - thresh = pe.Node(interface=fsl.Threshold(), - name='fsl_threshold_contrast') + thresh = pe.Node(interface=fsl.Threshold(), name="fsl_threshold_contrast") thresh.inputs.thresh = 0.95 - thresh.inputs.out_file = 'randomise_pipe_thresh_tstat.nii.gz' - wf.connect(select_tstat_files, 'out_file', thresh, 'in_file') + thresh.inputs.out_file = "randomise_pipe_thresh_tstat.nii.gz" + wf.connect(select_tstat_files, "out_file", thresh, "in_file") - thresh_bin = pe.Node(interface=fsl.UnaryMaths(), - name='fsl_threshold_bin_contrast') - thresh_bin.inputs.operation = 'bin' - wf.connect(thresh, 'out_file', thresh_bin, 'in_file') + thresh_bin = pe.Node(interface=fsl.UnaryMaths(), name="fsl_threshold_bin_contrast") + thresh_bin.inputs.operation = "bin" + wf.connect(thresh, "out_file", thresh_bin, "in_file") - apply_mask = pe.Node(interface=fsl.ApplyMask(), - name='fsl_applymask_contrast') - wf.connect(select_tstat_files, 'out_file', apply_mask, 'in_file') - wf.connect(thresh_bin, 'out_file', apply_mask, 'mask_file') + apply_mask = pe.Node(interface=fsl.ApplyMask(), name="fsl_applymask_contrast") + wf.connect(select_tstat_files, "out_file", apply_mask, "in_file") + wf.connect(thresh_bin, "out_file", apply_mask, "mask_file") - cluster = pe.Node(interface=fsl.Cluster(), - name='cluster_contrast') + cluster = pe.Node(interface=fsl.Cluster(), name="cluster_contrast") cluster.inputs.threshold = 0.0001 cluster.inputs.out_index_file = "index_file" cluster.inputs.out_localmax_txt_file = "lmax_contrast.txt" @@ -87,46 +95,43 @@ def prep_randomise_workflow(c, merged_file, mask_file, f_test, mat_file, cluster.inputs.out_pval_file = True cluster.inputs.out_size_file = True - wf.connect(apply_mask, 'out_file', cluster, 'in_file') + wf.connect(apply_mask, "out_file", cluster, "in_file") - ds = pe.Node(nio.DataSink(), name='fsl-randomise_sink') + ds = pe.Node(nio.DataSink(), name="fsl-randomise_sink") ds.inputs.base_directory = str(output_dir) - ds.inputs.container = '' - - wf.connect(randomise,'tstat_files', ds,'tstat_files') - wf.connect(randomise,'t_corrected_p_files', ds,'t_corrected_p_files') - wf.connect(select_tcorrp_files,'out_file', ds,'out_tcorr_corrected') - wf.connect(select_tstat_files,'out_file', ds,'out_tstat_corrected') - wf.connect(thresh,'out_file', ds,'randomise_pipe_thresh_tstat.nii.gz') - wf.connect(thresh_bin,'out_file', ds,'thresh_bin_out') - wf.connect(cluster,'index_file', ds,'index_file') - wf.connect(cluster,'threshold_file', ds,'threshold_file') - wf.connect(cluster,'localmax_txt_file', ds,'localmax_txt_file') - wf.connect(cluster,'localmax_vol_file', ds,'localmax_vol_file') - wf.connect(cluster,'max_file', ds,'max_file') - wf.connect(cluster,'mean_file', ds,'meal_file') - wf.connect(cluster,'pval_file', ds,'pval_file') - wf.connect(cluster,'size_file', ds,'size_file') + ds.inputs.container = "" + + wf.connect(randomise, "tstat_files", ds, "tstat_files") + wf.connect(randomise, "t_corrected_p_files", ds, "t_corrected_p_files") + wf.connect(select_tcorrp_files, "out_file", ds, "out_tcorr_corrected") + wf.connect(select_tstat_files, "out_file", ds, "out_tstat_corrected") + wf.connect(thresh, "out_file", ds, "randomise_pipe_thresh_tstat.nii.gz") + wf.connect(thresh_bin, "out_file", ds, "thresh_bin_out") + wf.connect(cluster, "index_file", ds, "index_file") + wf.connect(cluster, "threshold_file", ds, "threshold_file") + wf.connect(cluster, "localmax_txt_file", ds, "localmax_txt_file") + wf.connect(cluster, "localmax_vol_file", ds, "localmax_vol_file") + wf.connect(cluster, "max_file", ds, "max_file") + wf.connect(cluster, "mean_file", ds, "meal_file") + wf.connect(cluster, "pval_file", ds, "pval_file") + wf.connect(cluster, "size_file", ds, "size_file") wf.run() - def run(group_config_path): - import re import subprocess - subprocess.getoutput('source ~/.bashrc') + + subprocess.getoutput("source ~/.bashrc") import os - import sys - import pickle - import yaml + from CPAC.pipeline.cpac_group_runner import load_config_yml group_config_obj = load_config_yml(group_config_path) pipeline_output_folder = group_config_obj.pipeline_dir - if not group_config_obj.participant_list == None: + if group_config_obj.participant_list is not None: s_paths = group_config_obj.participant_list else: s_paths = [x for x in os.listdir(pipeline_output_folder) if os.path.isdir(x)] @@ -135,4 +140,11 @@ def run(group_config_path): out_file = randomise_merged_mask(s_paths) - prep_randomise_workflow(group_config_obj, merged_file=merged_file,mask_file=out_file,working_dir=None,output_dir=None,crash_dir=None) + prep_randomise_workflow( + group_config_obj, + merged_file=merged_file, + mask_file=out_file, + working_dir=None, + output_dir=None, + crash_dir=None, + ) diff --git a/CPAC/randomise/test_randomise.py b/CPAC/randomise/test_randomise.py index b61d34558d..06feda7672 100644 --- a/CPAC/randomise/test_randomise.py +++ b/CPAC/randomise/test_randomise.py @@ -1,23 +1,22 @@ import os -import glob -import nipype.interfaces.io as nio + import pytest +import nipype.interfaces.io as nio from CPAC.pipeline import nipype_pipeline_engine as pe from CPAC.pipeline.nipype_pipeline_engine.plugins import MultiProcPlugin -from nipype.interfaces.fsl import ImageStats @pytest.mark.skip(reason="test needs refactoring") @pytest.mark.parametrize( - 'inputs', - [['subjects', 'design_matrix_file', 'contrast_file', 'permutations']] + "inputs", [["subjects", "design_matrix_file", "contrast_file", "permutations"]] ) def test_run_randomize(inputs, output_dir=None, run=True): from . import pipeline - randomise_workflow = pe.Workflow(name='preproc') + + randomise_workflow = pe.Workflow(name="preproc") if output_dir is None: - output_dir = '' + output_dir = "" workflow_dir = os.path.join(output_dir, "randomise_results") randomise_workflow.base_dir = workflow_dir @@ -27,38 +26,39 @@ def test_run_randomize(inputs, output_dir=None, run=True): num_of_cores = 1 t_node = pipeline.create_randomise() - t_node.inputs.inputspec.subjects = '' - t_node.inputs.inputspec.design_matrix_file = '' - t_node.inputs.inputspec.constrast_file = '' + t_node.inputs.inputspec.subjects = "" + t_node.inputs.inputspec.design_matrix_file = "" + t_node.inputs.inputspec.constrast_file = "" # t_node.inputs.inputspec.f_constrast_file = '' t_node.inputs.inputspec.permutations = 5000 # t_node.inputs.inputspec.mask = '' # - dataSink = pe.Node(nio.DataSink(), name='dataSink_file') + dataSink = pe.Node(nio.DataSink(), name="dataSink_file") dataSink.inputs.base_directory = workflow_dir - randomise_workflow.connect(t_node, 'outputspec.index_file', - dataSink, 'index_file') + randomise_workflow.connect(t_node, "outputspec.index_file", dataSink, "index_file") # randomise_workflow.connect(t_node, 'outputspec.thresh_out', # dataSink,'threshold_file') - randomise_workflow.connect(t_node, 'outputspec.localmax_txt_file', - dataSink, 'localmax_txt_file') - randomise_workflow.connect(t_node, 'outputspec.localmax_vol_file', - dataSink, 'localmax_vol_file') - randomise_workflow.connect(t_node, 'outputspec.max_file', - dataSink, 'max_file') - randomise_workflow.connect(t_node, 'outputspec.mean_file', - dataSink, 'mean_file') - randomise_workflow.connect(t_node, 'outputspec.pval_file', - dataSink, 'pval_file') - randomise_workflow.connect(t_node, 'outputspec.size_file', - dataSink, 'size_file') - randomise_workflow.connect(t_node, 'outputspec.tstat_files', - dataSink, 'tstat_files') - randomise_workflow.connect(t_node, 'outputspec.t_corrected_p_files', - dataSink, 't_corrected_p_files') + randomise_workflow.connect( + t_node, "outputspec.localmax_txt_file", dataSink, "localmax_txt_file" + ) + randomise_workflow.connect( + t_node, "outputspec.localmax_vol_file", dataSink, "localmax_vol_file" + ) + randomise_workflow.connect(t_node, "outputspec.max_file", dataSink, "max_file") + randomise_workflow.connect(t_node, "outputspec.mean_file", dataSink, "mean_file") + randomise_workflow.connect(t_node, "outputspec.pval_file", dataSink, "pval_file") + randomise_workflow.connect(t_node, "outputspec.size_file", dataSink, "size_file") + randomise_workflow.connect( + t_node, "outputspec.tstat_files", dataSink, "tstat_files" + ) + randomise_workflow.connect( + t_node, "outputspec.t_corrected_p_files", dataSink, "t_corrected_p_files" + ) if run is True: - plugin_args = {'n_procs': num_of_cores} - randomise_workflow.run(plugin=MultiProcPlugin(plugin_args), - plugin_args=plugin_args) + plugin_args = {"n_procs": num_of_cores} + randomise_workflow.run( + plugin=MultiProcPlugin(plugin_args), plugin_args=plugin_args + ) + return None else: return randomise_workflow, randomise_workflow.base_dir diff --git a/CPAC/registration/__init__.py b/CPAC/registration/__init__.py index 2d5b934e1c..94585cdfc5 100644 --- a/CPAC/registration/__init__.py +++ b/CPAC/registration/__init__.py @@ -1,17 +1,19 @@ -from .registration import create_fsl_flirt_linear_reg, \ - create_fsl_fnirt_nonlinear_reg, \ - create_fsl_fnirt_nonlinear_reg_nhp, \ - create_register_func_to_anat, \ - create_register_func_to_anat_use_T2, \ - create_bbregister_func_to_anat, \ - create_wf_calculate_ants_warp +from .registration import ( + create_bbregister_func_to_anat, + create_fsl_flirt_linear_reg, + create_fsl_fnirt_nonlinear_reg, + create_fsl_fnirt_nonlinear_reg_nhp, + create_register_func_to_anat, + create_register_func_to_anat_use_T2, + create_wf_calculate_ants_warp, +) -from .output_func_to_standard import output_func_to_standard - -__all__ = ['create_fsl_flirt_linear_reg', \ - 'create_fsl_fnirt_nonlinear_reg', \ - 'create_fsl_fnirt_nonlinear_reg_nhp', \ - 'create_register_func_to_anat', \ - 'create_register_func_to_anat_use_T2', \ - 'create_bbregister_func_to_anat', \ - 'create_wf_calculate_ants_warp'] \ No newline at end of file +__all__ = [ + "create_fsl_flirt_linear_reg", + "create_fsl_fnirt_nonlinear_reg", + "create_fsl_fnirt_nonlinear_reg_nhp", + "create_register_func_to_anat", + "create_register_func_to_anat_use_T2", + "create_bbregister_func_to_anat", + "create_wf_calculate_ants_warp", +] diff --git a/CPAC/registration/output_func_to_standard.py b/CPAC/registration/output_func_to_standard.py index 10ed67b15b..2119798de0 100644 --- a/CPAC/registration/output_func_to_standard.py +++ b/CPAC/registration/output_func_to_standard.py @@ -1,34 +1,32 @@ -import nipype.interfaces.fsl as fsl -from CPAC.pipeline import nipype_pipeline_engine as pe +from nipype.interfaces import ants, c3, fsl +from nipype.interfaces.afni import utils as afni_utils import nipype.interfaces.utility as util -import nipype.interfaces.ants as ants -import nipype.interfaces.c3 as c3 + +from CPAC.func_preproc.utils import chunk_ts, split_ts_chunks +from CPAC.pipeline import nipype_pipeline_engine as pe from CPAC.registration.utils import ( change_itk_transform_type, check_transforms, - generate_inverse_transform_flags) - -from nipype.interfaces.afni import utils as afni_utils -from CPAC.func_preproc.utils import chunk_ts, split_ts_chunks + generate_inverse_transform_flags, +) from CPAC.utils.interfaces.function import Function -from CPAC.vmhc.utils import get_img_nvols # Todo: CC distcor is not implement for fsl apply xform func to mni, why ?? def fsl_apply_transform_func_to_mni( - workflow, - output_name, - func_key, - ref_key, - num_strat, - strat, - interpolation_method, - distcor=False, - map_node=False, - func_ts=False, - num_cpus=1 - ): + workflow, + output_name, + func_key, + ref_key, + num_strat, + strat, + interpolation_method, + distcor=False, + map_node=False, + func_ts=False, + num_cpus=1, +): """ Applies previously calculated FSL registration transforms to input images. This workflow employs the FSL applywarp tool: @@ -74,10 +72,9 @@ def fsl_apply_transform_func_to_mni( workflow : nipype.pipeline.engine.Workflow """ - strat_nodes = strat.get_nodes_names() - # if the input is a string, assume that it is resource pool key, + # if the input is a string, assume that it is resource pool key, # if it is a tuple, assume that it is a node, outfile pair, # otherwise, something funky is going on if isinstance(func_key, str): @@ -97,152 +94,145 @@ def fsl_apply_transform_func_to_mni( # parallelize time series warp application map_node = True - if map_node == True: + if map_node is True: # func_mni_warp - func_mni_warp = pe.MapNode(interface=fsl.ApplyWarp(), - name='func_mni_fsl_warp_{0}_{1:d}'.format(output_name, num_strat), - iterfield=['in_file'], - mem_gb=1.5) + func_mni_warp = pe.MapNode( + interface=fsl.ApplyWarp(), + name="func_mni_fsl_warp_{0}_{1:d}".format(output_name, num_strat), + iterfield=["in_file"], + mem_gb=1.5, + ) else: # func_mni_warp - func_mni_warp = pe.Node(interface=fsl.ApplyWarp(), - name='func_mni_fsl_warp_{0}_{1:d}'.format(output_name, num_strat)) + func_mni_warp = pe.Node( + interface=fsl.ApplyWarp(), + name="func_mni_fsl_warp_{0}_{1:d}".format(output_name, num_strat), + ) func_mni_warp.inputs.interp = interpolation_method # parallelize the apply warp, if multiple CPUs, and it's a time series! if int(num_cpus) > 1 and func_ts: - - node_id = '{0}_{1:d}'.format(output_name, num_strat) - - chunk_imports = ['import nibabel as nb'] - chunk = pe.Node(Function(input_names=['func_file', - 'n_cpus'], - output_names=['TR_ranges'], - function=chunk_ts, - imports=chunk_imports), - name=f'chunk_{node_id}') + node_id = "{0}_{1:d}".format(output_name, num_strat) + + chunk_imports = ["import nibabel as nb"] + chunk = pe.Node( + Function( + input_names=["func_file", "n_cpus"], + output_names=["TR_ranges"], + function=chunk_ts, + imports=chunk_imports, + ), + name=f"chunk_{node_id}", + ) chunk.inputs.n_cpus = int(num_cpus) - workflow.connect(func_node, func_file, chunk, 'func_file') + workflow.connect(func_node, func_file, chunk, "func_file") - split_imports = ['import os', 'import subprocess'] - split = pe.Node(Function(input_names=['func_file', - 'tr_ranges'], - output_names=['split_funcs'], - function=split_ts_chunks, - imports=split_imports), - name=f'split_{node_id}') + split_imports = ["import os", "import subprocess"] + split = pe.Node( + Function( + input_names=["func_file", "tr_ranges"], + output_names=["split_funcs"], + function=split_ts_chunks, + imports=split_imports, + ), + name=f"split_{node_id}", + ) - workflow.connect(func_node, func_file, split, 'func_file') - workflow.connect(chunk, 'TR_ranges', split, 'tr_ranges') + workflow.connect(func_node, func_file, split, "func_file") + workflow.connect(chunk, "TR_ranges", split, "tr_ranges") - workflow.connect(split, 'split_funcs', func_mni_warp, 'in_file') + workflow.connect(split, "split_funcs", func_mni_warp, "in_file") - func_concat = pe.Node(interface=afni_utils.TCat(), - name=f'func_concat{node_id}') - func_concat.inputs.outputtype = 'NIFTI_GZ' + func_concat = pe.Node(interface=afni_utils.TCat(), name=f"func_concat{node_id}") + func_concat.inputs.outputtype = "NIFTI_GZ" - workflow.connect(func_mni_warp, 'out_file', - func_concat, 'in_files') + workflow.connect(func_mni_warp, "out_file", func_concat, "in_files") - strat.update_resource_pool({ - output_name: (func_concat, 'out_file') - }) + strat.update_resource_pool({output_name: (func_concat, "out_file")}) else: - workflow.connect(func_node, func_file, - func_mni_warp, 'in_file') - strat.update_resource_pool({output_name: (func_mni_warp, 'out_file')}) - - workflow.connect(ref_node, ref_out_file, - func_mni_warp, 'ref_file') + workflow.connect(func_node, func_file, func_mni_warp, "in_file") + strat.update_resource_pool({output_name: (func_mni_warp, "out_file")}) - if 'anat_mni_fnirt_register' in strat_nodes: + workflow.connect(ref_node, ref_out_file, func_mni_warp, "ref_file") - node, out_file = strat['functional_to_anat_linear_xfm'] - workflow.connect(node, out_file, - func_mni_warp, 'premat') + if "anat_mni_fnirt_register" in strat_nodes: + node, out_file = strat["functional_to_anat_linear_xfm"] + workflow.connect(node, out_file, func_mni_warp, "premat") - node, out_file = strat['anatomical_to_mni_nonlinear_xfm'] - workflow.connect(node, out_file, - func_mni_warp, 'field_file') + node, out_file = strat["anatomical_to_mni_nonlinear_xfm"] + workflow.connect(node, out_file, func_mni_warp, "field_file") if output_name == "functional_to_standard": - write_composite_xfm = pe.Node(interface=fsl.ConvertWarp(), - name='combine_fsl_warps_{0}_{1:d}'.format(output_name,\ - num_strat)) + write_composite_xfm = pe.Node( + interface=fsl.ConvertWarp(), + name="combine_fsl_warps_{0}_{1:d}".format(output_name, num_strat), + ) - workflow.connect(ref_node, ref_out_file, - write_composite_xfm, 'reference') + workflow.connect(ref_node, ref_out_file, write_composite_xfm, "reference") - node, out_file = strat['functional_to_anat_linear_xfm'] - workflow.connect(node, out_file, - write_composite_xfm, 'premat') + node, out_file = strat["functional_to_anat_linear_xfm"] + workflow.connect(node, out_file, write_composite_xfm, "premat") - node, out_file = strat['anatomical_to_mni_nonlinear_xfm'] - workflow.connect(node, out_file, - write_composite_xfm, 'warp1') + node, out_file = strat["anatomical_to_mni_nonlinear_xfm"] + workflow.connect(node, out_file, write_composite_xfm, "warp1") strat.update_resource_pool( - {"functional_to_standard_xfm": (write_composite_xfm, - 'out_file')}) + {"functional_to_standard_xfm": (write_composite_xfm, "out_file")} + ) - elif 'anat_mni_flirt_register' in strat_nodes: - - if 'functional_to_mni_linear_xfm' not in strat: - - combine_transforms = pe.Node(interface=fsl.ConvertXFM(), - name='combine_fsl_xforms_{0}_{1:d}'.format(output_name,\ - num_strat)) + elif "anat_mni_flirt_register" in strat_nodes: + if "functional_to_mni_linear_xfm" not in strat: + combine_transforms = pe.Node( + interface=fsl.ConvertXFM(), + name="combine_fsl_xforms_{0}_{1:d}".format(output_name, num_strat), + ) combine_transforms.inputs.concat_xfm = True - node, out_file = strat['anatomical_to_mni_linear_xfm'] - workflow.connect(node, out_file, - combine_transforms, 'in_file2') + node, out_file = strat["anatomical_to_mni_linear_xfm"] + workflow.connect(node, out_file, combine_transforms, "in_file2") - node, out_file = strat['functional_to_anat_linear_xfm'] - workflow.connect(node, out_file, - combine_transforms, 'in_file') + node, out_file = strat["functional_to_anat_linear_xfm"] + workflow.connect(node, out_file, combine_transforms, "in_file") - strat.update_resource_pool({'functional_to_mni_linear_xfm': - (combine_transforms, 'out_file')}) + strat.update_resource_pool( + {"functional_to_mni_linear_xfm": (combine_transforms, "out_file")} + ) strat.append_name(combine_transforms.name) - combine_transforms, outfile = strat['functional_to_mni_linear_xfm'] + combine_transforms, outfile = strat["functional_to_mni_linear_xfm"] + + workflow.connect(combine_transforms, outfile, func_mni_warp, "premat") - workflow.connect(combine_transforms, outfile, - func_mni_warp, 'premat') - else: - raise ValueError( - 'Could not find flirt or fnirt registration in nodes') - + raise ValueError("Could not find flirt or fnirt registration in nodes") + strat.append_name(func_mni_warp.name) return workflow def ants_apply_warps_func_mni( - workflow, - output_name, - func_key, - ref_key, - num_strat, - strat, - interpolation_method='LanczosWindowedSinc', - distcor=False, - map_node=False, - inverse=False, - symmetry='asymmetric', - input_image_type=0, - num_ants_cores=1, - registration_template='t1', - func_type='non-ica-aroma', - num_cpus=1 - ): - + workflow, + output_name, + func_key, + ref_key, + num_strat, + strat, + interpolation_method="LanczosWindowedSinc", + distcor=False, + map_node=False, + inverse=False, + symmetry="asymmetric", + input_image_type=0, + num_ants_cores=1, + registration_template="t1", + func_type="non-ica-aroma", + num_cpus=1, +): """ Applies previously calculated ANTS registration transforms to input images. This workflow employs the antsApplyTransforms tool: @@ -260,7 +250,6 @@ def ants_apply_warps_func_mni( Notes ----- - Workflow Inputs:: workflow : Nipype workflow object @@ -270,7 +259,7 @@ def ants_apply_warps_func_mni( resource pool func_key : string resource pool key correspoding to the node containing the 3D or 4D - functional file to be written into MNI space, use 'leaf' for a + functional file to be written into MNI space, use 'leaf' for a leaf node ref_key : string resource pool key correspoding to the file path to the template brain @@ -281,15 +270,15 @@ def ants_apply_warps_func_mni( a strategy with one or more resource pools interpolation_method : str which interpolation to use when applying the warps, commonly used - options are 'Linear', 'Bspline', 'LanczosWindowedSinc' (default) + options are 'Linear', 'Bspline', 'LanczosWindowedSinc' (default) for derivatives and image data 'NearestNeighbor' for masks distcor : boolean - indicates whether a distortion correction transformation should be + indicates whether a distortion correction transformation should be added to the transforms, this of course requires that a distortion correction map exist in the resource pool map_node : boolean - indicates whether a mapnode should be used, if TRUE func_key is - expected to correspond to a list of resources that should each + indicates whether a mapnode should be used, if TRUE func_key is + expected to correspond to a list of resources that should each be written into standard space with the other parameters inverse : boolean writes the invrse of the transform, i.e. MNI->EPI instead of @@ -310,21 +299,21 @@ def ants_apply_warps_func_mni( num_cpus : int the number of CPUs dedicated to each participant workflow - this is used to determine how to parallelize the warp application step - + Workflow Outputs:: - + outputspec.output_image : string (nifti file) Normalized output file - + Workflow Graph: - + .. image:: :width: 500 - + Detailed Workflow Graph: - - .. image:: + + .. image:: :width: 500 Apply the functional-to-structural and structural-to-template warps to @@ -333,8 +322,7 @@ def ants_apply_warps_func_mni( Parameters ---------- """ - - # if the input is a string, assume that it is resource pool key, + # if the input is a string, assume that it is resource pool key, # if it is a tuple, assume that it is a node, outfile pair, # otherwise, something funky is going on if isinstance(func_key, str): @@ -350,209 +338,230 @@ def ants_apply_warps_func_mni( elif isinstance(ref_key, tuple): ref_node, ref_out = func_key - # when inverse is enabled, we want to update the name of various # nodes so that we know they were inverted - inverse_string = '' + inverse_string = "" if inverse is True: - inverse_string = '_inverse' + inverse_string = "_inverse" # make sure that resource pool has some required resources before proceeding - if 'fsl_mat_as_itk' not in strat and registration_template == 't1': - - fsl_reg_2_itk = pe.Node(c3.C3dAffineTool(), - name='fsl_reg_2_itk_{0}'.format(num_strat)) + if "fsl_mat_as_itk" not in strat and registration_template == "t1": + fsl_reg_2_itk = pe.Node( + c3.C3dAffineTool(), name="fsl_reg_2_itk_{0}".format(num_strat) + ) fsl_reg_2_itk.inputs.itk_transform = True fsl_reg_2_itk.inputs.fsl2ras = True # convert the .mat from linear Func->Anat to # ANTS format - node, out_file = strat['functional_to_anat_linear_xfm'] - workflow.connect(node, out_file, fsl_reg_2_itk, 'transform_file') + node, out_file = strat["functional_to_anat_linear_xfm"] + workflow.connect(node, out_file, fsl_reg_2_itk, "transform_file") - node, out_file = strat['anatomical_brain'] - workflow.connect(node, out_file, fsl_reg_2_itk, 'reference_file') + node, out_file = strat["anatomical_brain"] + workflow.connect(node, out_file, fsl_reg_2_itk, "reference_file") - ref_node, ref_out = strat['mean_functional'] - workflow.connect(ref_node, ref_out, fsl_reg_2_itk, 'source_file') + ref_node, ref_out = strat["mean_functional"] + workflow.connect(ref_node, ref_out, fsl_reg_2_itk, "source_file") - itk_imports = ['import os'] - change_transform = pe.Node(util.Function( - input_names=['input_affine_file'], - output_names=['updated_affine_file'], + itk_imports = ["import os"] + change_transform = pe.Node( + util.Function( + input_names=["input_affine_file"], + output_names=["updated_affine_file"], function=change_itk_transform_type, - imports=itk_imports), - name='change_transform_type_{0}'.format(num_strat)) + imports=itk_imports, + ), + name="change_transform_type_{0}".format(num_strat), + ) - workflow.connect(fsl_reg_2_itk, 'itk_transform', - change_transform, 'input_affine_file') + workflow.connect( + fsl_reg_2_itk, "itk_transform", change_transform, "input_affine_file" + ) - strat.update_resource_pool({ - 'fsl_mat_as_itk': (change_transform, 'updated_affine_file') - }) + strat.update_resource_pool( + {"fsl_mat_as_itk": (change_transform, "updated_affine_file")} + ) strat.append_name(fsl_reg_2_itk.name) # stack of transforms to be combined to acheive the desired transformation num_transforms = 5 - collect_transforms_key = \ - 'collect_transforms{0}'.format(inverse_string) + collect_transforms_key = "collect_transforms{0}".format(inverse_string) - if distcor is True and func_type not in 'ica-aroma': + if distcor is True and func_type not in "ica-aroma": num_transforms = 6 - collect_transforms_key = \ - 'collect_transforms{0}{1}'.format('_distcor', - inverse_string) + collect_transforms_key = "collect_transforms{0}{1}".format( + "_distcor", inverse_string + ) - if collect_transforms_key not in strat: - if registration_template == 't1': + if collect_transforms_key not in strat: + if registration_template == "t1": # handle both symmetric and asymmetric transforms - ants_transformation_dict = { - 'asymmetric': { - 'anatomical_to_mni_nonlinear_xfm': 'anatomical_to_mni_nonlinear_xfm', - 'mni_to_anatomical_nonlinear_xfm': 'mni_to_anatomical_nonlinear_xfm', - 'ants_affine_xfm': 'ants_affine_xfm', - 'ants_rigid_xfm': 'ants_rigid_xfm', - 'ants_initial_xfm': 'ants_initial_xfm', - 'blip_warp': 'blip_warp', - 'blip_warp_inverse': 'blip_warp_inverse', - 'fsl_mat_as_itk': 'fsl_mat_as_itk', - }, - 'symmetric': { - 'anatomical_to_mni_nonlinear_xfm': 'anatomical_to_symmetric_mni_nonlinear_xfm', - 'mni_to_anatomical_nonlinear_xfm':'symmetric_mni_to_anatomical_nonlinear_xfm', - 'ants_affine_xfm': 'ants_symmetric_affine_xfm', - 'ants_rigid_xfm': 'ants_symmetric_rigid_xfm', - 'ants_initial_xfm': 'ants_symmetric_initial_xfm', - 'blip_warp': 'blip_warp', - 'blip_warp_inverse': 'blip_warp_inverse', - 'fsl_mat_as_itk': 'fsl_mat_as_itk', - } - } + ants_transformation_dict = { + "asymmetric": { + "anatomical_to_mni_nonlinear_xfm": "anatomical_to_mni_nonlinear_xfm", + "mni_to_anatomical_nonlinear_xfm": "mni_to_anatomical_nonlinear_xfm", + "ants_affine_xfm": "ants_affine_xfm", + "ants_rigid_xfm": "ants_rigid_xfm", + "ants_initial_xfm": "ants_initial_xfm", + "blip_warp": "blip_warp", + "blip_warp_inverse": "blip_warp_inverse", + "fsl_mat_as_itk": "fsl_mat_as_itk", + }, + "symmetric": { + "anatomical_to_mni_nonlinear_xfm": "anatomical_to_symmetric_mni_nonlinear_xfm", + "mni_to_anatomical_nonlinear_xfm": "symmetric_mni_to_anatomical_nonlinear_xfm", + "ants_affine_xfm": "ants_symmetric_affine_xfm", + "ants_rigid_xfm": "ants_symmetric_rigid_xfm", + "ants_initial_xfm": "ants_symmetric_initial_xfm", + "blip_warp": "blip_warp", + "blip_warp_inverse": "blip_warp_inverse", + "fsl_mat_as_itk": "fsl_mat_as_itk", + }, + } # transforms to be concatenated, the first element of each tuple is - # the resource pool key related to the resource that should be - # connected in, and the second element is the input to which it + # the resource pool key related to the resource that should be + # connected in, and the second element is the input to which it # should be connected if inverse is True: - if distcor is True and func_type not in 'ica-aroma': + if distcor is True and func_type not in "ica-aroma": # Field file from anatomical nonlinear registration - transforms_to_combine = [\ - ('mni_to_anatomical_nonlinear_xfm', 'in6'), - ('ants_affine_xfm', 'in5'), - ('ants_rigid_xfm', 'in4'), - ('ants_initial_xfm', 'in3'), - ('fsl_mat_as_itk', 'in2'), - ('blip_warp_inverse', 'in1')] + transforms_to_combine = [ + ("mni_to_anatomical_nonlinear_xfm", "in6"), + ("ants_affine_xfm", "in5"), + ("ants_rigid_xfm", "in4"), + ("ants_initial_xfm", "in3"), + ("fsl_mat_as_itk", "in2"), + ("blip_warp_inverse", "in1"), + ] else: - transforms_to_combine = [\ - ('mni_to_anatomical_nonlinear_xfm', 'in5'), - ('ants_affine_xfm', 'in4'), - ('ants_rigid_xfm', 'in3'), - ('ants_initial_xfm', 'in2'), - ('fsl_mat_as_itk', 'in1')] + transforms_to_combine = [ + ("mni_to_anatomical_nonlinear_xfm", "in5"), + ("ants_affine_xfm", "in4"), + ("ants_rigid_xfm", "in3"), + ("ants_initial_xfm", "in2"), + ("fsl_mat_as_itk", "in1"), + ] else: - transforms_to_combine = [\ - ('anatomical_to_mni_nonlinear_xfm', 'in1'), - ('ants_affine_xfm', 'in2'), - ('ants_rigid_xfm', 'in3'), - ('ants_initial_xfm', 'in4'), - ('fsl_mat_as_itk', 'in5')] - - if distcor is True and func_type not in 'ica-aroma': - transforms_to_combine.append(('blip_warp', 'in6')) - - if registration_template == 'epi': + transforms_to_combine = [ + ("anatomical_to_mni_nonlinear_xfm", "in1"), + ("ants_affine_xfm", "in2"), + ("ants_rigid_xfm", "in3"), + ("ants_initial_xfm", "in4"), + ("fsl_mat_as_itk", "in5"), + ] + + if distcor is True and func_type not in "ica-aroma": + transforms_to_combine.append(("blip_warp", "in6")) + + if registration_template == "epi": # handle both symmetric and asymmetric transforms - ants_transformation_dict = { - 'asymmetric': { - 'func_to_epi_nonlinear_xfm': 'func_to_epi_nonlinear_xfm', - 'epi_to_func_nonlinear_xfm' : 'epi_to_func_nonlinear_xfm', - 'func_to_epi_ants_affine_xfm': 'func_to_epi_ants_affine_xfm', - 'func_to_epi_ants_rigid_xfm': 'func_to_epi_ants_rigid_xfm', - 'func_to_epi_ants_initial_xfm': 'func_to_epi_ants_initial_xfm', - # 'blip_warp': 'blip_warp', - # 'blip_warp_inverse': 'blip_warp_inverse', - # 'fsl_mat_as_itk': 'fsl_mat_as_itk', - }, - # 'symmetric': { - # 'func_to_epi_nonlinear_xfm': 'anatomical_to_mni_nonlinear_xfm', - # 'func_to_epi_ants_affine_xfm': 'func_to_epi_ants_affine_xfm', - # 'func_to_epi_ants_rigid_xfm': 'func_to_epi_ants_rigid_xfm', - # 'func_to_epi_ants_initial_xfm': 'ants_initial_xfm', - # 'blip_warp': 'blip_warp', - # 'blip_warp_inverse': 'blip_warp_inverse', - # 'fsl_mat_as_itk': 'fsl_mat_as_itk', - # } - } + ants_transformation_dict = { + "asymmetric": { + "func_to_epi_nonlinear_xfm": "func_to_epi_nonlinear_xfm", + "epi_to_func_nonlinear_xfm": "epi_to_func_nonlinear_xfm", + "func_to_epi_ants_affine_xfm": "func_to_epi_ants_affine_xfm", + "func_to_epi_ants_rigid_xfm": "func_to_epi_ants_rigid_xfm", + "func_to_epi_ants_initial_xfm": "func_to_epi_ants_initial_xfm", + # 'blip_warp': 'blip_warp', + # 'blip_warp_inverse': 'blip_warp_inverse', + # 'fsl_mat_as_itk': 'fsl_mat_as_itk', + }, + # 'symmetric': { + # 'func_to_epi_nonlinear_xfm': 'anatomical_to_mni_nonlinear_xfm', + # 'func_to_epi_ants_affine_xfm': 'func_to_epi_ants_affine_xfm', + # 'func_to_epi_ants_rigid_xfm': 'func_to_epi_ants_rigid_xfm', + # 'func_to_epi_ants_initial_xfm': 'ants_initial_xfm', + # 'blip_warp': 'blip_warp', + # 'blip_warp_inverse': 'blip_warp_inverse', + # 'fsl_mat_as_itk': 'fsl_mat_as_itk', + # } + } # transforms to be concatenated, the first element of each tuple is - # the resource pool key related to the resource that should be - # connected in, and the second element is the input to which it + # the resource pool key related to the resource that should be + # connected in, and the second element is the input to which it # should be connected if inverse is True: - if distcor is True and func_type not in 'ica-aroma': + if distcor is True and func_type not in "ica-aroma": # Field file from anatomical nonlinear registration - transforms_to_combine = [\ - ('epi_to_func_nonlinear_xfm', 'in4'), - ('func_to_epi_ants_affine_xfm', 'in3'), - ('func_to_epi_ants_rigid_xfm', 'in2'), - ('func_to_epi_ants_initial_xfm', 'in1')] + transforms_to_combine = [ + ("epi_to_func_nonlinear_xfm", "in4"), + ("func_to_epi_ants_affine_xfm", "in3"), + ("func_to_epi_ants_rigid_xfm", "in2"), + ("func_to_epi_ants_initial_xfm", "in1"), + ] else: - transforms_to_combine = [\ - ('epi_to_func_nonlinear_xfm', 'in4'), - ('func_to_epi_ants_affine_xfm', 'in3'), - ('func_to_epi_ants_rigid_xfm', 'in2'), - ('func_to_epi_ants_initial_xfm', 'in1')] + transforms_to_combine = [ + ("epi_to_func_nonlinear_xfm", "in4"), + ("func_to_epi_ants_affine_xfm", "in3"), + ("func_to_epi_ants_rigid_xfm", "in2"), + ("func_to_epi_ants_initial_xfm", "in1"), + ] else: - transforms_to_combine = [\ - ('func_to_epi_nonlinear_xfm', 'in1'), - ('func_to_epi_ants_affine_xfm', 'in2'), - ('func_to_epi_ants_rigid_xfm', 'in3'), - ('func_to_epi_ants_initial_xfm', 'in4')] + transforms_to_combine = [ + ("func_to_epi_nonlinear_xfm", "in1"), + ("func_to_epi_ants_affine_xfm", "in2"), + ("func_to_epi_ants_rigid_xfm", "in3"), + ("func_to_epi_ants_initial_xfm", "in4"), + ] # define the node - collect_transforms = pe.Node(util.Merge(num_transforms), - name='collect_transforms_{0}_{1}_{2}_{3}'.format(output_name, - inverse_string, - registration_template, - num_strat)) + collect_transforms = pe.Node( + util.Merge(num_transforms), + name="collect_transforms_{0}_{1}_{2}_{3}".format( + output_name, inverse_string, registration_template, num_strat + ), + ) # wire in the various transformations for transform_key, input_port in transforms_to_combine: - try: - node, out_file = strat[ants_transformation_dict[symmetry][transform_key]] - except KeyError: - raise Exception(locals()) - workflow.connect(node, out_file, collect_transforms, input_port) + try: + node, out_file = strat[ + ants_transformation_dict[symmetry][transform_key] + ] + except KeyError: + raise Exception(locals()) + workflow.connect(node, out_file, collect_transforms, input_port) # check transform list (if missing any init/rig/affine) and exclude Nonetype - check_transform = pe.Node(util.Function(input_names=['transform_list'], - output_names=['checked_transform_list', 'list_length'], - function=check_transforms), - name='check_transforms{0}_{1}_{2}_{3}'.format(output_name, - inverse_string, - registration_template, - num_strat)) - - workflow.connect(collect_transforms, 'out', check_transform, 'transform_list') + check_transform = pe.Node( + util.Function( + input_names=["transform_list"], + output_names=["checked_transform_list", "list_length"], + function=check_transforms, + ), + name="check_transforms{0}_{1}_{2}_{3}".format( + output_name, inverse_string, registration_template, num_strat + ), + ) + + workflow.connect(collect_transforms, "out", check_transform, "transform_list") # generate inverse transform flags, which depends on the number of transforms - inverse_transform_flags = pe.Node(util.Function(input_names=['transform_list'], - output_names=['inverse_transform_flags'], - function=generate_inverse_transform_flags), - name='inverse_transform_flags_{0}_{1}_{2}_{3}'.format(output_name, - inverse_string, - registration_template, - num_strat)) - - workflow.connect(check_transform, 'checked_transform_list', inverse_transform_flags, 'transform_list') - + inverse_transform_flags = pe.Node( + util.Function( + input_names=["transform_list"], + output_names=["inverse_transform_flags"], + function=generate_inverse_transform_flags, + ), + name="inverse_transform_flags_{0}_{1}_{2}_{3}".format( + output_name, inverse_string, registration_template, num_strat + ), + ) + + workflow.connect( + check_transform, + "checked_transform_list", + inverse_transform_flags, + "transform_list", + ) # set the output - strat.update_resource_pool({ - collect_transforms_key: (check_transform, 'checked_transform_list') - }) + strat.update_resource_pool( + {collect_transforms_key: (check_transform, "checked_transform_list")} + ) strat.append_name(check_transform.name) strat.append_name(inverse_transform_flags.name) @@ -564,29 +573,34 @@ def ants_apply_warps_func_mni( if map_node: apply_ants_warp = pe.MapNode( - interface=ants.ApplyTransforms(), - name='apply_ants_warp_{0}_mapnode_{1}_{2}_{3}'.format( - output_name, inverse_string, registration_template, - num_strat), - iterfield=['input_image'], - mem_gb=1, - mem_x=(1401462037888665 / 2361183241434822606848, - 'input_image')) + interface=ants.ApplyTransforms(), + name="apply_ants_warp_{0}_mapnode_{1}_{2}_{3}".format( + output_name, inverse_string, registration_template, num_strat + ), + iterfield=["input_image"], + mem_gb=1, + mem_x=(1401462037888665 / 2361183241434822606848, "input_image"), + ) else: apply_ants_warp = pe.Node( - interface=ants.ApplyTransforms(), - name='apply_ants_warp_{0}_{1}_{2}_{3}'.format( - output_name, inverse_string, registration_template, - num_strat), - mem_gb=1, - mem_x=(1401462037888665 / 2361183241434822606848, - 'input_image')) - - apply_ants_warp.inputs.out_postfix = '_antswarp' + interface=ants.ApplyTransforms(), + name="apply_ants_warp_{0}_{1}_{2}_{3}".format( + output_name, inverse_string, registration_template, num_strat + ), + mem_gb=1, + mem_x=(1401462037888665 / 2361183241434822606848, "input_image"), + ) + + apply_ants_warp.inputs.out_postfix = "_antswarp" apply_ants_warp.interface.num_threads = int(num_ants_cores) if inverse is True: - workflow.connect(inverse_transform_flags, 'inverse_transform_flags', apply_ants_warp, 'invert_transform_flags') + workflow.connect( + inverse_transform_flags, + "inverse_transform_flags", + apply_ants_warp, + "invert_transform_flags", + ) # input_image_type: # (0 or 1 or 2 or 3) @@ -597,36 +611,34 @@ def ants_apply_warps_func_mni( apply_ants_warp.inputs.interpolation = interpolation_method node, out_file = strat[ref_key] - workflow.connect(node, out_file, - apply_ants_warp, 'reference_image') + workflow.connect(node, out_file, apply_ants_warp, "reference_image") collect_node, collect_out = strat[collect_transforms_key] - workflow.connect(collect_node, collect_out, - apply_ants_warp, 'transforms') + workflow.connect(collect_node, collect_out, apply_ants_warp, "transforms") if output_name == "functional_to_standard": # write out the composite functional to standard transforms write_composite_xfm = pe.Node( - interface=ants.ApplyTransforms(), - name='write_composite_xfm_{0}_{1}_{2}_{3}'.format(output_name, - inverse_string, registration_template, num_strat), mem_gb=8.0) + interface=ants.ApplyTransforms(), + name="write_composite_xfm_{0}_{1}_{2}_{3}".format( + output_name, inverse_string, registration_template, num_strat + ), + mem_gb=8.0, + ) write_composite_xfm.inputs.print_out_composite_warp_file = True write_composite_xfm.inputs.output_image = "func_to_standard_xfm.nii.gz" - workflow.connect(input_node, input_out, - write_composite_xfm, 'input_image') + workflow.connect(input_node, input_out, write_composite_xfm, "input_image") write_composite_xfm.inputs.input_image_type = input_image_type write_composite_xfm.inputs.dimension = 3 write_composite_xfm.inputs.interpolation = interpolation_method node, out_file = strat[ref_key] - workflow.connect(node, out_file, - write_composite_xfm, 'reference_image') + workflow.connect(node, out_file, write_composite_xfm, "reference_image") collect_node, collect_out = strat[collect_transforms_key] - workflow.connect(collect_node, collect_out, - write_composite_xfm, 'transforms') + workflow.connect(collect_node, collect_out, write_composite_xfm, "transforms") # write_composite_inv_xfm = pe.Node( # interface=ants.ApplyTransforms(), @@ -654,122 +666,165 @@ def ants_apply_warps_func_mni( # workflow.connect(collect_node, collect_out, # write_composite_inv_xfm, 'transforms') - strat.update_resource_pool({ - "functional_to_standard_xfm": (write_composite_xfm, 'output_image') - }) - #"functional_to_standard_inverse-xfm": (write_composite_inv_xfm, 'output_image') - #}) + strat.update_resource_pool( + {"functional_to_standard_xfm": (write_composite_xfm, "output_image")} + ) + # "functional_to_standard_inverse-xfm": (write_composite_inv_xfm, 'output_image') + # }) # parallelize the apply warp, if multiple CPUs, and it's a time series! if int(num_cpus) > 1 and input_image_type == 3: - - node_id = f'_{output_name}_{inverse_string}_{registration_template}_{num_strat}' - - chunk_imports = ['import nibabel as nb'] - chunk = pe.Node(Function(input_names=['func_file', - 'n_cpus'], - output_names=['TR_ranges'], - function=chunk_ts, - imports=chunk_imports), - name=f'chunk_{node_id}') + node_id = f"_{output_name}_{inverse_string}_{registration_template}_{num_strat}" + + chunk_imports = ["import nibabel as nb"] + chunk = pe.Node( + Function( + input_names=["func_file", "n_cpus"], + output_names=["TR_ranges"], + function=chunk_ts, + imports=chunk_imports, + ), + name=f"chunk_{node_id}", + ) chunk.inputs.n_cpus = int(num_cpus) - workflow.connect(input_node, input_out, chunk, 'func_file') + workflow.connect(input_node, input_out, chunk, "func_file") - split_imports = ['import os', 'import subprocess'] - split = pe.Node(Function(input_names=['func_file', - 'tr_ranges'], - output_names=['split_funcs'], - function=split_ts_chunks, - imports=split_imports), - name=f'split_{node_id}') + split_imports = ["import os", "import subprocess"] + split = pe.Node( + Function( + input_names=["func_file", "tr_ranges"], + output_names=["split_funcs"], + function=split_ts_chunks, + imports=split_imports, + ), + name=f"split_{node_id}", + ) - workflow.connect(input_node, input_out, split, 'func_file') - workflow.connect(chunk, 'TR_ranges', split, 'tr_ranges') + workflow.connect(input_node, input_out, split, "func_file") + workflow.connect(chunk, "TR_ranges", split, "tr_ranges") - workflow.connect(split, 'split_funcs', apply_ants_warp, 'input_image') + workflow.connect(split, "split_funcs", apply_ants_warp, "input_image") - func_concat = pe.Node(interface=afni_utils.TCat(), - name=f'func_concat_{node_id}') - func_concat.inputs.outputtype = 'NIFTI_GZ' + func_concat = pe.Node( + interface=afni_utils.TCat(), name=f"func_concat_{node_id}" + ) + func_concat.inputs.outputtype = "NIFTI_GZ" - workflow.connect(apply_ants_warp, 'output_image', - func_concat, 'in_files') + workflow.connect(apply_ants_warp, "output_image", func_concat, "in_files") - strat.update_resource_pool({ - output_name: (func_concat, 'out_file') - }) + strat.update_resource_pool({output_name: (func_concat, "out_file")}) else: - workflow.connect(input_node, input_out, - apply_ants_warp, 'input_image') + workflow.connect(input_node, input_out, apply_ants_warp, "input_image") - strat.update_resource_pool({ - output_name: (apply_ants_warp, 'output_image') - }) + strat.update_resource_pool({output_name: (apply_ants_warp, "output_image")}) strat.append_name(apply_ants_warp.name) return workflow -def output_func_to_standard(workflow, func_key, ref_key, output_name, - strat, num_strat, pipeline_config_obj, input_image_type='func_derivative', - symmetry='asymmetric', inverse=False, registration_template='t1', - func_type='non-ica-aroma'): - - image_types = ['func_derivative', 'func_derivative_multi', - 'func_4d', 'func_mask'] +def output_func_to_standard( + workflow, + func_key, + ref_key, + output_name, + strat, + num_strat, + pipeline_config_obj, + input_image_type="func_derivative", + symmetry="asymmetric", + inverse=False, + registration_template="t1", + func_type="non-ica-aroma", +): + image_types = ["func_derivative", "func_derivative_multi", "func_4d", "func_mask"] if input_image_type not in image_types: - raise ValueError('Input image type {0} should be one of {1}'.format(\ - input_image_type, ', '.join(image_types))) + raise ValueError( + "Input image type {0} should be one of {1}".format( + input_image_type, ", ".join(image_types) + ) + ) nodes = strat.get_nodes_names() - - map_node = True if input_image_type == 'func_derivative_multi' else False - distcor = True if 'epi_distcorr' in nodes or \ - 'blip_correct' in nodes else False + map_node = True if input_image_type == "func_derivative_multi" else False - num_cpus = pipeline_config_obj.pipeline_setup['system_config']['max_cores_per_participant'] + distcor = True if "epi_distcorr" in nodes or "blip_correct" in nodes else False - if 'anat_mni_fnirt_register' in nodes or \ - 'anat_mni_flirt_register' in nodes or \ - 'func_to_epi_fsl' in nodes: + num_cpus = pipeline_config_obj.pipeline_setup["system_config"][ + "max_cores_per_participant" + ] - if input_image_type == 'map' or 'mask' in input_image_type: - interp = 'nn' + if ( + "anat_mni_fnirt_register" in nodes + or "anat_mni_flirt_register" in nodes + or "func_to_epi_fsl" in nodes + ): + if input_image_type == "map" or "mask" in input_image_type: + interp = "nn" else: - interp = pipeline_config_obj.functional_registration['2-func_registration_to_template']['FNIRT_pipelines']['interpolation'] - - func_ts = True if input_image_type == 'func_4d' else False - - fsl_apply_transform_func_to_mni(workflow, output_name, func_key, - ref_key, num_strat, strat, interp, distcor=distcor, - map_node=map_node, func_ts=func_ts, num_cpus=num_cpus) - - elif 'ANTS' in pipeline_config_obj.anatomical_preproc[ - 'registration_workflow' - ]['registration']['using']: - - if input_image_type == 'map' or 'mask' in input_image_type: - interp = 'NearestNeighbor' + interp = pipeline_config_obj.functional_registration[ + "2-func_registration_to_template" + ]["FNIRT_pipelines"]["interpolation"] + + func_ts = True if input_image_type == "func_4d" else False + + fsl_apply_transform_func_to_mni( + workflow, + output_name, + func_key, + ref_key, + num_strat, + strat, + interp, + distcor=distcor, + map_node=map_node, + func_ts=func_ts, + num_cpus=num_cpus, + ) + + elif ( + "ANTS" + in pipeline_config_obj.anatomical_preproc["registration_workflow"][ + "registration" + ]["using"] + ): + if input_image_type == "map" or "mask" in input_image_type: + interp = "NearestNeighbor" else: - interp = pipeline_config_obj.functional_registration['2-func_registration_to_template']['ANTs_pipelines']['interpolation'] - - image_type = 3 if input_image_type == 'func_4d' else 0 - - ants_apply_warps_func_mni(workflow, output_name, func_key, ref_key, - num_strat, strat, interpolation_method=interp, - distcor=distcor, map_node=map_node, inverse=inverse, - symmetry=symmetry, input_image_type=image_type, - num_ants_cores=pipeline_config_obj.pipeline_setup['system_config']['num_ants_threads'], - registration_template=registration_template, - func_type=func_type) + interp = pipeline_config_obj.functional_registration[ + "2-func_registration_to_template" + ]["ANTs_pipelines"]["interpolation"] + + image_type = 3 if input_image_type == "func_4d" else 0 + + ants_apply_warps_func_mni( + workflow, + output_name, + func_key, + ref_key, + num_strat, + strat, + interpolation_method=interp, + distcor=distcor, + map_node=map_node, + inverse=inverse, + symmetry=symmetry, + input_image_type=image_type, + num_ants_cores=pipeline_config_obj.pipeline_setup["system_config"][ + "num_ants_threads" + ], + registration_template=registration_template, + func_type=func_type, + ) else: - raise ValueError('Cannot determine whether a ANTS or FSL registration' \ - 'is desired, check your pipeline.') + raise ValueError( + "Cannot determine whether a ANTS or FSL registration" + "is desired, check your pipeline." + ) return workflow diff --git a/CPAC/registration/registration.py b/CPAC/registration/registration.py index cd3e69444a..ebd0784d0e 100644 --- a/CPAC/registration/registration.py +++ b/CPAC/registration/registration.py @@ -16,48 +16,58 @@ # License along with C-PAC. If not, see . # pylint: disable=too-many-lines,ungrouped-imports,wrong-import-order from typing import Optional -from CPAC.pipeline import nipype_pipeline_engine as pe -from CPAC.pipeline.nodeblock import nodeblock + from nipype.interfaces import afni, ants, c3, fsl, utility as util from nipype.interfaces.afni import utils as afni_utils from CPAC.anat_preproc.lesion_preproc import create_lesion_preproc from CPAC.func_preproc.utils import chunk_ts, split_ts_chunks -from CPAC.registration.utils import seperate_warps_list, \ - check_transforms, \ - generate_inverse_transform_flags, \ - single_ants_xfm_to_list, \ - interpolation_string, \ - change_itk_transform_type, \ - hardcoded_reg, \ - one_d_to_mat, \ - run_c3d, \ - run_c4d +from CPAC.pipeline import nipype_pipeline_engine as pe +from CPAC.pipeline.nodeblock import nodeblock +from CPAC.registration.utils import ( + change_itk_transform_type, + check_transforms, + generate_inverse_transform_flags, + hardcoded_reg, + interpolation_string, + one_d_to_mat, + run_c3d, + run_c4d, + seperate_warps_list, + single_ants_xfm_to_list, +) from CPAC.utils.interfaces.fsl import Merge as fslMerge from CPAC.utils.typing import LIST_OR_STR, TUPLE from CPAC.utils.utils import check_prov_for_motion_tool, check_prov_for_regtool -def apply_transform(wf_name, reg_tool, time_series=False, multi_input=False, - num_cpus=1, num_ants_cores=1): - +def apply_transform( + wf_name, + reg_tool, + time_series=False, + multi_input=False, + num_cpus=1, + num_ants_cores=1, +): if not reg_tool: - raise Exception("\n[!] Developer info: the 'reg_tool' parameter sent " - f"to the 'apply_transform' node for '{wf_name}' is " - f"empty.\n") + raise Exception( + "\n[!] Developer info: the 'reg_tool' parameter sent " + f"to the 'apply_transform' node for '{wf_name}' is " + f"empty.\n" + ) wf = pe.Workflow(name=wf_name) inputNode = pe.Node( - util.IdentityInterface(fields=['input_image', - 'reference', - 'transform', - 'interpolation']), - name='inputspec') + util.IdentityInterface( + fields=["input_image", "reference", "transform", "interpolation"] + ), + name="inputspec", + ) outputNode = pe.Node( - util.IdentityInterface(fields=['output_image']), - name='outputspec') + util.IdentityInterface(fields=["output_image"]), name="outputspec" + ) if int(num_cpus) > 1 and time_series: # parallelize time series warp application @@ -65,23 +75,22 @@ def apply_transform(wf_name, reg_tool, time_series=False, multi_input=False, # time series chunks multi_input = True - if reg_tool == 'ants': - + if reg_tool == "ants": if multi_input: - apply_warp = pe.MapNode(interface=ants.ApplyTransforms(), - name=f'apply_warp_{wf_name}', - iterfield=['input_image'], - mem_gb=0.7, - mem_x=(1708448960473801 / - 151115727451828646838272, - 'input_image')) + apply_warp = pe.MapNode( + interface=ants.ApplyTransforms(), + name=f"apply_warp_{wf_name}", + iterfield=["input_image"], + mem_gb=0.7, + mem_x=(1708448960473801 / 151115727451828646838272, "input_image"), + ) else: - apply_warp = pe.Node(interface=ants.ApplyTransforms(), - name=f'apply_warp_{wf_name}', - mem_gb=0.7, - mem_x=(1708448960473801 / - 151115727451828646838272, - 'input_image')) + apply_warp = pe.Node( + interface=ants.ApplyTransforms(), + name=f"apply_warp_{wf_name}", + mem_gb=0.7, + mem_x=(1708448960473801 / 151115727451828646838272, "input_image"), + ) apply_warp.inputs.dimension = 3 apply_warp.interface.num_threads = int(num_ants_cores) @@ -89,275 +98,313 @@ def apply_transform(wf_name, reg_tool, time_series=False, multi_input=False, if time_series: apply_warp.inputs.input_image_type = 3 - wf.connect(inputNode, 'reference', apply_warp, 'reference_image') + wf.connect(inputNode, "reference", apply_warp, "reference_image") - interp_string = pe.Node(util.Function(input_names=['interpolation', - 'reg_tool'], - output_names=['interpolation'], - function=interpolation_string), - name=f'interp_string', - mem_gb=2.5) + interp_string = pe.Node( + util.Function( + input_names=["interpolation", "reg_tool"], + output_names=["interpolation"], + function=interpolation_string, + ), + name="interp_string", + mem_gb=2.5, + ) interp_string.inputs.reg_tool = reg_tool - wf.connect(inputNode, 'interpolation', interp_string, 'interpolation') - wf.connect(interp_string, 'interpolation', - apply_warp, 'interpolation') - - ants_xfm_list = \ - pe.Node(util.Function(input_names=['transform'], - output_names=['transform_list'], - function=single_ants_xfm_to_list), - name=f'single_ants_xfm_to_list', - mem_gb=2.5) + wf.connect(inputNode, "interpolation", interp_string, "interpolation") + wf.connect(interp_string, "interpolation", apply_warp, "interpolation") + + ants_xfm_list = pe.Node( + util.Function( + input_names=["transform"], + output_names=["transform_list"], + function=single_ants_xfm_to_list, + ), + name="single_ants_xfm_to_list", + mem_gb=2.5, + ) - wf.connect(inputNode, 'transform', ants_xfm_list, 'transform') - wf.connect(ants_xfm_list, 'transform_list', apply_warp, 'transforms') + wf.connect(inputNode, "transform", ants_xfm_list, "transform") + wf.connect(ants_xfm_list, "transform_list", apply_warp, "transforms") # parallelize the apply warp, if multiple CPUs, and it's a time # series! if int(num_cpus) > 1 and time_series: + chunk_imports = ["import nibabel as nb"] + chunk = pe.Node( + util.Function( + input_names=["func_file", "n_chunks", "chunk_size"], + output_names=["TR_ranges"], + function=chunk_ts, + imports=chunk_imports, + ), + name=f"chunk_{wf_name}", + mem_gb=2.5, + ) - chunk_imports = ['import nibabel as nb'] - chunk = pe.Node(util.Function(input_names=['func_file', - 'n_chunks', - 'chunk_size'], - output_names=['TR_ranges'], - function=chunk_ts, - imports=chunk_imports), - name=f'chunk_{wf_name}', - mem_gb=2.5) - - #chunk.inputs.n_chunks = int(num_cpus) + # chunk.inputs.n_chunks = int(num_cpus) # 10-TR sized chunks chunk.inputs.chunk_size = 10 - wf.connect(inputNode, 'input_image', chunk, 'func_file') - - split_imports = ['import os', 'import subprocess'] - split = pe.Node(util.Function(input_names=['func_file', - 'tr_ranges'], - output_names=['split_funcs'], - function=split_ts_chunks, - imports=split_imports), - name=f'split_{wf_name}', - mem_gb=2.5) + wf.connect(inputNode, "input_image", chunk, "func_file") + + split_imports = ["import os", "import subprocess"] + split = pe.Node( + util.Function( + input_names=["func_file", "tr_ranges"], + output_names=["split_funcs"], + function=split_ts_chunks, + imports=split_imports, + ), + name=f"split_{wf_name}", + mem_gb=2.5, + ) - wf.connect(inputNode, 'input_image', split, 'func_file') - wf.connect(chunk, 'TR_ranges', split, 'tr_ranges') + wf.connect(inputNode, "input_image", split, "func_file") + wf.connect(chunk, "TR_ranges", split, "tr_ranges") - wf.connect(split, 'split_funcs', apply_warp, 'input_image') + wf.connect(split, "split_funcs", apply_warp, "input_image") - func_concat = pe.Node(interface=afni_utils.TCat(), - name=f'func_concat_{wf_name}', - mem_gb=2.5) - func_concat.inputs.outputtype = 'NIFTI_GZ' + func_concat = pe.Node( + interface=afni_utils.TCat(), name=f"func_concat_{wf_name}", mem_gb=2.5 + ) + func_concat.inputs.outputtype = "NIFTI_GZ" - wf.connect(apply_warp, 'output_image', func_concat, 'in_files') + wf.connect(apply_warp, "output_image", func_concat, "in_files") - wf.connect(func_concat, 'out_file', outputNode, 'output_image') + wf.connect(func_concat, "out_file", outputNode, "output_image") else: - wf.connect(inputNode, 'input_image', apply_warp, 'input_image') - wf.connect(apply_warp, 'output_image', outputNode, 'output_image') - - elif reg_tool == 'fsl': + wf.connect(inputNode, "input_image", apply_warp, "input_image") + wf.connect(apply_warp, "output_image", outputNode, "output_image") + elif reg_tool == "fsl": if multi_input: - apply_warp = pe.MapNode(interface=fsl.ApplyWarp(), - name=f'fsl_apply_warp', - iterfield=['in_file'], - mem_gb=2.5) + apply_warp = pe.MapNode( + interface=fsl.ApplyWarp(), + name="fsl_apply_warp", + iterfield=["in_file"], + mem_gb=2.5, + ) else: - apply_warp = pe.Node(interface=fsl.ApplyWarp(), - name='fsl_apply_warp', - mem_gb=2.5) - - interp_string = pe.Node(util.Function(input_names=['interpolation', - 'reg_tool'], - output_names=['interpolation'], - function=interpolation_string), - name=f'interp_string', - mem_gb=2.5) + apply_warp = pe.Node( + interface=fsl.ApplyWarp(), name="fsl_apply_warp", mem_gb=2.5 + ) + + interp_string = pe.Node( + util.Function( + input_names=["interpolation", "reg_tool"], + output_names=["interpolation"], + function=interpolation_string, + ), + name="interp_string", + mem_gb=2.5, + ) interp_string.inputs.reg_tool = reg_tool - wf.connect(inputNode, 'interpolation', interp_string, 'interpolation') - wf.connect(interp_string, 'interpolation', apply_warp, 'interp') + wf.connect(inputNode, "interpolation", interp_string, "interpolation") + wf.connect(interp_string, "interpolation", apply_warp, "interp") # mni to t1 - wf.connect(inputNode, 'reference', apply_warp, 'ref_file') + wf.connect(inputNode, "reference", apply_warp, "ref_file") # NOTE: C-PAC now converts all FSL xfm's to .nii, so even if the # inputNode 'transform' is a linear xfm, it's a .nii and must # go in as a warpfield file - wf.connect(inputNode, 'transform', apply_warp, 'field_file') + wf.connect(inputNode, "transform", apply_warp, "field_file") # parallelize the apply warp, if multiple CPUs, and it's a time # series! if int(num_cpus) > 1 and time_series: + chunk_imports = ["import nibabel as nb"] + chunk = pe.Node( + util.Function( + input_names=["func_file", "n_chunks", "chunk_size"], + output_names=["TR_ranges"], + function=chunk_ts, + imports=chunk_imports, + ), + name=f"chunk_{wf_name}", + mem_gb=2.5, + ) - chunk_imports = ['import nibabel as nb'] - chunk = pe.Node(util.Function(input_names=['func_file', - 'n_chunks', - 'chunk_size'], - output_names=['TR_ranges'], - function=chunk_ts, - imports=chunk_imports), - name=f'chunk_{wf_name}', - mem_gb=2.5) - - #chunk.inputs.n_chunks = int(num_cpus) + # chunk.inputs.n_chunks = int(num_cpus) # 10-TR sized chunks chunk.inputs.chunk_size = 10 - wf.connect(inputNode, 'input_image', chunk, 'func_file') - - split_imports = ['import os', 'import subprocess'] - split = pe.Node(util.Function(input_names=['func_file', - 'tr_ranges'], - output_names=['split_funcs'], - function=split_ts_chunks, - imports=split_imports), - name=f'split_{wf_name}', - mem_gb=2.5) + wf.connect(inputNode, "input_image", chunk, "func_file") + + split_imports = ["import os", "import subprocess"] + split = pe.Node( + util.Function( + input_names=["func_file", "tr_ranges"], + output_names=["split_funcs"], + function=split_ts_chunks, + imports=split_imports, + ), + name=f"split_{wf_name}", + mem_gb=2.5, + ) - wf.connect(inputNode, 'input_image', split, 'func_file') - wf.connect(chunk, 'TR_ranges', split, 'tr_ranges') + wf.connect(inputNode, "input_image", split, "func_file") + wf.connect(chunk, "TR_ranges", split, "tr_ranges") - wf.connect(split, 'split_funcs', apply_warp, 'in_file') + wf.connect(split, "split_funcs", apply_warp, "in_file") - func_concat = pe.Node(interface=afni_utils.TCat(), - name=f'func_concat{wf_name}') - func_concat.inputs.outputtype = 'NIFTI_GZ' + func_concat = pe.Node( + interface=afni_utils.TCat(), name=f"func_concat{wf_name}" + ) + func_concat.inputs.outputtype = "NIFTI_GZ" - wf.connect(apply_warp, 'out_file', func_concat, 'in_files') + wf.connect(apply_warp, "out_file", func_concat, "in_files") - wf.connect(func_concat, 'out_file', outputNode, 'output_image') + wf.connect(func_concat, "out_file", outputNode, "output_image") else: - wf.connect(inputNode, 'input_image', apply_warp, 'in_file') - wf.connect(apply_warp, 'out_file', outputNode, 'output_image') + wf.connect(inputNode, "input_image", apply_warp, "in_file") + wf.connect(apply_warp, "out_file", outputNode, "output_image") return wf -def transform_derivative(wf_name, label, reg_tool, num_cpus, num_ants_cores, - ants_interp=None, fsl_interp=None, opt=None): - '''Transform output derivatives to template space. +def transform_derivative( + wf_name, + label, + reg_tool, + num_cpus, + num_ants_cores, + ants_interp=None, + fsl_interp=None, + opt=None, +): + """Transform output derivatives to template space. This function is designed for use with the NodeBlock connection engine. - ''' - + """ wf = pe.Workflow(name=wf_name) - inputnode = pe.Node(util.IdentityInterface(fields=['in_file', - 'reference', - 'transform']), - name='inputspec') + inputnode = pe.Node( + util.IdentityInterface(fields=["in_file", "reference", "transform"]), + name="inputspec", + ) multi_input = False - if 'statmap' in label: + if "statmap" in label: multi_input = True stack = False - if 'correlations' in label: + if "correlations" in label: stack = True - apply_xfm = apply_transform(f'warp_{label}_to_template', reg_tool, - time_series=stack, - multi_input=multi_input, - num_cpus=num_cpus, - num_ants_cores=num_ants_cores) + apply_xfm = apply_transform( + f"warp_{label}_to_template", + reg_tool, + time_series=stack, + multi_input=multi_input, + num_cpus=num_cpus, + num_ants_cores=num_ants_cores, + ) - if reg_tool == 'ants': + if reg_tool == "ants": apply_xfm.inputs.inputspec.interpolation = ants_interp - elif reg_tool == 'fsl': + elif reg_tool == "fsl": apply_xfm.inputs.inputspec.interpolation = fsl_interp - wf.connect(inputnode, 'in_file', apply_xfm, 'inputspec.input_image') - wf.connect(inputnode, 'reference', apply_xfm, 'inputspec.reference') - wf.connect(inputnode, 'transform', apply_xfm, 'inputspec.transform') + wf.connect(inputnode, "in_file", apply_xfm, "inputspec.input_image") + wf.connect(inputnode, "reference", apply_xfm, "inputspec.reference") + wf.connect(inputnode, "transform", apply_xfm, "inputspec.transform") - outputnode = pe.Node(util.IdentityInterface(fields=['out_file']), - name='outputspec') + outputnode = pe.Node(util.IdentityInterface(fields=["out_file"]), name="outputspec") - wf.connect(apply_xfm, 'outputspec.output_image', outputnode, 'out_file') + wf.connect(apply_xfm, "outputspec.output_image", outputnode, "out_file") return wf -def convert_pedir(pedir, convert='xyz_to_int'): - '''FSL Flirt requires pedir input encoded as an int''' - if convert == 'xyz_to_int': - conv_dct = {'x': 1, 'y': 2, 'z': 3, 'x-': -1, 'y-': -2, 'z-': -3, - 'i': 1, 'j': 2, 'k': 3, 'i-': -1, 'j-': -2, 'k-': -3, - '-x': -1, '-i': -1, '-y': -2, - '-j': -2, '-z': -3, '-k': -3} - elif convert == 'ijk_to_xyz': - conv_dct = {'i': 'x', 'j': 'y', 'k': 'z', - 'i-': 'x-', 'j-': 'y-', 'k-': 'z-'} +def convert_pedir(pedir, convert="xyz_to_int"): + """FSL Flirt requires pedir input encoded as an int.""" + if convert == "xyz_to_int": + conv_dct = { + "x": 1, + "y": 2, + "z": 3, + "x-": -1, + "y-": -2, + "z-": -3, + "i": 1, + "j": 2, + "k": 3, + "i-": -1, + "j-": -2, + "k-": -3, + "-x": -1, + "-i": -1, + "-y": -2, + "-j": -2, + "-z": -3, + "-k": -3, + } + elif convert == "ijk_to_xyz": + conv_dct = {"i": "x", "j": "y", "k": "z", "i-": "x-", "j-": "y-", "k-": "z-"} if isinstance(pedir, bytes): pedir = pedir.decode() if not isinstance(pedir, str): - raise Exception("\n\nPhase-encoding direction must be a " - "string value.\n\nValue: {0}" - "\n\n".format(pedir)) + raise Exception( + "\n\nPhase-encoding direction must be a " + "string value.\n\nValue: {0}" + "\n\n".format(pedir) + ) if pedir not in conv_dct.keys(): - raise Exception("\n\nInvalid phase-encoding direction " - "entered: {0}\n\n".format(pedir)) - pedir = conv_dct[pedir] - return pedir - + raise Exception( + "\n\nInvalid phase-encoding direction " "entered: {0}\n\n".format(pedir) + ) + return conv_dct[pedir] -def create_fsl_flirt_linear_reg(name='fsl_flirt_linear_reg'): +def create_fsl_flirt_linear_reg(name="fsl_flirt_linear_reg"): linear_register = pe.Workflow(name=name) - inputspec = pe.Node(util.IdentityInterface(fields=['input_brain', - 'reference_brain', - 'interp', - 'ref_mask']), - name='inputspec') + inputspec = pe.Node( + util.IdentityInterface( + fields=["input_brain", "reference_brain", "interp", "ref_mask"] + ), + name="inputspec", + ) - outputspec = pe.Node(util.IdentityInterface(fields=['output_brain', - 'linear_xfm', - 'invlinear_xfm']), - name='outputspec') + outputspec = pe.Node( + util.IdentityInterface(fields=["output_brain", "linear_xfm", "invlinear_xfm"]), + name="outputspec", + ) - linear_reg = pe.Node(interface=fsl.FLIRT(), name='linear_reg_0') - linear_reg.inputs.cost = 'corratio' + linear_reg = pe.Node(interface=fsl.FLIRT(), name="linear_reg_0") + linear_reg.inputs.cost = "corratio" - inv_flirt_xfm = pe.Node(interface=fsl.utils.ConvertXFM(), - name='inv_linear_reg0_xfm') + inv_flirt_xfm = pe.Node( + interface=fsl.utils.ConvertXFM(), name="inv_linear_reg0_xfm" + ) inv_flirt_xfm.inputs.invert_xfm = True - linear_register.connect(inputspec, 'input_brain', - linear_reg, 'in_file') + linear_register.connect(inputspec, "input_brain", linear_reg, "in_file") - linear_register.connect(inputspec, 'reference_brain', - linear_reg, 'reference') + linear_register.connect(inputspec, "reference_brain", linear_reg, "reference") - linear_register.connect(inputspec, 'interp', - linear_reg, 'interp') + linear_register.connect(inputspec, "interp", linear_reg, "interp") - linear_register.connect(linear_reg, 'out_file', - outputspec, 'output_brain') + linear_register.connect(linear_reg, "out_file", outputspec, "output_brain") - linear_register.connect(linear_reg, 'out_matrix_file', - inv_flirt_xfm, 'in_file') + linear_register.connect(linear_reg, "out_matrix_file", inv_flirt_xfm, "in_file") - linear_register.connect(inv_flirt_xfm, 'out_file', - outputspec, 'invlinear_xfm') + linear_register.connect(inv_flirt_xfm, "out_file", outputspec, "invlinear_xfm") - linear_register.connect(linear_reg, 'out_matrix_file', - outputspec, 'linear_xfm') + linear_register.connect(linear_reg, "out_matrix_file", outputspec, "linear_xfm") return linear_register -def create_fsl_fnirt_nonlinear_reg(name='fsl_fnirt_nonlinear_reg'): +def create_fsl_fnirt_nonlinear_reg(name="fsl_fnirt_nonlinear_reg"): """ Performs non-linear registration of an input file to a reference file using FSL FNIRT. @@ -373,7 +420,6 @@ def create_fsl_fnirt_nonlinear_reg(name='fsl_fnirt_nonlinear_reg'): Notes ----- - Workflow Inputs:: inputspec.input_skull : string (nifti file) @@ -408,71 +454,68 @@ def create_fsl_fnirt_nonlinear_reg(name='fsl_fnirt_nonlinear_reg'): .. image:: ../images/nonlinear_register_detailed.dot.png :width: 500 """ - nonlinear_register = pe.Workflow(name=name) - inputspec = pe.Node(util.IdentityInterface(fields=['input_brain', - 'input_skull', - 'reference_brain', - 'reference_skull', - 'interp', - 'ref_mask', - 'linear_aff', - 'fnirt_config']), - name='inputspec') + inputspec = pe.Node( + util.IdentityInterface( + fields=[ + "input_brain", + "input_skull", + "reference_brain", + "reference_skull", + "interp", + "ref_mask", + "linear_aff", + "fnirt_config", + ] + ), + name="inputspec", + ) - outputspec = pe.Node(util.IdentityInterface(fields=['output_brain', - 'nonlinear_xfm']), - name='outputspec') + outputspec = pe.Node( + util.IdentityInterface(fields=["output_brain", "nonlinear_xfm"]), + name="outputspec", + ) - nonlinear_reg = pe.Node(interface=fsl.FNIRT(), - name='nonlinear_reg_1') + nonlinear_reg = pe.Node(interface=fsl.FNIRT(), name="nonlinear_reg_1") nonlinear_reg.inputs.fieldcoeff_file = True nonlinear_reg.inputs.jacobian_file = True - brain_warp = pe.Node(interface=fsl.ApplyWarp(), - name='brain_warp') + brain_warp = pe.Node(interface=fsl.ApplyWarp(), name="brain_warp") - nonlinear_register.connect(inputspec, 'input_skull', - nonlinear_reg, 'in_file') + nonlinear_register.connect(inputspec, "input_skull", nonlinear_reg, "in_file") - nonlinear_register.connect(inputspec, 'reference_skull', - nonlinear_reg, 'ref_file') + nonlinear_register.connect(inputspec, "reference_skull", nonlinear_reg, "ref_file") - nonlinear_register.connect(inputspec, 'interp', - brain_warp, 'interp') + nonlinear_register.connect(inputspec, "interp", brain_warp, "interp") - nonlinear_register.connect(inputspec, 'ref_mask', - nonlinear_reg, 'refmask_file') + nonlinear_register.connect(inputspec, "ref_mask", nonlinear_reg, "refmask_file") # FNIRT parameters are specified by FSL config file # ${FSLDIR}/etc/flirtsch/TI_2_MNI152_2mm.cnf (or user-specified) - nonlinear_register.connect(inputspec, 'fnirt_config', - nonlinear_reg, 'config_file') + nonlinear_register.connect(inputspec, "fnirt_config", nonlinear_reg, "config_file") - nonlinear_register.connect(inputspec, 'linear_aff', - nonlinear_reg, 'affine_file') + nonlinear_register.connect(inputspec, "linear_aff", nonlinear_reg, "affine_file") - nonlinear_register.connect(nonlinear_reg, 'fieldcoeff_file', - outputspec, 'nonlinear_xfm') + nonlinear_register.connect( + nonlinear_reg, "fieldcoeff_file", outputspec, "nonlinear_xfm" + ) - nonlinear_register.connect(inputspec, 'input_brain', - brain_warp, 'in_file') + nonlinear_register.connect(inputspec, "input_brain", brain_warp, "in_file") - nonlinear_register.connect(nonlinear_reg, 'fieldcoeff_file', - brain_warp, 'field_file') + nonlinear_register.connect( + nonlinear_reg, "fieldcoeff_file", brain_warp, "field_file" + ) - nonlinear_register.connect(inputspec, 'reference_brain', - brain_warp, 'ref_file') + nonlinear_register.connect(inputspec, "reference_brain", brain_warp, "ref_file") - nonlinear_register.connect(brain_warp, 'out_file', - outputspec, 'output_brain') + nonlinear_register.connect(brain_warp, "out_file", outputspec, "output_brain") return nonlinear_register -def create_fsl_fnirt_nonlinear_reg_nhp(name='fsl_fnirt_nonlinear_reg_nhp'): +def create_fsl_fnirt_nonlinear_reg_nhp(name="fsl_fnirt_nonlinear_reg_nhp"): """ Performs non-linear registration of an input file to a reference file using FSL FNIRT. @@ -488,7 +531,6 @@ def create_fsl_fnirt_nonlinear_reg_nhp(name='fsl_fnirt_nonlinear_reg_nhp'): Notes ----- - Workflow Inputs:: inputspec.input_skull : string (nifti file) @@ -525,129 +567,120 @@ def create_fsl_fnirt_nonlinear_reg_nhp(name='fsl_fnirt_nonlinear_reg_nhp'): .. image:: ../images/nonlinear_register_detailed.dot.png :width: 500 """ - nonlinear_register = pe.Workflow(name=name) - inputspec = pe.Node(util.IdentityInterface(fields=['input_brain', - 'input_skull', - 'reference_brain', - 'reference_skull', - 'interp', - 'ref_mask', - 'linear_aff', - 'fnirt_config']), - name='inputspec') - - outputspec = pe.Node(util.IdentityInterface(fields=['output_brain', - 'output_head', - 'output_mask', - 'output_biasfield', - 'nonlinear_xfm', - 'nonlinear_warp']), - name='outputspec') - - nonlinear_reg = pe.Node(interface=fsl.FNIRT(), - name='nonlinear_reg_1') + inputspec = pe.Node( + util.IdentityInterface( + fields=[ + "input_brain", + "input_skull", + "reference_brain", + "reference_skull", + "interp", + "ref_mask", + "linear_aff", + "fnirt_config", + ] + ), + name="inputspec", + ) + + outputspec = pe.Node( + util.IdentityInterface( + fields=[ + "output_brain", + "output_head", + "output_mask", + "output_biasfield", + "nonlinear_xfm", + "nonlinear_warp", + ] + ), + name="outputspec", + ) + + nonlinear_reg = pe.Node(interface=fsl.FNIRT(), name="nonlinear_reg_1") nonlinear_reg.inputs.fieldcoeff_file = True nonlinear_reg.inputs.jacobian_file = True nonlinear_reg.inputs.field_file = True - nonlinear_register.connect(inputspec, 'input_skull', - nonlinear_reg, 'in_file') + nonlinear_register.connect(inputspec, "input_skull", nonlinear_reg, "in_file") - nonlinear_register.connect(inputspec, 'reference_skull', - nonlinear_reg, 'ref_file') + nonlinear_register.connect(inputspec, "reference_skull", nonlinear_reg, "ref_file") - nonlinear_register.connect(inputspec, 'ref_mask', - nonlinear_reg, 'refmask_file') + nonlinear_register.connect(inputspec, "ref_mask", nonlinear_reg, "refmask_file") - nonlinear_register.connect(inputspec, 'fnirt_config', - nonlinear_reg, 'config_file') + nonlinear_register.connect(inputspec, "fnirt_config", nonlinear_reg, "config_file") - nonlinear_register.connect(inputspec, 'linear_aff', - nonlinear_reg, 'affine_file') + nonlinear_register.connect(inputspec, "linear_aff", nonlinear_reg, "affine_file") - brain_warp = pe.Node(interface=fsl.ApplyWarp(), - name='brain_warp') - brain_warp.inputs.interp = 'nn' + brain_warp = pe.Node(interface=fsl.ApplyWarp(), name="brain_warp") + brain_warp.inputs.interp = "nn" brain_warp.inputs.relwarp = True - nonlinear_register.connect(inputspec, 'input_brain', - brain_warp, 'in_file') + nonlinear_register.connect(inputspec, "input_brain", brain_warp, "in_file") - nonlinear_register.connect(nonlinear_reg, 'field_file', - brain_warp, 'field_file') + nonlinear_register.connect(nonlinear_reg, "field_file", brain_warp, "field_file") - nonlinear_register.connect(inputspec, 'reference_skull', - brain_warp, 'ref_file') + nonlinear_register.connect(inputspec, "reference_skull", brain_warp, "ref_file") - head_warp = pe.Node(interface=fsl.ApplyWarp(), - name='head_warp') - head_warp.inputs.interp = 'spline' + head_warp = pe.Node(interface=fsl.ApplyWarp(), name="head_warp") + head_warp.inputs.interp = "spline" head_warp.inputs.relwarp = True - nonlinear_register.connect(inputspec, 'input_brain', - head_warp, 'in_file') + nonlinear_register.connect(inputspec, "input_brain", head_warp, "in_file") - nonlinear_register.connect(nonlinear_reg, 'field_file', - head_warp, 'field_file') + nonlinear_register.connect(nonlinear_reg, "field_file", head_warp, "field_file") - nonlinear_register.connect(inputspec, 'reference_skull', - head_warp, 'ref_file') + nonlinear_register.connect(inputspec, "reference_skull", head_warp, "ref_file") - mask_warp = pe.Node(interface=fsl.ApplyWarp(), - name='mask_warp') - mask_warp.inputs.interp = 'nn' + mask_warp = pe.Node(interface=fsl.ApplyWarp(), name="mask_warp") + mask_warp.inputs.interp = "nn" mask_warp.inputs.relwarp = True - nonlinear_register.connect(inputspec, 'input_brain', - mask_warp, 'in_file') + nonlinear_register.connect(inputspec, "input_brain", mask_warp, "in_file") - nonlinear_register.connect(nonlinear_reg, 'field_file', - mask_warp, 'field_file') + nonlinear_register.connect(nonlinear_reg, "field_file", mask_warp, "field_file") - nonlinear_register.connect(inputspec, 'reference_skull', - mask_warp, 'ref_file') + nonlinear_register.connect(inputspec, "reference_skull", mask_warp, "ref_file") - biasfield_warp = pe.Node(interface=fsl.ApplyWarp(), - name='biasfield_warp') - biasfield_warp.inputs.interp = 'spline' + biasfield_warp = pe.Node(interface=fsl.ApplyWarp(), name="biasfield_warp") + biasfield_warp.inputs.interp = "spline" biasfield_warp.inputs.relwarp = True - nonlinear_register.connect(inputspec, 'input_brain', - biasfield_warp, 'in_file') + nonlinear_register.connect(inputspec, "input_brain", biasfield_warp, "in_file") - nonlinear_register.connect(nonlinear_reg, 'field_file', - biasfield_warp, 'field_file') + nonlinear_register.connect( + nonlinear_reg, "field_file", biasfield_warp, "field_file" + ) - nonlinear_register.connect(inputspec, 'reference_skull', - biasfield_warp, 'ref_file') + nonlinear_register.connect(inputspec, "reference_skull", biasfield_warp, "ref_file") - nonlinear_register.connect(nonlinear_reg, 'fieldcoeff_file', - outputspec, 'nonlinear_xfm') + nonlinear_register.connect( + nonlinear_reg, "fieldcoeff_file", outputspec, "nonlinear_xfm" + ) - nonlinear_register.connect(nonlinear_reg, 'field_file', - outputspec, 'nonlinear_warp') + nonlinear_register.connect( + nonlinear_reg, "field_file", outputspec, "nonlinear_warp" + ) - nonlinear_register.connect(brain_warp, 'out_file', - outputspec, 'output_brain') + nonlinear_register.connect(brain_warp, "out_file", outputspec, "output_brain") - nonlinear_register.connect(head_warp, 'out_file', - outputspec, 'output_head') + nonlinear_register.connect(head_warp, "out_file", outputspec, "output_head") - nonlinear_register.connect(mask_warp, 'out_file', - outputspec, 'output_mask') + nonlinear_register.connect(mask_warp, "out_file", outputspec, "output_mask") - nonlinear_register.connect(biasfield_warp, 'out_file', - outputspec, 'output_biasfield') + nonlinear_register.connect( + biasfield_warp, "out_file", outputspec, "output_biasfield" + ) return nonlinear_register -def create_register_func_to_anat(config, phase_diff_distcor=False, - name='register_func_to_anat'): - +def create_register_func_to_anat( + config, phase_diff_distcor=False, name="register_func_to_anat" +): """ Registers a functional scan in native space to anatomical space using a linear transform and does not include bbregister. @@ -668,7 +701,6 @@ def create_register_func_to_anat(config, phase_diff_distcor=False, Notes ----- - Workflow Inputs:: inputspec.func : string (nifti file) @@ -688,77 +720,93 @@ def create_register_func_to_anat(config, phase_diff_distcor=False, """ register_func_to_anat = pe.Workflow(name=name) - inputspec = pe.Node(util.IdentityInterface(fields=['func', - 'anat', - 'dof', - 'interp', - 'fieldmap', - 'fieldmapmask']), - name='inputspec') + inputspec = pe.Node( + util.IdentityInterface( + fields=["func", "anat", "dof", "interp", "fieldmap", "fieldmapmask"] + ), + name="inputspec", + ) inputNode_echospacing = pe.Node( - util.IdentityInterface(fields=['echospacing']), - name='echospacing_input') - - inputNode_pedir = pe.Node(util.IdentityInterface(fields=['pedir']), - name='pedir_input') + util.IdentityInterface(fields=["echospacing"]), name="echospacing_input" + ) - outputspec = pe.Node(util.IdentityInterface( - fields=['func_to_anat_linear_xfm_nobbreg', 'anat_func_nobbreg']), - name='outputspec') + inputNode_pedir = pe.Node( + util.IdentityInterface(fields=["pedir"]), name="pedir_input" + ) - linear_reg = pe.Node(interface=fsl.FLIRT(), - name='linear_func_to_anat') + outputspec = pe.Node( + util.IdentityInterface( + fields=["func_to_anat_linear_xfm_nobbreg", "anat_func_nobbreg"] + ), + name="outputspec", + ) - linear_reg.inputs.interp = config.registration_workflows['functional_registration']['coregistration']['interpolation'] - linear_reg.inputs.cost = config.registration_workflows['functional_registration']['coregistration']['cost'] - linear_reg.inputs.dof = config.registration_workflows['functional_registration']['coregistration']['dof'] - if config.registration_workflows['functional_registration']['coregistration']['arguments'] is not None: - linear_reg.inputs.args = config.registration_workflows['functional_registration']['coregistration']['arguments'] + linear_reg = pe.Node(interface=fsl.FLIRT(), name="linear_func_to_anat") + + linear_reg.inputs.interp = config.registration_workflows["functional_registration"][ + "coregistration" + ]["interpolation"] + linear_reg.inputs.cost = config.registration_workflows["functional_registration"][ + "coregistration" + ]["cost"] + linear_reg.inputs.dof = config.registration_workflows["functional_registration"][ + "coregistration" + ]["dof"] + if ( + config.registration_workflows["functional_registration"]["coregistration"][ + "arguments" + ] + is not None + ): + linear_reg.inputs.args = config.registration_workflows[ + "functional_registration" + ]["coregistration"]["arguments"] if phase_diff_distcor: - conv_pedir = \ - pe.Node(interface=util.Function(input_names=['pedir', - 'convert'], - output_names=['pedir'], - function=convert_pedir), - name='coreg_convert_pedir') - conv_pedir.inputs.convert = 'xyz_to_int' + conv_pedir = pe.Node( + interface=util.Function( + input_names=["pedir", "convert"], + output_names=["pedir"], + function=convert_pedir, + ), + name="coreg_convert_pedir", + ) + conv_pedir.inputs.convert = "xyz_to_int" - register_func_to_anat.connect(inputNode_pedir, 'pedir', - conv_pedir, 'pedir') - register_func_to_anat.connect(conv_pedir, 'pedir', - linear_reg, 'pedir') - register_func_to_anat.connect(inputspec, 'fieldmap', - linear_reg, 'fieldmap') - register_func_to_anat.connect(inputspec, 'fieldmapmask', - linear_reg, 'fieldmapmask') - register_func_to_anat.connect(inputNode_echospacing, 'echospacing', - linear_reg, 'echospacing') + register_func_to_anat.connect(inputNode_pedir, "pedir", conv_pedir, "pedir") + register_func_to_anat.connect(conv_pedir, "pedir", linear_reg, "pedir") + register_func_to_anat.connect(inputspec, "fieldmap", linear_reg, "fieldmap") + register_func_to_anat.connect( + inputspec, "fieldmapmask", linear_reg, "fieldmapmask" + ) + register_func_to_anat.connect( + inputNode_echospacing, "echospacing", linear_reg, "echospacing" + ) - register_func_to_anat.connect(inputspec, 'func', linear_reg, 'in_file') + register_func_to_anat.connect(inputspec, "func", linear_reg, "in_file") - register_func_to_anat.connect(inputspec, 'anat', linear_reg, 'reference') + register_func_to_anat.connect(inputspec, "anat", linear_reg, "reference") - register_func_to_anat.connect(inputspec, 'dof', linear_reg, 'dof') + register_func_to_anat.connect(inputspec, "dof", linear_reg, "dof") - register_func_to_anat.connect(inputspec, 'interp', linear_reg, 'interp') + register_func_to_anat.connect(inputspec, "interp", linear_reg, "interp") - register_func_to_anat.connect(linear_reg, 'out_matrix_file', - outputspec, - 'func_to_anat_linear_xfm_nobbreg') + register_func_to_anat.connect( + linear_reg, "out_matrix_file", outputspec, "func_to_anat_linear_xfm_nobbreg" + ) - register_func_to_anat.connect(linear_reg, 'out_file', - outputspec, 'anat_func_nobbreg') + register_func_to_anat.connect( + linear_reg, "out_file", outputspec, "anat_func_nobbreg" + ) return register_func_to_anat -def create_register_func_to_anat_use_T2(config, name='register_func_to_anat_use_T2'): +def create_register_func_to_anat_use_T2(config, name="register_func_to_anat_use_T2"): # for monkey data # ref: https://github.com/DCAN-Labs/dcan-macaque-pipeline/blob/master/fMRIVolume/GenericfMRIVolumeProcessingPipeline.sh#L287-L295 # https://github.com/HechengJin0/dcan-macaque-pipeline/blob/master/fMRIVolume/GenericfMRIVolumeProcessingPipeline.sh#L524-L535 - """ Registers a functional scan in native space to anatomical space using a linear transform and does not include bbregister, use T1 and T2 image. @@ -776,7 +824,6 @@ def create_register_func_to_anat_use_T2(config, name='register_func_to_anat_use_ Notes ----- - Workflow Inputs:: inputspec.func : string (nifti file) @@ -791,109 +838,130 @@ def create_register_func_to_anat_use_T2(config, name='register_func_to_anat_use_ outputspec.anat_func_nobbreg : string (nifti file) Functional scan registered to anatomical space """ - - register_func_to_anat_use_T2 = pe.Workflow(name=name) - inputspec = pe.Node(util.IdentityInterface(fields=['func', - 'T1_brain', - 'T2_head', - 'T2_brain']), - name='inputspec') + inputspec = pe.Node( + util.IdentityInterface(fields=["func", "T1_brain", "T2_head", "T2_brain"]), + name="inputspec", + ) - outputspec = pe.Node(util.IdentityInterface(fields=['func_to_anat_linear_xfm_nobbreg', - 'func_to_anat_linear_warp_nobbreg', - 'anat_func_nobbreg']), - name='outputspec') + outputspec = pe.Node( + util.IdentityInterface( + fields=[ + "func_to_anat_linear_xfm_nobbreg", + "func_to_anat_linear_warp_nobbreg", + "anat_func_nobbreg", + ] + ), + name="outputspec", + ) # ${FSLDIR}/bin/flirt -interp spline -dof 6 -in ${fMRIFolder}/${ScoutName}_gdc -ref ${T1wFolder}/${T2wRestoreImage} -omat "$fMRIFolder"/Scout2T2w.mat -out ${fMRIFolder}/Scout2T2w.nii.gz -searchrx -30 30 -searchry -30 30 -searchrz -30 30 -cost mutualinfo - linear_reg_func_to_t2 = pe.Node(interface=fsl.FLIRT(), - name='linear_reg_func_to_t2') - linear_reg_func_to_t2.inputs.interp = 'spline' - linear_reg_func_to_t2.inputs.cost = 'mutualinfo' + linear_reg_func_to_t2 = pe.Node(interface=fsl.FLIRT(), name="linear_reg_func_to_t2") + linear_reg_func_to_t2.inputs.interp = "spline" + linear_reg_func_to_t2.inputs.cost = "mutualinfo" linear_reg_func_to_t2.inputs.dof = 6 linear_reg_func_to_t2.inputs.searchr_x = [30, 30] linear_reg_func_to_t2.inputs.searchr_y = [30, 30] linear_reg_func_to_t2.inputs.searchr_z = [30, 30] - register_func_to_anat_use_T2.connect(inputspec, 'func', linear_reg_func_to_t2, 'in_file') + register_func_to_anat_use_T2.connect( + inputspec, "func", linear_reg_func_to_t2, "in_file" + ) - register_func_to_anat_use_T2.connect(inputspec, 'T2_head', linear_reg_func_to_t2, 'reference') + register_func_to_anat_use_T2.connect( + inputspec, "T2_head", linear_reg_func_to_t2, "reference" + ) # ${FSLDIR}/bin/convert_xfm -omat "$fMRIFolder"/T2w2Scout.mat -inverse "$fMRIFolder"/Scout2T2w.mat - invt = pe.Node(interface=fsl.ConvertXFM(), name='convert_xfm') + invt = pe.Node(interface=fsl.ConvertXFM(), name="convert_xfm") invt.inputs.invert_xfm = True - register_func_to_anat_use_T2.connect(linear_reg_func_to_t2, 'out_matrix_file', invt, 'in_file') + register_func_to_anat_use_T2.connect( + linear_reg_func_to_t2, "out_matrix_file", invt, "in_file" + ) # ${FSLDIR}/bin/applywarp --interp=nn -i ${T1wFolder}/${T2wRestoreImageBrain} -r ${fMRIFolder}/${ScoutName}_gdc --premat="$fMRIFolder"/T2w2Scout.mat -o ${fMRIFolder}/Scout_brain_mask.nii.gz - anat_to_func = pe.Node(interface=fsl.ApplyWarp(), - name='anat_to_func') - anat_to_func.inputs.interp = 'nn' + anat_to_func = pe.Node(interface=fsl.ApplyWarp(), name="anat_to_func") + anat_to_func.inputs.interp = "nn" - register_func_to_anat_use_T2.connect(inputspec, 'T2_brain', anat_to_func, 'in_file') - register_func_to_anat_use_T2.connect(inputspec, 'func', anat_to_func, 'ref_file') - register_func_to_anat_use_T2.connect(invt, 'out_file', anat_to_func, 'premat') + register_func_to_anat_use_T2.connect(inputspec, "T2_brain", anat_to_func, "in_file") + register_func_to_anat_use_T2.connect(inputspec, "func", anat_to_func, "ref_file") + register_func_to_anat_use_T2.connect(invt, "out_file", anat_to_func, "premat") # ${FSLDIR}/bin/fslmaths ${fMRIFolder}/Scout_brain_mask.nii.gz -bin ${fMRIFolder}/Scout_brain_mask.nii.gz - func_brain_mask = pe.Node(interface=fsl.maths.MathsCommand(), - name=f'func_brain_mask') - func_brain_mask.inputs.args = '-bin' + func_brain_mask = pe.Node( + interface=fsl.maths.MathsCommand(), name="func_brain_mask" + ) + func_brain_mask.inputs.args = "-bin" - register_func_to_anat_use_T2.connect(anat_to_func, 'out_file', func_brain_mask, 'in_file') + register_func_to_anat_use_T2.connect( + anat_to_func, "out_file", func_brain_mask, "in_file" + ) # ${FSLDIR}/bin/fslmaths ${fMRIFolder}/${ScoutName}_gdc -mas ${fMRIFolder}/Scout_brain_mask.nii.gz ${fMRIFolder}/Scout_brain_dc.nii.gz - func_brain = pe.Node(interface=fsl.MultiImageMaths(), - name='func_brain') + func_brain = pe.Node(interface=fsl.MultiImageMaths(), name="func_brain") func_brain.inputs.op_string = "-mas %s " - register_func_to_anat_use_T2.connect(inputspec, 'func', func_brain, 'in_file') - register_func_to_anat_use_T2.connect(func_brain_mask, 'out_file', func_brain, 'operand_files') + register_func_to_anat_use_T2.connect(inputspec, "func", func_brain, "in_file") + register_func_to_anat_use_T2.connect( + func_brain_mask, "out_file", func_brain, "operand_files" + ) # ## re-registering the maked brain to the T1 brain: # ${FSLDIR}/bin/flirt -interp spline -dof 6 -in ${fMRIFolder}/Scout_brain_dc.nii.gz -ref ${T1wFolder}/${T1wRestoreImageBrain} -omat "$fMRIFolder"/${ScoutName}_gdc2T1w_init.mat -out ${fMRIFolder}/${ScoutName}_gdc2T1w_brain_init -searchrx -30 30 -searchry -30 30 -searchrz -30 30 -cost mutualinfo - linear_reg_func_to_t1 = pe.Node(interface=fsl.FLIRT(), - name='linear_reg_func_to_t1') - linear_reg_func_to_t1.inputs.interp = 'spline' - linear_reg_func_to_t1.inputs.cost = 'mutualinfo' + linear_reg_func_to_t1 = pe.Node(interface=fsl.FLIRT(), name="linear_reg_func_to_t1") + linear_reg_func_to_t1.inputs.interp = "spline" + linear_reg_func_to_t1.inputs.cost = "mutualinfo" linear_reg_func_to_t1.inputs.dof = 6 linear_reg_func_to_t1.inputs.searchr_x = [30, 30] linear_reg_func_to_t1.inputs.searchr_y = [30, 30] linear_reg_func_to_t1.inputs.searchr_z = [30, 30] - register_func_to_anat_use_T2.connect(func_brain, 'out_file', linear_reg_func_to_t1, 'in_file') + register_func_to_anat_use_T2.connect( + func_brain, "out_file", linear_reg_func_to_t1, "in_file" + ) - register_func_to_anat_use_T2.connect(inputspec, 'T1_brain', linear_reg_func_to_t1, 'reference') + register_func_to_anat_use_T2.connect( + inputspec, "T1_brain", linear_reg_func_to_t1, "reference" + ) # #taking out warpfield as it is not being made without a fieldmap. # ${FSLDIR}/bin/convertwarp --relout --rel -r ${T1wFolder}/${T2wRestoreImage} --postmat=${fMRIFolder}/${ScoutName}_gdc2T1w_init.mat -o ${fMRIFolder}/${ScoutName}_gdc2T1w_init_warp - convert_warp = pe.Node(interface=fsl.ConvertWarp(), name='convert_warp') + convert_warp = pe.Node(interface=fsl.ConvertWarp(), name="convert_warp") convert_warp.inputs.out_relwarp = True convert_warp.inputs.relwarp = True - register_func_to_anat_use_T2.connect(linear_reg_func_to_t1, 'out_matrix_file', convert_warp, 'postmat') - - register_func_to_anat_use_T2.connect(inputspec, 'T2_head', convert_warp, 'reference') + register_func_to_anat_use_T2.connect( + linear_reg_func_to_t1, "out_matrix_file", convert_warp, "postmat" + ) + register_func_to_anat_use_T2.connect( + inputspec, "T2_head", convert_warp, "reference" + ) - register_func_to_anat_use_T2.connect(linear_reg_func_to_t1, 'out_matrix_file', - outputspec, - 'func_to_anat_linear_xfm_nobbreg') + register_func_to_anat_use_T2.connect( + linear_reg_func_to_t1, + "out_matrix_file", + outputspec, + "func_to_anat_linear_xfm_nobbreg", + ) - register_func_to_anat_use_T2.connect(convert_warp, 'out_file', - outputspec, - 'func_to_anat_linear_warp_nobbreg') + register_func_to_anat_use_T2.connect( + convert_warp, "out_file", outputspec, "func_to_anat_linear_warp_nobbreg" + ) - register_func_to_anat_use_T2.connect(linear_reg_func_to_t1, 'out_file', - outputspec, 'anat_func_nobbreg') + register_func_to_anat_use_T2.connect( + linear_reg_func_to_t1, "out_file", outputspec, "anat_func_nobbreg" + ) return register_func_to_anat_use_T2 -def create_bbregister_func_to_anat(phase_diff_distcor=False, - name='bbregister_func_to_anat'): - +def create_bbregister_func_to_anat( + phase_diff_distcor=False, name="bbregister_func_to_anat" +): """ Registers a functional scan in native space to structural. This is meant to be used after create_nonlinear_register() has been run and @@ -913,7 +981,6 @@ def create_bbregister_func_to_anat(phase_diff_distcor=False, Notes ----- - Workflow Inputs:: inputspec.func : string (nifti file) @@ -934,105 +1001,115 @@ def create_bbregister_func_to_anat(phase_diff_distcor=False, outputspec.anat_func : string (nifti file) Functional data in anatomical space """ - register_bbregister_func_to_anat = pe.Workflow(name=name) - inputspec = pe.Node(util.IdentityInterface(fields=['func', - 'anat', - 'linear_reg_matrix', - 'anat_wm_segmentation', - 'bbr_schedule', - 'bbr_wm_mask_args', - 'fieldmap', - 'fieldmapmask']), - name='inputspec') + inputspec = pe.Node( + util.IdentityInterface( + fields=[ + "func", + "anat", + "linear_reg_matrix", + "anat_wm_segmentation", + "bbr_schedule", + "bbr_wm_mask_args", + "fieldmap", + "fieldmapmask", + ] + ), + name="inputspec", + ) inputNode_echospacing = pe.Node( - util.IdentityInterface(fields=['echospacing']), - name='echospacing_input') + util.IdentityInterface(fields=["echospacing"]), name="echospacing_input" + ) - inputNode_pedir = pe.Node(util.IdentityInterface(fields=['pedir']), - name='pedir_input') + inputNode_pedir = pe.Node( + util.IdentityInterface(fields=["pedir"]), name="pedir_input" + ) - outputspec = pe.Node(util.IdentityInterface( - fields=['func_to_anat_linear_xfm', 'anat_func']), name='outputspec') + outputspec = pe.Node( + util.IdentityInterface(fields=["func_to_anat_linear_xfm", "anat_func"]), + name="outputspec", + ) - wm_bb_mask = pe.Node(interface=fsl.ImageMaths(), - name='wm_bb_mask') + wm_bb_mask = pe.Node(interface=fsl.ImageMaths(), name="wm_bb_mask") register_bbregister_func_to_anat.connect( - inputspec, 'bbr_wm_mask_args', - wm_bb_mask, 'op_string') + inputspec, "bbr_wm_mask_args", wm_bb_mask, "op_string" + ) - register_bbregister_func_to_anat.connect(inputspec, - 'anat_wm_segmentation', - wm_bb_mask, 'in_file') + register_bbregister_func_to_anat.connect( + inputspec, "anat_wm_segmentation", wm_bb_mask, "in_file" + ) def bbreg_args(bbreg_target): - return '-cost bbr -wmseg ' + bbreg_target + return "-cost bbr -wmseg " + bbreg_target - bbreg_func_to_anat = pe.Node(interface=fsl.FLIRT(), - name='bbreg_func_to_anat') + bbreg_func_to_anat = pe.Node(interface=fsl.FLIRT(), name="bbreg_func_to_anat") bbreg_func_to_anat.inputs.dof = 6 register_bbregister_func_to_anat.connect( - inputspec, 'bbr_schedule', - bbreg_func_to_anat, 'schedule') + inputspec, "bbr_schedule", bbreg_func_to_anat, "schedule" + ) register_bbregister_func_to_anat.connect( - wm_bb_mask, ('out_file', bbreg_args), - bbreg_func_to_anat, 'args') + wm_bb_mask, ("out_file", bbreg_args), bbreg_func_to_anat, "args" + ) register_bbregister_func_to_anat.connect( - inputspec, 'func', - bbreg_func_to_anat, 'in_file') + inputspec, "func", bbreg_func_to_anat, "in_file" + ) register_bbregister_func_to_anat.connect( - inputspec, 'anat', - bbreg_func_to_anat, 'reference') + inputspec, "anat", bbreg_func_to_anat, "reference" + ) register_bbregister_func_to_anat.connect( - inputspec, 'linear_reg_matrix', - bbreg_func_to_anat, 'in_matrix_file') + inputspec, "linear_reg_matrix", bbreg_func_to_anat, "in_matrix_file" + ) if phase_diff_distcor: - conv_pedir = \ - pe.Node(interface=util.Function(input_names=['pedir', - 'convert'], - output_names=['pedir'], - function=convert_pedir), - name='bbreg_convert_pedir') - conv_pedir.inputs.convert = 'xyz_to_int' - - register_bbregister_func_to_anat.connect(inputNode_pedir, 'pedir', - conv_pedir, 'pedir') - register_bbregister_func_to_anat.connect(conv_pedir, 'pedir', - bbreg_func_to_anat, 'pedir') + conv_pedir = pe.Node( + interface=util.Function( + input_names=["pedir", "convert"], + output_names=["pedir"], + function=convert_pedir, + ), + name="bbreg_convert_pedir", + ) + conv_pedir.inputs.convert = "xyz_to_int" + register_bbregister_func_to_anat.connect( - inputspec, 'fieldmap', - bbreg_func_to_anat, 'fieldmap') + inputNode_pedir, "pedir", conv_pedir, "pedir" + ) register_bbregister_func_to_anat.connect( - inputspec, 'fieldmapmask', - bbreg_func_to_anat, 'fieldmapmask') + conv_pedir, "pedir", bbreg_func_to_anat, "pedir" + ) + register_bbregister_func_to_anat.connect( + inputspec, "fieldmap", bbreg_func_to_anat, "fieldmap" + ) register_bbregister_func_to_anat.connect( - inputNode_echospacing, 'echospacing', - bbreg_func_to_anat, 'echospacing') + inputspec, "fieldmapmask", bbreg_func_to_anat, "fieldmapmask" + ) + register_bbregister_func_to_anat.connect( + inputNode_echospacing, "echospacing", bbreg_func_to_anat, "echospacing" + ) register_bbregister_func_to_anat.connect( - bbreg_func_to_anat, 'out_matrix_file', - outputspec, 'func_to_anat_linear_xfm') + bbreg_func_to_anat, "out_matrix_file", outputspec, "func_to_anat_linear_xfm" + ) register_bbregister_func_to_anat.connect( - bbreg_func_to_anat, 'out_file', - outputspec, 'anat_func') + bbreg_func_to_anat, "out_file", outputspec, "anat_func" + ) return register_bbregister_func_to_anat def create_wf_calculate_ants_warp( - name='create_wf_calculate_ants_warp', num_threads=1, reg_ants_skull=1 + name="create_wf_calculate_ants_warp", num_threads=1, reg_ants_skull=1 ): - ''' + """ Calculates the nonlinear ANTS registration transform. This workflow employs the antsRegistration tool: @@ -1050,7 +1127,6 @@ def create_wf_calculate_ants_warp( Notes ----- - Some of the inputs listed below are lists or lists of lists. This is because antsRegistration can perform multiple stages of calculations depending on how the user configures their registration. @@ -1157,814 +1233,1018 @@ def create_wf_calculate_ants_warp( .. image:: :width: 500 - ''' - + """ calc_ants_warp_wf = pe.Workflow(name=name) - inputspec = pe.Node(util.IdentityInterface( - fields=['moving_brain', - 'reference_brain', - 'moving_skull', - 'reference_skull', - 'reference_mask', - 'moving_mask', - 'fixed_image_mask', - 'ants_para', - 'interp']), - name='inputspec') - - outputspec = pe.Node(util.IdentityInterface( - fields=['ants_initial_xfm', - 'ants_rigid_xfm', - 'ants_affine_xfm', - 'warp_field', - 'inverse_warp_field', - 'composite_transform', - 'wait', - 'normalized_output_brain']), name='outputspec') + inputspec = pe.Node( + util.IdentityInterface( + fields=[ + "moving_brain", + "reference_brain", + "moving_skull", + "reference_skull", + "reference_mask", + "moving_mask", + "fixed_image_mask", + "ants_para", + "interp", + ] + ), + name="inputspec", + ) + + outputspec = pe.Node( + util.IdentityInterface( + fields=[ + "ants_initial_xfm", + "ants_rigid_xfm", + "ants_affine_xfm", + "warp_field", + "inverse_warp_field", + "composite_transform", + "wait", + "normalized_output_brain", + ] + ), + name="outputspec", + ) # use ANTS to warp the masked anatomical image to a template image - ''' + """ calculate_ants_warp = pe.Node(interface=ants.Registration(), name='calculate_ants_warp') calculate_ants_warp.inputs.output_warped_image = True calculate_ants_warp.inputs.initial_moving_transform_com = 0 - ''' - reg_imports = ['import os', 'import subprocess'] - calculate_ants_warp = \ - pe.Node(interface=util.Function(input_names=['moving_brain', - 'reference_brain', - 'moving_skull', - 'reference_skull', - 'ants_para', - 'moving_mask', - 'reference_mask', - 'fixed_image_mask', - 'interp', - 'reg_with_skull'], - output_names=['warp_list', - 'warped_image'], - function=hardcoded_reg, - imports=reg_imports), - name='calc_ants_warp', - mem_gb=2.8, - mem_x=(2e-7, 'moving_brain', 'xyz')) + """ + reg_imports = ["import os", "import subprocess"] + calculate_ants_warp = pe.Node( + interface=util.Function( + input_names=[ + "moving_brain", + "reference_brain", + "moving_skull", + "reference_skull", + "ants_para", + "moving_mask", + "reference_mask", + "fixed_image_mask", + "interp", + "reg_with_skull", + ], + output_names=["warp_list", "warped_image"], + function=hardcoded_reg, + imports=reg_imports, + ), + name="calc_ants_warp", + mem_gb=2.8, + mem_x=(2e-7, "moving_brain", "xyz"), + ) calculate_ants_warp.interface.num_threads = num_threads - select_forward_initial = pe.Node(util.Function( - input_names=['warp_list', 'selection'], - output_names=['selected_warp'], - function=seperate_warps_list), name='select_forward_initial') + select_forward_initial = pe.Node( + util.Function( + input_names=["warp_list", "selection"], + output_names=["selected_warp"], + function=seperate_warps_list, + ), + name="select_forward_initial", + ) select_forward_initial.inputs.selection = "Initial" - select_forward_rigid = pe.Node(util.Function( - input_names=['warp_list', 'selection'], - output_names=['selected_warp'], - function=seperate_warps_list), name='select_forward_rigid') + select_forward_rigid = pe.Node( + util.Function( + input_names=["warp_list", "selection"], + output_names=["selected_warp"], + function=seperate_warps_list, + ), + name="select_forward_rigid", + ) select_forward_rigid.inputs.selection = "Rigid" - select_forward_affine = pe.Node(util.Function( - input_names=['warp_list', 'selection'], - output_names=['selected_warp'], - function=seperate_warps_list), name='select_forward_affine') + select_forward_affine = pe.Node( + util.Function( + input_names=["warp_list", "selection"], + output_names=["selected_warp"], + function=seperate_warps_list, + ), + name="select_forward_affine", + ) select_forward_affine.inputs.selection = "Affine" - select_forward_warp = pe.Node(util.Function( - input_names=['warp_list', 'selection'], - output_names=['selected_warp'], - function=seperate_warps_list), name='select_forward_warp') + select_forward_warp = pe.Node( + util.Function( + input_names=["warp_list", "selection"], + output_names=["selected_warp"], + function=seperate_warps_list, + ), + name="select_forward_warp", + ) select_forward_warp.inputs.selection = "Warp" - select_inverse_warp = pe.Node(util.Function( - input_names=['warp_list', 'selection'], - output_names=['selected_warp'], - function=seperate_warps_list), name='select_inverse_warp') + select_inverse_warp = pe.Node( + util.Function( + input_names=["warp_list", "selection"], + output_names=["selected_warp"], + function=seperate_warps_list, + ), + name="select_inverse_warp", + ) select_inverse_warp.inputs.selection = "Inverse" calc_ants_warp_wf.connect( - inputspec, 'moving_brain', - calculate_ants_warp, 'moving_brain') + inputspec, "moving_brain", calculate_ants_warp, "moving_brain" + ) calc_ants_warp_wf.connect( - inputspec, 'reference_brain', - calculate_ants_warp, 'reference_brain') + inputspec, "reference_brain", calculate_ants_warp, "reference_brain" + ) if reg_ants_skull == 1: - calculate_ants_warp.inputs.reg_with_skull = 1 calc_ants_warp_wf.connect( - inputspec, 'moving_skull', - calculate_ants_warp, 'moving_skull') + inputspec, "moving_skull", calculate_ants_warp, "moving_skull" + ) calc_ants_warp_wf.connect( - inputspec, 'reference_skull', - calculate_ants_warp, 'reference_skull') + inputspec, "reference_skull", calculate_ants_warp, "reference_skull" + ) else: calc_ants_warp_wf.connect( - inputspec, 'moving_brain', - calculate_ants_warp, 'moving_skull') + inputspec, "moving_brain", calculate_ants_warp, "moving_skull" + ) calc_ants_warp_wf.connect( - inputspec, 'reference_brain', - calculate_ants_warp, 'reference_skull') + inputspec, "reference_brain", calculate_ants_warp, "reference_skull" + ) calc_ants_warp_wf.connect( - inputspec, 'fixed_image_mask', - calculate_ants_warp, 'fixed_image_mask') + inputspec, "fixed_image_mask", calculate_ants_warp, "fixed_image_mask" + ) - calc_ants_warp_wf.connect(inputspec, 'reference_mask', - calculate_ants_warp, 'reference_mask') + calc_ants_warp_wf.connect( + inputspec, "reference_mask", calculate_ants_warp, "reference_mask" + ) - calc_ants_warp_wf.connect(inputspec, 'moving_mask', - calculate_ants_warp, 'moving_mask') + calc_ants_warp_wf.connect( + inputspec, "moving_mask", calculate_ants_warp, "moving_mask" + ) - calc_ants_warp_wf.connect(inputspec, 'ants_para', - calculate_ants_warp, 'ants_para') + calc_ants_warp_wf.connect(inputspec, "ants_para", calculate_ants_warp, "ants_para") - calc_ants_warp_wf.connect( - inputspec, 'interp', - calculate_ants_warp, 'interp') + calc_ants_warp_wf.connect(inputspec, "interp", calculate_ants_warp, "interp") # inter-workflow connections calc_ants_warp_wf.connect( - calculate_ants_warp, 'warp_list', - select_forward_initial, 'warp_list') + calculate_ants_warp, "warp_list", select_forward_initial, "warp_list" + ) calc_ants_warp_wf.connect( - calculate_ants_warp, 'warp_list', - select_forward_rigid, 'warp_list') + calculate_ants_warp, "warp_list", select_forward_rigid, "warp_list" + ) calc_ants_warp_wf.connect( - calculate_ants_warp, 'warp_list', - select_forward_affine, 'warp_list') + calculate_ants_warp, "warp_list", select_forward_affine, "warp_list" + ) calc_ants_warp_wf.connect( - calculate_ants_warp, 'warp_list', - select_forward_warp, 'warp_list') + calculate_ants_warp, "warp_list", select_forward_warp, "warp_list" + ) calc_ants_warp_wf.connect( - calculate_ants_warp, 'warp_list', - select_inverse_warp, 'warp_list') + calculate_ants_warp, "warp_list", select_inverse_warp, "warp_list" + ) # connections to outputspec calc_ants_warp_wf.connect( - select_forward_initial, 'selected_warp', - outputspec, 'ants_initial_xfm') + select_forward_initial, "selected_warp", outputspec, "ants_initial_xfm" + ) calc_ants_warp_wf.connect( - select_forward_rigid, 'selected_warp', - outputspec, 'ants_rigid_xfm') + select_forward_rigid, "selected_warp", outputspec, "ants_rigid_xfm" + ) calc_ants_warp_wf.connect( - select_forward_affine, 'selected_warp', - outputspec, 'ants_affine_xfm') + select_forward_affine, "selected_warp", outputspec, "ants_affine_xfm" + ) calc_ants_warp_wf.connect( - select_forward_warp, 'selected_warp', - outputspec, 'warp_field') + select_forward_warp, "selected_warp", outputspec, "warp_field" + ) calc_ants_warp_wf.connect( - select_inverse_warp, 'selected_warp', - outputspec, 'inverse_warp_field') + select_inverse_warp, "selected_warp", outputspec, "inverse_warp_field" + ) calc_ants_warp_wf.connect( - calculate_ants_warp, 'warped_image', - outputspec, 'normalized_output_brain') + calculate_ants_warp, "warped_image", outputspec, "normalized_output_brain" + ) return calc_ants_warp_wf -def FSL_registration_connector(wf_name, cfg, orig="T1w", opt=None, - symmetric=False, template="T1w"): - +def FSL_registration_connector( + wf_name, cfg, orig="T1w", opt=None, symmetric=False, template="T1w" +): wf = pe.Workflow(name=wf_name) inputNode = pe.Node( - util.IdentityInterface(fields=['input_brain', - 'reference_brain', - 'input_head', - 'reference_head', - 'input_mask', - 'reference_mask', - 'transform', - 'interpolation', - 'fnirt_config']), - name='inputspec') - - sym = '' - symm = '' - if symmetric: - sym = 'sym' - symm = '_symmetric' + util.IdentityInterface( + fields=[ + "input_brain", + "reference_brain", + "input_head", + "reference_head", + "input_mask", + "reference_mask", + "transform", + "interpolation", + "fnirt_config", + ] + ), + name="inputspec", + ) - tmpl = '' - if template == 'EPI': - tmpl = 'EPI' + sym = "" + symm = "" + if symmetric: + sym = "sym" + symm = "_symmetric" - if opt == 'FSL' or opt == 'FSL-linear': + tmpl = "" + if template == "EPI": + tmpl = "EPI" + if opt in ("FSL", "FSL-linear"): flirt_reg_anat_mni = create_fsl_flirt_linear_reg( - f'anat_mni_flirt_register{symm}' + f"anat_mni_flirt_register{symm}" ) # Input registration parameters - wf.connect(inputNode, 'interpolation', - flirt_reg_anat_mni, 'inputspec.interp') + wf.connect(inputNode, "interpolation", flirt_reg_anat_mni, "inputspec.interp") - wf.connect(inputNode, 'input_brain', - flirt_reg_anat_mni, 'inputspec.input_brain') + wf.connect( + inputNode, "input_brain", flirt_reg_anat_mni, "inputspec.input_brain" + ) - wf.connect(inputNode, 'reference_brain', flirt_reg_anat_mni, - 'inputspec.reference_brain') + wf.connect( + inputNode, + "reference_brain", + flirt_reg_anat_mni, + "inputspec.reference_brain", + ) - write_lin_composite_xfm = pe.Node(interface=fsl.ConvertWarp(), - name=f'fsl_lin-warp_to_nii{symm}') + write_lin_composite_xfm = pe.Node( + interface=fsl.ConvertWarp(), name=f"fsl_lin-warp_to_nii{symm}" + ) - wf.connect(inputNode, 'reference_brain', - write_lin_composite_xfm, 'reference') + wf.connect(inputNode, "reference_brain", write_lin_composite_xfm, "reference") - wf.connect(flirt_reg_anat_mni, 'outputspec.linear_xfm', - write_lin_composite_xfm, 'premat') + wf.connect( + flirt_reg_anat_mni, + "outputspec.linear_xfm", + write_lin_composite_xfm, + "premat", + ) - write_invlin_composite_xfm = pe.Node(interface=fsl.ConvertWarp(), - name=f'fsl_invlin-warp_to_' - f'nii{symm}') + write_invlin_composite_xfm = pe.Node( + interface=fsl.ConvertWarp(), name=f"fsl_invlin-warp_to_" f"nii{symm}" + ) - wf.connect(inputNode, 'reference_brain', - write_invlin_composite_xfm, 'reference') + wf.connect( + inputNode, "reference_brain", write_invlin_composite_xfm, "reference" + ) - wf.connect(flirt_reg_anat_mni, 'outputspec.invlinear_xfm', - write_invlin_composite_xfm, 'premat') + wf.connect( + flirt_reg_anat_mni, + "outputspec.invlinear_xfm", + write_invlin_composite_xfm, + "premat", + ) outputs = { - f'space-{sym}template_desc-preproc_{orig}': ( - flirt_reg_anat_mni, 'outputspec.output_brain'), - f'from-{orig}_to-{sym}{tmpl}template_mode-image_desc-linear_xfm': ( - write_lin_composite_xfm, 'out_file'), - f'from-{sym}{tmpl}template_to-{orig}_mode-image_desc-linear_xfm': ( - write_invlin_composite_xfm, 'out_file'), - f'from-{orig}_to-{sym}{tmpl}template_mode-image_xfm': ( - write_lin_composite_xfm, 'out_file') + f"space-{sym}template_desc-preproc_{orig}": ( + flirt_reg_anat_mni, + "outputspec.output_brain", + ), + f"from-{orig}_to-{sym}{tmpl}template_mode-image_desc-linear_xfm": ( + write_lin_composite_xfm, + "out_file", + ), + f"from-{sym}{tmpl}template_to-{orig}_mode-image_desc-linear_xfm": ( + write_invlin_composite_xfm, + "out_file", + ), + f"from-{orig}_to-{sym}{tmpl}template_mode-image_xfm": ( + write_lin_composite_xfm, + "out_file", + ), } - - if opt == 'FSL': - if cfg.registration_workflows['anatomical_registration']['registration']['FSL-FNIRT']['ref_resolution'] == \ - cfg.registration_workflows['anatomical_registration']['resolution_for_anat']: + if opt == "FSL": + if ( + cfg.registration_workflows["anatomical_registration"]["registration"][ + "FSL-FNIRT" + ]["ref_resolution"] + == cfg.registration_workflows["anatomical_registration"][ + "resolution_for_anat" + ] + ): fnirt_reg_anat_mni = create_fsl_fnirt_nonlinear_reg( - f'anat_mni_fnirt_register{symm}' + f"anat_mni_fnirt_register{symm}" ) else: fnirt_reg_anat_mni = create_fsl_fnirt_nonlinear_reg_nhp( - f'anat_mni_fnirt_register{symm}' + f"anat_mni_fnirt_register{symm}" ) - wf.connect(inputNode, 'input_brain', - fnirt_reg_anat_mni, 'inputspec.input_brain') + wf.connect( + inputNode, "input_brain", fnirt_reg_anat_mni, "inputspec.input_brain" + ) - wf.connect(inputNode, 'reference_brain', - fnirt_reg_anat_mni, 'inputspec.reference_brain') + wf.connect( + inputNode, + "reference_brain", + fnirt_reg_anat_mni, + "inputspec.reference_brain", + ) - wf.connect(inputNode, 'input_head', - fnirt_reg_anat_mni, 'inputspec.input_skull') + wf.connect(inputNode, "input_head", fnirt_reg_anat_mni, "inputspec.input_skull") # NOTE: crossover from above opt block - wf.connect(flirt_reg_anat_mni, 'outputspec.linear_xfm', - fnirt_reg_anat_mni, 'inputspec.linear_aff') + wf.connect( + flirt_reg_anat_mni, + "outputspec.linear_xfm", + fnirt_reg_anat_mni, + "inputspec.linear_aff", + ) - wf.connect(inputNode, 'reference_head', - fnirt_reg_anat_mni, 'inputspec.reference_skull') + wf.connect( + inputNode, "reference_head", fnirt_reg_anat_mni, "inputspec.reference_skull" + ) - wf.connect(inputNode, 'reference_mask', - fnirt_reg_anat_mni, 'inputspec.ref_mask') + wf.connect( + inputNode, "reference_mask", fnirt_reg_anat_mni, "inputspec.ref_mask" + ) # assign the FSL FNIRT config file specified in pipeline config.yml - wf.connect(inputNode, 'fnirt_config', - fnirt_reg_anat_mni, 'inputspec.fnirt_config') + wf.connect( + inputNode, "fnirt_config", fnirt_reg_anat_mni, "inputspec.fnirt_config" + ) - if cfg.registration_workflows['anatomical_registration']['registration']['FSL-FNIRT']['ref_resolution'] == \ - cfg.registration_workflows['anatomical_registration']['resolution_for_anat']: + if ( + cfg.registration_workflows["anatomical_registration"]["registration"][ + "FSL-FNIRT" + ]["ref_resolution"] + == cfg.registration_workflows["anatomical_registration"][ + "resolution_for_anat" + ] + ): # NOTE: this is an UPDATE because of the opt block above added_outputs = { - f'space-{sym}template_desc-preproc_{orig}': ( - fnirt_reg_anat_mni, 'outputspec.output_brain'), - f'from-{orig}_to-{sym}{tmpl}template_mode-image_xfm': ( - fnirt_reg_anat_mni, 'outputspec.nonlinear_xfm') + f"space-{sym}template_desc-preproc_{orig}": ( + fnirt_reg_anat_mni, + "outputspec.output_brain", + ), + f"from-{orig}_to-{sym}{tmpl}template_mode-image_xfm": ( + fnirt_reg_anat_mni, + "outputspec.nonlinear_xfm", + ), } outputs.update(added_outputs) else: # NOTE: this is an UPDATE because of the opt block above added_outputs = { - f'space-{sym}template_desc-preproc_{orig}': ( - fnirt_reg_anat_mni, 'outputspec.output_brain'), - f'space-{sym}template_desc-head_{orig}': ( - fnirt_reg_anat_mni, 'outputspec.output_head'), - f'space-{sym}template_desc-{orig}_mask': ( - fnirt_reg_anat_mni, 'outputspec.output_mask'), - f'space-{sym}template_desc-T1wT2w_biasfield': ( - fnirt_reg_anat_mni, 'outputspec.output_biasfield'), - f'from-{orig}_to-{sym}{tmpl}template_mode-image_xfm': ( - fnirt_reg_anat_mni, 'outputspec.nonlinear_xfm'), - f'from-{orig}_to-{sym}{tmpl}template_mode-image_warp': ( - fnirt_reg_anat_mni, 'outputspec.nonlinear_warp') + f"space-{sym}template_desc-preproc_{orig}": ( + fnirt_reg_anat_mni, + "outputspec.output_brain", + ), + f"space-{sym}template_desc-head_{orig}": ( + fnirt_reg_anat_mni, + "outputspec.output_head", + ), + f"space-{sym}template_desc-{orig}_mask": ( + fnirt_reg_anat_mni, + "outputspec.output_mask", + ), + f"space-{sym}template_desc-T1wT2w_biasfield": ( + fnirt_reg_anat_mni, + "outputspec.output_biasfield", + ), + f"from-{orig}_to-{sym}{tmpl}template_mode-image_xfm": ( + fnirt_reg_anat_mni, + "outputspec.nonlinear_xfm", + ), + f"from-{orig}_to-{sym}{tmpl}template_mode-image_warp": ( + fnirt_reg_anat_mni, + "outputspec.nonlinear_warp", + ), } outputs.update(added_outputs) return (wf, outputs) -def ANTs_registration_connector(wf_name, cfg, params, orig="T1w", - symmetric=False, template="T1w"): - +def ANTs_registration_connector( + wf_name, cfg, params, orig="T1w", symmetric=False, template="T1w" +): wf = pe.Workflow(name=wf_name) inputNode = pe.Node( - util.IdentityInterface(fields=['input_brain', - 'reference_brain', - 'input_head', - 'reference_head', - 'input_mask', - 'reference_mask', - 'transform', - 'interpolation']), - name='inputspec') - - sym = '' - symm = '' + util.IdentityInterface( + fields=[ + "input_brain", + "reference_brain", + "input_head", + "reference_head", + "input_mask", + "reference_mask", + "transform", + "interpolation", + ] + ), + name="inputspec", + ) + + sym = "" + symm = "" if symmetric: - sym = 'sym' - symm = '_symmetric' + sym = "sym" + symm = "_symmetric" - tmpl = '' - if template == 'EPI': - tmpl = 'EPI' + tmpl = "" + if template == "EPI": + tmpl = "EPI" if params is None: - err_msg = '\n\n[!] C-PAC says: \nYou have selected ANTs as your ' \ - 'anatomical registration method.\n' \ - 'However, no ANTs parameters were specified.\n' \ - 'Please specify ANTs parameters properly and try again.' + err_msg = ( + "\n\n[!] C-PAC says: \nYou have selected ANTs as your " + "anatomical registration method.\n" + "However, no ANTs parameters were specified.\n" + "Please specify ANTs parameters properly and try again." + ) raise Exception(err_msg) - ants_reg_anat_mni = \ - create_wf_calculate_ants_warp( - f'anat_mni_ants_register{symm}', - num_threads=cfg.pipeline_setup['system_config'][ - 'num_ants_threads'], - reg_ants_skull=cfg['registration_workflows'][ - 'anatomical_registration']['reg_with_skull'] - ) + ants_reg_anat_mni = create_wf_calculate_ants_warp( + f"anat_mni_ants_register{symm}", + num_threads=cfg.pipeline_setup["system_config"]["num_ants_threads"], + reg_ants_skull=cfg["registration_workflows"]["anatomical_registration"][ + "reg_with_skull" + ], + ) ants_reg_anat_mni.inputs.inputspec.ants_para = params - wf.connect(inputNode, 'interpolation', - ants_reg_anat_mni, 'inputspec.interp') + wf.connect(inputNode, "interpolation", ants_reg_anat_mni, "inputspec.interp") # calculating the transform with the skullstripped is # reported to be better, but it requires very high # quality skullstripping. If skullstripping is imprecise # registration with skull is preferred - wf.connect(inputNode, 'input_brain', - ants_reg_anat_mni, 'inputspec.moving_brain') + wf.connect(inputNode, "input_brain", ants_reg_anat_mni, "inputspec.moving_brain") - wf.connect(inputNode, 'reference_brain', - ants_reg_anat_mni, 'inputspec.reference_brain') + wf.connect( + inputNode, "reference_brain", ants_reg_anat_mni, "inputspec.reference_brain" + ) - wf.connect(inputNode, 'input_head', - ants_reg_anat_mni, 'inputspec.moving_skull') + wf.connect(inputNode, "input_head", ants_reg_anat_mni, "inputspec.moving_skull") - wf.connect(inputNode, 'reference_head', - ants_reg_anat_mni, 'inputspec.reference_skull') + wf.connect( + inputNode, "reference_head", ants_reg_anat_mni, "inputspec.reference_skull" + ) - wf.connect(inputNode, 'input_mask', - ants_reg_anat_mni, 'inputspec.moving_mask') + wf.connect(inputNode, "input_mask", ants_reg_anat_mni, "inputspec.moving_mask") - wf.connect(inputNode, 'reference_mask', - ants_reg_anat_mni, 'inputspec.reference_mask') + wf.connect( + inputNode, "reference_mask", ants_reg_anat_mni, "inputspec.reference_mask" + ) ants_reg_anat_mni.inputs.inputspec.fixed_image_mask = None - if orig == 'T1w': - if cfg.registration_workflows['anatomical_registration'][ - 'registration']['ANTs']['use_lesion_mask']: + if orig == "T1w": + if cfg.registration_workflows["anatomical_registration"]["registration"][ + "ANTs" + ]["use_lesion_mask"]: # Create lesion preproc node to apply afni Refit and Resample - lesion_preproc = create_lesion_preproc( - wf_name=f'lesion_preproc{symm}' + lesion_preproc = create_lesion_preproc(wf_name=f"lesion_preproc{symm}") + wf.connect(inputNode, "lesion_mask", lesion_preproc, "inputspec.lesion") + wf.connect( + lesion_preproc, + "outputspec.reorient", + ants_reg_anat_mni, + "inputspec.fixed_image_mask", ) - wf.connect(inputNode, 'lesion_mask', - lesion_preproc, 'inputspec.lesion') - wf.connect(lesion_preproc, 'outputspec.reorient', - ants_reg_anat_mni, 'inputspec.fixed_image_mask') # combine the linear xfm's into one - makes it easier downstream write_composite_linear_xfm = pe.Node( interface=ants.ApplyTransforms(), - name=f'write_composite_linear{symm}_xfm', + name=f"write_composite_linear{symm}_xfm", mem_gb=1.155, - mem_x=(1708448960473801 / 1208925819614629174706176, 'input_image')) + mem_x=(1708448960473801 / 1208925819614629174706176, "input_image"), + ) write_composite_linear_xfm.inputs.print_out_composite_warp_file = True - write_composite_linear_xfm.inputs.output_image = \ + write_composite_linear_xfm.inputs.output_image = ( f"from-{orig}_to-{sym}{tmpl}template_mode-image_desc-linear_xfm.nii.gz" + ) - wf.connect(inputNode, 'input_brain', - write_composite_linear_xfm, 'input_image') + wf.connect(inputNode, "input_brain", write_composite_linear_xfm, "input_image") - wf.connect(inputNode, 'reference_brain', - write_composite_linear_xfm, 'reference_image') + wf.connect( + inputNode, "reference_brain", write_composite_linear_xfm, "reference_image" + ) - wf.connect(inputNode, 'interpolation', - write_composite_linear_xfm, 'interpolation') + wf.connect(inputNode, "interpolation", write_composite_linear_xfm, "interpolation") write_composite_linear_xfm.inputs.input_image_type = 0 write_composite_linear_xfm.inputs.dimension = 3 - collect_transforms = pe.Node(util.Merge(3), - name=f'collect_transforms{symm}', - mem_gb=0.8, - mem_x=(263474863123069 / - 37778931862957161709568, - 'in1')) + collect_transforms = pe.Node( + util.Merge(3), + name=f"collect_transforms{symm}", + mem_gb=0.8, + mem_x=(263474863123069 / 37778931862957161709568, "in1"), + ) - wf.connect(ants_reg_anat_mni, 'outputspec.ants_affine_xfm', - collect_transforms, 'in1') + wf.connect( + ants_reg_anat_mni, "outputspec.ants_affine_xfm", collect_transforms, "in1" + ) - wf.connect(ants_reg_anat_mni, 'outputspec.ants_rigid_xfm', - collect_transforms, 'in2') + wf.connect( + ants_reg_anat_mni, "outputspec.ants_rigid_xfm", collect_transforms, "in2" + ) - wf.connect(ants_reg_anat_mni, 'outputspec.ants_initial_xfm', - collect_transforms, 'in3') + wf.connect( + ants_reg_anat_mni, "outputspec.ants_initial_xfm", collect_transforms, "in3" + ) # check transform list to exclude Nonetype (missing) init/rig/affine check_transform = pe.Node( - util.Function(input_names=['transform_list'], - output_names=['checked_transform_list', - 'list_length'], - function=check_transforms), - name=f'check_transforms', - mem_gb=6) + util.Function( + input_names=["transform_list"], + output_names=["checked_transform_list", "list_length"], + function=check_transforms, + ), + name="check_transforms", + mem_gb=6, + ) - wf.connect(collect_transforms, 'out', check_transform, 'transform_list') + wf.connect(collect_transforms, "out", check_transform, "transform_list") - wf.connect(check_transform, 'checked_transform_list', - write_composite_linear_xfm, 'transforms') + wf.connect( + check_transform, + "checked_transform_list", + write_composite_linear_xfm, + "transforms", + ) # combine the linear xfm's into one - makes it easier downstream write_composite_invlinear_xfm = pe.Node( interface=ants.ApplyTransforms(), - name=f'write_composite_invlinear{symm}_xfm', + name=f"write_composite_invlinear{symm}_xfm", mem_gb=1.05, - mem_x=(1367826948979337 / 151115727451828646838272, 'input_image')) + mem_x=(1367826948979337 / 151115727451828646838272, "input_image"), + ) write_composite_invlinear_xfm.inputs.print_out_composite_warp_file = True - write_composite_invlinear_xfm.inputs.output_image = \ + write_composite_invlinear_xfm.inputs.output_image = ( f"from-{sym}{tmpl}template_to-{orig}_mode-image_desc-linear_xfm.nii.gz" + ) - wf.connect(inputNode, 'reference_brain', - write_composite_invlinear_xfm, 'input_image') + wf.connect( + inputNode, "reference_brain", write_composite_invlinear_xfm, "input_image" + ) - wf.connect(inputNode, 'input_brain', - write_composite_invlinear_xfm, 'reference_image') + wf.connect( + inputNode, "input_brain", write_composite_invlinear_xfm, "reference_image" + ) - wf.connect(inputNode, 'interpolation', - write_composite_invlinear_xfm, 'interpolation') + wf.connect( + inputNode, "interpolation", write_composite_invlinear_xfm, "interpolation" + ) write_composite_invlinear_xfm.inputs.input_image_type = 0 write_composite_invlinear_xfm.inputs.dimension = 3 - collect_inv_transforms = pe.Node(util.Merge(3), - name='collect_inv_transforms' - f'{symm}') + collect_inv_transforms = pe.Node( + util.Merge(3), name="collect_inv_transforms" f"{symm}" + ) - wf.connect(ants_reg_anat_mni, 'outputspec.ants_initial_xfm', - collect_inv_transforms, 'in1') + wf.connect( + ants_reg_anat_mni, "outputspec.ants_initial_xfm", collect_inv_transforms, "in1" + ) - wf.connect(ants_reg_anat_mni, 'outputspec.ants_rigid_xfm', - collect_inv_transforms, 'in2') + wf.connect( + ants_reg_anat_mni, "outputspec.ants_rigid_xfm", collect_inv_transforms, "in2" + ) - wf.connect(ants_reg_anat_mni, 'outputspec.ants_affine_xfm', - collect_inv_transforms, 'in3') + wf.connect( + ants_reg_anat_mni, "outputspec.ants_affine_xfm", collect_inv_transforms, "in3" + ) # check transform list to exclude Nonetype (missing) init/rig/affine check_invlinear_transform = pe.Node( - util.Function(input_names=['transform_list'], - output_names=['checked_transform_list', - 'list_length'], - function=check_transforms), - name=f'check_inv_transforms') + util.Function( + input_names=["transform_list"], + output_names=["checked_transform_list", "list_length"], + function=check_transforms, + ), + name="check_inv_transforms", + ) - wf.connect(collect_inv_transforms, 'out', - check_invlinear_transform, 'transform_list') + wf.connect( + collect_inv_transforms, "out", check_invlinear_transform, "transform_list" + ) - wf.connect(check_invlinear_transform, 'checked_transform_list', - write_composite_invlinear_xfm, 'transforms') + wf.connect( + check_invlinear_transform, + "checked_transform_list", + write_composite_invlinear_xfm, + "transforms", + ) # generate inverse transform flags, which depends on the # number of transforms inverse_transform_flags = pe.Node( - util.Function(input_names=['transform_list'], - output_names=['inverse_transform_flags'], - function=generate_inverse_transform_flags), - name=f'inverse_transform_flags') + util.Function( + input_names=["transform_list"], + output_names=["inverse_transform_flags"], + function=generate_inverse_transform_flags, + ), + name="inverse_transform_flags", + ) - wf.connect(check_invlinear_transform, 'checked_transform_list', - inverse_transform_flags, 'transform_list') + wf.connect( + check_invlinear_transform, + "checked_transform_list", + inverse_transform_flags, + "transform_list", + ) - wf.connect(inverse_transform_flags, 'inverse_transform_flags', - write_composite_invlinear_xfm, 'invert_transform_flags') + wf.connect( + inverse_transform_flags, + "inverse_transform_flags", + write_composite_invlinear_xfm, + "invert_transform_flags", + ) # combine ALL xfm's into one - makes it easier downstream write_composite_xfm = pe.Node( - interface=ants.ApplyTransforms(), - name=f'write_composite_{symm}xfm', - mem_gb=1.5) + interface=ants.ApplyTransforms(), name=f"write_composite_{symm}xfm", mem_gb=1.5 + ) write_composite_xfm.inputs.print_out_composite_warp_file = True - write_composite_xfm.inputs.output_image = \ + write_composite_xfm.inputs.output_image = ( f"from-{orig}_to-{sym}{tmpl}template_mode-image_xfm.nii.gz" + ) - wf.connect(inputNode, 'input_brain', write_composite_xfm, 'input_image') + wf.connect(inputNode, "input_brain", write_composite_xfm, "input_image") - wf.connect(inputNode, 'reference_brain', - write_composite_xfm, 'reference_image') + wf.connect(inputNode, "reference_brain", write_composite_xfm, "reference_image") - wf.connect(inputNode, 'interpolation', - write_composite_xfm, 'interpolation') + wf.connect(inputNode, "interpolation", write_composite_xfm, "interpolation") write_composite_xfm.inputs.input_image_type = 0 write_composite_xfm.inputs.dimension = 3 - collect_all_transforms = pe.Node(util.Merge(4), - name=f'collect_all_transforms' - f'{symm}') + collect_all_transforms = pe.Node( + util.Merge(4), name=f"collect_all_transforms" f"{symm}" + ) - wf.connect(ants_reg_anat_mni, 'outputspec.warp_field', - collect_all_transforms, 'in1') + wf.connect( + ants_reg_anat_mni, "outputspec.warp_field", collect_all_transforms, "in1" + ) - wf.connect(ants_reg_anat_mni, 'outputspec.ants_affine_xfm', - collect_all_transforms, 'in2') + wf.connect( + ants_reg_anat_mni, "outputspec.ants_affine_xfm", collect_all_transforms, "in2" + ) - wf.connect(ants_reg_anat_mni, 'outputspec.ants_rigid_xfm', - collect_all_transforms, 'in3') + wf.connect( + ants_reg_anat_mni, "outputspec.ants_rigid_xfm", collect_all_transforms, "in3" + ) - wf.connect(ants_reg_anat_mni, 'outputspec.ants_initial_xfm', - collect_all_transforms, 'in4') + wf.connect( + ants_reg_anat_mni, "outputspec.ants_initial_xfm", collect_all_transforms, "in4" + ) # check transform list to exclude Nonetype (missing) init/rig/affine check_all_transform = pe.Node( - util.Function(input_names=['transform_list'], - output_names=['checked_transform_list', - 'list_length'], - function=check_transforms), - name=f'check_all_transforms') + util.Function( + input_names=["transform_list"], + output_names=["checked_transform_list", "list_length"], + function=check_transforms, + ), + name="check_all_transforms", + ) - wf.connect(collect_all_transforms, 'out', - check_all_transform, 'transform_list') + wf.connect(collect_all_transforms, "out", check_all_transform, "transform_list") - wf.connect(check_all_transform, 'checked_transform_list', - write_composite_xfm, 'transforms') + wf.connect( + check_all_transform, "checked_transform_list", write_composite_xfm, "transforms" + ) # combine ALL xfm's into one - makes it easier downstream write_composite_inv_xfm = pe.Node( interface=ants.ApplyTransforms(), - name=f'write_composite_inv_{symm}xfm', + name=f"write_composite_inv_{symm}xfm", mem_gb=0.3, - mem_x=(6278549929741219 / 604462909807314587353088, 'input_image')) + mem_x=(6278549929741219 / 604462909807314587353088, "input_image"), + ) write_composite_inv_xfm.inputs.print_out_composite_warp_file = True - write_composite_inv_xfm.inputs.output_image = \ + write_composite_inv_xfm.inputs.output_image = ( f"from-{sym}{tmpl}template_to-{orig}_mode-image_xfm.nii.gz" + ) - wf.connect(inputNode, 'reference_brain', - write_composite_inv_xfm, 'input_image') + wf.connect(inputNode, "reference_brain", write_composite_inv_xfm, "input_image") - wf.connect(inputNode, 'input_brain', - write_composite_inv_xfm, 'reference_image') + wf.connect(inputNode, "input_brain", write_composite_inv_xfm, "reference_image") - wf.connect(inputNode, 'interpolation', - write_composite_inv_xfm, 'interpolation') + wf.connect(inputNode, "interpolation", write_composite_inv_xfm, "interpolation") write_composite_inv_xfm.inputs.input_image_type = 0 write_composite_inv_xfm.inputs.dimension = 3 - collect_all_inv_transforms = pe.Node(util.Merge(4), - name=f'collect_all_inv_transforms' - f'{symm}') + collect_all_inv_transforms = pe.Node( + util.Merge(4), name=f"collect_all_inv_transforms" f"{symm}" + ) - wf.connect(ants_reg_anat_mni, 'outputspec.ants_initial_xfm', - collect_all_inv_transforms, 'in1') + wf.connect( + ants_reg_anat_mni, + "outputspec.ants_initial_xfm", + collect_all_inv_transforms, + "in1", + ) - wf.connect(ants_reg_anat_mni, 'outputspec.ants_rigid_xfm', - collect_all_inv_transforms, 'in2') + wf.connect( + ants_reg_anat_mni, + "outputspec.ants_rigid_xfm", + collect_all_inv_transforms, + "in2", + ) - wf.connect(ants_reg_anat_mni, 'outputspec.ants_affine_xfm', - collect_all_inv_transforms, 'in3') + wf.connect( + ants_reg_anat_mni, + "outputspec.ants_affine_xfm", + collect_all_inv_transforms, + "in3", + ) - wf.connect(ants_reg_anat_mni, 'outputspec.inverse_warp_field', - collect_all_inv_transforms, 'in4') + wf.connect( + ants_reg_anat_mni, + "outputspec.inverse_warp_field", + collect_all_inv_transforms, + "in4", + ) # check transform list to exclude Nonetype (missing) init/rig/affine check_all_inv_transform = pe.Node( - util.Function(input_names=['transform_list'], - output_names=['checked_transform_list', - 'list_length'], - function=check_transforms), - name=f'check_all_inv_transforms') + util.Function( + input_names=["transform_list"], + output_names=["checked_transform_list", "list_length"], + function=check_transforms, + ), + name="check_all_inv_transforms", + ) - wf.connect(collect_all_inv_transforms, 'out', - check_all_inv_transform, 'transform_list') + wf.connect( + collect_all_inv_transforms, "out", check_all_inv_transform, "transform_list" + ) - wf.connect(check_all_inv_transform, 'checked_transform_list', - write_composite_inv_xfm, 'transforms') + wf.connect( + check_all_inv_transform, + "checked_transform_list", + write_composite_inv_xfm, + "transforms", + ) # generate inverse transform flags, which depends on the # number of transforms inverse_all_transform_flags = pe.Node( - util.Function(input_names=['transform_list'], - output_names=['inverse_transform_flags'], - function=generate_inverse_transform_flags), - name=f'inverse_all_transform_flags') + util.Function( + input_names=["transform_list"], + output_names=["inverse_transform_flags"], + function=generate_inverse_transform_flags, + ), + name="inverse_all_transform_flags", + ) - wf.connect(check_all_inv_transform, 'checked_transform_list', - inverse_all_transform_flags, 'transform_list') + wf.connect( + check_all_inv_transform, + "checked_transform_list", + inverse_all_transform_flags, + "transform_list", + ) - wf.connect(inverse_all_transform_flags, 'inverse_transform_flags', - write_composite_inv_xfm, 'invert_transform_flags') + wf.connect( + inverse_all_transform_flags, + "inverse_transform_flags", + write_composite_inv_xfm, + "invert_transform_flags", + ) outputs = { - f'space-{sym}template_desc-preproc_{orig}': ( - ants_reg_anat_mni, 'outputspec.normalized_output_brain'), - f'from-{orig}_to-{sym}{tmpl}template_mode-image_xfm': ( - write_composite_xfm, 'output_image'), - f'from-{sym}{tmpl}template_to-{orig}_mode-image_xfm': ( - write_composite_inv_xfm, 'output_image'), - f'from-{orig}_to-{sym}{tmpl}template_mode-image_desc-linear_xfm': ( - write_composite_linear_xfm, 'output_image'), - f'from-{sym}{tmpl}template_to-{orig}_mode-image_desc-linear_xfm': ( - write_composite_invlinear_xfm, 'output_image'), - f'from-{orig}_to-{sym}{tmpl}template_mode-image_desc-nonlinear_xfm': ( - ants_reg_anat_mni, 'outputspec.warp_field'), - f'from-{sym}{tmpl}template_to-{orig}_mode-image_desc-nonlinear_xfm': ( - ants_reg_anat_mni, 'outputspec.inverse_warp_field') + f"space-{sym}template_desc-preproc_{orig}": ( + ants_reg_anat_mni, + "outputspec.normalized_output_brain", + ), + f"from-{orig}_to-{sym}{tmpl}template_mode-image_xfm": ( + write_composite_xfm, + "output_image", + ), + f"from-{sym}{tmpl}template_to-{orig}_mode-image_xfm": ( + write_composite_inv_xfm, + "output_image", + ), + f"from-{orig}_to-{sym}{tmpl}template_mode-image_desc-linear_xfm": ( + write_composite_linear_xfm, + "output_image", + ), + f"from-{sym}{tmpl}template_to-{orig}_mode-image_desc-linear_xfm": ( + write_composite_invlinear_xfm, + "output_image", + ), + f"from-{orig}_to-{sym}{tmpl}template_mode-image_desc-nonlinear_xfm": ( + ants_reg_anat_mni, + "outputspec.warp_field", + ), + f"from-{sym}{tmpl}template_to-{orig}_mode-image_desc-nonlinear_xfm": ( + ants_reg_anat_mni, + "outputspec.inverse_warp_field", + ), } return (wf, outputs) -def bold_to_T1template_xfm_connector(wf_name, cfg, reg_tool, symmetric=False, - blip=False): - +def bold_to_T1template_xfm_connector( + wf_name, cfg, reg_tool, symmetric=False, blip=False +): wf = pe.Workflow(name=wf_name) inputNode = pe.Node( - util.IdentityInterface(fields=['input_brain', - 'mean_bold', - 'coreg_xfm', - 'T1w-brain-template_funcreg', - 'T1w_to_template_xfm', - 'template_to_T1w_xfm', - 'blip_warp']), - name='inputspec') - - sym = '' + util.IdentityInterface( + fields=[ + "input_brain", + "mean_bold", + "coreg_xfm", + "T1w-brain-template_funcreg", + "T1w_to_template_xfm", + "template_to_T1w_xfm", + "blip_warp", + ] + ), + name="inputspec", + ) + + sym = "" if symmetric: - sym = 'sym' + sym = "sym" - if reg_tool == 'ants': - fsl_reg_2_itk = pe.Node(c3.C3dAffineTool(), name='fsl_reg_2_itk') + if reg_tool == "ants": + fsl_reg_2_itk = pe.Node(c3.C3dAffineTool(), name="fsl_reg_2_itk") fsl_reg_2_itk.inputs.itk_transform = True fsl_reg_2_itk.inputs.fsl2ras = True # convert the .mat from linear Func->Anat to # ANTS format - wf.connect(inputNode, 'coreg_xfm', fsl_reg_2_itk, 'transform_file') + wf.connect(inputNode, "coreg_xfm", fsl_reg_2_itk, "transform_file") - wf.connect(inputNode, 'input_brain', fsl_reg_2_itk, 'reference_file') + wf.connect(inputNode, "input_brain", fsl_reg_2_itk, "reference_file") - wf.connect(inputNode, 'mean_bold', fsl_reg_2_itk, 'source_file') + wf.connect(inputNode, "mean_bold", fsl_reg_2_itk, "source_file") - itk_imports = ['import os'] - change_transform = pe.Node(util.Function( - input_names=['input_affine_file'], - output_names=['updated_affine_file'], - function=change_itk_transform_type, - imports=itk_imports), - name='change_transform_type') + itk_imports = ["import os"] + change_transform = pe.Node( + util.Function( + input_names=["input_affine_file"], + output_names=["updated_affine_file"], + function=change_itk_transform_type, + imports=itk_imports, + ), + name="change_transform_type", + ) - wf.connect(fsl_reg_2_itk, 'itk_transform', - change_transform, 'input_affine_file') + wf.connect( + fsl_reg_2_itk, "itk_transform", change_transform, "input_affine_file" + ) # combine ALL xfm's into one - makes it easier downstream write_composite_xfm = pe.Node( - interface=ants.ApplyTransforms(), - name=f'write_composite_xfm', - mem_gb=1.5) + interface=ants.ApplyTransforms(), name="write_composite_xfm", mem_gb=1.5 + ) write_composite_xfm.inputs.print_out_composite_warp_file = True - write_composite_xfm.inputs.output_image = \ + write_composite_xfm.inputs.output_image = ( f"from-bold_to-{sym}template_mode-image_xfm.nii.gz" + ) - wf.connect(inputNode, 'mean_bold', - write_composite_xfm, 'input_image') + wf.connect(inputNode, "mean_bold", write_composite_xfm, "input_image") - wf.connect(inputNode, 'T1w-brain-template_funcreg', - write_composite_xfm, 'reference_image') + wf.connect( + inputNode, + "T1w-brain-template_funcreg", + write_composite_xfm, + "reference_image", + ) write_composite_xfm.inputs.input_image_type = 0 write_composite_xfm.inputs.dimension = 3 - write_composite_xfm.inputs.interpolation = \ - cfg.registration_workflows['anatomical_registration'][ - 'registration']['ANTs']['interpolation'] + write_composite_xfm.inputs.interpolation = cfg.registration_workflows[ + "anatomical_registration" + ]["registration"]["ANTs"]["interpolation"] if not blip: - collect_all_transforms = pe.Node(util.Merge(2), - name='collect_all_transforms') + collect_all_transforms = pe.Node( + util.Merge(2), name="collect_all_transforms" + ) else: - collect_all_transforms = pe.Node(util.Merge(3), - name='collect_all_transforms') + collect_all_transforms = pe.Node( + util.Merge(3), name="collect_all_transforms" + ) - wf.connect(inputNode, 'blip_warp', - collect_all_transforms, 'in3') + wf.connect(inputNode, "blip_warp", collect_all_transforms, "in3") - wf.connect(inputNode, 'T1w_to_template_xfm', - collect_all_transforms, 'in1') + wf.connect(inputNode, "T1w_to_template_xfm", collect_all_transforms, "in1") - wf.connect(change_transform, 'updated_affine_file', - collect_all_transforms, 'in2') + wf.connect( + change_transform, "updated_affine_file", collect_all_transforms, "in2" + ) - wf.connect(collect_all_transforms, 'out', - write_composite_xfm, 'transforms') + wf.connect(collect_all_transforms, "out", write_composite_xfm, "transforms") write_composite_inv_xfm = pe.Node( - interface=ants.ApplyTransforms(), - name=f'write_composite_inv_xfm', - mem_gb=1.5) + interface=ants.ApplyTransforms(), name="write_composite_inv_xfm", mem_gb=1.5 + ) write_composite_inv_xfm.inputs.print_out_composite_warp_file = True write_composite_inv_xfm.inputs.invert_transform_flags = [True, False] - write_composite_inv_xfm.inputs.output_image = \ + write_composite_inv_xfm.inputs.output_image = ( f"from-{sym}template_to-bold_mode-image_xfm.nii.gz" + ) - wf.connect(inputNode, 'T1w-brain-template_funcreg', - write_composite_inv_xfm, 'input_image') + wf.connect( + inputNode, + "T1w-brain-template_funcreg", + write_composite_inv_xfm, + "input_image", + ) - wf.connect(inputNode, 'mean_bold', - write_composite_inv_xfm, 'reference_image') + wf.connect(inputNode, "mean_bold", write_composite_inv_xfm, "reference_image") write_composite_inv_xfm.inputs.input_image_type = 0 write_composite_inv_xfm.inputs.dimension = 3 - write_composite_inv_xfm.inputs.interpolation = \ - cfg.registration_workflows['anatomical_registration'][ - 'registration']['ANTs']['interpolation'] + write_composite_inv_xfm.inputs.interpolation = cfg.registration_workflows[ + "anatomical_registration" + ]["registration"]["ANTs"]["interpolation"] - collect_inv_transforms = pe.Node(util.Merge(2), - name='collect_inv_transforms') + collect_inv_transforms = pe.Node(util.Merge(2), name="collect_inv_transforms") - wf.connect(change_transform, 'updated_affine_file', - collect_inv_transforms, 'in1') + wf.connect( + change_transform, "updated_affine_file", collect_inv_transforms, "in1" + ) - wf.connect(inputNode, 'template_to_T1w_xfm', - collect_inv_transforms, 'in2') + wf.connect(inputNode, "template_to_T1w_xfm", collect_inv_transforms, "in2") - wf.connect(collect_inv_transforms, 'out', - write_composite_inv_xfm, 'transforms') + wf.connect(collect_inv_transforms, "out", write_composite_inv_xfm, "transforms") outputs = { - f'from-bold_to-{sym}template_mode-image_xfm': - (write_composite_xfm, 'output_image'), - f'from-{sym}template_to-bold_mode-image_xfm': - (write_composite_inv_xfm, 'output_image') + f"from-bold_to-{sym}template_mode-image_xfm": ( + write_composite_xfm, + "output_image", + ), + f"from-{sym}template_to-bold_mode-image_xfm": ( + write_composite_inv_xfm, + "output_image", + ), } - elif reg_tool == 'fsl': - - write_composite_xfm = pe.Node(interface=fsl.ConvertWarp(), - name='combine_fsl_warps') + elif reg_tool == "fsl": + write_composite_xfm = pe.Node( + interface=fsl.ConvertWarp(), name="combine_fsl_warps" + ) - wf.connect(inputNode, 'T1w-brain-template_funcreg', - write_composite_xfm, 'reference') + wf.connect( + inputNode, "T1w-brain-template_funcreg", write_composite_xfm, "reference" + ) if blip: - wf.connect(inputNode, 'coreg_xfm', - write_composite_xfm, 'postmat') - wf.connect(inputNode, 'blip_warp', - write_composite_xfm, 'warp1') - wf.connect(inputNode, 'T1w_to_template_xfm', - write_composite_xfm, 'warp2') + wf.connect(inputNode, "coreg_xfm", write_composite_xfm, "postmat") + wf.connect(inputNode, "blip_warp", write_composite_xfm, "warp1") + wf.connect(inputNode, "T1w_to_template_xfm", write_composite_xfm, "warp2") else: - wf.connect(inputNode, 'coreg_xfm', - write_composite_xfm, 'premat') - wf.connect(inputNode, 'T1w_to_template_xfm', - write_composite_xfm, 'warp1') + wf.connect(inputNode, "coreg_xfm", write_composite_xfm, "premat") + wf.connect(inputNode, "T1w_to_template_xfm", write_composite_xfm, "warp1") outputs = { - f'from-bold_to-{sym}template_mode-image_xfm': - (write_composite_xfm, 'out_file'), + f"from-bold_to-{sym}template_mode-image_xfm": ( + write_composite_xfm, + "out_file", + ), } return (wf, outputs) @@ -1992,10 +2272,8 @@ def bold_to_T1template_xfm_connector(wf_name, cfg, reg_tool, symmetric=False, "space-template_desc-head_T1w": {"Template": "T1w-template"}, "space-template_desc-T1w_mask": {"Template": "T1w-template"}, "space-template_desc-T1wT2w_biasfield": {"Template": "T1w-template"}, - "from-T1w_to-template_mode-image_desc-linear_xfm": { - "Template": "T1w-template"}, - "from-template_to-T1w_mode-image_desc-linear_xfm": { - "Template": "T1w-template"}, + "from-T1w_to-template_mode-image_desc-linear_xfm": {"Template": "T1w-template"}, + "from-template_to-T1w_mode-image_desc-linear_xfm": {"Template": "T1w-template"}, "from-T1w_to-template_mode-image_xfm": {"Template": "T1w-template"}, "from-T1w_to-template_mode-image_warp": {"Template": "T1w-template"}, "from-longitudinal_to-template_mode-image_desc-linear_xfm": { @@ -2004,61 +2282,62 @@ def bold_to_T1template_xfm_connector(wf_name, cfg, reg_tool, symmetric=False, "from-template_to-longitudinal_mode-image_desc-linear_xfm": { "Template": "T1w-template" }, - "from-longitudinal_to-template_mode-image_xfm": { - "Template": "T1w-template"}, + "from-longitudinal_to-template_mode-image_xfm": {"Template": "T1w-template"}, }, ) def register_FSL_anat_to_template(wf, cfg, strat_pool, pipe_num, opt=None): - - fsl, outputs = FSL_registration_connector(f'register_{opt}_anat_to_' - f'template_{pipe_num}', cfg, - orig='T1w', opt=opt) + fsl, outputs = FSL_registration_connector( + f"register_{opt}_anat_to_" f"template_{pipe_num}", cfg, orig="T1w", opt=opt + ) fsl.inputs.inputspec.interpolation = cfg.registration_workflows[ - 'anatomical_registration']['registration']['FSL-FNIRT'][ - 'interpolation'] + "anatomical_registration" + ]["registration"]["FSL-FNIRT"]["interpolation"] fsl.inputs.inputspec.fnirt_config = cfg.registration_workflows[ - 'anatomical_registration']['registration']['FSL-FNIRT'][ - 'fnirt_config'] + "anatomical_registration" + ]["registration"]["FSL-FNIRT"]["fnirt_config"] - connect, brain = \ - strat_pool.get_data(['desc-brain_T1w', - 'space-longitudinal_desc-brain_T1w'], - report_fetched=True) + connect, brain = strat_pool.get_data( + ["desc-brain_T1w", "space-longitudinal_desc-brain_T1w"], report_fetched=True + ) node, out = connect - wf.connect(node, out, fsl, 'inputspec.input_brain') - - if cfg.registration_workflows['anatomical_registration']['registration']['FSL-FNIRT']['ref_resolution'] == \ - cfg.registration_workflows['anatomical_registration']['resolution_for_anat']: - - node, out = strat_pool.get_data('T1w-brain-template') - wf.connect(node, out, fsl, 'inputspec.reference_brain') - - node, out = strat_pool.get_data('T1w-template') - wf.connect(node, out, fsl, 'inputspec.reference_head') + wf.connect(node, out, fsl, "inputspec.input_brain") + + if ( + cfg.registration_workflows["anatomical_registration"]["registration"][ + "FSL-FNIRT" + ]["ref_resolution"] + == cfg.registration_workflows["anatomical_registration"]["resolution_for_anat"] + ): + node, out = strat_pool.get_data("T1w-brain-template") + wf.connect(node, out, fsl, "inputspec.reference_brain") + + node, out = strat_pool.get_data("T1w-template") + wf.connect(node, out, fsl, "inputspec.reference_head") else: - node, out = strat_pool.get_data('FNIRT-T1w-brain-template') - wf.connect(node, out, fsl, 'inputspec.reference_brain') + node, out = strat_pool.get_data("FNIRT-T1w-brain-template") + wf.connect(node, out, fsl, "inputspec.reference_brain") - node, out = strat_pool.get_data('FNIRT-T1w-template') - wf.connect(node, out, fsl, 'inputspec.reference_head') + node, out = strat_pool.get_data("FNIRT-T1w-template") + wf.connect(node, out, fsl, "inputspec.reference_head") - node, out = strat_pool.get_data(["desc-preproc_T1w", - "space-longitudinal_desc-reorient_T1w"]) - wf.connect(node, out, fsl, 'inputspec.input_head') + node, out = strat_pool.get_data( + ["desc-preproc_T1w", "space-longitudinal_desc-reorient_T1w"] + ) + wf.connect(node, out, fsl, "inputspec.input_head") - node, out = strat_pool.get_data('template-ref-mask') - wf.connect(node, out, fsl, 'inputspec.reference_mask') + node, out = strat_pool.get_data("template-ref-mask") + wf.connect(node, out, fsl, "inputspec.reference_mask") - if 'space-longitudinal' in brain: + if "space-longitudinal" in brain: for key in outputs.keys(): - if 'from-T1w' in key: - new_key = key.replace('from-T1w', 'from-longitudinal') + if "from-T1w" in key: + new_key = key.replace("from-T1w", "from-longitudinal") outputs[new_key] = outputs[key] del outputs[key] - if 'to-T1w' in key: - new_key = key.replace('to-T1w', 'to-longitudinal') + if "to-T1w" in key: + new_key = key.replace("to-T1w", "to-longitudinal") outputs[new_key] = outputs[key] del outputs[key] @@ -2104,50 +2383,51 @@ def register_FSL_anat_to_template(wf, cfg, strat_pool, pipe_num, opt=None): }, }, ) -def register_symmetric_FSL_anat_to_template(wf, cfg, strat_pool, pipe_num, - opt=None): - - fsl, outputs = FSL_registration_connector(f'register_{opt}_anat_to_' - f'template_symmetric_' - f'{pipe_num}', cfg, orig='T1w', - opt=opt, symmetric=True) +def register_symmetric_FSL_anat_to_template(wf, cfg, strat_pool, pipe_num, opt=None): + fsl, outputs = FSL_registration_connector( + f"register_{opt}_anat_to_" f"template_symmetric_" f"{pipe_num}", + cfg, + orig="T1w", + opt=opt, + symmetric=True, + ) fsl.inputs.inputspec.interpolation = cfg.registration_workflows[ - 'anatomical_registration']['registration']['FSL-FNIRT'][ - 'interpolation'] + "anatomical_registration" + ]["registration"]["FSL-FNIRT"]["interpolation"] fsl.inputs.inputspec.fnirt_config = cfg.registration_workflows[ - 'anatomical_registration']['registration']['FSL-FNIRT'][ - 'fnirt_config'] + "anatomical_registration" + ]["registration"]["FSL-FNIRT"]["fnirt_config"] - connect, brain = \ - strat_pool.get_data(['desc-brain_T1w', - 'space-longitudinal_desc-brain_T1w'], - report_fetched=True) + connect, brain = strat_pool.get_data( + ["desc-brain_T1w", "space-longitudinal_desc-brain_T1w"], report_fetched=True + ) node, out = connect - wf.connect(node, out, fsl, 'inputspec.input_brain') + wf.connect(node, out, fsl, "inputspec.input_brain") - node, out = strat_pool.get_data('T1w-brain-template-symmetric') - wf.connect(node, out, fsl, 'inputspec.reference_brain') + node, out = strat_pool.get_data("T1w-brain-template-symmetric") + wf.connect(node, out, fsl, "inputspec.reference_brain") - node, out = strat_pool.get_data(["desc-preproc_T1w", - "space-longitudinal_desc-reorient_T1w"]) - wf.connect(node, out, fsl, 'inputspec.input_head') + node, out = strat_pool.get_data( + ["desc-preproc_T1w", "space-longitudinal_desc-reorient_T1w"] + ) + wf.connect(node, out, fsl, "inputspec.input_head") - node, out = strat_pool.get_data('T1w-template-symmetric') - wf.connect(node, out, fsl, 'inputspec.reference_head') + node, out = strat_pool.get_data("T1w-template-symmetric") + wf.connect(node, out, fsl, "inputspec.reference_head") - node, out = strat_pool.get_data('dilated-symmetric-brain-mask') - wf.connect(node, out, fsl, 'inputspec.reference_mask') + node, out = strat_pool.get_data("dilated-symmetric-brain-mask") + wf.connect(node, out, fsl, "inputspec.reference_mask") - if 'space-longitudinal' in brain: + if "space-longitudinal" in brain: for key in outputs.keys(): - if 'from-T1w' in key: - new_key = key.replace('from-T1w', 'from-longitudinal') + if "from-T1w" in key: + new_key = key.replace("from-T1w", "from-longitudinal") outputs[new_key] = outputs[key] del outputs[key] - if 'to-T1w' in key: - new_key = key.replace('to-T1w', 'to-longitudinal') + if "to-T1w" in key: + new_key = key.replace("to-T1w", "to-longitudinal") outputs[new_key] = outputs[key] del outputs[key] @@ -2156,8 +2436,7 @@ def register_symmetric_FSL_anat_to_template(wf, cfg, strat_pool, pipe_num, @nodeblock( name="register_FSL_EPI_to_template", - config=["registration_workflows", "functional_registration", - "EPI_registration"], + config=["registration_workflows", "functional_registration", "EPI_registration"], switch=["run"], option_key="using", option_val=["FSL", "FSL-linear"], @@ -2174,42 +2453,43 @@ def register_symmetric_FSL_anat_to_template(wf, cfg, strat_pool, pipe_num, "from-EPItemplate_to-bold_mode-image_desc-linear_xfm": { "Template": "EPI-template" }, - "from-bold_to-EPItemplate_mode-image_xfm": { - "Template": "EPI-template"}, + "from-bold_to-EPItemplate_mode-image_xfm": {"Template": "EPI-template"}, }, ) def register_FSL_EPI_to_template(wf, cfg, strat_pool, pipe_num, opt=None): - '''Directly register the mean functional to an EPI template. No T1w + """Directly register the mean functional to an EPI template. No T1w involved. - ''' - - fsl, outputs = FSL_registration_connector(f'register_{opt}_EPI_to_' - f'template_{pipe_num}', cfg, - orig='bold', opt=opt, - template='EPI') + """ + fsl, outputs = FSL_registration_connector( + f"register_{opt}_EPI_to_" f"template_{pipe_num}", + cfg, + orig="bold", + opt=opt, + template="EPI", + ) - fsl.inputs.inputspec.interpolation = cfg['registration_workflows'][ - 'functional_registration']['EPI_registration']['FSL-FNIRT'][ - 'interpolation'] + fsl.inputs.inputspec.interpolation = cfg["registration_workflows"][ + "functional_registration" + ]["EPI_registration"]["FSL-FNIRT"]["interpolation"] - fsl.inputs.inputspec.fnirt_config = cfg['registration_workflows'][ - 'functional_registration']['EPI_registration']['FSL-FNIRT'][ - 'fnirt_config'] + fsl.inputs.inputspec.fnirt_config = cfg["registration_workflows"][ + "functional_registration" + ]["EPI_registration"]["FSL-FNIRT"]["fnirt_config"] - node, out = strat_pool.get_data('sbref') - wf.connect(node, out, fsl, 'inputspec.input_brain') + node, out = strat_pool.get_data("sbref") + wf.connect(node, out, fsl, "inputspec.input_brain") - node, out = strat_pool.get_data('EPI-template') - wf.connect(node, out, fsl, 'inputspec.reference_brain') + node, out = strat_pool.get_data("EPI-template") + wf.connect(node, out, fsl, "inputspec.reference_brain") - node, out = strat_pool.get_data('sbref') - wf.connect(node, out, fsl, 'inputspec.input_head') + node, out = strat_pool.get_data("sbref") + wf.connect(node, out, fsl, "inputspec.input_head") - node, out = strat_pool.get_data('EPI-template') - wf.connect(node, out, fsl, 'inputspec.reference_head') + node, out = strat_pool.get_data("EPI-template") + wf.connect(node, out, fsl, "inputspec.reference_head") - node, out = strat_pool.get_data('EPI-template-mask') - wf.connect(node, out, fsl, 'inputspec.reference_mask') + node, out = strat_pool.get_data("EPI-template-mask") + wf.connect(node, out, fsl, "inputspec.reference_mask") return (wf, outputs) @@ -2245,130 +2525,141 @@ def register_FSL_EPI_to_template(wf, cfg, strat_pool, pipe_num, opt=None): outputs={ "space-template_desc-preproc_T1w": { "Description": "The preprocessed T1w brain transformed to " - "template space.", + "template space.", "Template": "T1w-template", }, "from-T1w_to-template_mode-image_desc-linear_xfm": { "Description": "Linear (affine) transform from T1w native space " - "to T1w-template space.", + "to T1w-template space.", "Template": "T1w-template", }, "from-template_to-T1w_mode-image_desc-linear_xfm": { "Description": "Linear (affine) transform from T1w-template space " - "to T1w native space.", + "to T1w native space.", "Template": "T1w-template", }, "from-T1w_to-template_mode-image_desc-nonlinear_xfm": { "Description": "Nonlinear (warp field) transform from T1w native " - "space to T1w-template space.", + "space to T1w-template space.", "Template": "T1w-template", }, "from-template_to-T1w_mode-image_desc-nonlinear_xfm": { "Description": "Nonlinear (warp field) transform from " - "T1w-template space to T1w native space.", + "T1w-template space to T1w native space.", "Template": "T1w-template", }, "from-T1w_to-template_mode-image_xfm": { "Description": "Composite (affine + warp field) transform from " - "T1w native space to T1w-template space.", + "T1w native space to T1w-template space.", "Template": "T1w-template", }, "from-template_to-T1w_mode-image_xfm": { "Description": "Composite (affine + warp field) transform from " - "T1w-template space to T1w native space.", + "T1w-template space to T1w native space.", "Template": "T1w-template", }, "from-longitudinal_to-template_mode-image_desc-linear_xfm": { "Description": "Linear (affine) transform from " - "longitudinal-template space to T1w-template " - "space.", + "longitudinal-template space to T1w-template " + "space.", "Template": "T1w-template", }, "from-template_to-longitudinal_mode-image_desc-linear_xfm": { "Description": "Linear (affine) transform from T1w-template " - "space to longitudinal-template space.", + "space to longitudinal-template space.", "Template": "T1w-template", }, "from-longitudinal_to-template_mode-image_desc-nonlinear_xfm": { "Description": "Nonlinear (warp field) transform from " - "longitudinal-template space to T1w-template " - "space.", + "longitudinal-template space to T1w-template " + "space.", "Template": "T1w-template", }, "from-template_to-longitudinal_mode-image_desc-nonlinear_xfm": { "Description": "Nonlinear (warp field) transform from " - "T1w-template space to longitudinal-template " - "space.", + "T1w-template space to longitudinal-template " + "space.", "Template": "T1w-template", }, "from-longitudinal_to-template_mode-image_xfm": { "Description": "Composite (affine + warp field) transform from " - "longitudinal-template space to T1w-template " - "space.", + "longitudinal-template space to T1w-template " + "space.", "Template": "T1w-template", }, "from-template_to-longitudinal_mode-image_xfm": { "Description": "Composite (affine + warp field) transform from " - "T1w-template space to longitudinal-template " - "space.", + "T1w-template space to longitudinal-template " + "space.", "Template": "T1w-template", }, }, ) def register_ANTs_anat_to_template(wf, cfg, strat_pool, pipe_num, opt=None): + params = cfg.registration_workflows["anatomical_registration"]["registration"][ + "ANTs" + ]["T1_registration"] - params = cfg.registration_workflows['anatomical_registration'][ - 'registration']['ANTs']['T1_registration'] - - ants_rc, outputs = ANTs_registration_connector('ANTS_T1_to_template_' - f'{pipe_num}', cfg, - params, orig='T1w') + ants_rc, outputs = ANTs_registration_connector( + "ANTS_T1_to_template_" f"{pipe_num}", cfg, params, orig="T1w" + ) ants_rc.inputs.inputspec.interpolation = cfg.registration_workflows[ - 'anatomical_registration']['registration']['ANTs']['interpolation'] + "anatomical_registration" + ]["registration"]["ANTs"]["interpolation"] - connect, brain = \ - strat_pool.get_data(['desc-preproc_T1w', - 'space-longitudinal_desc-brain_T1w'], - report_fetched=True) + connect, brain = strat_pool.get_data( + ["desc-preproc_T1w", "space-longitudinal_desc-brain_T1w"], report_fetched=True + ) node, out = connect - wf.connect(node, out, ants_rc, 'inputspec.input_brain') - - t1w_brain_template = strat_pool.node_data('T1w-brain-template') - wf.connect(t1w_brain_template.node, t1w_brain_template.out, - ants_rc, 'inputspec.reference_brain') + wf.connect(node, out, ants_rc, "inputspec.input_brain") + + t1w_brain_template = strat_pool.node_data("T1w-brain-template") + wf.connect( + t1w_brain_template.node, + t1w_brain_template.out, + ants_rc, + "inputspec.reference_brain", + ) # TODO check the order of T1w - node, out = strat_pool.get_data(["desc-restore_T1w", "desc-head_T1w", - "desc-preproc_T1w", - "space-longitudinal_desc-reorient_T1w"]) - wf.connect(node, out, ants_rc, 'inputspec.input_head') - + node, out = strat_pool.get_data( + [ + "desc-restore_T1w", + "desc-head_T1w", + "desc-preproc_T1w", + "space-longitudinal_desc-reorient_T1w", + ] + ) + wf.connect(node, out, ants_rc, "inputspec.input_head") - t1w_template = strat_pool.node_data('T1w-template') - wf.connect(t1w_template.node, t1w_template.out, - ants_rc, 'inputspec.reference_head') + t1w_template = strat_pool.node_data("T1w-template") + wf.connect(t1w_template.node, t1w_template.out, ants_rc, "inputspec.reference_head") - brain_mask = strat_pool.node_data(["space-T1w_desc-brain_mask", - "space-longitudinal_desc-brain_mask", - "space-T1w_desc-acpcbrain_mask"]) - wf.connect(brain_mask.node, brain_mask.out, - ants_rc, 'inputspec.input_mask') + brain_mask = strat_pool.node_data( + [ + "space-T1w_desc-brain_mask", + "space-longitudinal_desc-brain_mask", + "space-T1w_desc-acpcbrain_mask", + ] + ) + wf.connect(brain_mask.node, brain_mask.out, ants_rc, "inputspec.input_mask") - if strat_pool.check_rpool('T1w-brain-template-mask'): - node, out = strat_pool.get_data('T1w-brain-template-mask') - wf.connect(node, out, ants_rc, 'inputspec.reference_mask') + if strat_pool.check_rpool("T1w-brain-template-mask"): + node, out = strat_pool.get_data("T1w-brain-template-mask") + wf.connect(node, out, ants_rc, "inputspec.reference_mask") - if strat_pool.check_rpool('label-lesion_mask'): - node, out = strat_pool.get_data('label-lesion_mask') - wf.connect(node, out, ants_rc, 'inputspec.lesion_mask') + if strat_pool.check_rpool("label-lesion_mask"): + node, out = strat_pool.get_data("label-lesion_mask") + wf.connect(node, out, ants_rc, "inputspec.lesion_mask") - if 'space-longitudinal' in brain: + if "space-longitudinal" in brain: for key in outputs: - for direction in ['from', 'to']: - if f'{direction}-T1w' in key: - new_key = key.replace(f'{direction}-T1w', - f'{direction}-longitudinal') + for direction in ["from", "to"]: + if f"{direction}-T1w" in key: + new_key = key.replace( + f"{direction}-T1w", f"{direction}-longitudinal" + ) outputs[new_key] = outputs[key] del outputs[key] @@ -2384,8 +2675,7 @@ def register_ANTs_anat_to_template(wf, cfg, strat_pool, pipe_num, opt=None): inputs=[ ( ["desc-preproc_T1w", "space-longitudinal_desc-brain_T1w"], - ["space-T1w_desc-brain_mask", - "space-longitudinal_desc-brain_mask"], + ["space-T1w_desc-brain_mask", "space-longitudinal_desc-brain_mask"], [ "desc-head_T1w", "desc-preproc_T1w", @@ -2439,56 +2729,60 @@ def register_ANTs_anat_to_template(wf, cfg, strat_pool, pipe_num, opt=None): }, }, ) -def register_symmetric_ANTs_anat_to_template(wf, cfg, strat_pool, pipe_num, - opt=None): - - params = cfg.registration_workflows['anatomical_registration'][ - 'registration']['ANTs']['T1_registration'] - - ants, outputs = ANTs_registration_connector('ANTS_T1_to_template_' - f'symmetric_{pipe_num}', cfg, - params, orig='T1w', - symmetric=True) +def register_symmetric_ANTs_anat_to_template(wf, cfg, strat_pool, pipe_num, opt=None): + params = cfg.registration_workflows["anatomical_registration"]["registration"][ + "ANTs" + ]["T1_registration"] + + ants, outputs = ANTs_registration_connector( + "ANTS_T1_to_template_" f"symmetric_{pipe_num}", + cfg, + params, + orig="T1w", + symmetric=True, + ) ants.inputs.inputspec.interpolation = cfg.registration_workflows[ - 'anatomical_registration']['registration']['ANTs']['interpolation'] + "anatomical_registration" + ]["registration"]["ANTs"]["interpolation"] - connect, brain = \ - strat_pool.get_data(['desc-preproc_T1w', - 'space-longitudinal_desc-brain_T1w'], - report_fetched=True) + connect, brain = strat_pool.get_data( + ["desc-preproc_T1w", "space-longitudinal_desc-brain_T1w"], report_fetched=True + ) node, out = connect - wf.connect(node, out, ants, 'inputspec.input_brain') + wf.connect(node, out, ants, "inputspec.input_brain") - node, out = strat_pool.get_data('T1w-brain-template-symmetric') - wf.connect(node, out, ants, 'inputspec.reference_brain') + node, out = strat_pool.get_data("T1w-brain-template-symmetric") + wf.connect(node, out, ants, "inputspec.reference_brain") - node, out = strat_pool.get_data(["desc-head_T1w", "desc-preproc_T1w", - "space-longitudinal_desc-reorient_T1w"]) - wf.connect(node, out, ants, 'inputspec.input_head') + node, out = strat_pool.get_data( + ["desc-head_T1w", "desc-preproc_T1w", "space-longitudinal_desc-reorient_T1w"] + ) + wf.connect(node, out, ants, "inputspec.input_head") - node, out = strat_pool.get_data('T1w-template-symmetric') - wf.connect(node, out, ants, 'inputspec.reference_head') + node, out = strat_pool.get_data("T1w-template-symmetric") + wf.connect(node, out, ants, "inputspec.reference_head") - node, out = strat_pool.get_data(["space-T1w_desc-brain_mask", - "space-longitudinal_desc-brain_mask"]) - wf.connect(node, out, ants, 'inputspec.input_mask') + node, out = strat_pool.get_data( + ["space-T1w_desc-brain_mask", "space-longitudinal_desc-brain_mask"] + ) + wf.connect(node, out, ants, "inputspec.input_mask") - node, out = strat_pool.get_data('dilated-symmetric-brain-mask') - wf.connect(node, out, ants, 'inputspec.reference_mask') + node, out = strat_pool.get_data("dilated-symmetric-brain-mask") + wf.connect(node, out, ants, "inputspec.reference_mask") - if strat_pool.check_rpool('label-lesion_mask'): - node, out = strat_pool.get_data('label-lesion_mask') - wf.connect(node, out, ants, 'inputspec.lesion_mask') + if strat_pool.check_rpool("label-lesion_mask"): + node, out = strat_pool.get_data("label-lesion_mask") + wf.connect(node, out, ants, "inputspec.lesion_mask") - if 'space-longitudinal' in brain: + if "space-longitudinal" in brain: for key in outputs.keys(): - if 'from-T1w' in key: - new_key = key.replace('from-T1w', 'from-longitudinal') + if "from-T1w" in key: + new_key = key.replace("from-T1w", "from-longitudinal") outputs[new_key] = outputs[key] del outputs[key] - if 'to-T1w' in key: - new_key = key.replace('to-T1w', 'to-longitudinal') + if "to-T1w" in key: + new_key = key.replace("to-T1w", "to-longitudinal") outputs[new_key] = outputs[key] del outputs[key] @@ -2520,45 +2814,48 @@ def register_symmetric_ANTs_anat_to_template(wf, cfg, strat_pool, pipe_num, "from-EPItemplate_to-bold_mode-image_desc-nonlinear_xfm": { "Template": "EPI-template" }, - "from-bold_to-EPItemplate_mode-image_xfm": { - "Template": "EPI-template"}, - "from-EPItemplate_to-bold_mode-image_xfm": { - "Template": "EPI-template"}, + "from-bold_to-EPItemplate_mode-image_xfm": {"Template": "EPI-template"}, + "from-EPItemplate_to-bold_mode-image_xfm": {"Template": "EPI-template"}, }, ) def register_ANTs_EPI_to_template(wf, cfg, strat_pool, pipe_num, opt=None): - '''Directly register the mean functional to an EPI template. No T1w + """Directly register the mean functional to an EPI template. No T1w involved. - ''' - params = cfg.registration_workflows['functional_registration'][ - 'EPI_registration']['ANTs']['parameters'] - - ants, outputs = ANTs_registration_connector('ANTS_bold_to_EPI-template' - f'_{pipe_num}', cfg, params, - orig='bold', template='EPI') + """ + params = cfg.registration_workflows["functional_registration"]["EPI_registration"][ + "ANTs" + ]["parameters"] + + ants, outputs = ANTs_registration_connector( + "ANTS_bold_to_EPI-template" f"_{pipe_num}", + cfg, + params, + orig="bold", + template="EPI", + ) ants.inputs.inputspec.interpolation = cfg.registration_workflows[ - 'functional_registration']['EPI_registration']['ANTs'][ - 'interpolation'] + "functional_registration" + ]["EPI_registration"]["ANTs"]["interpolation"] - node, out = strat_pool.get_data('sbref') - wf.connect(node, out, ants, 'inputspec.input_brain') + node, out = strat_pool.get_data("sbref") + wf.connect(node, out, ants, "inputspec.input_brain") - node, out = strat_pool.get_data('EPI-template') - wf.connect(node, out, ants, 'inputspec.reference_brain') + node, out = strat_pool.get_data("EPI-template") + wf.connect(node, out, ants, "inputspec.reference_brain") - node, out = strat_pool.get_data('sbref') - wf.connect(node, out, ants, 'inputspec.input_head') + node, out = strat_pool.get_data("sbref") + wf.connect(node, out, ants, "inputspec.input_head") - node, out = strat_pool.get_data('EPI-template') - wf.connect(node, out, ants, 'inputspec.reference_head') + node, out = strat_pool.get_data("EPI-template") + wf.connect(node, out, ants, "inputspec.reference_head") - node, out = strat_pool.get_data('space-bold_desc-brain_mask') - wf.connect(node, out, ants, 'inputspec.input_mask') + node, out = strat_pool.get_data("space-bold_desc-brain_mask") + wf.connect(node, out, ants, "inputspec.input_mask") - if strat_pool.check_rpool('EPI-template-mask'): - node, out = strat_pool.get_data('EPI-template-mask') - wf.connect(node, out, ants, 'inputspec.reference_mask') + if strat_pool.check_rpool("EPI-template-mask"): + node, out = strat_pool.get_data("EPI-template-mask") + wf.connect(node, out, ants, "inputspec.reference_mask") return (wf, outputs) @@ -2585,8 +2882,7 @@ def register_ANTs_EPI_to_template(wf, cfg, strat_pool, pipe_num, opt=None): ( "desc-restore-brain_T1w", ["desc-preproc_T1w", "space-longitudinal_desc-brain_T1w"], - ["desc-restore_T1w", "desc-preproc_T1w", "desc-reorient_T1w", - "T1w"], + ["desc-restore_T1w", "desc-preproc_T1w", "desc-reorient_T1w", "T1w"], ["desc-preproc_T1w", "desc-reorient_T1w", "T1w"], "space-T1w_desc-brain_mask", "T1w-template", @@ -2604,16 +2900,12 @@ def register_ANTs_EPI_to_template(wf, cfg, strat_pool, pipe_num, opt=None): "from-template_to-T1w_mode-image_xfm": {"Template": "T1w-template"}, }, ) -def overwrite_transform_anat_to_template(wf, cfg, strat_pool, pipe_num, - opt=None): - - xfm_prov = strat_pool.get_cpac_provenance( - 'from-T1w_to-template_mode-image_xfm') +def overwrite_transform_anat_to_template(wf, cfg, strat_pool, pipe_num, opt=None): + xfm_prov = strat_pool.get_cpac_provenance("from-T1w_to-template_mode-image_xfm") reg_tool = check_prov_for_regtool(xfm_prov) - if opt.lower() == 'fsl' and reg_tool.lower() == 'ants': - + if opt.lower() == "fsl" and reg_tool.lower() == "ants": # Apply head-to-head transforms on brain using ABCD-style registration # Convert ANTs warps to FSL warps to be consistent with the functional registration # Ref: https://github.com/DCAN-Labs/DCAN-HCP/blob/master/PostFreeSurfer/scripts/AtlasRegistrationToMNI152_ANTsbased.sh#L134-L172 @@ -2624,20 +2916,22 @@ def overwrite_transform_anat_to_template(wf, cfg, strat_pool, pipe_num, # -t ${WD}/xfms/T1w_to_MNI_1Rigid.mat \ # -t ${WD}/xfms/T1w_to_MNI_0DerivedInitialMovingTranslation.mat \ # -o [${WD}/xfms/ANTs_CombinedWarp.nii.gz,1] - ants_apply_warp_t1_to_template = pe.Node(interface=ants.ApplyTransforms(), - name=f'ANTS-ABCD_T1_to_template_{pipe_num}') + ants_apply_warp_t1_to_template = pe.Node( + interface=ants.ApplyTransforms(), + name=f"ANTS-ABCD_T1_to_template_{pipe_num}", + ) ants_apply_warp_t1_to_template.inputs.dimension = 3 ants_apply_warp_t1_to_template.inputs.print_out_composite_warp_file = True - ants_apply_warp_t1_to_template.inputs.output_image = 'ANTs_CombinedWarp.nii.gz' + ants_apply_warp_t1_to_template.inputs.output_image = "ANTs_CombinedWarp.nii.gz" - node, out = strat_pool.get_data(['desc-restore_T1w', 'desc-preproc_T1w']) - wf.connect(node, out, ants_apply_warp_t1_to_template, 'input_image') + node, out = strat_pool.get_data(["desc-restore_T1w", "desc-preproc_T1w"]) + wf.connect(node, out, ants_apply_warp_t1_to_template, "input_image") - node, out = strat_pool.get_data('T1w-template') - wf.connect(node, out, ants_apply_warp_t1_to_template, 'reference_image') + node, out = strat_pool.get_data("T1w-template") + wf.connect(node, out, ants_apply_warp_t1_to_template, "reference_image") - node, out = strat_pool.get_data('from-T1w_to-template_mode-image_xfm') - wf.connect(node, out, ants_apply_warp_t1_to_template, 'transforms') + node, out = strat_pool.get_data("from-T1w_to-template_mode-image_xfm") + wf.connect(node, out, ants_apply_warp_t1_to_template, "transforms") # antsApplyTransforms -d 3 -i ${T1wImage}.nii.gz -r ${Reference} \ # -t [${WD}/xfms/T1w_to_MNI_0DerivedInitialMovingTranslation.mat,1] \ @@ -2647,161 +2941,182 @@ def overwrite_transform_anat_to_template(wf, cfg, strat_pool, pipe_num, # -o [${WD}/xfms/ANTs_CombinedInvWarp.nii.gz,1] # T1wImage is ACPC aligned head - ants_apply_warp_template_to_t1 = pe.Node(interface=ants.ApplyTransforms(), - name=f'ANTS-ABCD_template_to_T1_{pipe_num}') + ants_apply_warp_template_to_t1 = pe.Node( + interface=ants.ApplyTransforms(), + name=f"ANTS-ABCD_template_to_T1_{pipe_num}", + ) ants_apply_warp_template_to_t1.inputs.dimension = 3 ants_apply_warp_template_to_t1.inputs.print_out_composite_warp_file = True - ants_apply_warp_template_to_t1.inputs.output_image = 'ANTs_CombinedInvWarp.nii.gz' + ants_apply_warp_template_to_t1.inputs.output_image = ( + "ANTs_CombinedInvWarp.nii.gz" + ) - node, out = strat_pool.get_data('desc-preproc_T1w') - wf.connect(node, out, ants_apply_warp_template_to_t1, 'input_image') + node, out = strat_pool.get_data("desc-preproc_T1w") + wf.connect(node, out, ants_apply_warp_template_to_t1, "input_image") - node, out = strat_pool.get_data('T1w-template') - wf.connect(node, out, ants_apply_warp_template_to_t1, 'reference_image') + node, out = strat_pool.get_data("T1w-template") + wf.connect(node, out, ants_apply_warp_template_to_t1, "reference_image") - node, out = strat_pool.get_data('from-template_to-T1w_mode-image_xfm') - wf.connect(node, out, ants_apply_warp_template_to_t1, 'transforms') + node, out = strat_pool.get_data("from-template_to-T1w_mode-image_xfm") + wf.connect(node, out, ants_apply_warp_template_to_t1, "transforms") # c4d -mcs ${WD}/xfms/ANTs_CombinedWarp.nii.gz -oo ${WD}/xfms/e1.nii.gz ${WD}/xfms/e2.nii.gz ${WD}/xfms/e3.nii.gz # -mcs: -multicomponent-split, -oo: -output-multiple - split_combined_warp = pe.Node(util.Function(input_names=['input', - 'output_name'], - output_names=['output1', - 'output2', - 'output3'], - function=run_c4d), - name=f'split_combined_warp_{pipe_num}') - split_combined_warp.inputs.output_name = 'e' - - wf.connect(ants_apply_warp_t1_to_template, 'output_image', - split_combined_warp, 'input') + split_combined_warp = pe.Node( + util.Function( + input_names=["input", "output_name"], + output_names=["output1", "output2", "output3"], + function=run_c4d, + ), + name=f"split_combined_warp_{pipe_num}", + ) + split_combined_warp.inputs.output_name = "e" + + wf.connect( + ants_apply_warp_t1_to_template, "output_image", split_combined_warp, "input" + ) # c4d -mcs ${WD}/xfms/ANTs_CombinedInvWarp.nii.gz -oo ${WD}/xfms/e1inv.nii.gz ${WD}/xfms/e2inv.nii.gz ${WD}/xfms/e3inv.nii.gz - split_combined_inv_warp = pe.Node(util.Function(input_names=['input', - 'output_name'], - output_names=['output1', - 'output2', - 'output3'], - function=run_c4d), - name=f'split_combined_inv_warp_{pipe_num}') - split_combined_inv_warp.inputs.output_name = 'einv' - - wf.connect(ants_apply_warp_template_to_t1, 'output_image', - split_combined_inv_warp, 'input') + split_combined_inv_warp = pe.Node( + util.Function( + input_names=["input", "output_name"], + output_names=["output1", "output2", "output3"], + function=run_c4d, + ), + name=f"split_combined_inv_warp_{pipe_num}", + ) + split_combined_inv_warp.inputs.output_name = "einv" + + wf.connect( + ants_apply_warp_template_to_t1, + "output_image", + split_combined_inv_warp, + "input", + ) # fslmaths ${WD}/xfms/e2.nii.gz -mul -1 ${WD}/xfms/e-2.nii.gz - change_e2_sign = pe.Node(interface=fsl.maths.MathsCommand(), - name=f'change_e2_sign_{pipe_num}') - change_e2_sign.inputs.args = '-mul -1' + change_e2_sign = pe.Node( + interface=fsl.maths.MathsCommand(), name=f"change_e2_sign_{pipe_num}" + ) + change_e2_sign.inputs.args = "-mul -1" - wf.connect(split_combined_warp, 'output2', - change_e2_sign, 'in_file') + wf.connect(split_combined_warp, "output2", change_e2_sign, "in_file") # fslmaths ${WD}/xfms/e2inv.nii.gz -mul -1 ${WD}/xfms/e-2inv.nii.gz - change_e2inv_sign = pe.Node(interface=fsl.maths.MathsCommand(), - name=f'change_e2inv_sign_{pipe_num}') - change_e2inv_sign.inputs.args = '-mul -1' + change_e2inv_sign = pe.Node( + interface=fsl.maths.MathsCommand(), name=f"change_e2inv_sign_{pipe_num}" + ) + change_e2inv_sign.inputs.args = "-mul -1" - wf.connect(split_combined_inv_warp, 'output2', - change_e2inv_sign, 'in_file') + wf.connect(split_combined_inv_warp, "output2", change_e2inv_sign, "in_file") # fslmerge -t ${OutputTransform} ${WD}/xfms/e1.nii.gz ${WD}/xfms/e-2.nii.gz ${WD}/xfms/e3.nii.gz - merge_xfms_to_list = pe.Node(util.Merge(3), - name=f'merge_t1_to_template_xfms_to_list_{pipe_num}') + merge_xfms_to_list = pe.Node( + util.Merge(3), name=f"merge_t1_to_template_xfms_to_list_{pipe_num}" + ) - wf.connect(split_combined_warp, 'output1', - merge_xfms_to_list, 'in1') - wf.connect(change_e2_sign, 'out_file', - merge_xfms_to_list, 'in2') - wf.connect(split_combined_warp, 'output3', - merge_xfms_to_list, 'in3') + wf.connect(split_combined_warp, "output1", merge_xfms_to_list, "in1") + wf.connect(change_e2_sign, "out_file", merge_xfms_to_list, "in2") + wf.connect(split_combined_warp, "output3", merge_xfms_to_list, "in3") - merge_xfms = pe.Node(interface=fslMerge(), - name=f'merge_t1_to_template_xfms_{pipe_num}') - merge_xfms.inputs.dimension = 't' + merge_xfms = pe.Node( + interface=fslMerge(), name=f"merge_t1_to_template_xfms_{pipe_num}" + ) + merge_xfms.inputs.dimension = "t" - wf.connect(merge_xfms_to_list, 'out', - merge_xfms, 'in_files') + wf.connect(merge_xfms_to_list, "out", merge_xfms, "in_files") # fslmerge -t ${OutputInvTransform} ${WD}/xfms/e1inv.nii.gz ${WD}/xfms/e-2inv.nii.gz ${WD}/xfms/e3inv.nii.gz - merge_inv_xfms_to_list = pe.Node(util.Merge(3), - name=f'merge_template_to_t1_xfms_to_list_{pipe_num}') + merge_inv_xfms_to_list = pe.Node( + util.Merge(3), name=f"merge_template_to_t1_xfms_to_list_{pipe_num}" + ) - wf.connect(split_combined_inv_warp, 'output1', - merge_inv_xfms_to_list, 'in1') - wf.connect(change_e2inv_sign, 'out_file', - merge_inv_xfms_to_list, 'in2') - wf.connect(split_combined_inv_warp, 'output3', - merge_inv_xfms_to_list, 'in3') + wf.connect(split_combined_inv_warp, "output1", merge_inv_xfms_to_list, "in1") + wf.connect(change_e2inv_sign, "out_file", merge_inv_xfms_to_list, "in2") + wf.connect(split_combined_inv_warp, "output3", merge_inv_xfms_to_list, "in3") - merge_inv_xfms = pe.Node(interface=fslMerge(), - name=f'merge_template_to_t1_xfms_{pipe_num}') - merge_inv_xfms.inputs.dimension = 't' + merge_inv_xfms = pe.Node( + interface=fslMerge(), name=f"merge_template_to_t1_xfms_{pipe_num}" + ) + merge_inv_xfms.inputs.dimension = "t" - wf.connect(merge_inv_xfms_to_list, 'out', - merge_inv_xfms, 'in_files') + wf.connect(merge_inv_xfms_to_list, "out", merge_inv_xfms, "in_files") # applywarp --rel --interp=spline -i ${T1wRestore} -r ${Reference} -w ${OutputTransform} -o ${OutputT1wImageRestore} - fsl_apply_warp_t1_to_template = pe.Node(interface=fsl.ApplyWarp(), - name=f'FSL-ABCD_T1_to_template_{pipe_num}') + fsl_apply_warp_t1_to_template = pe.Node( + interface=fsl.ApplyWarp(), name=f"FSL-ABCD_T1_to_template_{pipe_num}" + ) fsl_apply_warp_t1_to_template.inputs.relwarp = True - fsl_apply_warp_t1_to_template.inputs.interp = 'spline' + fsl_apply_warp_t1_to_template.inputs.interp = "spline" - node, out = strat_pool.get_data(['desc-restore_T1w', 'desc-preproc_T1w']) - wf.connect(node, out, fsl_apply_warp_t1_to_template, 'in_file') + node, out = strat_pool.get_data(["desc-restore_T1w", "desc-preproc_T1w"]) + wf.connect(node, out, fsl_apply_warp_t1_to_template, "in_file") - node, out = strat_pool.get_data('T1w-template') - wf.connect(node, out, fsl_apply_warp_t1_to_template, 'ref_file') + node, out = strat_pool.get_data("T1w-template") + wf.connect(node, out, fsl_apply_warp_t1_to_template, "ref_file") - wf.connect(merge_xfms, 'merged_file', - fsl_apply_warp_t1_to_template, 'field_file') + wf.connect( + merge_xfms, "merged_file", fsl_apply_warp_t1_to_template, "field_file" + ) # applywarp --rel --interp=nn -i ${T1wRestoreBrain} -r ${Reference} -w ${OutputTransform} -o ${OutputT1wImageRestoreBrain} - fsl_apply_warp_t1_brain_to_template = pe.Node(interface=fsl.ApplyWarp(), - name=f'FSL-ABCD_T1_brain_to_template_{pipe_num}') + fsl_apply_warp_t1_brain_to_template = pe.Node( + interface=fsl.ApplyWarp(), name=f"FSL-ABCD_T1_brain_to_template_{pipe_num}" + ) fsl_apply_warp_t1_brain_to_template.inputs.relwarp = True - fsl_apply_warp_t1_brain_to_template.inputs.interp = 'nn' + fsl_apply_warp_t1_brain_to_template.inputs.interp = "nn" # TODO connect T1wRestoreBrain, check T1wRestoreBrain quality - node, out = strat_pool.get_data('desc-preproc_T1w') - wf.connect(node, out, fsl_apply_warp_t1_brain_to_template, 'in_file') + node, out = strat_pool.get_data("desc-preproc_T1w") + wf.connect(node, out, fsl_apply_warp_t1_brain_to_template, "in_file") - node, out = strat_pool.get_data('T1w-template') - wf.connect(node, out, fsl_apply_warp_t1_brain_to_template, 'ref_file') + node, out = strat_pool.get_data("T1w-template") + wf.connect(node, out, fsl_apply_warp_t1_brain_to_template, "ref_file") - wf.connect(merge_xfms, 'merged_file', - fsl_apply_warp_t1_brain_to_template, 'field_file') + wf.connect( + merge_xfms, "merged_file", fsl_apply_warp_t1_brain_to_template, "field_file" + ) - fsl_apply_warp_t1_brain_mask_to_template = pe.Node(interface=fsl.ApplyWarp(), - name=f'FSL-ABCD_T1_brain_mask_to_template_{pipe_num}') + fsl_apply_warp_t1_brain_mask_to_template = pe.Node( + interface=fsl.ApplyWarp(), + name=f"FSL-ABCD_T1_brain_mask_to_template_{pipe_num}", + ) fsl_apply_warp_t1_brain_mask_to_template.inputs.relwarp = True - fsl_apply_warp_t1_brain_mask_to_template.inputs.interp = 'nn' + fsl_apply_warp_t1_brain_mask_to_template.inputs.interp = "nn" - node, out = strat_pool.get_data('space-T1w_desc-brain_mask') - wf.connect(node, out, fsl_apply_warp_t1_brain_mask_to_template, 'in_file') + node, out = strat_pool.get_data("space-T1w_desc-brain_mask") + wf.connect(node, out, fsl_apply_warp_t1_brain_mask_to_template, "in_file") - node, out = strat_pool.get_data('T1w-template') - wf.connect(node, out, fsl_apply_warp_t1_brain_mask_to_template, 'ref_file') + node, out = strat_pool.get_data("T1w-template") + wf.connect(node, out, fsl_apply_warp_t1_brain_mask_to_template, "ref_file") - wf.connect(merge_xfms, 'merged_file', - fsl_apply_warp_t1_brain_mask_to_template, 'field_file') + wf.connect( + merge_xfms, + "merged_file", + fsl_apply_warp_t1_brain_mask_to_template, + "field_file", + ) # fslmaths ${OutputT1wImageRestore} -mas ${OutputT1wImageRestoreBrain} ${OutputT1wImageRestoreBrain} - apply_mask = pe.Node(interface=fsl.maths.ApplyMask(), - name=f'get_t1_brain_{pipe_num}') + apply_mask = pe.Node( + interface=fsl.maths.ApplyMask(), name=f"get_t1_brain_{pipe_num}" + ) - wf.connect(fsl_apply_warp_t1_to_template, 'out_file', - apply_mask, 'in_file') + wf.connect(fsl_apply_warp_t1_to_template, "out_file", apply_mask, "in_file") - wf.connect(fsl_apply_warp_t1_brain_to_template, 'out_file', - apply_mask, 'mask_file') + wf.connect( + fsl_apply_warp_t1_brain_to_template, "out_file", apply_mask, "mask_file" + ) outputs = { - 'space-template_desc-preproc_T1w': (apply_mask, 'out_file'), - 'space-template_desc-head_T1w': (fsl_apply_warp_t1_to_template, 'out_file'), - 'space-template_desc-T1w_mask': (fsl_apply_warp_t1_brain_mask_to_template, 'out_file'), - 'from-T1w_to-template_mode-image_xfm': (merge_xfms, 'merged_file'), - 'from-template_to-T1w_mode-image_xfm': (merge_inv_xfms, 'merged_file') + "space-template_desc-preproc_T1w": (apply_mask, "out_file"), + "space-template_desc-head_T1w": (fsl_apply_warp_t1_to_template, "out_file"), + "space-template_desc-T1w_mask": ( + fsl_apply_warp_t1_brain_mask_to_template, + "out_file", + ), + "from-T1w_to-template_mode-image_xfm": (merge_xfms, "merged_file"), + "from-template_to-T1w_mode-image_xfm": (merge_inv_xfms, "merged_file"), } return (wf, outputs) @@ -2822,40 +3137,37 @@ def overwrite_transform_anat_to_template(wf, cfg, strat_pool, pipe_num, outputs=["sbref"], ) def coregistration_prep_vol(wf, cfg, strat_pool, pipe_num, opt=None): - - get_func_volume = pe.Node(interface=afni.Calc(), - name=f'get_func_volume_{pipe_num}') + get_func_volume = pe.Node(interface=afni.Calc(), name=f"get_func_volume_{pipe_num}") get_func_volume.inputs.set( - expr='a', - single_idx=cfg.registration_workflows['functional_registration']['coregistration'][ - 'func_input_prep']['Selected Functional Volume']['func_reg_input_volume'], - outputtype='NIFTI_GZ' + expr="a", + single_idx=cfg.registration_workflows["functional_registration"][ + "coregistration" + ]["func_input_prep"]["Selected Functional Volume"]["func_reg_input_volume"], + outputtype="NIFTI_GZ", ) - if not cfg.registration_workflows['functional_registration'][ - 'coregistration']['func_input_prep']['reg_with_skull']: + if not cfg.registration_workflows["functional_registration"]["coregistration"][ + "func_input_prep" + ]["reg_with_skull"]: node, out = strat_pool.get_data("desc-brain_bold") else: # TODO check which file is functional_skull_leaf # TODO add a function to choose brain or skull? node, out = strat_pool.get_data(["desc-motion_bold", "bold"]) - wf.connect(node, out, get_func_volume, 'in_file_a') + wf.connect(node, out, get_func_volume, "in_file_a") - coreg_input = (get_func_volume, 'out_file') + coreg_input = (get_func_volume, "out_file") - outputs = { - 'sbref': coreg_input - } + outputs = {"sbref": coreg_input} return (wf, outputs) @nodeblock( name="coregistration_prep_mean", - switch=[["functional_preproc", "run"], - ["functional_preproc", "coreg_prep", "run"]], + switch=[["functional_preproc", "run"], ["functional_preproc", "coreg_prep", "run"]], option_key=[ "registration_workflows", "functional_registration", @@ -2868,38 +3180,34 @@ def coregistration_prep_vol(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["sbref"], ) def coregistration_prep_mean(wf, cfg, strat_pool, pipe_num, opt=None): - coreg_input = strat_pool.get_data("desc-mean_bold") # TODO add mean skull - if cfg.registration_workflows['functional_registration'][ - 'coregistration']['func_input_prep']['Mean Functional'][ - 'n4_correct_func']: + if cfg.registration_workflows["functional_registration"]["coregistration"][ + "func_input_prep" + ]["Mean Functional"]["n4_correct_func"]: n4_correct_func = pe.Node( - interface= - ants.N4BiasFieldCorrection(dimension=3, - copy_header=True, - bspline_fitting_distance=200), + interface=ants.N4BiasFieldCorrection( + dimension=3, copy_header=True, bspline_fitting_distance=200 + ), shrink_factor=2, - name=f'func_mean_n4_corrected_{pipe_num}') - n4_correct_func.inputs.args = '-r True' + name=f"func_mean_n4_corrected_{pipe_num}", + ) + n4_correct_func.inputs.args = "-r True" node, out = coreg_input - wf.connect(node, out, n4_correct_func, 'input_image') + wf.connect(node, out, n4_correct_func, "input_image") - coreg_input = (n4_correct_func, 'output_image') + coreg_input = (n4_correct_func, "output_image") - outputs = { - 'sbref': coreg_input - } + outputs = {"sbref": coreg_input} return (wf, outputs) @nodeblock( name="coregistration_prep_fmriprep", - switch=[["functional_preproc", "run"], - ["functional_preproc", "coreg_prep", "run"]], + switch=[["functional_preproc", "run"], ["functional_preproc", "coreg_prep", "run"]], option_key=[ "registration_workflows", "functional_registration", @@ -2912,12 +3220,9 @@ def coregistration_prep_mean(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["sbref"], ) def coregistration_prep_fmriprep(wf, cfg, strat_pool, pipe_num, opt=None): - coreg_input = strat_pool.get_data("desc-ref_bold") - outputs = { - 'sbref': coreg_input - } + outputs = {"sbref": coreg_input} return (wf, outputs) @@ -2956,164 +3261,195 @@ def coregistration_prep_fmriprep(wf, cfg, strat_pool, pipe_num, opt=None): ) def coregistration(wf, cfg, strat_pool, pipe_num, opt=None): diff_complete = False - if strat_pool.check_rpool("despiked-fieldmap") and \ - strat_pool.check_rpool("fieldmap-mask"): + if strat_pool.check_rpool("despiked-fieldmap") and strat_pool.check_rpool( + "fieldmap-mask" + ): diff_complete = True - if strat_pool.check_rpool('T2w') and cfg.anatomical_preproc['run_t2']: + if strat_pool.check_rpool("T2w") and cfg.anatomical_preproc["run_t2"]: # monkey data - func_to_anat = create_register_func_to_anat_use_T2(cfg, - f'func_to_anat_FLIRT_' - f'{pipe_num}') + func_to_anat = create_register_func_to_anat_use_T2( + cfg, f"func_to_anat_FLIRT_" f"{pipe_num}" + ) # https://github.com/DCAN-Labs/dcan-macaque-pipeline/blob/master/fMRIVolume/GenericfMRIVolumeProcessingPipeline.sh#L177 # fslmaths "$fMRIFolder"/"$NameOffMRI"_mc -Tmean "$fMRIFolder"/"$ScoutName"_gdc - func_mc_mean = pe.Node(interface=afni_utils.TStat(), - name=f'func_motion_corrected_mean_{pipe_num}') + func_mc_mean = pe.Node( + interface=afni_utils.TStat(), name=f"func_motion_corrected_mean_{pipe_num}" + ) - func_mc_mean.inputs.options = '-mean' - func_mc_mean.inputs.outputtype = 'NIFTI_GZ' + func_mc_mean.inputs.options = "-mean" + func_mc_mean.inputs.outputtype = "NIFTI_GZ" node, out = strat_pool.get_data("desc-motion_bold") - wf.connect(node, out, func_mc_mean, 'in_file') + wf.connect(node, out, func_mc_mean, "in_file") - wf.connect(func_mc_mean, 'out_file', func_to_anat, 'inputspec.func') + wf.connect(func_mc_mean, "out_file", func_to_anat, "inputspec.func") - node, out = strat_pool.get_data('desc-preproc_T1w') - wf.connect(node, out, func_to_anat, 'inputspec.T1_brain') + node, out = strat_pool.get_data("desc-preproc_T1w") + wf.connect(node, out, func_to_anat, "inputspec.T1_brain") - node, out = strat_pool.get_data('desc-head_T2w') - wf.connect(node, out, func_to_anat, 'inputspec.T2_head') + node, out = strat_pool.get_data("desc-head_T2w") + wf.connect(node, out, func_to_anat, "inputspec.T2_head") - node, out = strat_pool.get_data('desc-preproc_T2w') - wf.connect(node, out, func_to_anat, 'inputspec.T2_brain') + node, out = strat_pool.get_data("desc-preproc_T2w") + wf.connect(node, out, func_to_anat, "inputspec.T2_brain") else: # if field map-based distortion correction is on, but BBR is off, # send in the distortion correction files here - func_to_anat = create_register_func_to_anat(cfg, diff_complete, - f'func_to_anat_FLIRT_' - f'{pipe_num}') + func_to_anat = create_register_func_to_anat( + cfg, diff_complete, f"func_to_anat_FLIRT_" f"{pipe_num}" + ) func_to_anat.inputs.inputspec.dof = cfg.registration_workflows[ - 'functional_registration']['coregistration']['dof'] + "functional_registration" + ]["coregistration"]["dof"] func_to_anat.inputs.inputspec.interp = cfg.registration_workflows[ - 'functional_registration']['coregistration']['interpolation'] - - node, out = strat_pool.get_data('sbref') - wf.connect(node, out, func_to_anat, 'inputspec.func') - - if cfg.registration_workflows['functional_registration'][ - 'coregistration']['reference'] == 'brain': + "functional_registration" + ]["coregistration"]["interpolation"] + + node, out = strat_pool.get_data("sbref") + wf.connect(node, out, func_to_anat, "inputspec.func") + + if ( + cfg.registration_workflows["functional_registration"]["coregistration"][ + "reference" + ] + == "brain" + ): # TODO: use JSON meta-data to confirm - node, out = strat_pool.get_data('desc-preproc_T1w') - elif cfg.registration_workflows['functional_registration'][ - 'coregistration']['reference'] == 'restore-brain': - node, out = strat_pool.get_data('desc-restore-brain_T1w') - wf.connect(node, out, func_to_anat, 'inputspec.anat') + node, out = strat_pool.get_data("desc-preproc_T1w") + elif ( + cfg.registration_workflows["functional_registration"]["coregistration"][ + "reference" + ] + == "restore-brain" + ): + node, out = strat_pool.get_data("desc-restore-brain_T1w") + wf.connect(node, out, func_to_anat, "inputspec.anat") if diff_complete: - node, out = strat_pool.get_data('effectiveEchoSpacing') - wf.connect(node, out, func_to_anat, 'echospacing_input.echospacing') + node, out = strat_pool.get_data("effectiveEchoSpacing") + wf.connect(node, out, func_to_anat, "echospacing_input.echospacing") - node, out = strat_pool.get_data('pe-direction') - wf.connect(node, out, func_to_anat, 'pedir_input.pedir') + node, out = strat_pool.get_data("pe-direction") + wf.connect(node, out, func_to_anat, "pedir_input.pedir") node, out = strat_pool.get_data("despiked-fieldmap") - wf.connect(node, out, func_to_anat, 'inputspec.fieldmap') + wf.connect(node, out, func_to_anat, "inputspec.fieldmap") node, out = strat_pool.get_data("fieldmap-mask") - wf.connect(node, out, func_to_anat, 'inputspec.fieldmapmask') + wf.connect(node, out, func_to_anat, "inputspec.fieldmapmask") - if strat_pool.check_rpool('T2w') and cfg.anatomical_preproc['run_t2']: + if strat_pool.check_rpool("T2w") and cfg.anatomical_preproc["run_t2"]: outputs = { - 'space-T1w_sbref': - (func_to_anat, 'outputspec.anat_func_nobbreg'), - 'from-bold_to-T1w_mode-image_desc-linear_xfm': - (func_to_anat, 'outputspec.func_to_anat_linear_xfm_nobbreg'), - 'from-bold_to-T1w_mode-image_desc-linear_warp': - (func_to_anat, 'outputspec.func_to_anat_linear_warp_nobbreg') + "space-T1w_sbref": (func_to_anat, "outputspec.anat_func_nobbreg"), + "from-bold_to-T1w_mode-image_desc-linear_xfm": ( + func_to_anat, + "outputspec.func_to_anat_linear_xfm_nobbreg", + ), + "from-bold_to-T1w_mode-image_desc-linear_warp": ( + func_to_anat, + "outputspec.func_to_anat_linear_warp_nobbreg", + ), } else: outputs = { - 'space-T1w_sbref': - (func_to_anat, 'outputspec.anat_func_nobbreg'), - 'from-bold_to-T1w_mode-image_desc-linear_xfm': - (func_to_anat, 'outputspec.func_to_anat_linear_xfm_nobbreg') + "space-T1w_sbref": (func_to_anat, "outputspec.anat_func_nobbreg"), + "from-bold_to-T1w_mode-image_desc-linear_xfm": ( + func_to_anat, + "outputspec.func_to_anat_linear_xfm_nobbreg", + ), } - if True in cfg.registration_workflows['functional_registration'][ - 'coregistration']["boundary_based_registration"]["run"]: - - func_to_anat_bbreg = create_bbregister_func_to_anat(diff_complete, - f'func_to_anat_' - f'bbreg_' - f'{pipe_num}') - func_to_anat_bbreg.inputs.inputspec.bbr_schedule = \ - cfg.registration_workflows['functional_registration'][ - 'coregistration']['boundary_based_registration'][ - 'bbr_schedule'] - - func_to_anat_bbreg.inputs.inputspec.bbr_wm_mask_args = \ - cfg.registration_workflows['functional_registration'][ - 'coregistration']['boundary_based_registration'][ - 'bbr_wm_mask_args'] - - node, out = strat_pool.get_data('sbref') - wf.connect(node, out, func_to_anat_bbreg, 'inputspec.func') - - if cfg.registration_workflows['functional_registration'][ - 'coregistration']['boundary_based_registration'][ - 'reference'] == 'whole-head': - node, out = strat_pool.get_data('desc-head_T1w') - wf.connect(node, out, func_to_anat_bbreg, 'inputspec.anat') - - elif cfg.registration_workflows['functional_registration'][ - 'coregistration']['boundary_based_registration'][ - 'reference'] == 'brain': - node, out = strat_pool.get_data('desc-preproc_T1w') - wf.connect(node, out, func_to_anat_bbreg, 'inputspec.anat') - - wf.connect(func_to_anat, 'outputspec.func_to_anat_linear_xfm_nobbreg', - func_to_anat_bbreg, 'inputspec.linear_reg_matrix') - - if strat_pool.check_rpool('space-bold_label-WM_mask'): + if ( + True + in cfg.registration_workflows["functional_registration"]["coregistration"][ + "boundary_based_registration" + ]["run"] + ): + func_to_anat_bbreg = create_bbregister_func_to_anat( + diff_complete, f"func_to_anat_" f"bbreg_" f"{pipe_num}" + ) + func_to_anat_bbreg.inputs.inputspec.bbr_schedule = cfg.registration_workflows[ + "functional_registration" + ]["coregistration"]["boundary_based_registration"]["bbr_schedule"] + + func_to_anat_bbreg.inputs.inputspec.bbr_wm_mask_args = ( + cfg.registration_workflows["functional_registration"]["coregistration"][ + "boundary_based_registration" + ]["bbr_wm_mask_args"] + ) + + node, out = strat_pool.get_data("sbref") + wf.connect(node, out, func_to_anat_bbreg, "inputspec.func") + + if ( + cfg.registration_workflows["functional_registration"]["coregistration"][ + "boundary_based_registration" + ]["reference"] + == "whole-head" + ): + node, out = strat_pool.get_data("desc-head_T1w") + wf.connect(node, out, func_to_anat_bbreg, "inputspec.anat") + + elif ( + cfg.registration_workflows["functional_registration"]["coregistration"][ + "boundary_based_registration" + ]["reference"] + == "brain" + ): + node, out = strat_pool.get_data("desc-preproc_T1w") + wf.connect(node, out, func_to_anat_bbreg, "inputspec.anat") + + wf.connect( + func_to_anat, + "outputspec.func_to_anat_linear_xfm_nobbreg", + func_to_anat_bbreg, + "inputspec.linear_reg_matrix", + ) + + if strat_pool.check_rpool("space-bold_label-WM_mask"): node, out = strat_pool.get_data(["space-bold_label-WM_mask"]) - wf.connect(node, out, - func_to_anat_bbreg, 'inputspec.anat_wm_segmentation') + wf.connect(node, out, func_to_anat_bbreg, "inputspec.anat_wm_segmentation") else: - if cfg.registration_workflows['functional_registration'][ - 'coregistration']['boundary_based_registration']['bbr_wm_map'] == 'probability_map': - node, out = strat_pool.get_data(["label-WM_probseg", - "label-WM_mask"]) - elif cfg.registration_workflows['functional_registration'][ - 'coregistration']['boundary_based_registration']['bbr_wm_map'] == 'partial_volume_map': - node, out = strat_pool.get_data(["label-WM_pveseg", - "label-WM_mask"]) - wf.connect(node, out, - func_to_anat_bbreg, 'inputspec.anat_wm_segmentation') + if ( + cfg.registration_workflows["functional_registration"]["coregistration"][ + "boundary_based_registration" + ]["bbr_wm_map"] + == "probability_map" + ): + node, out = strat_pool.get_data(["label-WM_probseg", "label-WM_mask"]) + elif ( + cfg.registration_workflows["functional_registration"]["coregistration"][ + "boundary_based_registration" + ]["bbr_wm_map"] + == "partial_volume_map" + ): + node, out = strat_pool.get_data(["label-WM_pveseg", "label-WM_mask"]) + wf.connect(node, out, func_to_anat_bbreg, "inputspec.anat_wm_segmentation") if diff_complete: - node, out = strat_pool.get_data('effectiveEchoSpacing') - wf.connect(node, out, - func_to_anat_bbreg, 'echospacing_input.echospacing') + node, out = strat_pool.get_data("effectiveEchoSpacing") + wf.connect(node, out, func_to_anat_bbreg, "echospacing_input.echospacing") - node, out = strat_pool.get_data('pe-direction') - wf.connect(node, out, func_to_anat_bbreg, 'pedir_input.pedir') + node, out = strat_pool.get_data("pe-direction") + wf.connect(node, out, func_to_anat_bbreg, "pedir_input.pedir") node, out = strat_pool.get_data("despiked-fieldmap") - wf.connect(node, out, func_to_anat_bbreg, 'inputspec.fieldmap') + wf.connect(node, out, func_to_anat_bbreg, "inputspec.fieldmap") node, out = strat_pool.get_data("fieldmap-mask") - wf.connect(node, out, - func_to_anat_bbreg, 'inputspec.fieldmapmask') + wf.connect(node, out, func_to_anat_bbreg, "inputspec.fieldmapmask") outputs = { - 'space-T1w_sbref': - (func_to_anat_bbreg, 'outputspec.anat_func'), - 'from-bold_to-T1w_mode-image_desc-linear_xfm': - (func_to_anat_bbreg, 'outputspec.func_to_anat_linear_xfm') + "space-T1w_sbref": (func_to_anat_bbreg, "outputspec.anat_func"), + "from-bold_to-T1w_mode-image_desc-linear_xfm": ( + func_to_anat_bbreg, + "outputspec.func_to_anat_linear_xfm", + ), } return (wf, outputs) @@ -3153,51 +3489,48 @@ def coregistration(wf, cfg, strat_pool, pipe_num, opt=None): }, ) def create_func_to_T1template_xfm(wf, cfg, strat_pool, pipe_num, opt=None): - '''Condense the BOLD-to-T1 coregistration transform and the T1-to-template + """Condense the BOLD-to-T1 coregistration transform and the T1-to-template transform into one transform matrix. - ''' - xfm_prov = strat_pool.get_cpac_provenance( - 'from-T1w_to-template_mode-image_xfm') + """ + xfm_prov = strat_pool.get_cpac_provenance("from-T1w_to-template_mode-image_xfm") reg_tool = check_prov_for_regtool(xfm_prov) - xfm, outputs = bold_to_T1template_xfm_connector('create_func_to_T1w' - f'template_xfm_{pipe_num}', - cfg, reg_tool, - symmetric=False) + xfm, outputs = bold_to_T1template_xfm_connector( + "create_func_to_T1w" f"template_xfm_{pipe_num}", cfg, reg_tool, symmetric=False + ) - node, out = strat_pool.get_data( - 'from-bold_to-T1w_mode-image_desc-linear_xfm') - wf.connect(node, out, xfm, 'inputspec.coreg_xfm') + node, out = strat_pool.get_data("from-bold_to-T1w_mode-image_desc-linear_xfm") + wf.connect(node, out, xfm, "inputspec.coreg_xfm") - node, out = strat_pool.get_data('desc-brain_T1w') - wf.connect(node, out, xfm, 'inputspec.input_brain') + node, out = strat_pool.get_data("desc-brain_T1w") + wf.connect(node, out, xfm, "inputspec.input_brain") - node, out = strat_pool.get_data('sbref') - wf.connect(node, out, xfm, 'inputspec.mean_bold') + node, out = strat_pool.get_data("sbref") + wf.connect(node, out, xfm, "inputspec.mean_bold") - node, out = strat_pool.get_data('T1w-brain-template-funcreg') - wf.connect(node, out, xfm, 'inputspec.T1w-brain-template_funcreg') + node, out = strat_pool.get_data("T1w-brain-template-funcreg") + wf.connect(node, out, xfm, "inputspec.T1w-brain-template_funcreg") - node, out = strat_pool.get_data('from-T1w_to-template_mode-image_xfm') - wf.connect(node, out, xfm, 'inputspec.T1w_to_template_xfm') + node, out = strat_pool.get_data("from-T1w_to-template_mode-image_xfm") + wf.connect(node, out, xfm, "inputspec.T1w_to_template_xfm") # FNIRT pipelines don't have an inverse nonlinear warp, make optional - if strat_pool.check_rpool('from-template_to-T1w_mode-image_xfm'): - node, out = strat_pool.get_data('from-template_to-T1w_mode-image_xfm') - wf.connect(node, out, xfm, 'inputspec.template_to_T1w_xfm') - - if strat_pool.check_rpool('ants-blip-warp'): - if reg_tool == 'ants': - node, out = strat_pool.get_data('ants-blip-warp') - wf.connect(node, out, xfm, 'inputspec.blip_warp') - elif reg_tool == 'fsl': + if strat_pool.check_rpool("from-template_to-T1w_mode-image_xfm"): + node, out = strat_pool.get_data("from-template_to-T1w_mode-image_xfm") + wf.connect(node, out, xfm, "inputspec.template_to_T1w_xfm") + + if strat_pool.check_rpool("ants-blip-warp"): + if reg_tool == "ants": + node, out = strat_pool.get_data("ants-blip-warp") + wf.connect(node, out, xfm, "inputspec.blip_warp") + elif reg_tool == "fsl": # apply the ants blip warp separately pass - elif strat_pool.check_rpool('fsl-blip-warp'): - if reg_tool == 'fsl': - node, out = strat_pool.get_data('fsl-blip-warp') - wf.connect(node, out, xfm, 'inputspec.blip_warp') - elif reg_tool == 'ants': + elif strat_pool.check_rpool("fsl-blip-warp"): + if reg_tool == "fsl": + node, out = strat_pool.get_data("fsl-blip-warp") + wf.connect(node, out, xfm, "inputspec.blip_warp") + elif reg_tool == "ants": # apply the fsl blip warp separately pass @@ -3232,42 +3565,39 @@ def create_func_to_T1template_xfm(wf, cfg, strat_pool, pipe_num, opt=None): }, }, ) -def create_func_to_T1template_symmetric_xfm(wf, cfg, strat_pool, pipe_num, - opt=None): - '''Condense the BOLD-to-T1 coregistration transform and the T1-to- +def create_func_to_T1template_symmetric_xfm(wf, cfg, strat_pool, pipe_num, opt=None): + """Condense the BOLD-to-T1 coregistration transform and the T1-to- symmetric-template transform into one transform matrix. - ''' - - xfm_prov = strat_pool.get_cpac_provenance( - 'from-T1w_to-symtemplate_mode-image_xfm') + """ + xfm_prov = strat_pool.get_cpac_provenance("from-T1w_to-symtemplate_mode-image_xfm") reg_tool = check_prov_for_regtool(xfm_prov) - xfm, outputs = bold_to_T1template_xfm_connector('create_func_to_T1wsymtem' - f'plate_xfm_{pipe_num}', - cfg, reg_tool, - symmetric=True) + xfm, outputs = bold_to_T1template_xfm_connector( + "create_func_to_T1wsymtem" f"plate_xfm_{pipe_num}", + cfg, + reg_tool, + symmetric=True, + ) - node, out = strat_pool.get_data( - 'from-bold_to-T1w_mode-image_desc-linear_xfm') - wf.connect(node, out, xfm, 'inputspec.coreg_xfm') + node, out = strat_pool.get_data("from-bold_to-T1w_mode-image_desc-linear_xfm") + wf.connect(node, out, xfm, "inputspec.coreg_xfm") - node, out = strat_pool.get_data('desc-brain_T1w') - wf.connect(node, out, xfm, 'inputspec.input_brain') + node, out = strat_pool.get_data("desc-brain_T1w") + wf.connect(node, out, xfm, "inputspec.input_brain") - node, out = strat_pool.get_data('sbref') - wf.connect(node, out, xfm, 'inputspec.mean_bold') + node, out = strat_pool.get_data("sbref") + wf.connect(node, out, xfm, "inputspec.mean_bold") - node, out = strat_pool.get_data('T1w-brain-template-symmetric-deriv') - wf.connect(node, out, xfm, 'inputspec.T1w-brain-template_funcreg') + node, out = strat_pool.get_data("T1w-brain-template-symmetric-deriv") + wf.connect(node, out, xfm, "inputspec.T1w-brain-template_funcreg") - node, out = strat_pool.get_data('from-T1w_to-symtemplate_mode-image_xfm') - wf.connect(node, out, xfm, 'inputspec.T1w_to_template_xfm') + node, out = strat_pool.get_data("from-T1w_to-symtemplate_mode-image_xfm") + wf.connect(node, out, xfm, "inputspec.T1w_to_template_xfm") # FNIRT pipelines don't have an inverse nonlinear warp, make optional - if strat_pool.check_rpool('from-symtemplate_to-T1w_mode-image_xfm'): - node, out = \ - strat_pool.get_data('from-symtemplate_to-T1w_mode-image_xfm') - wf.connect(node, out, xfm, 'inputspec.template_to_T1w_xfm') + if strat_pool.check_rpool("from-symtemplate_to-T1w_mode-image_xfm"): + node, out = strat_pool.get_data("from-symtemplate_to-T1w_mode-image_xfm") + wf.connect(node, out, xfm, "inputspec.template_to_T1w_xfm") return (wf, outputs) @@ -3305,109 +3635,107 @@ def create_func_to_T1template_symmetric_xfm(wf, cfg, strat_pool, pipe_num, ], outputs=["sbref", "desc-preproc_bold", "desc-stc_bold", "bold"], ) -def apply_phasediff_to_timeseries_separately(wf, cfg, strat_pool, pipe_num, - opt=None): - - outputs = {'desc-preproc_bold': strat_pool.get_data("desc-preproc_bold")} +def apply_phasediff_to_timeseries_separately(wf, cfg, strat_pool, pipe_num, opt=None): + outputs = {"desc-preproc_bold": strat_pool.get_data("desc-preproc_bold")} if not strat_pool.check_rpool("despiked-fieldmap"): return (wf, outputs) - invert_coreg_xfm = pe.Node(interface=fsl.ConvertXFM(), - name=f'invert_coreg_xfm_{pipe_num}') + invert_coreg_xfm = pe.Node( + interface=fsl.ConvertXFM(), name=f"invert_coreg_xfm_{pipe_num}" + ) invert_coreg_xfm.inputs.invert_xfm = True node, out = strat_pool.get_data("from-bold_to-T1w_mode-image_desc-linear_xfm") - wf.connect(node, out, invert_coreg_xfm, 'in_file') + wf.connect(node, out, invert_coreg_xfm, "in_file") - warp_fmap = pe.Node(interface=fsl.ApplyWarp(), - name=f'warp_fmap_{pipe_num}') + warp_fmap = pe.Node(interface=fsl.ApplyWarp(), name=f"warp_fmap_{pipe_num}") - node, out = strat_pool.get_data('despiked-fieldmap') - wf.connect(node, out, warp_fmap, 'in_file') + node, out = strat_pool.get_data("despiked-fieldmap") + wf.connect(node, out, warp_fmap, "in_file") - node, out = strat_pool.get_data('sbref') - wf.connect(node, out, warp_fmap, 'ref_file') + node, out = strat_pool.get_data("sbref") + wf.connect(node, out, warp_fmap, "ref_file") - wf.connect(invert_coreg_xfm, 'out_file', warp_fmap, 'premat') + wf.connect(invert_coreg_xfm, "out_file", warp_fmap, "premat") - mask_fmap = pe.Node(interface=fsl.maths.MathsCommand(), - name=f'mask_fmap_{pipe_num}') - mask_fmap.inputs.args = '-abs -bin' + mask_fmap = pe.Node( + interface=fsl.maths.MathsCommand(), name=f"mask_fmap_{pipe_num}" + ) + mask_fmap.inputs.args = "-abs -bin" - wf.connect(warp_fmap, 'out_file', mask_fmap, 'in_file') + wf.connect(warp_fmap, "out_file", mask_fmap, "in_file") - conv_pedir = \ - pe.Node(interface=util.Function(input_names=['pedir', - 'convert'], - output_names=['pedir'], - function=convert_pedir), - name=f'apply_phasediff_convert_pedir_{pipe_num}') - conv_pedir.inputs.convert = 'ijk_to_xyz' + conv_pedir = pe.Node( + interface=util.Function( + input_names=["pedir", "convert"], + output_names=["pedir"], + function=convert_pedir, + ), + name=f"apply_phasediff_convert_pedir_{pipe_num}", + ) + conv_pedir.inputs.convert = "ijk_to_xyz" - node, out = strat_pool.get_data('pe-direction') - wf.connect(node, out, conv_pedir, 'pedir') + node, out = strat_pool.get_data("pe-direction") + wf.connect(node, out, conv_pedir, "pedir") - fugue_saveshift = pe.Node(interface=fsl.FUGUE(), - name=f'fugue_saveshift_{pipe_num}') + fugue_saveshift = pe.Node(interface=fsl.FUGUE(), name=f"fugue_saveshift_{pipe_num}") fugue_saveshift.inputs.save_shift = True - wf.connect(warp_fmap, 'out_file', fugue_saveshift, 'fmap_in_file') - wf.connect(mask_fmap, 'out_file', fugue_saveshift, 'mask_file') + wf.connect(warp_fmap, "out_file", fugue_saveshift, "fmap_in_file") + wf.connect(mask_fmap, "out_file", fugue_saveshift, "mask_file") # FSL calls effective echo spacing = dwell time (not accurate) - node, out = strat_pool.get_data('effectiveEchoSpacing') - wf.connect(node, out, fugue_saveshift, 'dwell_time') + node, out = strat_pool.get_data("effectiveEchoSpacing") + wf.connect(node, out, fugue_saveshift, "dwell_time") - wf.connect(conv_pedir, 'pedir', fugue_saveshift, 'unwarp_direction') + wf.connect(conv_pedir, "pedir", fugue_saveshift, "unwarp_direction") - shift_warp = pe.Node(interface=fsl.ConvertWarp(), - name=f'shift_warp_{pipe_num}') + shift_warp = pe.Node(interface=fsl.ConvertWarp(), name=f"shift_warp_{pipe_num}") shift_warp.inputs.out_relwarp = True - wf.connect(fugue_saveshift, 'shift_out_file', shift_warp, 'shift_in_file') + wf.connect(fugue_saveshift, "shift_out_file", shift_warp, "shift_in_file") - node, out = strat_pool.get_data('sbref') - wf.connect(node, out, shift_warp, 'reference') + node, out = strat_pool.get_data("sbref") + wf.connect(node, out, shift_warp, "reference") - wf.connect(conv_pedir, 'pedir', shift_warp, 'shift_direction') + wf.connect(conv_pedir, "pedir", shift_warp, "shift_direction") - warp_bold = pe.Node(interface=fsl.ApplyWarp(), - name=f'warp_bold_phasediff_{pipe_num}') + warp_bold = pe.Node( + interface=fsl.ApplyWarp(), name=f"warp_bold_phasediff_{pipe_num}" + ) warp_bold.inputs.relwarp = True - warp_bold.inputs.interp = 'spline' - - if opt == 'default': - node, out = strat_pool.get_data('desc-preproc_bold') - out_label = 'desc-preproc_bold' - elif opt == 'single_step_resampling_from_stc': - node, out = strat_pool.get_data('desc-stc_bold') - out_label = 'desc-stc_bold' - elif opt == 'abcd': - node, out = strat_pool.get_data('bold') - out_label = 'bold' - - wf.connect(node, out, warp_bold, 'in_file') - - node, out = strat_pool.get_data('sbref') - wf.connect(node, out, warp_bold, 'ref_file') - - wf.connect(shift_warp, 'out_file', warp_bold, 'field_file') - - warp_sbref = pe.Node(interface=fsl.ApplyWarp(), - name=f'warp_sbref_phasediff_{pipe_num}') + warp_bold.inputs.interp = "spline" + + if opt == "default": + node, out = strat_pool.get_data("desc-preproc_bold") + out_label = "desc-preproc_bold" + elif opt == "single_step_resampling_from_stc": + node, out = strat_pool.get_data("desc-stc_bold") + out_label = "desc-stc_bold" + elif opt == "abcd": + node, out = strat_pool.get_data("bold") + out_label = "bold" + + wf.connect(node, out, warp_bold, "in_file") + + node, out = strat_pool.get_data("sbref") + wf.connect(node, out, warp_bold, "ref_file") + + wf.connect(shift_warp, "out_file", warp_bold, "field_file") + + warp_sbref = pe.Node( + interface=fsl.ApplyWarp(), name=f"warp_sbref_phasediff_{pipe_num}" + ) warp_sbref.inputs.relwarp = True - warp_sbref.inputs.interp = 'spline' + warp_sbref.inputs.interp = "spline" - node, out = strat_pool.get_data('sbref') - wf.connect(node, out, warp_sbref, 'in_file') - wf.connect(node, out, warp_sbref, 'ref_file') + node, out = strat_pool.get_data("sbref") + wf.connect(node, out, warp_sbref, "in_file") + wf.connect(node, out, warp_sbref, "ref_file") - wf.connect(shift_warp, 'out_file', warp_sbref, 'field_file') + wf.connect(shift_warp, "out_file", warp_sbref, "field_file") - outputs = { - out_label: (warp_bold, 'out_file'), - 'sbref': (warp_sbref, 'out_file') - } + outputs = {out_label: (warp_bold, "out_file"), "sbref": (warp_sbref, "out_file")} return (wf, outputs) @@ -3444,69 +3772,67 @@ def apply_phasediff_to_timeseries_separately(wf, cfg, strat_pool, pipe_num, ], outputs=["desc-preproc_bold", "desc-stc_bold", "bold"], ) -def apply_blip_to_timeseries_separately(wf, cfg, strat_pool, pipe_num, - opt=None): - - xfm_prov = strat_pool.get_cpac_provenance( - 'from-bold_to-template_mode-image_xfm') +def apply_blip_to_timeseries_separately(wf, cfg, strat_pool, pipe_num, opt=None): + xfm_prov = strat_pool.get_cpac_provenance("from-bold_to-template_mode-image_xfm") reg_tool = check_prov_for_regtool(xfm_prov) - outputs = {'desc-preproc_bold': strat_pool.get_data("desc-preproc_bold")} + outputs = {"desc-preproc_bold": strat_pool.get_data("desc-preproc_bold")} if strat_pool.check_rpool("ants-blip-warp"): - if reg_tool == 'fsl': + if reg_tool == "fsl": blip_node, blip_out = strat_pool.get_data("ants-blip-warp") - reg_tool = 'ants' + reg_tool = "ants" else: return (wf, outputs) elif strat_pool.check_rpool("fsl-blip-warp"): - if reg_tool == 'ants': + if reg_tool == "ants": blip_node, blip_out = strat_pool.get_data("fsl-blip-warp") - reg_tool = 'fsl' + reg_tool = "fsl" else: return (wf, outputs) else: return (wf, outputs) - num_cpus = cfg.pipeline_setup['system_config'][ - 'max_cores_per_participant'] + num_cpus = cfg.pipeline_setup["system_config"]["max_cores_per_participant"] - num_ants_cores = cfg.pipeline_setup['system_config']['num_ants_threads'] + num_ants_cores = cfg.pipeline_setup["system_config"]["num_ants_threads"] - apply_xfm = apply_transform(f'warp_ts_to_blip_sep_{pipe_num}', reg_tool, - time_series=True, num_cpus=num_cpus, - num_ants_cores=num_ants_cores) + apply_xfm = apply_transform( + f"warp_ts_to_blip_sep_{pipe_num}", + reg_tool, + time_series=True, + num_cpus=num_cpus, + num_ants_cores=num_ants_cores, + ) - if reg_tool == 'ants': + if reg_tool == "ants": apply_xfm.inputs.inputspec.interpolation = cfg.registration_workflows[ - 'functional_registration']['func_registration_to_template'][ - 'ANTs_pipelines']['interpolation'] - elif reg_tool == 'fsl': + "functional_registration" + ]["func_registration_to_template"]["ANTs_pipelines"]["interpolation"] + elif reg_tool == "fsl": apply_xfm.inputs.inputspec.interpolation = cfg.registration_workflows[ - 'functional_registration']['func_registration_to_template'][ - 'FNIRT_pipelines']['interpolation'] + "functional_registration" + ]["func_registration_to_template"]["FNIRT_pipelines"]["interpolation"] - connect = strat_pool.get_data("desc-preproc_bold") + strat_pool.get_data("desc-preproc_bold") - if opt == 'default': - node, out = strat_pool.get_data('desc-preproc_bold') - out_label = 'desc-preproc_bold' - elif opt == 'single_step_resampling_from_stc': - node, out = strat_pool.get_data('desc-stc_bold') - out_label = 'desc-stc_bold' - elif opt == 'abcd': - node, out = strat_pool.get_data('bold') - out_label = 'bold' + if opt == "default": + node, out = strat_pool.get_data("desc-preproc_bold") + out_label = "desc-preproc_bold" + elif opt == "single_step_resampling_from_stc": + node, out = strat_pool.get_data("desc-stc_bold") + out_label = "desc-stc_bold" + elif opt == "abcd": + node, out = strat_pool.get_data("bold") + out_label = "bold" - wf.connect(node, out, apply_xfm, 'inputspec.input_image') + wf.connect(node, out, apply_xfm, "inputspec.input_image") node, out = strat_pool.get_data("sbref") - wf.connect(node, out, apply_xfm, 'inputspec.reference') + wf.connect(node, out, apply_xfm, "inputspec.reference") - wf.connect(blip_node, blip_out, apply_xfm, 'inputspec.transform') + wf.connect(blip_node, blip_out, apply_xfm, "inputspec.transform") - outputs = { - out_label: (apply_xfm, 'outputspec.output_image') - } + outputs = {out_label: (apply_xfm, "outputspec.output_image")} return (wf, outputs) @@ -3526,42 +3852,41 @@ def apply_blip_to_timeseries_separately(wf, cfg, strat_pool, pipe_num, outputs={"space-template_desc-head_T1w": {"Template": "T1w-template"}}, ) def warp_wholeheadT1_to_template(wf, cfg, strat_pool, pipe_num, opt=None): - - xfm_prov = strat_pool.get_cpac_provenance( - 'from-T1w_to-template_mode-image_xfm') + xfm_prov = strat_pool.get_cpac_provenance("from-T1w_to-template_mode-image_xfm") reg_tool = check_prov_for_regtool(xfm_prov) - num_cpus = cfg.pipeline_setup['system_config'][ - 'max_cores_per_participant'] + num_cpus = cfg.pipeline_setup["system_config"]["max_cores_per_participant"] - num_ants_cores = cfg.pipeline_setup['system_config']['num_ants_threads'] + num_ants_cores = cfg.pipeline_setup["system_config"]["num_ants_threads"] - apply_xfm = apply_transform(f'warp_wholehead_T1w_to_T1template_{pipe_num}', - reg_tool, time_series=False, num_cpus=num_cpus, - num_ants_cores=num_ants_cores) + apply_xfm = apply_transform( + f"warp_wholehead_T1w_to_T1template_{pipe_num}", + reg_tool, + time_series=False, + num_cpus=num_cpus, + num_ants_cores=num_ants_cores, + ) - if reg_tool == 'ants': + if reg_tool == "ants": apply_xfm.inputs.inputspec.interpolation = cfg.registration_workflows[ - 'functional_registration']['func_registration_to_template'][ - 'ANTs_pipelines']['interpolation'] - elif reg_tool == 'fsl': + "functional_registration" + ]["func_registration_to_template"]["ANTs_pipelines"]["interpolation"] + elif reg_tool == "fsl": apply_xfm.inputs.inputspec.interpolation = cfg.registration_workflows[ - 'functional_registration']['func_registration_to_template'][ - 'FNIRT_pipelines']['interpolation'] + "functional_registration" + ]["func_registration_to_template"]["FNIRT_pipelines"]["interpolation"] connect = strat_pool.get_data("desc-head_T1w") node, out = connect - wf.connect(node, out, apply_xfm, 'inputspec.input_image') + wf.connect(node, out, apply_xfm, "inputspec.input_image") node, out = strat_pool.get_data("T1w-template") - wf.connect(node, out, apply_xfm, 'inputspec.reference') + wf.connect(node, out, apply_xfm, "inputspec.reference") node, out = strat_pool.get_data("from-T1w_to-template_mode-image_xfm") - wf.connect(node, out, apply_xfm, 'inputspec.transform') + wf.connect(node, out, apply_xfm, "inputspec.transform") - outputs = { - 'space-template_desc-head_T1w': (apply_xfm, 'outputspec.output_image') - } + outputs = {"space-template_desc-head_T1w": (apply_xfm, "outputspec.output_image")} return (wf, outputs) @@ -3580,22 +3905,23 @@ def warp_wholeheadT1_to_template(wf, cfg, strat_pool, pipe_num, opt=None): outputs={"space-template_desc-brain_mask": {"Template": "T1w-template"}}, ) def warp_T1mask_to_template(wf, cfg, strat_pool, pipe_num, opt=None): - - xfm_prov = strat_pool.get_cpac_provenance( - 'from-T1w_to-template_mode-image_xfm') + xfm_prov = strat_pool.get_cpac_provenance("from-T1w_to-template_mode-image_xfm") reg_tool = check_prov_for_regtool(xfm_prov) - num_cpus = cfg.pipeline_setup['system_config'][ - 'max_cores_per_participant'] + num_cpus = cfg.pipeline_setup["system_config"]["max_cores_per_participant"] - num_ants_cores = cfg.pipeline_setup['system_config']['num_ants_threads'] + num_ants_cores = cfg.pipeline_setup["system_config"]["num_ants_threads"] - apply_xfm = apply_transform(f'warp_T1mask_to_T1template_{pipe_num}', - reg_tool, time_series=False, num_cpus=num_cpus, - num_ants_cores=num_ants_cores) + apply_xfm = apply_transform( + f"warp_T1mask_to_T1template_{pipe_num}", + reg_tool, + time_series=False, + num_cpus=num_cpus, + num_ants_cores=num_ants_cores, + ) apply_xfm.inputs.inputspec.interpolation = "NearestNeighbor" - ''' + """ if reg_tool == 'ants': apply_xfm.inputs.inputspec.interpolation = cfg.registration_workflows[ 'functional_registration']['func_registration_to_template'][ @@ -3604,20 +3930,18 @@ def warp_T1mask_to_template(wf, cfg, strat_pool, pipe_num, opt=None): apply_xfm.inputs.inputspec.interpolation = cfg.registration_workflows[ 'functional_registration']['func_registration_to_template'][ 'FNIRT_pipelines']['interpolation'] - ''' + """ connect = strat_pool.get_data("space-T1w_desc-brain_mask") node, out = connect - wf.connect(node, out, apply_xfm, 'inputspec.input_image') + wf.connect(node, out, apply_xfm, "inputspec.input_image") node, out = strat_pool.get_data("T1w-template") - wf.connect(node, out, apply_xfm, 'inputspec.reference') + wf.connect(node, out, apply_xfm, "inputspec.reference") node, out = strat_pool.get_data("from-T1w_to-template_mode-image_xfm") - wf.connect(node, out, apply_xfm, 'inputspec.transform') + wf.connect(node, out, apply_xfm, "inputspec.transform") - outputs = { - 'space-template_desc-brain_mask': (apply_xfm, 'outputspec.output_image') - } + outputs = {"space-template_desc-brain_mask": (apply_xfm, "outputspec.output_image")} return (wf, outputs) @@ -3637,46 +3961,46 @@ def warp_T1mask_to_template(wf, cfg, strat_pool, pipe_num, opt=None): "T1w-brain-template-funcreg", ], outputs={ - "space-template_desc-preproc_bold": { - "Template": "T1w-brain-template-funcreg"} + "space-template_desc-preproc_bold": {"Template": "T1w-brain-template-funcreg"} }, ) def warp_timeseries_to_T1template(wf, cfg, strat_pool, pipe_num, opt=None): - - xfm_prov = strat_pool.get_cpac_provenance( - 'from-bold_to-template_mode-image_xfm') + xfm_prov = strat_pool.get_cpac_provenance("from-bold_to-template_mode-image_xfm") reg_tool = check_prov_for_regtool(xfm_prov) - num_cpus = cfg.pipeline_setup['system_config'][ - 'max_cores_per_participant'] + num_cpus = cfg.pipeline_setup["system_config"]["max_cores_per_participant"] - num_ants_cores = cfg.pipeline_setup['system_config']['num_ants_threads'] + num_ants_cores = cfg.pipeline_setup["system_config"]["num_ants_threads"] - apply_xfm = apply_transform(f'warp_ts_to_T1template_{pipe_num}', reg_tool, - time_series=True, num_cpus=num_cpus, - num_ants_cores=num_ants_cores) + apply_xfm = apply_transform( + f"warp_ts_to_T1template_{pipe_num}", + reg_tool, + time_series=True, + num_cpus=num_cpus, + num_ants_cores=num_ants_cores, + ) - if reg_tool == 'ants': + if reg_tool == "ants": apply_xfm.inputs.inputspec.interpolation = cfg.registration_workflows[ - 'functional_registration']['func_registration_to_template'][ - 'ANTs_pipelines']['interpolation'] - elif reg_tool == 'fsl': + "functional_registration" + ]["func_registration_to_template"]["ANTs_pipelines"]["interpolation"] + elif reg_tool == "fsl": apply_xfm.inputs.inputspec.interpolation = cfg.registration_workflows[ - 'functional_registration']['func_registration_to_template'][ - 'FNIRT_pipelines']['interpolation'] + "functional_registration" + ]["func_registration_to_template"]["FNIRT_pipelines"]["interpolation"] connect = strat_pool.get_data("desc-preproc_bold") node, out = connect - wf.connect(node, out, apply_xfm, 'inputspec.input_image') + wf.connect(node, out, apply_xfm, "inputspec.input_image") node, out = strat_pool.get_data("T1w-brain-template-funcreg") - wf.connect(node, out, apply_xfm, 'inputspec.reference') + wf.connect(node, out, apply_xfm, "inputspec.reference") node, out = strat_pool.get_data("from-bold_to-template_mode-image_xfm") - wf.connect(node, out, apply_xfm, 'inputspec.transform') + wf.connect(node, out, apply_xfm, "inputspec.transform") outputs = { - 'space-template_desc-preproc_bold': (apply_xfm, 'outputspec.output_image') + "space-template_desc-preproc_bold": (apply_xfm, "outputspec.output_image") } return (wf, outputs) @@ -3702,43 +4026,46 @@ def warp_timeseries_to_T1template(wf, cfg, strat_pool, pipe_num, opt=None): } }, ) -def warp_timeseries_to_T1template_deriv(wf, cfg, strat_pool, pipe_num, - opt=None): - xfm_prov = strat_pool.get_cpac_provenance( - 'from-bold_to-template_mode-image_xfm') +def warp_timeseries_to_T1template_deriv(wf, cfg, strat_pool, pipe_num, opt=None): + xfm_prov = strat_pool.get_cpac_provenance("from-bold_to-template_mode-image_xfm") reg_tool = check_prov_for_regtool(xfm_prov) - num_cpus = cfg.pipeline_setup['system_config'][ - 'max_cores_per_participant'] + num_cpus = cfg.pipeline_setup["system_config"]["max_cores_per_participant"] - num_ants_cores = cfg.pipeline_setup['system_config']['num_ants_threads'] + num_ants_cores = cfg.pipeline_setup["system_config"]["num_ants_threads"] - apply_xfm = apply_transform(f'warp_ts_to_T1template_{pipe_num}', reg_tool, - time_series=True, num_cpus=num_cpus, - num_ants_cores=num_ants_cores) + apply_xfm = apply_transform( + f"warp_ts_to_T1template_{pipe_num}", + reg_tool, + time_series=True, + num_cpus=num_cpus, + num_ants_cores=num_ants_cores, + ) - if reg_tool == 'ants': + if reg_tool == "ants": apply_xfm.inputs.inputspec.interpolation = cfg.registration_workflows[ - 'functional_registration']['func_registration_to_template'][ - 'ANTs_pipelines']['interpolation'] - elif reg_tool == 'fsl': + "functional_registration" + ]["func_registration_to_template"]["ANTs_pipelines"]["interpolation"] + elif reg_tool == "fsl": apply_xfm.inputs.inputspec.interpolation = cfg.registration_workflows[ - 'functional_registration']['func_registration_to_template'][ - 'FNIRT_pipelines']['interpolation'] + "functional_registration" + ]["func_registration_to_template"]["FNIRT_pipelines"]["interpolation"] connect = strat_pool.get_data("desc-preproc_bold") node, out = connect - wf.connect(node, out, apply_xfm, 'inputspec.input_image') + wf.connect(node, out, apply_xfm, "inputspec.input_image") node, out = strat_pool.get_data("T1w-brain-template-deriv") - wf.connect(node, out, apply_xfm, 'inputspec.reference') + wf.connect(node, out, apply_xfm, "inputspec.reference") node, out = strat_pool.get_data("from-bold_to-template_mode-image_xfm") - wf.connect(node, out, apply_xfm, 'inputspec.transform') + wf.connect(node, out, apply_xfm, "inputspec.transform") outputs = { - 'space-template_res-derivative_desc-preproc_bold': - (apply_xfm, 'outputspec.output_image') + "space-template_res-derivative_desc-preproc_bold": ( + apply_xfm, + "outputspec.output_image", + ) } return (wf, outputs) @@ -3755,8 +4082,7 @@ def warp_timeseries_to_T1template_deriv(wf, cfg, strat_pool, pipe_num, option_key=["apply_transform", "using"], option_val="abcd", inputs=[ - ("desc-preproc_bold", "bold", "motion-basefile", - "coordinate-transformation"), + ("desc-preproc_bold", "bold", "motion-basefile", "coordinate-transformation"), "from-T1w_to-template_mode-image_xfm", "from-bold_to-T1w_mode-image_desc-linear_xfm", "from-bold_to-template_mode-image_xfm", @@ -3767,270 +4093,289 @@ def warp_timeseries_to_T1template_deriv(wf, cfg, strat_pool, pipe_num, "T1w-brain-template-funcreg", ], outputs={ - "space-template_desc-preproc_bold": { - "Template": "T1w-brain-template-funcreg"}, - "space-template_desc-scout_bold": { - "Template": "T1w-brain-template-funcreg"}, - "space-template_desc-head_bold": { - "Template": "T1w-brain-template-funcreg"}, + "space-template_desc-preproc_bold": {"Template": "T1w-brain-template-funcreg"}, + "space-template_desc-scout_bold": {"Template": "T1w-brain-template-funcreg"}, + "space-template_desc-head_bold": {"Template": "T1w-brain-template-funcreg"}, }, ) -def warp_timeseries_to_T1template_abcd(wf, cfg, strat_pool, pipe_num, opt=None - ): +def warp_timeseries_to_T1template_abcd(wf, cfg, strat_pool, pipe_num, opt=None): # Apply motion correction, coreg, anat-to-template transforms on raw functional timeseries using ABCD-style registration # Ref: https://github.com/DCAN-Labs/DCAN-HCP/blob/master/fMRIVolume/scripts/OneStepResampling.sh#L168-L197 # https://github.com/DCAN-Labs/DCAN-HCP/blob/master/fMRIVolume/scripts/DistortionCorrectionAndEPIToT1wReg_FLIRTBBRAndFreeSurferBBRbased.sh#L548 # convertwarp --relout --rel -m ${WD}/fMRI2str.mat --ref=${T1wImage} --out=${WD}/fMRI2str.nii.gz - convert_func_to_anat_linear_warp = pe.Node(interface=fsl.ConvertWarp(), - name=f'convert_func_to_anat_linear_warp_{pipe_num}') + convert_func_to_anat_linear_warp = pe.Node( + interface=fsl.ConvertWarp(), name=f"convert_func_to_anat_linear_warp_{pipe_num}" + ) convert_func_to_anat_linear_warp.inputs.out_relwarp = True convert_func_to_anat_linear_warp.inputs.relwarp = True - - node, out = strat_pool.get_data('desc-preproc_T1w') - wf.connect(node, out, convert_func_to_anat_linear_warp, 'reference') - - if strat_pool.check_rpool('fsl-blip-warp'): - node, out = strat_pool.get_data('from-bold_to-T1w_mode-image_desc-linear_xfm') - wf.connect(node, out, convert_func_to_anat_linear_warp, 'postmat') - - node, out = strat_pool.get_data('fsl-blip-warp') - wf.connect(node, out, convert_func_to_anat_linear_warp, 'warp1') + + node, out = strat_pool.get_data("desc-preproc_T1w") + wf.connect(node, out, convert_func_to_anat_linear_warp, "reference") + + if strat_pool.check_rpool("fsl-blip-warp"): + node, out = strat_pool.get_data("from-bold_to-T1w_mode-image_desc-linear_xfm") + wf.connect(node, out, convert_func_to_anat_linear_warp, "postmat") + + node, out = strat_pool.get_data("fsl-blip-warp") + wf.connect(node, out, convert_func_to_anat_linear_warp, "warp1") else: - node, out = strat_pool.get_data('from-bold_to-T1w_mode-image_desc-linear_xfm') - wf.connect(node, out, convert_func_to_anat_linear_warp, 'premat') + node, out = strat_pool.get_data("from-bold_to-T1w_mode-image_desc-linear_xfm") + wf.connect(node, out, convert_func_to_anat_linear_warp, "premat") # https://github.com/DCAN-Labs/DCAN-HCP/blob/master/fMRIVolume/scripts/OneStepResampling.sh#L140 # convertwarp --relout --rel --warp1=${fMRIToStructuralInput} --warp2=${StructuralToStandard} --ref=${WD}/${T1wImageFile}.${FinalfMRIResolution} --out=${OutputTransform} - convert_func_to_standard_warp = pe.Node(interface=fsl.ConvertWarp(), - name=f'convert_func_to_standard_warp_{pipe_num}') + convert_func_to_standard_warp = pe.Node( + interface=fsl.ConvertWarp(), name=f"convert_func_to_standard_warp_{pipe_num}" + ) convert_func_to_standard_warp.inputs.out_relwarp = True convert_func_to_standard_warp.inputs.relwarp = True - wf.connect(convert_func_to_anat_linear_warp, 'out_file', - convert_func_to_standard_warp, 'warp1') + wf.connect( + convert_func_to_anat_linear_warp, + "out_file", + convert_func_to_standard_warp, + "warp1", + ) - node, out = strat_pool.get_data('from-T1w_to-template_mode-image_xfm') - wf.connect(node, out, convert_func_to_standard_warp, 'warp2') + node, out = strat_pool.get_data("from-T1w_to-template_mode-image_xfm") + wf.connect(node, out, convert_func_to_standard_warp, "warp2") - node, out = strat_pool.get_data('space-template_res-bold_desc-brain_T1w') - wf.connect(node, out, convert_func_to_standard_warp, 'reference') + node, out = strat_pool.get_data("space-template_res-bold_desc-brain_T1w") + wf.connect(node, out, convert_func_to_standard_warp, "reference") # TODO add condition: if no gradient distortion # https://github.com/DCAN-Labs/DCAN-HCP/blob/master/fMRIVolume/GenericfMRIVolumeProcessingPipeline.sh#L283-L284 # fslroi "$fMRIFolder"/"$NameOffMRI"_gdc "$fMRIFolder"/"$NameOffMRI"_gdc_warp 0 3 - extract_func_roi = pe.Node(interface=fsl.ExtractROI(), - name=f'extract_func_roi_{pipe_num}') + extract_func_roi = pe.Node( + interface=fsl.ExtractROI(), name=f"extract_func_roi_{pipe_num}" + ) extract_func_roi.inputs.t_min = 0 extract_func_roi.inputs.t_size = 3 - node, out = strat_pool.get_data('bold') - wf.connect(node, out, extract_func_roi, 'in_file') + node, out = strat_pool.get_data("bold") + wf.connect(node, out, extract_func_roi, "in_file") # fslmaths "$fMRIFolder"/"$NameOffMRI"_gdc_warp -mul 0 "$fMRIFolder"/"$NameOffMRI"_gdc_warp - multiply_func_roi_by_zero = pe.Node(interface=fsl.maths.MathsCommand(), - name=f'multiply_func_roi_by_zero_{pipe_num}') + multiply_func_roi_by_zero = pe.Node( + interface=fsl.maths.MathsCommand(), name=f"multiply_func_roi_by_zero_{pipe_num}" + ) - multiply_func_roi_by_zero.inputs.args = '-mul 0' + multiply_func_roi_by_zero.inputs.args = "-mul 0" - wf.connect(extract_func_roi, 'roi_file', - multiply_func_roi_by_zero, 'in_file') + wf.connect(extract_func_roi, "roi_file", multiply_func_roi_by_zero, "in_file") # https://github.com/DCAN-Labs/DCAN-HCP/blob/master/fMRIVolume/scripts/OneStepResampling.sh#L168-L193 # fslsplit ${InputfMRI} ${WD}/prevols/vol -t - split_func = pe.Node(interface=fsl.Split(), - name=f'split_func_{pipe_num}') + split_func = pe.Node(interface=fsl.Split(), name=f"split_func_{pipe_num}") - split_func.inputs.dimension = 't' + split_func.inputs.dimension = "t" - node, out = strat_pool.get_data('bold') - wf.connect(node, out, split_func, 'in_file') + node, out = strat_pool.get_data("bold") + wf.connect(node, out, split_func, "in_file") ### Loop starts! ### # convertwarp --relout --rel --ref=${WD}/prevols/vol${vnum}.nii.gz --warp1=${GradientDistortionField} --postmat=${MotionMatrixFolder}/${MotionMatrixPrefix}${vnum} --out=${MotionMatrixFolder}/${MotionMatrixPrefix}${vnum}_gdc_warp.nii.gz - convert_motion_distortion_warp = pe.MapNode(interface=fsl.ConvertWarp(), - name=f'convert_motion_distortion_warp_{pipe_num}', - iterfield=['reference', 'postmat']) + convert_motion_distortion_warp = pe.MapNode( + interface=fsl.ConvertWarp(), + name=f"convert_motion_distortion_warp_{pipe_num}", + iterfield=["reference", "postmat"], + ) convert_motion_distortion_warp.inputs.out_relwarp = True convert_motion_distortion_warp.inputs.relwarp = True - wf.connect(multiply_func_roi_by_zero, 'out_file', - convert_motion_distortion_warp, 'warp1') + wf.connect( + multiply_func_roi_by_zero, "out_file", convert_motion_distortion_warp, "warp1" + ) - wf.connect(split_func, 'out_files', - convert_motion_distortion_warp, 'reference') + wf.connect(split_func, "out_files", convert_motion_distortion_warp, "reference") - node, out = strat_pool.get_data('coordinate-transformation') - wf.connect(node, out, convert_motion_distortion_warp, 'postmat') + node, out = strat_pool.get_data("coordinate-transformation") + wf.connect(node, out, convert_motion_distortion_warp, "postmat") # convertwarp --relout --rel --ref=${WD}/${T1wImageFile}.${FinalfMRIResolution} --warp1=${MotionMatrixFolder}/${MotionMatrixPrefix}${vnum}_gdc_warp.nii.gz --warp2=${OutputTransform} --out=${MotionMatrixFolder}/${MotionMatrixPrefix}${vnum}_all_warp.nii.gz - convert_registration_warp = pe.MapNode(interface=fsl.ConvertWarp(), - name=f'convert_registration_warp_{pipe_num}', - iterfield=['warp1']) + convert_registration_warp = pe.MapNode( + interface=fsl.ConvertWarp(), + name=f"convert_registration_warp_{pipe_num}", + iterfield=["warp1"], + ) convert_registration_warp.inputs.out_relwarp = True convert_registration_warp.inputs.relwarp = True - node, out = strat_pool.get_data('space-template_res-bold_desc-brain_T1w') - wf.connect(node, out, convert_registration_warp, 'reference') + node, out = strat_pool.get_data("space-template_res-bold_desc-brain_T1w") + wf.connect(node, out, convert_registration_warp, "reference") - wf.connect(convert_motion_distortion_warp, 'out_file', - convert_registration_warp, 'warp1') + wf.connect( + convert_motion_distortion_warp, "out_file", convert_registration_warp, "warp1" + ) - wf.connect(convert_func_to_standard_warp, 'out_file', - convert_registration_warp, 'warp2') + wf.connect( + convert_func_to_standard_warp, "out_file", convert_registration_warp, "warp2" + ) # fslmaths ${WD}/prevols/vol${vnum}.nii.gz -mul 0 -add 1 ${WD}/prevols/vol${vnum}_mask.nii.gz - generate_vol_mask = pe.MapNode(interface=fsl.maths.MathsCommand(), - name=f'generate_mask_{pipe_num}', - iterfield=['in_file']) + generate_vol_mask = pe.MapNode( + interface=fsl.maths.MathsCommand(), + name=f"generate_mask_{pipe_num}", + iterfield=["in_file"], + ) - generate_vol_mask.inputs.args = '-mul 0 -add 1' + generate_vol_mask.inputs.args = "-mul 0 -add 1" - wf.connect(split_func, 'out_files', - generate_vol_mask, 'in_file') + wf.connect(split_func, "out_files", generate_vol_mask, "in_file") # applywarp --rel --interp=spline --in=${WD}/prevols/vol${vnum}.nii.gz --warp=${MotionMatrixFolder}/${MotionMatrixPrefix}${vnum}_all_warp.nii.gz --ref=${WD}/${T1wImageFile}.${FinalfMRIResolution} --out=${WD}/postvols/vol${vnum}.nii.gz - applywarp_func_to_standard = pe.MapNode(interface=fsl.ApplyWarp(), - name=f'applywarp_func_to_standard_{pipe_num}', - iterfield=['in_file', 'field_file']) + applywarp_func_to_standard = pe.MapNode( + interface=fsl.ApplyWarp(), + name=f"applywarp_func_to_standard_{pipe_num}", + iterfield=["in_file", "field_file"], + ) applywarp_func_to_standard.inputs.relwarp = True - applywarp_func_to_standard.inputs.interp = 'spline' + applywarp_func_to_standard.inputs.interp = "spline" - wf.connect(split_func, 'out_files', - applywarp_func_to_standard, 'in_file') + wf.connect(split_func, "out_files", applywarp_func_to_standard, "in_file") - wf.connect(convert_registration_warp, 'out_file', - applywarp_func_to_standard, 'field_file') + wf.connect( + convert_registration_warp, "out_file", applywarp_func_to_standard, "field_file" + ) - node, out = strat_pool.get_data('space-template_res-bold_desc-brain_T1w') - wf.connect(node, out, - applywarp_func_to_standard, 'ref_file') + node, out = strat_pool.get_data("space-template_res-bold_desc-brain_T1w") + wf.connect(node, out, applywarp_func_to_standard, "ref_file") # applywarp --rel --interp=nn --in=${WD}/prevols/vol${vnum}_mask.nii.gz --warp=${MotionMatrixFolder}/${MotionMatrixPrefix}${vnum}_all_warp.nii.gz --ref=${WD}/${T1wImageFile}.${FinalfMRIResolution} --out=${WD}/postvols/vol${vnum}_mask.nii.gz - applywarp_func_mask_to_standard = pe.MapNode(interface=fsl.ApplyWarp(), - name=f'applywarp_func_mask_to_standard_{pipe_num}', - iterfield=['in_file', 'field_file']) + applywarp_func_mask_to_standard = pe.MapNode( + interface=fsl.ApplyWarp(), + name=f"applywarp_func_mask_to_standard_{pipe_num}", + iterfield=["in_file", "field_file"], + ) applywarp_func_mask_to_standard.inputs.relwarp = True - applywarp_func_mask_to_standard.inputs.interp = 'nn' + applywarp_func_mask_to_standard.inputs.interp = "nn" - wf.connect(generate_vol_mask, 'out_file', - applywarp_func_mask_to_standard, 'in_file') + wf.connect( + generate_vol_mask, "out_file", applywarp_func_mask_to_standard, "in_file" + ) - wf.connect(convert_registration_warp, 'out_file', - applywarp_func_mask_to_standard, 'field_file') + wf.connect( + convert_registration_warp, + "out_file", + applywarp_func_mask_to_standard, + "field_file", + ) - node, out = strat_pool.get_data('space-template_res-bold_desc-brain_T1w') - wf.connect(node, out, - applywarp_func_mask_to_standard, 'ref_file') + node, out = strat_pool.get_data("space-template_res-bold_desc-brain_T1w") + wf.connect(node, out, applywarp_func_mask_to_standard, "ref_file") ### Loop ends! ### # fslmerge -tr ${OutputfMRI} $FrameMergeSTRING $TR_vol - merge_func_to_standard = pe.Node(interface=fslMerge(), - name=f'merge_func_to_standard_{pipe_num}') + merge_func_to_standard = pe.Node( + interface=fslMerge(), name=f"merge_func_to_standard_{pipe_num}" + ) - merge_func_to_standard.inputs.dimension = 't' + merge_func_to_standard.inputs.dimension = "t" - wf.connect(applywarp_func_to_standard, 'out_file', - merge_func_to_standard, 'in_files') + wf.connect( + applywarp_func_to_standard, "out_file", merge_func_to_standard, "in_files" + ) # fslmerge -tr ${OutputfMRI}_mask $FrameMergeSTRINGII $TR_vol - merge_func_mask_to_standard = pe.Node(interface=fslMerge(), - name='merge_func_mask_to_' - f'standard_{pipe_num}') + merge_func_mask_to_standard = pe.Node( + interface=fslMerge(), name="merge_func_mask_to_" f"standard_{pipe_num}" + ) - merge_func_mask_to_standard.inputs.dimension = 't' + merge_func_mask_to_standard.inputs.dimension = "t" - wf.connect(applywarp_func_mask_to_standard, 'out_file', - merge_func_mask_to_standard, 'in_files') + wf.connect( + applywarp_func_mask_to_standard, + "out_file", + merge_func_mask_to_standard, + "in_files", + ) # fslmaths ${OutputfMRI}_mask -Tmin ${OutputfMRI}_mask - find_min_mask = pe.Node(interface=fsl.maths.MathsCommand(), - name=f'find_min_mask_{pipe_num}') + find_min_mask = pe.Node( + interface=fsl.maths.MathsCommand(), name=f"find_min_mask_{pipe_num}" + ) - find_min_mask.inputs.args = '-Tmin' + find_min_mask.inputs.args = "-Tmin" - wf.connect(merge_func_mask_to_standard, 'merged_file', - find_min_mask, 'in_file') + wf.connect(merge_func_mask_to_standard, "merged_file", find_min_mask, "in_file") # Combine transformations: gradient non-linearity distortion + fMRI_dc to standard # convertwarp --relout --rel --ref=${WD}/${T1wImageFile}.${FinalfMRIResolution} --warp1=${GradientDistortionField} --warp2=${OutputTransform} --out=${WD}/Scout_gdc_MNI_warp.nii.gz - convert_dc_warp = pe.Node(interface=fsl.ConvertWarp(), - name=f'convert_dc_warp_{pipe_num}') + convert_dc_warp = pe.Node( + interface=fsl.ConvertWarp(), name=f"convert_dc_warp_{pipe_num}" + ) convert_dc_warp.inputs.out_relwarp = True convert_dc_warp.inputs.relwarp = True - node, out = strat_pool.get_data('space-template_res-bold_desc-brain_T1w') - wf.connect(node, out, convert_dc_warp, 'reference') + node, out = strat_pool.get_data("space-template_res-bold_desc-brain_T1w") + wf.connect(node, out, convert_dc_warp, "reference") - wf.connect(multiply_func_roi_by_zero, 'out_file', - convert_dc_warp, 'warp1') + wf.connect(multiply_func_roi_by_zero, "out_file", convert_dc_warp, "warp1") - wf.connect(convert_func_to_standard_warp, 'out_file', - convert_dc_warp, 'warp2') + wf.connect(convert_func_to_standard_warp, "out_file", convert_dc_warp, "warp2") # applywarp --rel --interp=spline --in=${ScoutInput} -w ${WD}/Scout_gdc_MNI_warp.nii.gz -r ${WD}/${T1wImageFile}.${FinalfMRIResolution} -o ${ScoutOutput} - applywarp_scout = pe.Node(interface=fsl.ApplyWarp(), - name=f'applywarp_scout_input_{pipe_num}') + applywarp_scout = pe.Node( + interface=fsl.ApplyWarp(), name=f"applywarp_scout_input_{pipe_num}" + ) applywarp_scout.inputs.relwarp = True - applywarp_scout.inputs.interp = 'spline' + applywarp_scout.inputs.interp = "spline" - node, out = strat_pool.get_data('motion-basefile') - wf.connect(node, out, applywarp_scout, 'in_file') + node, out = strat_pool.get_data("motion-basefile") + wf.connect(node, out, applywarp_scout, "in_file") - node, out = strat_pool.get_data('space-template_res-bold_desc-brain_T1w') - wf.connect(node, out, applywarp_scout, 'ref_file') + node, out = strat_pool.get_data("space-template_res-bold_desc-brain_T1w") + wf.connect(node, out, applywarp_scout, "ref_file") - wf.connect(convert_dc_warp, 'out_file', applywarp_scout, 'field_file') + wf.connect(convert_dc_warp, "out_file", applywarp_scout, "field_file") # https://github.com/DCAN-Labs/DCAN-HCP/blob/master/fMRIVolume/scripts/IntensityNormalization.sh#L124-L127 # fslmaths ${InputfMRI} -mas ${BrainMask} -mas ${InputfMRI}_mask -thr 0 -ing 10000 ${OutputfMRI} -odt float - merge_func_mask = pe.Node(util.Merge(2), - name=f'merge_func_mask_{pipe_num}') + merge_func_mask = pe.Node(util.Merge(2), name=f"merge_func_mask_{pipe_num}") - node, out = strat_pool.get_data('space-template_desc-bold_mask') - wf.connect(node, out, merge_func_mask, 'in1') + node, out = strat_pool.get_data("space-template_desc-bold_mask") + wf.connect(node, out, merge_func_mask, "in1") - wf.connect(find_min_mask, 'out_file', merge_func_mask, 'in2') + wf.connect(find_min_mask, "out_file", merge_func_mask, "in2") - extract_func_brain = pe.Node(interface=fsl.MultiImageMaths(), - name=f'extract_func_brain_{pipe_num}') + extract_func_brain = pe.Node( + interface=fsl.MultiImageMaths(), name=f"extract_func_brain_{pipe_num}" + ) - extract_func_brain.inputs.op_string = '-mas %s -mas %s -thr 0 -ing 10000' - extract_func_brain.inputs.output_datatype = 'float' + extract_func_brain.inputs.op_string = "-mas %s -mas %s -thr 0 -ing 10000" + extract_func_brain.inputs.output_datatype = "float" - wf.connect(merge_func_to_standard, 'merged_file', - extract_func_brain, 'in_file') + wf.connect(merge_func_to_standard, "merged_file", extract_func_brain, "in_file") - wf.connect(merge_func_mask, 'out', - extract_func_brain, 'operand_files') + wf.connect(merge_func_mask, "out", extract_func_brain, "operand_files") # fslmaths ${ScoutInput} -mas ${BrainMask} -mas ${InputfMRI}_mask -thr 0 -ing 10000 ${ScoutOutput} -odt float - extract_scout_brain = pe.Node(interface=fsl.MultiImageMaths(), - name=f'extract_scout_brain_{pipe_num}') + extract_scout_brain = pe.Node( + interface=fsl.MultiImageMaths(), name=f"extract_scout_brain_{pipe_num}" + ) - extract_scout_brain.inputs.op_string = '-mas %s -mas %s -thr 0 -ing 10000' - extract_scout_brain.inputs.output_datatype = 'float' + extract_scout_brain.inputs.op_string = "-mas %s -mas %s -thr 0 -ing 10000" + extract_scout_brain.inputs.output_datatype = "float" - wf.connect(applywarp_scout, 'out_file', - extract_scout_brain, 'in_file') + wf.connect(applywarp_scout, "out_file", extract_scout_brain, "in_file") - wf.connect(merge_func_mask, 'out', - extract_scout_brain, 'operand_files') + wf.connect(merge_func_mask, "out", extract_scout_brain, "operand_files") outputs = { - 'space-template_desc-preproc_bold': (extract_func_brain, 'out_file'), - 'space-template_desc-scout_bold': (extract_scout_brain, 'out_file'), - 'space-template_desc-head_bold': (merge_func_to_standard, 'merged_file') + "space-template_desc-preproc_bold": (extract_func_brain, "out_file"), + "space-template_desc-scout_bold": (extract_scout_brain, "out_file"), + "space-template_desc-head_bold": (merge_func_to_standard, "merged_file"), } return (wf, outputs) @@ -4063,265 +4408,302 @@ def warp_timeseries_to_T1template_abcd(wf, cfg, strat_pool, pipe_num, opt=None "space-template_desc-bold_mask": {"Template": "T1w-template"}, }, ) -def warp_timeseries_to_T1template_dcan_nhp(wf, cfg, strat_pool, pipe_num, - opt=None): +def warp_timeseries_to_T1template_dcan_nhp(wf, cfg, strat_pool, pipe_num, opt=None): # Apply motion correction, coreg, anat-to-template transforms on raw functional timeseries # Ref: https://github.com/DCAN-Labs/dcan-macaque-pipeline/blob/master/fMRIVolume/scripts/OneStepResampling.sh # https://github.com/DCAN-Labs/dcan-macaque-pipeline/blob/master/fMRIVolume/scripts/OneStepResampling.sh#L131 # ${FSLDIR}/bin/flirt -interp spline -in ${T1wImage} -ref ${T1wImage} -applyisoxfm $FinalfMRIResolution -out ${WD}/${T1wImageFile}.${FinalfMRIResolution} - anat_resample = pe.Node(interface=fsl.FLIRT(), - name=f'anat_resample_func_res_{pipe_num}' - ) - anat_resample.inputs.apply_isoxfm = float(cfg.registration_workflows['functional_registration']['func_registration_to_template']['output_resolution']['func_preproc_outputs'].replace("mm", "")) - anat_resample.inputs.interp = 'spline' + anat_resample = pe.Node( + interface=fsl.FLIRT(), name=f"anat_resample_func_res_{pipe_num}" + ) + anat_resample.inputs.apply_isoxfm = float( + cfg.registration_workflows["functional_registration"][ + "func_registration_to_template" + ]["output_resolution"]["func_preproc_outputs"].replace("mm", "") + ) + anat_resample.inputs.interp = "spline" - node, out = strat_pool.get_data('space-template_desc-head_T1w') - wf.connect(node, out, anat_resample, 'in_file') - wf.connect(node, out, anat_resample, 'reference') + node, out = strat_pool.get_data("space-template_desc-head_T1w") + wf.connect(node, out, anat_resample, "in_file") + wf.connect(node, out, anat_resample, "reference") # ${FSLDIR}/bin/applywarp --rel --interp=spline -i ${T1wImage} -r ${ResampRefIm} --premat=$FSLDIR/etc/flirtsch/ident.mat -o ${WD}/${T1wImageFile}.${FinalfMRIResolution} - applywarp_anat_res = pe.Node(interface=fsl.ApplyWarp(), - name=f'anat_func_res_{pipe_num}') + applywarp_anat_res = pe.Node( + interface=fsl.ApplyWarp(), name=f"anat_func_res_{pipe_num}" + ) applywarp_anat_res.inputs.relwarp = True - applywarp_anat_res.inputs.interp = 'spline' - applywarp_anat_res.inputs.premat = cfg.registration_workflows['anatomical_registration']['registration']['FSL-FNIRT']['identity_matrix'] + applywarp_anat_res.inputs.interp = "spline" + applywarp_anat_res.inputs.premat = cfg.registration_workflows[ + "anatomical_registration" + ]["registration"]["FSL-FNIRT"]["identity_matrix"] - node, out = strat_pool.get_data('space-template_desc-head_T1w') - wf.connect(node, out, applywarp_anat_res, 'in_file') - wf.connect(anat_resample, 'out_file', applywarp_anat_res, 'ref_file') + node, out = strat_pool.get_data("space-template_desc-head_T1w") + wf.connect(node, out, applywarp_anat_res, "in_file") + wf.connect(anat_resample, "out_file", applywarp_anat_res, "ref_file") # https://github.com/DCAN-Labs/dcan-macaque-pipeline/blob/master/fMRIVolume/scripts/OneStepResampling.sh#L136-L138 # Create brain masks in this space (changing resolution) # ${FSLDIR}/bin/applywarp --rel --interp=nn -i ${FreeSurferBrainMask}.nii.gz -r ${WD}/${T1wImageFile}.${FinalfMRIResolution} --premat=$FSLDIR/etc/flirtsch/ident.mat -o ${WD}/${FreeSurferBrainMaskFile}.${FinalfMRIResolution}.nii.gz - applywarp_anat_mask_res = pe.Node(interface=fsl.ApplyWarp(), - name=f'anat_mask_func_res_{pipe_num}') + applywarp_anat_mask_res = pe.Node( + interface=fsl.ApplyWarp(), name=f"anat_mask_func_res_{pipe_num}" + ) applywarp_anat_mask_res.inputs.relwarp = True - applywarp_anat_mask_res.inputs.interp = 'nn' - applywarp_anat_mask_res.inputs.premat = cfg.registration_workflows['anatomical_registration']['registration']['FSL-FNIRT']['identity_matrix'] + applywarp_anat_mask_res.inputs.interp = "nn" + applywarp_anat_mask_res.inputs.premat = cfg.registration_workflows[ + "anatomical_registration" + ]["registration"]["FSL-FNIRT"]["identity_matrix"] - node, out = strat_pool.get_data('space-template_desc-T1w_mask') - wf.connect(node, out, applywarp_anat_mask_res, 'in_file') - wf.connect(applywarp_anat_res, 'out_file', applywarp_anat_mask_res, 'ref_file') + node, out = strat_pool.get_data("space-template_desc-T1w_mask") + wf.connect(node, out, applywarp_anat_mask_res, "in_file") + wf.connect(applywarp_anat_res, "out_file", applywarp_anat_mask_res, "ref_file") # ${FSLDIR}/bin/fslmaths ${WD}/${T1wImageFile}.${FinalfMRIResolution} -mas ${WD}/${FreeSurferBrainMaskFile}.${FinalfMRIResolution}.nii.gz ${WD}/${FreeSurferBrainMaskFile}.${FinalfMRIResolution}.nii.gz - T1_brain_res = pe.Node(interface=fsl.MultiImageMaths(), - name=f't1_brain_func_res_{pipe_num}') + T1_brain_res = pe.Node( + interface=fsl.MultiImageMaths(), name=f"t1_brain_func_res_{pipe_num}" + ) T1_brain_res.inputs.op_string = "-mas %s " - wf.connect(applywarp_anat_res, 'out_file', T1_brain_res, 'in_file') - wf.connect(applywarp_anat_mask_res, 'out_file', T1_brain_res, 'operand_files') + wf.connect(applywarp_anat_res, "out_file", T1_brain_res, "in_file") + wf.connect(applywarp_anat_mask_res, "out_file", T1_brain_res, "operand_files") # Create versions of the biasfield (changing resolution) # ${FSLDIR}/bin/applywarp --rel --interp=spline -i ${BiasField} -r ${WD}/${FreeSurferBrainMaskFile}.${FinalfMRIResolution}.nii.gz --premat=$FSLDIR/etc/flirtsch/ident.mat -o ${WD}/${BiasFieldFile}.${FinalfMRIResolution} - applywarp_bias_field_res = pe.Node(interface=fsl.ApplyWarp(), - name=f'biasfiled_func_res_{pipe_num}') + applywarp_bias_field_res = pe.Node( + interface=fsl.ApplyWarp(), name=f"biasfiled_func_res_{pipe_num}" + ) applywarp_bias_field_res.inputs.relwarp = True - applywarp_bias_field_res.inputs.interp = 'spline' - applywarp_bias_field_res.inputs.premat = cfg.registration_workflows['anatomical_registration']['registration']['FSL-FNIRT']['identity_matrix'] + applywarp_bias_field_res.inputs.interp = "spline" + applywarp_bias_field_res.inputs.premat = cfg.registration_workflows[ + "anatomical_registration" + ]["registration"]["FSL-FNIRT"]["identity_matrix"] - node, out = strat_pool.get_data('space-template_desc-T1wT2w_biasfield') - wf.connect(node, out, applywarp_bias_field_res, 'in_file') - wf.connect(T1_brain_res, 'out_file', applywarp_bias_field_res, 'ref_file') + node, out = strat_pool.get_data("space-template_desc-T1wT2w_biasfield") + wf.connect(node, out, applywarp_bias_field_res, "in_file") + wf.connect(T1_brain_res, "out_file", applywarp_bias_field_res, "ref_file") # ${FSLDIR}/bin/fslmaths ${WD}/${BiasFieldFile}.${FinalfMRIResolution} -thr 0.1 ${WD}/${BiasFieldFile}.${FinalfMRIResolution} - biasfield_thr = pe.Node(interface=fsl.MultiImageMaths(), - name=f'biasfiedl_thr_{pipe_num}') + biasfield_thr = pe.Node( + interface=fsl.MultiImageMaths(), name=f"biasfiedl_thr_{pipe_num}" + ) biasfield_thr.inputs.op_string = "-thr 0.1" - wf.connect(applywarp_bias_field_res, 'out_file', biasfield_thr, 'in_file') + wf.connect(applywarp_bias_field_res, "out_file", biasfield_thr, "in_file") # https://github.com/DCAN-Labs/dcan-macaque-pipeline/blob/master/fMRIVolume/scripts/OneStepResampling.sh#L144-L146 # convertwarp --relout --rel --warp1=${fMRIToStructuralInput} --warp2=${StructuralToStandard} --ref=${WD}/${T1wImageFile}.${FinalfMRIResolution} --out=${OutputTransform} - convert_func_to_standard_warp = pe.Node(interface=fsl.ConvertWarp(), - name=f'convert_func_to_standard_warp_{pipe_num}') + convert_func_to_standard_warp = pe.Node( + interface=fsl.ConvertWarp(), name=f"convert_func_to_standard_warp_{pipe_num}" + ) convert_func_to_standard_warp.inputs.out_relwarp = True convert_func_to_standard_warp.inputs.relwarp = True - node, out = strat_pool.get_data('from-bold_to-T1w_mode-image_desc-linear_warp') - wf.connect(node, out, convert_func_to_standard_warp, 'warp1') + node, out = strat_pool.get_data("from-bold_to-T1w_mode-image_desc-linear_warp") + wf.connect(node, out, convert_func_to_standard_warp, "warp1") - node, out = strat_pool.get_data('from-T1w_to-template_mode-image_warp') - wf.connect(node, out, convert_func_to_standard_warp, 'warp2') + node, out = strat_pool.get_data("from-T1w_to-template_mode-image_warp") + wf.connect(node, out, convert_func_to_standard_warp, "warp2") - wf.connect(applywarp_anat_res, 'out_file', convert_func_to_standard_warp, 'reference') + wf.connect( + applywarp_anat_res, "out_file", convert_func_to_standard_warp, "reference" + ) # https://github.com/DCAN-Labs/dcan-macaque-pipeline/blob/master/fMRIVolume/GenericfMRIVolumeProcessingPipeline.sh#L157-L158 # fslroi "$fMRIFolder"/"$NameOffMRI"_gdc "$fMRIFolder"/"$NameOffMRI"_gdc_warp 0 3 - extract_func_roi = pe.Node(interface=fsl.ExtractROI(), - name=f'extract_func_roi_{pipe_num}') + extract_func_roi = pe.Node( + interface=fsl.ExtractROI(), name=f"extract_func_roi_{pipe_num}" + ) extract_func_roi.inputs.t_min = 0 extract_func_roi.inputs.t_size = 3 - node, out = strat_pool.get_data(['desc-reorient_bold', 'bold']) - wf.connect(node, out, extract_func_roi, 'in_file') + node, out = strat_pool.get_data(["desc-reorient_bold", "bold"]) + wf.connect(node, out, extract_func_roi, "in_file") # fslmaths "$fMRIFolder"/"$NameOffMRI"_gdc_warp -mul 0 "$fMRIFolder"/"$NameOffMRI"_gdc_warp - multiply_func_roi_by_zero = pe.Node(interface=fsl.maths.MathsCommand(), - name=f'multiply_func_roi_by_zero_{pipe_num}') + multiply_func_roi_by_zero = pe.Node( + interface=fsl.maths.MathsCommand(), name=f"multiply_func_roi_by_zero_{pipe_num}" + ) - multiply_func_roi_by_zero.inputs.args = '-mul 0' + multiply_func_roi_by_zero.inputs.args = "-mul 0" - wf.connect(extract_func_roi, 'roi_file', - multiply_func_roi_by_zero, 'in_file') + wf.connect(extract_func_roi, "roi_file", multiply_func_roi_by_zero, "in_file") # https://github.com/DCAN-Labs/dcan-macaque-pipeline/blob/master/fMRIVolume/scripts/OneStepResampling.sh#L173 # fslsplit ${InputfMRI} ${WD}/prevols/vol -t - split_func = pe.Node(interface=fsl.Split(), - name=f'split_func_{pipe_num}') + split_func = pe.Node(interface=fsl.Split(), name=f"split_func_{pipe_num}") - split_func.inputs.dimension = 't' + split_func.inputs.dimension = "t" - node, out = strat_pool.get_data(['desc-reorient_bold', 'bold']) - wf.connect(node, out, split_func, 'in_file') + node, out = strat_pool.get_data(["desc-reorient_bold", "bold"]) + wf.connect(node, out, split_func, "in_file") ### Loop starts! ### # convertwarp --relout --rel --ref=${WD}/prevols/vol${vnum}.nii.gz --warp1=${GradientDistortionField} --postmat=${MotionMatrixFolder}/${MotionMatrixPrefix}${vnum} --out=${MotionMatrixFolder}/${MotionMatrixPrefix}${vnum}_gdc_warp.nii.gz - convert_motion_distortion_warp = pe.MapNode(interface=fsl.ConvertWarp(), - name=f'convert_motion_distortion_warp_{pipe_num}', - iterfield=['reference', 'postmat']) + convert_motion_distortion_warp = pe.MapNode( + interface=fsl.ConvertWarp(), + name=f"convert_motion_distortion_warp_{pipe_num}", + iterfield=["reference", "postmat"], + ) convert_motion_distortion_warp.inputs.out_relwarp = True convert_motion_distortion_warp.inputs.relwarp = True - wf.connect(multiply_func_roi_by_zero, 'out_file', - convert_motion_distortion_warp, 'warp1') + wf.connect( + multiply_func_roi_by_zero, "out_file", convert_motion_distortion_warp, "warp1" + ) - wf.connect(split_func, 'out_files', - convert_motion_distortion_warp, 'reference') + wf.connect(split_func, "out_files", convert_motion_distortion_warp, "reference") - node, out = strat_pool.get_data('coordinate-transformation') - wf.connect(node, out, convert_motion_distortion_warp, 'postmat') + node, out = strat_pool.get_data("coordinate-transformation") + wf.connect(node, out, convert_motion_distortion_warp, "postmat") # convertwarp --relout --rel --ref=${WD}/${T1wImageFile}.${FinalfMRIResolution} --warp1=${MotionMatrixFolder}/${MotionMatrixPrefix}${vnum}_gdc_warp.nii.gz --warp2=${OutputTransform} --out=${MotionMatrixFolder}/${MotionMatrixPrefix}${vnum}_all_warp.nii.gz - convert_registration_warp = pe.MapNode(interface=fsl.ConvertWarp(), - name=f'convert_registration_warp_{pipe_num}', - iterfield=['warp1']) + convert_registration_warp = pe.MapNode( + interface=fsl.ConvertWarp(), + name=f"convert_registration_warp_{pipe_num}", + iterfield=["warp1"], + ) convert_registration_warp.inputs.out_relwarp = True convert_registration_warp.inputs.relwarp = True - wf.connect(applywarp_anat_res, 'out_file', convert_registration_warp, 'reference') + wf.connect(applywarp_anat_res, "out_file", convert_registration_warp, "reference") - wf.connect(convert_motion_distortion_warp, 'out_file', - convert_registration_warp, 'warp1') + wf.connect( + convert_motion_distortion_warp, "out_file", convert_registration_warp, "warp1" + ) - wf.connect(convert_func_to_standard_warp, 'out_file', - convert_registration_warp, 'warp2') + wf.connect( + convert_func_to_standard_warp, "out_file", convert_registration_warp, "warp2" + ) # fslmaths ${WD}/prevols/vol${vnum}.nii.gz -mul 0 -add 1 ${WD}/prevols/vol${vnum}_mask.nii.gz - generate_vol_mask = pe.MapNode(interface=fsl.maths.MathsCommand(), - name=f'generate_mask_{pipe_num}', - iterfield=['in_file']) + generate_vol_mask = pe.MapNode( + interface=fsl.maths.MathsCommand(), + name=f"generate_mask_{pipe_num}", + iterfield=["in_file"], + ) - generate_vol_mask.inputs.args = '-mul 0 -add 1' + generate_vol_mask.inputs.args = "-mul 0 -add 1" - wf.connect(split_func, 'out_files', - generate_vol_mask, 'in_file') + wf.connect(split_func, "out_files", generate_vol_mask, "in_file") # applywarp --rel --interp=spline --in=${WD}/prevols/vol${vnum}.nii.gz --warp=${MotionMatrixFolder}/${MotionMatrixPrefix}${vnum}_all_warp.nii.gz --ref=${WD}/${T1wImageFile}.${FinalfMRIResolution} --out=${WD}/postvols/vol${vnum}.nii.gz - applywarp_func_to_standard = pe.MapNode(interface=fsl.ApplyWarp(), - name=f'applywarp_func_to_standard_{pipe_num}', - iterfield=['in_file', 'field_file']) + applywarp_func_to_standard = pe.MapNode( + interface=fsl.ApplyWarp(), + name=f"applywarp_func_to_standard_{pipe_num}", + iterfield=["in_file", "field_file"], + ) applywarp_func_to_standard.inputs.relwarp = True - applywarp_func_to_standard.inputs.interp = 'spline' + applywarp_func_to_standard.inputs.interp = "spline" - wf.connect(split_func, 'out_files', - applywarp_func_to_standard, 'in_file') + wf.connect(split_func, "out_files", applywarp_func_to_standard, "in_file") - wf.connect(convert_registration_warp, 'out_file', - applywarp_func_to_standard, 'field_file') + wf.connect( + convert_registration_warp, "out_file", applywarp_func_to_standard, "field_file" + ) - wf.connect(applywarp_anat_res, 'out_file', - applywarp_func_to_standard, 'ref_file') + wf.connect(applywarp_anat_res, "out_file", applywarp_func_to_standard, "ref_file") # applywarp --rel --interp=nn --in=${WD}/prevols/vol${vnum}_mask.nii.gz --warp=${MotionMatrixFolder}/${MotionMatrixPrefix}${vnum}_all_warp.nii.gz --ref=${WD}/${T1wImageFile}.${FinalfMRIResolution} --out=${WD}/postvols/vol${vnum}_mask.nii.gz - applywarp_func_mask_to_standard = pe.MapNode(interface=fsl.ApplyWarp(), - name=f'applywarp_func_mask_to_standard_{pipe_num}', - iterfield=['in_file', 'field_file']) + applywarp_func_mask_to_standard = pe.MapNode( + interface=fsl.ApplyWarp(), + name=f"applywarp_func_mask_to_standard_{pipe_num}", + iterfield=["in_file", "field_file"], + ) applywarp_func_mask_to_standard.inputs.relwarp = True - applywarp_func_mask_to_standard.inputs.interp = 'nn' + applywarp_func_mask_to_standard.inputs.interp = "nn" - wf.connect(generate_vol_mask, 'out_file', - applywarp_func_mask_to_standard, 'in_file') + wf.connect( + generate_vol_mask, "out_file", applywarp_func_mask_to_standard, "in_file" + ) - wf.connect(convert_registration_warp, 'out_file', - applywarp_func_mask_to_standard, 'field_file') + wf.connect( + convert_registration_warp, + "out_file", + applywarp_func_mask_to_standard, + "field_file", + ) - wf.connect(applywarp_anat_res, 'out_file', - applywarp_func_mask_to_standard, 'ref_file') + wf.connect( + applywarp_anat_res, "out_file", applywarp_func_mask_to_standard, "ref_file" + ) ### Loop ends! ### # fslmerge -tr ${OutputfMRI} $FrameMergeSTRING $TR_vol - merge_func_to_standard = pe.Node(interface=fslMerge(), - name=f'merge_func_to_standard_{pipe_num}') + merge_func_to_standard = pe.Node( + interface=fslMerge(), name=f"merge_func_to_standard_{pipe_num}" + ) - merge_func_to_standard.inputs.dimension = 't' + merge_func_to_standard.inputs.dimension = "t" - wf.connect(applywarp_func_to_standard, 'out_file', - merge_func_to_standard, 'in_files') + wf.connect( + applywarp_func_to_standard, "out_file", merge_func_to_standard, "in_files" + ) # fslmerge -tr ${OutputfMRI}_mask $FrameMergeSTRINGII $TR_vol - merge_func_mask_to_standard = pe.Node(interface=fslMerge(), - name='merge_func_mask_to_' - f'standard_{pipe_num}') + merge_func_mask_to_standard = pe.Node( + interface=fslMerge(), name="merge_func_mask_to_" f"standard_{pipe_num}" + ) - merge_func_mask_to_standard.inputs.dimension = 't' + merge_func_mask_to_standard.inputs.dimension = "t" - wf.connect(applywarp_func_mask_to_standard, 'out_file', - merge_func_mask_to_standard, 'in_files') + wf.connect( + applywarp_func_mask_to_standard, + "out_file", + merge_func_mask_to_standard, + "in_files", + ) # fslmaths ${OutputfMRI}_mask -Tmin ${OutputfMRI}_mask - find_min_mask = pe.Node(interface=fsl.maths.MathsCommand(), - name=f'find_min_mask_{pipe_num}') + find_min_mask = pe.Node( + interface=fsl.maths.MathsCommand(), name=f"find_min_mask_{pipe_num}" + ) - find_min_mask.inputs.args = '-Tmin' + find_min_mask.inputs.args = "-Tmin" - wf.connect(merge_func_mask_to_standard, 'merged_file', - find_min_mask, 'in_file') + wf.connect(merge_func_mask_to_standard, "merged_file", find_min_mask, "in_file") # https://github.com/DCAN-Labs/dcan-macaque-pipeline/blob/master/fMRIVolume/scripts/IntensityNormalization.sh#L113-L119 # fslmaths ${InputfMRI} -div ${BiasField} $jacobiancom -mas ${BrainMask} -mas ${InputfMRI}_mask -ing 10000 ${OutputfMRI} -odt float - merge_func_mask = pe.Node(util.Merge(3), - name=f'merge_operand_files_{pipe_num}') + merge_func_mask = pe.Node(util.Merge(3), name=f"merge_operand_files_{pipe_num}") - wf.connect(biasfield_thr, 'out_file', merge_func_mask, 'in1') + wf.connect(biasfield_thr, "out_file", merge_func_mask, "in1") - wf.connect(applywarp_anat_mask_res, 'out_file', merge_func_mask, 'in2') + wf.connect(applywarp_anat_mask_res, "out_file", merge_func_mask, "in2") - wf.connect(find_min_mask, 'out_file', merge_func_mask, 'in3') + wf.connect(find_min_mask, "out_file", merge_func_mask, "in3") + extract_func_brain = pe.Node( + interface=fsl.MultiImageMaths(), name=f"extract_func_brain_{pipe_num}" + ) - extract_func_brain = pe.Node(interface=fsl.MultiImageMaths(), - name=f'extract_func_brain_{pipe_num}') - - extract_func_brain.inputs.op_string = '-div %s -mas %s -mas %s -ing 10000' - extract_func_brain.inputs.output_datatype = 'float' + extract_func_brain.inputs.op_string = "-div %s -mas %s -mas %s -ing 10000" + extract_func_brain.inputs.output_datatype = "float" - wf.connect(merge_func_to_standard, 'merged_file', - extract_func_brain, 'in_file') + wf.connect(merge_func_to_standard, "merged_file", extract_func_brain, "in_file") - wf.connect(merge_func_mask, 'out', - extract_func_brain, 'operand_files') + wf.connect(merge_func_mask, "out", extract_func_brain, "operand_files") - func_mask_final = pe.Node(interface=fsl.MultiImageMaths(), - name=f'func_mask_final_{pipe_num}') + func_mask_final = pe.Node( + interface=fsl.MultiImageMaths(), name=f"func_mask_final_{pipe_num}" + ) func_mask_final.inputs.op_string = "-mas %s " - wf.connect(applywarp_anat_mask_res, 'out_file', func_mask_final, 'in_file') + wf.connect(applywarp_anat_mask_res, "out_file", func_mask_final, "in_file") - wf.connect(find_min_mask, 'out_file', func_mask_final, 'operand_files') + wf.connect(find_min_mask, "out_file", func_mask_final, "operand_files") outputs = { - 'space-template_desc-preproc_bold': (extract_func_brain, 'out_file'), - 'space-template_desc-bold_mask': (func_mask_final, 'out_file') + "space-template_desc-preproc_bold": (extract_func_brain, "out_file"), + "space-template_desc-bold_mask": (func_mask_final, "out_file"), } return (wf, outputs) @@ -4356,14 +4738,10 @@ def warp_timeseries_to_T1template_dcan_nhp(wf, cfg, strat_pool, pipe_num, ) ], outputs={ - "space-template_desc-preproc_bold": { - "Template": "T1w-brain-template-funcreg"}, - "space-template_desc-brain_bold": { - "Template": "T1w-brain-template-funcreg"}, - "space-template_desc-bold_mask": { - "Template": "T1w-brain-template-funcreg"}, - "space-template_desc-head_bold": { - "Template": "T1w-brain-template-funcreg"}, + "space-template_desc-preproc_bold": {"Template": "T1w-brain-template-funcreg"}, + "space-template_desc-brain_bold": {"Template": "T1w-brain-template-funcreg"}, + "space-template_desc-bold_mask": {"Template": "T1w-brain-template-funcreg"}, + "space-template_desc-head_bold": {"Template": "T1w-brain-template-funcreg"}, "space-template_res-derivative_desc-preproc_bold": { "Template": "T1w-brain-template-deriv" }, @@ -4372,12 +4750,13 @@ def warp_timeseries_to_T1template_dcan_nhp(wf, cfg, strat_pool, pipe_num, }, }, ) -def single_step_resample_timeseries_to_T1template(wf, cfg, strat_pool, - pipe_num, opt=None): - ''' +def single_step_resample_timeseries_to_T1template( + wf, cfg, strat_pool, pipe_num, opt=None +): + """ Apply motion correction, coreg, anat-to-template transforms on slice-time corrected functional timeseries based on fMRIPrep - pipeline + pipeline. Copyright (c) 2015-2018, the CRN developers team. All rights reserved. @@ -4411,213 +4790,238 @@ def single_step_resample_timeseries_to_T1template(wf, cfg, strat_pool, OF THE POSSIBILITY OF SUCH DAMAGE. Ref: https://github.com/nipreps/fmriprep/blob/84a6005b/fmriprep/workflows/bold/resampling.py#L159-L419 - ''' # noqa: 501 - xfm_prov = strat_pool.get_cpac_provenance( - 'from-T1w_to-template_mode-image_xfm') + """ + xfm_prov = strat_pool.get_cpac_provenance("from-T1w_to-template_mode-image_xfm") reg_tool = check_prov_for_regtool(xfm_prov) - bbr2itk = pe.Node(util.Function(input_names=['reference_file', - 'source_file', - 'transform_file'], - output_names=['itk_transform'], - function=run_c3d), - name=f'convert_bbr2itk_{pipe_num}') - - if cfg.registration_workflows['functional_registration'][ - 'coregistration']['boundary_based_registration'][ - 'reference'] == 'whole-head': - node, out = strat_pool.get_data('T1w') - wf.connect(node, out, bbr2itk, 'reference_file') + bbr2itk = pe.Node( + util.Function( + input_names=["reference_file", "source_file", "transform_file"], + output_names=["itk_transform"], + function=run_c3d, + ), + name=f"convert_bbr2itk_{pipe_num}", + ) - elif cfg.registration_workflows['functional_registration'][ - 'coregistration']['boundary_based_registration'][ - 'reference'] == 'brain': - node, out = strat_pool.get_data('desc-preproc_T1w') - wf.connect(node, out, bbr2itk, 'reference_file') + if ( + cfg.registration_workflows["functional_registration"]["coregistration"][ + "boundary_based_registration" + ]["reference"] + == "whole-head" + ): + node, out = strat_pool.get_data("T1w") + wf.connect(node, out, bbr2itk, "reference_file") + + elif ( + cfg.registration_workflows["functional_registration"]["coregistration"][ + "boundary_based_registration" + ]["reference"] + == "brain" + ): + node, out = strat_pool.get_data("desc-preproc_T1w") + wf.connect(node, out, bbr2itk, "reference_file") - node, out = strat_pool.get_data('sbref') - wf.connect(node, out, bbr2itk, 'source_file') + node, out = strat_pool.get_data("sbref") + wf.connect(node, out, bbr2itk, "source_file") - node, out = strat_pool.get_data( - 'from-bold_to-T1w_mode-image_desc-linear_xfm') - wf.connect(node, out, bbr2itk, 'transform_file') + node, out = strat_pool.get_data("from-bold_to-T1w_mode-image_desc-linear_xfm") + wf.connect(node, out, bbr2itk, "transform_file") - split_func = pe.Node(interface=fsl.Split(), - name=f'split_func_{pipe_num}') + split_func = pe.Node(interface=fsl.Split(), name=f"split_func_{pipe_num}") - split_func.inputs.dimension = 't' + split_func.inputs.dimension = "t" - node, out = strat_pool.get_data('desc-stc_bold') - wf.connect(node, out, split_func, 'in_file') + node, out = strat_pool.get_data("desc-stc_bold") + wf.connect(node, out, split_func, "in_file") ### Loop starts! ### - motionxfm2itk = pe.MapNode(util.Function( - input_names=['reference_file', - 'source_file', - 'transform_file'], - output_names=['itk_transform'], - function=run_c3d), - name=f'convert_motionxfm2itk_{pipe_num}', - iterfield=['transform_file']) - - node, out = strat_pool.get_data('motion-basefile') - wf.connect(node, out, motionxfm2itk, 'reference_file') - wf.connect(node, out, motionxfm2itk, 'source_file') - - node, out = strat_pool.get_data('coordinate-transformation') + motionxfm2itk = pe.MapNode( + util.Function( + input_names=["reference_file", "source_file", "transform_file"], + output_names=["itk_transform"], + function=run_c3d, + ), + name=f"convert_motionxfm2itk_{pipe_num}", + iterfield=["transform_file"], + ) + + node, out = strat_pool.get_data("motion-basefile") + wf.connect(node, out, motionxfm2itk, "reference_file") + wf.connect(node, out, motionxfm2itk, "source_file") + + node, out = strat_pool.get_data("coordinate-transformation") motion_correct_tool = check_prov_for_motion_tool( - strat_pool.get_cpac_provenance('coordinate-transformation')) - if motion_correct_tool == 'mcflirt': - wf.connect(node, out, motionxfm2itk, 'transform_file') - elif motion_correct_tool == '3dvolreg': - convert_transform = pe.Node(util.Function( - input_names=['one_d_filename'], - output_names=['transform_directory'], - function=one_d_to_mat, - imports=['import os', 'import numpy as np']), - name=f'convert_transform_{pipe_num}') - wf.connect(node, out, convert_transform, 'one_d_filename') - wf.connect(convert_transform, 'transform_directory', - motionxfm2itk, 'transform_file') + strat_pool.get_cpac_provenance("coordinate-transformation") + ) + if motion_correct_tool == "mcflirt": + wf.connect(node, out, motionxfm2itk, "transform_file") + elif motion_correct_tool == "3dvolreg": + convert_transform = pe.Node( + util.Function( + input_names=["one_d_filename"], + output_names=["transform_directory"], + function=one_d_to_mat, + imports=["import os", "import numpy as np"], + ), + name=f"convert_transform_{pipe_num}", + ) + wf.connect(node, out, convert_transform, "one_d_filename") + wf.connect( + convert_transform, "transform_directory", motionxfm2itk, "transform_file" + ) merge_num = 4 blip = False - if strat_pool.check_rpool('ants-blip-warp') and reg_tool == 'ants': - blip_node, blip_out = strat_pool.get_data('ants-blip-warp') + if strat_pool.check_rpool("ants-blip-warp") and reg_tool == "ants": + blip_node, blip_out = strat_pool.get_data("ants-blip-warp") merge_num = 5 blip = True - elif strat_pool.check_rpool('fsl-blip-warp') and reg_tool == 'fsl': - blip_node, blip_out = strat_pool.get_data('fsl-blip-warp') + elif strat_pool.check_rpool("fsl-blip-warp") and reg_tool == "fsl": + blip_node, blip_out = strat_pool.get_data("fsl-blip-warp") merge_num = 5 blip = True - collectxfm = pe.MapNode(util.Merge(merge_num), - name=f'collectxfm_func_to_standard_{pipe_num}', - iterfield=[f'in{merge_num}']) + collectxfm = pe.MapNode( + util.Merge(merge_num), + name=f"collectxfm_func_to_standard_{pipe_num}", + iterfield=[f"in{merge_num}"], + ) - node, out = strat_pool.get_data('from-T1w_to-template_mode-image_xfm') - wf.connect(node, out, collectxfm, 'in1') + node, out = strat_pool.get_data("from-T1w_to-template_mode-image_xfm") + wf.connect(node, out, collectxfm, "in1") - wf.connect(bbr2itk, 'itk_transform', - collectxfm, 'in2') + wf.connect(bbr2itk, "itk_transform", collectxfm, "in2") - collectxfm.inputs.in3 = 'identity' + collectxfm.inputs.in3 = "identity" if blip: - wf.connect(blip_node, blip_out, collectxfm, 'in4') + wf.connect(blip_node, blip_out, collectxfm, "in4") - wf.connect(motionxfm2itk, 'itk_transform', - collectxfm, f'in{merge_num}') + wf.connect(motionxfm2itk, "itk_transform", collectxfm, f"in{merge_num}") applyxfm_func_to_standard = pe.MapNode( interface=ants.ApplyTransforms(), - name=f'applyxfm_func_to_standard_{pipe_num}', - iterfield=['input_image', 'transforms']) + name=f"applyxfm_func_to_standard_{pipe_num}", + iterfield=["input_image", "transforms"], + ) applyxfm_func_to_standard.inputs.float = True - applyxfm_func_to_standard.inputs.interpolation = 'LanczosWindowedSinc' + applyxfm_func_to_standard.inputs.interpolation = "LanczosWindowedSinc" applyxfm_derivfunc_to_standard = pe.MapNode( interface=ants.ApplyTransforms(), - name=f'applyxfm_derivfunc_to_standard_{pipe_num}', - iterfield=['input_image', 'transforms']) + name=f"applyxfm_derivfunc_to_standard_{pipe_num}", + iterfield=["input_image", "transforms"], + ) applyxfm_derivfunc_to_standard.inputs.float = True - applyxfm_derivfunc_to_standard.inputs.interpolation = 'LanczosWindowedSinc' + applyxfm_derivfunc_to_standard.inputs.interpolation = "LanczosWindowedSinc" + + wf.connect(split_func, "out_files", applyxfm_func_to_standard, "input_image") + wf.connect(split_func, "out_files", applyxfm_derivfunc_to_standard, "input_image") - wf.connect(split_func, 'out_files', - applyxfm_func_to_standard, 'input_image') - wf.connect(split_func, 'out_files', - applyxfm_derivfunc_to_standard, 'input_image') + node, out = strat_pool.get_data("T1w-brain-template-funcreg") + wf.connect(node, out, applyxfm_func_to_standard, "reference_image") - node, out = strat_pool.get_data('T1w-brain-template-funcreg') - wf.connect(node, out, applyxfm_func_to_standard, 'reference_image') - - node, out = strat_pool.get_data('T1w-brain-template-deriv') - wf.connect(node, out, applyxfm_derivfunc_to_standard, 'reference_image') + node, out = strat_pool.get_data("T1w-brain-template-deriv") + wf.connect(node, out, applyxfm_derivfunc_to_standard, "reference_image") - wf.connect(collectxfm, 'out', applyxfm_func_to_standard, 'transforms') - wf.connect(collectxfm, 'out', applyxfm_derivfunc_to_standard, 'transforms') + wf.connect(collectxfm, "out", applyxfm_func_to_standard, "transforms") + wf.connect(collectxfm, "out", applyxfm_derivfunc_to_standard, "transforms") ### Loop ends! ### - merge_func_to_standard = pe.Node(interface=fslMerge(), - name=f'merge_func_to_standard_{pipe_num}') - merge_func_to_standard.inputs.dimension = 't' + merge_func_to_standard = pe.Node( + interface=fslMerge(), name=f"merge_func_to_standard_{pipe_num}" + ) + merge_func_to_standard.inputs.dimension = "t" - wf.connect(applyxfm_func_to_standard, 'output_image', - merge_func_to_standard, 'in_files') + wf.connect( + applyxfm_func_to_standard, "output_image", merge_func_to_standard, "in_files" + ) merge_derivfunc_to_standard = pe.Node( - interface=fslMerge(), name=f'merge_derivfunc_to_standard_{pipe_num}') - merge_derivfunc_to_standard.inputs.dimension = 't' + interface=fslMerge(), name=f"merge_derivfunc_to_standard_{pipe_num}" + ) + merge_derivfunc_to_standard.inputs.dimension = "t" - wf.connect(applyxfm_derivfunc_to_standard, 'output_image', - merge_derivfunc_to_standard, 'in_files') + wf.connect( + applyxfm_derivfunc_to_standard, + "output_image", + merge_derivfunc_to_standard, + "in_files", + ) applyxfm_func_mask_to_standard = pe.Node( interface=ants.ApplyTransforms(), - name=f'applyxfm_func_mask_to_standard_{pipe_num}') - applyxfm_func_mask_to_standard.inputs.interpolation = 'MultiLabel' + name=f"applyxfm_func_mask_to_standard_{pipe_num}", + ) + applyxfm_func_mask_to_standard.inputs.interpolation = "MultiLabel" - node, out = strat_pool.get_data('space-bold_desc-brain_mask') - wf.connect(node, out, applyxfm_func_mask_to_standard, 'input_image') + node, out = strat_pool.get_data("space-bold_desc-brain_mask") + wf.connect(node, out, applyxfm_func_mask_to_standard, "input_image") - node, out = strat_pool.get_data('T1w-brain-template-funcreg') - wf.connect(node, out, applyxfm_func_mask_to_standard, 'reference_image') + node, out = strat_pool.get_data("T1w-brain-template-funcreg") + wf.connect(node, out, applyxfm_func_mask_to_standard, "reference_image") collectxfm_mask = pe.Node( - util.Merge(2), name=f'collectxfm_func_mask_to_standard_{pipe_num}') + util.Merge(2), name=f"collectxfm_func_mask_to_standard_{pipe_num}" + ) - node, out = strat_pool.get_data('from-T1w_to-template_mode-image_xfm') - wf.connect(node, out, collectxfm_mask, 'in1') + node, out = strat_pool.get_data("from-T1w_to-template_mode-image_xfm") + wf.connect(node, out, collectxfm_mask, "in1") - wf.connect(bbr2itk, 'itk_transform', collectxfm_mask, 'in2') + wf.connect(bbr2itk, "itk_transform", collectxfm_mask, "in2") - wf.connect(collectxfm_mask, 'out', - applyxfm_func_mask_to_standard, 'transforms') + wf.connect(collectxfm_mask, "out", applyxfm_func_mask_to_standard, "transforms") applyxfm_deriv_mask_to_standard = pe.Node( interface=ants.ApplyTransforms(), - name=f'applyxfm_deriv_mask_to_standard_{pipe_num}') - applyxfm_deriv_mask_to_standard.inputs.interpolation = 'MultiLabel' + name=f"applyxfm_deriv_mask_to_standard_{pipe_num}", + ) + applyxfm_deriv_mask_to_standard.inputs.interpolation = "MultiLabel" - node, out = strat_pool.get_data('space-bold_desc-brain_mask') - wf.connect(node, out, applyxfm_deriv_mask_to_standard, 'input_image') + node, out = strat_pool.get_data("space-bold_desc-brain_mask") + wf.connect(node, out, applyxfm_deriv_mask_to_standard, "input_image") - node, out = strat_pool.get_data('T1w-brain-template-deriv') - wf.connect(node, out, applyxfm_deriv_mask_to_standard, 'reference_image') + node, out = strat_pool.get_data("T1w-brain-template-deriv") + wf.connect(node, out, applyxfm_deriv_mask_to_standard, "reference_image") collectxfm_deriv_mask = pe.Node( - util.Merge(2), name=f'collectxfm_deriv_mask_to_standard_{pipe_num}') + util.Merge(2), name=f"collectxfm_deriv_mask_to_standard_{pipe_num}" + ) - node, out = strat_pool.get_data('from-T1w_to-template_mode-image_xfm') - wf.connect(node, out, collectxfm_deriv_mask, 'in1') + node, out = strat_pool.get_data("from-T1w_to-template_mode-image_xfm") + wf.connect(node, out, collectxfm_deriv_mask, "in1") - wf.connect(bbr2itk, 'itk_transform', - collectxfm_deriv_mask, 'in2') + wf.connect(bbr2itk, "itk_transform", collectxfm_deriv_mask, "in2") - wf.connect(collectxfm_deriv_mask, 'out', - applyxfm_deriv_mask_to_standard, 'transforms') + wf.connect( + collectxfm_deriv_mask, "out", applyxfm_deriv_mask_to_standard, "transforms" + ) - apply_mask = pe.Node(interface=fsl.maths.ApplyMask(), - name=f'get_func_brain_to_standard_{pipe_num}') + apply_mask = pe.Node( + interface=fsl.maths.ApplyMask(), name=f"get_func_brain_to_standard_{pipe_num}" + ) - wf.connect(merge_func_to_standard, 'merged_file', - apply_mask, 'in_file') + wf.connect(merge_func_to_standard, "merged_file", apply_mask, "in_file") - wf.connect(applyxfm_func_mask_to_standard, 'output_image', - apply_mask, 'mask_file') + wf.connect(applyxfm_func_mask_to_standard, "output_image", apply_mask, "mask_file") outputs = { - 'space-template_desc-head_bold': (merge_func_to_standard, - 'merged_file'), - 'space-template_desc-brain_bold': (apply_mask, 'out_file'), - 'space-template_desc-preproc_bold': (apply_mask, 'out_file'), - 'space-template_desc-bold_mask': (applyxfm_func_mask_to_standard, - 'output_image'), - 'space-template_res-derivative_desc-preproc_bold': - (merge_derivfunc_to_standard, 'merged_file'), - 'space-template_res-derivative_desc-bold_mask': - (applyxfm_deriv_mask_to_standard, 'output_image') + "space-template_desc-head_bold": (merge_func_to_standard, "merged_file"), + "space-template_desc-brain_bold": (apply_mask, "out_file"), + "space-template_desc-preproc_bold": (apply_mask, "out_file"), + "space-template_desc-bold_mask": ( + applyxfm_func_mask_to_standard, + "output_image", + ), + "space-template_res-derivative_desc-preproc_bold": ( + merge_derivfunc_to_standard, + "merged_file", + ), + "space-template_res-derivative_desc-bold_mask": ( + applyxfm_deriv_mask_to_standard, + "output_image", + ), } return (wf, outputs) @@ -4638,18 +5042,24 @@ def single_step_resample_timeseries_to_T1template(wf, cfg, strat_pool, outputs={ "space-template_sbref": { "Description": "Single-volume sbref of the BOLD time-series " - "transformed to template space.", + "transformed to template space.", "Template": "T1w-brain-template-funcreg", } }, ) def warp_sbref_to_T1template(wf, cfg, strat_pool, pipe_num, opt=None): - xfm = 'from-bold_to-template_mode-image_xfm' + xfm = "from-bold_to-template_mode-image_xfm" wf, apply_xfm = warp_resource_to_template( - wf, cfg, strat_pool, pipe_num, 'sbref', xfm, - reference='T1w-brain-template-funcreg', time_series=False)[:2] - outputs = {'space-template_sbref': - (apply_xfm, 'outputspec.output_image')} + wf, + cfg, + strat_pool, + pipe_num, + "sbref", + xfm, + reference="T1w-brain-template-funcreg", + time_series=False, + )[:2] + outputs = {"space-template_sbref": (apply_xfm, "outputspec.output_image")} return _warp_return(wf, apply_xfm, outputs) @@ -4677,15 +5087,22 @@ def warp_sbref_to_T1template(wf, cfg, strat_pool, pipe_num, opt=None): "T1w-brain-template-funcreg", ], outputs={ - "space-template_desc-bold_mask": { - "Template": "T1w-brain-template-funcreg"}}) + "space-template_desc-bold_mask": {"Template": "T1w-brain-template-funcreg"} + }, +) def warp_bold_mask_to_T1template(wf, cfg, strat_pool, pipe_num, opt=None): - xfm = 'from-bold_to-template_mode-image_xfm' + xfm = "from-bold_to-template_mode-image_xfm" wf, apply_xfm = warp_resource_to_template( - wf, cfg, strat_pool, pipe_num, 'space-bold_desc-brain_mask', xfm, - reference='T1w-brain-template-funcreg', time_series=False)[:2] - outputs = {'space-template_desc-bold_mask': - (apply_xfm, 'outputspec.output_image')} + wf, + cfg, + strat_pool, + pipe_num, + "space-bold_desc-brain_mask", + xfm, + reference="T1w-brain-template-funcreg", + time_series=False, + )[:2] + outputs = {"space-template_desc-bold_mask": (apply_xfm, "outputspec.output_image")} return _warp_return(wf, apply_xfm, outputs) @@ -4719,15 +5136,26 @@ def warp_bold_mask_to_T1template(wf, cfg, strat_pool, pipe_num, opt=None): }, ) def warp_deriv_mask_to_T1template(wf, cfg, strat_pool, pipe_num, opt=None): - '''Transform the BOLD mask to template space and to the resolution set for + """Transform the BOLD mask to template space and to the resolution set for the derivative outputs. - ''' - xfm = 'from-bold_to-template_mode-image_xfm' + """ + xfm = "from-bold_to-template_mode-image_xfm" wf, apply_xfm = warp_resource_to_template( - wf, cfg, strat_pool, pipe_num, 'space-bold_desc-brain_mask', xfm, - reference='T1w-brain-template-deriv', time_series=False)[:2] - outputs = {'space-template_res-derivative_desc-bold_mask': - (apply_xfm, 'outputspec.output_image')} + wf, + cfg, + strat_pool, + pipe_num, + "space-bold_desc-brain_mask", + xfm, + reference="T1w-brain-template-deriv", + time_series=False, + )[:2] + outputs = { + "space-template_res-derivative_desc-bold_mask": ( + apply_xfm, + "outputspec.output_image", + ) + } return _warp_return(wf, apply_xfm, outputs) @@ -4746,12 +5174,11 @@ def warp_deriv_mask_to_T1template(wf, cfg, strat_pool, pipe_num, opt=None): outputs={"space-template_desc-preproc_bold": {"Template": "EPI-template"}}, ) def warp_timeseries_to_EPItemplate(wf, cfg, strat_pool, pipe_num, opt=None): - xfm = 'from-bold_to-EPItemplate_mode-image_xfm' + xfm = "from-bold_to-EPItemplate_mode-image_xfm" wf, apply_xfm, resource = warp_resource_to_template( - wf, cfg, strat_pool, pipe_num, 'desc-preproc_bold', xfm, - time_series=True) - outputs = {f'space-template_{resource}': - (apply_xfm, 'outputspec.output_image')} + wf, cfg, strat_pool, pipe_num, "desc-preproc_bold", xfm, time_series=True + ) + outputs = {f"space-template_{resource}": (apply_xfm, "outputspec.output_image")} return _warp_return(wf, apply_xfm, outputs) @@ -4770,12 +5197,11 @@ def warp_timeseries_to_EPItemplate(wf, cfg, strat_pool, pipe_num, opt=None): outputs={"space-template_desc-mean_bold": {"Template": "EPI-template"}}, ) def warp_bold_mean_to_EPItemplate(wf, cfg, strat_pool, pipe_num, opt=None): - xfm = 'from-bold_to-EPItemplate_mode-image_xfm' + xfm = "from-bold_to-EPItemplate_mode-image_xfm" wf, apply_xfm = warp_resource_to_template( - wf, cfg, strat_pool, pipe_num, 'desc-mean_bold', xfm, - time_series=False)[:2] - outputs = {'space-template_desc-mean_bold': - (apply_xfm, 'outputspec.output_image')} + wf, cfg, strat_pool, pipe_num, "desc-mean_bold", xfm, time_series=False + )[:2] + outputs = {"space-template_desc-mean_bold": (apply_xfm, "outputspec.output_image")} return _warp_return(wf, apply_xfm, outputs) @@ -4788,19 +5214,23 @@ def warp_bold_mean_to_EPItemplate(wf, cfg, strat_pool, pipe_num, opt=None): ], switch=["run_EPI"], inputs=[ - ("space-bold_desc-brain_mask", - "from-bold_to-EPItemplate_mode-image_xfm"), + ("space-bold_desc-brain_mask", "from-bold_to-EPItemplate_mode-image_xfm"), "EPI-template", ], outputs={"space-template_desc-bold_mask": {"Template": "EPI-template"}}, ) def warp_bold_mask_to_EPItemplate(wf, cfg, strat_pool, pipe_num, opt=None): - xfm = 'from-bold_to-EPItemplate_mode-image_xfm' + xfm = "from-bold_to-EPItemplate_mode-image_xfm" wf, apply_xfm = warp_resource_to_template( - wf, cfg, strat_pool, pipe_num, 'space-bold_desc-brain_mask', xfm, - time_series=False)[:2] - outputs = {'space-template_desc-bold_mask': - (apply_xfm, 'outputspec.output_image')} + wf, + cfg, + strat_pool, + pipe_num, + "space-bold_desc-brain_mask", + xfm, + time_series=False, + )[:2] + outputs = {"space-template_desc-bold_mask": (apply_xfm, "outputspec.output_image")} return _warp_return(wf, apply_xfm, outputs) @@ -4813,25 +5243,33 @@ def warp_bold_mask_to_EPItemplate(wf, cfg, strat_pool, pipe_num, opt=None): ], switch=["run_EPI"], inputs=[ - ("space-bold_desc-brain_mask", - "from-bold_to-EPItemplate_mode-image_xfm"), + ("space-bold_desc-brain_mask", "from-bold_to-EPItemplate_mode-image_xfm"), "EPI-template", ], outputs={ - "space-template_res-derivative_desc-bold_mask": { - "Template": "EPI-template"} + "space-template_res-derivative_desc-bold_mask": {"Template": "EPI-template"} }, ) def warp_deriv_mask_to_EPItemplate(wf, cfg, strat_pool, pipe_num, opt=None): - '''Transform the BOLD mask to template space and to the resolution set for + """Transform the BOLD mask to template space and to the resolution set for the derivative outputs. - ''' - xfm = 'from-bold_to-EPItemplate_mode-image_xfm' + """ + xfm = "from-bold_to-EPItemplate_mode-image_xfm" wf, apply_xfm = warp_resource_to_template( - wf, cfg, strat_pool, pipe_num, 'space-bold_desc-brain_mask', xfm, - time_series=False)[:2] - outputs = {'space-template_res-derivative_desc-bold_mask': - (apply_xfm, 'outputspec.output_image')} + wf, + cfg, + strat_pool, + pipe_num, + "space-bold_desc-brain_mask", + xfm, + time_series=False, + )[:2] + outputs = { + "space-template_res-derivative_desc-bold_mask": ( + apply_xfm, + "outputspec.output_image", + ) + } return _warp_return(wf, apply_xfm, outputs) @@ -4854,9 +5292,14 @@ def warp_deriv_mask_to_EPItemplate(wf, cfg, strat_pool, pipe_num, opt=None): }, ) def warp_tissuemask_to_T1template(wf, cfg, strat_pool, pipe_num, opt=None): - return warp_tissuemask_to_template(wf, cfg, strat_pool, pipe_num, - xfm='from-T1w_to-template_mode-image_' - 'xfm', template_space='T1') + return warp_tissuemask_to_template( + wf, + cfg, + strat_pool, + pipe_num, + xfm="from-T1w_to-template_mode-image_" "xfm", + template_space="T1", + ) @nodeblock( @@ -4883,15 +5326,18 @@ def warp_tissuemask_to_T1template(wf, cfg, strat_pool, pipe_num, opt=None): }, ) def warp_tissuemask_to_EPItemplate(wf, cfg, strat_pool, pipe_num, opt=None): - return warp_tissuemask_to_template(wf, cfg, strat_pool, pipe_num, - xfm='from-bold_to-EPItemplate_' - 'mode-image_xfm', - template_space='EPI') + return warp_tissuemask_to_template( + wf, + cfg, + strat_pool, + pipe_num, + xfm="from-bold_to-EPItemplate_" "mode-image_xfm", + template_space="EPI", + ) -def warp_tissuemask_to_template(wf, cfg, strat_pool, pipe_num, xfm, - template_space): - '''Function to apply transforms to tissue masks +def warp_tissuemask_to_template(wf, cfg, strat_pool, pipe_num, xfm, template_space): + """Function to apply transforms to tissue masks. Parameters ---------- @@ -4909,27 +5355,42 @@ def warp_tissuemask_to_template(wf, cfg, strat_pool, pipe_num, xfm, wf : nipype.pipeline.engine.workflows.Workflow outputs : dict - ''' - tissue_types = ['CSF', 'WM', 'GM'] + """ + tissue_types = ["CSF", "WM", "GM"] apply_xfm = {} for tissue in tissue_types: wf, apply_xfm[tissue] = warp_resource_to_template( - wf, cfg, strat_pool, pipe_num, f'label-{tissue}_mask', xfm, - time_series=False)[:2] - if template_space == 'T1': - template_space = '' - outputs = {f'space-{template_space}template_label-{tissue}_mask': ( - apply_xfm[tissue], 'outputspec.output_image') for - tissue in tissue_types} + wf, + cfg, + strat_pool, + pipe_num, + f"label-{tissue}_mask", + xfm, + time_series=False, + )[:2] + if template_space == "T1": + template_space = "" + outputs = { + f"space-{template_space}template_label-{tissue}_mask": ( + apply_xfm[tissue], + "outputspec.output_image", + ) + for tissue in tissue_types + } return _warp_return(wf, apply_xfm, outputs) -def warp_resource_to_template(wf: pe.Workflow, cfg, strat_pool, pipe_num: int, - input_resource: LIST_OR_STR, xfm: str, - reference: Optional[str] = None, - time_series: Optional[bool] = False - ) -> TUPLE[pe.Workflow, pe.Workflow, str]: - '''Function to warp a resource into a template space +def warp_resource_to_template( + wf: pe.Workflow, + cfg, + strat_pool, + pipe_num: int, + input_resource: LIST_OR_STR, + xfm: str, + reference: Optional[str] = None, + time_series: Optional[bool] = False, +) -> TUPLE[pe.Workflow, pe.Workflow, str]: + """Function to warp a resource into a template space. Parameters ---------- @@ -4964,54 +5425,59 @@ def warp_resource_to_template(wf: pe.Workflow, cfg, strat_pool, pipe_num: int, resource : str key of input resource in strat_pool - ''' + """ # determine space we're warping to - template_space = xfm.split('_to-', 1)[1].split('template')[0] - if template_space == '': - template_space = 'T1w' + template_space = xfm.split("_to-", 1)[1].split("template")[0] + if template_space == "": + template_space = "T1w" # determine tool used for registration xfm_prov = strat_pool.get_cpac_provenance(xfm) reg_tool = check_prov_for_regtool(xfm_prov) # set 'resource' if strat_pool.check_rpool(input_resource): - resource, input_resource = strat_pool.get_data(input_resource, - report_fetched=True) + resource, input_resource = strat_pool.get_data( + input_resource, report_fetched=True + ) else: return wf, None, input_resource # set 'reference' if not passed and determine subworkflow name if reference is None: subwf_input_name = input_resource - reference = f'{template_space}-template' + reference = f"{template_space}-template" else: - subwf_input_name = '-'.join([ - reference.split('-')[-1].split('_')[-1], - input_resource.split('-')[-1].split('_')[-1]]) + subwf_input_name = "-".join( + [ + reference.split("-")[-1].split("_")[-1], + input_resource.split("-")[-1].split("_")[-1], + ] + ) # set up 'apply_transform' subworkflow - apply_xfm = apply_transform(f'warp_{subwf_input_name}_to_' - f'{template_space}template_{pipe_num}', - reg_tool, time_series=time_series, - num_cpus=cfg.pipeline_setup['system_config'][ - 'max_cores_per_participant'], - num_ants_cores=cfg.pipeline_setup[ - 'system_config']['num_ants_threads']) + apply_xfm = apply_transform( + f"warp_{subwf_input_name}_to_" f"{template_space}template_{pipe_num}", + reg_tool, + time_series=time_series, + num_cpus=cfg.pipeline_setup["system_config"]["max_cores_per_participant"], + num_ants_cores=cfg.pipeline_setup["system_config"]["num_ants_threads"], + ) # set appropriate 'interpolation' input based on registration tool - if reg_tool == 'ants': - apply_xfm.inputs.inputspec.interpolation = 'NearestNeighbor' - elif reg_tool == 'fsl': - apply_xfm.inputs.inputspec.interpolation = 'nn' + if reg_tool == "ants": + apply_xfm.inputs.inputspec.interpolation = "NearestNeighbor" + elif reg_tool == "fsl": + apply_xfm.inputs.inputspec.interpolation = "nn" # connect nodes to subworkflow node, out = resource - wf.connect(node, out, apply_xfm, 'inputspec.input_image') + wf.connect(node, out, apply_xfm, "inputspec.input_image") node, out = strat_pool.get_data(reference) - wf.connect(node, out, apply_xfm, 'inputspec.reference') + wf.connect(node, out, apply_xfm, "inputspec.reference") node, out = strat_pool.get_data(xfm) - wf.connect(node, out, apply_xfm, 'inputspec.transform') + wf.connect(node, out, apply_xfm, "inputspec.transform") return wf, apply_xfm, input_resource -def _warp_return(wf: pe.Workflow, apply_xfm: Optional[pe.Workflow], - outputs: dict) -> TUPLE[pe.Workflow, dict]: - """Check if we have a transform to apply; if not, don't add the outputs""" +def _warp_return( + wf: pe.Workflow, apply_xfm: Optional[pe.Workflow], outputs: dict +) -> TUPLE[pe.Workflow, dict]: + """Check if we have a transform to apply; if not, don't add the outputs.""" if apply_xfm is None: return wf, {} return wf, outputs diff --git a/CPAC/registration/tests/mocks.py b/CPAC/registration/tests/mocks.py index b0a1000499..7fcdf789ef 100644 --- a/CPAC/registration/tests/mocks.py +++ b/CPAC/registration/tests/mocks.py @@ -1,111 +1,162 @@ import os + from nipype.interfaces import utility as util + from CPAC.pipeline import nipype_pipeline_engine as pe from CPAC.utils.configuration import Configuration from CPAC.utils.datasource import resolve_resolution from CPAC.utils.interfaces.function import Function from CPAC.utils.strategy import Strategy + def file_node(path, file_node_num=0): input_node = pe.Node( - util.IdentityInterface(fields=['file']), name='file_node_{0}'.format(file_node_num) + util.IdentityInterface(fields=["file"]), + name="file_node_{0}".format(file_node_num), ) input_node.inputs.file = path - return input_node, 'file' + return input_node, "file" + -def configuration_strategy_mock( method = 'FSL' ): +def configuration_strategy_mock(method="FSL"): fsldir = os.environ.get("FSLDIR") # mock the config dictionary - c = Configuration({ - "num_ants_threads": 4, - "workingDirectory": "/scratch/pipeline_tests", - "crashLogDirectory": "/scratch", - "outputDirectory": "/output/output/pipeline_analysis_nuisance/sub-M10978008_ses-NFB3", - "resolution_for_func_preproc": "3mm", - "resolution_for_func_derivative": "3mm", - "template_for_resample": f"{fsldir}/data/standard/" - "MNI152_T1_1mm_brain.nii.gz", - "template_brain_only_for_func": f"{fsldir}/data/standard/" - r"MNI152_T1_${func_resolution}_" - "brain.nii.gz", - "template_skull_for_func": f"{fsldir}/data/standard/" - r"MNI152_T1_${func_resolution}.nii.gz", - "identityMatrix": f"{fsldir}/etc/flirtsch/ident.mat", - "funcRegFSLinterpolation": "sinc", - "funcRegANTSinterpolation": "LanczosWindowedSinc" - }) - - if method == 'ANTS': - c.update('regOption', 'ANTS') + c = Configuration( + { + "num_ants_threads": 4, + "workingDirectory": "/scratch/pipeline_tests", + "crashLogDirectory": "/scratch", + "outputDirectory": "/output/output/pipeline_analysis_nuisance/sub-M10978008_ses-NFB3", + "resolution_for_func_preproc": "3mm", + "resolution_for_func_derivative": "3mm", + "template_for_resample": f"{fsldir}/data/standard/" + "MNI152_T1_1mm_brain.nii.gz", + "template_brain_only_for_func": f"{fsldir}/data/standard/" + r"MNI152_T1_${func_resolution}_" + "brain.nii.gz", + "template_skull_for_func": f"{fsldir}/data/standard/" + r"MNI152_T1_${func_resolution}.nii.gz", + "identityMatrix": f"{fsldir}/etc/flirtsch/ident.mat", + "funcRegFSLinterpolation": "sinc", + "funcRegANTSinterpolation": "LanczosWindowedSinc", + } + ) + + if method == "ANTS": + c.update("regOption", "ANTS") else: - c.update('regOption', 'FSL') + c.update("regOption", "FSL") # mock the strategy strat = Strategy() resource_dict = { - "mean_functional": os.path.join(c.outputDirectory, - "mean_functional/sub-M10978008_ses-NFB3_task-test_bold_calc_tshift_resample_volreg_calc_tstat.nii.gz"), - "motion_correct": os.path.join(c.outputDirectory, - "motion_correct/_scan_test/sub-M10978008_ses-NFB3_task-test_bold_calc_tshift_resample_volreg.nii.gz"), - "anatomical_brain": os.path.join(c.outputDirectory, - "anatomical_brain/sub-M10978008_ses-NFB3_acq-ao_brain_resample.nii.gz"), - "ants_initial_xfm": os.path.join(c.outputDirectory, - "ants_initial_xfm/transform0DerivedInitialMovingTranslation.mat"), - "ants_affine_xfm": os.path.join(c.outputDirectory, - "ants_affine_xfm/transform2Affine.mat"), - "ants_rigid_xfm": os.path.join(c.outputDirectory, - "ants_rigid_xfm/transform1Rigid.mat"), - "anatomical_to_mni_linear_xfm": os.path.join(c.outputDirectory, - "anatomical_to_mni_linear_xfm/sub-M10978008_ses-NFB3_T1w_resample_calc_flirt.mat"), - "functional_to_anat_linear_xfm": os.path.join(c.outputDirectory, - "functional_to_anat_linear_xfm/_scan_test/sub-M10978008_ses-NFB3_task-test_bold_calc_tshift_resample_volreg_calc_tstat_flirt.mat"), - 'ants_symm_warp_field': os.path.join(c.outputDirectory, - "anatomical_to_symmetric_mni_nonlinear_xfm/transform3Warp.nii.gz"), - 'ants_symm_affine_xfm': os.path.join(c.outputDirectory, - "ants_symmetric_affine_xfm/transform2Affine.mat"), - 'ants_symm_rigid_xfm': os.path.join(c.outputDirectory, - "ants_symmetric_rigid_xfm/transform1Rigid.mat"), - 'ants_symm_initial_xfm': os.path.join(c.outputDirectory, - "ants_symmetric_initial_xfm/transform0DerivedInitialMovingTranslation.mat"), - "dr_tempreg_maps_files": [os.path.join('/scratch', 'resting_preproc_sub-M10978008_ses-NFB3_cpac105', 'temporal_dual_regression_0/_scan_test/_selector_CSF-2mmE-M_aC-WM-2mmE-DPC5_G-M_M-SDB_P-2/_spatial_map_PNAS_Smith09_rsn10_spatial_map_file_..cpac_templates..PNAS_Smith09_rsn10.nii.gz/split_raw_volumes/temp_reg_map_000{0}.nii.gz'.format(n)) for n in range(10)] + "mean_functional": os.path.join( + c.outputDirectory, + "mean_functional/sub-M10978008_ses-NFB3_task-test_bold_calc_tshift_resample_volreg_calc_tstat.nii.gz", + ), + "motion_correct": os.path.join( + c.outputDirectory, + "motion_correct/_scan_test/sub-M10978008_ses-NFB3_task-test_bold_calc_tshift_resample_volreg.nii.gz", + ), + "anatomical_brain": os.path.join( + c.outputDirectory, + "anatomical_brain/sub-M10978008_ses-NFB3_acq-ao_brain_resample.nii.gz", + ), + "ants_initial_xfm": os.path.join( + c.outputDirectory, + "ants_initial_xfm/transform0DerivedInitialMovingTranslation.mat", + ), + "ants_affine_xfm": os.path.join( + c.outputDirectory, "ants_affine_xfm/transform2Affine.mat" + ), + "ants_rigid_xfm": os.path.join( + c.outputDirectory, "ants_rigid_xfm/transform1Rigid.mat" + ), + "anatomical_to_mni_linear_xfm": os.path.join( + c.outputDirectory, + "anatomical_to_mni_linear_xfm/sub-M10978008_ses-NFB3_T1w_resample_calc_flirt.mat", + ), + "functional_to_anat_linear_xfm": os.path.join( + c.outputDirectory, + "functional_to_anat_linear_xfm/_scan_test/sub-M10978008_ses-NFB3_task-test_bold_calc_tshift_resample_volreg_calc_tstat_flirt.mat", + ), + "ants_symm_warp_field": os.path.join( + c.outputDirectory, + "anatomical_to_symmetric_mni_nonlinear_xfm/transform3Warp.nii.gz", + ), + "ants_symm_affine_xfm": os.path.join( + c.outputDirectory, "ants_symmetric_affine_xfm/transform2Affine.mat" + ), + "ants_symm_rigid_xfm": os.path.join( + c.outputDirectory, "ants_symmetric_rigid_xfm/transform1Rigid.mat" + ), + "ants_symm_initial_xfm": os.path.join( + c.outputDirectory, + "ants_symmetric_initial_xfm/transform0DerivedInitialMovingTranslation.mat", + ), + "dr_tempreg_maps_files": [ + os.path.join( + "/scratch", + "resting_preproc_sub-M10978008_ses-NFB3_cpac105", + "temporal_dual_regression_0/_scan_test/_selector_CSF-2mmE-M_aC-WM-2mmE-DPC5_G-M_M-SDB_P-2/_spatial_map_PNAS_Smith09_rsn10_spatial_map_file_..cpac_templates..PNAS_Smith09_rsn10.nii.gz/split_raw_volumes/temp_reg_map_000{0}.nii.gz".format( + n + ), + ) + for n in range(10) + ], } - if method == 'ANTS': - resource_dict["anatomical_to_mni_nonlinear_xfm"] = os.path.join(c.outputDirectory, - "anatomical_to_mni_nonlinear_xfm/transform3Warp.nii.gz") + if method == "ANTS": + resource_dict["anatomical_to_mni_nonlinear_xfm"] = os.path.join( + c.outputDirectory, "anatomical_to_mni_nonlinear_xfm/transform3Warp.nii.gz" + ) else: - resource_dict["anatomical_to_mni_nonlinear_xfm"] = os.path.join(c.outputDirectory, - "anatomical_to_mni_nonlinear_xfm/sub-M10978008_ses-NFB3_T1w_resample_fieldwarp.nii.gz") - + resource_dict["anatomical_to_mni_nonlinear_xfm"] = os.path.join( + c.outputDirectory, + "anatomical_to_mni_nonlinear_xfm/sub-M10978008_ses-NFB3_T1w_resample_fieldwarp.nii.gz", + ) + file_node_num = 0 for resource, filepath in resource_dict.items(): - strat.update_resource_pool({ - resource: file_node(filepath, file_node_num) - }) - strat.append_name(resource+'_0') + strat.update_resource_pool({resource: file_node(filepath, file_node_num)}) + strat.append_name(resource + "_0") file_node_num += 1 templates_for_resampling = [ - (c.resolution_for_func_preproc, c.template_brain_only_for_func, - 'template_brain_for_func_preproc', 'resolution_for_func_preproc'), - (c.resolution_for_func_preproc, c.template_brain_only_for_func, - 'template_skull_for_func_preproc', 'resolution_for_func_preproc') + ( + c.resolution_for_func_preproc, + c.template_brain_only_for_func, + "template_brain_for_func_preproc", + "resolution_for_func_preproc", + ), + ( + c.resolution_for_func_preproc, + c.template_brain_only_for_func, + "template_skull_for_func_preproc", + "resolution_for_func_preproc", + ), ] for resolution, template, template_name, tag in templates_for_resampling: - resampled_template = pe.Node(Function(input_names = ['resolution', 'template', 'template_name', 'tag'], - output_names = ['resampled_template'], - function = resolve_resolution, - as_module = True), - name = 'resampled_' + template_name) + resampled_template = pe.Node( + Function( + input_names=["resolution", "template", "template_name", "tag"], + output_names=["resampled_template"], + function=resolve_resolution, + as_module=True, + ), + name="resampled_" + template_name, + ) resampled_template.inputs.resolution = resolution resampled_template.inputs.template = template resampled_template.inputs.template_name = template_name resampled_template.inputs.tag = tag - strat.update_resource_pool({template_name: (resampled_template, 'resampled_template')}) - strat.append_name('resampled_template_0') + strat.update_resource_pool( + {template_name: (resampled_template, "resampled_template")} + ) + strat.append_name("resampled_template_0") return c, strat diff --git a/CPAC/registration/tests/test_ants_apply_warp.py b/CPAC/registration/tests/test_ants_apply_warp.py index 0e3426a4f9..245cfc67a3 100644 --- a/CPAC/registration/tests/test_ants_apply_warp.py +++ b/CPAC/registration/tests/test_ants_apply_warp.py @@ -1,237 +1,288 @@ import os + +import pytest + from CPAC.pipeline import nipype_pipeline_engine as pe +import CPAC.utils.test_init as test_utils from ..output_func_to_standard import ants_apply_warps_func_mni from .mocks import configuration_strategy_mock -import CPAC.utils.test_init as test_utils -import nibabel as nb -import pytest + @pytest.mark.skip(reason="no way of currently testing this") def test_ants_apply_warp_func_mni(): - - test_name = 'test_ants_apply_warps_func_mni' + test_name = "test_ants_apply_warps_func_mni" # get the config and strat for the mock c, strat = configuration_strategy_mock() num_strat = 0 - node, out = strat['mean_functional'] + node, out = strat["mean_functional"] mean_functional = node.inputs.file # build the workflow - workflow = pe.Workflow(name='test_ants_apply_warps_func_mni') + workflow = pe.Workflow(name="test_ants_apply_warps_func_mni") workflow.base_dir = c.workingDirectory - workflow.config['execution'] = { - 'hash_method': 'timestamp', - 'crashdump_dir': os.path.abspath(c.crashLogDirectory) + workflow.config["execution"] = { + "hash_method": "timestamp", + "crashdump_dir": os.path.abspath(c.crashLogDirectory), } - workflow = ants_apply_warps_func_mni(workflow, - 'mean_functional_to_standard', - 'mean_functional', - 'template_brain_for_func_preproc', - num_strat, strat, - interpolation_method = c.funcRegANTSinterpolation, - distcor=False, - map_node=False, - inverse=False, - input_image_type=0, - num_ants_cores=1) - - workflow = ants_apply_warps_func_mni(workflow, - 'mean_functional_standard_to_original', - 'mean_functional_to_standard', - 'mean_functional', - num_strat, strat, - interpolation_method = c.funcRegANTSinterpolation, - distcor=False, - map_node=False, - inverse=True, - input_image_type=0, - num_ants_cores=1) + workflow = ants_apply_warps_func_mni( + workflow, + "mean_functional_to_standard", + "mean_functional", + "template_brain_for_func_preproc", + num_strat, + strat, + interpolation_method=c.funcRegANTSinterpolation, + distcor=False, + map_node=False, + inverse=False, + input_image_type=0, + num_ants_cores=1, + ) + + workflow = ants_apply_warps_func_mni( + workflow, + "mean_functional_standard_to_original", + "mean_functional_to_standard", + "mean_functional", + num_strat, + strat, + interpolation_method=c.funcRegANTSinterpolation, + distcor=False, + map_node=False, + inverse=True, + input_image_type=0, + num_ants_cores=1, + ) workflow.run() - mean_functional_after_transform=os.path.join(c.workingDirectory, test_name, - 'apply_ants_warp_mean_functional_standard_to_original_inverse_0', - 'sub-M10978008_ses-NFB3_task-test_bold_calc_tshift_resample_volreg_calc_tstat_antswarp_antswarp.nii.gz') + mean_functional_after_transform = os.path.join( + c.workingDirectory, + test_name, + "apply_ants_warp_mean_functional_standard_to_original_inverse_0", + "sub-M10978008_ses-NFB3_task-test_bold_calc_tshift_resample_volreg_calc_tstat_antswarp_antswarp.nii.gz", + ) - assert(test_utils.pearson_correlation(mean_functional, mean_functional_after_transform) > .99) + assert ( + test_utils.pearson_correlation(mean_functional, mean_functional_after_transform) + > 0.99 + ) @pytest.mark.skip(reason="no way of currently testing this") def test_ants_apply_warps_func_mni_mapnode(): - - test_name = 'test_ants_apply_warps_func_mni_mapnode' + test_name = "test_ants_apply_warps_func_mni_mapnode" # get the config and strat for the mock c, strat = configuration_strategy_mock() num_strat = 0 - node, out = strat['dr_tempreg_maps_files'] + node, out = strat["dr_tempreg_maps_files"] dr_spatmaps = node.inputs.file # build the workflow - workflow = pe.Workflow(name='test_ants_apply_warps_func_mni_mapnode') + workflow = pe.Workflow(name="test_ants_apply_warps_func_mni_mapnode") workflow.base_dir = c.workingDirectory - workflow.config['execution'] = { - 'hash_method': 'timestamp', - 'crashdump_dir': os.path.abspath(c.crashLogDirectory) + workflow.config["execution"] = { + "hash_method": "timestamp", + "crashdump_dir": os.path.abspath(c.crashLogDirectory), } - workflow = ants_apply_warps_func_mni(workflow, - 'dr_tempreg_maps_to_standard', - 'dr_tempreg_maps_files', - 'template_brain_for_func_preproc', - num_strat, strat, - interpolation_method = c.funcRegANTSinterpolation, - distcor=False, - map_node=True, - inverse=False, - input_image_type=0, - num_ants_cores=1) - - workflow = ants_apply_warps_func_mni(workflow, - 'dr_tempreg_maps_standard_to_original', - 'dr_tempreg_maps_to_standard', - 'mean_functional', - num_strat, strat, - interpolation_method = c.funcRegANTSinterpolation, - distcor=False, - map_node=True, - inverse=True, - input_image_type=0, - num_ants_cores=8) + workflow = ants_apply_warps_func_mni( + workflow, + "dr_tempreg_maps_to_standard", + "dr_tempreg_maps_files", + "template_brain_for_func_preproc", + num_strat, + strat, + interpolation_method=c.funcRegANTSinterpolation, + distcor=False, + map_node=True, + inverse=False, + input_image_type=0, + num_ants_cores=1, + ) + + workflow = ants_apply_warps_func_mni( + workflow, + "dr_tempreg_maps_standard_to_original", + "dr_tempreg_maps_to_standard", + "mean_functional", + num_strat, + strat, + interpolation_method=c.funcRegANTSinterpolation, + distcor=False, + map_node=True, + inverse=True, + input_image_type=0, + num_ants_cores=8, + ) workflow.run() - dr_spatmaps_after_transform=[os.path.join(c.workingDirectory, test_name, - 'apply_ants_warp_dr_tempreg_maps_standard_to_original_mapnode_inverse_0', - 'mapflow', '_apply_ants_warp_dr_tempreg_maps_standard_to_original_mapnode_inverse_0{0}'.format(n), - 'temp_reg_map_000{0}_antswarp_antswarp.nii.gz'.format(n)) - for n in range(0,10)] - - - test_results = [ test_utils.pearson_correlation(orig_file, xformed_file) > 0.99 \ - for orig_file, xformed_file in zip(dr_spatmaps, dr_spatmaps_after_transform)] + dr_spatmaps_after_transform = [ + os.path.join( + c.workingDirectory, + test_name, + "apply_ants_warp_dr_tempreg_maps_standard_to_original_mapnode_inverse_0", + "mapflow", + "_apply_ants_warp_dr_tempreg_maps_standard_to_original_mapnode_inverse_0{0}".format( + n + ), + "temp_reg_map_000{0}_antswarp_antswarp.nii.gz".format(n), + ) + for n in range(0, 10) + ] + + test_results = [ + test_utils.pearson_correlation(orig_file, xformed_file) > 0.99 + for orig_file, xformed_file in zip(dr_spatmaps, dr_spatmaps_after_transform) + ] assert all(test_results) -@pytest.mark.skip(reason='needs refactoring') +@pytest.mark.skip(reason="needs refactoring") def test_ants_apply_warp_func_mni_symm(): - - test_name = 'test_ants_apply_warps_func_mni_symm' + test_name = "test_ants_apply_warps_func_mni_symm" # get the config and strat for the mock c, strat = configuration_strategy_mock() num_strat = 0 - node, out = strat['mean_functional'] + node, out = strat["mean_functional"] mean_functional = node.inputs.file # build the workflow workflow = pe.Workflow(name=test_name) workflow.base_dir = c.workingDirectory - workflow.config['execution'] = { - 'hash_method': 'timestamp', - 'crashdump_dir': os.path.abspath(c.crashLogDirectory) + workflow.config["execution"] = { + "hash_method": "timestamp", + "crashdump_dir": os.path.abspath(c.crashLogDirectory), } - workflow = ants_apply_warps_func_mni(workflow, - 'mean_functional_to_standard_symm', - 'mean_functional', - 'template_brain_for_func_preproc', - num_strat, strat, - interpolation_method = c.funcRegANTSinterpolation, - distcor=False, - map_node=False, - inverse=False, - symmetry='symmetric', - input_image_type=0, - num_ants_cores=8) - - workflow = ants_apply_warps_func_mni(workflow, - 'mean_functional_standard_to_original_symm', - 'mean_functional_to_standard_symm', - 'mean_functional', - num_strat, strat, - interpolation_method = c.funcRegANTSinterpolation, - distcor=False, - map_node=False, - inverse=True, - symmetry='symmetric', - input_image_type=0, - num_ants_cores=1) - - retval = workflow.run() - - mean_functional_after_transform=os.path.join(c.workingDirectory, test_name, - 'apply_ants_warp_mean_functional_standard_to_original_symm_inverse_0', - 'sub-M10978008_ses-NFB3_task-test_bold_calc_tshift_resample_volreg_calc_tstat_antswarp_antswarp.nii.gz') - - assert(test_utils.pearson_correlation(mean_functional, mean_functional_after_transform) > .93) - - -@pytest.mark.skip(reason='needs refactoring') -def test_ants_apply_warps_func_mni_mapnode_symm(): + workflow = ants_apply_warps_func_mni( + workflow, + "mean_functional_to_standard_symm", + "mean_functional", + "template_brain_for_func_preproc", + num_strat, + strat, + interpolation_method=c.funcRegANTSinterpolation, + distcor=False, + map_node=False, + inverse=False, + symmetry="symmetric", + input_image_type=0, + num_ants_cores=8, + ) + + workflow = ants_apply_warps_func_mni( + workflow, + "mean_functional_standard_to_original_symm", + "mean_functional_to_standard_symm", + "mean_functional", + num_strat, + strat, + interpolation_method=c.funcRegANTSinterpolation, + distcor=False, + map_node=False, + inverse=True, + symmetry="symmetric", + input_image_type=0, + num_ants_cores=1, + ) + + workflow.run() + + mean_functional_after_transform = os.path.join( + c.workingDirectory, + test_name, + "apply_ants_warp_mean_functional_standard_to_original_symm_inverse_0", + "sub-M10978008_ses-NFB3_task-test_bold_calc_tshift_resample_volreg_calc_tstat_antswarp_antswarp.nii.gz", + ) + + assert ( + test_utils.pearson_correlation(mean_functional, mean_functional_after_transform) + > 0.93 + ) - test_name = 'test_ants_apply_warps_func_mni_mapnode_symm' + +@pytest.mark.skip(reason="needs refactoring") +def test_ants_apply_warps_func_mni_mapnode_symm(): + test_name = "test_ants_apply_warps_func_mni_mapnode_symm" # get the config and strat for the mock c, strat = configuration_strategy_mock() num_strat = 0 - node, out = strat['dr_tempreg_maps_files'] + node, out = strat["dr_tempreg_maps_files"] dr_spatmaps = node.inputs.file # build the workflow workflow = pe.Workflow(name=test_name) workflow.base_dir = c.workingDirectory - workflow.config['execution'] = { - 'hash_method': 'timestamp', - 'crashdump_dir': os.path.abspath(c.crashLogDirectory) + workflow.config["execution"] = { + "hash_method": "timestamp", + "crashdump_dir": os.path.abspath(c.crashLogDirectory), } - workflow = ants_apply_warps_func_mni(workflow, - 'dr_tempreg_maps_to_standard_symm', - 'dr_tempreg_maps_files', - 'template_brain_for_func_preproc', - num_strat, strat, - interpolation_method = c.funcRegANTSinterpolation, - distcor=False, - map_node=True, - inverse=False, - symmetry='symmetric', - input_image_type=0, - num_ants_cores=8) - - workflow = ants_apply_warps_func_mni(workflow, - 'dr_tempreg_maps_standard_symm_to_original', - 'dr_tempreg_maps_to_standard_symm', - 'mean_functional', - num_strat, strat, - interpolation_method = c.funcRegANTSinterpolation, - distcor=False, - map_node=True, - inverse=True, - symmetry='symmetric', - input_image_type=0, - num_ants_cores=8) + workflow = ants_apply_warps_func_mni( + workflow, + "dr_tempreg_maps_to_standard_symm", + "dr_tempreg_maps_files", + "template_brain_for_func_preproc", + num_strat, + strat, + interpolation_method=c.funcRegANTSinterpolation, + distcor=False, + map_node=True, + inverse=False, + symmetry="symmetric", + input_image_type=0, + num_ants_cores=8, + ) + + workflow = ants_apply_warps_func_mni( + workflow, + "dr_tempreg_maps_standard_symm_to_original", + "dr_tempreg_maps_to_standard_symm", + "mean_functional", + num_strat, + strat, + interpolation_method=c.funcRegANTSinterpolation, + distcor=False, + map_node=True, + inverse=True, + symmetry="symmetric", + input_image_type=0, + num_ants_cores=8, + ) workflow.run() - dr_spatmaps_after_transform=[os.path.join(c.workingDirectory, test_name, - 'apply_ants_warp_dr_tempreg_maps_standard_symm_to_original_mapnode_inverse_0', - 'mapflow', '_apply_ants_warp_dr_tempreg_maps_standard_symm_to_original_mapnode_inverse_0{0}'.format(n), - 'temp_reg_map_000{0}_antswarp_antswarp.nii.gz'.format(n)) - for n in range(0,10)] - - r = [test_utils.pearson_correlation(orig_file, xformed_file) \ - for orig_file, xformed_file in zip(dr_spatmaps, dr_spatmaps_after_transform)] - - print(r) - test_results = [ r_value > 0.93 for r_value in r ] + dr_spatmaps_after_transform = [ + os.path.join( + c.workingDirectory, + test_name, + "apply_ants_warp_dr_tempreg_maps_standard_symm_to_original_mapnode_inverse_0", + "mapflow", + "_apply_ants_warp_dr_tempreg_maps_standard_symm_to_original_mapnode_inverse_0{0}".format( + n + ), + "temp_reg_map_000{0}_antswarp_antswarp.nii.gz".format(n), + ) + for n in range(0, 10) + ] + + r = [ + test_utils.pearson_correlation(orig_file, xformed_file) + for orig_file, xformed_file in zip(dr_spatmaps, dr_spatmaps_after_transform) + ] + + test_results = [r_value > 0.93 for r_value in r] assert all(test_results) - diff --git a/CPAC/registration/tests/test_apply_transform.py b/CPAC/registration/tests/test_apply_transform.py index 6ea153ff33..7dc98890da 100644 --- a/CPAC/registration/tests/test_apply_transform.py +++ b/CPAC/registration/tests/test_apply_transform.py @@ -1,93 +1,117 @@ import os + import pytest -from .mocks import configuration_strategy_mock + from CPAC.pipeline import nipype_pipeline_engine as pe from ..output_func_to_standard import fsl_apply_transform_func_to_mni +from .mocks import configuration_strategy_mock -@pytest.mark.skip(reason='needs refactoring') +@pytest.mark.skip(reason="needs refactoring") def test_fsl_apply_transform_func_to_mni_nonlinear(): + c, strat = configuration_strategy_mock(method="FSL") - c, strat = configuration_strategy_mock(method='FSL') - - strat.append_name('anat_mni_fnirt_register_0') + strat.append_name("anat_mni_fnirt_register_0") # build the workflow - workflow = pe.Workflow(name='test_fsl_apply_transform_func_to_mni_nonlinear') + workflow = pe.Workflow(name="test_fsl_apply_transform_func_to_mni_nonlinear") workflow.base_dir = c.workingDirectory - workflow.config['execution'] = { - 'hash_method': 'timestamp', - 'crashdump_dir': os.path.abspath(c.crashLogDirectory) + workflow.config["execution"] = { + "hash_method": "timestamp", + "crashdump_dir": os.path.abspath(c.crashLogDirectory), } - workflow = fsl_apply_transform_func_to_mni(workflow, 'mean_functional_to_standard', - 'mean_functional', 'template_brain_for_func_preproc', 0, strat, - c.funcRegFSLinterpolation) + workflow = fsl_apply_transform_func_to_mni( + workflow, + "mean_functional_to_standard", + "mean_functional", + "template_brain_for_func_preproc", + 0, + strat, + c.funcRegFSLinterpolation, + ) workflow.run() -@pytest.mark.skip(reason='needs refactoring') +@pytest.mark.skip(reason="needs refactoring") def test_fsl_apply_transform_func_to_mni_nonlinear_mapnode(): + c, strat = configuration_strategy_mock(method="FSL") - c, strat = configuration_strategy_mock(method='FSL') - - strat.append_name('anat_mni_fnirt_register_0') + strat.append_name("anat_mni_fnirt_register_0") # build the workflow - workflow = pe.Workflow(name='test_fsl_apply_transform_func_to_mni_nonlinear') + workflow = pe.Workflow(name="test_fsl_apply_transform_func_to_mni_nonlinear") workflow.base_dir = c.workingDirectory - workflow.config['execution'] = { - 'hash_method': 'timestamp', - 'crashdump_dir': os.path.abspath(c.crashLogDirectory) + workflow.config["execution"] = { + "hash_method": "timestamp", + "crashdump_dir": os.path.abspath(c.crashLogDirectory), } - workflow = fsl_apply_transform_func_to_mni(workflow, 'dr_tempreg_to_standard', - 'dr_tempreg_maps_files', 'template_brain_for_func_preproc', 0, strat, - c.funcRegFSLinterpolation, map_node=True) + workflow = fsl_apply_transform_func_to_mni( + workflow, + "dr_tempreg_to_standard", + "dr_tempreg_maps_files", + "template_brain_for_func_preproc", + 0, + strat, + c.funcRegFSLinterpolation, + map_node=True, + ) workflow.run() -@pytest.mark.skip(reason='needs refactoring') +@pytest.mark.skip(reason="needs refactoring") def test_fsl_apply_transform_func_to_mni_linear(): + c, strat = configuration_strategy_mock(method="FSL") - c, strat = configuration_strategy_mock(method='FSL') - - strat.append_name('anat_mni_flirt_register_0') + strat.append_name("anat_mni_flirt_register_0") # build the workflow - workflow = pe.Workflow(name='test_fsl_apply_transform_func_to_mni_linear') + workflow = pe.Workflow(name="test_fsl_apply_transform_func_to_mni_linear") workflow.base_dir = c.workingDirectory - workflow.config['execution'] = { - 'hash_method': 'timestamp', - 'crashdump_dir': os.path.abspath(c.crashLogDirectory) + workflow.config["execution"] = { + "hash_method": "timestamp", + "crashdump_dir": os.path.abspath(c.crashLogDirectory), } - workflow = fsl_apply_transform_func_to_mni(workflow, 'mean_functional_to_standard', - 'mean_functional', 'template_brain_for_func_preproc', 0, strat, - c.funcRegFSLinterpolation) + workflow = fsl_apply_transform_func_to_mni( + workflow, + "mean_functional_to_standard", + "mean_functional", + "template_brain_for_func_preproc", + 0, + strat, + c.funcRegFSLinterpolation, + ) workflow.run() -@pytest.mark.skip(reason='needs refactoring') +@pytest.mark.skip(reason="needs refactoring") def test_fsl_apply_transform_func_to_mni_linear_mapnode(): + c, strat = configuration_strategy_mock(method="FSL") - c, strat = configuration_strategy_mock(method='FSL') - - strat.append_name('anat_mni_flirt_register_0') + strat.append_name("anat_mni_flirt_register_0") # build the workflow - workflow = pe.Workflow(name='test_fsl_apply_transform_func_to_mni_linear_mapnode') + workflow = pe.Workflow(name="test_fsl_apply_transform_func_to_mni_linear_mapnode") workflow.base_dir = c.workingDirectory - workflow.config['execution'] = { - 'hash_method': 'timestamp', - 'crashdump_dir': os.path.abspath(c.crashLogDirectory) + workflow.config["execution"] = { + "hash_method": "timestamp", + "crashdump_dir": os.path.abspath(c.crashLogDirectory), } - workflow = fsl_apply_transform_func_to_mni(workflow, 'dr_tempreg_to_standard', - 'dr_tempreg_maps_files', 'template_brain_for_func_preproc', 0, strat, - c.funcRegFSLinterpolation, map_node=True) + workflow = fsl_apply_transform_func_to_mni( + workflow, + "dr_tempreg_to_standard", + "dr_tempreg_maps_files", + "template_brain_for_func_preproc", + 0, + strat, + c.funcRegFSLinterpolation, + map_node=True, + ) workflow.run() diff --git a/CPAC/registration/tests/test_output_func_to_standard.py b/CPAC/registration/tests/test_output_func_to_standard.py index 782fe8df4c..22c077f2ef 100644 --- a/CPAC/registration/tests/test_output_func_to_standard.py +++ b/CPAC/registration/tests/test_output_func_to_standard.py @@ -1,141 +1,183 @@ import os -import nibabel as nb -import numpy as np + import pytest from CPAC.registration import output_func_to_standard -from .mocks import configuration_strategy_mock -import nipype.interfaces.afni.utils as afni_utils -import nipype.interfaces.utility as util import CPAC.utils.test_init as test_utils +from .mocks import configuration_strategy_mock -@pytest.mark.skip(reason='needs refactoring') +@pytest.mark.skip(reason="needs refactoring") def test_output_func_to_standard_ANTS(): - - test_name = 'test_output_func_to_standard_ANTS' + test_name = "test_output_func_to_standard_ANTS" # get the config and strat for the mock - c, strat = configuration_strategy_mock(method='ANTS') + c, strat = configuration_strategy_mock(method="ANTS") num_strat = 0 # build the workflow - workflow = pe.Workflow(name='test_output_func_to_standard_ANTS') + workflow = pe.Workflow(name="test_output_func_to_standard_ANTS") workflow.base_dir = c.workingDirectory - workflow.config['execution'] = { - 'hash_method': 'timestamp', - 'crashdump_dir': os.path.abspath(c.crashLogDirectory) + workflow.config["execution"] = { + "hash_method": "timestamp", + "crashdump_dir": os.path.abspath(c.crashLogDirectory), } - output_func_to_standard(workflow, - 'mean_functional', - 'template_brain_for_func_preproc', - 'mean_functional_to_standard', - strat, num_strat, c, input_image_type='func_derivative') - - out1_name = os.path.join(c.workingDirectory, test_name, - 'apply_ants_warp_mean_functional_to_standard_0', - 'sub-M10978008_ses-NFB3_task-test_bold_calc_tshift_resample_volreg_calc_tstat_antswarp.nii.gz') - - in_node, in_file = strat['mean_functional'] - output_func_to_standard(workflow, - (in_node, in_file), - 'template_brain_for_func_preproc', - 'mean_functional_to_standard_node', - strat, num_strat, c, input_image_type='func_derivative') - - out2_name = os.path.join(c.workingDirectory, test_name, - 'apply_ants_warp_mean_functional_to_standard_node_0', - 'sub-M10978008_ses-NFB3_task-test_bold_calc_tshift_resample_volreg_calc_tstat_antswarp.nii.gz') + output_func_to_standard( + workflow, + "mean_functional", + "template_brain_for_func_preproc", + "mean_functional_to_standard", + strat, + num_strat, + c, + input_image_type="func_derivative", + ) + + out1_name = os.path.join( + c.workingDirectory, + test_name, + "apply_ants_warp_mean_functional_to_standard_0", + "sub-M10978008_ses-NFB3_task-test_bold_calc_tshift_resample_volreg_calc_tstat_antswarp.nii.gz", + ) + + in_node, in_file = strat["mean_functional"] + output_func_to_standard( + workflow, + (in_node, in_file), + "template_brain_for_func_preproc", + "mean_functional_to_standard_node", + strat, + num_strat, + c, + input_image_type="func_derivative", + ) + + out2_name = os.path.join( + c.workingDirectory, + test_name, + "apply_ants_warp_mean_functional_to_standard_node_0", + "sub-M10978008_ses-NFB3_task-test_bold_calc_tshift_resample_volreg_calc_tstat_antswarp.nii.gz", + ) workflow.run() - assert(test_utils.pearson_correlation(out1_name, out2_name) > 0.99) + assert test_utils.pearson_correlation(out1_name, out2_name) > 0.99 -@pytest.mark.skip(reason='needs refactoring') +@pytest.mark.skip(reason="needs refactoring") def test_output_func_to_standard_FSL_linear(): - - test_name = 'test_output_func_to_standard_FSL_linear' + test_name = "test_output_func_to_standard_FSL_linear" # get the config and strat for the mock - c, strat = configuration_strategy_mock(method='FSL') - strat.append_name('anat_mni_flirt_register_0') + c, strat = configuration_strategy_mock(method="FSL") + strat.append_name("anat_mni_flirt_register_0") num_strat = 0 # build the workflow - workflow = pe.Workflow(name='test_output_func_to_standard_FSL_linear') + workflow = pe.Workflow(name="test_output_func_to_standard_FSL_linear") workflow.base_dir = c.workingDirectory - workflow.config['execution'] = { - 'hash_method': 'timestamp', - 'crashdump_dir': os.path.abspath(c.crashLogDirectory) + workflow.config["execution"] = { + "hash_method": "timestamp", + "crashdump_dir": os.path.abspath(c.crashLogDirectory), } - output_func_to_standard(workflow, - 'mean_functional', - 'template_brain_for_func_preproc', - 'mean_functional_to_standard', - strat, num_strat, c, input_image_type='func_derivative') - - out1_name = os.path.join(c.workingDirectory, test_name, - 'func_mni_fsl_warp_mean_functional_to_standard_0', - 'sub-M10978008_ses-NFB3_task-test_bold_calc_tshift_resample_volreg_calc_tstat_warp.nii.gz') - - func_node, func_out = strat['mean_functional'] - output_func_to_standard(workflow, - (func_node, func_out), - 'template_brain_for_func_preproc', - 'mean_functional_to_standard_node', - strat, num_strat, c, input_image_type='func_derivative') - - out2_name = os.path.join(c.workingDirectory, test_name, - 'func_mni_fsl_warp_mean_functional_to_standard_node_0', - 'sub-M10978008_ses-NFB3_task-test_bold_calc_tshift_resample_volreg_calc_tstat_warp.nii.gz') + output_func_to_standard( + workflow, + "mean_functional", + "template_brain_for_func_preproc", + "mean_functional_to_standard", + strat, + num_strat, + c, + input_image_type="func_derivative", + ) + + out1_name = os.path.join( + c.workingDirectory, + test_name, + "func_mni_fsl_warp_mean_functional_to_standard_0", + "sub-M10978008_ses-NFB3_task-test_bold_calc_tshift_resample_volreg_calc_tstat_warp.nii.gz", + ) + + func_node, func_out = strat["mean_functional"] + output_func_to_standard( + workflow, + (func_node, func_out), + "template_brain_for_func_preproc", + "mean_functional_to_standard_node", + strat, + num_strat, + c, + input_image_type="func_derivative", + ) + + out2_name = os.path.join( + c.workingDirectory, + test_name, + "func_mni_fsl_warp_mean_functional_to_standard_node_0", + "sub-M10978008_ses-NFB3_task-test_bold_calc_tshift_resample_volreg_calc_tstat_warp.nii.gz", + ) workflow.run() - assert(test_utils.pearson_correlation(out1_name, out2_name) > 0.99) + assert test_utils.pearson_correlation(out1_name, out2_name) > 0.99 -@pytest.mark.skip(reason='needs refactoring') +@pytest.mark.skip(reason="needs refactoring") def test_output_func_to_standard_FSL_nonlinear(): - - test_name = 'test_output_func_to_standard_FSL_nonlinear' + test_name = "test_output_func_to_standard_FSL_nonlinear" # get the config and strat for the mock - c, strat = configuration_strategy_mock(method='FSL') - strat.append_name('anat_mni_fnirt_register_0') + c, strat = configuration_strategy_mock(method="FSL") + strat.append_name("anat_mni_fnirt_register_0") num_strat = 0 # build the workflow - workflow = pe.Workflow(name='test_output_func_to_standard_FSL_nonlinear') + workflow = pe.Workflow(name="test_output_func_to_standard_FSL_nonlinear") workflow.base_dir = c.workingDirectory - workflow.config['execution'] = { - 'hash_method': 'timestamp', - 'crashdump_dir': os.path.abspath(c.crashLogDirectory) + workflow.config["execution"] = { + "hash_method": "timestamp", + "crashdump_dir": os.path.abspath(c.crashLogDirectory), } - output_func_to_standard(workflow, - 'mean_functional', - 'template_brain_for_func_preproc', - 'mean_functional_to_standard', - strat, num_strat, c, input_image_type='func_derivative') - - out1_name = os.path.join(c.workingDirectory, test_name, - 'func_mni_fsl_warp_mean_functional_to_standard_0', - 'sub-M10978008_ses-NFB3_task-test_bold_calc_tshift_resample_volreg_calc_tstat_warp.nii.gz') - - node, out_file = strat['mean_functional'] - output_func_to_standard(workflow, - (node, out_file), - 'template_brain_for_func_preproc', - 'mean_functional_to_standard_node', - strat, num_strat, c, input_image_type='func_derivative') - - out2_name = os.path.join(c.workingDirectory, test_name, - 'func_mni_fsl_warp_mean_functional_to_standard_node_0', - 'sub-M10978008_ses-NFB3_task-test_bold_calc_tshift_resample_volreg_calc_tstat_warp.nii.gz') + output_func_to_standard( + workflow, + "mean_functional", + "template_brain_for_func_preproc", + "mean_functional_to_standard", + strat, + num_strat, + c, + input_image_type="func_derivative", + ) + + out1_name = os.path.join( + c.workingDirectory, + test_name, + "func_mni_fsl_warp_mean_functional_to_standard_0", + "sub-M10978008_ses-NFB3_task-test_bold_calc_tshift_resample_volreg_calc_tstat_warp.nii.gz", + ) + + node, out_file = strat["mean_functional"] + output_func_to_standard( + workflow, + (node, out_file), + "template_brain_for_func_preproc", + "mean_functional_to_standard_node", + strat, + num_strat, + c, + input_image_type="func_derivative", + ) + + out2_name = os.path.join( + c.workingDirectory, + test_name, + "func_mni_fsl_warp_mean_functional_to_standard_node_0", + "sub-M10978008_ses-NFB3_task-test_bold_calc_tshift_resample_volreg_calc_tstat_warp.nii.gz", + ) workflow.run() - assert(test_utils.pearson_correlation(out1_name, out2_name) > .99) + assert test_utils.pearson_correlation(out1_name, out2_name) > 0.99 diff --git a/CPAC/registration/tests/test_registration.py b/CPAC/registration/tests/test_registration.py index 430e71ef7b..58741da445 100755 --- a/CPAC/registration/tests/test_registration.py +++ b/CPAC/registration/tests/test_registration.py @@ -1,13 +1,13 @@ import pytest -@pytest.mark.skip(reason='needs refactoring') +@pytest.mark.skip(reason="needs refactoring") def test_nonlinear_register(): - from ..registration import create_nonlinear_register - + from nipype.interfaces import fsl + from CPAC.pipeline import nipype_pipeline_engine as pe - import nipype.interfaces.fsl as fsl - + from ..registration import create_nonlinear_register + ## necessary inputs ## -input_brain ## -input_skull @@ -15,146 +15,141 @@ def test_nonlinear_register(): ## -reference_skull ## -fnirt_config ## -fnirt_warp_res - + ## input_brain - anat_bet_file = '/home/data/Projects/nuisance_reliability_paper/working_dir_CPAC_order/resting_preproc/anatpreproc/_session_id_NYU_TRT_session1_subject_id_sub05676/anat_skullstrip/mprage_anonymized_RPI_3dT.nii.gz' - + anat_bet_file = "/home/data/Projects/nuisance_reliability_paper/working_dir_CPAC_order/resting_preproc/anatpreproc/_session_id_NYU_TRT_session1_subject_id_sub05676/anat_skullstrip/mprage_anonymized_RPI_3dT.nii.gz" + ## input_skull - + ## reference_brain - mni_file = '/usr/share/fsl/4.1/data/standard/MNI152_T1_3mm_brain.nii.gz' - + ## reference_skull - + ## fnirt_config - fnirt_config = 'T1_2_MNI152_3mm' - + ## fnirt_warp_res - fnirt_warp_res = None - - #?? what is this for?: - func_file = '/home/data/Projects/nuisance_reliability_paper/working_dir_CPAC_order/resting_preproc/nuisance_preproc/_session_id_NYU_TRT_session1_subject_id_sub05676/_csf_threshold_0.4/_gm_threshold_0.2/_wm_threshold_0.66/_run_scrubbing_False/_nc_5/_selector_6.7/regress_nuisance/mapflow/_regress_nuisance0/residual.nii.gz' - - - mni_workflow = pe.Workflow(name='mni_workflow') - - linear_reg = pe.Node(interface=fsl.FLIRT(), - name='linear_reg_0') - linear_reg.inputs.cost = 'corratio' + + # ?? what is this for?: + func_file = "/home/data/Projects/nuisance_reliability_paper/working_dir_CPAC_order/resting_preproc/nuisance_preproc/_session_id_NYU_TRT_session1_subject_id_sub05676/_csf_threshold_0.4/_gm_threshold_0.2/_wm_threshold_0.66/_run_scrubbing_False/_nc_5/_selector_6.7/regress_nuisance/mapflow/_regress_nuisance0/residual.nii.gz" + + mni_workflow = pe.Workflow(name="mni_workflow") + + linear_reg = pe.Node(interface=fsl.FLIRT(), name="linear_reg_0") + linear_reg.inputs.cost = "corratio" linear_reg.inputs.dof = 6 - linear_reg.inputs.interp = 'nearestneighbour' - + linear_reg.inputs.interp = "nearestneighbour" + linear_reg.inputs.in_file = func_file linear_reg.inputs.reference = anat_bet_file - - #T1 to MNI Node + + # T1 to MNI Node c = create_nonlinear_register() c.inputs.inputspec.input = anat_bet_file - c.inputs.inputspec.reference = '/usr/share/fsl/4.1/data/standard/MNI152_T1_3mm_brain.nii.gz' - c.inputs.inputspec.fnirt_config = 'T1_2_MNI152_3mm' - - #EPI to MNI warp Node - mni_warp = pe.Node(interface=fsl.ApplyWarp(), - name='mni_warp') - mni_warp.inputs.ref_file = '/usr/share/fsl/4.1/data/standard/MNI152_T1_3mm_brain.nii.gz' + c.inputs.inputspec.reference = ( + "/usr/share/fsl/4.1/data/standard/MNI152_T1_3mm_brain.nii.gz" + ) + c.inputs.inputspec.fnirt_config = "T1_2_MNI152_3mm" + + # EPI to MNI warp Node + mni_warp = pe.Node(interface=fsl.ApplyWarp(), name="mni_warp") + mni_warp.inputs.ref_file = ( + "/usr/share/fsl/4.1/data/standard/MNI152_T1_3mm_brain.nii.gz" + ) mni_warp.inputs.in_file = func_file - mni_workflow.connect(c, 'outputspec.nonlinear_xfm', - mni_warp, 'field_file') - mni_workflow.connect(linear_reg, 'out_matrix_file', - mni_warp, 'premat') - - mni_workflow.base_dir = './' - mni_workflow.run() + mni_workflow.connect(c, "outputspec.nonlinear_xfm", mni_warp, "field_file") + mni_workflow.connect(linear_reg, "out_matrix_file", mni_warp, "premat") + + mni_workflow.base_dir = "./" + mni_workflow.run() -@pytest.mark.skip(reason='needs refactoring') +@pytest.mark.skip(reason="needs refactoring") def test_registration(): - from ..registration import create_nonlinear_register - from ..registration import create_register_func_to_mni - from CPAC.pipeline import nipype_pipeline_engine as pe - import nipype.interfaces.fsl as fsl - - func_file = '/home/data/Projects/nuisance_reliability_paper/working_dir_CPAC_order/resting_preproc/nuisance_preproc/_session_id_NYU_TRT_session1_subject_id_sub05676/_csf_threshold_0.4/_gm_threshold_0.2/_wm_threshold_0.66/_run_scrubbing_False/_nc_5/_selector_6.7/regress_nuisance/mapflow/_regress_nuisance0/residual.nii.gz' - anat_skull_file = '/home/data/Projects/nuisance_reliability_paper/working_dir_CPAC_order/resting_preproc/anatpreproc/_session_id_NYU_TRT_session1_subject_id_sub05676/anat_reorient/mprage_anonymized_RPI.nii.gz' - anat_bet_file = '/home/data/Projects/nuisance_reliability_paper/working_dir_CPAC_order/resting_preproc/anatpreproc/_session_id_NYU_TRT_session1_subject_id_sub05676/anat_skullstrip/mprage_anonymized_RPI_3dT.nii.gz' - mni_brain_file = '/usr/share/fsl/4.1/data/standard/MNI152_T1_3mm_brain.nii.gz' - mni_skull_file = '/usr/share/fsl/4.1/data/standard/MNI152_T1_3mm.nii.gz' - - - mni_workflow = pe.Workflow(name='mni_workflow') - + from ..registration import create_nonlinear_register, create_register_func_to_mni + + func_file = "/home/data/Projects/nuisance_reliability_paper/working_dir_CPAC_order/resting_preproc/nuisance_preproc/_session_id_NYU_TRT_session1_subject_id_sub05676/_csf_threshold_0.4/_gm_threshold_0.2/_wm_threshold_0.66/_run_scrubbing_False/_nc_5/_selector_6.7/regress_nuisance/mapflow/_regress_nuisance0/residual.nii.gz" + anat_skull_file = "/home/data/Projects/nuisance_reliability_paper/working_dir_CPAC_order/resting_preproc/anatpreproc/_session_id_NYU_TRT_session1_subject_id_sub05676/anat_reorient/mprage_anonymized_RPI.nii.gz" + anat_bet_file = "/home/data/Projects/nuisance_reliability_paper/working_dir_CPAC_order/resting_preproc/anatpreproc/_session_id_NYU_TRT_session1_subject_id_sub05676/anat_skullstrip/mprage_anonymized_RPI_3dT.nii.gz" + mni_brain_file = "/usr/share/fsl/4.1/data/standard/MNI152_T1_3mm_brain.nii.gz" + mni_skull_file = "/usr/share/fsl/4.1/data/standard/MNI152_T1_3mm.nii.gz" + + mni_workflow = pe.Workflow(name="mni_workflow") + nr = create_nonlinear_register() nr.inputs.inputspec.input_brain = anat_bet_file nr.inputs.inputspec.input_skull = anat_skull_file nr.inputs.inputspec.reference_brain = mni_brain_file nr.inputs.inputspec.reference_skull = mni_skull_file - nr.inputs.inputspec.fnirt_config = '/usr/share/fsl/4.1/etc/flirtsch/T1_2_MNI152_3mm.cnf' + nr.inputs.inputspec.fnirt_config = ( + "/usr/share/fsl/4.1/etc/flirtsch/T1_2_MNI152_3mm.cnf" + ) func2mni = create_register_func_to_mni() func2mni.inputs.inputspec.func = func_file func2mni.inputs.inputspec.mni = mni_brain_file func2mni.inputs.inputspec.anat = anat_bet_file - - mni_workflow.connect(nr, 'outputspec.nonlinear_xfm', - func2mni, 'inputspec.anat_to_mni_xfm') - mni_workflow.base_dir = './mni_05676_3' + + mni_workflow.connect( + nr, "outputspec.nonlinear_xfm", func2mni, "inputspec.anat_to_mni_xfm" + ) + mni_workflow.base_dir = "./mni_05676_3" mni_workflow.run() -@pytest.mark.skip(reason='needs refactoring') +@pytest.mark.skip(reason="needs refactoring") def test_registration_lesion(): import os - from CPAC.pipeline import nipype_pipeline_engine as pe - from ..registration import create_wf_calculate_ants_warp + from CPAC.anat_preproc.anat_preproc import create_anat_preproc from CPAC.anat_preproc.lesion_preproc import create_lesion_preproc + from CPAC.pipeline import nipype_pipeline_engine as pe + from ..registration import create_wf_calculate_ants_warp # Skull stripped anat image - anat_file = '/bids_dataset/sub-0027228/ses-1/anat/sub-0027228_ses-1_run-1_T1w.nii.gz' - lesion_file = '/bids_dataset/sub-0027228/ses-1/anat/sub-0027228_ses-1_run-1_T1w_lesion-mask.nii.gz' - mni_brain_file = (f'{os.environ.get("FSLDIR")}/data/standard/' - 'MNI152_T1_3mm_brain.nii.gz') + anat_file = ( + "/bids_dataset/sub-0027228/ses-1/anat/sub-0027228_ses-1_run-1_T1w.nii.gz" + ) + lesion_file = "/bids_dataset/sub-0027228/ses-1/anat/sub-0027228_ses-1_run-1_T1w_lesion-mask.nii.gz" + mni_brain_file = ( + f'{os.environ.get("FSLDIR")}/data/standard/' 'MNI152_T1_3mm_brain.nii.gz' + ) if not os.path.exists(anat_file): - raise IOError(anat_file + ' not found') + raise IOError(anat_file + " not found") if not os.path.exists(lesion_file): - raise IOError(lesion_file + ' not found') + raise IOError(lesion_file + " not found") if not os.path.exists(mni_brain_file): - raise IOError(mni_brain_file + ' not found') + raise IOError(mni_brain_file + " not found") - wf = pe.Workflow(name='test_reg_lesion') + wf = pe.Workflow(name="test_reg_lesion") - anat_preproc = create_anat_preproc(method='mask', - already_skullstripped=True, - wf_name='anat_preproc') + anat_preproc = create_anat_preproc( + method="mask", already_skullstripped=True, wf_name="anat_preproc" + ) anat_preproc.inputs.inputspec.anat = anat_file - lesion_preproc = create_lesion_preproc( - wf_name='lesion_preproc' - ) + lesion_preproc = create_lesion_preproc(wf_name="lesion_preproc") lesion_preproc.inputs.inputspec.lesion = lesion_file - ants_reg_anat_mni = \ - create_wf_calculate_ants_warp( - 'anat_mni_ants_register', - 0, - num_threads=4 - ) + ants_reg_anat_mni = create_wf_calculate_ants_warp( + "anat_mni_ants_register", 0, num_threads=4 + ) # pass the reference file ants_reg_anat_mni.inputs.inputspec.reference_brain = mni_brain_file wf.connect( - anat_preproc, 'outputspec.reorient', - ants_reg_anat_mni, 'inputspec.moving_brain' + anat_preproc, "outputspec.reorient", ants_reg_anat_mni, "inputspec.moving_brain" ) wf.connect( - lesion_preproc, 'outputspec.reorient', - ants_reg_anat_mni, 'inputspec.fixed_image_mask' + lesion_preproc, + "outputspec.reorient", + ants_reg_anat_mni, + "inputspec.fixed_image_mask", ) ants_reg_anat_mni.inputs.inputspec.set( @@ -162,30 +157,22 @@ def test_registration_lesion(): use_histogram_matching=True, winsorize_lower_quantile=0.01, winsorize_upper_quantile=0.99, - metric=['MI', 'MI', 'CC'], + metric=["MI", "MI", "CC"], metric_weight=[1, 1, 1], radius_or_number_of_bins=[32, 32, 4], - sampling_strategy=['Regular', 'Regular', None], + sampling_strategy=["Regular", "Regular", None], sampling_percentage=[0.25, 0.25, None], number_of_iterations=[ [1000, 500, 250, 100], [1000, 500, 250, 100], - [100, 100, 70, 20] + [100, 100, 70, 20], ], convergence_threshold=[1e-8, 1e-8, 1e-9], convergence_window_size=[10, 10, 15], - transforms=['Rigid', 'Affine', 'SyN'], + transforms=["Rigid", "Affine", "SyN"], transform_parameters=[[0.1], [0.1], [0.1, 3, 0]], - shrink_factors=[ - [8, 4, 2, 1], - [8, 4, 2, 1], - [6, 4, 2, 1] - ], - smoothing_sigmas=[ - [3, 2, 1, 0], - [3, 2, 1, 0], - [3, 2, 1, 0] - ] + shrink_factors=[[8, 4, 2, 1], [8, 4, 2, 1], [6, 4, 2, 1]], + smoothing_sigmas=[[3, 2, 1, 0], [3, 2, 1, 0], [3, 2, 1, 0]], ) wf.run() diff --git a/CPAC/registration/utils.py b/CPAC/registration/utils.py index 1185f0190b..5fd7310d57 100644 --- a/CPAC/registration/utils.py +++ b/CPAC/registration/utils.py @@ -4,32 +4,30 @@ def single_ants_xfm_to_list(transform): - transform_list = [transform] - return transform_list + return [transform] def interpolation_string(interpolation, reg_tool): - if reg_tool == 'ants': + if reg_tool == "ants": pass - elif reg_tool == 'fsl': + elif reg_tool == "fsl": # translate to FSL # warning: flirt requires 'nearestneighbour', but FSL applywarp uses # 'nn', so this is designed for applywarp, as all FSL xfm's # in C-PAC are now converted to .nii.gz - interpolation = interpolation.replace('NearestNeighbor', 'nn') + interpolation = interpolation.replace("NearestNeighbor", "nn") return interpolation def combine_inputs_into_list(input1, input2, input3): - outputs_list = [input1, input2, input3] - return outputs_list + return [input1, input2, input3] def seperate_warps_list(warp_list, selection): selected_warp = None for warp in warp_list: - if selection == 'Warp': - if '3Warp' in warp or '2Warp' in warp or '1Warp' in warp: + if selection == "Warp": + if "3Warp" in warp or "2Warp" in warp or "1Warp" in warp: selected_warp = warp else: if selection in warp: @@ -39,395 +37,553 @@ def seperate_warps_list(warp_list, selection): def check_transforms(transform_list): transform_number = list(filter(None, transform_list)) - return [(transform_number[index]) for index in - range(len(transform_number))], len(transform_number) + return [(transform_number[index]) for index in range(len(transform_number))], len( + transform_number + ) def generate_inverse_transform_flags(transform_list): inverse_transform_flags = [] for transform in transform_list: # check `blip_warp_inverse` file name and rename it - if 'WARPINV' in transform: + if "WARPINV" in transform: inverse_transform_flags.append(False) - if 'updated_affine' in transform: + if "updated_affine" in transform: inverse_transform_flags.append(True) - if 'Initial' in transform: + if "Initial" in transform: inverse_transform_flags.append(True) - if 'Rigid' in transform: + if "Rigid" in transform: inverse_transform_flags.append(True) - if 'Affine' in transform: + if "Affine" in transform: inverse_transform_flags.append(True) - if 'InverseWarp' in transform: + if "InverseWarp" in transform: inverse_transform_flags.append(False) return inverse_transform_flags -def hardcoded_reg(moving_brain, reference_brain, moving_skull, - reference_skull, ants_para, moving_mask=None, - reference_mask=None, fixed_image_mask=None, interp=None, - reg_with_skull=0): +def hardcoded_reg( + moving_brain, + reference_brain, + moving_skull, + reference_skull, + ants_para, + moving_mask=None, + reference_mask=None, + fixed_image_mask=None, + interp=None, + reg_with_skull=0, +): # TODO: expand transforms to cover all in ANTs para regcmd = ["antsRegistration"] for para_index in range(len(ants_para)): for para_type in ants_para[para_index]: - if para_type == 'dimensionality': + if para_type == "dimensionality": if ants_para[para_index][para_type] not in [2, 3, 4]: - err_msg = 'Dimensionality specified in ANTs parameters: %d, is not supported. ' \ - 'Change to 2, 3, or 4 and try again' % \ - ants_para[para_index][para_type] + err_msg = ( + "Dimensionality specified in ANTs parameters: %d, is not supported. " + "Change to 2, 3, or 4 and try again" + % ants_para[para_index][para_type] + ) raise Exception(err_msg) else: regcmd.append("--dimensionality") regcmd.append(str(ants_para[para_index][para_type])) - elif para_type == 'verbose': + elif para_type == "verbose": if ants_para[para_index][para_type] not in [0, 1]: - err_msg = 'Verbose output option in ANTs parameters: %d, is not supported. ' \ - 'Change to 0 or 1 and try again' % \ - ants_para[para_index][para_type] + err_msg = ( + "Verbose output option in ANTs parameters: %d, is not supported. " + "Change to 0 or 1 and try again" + % ants_para[para_index][para_type] + ) raise Exception(err_msg) else: regcmd.append("--verbose") regcmd.append(str(ants_para[para_index][para_type])) - elif para_type == 'float': + elif para_type == "float": if ants_para[para_index][para_type] not in [0, 1]: - err_msg = 'Float option in ANTs parameters: %d, is not supported. ' \ - 'Change to 0 or 1 and try again' % \ - ants_para[para_index][para_type] + err_msg = ( + "Float option in ANTs parameters: %d, is not supported. " + "Change to 0 or 1 and try again" + % ants_para[para_index][para_type] + ) raise Exception(err_msg) else: regcmd.append("--float") regcmd.append(str(ants_para[para_index][para_type])) - elif para_type == 'collapse-output-transforms': + elif para_type == "collapse-output-transforms": if ants_para[para_index][para_type] not in [0, 1]: - err_msg = 'collapse-output-transforms specified in ANTs parameters: %d, is not supported. ' \ - 'Change to 0 or 1 and try again' % \ - ants_para[para_index][para_type] + err_msg = ( + "collapse-output-transforms specified in ANTs parameters: %d, is not supported. " + "Change to 0 or 1 and try again" + % ants_para[para_index][para_type] + ) raise Exception(err_msg) else: regcmd.append("--collapse-output-transforms") regcmd.append(str(ants_para[para_index][para_type])) - elif para_type == 'winsorize-image-intensities': - if ants_para[para_index][para_type]['lowerQuantile'] is None or ants_para[para_index][para_type]['upperQuantile'] is None: - err_msg = 'Please specifiy lowerQuantile and upperQuantile of ANTs parameters --winsorize-image-intensities in pipeline config. ' + elif para_type == "winsorize-image-intensities": + if ( + ants_para[para_index][para_type]["lowerQuantile"] is None + or ants_para[para_index][para_type]["upperQuantile"] is None + ): + err_msg = "Please specifiy lowerQuantile and upperQuantile of ANTs parameters --winsorize-image-intensities in pipeline config. " raise Exception(err_msg) else: regcmd.append("--winsorize-image-intensities") - regcmd.append("[{0},{1}]".format(ants_para[para_index][para_type]['lowerQuantile'], - ants_para[para_index][para_type]['upperQuantile'])) - - elif para_type == 'initial-moving-transform': - if ants_para[para_index][para_type][ - 'initializationFeature'] is None: - err_msg = 'Please specifiy initializationFeature of ANTs parameters in pipeline config. ' + regcmd.append( + "[{0},{1}]".format( + ants_para[para_index][para_type]["lowerQuantile"], + ants_para[para_index][para_type]["upperQuantile"], + ) + ) + + elif para_type == "initial-moving-transform": + if ants_para[para_index][para_type]["initializationFeature"] is None: + err_msg = "Please specifiy initializationFeature of ANTs parameters in pipeline config. " raise Exception(err_msg) else: regcmd.append("--initial-moving-transform") if reg_with_skull == 1: - regcmd.append("[{0},{1},{2}]".format( - reference_skull, moving_skull, - ants_para[para_index][para_type][ - 'initializationFeature'])) + regcmd.append( + "[{0},{1},{2}]".format( + reference_skull, + moving_skull, + ants_para[para_index][para_type][ + "initializationFeature" + ], + ) + ) else: - regcmd.append("[{0},{1},{2}]".format( - reference_brain, moving_brain, - ants_para[para_index][para_type][ - 'initializationFeature'])) - - elif para_type == 'transforms': - for trans_index in range( - len(ants_para[para_index][para_type])): - for trans_type in ants_para[para_index][para_type][ - trans_index]: + regcmd.append( + "[{0},{1},{2}]".format( + reference_brain, + moving_brain, + ants_para[para_index][para_type][ + "initializationFeature" + ], + ) + ) + + elif para_type == "transforms": + for trans_index in range(len(ants_para[para_index][para_type])): + for trans_type in ants_para[para_index][para_type][trans_index]: regcmd.append("--transform") - if trans_type == 'Rigid' or trans_type == 'Affine': - if ants_para[para_index][para_type][trans_index][ - trans_type]['gradientStep'] is None: - err_msg = 'Please specifiy % s Gradient Step of ANTs parameters in pipeline config. ' % trans_type + if trans_type in ("Rigid", "Affine"): + if ( + ants_para[para_index][para_type][trans_index][ + trans_type + ]["gradientStep"] + is None + ): + err_msg = ( + "Please specifiy % s Gradient Step of ANTs parameters in pipeline config. " + % trans_type + ) raise Exception(err_msg) else: - regcmd.append("{0}[{1}]".format( - trans_type, - ants_para[para_index][para_type][ - trans_index][trans_type][ - 'gradientStep'])) - - if trans_type == 'SyN': - if ants_para[para_index][para_type][trans_index][ - trans_type]['gradientStep'] is None: - err_msg = 'Please specifiy % s Gradient Step of ANTs parameters in pipeline config. ' % trans_type + regcmd.append( + "{0}[{1}]".format( + trans_type, + ants_para[para_index][para_type][trans_index][ + trans_type + ]["gradientStep"], + ) + ) + + if trans_type == "SyN": + if ( + ants_para[para_index][para_type][trans_index][ + trans_type + ]["gradientStep"] + is None + ): + err_msg = ( + "Please specifiy % s Gradient Step of ANTs parameters in pipeline config. " + % trans_type + ) raise Exception(err_msg) else: SyN_para = [] - SyN_para.append("{0}".format( - ants_para[para_index][para_type][ - trans_index][trans_type][ - 'gradientStep'])) - if ants_para[para_index][para_type][ - trans_index][trans_type][ - 'updateFieldVarianceInVoxelSpace'] is not None: - SyN_para.append("{0}".format( - ants_para[para_index][para_type][ - trans_index][trans_type][ - 'updateFieldVarianceInVoxelSpace'])) - if ants_para[para_index][para_type][ - trans_index][trans_type][ - 'totalFieldVarianceInVoxelSpace'] is not None: - SyN_para.append("{0}".format( - ants_para[para_index][para_type][ - trans_index][trans_type][ - 'totalFieldVarianceInVoxelSpace'])) - SyN_para = ','.join([str(elem) - for elem in SyN_para]) - regcmd.append("{0}[{1}]".format( - trans_type, SyN_para)) - - if ants_para[para_index][para_type][trans_index][ - trans_type]['metric']['type'] == 'MI': - if ants_para[para_index][para_type][trans_index][ - trans_type]['metric'][ - 'metricWeight'] is None or \ + SyN_para.append( + "{0}".format( + ants_para[para_index][para_type][trans_index][ + trans_type + ]["gradientStep"] + ) + ) + if ( + ants_para[para_index][para_type][trans_index][ + trans_type + ]["updateFieldVarianceInVoxelSpace"] + is not None + ): + SyN_para.append( + "{0}".format( + ants_para[para_index][para_type][ + trans_index + ][trans_type][ + "updateFieldVarianceInVoxelSpace" + ] + ) + ) + if ( + ants_para[para_index][para_type][trans_index][ + trans_type + ]["totalFieldVarianceInVoxelSpace"] + is not None + ): + SyN_para.append( + "{0}".format( ants_para[para_index][para_type][ - trans_index][trans_type][ - 'metric'][ - 'numberOfBins'] is None: - err_msg = 'Please specifiy metricWeight and numberOfBins for metric MI of ANTs parameters in pipeline config.' + trans_index + ][trans_type][ + "totalFieldVarianceInVoxelSpace" + ] + ) + ) + SyN_para = ",".join([str(elem) for elem in SyN_para]) + regcmd.append("{0}[{1}]".format(trans_type, SyN_para)) + + if ( + ants_para[para_index][para_type][trans_index][trans_type][ + "metric" + ]["type"] + == "MI" + ): + if ( + ants_para[para_index][para_type][trans_index][ + trans_type + ]["metric"]["metricWeight"] + is None + or ants_para[para_index][para_type][trans_index][ + trans_type + ]["metric"]["numberOfBins"] + is None + ): + err_msg = "Please specifiy metricWeight and numberOfBins for metric MI of ANTs parameters in pipeline config." raise Exception(err_msg) else: MI_para = [] - MI_para.append("{0},{1}".format( - ants_para[para_index][para_type][ - trans_index][trans_type]['metric'] - ['metricWeight'], - ants_para[para_index][para_type][ - trans_index][trans_type]['metric'][ - 'numberOfBins'])) - if 'samplingStrategy' in \ - ants_para[para_index][para_type][ - trans_index][trans_type][ - 'metric'] and \ - ants_para[para_index][ - para_type][trans_index][ - trans_type]['metric'][ - 'samplingStrategy'] in [ - 'None', 'Regular', 'Random']: - MI_para.append("{0}".format( - ants_para[para_index][para_type][ - trans_index][trans_type][ - 'metric']['samplingStrategy'])) - if 'samplingPercentage' in \ - ants_para[para_index][para_type][ - trans_index][trans_type][ - 'metric'] and \ - ants_para[para_index][ - para_type][trans_index][ - trans_type]['metric'][ - 'samplingPercentage'] is not None: - MI_para.append("{0}".format( - ants_para[para_index][para_type][ - trans_index][trans_type][ - 'metric']['samplingPercentage'])) - MI_para = ','.join( - [str(elem) for elem in MI_para]) + MI_para.append( + "{0},{1}".format( + ants_para[para_index][para_type][trans_index][ + trans_type + ]["metric"]["metricWeight"], + ants_para[para_index][para_type][trans_index][ + trans_type + ]["metric"]["numberOfBins"], + ) + ) + if "samplingStrategy" in ants_para[para_index][ + para_type + ][trans_index][trans_type]["metric"] and ants_para[ + para_index + ][para_type][trans_index][trans_type]["metric"][ + "samplingStrategy" + ] in ["None", "Regular", "Random"]: + MI_para.append( + "{0}".format( + ants_para[para_index][para_type][ + trans_index + ][trans_type]["metric"]["samplingStrategy"] + ) + ) + if ( + "samplingPercentage" + in ants_para[para_index][para_type][trans_index][ + trans_type + ]["metric"] + and ants_para[para_index][para_type][trans_index][ + trans_type + ]["metric"]["samplingPercentage"] + is not None + ): + MI_para.append( + "{0}".format( + ants_para[para_index][para_type][ + trans_index + ][trans_type]["metric"][ + "samplingPercentage" + ] + ) + ) + MI_para = ",".join([str(elem) for elem in MI_para]) regcmd.append("--metric") if reg_with_skull == 1: - regcmd.append("MI[{0},{1},{2}]".format( - reference_skull, moving_skull, MI_para)) + regcmd.append( + "MI[{0},{1},{2}]".format( + reference_skull, moving_skull, MI_para + ) + ) else: - regcmd.append("MI[{0},{1},{2}]".format( - reference_brain, moving_brain, MI_para)) - - if ants_para[para_index][para_type][trans_index][ - trans_type]['metric']['type'] == 'CC': - if ants_para[para_index][para_type][trans_index][ - trans_type]['metric'][ - 'metricWeight'] is None or \ - ants_para[para_index][para_type][ - trans_index][trans_type][ - 'metric']['radius'] is None: - err_msg = 'Please specifiy metricWeight and radius for metric CC of ANTs parameters in pipeline config.' + regcmd.append( + "MI[{0},{1},{2}]".format( + reference_brain, moving_brain, MI_para + ) + ) + + if ( + ants_para[para_index][para_type][trans_index][trans_type][ + "metric" + ]["type"] + == "CC" + ): + if ( + ants_para[para_index][para_type][trans_index][ + trans_type + ]["metric"]["metricWeight"] + is None + or ants_para[para_index][para_type][trans_index][ + trans_type + ]["metric"]["radius"] + is None + ): + err_msg = "Please specifiy metricWeight and radius for metric CC of ANTs parameters in pipeline config." raise Exception(err_msg) else: CC_para = [] - CC_para.append("{0},{1}".format( - ants_para[para_index][para_type][ - trans_index][trans_type]['metric'] - ['metricWeight'], - ants_para[para_index][para_type][ - trans_index][trans_type]['metric'][ - 'radius'])) - if 'samplingStrategy' in \ - ants_para[para_index][para_type][ - trans_index][trans_type][ - 'metric'] and \ - ants_para[para_index][ - para_type][trans_index][ - trans_type]['metric'][ - 'samplingStrategy'] in [ - 'None', 'Regular', 'Random']: - CC_para.append("{0}".format( - ants_para[para_index][para_type][ - trans_index][trans_type][ - 'metric']['samplingStrategy'])) - if 'samplingPercentage' in \ - ants_para[para_index][para_type][ - trans_index][trans_type][ - 'metric'] and \ - ants_para[para_index][ - para_type][trans_index][ - trans_type]['metric'][ - 'samplingPercentage'] is not None: - CC_para.append("{0}".format( - ants_para[para_index][para_type][ - trans_index][trans_type][ - 'metric']['samplingPercentage'])) - CC_para = ','.join( - [str(elem) for elem in CC_para]) + CC_para.append( + "{0},{1}".format( + ants_para[para_index][para_type][trans_index][ + trans_type + ]["metric"]["metricWeight"], + ants_para[para_index][para_type][trans_index][ + trans_type + ]["metric"]["radius"], + ) + ) + if "samplingStrategy" in ants_para[para_index][ + para_type + ][trans_index][trans_type]["metric"] and ants_para[ + para_index + ][para_type][trans_index][trans_type]["metric"][ + "samplingStrategy" + ] in ["None", "Regular", "Random"]: + CC_para.append( + "{0}".format( + ants_para[para_index][para_type][ + trans_index + ][trans_type]["metric"]["samplingStrategy"] + ) + ) + if ( + "samplingPercentage" + in ants_para[para_index][para_type][trans_index][ + trans_type + ]["metric"] + and ants_para[para_index][para_type][trans_index][ + trans_type + ]["metric"]["samplingPercentage"] + is not None + ): + CC_para.append( + "{0}".format( + ants_para[para_index][para_type][ + trans_index + ][trans_type]["metric"][ + "samplingPercentage" + ] + ) + ) + CC_para = ",".join([str(elem) for elem in CC_para]) regcmd.append("--metric") - regcmd.append("CC[{0},{1},{2}]".format( - reference_skull, moving_skull, CC_para)) - - if 'convergence' in \ - ants_para[para_index][para_type][trans_index][ - trans_type]: + regcmd.append( + "CC[{0},{1},{2}]".format( + reference_skull, moving_skull, CC_para + ) + ) + + if ( + "convergence" + in ants_para[para_index][para_type][trans_index][trans_type] + ): convergence_para = [] - if ants_para[para_index][para_type][trans_index][ - trans_type]['convergence'][ - 'iteration'] is None: - err_msg = 'Please specifiy convergence iteration of ANTs parameters in pipeline config.' + if ( + ants_para[para_index][para_type][trans_index][ + trans_type + ]["convergence"]["iteration"] + is None + ): + err_msg = "Please specifiy convergence iteration of ANTs parameters in pipeline config." raise Exception(err_msg) else: - convergence_para.append("{0}".format( - ants_para[para_index][para_type][ - trans_index][trans_type][ - 'convergence']['iteration'])) - if 'convergenceThreshold' in \ - ants_para[para_index][para_type][ - trans_index][trans_type][ - 'convergence'] and \ - ants_para[para_index][ - para_type][trans_index][ - trans_type][ - 'convergence'][ - 'convergenceThreshold'] is not None: - convergence_para.append("{0}".format( - ants_para[para_index][para_type][ - trans_index][trans_type][ - 'convergence'][ - 'convergenceThreshold'])) - if 'convergenceWindowSize' in \ - ants_para[para_index][para_type][ - trans_index][trans_type][ - 'convergence'] and \ - ants_para[para_index][ - para_type][trans_index][ - trans_type][ - 'convergence'][ - 'convergenceWindowSize'] is not None: - convergence_para.append("{0}".format( - ants_para[para_index][para_type][ - trans_index][trans_type][ - 'convergence'][ - 'convergenceWindowSize'])) - convergence_para = ','.join( - [str(elem) for elem in convergence_para]) + convergence_para.append( + "{0}".format( + ants_para[para_index][para_type][trans_index][ + trans_type + ]["convergence"]["iteration"] + ) + ) + if ( + "convergenceThreshold" + in ants_para[para_index][para_type][trans_index][ + trans_type + ]["convergence"] + and ants_para[para_index][para_type][trans_index][ + trans_type + ]["convergence"]["convergenceThreshold"] + is not None + ): + convergence_para.append( + "{0}".format( + ants_para[para_index][para_type][ + trans_index + ][trans_type]["convergence"][ + "convergenceThreshold" + ] + ) + ) + if ( + "convergenceWindowSize" + in ants_para[para_index][para_type][trans_index][ + trans_type + ]["convergence"] + and ants_para[para_index][para_type][trans_index][ + trans_type + ]["convergence"]["convergenceWindowSize"] + is not None + ): + convergence_para.append( + "{0}".format( + ants_para[para_index][para_type][ + trans_index + ][trans_type]["convergence"][ + "convergenceWindowSize" + ] + ) + ) + convergence_para = ",".join( + [str(elem) for elem in convergence_para] + ) regcmd.append("--convergence") - regcmd.append( - "[{0}]".format(convergence_para)) - - if 'smoothing-sigmas' in \ - ants_para[para_index][para_type][trans_index][ - trans_type] and \ - ants_para[para_index][para_type][ - trans_index][trans_type][ - 'smoothing-sigmas'] is not None: + regcmd.append("[{0}]".format(convergence_para)) + + if ( + "smoothing-sigmas" + in ants_para[para_index][para_type][trans_index][trans_type] + and ants_para[para_index][para_type][trans_index][ + trans_type + ]["smoothing-sigmas"] + is not None + ): regcmd.append("--smoothing-sigmas") - regcmd.append("{0}".format( - ants_para[para_index][para_type][trans_index][ - trans_type]['smoothing-sigmas'])) - - if 'shrink-factors' in \ - ants_para[para_index][para_type][trans_index][ - trans_type] and \ - ants_para[para_index][para_type][ - trans_index][trans_type][ - 'shrink-factors'] is not None: + regcmd.append( + "{0}".format( + ants_para[para_index][para_type][trans_index][ + trans_type + ]["smoothing-sigmas"] + ) + ) + + if ( + "shrink-factors" + in ants_para[para_index][para_type][trans_index][trans_type] + and ants_para[para_index][para_type][trans_index][ + trans_type + ]["shrink-factors"] + is not None + ): regcmd.append("--shrink-factors") - regcmd.append("{0}".format( - ants_para[para_index][para_type][trans_index][ - trans_type]['shrink-factors'])) - - if 'use-histogram-matching' in \ - ants_para[para_index][para_type][trans_index][ - trans_type]: + regcmd.append( + "{0}".format( + ants_para[para_index][para_type][trans_index][ + trans_type + ]["shrink-factors"] + ) + ) + + if ( + "use-histogram-matching" + in ants_para[para_index][para_type][trans_index][trans_type] + ): if ants_para[para_index][para_type][trans_index][ - trans_type]['use-histogram-matching']: + trans_type + ]["use-histogram-matching"]: regcmd.append("--use-histogram-matching") regcmd.append("1") - if 'winsorize-image-intensities' in \ - ants_para[para_index][para_type][trans_index][ - trans_type] and \ - ants_para[para_index][para_type][ - trans_index][trans_type][ - 'winsorize-image-intensities'][ - 'lowerQuantile'] is not None and \ - ants_para[para_index][para_type][ - trans_index][trans_type][ - 'winsorize-image-intensities'][ - 'upperQuantile'] is not None: + if ( + "winsorize-image-intensities" + in ants_para[para_index][para_type][trans_index][trans_type] + and ants_para[para_index][para_type][trans_index][ + trans_type + ]["winsorize-image-intensities"]["lowerQuantile"] + is not None + and ants_para[para_index][para_type][trans_index][ + trans_type + ]["winsorize-image-intensities"]["upperQuantile"] + is not None + ): regcmd.append("--winsorize-image-intensities") - regcmd.append("[{0},{1}]".format( - ants_para[para_index][para_type][trans_index][ - trans_type]['winsorize-image-intensities'] - ['lowerQuantile'], - ants_para[para_index][para_type][trans_index][ - trans_type][ - 'winsorize-image-intensities'][ - 'upperQuantile'])) - - if 'masks' in ants_para[para_index][para_type][trans_index][ - trans_type] and ants_para[para_index][para_type][ - trans_index][trans_type]['masks'] is not None: - if ants_para[para_index][para_type][trans_index][trans_type]['masks']: + regcmd.append( + "[{0},{1}]".format( + ants_para[para_index][para_type][trans_index][ + trans_type + ]["winsorize-image-intensities"]["lowerQuantile"], + ants_para[para_index][para_type][trans_index][ + trans_type + ]["winsorize-image-intensities"]["upperQuantile"], + ) + ) + + if ( + "masks" + in ants_para[para_index][para_type][trans_index][trans_type] + and ants_para[para_index][para_type][trans_index][ + trans_type + ]["masks"] + is not None + ): + if ants_para[para_index][para_type][trans_index][ + trans_type + ]["masks"]: regcmd.append("--masks") - regcmd.append("[{0},{1}]".format(reference_mask, moving_mask)) + regcmd.append( + "[{0},{1}]".format(reference_mask, moving_mask) + ) else: regcmd.append("--masks") regcmd.append("[NULL,NULL]") - elif para_type == 'masks': + elif para_type == "masks": # lesion preproc has if fixed_image_mask is not None: regcmd.append("--masks") regcmd.append(str(fixed_image_mask)) else: - if ants_para[para_index][para_type][ - 'fixed_image_mask'] == False and \ - ants_para[para_index][para_type][ - 'moving_image_mask'] == True: - err_msg = 'Masks option in ANTs parameters: %d is not supported. ' \ - 'Please set `fixed_image_mask` as True. ' \ - 'Or set both `fixed_image_mask` and `moving_image_mask` as False' % \ - ants_para[para_index][para_type] + if ( + ants_para[para_index][para_type]["fixed_image_mask"] is False + and ants_para[para_index][para_type]["moving_image_mask"] + is True + ): + err_msg = ( + "Masks option in ANTs parameters: %d is not supported. " + "Please set `fixed_image_mask` as True. " + "Or set both `fixed_image_mask` and `moving_image_mask` as False" + % ants_para[para_index][para_type] + ) raise Exception(err_msg) - elif ants_para[para_index][para_type][ - 'fixed_image_mask'] == True and \ - ants_para[para_index][para_type][ - 'moving_image_mask'] == True: + elif ( + ants_para[para_index][para_type]["fixed_image_mask"] is True + and ants_para[para_index][para_type]["moving_image_mask"] + is True + ): regcmd.append("--masks") - regcmd.append('[' + str(reference_mask) + ',' + str( - moving_mask) + ']') - elif ants_para[para_index][para_type][ - 'fixed_image_mask'] == True and \ - ants_para[para_index][para_type][ - 'moving_image_mask'] == False: + regcmd.append( + "[" + str(reference_mask) + "," + str(moving_mask) + "]" + ) + elif ( + ants_para[para_index][para_type]["fixed_image_mask"] is True + and ants_para[para_index][para_type]["moving_image_mask"] + is False + ): regcmd.append("--masks") - regcmd.append('[' + str(reference_mask) + ']') + regcmd.append("[" + str(reference_mask) + "]") else: continue @@ -439,20 +595,22 @@ def hardcoded_reg(moving_brain, reference_brain, moving_skull, regcmd.append("[transform,transform_Warped.nii.gz]") # write out the actual command-line entry for testing/validation later - command_file = os.path.join(os.getcwd(), 'command.txt') - with open(command_file, 'wt') as f: - f.write(' '.join(regcmd)) + command_file = os.path.join(os.getcwd(), "command.txt") + with open(command_file, "wt") as f: + f.write(" ".join(regcmd)) try: - retcode = subprocess.check_output(regcmd) + subprocess.check_output(regcmd) except Exception as e: - raise Exception('[!] ANTS registration did not complete successfully.' - '\n\nError details:\n{0}\n{1}\n'.format(e, e.output)) + raise Exception( + "[!] ANTS registration did not complete successfully." + "\n\nError details:\n{0}\n{1}\n".format(e, e.output) + ) warp_list = [] warped_image = None - files = [f for f in os.listdir('.') if os.path.isfile(f)] + files = [f for f in os.listdir(".") if os.path.isfile(f)] for f in files: if ("transform" in f) and ("Warped" not in f): @@ -461,9 +619,11 @@ def hardcoded_reg(moving_brain, reference_brain, moving_skull, warped_image = os.getcwd() + "/" + f if not warped_image: - raise Exception("\n\n[!] No registration output file found. ANTS " - "registration may not have completed " - "successfully.\n\n") + raise Exception( + "\n\n[!] No registration output file found. ANTS " + "registration may not have completed " + "successfully.\n\n" + ) return warp_list, warped_image @@ -471,27 +631,26 @@ def hardcoded_reg(moving_brain, reference_brain, moving_skull, def change_itk_transform_type(input_affine_file): """ this function takes in the affine.txt produced by the c3d_affine_tool - (which converted an FSL FLIRT affine.mat into the affine.txt) + (which converted an FSL FLIRT affine.mat into the affine.txt). it then modifies the 'Transform Type' of this affine.txt so that it is compatible with the antsApplyTransforms tool and produces a new affine file titled 'updated_affine.txt' """ - new_file_lines = [] with open(input_affine_file) as f: for line in f: - if 'Transform:' in line: - if 'MatrixOffsetTransformBase_double_3_3' in line: - transform_line = 'Transform: AffineTransform_double_3_3\n' + if "Transform:" in line: + if "MatrixOffsetTransformBase_double_3_3" in line: + transform_line = "Transform: AffineTransform_double_3_3\n" new_file_lines.append(transform_line) else: new_file_lines.append(line) - updated_affine_file = os.path.join(os.getcwd(), 'updated_affine.txt') + updated_affine_file = os.path.join(os.getcwd(), "updated_affine.txt") - with open(updated_affine_file, 'wt') as f: + with open(updated_affine_file, "wt") as f: for line in new_file_lines: f.write(line) @@ -499,7 +658,7 @@ def change_itk_transform_type(input_affine_file): def one_d_to_mat(one_d_filename): - """Convert a .1D file to a .mat directory + """Convert a .1D file to a .mat directory. Parameters ---------- @@ -511,130 +670,177 @@ def one_d_to_mat(one_d_filename): mat_filenames : list of str The of paths in the .mat directory created """ - mat_dirname = one_d_filename.replace('.1D', '.mat') - with open(one_d_filename, 'r') as one_d_file: - rows = [np.reshape(row, (4, 4)).astype('float') for row in [[ - term.strip() for term in row.split(' ') if term.strip() - ] + [0, 0, 0, 1] for row in [ - line.strip() for line in one_d_file.readlines() if - not line.startswith('#')]]] + mat_dirname = one_d_filename.replace(".1D", ".mat") + with open(one_d_filename, "r") as one_d_file: + rows = [ + np.reshape(row, (4, 4)).astype("float") + for row in [ + [term.strip() for term in row.split(" ") if term.strip()] + [0, 0, 0, 1] + for row in [ + line.strip() + for line in one_d_file.readlines() + if not line.startswith("#") + ] + ] + ] try: os.mkdir(mat_dirname) except FileExistsError: pass for i, row in enumerate(rows): - np.savetxt(os.path.join(mat_dirname, f'MAT_{i:04}'), - row, fmt='%.5f', delimiter=' ') - mat_filenames = [os.path.join(mat_dirname, filename) for - filename in os.listdir(mat_dirname)] + np.savetxt( + os.path.join(mat_dirname, f"MAT_{i:04}"), row, fmt="%.5f", delimiter=" " + ) + mat_filenames = [ + os.path.join(mat_dirname, filename) for filename in os.listdir(mat_dirname) + ] mat_filenames.sort() return mat_filenames -def run_ants_apply_warp(moving_image, reference, initial=None, rigid=None, - affine=None, nonlinear=None, func_to_anat=None, - anatomical_brain=None, dim=3, interp='Linear', - inverse=False): +def run_ants_apply_warp( + moving_image, + reference, + initial=None, + rigid=None, + affine=None, + nonlinear=None, + func_to_anat=None, + anatomical_brain=None, + dim=3, + interp="Linear", + inverse=False, +): """Apply a transform using ANTs transforms.""" - import os import subprocess if func_to_anat: # this assumes the func->anat affine transform is FSL-based and needs # to be converted to ITK format via c3d_affine_tool - cmd = ['c3d_affine_tool', '-ref', anatomical_brain, '-src', - moving_image, func_to_anat, '-fsl2ras', '-oitk', 'affine.txt'] - retcode = subprocess.check_output(cmd) - func_to_anat = change_itk_transform_type(os.path.join(os.getcwd(), - 'affine.txt')) - - out_image = os.path.join(os.getcwd(), moving_image[moving_image.rindex( - '/') + 1:moving_image.rindex('.nii.gz')] + '_warp.nii.gz') - - cmd = ['antsApplyTransforms', '-d', str(dim), '-i', moving_image, '-r', - reference, '-o', out_image, '-n', interp] + cmd = [ + "c3d_affine_tool", + "-ref", + anatomical_brain, + "-src", + moving_image, + func_to_anat, + "-fsl2ras", + "-oitk", + "affine.txt", + ] + subprocess.check_output(cmd) + func_to_anat = change_itk_transform_type( + os.path.join(os.getcwd(), "affine.txt") + ) + + out_image = os.path.join( + os.getcwd(), + moving_image[moving_image.rindex("/") + 1 : moving_image.rindex(".nii.gz")] + + "_warp.nii.gz", + ) + + cmd = [ + "antsApplyTransforms", + "-d", + str(dim), + "-i", + moving_image, + "-r", + reference, + "-o", + out_image, + "-n", + interp, + ] if nonlinear: - cmd.append('-t') + cmd.append("-t") if inverse: - cmd.append('[{0}, {1}]'.format(os.path.abspath(nonlinear), '1')) + cmd.append("[{0}, {1}]".format(os.path.abspath(nonlinear), "1")) else: cmd.append(os.path.abspath(nonlinear)) if affine: - cmd.append('-t') + cmd.append("-t") if inverse: - cmd.append('[{0}, {1}]'.format(os.path.abspath(affine), '1')) + cmd.append("[{0}, {1}]".format(os.path.abspath(affine), "1")) else: cmd.append(os.path.abspath(affine)) if rigid: - cmd.append('-t') + cmd.append("-t") if inverse: - cmd.append('[{0}, {1}]'.format(os.path.abspath(rigid), '1')) + cmd.append("[{0}, {1}]".format(os.path.abspath(rigid), "1")) else: cmd.append(os.path.abspath(rigid)) if initial: - cmd.append('-t') + cmd.append("-t") if inverse: - cmd.append('[{0}, {1}]'.format(os.path.abspath(initial), '1')) + cmd.append("[{0}, {1}]".format(os.path.abspath(initial), "1")) else: cmd.append(os.path.abspath(initial)) if func_to_anat: - cmd.append('-t') + cmd.append("-t") if inverse: - cmd.append( - '[{0}, {1}]'.format(os.path.abspath(func_to_anat), '1')) + cmd.append("[{0}, {1}]".format(os.path.abspath(func_to_anat), "1")) else: cmd.append(os.path.abspath(func_to_anat)) - retcode = subprocess.check_output(cmd) + subprocess.check_output(cmd) return out_image -def cpac_ants_apply_nonlinear_inverse_warp(cpac_dir, moving_image, reference, - dim=3, interp='Linear'): +def cpac_ants_apply_nonlinear_inverse_warp( + cpac_dir, moving_image, reference, dim=3, interp="Linear" +): """Run antsApplyTransforms for inverse warping when given a C-PAC output - directory.""" - + directory. + """ import os cpac_dir = os.path.abspath(cpac_dir) for dir in os.listdir(cpac_dir): - if 'ants_initial_xfm' in dir: + if "ants_initial_xfm" in dir: pass # run_ants_apply_warp() def run_c3d(reference_file, source_file, transform_file): - import os import subprocess - itk_transform = os.path.join(os.getcwd(), 'affine.txt') - - cmd = ['c3d_affine_tool', '-ref', reference_file, '-src', - source_file, transform_file, '-fsl2ras', '-oitk', itk_transform] - retcode = subprocess.check_output(cmd) + itk_transform = os.path.join(os.getcwd(), "affine.txt") + + cmd = [ + "c3d_affine_tool", + "-ref", + reference_file, + "-src", + source_file, + transform_file, + "-fsl2ras", + "-oitk", + itk_transform, + ] + subprocess.check_output(cmd) return itk_transform def run_c4d(input, output_name): - import os - output1 = os.path.join(os.getcwd(), output_name+'1.nii.gz') - output2 = os.path.join(os.getcwd(), output_name+'2.nii.gz') - output3 = os.path.join(os.getcwd(), output_name+'3.nii.gz') + output1 = os.path.join(os.getcwd(), output_name + "1.nii.gz") + output2 = os.path.join(os.getcwd(), output_name + "2.nii.gz") + output3 = os.path.join(os.getcwd(), output_name + "3.nii.gz") - cmd = 'c4d -mcs %s -oo %s %s %s' % (input, output1, output2, output3) + cmd = "c4d -mcs %s -oo %s %s %s" % (input, output1, output2, output3) os.system(cmd) return output1, output2, output3 diff --git a/CPAC/reho/__init__.py b/CPAC/reho/__init__.py index d40a7cb168..3cd1d1c07e 100644 --- a/CPAC/reho/__init__.py +++ b/CPAC/reho/__init__.py @@ -1,11 +1,4 @@ from .reho import create_reho +from .utils import compute_reho, f_kendall, getOpString -from .utils import f_kendall, \ - compute_reho, \ - getOpString - - -__all__ = ['create_reho', \ - 'f_kendall', \ - 'getOpString', \ - 'compute_reho'] +__all__ = ["create_reho", "f_kendall", "getOpString", "compute_reho"] diff --git a/CPAC/reho/reho.py b/CPAC/reho/reho.py index e170d5c2e1..fe11205907 100644 --- a/CPAC/reho/reho.py +++ b/CPAC/reho/reho.py @@ -1,21 +1,19 @@ # coding: utf-8 +import nipype.interfaces.utility as util + from CPAC.pipeline import nipype_pipeline_engine as pe from CPAC.pipeline.nodeblock import nodeblock -import nipype.interfaces.fsl as fsl -import nipype.interfaces.utility as util from CPAC.reho.utils import * def create_reho(wf_name): - """ - Regional Homogeneity(ReHo) approach to fMRI data analysis + Regional Homogeneity(ReHo) approach to fMRI data analysis. This workflow computes the ReHo map, z-score on map Parameters ---------- - None Returns @@ -25,7 +23,6 @@ def create_reho(wf_name): Notes ----- - `Source `_ Workflow Inputs: :: @@ -84,30 +81,38 @@ def create_reho(wf_name): >>> wf.inputs.inputspec.cluster_size = 27 >>> wf.run() # doctest: +SKIP """ - reHo = pe.Workflow(name=wf_name) - inputNode = pe.Node(util.IdentityInterface(fields=['cluster_size', - 'rest_res_filt', - 'rest_mask']), - name='inputspec') - - outputNode = pe.Node(util.IdentityInterface(fields=['raw_reho_map']), - name='outputspec') - - reho_imports = ['import os', 'import sys', 'import nibabel as nb', - 'import numpy as np', - 'from CPAC.reho.utils import f_kendall'] - raw_reho_map = pe.Node(util.Function(input_names=['in_file', 'mask_file', - 'cluster_size'], - output_names=['out_file'], - function=compute_reho, - imports=reho_imports), - name='reho_map', mem_gb=6.0) - - reHo.connect(inputNode, 'rest_res_filt', raw_reho_map, 'in_file') - reHo.connect(inputNode, 'rest_mask', raw_reho_map, 'mask_file') - reHo.connect(inputNode, 'cluster_size', raw_reho_map, 'cluster_size') - reHo.connect(raw_reho_map, 'out_file', outputNode, 'raw_reho_map') + inputNode = pe.Node( + util.IdentityInterface(fields=["cluster_size", "rest_res_filt", "rest_mask"]), + name="inputspec", + ) + + outputNode = pe.Node( + util.IdentityInterface(fields=["raw_reho_map"]), name="outputspec" + ) + + reho_imports = [ + "import os", + "import sys", + "import nibabel as nb", + "import numpy as np", + "from CPAC.reho.utils import f_kendall", + ] + raw_reho_map = pe.Node( + util.Function( + input_names=["in_file", "mask_file", "cluster_size"], + output_names=["out_file"], + function=compute_reho, + imports=reho_imports, + ), + name="reho_map", + mem_gb=6.0, + ) + + reHo.connect(inputNode, "rest_res_filt", raw_reho_map, "in_file") + reHo.connect(inputNode, "rest_mask", raw_reho_map, "mask_file") + reHo.connect(inputNode, "cluster_size", raw_reho_map, "cluster_size") + reHo.connect(raw_reho_map, "out_file", outputNode, "raw_reho_map") return reHo @@ -120,27 +125,27 @@ def create_reho(wf_name): outputs=["reho"], ) def reho(wf, cfg, strat_pool, pipe_num, opt=None): - cluster_size = cfg.regional_homogeneity['cluster_size'] + cluster_size = cfg.regional_homogeneity["cluster_size"] # Check the cluster size is supported if cluster_size not in [7, 19, 27]: - err_msg = 'Cluster size specified: %d, is not ' \ - 'supported. Change to 7, 19, or 27 and try ' \ - 'again' % cluster_size + err_msg = ( + "Cluster size specified: %d, is not " + "supported. Change to 7, 19, or 27 and try " + "again" % cluster_size + ) raise Exception(err_msg) - reho = create_reho(f'reho_{pipe_num}') + reho = create_reho(f"reho_{pipe_num}") reho.inputs.inputspec.cluster_size = cluster_size node, out = strat_pool.get_data("desc-preproc_bold") - wf.connect(node, out, reho, 'inputspec.rest_res_filt') + wf.connect(node, out, reho, "inputspec.rest_res_filt") - node, out_file = strat_pool.get_data('space-bold_desc-brain_mask') - wf.connect(node, out_file, reho, 'inputspec.rest_mask') + node, out_file = strat_pool.get_data("space-bold_desc-brain_mask") + wf.connect(node, out_file, reho, "inputspec.rest_mask") - outputs = { - 'reho': (reho, 'outputspec.raw_reho_map') - } + outputs = {"reho": (reho, "outputspec.raw_reho_map")} return (wf, outputs) @@ -150,37 +155,48 @@ def reho(wf, cfg, strat_pool, pipe_num, opt=None): config=["regional_homogeneity"], switch=["run"], inputs=[ - ["space-template_res-derivative_desc-preproc_bold", - "space-template_desc-preproc_bold"], - ["space-template_res-derivative_desc-bold_mask", - "space-template_desc-brain_mask"], + [ + "space-template_res-derivative_desc-preproc_bold", + "space-template_desc-preproc_bold", + ], + [ + "space-template_res-derivative_desc-bold_mask", + "space-template_desc-brain_mask", + ], ], outputs=["space-template_reho"], ) def reho_space_template(wf, cfg, strat_pool, pipe_num, opt=None): - - cluster_size = cfg.regional_homogeneity['cluster_size'] + cluster_size = cfg.regional_homogeneity["cluster_size"] # Check the cluster size is supported if cluster_size not in [7, 19, 27]: - err_msg = 'Cluster size specified: %d, is not ' \ - 'supported. Change to 7, 19, or 27 and try ' \ - 'again' % cluster_size + err_msg = ( + "Cluster size specified: %d, is not " + "supported. Change to 7, 19, or 27 and try " + "again" % cluster_size + ) raise Exception(err_msg) - reho = create_reho(f'reho_{pipe_num}') + reho = create_reho(f"reho_{pipe_num}") reho.inputs.inputspec.cluster_size = cluster_size - node, out = strat_pool.get_data(["space-template_res-derivative_desc-preproc_bold", - "space-template_desc-preproc_bold"]) - wf.connect(node, out, reho, 'inputspec.rest_res_filt') - - node, out_file = strat_pool.get_data(['space-template_res-derivative_desc-bold_mask', - 'space-template_desc-brain_mask']) - wf.connect(node, out_file, reho, 'inputspec.rest_mask') - - outputs = { - 'space-template_reho': (reho, 'outputspec.raw_reho_map') - } + node, out = strat_pool.get_data( + [ + "space-template_res-derivative_desc-preproc_bold", + "space-template_desc-preproc_bold", + ] + ) + wf.connect(node, out, reho, "inputspec.rest_res_filt") + + node, out_file = strat_pool.get_data( + [ + "space-template_res-derivative_desc-bold_mask", + "space-template_desc-brain_mask", + ] + ) + wf.connect(node, out_file, reho, "inputspec.rest_mask") + + outputs = {"space-template_reho": (reho, "outputspec.raw_reho_map")} return (wf, outputs) diff --git a/CPAC/reho/utils.py b/CPAC/reho/utils.py index a0a80692f7..3a69c9abee 100644 --- a/CPAC/reho/utils.py +++ b/CPAC/reho/utils.py @@ -1,14 +1,10 @@ - - def getOpString(mean, std_dev): - """ Generate the Operand String to be used in workflow nodes to supply - mean and std deviation to alff workflow nodes + mean and std deviation to alff workflow nodes. Parameters ---------- - mean : string mean value in string format @@ -18,40 +14,33 @@ def getOpString(mean, std_dev): Returns ------- - op_string : string """ - str1 = "-sub %f -div %f" % (float(mean), float(std_dev)) - op_string = str1 + " -mas %s" - - return op_string + return str1 + " -mas %s" def f_kendall(timeseries_matrix): - """ Calculates the Kendall's coefficient of concordance for a number of - time-series in the input matrix + time-series in the input matrix. Parameters ---------- - timeseries_matrix : ndarray A matrix of ranks of a subset subject's brain voxels Returns ------- - kcc : float Kendall's coefficient of concordance on the given input matrix """ - import numpy as np + nk = timeseries_matrix.shape n = nk[0] @@ -60,23 +49,19 @@ def f_kendall(timeseries_matrix): sr = np.sum(timeseries_matrix, 1) sr_bar = np.mean(sr) - s = np.sum(np.power(sr, 2)) - n*np.power(sr_bar, 2) - - kcc = 12 *s/np.power(k, 2)/(np.power(n, 3) - n) + s = np.sum(np.power(sr, 2)) - n * np.power(sr_bar, 2) - return kcc + return 12 * s / np.power(k, 2) / (np.power(n, 3) - n) def compute_reho(in_file, mask_file, cluster_size): - """ Computes the ReHo Map, by computing tied ranks of the timepoints, followed by computing Kendall's coefficient concordance(KCC) of a - timeseries with its neighbours + timeseries with its neighbours. Parameters ---------- - in_file : nifti file 4D EPI File @@ -90,19 +75,15 @@ def compute_reho(in_file, mask_file, cluster_size): Returns ------- - out_file : nifti file ReHo map of the input EPI image """ - - out_file = None - - res_fname = (in_file) - res_mask_fname = (mask_file) + res_fname = in_file + res_mask_fname = mask_file CUTNUMBER = 10 - if not (cluster_size == 27 or cluster_size == 19 or cluster_size == 7): + if cluster_size not in (27, 19, 7): cluster_size = 27 nvoxel = cluster_size @@ -113,34 +94,33 @@ def compute_reho(in_file, mask_file, cluster_size): res_data = res_img.get_fdata() res_mask_data = res_mask_img.get_fdata() - print(res_data.shape) (n_x, n_y, n_z, n_t) = res_data.shape # "flatten" each volume of the timeseries into one big array instead of # x,y,z - produces (timepoints, N voxels) shaped data array - res_data = np.reshape(res_data, (n_x*n_y*n_z, n_t), order='F').T + res_data = np.reshape(res_data, (n_x * n_y * n_z, n_t), order="F").T # create a blank array of zeroes of size n_voxels, one for each time point - Ranks_res_data = np.tile((np.zeros((1, (res_data.shape)[1]))), - [(res_data.shape)[0], 1]) + Ranks_res_data = np.tile( + (np.zeros((1, (res_data.shape)[1]))), [(res_data.shape)[0], 1] + ) # divide the number of total voxels by the cutnumber (set to 10) # ex. end up with a number in the thousands if there are tens of thousands # of voxels - segment_length = np.ceil(float((res_data.shape)[1])/float(CUTNUMBER)) + segment_length = np.ceil(float((res_data.shape)[1]) / float(CUTNUMBER)) for icut in range(0, CUTNUMBER): - segment = None # create a Numpy array of evenly spaced values from the segment # starting point up until the segment_length integer if not (icut == (CUTNUMBER - 1)): - segment = np.array(np.arange(icut * segment_length, - (icut+1) * segment_length)) + segment = np.array( + np.arange(icut * segment_length, (icut + 1) * segment_length) + ) else: - segment = np.array(np.arange(icut * segment_length, - (res_data.shape[1]))) + segment = np.array(np.arange(icut * segment_length, (res_data.shape[1]))) segment = np.int64(segment[np.newaxis]) @@ -151,8 +131,8 @@ def compute_reho(in_file, mask_file, cluster_size): # run a merge sort across the time axis, re-ordering the flattened # volume voxel arrays - res_data_sorted = np.sort(res_data_piece, 0, kind='mergesort') - sort_index = np.argsort(res_data_piece, axis=0, kind='mergesort') + res_data_sorted = np.sort(res_data_piece, 0, kind="mergesort") + sort_index = np.argsort(res_data_piece, axis=0, kind="mergesort") # subtract each volume from each other db = np.diff(res_data_sorted, 1, 0) @@ -170,11 +150,9 @@ def compute_reho(in_file, mask_file, cluster_size): sorted_ranks = np.tile(temp_array, [1, nvoxels_piece]) if np.any(sumdb[:]): - tie_adjust_index = np.flatnonzero(sumdb) for i in range(0, len(tie_adjust_index)): - ranks = sorted_ranks[:, tie_adjust_index[i]] ties = db[:, tie_adjust_index[i]] @@ -183,40 +161,45 @@ def compute_reho(in_file, mask_file, cluster_size): maxties = len(tieloc) tiecount = 0 - while(tiecount < maxties -1): + while tiecount < maxties - 1: tiestart = tieloc[tiecount] ntied = 2 - while(tieloc[tiecount + 1] == (tieloc[tiecount] + 1)): + while tieloc[tiecount + 1] == (tieloc[tiecount] + 1): tiecount += 1 ntied += 1 - ranks[tiestart:tiestart + ntied] = np.ceil(np.float32(np.sum(ranks[tiestart:tiestart + ntied ]))/np.float32(ntied)) + ranks[tiestart : tiestart + ntied] = np.ceil( + np.float32(np.sum(ranks[tiestart : tiestart + ntied])) + / np.float32(ntied) + ) tiecount += 1 sorted_ranks[:, tie_adjust_index[i]] = ranks del db, sumdb - sort_index_base = np.tile(np.multiply(np.arange(0, nvoxels_piece), n_t), [n_t, 1]) + sort_index_base = np.tile( + np.multiply(np.arange(0, nvoxels_piece), n_t), [n_t, 1] + ) sort_index += sort_index_base del sort_index_base ranks_piece = np.zeros((n_t, nvoxels_piece)) - ranks_piece = ranks_piece.flatten(order='F') - sort_index = sort_index.flatten(order='F') - sorted_ranks = sorted_ranks.flatten(order='F') + ranks_piece = ranks_piece.flatten(order="F") + sort_index = sort_index.flatten(order="F") + sorted_ranks = sorted_ranks.flatten(order="F") ranks_piece[sort_index] = np.array(sorted_ranks) - ranks_piece = np.reshape(ranks_piece, (n_t, nvoxels_piece), order='F') + ranks_piece = np.reshape(ranks_piece, (n_t, nvoxels_piece), order="F") del sort_index, sorted_ranks Ranks_res_data[:, segment[0]] = ranks_piece - sys.stdout.write('.') + sys.stdout.write(".") - Ranks_res_data = np.reshape(Ranks_res_data, (n_t, n_x, n_y, n_z), order='F') + Ranks_res_data = np.reshape(Ranks_res_data, (n_t, n_x, n_y, n_z), order="F") K = np.zeros((n_x, n_y, n_z)) @@ -233,7 +216,6 @@ def compute_reho(in_file, mask_file, cluster_size): mask_cluster[2, 2, 2] = 0 elif nvoxel == 7: - mask_cluster[0, 0, 0] = 0 mask_cluster[0, 1, 0] = 0 mask_cluster[0, 2, 0] = 0 @@ -256,27 +238,26 @@ def compute_reho(in_file, mask_file, cluster_size): mask_cluster[2, 2, 2] = 0 for i in range(1, n_x - 1): - for j in range(1, n_y -1): - for k in range(1, n_z -1): + for j in range(1, n_y - 1): + for k in range(1, n_z - 1): + block = Ranks_res_data[:, i - 1 : i + 2, j - 1 : j + 2, k - 1 : k + 2] + mask_block = res_mask_data[i - 1 : i + 2, j - 1 : j + 2, k - 1 : k + 2] - block = Ranks_res_data[:, i-1:i+2, j-1:j+2, k-1:k+2] - mask_block = res_mask_data[i-1:i+2, j-1:j+2, k-1:k+2] - - if not(int(mask_block[1, 1, 1]) == 0): - - if nvoxel == 19 or nvoxel == 7: + if not (int(mask_block[1, 1, 1]) == 0): + if nvoxel in (19, 7): mask_block = np.multiply(mask_block, mask_cluster) - R_block = np.reshape(block, (block.shape[0], 27), - order='F') - mask_R_block = R_block[:, np.argwhere(np.reshape(mask_block, (1, 27), order='F') > 0)[:, 1]] + R_block = np.reshape(block, (block.shape[0], 27), order="F") + mask_R_block = R_block[ + :, + np.argwhere(np.reshape(mask_block, (1, 27), order="F") > 0)[ + :, 1 + ], + ] K[i, j, k] = f_kendall(mask_R_block) - img = nb.Nifti1Image(K, header=res_img.header, - affine=res_img.affine) - reho_file = os.path.join(os.getcwd(), 'ReHo.nii.gz') + img = nb.Nifti1Image(K, header=res_img.header, affine=res_img.affine) + reho_file = os.path.join(os.getcwd(), "ReHo.nii.gz") img.to_filename(reho_file) - out_file = reho_file - - return out_file + return reho_file diff --git a/CPAC/resources/configs/data_settings_template.yml b/CPAC/resources/configs/data_settings_template.yml index 1d6c1b912e..0448605113 100644 --- a/CPAC/resources/configs/data_settings_template.yml +++ b/CPAC/resources/configs/data_settings_template.yml @@ -20,26 +20,26 @@ bidsBaseDir: None # File Path Template for Anatomical Files # Custom Data Format only. -# +# # Place tags for the appropriate data directory levels with the tags {site}, {participant}, and {session}. Only {participant} is required. -# +# # Examples: # /data/{site}/{participant}/{session}/anat/mprage.nii.gz # /data/{site}/{participant}/anat.nii.gz -# +# # See the User Guide for more detailed instructions. anatomicalTemplate: None # File Path Template for Functional Files # Custom Data Format only. -# +# # Place tags for the appropriate data directory levels with the tags {site}, {participant}, {session}, and {series}. Only {participant} is required. -# +# # Examples: # /data/{site}/{participant}/{session}/func/{series}_bold.nii.gz # /data/{site}/{participant}/{series}/func.nii.gz -# +# # See the User Guide for more detailed instructions. functionalTemplate: None @@ -105,7 +105,7 @@ brain_mask_template: None # # Examples: # /recon-all_path/{participant} -# +# freesurfer_dir: None @@ -134,66 +134,64 @@ fieldMapMagnitude: None # Include only a sub-set of the participants present in the folders defined above. -# +# # List participants in this box (ex: sub101, sub102) or provide the path to a text file with one participant ID on each line. -# +# # If 'None' is specified, CPAC will include all participants. subjectList: None # Exclude a sub-set of the participants present in the folders defined above. -# +# # List participants in this box (ex: sub101, sub102) or provide the path to a text file with one participant ID on each line. -# +# # If 'None' is specified, CPAC will not exclude any participants. exclusionSubjectList: None # Include only a sub-set of the sites present in the folders defined above. -# +# # List sites in this box (ex: NYU, UCLA) or provide the path to a text file with one site name on each line. -# +# # If 'None' is specified, CPAC will include all sites. siteList: None # Exclude a sub-set of the sites present in the folders defined above. -# +# # List sites in this box (ex: NYU, UCLA) or provide the path to a text file with one site name on each line. -# +# # If 'None' is specified, CPAC will include all sites. exclusionSiteList: None # Include only a sub-set of the sessions present in the folders defined above. -# +# # List sessions in this box (ex: session-1, session-2) or provide the path to a text file with one session name on each line. -# +# # If 'None' is specified, CPAC will include all sessions. sessionList: None # Exclude a sub-set of the sessions present in the folders defined above. -# +# # List sessions in this box (ex: session-1, session-2) or provide the path to a text file with one session name on each line. -# +# # If 'None' is specified, CPAC will include all sessions. exclusionSessionList: None # Include only a sub-set of the series present in the folders defined above. -# +# # List series in this box (ex: func-1, func-2) or provide the path to a text file with one series name on each line. -# +# # If 'None' is specified, CPAC will include all series. scanList: None # Exclude a sub-set of the series present in the folders defined above. -# +# # List series in this box (ex: func-1, func-2) or provide the path to a text file with one series name on each line. -# +# # If 'None' is specified, CPAC will include all series. exclusionScanList: None - - diff --git a/CPAC/resources/configs/group_config_template.yml b/CPAC/resources/configs/group_config_template.yml index 996c2386f0..c2b8074eb7 100644 --- a/CPAC/resources/configs/group_config_template.yml +++ b/CPAC/resources/configs/group_config_template.yml @@ -34,7 +34,7 @@ pipeline_setup: remove_working_dir: True log_directory: - + # Whether to write log details of the pipeline run to the logging files. run_logging: True @@ -315,20 +315,20 @@ qpp: # Run Quasi Periodic Pattern Analysis run: [1] - scan_inclusion: + scan_inclusion: + + session_inclusion: - session_inclusion: - stratification: permutations: 100 - + window: 30 initial_threshold: 0.2 - + final_threshold: 0.3 - + initial_threshold_iterations : 20 qpp_iterations : 15 diff --git a/CPAC/resources/configs/pipeline_config_default.yml b/CPAC/resources/configs/pipeline_config_default.yml index ef94cd0095..55c344010c 100644 --- a/CPAC/resources/configs/pipeline_config_default.yml +++ b/CPAC/resources/configs/pipeline_config_default.yml @@ -200,7 +200,7 @@ surface_analysis: # Will run Freesurfer for surface-based analysis. Will output traditional Freesurfer derivatives. # If you wish to employ Freesurfer outputs for brain masking or tissue segmentation in the voxel-based pipeline, # select those 'Freesurfer-' labeled options further below in anatomical_preproc. - freesurfer: + freesurfer: run_reconall: Off @@ -212,7 +212,7 @@ surface_analysis: # Run ABCD-HCP post FreeSurfer and fMRISurface pipeline - post_freesurfer: + post_freesurfer: run: Off @@ -277,7 +277,7 @@ anatomical_preproc: run_t2: Off # Non-local means filtering via ANTs DenoiseImage - non_local_means_filtering: + non_local_means_filtering: # this is a fork option run: [Off] @@ -295,12 +295,12 @@ anatomical_preproc: shrink_factor: 2 # Bias field correction based on square root of T1w * T2w - t1t2_bias_field_correction: + t1t2_bias_field_correction: + + run: Off - run: Off - BiasFieldSmoothingSigma: 5 - + acpc_alignment: run: Off @@ -313,17 +313,17 @@ anatomical_preproc: # Default: 150mm for human data. brain_size: 150 - # Choose a tool to crop the FOV in ACPC alignment. - # Using FSL's robustfov or flirt command. - # Default: robustfov for human data, flirt for monkey data. + # Choose a tool to crop the FOV in ACPC alignment. + # Using FSL's robustfov or flirt command. + # Default: robustfov for human data, flirt for monkey data. FOV_crop: robustfov - + # ACPC Target # options: 'brain' or 'whole-head' # note: 'brain' requires T1w_brain_ACPC_template below to be populated acpc_target: 'whole-head' - # Run ACPC alignment on brain mask + # Run ACPC alignment on brain mask # If the brain mask is in native space, turn it on # If the brain mask is ACPC aligned, turn it off align_brain_mask: Off @@ -335,7 +335,7 @@ anatomical_preproc: T2w_brain_ACPC_template: None brain_extraction: - + run: On # using: ['3dSkullStrip', 'BET', 'UNet', 'niworkflows-ants', 'FreeSurfer-ABCD', 'FreeSurfer-BET-Tight', 'FreeSurfer-BET-Loose', 'FreeSurfer-Brainmask'] @@ -713,21 +713,21 @@ registration_workflows: fnirt_config: T1_2_MNI152_2mm # The resolution to which anatomical images should be transformed during registration. - # This is the resolution at which processed anatomical files will be output. + # This is the resolution at which processed anatomical files will be output. # specifically for monkey pipeline ref_resolution: 2mm # Reference mask for FSL registration. ref_mask: $FSLDIR/data/standard/MNI152_T1_${resolution_for_anat}_brain_mask_dil.nii.gz - + # Template to be used during registration. - # It is for monkey pipeline specifically. + # It is for monkey pipeline specifically. FNIRT_T1w_brain_template: None # Template to be used during registration. - # It is for monkey pipeline specifically. + # It is for monkey pipeline specifically. FNIRT_T1w_template: None - + # Interpolation method for writing out transformed anatomical images. # Possible values: trilinear, sinc, spline interpolation: sinc @@ -924,10 +924,10 @@ registration_workflows: # these options modify the application (to the functional data), not the calculation, of the # T1-to-template and EPI-to-template transforms calculated earlier during registration - + # apply the functional-to-template (T1 template) registration transform to the functional data run: On - + # apply the functional-to-template (EPI template) registration transform to the functional data run_EPI: Off @@ -948,7 +948,7 @@ registration_workflows: # thus, a higher resolution may not result in a large increase in RAM needs as above func_derivative_outputs: 3mm - target_template: + target_template: # choose which template space to transform derivatives towards # using: ['T1_template', 'EPI_template'] # this is a fork point @@ -1016,11 +1016,11 @@ functional_preproc: run: On - update_header: + update_header: # Convert raw data from LPI to RPI run: On - + truncation: # First timepoint to include in analysis. @@ -1065,10 +1065,10 @@ functional_preproc: tzero: None motion_estimates_and_correction: - + run: On - motion_estimates: + motion_estimates: # calculate motion statistics BEFORE slice-timing correction calculate_motion_first: Off @@ -1208,48 +1208,48 @@ functional_preproc: # Set the threshold value for the skull-stripping of the magnitude file. Depending on the data, a tighter extraction may be necessary in order to prevent noisy voxels from interfering with preparing the field map. # The default value is 0.6. fmap_skullstrip_AFNI_threshold: 0.6 - + Blip-FSL-TOPUP: - + # (approximate) resolution (in mm) of warp basis for the different sub-sampling levels, default 10 warpres: 10 - + # sub-sampling scheme, default 1 subsamp: 1 - + # FWHM (in mm) of gaussian smoothing kernel, default 8 fwhm: 8 - + # Max # of non-linear iterations, default 5 miter: 5 - + # Weight of regularisation, default depending on --ssqlambda and --regmod switches. See user documentation. lambda: 1 - + # If set (=1), lambda is weighted by current ssq, default 1 ssqlambda: 1 - + # Model for regularisation of warp-field [membrane_energy bending_energy], default bending_energy regmod: bending_energy - + # Estimate movements if set, default 1 (true) estmov: 1 - + # Minimisation method 0=Levenberg-Marquardt, 1=Scaled Conjugate Gradient, default 0 (LM) minmet: 0 - + # Order of spline, 2->Qadratic spline, 3->Cubic spline. Default=3 splineorder: 3 - + # Precision for representing Hessian, double or float. Default double numprec: double - + # Image interpolation model, linear or spline. Default spline interp: spline - + # If set (=1), the images are individually scaled to a common mean, default 0 (false) scale: 0 - + # If set (=1), the calculations are done in a different grid, default 1 (true) regrid: 1 @@ -1274,7 +1274,7 @@ functional_preproc: functional_mean_boolean: Off # Set an intensity threshold to improve skull stripping performances of FSL BET on rodent scans. - functional_mean_thr: + functional_mean_thr: run: Off threshold_value: 98 @@ -1342,7 +1342,7 @@ functional_preproc: # Normalize functional image run: On - + coreg_prep: # Generate sbref diff --git a/CPAC/resources/configs/scan_parameters_template.csv b/CPAC/resources/configs/scan_parameters_template.csv index f7e5002bb4..52c10e49e0 100755 --- a/CPAC/resources/configs/scan_parameters_template.csv +++ b/CPAC/resources/configs/scan_parameters_template.csv @@ -1 +1 @@ -Site,Participant,Session,Series,TR (seconds),TE (seconds),Reference (slice no),Acquisition (pattern),FirstTR (start volume index),LastTR (final volume index) \ No newline at end of file +Site,Participant,Session,Series,TR (seconds),TE (seconds),Reference (slice no),Acquisition (pattern),FirstTR (start volume index),LastTR (final volume index) diff --git a/CPAC/resources/configs/test_configs/pipe-test_ABCD.yml b/CPAC/resources/configs/test_configs/pipe-test_ABCD.yml index 39023ec11d..856ad2b2f0 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_ABCD.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_ABCD.yml @@ -10,16 +10,16 @@ FROM: default -pipeline_setup: +pipeline_setup: # Name for this pipeline configuration - useful for identification. pipeline_name: analysis - log_directory: + log_directory: path: /tmp - system_config: + system_config: # The maximum amount of memory each participant's workflow can allocate. # Use this to place an upper bound of memory usage. @@ -30,7 +30,7 @@ pipeline_setup: # 'Number of Participants to Run Simultaneously' is as much RAM you can safely allocate. maximum_memory_per_participant: 8 -anatomical_preproc: +anatomical_preproc: # Non-local means filtering via ANTs DenoiseImage non_local_means_filtering: @@ -40,43 +40,43 @@ anatomical_preproc: n4_bias_field_correction: run: On - acpc_alignment: + acpc_alignment: run: On # ACPC aligned template T1w_ACPC_template: /ABCD_pipeline_template/MNI152_T1_1mm.nii.gz - brain_extraction: + brain_extraction: # using: ['3dSkullStrip', 'BET', 'UNet', 'niworkflows-ants'] # this is a fork option using: [FreeSurfer-ABCD] - FSL-BET: + FSL-BET: # Set the threshold value controling the brain vs non-brain voxels, default is 0.5 frac: 0.3 -segmentation: +segmentation: - tissue_segmentation: + tissue_segmentation: # using: ['FSL-FAST', 'Template_Based', 'FreeSurfer', 'ANTs_Prior_Based'] # this is a fork point using: [FreeSurfer] # option parameters - FSL-FAST: + FSL-FAST: - use_priors: + use_priors: # Full path to a directory containing binarized prior probability maps. # These maps are included as part of the 'Image Resource Files' package available on the Install page of the User Guide. # It is not necessary to change this path unless you intend to use non-standard priors. priors_path: $FSLDIR/data/standard/tissuepriors/2mm - Template_Based: + Template_Based: # These masks should be in the same space of your registration template, e.g. if # you choose 'EPI Template' , below tissue masks should also be EPI template tissue masks. @@ -96,9 +96,9 @@ segmentation: # Full path to a binarized CSF mask. CSF: $FSLDIR/data/standard/tissuepriors/2mm/avg152T1_csf_bin.nii.gz -registration_workflows: +registration_workflows: - anatomical_registration: + anatomical_registration: # The resolution to which anatomical images should be transformed during registration. # This is the resolution at which processed anatomical files will be output. @@ -112,13 +112,13 @@ registration_workflows: # It is not necessary to change this path unless you intend to use a non-standard template. T1w_brain_template_mask: /ABCD_pipeline_template/MNI152_T1_${resolution_for_anat}_brain_mask.nii.gz - registration: + registration: # option parameters - ANTs: + ANTs: # ANTs parameters for T1-template-based registration - T1_registration: + T1_registration: - verbose: 1 - float: 0 - collapse-output-transforms: 0 @@ -182,19 +182,19 @@ registration_workflows: # Possible values: Linear, BSpline, LanczosWindowedSinc interpolation: Linear - functional_registration: + functional_registration: - EPI_registration: + EPI_registration: - ANTs: + ANTs: # EPI registration configuration - synonymous with T1_registration # parameters under anatomical registration above - parameters: + parameters: - func_registration_to_template: + func_registration_to_template: - output_resolution: + output_resolution: # The resolution (in mm) to which the preprocessed, registered functional timeseries outputs are written into. # NOTE: @@ -211,10 +211,10 @@ registration_workflows: # thus, a higher resolution may not result in a large increase in RAM needs as above func_derivative_outputs: 2mm - target_template: + target_template: # option parameters - T1_template: + T1_template: # Standard Skull Stripped Template. Used as a reference image for functional registration. # This can be different than the template used as the reference/fixed for T1-to-template registration. @@ -228,29 +228,29 @@ registration_workflows: # It is not necessary to change this path unless you intend to use a non-standard template. T1w_brain_template_mask_funcreg: /ABCD_pipeline_template/MNI152_T1_${func_resolution}_brain_mask.nii.gz - ANTs_pipelines: + ANTs_pipelines: # Interpolation method for writing out transformed functional images. # Possible values: Linear, BSpline, LanczosWindowedSinc interpolation: Linear -functional_preproc: +functional_preproc: - motion_estimates_and_correction: + motion_estimates_and_correction: - motion_correction: + motion_correction: # using: ['3dvolreg', 'mcflirt'] # this is a fork point using: [mcflirt] # option parameters - AFNI-3dvolreg: + AFNI-3dvolreg: # This option is useful when aligning high-resolution datasets that may need more alignment than a few voxels. functional_volreg_twopass: On - distortion_correction: + distortion_correction: # using: ['PhaseDiff', 'Blip'] # PhaseDiff - Perform field map correction using a single phase difference image, a subtraction of the two phase images from each echo. Default scanner for this method is SIEMENS. @@ -260,22 +260,22 @@ functional_preproc: # for example, phase-difference field maps will lead to phase-difference distortion correction, and phase-encoding direction field maps will lead to blip-up/blip-down using: [] - func_masking: + func_masking: # using: ['AFNI', 'FSL', 'FSL_AFNI', 'Anatomical_Refined', 'Anatomical_Based'] # this is a fork point using: [Anatomical_Based] -nuisance_corrections: +nuisance_corrections: - 2-nuisance_regression: + 2-nuisance_regression: # this is a fork point # run: [On, Off] - this will run both and fork the pipeline run: [Off] # Select which nuisance signal corrections to apply - Regressors: + Regressors: - Motion: include_delayed: On include_delayed_squared: On @@ -286,36 +286,36 @@ nuisance_corrections: # used in CSF mask refinement for CSF signal-related regressions lateral_ventricles_mask: $FSLDIR/data/atlases/HarvardOxford/HarvardOxford-lateral-ventricles-thr25-2mm.nii.gz -timeseries_extraction: +timeseries_extraction: run: Off -seed_based_correlation_analysis: +seed_based_correlation_analysis: # SCA - Seed-Based Correlation Analysis # For each extracted ROI Average time series, CPAC will generate a whole-brain correlation map. # It should be noted that for a given seed/ROI, SCA maps for ROI Average time series will be the same. run: Off -amplitude_low_frequency_fluctuation: +amplitude_low_frequency_fluctuation: # ALFF & f/ALFF # Calculate Amplitude of Low Frequency Fluctuations (ALFF) and and fractional ALFF (f/ALFF) for all voxels. run: Off -regional_homogeneity: +regional_homogeneity: # ReHo # Calculate Regional Homogeneity (ReHo) for all voxels. run: Off -voxel_mirrored_homotopic_connectivity: +voxel_mirrored_homotopic_connectivity: # VMHC # Calculate Voxel-mirrored Homotopic Connectivity (VMHC) for all voxels. run: Off - symmetric_registration: + symmetric_registration: # Included as part of the 'Image Resource Files' package available on the Install page of the User Guide. # It is not necessary to change this path unless you intend to use a non-standard symmetric template. @@ -340,7 +340,7 @@ voxel_mirrored_homotopic_connectivity: # PACKAGE INTEGRATIONS # -------------------- -PyPEER: +PyPEER: # Template-space eye mask eye_mask_path: $FSLDIR/data/standard/MNI152_T1_${func_resolution}_eye_mask.nii.gz diff --git a/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-AllNuis.yml b/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-AllNuis.yml index e68f31d827..dbb0c96834 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-AllNuis.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-AllNuis.yml @@ -10,12 +10,12 @@ FROM: default -pipeline_setup: +pipeline_setup: # Name for this pipeline configuration - useful for identification. pipeline_name: pipe-test_ANTs-3dSk-AllNuis - output_directory: + output_directory: # Directory where C-PAC should write out processed data, logs, and crash reports. # - If running in a container (Singularity/Docker), you can simply set this to an arbitrary @@ -23,7 +23,7 @@ pipeline_setup: # - If running outside a container, this should be a full path to a directory. path: ./cpac_runs/output - working_directory: + working_directory: # Directory where C-PAC should store temporary and intermediate files. # - This directory must be saved if you wish to re-run your pipeline from where you left off (if not completed). @@ -39,17 +39,17 @@ pipeline_setup: # This saves disk space, but any additional preprocessing or analysis will have to be completely re-run. remove_working_dir: Off - log_directory: + log_directory: path: ./cpac_runs/log - system_config: + system_config: # Select Off if you intend to run CPAC on a single machine. # If set to On, CPAC will attempt to submit jobs through the job scheduler / resource manager selected below. - on_grid: + on_grid: - SGE: + SGE: # SGE Parallel Environment to use when running CPAC. # Only applies when you are running on a grid or compute cluster using SGE. @@ -68,32 +68,32 @@ pipeline_setup: # If you have specified an FSL path in your .bashrc file, this path will be set automatically. FSLDIR: FSLDIR -anatomical_preproc: +anatomical_preproc: - brain_extraction: + brain_extraction: - FSL-BET: + FSL-BET: # Mask created along with skull stripping. It should be `On`, if selected functionalMasking : ['Anatomical_Refined'] and `FSL` as skull-stripping method. mask_boolean: Off -segmentation: +segmentation: - tissue_segmentation: + tissue_segmentation: # option parameters - FSL-FAST: + FSL-FAST: - use_priors: + use_priors: # Full path to a directory containing binarized prior probability maps. # These maps are included as part of the 'Image Resource Files' package available on the Install page of the User Guide. # It is not necessary to change this path unless you intend to use non-standard priors. priors_path: $FSLDIR/data/standard/tissuepriors/2mm -registration_workflows: +registration_workflows: - anatomical_registration: + anatomical_registration: # Template to be used during registration. # It is not necessary to change this path unless you intend to use a non-standard template. @@ -106,34 +106,34 @@ registration_workflows: # Register skull-on anatomical image to a template. reg_with_skull: Off - registration: + registration: - FSL-FNIRT: + FSL-FNIRT: # Reference mask for FSL registration. ref_mask: $FSLDIR/data/standard/MNI152_T1_${resolution_for_anat}_brain_mask_dil.nii.gz - functional_registration: + functional_registration: - coregistration: + coregistration: - boundary_based_registration: + boundary_based_registration: # Standard FSL 5.0 Scheduler used for Boundary Based Registration. # It is not necessary to change this path unless you intend to use non-standard MNI registration. bbr_schedule: $FSLDIR/etc/flirtsch/bbr.sch - EPI_registration: + EPI_registration: - FSL-FNIRT: + FSL-FNIRT: # Identity matrix used during FSL-based resampling of BOLD-space data throughout the pipeline. # It is not necessary to change this path unless you intend to use a different template. identity_matrix: $FSLDIR/etc/flirtsch/ident.mat - func_registration_to_template: + func_registration_to_template: - output_resolution: + output_resolution: # The resolution (in mm) to which the registered derivative outputs are written into. # NOTE: @@ -141,10 +141,10 @@ registration_workflows: # thus, a higher resolution may not result in a large increase in RAM needs as above func_derivative_outputs: 2mm - target_template: + target_template: # option parameters - T1_template: + T1_template: # Standard Skull Stripped Template. Used as a reference image for functional registration. # This can be different than the template used as the reference/fixed for T1-to-template registration. @@ -154,18 +154,18 @@ registration_workflows: # This can be different than the template used as the reference/fixed for T1-to-template registration. T1w_template_funcreg: $FSLDIR/data/standard/MNI152_T1_${func_resolution}.nii.gz -nuisance_corrections: +nuisance_corrections: - 1-ICA-AROMA: + 1-ICA-AROMA: # this is a fork point # run: [On, Off] - this will run both and fork the pipeline run: [On] - 2-nuisance_regression: + 2-nuisance_regression: # Select which nuisance signal corrections to apply - Regressors: + Regressors: - Bandpass: bottom_frequency: 0.01 top_frequency: 0.1 @@ -281,28 +281,28 @@ nuisance_corrections: # used in CSF mask refinement for CSF signal-related regressions lateral_ventricles_mask: $FSLDIR/data/atlases/HarvardOxford/HarvardOxford-lateral-ventricles-thr25-2mm.nii.gz -timeseries_extraction: +timeseries_extraction: # Enter paths to region-of-interest (ROI) NIFTI files (.nii or .nii.gz) to be used for time-series extraction, and then select which types of analyses to run. # Denote which analyses to run for each ROI path by listing the names below. For example, if you wish to run Avg and SpatialReg, you would enter: '/path/to/ROI.nii.gz': Avg, SpatialReg # available analyses: # /path/to/atlas.nii.gz: Avg, Voxel, SpatialReg - tse_roi_paths: + tse_roi_paths: s3://fcp-indi/resources/cpac/resources/rois_2mm.nii.gz: Avg, Voxel, SpatialReg -seed_based_correlation_analysis: +seed_based_correlation_analysis: # Enter paths to region-of-interest (ROI) NIFTI files (.nii or .nii.gz) to be used for seed-based correlation analysis, and then select which types of analyses to run. # Denote which analyses to run for each ROI path by listing the names below. For example, if you wish to run Avg and MultReg, you would enter: '/path/to/ROI.nii.gz': Avg, MultReg # available analyses: # /path/to/atlas.nii.gz: Avg, DualReg, MultReg - sca_roi_paths: + sca_roi_paths: s3://fcp-indi/resources/cpac/resources/rois_2mm.nii.gz: Avg, MultReg s3://fcp-indi/resources/cpac/resources/PNAS_Smith09_rsn10.nii.gz: DualReg -voxel_mirrored_homotopic_connectivity: +voxel_mirrored_homotopic_connectivity: - symmetric_registration: + symmetric_registration: # Included as part of the 'Image Resource Files' package available on the Install page of the User Guide. # It is not necessary to change this path unless you intend to use a non-standard symmetric template. @@ -316,7 +316,7 @@ voxel_mirrored_homotopic_connectivity: # It is not necessary to change this path unless you intend to use a non-standard symmetric template. dilated_symmetric_brain_mask: $FSLDIR/data/standard/MNI152_T1_${resolution_for_anat}_brain_mask_symmetric_dil.nii.gz -network_centrality: +network_centrality: # Maximum amount of RAM (in GB) to be used when calculating Degree Centrality. # Calculating Eigenvector Centrality will require additional memory based on the size of the mask or number of ROI nodes. @@ -325,7 +325,7 @@ network_centrality: # Full path to a NIFTI file describing the mask. Centrality will be calculated for all voxels within the mask. template_specification_file: s3://fcp-indi/resources/cpac/resources/mask-thr50-3mm.nii.gz - eigenvector_centrality: + eigenvector_centrality: # Enable/Disable eigenvector centrality by selecting the connectivity weights # weight_options: ['Binarized', 'Weighted'] @@ -333,7 +333,7 @@ network_centrality: # weight_options: [] weight_options: [Binarized, Weighted] - local_functional_connectivity_density: + local_functional_connectivity_density: # Select the type of threshold used when creating the lFCD adjacency matrix. # options: diff --git a/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-DistCorr3dSk.yml b/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-DistCorr3dSk.yml index 510e3a7ef9..73f9cb6db5 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-DistCorr3dSk.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-DistCorr3dSk.yml @@ -10,12 +10,12 @@ FROM: default -pipeline_setup: +pipeline_setup: # Name for this pipeline configuration - useful for identification. pipeline_name: pipe-test_ANTs-3dSk-DistCorr3dSk - output_directory: + output_directory: # Directory where C-PAC should write out processed data, logs, and crash reports. # - If running in a container (Singularity/Docker), you can simply set this to an arbitrary @@ -23,7 +23,7 @@ pipeline_setup: # - If running outside a container, this should be a full path to a directory. path: ./cpac_runs/output - working_directory: + working_directory: # Directory where C-PAC should store temporary and intermediate files. # - This directory must be saved if you wish to re-run your pipeline from where you left off (if not completed). @@ -39,17 +39,17 @@ pipeline_setup: # This saves disk space, but any additional preprocessing or analysis will have to be completely re-run. remove_working_dir: Off - log_directory: + log_directory: path: ./cpac_runs/log - system_config: + system_config: # Select Off if you intend to run CPAC on a single machine. # If set to On, CPAC will attempt to submit jobs through the job scheduler / resource manager selected below. - on_grid: + on_grid: - SGE: + SGE: # SGE Parallel Environment to use when running CPAC. # Only applies when you are running on a grid or compute cluster using SGE. @@ -68,32 +68,32 @@ pipeline_setup: # If you have specified an FSL path in your .bashrc file, this path will be set automatically. FSLDIR: FSLDIR -anatomical_preproc: +anatomical_preproc: - brain_extraction: + brain_extraction: - FSL-BET: + FSL-BET: # Mask created along with skull stripping. It should be `On`, if selected functionalMasking : ['Anatomical_Refined'] and `FSL` as skull-stripping method. mask_boolean: Off -segmentation: +segmentation: - tissue_segmentation: + tissue_segmentation: # option parameters - FSL-FAST: + FSL-FAST: - use_priors: + use_priors: # Full path to a directory containing binarized prior probability maps. # These maps are included as part of the 'Image Resource Files' package available on the Install page of the User Guide. # It is not necessary to change this path unless you intend to use non-standard priors. priors_path: $FSLDIR/data/standard/tissuepriors/2mm -registration_workflows: +registration_workflows: - anatomical_registration: + anatomical_registration: # Template to be used during registration. # It is not necessary to change this path unless you intend to use a non-standard template. @@ -106,34 +106,34 @@ registration_workflows: # Register skull-on anatomical image to a template. reg_with_skull: Off - registration: + registration: - FSL-FNIRT: + FSL-FNIRT: # Reference mask for FSL registration. ref_mask: $FSLDIR/data/standard/MNI152_T1_${resolution_for_anat}_brain_mask_dil.nii.gz - functional_registration: + functional_registration: - coregistration: + coregistration: - boundary_based_registration: + boundary_based_registration: # Standard FSL 5.0 Scheduler used for Boundary Based Registration. # It is not necessary to change this path unless you intend to use non-standard MNI registration. bbr_schedule: $FSLDIR/etc/flirtsch/bbr.sch - EPI_registration: + EPI_registration: - FSL-FNIRT: + FSL-FNIRT: # Identity matrix used during FSL-based resampling of BOLD-space data throughout the pipeline. # It is not necessary to change this path unless you intend to use a different template. identity_matrix: $FSLDIR/etc/flirtsch/ident.mat - func_registration_to_template: + func_registration_to_template: - output_resolution: + output_resolution: # The resolution (in mm) to which the registered derivative outputs are written into. # NOTE: @@ -141,10 +141,10 @@ registration_workflows: # thus, a higher resolution may not result in a large increase in RAM needs as above func_derivative_outputs: 2mm - target_template: + target_template: # option parameters - T1_template: + T1_template: # Standard Skull Stripped Template. Used as a reference image for functional registration. # This can be different than the template used as the reference/fixed for T1-to-template registration. @@ -154,23 +154,23 @@ registration_workflows: # This can be different than the template used as the reference/fixed for T1-to-template registration. T1w_template_funcreg: $FSLDIR/data/standard/MNI152_T1_${func_resolution}.nii.gz -functional_preproc: +functional_preproc: - distortion_correction: + distortion_correction: # option parameters - PhaseDiff: + PhaseDiff: # Since the quality of the distortion heavily relies on the skull-stripping step, we provide a choice of method ('AFNI' for AFNI 3dSkullStrip or 'BET' for FSL BET). # Options: 'BET' or 'AFNI' fmap_skullstrip_option: AFNI -nuisance_corrections: +nuisance_corrections: - 2-nuisance_regression: + 2-nuisance_regression: # Select which nuisance signal corrections to apply - Regressors: + Regressors: - Bandpass: bottom_frequency: 0.01 top_frequency: 0.1 @@ -286,28 +286,28 @@ nuisance_corrections: # used in CSF mask refinement for CSF signal-related regressions lateral_ventricles_mask: $FSLDIR/data/atlases/HarvardOxford/HarvardOxford-lateral-ventricles-thr25-2mm.nii.gz -timeseries_extraction: +timeseries_extraction: # Enter paths to region-of-interest (ROI) NIFTI files (.nii or .nii.gz) to be used for time-series extraction, and then select which types of analyses to run. # Denote which analyses to run for each ROI path by listing the names below. For example, if you wish to run Avg and SpatialReg, you would enter: '/path/to/ROI.nii.gz': Avg, SpatialReg # available analyses: # /path/to/atlas.nii.gz: Avg, Voxel, SpatialReg - tse_roi_paths: + tse_roi_paths: s3://fcp-indi/resources/cpac/resources/rois_2mm.nii.gz: Avg, Voxel, SpatialReg -seed_based_correlation_analysis: +seed_based_correlation_analysis: # Enter paths to region-of-interest (ROI) NIFTI files (.nii or .nii.gz) to be used for seed-based correlation analysis, and then select which types of analyses to run. # Denote which analyses to run for each ROI path by listing the names below. For example, if you wish to run Avg and MultReg, you would enter: '/path/to/ROI.nii.gz': Avg, MultReg # available analyses: # /path/to/atlas.nii.gz: Avg, DualReg, MultReg - sca_roi_paths: + sca_roi_paths: s3://fcp-indi/resources/cpac/resources/rois_2mm.nii.gz: Avg, MultReg s3://fcp-indi/resources/cpac/resources/PNAS_Smith09_rsn10.nii.gz: DualReg -voxel_mirrored_homotopic_connectivity: +voxel_mirrored_homotopic_connectivity: - symmetric_registration: + symmetric_registration: # Included as part of the 'Image Resource Files' package available on the Install page of the User Guide. # It is not necessary to change this path unless you intend to use a non-standard symmetric template. @@ -321,7 +321,7 @@ voxel_mirrored_homotopic_connectivity: # It is not necessary to change this path unless you intend to use a non-standard symmetric template. dilated_symmetric_brain_mask: $FSLDIR/data/standard/MNI152_T1_${resolution_for_anat}_brain_mask_symmetric_dil.nii.gz -network_centrality: +network_centrality: # Maximum amount of RAM (in GB) to be used when calculating Degree Centrality. # Calculating Eigenvector Centrality will require additional memory based on the size of the mask or number of ROI nodes. @@ -330,7 +330,7 @@ network_centrality: # Full path to a NIFTI file describing the mask. Centrality will be calculated for all voxels within the mask. template_specification_file: s3://fcp-indi/resources/cpac/resources/mask-thr50-3mm.nii.gz - eigenvector_centrality: + eigenvector_centrality: # Enable/Disable eigenvector centrality by selecting the connectivity weights # weight_options: ['Binarized', 'Weighted'] @@ -338,7 +338,7 @@ network_centrality: # weight_options: [] weight_options: [Binarized, Weighted] - local_functional_connectivity_density: + local_functional_connectivity_density: # Select the type of threshold used when creating the lFCD adjacency matrix. # options: diff --git a/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-DistCorrBET.yml b/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-DistCorrBET.yml index e6940dc729..83d6ca70b0 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-DistCorrBET.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_ANTs-3dSk-DistCorrBET.yml @@ -10,12 +10,12 @@ FROM: default -pipeline_setup: +pipeline_setup: # Name for this pipeline configuration - useful for identification. pipeline_name: pipe-test_ANTs-3dSk-DistCorrBET - output_directory: + output_directory: # Directory where C-PAC should write out processed data, logs, and crash reports. # - If running in a container (Singularity/Docker), you can simply set this to an arbitrary @@ -23,7 +23,7 @@ pipeline_setup: # - If running outside a container, this should be a full path to a directory. path: ./cpac_runs/output - working_directory: + working_directory: # Directory where C-PAC should store temporary and intermediate files. # - This directory must be saved if you wish to re-run your pipeline from where you left off (if not completed). @@ -39,17 +39,17 @@ pipeline_setup: # This saves disk space, but any additional preprocessing or analysis will have to be completely re-run. remove_working_dir: Off - log_directory: + log_directory: path: ./cpac_runs/log - system_config: + system_config: # Select Off if you intend to run CPAC on a single machine. # If set to On, CPAC will attempt to submit jobs through the job scheduler / resource manager selected below. - on_grid: + on_grid: - SGE: + SGE: # SGE Parallel Environment to use when running CPAC. # Only applies when you are running on a grid or compute cluster using SGE. @@ -68,32 +68,32 @@ pipeline_setup: # If you have specified an FSL path in your .bashrc file, this path will be set automatically. FSLDIR: FSLDIR -anatomical_preproc: +anatomical_preproc: - brain_extraction: + brain_extraction: - FSL-BET: + FSL-BET: # Mask created along with skull stripping. It should be `On`, if selected functionalMasking : ['Anatomical_Refined'] and `FSL` as skull-stripping method. mask_boolean: Off -segmentation: +segmentation: - tissue_segmentation: + tissue_segmentation: # option parameters - FSL-FAST: + FSL-FAST: - use_priors: + use_priors: # Full path to a directory containing binarized prior probability maps. # These maps are included as part of the 'Image Resource Files' package available on the Install page of the User Guide. # It is not necessary to change this path unless you intend to use non-standard priors. priors_path: $FSLDIR/data/standard/tissuepriors/2mm -registration_workflows: +registration_workflows: - anatomical_registration: + anatomical_registration: # Template to be used during registration. # It is not necessary to change this path unless you intend to use a non-standard template. @@ -106,34 +106,34 @@ registration_workflows: # Register skull-on anatomical image to a template. reg_with_skull: Off - registration: + registration: - FSL-FNIRT: + FSL-FNIRT: # Reference mask for FSL registration. ref_mask: $FSLDIR/data/standard/MNI152_T1_${resolution_for_anat}_brain_mask_dil.nii.gz - functional_registration: + functional_registration: - coregistration: + coregistration: - boundary_based_registration: + boundary_based_registration: # Standard FSL 5.0 Scheduler used for Boundary Based Registration. # It is not necessary to change this path unless you intend to use non-standard MNI registration. bbr_schedule: $FSLDIR/etc/flirtsch/bbr.sch - EPI_registration: + EPI_registration: - FSL-FNIRT: + FSL-FNIRT: # Identity matrix used during FSL-based resampling of BOLD-space data throughout the pipeline. # It is not necessary to change this path unless you intend to use a different template. identity_matrix: $FSLDIR/etc/flirtsch/ident.mat - func_registration_to_template: + func_registration_to_template: - output_resolution: + output_resolution: # The resolution (in mm) to which the registered derivative outputs are written into. # NOTE: @@ -141,10 +141,10 @@ registration_workflows: # thus, a higher resolution may not result in a large increase in RAM needs as above func_derivative_outputs: 2mm - target_template: + target_template: # option parameters - T1_template: + T1_template: # Standard Skull Stripped Template. Used as a reference image for functional registration. # This can be different than the template used as the reference/fixed for T1-to-template registration. @@ -154,12 +154,12 @@ registration_workflows: # This can be different than the template used as the reference/fixed for T1-to-template registration. T1w_template_funcreg: $FSLDIR/data/standard/MNI152_T1_${func_resolution}.nii.gz -nuisance_corrections: +nuisance_corrections: - 2-nuisance_regression: + 2-nuisance_regression: # Select which nuisance signal corrections to apply - Regressors: + Regressors: - Bandpass: bottom_frequency: 0.01 top_frequency: 0.1 @@ -275,28 +275,28 @@ nuisance_corrections: # used in CSF mask refinement for CSF signal-related regressions lateral_ventricles_mask: $FSLDIR/data/atlases/HarvardOxford/HarvardOxford-lateral-ventricles-thr25-2mm.nii.gz -timeseries_extraction: +timeseries_extraction: # Enter paths to region-of-interest (ROI) NIFTI files (.nii or .nii.gz) to be used for time-series extraction, and then select which types of analyses to run. # Denote which analyses to run for each ROI path by listing the names below. For example, if you wish to run Avg and SpatialReg, you would enter: '/path/to/ROI.nii.gz': Avg, SpatialReg # available analyses: # /path/to/atlas.nii.gz: Avg, Voxel, SpatialReg - tse_roi_paths: + tse_roi_paths: s3://fcp-indi/resources/cpac/resources/rois_2mm.nii.gz: Avg, Voxel, SpatialReg -seed_based_correlation_analysis: +seed_based_correlation_analysis: # Enter paths to region-of-interest (ROI) NIFTI files (.nii or .nii.gz) to be used for seed-based correlation analysis, and then select which types of analyses to run. # Denote which analyses to run for each ROI path by listing the names below. For example, if you wish to run Avg and MultReg, you would enter: '/path/to/ROI.nii.gz': Avg, MultReg # available analyses: # /path/to/atlas.nii.gz: Avg, DualReg, MultReg - sca_roi_paths: + sca_roi_paths: s3://fcp-indi/resources/cpac/resources/rois_2mm.nii.gz: Avg, MultReg s3://fcp-indi/resources/cpac/resources/PNAS_Smith09_rsn10.nii.gz: DualReg -voxel_mirrored_homotopic_connectivity: +voxel_mirrored_homotopic_connectivity: - symmetric_registration: + symmetric_registration: # Included as part of the 'Image Resource Files' package available on the Install page of the User Guide. # It is not necessary to change this path unless you intend to use a non-standard symmetric template. @@ -310,7 +310,7 @@ voxel_mirrored_homotopic_connectivity: # It is not necessary to change this path unless you intend to use a non-standard symmetric template. dilated_symmetric_brain_mask: $FSLDIR/data/standard/MNI152_T1_${resolution_for_anat}_brain_mask_symmetric_dil.nii.gz -network_centrality: +network_centrality: # Maximum amount of RAM (in GB) to be used when calculating Degree Centrality. # Calculating Eigenvector Centrality will require additional memory based on the size of the mask or number of ROI nodes. @@ -319,7 +319,7 @@ network_centrality: # Full path to a NIFTI file describing the mask. Centrality will be calculated for all voxels within the mask. template_specification_file: s3://fcp-indi/resources/cpac/resources/mask-thr50-3mm.nii.gz - eigenvector_centrality: + eigenvector_centrality: # Enable/Disable eigenvector centrality by selecting the connectivity weights # weight_options: ['Binarized', 'Weighted'] @@ -327,7 +327,7 @@ network_centrality: # weight_options: [] weight_options: [Binarized, Weighted] - local_functional_connectivity_density: + local_functional_connectivity_density: # Select the type of threshold used when creating the lFCD adjacency matrix. # options: diff --git a/CPAC/resources/configs/test_configs/pipe-test_ANTs-BET-AllNuis.yml b/CPAC/resources/configs/test_configs/pipe-test_ANTs-BET-AllNuis.yml index 0e9e8f34c5..6e970bfd63 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_ANTs-BET-AllNuis.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_ANTs-BET-AllNuis.yml @@ -10,12 +10,12 @@ FROM: default -pipeline_setup: +pipeline_setup: # Name for this pipeline configuration - useful for identification. pipeline_name: pipe-test_ANTs-BET-AllNuis - output_directory: + output_directory: # Directory where C-PAC should write out processed data, logs, and crash reports. # - If running in a container (Singularity/Docker), you can simply set this to an arbitrary @@ -23,7 +23,7 @@ pipeline_setup: # - If running outside a container, this should be a full path to a directory. path: ./cpac_runs/output - working_directory: + working_directory: # Directory where C-PAC should store temporary and intermediate files. # - This directory must be saved if you wish to re-run your pipeline from where you left off (if not completed). @@ -39,17 +39,17 @@ pipeline_setup: # This saves disk space, but any additional preprocessing or analysis will have to be completely re-run. remove_working_dir: Off - log_directory: + log_directory: path: ./cpac_runs/log - system_config: + system_config: # Select Off if you intend to run CPAC on a single machine. # If set to On, CPAC will attempt to submit jobs through the job scheduler / resource manager selected below. - on_grid: + on_grid: - SGE: + SGE: # SGE Parallel Environment to use when running CPAC. # Only applies when you are running on a grid or compute cluster using SGE. @@ -68,36 +68,36 @@ pipeline_setup: # If you have specified an FSL path in your .bashrc file, this path will be set automatically. FSLDIR: FSLDIR -anatomical_preproc: +anatomical_preproc: - brain_extraction: + brain_extraction: # using: ['3dSkullStrip', 'BET', 'UNet', 'niworkflows-ants'] # this is a fork option using: [BET] - FSL-BET: + FSL-BET: # Mask created along with skull stripping. It should be `On`, if selected functionalMasking : ['Anatomical_Refined'] and `FSL` as skull-stripping method. mask_boolean: Off -segmentation: +segmentation: - tissue_segmentation: + tissue_segmentation: # option parameters - FSL-FAST: + FSL-FAST: - use_priors: + use_priors: # Full path to a directory containing binarized prior probability maps. # These maps are included as part of the 'Image Resource Files' package available on the Install page of the User Guide. # It is not necessary to change this path unless you intend to use non-standard priors. priors_path: $FSLDIR/data/standard/tissuepriors/2mm -registration_workflows: +registration_workflows: - anatomical_registration: + anatomical_registration: # Template to be used during registration. # It is not necessary to change this path unless you intend to use a non-standard template. @@ -110,34 +110,34 @@ registration_workflows: # Register skull-on anatomical image to a template. reg_with_skull: Off - registration: + registration: - FSL-FNIRT: + FSL-FNIRT: # Reference mask for FSL registration. ref_mask: $FSLDIR/data/standard/MNI152_T1_${resolution_for_anat}_brain_mask_dil.nii.gz - functional_registration: + functional_registration: - coregistration: + coregistration: - boundary_based_registration: + boundary_based_registration: # Standard FSL 5.0 Scheduler used for Boundary Based Registration. # It is not necessary to change this path unless you intend to use non-standard MNI registration. bbr_schedule: $FSLDIR/etc/flirtsch/bbr.sch - EPI_registration: + EPI_registration: - FSL-FNIRT: + FSL-FNIRT: # Identity matrix used during FSL-based resampling of BOLD-space data throughout the pipeline. # It is not necessary to change this path unless you intend to use a different template. identity_matrix: $FSLDIR/etc/flirtsch/ident.mat - func_registration_to_template: + func_registration_to_template: - output_resolution: + output_resolution: # The resolution (in mm) to which the registered derivative outputs are written into. # NOTE: @@ -145,10 +145,10 @@ registration_workflows: # thus, a higher resolution may not result in a large increase in RAM needs as above func_derivative_outputs: 2mm - target_template: + target_template: # option parameters - T1_template: + T1_template: # Standard Skull Stripped Template. Used as a reference image for functional registration. # This can be different than the template used as the reference/fixed for T1-to-template registration. @@ -158,12 +158,12 @@ registration_workflows: # This can be different than the template used as the reference/fixed for T1-to-template registration. T1w_template_funcreg: $FSLDIR/data/standard/MNI152_T1_${func_resolution}.nii.gz -nuisance_corrections: +nuisance_corrections: - 2-nuisance_regression: + 2-nuisance_regression: # Select which nuisance signal corrections to apply - Regressors: + Regressors: - Bandpass: bottom_frequency: 0.01 top_frequency: 0.1 @@ -279,28 +279,28 @@ nuisance_corrections: # used in CSF mask refinement for CSF signal-related regressions lateral_ventricles_mask: $FSLDIR/data/atlases/HarvardOxford/HarvardOxford-lateral-ventricles-thr25-2mm.nii.gz -timeseries_extraction: +timeseries_extraction: # Enter paths to region-of-interest (ROI) NIFTI files (.nii or .nii.gz) to be used for time-series extraction, and then select which types of analyses to run. # Denote which analyses to run for each ROI path by listing the names below. For example, if you wish to run Avg and SpatialReg, you would enter: '/path/to/ROI.nii.gz': Avg, SpatialReg # available analyses: # /path/to/atlas.nii.gz: Avg, Voxel, SpatialReg - tse_roi_paths: + tse_roi_paths: s3://fcp-indi/resources/cpac/resources/rois_2mm.nii.gz: Avg, Voxel, SpatialReg -seed_based_correlation_analysis: +seed_based_correlation_analysis: # Enter paths to region-of-interest (ROI) NIFTI files (.nii or .nii.gz) to be used for seed-based correlation analysis, and then select which types of analyses to run. # Denote which analyses to run for each ROI path by listing the names below. For example, if you wish to run Avg and MultReg, you would enter: '/path/to/ROI.nii.gz': Avg, MultReg # available analyses: # /path/to/atlas.nii.gz: Avg, DualReg, MultReg - sca_roi_paths: + sca_roi_paths: s3://fcp-indi/resources/cpac/resources/rois_2mm.nii.gz: Avg, MultReg s3://fcp-indi/resources/cpac/resources/PNAS_Smith09_rsn10.nii.gz: DualReg -voxel_mirrored_homotopic_connectivity: +voxel_mirrored_homotopic_connectivity: - symmetric_registration: + symmetric_registration: # Included as part of the 'Image Resource Files' package available on the Install page of the User Guide. # It is not necessary to change this path unless you intend to use a non-standard symmetric template. @@ -314,7 +314,7 @@ voxel_mirrored_homotopic_connectivity: # It is not necessary to change this path unless you intend to use a non-standard symmetric template. dilated_symmetric_brain_mask: $FSLDIR/data/standard/MNI152_T1_${resolution_for_anat}_brain_mask_symmetric_dil.nii.gz -network_centrality: +network_centrality: # Maximum amount of RAM (in GB) to be used when calculating Degree Centrality. # Calculating Eigenvector Centrality will require additional memory based on the size of the mask or number of ROI nodes. @@ -323,7 +323,7 @@ network_centrality: # Full path to a NIFTI file describing the mask. Centrality will be calculated for all voxels within the mask. template_specification_file: s3://fcp-indi/resources/cpac/resources/mask-thr50-3mm.nii.gz - eigenvector_centrality: + eigenvector_centrality: # Enable/Disable eigenvector centrality by selecting the connectivity weights # weight_options: ['Binarized', 'Weighted'] @@ -331,7 +331,7 @@ network_centrality: # weight_options: [] weight_options: [Binarized, Weighted] - local_functional_connectivity_density: + local_functional_connectivity_density: # Select the type of threshold used when creating the lFCD adjacency matrix. # options: diff --git a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-3dSk-AllNuis.yml b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-3dSk-AllNuis.yml index 3326d3427f..adc9bc18f0 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-3dSk-AllNuis.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-3dSk-AllNuis.yml @@ -10,12 +10,12 @@ FROM: default -pipeline_setup: +pipeline_setup: # Name for this pipeline configuration - useful for identification. pipeline_name: pipe-test_FNIRT-3dSk-AllNuis - output_directory: + output_directory: # Directory where C-PAC should write out processed data, logs, and crash reports. # - If running in a container (Singularity/Docker), you can simply set this to an arbitrary @@ -23,7 +23,7 @@ pipeline_setup: # - If running outside a container, this should be a full path to a directory. path: ./cpac_runs/output - working_directory: + working_directory: # Directory where C-PAC should store temporary and intermediate files. # - This directory must be saved if you wish to re-run your pipeline from where you left off (if not completed). @@ -39,17 +39,17 @@ pipeline_setup: # This saves disk space, but any additional preprocessing or analysis will have to be completely re-run. remove_working_dir: Off - log_directory: + log_directory: path: ./cpac_runs/log - system_config: + system_config: # Select Off if you intend to run CPAC on a single machine. # If set to On, CPAC will attempt to submit jobs through the job scheduler / resource manager selected below. - on_grid: + on_grid: - SGE: + SGE: # SGE Parallel Environment to use when running CPAC. # Only applies when you are running on a grid or compute cluster using SGE. @@ -68,32 +68,32 @@ pipeline_setup: # If you have specified an FSL path in your .bashrc file, this path will be set automatically. FSLDIR: FSLDIR -anatomical_preproc: +anatomical_preproc: - brain_extraction: + brain_extraction: - FSL-BET: + FSL-BET: # Mask created along with skull stripping. It should be `On`, if selected functionalMasking : ['Anatomical_Refined'] and `FSL` as skull-stripping method. mask_boolean: Off -segmentation: +segmentation: - tissue_segmentation: + tissue_segmentation: # option parameters - FSL-FAST: + FSL-FAST: - use_priors: + use_priors: # Full path to a directory containing binarized prior probability maps. # These maps are included as part of the 'Image Resource Files' package available on the Install page of the User Guide. # It is not necessary to change this path unless you intend to use non-standard priors. priors_path: $FSLDIR/data/standard/tissuepriors/2mm -registration_workflows: +registration_workflows: - anatomical_registration: + anatomical_registration: # Template to be used during registration. # It is not necessary to change this path unless you intend to use a non-standard template. @@ -106,39 +106,39 @@ registration_workflows: # Register skull-on anatomical image to a template. reg_with_skull: Off - registration: + registration: # using: ['ANTS', 'FSL', 'FSL-linear'] # this is a fork point # selecting both ['ANTS', 'FSL'] will run both and fork the pipeline using: [FSL] - FSL-FNIRT: + FSL-FNIRT: # Reference mask for FSL registration. ref_mask: $FSLDIR/data/standard/MNI152_T1_${resolution_for_anat}_brain_mask_dil.nii.gz - functional_registration: + functional_registration: - coregistration: + coregistration: - boundary_based_registration: + boundary_based_registration: # Standard FSL 5.0 Scheduler used for Boundary Based Registration. # It is not necessary to change this path unless you intend to use non-standard MNI registration. bbr_schedule: $FSLDIR/etc/flirtsch/bbr.sch - EPI_registration: + EPI_registration: - FSL-FNIRT: + FSL-FNIRT: # Identity matrix used during FSL-based resampling of BOLD-space data throughout the pipeline. # It is not necessary to change this path unless you intend to use a different template. identity_matrix: $FSLDIR/etc/flirtsch/ident.mat - func_registration_to_template: + func_registration_to_template: - output_resolution: + output_resolution: # The resolution (in mm) to which the registered derivative outputs are written into. # NOTE: @@ -146,10 +146,10 @@ registration_workflows: # thus, a higher resolution may not result in a large increase in RAM needs as above func_derivative_outputs: 2mm - target_template: + target_template: # option parameters - T1_template: + T1_template: # Standard Skull Stripped Template. Used as a reference image for functional registration. # This can be different than the template used as the reference/fixed for T1-to-template registration. @@ -159,18 +159,18 @@ registration_workflows: # This can be different than the template used as the reference/fixed for T1-to-template registration. T1w_template_funcreg: $FSLDIR/data/standard/MNI152_T1_${func_resolution}.nii.gz -nuisance_corrections: +nuisance_corrections: - 1-ICA-AROMA: + 1-ICA-AROMA: # this is a fork point # run: [On, Off] - this will run both and fork the pipeline run: [On] - 2-nuisance_regression: + 2-nuisance_regression: # Select which nuisance signal corrections to apply - Regressors: + Regressors: - Bandpass: bottom_frequency: 0.01 top_frequency: 0.1 @@ -286,28 +286,28 @@ nuisance_corrections: # used in CSF mask refinement for CSF signal-related regressions lateral_ventricles_mask: $FSLDIR/data/atlases/HarvardOxford/HarvardOxford-lateral-ventricles-thr25-2mm.nii.gz -timeseries_extraction: +timeseries_extraction: # Enter paths to region-of-interest (ROI) NIFTI files (.nii or .nii.gz) to be used for time-series extraction, and then select which types of analyses to run. # Denote which analyses to run for each ROI path by listing the names below. For example, if you wish to run Avg and SpatialReg, you would enter: '/path/to/ROI.nii.gz': Avg, SpatialReg # available analyses: # /path/to/atlas.nii.gz: Avg, Voxel, SpatialReg - tse_roi_paths: + tse_roi_paths: s3://fcp-indi/resources/cpac/resources/rois_2mm.nii.gz: Avg, Voxel, SpatialReg -seed_based_correlation_analysis: +seed_based_correlation_analysis: # Enter paths to region-of-interest (ROI) NIFTI files (.nii or .nii.gz) to be used for seed-based correlation analysis, and then select which types of analyses to run. # Denote which analyses to run for each ROI path by listing the names below. For example, if you wish to run Avg and MultReg, you would enter: '/path/to/ROI.nii.gz': Avg, MultReg # available analyses: # /path/to/atlas.nii.gz: Avg, DualReg, MultReg - sca_roi_paths: + sca_roi_paths: s3://fcp-indi/resources/cpac/resources/rois_2mm.nii.gz: Avg, MultReg s3://fcp-indi/resources/cpac/resources/PNAS_Smith09_rsn10.nii.gz: DualReg -voxel_mirrored_homotopic_connectivity: +voxel_mirrored_homotopic_connectivity: - symmetric_registration: + symmetric_registration: # Included as part of the 'Image Resource Files' package available on the Install page of the User Guide. # It is not necessary to change this path unless you intend to use a non-standard symmetric template. @@ -321,7 +321,7 @@ voxel_mirrored_homotopic_connectivity: # It is not necessary to change this path unless you intend to use a non-standard symmetric template. dilated_symmetric_brain_mask: $FSLDIR/data/standard/MNI152_T1_${resolution_for_anat}_brain_mask_symmetric_dil.nii.gz -network_centrality: +network_centrality: # Maximum amount of RAM (in GB) to be used when calculating Degree Centrality. # Calculating Eigenvector Centrality will require additional memory based on the size of the mask or number of ROI nodes. @@ -330,7 +330,7 @@ network_centrality: # Full path to a NIFTI file describing the mask. Centrality will be calculated for all voxels within the mask. template_specification_file: s3://fcp-indi/resources/cpac/resources/mask-thr50-3mm.nii.gz - eigenvector_centrality: + eigenvector_centrality: # Enable/Disable eigenvector centrality by selecting the connectivity weights # weight_options: ['Binarized', 'Weighted'] @@ -338,7 +338,7 @@ network_centrality: # weight_options: [] weight_options: [Binarized, Weighted] - local_functional_connectivity_density: + local_functional_connectivity_density: # Select the type of threshold used when creating the lFCD adjacency matrix. # options: diff --git a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-BASC.yml b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-BASC.yml index 8f4faae6c1..c134253199 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-BASC.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-BASC.yml @@ -10,12 +10,12 @@ FROM: default -pipeline_setup: +pipeline_setup: # Name for this pipeline configuration - useful for identification. pipeline_name: pipe-test_FNIRT-BET-AllNuis - output_directory: + output_directory: # Directory where C-PAC should write out processed data, logs, and crash reports. # - If running in a container (Singularity/Docker), you can simply set this to an arbitrary @@ -23,7 +23,7 @@ pipeline_setup: # - If running outside a container, this should be a full path to a directory. path: ./cpac_runs/output - working_directory: + working_directory: # Directory where C-PAC should store temporary and intermediate files. # - This directory must be saved if you wish to re-run your pipeline from where you left off (if not completed). @@ -39,17 +39,17 @@ pipeline_setup: # This saves disk space, but any additional preprocessing or analysis will have to be completely re-run. remove_working_dir: Off - log_directory: + log_directory: path: ./cpac_runs/log - system_config: + system_config: # Select Off if you intend to run CPAC on a single machine. # If set to On, CPAC will attempt to submit jobs through the job scheduler / resource manager selected below. - on_grid: + on_grid: - SGE: + SGE: # SGE Parallel Environment to use when running CPAC. # Only applies when you are running on a grid or compute cluster using SGE. @@ -68,36 +68,36 @@ pipeline_setup: # If you have specified an FSL path in your .bashrc file, this path will be set automatically. FSLDIR: FSLDIR -anatomical_preproc: +anatomical_preproc: - brain_extraction: + brain_extraction: # using: ['3dSkullStrip', 'BET', 'UNet', 'niworkflows-ants'] # this is a fork option using: [BET] - FSL-BET: + FSL-BET: # Mask created along with skull stripping. It should be `On`, if selected functionalMasking : ['Anatomical_Refined'] and `FSL` as skull-stripping method. mask_boolean: Off -segmentation: +segmentation: - tissue_segmentation: + tissue_segmentation: # option parameters - FSL-FAST: + FSL-FAST: - use_priors: + use_priors: # Full path to a directory containing binarized prior probability maps. # These maps are included as part of the 'Image Resource Files' package available on the Install page of the User Guide. # It is not necessary to change this path unless you intend to use non-standard priors. priors_path: $FSLDIR/data/standard/tissuepriors/2mm -registration_workflows: +registration_workflows: - anatomical_registration: + anatomical_registration: # Template to be used during registration. # It is not necessary to change this path unless you intend to use a non-standard template. @@ -110,39 +110,39 @@ registration_workflows: # Register skull-on anatomical image to a template. reg_with_skull: Off - registration: + registration: # using: ['ANTS', 'FSL', 'FSL-linear'] # this is a fork point # selecting both ['ANTS', 'FSL'] will run both and fork the pipeline using: [FSL] - FSL-FNIRT: + FSL-FNIRT: # Reference mask for FSL registration. ref_mask: $FSLDIR/data/standard/MNI152_T1_${resolution_for_anat}_brain_mask_dil.nii.gz - functional_registration: + functional_registration: - coregistration: + coregistration: - boundary_based_registration: + boundary_based_registration: # Standard FSL 5.0 Scheduler used for Boundary Based Registration. # It is not necessary to change this path unless you intend to use non-standard MNI registration. bbr_schedule: $FSLDIR/etc/flirtsch/bbr.sch - EPI_registration: + EPI_registration: - FSL-FNIRT: + FSL-FNIRT: # Identity matrix used during FSL-based resampling of BOLD-space data throughout the pipeline. # It is not necessary to change this path unless you intend to use a different template. identity_matrix: $FSLDIR/etc/flirtsch/ident.mat - func_registration_to_template: + func_registration_to_template: - output_resolution: + output_resolution: # The resolution (in mm) to which the registered derivative outputs are written into. # NOTE: @@ -150,10 +150,10 @@ registration_workflows: # thus, a higher resolution may not result in a large increase in RAM needs as above func_derivative_outputs: 2mm - target_template: + target_template: # option parameters - T1_template: + T1_template: # Standard Skull Stripped Template. Used as a reference image for functional registration. # This can be different than the template used as the reference/fixed for T1-to-template registration. @@ -163,12 +163,12 @@ registration_workflows: # This can be different than the template used as the reference/fixed for T1-to-template registration. T1w_template_funcreg: $FSLDIR/data/standard/MNI152_T1_${func_resolution}.nii.gz -nuisance_corrections: +nuisance_corrections: - 2-nuisance_regression: + 2-nuisance_regression: # Select which nuisance signal corrections to apply - Regressors: + Regressors: - Bandpass: bottom_frequency: 0.01 top_frequency: 0.1 @@ -284,28 +284,28 @@ nuisance_corrections: # used in CSF mask refinement for CSF signal-related regressions lateral_ventricles_mask: $FSLDIR/data/atlases/HarvardOxford/HarvardOxford-lateral-ventricles-thr25-2mm.nii.gz -timeseries_extraction: +timeseries_extraction: # Enter paths to region-of-interest (ROI) NIFTI files (.nii or .nii.gz) to be used for time-series extraction, and then select which types of analyses to run. # Denote which analyses to run for each ROI path by listing the names below. For example, if you wish to run Avg and SpatialReg, you would enter: '/path/to/ROI.nii.gz': Avg, SpatialReg # available analyses: # /path/to/atlas.nii.gz: Avg, Voxel, SpatialReg - tse_roi_paths: + tse_roi_paths: s3://fcp-indi/resources/cpac/resources/rois_2mm.nii.gz: Avg, Voxel, SpatialReg -seed_based_correlation_analysis: +seed_based_correlation_analysis: # Enter paths to region-of-interest (ROI) NIFTI files (.nii or .nii.gz) to be used for seed-based correlation analysis, and then select which types of analyses to run. # Denote which analyses to run for each ROI path by listing the names below. For example, if you wish to run Avg and MultReg, you would enter: '/path/to/ROI.nii.gz': Avg, MultReg # available analyses: # /path/to/atlas.nii.gz: Avg, DualReg, MultReg - sca_roi_paths: + sca_roi_paths: s3://fcp-indi/resources/cpac/resources/rois_2mm.nii.gz: Avg, MultReg s3://fcp-indi/resources/cpac/resources/PNAS_Smith09_rsn10.nii.gz: DualReg -voxel_mirrored_homotopic_connectivity: +voxel_mirrored_homotopic_connectivity: - symmetric_registration: + symmetric_registration: # Included as part of the 'Image Resource Files' package available on the Install page of the User Guide. # It is not necessary to change this path unless you intend to use a non-standard symmetric template. @@ -319,7 +319,7 @@ voxel_mirrored_homotopic_connectivity: # It is not necessary to change this path unless you intend to use a non-standard symmetric template. dilated_symmetric_brain_mask: $FSLDIR/data/standard/MNI152_T1_${resolution_for_anat}_brain_mask_symmetric_dil.nii.gz -network_centrality: +network_centrality: # Maximum amount of RAM (in GB) to be used when calculating Degree Centrality. # Calculating Eigenvector Centrality will require additional memory based on the size of the mask or number of ROI nodes. @@ -328,7 +328,7 @@ network_centrality: # Full path to a NIFTI file describing the mask. Centrality will be calculated for all voxels within the mask. template_specification_file: s3://fcp-indi/resources/cpac/resources/mask-thr50-3mm.nii.gz - eigenvector_centrality: + eigenvector_centrality: # Enable/Disable eigenvector centrality by selecting the connectivity weights # weight_options: ['Binarized', 'Weighted'] @@ -336,7 +336,7 @@ network_centrality: # weight_options: [] weight_options: [Binarized, Weighted] - local_functional_connectivity_density: + local_functional_connectivity_density: # Select the type of threshold used when creating the lFCD adjacency matrix. # options: diff --git a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-ISC-voxel.yml b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-ISC-voxel.yml index 8f4faae6c1..c134253199 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-ISC-voxel.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-ISC-voxel.yml @@ -10,12 +10,12 @@ FROM: default -pipeline_setup: +pipeline_setup: # Name for this pipeline configuration - useful for identification. pipeline_name: pipe-test_FNIRT-BET-AllNuis - output_directory: + output_directory: # Directory where C-PAC should write out processed data, logs, and crash reports. # - If running in a container (Singularity/Docker), you can simply set this to an arbitrary @@ -23,7 +23,7 @@ pipeline_setup: # - If running outside a container, this should be a full path to a directory. path: ./cpac_runs/output - working_directory: + working_directory: # Directory where C-PAC should store temporary and intermediate files. # - This directory must be saved if you wish to re-run your pipeline from where you left off (if not completed). @@ -39,17 +39,17 @@ pipeline_setup: # This saves disk space, but any additional preprocessing or analysis will have to be completely re-run. remove_working_dir: Off - log_directory: + log_directory: path: ./cpac_runs/log - system_config: + system_config: # Select Off if you intend to run CPAC on a single machine. # If set to On, CPAC will attempt to submit jobs through the job scheduler / resource manager selected below. - on_grid: + on_grid: - SGE: + SGE: # SGE Parallel Environment to use when running CPAC. # Only applies when you are running on a grid or compute cluster using SGE. @@ -68,36 +68,36 @@ pipeline_setup: # If you have specified an FSL path in your .bashrc file, this path will be set automatically. FSLDIR: FSLDIR -anatomical_preproc: +anatomical_preproc: - brain_extraction: + brain_extraction: # using: ['3dSkullStrip', 'BET', 'UNet', 'niworkflows-ants'] # this is a fork option using: [BET] - FSL-BET: + FSL-BET: # Mask created along with skull stripping. It should be `On`, if selected functionalMasking : ['Anatomical_Refined'] and `FSL` as skull-stripping method. mask_boolean: Off -segmentation: +segmentation: - tissue_segmentation: + tissue_segmentation: # option parameters - FSL-FAST: + FSL-FAST: - use_priors: + use_priors: # Full path to a directory containing binarized prior probability maps. # These maps are included as part of the 'Image Resource Files' package available on the Install page of the User Guide. # It is not necessary to change this path unless you intend to use non-standard priors. priors_path: $FSLDIR/data/standard/tissuepriors/2mm -registration_workflows: +registration_workflows: - anatomical_registration: + anatomical_registration: # Template to be used during registration. # It is not necessary to change this path unless you intend to use a non-standard template. @@ -110,39 +110,39 @@ registration_workflows: # Register skull-on anatomical image to a template. reg_with_skull: Off - registration: + registration: # using: ['ANTS', 'FSL', 'FSL-linear'] # this is a fork point # selecting both ['ANTS', 'FSL'] will run both and fork the pipeline using: [FSL] - FSL-FNIRT: + FSL-FNIRT: # Reference mask for FSL registration. ref_mask: $FSLDIR/data/standard/MNI152_T1_${resolution_for_anat}_brain_mask_dil.nii.gz - functional_registration: + functional_registration: - coregistration: + coregistration: - boundary_based_registration: + boundary_based_registration: # Standard FSL 5.0 Scheduler used for Boundary Based Registration. # It is not necessary to change this path unless you intend to use non-standard MNI registration. bbr_schedule: $FSLDIR/etc/flirtsch/bbr.sch - EPI_registration: + EPI_registration: - FSL-FNIRT: + FSL-FNIRT: # Identity matrix used during FSL-based resampling of BOLD-space data throughout the pipeline. # It is not necessary to change this path unless you intend to use a different template. identity_matrix: $FSLDIR/etc/flirtsch/ident.mat - func_registration_to_template: + func_registration_to_template: - output_resolution: + output_resolution: # The resolution (in mm) to which the registered derivative outputs are written into. # NOTE: @@ -150,10 +150,10 @@ registration_workflows: # thus, a higher resolution may not result in a large increase in RAM needs as above func_derivative_outputs: 2mm - target_template: + target_template: # option parameters - T1_template: + T1_template: # Standard Skull Stripped Template. Used as a reference image for functional registration. # This can be different than the template used as the reference/fixed for T1-to-template registration. @@ -163,12 +163,12 @@ registration_workflows: # This can be different than the template used as the reference/fixed for T1-to-template registration. T1w_template_funcreg: $FSLDIR/data/standard/MNI152_T1_${func_resolution}.nii.gz -nuisance_corrections: +nuisance_corrections: - 2-nuisance_regression: + 2-nuisance_regression: # Select which nuisance signal corrections to apply - Regressors: + Regressors: - Bandpass: bottom_frequency: 0.01 top_frequency: 0.1 @@ -284,28 +284,28 @@ nuisance_corrections: # used in CSF mask refinement for CSF signal-related regressions lateral_ventricles_mask: $FSLDIR/data/atlases/HarvardOxford/HarvardOxford-lateral-ventricles-thr25-2mm.nii.gz -timeseries_extraction: +timeseries_extraction: # Enter paths to region-of-interest (ROI) NIFTI files (.nii or .nii.gz) to be used for time-series extraction, and then select which types of analyses to run. # Denote which analyses to run for each ROI path by listing the names below. For example, if you wish to run Avg and SpatialReg, you would enter: '/path/to/ROI.nii.gz': Avg, SpatialReg # available analyses: # /path/to/atlas.nii.gz: Avg, Voxel, SpatialReg - tse_roi_paths: + tse_roi_paths: s3://fcp-indi/resources/cpac/resources/rois_2mm.nii.gz: Avg, Voxel, SpatialReg -seed_based_correlation_analysis: +seed_based_correlation_analysis: # Enter paths to region-of-interest (ROI) NIFTI files (.nii or .nii.gz) to be used for seed-based correlation analysis, and then select which types of analyses to run. # Denote which analyses to run for each ROI path by listing the names below. For example, if you wish to run Avg and MultReg, you would enter: '/path/to/ROI.nii.gz': Avg, MultReg # available analyses: # /path/to/atlas.nii.gz: Avg, DualReg, MultReg - sca_roi_paths: + sca_roi_paths: s3://fcp-indi/resources/cpac/resources/rois_2mm.nii.gz: Avg, MultReg s3://fcp-indi/resources/cpac/resources/PNAS_Smith09_rsn10.nii.gz: DualReg -voxel_mirrored_homotopic_connectivity: +voxel_mirrored_homotopic_connectivity: - symmetric_registration: + symmetric_registration: # Included as part of the 'Image Resource Files' package available on the Install page of the User Guide. # It is not necessary to change this path unless you intend to use a non-standard symmetric template. @@ -319,7 +319,7 @@ voxel_mirrored_homotopic_connectivity: # It is not necessary to change this path unless you intend to use a non-standard symmetric template. dilated_symmetric_brain_mask: $FSLDIR/data/standard/MNI152_T1_${resolution_for_anat}_brain_mask_symmetric_dil.nii.gz -network_centrality: +network_centrality: # Maximum amount of RAM (in GB) to be used when calculating Degree Centrality. # Calculating Eigenvector Centrality will require additional memory based on the size of the mask or number of ROI nodes. @@ -328,7 +328,7 @@ network_centrality: # Full path to a NIFTI file describing the mask. Centrality will be calculated for all voxels within the mask. template_specification_file: s3://fcp-indi/resources/cpac/resources/mask-thr50-3mm.nii.gz - eigenvector_centrality: + eigenvector_centrality: # Enable/Disable eigenvector centrality by selecting the connectivity weights # weight_options: ['Binarized', 'Weighted'] @@ -336,7 +336,7 @@ network_centrality: # weight_options: [] weight_options: [Binarized, Weighted] - local_functional_connectivity_density: + local_functional_connectivity_density: # Select the type of threshold used when creating the lFCD adjacency matrix. # options: diff --git a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-ISC.yml b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-ISC.yml index 8f4faae6c1..c134253199 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-ISC.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-ISC.yml @@ -10,12 +10,12 @@ FROM: default -pipeline_setup: +pipeline_setup: # Name for this pipeline configuration - useful for identification. pipeline_name: pipe-test_FNIRT-BET-AllNuis - output_directory: + output_directory: # Directory where C-PAC should write out processed data, logs, and crash reports. # - If running in a container (Singularity/Docker), you can simply set this to an arbitrary @@ -23,7 +23,7 @@ pipeline_setup: # - If running outside a container, this should be a full path to a directory. path: ./cpac_runs/output - working_directory: + working_directory: # Directory where C-PAC should store temporary and intermediate files. # - This directory must be saved if you wish to re-run your pipeline from where you left off (if not completed). @@ -39,17 +39,17 @@ pipeline_setup: # This saves disk space, but any additional preprocessing or analysis will have to be completely re-run. remove_working_dir: Off - log_directory: + log_directory: path: ./cpac_runs/log - system_config: + system_config: # Select Off if you intend to run CPAC on a single machine. # If set to On, CPAC will attempt to submit jobs through the job scheduler / resource manager selected below. - on_grid: + on_grid: - SGE: + SGE: # SGE Parallel Environment to use when running CPAC. # Only applies when you are running on a grid or compute cluster using SGE. @@ -68,36 +68,36 @@ pipeline_setup: # If you have specified an FSL path in your .bashrc file, this path will be set automatically. FSLDIR: FSLDIR -anatomical_preproc: +anatomical_preproc: - brain_extraction: + brain_extraction: # using: ['3dSkullStrip', 'BET', 'UNet', 'niworkflows-ants'] # this is a fork option using: [BET] - FSL-BET: + FSL-BET: # Mask created along with skull stripping. It should be `On`, if selected functionalMasking : ['Anatomical_Refined'] and `FSL` as skull-stripping method. mask_boolean: Off -segmentation: +segmentation: - tissue_segmentation: + tissue_segmentation: # option parameters - FSL-FAST: + FSL-FAST: - use_priors: + use_priors: # Full path to a directory containing binarized prior probability maps. # These maps are included as part of the 'Image Resource Files' package available on the Install page of the User Guide. # It is not necessary to change this path unless you intend to use non-standard priors. priors_path: $FSLDIR/data/standard/tissuepriors/2mm -registration_workflows: +registration_workflows: - anatomical_registration: + anatomical_registration: # Template to be used during registration. # It is not necessary to change this path unless you intend to use a non-standard template. @@ -110,39 +110,39 @@ registration_workflows: # Register skull-on anatomical image to a template. reg_with_skull: Off - registration: + registration: # using: ['ANTS', 'FSL', 'FSL-linear'] # this is a fork point # selecting both ['ANTS', 'FSL'] will run both and fork the pipeline using: [FSL] - FSL-FNIRT: + FSL-FNIRT: # Reference mask for FSL registration. ref_mask: $FSLDIR/data/standard/MNI152_T1_${resolution_for_anat}_brain_mask_dil.nii.gz - functional_registration: + functional_registration: - coregistration: + coregistration: - boundary_based_registration: + boundary_based_registration: # Standard FSL 5.0 Scheduler used for Boundary Based Registration. # It is not necessary to change this path unless you intend to use non-standard MNI registration. bbr_schedule: $FSLDIR/etc/flirtsch/bbr.sch - EPI_registration: + EPI_registration: - FSL-FNIRT: + FSL-FNIRT: # Identity matrix used during FSL-based resampling of BOLD-space data throughout the pipeline. # It is not necessary to change this path unless you intend to use a different template. identity_matrix: $FSLDIR/etc/flirtsch/ident.mat - func_registration_to_template: + func_registration_to_template: - output_resolution: + output_resolution: # The resolution (in mm) to which the registered derivative outputs are written into. # NOTE: @@ -150,10 +150,10 @@ registration_workflows: # thus, a higher resolution may not result in a large increase in RAM needs as above func_derivative_outputs: 2mm - target_template: + target_template: # option parameters - T1_template: + T1_template: # Standard Skull Stripped Template. Used as a reference image for functional registration. # This can be different than the template used as the reference/fixed for T1-to-template registration. @@ -163,12 +163,12 @@ registration_workflows: # This can be different than the template used as the reference/fixed for T1-to-template registration. T1w_template_funcreg: $FSLDIR/data/standard/MNI152_T1_${func_resolution}.nii.gz -nuisance_corrections: +nuisance_corrections: - 2-nuisance_regression: + 2-nuisance_regression: # Select which nuisance signal corrections to apply - Regressors: + Regressors: - Bandpass: bottom_frequency: 0.01 top_frequency: 0.1 @@ -284,28 +284,28 @@ nuisance_corrections: # used in CSF mask refinement for CSF signal-related regressions lateral_ventricles_mask: $FSLDIR/data/atlases/HarvardOxford/HarvardOxford-lateral-ventricles-thr25-2mm.nii.gz -timeseries_extraction: +timeseries_extraction: # Enter paths to region-of-interest (ROI) NIFTI files (.nii or .nii.gz) to be used for time-series extraction, and then select which types of analyses to run. # Denote which analyses to run for each ROI path by listing the names below. For example, if you wish to run Avg and SpatialReg, you would enter: '/path/to/ROI.nii.gz': Avg, SpatialReg # available analyses: # /path/to/atlas.nii.gz: Avg, Voxel, SpatialReg - tse_roi_paths: + tse_roi_paths: s3://fcp-indi/resources/cpac/resources/rois_2mm.nii.gz: Avg, Voxel, SpatialReg -seed_based_correlation_analysis: +seed_based_correlation_analysis: # Enter paths to region-of-interest (ROI) NIFTI files (.nii or .nii.gz) to be used for seed-based correlation analysis, and then select which types of analyses to run. # Denote which analyses to run for each ROI path by listing the names below. For example, if you wish to run Avg and MultReg, you would enter: '/path/to/ROI.nii.gz': Avg, MultReg # available analyses: # /path/to/atlas.nii.gz: Avg, DualReg, MultReg - sca_roi_paths: + sca_roi_paths: s3://fcp-indi/resources/cpac/resources/rois_2mm.nii.gz: Avg, MultReg s3://fcp-indi/resources/cpac/resources/PNAS_Smith09_rsn10.nii.gz: DualReg -voxel_mirrored_homotopic_connectivity: +voxel_mirrored_homotopic_connectivity: - symmetric_registration: + symmetric_registration: # Included as part of the 'Image Resource Files' package available on the Install page of the User Guide. # It is not necessary to change this path unless you intend to use a non-standard symmetric template. @@ -319,7 +319,7 @@ voxel_mirrored_homotopic_connectivity: # It is not necessary to change this path unless you intend to use a non-standard symmetric template. dilated_symmetric_brain_mask: $FSLDIR/data/standard/MNI152_T1_${resolution_for_anat}_brain_mask_symmetric_dil.nii.gz -network_centrality: +network_centrality: # Maximum amount of RAM (in GB) to be used when calculating Degree Centrality. # Calculating Eigenvector Centrality will require additional memory based on the size of the mask or number of ROI nodes. @@ -328,7 +328,7 @@ network_centrality: # Full path to a NIFTI file describing the mask. Centrality will be calculated for all voxels within the mask. template_specification_file: s3://fcp-indi/resources/cpac/resources/mask-thr50-3mm.nii.gz - eigenvector_centrality: + eigenvector_centrality: # Enable/Disable eigenvector centrality by selecting the connectivity weights # weight_options: ['Binarized', 'Weighted'] @@ -336,7 +336,7 @@ network_centrality: # weight_options: [] weight_options: [Binarized, Weighted] - local_functional_connectivity_density: + local_functional_connectivity_density: # Select the type of threshold used when creating the lFCD adjacency matrix. # options: diff --git a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-MDMR.yml b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-MDMR.yml index 8f4faae6c1..c134253199 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-MDMR.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis-MDMR.yml @@ -10,12 +10,12 @@ FROM: default -pipeline_setup: +pipeline_setup: # Name for this pipeline configuration - useful for identification. pipeline_name: pipe-test_FNIRT-BET-AllNuis - output_directory: + output_directory: # Directory where C-PAC should write out processed data, logs, and crash reports. # - If running in a container (Singularity/Docker), you can simply set this to an arbitrary @@ -23,7 +23,7 @@ pipeline_setup: # - If running outside a container, this should be a full path to a directory. path: ./cpac_runs/output - working_directory: + working_directory: # Directory where C-PAC should store temporary and intermediate files. # - This directory must be saved if you wish to re-run your pipeline from where you left off (if not completed). @@ -39,17 +39,17 @@ pipeline_setup: # This saves disk space, but any additional preprocessing or analysis will have to be completely re-run. remove_working_dir: Off - log_directory: + log_directory: path: ./cpac_runs/log - system_config: + system_config: # Select Off if you intend to run CPAC on a single machine. # If set to On, CPAC will attempt to submit jobs through the job scheduler / resource manager selected below. - on_grid: + on_grid: - SGE: + SGE: # SGE Parallel Environment to use when running CPAC. # Only applies when you are running on a grid or compute cluster using SGE. @@ -68,36 +68,36 @@ pipeline_setup: # If you have specified an FSL path in your .bashrc file, this path will be set automatically. FSLDIR: FSLDIR -anatomical_preproc: +anatomical_preproc: - brain_extraction: + brain_extraction: # using: ['3dSkullStrip', 'BET', 'UNet', 'niworkflows-ants'] # this is a fork option using: [BET] - FSL-BET: + FSL-BET: # Mask created along with skull stripping. It should be `On`, if selected functionalMasking : ['Anatomical_Refined'] and `FSL` as skull-stripping method. mask_boolean: Off -segmentation: +segmentation: - tissue_segmentation: + tissue_segmentation: # option parameters - FSL-FAST: + FSL-FAST: - use_priors: + use_priors: # Full path to a directory containing binarized prior probability maps. # These maps are included as part of the 'Image Resource Files' package available on the Install page of the User Guide. # It is not necessary to change this path unless you intend to use non-standard priors. priors_path: $FSLDIR/data/standard/tissuepriors/2mm -registration_workflows: +registration_workflows: - anatomical_registration: + anatomical_registration: # Template to be used during registration. # It is not necessary to change this path unless you intend to use a non-standard template. @@ -110,39 +110,39 @@ registration_workflows: # Register skull-on anatomical image to a template. reg_with_skull: Off - registration: + registration: # using: ['ANTS', 'FSL', 'FSL-linear'] # this is a fork point # selecting both ['ANTS', 'FSL'] will run both and fork the pipeline using: [FSL] - FSL-FNIRT: + FSL-FNIRT: # Reference mask for FSL registration. ref_mask: $FSLDIR/data/standard/MNI152_T1_${resolution_for_anat}_brain_mask_dil.nii.gz - functional_registration: + functional_registration: - coregistration: + coregistration: - boundary_based_registration: + boundary_based_registration: # Standard FSL 5.0 Scheduler used for Boundary Based Registration. # It is not necessary to change this path unless you intend to use non-standard MNI registration. bbr_schedule: $FSLDIR/etc/flirtsch/bbr.sch - EPI_registration: + EPI_registration: - FSL-FNIRT: + FSL-FNIRT: # Identity matrix used during FSL-based resampling of BOLD-space data throughout the pipeline. # It is not necessary to change this path unless you intend to use a different template. identity_matrix: $FSLDIR/etc/flirtsch/ident.mat - func_registration_to_template: + func_registration_to_template: - output_resolution: + output_resolution: # The resolution (in mm) to which the registered derivative outputs are written into. # NOTE: @@ -150,10 +150,10 @@ registration_workflows: # thus, a higher resolution may not result in a large increase in RAM needs as above func_derivative_outputs: 2mm - target_template: + target_template: # option parameters - T1_template: + T1_template: # Standard Skull Stripped Template. Used as a reference image for functional registration. # This can be different than the template used as the reference/fixed for T1-to-template registration. @@ -163,12 +163,12 @@ registration_workflows: # This can be different than the template used as the reference/fixed for T1-to-template registration. T1w_template_funcreg: $FSLDIR/data/standard/MNI152_T1_${func_resolution}.nii.gz -nuisance_corrections: +nuisance_corrections: - 2-nuisance_regression: + 2-nuisance_regression: # Select which nuisance signal corrections to apply - Regressors: + Regressors: - Bandpass: bottom_frequency: 0.01 top_frequency: 0.1 @@ -284,28 +284,28 @@ nuisance_corrections: # used in CSF mask refinement for CSF signal-related regressions lateral_ventricles_mask: $FSLDIR/data/atlases/HarvardOxford/HarvardOxford-lateral-ventricles-thr25-2mm.nii.gz -timeseries_extraction: +timeseries_extraction: # Enter paths to region-of-interest (ROI) NIFTI files (.nii or .nii.gz) to be used for time-series extraction, and then select which types of analyses to run. # Denote which analyses to run for each ROI path by listing the names below. For example, if you wish to run Avg and SpatialReg, you would enter: '/path/to/ROI.nii.gz': Avg, SpatialReg # available analyses: # /path/to/atlas.nii.gz: Avg, Voxel, SpatialReg - tse_roi_paths: + tse_roi_paths: s3://fcp-indi/resources/cpac/resources/rois_2mm.nii.gz: Avg, Voxel, SpatialReg -seed_based_correlation_analysis: +seed_based_correlation_analysis: # Enter paths to region-of-interest (ROI) NIFTI files (.nii or .nii.gz) to be used for seed-based correlation analysis, and then select which types of analyses to run. # Denote which analyses to run for each ROI path by listing the names below. For example, if you wish to run Avg and MultReg, you would enter: '/path/to/ROI.nii.gz': Avg, MultReg # available analyses: # /path/to/atlas.nii.gz: Avg, DualReg, MultReg - sca_roi_paths: + sca_roi_paths: s3://fcp-indi/resources/cpac/resources/rois_2mm.nii.gz: Avg, MultReg s3://fcp-indi/resources/cpac/resources/PNAS_Smith09_rsn10.nii.gz: DualReg -voxel_mirrored_homotopic_connectivity: +voxel_mirrored_homotopic_connectivity: - symmetric_registration: + symmetric_registration: # Included as part of the 'Image Resource Files' package available on the Install page of the User Guide. # It is not necessary to change this path unless you intend to use a non-standard symmetric template. @@ -319,7 +319,7 @@ voxel_mirrored_homotopic_connectivity: # It is not necessary to change this path unless you intend to use a non-standard symmetric template. dilated_symmetric_brain_mask: $FSLDIR/data/standard/MNI152_T1_${resolution_for_anat}_brain_mask_symmetric_dil.nii.gz -network_centrality: +network_centrality: # Maximum amount of RAM (in GB) to be used when calculating Degree Centrality. # Calculating Eigenvector Centrality will require additional memory based on the size of the mask or number of ROI nodes. @@ -328,7 +328,7 @@ network_centrality: # Full path to a NIFTI file describing the mask. Centrality will be calculated for all voxels within the mask. template_specification_file: s3://fcp-indi/resources/cpac/resources/mask-thr50-3mm.nii.gz - eigenvector_centrality: + eigenvector_centrality: # Enable/Disable eigenvector centrality by selecting the connectivity weights # weight_options: ['Binarized', 'Weighted'] @@ -336,7 +336,7 @@ network_centrality: # weight_options: [] weight_options: [Binarized, Weighted] - local_functional_connectivity_density: + local_functional_connectivity_density: # Select the type of threshold used when creating the lFCD adjacency matrix. # options: diff --git a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis.yml b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis.yml index 8f4faae6c1..c134253199 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_FNIRT-BET-AllNuis.yml @@ -10,12 +10,12 @@ FROM: default -pipeline_setup: +pipeline_setup: # Name for this pipeline configuration - useful for identification. pipeline_name: pipe-test_FNIRT-BET-AllNuis - output_directory: + output_directory: # Directory where C-PAC should write out processed data, logs, and crash reports. # - If running in a container (Singularity/Docker), you can simply set this to an arbitrary @@ -23,7 +23,7 @@ pipeline_setup: # - If running outside a container, this should be a full path to a directory. path: ./cpac_runs/output - working_directory: + working_directory: # Directory where C-PAC should store temporary and intermediate files. # - This directory must be saved if you wish to re-run your pipeline from where you left off (if not completed). @@ -39,17 +39,17 @@ pipeline_setup: # This saves disk space, but any additional preprocessing or analysis will have to be completely re-run. remove_working_dir: Off - log_directory: + log_directory: path: ./cpac_runs/log - system_config: + system_config: # Select Off if you intend to run CPAC on a single machine. # If set to On, CPAC will attempt to submit jobs through the job scheduler / resource manager selected below. - on_grid: + on_grid: - SGE: + SGE: # SGE Parallel Environment to use when running CPAC. # Only applies when you are running on a grid or compute cluster using SGE. @@ -68,36 +68,36 @@ pipeline_setup: # If you have specified an FSL path in your .bashrc file, this path will be set automatically. FSLDIR: FSLDIR -anatomical_preproc: +anatomical_preproc: - brain_extraction: + brain_extraction: # using: ['3dSkullStrip', 'BET', 'UNet', 'niworkflows-ants'] # this is a fork option using: [BET] - FSL-BET: + FSL-BET: # Mask created along with skull stripping. It should be `On`, if selected functionalMasking : ['Anatomical_Refined'] and `FSL` as skull-stripping method. mask_boolean: Off -segmentation: +segmentation: - tissue_segmentation: + tissue_segmentation: # option parameters - FSL-FAST: + FSL-FAST: - use_priors: + use_priors: # Full path to a directory containing binarized prior probability maps. # These maps are included as part of the 'Image Resource Files' package available on the Install page of the User Guide. # It is not necessary to change this path unless you intend to use non-standard priors. priors_path: $FSLDIR/data/standard/tissuepriors/2mm -registration_workflows: +registration_workflows: - anatomical_registration: + anatomical_registration: # Template to be used during registration. # It is not necessary to change this path unless you intend to use a non-standard template. @@ -110,39 +110,39 @@ registration_workflows: # Register skull-on anatomical image to a template. reg_with_skull: Off - registration: + registration: # using: ['ANTS', 'FSL', 'FSL-linear'] # this is a fork point # selecting both ['ANTS', 'FSL'] will run both and fork the pipeline using: [FSL] - FSL-FNIRT: + FSL-FNIRT: # Reference mask for FSL registration. ref_mask: $FSLDIR/data/standard/MNI152_T1_${resolution_for_anat}_brain_mask_dil.nii.gz - functional_registration: + functional_registration: - coregistration: + coregistration: - boundary_based_registration: + boundary_based_registration: # Standard FSL 5.0 Scheduler used for Boundary Based Registration. # It is not necessary to change this path unless you intend to use non-standard MNI registration. bbr_schedule: $FSLDIR/etc/flirtsch/bbr.sch - EPI_registration: + EPI_registration: - FSL-FNIRT: + FSL-FNIRT: # Identity matrix used during FSL-based resampling of BOLD-space data throughout the pipeline. # It is not necessary to change this path unless you intend to use a different template. identity_matrix: $FSLDIR/etc/flirtsch/ident.mat - func_registration_to_template: + func_registration_to_template: - output_resolution: + output_resolution: # The resolution (in mm) to which the registered derivative outputs are written into. # NOTE: @@ -150,10 +150,10 @@ registration_workflows: # thus, a higher resolution may not result in a large increase in RAM needs as above func_derivative_outputs: 2mm - target_template: + target_template: # option parameters - T1_template: + T1_template: # Standard Skull Stripped Template. Used as a reference image for functional registration. # This can be different than the template used as the reference/fixed for T1-to-template registration. @@ -163,12 +163,12 @@ registration_workflows: # This can be different than the template used as the reference/fixed for T1-to-template registration. T1w_template_funcreg: $FSLDIR/data/standard/MNI152_T1_${func_resolution}.nii.gz -nuisance_corrections: +nuisance_corrections: - 2-nuisance_regression: + 2-nuisance_regression: # Select which nuisance signal corrections to apply - Regressors: + Regressors: - Bandpass: bottom_frequency: 0.01 top_frequency: 0.1 @@ -284,28 +284,28 @@ nuisance_corrections: # used in CSF mask refinement for CSF signal-related regressions lateral_ventricles_mask: $FSLDIR/data/atlases/HarvardOxford/HarvardOxford-lateral-ventricles-thr25-2mm.nii.gz -timeseries_extraction: +timeseries_extraction: # Enter paths to region-of-interest (ROI) NIFTI files (.nii or .nii.gz) to be used for time-series extraction, and then select which types of analyses to run. # Denote which analyses to run for each ROI path by listing the names below. For example, if you wish to run Avg and SpatialReg, you would enter: '/path/to/ROI.nii.gz': Avg, SpatialReg # available analyses: # /path/to/atlas.nii.gz: Avg, Voxel, SpatialReg - tse_roi_paths: + tse_roi_paths: s3://fcp-indi/resources/cpac/resources/rois_2mm.nii.gz: Avg, Voxel, SpatialReg -seed_based_correlation_analysis: +seed_based_correlation_analysis: # Enter paths to region-of-interest (ROI) NIFTI files (.nii or .nii.gz) to be used for seed-based correlation analysis, and then select which types of analyses to run. # Denote which analyses to run for each ROI path by listing the names below. For example, if you wish to run Avg and MultReg, you would enter: '/path/to/ROI.nii.gz': Avg, MultReg # available analyses: # /path/to/atlas.nii.gz: Avg, DualReg, MultReg - sca_roi_paths: + sca_roi_paths: s3://fcp-indi/resources/cpac/resources/rois_2mm.nii.gz: Avg, MultReg s3://fcp-indi/resources/cpac/resources/PNAS_Smith09_rsn10.nii.gz: DualReg -voxel_mirrored_homotopic_connectivity: +voxel_mirrored_homotopic_connectivity: - symmetric_registration: + symmetric_registration: # Included as part of the 'Image Resource Files' package available on the Install page of the User Guide. # It is not necessary to change this path unless you intend to use a non-standard symmetric template. @@ -319,7 +319,7 @@ voxel_mirrored_homotopic_connectivity: # It is not necessary to change this path unless you intend to use a non-standard symmetric template. dilated_symmetric_brain_mask: $FSLDIR/data/standard/MNI152_T1_${resolution_for_anat}_brain_mask_symmetric_dil.nii.gz -network_centrality: +network_centrality: # Maximum amount of RAM (in GB) to be used when calculating Degree Centrality. # Calculating Eigenvector Centrality will require additional memory based on the size of the mask or number of ROI nodes. @@ -328,7 +328,7 @@ network_centrality: # Full path to a NIFTI file describing the mask. Centrality will be calculated for all voxels within the mask. template_specification_file: s3://fcp-indi/resources/cpac/resources/mask-thr50-3mm.nii.gz - eigenvector_centrality: + eigenvector_centrality: # Enable/Disable eigenvector centrality by selecting the connectivity weights # weight_options: ['Binarized', 'Weighted'] @@ -336,7 +336,7 @@ network_centrality: # weight_options: [] weight_options: [Binarized, Weighted] - local_functional_connectivity_density: + local_functional_connectivity_density: # Select the type of threshold used when creating the lFCD adjacency matrix. # options: diff --git a/CPAC/resources/configs/test_configs/pipe-test_all.yml b/CPAC/resources/configs/test_configs/pipe-test_all.yml index b6feb9c42c..94fe0016e9 100644 --- a/CPAC/resources/configs/test_configs/pipe-test_all.yml +++ b/CPAC/resources/configs/test_configs/pipe-test_all.yml @@ -10,12 +10,12 @@ FROM: default -pipeline_setup: +pipeline_setup: # Name for this pipeline configuration - useful for identification. pipeline_name: pipe-test_all - output_directory: + output_directory: # Directory where C-PAC should write out processed data, logs, and crash reports. # - If running in a container (Singularity/Docker), you can simply set this to an arbitrary @@ -23,7 +23,7 @@ pipeline_setup: # - If running outside a container, this should be a full path to a directory. path: ./cpac_runs/output - working_directory: + working_directory: # Directory where C-PAC should store temporary and intermediate files. # - This directory must be saved if you wish to re-run your pipeline from where you left off (if not completed). @@ -39,17 +39,17 @@ pipeline_setup: # This saves disk space, but any additional preprocessing or analysis will have to be completely re-run. remove_working_dir: Off - log_directory: + log_directory: path: ./cpac_runs/log - system_config: + system_config: # Select Off if you intend to run CPAC on a single machine. # If set to On, CPAC will attempt to submit jobs through the job scheduler / resource manager selected below. - on_grid: + on_grid: - SGE: + SGE: # SGE Parallel Environment to use when running CPAC. # Only applies when you are running on a grid or compute cluster using SGE. @@ -68,32 +68,32 @@ pipeline_setup: # If you have specified an FSL path in your .bashrc file, this path will be set automatically. FSLDIR: FSLDIR -anatomical_preproc: +anatomical_preproc: - brain_extraction: + brain_extraction: - FSL-BET: + FSL-BET: # Mask created along with skull stripping. It should be `On`, if selected functionalMasking : ['Anatomical_Refined'] and `FSL` as skull-stripping method. mask_boolean: Off -segmentation: +segmentation: - tissue_segmentation: + tissue_segmentation: # option parameters - FSL-FAST: + FSL-FAST: - use_priors: + use_priors: # Full path to a directory containing binarized prior probability maps. # These maps are included as part of the 'Image Resource Files' package available on the Install page of the User Guide. # It is not necessary to change this path unless you intend to use non-standard priors. priors_path: $FSLDIR/data/standard/tissuepriors/2mm -registration_workflows: +registration_workflows: - anatomical_registration: + anatomical_registration: # Template to be used during registration. # It is not necessary to change this path unless you intend to use a non-standard template. @@ -106,39 +106,39 @@ registration_workflows: # Register skull-on anatomical image to a template. reg_with_skull: Off - registration: + registration: # using: ['ANTS', 'FSL', 'FSL-linear'] # this is a fork point # selecting both ['ANTS', 'FSL'] will run both and fork the pipeline using: [ANTS, FSL] - FSL-FNIRT: + FSL-FNIRT: # Reference mask for FSL registration. ref_mask: $FSLDIR/data/standard/MNI152_T1_${resolution_for_anat}_brain_mask_dil.nii.gz - functional_registration: + functional_registration: - coregistration: + coregistration: - boundary_based_registration: + boundary_based_registration: # Standard FSL 5.0 Scheduler used for Boundary Based Registration. # It is not necessary to change this path unless you intend to use non-standard MNI registration. bbr_schedule: $FSLDIR/etc/flirtsch/bbr.sch - EPI_registration: + EPI_registration: - FSL-FNIRT: + FSL-FNIRT: # Identity matrix used during FSL-based resampling of BOLD-space data throughout the pipeline. # It is not necessary to change this path unless you intend to use a different template. identity_matrix: $FSLDIR/etc/flirtsch/ident.mat - func_registration_to_template: + func_registration_to_template: - output_resolution: + output_resolution: # The resolution (in mm) to which the registered derivative outputs are written into. # NOTE: @@ -146,10 +146,10 @@ registration_workflows: # thus, a higher resolution may not result in a large increase in RAM needs as above func_derivative_outputs: 2mm - target_template: + target_template: # option parameters - T1_template: + T1_template: # Standard Skull Stripped Template. Used as a reference image for functional registration. # This can be different than the template used as the reference/fixed for T1-to-template registration. @@ -159,18 +159,18 @@ registration_workflows: # This can be different than the template used as the reference/fixed for T1-to-template registration. T1w_template_funcreg: $FSLDIR/data/standard/MNI152_T1_${func_resolution}.nii.gz -nuisance_corrections: +nuisance_corrections: - 1-ICA-AROMA: + 1-ICA-AROMA: # this is a fork point # run: [On, Off] - this will run both and fork the pipeline run: [On] - 2-nuisance_regression: + 2-nuisance_regression: # Select which nuisance signal corrections to apply - Regressors: + Regressors: - Bandpass: bottom_frequency: 0.01 top_frequency: 0.1 @@ -286,28 +286,28 @@ nuisance_corrections: # used in CSF mask refinement for CSF signal-related regressions lateral_ventricles_mask: $FSLDIR/data/atlases/HarvardOxford/HarvardOxford-lateral-ventricles-thr25-2mm.nii.gz -timeseries_extraction: +timeseries_extraction: # Enter paths to region-of-interest (ROI) NIFTI files (.nii or .nii.gz) to be used for time-series extraction, and then select which types of analyses to run. # Denote which analyses to run for each ROI path by listing the names below. For example, if you wish to run Avg and SpatialReg, you would enter: '/path/to/ROI.nii.gz': Avg, SpatialReg # available analyses: # /path/to/atlas.nii.gz: Avg, Voxel, SpatialReg - tse_roi_paths: + tse_roi_paths: s3://fcp-indi/resources/cpac/resources/rois_2mm.nii.gz: Avg, Voxel, SpatialReg -seed_based_correlation_analysis: +seed_based_correlation_analysis: # Enter paths to region-of-interest (ROI) NIFTI files (.nii or .nii.gz) to be used for seed-based correlation analysis, and then select which types of analyses to run. # Denote which analyses to run for each ROI path by listing the names below. For example, if you wish to run Avg and MultReg, you would enter: '/path/to/ROI.nii.gz': Avg, MultReg # available analyses: # /path/to/atlas.nii.gz: Avg, DualReg, MultReg - sca_roi_paths: + sca_roi_paths: s3://fcp-indi/resources/cpac/resources/rois_2mm.nii.gz: Avg, MultReg s3://fcp-indi/resources/cpac/resources/PNAS_Smith09_rsn10.nii.gz: DualReg -voxel_mirrored_homotopic_connectivity: +voxel_mirrored_homotopic_connectivity: - symmetric_registration: + symmetric_registration: # Included as part of the 'Image Resource Files' package available on the Install page of the User Guide. # It is not necessary to change this path unless you intend to use a non-standard symmetric template. @@ -321,7 +321,7 @@ voxel_mirrored_homotopic_connectivity: # It is not necessary to change this path unless you intend to use a non-standard symmetric template. dilated_symmetric_brain_mask: $FSLDIR/data/standard/MNI152_T1_${resolution_for_anat}_brain_mask_symmetric_dil.nii.gz -network_centrality: +network_centrality: # Maximum amount of RAM (in GB) to be used when calculating Degree Centrality. # Calculating Eigenvector Centrality will require additional memory based on the size of the mask or number of ROI nodes. @@ -330,7 +330,7 @@ network_centrality: # Full path to a NIFTI file describing the mask. Centrality will be calculated for all voxels within the mask. template_specification_file: s3://fcp-indi/resources/cpac/resources/mask-thr50-3mm.nii.gz - eigenvector_centrality: + eigenvector_centrality: # Enable/Disable eigenvector centrality by selecting the connectivity weights # weight_options: ['Binarized', 'Weighted'] @@ -338,7 +338,7 @@ network_centrality: # weight_options: [] weight_options: [Binarized, Weighted] - local_functional_connectivity_density: + local_functional_connectivity_density: # Select the type of threshold used when creating the lFCD adjacency matrix. # options: diff --git a/CPAC/resources/cpac_outputs.tsv b/CPAC/resources/cpac_outputs.tsv index 03ddd5de73..cf6a492380 100644 --- a/CPAC/resources/cpac_outputs.tsv +++ b/CPAC/resources/cpac_outputs.tsv @@ -1,220 +1,220 @@ Resource Type Space Sub-Directory File To Smooth To z-std 4D Time Series Optional: Debugging Multi-File -alff alff functional func NIfTI Yes Yes -desc-sm_alff alff functional func NIfTI Yes -desc-sm-zstd_alff alff functional func NIfTI -desc-zstd_alff alff functional func NIfTI -space-template_alff alff template func NIfTI Yes Yes -space-template_desc-sm_alff alff template func NIfTI Yes -space-template_desc-sm-zstd_alff alff template func NIfTI -space-template_desc-zstd_alff alff template func NIfTI -desc-brain_bold bold functional func NIfTI Yes Yes -desc-mean_bold bold functional func NIfTI -desc-motion_bold bold functional func NIfTI Yes Yes -desc-preproc_bold bold functional func NIfTI Yes -desc-sm_bold bold functional func NIfTI Yes Yes -sbref bold functional func NIfTI -space-EPItemplate_bold bold EPI template func NIfTI Yes -space-EPItemplate_desc-brain_bold bold EPI template func NIfTI Yes Yes -space-EPItemplate_desc-mean_bold bold EPI template func NIfTI -space-EPItemplate_desc-preproc_bold bold EPI template func NIfTI Yes -space-symtemplate_desc-sm_bold bold symmetric template func NIfTI Yes Yes -space-T1w_sbref bold T1w func NIfTI -space-template_bold bold template func NIfTI Yes -space-template_desc-brain_bold bold template func NIfTI Yes Yes -space-template_desc-head_bold bold template func NIfTI Yes -space-template_desc-mean_bold bold template func NIfTI -space-template_desc-preproc_bold bold template func NIfTI Yes -space-template_desc-scout_bold bold template func NIfTI -space-template_sbref bold template func NIfTI -space-template_desc-DualReg_correlations correlation template func NIfTI -space-template_desc-MeanSCA_correlations correlation template func NIfTI -space-template_desc-MultReg_correlations correlation template func NIfTI -space-template_desc-ndmg_correlations correlation template func NIfTI -space-template_desc-PearsonAfni_correlations correlation template func tsv -space-template_desc-PartialAfni_correlations correlation template func tsv -space-template_desc-PearsonNilearn_correlations correlation template func tsv -space-template_desc-PartialNilearn_correlations correlation template func tsv -space-template_dcb degree-centrality template func NIfTI Yes Yes -space-template_desc-sm_dcb degree-centrality template func NIfTI Yes -space-template_desc-sm-zstd_dcb degree-centrality template func NIfTI -space-template_desc-zstd_dcb degree-centrality template func NIfTI -space-template_dcw degree-centrality template func NIfTI Yes Yes -space-template_desc-sm_dcw degree-centrality template func NIfTI Yes -space-template_desc-sm-zstd_dcw degree-centrality template func NIfTI -space-template_desc-zstd_dcw degree-centrality template func NIfTI -space-template_ecb eigen-centrality template func NIfTI Yes Yes -space-template_desc-sm_ecb eigen-centrality template func NIfTI Yes -space-template_desc-sm-zstd_ecb eigen-centrality template func NIfTI -space-template_desc-zstd_ecb eigen-centrality template func NIfTI -space-template_ecw eigen-centrality template func NIfTI Yes Yes -space-template_desc-sm_ecw eigen-centrality template func NIfTI Yes -space-template_desc-sm-zstd_ecw eigen-centrality template func NIfTI -space-template_desc-zstd_ecw eigen-centrality template func NIfTI -desc-sm_falff falff functional func NIfTI Yes -desc-sm-zstd_falff falff functional func NIfTI -desc-zstd_falff falff functional func NIfTI -falff falff functional func NIfTI Yes Yes -space-template_desc-sm_falff falff template func NIfTI Yes -space-template_desc-sm-zstd_falff falff template func NIfTI -space-template_desc-zstd_falff falff template func NIfTI -space-template_falff falff template func NIfTI Yes Yes -space-template_lfcdb lfcd template func NIfTI Yes Yes -space-template_desc-sm_lfcdb lfcd template func NIfTI Yes -space-template_desc-sm-zstd_lfcdb lfcd template func NIfTI -space-template_desc-zstd_lfcdb lfcd template func NIfTI -space-template_lfcdw lfcd template func NIfTI Yes Yes -space-template_desc-sm_lfcdw lfcd template func NIfTI Yes -space-template_desc-sm-zstd_lfcdw lfcd template func NIfTI -space-template_desc-zstd_lfcdw lfcd template func NIfTI -space-EPItemplate_desc-bold_mask mask EPI template func NIfTI -space-EPItemplate_res-derivative_desc-bold_mask mask EPI template func NIfTI -space-bold_desc-brain_mask mask functional func NIfTI -space-bold_desc-eroded_mask mask functional func NIfTI -space-bold_label-CSF_desc-eroded_mask mask functional func NIfTI -space-bold_label-CSF_mask mask functional func NIfTI -space-bold_label-GM_desc-eroded_mask mask functional func NIfTI -space-bold_label-GM_mask mask functional func NIfTI -space-bold_label-WM_desc-eroded_mask mask functional func NIfTI -space-bold_label-WM_mask mask functional func NIfTI -space-longitudinal_desc-brain_mask mask longitudinal T1w anat NIfTI -space-longitudinal_label-CSF_desc-preproc_mask mask longitudinal T1w anat NIfTI -space-longitudinal_label-CSF_mask mask longitudinal T1w anat NIfTI -space-longitudinal_label-GM_desc-preproc_mask mask longitudinal T1w anat NIfTI -space-longitudinal_label-GM_mask mask longitudinal T1w anat NIfTI -space-longitudinal_label-WM_desc-preproc_mask mask longitudinal T1w anat NIfTI -space-longitudinal_label-WM_mask mask longitudinal T1w anat NIfTI -label-CSF_desc-eroded_mask mask T1w anat NIfTI -label-CSF_desc-preproc_mask mask T1w anat NIfTI -label-CSF_mask mask T1w anat NIfTI -label-GM_desc-eroded_mask mask T1w anat NIfTI -label-GM_desc-preproc_mask mask T1w anat NIfTI -label-GM_mask mask T1w anat NIfTI -label-WM_desc-eroded_mask mask T1w anat NIfTI -label-WM_desc-preproc_mask mask T1w anat NIfTI -label-WM_mask mask T1w anat NIfTI -space-T1w_desc-acpcbrain_mask mask T1w anat NIfTI -space-T1w_desc-brain_mask mask T1w anat NIfTI -space-T1w_desc-eroded_mask mask T1w anat NIfTI -space-template_desc-brain_mask mask template anat NIfTI -space-template_desc-bold_mask mask template func NIfTI -space-template_res-derivative_desc-bold_mask mask template func NIfTI -motion motion func TSV -desc-summary_motion motion func TSV -motion-filter-plot motion func png -desc-movementParameters_motion motion func TSV -desc-movementParametersUnfiltered_motion motion func TSV -label-CSF_probseg probseg T1w anat NIfTI -label-GM_probseg probseg T1w anat NIfTI -label-WM_probseg probseg T1w anat NIfTI -desc-T1wAxial_quality qc anat png -desc-T1wSagittal_quality qc anat png -desc-dsegAxial_quality qc anat png -desc-dsegSagittal_quality qc anat png -desc-boldAxial_quality qc func png -desc-boldSagittal_quality qc func png -desc-boldCarpet_quality qc func png -desc-framewiseDisplacementJenkinsonPlot_quality qc func png -desc-movementParametersTrans_quality qc func png -desc-movementParametersRot_quality qc func png -desc-boldSnrAxial_quality qc func png -desc-boldSnrSagittal_quality qc func png -desc-boldSnrHist_quality qc func png -desc-boldSnr_quality qc func png -space-template_desc-xcp_quality qc func tsv -desc-confounds_timeseries regressors func 1D -desc-sm_reho reho functional func NIfTI Yes -desc-sm-zstd_reho reho functional func NIfTI -desc-zstd_reho reho functional func NIfTI -reho reho functional func NIfTI Yes Yes -space-template_desc-sm_reho reho template func NIfTI Yes -space-template_desc-sm-zstd_reho reho template func NIfTI -space-template_desc-zstd_reho reho template func NIfTI -space-template_reho reho template func NIfTI Yes Yes -desc-DualReg_statmap statistic template func NIfTI -desc-MultReg_statmap statistic template func NIfTI +alff alff functional func NIfTI Yes Yes +desc-sm_alff alff functional func NIfTI Yes +desc-sm-zstd_alff alff functional func NIfTI +desc-zstd_alff alff functional func NIfTI +space-template_alff alff template func NIfTI Yes Yes +space-template_desc-sm_alff alff template func NIfTI Yes +space-template_desc-sm-zstd_alff alff template func NIfTI +space-template_desc-zstd_alff alff template func NIfTI +desc-brain_bold bold functional func NIfTI Yes Yes +desc-mean_bold bold functional func NIfTI +desc-motion_bold bold functional func NIfTI Yes Yes +desc-preproc_bold bold functional func NIfTI Yes +desc-sm_bold bold functional func NIfTI Yes Yes +sbref bold functional func NIfTI +space-EPItemplate_bold bold EPI template func NIfTI Yes +space-EPItemplate_desc-brain_bold bold EPI template func NIfTI Yes Yes +space-EPItemplate_desc-mean_bold bold EPI template func NIfTI +space-EPItemplate_desc-preproc_bold bold EPI template func NIfTI Yes +space-symtemplate_desc-sm_bold bold symmetric template func NIfTI Yes Yes +space-T1w_sbref bold T1w func NIfTI +space-template_bold bold template func NIfTI Yes +space-template_desc-brain_bold bold template func NIfTI Yes Yes +space-template_desc-head_bold bold template func NIfTI Yes +space-template_desc-mean_bold bold template func NIfTI +space-template_desc-preproc_bold bold template func NIfTI Yes +space-template_desc-scout_bold bold template func NIfTI +space-template_sbref bold template func NIfTI +space-template_desc-DualReg_correlations correlation template func NIfTI +space-template_desc-MeanSCA_correlations correlation template func NIfTI +space-template_desc-MultReg_correlations correlation template func NIfTI +space-template_desc-ndmg_correlations correlation template func NIfTI +space-template_desc-PearsonAfni_correlations correlation template func tsv +space-template_desc-PartialAfni_correlations correlation template func tsv +space-template_desc-PearsonNilearn_correlations correlation template func tsv +space-template_desc-PartialNilearn_correlations correlation template func tsv +space-template_dcb degree-centrality template func NIfTI Yes Yes +space-template_desc-sm_dcb degree-centrality template func NIfTI Yes +space-template_desc-sm-zstd_dcb degree-centrality template func NIfTI +space-template_desc-zstd_dcb degree-centrality template func NIfTI +space-template_dcw degree-centrality template func NIfTI Yes Yes +space-template_desc-sm_dcw degree-centrality template func NIfTI Yes +space-template_desc-sm-zstd_dcw degree-centrality template func NIfTI +space-template_desc-zstd_dcw degree-centrality template func NIfTI +space-template_ecb eigen-centrality template func NIfTI Yes Yes +space-template_desc-sm_ecb eigen-centrality template func NIfTI Yes +space-template_desc-sm-zstd_ecb eigen-centrality template func NIfTI +space-template_desc-zstd_ecb eigen-centrality template func NIfTI +space-template_ecw eigen-centrality template func NIfTI Yes Yes +space-template_desc-sm_ecw eigen-centrality template func NIfTI Yes +space-template_desc-sm-zstd_ecw eigen-centrality template func NIfTI +space-template_desc-zstd_ecw eigen-centrality template func NIfTI +desc-sm_falff falff functional func NIfTI Yes +desc-sm-zstd_falff falff functional func NIfTI +desc-zstd_falff falff functional func NIfTI +falff falff functional func NIfTI Yes Yes +space-template_desc-sm_falff falff template func NIfTI Yes +space-template_desc-sm-zstd_falff falff template func NIfTI +space-template_desc-zstd_falff falff template func NIfTI +space-template_falff falff template func NIfTI Yes Yes +space-template_lfcdb lfcd template func NIfTI Yes Yes +space-template_desc-sm_lfcdb lfcd template func NIfTI Yes +space-template_desc-sm-zstd_lfcdb lfcd template func NIfTI +space-template_desc-zstd_lfcdb lfcd template func NIfTI +space-template_lfcdw lfcd template func NIfTI Yes Yes +space-template_desc-sm_lfcdw lfcd template func NIfTI Yes +space-template_desc-sm-zstd_lfcdw lfcd template func NIfTI +space-template_desc-zstd_lfcdw lfcd template func NIfTI +space-EPItemplate_desc-bold_mask mask EPI template func NIfTI +space-EPItemplate_res-derivative_desc-bold_mask mask EPI template func NIfTI +space-bold_desc-brain_mask mask functional func NIfTI +space-bold_desc-eroded_mask mask functional func NIfTI +space-bold_label-CSF_desc-eroded_mask mask functional func NIfTI +space-bold_label-CSF_mask mask functional func NIfTI +space-bold_label-GM_desc-eroded_mask mask functional func NIfTI +space-bold_label-GM_mask mask functional func NIfTI +space-bold_label-WM_desc-eroded_mask mask functional func NIfTI +space-bold_label-WM_mask mask functional func NIfTI +space-longitudinal_desc-brain_mask mask longitudinal T1w anat NIfTI +space-longitudinal_label-CSF_desc-preproc_mask mask longitudinal T1w anat NIfTI +space-longitudinal_label-CSF_mask mask longitudinal T1w anat NIfTI +space-longitudinal_label-GM_desc-preproc_mask mask longitudinal T1w anat NIfTI +space-longitudinal_label-GM_mask mask longitudinal T1w anat NIfTI +space-longitudinal_label-WM_desc-preproc_mask mask longitudinal T1w anat NIfTI +space-longitudinal_label-WM_mask mask longitudinal T1w anat NIfTI +label-CSF_desc-eroded_mask mask T1w anat NIfTI +label-CSF_desc-preproc_mask mask T1w anat NIfTI +label-CSF_mask mask T1w anat NIfTI +label-GM_desc-eroded_mask mask T1w anat NIfTI +label-GM_desc-preproc_mask mask T1w anat NIfTI +label-GM_mask mask T1w anat NIfTI +label-WM_desc-eroded_mask mask T1w anat NIfTI +label-WM_desc-preproc_mask mask T1w anat NIfTI +label-WM_mask mask T1w anat NIfTI +space-T1w_desc-acpcbrain_mask mask T1w anat NIfTI +space-T1w_desc-brain_mask mask T1w anat NIfTI +space-T1w_desc-eroded_mask mask T1w anat NIfTI +space-template_desc-brain_mask mask template anat NIfTI +space-template_desc-bold_mask mask template func NIfTI +space-template_res-derivative_desc-bold_mask mask template func NIfTI +motion motion func TSV +desc-summary_motion motion func TSV +motion-filter-plot motion func png +desc-movementParameters_motion motion func TSV +desc-movementParametersUnfiltered_motion motion func TSV +label-CSF_probseg probseg T1w anat NIfTI +label-GM_probseg probseg T1w anat NIfTI +label-WM_probseg probseg T1w anat NIfTI +desc-T1wAxial_quality qc anat png +desc-T1wSagittal_quality qc anat png +desc-dsegAxial_quality qc anat png +desc-dsegSagittal_quality qc anat png +desc-boldAxial_quality qc func png +desc-boldSagittal_quality qc func png +desc-boldCarpet_quality qc func png +desc-framewiseDisplacementJenkinsonPlot_quality qc func png +desc-movementParametersTrans_quality qc func png +desc-movementParametersRot_quality qc func png +desc-boldSnrAxial_quality qc func png +desc-boldSnrSagittal_quality qc func png +desc-boldSnrHist_quality qc func png +desc-boldSnr_quality qc func png +space-template_desc-xcp_quality qc func tsv +desc-confounds_timeseries regressors func 1D +desc-sm_reho reho functional func NIfTI Yes +desc-sm-zstd_reho reho functional func NIfTI +desc-zstd_reho reho functional func NIfTI +reho reho functional func NIfTI Yes Yes +space-template_desc-sm_reho reho template func NIfTI Yes +space-template_desc-sm-zstd_reho reho template func NIfTI +space-template_desc-zstd_reho reho template func NIfTI +space-template_reho reho template func NIfTI Yes Yes +desc-DualReg_statmap statistic template func NIfTI +desc-MultReg_statmap statistic template func NIfTI hemi-L_desc-surfaceMap_thickness surface-derived anat Yes hemi-R_desc-surfaceMap_thickness surface-derived anat Yes hemi-L_desc-surfaceMap_volume surface-derived anat Yes hemi-R_desc-surfaceMap_volume surface-derived anat Yes -hemi-L_desc-surfaceMesh_pial surface-derived anat -hemi-R_desc-surfaceMesh_pial surface-derived anat -raw-average surface-derived anat -hemi-L_desc-surfaceMesh_smoothwm surface-derived anat -hemi-R_desc-surfaceMesh_smoothwm surface-derived anat -atlas-DesikanKilliany_space-fsLR_den-32k_dlabel surface-derived anat -atlas-Destrieux_space-fsLR_den-32k_dlabel surface-derived anat -atlas-DesikanKilliany_space-fsLR_den-164k_dlabel surface-derived anat -atlas-Destrieux_space-fsLR_den-164k_dlabel surface-derived anat -space-fsLR_den-32k_bold-dtseries surface-derived func +hemi-L_desc-surfaceMesh_pial surface-derived anat +hemi-R_desc-surfaceMesh_pial surface-derived anat +raw-average surface-derived anat +hemi-L_desc-surfaceMesh_smoothwm surface-derived anat +hemi-R_desc-surfaceMesh_smoothwm surface-derived anat +atlas-DesikanKilliany_space-fsLR_den-32k_dlabel surface-derived anat +atlas-Destrieux_space-fsLR_den-32k_dlabel surface-derived anat +atlas-DesikanKilliany_space-fsLR_den-164k_dlabel surface-derived anat +atlas-Destrieux_space-fsLR_den-164k_dlabel surface-derived anat +space-fsLR_den-32k_bold-dtseries surface-derived func hemi-L_desc-surfaceMesh_sphere surface-derived anat Yes hemi-R_desc-surfaceMesh_sphere surface-derived anat Yes hemi-L_desc-surfaceMap_sulc surface-derived anat Yes hemi-R_desc-surfaceMap_sulc surface-derived anat Yes -hemi-L_desc-surface_curv surface-derived anat -hemi-R_desc-surface_curv surface-derived anat +hemi-L_desc-surface_curv surface-derived anat +hemi-R_desc-surface_curv surface-derived anat hemi-L_desc-surfaceMesh_white surface-derived anat Yes hemi-R_desc-surfaceMesh_white surface-derived anat Yes wmparc surface-derived anat Yes -space-symtemplate_desc-brain_T1w T1w symmetric template anat NIfTI Yes -desc-brain_T1w T1w T1w anat NIfTI Yes -desc-head_T1w T1w T1w anat NIfTI -desc-preproc_T1w T1w T1w anat NIfTI -desc-reorient_T1w T1w T1w anat NIfTI Yes -desc-restore_T1w T1w T1w anat NIfTI -desc-restore-brain_T1w T1w T1w anat NIfTI -space-template_desc-brain_T1w T1w template anat NIfTI Yes -space-template_desc-preproc_T1w T1w template anat NIfTI -space-template_desc-head_T1w T1w template anat NIfTI -space-template_desc-T1w_mask mask template anat NIfTI -space-template_desc-Mean_timeseries timeseries func 1D -desc-MeanSCA_timeseries timeseries func 1D -desc-SpatReg_timeseries timeseries func 1D -desc-Voxel_timeseries timeseries func 1D -space-longitudinal_label-CSF_probseg tissue probability longitudinal T1w anat NIfTI -space-longitudinal_label-GM_probseg tissue probability longitudinal T1w anat NIfTI -space-longitudinal_label-WM_probseg tissue probability longitudinal T1w anat NIfTI -vmhc vmhc symmetric template func NIfTI -blip-warp xfm func NIfTI -from-bold_to-EPItemplate_mode-image_desc-linear_xfm xfm func NIfTI -from-bold_to-EPItemplate_mode-image_desc-nonlinear_xfm xfm func NIfTI -from-bold_to-EPItemplate_mode-image_xfm xfm func NIfTI -from-bold_to-symtemplate_mode-image_xfm xfm func NIfTI -from-bold_to-T1w_mode-image_desc-linear_xfm xfm func NIfTI -from-bold_to-template_mode-image_xfm xfm func NIfTI -from-EPItemplate_to-bold_mode-image_desc-linear_xfm xfm func NIfTI -from-EPItemplate_to-bold_mode-image_desc-nonlinear_xfm xfm func NIfTI -from-longitudinal_to-symtemplate_mode-image_desc-linear_xfm xfm anat NIfTI -from-longitudinal_to-symtemplate_mode-image_desc-nonlinear_xfm xfm anat NIfTI -from-longitudinal_to-symtemplate_mode-image_xfm xfm anat NIfTI -from-longitudinal_to-template_mode-image_desc-linear_xfm xfm anat NIfTI -from-longitudinal_to-template_mode-image_desc-nonlinear_xfm xfm anat NIfTI -from-longitudinal_to-template_mode-image_xfm xfm anat NIfTI -from-symtemplate_to-bold_mode-image_xfm xfm func NIfTI -from-symtemplate_to-longitudinal_mode-image_desc-linear_xfm xfm anat NIfTI -from-symtemplate_to-longitudinal_mode-image_desc-nonlinear_xfm xfm anat NIfTI -from-symtemplate_to-longitudinal_mode-image_xfm xfm anat NIfTI -from-symtemplate_to-T1w_mode-image_desc-linear_xfm xfm anat NIfTI -from-symtemplate_to-T1w_mode-image_desc-nonlinear_xfm xfm anat NIfTI -from-symtemplate_to-T1w_mode-image_xfm xfm anat NIfTI -from-T1w_to-symtemplate_mode-image_desc-linear_xfm xfm anat NIfTI -from-T1w_to-symtemplate_mode-image_desc-nonlinear_xfm xfm anat NIfTI -from-T1w_to-symtemplate_mode-image_xfm xfm anat NIfTI -from-T1w_to-template_mode-image_desc-linear_xfm xfm anat NIfTI -from-T1w_to-template_mode-image_desc-nonlinear_xfm xfm anat NIfTI -from-T1w_to-template_mode-image_xfm xfm anat NIfTI -from-template_to-bold_mode-image_xfm xfm func NIfTI -from-template_to-longitudinal_mode-image_desc-linear_xfm xfm anat NIfTI -from-template_to-longitudinal_mode-image_desc-nonlinear_xfm xfm anat NIfTI -from-template_to-longitudinal_mode-image_xfm xfm anat NIfTI -from-template_to-T1w_mode-image_desc-linear_xfm xfm anat NIfTI -from-template_to-T1w_mode-image_desc-nonlinear_xfm xfm anat NIfTI -from-template_to-T1w_mode-image_xfm xfm anat NIfTI -space-template_label-CSF_mask mask template anat NIfTI -space-template_label-WM_mask mask template anat NIfTI -space-template_label-GM_mask mask template anat NIfTI -space-EPItemplate_label-CSF_mask mask template func NIfTI -space-EPItemplate_label-WM_mask mask template func NIfTI -space-EPItemplate_label-GM_mask mask template func NIfTI -mdmr group functional group_analysis NIfTI -desc-zstd-mdmr group functional group_analysis NIfTI Yes +space-symtemplate_desc-brain_T1w T1w symmetric template anat NIfTI Yes +desc-brain_T1w T1w T1w anat NIfTI Yes +desc-head_T1w T1w T1w anat NIfTI +desc-preproc_T1w T1w T1w anat NIfTI +desc-reorient_T1w T1w T1w anat NIfTI Yes +desc-restore_T1w T1w T1w anat NIfTI +desc-restore-brain_T1w T1w T1w anat NIfTI +space-template_desc-brain_T1w T1w template anat NIfTI Yes +space-template_desc-preproc_T1w T1w template anat NIfTI +space-template_desc-head_T1w T1w template anat NIfTI +space-template_desc-T1w_mask mask template anat NIfTI +space-template_desc-Mean_timeseries timeseries func 1D +desc-MeanSCA_timeseries timeseries func 1D +desc-SpatReg_timeseries timeseries func 1D +desc-Voxel_timeseries timeseries func 1D +space-longitudinal_label-CSF_probseg tissue probability longitudinal T1w anat NIfTI +space-longitudinal_label-GM_probseg tissue probability longitudinal T1w anat NIfTI +space-longitudinal_label-WM_probseg tissue probability longitudinal T1w anat NIfTI +vmhc vmhc symmetric template func NIfTI +blip-warp xfm func NIfTI +from-bold_to-EPItemplate_mode-image_desc-linear_xfm xfm func NIfTI +from-bold_to-EPItemplate_mode-image_desc-nonlinear_xfm xfm func NIfTI +from-bold_to-EPItemplate_mode-image_xfm xfm func NIfTI +from-bold_to-symtemplate_mode-image_xfm xfm func NIfTI +from-bold_to-T1w_mode-image_desc-linear_xfm xfm func NIfTI +from-bold_to-template_mode-image_xfm xfm func NIfTI +from-EPItemplate_to-bold_mode-image_desc-linear_xfm xfm func NIfTI +from-EPItemplate_to-bold_mode-image_desc-nonlinear_xfm xfm func NIfTI +from-longitudinal_to-symtemplate_mode-image_desc-linear_xfm xfm anat NIfTI +from-longitudinal_to-symtemplate_mode-image_desc-nonlinear_xfm xfm anat NIfTI +from-longitudinal_to-symtemplate_mode-image_xfm xfm anat NIfTI +from-longitudinal_to-template_mode-image_desc-linear_xfm xfm anat NIfTI +from-longitudinal_to-template_mode-image_desc-nonlinear_xfm xfm anat NIfTI +from-longitudinal_to-template_mode-image_xfm xfm anat NIfTI +from-symtemplate_to-bold_mode-image_xfm xfm func NIfTI +from-symtemplate_to-longitudinal_mode-image_desc-linear_xfm xfm anat NIfTI +from-symtemplate_to-longitudinal_mode-image_desc-nonlinear_xfm xfm anat NIfTI +from-symtemplate_to-longitudinal_mode-image_xfm xfm anat NIfTI +from-symtemplate_to-T1w_mode-image_desc-linear_xfm xfm anat NIfTI +from-symtemplate_to-T1w_mode-image_desc-nonlinear_xfm xfm anat NIfTI +from-symtemplate_to-T1w_mode-image_xfm xfm anat NIfTI +from-T1w_to-symtemplate_mode-image_desc-linear_xfm xfm anat NIfTI +from-T1w_to-symtemplate_mode-image_desc-nonlinear_xfm xfm anat NIfTI +from-T1w_to-symtemplate_mode-image_xfm xfm anat NIfTI +from-T1w_to-template_mode-image_desc-linear_xfm xfm anat NIfTI +from-T1w_to-template_mode-image_desc-nonlinear_xfm xfm anat NIfTI +from-T1w_to-template_mode-image_xfm xfm anat NIfTI +from-template_to-bold_mode-image_xfm xfm func NIfTI +from-template_to-longitudinal_mode-image_desc-linear_xfm xfm anat NIfTI +from-template_to-longitudinal_mode-image_desc-nonlinear_xfm xfm anat NIfTI +from-template_to-longitudinal_mode-image_xfm xfm anat NIfTI +from-template_to-T1w_mode-image_desc-linear_xfm xfm anat NIfTI +from-template_to-T1w_mode-image_desc-nonlinear_xfm xfm anat NIfTI +from-template_to-T1w_mode-image_xfm xfm anat NIfTI +space-template_label-CSF_mask mask template anat NIfTI +space-template_label-WM_mask mask template anat NIfTI +space-template_label-GM_mask mask template anat NIfTI +space-EPItemplate_label-CSF_mask mask template func NIfTI +space-EPItemplate_label-WM_mask mask template func NIfTI +space-EPItemplate_label-GM_mask mask template func NIfTI +mdmr group functional group_analysis NIfTI +desc-zstd-mdmr group functional group_analysis NIfTI Yes dseg anat diff --git a/CPAC/resources/global/scripts/log.shlib b/CPAC/resources/global/scripts/log.shlib index 274af3eb2b..869890a7a1 100644 --- a/CPAC/resources/global/scripts/log.shlib +++ b/CPAC/resources/global/scripts/log.shlib @@ -137,13 +137,13 @@ log_Msg() fi fi - # always add the message/parameters specified + # always add the message/parameters specified if [ -z "${msg}" ]; then msg+="${parameters}" else msg+=": ${parameters}" fi - + echo "${msg}" } diff --git a/CPAC/resources/templates/BIDS_identifiers.tsv b/CPAC/resources/templates/BIDS_identifiers.tsv index b43c6a1c9f..ed96cb3942 100644 --- a/CPAC/resources/templates/BIDS_identifiers.tsv +++ b/CPAC/resources/templates/BIDS_identifiers.tsv @@ -1,26 +1,26 @@ -/code/CPAC/resources/templates/tpl-MNI152NLin2009cAsym_(res-){0,1}[0-9]+(\.[0-9]*){0,1}[a-z]*(x[0-9]+(\.[0-9]*){0,1}[a-z]*)*_T1w_reference.nii.gz MNI152NLin2009cAsym -/code/CPAC/resources/templates/tpl-MNI152NLin2009cAsym_(res-){0,1}[0-9]+(\.[0-9]*){0,1}[a-z]*(x[0-9]+(\.[0-9]*){0,1}[a-z]*)*_desc-brain_T1w.nii.gz MNI152NLin2009cAsym -/code/CPAC/resources/templates/tpl-MNI152NLin2009cAsym_(res-){0,1}[0-9]+(\.[0-9]*){0,1}[a-z]*(x[0-9]+(\.[0-9]*){0,1}[a-z]*)*_desc-brain_mask.nii.gz MNI152NLin2009cAsym -/code/CPAC/resources/templates/tpl-MNI152NLin2009cAsym_(res-){0,1}[0-9]+(\.[0-9]*){0,1}[a-z]*(x[0-9]+(\.[0-9]*){0,1}[a-z]*)*_desc-fMRIPrep_boldref.nii.gz MNI152NLin2009cAsym -/code/CPAC/resources/templates/tpl-MNI152NLin2009cAsym_(res-){0,1}[0-9]+(\.[0-9]*){0,1}[a-z]*(x[0-9]+(\.[0-9]*){0,1}[a-z]*)*_label-brain_probseg.nii.gz MNI152NLin2009cAsym -/code/CPAC/resources/templates/mni_icbm152_t1_tal_nlin_asym_09c.nii MNI152NLin2009cAsym -$FSLDIR/data/standard/MNI152_T1_(res-){0,1}[0-9]+(\.[0-9]*){0,1}[a-z]*(x[0-9]+(\.[0-9]*){0,1}[a-z]*)*.nii.gz MNI152NLin6ASym -$FSLDIR/data/standard/MNI152_T1_(res-){0,1}[0-9]+(\.[0-9]*){0,1}[a-z]*(x[0-9]+(\.[0-9]*){0,1}[a-z]*)*_brain.nii.gz MNI152NLin6ASym -$FSLDIR/data/standard/MNI152_T1_(res-){0,1}[0-9]+(\.[0-9]*){0,1}[a-z]*(x[0-9]+(\.[0-9]*){0,1}[a-z]*)*_brain_mask.nii.gz MNI152NLin6ASym -$FSLDIR/data/standard/MNI152_T1_(res-){0,1}[0-9]+(\.[0-9]*){0,1}[a-z]*(x[0-9]+(\.[0-9]*){0,1}[a-z]*)*_brain_mask_dil.nii.gz MNI152NLin6ASym -$FSLDIR/data/standard/MNI152_T1_(res-){0,1}[0-9]+(\.[0-9]*){0,1}[a-z]*(x[0-9]+(\.[0-9]*){0,1}[a-z]*)*_symmetric.nii.gz MNI152NLin6Sym -$FSLDIR/data/standard/MNI152_T1_(res-){0,1}[0-9]+(\.[0-9]*){0,1}[a-z]*(x[0-9]+(\.[0-9]*){0,1}[a-z]*)*_brain_symmetric.nii.gz MNI152NLin6Sym -$FSLDIR/data/standard/MNI152_T1_(res-){0,1}[0-9]+(\.[0-9]*){0,1}[a-z]*(x[0-9]+(\.[0-9]*){0,1}[a-z]*)*_brain_mask_symmetric.nii.gz MNI152NLin6Sym -$FSLDIR/data/standard/MNI152_T1_(res-){0,1}[0-9]+(\.[0-9]*){0,1}[a-z]*(x[0-9]+(\.[0-9]*){0,1}[a-z]*)*_brain_mask_symmetric_dil.nii.gz MNI152NLin6Sym -/ndmg_atlases/label/Human/AAL_space-MNI152NLin6_res-2x2x2.nii.gz AAL -/ndmg_atlases/label/Human/Brodmann_space-MNI152NLin6_res-2x2x2.nii.gz Brodmann +/code/CPAC/resources/templates/tpl-MNI152NLin2009cAsym_(res-){0,1}[0-9]+(\.[0-9]*){0,1}[a-z]*(x[0-9]+(\.[0-9]*){0,1}[a-z]*)*_T1w_reference.nii.gz MNI152NLin2009cAsym +/code/CPAC/resources/templates/tpl-MNI152NLin2009cAsym_(res-){0,1}[0-9]+(\.[0-9]*){0,1}[a-z]*(x[0-9]+(\.[0-9]*){0,1}[a-z]*)*_desc-brain_T1w.nii.gz MNI152NLin2009cAsym +/code/CPAC/resources/templates/tpl-MNI152NLin2009cAsym_(res-){0,1}[0-9]+(\.[0-9]*){0,1}[a-z]*(x[0-9]+(\.[0-9]*){0,1}[a-z]*)*_desc-brain_mask.nii.gz MNI152NLin2009cAsym +/code/CPAC/resources/templates/tpl-MNI152NLin2009cAsym_(res-){0,1}[0-9]+(\.[0-9]*){0,1}[a-z]*(x[0-9]+(\.[0-9]*){0,1}[a-z]*)*_desc-fMRIPrep_boldref.nii.gz MNI152NLin2009cAsym +/code/CPAC/resources/templates/tpl-MNI152NLin2009cAsym_(res-){0,1}[0-9]+(\.[0-9]*){0,1}[a-z]*(x[0-9]+(\.[0-9]*){0,1}[a-z]*)*_label-brain_probseg.nii.gz MNI152NLin2009cAsym +/code/CPAC/resources/templates/mni_icbm152_t1_tal_nlin_asym_09c.nii MNI152NLin2009cAsym +$FSLDIR/data/standard/MNI152_T1_(res-){0,1}[0-9]+(\.[0-9]*){0,1}[a-z]*(x[0-9]+(\.[0-9]*){0,1}[a-z]*)*.nii.gz MNI152NLin6ASym +$FSLDIR/data/standard/MNI152_T1_(res-){0,1}[0-9]+(\.[0-9]*){0,1}[a-z]*(x[0-9]+(\.[0-9]*){0,1}[a-z]*)*_brain.nii.gz MNI152NLin6ASym +$FSLDIR/data/standard/MNI152_T1_(res-){0,1}[0-9]+(\.[0-9]*){0,1}[a-z]*(x[0-9]+(\.[0-9]*){0,1}[a-z]*)*_brain_mask.nii.gz MNI152NLin6ASym +$FSLDIR/data/standard/MNI152_T1_(res-){0,1}[0-9]+(\.[0-9]*){0,1}[a-z]*(x[0-9]+(\.[0-9]*){0,1}[a-z]*)*_brain_mask_dil.nii.gz MNI152NLin6ASym +$FSLDIR/data/standard/MNI152_T1_(res-){0,1}[0-9]+(\.[0-9]*){0,1}[a-z]*(x[0-9]+(\.[0-9]*){0,1}[a-z]*)*_symmetric.nii.gz MNI152NLin6Sym +$FSLDIR/data/standard/MNI152_T1_(res-){0,1}[0-9]+(\.[0-9]*){0,1}[a-z]*(x[0-9]+(\.[0-9]*){0,1}[a-z]*)*_brain_symmetric.nii.gz MNI152NLin6Sym +$FSLDIR/data/standard/MNI152_T1_(res-){0,1}[0-9]+(\.[0-9]*){0,1}[a-z]*(x[0-9]+(\.[0-9]*){0,1}[a-z]*)*_brain_mask_symmetric.nii.gz MNI152NLin6Sym +$FSLDIR/data/standard/MNI152_T1_(res-){0,1}[0-9]+(\.[0-9]*){0,1}[a-z]*(x[0-9]+(\.[0-9]*){0,1}[a-z]*)*_brain_mask_symmetric_dil.nii.gz MNI152NLin6Sym +/ndmg_atlases/label/Human/AAL_space-MNI152NLin6_res-2x2x2.nii.gz AAL +/ndmg_atlases/label/Human/Brodmann_space-MNI152NLin6_res-2x2x2.nii.gz Brodmann /cpac_templates/CC200.nii.gz CC 200 /cpac_templates/CC400.nii.gz CC 400 -/ndmg_atlases/label/Human/Glasser_space-MNI152NLin6_res-2x2x2.nii.gz Glasser -/ndmg_atlases/label/Human/Slab907_space-MNI152NLin6_res-2x2x2.nii.gz Slab +/ndmg_atlases/label/Human/Glasser_space-MNI152NLin6_res-2x2x2.nii.gz Glasser +/ndmg_atlases/label/Human/Slab907_space-MNI152NLin6_res-2x2x2.nii.gz Slab /ndmg_atlases/label/Human/HarvardOxfordcort-maxprob-thr25_space-MNI152NLin6_res-2x2x2.nii.gz HOCPA th25 /ndmg_atlases/label/Human/HarvardOxfordsub-maxprob-thr25_space-MNI152NLin6_res-2x2x2.nii.gz HOSPA th25 -/ndmg_atlases/label/Human/Juelich_space-MNI152NLin6_res-2x2x2.nii.gz Juelich +/ndmg_atlases/label/Human/Juelich_space-MNI152NLin6_res-2x2x2.nii.gz Juelich /ndmg_atlases/label/Human/Schaefer[^_-]*200.*.nii(\.gz){0,1} Schaefer2018 p200n17 /ndmg_atlases/label/Human/Schaefer[^_-]*300.*.nii(\.gz){0,1} Schaefer2018 p300n17 /ndmg_atlases/label/Human/Schaefer[^_-]*400.*.nii(\.gz){0,1} Schaefer2018 p400n17 diff --git a/CPAC/resources/templates/__init__.py b/CPAC/resources/templates/__init__.py index 35eb3ce592..57e1ac929e 100644 --- a/CPAC/resources/templates/__init__.py +++ b/CPAC/resources/templates/__init__.py @@ -1,4 +1,4 @@ -'''Template resources for C-PAC''' +"""Template resources for C-PAC.""" from .lookup_table import format_identifier, lookup_identifier -__all__ = ['format_identifier', 'lookup_identifier'] +__all__ = ["format_identifier", "lookup_identifier"] diff --git a/CPAC/resources/templates/lookup_table.py b/CPAC/resources/templates/lookup_table.py index ce81e7fd29..88f82191fc 100644 --- a/CPAC/resources/templates/lookup_table.py +++ b/CPAC/resources/templates/lookup_table.py @@ -16,22 +16,32 @@ # License along with C-PAC. If not, see . """Utilities for determining BIDS standard template identifiers (https://bids-specification.readthedocs.io/en/stable/99-appendices/08-coordinate-systems.html#standard-template-identifiers) -from in-container template paths""" +from in-container template paths. +""" from os import environ, path as op from re import findall, search from typing import Optional + from bids.layout import parse_file_entities from numpy import loadtxt + from CPAC.utils.typing import TUPLE -LOOKUP_TABLE = {row[0].replace(r'$FSLDIR', environ['FSLDIR']): ( - row[1], str(row[2]) if row[2] else None) for row in - loadtxt(op.join(op.dirname(__file__), 'BIDS_identifiers.tsv'), - dtype='str', delimiter='\t')} +LOOKUP_TABLE = { + row[0].replace(r"$FSLDIR", environ["FSLDIR"]): ( + row[1], + str(row[2]) if row[2] else None, + ) + for row in loadtxt( + op.join(op.dirname(__file__), "BIDS_identifiers.tsv"), + dtype="str", + delimiter="\t", + ) +} def format_identifier(identifier: str, desc: Optional[str] = None) -> str: - '''Function to create an identifier string from a name and description + """Function to create an identifier string from a name and description. Parameters ---------- @@ -49,16 +59,16 @@ def format_identifier(identifier: str, desc: Optional[str] = None) -> str: 'CC_desc-200' >>> format_identifier('AAL') 'AAL' - ''' + """ if desc: - return f'{identifier}_desc-{desc}' + return f"{identifier}_desc-{desc}" return identifier def lookup_identifier(template_path: str) -> TUPLE[str, None]: - '''Function to return a standard template identifier for a packaged + """Function to return a standard template identifier for a packaged template, if known. Otherwise, returns the literal string - 'template' + 'template'. Parameters ---------- @@ -87,20 +97,19 @@ def lookup_identifier(template_path: str) -> TUPLE[str, None]: ... 'tpl-MNI152NLin2009cAsym_res-02_atlas-Schaefer2018_' ... 'desc-200Parcels17Networks_dseg.nii.gz') ('Schaefer2018', '200Parcels17Networks') - ''' - if r'$' in template_path: - bash_var_pattern = r'(\$[\w]+(?=/|\s)|\${\w+})' + """ + if r"$" in template_path: + bash_var_pattern = r"(\$[\w]+(?=/|\s)|\${\w+})" matches = findall(bash_var_pattern, template_path) if matches is not None: for match in matches: - bash_var = match.lstrip('${').rstrip('}') + bash_var = match.lstrip("${").rstrip("}") if bash_var in environ: - template_path = template_path.replace(match, - environ[bash_var]) + template_path = template_path.replace(match, environ[bash_var]) for key, value in LOOKUP_TABLE.items(): if search(key, template_path) is not None: return value _bidsy = parse_file_entities(template_path) - if 'atlas' in _bidsy: - return _bidsy['atlas'], _bidsy.get('desc') - return 'template', None + if "atlas" in _bidsy: + return _bidsy["atlas"], _bidsy.get("desc") + return "template", None diff --git a/CPAC/resources/templates/ndmg_atlases.csv b/CPAC/resources/templates/ndmg_atlases.csv index 15ac6b0ba6..87ba249cad 100644 --- a/CPAC/resources/templates/ndmg_atlases.csv +++ b/CPAC/resources/templates/ndmg_atlases.csv @@ -19,4 +19,4 @@ "yeo-7_space-MNI152NLin6_res-1x1x1.nii.gz","Yeo-7_space-MNI152NLin6_res-1x1x1.nii.gz" "yeo-7-liberal_space-MNI152NLin6_res-1x1x1.nii.gz","Yeo-7-liberal_space-MNI152NLin6_res-1x1x1.nii.gz" "yeo-17_space-MNI152NLin6_res-1x1x1.nii.gz","Yeo-17_space-MNI152NLin6_res-1x1x1.nii.gz" -"yeo-17-liberal_space-MNI152NLin6_res-1x1x1.nii.gz","Yeo-17-liberal_space-MNI152NLin6_res-1x1x1.nii.gz" \ No newline at end of file +"yeo-17-liberal_space-MNI152NLin6_res-1x1x1.nii.gz","Yeo-17-liberal_space-MNI152NLin6_res-1x1x1.nii.gz" diff --git a/CPAC/resources/tests/test_permissions.py b/CPAC/resources/tests/test_permissions.py index fe881e0c3e..390b00e6d0 100644 --- a/CPAC/resources/tests/test_permissions.py +++ b/CPAC/resources/tests/test_permissions.py @@ -14,16 +14,17 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -"""Tests for appropriate permissions on resources""" +"""Tests for appropriate permissions on resources.""" from os import environ from pathlib import Path import stat + import pytest -@pytest.mark.parametrize('template', - list(Path(f'{environ.get("FSLDIR")}/data/standard' - ).glob('*.nii.gz'))) +@pytest.mark.parametrize( + "template", list(Path(f'{environ.get("FSLDIR")}/data/standard').glob("*.nii.gz")) +) def test_read_fsl_templates(template): - """For each FSL template, make sure its permissions include 444""" - assert stat.filemode(template.stat().st_mode).count('r') == 3 + """For each FSL template, make sure its permissions include 444.""" + assert stat.filemode(template.stat().st_mode).count("r") == 3 diff --git a/CPAC/resources/tests/test_templates.py b/CPAC/resources/tests/test_templates.py index 40a598cfab..8b8d316d1d 100644 --- a/CPAC/resources/tests/test_templates.py +++ b/CPAC/resources/tests/test_templates.py @@ -14,29 +14,37 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -"""Tests for packaged templates""" +"""Tests for packaged templates.""" import os + import pytest + from CPAC.pipeline import ALL_PIPELINE_CONFIGS -from CPAC.pipeline.engine import ingress_pipeconfig_paths, ResourcePool +from CPAC.pipeline.engine import ResourcePool, ingress_pipeconfig_paths from CPAC.utils.configuration import Preconfiguration from CPAC.utils.datasource import get_highest_local_res -@pytest.mark.parametrize('pipeline', ALL_PIPELINE_CONFIGS) +@pytest.mark.parametrize("pipeline", ALL_PIPELINE_CONFIGS) def test_packaged_path_exists(pipeline): """ Check that all local templates are included in image at at - least one resolution + least one resolution. """ - rpool = ingress_pipeconfig_paths(Preconfiguration(pipeline), - ResourcePool(), 'pytest') + rpool = ingress_pipeconfig_paths( + Preconfiguration(pipeline), ResourcePool(), "pytest" + ) for resource in rpool.rpool.values(): - node = list(resource.values())[0].get('data')[0] - if hasattr(node.inputs, 'template' - ) and not node.inputs.template.startswith('s3:'): - if not pipeline == 'rodent' and node.inputs.template.startswith( - '/template/study_based'): - assert (os.path.exists(node.inputs.template) or - get_highest_local_res(node.inputs.template, - node.inputs.resolution).exists()) + node = next(iter(resource.values())).get("data")[0] + if hasattr(node.inputs, "template") and not node.inputs.template.startswith( + "s3:" + ): + if not pipeline == "rodent" and node.inputs.template.startswith( + "/template/study_based" + ): + assert ( + os.path.exists(node.inputs.template) + or get_highest_local_res( + node.inputs.template, node.inputs.resolution + ).exists() + ) diff --git a/CPAC/sca/__init__.py b/CPAC/sca/__init__.py index 1e61826749..73cc43c1fe 100644 --- a/CPAC/sca/__init__.py +++ b/CPAC/sca/__init__.py @@ -1,12 +1,11 @@ -from .sca import create_sca -from .sca import create_temporal_reg - -from .utils import compute_fisher_z_score -from .utils import check_ts, map_to_roi +from .sca import create_sca, create_temporal_reg +from .utils import check_ts, compute_fisher_z_score, map_to_roi # List all functions -__all__ = ['create_sca', \ - 'compute_fisher_z_score', \ - 'create_temporal_reg', \ - 'check_ts', \ - 'map_to_roi'] +__all__ = [ + "create_sca", + "compute_fisher_z_score", + "create_temporal_reg", + "check_ts", + "map_to_roi", +] diff --git a/CPAC/sca/sca.py b/CPAC/sca/sca.py index 26755f1d89..01e35b17c3 100644 --- a/CPAC/sca/sca.py +++ b/CPAC/sca/sca.py @@ -14,21 +14,27 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -from CPAC.pipeline.nodeblock import nodeblock -from nipype.interfaces.afni import preprocess -from CPAC.pipeline import nipype_pipeline_engine as pe from nipype.interfaces import fsl, utility as util +from nipype.interfaces.afni import preprocess +from CPAC.pipeline import nipype_pipeline_engine as pe +from CPAC.pipeline.nodeblock import nodeblock from CPAC.sca.utils import * -# from CPAC.utils.utils import extract_one_d -from CPAC.utils.datasource import resample_func_roi, \ - create_roi_mask_dataflow, create_spatial_map_dataflow +from CPAC.timeseries.timeseries_analysis import ( + get_roi_timeseries, + get_spatial_map_timeseries, + resample_function, +) -from CPAC.timeseries.timeseries_analysis import get_roi_timeseries, \ - get_spatial_map_timeseries, resample_function +# from CPAC.utils.utils import extract_one_d +from CPAC.utils.datasource import ( + create_roi_mask_dataflow, + create_spatial_map_dataflow, + resample_func_roi, +) -def create_sca(name_sca='sca'): +def create_sca(name_sca="sca"): """ Map of the correlations of the Region of Interest(Seed in native or MNI space) with the rest of brain voxels. The map is normalized to contain Z-scores, mapped in standard space and treated with spatial smoothing. @@ -93,89 +99,91 @@ def create_sca(name_sca='sca'): Examples -------- - >>> sca_w = create_sca("sca_wf") >>> sca_w.inputs.inputspec.functional_file = '/home/data/subject/func/rest_bandpassed.nii.gz' # doctest: +SKIP >>> sca_w.inputs.inputspec.timeseries_one_d = '/home/data/subject/func/ts.1D' # doctest: +SKIP >>> sca_w.run() # doctest: +SKIP """ - - from CPAC.utils.utils import get_roi_num_list - sca = pe.Workflow(name=name_sca) - inputNode = pe.Node(util.IdentityInterface(fields=['timeseries_one_d', - 'functional_file',]), - name='inputspec') + inputNode = pe.Node( + util.IdentityInterface( + fields=[ + "timeseries_one_d", + "functional_file", + ] + ), + name="inputspec", + ) - outputNode = pe.Node(util.IdentityInterface(fields=[ - 'correlation_stack', - 'correlation_files', - 'Z_score', - ]), - name='outputspec') + outputNode = pe.Node( + util.IdentityInterface( + fields=[ + "correlation_stack", + "correlation_files", + "Z_score", + ] + ), + name="outputspec", + ) # 2. Compute voxel-wise correlation with Seed Timeseries - corr = pe.Node(interface=preprocess.TCorr1D(), - name='3dTCorr1D', mem_gb=3.0) + corr = pe.Node(interface=preprocess.TCorr1D(), name="3dTCorr1D", mem_gb=3.0) corr.inputs.pearson = True - corr.inputs.outputtype = 'NIFTI_GZ' + corr.inputs.outputtype = "NIFTI_GZ" - sca.connect(inputNode, 'timeseries_one_d', - corr, 'y_1d') - sca.connect(inputNode, 'functional_file', - corr, 'xset') + sca.connect(inputNode, "timeseries_one_d", corr, "y_1d") + sca.connect(inputNode, "functional_file", corr, "xset") # Transform the sub-bricks into volumes try: - concat = pe.Node(interface=preprocess.TCat(), name='3dTCat') + concat = pe.Node(interface=preprocess.TCat(), name="3dTCat") except AttributeError: from nipype.interfaces.afni import utils as afni_utils - concat = pe.Node(interface=afni_utils.TCat(), name='3dTCat') - concat.inputs.outputtype = 'NIFTI_GZ' + concat = pe.Node(interface=afni_utils.TCat(), name="3dTCat") + + concat.inputs.outputtype = "NIFTI_GZ" # also write out volumes as individual files - #split = pe.Node(interface=fsl.Split(), name='split_raw_volumes_sca') - #split.inputs.dimension = 't' - #split.inputs.out_base_name = 'sca_' + # split = pe.Node(interface=fsl.Split(), name='split_raw_volumes_sca') + # split.inputs.dimension = 't' + # split.inputs.out_base_name = 'sca_' - #get_roi_num_list = pe.Node(util.Function(input_names=['timeseries_file', + # get_roi_num_list = pe.Node(util.Function(input_names=['timeseries_file', # 'prefix'], # output_names=['roi_list'], # function=get_roi_num_list), # name='get_roi_num_list') - #get_roi_num_list.inputs.prefix = "sca" + # get_roi_num_list.inputs.prefix = "sca" - #sca.connect(inputNode, 'timeseries_one_d', get_roi_num_list, + # sca.connect(inputNode, 'timeseries_one_d', get_roi_num_list, # 'timeseries_file') - #rename_rois = pe.MapNode(interface=util.Rename(), name='output_rois', + # rename_rois = pe.MapNode(interface=util.Rename(), name='output_rois', # iterfield=['in_file', 'format_string']) - #rename_rois.inputs.keep_ext = True + # rename_rois.inputs.keep_ext = True - #sca.connect(split, 'out_files', rename_rois, 'in_file') - #sca.connect(get_roi_num_list, 'roi_list', rename_rois, 'format_string') + # sca.connect(split, 'out_files', rename_rois, 'in_file') + # sca.connect(get_roi_num_list, 'roi_list', rename_rois, 'format_string') - sca.connect(corr, 'out_file', concat, 'in_files') - #sca.connect(concat, 'out_file', split, 'in_file') - sca.connect(concat, 'out_file', - outputNode, 'correlation_stack') - #sca.connect(rename_rois, 'out_file', outputNode, + sca.connect(corr, "out_file", concat, "in_files") + # sca.connect(concat, 'out_file', split, 'in_file') + sca.connect(concat, "out_file", outputNode, "correlation_stack") + # sca.connect(rename_rois, 'out_file', outputNode, # 'correlation_files') return sca -def create_temporal_reg(wflow_name='temporal_reg', which='SR'): - """ +def create_temporal_reg(wflow_name="temporal_reg", which="SR"): + r""" Temporal multiple regression workflow Provides a spatial map of parameter estimates corresponding to each - provided timeseries in a timeseries.txt file as regressors + provided timeseries in a timeseries.txt file as regressors. Parameters ---------- - wflow_name : a string Name of the temporal regression workflow @@ -191,7 +199,6 @@ def create_temporal_reg(wflow_name='temporal_reg', which='SR'): Returns ------- - wflow : workflow temporal multiple regression Workflow @@ -200,7 +207,6 @@ def create_temporal_reg(wflow_name='temporal_reg', which='SR'): Notes ----- - `Source `_ Workflow Inputs:: @@ -264,7 +270,6 @@ def create_temporal_reg(wflow_name='temporal_reg', which='SR'): Examples -------- - >>> tr_wf = create_temporal_reg('temporal-regression') >>> tr_wf.inputs.inputspec.subject_rest = '/home/data/subject/func/rest_bandpassed.nii.gz' # doctest: +SKIP >>> tr_wf.inputs.inputspec.subject_timeseries = '/home/data/subject/func/timeseries.txt' # doctest: +SKIP @@ -274,47 +279,56 @@ def create_temporal_reg(wflow_name='temporal_reg', which='SR'): >>> tr_wf.run() # doctest: +SKIP """ - wflow = pe.Workflow(name=wflow_name) - inputNode = pe.Node(util.IdentityInterface - (fields=['subject_rest', - 'subject_timeseries', - 'subject_mask', - 'demean', - 'normalize']), - name='inputspec') - - outputNode = pe.Node(util.IdentityInterface - (fields=['temp_reg_map', - 'temp_reg_map_files', - 'temp_reg_map_z', - 'temp_reg_map_z_files']), - name='outputspec') - - check_timeseries = pe.Node(util.Function(input_names=['in_file'], - output_names=['out_file'], - function=check_ts), - name='check_timeseries') - - wflow.connect(inputNode, 'subject_timeseries', - check_timeseries, 'in_file') - - temporalReg = pe.Node(interface=fsl.GLM(), name='temporal_regression', - mem_gb=4.0) - temporalReg.inputs.out_file = 'temp_reg_map.nii.gz' - temporalReg.inputs.out_z_name = 'temp_reg_map_z.nii.gz' - - wflow.connect(inputNode, 'subject_rest', temporalReg, 'in_file') - wflow.connect(check_timeseries, 'out_file', temporalReg, 'design') - wflow.connect(inputNode, 'demean', temporalReg, 'demean') - wflow.connect(inputNode, 'normalize', temporalReg, 'des_norm') - wflow.connect(inputNode, 'subject_mask', temporalReg, 'mask') - - wflow.connect(temporalReg, 'out_file', outputNode, 'temp_reg_map') - wflow.connect(temporalReg, 'out_z', outputNode, 'temp_reg_map_z') - - ''' + inputNode = pe.Node( + util.IdentityInterface( + fields=[ + "subject_rest", + "subject_timeseries", + "subject_mask", + "demean", + "normalize", + ] + ), + name="inputspec", + ) + + outputNode = pe.Node( + util.IdentityInterface( + fields=[ + "temp_reg_map", + "temp_reg_map_files", + "temp_reg_map_z", + "temp_reg_map_z_files", + ] + ), + name="outputspec", + ) + + check_timeseries = pe.Node( + util.Function( + input_names=["in_file"], output_names=["out_file"], function=check_ts + ), + name="check_timeseries", + ) + + wflow.connect(inputNode, "subject_timeseries", check_timeseries, "in_file") + + temporalReg = pe.Node(interface=fsl.GLM(), name="temporal_regression", mem_gb=4.0) + temporalReg.inputs.out_file = "temp_reg_map.nii.gz" + temporalReg.inputs.out_z_name = "temp_reg_map_z.nii.gz" + + wflow.connect(inputNode, "subject_rest", temporalReg, "in_file") + wflow.connect(check_timeseries, "out_file", temporalReg, "design") + wflow.connect(inputNode, "demean", temporalReg, "demean") + wflow.connect(inputNode, "normalize", temporalReg, "des_norm") + wflow.connect(inputNode, "subject_mask", temporalReg, "mask") + + wflow.connect(temporalReg, "out_file", outputNode, "temp_reg_map") + wflow.connect(temporalReg, "out_z", outputNode, "temp_reg_map_z") + + """ split = pe.Node(interface=fsl.Split(), name='split_raw_volumes') split.inputs.dimension = 't' split.inputs.out_base_name = 'temp_reg_map_' @@ -388,7 +402,7 @@ def create_temporal_reg(wflow_name='temporal_reg', which='SR'): wflow.connect(rename_maps_zstat, 'out_file', outputNode, 'temp_reg_map_z_files') - ''' + """ return wflow @@ -405,69 +419,84 @@ def create_temporal_reg(wflow_name='temporal_reg', which='SR'): ], ) def SCA_AVG(wf, cfg, strat_pool, pipe_num, opt=None): - '''Run Seed-Based Correlation Analysis.''' - + """Run Seed-Based Correlation Analysis.""" # same workflow, except to run TSE and send it to the resource # pool so that it will not get sent to SCA resample_functional_roi_for_sca = pe.Node( - util.Function(input_names=['in_func', - 'in_roi', - 'realignment', - 'identity_matrix'], - output_names=['out_func', 'out_roi'], - function=resample_func_roi, - as_module=True), - name=f'resample_functional_roi_for_sca_{pipe_num}') - - resample_functional_roi_for_sca.inputs.realignment = \ - cfg.timeseries_extraction['realignment'] - resample_functional_roi_for_sca.inputs.identity_matrix = \ - cfg.registration_workflows['functional_registration'][ - 'func_registration_to_template']['FNIRT_pipelines']['identity_matrix'] + util.Function( + input_names=["in_func", "in_roi", "realignment", "identity_matrix"], + output_names=["out_func", "out_roi"], + function=resample_func_roi, + as_module=True, + ), + name=f"resample_functional_roi_for_sca_{pipe_num}", + ) + + resample_functional_roi_for_sca.inputs.realignment = cfg.timeseries_extraction[ + "realignment" + ] + resample_functional_roi_for_sca.inputs.identity_matrix = cfg.registration_workflows[ + "functional_registration" + ]["func_registration_to_template"]["FNIRT_pipelines"]["identity_matrix"] roi_dataflow_for_sca = create_roi_mask_dataflow( - cfg.seed_based_correlation_analysis['sca_atlases']['Avg'], - f'roi_dataflow_for_sca_{pipe_num}' + cfg.seed_based_correlation_analysis["sca_atlases"]["Avg"], + f"roi_dataflow_for_sca_{pipe_num}", ) roi_dataflow_for_sca.inputs.inputspec.set( - creds_path=cfg.pipeline_setup['input_creds_path'], - dl_dir=cfg.pipeline_setup['working_directory']['path'] + creds_path=cfg.pipeline_setup["input_creds_path"], + dl_dir=cfg.pipeline_setup["working_directory"]["path"], ) - roi_timeseries_for_sca = get_roi_timeseries( - f'roi_timeseries_for_sca_{pipe_num}') + roi_timeseries_for_sca = get_roi_timeseries(f"roi_timeseries_for_sca_{pipe_num}") node, out = strat_pool.get_data("space-template_desc-preproc_bold") # resample the input functional file to roi - wf.connect(node, out, - resample_functional_roi_for_sca, 'in_func') - wf.connect(roi_dataflow_for_sca, 'outputspec.out_file', - resample_functional_roi_for_sca, 'in_roi') + wf.connect(node, out, resample_functional_roi_for_sca, "in_func") + wf.connect( + roi_dataflow_for_sca, + "outputspec.out_file", + resample_functional_roi_for_sca, + "in_roi", + ) # connect it to the roi_timeseries - wf.connect(resample_functional_roi_for_sca, 'out_roi', - roi_timeseries_for_sca, 'input_roi.roi') - wf.connect(resample_functional_roi_for_sca, 'out_func', - roi_timeseries_for_sca, 'inputspec.rest') + wf.connect( + resample_functional_roi_for_sca, + "out_roi", + roi_timeseries_for_sca, + "input_roi.roi", + ) + wf.connect( + resample_functional_roi_for_sca, + "out_func", + roi_timeseries_for_sca, + "inputspec.rest", + ) - sca_roi = create_sca(f'sca_roi_{pipe_num}') + sca_roi = create_sca(f"sca_roi_{pipe_num}") node, out = strat_pool.get_data("space-template_desc-preproc_bold") - wf.connect(node, out, sca_roi, 'inputspec.functional_file') - - wf.connect(roi_timeseries_for_sca, 'outputspec.roi_csv', - #('outputspec.roi_outputs', extract_one_d), - sca_roi, 'inputspec.timeseries_one_d') + wf.connect(node, out, sca_roi, "inputspec.functional_file") + + wf.connect( + roi_timeseries_for_sca, + "outputspec.roi_csv", + # ('outputspec.roi_outputs', extract_one_d), + sca_roi, + "inputspec.timeseries_one_d", + ) outputs = { - 'desc-MeanSCA_timeseries': - (roi_timeseries_for_sca, 'outputspec.roi_csv'), - #('outputspec.roi_outputs', - # extract_one_d)), - 'space-template_desc-MeanSCA_correlations': - (sca_roi, 'outputspec.correlation_stack'), - 'atlas_name': (roi_dataflow_for_sca, 'outputspec.out_name') + "desc-MeanSCA_timeseries": (roi_timeseries_for_sca, "outputspec.roi_csv"), + # ('outputspec.roi_outputs', + # extract_one_d)), + "space-template_desc-MeanSCA_correlations": ( + sca_roi, + "outputspec.correlation_stack", + ), + "atlas_name": (roi_dataflow_for_sca, "outputspec.out_name"), } return (wf, outputs) @@ -477,8 +506,7 @@ def SCA_AVG(wf, cfg, strat_pool, pipe_num, opt=None): name="dual_regression", config=["seed_based_correlation_analysis"], switch=["run"], - inputs=["space-template_desc-preproc_bold", - "space-template_desc-bold_mask"], + inputs=["space-template_desc-preproc_bold", "space-template_desc-bold_mask"], outputs=[ "space-template_desc-DualReg_correlations", "desc-DualReg_statmap", @@ -486,74 +514,81 @@ def SCA_AVG(wf, cfg, strat_pool, pipe_num, opt=None): ], ) def dual_regression(wf, cfg, strat_pool, pipe_num, opt=None): - ''' - Run Dual Regression - spatial regression and then temporal regression. - ''' + """Run Dual Regression - spatial regression and then temporal regression.""" resample_spatial_map_to_native_space_for_dr = pe.Node( interface=fsl.FLIRT(), - name=f'resample_spatial_map_to_native_space_for_DR_{pipe_num}' + name=f"resample_spatial_map_to_native_space_for_DR_{pipe_num}", ) resample_spatial_map_to_native_space_for_dr.inputs.set( - interp='nearestneighbour', + interp="nearestneighbour", apply_xfm=True, - in_matrix_file= - cfg.registration_workflows['functional_registration'][ - 'func_registration_to_template']['FNIRT_pipelines'][ - 'identity_matrix'] + in_matrix_file=cfg.registration_workflows["functional_registration"][ + "func_registration_to_template" + ]["FNIRT_pipelines"]["identity_matrix"], ) spatial_map_dataflow_for_dr = create_spatial_map_dataflow( - cfg.seed_based_correlation_analysis['sca_atlases']['DualReg'], - f'spatial_map_dataflow_for_DR_{pipe_num}' + cfg.seed_based_correlation_analysis["sca_atlases"]["DualReg"], + f"spatial_map_dataflow_for_DR_{pipe_num}", ) spatial_map_dataflow_for_dr.inputs.inputspec.set( - creds_path=cfg.pipeline_setup['input_creds_path'], - dl_dir=cfg.pipeline_setup['working_directory']['path'] + creds_path=cfg.pipeline_setup["input_creds_path"], + dl_dir=cfg.pipeline_setup["working_directory"]["path"], ) spatial_map_timeseries_for_dr = get_spatial_map_timeseries( - f'spatial_map_timeseries_for_DR_{pipe_num}' + f"spatial_map_timeseries_for_DR_{pipe_num}" ) spatial_map_timeseries_for_dr.inputs.inputspec.demean = True # resample the input functional file and functional mask # to spatial map node, out = strat_pool.get_data("space-template_desc-preproc_bold") - wf.connect(node, out, - resample_spatial_map_to_native_space_for_dr, 'reference') - wf.connect(node, out, - spatial_map_timeseries_for_dr, 'inputspec.subject_rest') - - wf.connect(spatial_map_dataflow_for_dr, 'select_spatial_map.out_file', - resample_spatial_map_to_native_space_for_dr, 'in_file') + wf.connect(node, out, resample_spatial_map_to_native_space_for_dr, "reference") + wf.connect(node, out, spatial_map_timeseries_for_dr, "inputspec.subject_rest") + + wf.connect( + spatial_map_dataflow_for_dr, + "select_spatial_map.out_file", + resample_spatial_map_to_native_space_for_dr, + "in_file", + ) # connect it to the spatial_map_timeseries - wf.connect(resample_spatial_map_to_native_space_for_dr, 'out_file', - spatial_map_timeseries_for_dr, 'inputspec.spatial_map' + wf.connect( + resample_spatial_map_to_native_space_for_dr, + "out_file", + spatial_map_timeseries_for_dr, + "inputspec.spatial_map", ) - dr_temp_reg = create_temporal_reg(f'temporal_regression_{pipe_num}') - dr_temp_reg.inputs.inputspec.normalize = \ - cfg.seed_based_correlation_analysis['norm_timeseries_for_DR'] + dr_temp_reg = create_temporal_reg(f"temporal_regression_{pipe_num}") + dr_temp_reg.inputs.inputspec.normalize = cfg.seed_based_correlation_analysis[ + "norm_timeseries_for_DR" + ] dr_temp_reg.inputs.inputspec.demean = True - wf.connect(spatial_map_timeseries_for_dr, 'outputspec.subject_timeseries', - dr_temp_reg, 'inputspec.subject_timeseries') + wf.connect( + spatial_map_timeseries_for_dr, + "outputspec.subject_timeseries", + dr_temp_reg, + "inputspec.subject_timeseries", + ) node, out = strat_pool.get_data("space-template_desc-preproc_bold") - wf.connect(node, out, dr_temp_reg, 'inputspec.subject_rest') + wf.connect(node, out, dr_temp_reg, "inputspec.subject_rest") node, out = strat_pool.get_data("space-template_desc-bold_mask") - wf.connect(node, out, dr_temp_reg, 'inputspec.subject_mask') + wf.connect(node, out, dr_temp_reg, "inputspec.subject_mask") outputs = { - 'space-template_desc-DualReg_correlations': - (dr_temp_reg, 'outputspec.temp_reg_map'), - 'desc-DualReg_statmap': - (dr_temp_reg, 'outputspec.temp_reg_map_z'), - 'atlas_name': - (spatial_map_dataflow_for_dr, 'select_spatial_map.out_name') + "space-template_desc-DualReg_correlations": ( + dr_temp_reg, + "outputspec.temp_reg_map", + ), + "desc-DualReg_statmap": (dr_temp_reg, "outputspec.temp_reg_map_z"), + "atlas_name": (spatial_map_dataflow_for_dr, "select_spatial_map.out_name"), } return (wf, outputs) @@ -563,8 +598,7 @@ def dual_regression(wf, cfg, strat_pool, pipe_num, opt=None): name="multiple_regression", config=["seed_based_correlation_analysis"], switch=["run"], - inputs=["space-template_desc-preproc_bold", - "space-template_desc-bold_mask"], + inputs=["space-template_desc-preproc_bold", "space-template_desc-bold_mask"], outputs=[ "space-template_desc-MultReg_correlations", "desc-MultReg_statmap", @@ -572,77 +606,95 @@ def dual_regression(wf, cfg, strat_pool, pipe_num, opt=None): ], ) def multiple_regression(wf, cfg, strat_pool, pipe_num, opt=None): - '''Run Multiple Regression.''' - + """Run Multiple Regression.""" # same workflow, except to run TSE and send it to the resource # pool so that it will not get sent to SCA resample_functional_roi_for_multreg = pe.Node( - resample_function(), - name=f'resample_functional_roi_for_multreg_{pipe_num}') + resample_function(), name=f"resample_functional_roi_for_multreg_{pipe_num}" + ) - resample_functional_roi_for_multreg.inputs.realignment = \ - cfg.timeseries_extraction['realignment'] - resample_functional_roi_for_multreg.inputs.identity_matrix = \ - cfg.registration_workflows['functional_registration'][ - 'func_registration_to_template']['FNIRT_pipelines']['identity_matrix'] + resample_functional_roi_for_multreg.inputs.realignment = cfg.timeseries_extraction[ + "realignment" + ] + resample_functional_roi_for_multreg.inputs.identity_matrix = ( + cfg.registration_workflows["functional_registration"][ + "func_registration_to_template" + ]["FNIRT_pipelines"]["identity_matrix"] + ) roi_dataflow_for_multreg = create_roi_mask_dataflow( - cfg.seed_based_correlation_analysis['sca_atlases']['MultReg'], - f'roi_dataflow_for_mult_reg_{pipe_num}') + cfg.seed_based_correlation_analysis["sca_atlases"]["MultReg"], + f"roi_dataflow_for_mult_reg_{pipe_num}", + ) roi_dataflow_for_multreg.inputs.inputspec.set( - creds_path=cfg.pipeline_setup['input_creds_path'], - dl_dir=cfg.pipeline_setup['working_directory']['path'] + creds_path=cfg.pipeline_setup["input_creds_path"], + dl_dir=cfg.pipeline_setup["working_directory"]["path"], ) roi_timeseries_for_multreg = get_roi_timeseries( - f'roi_timeseries_for_mult_reg_{pipe_num}') + f"roi_timeseries_for_mult_reg_{pipe_num}" + ) node, out = strat_pool.get_data("space-template_desc-preproc_bold") # resample the input functional file to roi - wf.connect(node, out, resample_functional_roi_for_multreg, 'in_func') - wf.connect(roi_dataflow_for_multreg, - 'outputspec.out_file', - resample_functional_roi_for_multreg, - 'in_roi') + wf.connect(node, out, resample_functional_roi_for_multreg, "in_func") + wf.connect( + roi_dataflow_for_multreg, + "outputspec.out_file", + resample_functional_roi_for_multreg, + "in_roi", + ) # connect it to the roi_timeseries - wf.connect(resample_functional_roi_for_multreg, - 'out_roi', - roi_timeseries_for_multreg, - 'input_roi.roi') - wf.connect(resample_functional_roi_for_multreg, - 'out_func', - roi_timeseries_for_multreg, - 'inputspec.rest') - - sc_temp_reg = create_temporal_reg( - f'temporal_regression_sca_{pipe_num}', - which='RT') - sc_temp_reg.inputs.inputspec.normalize = \ - cfg.seed_based_correlation_analysis['norm_timeseries_for_DR'] - sc_temp_reg.inputs.inputspec.demean = True + wf.connect( + resample_functional_roi_for_multreg, + "out_roi", + roi_timeseries_for_multreg, + "input_roi.roi", + ) + wf.connect( + resample_functional_roi_for_multreg, + "out_func", + roi_timeseries_for_multreg, + "inputspec.rest", + ) - node, out = strat_pool.get_data(["space-template_desc-cleaned_bold", - "space-template_desc-brain_bold", - "space-template_desc-motion_bold", - "space-template_desc-preproc_bold", - "space-template_bold"]) - wf.connect(node, out, sc_temp_reg, 'inputspec.subject_rest') + sc_temp_reg = create_temporal_reg(f"temporal_regression_sca_{pipe_num}", which="RT") + sc_temp_reg.inputs.inputspec.normalize = cfg.seed_based_correlation_analysis[ + "norm_timeseries_for_DR" + ] + sc_temp_reg.inputs.inputspec.demean = True - wf.connect(roi_timeseries_for_multreg, 'outputspec.roi_csv', - #('outputspec.roi_outputs', extract_one_d), - sc_temp_reg, 'inputspec.subject_timeseries') + node, out = strat_pool.get_data( + [ + "space-template_desc-cleaned_bold", + "space-template_desc-brain_bold", + "space-template_desc-motion_bold", + "space-template_desc-preproc_bold", + "space-template_bold", + ] + ) + wf.connect(node, out, sc_temp_reg, "inputspec.subject_rest") + + wf.connect( + roi_timeseries_for_multreg, + "outputspec.roi_csv", + # ('outputspec.roi_outputs', extract_one_d), + sc_temp_reg, + "inputspec.subject_timeseries", + ) - node, out = strat_pool.get_data('space-template_desc-bold_mask') - wf.connect(node, out, sc_temp_reg, 'inputspec.subject_mask') + node, out = strat_pool.get_data("space-template_desc-bold_mask") + wf.connect(node, out, sc_temp_reg, "inputspec.subject_mask") outputs = { - 'space-template_desc-MultReg_correlations': - (sc_temp_reg, 'outputspec.temp_reg_map'), - 'desc-MultReg_statmap': - (sc_temp_reg, 'outputspec.temp_reg_map_z'), - 'atlas_name': (roi_dataflow_for_multreg, 'outputspec.out_name') + "space-template_desc-MultReg_correlations": ( + sc_temp_reg, + "outputspec.temp_reg_map", + ), + "desc-MultReg_statmap": (sc_temp_reg, "outputspec.temp_reg_map_z"), + "atlas_name": (roi_dataflow_for_multreg, "outputspec.out_name"), } return (wf, outputs) diff --git a/CPAC/sca/tests/test_sca.py b/CPAC/sca/tests/test_sca.py index 6a6ba71489..cfbf788c39 100755 --- a/CPAC/sca/tests/test_sca.py +++ b/CPAC/sca/tests/test_sca.py @@ -1,22 +1,23 @@ -from CPAC.pipeline import nipype_pipeline_engine as pe -import nipype.algorithms.rapidart as ra -import nipype.interfaces.afni as afni -import nipype.interfaces.fsl as fsl -import nipype.interfaces.io as nio -import nipype.interfaces.utility as util - class TestSCA(object): - - def __init__(self, rest_res_filt, ref, standard, fwhm, seed_list, rest_mask2standard, premat, postmat, extraction_space): - + def __init__( + self, + rest_res_filt, + ref, + standard, + fwhm, + seed_list, + rest_mask2standard, + premat, + postmat, + extraction_space, + ): """ Initialize Inputs - Constructor call to initialize the inputs to sca_preproc workflow + Constructor call to initialize the inputs to sca_preproc workflow. Parameters ---------- - seed_list : a list of existing nifti files A list of seeds/ ROI iin MNI space. @@ -40,12 +41,10 @@ def __init__(self, rest_res_filt, ref, standard, fwhm, seed_list, rest_mask2stan Returns ------- - Nothing """ - self.rest_res_filt = rest_res_filt self.ref = ref self.standard = standard @@ -57,39 +56,32 @@ def __init__(self, rest_res_filt, ref, standard, fwhm, seed_list, rest_mask2stan self.extraction_space = extraction_space self.sca_preproc = self.setup_workflow() - def setup_workflow(self): """ - Set up the workflow object by initializing it + Set up the workflow object by initializing it. Parameters ---------- - self Returns ------- - None """ - self.sca_preproc = sca_preproc(self.extraction_space) - def teardown_workflow(self): """ - Tear down the workflow by deleting the workflow object + Tear down the workflow by deleting the workflow object. Parameters ---------- - self Returns ------- - None """ @@ -98,21 +90,18 @@ def teardown_workflow(self): def test_inputs(self): """ - Test the workflow inputs + Test the workflow inputs. Parameters ---------- - self Returns ------- - Generates exceptions if any of the input tests fails Notes ----- - seed_list : Verify that inputs seeds lie inside the MNI brain(Not clear on how to verify it without the user supplied co-ordinates. @@ -145,27 +134,23 @@ def test_inputs(self): """ - assert False def test_output(self): """ - Run the sca_preproc workflow and test all the outputs + Run the sca_preproc workflow and test all the outputs. Parameters ---------- - self Returns ------- - generate exceptions for each outputs that fail the test Notes ----- - correlations_verification : (a boolean value) Reports if correlations command works as expected. @@ -241,29 +226,23 @@ def test_output(self): """ - assert False - - def test_warp_to_native(self): """ Checks if warping to native space works as expected - Set up the workflow object by initializing it + Set up the workflow object by initializing it. Parameters ---------- - self Returns ------- - generates an exception if warp_to_native_test fails Notes ----- - On Real Data - Compute the spatial correlation of the registered Image(outputs of the warp_to_native node) with the reference nifti image (ref) @@ -277,27 +256,22 @@ def test_warp_to_native(self): """ - assert False - def test_time_series(self): """ - checks if extraction of timeseries works as expected + checks if extraction of timeseries works as expected. Parameters ---------- - self Returns ------- - generates an exception if time_series_test fails Notes ----- - On Real Data - CASE_extraction_space_MNI: Extract the mean time series from the rest_res_filt in MNI space(do it for all the seeds in MNI space) @@ -312,27 +286,22 @@ def test_time_series(self): - Not sure """ - assert False - def test_print_timeseries_to_file(self): """ - checks if outputing timeseries list to a file works as expected + checks if outputing timeseries list to a file works as expected. Parameters ---------- - self Returns ------- - generates an exception if print_timeseries_to_file_test fails Notes ----- - On Real Data - Read the timeseries from the timeseries file (output of print_timeseries_to_file node). @@ -343,29 +312,22 @@ def test_print_timeseries_to_file(self): Not sure """ - assert False - def test_warp_filt(self): - - """ - checks if warping from functional space to MNI space works as expected + checks if warping from functional space to MNI space works as expected. Parameters ---------- - self Returns ------- - generates an exception if warp_filt_test fails Notes ----- - On Real Data - Compute the spatial correlation of the registered Image(output of warp_filt node) with the reference nifti image (standard) @@ -379,28 +341,22 @@ def test_warp_filt(self): Not Sure """ - assert False - def test_z_trans(self): - """ - checks if fisher Z transformation works as expected + checks if fisher Z transformation works as expected. Parameters ---------- - self Returns ------- - generates an exception if z_trans_test fails Notes ----- - On Real Data - Measure the variance in z scores in a subject and between subjects as the correlation values vary, is approximately constant @@ -412,26 +368,20 @@ def test_z_trans(self): """ assert False - def test_warp_to_standard(self): - """ - checks if warping from functional space to MNI space works as expected - + checks if warping from functional space to MNI space works as expected. Parameters ---------- - self Returns ------- - generates an exception if warp_to_standard_test fails Notes ----- - On Real Data - Compute the spatial correlation of the registered Image(output of warp_to_standard node) with the reference nifti image (standard) @@ -448,25 +398,20 @@ def test_warp_to_standard(self): """ assert False - def test_corr(self): - """ - tests if computing correlations works as expected + tests if computing correlations works as expected. Parameters ---------- - self Returns ------- - generates an exception if corr_test fails Notes ----- - On Real Data - Extract mean TimeSeries using the ROI from rest_res_filt nifti image(use the seed and nifi image in appropriate co-ordinate space for extraction_space = 'mni' & 'native') @@ -484,25 +429,20 @@ def test_corr(self): """ assert False - def test_smooth_mni(self): - """ - test if spatial smoothing works as expected + test if spatial smoothing works as expected. Parameters ---------- - self Returns ------- - generates an exception if smooth_mni_test fails Notes ----- - On Real Data - Verify that the intensiy distribution in smoothed image is still centered at zero and have unit variance diff --git a/CPAC/sca/utils.py b/CPAC/sca/utils.py index a75062e702..d09601ad7d 100644 --- a/CPAC/sca/utils.py +++ b/CPAC/sca/utils.py @@ -1,6 +1,4 @@ import os -from CPAC.pipeline import nipype_pipeline_engine as pe -import nipype.interfaces.utility as util def compute_fisher_z_score(correlation_file, timeseries_one_d): @@ -8,7 +6,7 @@ def compute_fisher_z_score(correlation_file, timeseries_one_d): Computes the fisher z transform of the input correlation map If the correlation map contains data for multiple ROIs then the function returns z score for each ROI as a seperate nifti - file + file. Parameters @@ -21,16 +19,22 @@ def compute_fisher_z_score(correlation_file, timeseries_one_d): out_file : list (nifti files) list of z_scores for mask or ROI """ + import os - import nibabel as nb import numpy as np - import os + import nibabel as nib roi_numbers = [] - if '#' in open(timeseries_one_d, 'r').readline().rstrip('\r\n'): - roi_numbers = open(timeseries_one_d, 'r').readline().rstrip('\r\n').replace('#', '').split('\t') - - corr_img = nb.load(correlation_file) + if "#" in open(timeseries_one_d, "r").readline().rstrip("\r\n"): + roi_numbers = ( + open(timeseries_one_d, "r") + .readline() + .rstrip("\r\n") + .replace("#", "") + .split("\t") + ) + + corr_img = nib.load(correlation_file) corr_data = corr_img.get_fdata() hdr = corr_img.header @@ -42,25 +46,27 @@ def compute_fisher_z_score(correlation_file, timeseries_one_d): out_file = [] if len(dims) == 5 or len(roi_numbers) > 0: - if len(dims) == 5: x, y, z, one, roi_number = dims - corr_data = np.reshape(corr_data, (x * y * z, roi_number), order='F') + corr_data = np.reshape(corr_data, (x * y * z, roi_number), order="F") for i in range(0, len(roi_numbers)): - sub_data = corr_data if len(dims) == 5: - sub_data = np.reshape(corr_data[:, i], (x, y, z), order='F') - - sub_img = nb.Nifti1Image(sub_data, header=corr_img.header, affine=corr_img.affine) - sub_z_score_file = os.path.join(os.getcwd(), 'z_score_ROI_number_%s.nii.gz' % (roi_numbers[i])) + sub_data = np.reshape(corr_data[:, i], (x, y, z), order="F") + + sub_img = nib.Nifti1Image( + sub_data, header=corr_img.header, affine=corr_img.affine + ) + sub_z_score_file = os.path.join( + os.getcwd(), "z_score_ROI_number_%s.nii.gz" % (roi_numbers[i]) + ) sub_img.to_filename(sub_z_score_file) out_file.append(sub_z_score_file) else: - z_score_img = nb.Nifti1Image(corr_data, header=hdr, affine=corr_img.affine) - z_score_file = os.path.join(os.getcwd(), 'z_score.nii.gz') + z_score_img = nib.Nifti1Image(corr_data, header=hdr, affine=corr_img.affine) + z_score_file = os.path.join(os.getcwd(), "z_score.nii.gz") z_score_img.to_filename(z_score_file) out_file.append(z_score_file) @@ -69,37 +75,42 @@ def compute_fisher_z_score(correlation_file, timeseries_one_d): def check_ts(in_file): import os + import numpy as np - if in_file.endswith('.txt'): + if in_file.endswith(".txt"): try: timepoints, rois = np.loadtxt(in_file).shape except ValueError: timepoints = np.loadtxt(in_file).shape[0] rois = 1 out_file = in_file - elif in_file.endswith('.csv') or in_file.endswith('.1D'): - csv_array = np.genfromtxt(in_file, delimiter=',') + elif in_file.endswith(".csv") or in_file.endswith(".1D"): + csv_array = np.genfromtxt(in_file, delimiter=",") if np.isnan(csv_array[0][0]): csv_array = csv_array[1:] timepoints, rois = csv_array.shape # multiple regression (fsl_glm) needs this format for -design input - out_file = os.path.join(os.getcwd(), - os.path.basename(in_file).replace('.csv', '.txt')) - np.savetxt(out_file, csv_array, delimiter='\t') + out_file = os.path.join( + os.getcwd(), os.path.basename(in_file).replace(".csv", ".txt") + ) + np.savetxt(out_file, csv_array, delimiter="\t") if rois > timepoints: - message = ('\n\n\n****The number of timepoints (' + str(timepoints) - + ') is smaller than the number of ROIs to run (' - + str(rois) + ') - therefore the GLM is' - + ' underspecified and can\'t run.****\n\n\n') - print(message) + message = ( + "\n\n\n****The number of timepoints (" + + str(timepoints) + + ") is smaller than the number of ROIs to run (" + + str(rois) + + ") - therefore the GLM is" + + " underspecified and can't run.****\n\n\n" + ) raise Exception(message) else: return out_file def map_to_roi(timeseries, maps): - """ + r""" Renames the outputs of the temporal multiple regression workflow for sca according to the header information of the timeseries.txt file that was passed @@ -107,14 +118,16 @@ def map_to_roi(timeseries, maps): (which = 'RT') when calling the temporal regression workflow. If you run the temporal regression workflow manually, don\'t set (which = 'RT') unless you provide a timeseries.txt file with a header - containing the names of the timeseries - Parameters + containing the names of the timeseries. + + Parameters. ---------- timeseries : string Input timeseries.txt file maps : List (nifti files) List of output files generated by the temporal regression workflow if (which == 'RT') + Returns ------- labels : List (strings) @@ -123,27 +136,31 @@ def map_to_roi(timeseries, maps): List of output files generated by the temporal regression workflow if (which == 'RT') """ - import numpy as np import pandas as pd from nipype import logging - logger = logging.getLogger('nipype.workflow') + + logger = logging.getLogger("nipype.workflow") testMat = pd.read_csv(timeseries) timepoints, rois = testMat.shape if rois > timepoints: - logger.warning('The number of timepoints is smaller than the number ' - 'of ROIs to run - therefore the GLM is ' - 'underspecified and can\'t run.') + logger.warning( + "The number of timepoints is smaller than the number " + "of ROIs to run - therefore the GLM is " + "underspecified and can't run." + ) # pull in the ROI labels from the header of the extracted timeseries # CSV file with open(timeseries, "r") as f: roi_file_lines = f.read().splitlines() - roi_err = "\n\n[!] The output of 3dROIstats, used in extracting " \ - "the timeseries, was not in the expected format.\n\nROI " \ - "output file: {0}\n\n".format(timeseries) + roi_err = ( + "\n\n[!] The output of 3dROIstats, used in extracting " + "the timeseries, was not in the expected format.\n\nROI " + "output file: {0}\n\n".format(timeseries) + ) for line in roi_file_lines: if "Mean_" in line: @@ -152,10 +169,12 @@ def map_to_roi(timeseries, maps): # clear out any blank strings/non ROI labels in the list roi_list = [x for x in roi_list if "Mean" in x] # rename labels - roi_list = \ - [x.replace("Mean", - "sca_tempreg_z_maps_roi").replace(" ", "").replace("#", "") - for x in roi_list] + roi_list = [ + x.replace("Mean", "sca_tempreg_z_maps_roi") + .replace(" ", "") + .replace("#", "") + for x in roi_list + ] except: raise Exception(roi_err) break @@ -166,7 +185,7 @@ def map_to_roi(timeseries, maps): for roi_label in roi_list: new_labels.append(os.path.join(os.getcwd(), roi_label)) - numMaps = len(maps) + len(maps) maps.sort() # if not numMaps / 2 == rois: diff --git a/CPAC/scrubbing/__init__.py b/CPAC/scrubbing/__init__.py index a64a607eab..538f9ec299 100644 --- a/CPAC/scrubbing/__init__.py +++ b/CPAC/scrubbing/__init__.py @@ -1,7 +1,3 @@ -from .scrubbing import create_scrubbing_preproc, \ - get_mov_parameters, \ - get_indx +from .scrubbing import create_scrubbing_preproc, get_indx, get_mov_parameters -__all__ = ['create_scrubbing_preproc', \ - 'get_mov_parameters', \ - 'get_indx'] +__all__ = ["create_scrubbing_preproc", "get_mov_parameters", "get_indx"] diff --git a/CPAC/scrubbing/scrubbing.py b/CPAC/scrubbing/scrubbing.py index 6082312c88..b000438902 100644 --- a/CPAC/scrubbing/scrubbing.py +++ b/CPAC/scrubbing/scrubbing.py @@ -1,58 +1,58 @@ -import nipype.interfaces.afni.preprocess as e_afni -from CPAC.pipeline import nipype_pipeline_engine as pe import nipype.interfaces.utility as util +from CPAC.pipeline import nipype_pipeline_engine as pe + -def create_scrubbing_preproc(wf_name = 'scrubbing'): +def create_scrubbing_preproc(wf_name="scrubbing"): """ This workflow essentially takes the list of offending timepoints that are to be removed and removes it from the motion corrected input image. Also, it removes the information of discarded time points from the movement parameters file obtained during motion correction. - + Parameters ---------- wf_name : string Name of the workflow - + Returns ------- scrub : object Scrubbing workfow object - + Notes ----- `Source `_ - + Workflow Inputs:: - + inputspec.frames_in_ID : string (mat file) path to file containing list of time points for which FD > threshold inputspec.movement_parameters : string (mat file) path to file containing 1D file containing six movement/motion parameters - (3 Translation, 3 Rotations) in different columns + (3 Translation, 3 Rotations) in different columns inputspec.preprocessed : string (nifti file) preprocessed input image path - + Workflow Outputs:: - + outputspec.preprocessed : string (nifti file) - preprocessed scrubbed output image + preprocessed scrubbed output image outputspec.scrubbed_movement_parameters : string (mat file) path to 1D file containing six movement/motion parameters for the timepoints which are not discarded by scrubbing - + Order of Commands: - + - Remove all movement parameters for all the time frames other than those that are present in the frames_in_1D file - + - Remove the discarded timepoints from the input image. For details see `3dcalc `_:: - - 3dcalc -a bandpassed_demeaned_filtered.nii.gz[0,1,5,6,7,8,9,10,15,16,17,18,19,20,24,25,287,288,289,290,291,292,293,294,295] + + 3dcalc -a bandpassed_demeaned_filtered.nii.gz[0,1,5,6,7,8,9,10,15,16,17,18,19,20,24,25,287,288,289,290,291,292,293,294,295] -expr 'a' -prefix bandpassed_demeaned_filtered_3dc.nii.gz - + High Level Workflow Graph: - + .. exec:: from CPAC.scrubbing import create_scrubbing_preproc wf = create_scrubbing_preproc() @@ -63,12 +63,12 @@ def create_scrubbing_preproc(wf_name = 'scrubbing'): .. image:: ../../images/generated/scrubbing.png :width: 500 - + Detailed Workflow Graph: - + .. image:: ../../images/generated/scrubbing_detailed.png :width: 500 - + Example ------- >>> from CPAC import scrubbing @@ -77,54 +77,76 @@ def create_scrubbing_preproc(wf_name = 'scrubbing'): >>> sc.inputs.inputpsec.movement_parameters = 'rest_mc.1D' # doctest: +SKIP >>> sc.inputs.inputpsec.preprocessed = 'rest_pp.nii.gz' # doctest: +SKIP >>> sc.run() # doctest: +SKIP - - """ + """ scrub = pe.Workflow(name=wf_name) - inputNode = pe.Node(util.IdentityInterface(fields=['frames_in_1D', - 'movement_parameters', - 'preprocessed' - ]), - name='inputspec') - - outputNode = pe.Node(util.IdentityInterface(fields=['preprocessed', - 'scrubbed_movement_parameters']), - name='outputspec') - - craft_scrub_input = pe.Node(util.Function(input_names=['scrub_input', 'frames_in_1D_file'], - output_names=['scrub_input_string'], - function=get_indx), - name = 'scrubbing_craft_input_string') - - scrubbed_movement_parameters = pe.Node(util.Function(input_names=['infile_a', 'infile_b'], - output_names=['out_file'], - function=get_mov_parameters), - name='scrubbed_movement_parameters') + inputNode = pe.Node( + util.IdentityInterface( + fields=["frames_in_1D", "movement_parameters", "preprocessed"] + ), + name="inputspec", + ) + + outputNode = pe.Node( + util.IdentityInterface(fields=["preprocessed", "scrubbed_movement_parameters"]), + name="outputspec", + ) + + craft_scrub_input = pe.Node( + util.Function( + input_names=["scrub_input", "frames_in_1D_file"], + output_names=["scrub_input_string"], + function=get_indx, + ), + name="scrubbing_craft_input_string", + ) + + scrubbed_movement_parameters = pe.Node( + util.Function( + input_names=["infile_a", "infile_b"], + output_names=["out_file"], + function=get_mov_parameters, + ), + name="scrubbed_movement_parameters", + ) # THIS commented out until Nipype has an input for this interface that # allows for the selection of specific volumes to include - #scrubbed_preprocessed = pe.Node(interface=e_afni.Calc(), + # scrubbed_preprocessed = pe.Node(interface=e_afni.Calc(), # name='scrubbed_preprocessed') - #scrubbed_preprocessed.inputs.expr = 'a' - #scrubbed_preprocessed.inputs.outputtype = 'NIFTI_GZ' - - scrubbed_preprocessed = pe.Node(util.Function(input_names=['scrub_input'], - output_names=['scrubbed_image'], - function=scrub_image), - name='scrubbed_preprocessed') - - scrub.connect(inputNode, 'preprocessed', craft_scrub_input, 'scrub_input') - scrub.connect(inputNode, 'frames_in_1D', craft_scrub_input, 'frames_in_1D_file') - - scrub.connect(craft_scrub_input, 'scrub_input_string', scrubbed_preprocessed, 'scrub_input') - - scrub.connect(inputNode, 'movement_parameters', scrubbed_movement_parameters, 'infile_b') - scrub.connect(inputNode, 'frames_in_1D', scrubbed_movement_parameters, 'infile_a' ) - - scrub.connect(scrubbed_preprocessed, 'scrubbed_image', outputNode, 'preprocessed') - scrub.connect(scrubbed_movement_parameters, 'out_file', outputNode, 'scrubbed_movement_parameters') + # scrubbed_preprocessed.inputs.expr = 'a' + # scrubbed_preprocessed.inputs.outputtype = 'NIFTI_GZ' + + scrubbed_preprocessed = pe.Node( + util.Function( + input_names=["scrub_input"], + output_names=["scrubbed_image"], + function=scrub_image, + ), + name="scrubbed_preprocessed", + ) + + scrub.connect(inputNode, "preprocessed", craft_scrub_input, "scrub_input") + scrub.connect(inputNode, "frames_in_1D", craft_scrub_input, "frames_in_1D_file") + + scrub.connect( + craft_scrub_input, "scrub_input_string", scrubbed_preprocessed, "scrub_input" + ) + + scrub.connect( + inputNode, "movement_parameters", scrubbed_movement_parameters, "infile_b" + ) + scrub.connect(inputNode, "frames_in_1D", scrubbed_movement_parameters, "infile_a") + + scrub.connect(scrubbed_preprocessed, "scrubbed_image", outputNode, "preprocessed") + scrub.connect( + scrubbed_movement_parameters, + "out_file", + outputNode, + "scrubbed_movement_parameters", + ) return scrub @@ -132,45 +154,45 @@ def create_scrubbing_preproc(wf_name = 'scrubbing'): def get_mov_parameters(infile_a, infile_b): """ Method to get the new movement parameters - file after removing the offending time frames - (i.e., those exceeding FD 0.5mm/0.2mm threshold) - + file after removing the offending time frames + (i.e., those exceeding FD 0.5mm/0.2mm threshold). + Parameters ---------- infile_a : string path to file containing the valid time frames - + infile_b : string - path to the file containing motion parameters - + path to the file containing motion parameters + Returns ------- out_file : string path to the file containing motion parameters - for the valid time frames - + for the valid time frames + """ import os import warnings - out_file = os.path.join(os.getcwd(), 'rest_mc_scrubbed.1D') + out_file = os.path.join(os.getcwd(), "rest_mc_scrubbed.1D") - f1= open(infile_a) - f2=open(infile_b) - l1=f1.readline() - l2=f2.readlines() + f1 = open(infile_a) + f2 = open(infile_b) + l1 = f1.readline() + l2 = f2.readlines() f1.close() f2.close() if l1: - l1=l1.rstrip(',').split(',') - warnings.warn("number of timepoints remaining after scrubbing -> %d"%len(l1)) + l1 = l1.rstrip(",").split(",") + warnings.warn("number of timepoints remaining after scrubbing -> %d" % len(l1)) else: raise Exception("No time points remaining after scrubbing.") - f = open(out_file, 'a') + f = open(out_file, "a") for l in l1: - data=l2[int(l.strip())] + data = l2[int(l.strip())] f.write(data) f.close() return out_file @@ -178,34 +200,31 @@ def get_mov_parameters(infile_a, infile_b): def get_indx(scrub_input, frames_in_1D_file): """ - Method to get the list of time - frames that are to be included - + Method to get the list of time + frames that are to be included. + Parameters ---------- in_file : string path to file containing the valid time frames - + Returns ------- scrub_input_string : string input string for 3dCalc in scrubbing workflow, looks something like " 4dfile.nii.gz[0,1,2,..100] " - - """ - with open(frames_in_1D_file, 'r') as f: + """ + with open(frames_in_1D_file, "r") as f: line = f.readline() - line = line.strip(',') + line = line.strip(",") if line: indx = map(int, line.split(",")) else: raise Exception("No time points remaining after scrubbing.") - scrub_input_string = scrub_input + str(indx).replace(" ", "") - - return scrub_input_string + return scrub_input + str(indx).replace(" ", "") def scrub_image(scrub_input): @@ -214,24 +233,23 @@ def scrub_image(scrub_input): the Nipype interface for 3dcalc because functionality is needed for specifying an input file with specifically-selected volumes. For example: input.nii.gz[2,3,4,..98], etc. - + Parameters ---------- scrub_input : string path to 4D file to be scrubbed, plus with selected volumes to be included - + Returns ------- scrubbed_image : string path to the scrubbed 4D file - - """ + """ import os - os.system("3dcalc -a %s -expr 'a' -prefix scrubbed_preprocessed.nii.gz" % scrub_input) - - scrubbed_image = os.path.join(os.getcwd(), "scrubbed_preprocessed.nii.gz") + os.system( + "3dcalc -a %s -expr 'a' -prefix scrubbed_preprocessed.nii.gz" % scrub_input + ) - return scrubbed_image + return os.path.join(os.getcwd(), "scrubbed_preprocessed.nii.gz") diff --git a/CPAC/seg_preproc/__init__.py b/CPAC/seg_preproc/__init__.py index 8b50288e8e..cc1143635a 100644 --- a/CPAC/seg_preproc/__init__.py +++ b/CPAC/seg_preproc/__init__.py @@ -1,30 +1,34 @@ -from .seg_preproc import process_segment_map, \ - tissue_mask_template_to_t1, \ - create_seg_preproc_antsJointLabel_method - - -from .utils import check_if_file_is_empty, \ - pick_wm_prob_0, \ - pick_wm_prob_1, \ - pick_wm_prob_2, \ - pick_wm_class_0, \ - pick_wm_class_1, \ - pick_wm_class_2, \ - mask_erosion, \ - erosion, \ - hardcoded_antsJointLabelFusion +from .seg_preproc import ( + create_seg_preproc_antsJointLabel_method, + process_segment_map, + tissue_mask_template_to_t1, +) +from .utils import ( + check_if_file_is_empty, + erosion, + hardcoded_antsJointLabelFusion, + mask_erosion, + pick_wm_class_0, + pick_wm_class_1, + pick_wm_class_2, + pick_wm_prob_0, + pick_wm_prob_1, + pick_wm_prob_2, +) # List all functions -__all__ = ['process_segment_map', - 'check_if_file_is_empty', - 'pick_wm_prob_0', - 'pick_wm_prob_1', - 'pick_wm_prob_2', - 'pick_wm_class_0', - 'pick_wm_class_1', - 'pick_wm_class_2', - 'mask_erosion', - 'erosion', - 'hardcoded_antsJointLabelFusion', - 'tissue_mask_template_to_t1', - 'create_seg_preproc_antsJointLabel_method'] +__all__ = [ + "process_segment_map", + "check_if_file_is_empty", + "pick_wm_prob_0", + "pick_wm_prob_1", + "pick_wm_prob_2", + "pick_wm_class_0", + "pick_wm_class_1", + "pick_wm_class_2", + "mask_erosion", + "erosion", + "hardcoded_antsJointLabelFusion", + "tissue_mask_template_to_t1", + "create_seg_preproc_antsJointLabel_method", +] diff --git a/CPAC/seg_preproc/seg_preproc.py b/CPAC/seg_preproc/seg_preproc.py index 78d40703d9..0302a4c86f 100644 --- a/CPAC/seg_preproc/seg_preproc.py +++ b/CPAC/seg_preproc/seg_preproc.py @@ -1,24 +1,24 @@ -from CPAC.pipeline.nodeblock import nodeblock -from nipype.interfaces.utility import Function from nipype.interfaces import ants, freesurfer, fsl, utility as util +from nipype.interfaces.utility import Function -from CPAC.anat_preproc.utils import freesurfer_hemispheres, mri_convert +from CPAC.anat_preproc.utils import mri_convert from CPAC.pipeline import nipype_pipeline_engine as pe +from CPAC.pipeline.nodeblock import nodeblock from CPAC.registration.registration import apply_transform -from CPAC.registration.utils import ( - check_transforms, - generate_inverse_transform_flags) +from CPAC.registration.utils import check_transforms, generate_inverse_transform_flags from CPAC.seg_preproc.utils import ( check_if_file_is_empty, - pick_wm_prob_0, - pick_wm_prob_1, - pick_wm_prob_2, + hardcoded_antsJointLabelFusion, pick_wm_class_0, pick_wm_class_1, pick_wm_class_2, - hardcoded_antsJointLabelFusion) -from CPAC.utils.interfaces.function.seg_preproc import \ - pick_tissue_from_labels_file_interface + pick_wm_prob_0, + pick_wm_prob_1, + pick_wm_prob_2, +) +from CPAC.utils.interfaces.function.seg_preproc import ( + pick_tissue_from_labels_file_interface, +) from CPAC.utils.utils import check_prov_for_regtool @@ -51,7 +51,6 @@ def process_segment_map(wf_name, use_priors, use_custom_threshold, reg_tool): Notes ----- - `Source `_ @@ -136,22 +135,27 @@ def process_segment_map(wf_name, use_priors, use_custom_threshold, reg_tool): preproc = pe.Workflow(name=wf_name) inputNode = pe.Node( - util.IdentityInterface(fields=['tissue_prior', - 'threshold', - 'erosion_prop', - 'mask_erosion_mm', - 'erosion_mm', - 'brain', - 'brain_mask', - 'tissue_class_file', - 'probability_tissue_map', - 'template_to_T1_xfm']), - name='inputspec') + util.IdentityInterface( + fields=[ + "tissue_prior", + "threshold", + "erosion_prop", + "mask_erosion_mm", + "erosion_mm", + "brain", + "brain_mask", + "tissue_class_file", + "probability_tissue_map", + "template_to_T1_xfm", + ] + ), + name="inputspec", + ) outputNode = pe.Node( - util.IdentityInterface(fields=['segment_mask', - 'probability_tissue_map']), - name='outputspec') + util.IdentityInterface(fields=["segment_mask", "probability_tissue_map"]), + name="outputspec", + ) # FSL-FAST # 'tissue_class_files' output is a list of individual binary tissue masks @@ -160,68 +164,73 @@ def process_segment_map(wf_name, use_priors, use_custom_threshold, reg_tool): # triggered by 'probability_maps' boolean input (-p) def form_threshold_string(threshold): - return '-thr %f ' % (threshold) + return "-thr %f " % (threshold) if not use_custom_threshold: # already binary tissue mask - input_1, value_1 = (inputNode, 'tissue_class_file') + input_1, value_1 = (inputNode, "tissue_class_file") else: # probability map - input_1, value_1 = (inputNode, 'probability_tissue_map') - - + input_1, value_1 = (inputNode, "probability_tissue_map") if use_priors: - apply_xfm = apply_transform(f'seg_tissue_priors_template_to_T1', - reg_tool=reg_tool) + apply_xfm = apply_transform( + "seg_tissue_priors_template_to_T1", reg_tool=reg_tool + ) apply_xfm.inputs.inputspec.interpolation = "NearestNeighbor" - preproc.connect(inputNode, 'tissue_prior', apply_xfm, - 'inputspec.input_image') - preproc.connect(inputNode, 'brain', apply_xfm, - 'inputspec.reference') - preproc.connect(inputNode, 'template_to_T1_xfm', apply_xfm, - 'inputspec.transform') + preproc.connect(inputNode, "tissue_prior", apply_xfm, "inputspec.input_image") + preproc.connect(inputNode, "brain", apply_xfm, "inputspec.reference") + preproc.connect( + inputNode, "template_to_T1_xfm", apply_xfm, "inputspec.transform" + ) overlap_segmentmap_with_prior = pe.Node( interface=fsl.MultiImageMaths(), - name='overlap_%s_map_with_prior' % (wf_name), + name="overlap_%s_map_with_prior" % (wf_name), mem_gb=1.775, - mem_x=(5022839943792975 / 2417851639229258349412352, 'in_file')) - overlap_segmentmap_with_prior.inputs.op_string = '-mas %s ' + mem_x=(5022839943792975 / 2417851639229258349412352, "in_file"), + ) + overlap_segmentmap_with_prior.inputs.op_string = "-mas %s " - preproc.connect(input_1, value_1, - overlap_segmentmap_with_prior, 'in_file') + preproc.connect(input_1, value_1, overlap_segmentmap_with_prior, "in_file") - preproc.connect(apply_xfm, 'outputspec.output_image', - overlap_segmentmap_with_prior, 'operand_files') + preproc.connect( + apply_xfm, + "outputspec.output_image", + overlap_segmentmap_with_prior, + "operand_files", + ) - input_1, value_1 = (overlap_segmentmap_with_prior, 'out_file') + input_1, value_1 = (overlap_segmentmap_with_prior, "out_file") if use_custom_threshold: segmentmap_threshold = pe.Node( - interface=fsl.ImageMaths(), - name='threshold_segmentmap_%s' % (wf_name)) - preproc.connect(inputNode, ('threshold', form_threshold_string), - segmentmap_threshold, 'op_string') + interface=fsl.ImageMaths(), name="threshold_segmentmap_%s" % (wf_name) + ) + preproc.connect( + inputNode, + ("threshold", form_threshold_string), + segmentmap_threshold, + "op_string", + ) - preproc.connect(input_1, value_1, segmentmap_threshold, 'in_file') + preproc.connect(input_1, value_1, segmentmap_threshold, "in_file") # these are the probability maps, not the binary tissue masks - input_1, value_1 = (segmentmap_threshold, 'out_file') + input_1, value_1 = (segmentmap_threshold, "out_file") - binarize_threshold_segmentmap = pe.Node(interface=fsl.ImageMaths(), - name='binarize_%s' % ( - wf_name)) - binarize_threshold_segmentmap.inputs.op_string = '-bin ' + binarize_threshold_segmentmap = pe.Node( + interface=fsl.ImageMaths(), name="binarize_%s" % (wf_name) + ) + binarize_threshold_segmentmap.inputs.op_string = "-bin " - preproc.connect(input_1, value_1, - binarize_threshold_segmentmap, 'in_file') + preproc.connect(input_1, value_1, binarize_threshold_segmentmap, "in_file") - input_1, value_1 = (binarize_threshold_segmentmap, 'out_file') + input_1, value_1 = (binarize_threshold_segmentmap, "out_file") # regardless of input, they are binary tissue masks now - preproc.connect(input_1, value_1, outputNode, 'segment_mask') + preproc.connect(input_1, value_1, outputNode, "segment_mask") return preproc @@ -232,91 +241,121 @@ def tissue_mask_template_to_t1(wf_name, use_ants): preproc = pe.Workflow(name=wf_name) inputNode = pe.Node( - util.IdentityInterface(fields=['brain', - 'standard2highres_init', - 'standard2highres_mat', - 'standard2highres_rig', - 'tissue_mask_template']), - name='inputspec') + util.IdentityInterface( + fields=[ + "brain", + "standard2highres_init", + "standard2highres_mat", + "standard2highres_rig", + "tissue_mask_template", + ] + ), + name="inputspec", + ) outputNode = pe.Node( - util.IdentityInterface(fields=['segment_mask_temp2t1']), - name='outputspec') + util.IdentityInterface(fields=["segment_mask_temp2t1"]), name="outputspec" + ) if use_ants: collect_linear_transforms = pe.Node( - util.Merge(3), - name='{0}_collect_linear_transforms'.format(wf_name)) + util.Merge(3), name="{0}_collect_linear_transforms".format(wf_name) + ) - preproc.connect(inputNode, 'standard2highres_init', - collect_linear_transforms, 'in1') - preproc.connect(inputNode, 'standard2highres_rig', - collect_linear_transforms, 'in2') - preproc.connect(inputNode, 'standard2highres_mat', - collect_linear_transforms, 'in3') + preproc.connect( + inputNode, "standard2highres_init", collect_linear_transforms, "in1" + ) + preproc.connect( + inputNode, "standard2highres_rig", collect_linear_transforms, "in2" + ) + preproc.connect( + inputNode, "standard2highres_mat", collect_linear_transforms, "in3" + ) # check transform list to exclude Nonetype (missing) init/rig/affine check_transform = pe.Node( - util.Function(input_names=['transform_list'], - output_names=['checked_transform_list', - 'list_length'], - function=check_transforms), - name='{0}_check_transforms'.format(wf_name)) + util.Function( + input_names=["transform_list"], + output_names=["checked_transform_list", "list_length"], + function=check_transforms, + ), + name="{0}_check_transforms".format(wf_name), + ) - preproc.connect(collect_linear_transforms, 'out', - check_transform, 'transform_list') + preproc.connect( + collect_linear_transforms, "out", check_transform, "transform_list" + ) # generate inverse transform flags, which depends on the # number of transforms inverse_transform_flags = pe.Node( - util.Function(input_names=['transform_list'], - output_names=['inverse_transform_flags'], - function=generate_inverse_transform_flags), - name='{0}_inverse_transform_flags'.format(wf_name)) + util.Function( + input_names=["transform_list"], + output_names=["inverse_transform_flags"], + function=generate_inverse_transform_flags, + ), + name="{0}_inverse_transform_flags".format(wf_name), + ) - preproc.connect(check_transform, 'checked_transform_list', - inverse_transform_flags, 'transform_list') + preproc.connect( + check_transform, + "checked_transform_list", + inverse_transform_flags, + "transform_list", + ) # mni to t1 - tissueprior_mni_to_t1 = pe.Node(interface=ants.ApplyTransforms(), - name='{0}_mni_to_t1'.format(wf_name)) + tissueprior_mni_to_t1 = pe.Node( + interface=ants.ApplyTransforms(), name="{0}_mni_to_t1".format(wf_name) + ) - tissueprior_mni_to_t1.inputs.interpolation = 'NearestNeighbor' + tissueprior_mni_to_t1.inputs.interpolation = "NearestNeighbor" - preproc.connect(inverse_transform_flags, 'inverse_transform_flags', - tissueprior_mni_to_t1, 'invert_transform_flags') - preproc.connect(inputNode, 'brain', - tissueprior_mni_to_t1, 'reference_image') - preproc.connect(check_transform, 'checked_transform_list', - tissueprior_mni_to_t1, 'transforms') - preproc.connect(inputNode, 'tissue_mask_template', - tissueprior_mni_to_t1, 'input_image') + preproc.connect( + inverse_transform_flags, + "inverse_transform_flags", + tissueprior_mni_to_t1, + "invert_transform_flags", + ) + preproc.connect(inputNode, "brain", tissueprior_mni_to_t1, "reference_image") + preproc.connect( + check_transform, + "checked_transform_list", + tissueprior_mni_to_t1, + "transforms", + ) + preproc.connect( + inputNode, "tissue_mask_template", tissueprior_mni_to_t1, "input_image" + ) - preproc.connect(tissueprior_mni_to_t1, 'output_image', - outputNode, 'segment_mask_temp2t1') + preproc.connect( + tissueprior_mni_to_t1, "output_image", outputNode, "segment_mask_temp2t1" + ) else: - tissueprior_mni_to_t1 = pe.Node(interface=fsl.FLIRT(), - name='{0}_mni_to_t1'.format(wf_name)) + tissueprior_mni_to_t1 = pe.Node( + interface=fsl.FLIRT(), name="{0}_mni_to_t1".format(wf_name) + ) tissueprior_mni_to_t1.inputs.apply_xfm = True - tissueprior_mni_to_t1.inputs.interp = 'nearestneighbour' + tissueprior_mni_to_t1.inputs.interp = "nearestneighbour" # mni to t1 - preproc.connect(inputNode, 'tissue_mask_template', - tissueprior_mni_to_t1, 'in_file') - preproc.connect(inputNode, 'brain', tissueprior_mni_to_t1, - 'reference') - preproc.connect(inputNode, 'standard2highres_mat', - tissueprior_mni_to_t1, 'in_matrix_file') + preproc.connect( + inputNode, "tissue_mask_template", tissueprior_mni_to_t1, "in_file" + ) + preproc.connect(inputNode, "brain", tissueprior_mni_to_t1, "reference") + preproc.connect( + inputNode, "standard2highres_mat", tissueprior_mni_to_t1, "in_matrix_file" + ) - preproc.connect(tissueprior_mni_to_t1, 'out_file', - outputNode, 'segment_mask_temp2t1') + preproc.connect( + tissueprior_mni_to_t1, "out_file", outputNode, "segment_mask_temp2t1" + ) return preproc -def create_seg_preproc_antsJointLabel_method( - wf_name='seg_preproc_templated_based'): +def create_seg_preproc_antsJointLabel_method(wf_name="seg_preproc_templated_based"): """ Generate the subject's cerebral spinal fluids, white matter and gray matter mask based on provided template, if selected to do so. @@ -333,7 +372,6 @@ def create_seg_preproc_antsJointLabel_method( Notes ----- - Workflow Inputs: :: inputspec.brain : string (existing nifti file) @@ -356,61 +394,81 @@ def create_seg_preproc_antsJointLabel_method( outputspec.wm_mask : string (nifti file) outputs White Matter mask """ - preproc = pe.Workflow(name=wf_name) - inputNode = pe.Node(util.IdentityInterface(fields=['anatomical_brain', - 'anatomical_brain_mask', - 'template_brain_list', - 'template_segmentation' - '_list', - 'csf_label', - 'gm_label', - 'wm_label']), - name='inputspec') - - outputNode = pe.Node(util.IdentityInterface(fields=['csf_mask', - 'gm_mask', - 'wm_mask']), - name='outputspec') + inputNode = pe.Node( + util.IdentityInterface( + fields=[ + "anatomical_brain", + "anatomical_brain_mask", + "template_brain_list", + "template_segmentation" "_list", + "csf_label", + "gm_label", + "wm_label", + ] + ), + name="inputspec", + ) + + outputNode = pe.Node( + util.IdentityInterface(fields=["csf_mask", "gm_mask", "wm_mask"]), + name="outputspec", + ) seg_preproc_antsJointLabel = pe.Node( - util.Function(input_names=['anatomical_brain', - 'anatomical_brain_mask', - 'template_brain_list', - 'template_segmentation_list'], - output_names=['multiatlas_Intensity', - 'multiatlas_Labels'], - function=hardcoded_antsJointLabelFusion), - name='{0}_antsJointLabel'.format(wf_name)) - - preproc.connect(inputNode, 'anatomical_brain', - seg_preproc_antsJointLabel, 'anatomical_brain') - preproc.connect(inputNode, 'anatomical_brain_mask', - seg_preproc_antsJointLabel, 'anatomical_brain_mask') - preproc.connect(inputNode, 'template_brain_list', - seg_preproc_antsJointLabel, 'template_brain_list') - preproc.connect(inputNode, 'template_segmentation_list', - seg_preproc_antsJointLabel, 'template_segmentation_list') - - pick_tissue = pe.Node(pick_tissue_from_labels_file_interface(), - name='{0}_tissue_mask'.format(wf_name)) - - preproc.connect(seg_preproc_antsJointLabel, 'multiatlas_Labels', - pick_tissue, 'multiatlas_Labels') - preproc.connect(inputNode, 'csf_label', - pick_tissue, 'csf_label') - preproc.connect(inputNode, 'gm_label', - pick_tissue, 'gm_label') - preproc.connect(inputNode, 'wm_label', - pick_tissue, 'wm_label') - - preproc.connect(pick_tissue, 'csf_mask', - outputNode, 'csf_mask') - preproc.connect(pick_tissue, 'gm_mask', - outputNode, 'gm_mask') - preproc.connect(pick_tissue, 'wm_mask', - outputNode, 'wm_mask') + util.Function( + input_names=[ + "anatomical_brain", + "anatomical_brain_mask", + "template_brain_list", + "template_segmentation_list", + ], + output_names=["multiatlas_Intensity", "multiatlas_Labels"], + function=hardcoded_antsJointLabelFusion, + ), + name="{0}_antsJointLabel".format(wf_name), + ) + + preproc.connect( + inputNode, "anatomical_brain", seg_preproc_antsJointLabel, "anatomical_brain" + ) + preproc.connect( + inputNode, + "anatomical_brain_mask", + seg_preproc_antsJointLabel, + "anatomical_brain_mask", + ) + preproc.connect( + inputNode, + "template_brain_list", + seg_preproc_antsJointLabel, + "template_brain_list", + ) + preproc.connect( + inputNode, + "template_segmentation_list", + seg_preproc_antsJointLabel, + "template_segmentation_list", + ) + + pick_tissue = pe.Node( + pick_tissue_from_labels_file_interface(), name="{0}_tissue_mask".format(wf_name) + ) + + preproc.connect( + seg_preproc_antsJointLabel, + "multiatlas_Labels", + pick_tissue, + "multiatlas_Labels", + ) + preproc.connect(inputNode, "csf_label", pick_tissue, "csf_label") + preproc.connect(inputNode, "gm_label", pick_tissue, "gm_label") + preproc.connect(inputNode, "wm_label", pick_tissue, "wm_label") + + preproc.connect(pick_tissue, "csf_mask", outputNode, "csf_mask") + preproc.connect(pick_tissue, "gm_mask", outputNode, "gm_mask") + preproc.connect(pick_tissue, "wm_mask", outputNode, "wm_mask") return preproc @@ -465,51 +523,65 @@ def tissue_seg_fsl_fast(wf, cfg, strat_pool, pipe_num, opt=None): # 'probability_maps' output is a list of individual probability maps # triggered by 'probability_maps' boolean input (-p) - segment = pe.Node(interface=fsl.FAST(), - name=f'segment_{pipe_num}', - mem_gb=3.48, - mem_x=(3444233104315183 / 19342813113834066795298816, - 'in_files')) + segment = pe.Node( + interface=fsl.FAST(), + name=f"segment_{pipe_num}", + mem_gb=3.48, + mem_x=(3444233104315183 / 19342813113834066795298816, "in_files"), + ) segment.inputs.img_type = 1 segment.inputs.segments = True segment.inputs.probability_maps = True - segment.inputs.out_basename = 'segment' - - check_wm = pe.Node(name='check_wm', - interface=Function(function=check_if_file_is_empty, - input_names=['in_file'], - output_names=['out_file'])) - check_gm = pe.Node(name='check_gm', - interface=Function(function=check_if_file_is_empty, - input_names=['in_file'], - output_names=['out_file'])) - check_csf = pe.Node(name='check_csf', interface=Function( - function=check_if_file_is_empty, input_names=['in_file'], - output_names=['out_file'])) - - connect, resource = \ - strat_pool.get_data(["desc-brain_T1w", - "space-longitudinal_desc-brain_T1w"], - report_fetched=True) - node, out = connect - wf.connect(node, out, segment, 'in_files') + segment.inputs.out_basename = "segment" + + pe.Node( + name="check_wm", + interface=Function( + function=check_if_file_is_empty, + input_names=["in_file"], + output_names=["out_file"], + ), + ) + pe.Node( + name="check_gm", + interface=Function( + function=check_if_file_is_empty, + input_names=["in_file"], + output_names=["out_file"], + ), + ) + pe.Node( + name="check_csf", + interface=Function( + function=check_if_file_is_empty, + input_names=["in_file"], + output_names=["out_file"], + ), + ) + connect, resource = strat_pool.get_data( + ["desc-brain_T1w", "space-longitudinal_desc-brain_T1w"], report_fetched=True + ) + node, out = connect + wf.connect(node, out, segment, "in_files") - use_custom_threshold = cfg['segmentation']['tissue_segmentation'][ - 'FSL-FAST']['thresholding'][ - 'use'] == 'Custom' + use_custom_threshold = ( + cfg["segmentation"]["tissue_segmentation"]["FSL-FAST"]["thresholding"]["use"] + == "Custom" + ) - use_priors = cfg['segmentation']['tissue_segmentation'][ - 'FSL-FAST']['use_priors']['run'] + use_priors = cfg["segmentation"]["tissue_segmentation"]["FSL-FAST"]["use_priors"][ + "run" + ] - long = '' - if 'space-longitudinal' in resource: - long = 'space-longitudinal_' + long = "" + if "space-longitudinal" in resource: + long = "space-longitudinal_" if use_priors: - xfm = 'from-template_to-T1w_mode-image_desc-linear_xfm' - if 'space-longitudinal' in resource: - xfm = 'from-template_to-longitudinal_mode-image_desc-linear_xfm' + xfm = "from-template_to-T1w_mode-image_desc-linear_xfm" + if "space-longitudinal" in resource: + xfm = "from-template_to-longitudinal_mode-image_desc-linear_xfm" xfm_prov = strat_pool.get_cpac_provenance(xfm) reg_tool = check_prov_for_regtool(xfm_prov) else: @@ -517,113 +589,140 @@ def tissue_seg_fsl_fast(wf, cfg, strat_pool, pipe_num, opt=None): reg_tool = None xfm = None - - process_csf = process_segment_map(f'CSF_{pipe_num}', use_priors, - use_custom_threshold, reg_tool) - process_csf.inputs.inputspec.threshold = cfg['segmentation'][ - 'tissue_segmentation']['FSL-FAST']['thresholding']['Custom'][ - 'CSF_threshold_value'] - - get_pve_csf = pe.Node(interface=fsl.maths.MathsCommand(), - name=f'get_pve_csf_{pipe_num}') - get_pve_csf.inputs.args = '-thr 0.5 -uthr 1.5 -bin' - wf.connect(segment, 'partial_volume_map', get_pve_csf, 'in_file') - - get_pve_gm = pe.Node(interface=fsl.maths.MathsCommand(), - name=f'get_pve_gm_{pipe_num}') - get_pve_gm.inputs.args = '-thr 1.5 -uthr 2.5 -bin' - wf.connect(segment, 'partial_volume_map', get_pve_gm, 'in_file') - - get_pve_wm = pe.Node(interface=fsl.maths.MathsCommand(), - name=f'get_pve_wm_{pipe_num}') - get_pve_wm.inputs.args = '-thr 2.5 -uthr 3.5 -bin' - wf.connect(segment, 'partial_volume_map', get_pve_wm, 'in_file') + process_csf = process_segment_map( + f"CSF_{pipe_num}", use_priors, use_custom_threshold, reg_tool + ) + process_csf.inputs.inputspec.threshold = cfg["segmentation"]["tissue_segmentation"][ + "FSL-FAST" + ]["thresholding"]["Custom"]["CSF_threshold_value"] + + get_pve_csf = pe.Node( + interface=fsl.maths.MathsCommand(), name=f"get_pve_csf_{pipe_num}" + ) + get_pve_csf.inputs.args = "-thr 0.5 -uthr 1.5 -bin" + wf.connect(segment, "partial_volume_map", get_pve_csf, "in_file") + + get_pve_gm = pe.Node( + interface=fsl.maths.MathsCommand(), name=f"get_pve_gm_{pipe_num}" + ) + get_pve_gm.inputs.args = "-thr 1.5 -uthr 2.5 -bin" + wf.connect(segment, "partial_volume_map", get_pve_gm, "in_file") + + get_pve_wm = pe.Node( + interface=fsl.maths.MathsCommand(), name=f"get_pve_wm_{pipe_num}" + ) + get_pve_wm.inputs.args = "-thr 2.5 -uthr 3.5 -bin" + wf.connect(segment, "partial_volume_map", get_pve_wm, "in_file") if use_priors: - node, out = strat_pool.get_data('CSF-path') - wf.connect(node, out, process_csf, 'inputspec.tissue_prior') + node, out = strat_pool.get_data("CSF-path") + wf.connect(node, out, process_csf, "inputspec.tissue_prior") - process_gm = process_segment_map(f'GM_{pipe_num}', use_priors, - use_custom_threshold, reg_tool) - process_gm.inputs.inputspec.threshold = cfg['segmentation'][ - 'tissue_segmentation']['FSL-FAST']['thresholding']['Custom'][ - 'GM_threshold_value'] + process_gm = process_segment_map( + f"GM_{pipe_num}", use_priors, use_custom_threshold, reg_tool + ) + process_gm.inputs.inputspec.threshold = cfg["segmentation"]["tissue_segmentation"][ + "FSL-FAST" + ]["thresholding"]["Custom"]["GM_threshold_value"] if use_priors: - node, out = strat_pool.get_data('GM-path') - wf.connect(node, out, process_gm, 'inputspec.tissue_prior') + node, out = strat_pool.get_data("GM-path") + wf.connect(node, out, process_gm, "inputspec.tissue_prior") - process_wm = process_segment_map(f'WM_{pipe_num}', use_priors, - use_custom_threshold, reg_tool) - process_wm.inputs.inputspec.threshold = cfg['segmentation'][ - 'tissue_segmentation']['FSL-FAST']['thresholding']['Custom'][ - 'WM_threshold_value'] + process_wm = process_segment_map( + f"WM_{pipe_num}", use_priors, use_custom_threshold, reg_tool + ) + process_wm.inputs.inputspec.threshold = cfg["segmentation"]["tissue_segmentation"][ + "FSL-FAST" + ]["thresholding"]["Custom"]["WM_threshold_value"] if use_priors: - node, out = strat_pool.get_data('WM-path') - wf.connect(node, out, process_wm, 'inputspec.tissue_prior') - - node, out = strat_pool.get_data(["desc-brain_T1w", - "space-longitudinal_desc-brain_T1w"]) - wf.connect(node, out, process_csf, 'inputspec.brain') - wf.connect(node, out, process_gm, 'inputspec.brain') - wf.connect(node, out, process_wm, 'inputspec.brain') - - node, out = strat_pool.get_data(["space-T1w_desc-brain_mask", - "space-longitudinal_desc-brain_mask"]) - wf.connect(node, out, process_csf, 'inputspec.brain_mask') - wf.connect(node, out, process_gm, 'inputspec.brain_mask') - wf.connect(node, out, process_wm, 'inputspec.brain_mask') + node, out = strat_pool.get_data("WM-path") + wf.connect(node, out, process_wm, "inputspec.tissue_prior") + + node, out = strat_pool.get_data( + ["desc-brain_T1w", "space-longitudinal_desc-brain_T1w"] + ) + wf.connect(node, out, process_csf, "inputspec.brain") + wf.connect(node, out, process_gm, "inputspec.brain") + wf.connect(node, out, process_wm, "inputspec.brain") + + node, out = strat_pool.get_data( + ["space-T1w_desc-brain_mask", "space-longitudinal_desc-brain_mask"] + ) + wf.connect(node, out, process_csf, "inputspec.brain_mask") + wf.connect(node, out, process_gm, "inputspec.brain_mask") + wf.connect(node, out, process_wm, "inputspec.brain_mask") if use_priors: node, out = strat_pool.get_data(xfm) - wf.connect(node, out, process_csf, 'inputspec.template_to_T1_xfm') - wf.connect(node, out, process_gm, 'inputspec.template_to_T1_xfm') - wf.connect(node, out, process_wm, 'inputspec.template_to_T1_xfm') - - wf.connect(segment, ('tissue_class_files', pick_wm_class_0), - process_csf, 'inputspec.tissue_class_file') - wf.connect(segment, ('probability_maps', pick_wm_prob_0), - process_csf, 'inputspec.probability_tissue_map') - - wf.connect(segment, ('tissue_class_files', pick_wm_class_1), - process_gm, 'inputspec.tissue_class_file') - wf.connect(segment, ('probability_maps', pick_wm_prob_1), - process_gm, 'inputspec.probability_tissue_map') - - wf.connect(segment, ('tissue_class_files', pick_wm_class_2), - process_wm, 'inputspec.tissue_class_file') - wf.connect(segment, ('probability_maps', pick_wm_prob_2), - process_wm, 'inputspec.probability_tissue_map') - - get_csf = pe.Node(util.Function(input_names=['probability_maps'], - output_names=['filename'], - function=pick_wm_prob_0), - name=f'get_csf_{pipe_num}') + wf.connect(node, out, process_csf, "inputspec.template_to_T1_xfm") + wf.connect(node, out, process_gm, "inputspec.template_to_T1_xfm") + wf.connect(node, out, process_wm, "inputspec.template_to_T1_xfm") + + wf.connect( + segment, + ("tissue_class_files", pick_wm_class_0), + process_csf, + "inputspec.tissue_class_file", + ) + wf.connect( + segment, + ("probability_maps", pick_wm_prob_0), + process_csf, + "inputspec.probability_tissue_map", + ) + + wf.connect( + segment, + ("tissue_class_files", pick_wm_class_1), + process_gm, + "inputspec.tissue_class_file", + ) + wf.connect( + segment, + ("probability_maps", pick_wm_prob_1), + process_gm, + "inputspec.probability_tissue_map", + ) + + wf.connect( + segment, + ("tissue_class_files", pick_wm_class_2), + process_wm, + "inputspec.tissue_class_file", + ) + wf.connect( + segment, + ("probability_maps", pick_wm_prob_2), + process_wm, + "inputspec.probability_tissue_map", + ) + + get_csf = pe.Node( + util.Function( + input_names=["probability_maps"], + output_names=["filename"], + function=pick_wm_prob_0, + ), + name=f"get_csf_{pipe_num}", + ) - wf.connect(segment, 'probability_maps', get_csf, 'probability_maps') + wf.connect(segment, "probability_maps", get_csf, "probability_maps") outputs = { - f'{long}label-CSF_probseg': (get_csf, 'filename'), - f'{long}label-GM_probseg': - (segment, ('probability_maps', pick_wm_prob_1)), - f'{long}label-WM_probseg': - (segment, ('probability_maps', pick_wm_prob_2)), - f'{long}label-CSF_mask': - (segment, ('tissue_class_files', pick_wm_class_0)), - f'{long}label-GM_mask': - (segment, ('tissue_class_files', pick_wm_class_1)), - f'{long}label-WM_mask': - (segment, ('tissue_class_files', pick_wm_class_2)), - f'{long}label-CSF_desc-preproc_mask': - (process_csf, 'outputspec.segment_mask'), - f'{long}label-GM_desc-preproc_mask': - (process_gm, 'outputspec.segment_mask'), - f'{long}label-WM_desc-preproc_mask': - (process_wm, 'outputspec.segment_mask'), - f'{long}label-CSF_pveseg': (get_pve_csf, 'out_file'), - f'{long}label-GM_pveseg': (get_pve_gm, 'out_file'), - f'{long}label-WM_pveseg': (get_pve_wm, 'out_file'), + f"{long}label-CSF_probseg": (get_csf, "filename"), + f"{long}label-GM_probseg": (segment, ("probability_maps", pick_wm_prob_1)), + f"{long}label-WM_probseg": (segment, ("probability_maps", pick_wm_prob_2)), + f"{long}label-CSF_mask": (segment, ("tissue_class_files", pick_wm_class_0)), + f"{long}label-GM_mask": (segment, ("tissue_class_files", pick_wm_class_1)), + f"{long}label-WM_mask": (segment, ("tissue_class_files", pick_wm_class_2)), + f"{long}label-CSF_desc-preproc_mask": (process_csf, "outputspec.segment_mask"), + f"{long}label-GM_desc-preproc_mask": (process_gm, "outputspec.segment_mask"), + f"{long}label-WM_desc-preproc_mask": (process_wm, "outputspec.segment_mask"), + f"{long}label-CSF_pveseg": (get_pve_csf, "out_file"), + f"{long}label-GM_pveseg": (get_pve_gm, "out_file"), + f"{long}label-WM_pveseg": (get_pve_wm, "out_file"), } return (wf, outputs) @@ -639,46 +738,41 @@ def tissue_seg_fsl_fast(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["label-CSF_mask", "label-GM_mask", "label-WM_mask"], ) def tissue_seg_T1_template_based(wf, cfg, strat_pool, pipe_num, opt=None): - xfm_prov = strat_pool.get_cpac_provenance( - 'from-template_to-T1w_mode-image_desc-linear_xfm') + "from-template_to-T1w_mode-image_desc-linear_xfm" + ) reg_tool = check_prov_for_regtool(xfm_prov) - use_ants = reg_tool == 'ants' - - csf_template2t1 = tissue_mask_template_to_t1(f'CSF_{pipe_num}', - use_ants) - csf_template2t1.inputs.inputspec.tissue_mask_template = cfg[ - 'segmentation']['tissue_segmentation']['Template_Based']['CSF'] - - gm_template2t1 = tissue_mask_template_to_t1(f'GM_{pipe_num}', - use_ants) - gm_template2t1.inputs.inputspec.tissue_mask_template = cfg[ - 'segmentation']['tissue_segmentation']['Template_Based']['GRAY'] - - wm_template2t1 = tissue_mask_template_to_t1(f'WM_{pipe_num}', - use_ants) - wm_template2t1.inputs.inputspec.tissue_mask_template = cfg[ - 'segmentation']['tissue_segmentation']['Template_Based']['WHITE'] - - node, out = strat_pool.get_data('desc-brain_T1w') - wf.connect(node, out, csf_template2t1, 'inputspec.brain') - wf.connect(node, out, gm_template2t1, 'inputspec.brain') - wf.connect(node, out, wm_template2t1, 'inputspec.brain') - - node, out = \ - strat_pool.get_data('from-template_to-T1w_mode-image_desc-linear_xfm') - wf.connect(node, out, - csf_template2t1, 'inputspec.standard2highres_mat') - wf.connect(node, out, - wm_template2t1, 'inputspec.standard2highres_mat') - wf.connect(node, out, - gm_template2t1, 'inputspec.standard2highres_mat') + use_ants = reg_tool == "ants" + + csf_template2t1 = tissue_mask_template_to_t1(f"CSF_{pipe_num}", use_ants) + csf_template2t1.inputs.inputspec.tissue_mask_template = cfg["segmentation"][ + "tissue_segmentation" + ]["Template_Based"]["CSF"] + + gm_template2t1 = tissue_mask_template_to_t1(f"GM_{pipe_num}", use_ants) + gm_template2t1.inputs.inputspec.tissue_mask_template = cfg["segmentation"][ + "tissue_segmentation" + ]["Template_Based"]["GRAY"] + + wm_template2t1 = tissue_mask_template_to_t1(f"WM_{pipe_num}", use_ants) + wm_template2t1.inputs.inputspec.tissue_mask_template = cfg["segmentation"][ + "tissue_segmentation" + ]["Template_Based"]["WHITE"] + + node, out = strat_pool.get_data("desc-brain_T1w") + wf.connect(node, out, csf_template2t1, "inputspec.brain") + wf.connect(node, out, gm_template2t1, "inputspec.brain") + wf.connect(node, out, wm_template2t1, "inputspec.brain") + + node, out = strat_pool.get_data("from-template_to-T1w_mode-image_desc-linear_xfm") + wf.connect(node, out, csf_template2t1, "inputspec.standard2highres_mat") + wf.connect(node, out, wm_template2t1, "inputspec.standard2highres_mat") + wf.connect(node, out, gm_template2t1, "inputspec.standard2highres_mat") outputs = { - 'label-CSF_mask': ( - csf_template2t1, 'outputspec.segment_mask_temp2t1'), - 'label-GM_mask': (gm_template2t1, 'outputspec.segment_mask_temp2t1'), - 'label-WM_mask': (wm_template2t1, 'outputspec.segment_mask_temp2t1') + "label-CSF_mask": (csf_template2t1, "outputspec.segment_mask_temp2t1"), + "label-GM_mask": (gm_template2t1, "outputspec.segment_mask_temp2t1"), + "label-WM_mask": (wm_template2t1, "outputspec.segment_mask_temp2t1"), } return (wf, outputs) @@ -698,46 +792,46 @@ def tissue_seg_T1_template_based(wf, cfg, strat_pool, pipe_num, opt=None): ], ) def tissue_seg_EPI_template_based(wf, cfg, strat_pool, pipe_num, opt=None): - xfm_prov = strat_pool.get_cpac_provenance( - 'from-EPItemplate_to-bold_mode-image_desc-linear_xfm') + "from-EPItemplate_to-bold_mode-image_desc-linear_xfm" + ) reg_tool = check_prov_for_regtool(xfm_prov) - use_ants = reg_tool == 'ants' - - csf_template2t1 = tissue_mask_template_to_t1('CSF', use_ants) - csf_template2t1.inputs.inputspec.tissue_mask_template = cfg[ - 'segmentation']['tissue_segmentation']['Template_Based']['CSF'] - - gm_template2t1 = tissue_mask_template_to_t1('GM', use_ants) - gm_template2t1.inputs.inputspec.tissue_mask_template = cfg[ - 'segmentation']['tissue_segmentation']['Template_Based']['GRAY'] - - wm_template2t1 = tissue_mask_template_to_t1('WM', use_ants) - wm_template2t1.inputs.inputspec.tissue_mask_template = cfg[ - 'segmentation']['tissue_segmentation']['Template_Based']['WHITE'] - - node, out = strat_pool.get_data('desc-mean_bold') - wf.connect(node, out, csf_template2t1, 'inputspec.brain') - wf.connect(node, out, gm_template2t1, 'inputspec.brain') - wf.connect(node, out, wm_template2t1, 'inputspec.brain') - - node, out = \ - strat_pool.get_data( - 'from-EPItemplate_to-bold_mode-image_desc-linear_xfm') - wf.connect(node, out, - csf_template2t1, 'inputspec.standard2highres_mat') - wf.connect(node, out, - wm_template2t1, 'inputspec.standard2highres_mat') - wf.connect(node, out, - gm_template2t1, 'inputspec.standard2highres_mat') + use_ants = reg_tool == "ants" + + csf_template2t1 = tissue_mask_template_to_t1("CSF", use_ants) + csf_template2t1.inputs.inputspec.tissue_mask_template = cfg["segmentation"][ + "tissue_segmentation" + ]["Template_Based"]["CSF"] + + gm_template2t1 = tissue_mask_template_to_t1("GM", use_ants) + gm_template2t1.inputs.inputspec.tissue_mask_template = cfg["segmentation"][ + "tissue_segmentation" + ]["Template_Based"]["GRAY"] + + wm_template2t1 = tissue_mask_template_to_t1("WM", use_ants) + wm_template2t1.inputs.inputspec.tissue_mask_template = cfg["segmentation"][ + "tissue_segmentation" + ]["Template_Based"]["WHITE"] + + node, out = strat_pool.get_data("desc-mean_bold") + wf.connect(node, out, csf_template2t1, "inputspec.brain") + wf.connect(node, out, gm_template2t1, "inputspec.brain") + wf.connect(node, out, wm_template2t1, "inputspec.brain") + + node, out = strat_pool.get_data( + "from-EPItemplate_to-bold_mode-image_desc-linear_xfm" + ) + wf.connect(node, out, csf_template2t1, "inputspec.standard2highres_mat") + wf.connect(node, out, wm_template2t1, "inputspec.standard2highres_mat") + wf.connect(node, out, gm_template2t1, "inputspec.standard2highres_mat") outputs = { - 'space-bold_label-CSF_mask': (csf_template2t1, - 'outputspec.segment_mask_temp2t1'), - 'space-bold_label-GM_mask': (gm_template2t1, - 'outputspec.segment_mask_temp2t1'), - 'space-bold_label-WM_mask': (wm_template2t1, - 'outputspec.segment_mask_temp2t1') + "space-bold_label-CSF_mask": ( + csf_template2t1, + "outputspec.segment_mask_temp2t1", + ), + "space-bold_label-GM_mask": (gm_template2t1, "outputspec.segment_mask_temp2t1"), + "space-bold_label-WM_mask": (wm_template2t1, "outputspec.segment_mask_temp2t1"), } return (wf, outputs) @@ -758,45 +852,43 @@ def tissue_seg_EPI_template_based(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["label-CSF_mask", "label-GM_mask", "label-WM_mask"], ) def tissue_seg_ants_prior(wf, cfg, strat_pool, pipe_num, opt=None): - - seg_preproc_ants_prior_based = \ - create_seg_preproc_antsJointLabel_method(wf_name=f'seg_preproc_' - f'ants_prior_' - f'{pipe_num}') - - seg_preproc_ants_prior_based.inputs.inputspec.template_brain_list = \ - cfg['segmentation']['tissue_segmentation']['ANTs_Prior_Based'][ - 'template_brain_list'] - seg_preproc_ants_prior_based.inputs.inputspec.template_segmentation_list = \ - cfg['segmentation']['tissue_segmentation']['ANTs_Prior_Based'][ - 'template_segmentation_list'] - - seg_preproc_ants_prior_based.inputs.inputspec.csf_label = cfg[ - 'segmentation']['tissue_segmentation']['ANTs_Prior_Based'][ - 'CSF_label'] - - seg_preproc_ants_prior_based.inputs.inputspec.gm_label = cfg[ - 'segmentation']['tissue_segmentation']['ANTs_Prior_Based'][ - 'GM_label'] - - seg_preproc_ants_prior_based.inputs.inputspec.wm_label = cfg[ - 'segmentation']['tissue_segmentation']['ANTs_Prior_Based'][ - 'WM_label'] - - node, out = strat_pool.get_data('desc-brain_T1w') - wf.connect(node, out, - seg_preproc_ants_prior_based, 'inputspec.anatomical_brain') - - node, out = strat_pool.get_data(['space-T1w_desc-brain_mask', - 'space-T1w_desc-acpcbrain_mask']) - wf.connect(node, out, seg_preproc_ants_prior_based, - 'inputspec.anatomical_brain_mask') + seg_preproc_ants_prior_based = create_seg_preproc_antsJointLabel_method( + wf_name=f"seg_preproc_" f"ants_prior_" f"{pipe_num}" + ) + + seg_preproc_ants_prior_based.inputs.inputspec.template_brain_list = cfg[ + "segmentation" + ]["tissue_segmentation"]["ANTs_Prior_Based"]["template_brain_list"] + seg_preproc_ants_prior_based.inputs.inputspec.template_segmentation_list = cfg[ + "segmentation" + ]["tissue_segmentation"]["ANTs_Prior_Based"]["template_segmentation_list"] + + seg_preproc_ants_prior_based.inputs.inputspec.csf_label = cfg["segmentation"][ + "tissue_segmentation" + ]["ANTs_Prior_Based"]["CSF_label"] + + seg_preproc_ants_prior_based.inputs.inputspec.gm_label = cfg["segmentation"][ + "tissue_segmentation" + ]["ANTs_Prior_Based"]["GM_label"] + + seg_preproc_ants_prior_based.inputs.inputspec.wm_label = cfg["segmentation"][ + "tissue_segmentation" + ]["ANTs_Prior_Based"]["WM_label"] + + node, out = strat_pool.get_data("desc-brain_T1w") + wf.connect(node, out, seg_preproc_ants_prior_based, "inputspec.anatomical_brain") + + node, out = strat_pool.get_data( + ["space-T1w_desc-brain_mask", "space-T1w_desc-acpcbrain_mask"] + ) + wf.connect( + node, out, seg_preproc_ants_prior_based, "inputspec.anatomical_brain_mask" + ) outputs = { - 'label-CSF_mask': ( - seg_preproc_ants_prior_based, 'outputspec.csf_mask'), - 'label-GM_mask': (seg_preproc_ants_prior_based, 'outputspec.gm_mask'), - 'label-WM_mask': (seg_preproc_ants_prior_based, 'outputspec.wm_mask') + "label-CSF_mask": (seg_preproc_ants_prior_based, "outputspec.csf_mask"), + "label-GM_mask": (seg_preproc_ants_prior_based, "outputspec.gm_mask"), + "label-WM_mask": (seg_preproc_ants_prior_based, "outputspec.wm_mask"), } return (wf, outputs) @@ -836,67 +928,72 @@ def tissue_seg_ants_prior(wf, cfg, strat_pool, pipe_num, opt=None): ], ) def tissue_seg_freesurfer(wf, cfg, strat_pool, pipe_num, opt=None): + node, out = strat_pool.get_data("freesurfer-subject-dir") - node, out = strat_pool.get_data('freesurfer-subject-dir') - - fs_aseg_to_native = pe.Node(interface=freesurfer.ApplyVolTransform(), - name=f'fs_aseg_to_native_{pipe_num}') + fs_aseg_to_native = pe.Node( + interface=freesurfer.ApplyVolTransform(), name=f"fs_aseg_to_native_{pipe_num}" + ) fs_aseg_to_native.inputs.reg_header = True - fs_aseg_to_native.inputs.interp = 'nearest' + fs_aseg_to_native.inputs.interp = "nearest" - wf.connect(node, out, fs_aseg_to_native, 'subjects_dir') + wf.connect(node, out, fs_aseg_to_native, "subjects_dir") - node, out = strat_pool.get_data('pipeline-fs_subcortical-seg') - wf.connect(node, out, fs_aseg_to_native, 'source_file') + node, out = strat_pool.get_data("pipeline-fs_subcortical-seg") + wf.connect(node, out, fs_aseg_to_native, "source_file") - node, out = strat_pool.get_data('pipeline-fs_raw-average') - wf.connect(node, out, fs_aseg_to_native, 'target_file') + node, out = strat_pool.get_data("pipeline-fs_raw-average") + wf.connect(node, out, fs_aseg_to_native, "target_file") - fs_aseg_to_nifti = pe.Node(util.Function(input_names=['in_file'], - output_names=['out_file'], - function=mri_convert), - name=f'fs_aseg_to_nifti_{pipe_num}') - fs_aseg_to_nifti.inputs.args = '-rt nearest' + fs_aseg_to_nifti = pe.Node( + util.Function( + input_names=["in_file"], output_names=["out_file"], function=mri_convert + ), + name=f"fs_aseg_to_nifti_{pipe_num}", + ) + fs_aseg_to_nifti.inputs.args = "-rt nearest" - wf.connect(fs_aseg_to_native, 'transformed_file', - fs_aseg_to_nifti, 'in_file') + wf.connect(fs_aseg_to_native, "transformed_file", fs_aseg_to_nifti, "in_file") - pick_tissue = pe.Node(pick_tissue_from_labels_file_interface(), - name=f'select_fs_tissue_{pipe_num}') + pick_tissue = pe.Node( + pick_tissue_from_labels_file_interface(), name=f"select_fs_tissue_{pipe_num}" + ) - pick_tissue.inputs.csf_label = cfg['segmentation'][ - 'tissue_segmentation']['FreeSurfer']['CSF_label'] - pick_tissue.inputs.gm_label = cfg['segmentation'][ - 'tissue_segmentation']['FreeSurfer']['GM_label'] - pick_tissue.inputs.wm_label = cfg['segmentation'][ - 'tissue_segmentation']['FreeSurfer']['WM_label'] + pick_tissue.inputs.csf_label = cfg["segmentation"]["tissue_segmentation"][ + "FreeSurfer" + ]["CSF_label"] + pick_tissue.inputs.gm_label = cfg["segmentation"]["tissue_segmentation"][ + "FreeSurfer" + ]["GM_label"] + pick_tissue.inputs.wm_label = cfg["segmentation"]["tissue_segmentation"][ + "FreeSurfer" + ]["WM_label"] - wf.connect(fs_aseg_to_nifti, 'out_file', pick_tissue, 'multiatlas_Labels') + wf.connect(fs_aseg_to_nifti, "out_file", pick_tissue, "multiatlas_Labels") erode_tissues = {} - if cfg['segmentation']['tissue_segmentation']['FreeSurfer']['erode'] > 0: - for tissue in ['csf', 'wm', 'gm']: + if cfg["segmentation"]["tissue_segmentation"]["FreeSurfer"]["erode"] > 0: + for tissue in ["csf", "wm", "gm"]: erode_tissues[tissue] = pe.Node( - interface=freesurfer.model.Binarize(), - name=f'erode_{tissue}_{pipe_num}') + interface=freesurfer.model.Binarize(), name=f"erode_{tissue}_{pipe_num}" + ) erode_tissues[tissue].inputs.match = [1] - erode_tissues[tissue].inputs.erode = cfg['segmentation'][ - 'tissue_segmentation']['FreeSurfer']['erode'] - wf.connect(pick_tissue, f'{tissue}_mask', erode_tissues[tissue], - 'in_file') + erode_tissues[tissue].inputs.erode = cfg["segmentation"][ + "tissue_segmentation" + ]["FreeSurfer"]["erode"] + wf.connect(pick_tissue, f"{tissue}_mask", erode_tissues[tissue], "in_file") if erode_tissues: outputs = { - 'label-CSF_mask': (erode_tissues['csf'], 'binary_file'), - 'label-WM_mask': (erode_tissues['wm'], 'binary_file'), - 'label-GM_mask': (erode_tissues['gm'], 'binary_file') - } + "label-CSF_mask": (erode_tissues["csf"], "binary_file"), + "label-WM_mask": (erode_tissues["wm"], "binary_file"), + "label-GM_mask": (erode_tissues["gm"], "binary_file"), + } else: outputs = { - 'label-CSF_mask': (pick_tissue, 'csf_mask'), - 'label-WM_mask': (pick_tissue, 'wm_mask'), - 'label-GM_mask': (pick_tissue, 'gm_mask') + "label-CSF_mask": (pick_tissue, "csf_mask"), + "label-WM_mask": (pick_tissue, "wm_mask"), + "label-GM_mask": (pick_tissue, "gm_mask"), } return (wf, outputs) diff --git a/CPAC/seg_preproc/utils.py b/CPAC/seg_preproc/utils.py index b594200b6a..578a088b4a 100644 --- a/CPAC/seg_preproc/utils.py +++ b/CPAC/seg_preproc/utils.py @@ -1,9 +1,9 @@ # Import packages import os -from CPAC.pipeline import nipype_pipeline_engine as pe -import scipy.ndimage as nd + import numpy as np -import nibabel as nb +import nibabel as nib +import scipy.ndimage as nd def check_if_file_is_empty(in_file): @@ -11,28 +11,29 @@ def check_if_file_is_empty(in_file): Parameters ---------- - in_file : nii file (string) regressor file Returns ------- - in_file : string return same file """ - import nibabel as nb import numpy as np + import nibabel as nb + nii = nb.load(in_file) data = nii.get_fdata() if data.size == 0 or np.all(data == 0) or np.all(data == np.nan): - raise ValueError('File {0} is empty. Use a lower threshold or turn ' - 'off regressors.'.format(in_file)) + raise ValueError( + "File {0} is empty. Use a lower threshold or turn " + "off regressors.".format(in_file) + ) return in_file def _erode(roi_mask, erosion_mm, erosion_prop): - """Function to perform in-common erosion steps + """Function to perform in-common erosion steps. Parameters ---------- @@ -56,11 +57,12 @@ def _erode(roi_mask, erosion_mm, erosion_prop): mask_data : numpy array eroded mask data """ - mask_img = nb.load(roi_mask) + mask_img = nib.load(roi_mask) mask_data = mask_img.get_fdata() orig_vol = np.sum(mask_data > 0) - erode = ((erosion_mm is not None and erosion_mm > 0) or - (erosion_prop is not None and 0 < erosion_prop < 1)) + erode = (erosion_mm is not None and erosion_mm > 0) or ( + erosion_prop is not None and 0 < erosion_prop < 1 + ) if erode: if erosion_mm: iter_n = max(int(erosion_mm / max(mask_img.header.get_zooms())), 1) @@ -73,17 +75,15 @@ def _erode(roi_mask, erosion_mm, erosion_prop): def pick_wm_prob_0(probability_maps): """Returns the csf probability map from the list of segmented - probability maps + probability maps. Parameters ---------- - probability_maps : list (string) List of Probability Maps Returns ------- - file : string Path to segment_prob_0.nii.gz is returned """ @@ -97,17 +97,15 @@ def pick_wm_prob_0(probability_maps): def pick_wm_prob_1(probability_maps): - """Returns the gray matter probability map from the list of segmented probability maps + """Returns the gray matter probability map from the list of segmented probability maps. Parameters ---------- - probability_maps : list (string) List of Probability Maps Returns ------- - file : string Path to segment_prob_1.nii.gz is returned """ @@ -121,17 +119,15 @@ def pick_wm_prob_1(probability_maps): def pick_wm_prob_2(probability_maps): - """Returns the white matter probability map from the list of segmented probability maps + """Returns the white matter probability map from the list of segmented probability maps. Parameters ---------- - probability_maps : list (string) List of Probability Maps Returns ------- - file : string Path to segment_prob_2.nii.gz is returned """ @@ -145,17 +141,15 @@ def pick_wm_prob_2(probability_maps): def pick_wm_class_0(tissue_class_files): - """Returns the csf tissue class file from the list of segmented tissue class files + """Returns the csf tissue class file from the list of segmented tissue class files. Parameters ---------- - tissue_class_files : list (string) List of tissue class files Returns ------- - file : string Path to segment_seg_0.nii.gz is returned """ @@ -169,17 +163,15 @@ def pick_wm_class_0(tissue_class_files): def pick_wm_class_1(tissue_class_files): - """Returns the gray matter tissue class file from the list of segmented tissue class files + """Returns the gray matter tissue class file from the list of segmented tissue class files. Parameters ---------- - tissue_class_files : list (string) List of tissue class files Returns ------- - file : string Path to segment_seg_1.nii.gz is returned """ @@ -193,17 +185,15 @@ def pick_wm_class_1(tissue_class_files): def pick_wm_class_2(tissue_class_files): - """Returns the white matter tissue class file from the list of segmented tissue class files + """Returns the white matter tissue class file from the list of segmented tissue class files. Parameters ---------- - tissue_class_files : list (string) List of tissue class files Returns ------- - file : string Path to segment_seg_2.nii.gz is returned """ @@ -216,9 +206,10 @@ def pick_wm_class_2(tissue_class_files): return None -def mask_erosion(roi_mask=None, skullstrip_mask=None, mask_erosion_mm=None, - mask_erosion_prop=None): - """Returns eroded segment mask and skull-stripped brain mask +def mask_erosion( + roi_mask=None, skullstrip_mask=None, mask_erosion_mm=None, mask_erosion_prop=None +): + """Returns eroded segment mask and skull-stripped brain mask. # This functionality is adapted from poldracklab/niworkflows: # https://github.com/poldracklab/niworkflows/blob/master/niworkflows/interfaces/utils.py @@ -228,7 +219,6 @@ def mask_erosion(roi_mask=None, skullstrip_mask=None, mask_erosion_mm=None, Parameters ---------- - roi_mask : string Path to binarized segment mask @@ -240,35 +230,34 @@ def mask_erosion(roi_mask=None, skullstrip_mask=None, mask_erosion_mm=None, Returns ------- - output_roi_mask : string Path to eroded segment mask eroded_skullstrip_mask : string Path to eroded skull-stripped brain mask """ - roi_mask_img = nb.load(roi_mask) + roi_mask_img = nib.load(roi_mask) roi_mask_data = roi_mask_img.get_fdata() skullstrip_mask_img, erode_in, skullstrip_mask_data = _erode( - skullstrip_mask, mask_erosion_mm, mask_erosion_prop) + skullstrip_mask, mask_erosion_mm, mask_erosion_prop + ) if erode_in: # pylint: disable=invalid-unary-operand-type roi_mask_data[~skullstrip_mask_data] = 0 hdr = roi_mask_img.header - output_roi_mask_img = nb.Nifti1Image(roi_mask_data, header=hdr, - affine=roi_mask_img.affine) - output_roi_mask = os.path.join(os.getcwd(), - 'segment_tissue_eroded_mask.nii.gz') + output_roi_mask_img = nib.Nifti1Image( + roi_mask_data, header=hdr, affine=roi_mask_img.affine + ) + output_roi_mask = os.path.join(os.getcwd(), "segment_tissue_eroded_mask.nii.gz") output_roi_mask_img.to_filename(output_roi_mask) hdr = skullstrip_mask_img.header - output_skullstrip_mask_img = nb.Nifti1Image( - skullstrip_mask_data, header=hdr, - affine=skullstrip_mask_img.affine) - eroded_skullstrip_mask = os.path.join(os.getcwd(), - 'eroded_skullstrip_mask.nii.gz') + output_skullstrip_mask_img = nib.Nifti1Image( + skullstrip_mask_data, header=hdr, affine=skullstrip_mask_img.affine + ) + eroded_skullstrip_mask = os.path.join(os.getcwd(), "eroded_skullstrip_mask.nii.gz") output_skullstrip_mask_img.to_filename(eroded_skullstrip_mask) @@ -281,7 +270,7 @@ def mask_erosion(roi_mask=None, skullstrip_mask=None, mask_erosion_mm=None, # https://poldracklab.stanford.edu/ # We are temporarily maintaining our own copy for more granular control. def erosion(roi_mask=None, erosion_mm=None, erosion_prop=None): - """Returns eroded tissue segment mask + """Returns eroded tissue segment mask. Parameters ---------- @@ -299,23 +288,24 @@ def erosion(roi_mask=None, erosion_mm=None, erosion_prop=None): roi_mask_img, _, roi_mask_data = _erode(roi_mask, erosion_mm, erosion_prop) hdr = roi_mask_img.header - output_img = nb.Nifti1Image(roi_mask_data, header=hdr, - affine=roi_mask_img.affine) - eroded_roi_mask = os.path.join(os.getcwd(), 'segment_tissue_mask.nii.gz') + output_img = nib.Nifti1Image(roi_mask_data, header=hdr, affine=roi_mask_img.affine) + eroded_roi_mask = os.path.join(os.getcwd(), "segment_tissue_mask.nii.gz") output_img.to_filename(eroded_roi_mask) return eroded_roi_mask -def hardcoded_antsJointLabelFusion(anatomical_brain, anatomical_brain_mask, - template_brain_list, - template_segmentation_list): - """run antsJointLabelFusion.sh +def hardcoded_antsJointLabelFusion( + anatomical_brain, + anatomical_brain_mask, + template_brain_list, + template_segmentation_list, +): + """Run antsJointLabelFusion.sh. Parameters ---------- - anatomical_brain : string (nifti file) Target image to be labeled. @@ -330,7 +320,6 @@ def hardcoded_antsJointLabelFusion(anatomical_brain, anatomical_brain_mask, Returns ------- - multiatlas_Intensity : string (nifti file) multiatlas_Labels : string (nifti file) @@ -341,45 +330,50 @@ def hardcoded_antsJointLabelFusion(anatomical_brain, anatomical_brain_mask, cmd = ["${ANTSPATH}${ANTSPATH:+/}antsJointLabelFusion.sh"] cmd.append( " -d 3 -o ants_multiatlas_ -t {0} -x {1} -y b -c 0".format( - anatomical_brain, anatomical_brain_mask)) - - if (not len(template_brain_list) == len(template_segmentation_list)): - err_msg = '\n\n[!] C-PAC says: Please check ANTs Prior-based ' \ - 'Segmentation setting. For performing ANTs Prior-based ' \ - 'segmentation method the number of specified ' \ - 'segmentations should be identical to the number of atlas ' \ - 'image sets.\n\n' + anatomical_brain, anatomical_brain_mask + ) + ) + + if not len(template_brain_list) == len(template_segmentation_list): + err_msg = ( + "\n\n[!] C-PAC says: Please check ANTs Prior-based " + "Segmentation setting. For performing ANTs Prior-based " + "segmentation method the number of specified " + "segmentations should be identical to the number of atlas " + "image sets.\n\n" + ) raise Exception(err_msg) else: for index in range(len(template_brain_list)): cmd.append( " -g {0} -l {1}".format( - template_brain_list[index], - template_segmentation_list[index])) + template_brain_list[index], template_segmentation_list[index] + ) + ) # write out the actual command-line entry for testing/validation later - command_file = os.path.join(os.getcwd(), 'command.txt') - with open(command_file, 'wt') as f: - f.write(' '.join(cmd)) + command_file = os.path.join(os.getcwd(), "command.txt") + with open(command_file, "wt") as f: + f.write(" ".join(cmd)) str = "" bash_cmd = str.join(cmd) try: - retcode = subprocess.check_output(bash_cmd, shell=True) \ - # noqa: F841 # pylint: disable=unused-variable + subprocess.check_output(bash_cmd, shell=True) + # pylint: disable=unused-variable except Exception as e: # pylint: disable=broad-except,invalid-name # pylint: disable=raise-missing-from - raise Exception('[!] antsJointLabel segmentation method did not ' - 'complete successfully.\n\nError ' - 'details:\n{0}\n{1}\n'.format( - e, - getattr(e, 'output', ''))) + raise Exception( + "[!] antsJointLabel segmentation method did not " + "complete successfully.\n\nError " + "details:\n{0}\n{1}\n".format(e, getattr(e, "output", "")) + ) multiatlas_Intensity = None multiatlas_Labels = None - files = [f for f in os.listdir('.') if os.path.isfile(f)] + files = [f for f in os.listdir(".") if os.path.isfile(f)] for f in files: if "Intensity" in f: @@ -388,18 +382,21 @@ def hardcoded_antsJointLabelFusion(anatomical_brain, anatomical_brain_mask, multiatlas_Labels = os.getcwd() + "/" + f if not multiatlas_Labels: - raise Exception("\n\n[!] No multiatlas labels file found. " - "antsJointLabelFusion may not have completed " - "successfully.\n\n") + raise Exception( + "\n\n[!] No multiatlas labels file found. " + "antsJointLabelFusion may not have completed " + "successfully.\n\n" + ) return multiatlas_Intensity, multiatlas_Labels -def pick_tissue_from_labels_file(multiatlas_Labels, csf_label=[4,14,15,24,43], - gm_label=[3,42], wm_label=[2,41]): +def pick_tissue_from_labels_file( + multiatlas_Labels, csf_label=[4, 14, 15, 24, 43], gm_label=[3, 42], wm_label=[2, 41] +): """Pick tissue mask from multiatlas labels file based off of FreeSurferColorLUT https://surfer.nmr.mgh.harvard.edu/fswiki/FsTutorial/AnatomicalROI/FreeSurferColorLUT - or user provided label value + or user provided label value. Parameters ---------- @@ -424,10 +421,11 @@ def pick_tissue_from_labels_file(multiatlas_Labels, csf_label=[4,14,15,24,43], """ # pylint: disable=import-outside-toplevel,redefined-outer-name,reimported import os - import nibabel as nb + import numpy as np + import nibabel as nib - img = nb.load(multiatlas_Labels) + img = nib.load(multiatlas_Labels) data = img.get_fdata() # pick tissue mask from multiatlas labels file @@ -449,16 +447,16 @@ def pick_tissue_from_labels_file(multiatlas_Labels, csf_label=[4,14,15,24,43], wm[np.where(np.in1d(data, np.array(wm_label)))] = 1 wm = wm.reshape(data.shape) - save_img_csf = nb.Nifti1Image(csf, header=img.header, affine=img.affine) - save_img_gm = nb.Nifti1Image(gm, header=img.header, affine=img.affine) - save_img_wm = nb.Nifti1Image(wm, header=img.header, affine=img.affine) + save_img_csf = nib.Nifti1Image(csf, header=img.header, affine=img.affine) + save_img_gm = nib.Nifti1Image(gm, header=img.header, affine=img.affine) + save_img_wm = nib.Nifti1Image(wm, header=img.header, affine=img.affine) - save_img_csf.to_filename('csf_mask.nii.gz') - save_img_gm.to_filename('gm_mask.nii.gz') - save_img_wm.to_filename('wm_mask.nii.gz') + save_img_csf.to_filename("csf_mask.nii.gz") + save_img_gm.to_filename("gm_mask.nii.gz") + save_img_wm.to_filename("wm_mask.nii.gz") - csf_mask = os.path.join(os.getcwd(), 'csf_mask.nii.gz') - gm_mask = os.path.join(os.getcwd(), 'gm_mask.nii.gz') - wm_mask = os.path.join(os.getcwd(), 'wm_mask.nii.gz') + csf_mask = os.path.join(os.getcwd(), "csf_mask.nii.gz") + gm_mask = os.path.join(os.getcwd(), "gm_mask.nii.gz") + wm_mask = os.path.join(os.getcwd(), "wm_mask.nii.gz") return csf_mask, gm_mask, wm_mask diff --git a/CPAC/surface/PostFreeSurfer/FreeSurfer2CaretConvertAndRegisterNonlinear.sh b/CPAC/surface/PostFreeSurfer/FreeSurfer2CaretConvertAndRegisterNonlinear.sh index 1f53e25bdf..a95a8e5f48 100644 --- a/CPAC/surface/PostFreeSurfer/FreeSurfer2CaretConvertAndRegisterNonlinear.sh +++ b/CPAC/surface/PostFreeSurfer/FreeSurfer2CaretConvertAndRegisterNonlinear.sh @@ -173,7 +173,7 @@ cd ${StudyFolder} #Convert FreeSurfer Volumes for Image in wmparc aparc.a2009s+aseg aparc+aseg ; do if [ -e "$FreeSurferFolder"/mri/"$Image".mgz ] ; then - + mri_convert -rt nearest -rl "$T1wFolder"/"$T1wImage".nii.gz "$FreeSurferFolder"/mri/"$Image".mgz "$T1wFolder"/"$Image"_1mm.nii.gz applywarp --rel --interp=nn -i "$T1wFolder"/"$Image"_1mm.nii.gz -r "$AtlasSpaceFolder"/"$AtlasSpaceT1wImage" --premat=$FSLDIR/etc/flirtsch/ident.mat -o "$T1wFolder"/"$Image".nii.gz applywarp --rel --interp=nn -i "$T1wFolder"/"$Image"_1mm.nii.gz -r "$AtlasSpaceFolder"/"$AtlasSpaceT1wImage" -w "$AtlasTransform" -o "$AtlasSpaceFolder"/"$Image".nii.gz @@ -229,7 +229,7 @@ for GrayordinatesResolution in ${GrayordinatesResolutions} ; do applywarp --interp=spline -i "$AtlasSpaceFolder"/"$AtlasSpaceT2wImage".nii.gz -r "$AtlasSpaceFolder"/ROIs/Atlas_ROIs."$GrayordinatesResolution".nii.gz -o "$AtlasSpaceFolder"/"$AtlasSpaceT2wImage"."$GrayordinatesResolution".nii.gz fi applywarp --interp=spline -i "$AtlasSpaceFolder"/"$AtlasSpaceT1wImage".nii.gz -r "$AtlasSpaceFolder"/ROIs/Atlas_ROIs."$GrayordinatesResolution".nii.gz -o "$AtlasSpaceFolder"/"$AtlasSpaceT1wImage"."$GrayordinatesResolution".nii.gz -done +done #Loop through left and right hemispheres for Hemisphere in L R ; do @@ -349,7 +349,7 @@ for Hemisphere in L R ; do wb_command -metric-merge "$AtlasSpaceFolder"/"$NativeFolder"/"$Subject"."$Hemisphere".StrainJ_FS.native.shape.gii -metric "$AtlasSpaceFolder"/"$NativeFolder"/"$Subject"."$Hemisphere".Strain_FS.native.shape.gii -column 1 wb_command -metric-merge "$AtlasSpaceFolder"/"$NativeFolder"/"$Subject"."$Hemisphere".StrainR_FS.native.shape.gii -metric "$AtlasSpaceFolder"/"$NativeFolder"/"$Subject"."$Hemisphere".Strain_FS.native.shape.gii -column 2 wb_command -metric-math "ln(var) / ln (2)" "$AtlasSpaceFolder"/"$NativeFolder"/"$Subject"."$Hemisphere".StrainJ_FS.native.shape.gii -var var "$AtlasSpaceFolder"/"$NativeFolder"/"$Subject"."$Hemisphere".StrainJ_FS.native.shape.gii - wb_command -metric-math "ln(var) / ln (2)" "$AtlasSpaceFolder"/"$NativeFolder"/"$Subject"."$Hemisphere".StrainR_FS.native.shape.gii -var var "$AtlasSpaceFolder"/"$NativeFolder"/"$Subject"."$Hemisphere".StrainR_FS.native.shape.gii + wb_command -metric-math "ln(var) / ln (2)" "$AtlasSpaceFolder"/"$NativeFolder"/"$Subject"."$Hemisphere".StrainR_FS.native.shape.gii -var var "$AtlasSpaceFolder"/"$NativeFolder"/"$Subject"."$Hemisphere".StrainR_FS.native.shape.gii rm "$AtlasSpaceFolder"/"$NativeFolder"/"$Subject"."$Hemisphere".Strain_FS.native.shape.gii #If desired, run MSMSulc folding-based registration to FS_LR initialized with FS affine @@ -389,7 +389,7 @@ for Hemisphere in L R ; do wb_command -metric-merge "$AtlasSpaceFolder"/"$NativeFolder"/"$Subject"."$Hemisphere".StrainJ_MSMSulc.native.shape.gii -metric "$AtlasSpaceFolder"/"$NativeFolder"/"$Subject"."$Hemisphere".Strain_MSMSulc.native.shape.gii -column 1 wb_command -metric-merge "$AtlasSpaceFolder"/"$NativeFolder"/"$Subject"."$Hemisphere".StrainR_MSMSulc.native.shape.gii -metric "$AtlasSpaceFolder"/"$NativeFolder"/"$Subject"."$Hemisphere".Strain_MSMSulc.native.shape.gii -column 2 wb_command -metric-math "ln(var) / ln (2)" "$AtlasSpaceFolder"/"$NativeFolder"/"$Subject"."$Hemisphere".StrainJ_MSMSulc.native.shape.gii -var var "$AtlasSpaceFolder"/"$NativeFolder"/"$Subject"."$Hemisphere".StrainJ_MSMSulc.native.shape.gii - wb_command -metric-math "ln(var) / ln (2)" "$AtlasSpaceFolder"/"$NativeFolder"/"$Subject"."$Hemisphere".StrainR_MSMSulc.native.shape.gii -var var "$AtlasSpaceFolder"/"$NativeFolder"/"$Subject"."$Hemisphere".StrainR_MSMSulc.native.shape.gii + wb_command -metric-math "ln(var) / ln (2)" "$AtlasSpaceFolder"/"$NativeFolder"/"$Subject"."$Hemisphere".StrainR_MSMSulc.native.shape.gii -var var "$AtlasSpaceFolder"/"$NativeFolder"/"$Subject"."$Hemisphere".StrainR_MSMSulc.native.shape.gii rm "$AtlasSpaceFolder"/"$NativeFolder"/"$Subject"."$Hemisphere".Strain_MSMSulc.native.shape.gii RegSphere="${AtlasSpaceFolder}/${NativeFolder}/${Subject}.${Hemisphere}.sphere.MSMSulc.native.surf.gii" @@ -611,7 +611,7 @@ for LowResMesh in ${LowResMeshes} ; do # roi_left - path to file of ROI vertices to use from left surface # right_metric - path to right hemisphere VA metric file # roi_right - path to file of ROI vertices to use from right surface - + left_metric=${DownSampleT1wFolder}/${Subject}.L.midthickness_va.${LowResMesh}k_fs_LR.shape.gii roi_left=${DownSampleFolder}/${Subject}.L.atlasroi.${LowResMesh}k_fs_LR.shape.gii right_metric=${DownSampleT1wFolder}/${Subject}.R.midthickness_va.${LowResMesh}k_fs_LR.shape.gii @@ -622,7 +622,7 @@ for LowResMesh in ${LowResMeshes} ; do -roi-left ${roi_left} \ -right-metric ${right_metric} \ -roi-right ${roi_right} - + # VAMean - mean of surface area accounted for for each vertex - used for normalization VAMean=$(wb_command -cifti-stats ${midthickness_va_file} -reduce MEAN) echo "VAMean: ${VAMean}" @@ -636,5 +636,3 @@ done echo "Done creating midthickness Vertex Area (VA) maps" echo "END" - - diff --git a/CPAC/surface/PostFreeSurfer/run.sh b/CPAC/surface/PostFreeSurfer/run.sh index 5123114c86..f146eff24d 100644 --- a/CPAC/surface/PostFreeSurfer/run.sh +++ b/CPAC/surface/PostFreeSurfer/run.sh @@ -31,7 +31,7 @@ InflateExtraScale=1 T1wImage="T1w_acpc_dc" T1wFolder="T1w" #Location of T1w images T2wFolder="T2w" #Location of T1w images -T2wImage="T2w_acpc_dc" +T2wImage="T2w_acpc_dc" AtlasSpaceFolder="MNINonLinear" NativeFolder="Native" FreeSurferInput="T1w_acpc_dc_restore_1mm" @@ -69,8 +69,8 @@ OutputOrigT2wToStandard="OrigT2w2standard.nii.gz" BiasFieldOutput="BiasField" Jacobian="NonlinearRegJacobians.nii.gz" -T1wFolder="$StudyFolder"/"$T1wFolder" -T2wFolder="$StudyFolder"/"$T2wFolder" +T1wFolder="$StudyFolder"/"$T1wFolder" +T2wFolder="$StudyFolder"/"$T2wFolder" AtlasSpaceFolder="$StudyFolder"/"$AtlasSpaceFolder" AtlasTransform="$AtlasSpaceFolder"/xfms/"$AtlasTransform" InverseAtlasTransform="$AtlasSpaceFolder"/xfms/"$InverseAtlasTransform" @@ -108,4 +108,4 @@ fi echo "$StudyFolder" "$Subject" "$T1wFolder" "$AtlasSpaceFolder" "$NativeFolder" "$FreeSurferFolder" "$FreeSurferInput" "$T1wRestoreImage" "$T2wRestoreImage" "$SurfaceAtlasDIR" "$HighResMesh" "$LowResMeshes" "$AtlasTransform" "$InverseAtlasTransform" "$AtlasSpaceT1wImage" "$AtlasSpaceT2wImage" "$T1wImageBrainMask" "$FreeSurferLabels" "$GrayordinatesSpaceDIR" "$GrayordinatesResolutions" "$SubcorticalGrayLabels" "$RegName" "$InflateExtraScale" "$useT2" -bash /code/CPAC/surface/PostFreeSurfer/FreeSurfer2CaretConvertAndRegisterNonlinear.sh "$StudyFolder" "$Subject" "$T1wFolder" "$AtlasSpaceFolder" "$NativeFolder" "$FreeSurferFolder" "$FreeSurferInput" "$T1wRestoreImage" "$T2wRestoreImage" "$SurfaceAtlasDIR" "$HighResMesh" "$LowResMeshes" "$AtlasTransform" "$InverseAtlasTransform" "$AtlasSpaceT1wImage" "$AtlasSpaceT2wImage" "$T1wImageBrainMask" "$FreeSurferLabels" "$GrayordinatesSpaceDIR" "$GrayordinatesResolutions" "$SubcorticalGrayLabels" "$RegName" "$InflateExtraScale" "$useT2" \ No newline at end of file +bash /code/CPAC/surface/PostFreeSurfer/FreeSurfer2CaretConvertAndRegisterNonlinear.sh "$StudyFolder" "$Subject" "$T1wFolder" "$AtlasSpaceFolder" "$NativeFolder" "$FreeSurferFolder" "$FreeSurferInput" "$T1wRestoreImage" "$T2wRestoreImage" "$SurfaceAtlasDIR" "$HighResMesh" "$LowResMeshes" "$AtlasTransform" "$InverseAtlasTransform" "$AtlasSpaceT1wImage" "$AtlasSpaceT2wImage" "$T1wImageBrainMask" "$FreeSurferLabels" "$GrayordinatesSpaceDIR" "$GrayordinatesResolutions" "$SubcorticalGrayLabels" "$RegName" "$InflateExtraScale" "$useT2" diff --git a/CPAC/surface/fMRISurface/CreateDenseTimeseries.sh b/CPAC/surface/fMRISurface/CreateDenseTimeseries.sh index b78de736c0..1c4a078f8b 100644 --- a/CPAC/surface/fMRISurface/CreateDenseTimeseries.sh +++ b/CPAC/surface/fMRISurface/CreateDenseTimeseries.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash set -e echo -e "\n START: CreateDenseTimeSeries" diff --git a/CPAC/surface/fMRISurface/RibbonVolumeToSurfaceMapping.sh b/CPAC/surface/fMRISurface/RibbonVolumeToSurfaceMapping.sh index ea3f47070e..99b5064c48 100644 --- a/CPAC/surface/fMRISurface/RibbonVolumeToSurfaceMapping.sh +++ b/CPAC/surface/fMRISurface/RibbonVolumeToSurfaceMapping.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash set -e echo -e "\n START: RibbonVolumeToSurfaceMapping" @@ -90,4 +90,3 @@ for Hemisphere in L R ; do done echo " END: RibbonVolumeToSurfaceMapping" - diff --git a/CPAC/surface/fMRISurface/SubcorticalProcessing.sh b/CPAC/surface/fMRISurface/SubcorticalProcessing.sh index c0ac90d704..a90a4dc8b1 100644 --- a/CPAC/surface/fMRISurface/SubcorticalProcessing.sh +++ b/CPAC/surface/fMRISurface/SubcorticalProcessing.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash set -e script_name="SubcorticalProcessing.sh" echo "${script_name}: START" diff --git a/CPAC/surface/fMRISurface/SurfaceSmoothing.sh b/CPAC/surface/fMRISurface/SurfaceSmoothing.sh index 7994c98471..36af8489ae 100644 --- a/CPAC/surface/fMRISurface/SurfaceSmoothing.sh +++ b/CPAC/surface/fMRISurface/SurfaceSmoothing.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash set -e echo -e "\n START: SurfaceSmoothing" @@ -16,4 +16,3 @@ for Hemisphere in L R ; do done echo " END: SurfaceSmoothing" - diff --git a/CPAC/surface/fMRISurface/run.sh b/CPAC/surface/fMRISurface/run.sh index b3837b97b0..27d020e8db 100755 --- a/CPAC/surface/fMRISurface/run.sh +++ b/CPAC/surface/fMRISurface/run.sh @@ -58,4 +58,4 @@ bash /code/CPAC/surface/fMRISurface/SubcorticalProcessing.sh "$AtlasSpaceFolder" #Generation of Dense Timeseries echo "Generation of Dense Timeseries" -bash /code/CPAC/surface/fMRISurface/CreateDenseTimeseries.sh "$AtlasSpaceFolder"/"$DownSampleFolder" "$Subject" "$LowResMesh" "$ResultsFolder"/"$NameOffMRI" "$SmoothingFWHM" "$ROIFolder" "$ResultsFolder"/"$OutputAtlasDenseTimeseries" "$GrayordinatesResolution" \ No newline at end of file +bash /code/CPAC/surface/fMRISurface/CreateDenseTimeseries.sh "$AtlasSpaceFolder"/"$DownSampleFolder" "$Subject" "$LowResMesh" "$ResultsFolder"/"$NameOffMRI" "$SmoothingFWHM" "$ROIFolder" "$ResultsFolder"/"$OutputAtlasDenseTimeseries" "$GrayordinatesResolution" diff --git a/CPAC/surface/surf_preproc.py b/CPAC/surface/surf_preproc.py index a540846041..c716e90f67 100644 --- a/CPAC/surface/surf_preproc.py +++ b/CPAC/surface/surf_preproc.py @@ -1,28 +1,31 @@ import os -from CPAC.pipeline.nodeblock import nodeblock + import nipype.interfaces.utility as util -from CPAC.utils.interfaces.function import Function + from CPAC.pipeline import nipype_pipeline_engine as pe +from CPAC.pipeline.nodeblock import nodeblock -def run_surface(post_freesurfer_folder, - freesurfer_folder, - subject, - t1w_restore_image, - atlas_space_t1w_image, - atlas_transform, - inverse_atlas_transform, - atlas_space_bold, - scout_bold, - surf_atlas_dir, - gray_ordinates_dir, - gray_ordinates_res, - high_res_mesh, - low_res_mesh, - subcortical_gray_labels, - freesurfer_labels, - fmri_res, - smooth_fwhm): +def run_surface( + post_freesurfer_folder, + freesurfer_folder, + subject, + t1w_restore_image, + atlas_space_t1w_image, + atlas_transform, + inverse_atlas_transform, + atlas_space_bold, + scout_bold, + surf_atlas_dir, + gray_ordinates_dir, + gray_ordinates_res, + high_res_mesh, + low_res_mesh, + subcortical_gray_labels, + freesurfer_labels, + fmri_res, + smooth_fwhm, +): """ Returns ------- @@ -35,147 +38,255 @@ def run_surface(post_freesurfer_folder, destrieux : str Path to the Destrieux parcellation file. """ - - from CPAC.utils.monitoring.custom_logging import log_subprocess import os - - recon_all_path = os.path.join(freesurfer_folder, 'recon_all') - + + from CPAC.utils.monitoring.custom_logging import log_subprocess + + recon_all_path = os.path.join(freesurfer_folder, "recon_all") + if os.path.isdir(recon_all_path): freesurfer_folder = recon_all_path # DCAN-HCP PostFreeSurfer # Ref: https://github.com/DCAN-Labs/DCAN-HCP/blob/master/PostFreeSurfer/PostFreeSurferPipeline.sh - cmd = ['bash', '/code/CPAC/surface/PostFreeSurfer/run.sh', '--post_freesurfer_folder', post_freesurfer_folder, \ - '--freesurfer_folder', freesurfer_folder, '--subject', subject, \ - '--t1w_restore', t1w_restore_image, '--atlas_t1w', atlas_space_t1w_image, \ - '--atlas_transform', atlas_transform, '--inverse_atlas_transform', inverse_atlas_transform, \ - '--surfatlasdir', surf_atlas_dir, '--grayordinatesdir', gray_ordinates_dir, '--grayordinatesres', gray_ordinates_res, \ - '--hiresmesh', high_res_mesh, '--lowresmesh', low_res_mesh, \ - '--subcortgraylabels', subcortical_gray_labels, '--freesurferlabels', freesurfer_labels] + cmd = [ + "bash", + "/code/CPAC/surface/PostFreeSurfer/run.sh", + "--post_freesurfer_folder", + post_freesurfer_folder, + "--freesurfer_folder", + freesurfer_folder, + "--subject", + subject, + "--t1w_restore", + t1w_restore_image, + "--atlas_t1w", + atlas_space_t1w_image, + "--atlas_transform", + atlas_transform, + "--inverse_atlas_transform", + inverse_atlas_transform, + "--surfatlasdir", + surf_atlas_dir, + "--grayordinatesdir", + gray_ordinates_dir, + "--grayordinatesres", + gray_ordinates_res, + "--hiresmesh", + high_res_mesh, + "--lowresmesh", + low_res_mesh, + "--subcortgraylabels", + subcortical_gray_labels, + "--freesurferlabels", + freesurfer_labels, + ] log_subprocess(cmd) # DCAN-HCP fMRISurface # https://github.com/DCAN-Labs/DCAN-HCP/blob/master/fMRISurface/GenericfMRISurfaceProcessingPipeline.sh - cmd = ['bash', '/code/CPAC/surface/fMRISurface/run.sh', - '--post_freesurfer_folder', post_freesurfer_folder, - '--subject', subject, '--fmri', atlas_space_bold, '--scout', - scout_bold, '--lowresmesh', low_res_mesh, '--grayordinatesres', - gray_ordinates_res, '--fmrires', fmri_res, '--smoothingFWHM', - smooth_fwhm] + cmd = [ + "bash", + "/code/CPAC/surface/fMRISurface/run.sh", + "--post_freesurfer_folder", + post_freesurfer_folder, + "--subject", + subject, + "--fmri", + atlas_space_bold, + "--scout", + scout_bold, + "--lowresmesh", + low_res_mesh, + "--grayordinatesres", + gray_ordinates_res, + "--fmrires", + fmri_res, + "--smoothingFWHM", + smooth_fwhm, + ] log_subprocess(cmd) - dtseries = os.path.join(post_freesurfer_folder, - 'MNINonLinear/Results/task-rest01/' - 'task-rest01_Atlas.dtseries.nii') - aparc = {'desikan_killiany': { - 164: os.path.join(post_freesurfer_folder, 'MNINonLinear', - f'{subject}.aparc.164k_fs_LR.dlabel.nii'), - 32: os.path.join(post_freesurfer_folder, 'MNINonLinear', - 'fsaverage_LR32k', - f'{subject}.aparc.32k_fs_LR.dlabel.nii')}, - 'destrieux': { - 164: os.path.join(post_freesurfer_folder, 'MNINonLinear', - f'{subject}.aparc.a2009s.164k_fs_LR.dlabel.nii'), - 32: os.path.join(post_freesurfer_folder, 'MNINonLinear', - 'fsaverage_LR32k', - f'{subject}.aparc.a2009s.32k_fs_LR.dlabel.nii')}} - - return (dtseries, aparc['desikan_killiany'][164], aparc['destrieux'][164], - aparc['desikan_killiany'][32], aparc['destrieux'][32]) + dtseries = os.path.join( + post_freesurfer_folder, + "MNINonLinear/Results/task-rest01/" "task-rest01_Atlas.dtseries.nii", + ) + aparc = { + "desikan_killiany": { + 164: os.path.join( + post_freesurfer_folder, + "MNINonLinear", + f"{subject}.aparc.164k_fs_LR.dlabel.nii", + ), + 32: os.path.join( + post_freesurfer_folder, + "MNINonLinear", + "fsaverage_LR32k", + f"{subject}.aparc.32k_fs_LR.dlabel.nii", + ), + }, + "destrieux": { + 164: os.path.join( + post_freesurfer_folder, + "MNINonLinear", + f"{subject}.aparc.a2009s.164k_fs_LR.dlabel.nii", + ), + 32: os.path.join( + post_freesurfer_folder, + "MNINonLinear", + "fsaverage_LR32k", + f"{subject}.aparc.a2009s.32k_fs_LR.dlabel.nii", + ), + }, + } + + return ( + dtseries, + aparc["desikan_killiany"][164], + aparc["destrieux"][164], + aparc["desikan_killiany"][32], + aparc["destrieux"][32], + ) def surface_connector(wf, cfg, strat_pool, pipe_num, opt): + surf = pe.Node( + util.Function( + input_names=[ + "post_freesurfer_folder", + "freesurfer_folder", + "subject", + "t1w_restore_image", + "atlas_space_t1w_image", + "atlas_transform", + "inverse_atlas_transform", + "atlas_space_bold", + "scout_bold", + "surf_atlas_dir", + "gray_ordinates_dir", + "gray_ordinates_res", + "high_res_mesh", + "low_res_mesh", + "subcortical_gray_labels", + "freesurfer_labels", + "fmri_res", + "smooth_fwhm", + ], + output_names=[ + "dtseries", + "desikan_killiany_164", + "destrieux_164", + "desikan_killiany_32", + "destrieux_32", + ], + function=run_surface, + ), + name=f"post_freesurfer_{pipe_num}", + ) + + surf.inputs.subject = cfg["subject_id"] + + surf.inputs.post_freesurfer_folder = os.path.join( + cfg.pipeline_setup["working_directory"]["path"], + "cpac_" + cfg["subject_id"], + f"post_freesurfer_{pipe_num}", + ) - surf = pe.Node(util.Function(input_names=['post_freesurfer_folder', - 'freesurfer_folder', - 'subject', - 't1w_restore_image', - 'atlas_space_t1w_image', - 'atlas_transform', - 'inverse_atlas_transform', - 'atlas_space_bold', - 'scout_bold', - 'surf_atlas_dir', - 'gray_ordinates_dir', - 'gray_ordinates_res', - 'high_res_mesh', - 'low_res_mesh', - 'subcortical_gray_labels', - 'freesurfer_labels', - 'fmri_res', - 'smooth_fwhm'], - output_names=['dtseries', - 'desikan_killiany_164', - 'destrieux_164', - 'desikan_killiany_32', - 'destrieux_32'], - function=run_surface), - name=f'post_freesurfer_{pipe_num}') - - - surf.inputs.subject = cfg['subject_id'] - - surf.inputs.post_freesurfer_folder = os.path.join(cfg.pipeline_setup['working_directory']['path'], - 'cpac_'+cfg['subject_id'], - f'post_freesurfer_{pipe_num}') - - surf.inputs.surf_atlas_dir = cfg.surface_analysis['post_freesurfer']['surf_atlas_dir'] - surf.inputs.gray_ordinates_dir = cfg.surface_analysis['post_freesurfer']['gray_ordinates_dir'] - surf.inputs.subcortical_gray_labels = cfg.surface_analysis['post_freesurfer']['subcortical_gray_labels'] - surf.inputs.freesurfer_labels = cfg.surface_analysis['post_freesurfer']['freesurfer_labels'] + surf.inputs.surf_atlas_dir = cfg.surface_analysis["post_freesurfer"][ + "surf_atlas_dir" + ] + surf.inputs.gray_ordinates_dir = cfg.surface_analysis["post_freesurfer"][ + "gray_ordinates_dir" + ] + surf.inputs.subcortical_gray_labels = cfg.surface_analysis["post_freesurfer"][ + "subcortical_gray_labels" + ] + surf.inputs.freesurfer_labels = cfg.surface_analysis["post_freesurfer"][ + "freesurfer_labels" + ] # convert integers to strings as subprocess requires string inputs - surf.inputs.gray_ordinates_res = str(cfg.surface_analysis['post_freesurfer']['gray_ordinates_res']) - surf.inputs.high_res_mesh = str(cfg.surface_analysis['post_freesurfer']['high_res_mesh']) - surf.inputs.low_res_mesh = str(cfg.surface_analysis['post_freesurfer']['low_res_mesh']) - surf.inputs.fmri_res = str(cfg.surface_analysis['post_freesurfer']['fmri_res']) - surf.inputs.smooth_fwhm = str(cfg.surface_analysis['post_freesurfer']['smooth_fwhm']) - - restore = ["pipeline-fs_desc-restore_T1w", "desc-preproc_T1w", "desc-reorient_T1w", "T1w", - "space-longitudinal_desc-reorient_T1w"] - space_temp = ["space-template_desc-head_T1w", "space-template_desc-brain_T1w", "space-template_desc-T1w_mask"] - atlas_xfm = ["from-T1w_to-template_mode-image_xfm", "from-T1w_to-template_mode-image_desc-linear_xfm"] - atlas_xfm_inv = ["from-template_to-T1w_mode-image_xfm", "from-template_to-T1w_mode-image_desc-linear_xfm"] - atlas_space_bold = ["space-template_desc-brain_bold", "space-template_desc-preproc_bold"] - scout_bold = ["space-template_desc-scout_bold", "space-template_desc-cleaned_bold", "space-template_desc-brain_bold", - "space-template_desc-preproc_bold", "space-template_desc-motion_bold", "space-template_bold"] - - - node, out = strat_pool.get_data('freesurfer-subject-dir') - wf.connect(node, out, surf, 'freesurfer_folder') - - - node, out = strat_pool.get_data(restore) - wf.connect(node, out, surf, 't1w_restore_image') - - - node, out = strat_pool.get_data(space_temp) - wf.connect(node, out, surf, 'atlas_space_t1w_image') - - node, out = strat_pool.get_data(atlas_xfm) - wf.connect(node, out, surf, 'atlas_transform') - - node, out = strat_pool.get_data(atlas_xfm_inv) - wf.connect(node, out, surf, 'inverse_atlas_transform') - - node, out = strat_pool.get_data(atlas_space_bold) - wf.connect(node, out, surf, 'atlas_space_bold') + surf.inputs.gray_ordinates_res = str( + cfg.surface_analysis["post_freesurfer"]["gray_ordinates_res"] + ) + surf.inputs.high_res_mesh = str( + cfg.surface_analysis["post_freesurfer"]["high_res_mesh"] + ) + surf.inputs.low_res_mesh = str( + cfg.surface_analysis["post_freesurfer"]["low_res_mesh"] + ) + surf.inputs.fmri_res = str(cfg.surface_analysis["post_freesurfer"]["fmri_res"]) + surf.inputs.smooth_fwhm = str( + cfg.surface_analysis["post_freesurfer"]["smooth_fwhm"] + ) + + restore = [ + "pipeline-fs_desc-restore_T1w", + "desc-preproc_T1w", + "desc-reorient_T1w", + "T1w", + "space-longitudinal_desc-reorient_T1w", + ] + space_temp = [ + "space-template_desc-head_T1w", + "space-template_desc-brain_T1w", + "space-template_desc-T1w_mask", + ] + atlas_xfm = [ + "from-T1w_to-template_mode-image_xfm", + "from-T1w_to-template_mode-image_desc-linear_xfm", + ] + atlas_xfm_inv = [ + "from-template_to-T1w_mode-image_xfm", + "from-template_to-T1w_mode-image_desc-linear_xfm", + ] + atlas_space_bold = [ + "space-template_desc-brain_bold", + "space-template_desc-preproc_bold", + ] + scout_bold = [ + "space-template_desc-scout_bold", + "space-template_desc-cleaned_bold", + "space-template_desc-brain_bold", + "space-template_desc-preproc_bold", + "space-template_desc-motion_bold", + "space-template_bold", + ] + + node, out = strat_pool.get_data("freesurfer-subject-dir") + wf.connect(node, out, surf, "freesurfer_folder") + + node, out = strat_pool.get_data(restore) + wf.connect(node, out, surf, "t1w_restore_image") + + node, out = strat_pool.get_data(space_temp) + wf.connect(node, out, surf, "atlas_space_t1w_image") + + node, out = strat_pool.get_data(atlas_xfm) + wf.connect(node, out, surf, "atlas_transform") + + node, out = strat_pool.get_data(atlas_xfm_inv) + wf.connect(node, out, surf, "inverse_atlas_transform") + + node, out = strat_pool.get_data(atlas_space_bold) + wf.connect(node, out, surf, "atlas_space_bold") node, out = strat_pool.get_data(scout_bold) - wf.connect(node, out, surf, 'scout_bold') + wf.connect(node, out, surf, "scout_bold") outputs = { - 'atlas-DesikanKilliany_space-fsLR_den-32k_dlabel': (surf, - 'desikan_' - 'killiany_32'), - 'atlas-Destrieux_space-fsLR_den-32k_dlabel': (surf, 'destrieux_32'), - 'atlas-DesikanKilliany_space-fsLR_den-164k_dlabel': (surf, - 'desikan_' - 'killiany_164'), - 'atlas-Destrieux_space-fsLR_den-164k_dlabel': (surf, 'destrieux_164'), - 'space-fsLR_den-32k_bold-dtseries': (surf, 'dtseries') + "atlas-DesikanKilliany_space-fsLR_den-32k_dlabel": ( + surf, + "desikan_" "killiany_32", + ), + "atlas-Destrieux_space-fsLR_den-32k_dlabel": (surf, "destrieux_32"), + "atlas-DesikanKilliany_space-fsLR_den-164k_dlabel": ( + surf, + "desikan_" "killiany_164", + ), + "atlas-Destrieux_space-fsLR_den-164k_dlabel": (surf, "destrieux_164"), + "space-fsLR_den-32k_bold-dtseries": (surf, "dtseries"), } return wf, outputs @@ -230,4 +341,3 @@ def surface_postproc(wf, cfg, strat_pool, pipe_num, opt=None): wf, outputs = surface_connector(wf, cfg, strat_pool, pipe_num, opt) return (wf, outputs) - diff --git a/CPAC/surface/tests/test_config.py b/CPAC/surface/tests/test_config.py index db5f223b78..684188dd6a 100644 --- a/CPAC/surface/tests/test_config.py +++ b/CPAC/surface/tests/test_config.py @@ -1,7 +1,6 @@ -""" -Tests for surface configuration -""" +"""Tests for surface configuration.""" import os + import pkg_resources as p import pytest import yaml @@ -9,22 +8,23 @@ from CPAC.pipeline.cpac_pipeline import run_workflow from CPAC.utils.configuration import Configuration -@pytest.mark.skip(reason='timing out for unrelated reasons') + +@pytest.mark.skip(reason="timing out for unrelated reasons") @pytest.mark.timeout(60) def test_duplicate_freesurfer(tmp_path): - """The pipeline should build fast if freesurfer is not self-duplicating""" - config = Configuration(yaml.safe_load('FROM: abcd-options')) - with open(p.resource_filename( - "CPAC", - os.path.join( - "resources", - "configs", - "data_config_S3-BIDS-ABIDE.yml" - ) - ), 'r') as data_config: + """The pipeline should build fast if freesurfer is not self-duplicating.""" + config = Configuration(yaml.safe_load("FROM: abcd-options")) + with open( + p.resource_filename( + "CPAC", + os.path.join("resources", "configs", "data_config_S3-BIDS-ABIDE.yml"), + ), + "r", + ) as data_config: sub_dict = yaml.safe_load(data_config)[0] - for directory in ['output', 'working', 'log', 'crash_log']: - directory_key = ['pipeline_setup', f'{directory}_directory', 'path'] + for directory in ["output", "working", "log", "crash_log"]: + directory_key = ["pipeline_setup", f"{directory}_directory", "path"] config[directory_key] = os.path.join( - tmp_path, config[directory_key].lstrip('/')) + tmp_path, config[directory_key].lstrip("/") + ) run_workflow(sub_dict, config, False, test_config=True) diff --git a/CPAC/surface/tests/test_installation.py b/CPAC/surface/tests/test_installation.py index cc4a62a190..32a1566d19 100644 --- a/CPAC/surface/tests/test_installation.py +++ b/CPAC/surface/tests/test_installation.py @@ -14,17 +14,20 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -"""Tests for requisite surface prerequisites""" +"""Tests for requisite surface prerequisites.""" import os + import pytest + from CPAC.utils.tests.test_utils import _installation_check @pytest.mark.parametrize("executable", ["bc", "csh"]) -@pytest.mark.skipif("FREESURFER_HOME" not in os.environ or - not os.path.exists(os.environ['FREESURFER_HOME']), - reason="We don't need these dependencies if we don't" - "have FreeSurfer.") +@pytest.mark.skipif( + "FREESURFER_HOME" not in os.environ + or not os.path.exists(os.environ["FREESURFER_HOME"]), + reason="We don't need these dependencies if we don't" "have FreeSurfer.", +) def test_executable(executable): - """Make sure executable is installed""" + """Make sure executable is installed.""" _installation_check(executable, "--version") diff --git a/CPAC/timeseries/__init__.py b/CPAC/timeseries/__init__.py index b49fc306c4..9ef8374d48 100644 --- a/CPAC/timeseries/__init__.py +++ b/CPAC/timeseries/__init__.py @@ -1,15 +1,19 @@ -from .timeseries_analysis import get_voxel_timeseries, \ - get_roi_timeseries, \ - get_vertices_timeseries, \ - gen_vertices_timeseries, \ - gen_voxel_timeseries, \ - gen_roi_timeseries, \ - get_spatial_map_timeseries +from .timeseries_analysis import ( + gen_roi_timeseries, + gen_vertices_timeseries, + gen_voxel_timeseries, + get_roi_timeseries, + get_spatial_map_timeseries, + get_vertices_timeseries, + get_voxel_timeseries, +) -__all__ = ['get_voxel_timeseries', \ - 'get_roi_timeseries', \ - 'get_vertices_timeseries', \ - 'gen_vertices_timeseries', \ - 'gen_voxel_timeseries', \ - 'gen_roi_timeseries', \ - 'get_spatial_map_timeseries'] +__all__ = [ + "get_voxel_timeseries", + "get_roi_timeseries", + "get_vertices_timeseries", + "gen_vertices_timeseries", + "gen_voxel_timeseries", + "gen_roi_timeseries", + "get_spatial_map_timeseries", +] diff --git a/CPAC/timeseries/timeseries_analysis.py b/CPAC/timeseries/timeseries_analysis.py index 5357ce5d0d..c3b38fe6f6 100644 --- a/CPAC/timeseries/timeseries_analysis.py +++ b/CPAC/timeseries/timeseries_analysis.py @@ -14,24 +14,28 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -from CPAC.pipeline.nodeblock import nodeblock -import nipype.interfaces.utility as util from nipype.interfaces import afni, fsl +import nipype.interfaces.utility as util from nipype.interfaces.utility import Function -from CPAC.connectome.connectivity_matrix import create_connectome_afni, \ - create_connectome_nilearn, \ - get_connectome_method +from CPAC.connectome.connectivity_matrix import ( + create_connectome_afni, + create_connectome_nilearn, + get_connectome_method, +) from CPAC.pipeline import nipype_pipeline_engine as pe -from CPAC.utils.datasource import create_roi_mask_dataflow, \ - create_spatial_map_dataflow, \ - resample_func_roi +from CPAC.pipeline.nodeblock import nodeblock +from CPAC.utils.datasource import ( + create_roi_mask_dataflow, + create_spatial_map_dataflow, + resample_func_roi, +) -def get_voxel_timeseries(wf_name='voxel_timeseries'): +def get_voxel_timeseries(wf_name="voxel_timeseries"): """ Workflow to extract time series for each voxel - in the data that is present in the input mask + in the data that is present in the input mask. Parameters ---------- @@ -76,33 +80,32 @@ def get_voxel_timeseries(wf_name='voxel_timeseries'): >>> wf.run() # doctest: +SKIP """ - wflow = pe.Workflow(name=wf_name) - inputNode = pe.Node(util.IdentityInterface(fields=['rest', - 'output_type']), - name='inputspec') - inputNode_mask = pe.Node(util.IdentityInterface(fields=['mask']), - name='input_mask') + inputNode = pe.Node( + util.IdentityInterface(fields=["rest", "output_type"]), name="inputspec" + ) + inputNode_mask = pe.Node(util.IdentityInterface(fields=["mask"]), name="input_mask") - outputNode = pe.Node(util.IdentityInterface(fields=['mask_outputs']), - name='outputspec') + outputNode = pe.Node( + util.IdentityInterface(fields=["mask_outputs"]), name="outputspec" + ) - timeseries_voxel = pe.Node(util.Function(input_names=['data_file', - 'template'], - output_names=['oneD_file'], - function=gen_voxel_timeseries), - name='timeseries_voxel') + timeseries_voxel = pe.Node( + util.Function( + input_names=["data_file", "template"], + output_names=["oneD_file"], + function=gen_voxel_timeseries, + ), + name="timeseries_voxel", + ) - wflow.connect(inputNode, 'rest', - timeseries_voxel, 'data_file') - #wflow.connect(inputNode, 'output_type', + wflow.connect(inputNode, "rest", timeseries_voxel, "data_file") + # wflow.connect(inputNode, 'output_type', # timeseries_voxel, 'output_type') - wflow.connect(inputNode_mask, 'mask', - timeseries_voxel, 'template') + wflow.connect(inputNode_mask, "mask", timeseries_voxel, "template") - wflow.connect(timeseries_voxel, 'oneD_file', - outputNode, 'mask_outputs') + wflow.connect(timeseries_voxel, "oneD_file", outputNode, "mask_outputs") return wflow @@ -131,10 +134,11 @@ def clean_roi_csv(roi_csv): path to CSV """ import os - import pandas as pd + import numpy as np + import pandas as pd - with open(roi_csv, 'r') as f: + with open(roi_csv, "r") as f: csv_lines = f.readlines() # flag whether to re-write @@ -142,29 +146,29 @@ def clean_roi_csv(roi_csv): edited_lines = [] for line in csv_lines: - line = line.replace('\t\t\t', '') - line = line.replace('\t\t', '') - line = line.replace('\t', ',') - line = line.replace('#,', '#') - if '#' in line: - if '/' in line and '.' in line: + line = line.replace("\t\t\t", "") + line = line.replace("\t\t", "") + line = line.replace("\t", ",") + line = line.replace("#,", "#") + if "#" in line: + if "/" in line and "." in line: modified = True continue - if 'Sub-brick' in line: + if "Sub-brick" in line: modified = True continue edited_lines.append(line) if modified: edited_roi_csv = os.path.join(os.getcwd(), os.path.basename(roi_csv)) - with open(edited_roi_csv, 'wt') as f: + with open(edited_roi_csv, "wt") as f: for line in edited_lines: f.write(line) edited_roi_csv = edited_roi_csv else: edited_roi_csv = roi_csv - data = pd.read_csv(edited_roi_csv, sep=',', header=1) + data = pd.read_csv(edited_roi_csv, sep=",", header=1) data = data.dropna(axis=1) roi_array = np.transpose(data.values) @@ -172,23 +176,22 @@ def clean_roi_csv(roi_csv): def write_roi_npz(roi_csv, out_type=None): - roi_npz = None roi_outputs = [roi_csv[0]] if not out_type: return roi_outputs elif out_type[1]: - np_roi_data = genfromtxt(roi_csv[0], delimiter=',') - roi_npz = os.path.join(os.getcwd(), 'roi_stats.npz') - with open(roi_npz, 'wb') as f: + np_roi_data = genfromtxt(roi_csv[0], delimiter=",") + roi_npz = os.path.join(os.getcwd(), "roi_stats.npz") + with open(roi_npz, "wb") as f: np.savez(f, np_roi_data) roi_outputs.append(roi_npz) return roi_outputs -def get_roi_timeseries(wf_name='roi_timeseries'): +def get_roi_timeseries(wf_name="roi_timeseries"): """ Workflow to extract timeseries for each node in the ROI mask. For each node, mean across all the timepoint is calculated and stored @@ -238,67 +241,66 @@ def get_roi_timeseries(wf_name='roi_timeseries'): >>> wf.run() # doctest: +SKIP """ - wflow = pe.Workflow(name=wf_name) - inputNode = pe.Node(util.IdentityInterface(fields=['rest']), - name='inputspec') + inputNode = pe.Node(util.IdentityInterface(fields=["rest"]), name="inputspec") - inputnode_roi = pe.Node(util.IdentityInterface(fields=['roi']), - name='input_roi') + inputnode_roi = pe.Node(util.IdentityInterface(fields=["roi"]), name="input_roi") - outputNode = pe.Node(util.IdentityInterface(fields=['roi_ts', - 'roi_csv']), - name='outputspec') + outputNode = pe.Node( + util.IdentityInterface(fields=["roi_ts", "roi_csv"]), name="outputspec" + ) - timeseries_roi = pe.Node(interface=afni.ROIStats(), - name='3dROIstats', - mem_gb=0.4, - mem_x=(756789500459879 / 37778931862957161709568, - 'in_file')) + timeseries_roi = pe.Node( + interface=afni.ROIStats(), + name="3dROIstats", + mem_gb=0.4, + mem_x=(756789500459879 / 37778931862957161709568, "in_file"), + ) timeseries_roi.inputs.quiet = False timeseries_roi.inputs.args = "-1Dformat" # TODO: add -mask_f2short for float parcellation mask # if parcellation mask has float values # timeseries_roi.inputs.mask_f2short = True - wflow.connect(inputNode, 'rest', - timeseries_roi, 'in_file') + wflow.connect(inputNode, "rest", timeseries_roi, "in_file") - wflow.connect(inputnode_roi, 'roi', - timeseries_roi, 'mask_file') + wflow.connect(inputnode_roi, "roi", timeseries_roi, "mask_file") - clean_csv_imports = ['import os'] - clean_csv = pe.Node(util.Function(input_names=['roi_csv'], - output_names=['roi_array', - 'edited_roi_csv'], - function=clean_roi_csv, - imports=clean_csv_imports), - name='clean_roi_csv') + clean_csv_imports = ["import os"] + clean_csv = pe.Node( + util.Function( + input_names=["roi_csv"], + output_names=["roi_array", "edited_roi_csv"], + function=clean_roi_csv, + imports=clean_csv_imports, + ), + name="clean_roi_csv", + ) - wflow.connect(timeseries_roi, 'out_file', clean_csv, 'roi_csv') - wflow.connect(clean_csv, 'roi_array', outputNode, 'roi_ts') - wflow.connect(clean_csv, 'edited_roi_csv', outputNode, 'roi_csv') + wflow.connect(timeseries_roi, "out_file", clean_csv, "roi_csv") + wflow.connect(clean_csv, "roi_array", outputNode, "roi_ts") + wflow.connect(clean_csv, "edited_roi_csv", outputNode, "roi_csv") - #write_npz_imports = ['import os', 'import numpy as np', + # write_npz_imports = ['import os', 'import numpy as np', # 'from numpy import genfromtxt'] - #write_npz = pe.Node(util.Function(input_names=['roi_csv', 'out_type'], + # write_npz = pe.Node(util.Function(input_names=['roi_csv', 'out_type'], # output_names=['roi_output_npz'], # function=write_roi_npz, # imports=write_npz_imports), # name='write_roi_npz') - #wflow.connect(clean_csv, 'edited_roi_csv', write_npz, 'roi_csv') - #wflow.connect(inputNode, 'output_type', write_npz, 'out_type') - #wflow.connect(write_npz, 'roi_output_npz', outputNode, 'roi_outputs') + # wflow.connect(clean_csv, 'edited_roi_csv', write_npz, 'roi_csv') + # wflow.connect(inputNode, 'output_type', write_npz, 'out_type') + # wflow.connect(write_npz, 'roi_output_npz', outputNode, 'roi_outputs') return wflow -def get_spatial_map_timeseries(wf_name='spatial_map_timeseries'): +def get_spatial_map_timeseries(wf_name="spatial_map_timeseries"): """ Workflow to regress each provided spatial map to the subjects functional 4D file in order - to return a timeseries for each of the maps + to return a timeseries for each of the maps. Parameters ---------- @@ -345,40 +347,40 @@ def get_spatial_map_timeseries(wf_name='spatial_map_timeseries'): >>> wf.run() # doctest: +SKIP """ - wflow = pe.Workflow(name=wf_name) - inputNode = pe.Node(util.IdentityInterface - (fields=['subject_rest', - 'subject_mask', - 'spatial_map', - 'demean']), - name='inputspec') + inputNode = pe.Node( + util.IdentityInterface( + fields=["subject_rest", "subject_mask", "spatial_map", "demean"] + ), + name="inputspec", + ) - outputNode = pe.Node(util.IdentityInterface( - fields=['subject_timeseries']), - name='outputspec') + outputNode = pe.Node( + util.IdentityInterface(fields=["subject_timeseries"]), name="outputspec" + ) - spatialReg = pe.Node(interface=fsl.GLM(), - name='spatial_regression', - mem_gb=0.2, - mem_x=(1708448960473801 / 302231454903657293676544, - 'in_file')) + spatialReg = pe.Node( + interface=fsl.GLM(), + name="spatial_regression", + mem_gb=0.2, + mem_x=(1708448960473801 / 302231454903657293676544, "in_file"), + ) - spatialReg.inputs.out_file = 'spatial_map_timeseries.txt' + spatialReg.inputs.out_file = "spatial_map_timeseries.txt" - wflow.connect(inputNode, 'subject_rest', spatialReg, 'in_file') - wflow.connect(inputNode, 'subject_mask', spatialReg, 'mask') - wflow.connect(inputNode, 'spatial_map', spatialReg, 'design') - wflow.connect(inputNode, 'demean', spatialReg, 'demean') - wflow.connect(spatialReg, 'out_file', outputNode, 'subject_timeseries') + wflow.connect(inputNode, "subject_rest", spatialReg, "in_file") + wflow.connect(inputNode, "subject_mask", spatialReg, "mask") + wflow.connect(inputNode, "spatial_map", spatialReg, "design") + wflow.connect(inputNode, "demean", spatialReg, "demean") + wflow.connect(spatialReg, "out_file", outputNode, "subject_timeseries") return wflow -def get_vertices_timeseries(wf_name='vertices_timeseries'): +def get_vertices_timeseries(wf_name="vertices_timeseries"): """ - Workflow to get vertices time series from a FreeSurfer surface file + Workflow to get vertices time series from a FreeSurfer surface file. Parameters ---------- @@ -416,36 +418,37 @@ def get_vertices_timeseries(wf_name='vertices_timeseries'): >>> wf.base_dir = './' >>> wf.run() # doctest: +SKIP """ - wflow = pe.Workflow(name=wf_name) - inputNode = pe.Node(util.IdentityInterface(fields=['lh_surface_file', - 'rh_surface_file']), - name='inputspec') + inputNode = pe.Node( + util.IdentityInterface(fields=["lh_surface_file", "rh_surface_file"]), + name="inputspec", + ) - timeseries_surface = pe.Node(util.Function(input_names=['rh_surface_file', - 'lh_surface_file'], - output_names=['out_file'], - function=gen_vertices_timeseries), - name='timeseries_surface') + timeseries_surface = pe.Node( + util.Function( + input_names=["rh_surface_file", "lh_surface_file"], + output_names=["out_file"], + function=gen_vertices_timeseries, + ), + name="timeseries_surface", + ) - outputNode = pe.Node(util.IdentityInterface(fields=['surface_outputs']), - name='outputspec') + outputNode = pe.Node( + util.IdentityInterface(fields=["surface_outputs"]), name="outputspec" + ) - wflow.connect(inputNode, 'rh_surface_file', - timeseries_surface, 'rh_surface_file') - wflow.connect(inputNode, 'lh_surface_file', - timeseries_surface, 'lh_surface_file') + wflow.connect(inputNode, "rh_surface_file", timeseries_surface, "rh_surface_file") + wflow.connect(inputNode, "lh_surface_file", timeseries_surface, "lh_surface_file") - wflow.connect(timeseries_surface, 'out_file', - outputNode, 'surface_outputs') + wflow.connect(timeseries_surface, "out_file", outputNode, "surface_outputs") return wflow -def get_normalized_moments(wf_name='normalized_moments'): +def get_normalized_moments(wf_name="normalized_moments"): """ - Workflow to calculate the normalized moments for skewedness calculations + Workflow to calculate the normalized moments for skewedness calculations. Parameters ---------- @@ -479,23 +482,24 @@ def get_normalized_moments(wf_name='normalized_moments'): >>> wf.base_dir = './' # doctest: +SKIP >>> wf.run() # doctest: +SKIP """ - wflow = pe.Workflow(name=wf_name) - inputNode = pe.Node(util.IdentityInterface(fields=['spatial_timeseries']), - name='inputspec') + inputNode = pe.Node( + util.IdentityInterface(fields=["spatial_timeseries"]), name="inputspec" + ) # calculate normalized moments # output of this node is a list, 'moments' - norm_moments = pe.Node(util.CalculateNormalizedMoments(moment='3'), - name='norm_moments') + norm_moments = pe.Node( + util.CalculateNormalizedMoments(moment="3"), name="norm_moments" + ) - outputNode = pe.Node(util.IdentityInterface(fields=['moments_outputs']), - name='outputspec') + outputNode = pe.Node( + util.IdentityInterface(fields=["moments_outputs"]), name="outputspec" + ) - wflow.connect(inputNode, 'spatial_timeseries', - norm_moments, 'timeseries_file') - wflow.connect(norm_moments, 'moments', outputNode, 'moments_outputs') + wflow.connect(inputNode, "spatial_timeseries", norm_moments, "timeseries_file") + wflow.connect(norm_moments, "moments", outputNode, "moments_outputs") return wflow @@ -503,7 +507,7 @@ def get_normalized_moments(wf_name='normalized_moments'): def gen_roi_timeseries(data_file, template, output_type): """ Method to extract mean of voxel across - all timepoints for each node in roi mask + all timepoints for each node in roi mask. Parameters ---------- @@ -528,23 +532,26 @@ def gen_roi_timeseries(data_file, template, output_type): Exception """ - import nibabel as nib import csv - import numpy as np import os import shutil + import numpy as np + import nibabel as nib + unit_data = nib.load(template).get_fdata() # Cast as rounded-up integer unit_data = np.int64(np.ceil(unit_data)) datafile = nib.load(data_file) img_data = datafile.get_fdata() - vol = img_data.shape[3] + img_data.shape[3] if unit_data.shape != img_data.shape[:3]: - raise Exception('\n\n[!] CPAC says: Invalid Shape Error.' - 'Please check the voxel dimensions. ' - 'Data and roi should have the same shape.\n\n') + raise Exception( + "\n\n[!] CPAC says: Invalid Shape Error." + "Please check the voxel dimensions. " + "Data and roi should have the same shape.\n\n" + ) nodes = np.unique(unit_data).tolist() sorted_list = [] @@ -552,36 +559,32 @@ def gen_roi_timeseries(data_file, template, output_type): out_list = [] # extracting filename from input template - tmp_file = os.path.splitext( - os.path.basename(template))[0] + tmp_file = os.path.splitext(os.path.basename(template))[0] tmp_file = os.path.splitext(tmp_file)[0] - oneD_file = os.path.abspath('roi_' + tmp_file + '.1D') - txt_file = os.path.abspath('roi_' + tmp_file + '.txt') - csv_file = os.path.abspath('roi_' + tmp_file + '.csv') - numpy_file = os.path.abspath('roi_' + tmp_file + '.npz') + oneD_file = os.path.abspath("roi_" + tmp_file + ".1D") + txt_file = os.path.abspath("roi_" + tmp_file + ".txt") + os.path.abspath("roi_" + tmp_file + ".csv") + os.path.abspath("roi_" + tmp_file + ".npz") nodes.sort() for n in nodes: if n > 0: node_array = img_data[unit_data == n] - node_str = 'node_{0}'.format(n) + node_str = "node_{0}".format(n) avg = np.mean(node_array, axis=0) avg = np.round(avg, 6) - list1 = [n] + avg.tolist() + list1 = [n, *avg.tolist()] sorted_list.append(list1) node_dict[node_str] = avg.tolist() # writing to 1Dfile - print("writing 1D file..") - f = open(oneD_file, 'w') - writer = csv.writer(f, delimiter=',') + f = open(oneD_file, "w") + writer = csv.writer(f, delimiter=",") value_list = [] - new_keys = sorted([ - int(float(key.split('node_')[1])) for key in node_dict - ]) + new_keys = sorted([int(float(key.split("node_")[1])) for key in node_dict]) roi_number_list = [str(n) for n in new_keys] @@ -590,7 +593,7 @@ def gen_roi_timeseries(data_file, template, output_type): roi_number_str.append("#" + number) for key in new_keys: - value_list.append(str('{0}\n'.format(node_dict['node_{0}'.format(key)]))) + value_list.append(str("{0}\n".format(node_dict["node_{0}".format(key)]))) column_list = list(zip(*value_list)) @@ -607,7 +610,7 @@ def gen_roi_timeseries(data_file, template, output_type): out_list.append(txt_file) # if csv is required - ''' + """ if output_type[0]: print("writing csv file..") f = open(csv_file, 'wt') @@ -625,7 +628,7 @@ def gen_roi_timeseries(data_file, template, output_type): out_list.append(numpy_file) return out_list - ''' + """ return oneD_file @@ -633,7 +636,7 @@ def gen_roi_timeseries(data_file, template, output_type): def gen_voxel_timeseries(data_file, template): """ Method to extract timeseries for each voxel - in the data that is present in the input mask + in the data that is present in the input mask. Parameters ---------- @@ -661,11 +664,12 @@ def gen_voxel_timeseries(data_file, template): Exception """ - import nibabel as nib - import numpy as np import csv import os + import numpy as np + import nibabel as nib + unit = nib.load(template) unit_data = unit.get_fdata() datafile = nib.load(data_file) @@ -674,13 +678,11 @@ def gen_voxel_timeseries(data_file, template): qform = header_data.get_qform() sorted_list = [] vol_dict = {} - out_list = [] - tmp_file = os.path.splitext( - os.path.basename(template))[0] + tmp_file = os.path.splitext(os.path.basename(template))[0] tmp_file = os.path.splitext(tmp_file)[0] - oneD_file = os.path.abspath('mask_' + tmp_file + '.1D') - f = open(oneD_file, 'wt') + oneD_file = os.path.abspath("mask_" + tmp_file + ".1D") + f = open(oneD_file, "wt") x, y, z = unit_data.shape @@ -688,22 +690,21 @@ def gen_voxel_timeseries(data_file, template): node_array = node_array.T time_points = node_array.shape[0] for t in range(0, time_points): - string = 'vol {0}'.format(t) + string = "vol {0}".format(t) vol_dict[string] = node_array[t] f.write(str(np.round(np.mean(node_array[t]), 6))) - f.write('\n') + f.write("\n") val = node_array[t].tolist() val.insert(0, t) sorted_list.append(val) f.close() - csv_file = os.path.abspath('mask_' + tmp_file + '.csv') - f = open(csv_file, 'wt') - writer = csv.writer(f, delimiter=str(','), - quoting=csv.QUOTE_MINIMAL) + csv_file = os.path.abspath("mask_" + tmp_file + ".csv") + f = open(csv_file, "wt") + writer = csv.writer(f, delimiter=str(","), quoting=csv.QUOTE_MINIMAL) one = np.array([1]) - headers = ['volume/xyz'] + headers = ["volume/xyz"] cordinates = np.argwhere(unit_data != 0) for val in range(len(cordinates)): ijk_mat = np.concatenate([cordinates[val], one]) @@ -715,7 +716,7 @@ def gen_voxel_timeseries(data_file, template): writer.writerows(sorted_list) f.close() - #if output_type[1]: + # if output_type[1]: # numpy_file = os.path.abspath('mask_' + tmp_file + '.npz') # np.savez(numpy_file, **dict(vol_dict)) # out_list.append(numpy_file) @@ -723,11 +724,10 @@ def gen_voxel_timeseries(data_file, template): return oneD_file -def gen_vertices_timeseries(rh_surface_file, - lh_surface_file): +def gen_vertices_timeseries(rh_surface_file, lh_surface_file): """ Method to extract timeseries from vertices - of a freesurfer surface file + of a freesurfer surface file. Parameters ---------- @@ -742,61 +742,59 @@ def gen_vertices_timeseries(rh_surface_file, list of vertices timeseries csv files """ + import os import gradunwarp import numpy as np - import os out_list = [] - rh_file = os.path.splitext( - os.path.basename(rh_surface_file))[0] + '_rh.csv' + rh_file = os.path.splitext(os.path.basename(rh_surface_file))[0] + "_rh.csv" mghobj1 = gradunwarp.mgh.MGH() mghobj1.load(rh_surface_file) vol = mghobj1.vol (x, y) = vol.shape -# print "rh shape", x, y + # print "rh shape", x, y - np.savetxt(rh_file, vol, delimiter='\t') + np.savetxt(rh_file, vol, delimiter="\t") out_list.append(rh_file) - lh_file = os.path.splitext(os.path.basename(lh_surface_file))[0] + '_lh.csv' + lh_file = os.path.splitext(os.path.basename(lh_surface_file))[0] + "_lh.csv" mghobj2 = gradunwarp.mgh.MGH() mghobj2.load(lh_surface_file) vol = mghobj2.vol (x, y) = vol.shape -# print "lh shape", x, y + # print "lh shape", x, y - np.savetxt(lh_file, - vol, - delimiter=',') + np.savetxt(lh_file, vol, delimiter=",") out_list.append(lh_file) return out_list -def resample_function() -> 'Function': +def resample_function() -> "Function": """ Returns a Function interface for - `CPAC.utils.datasource.resample_func_roi` + `CPAC.utils.datasource.resample_func_roi`. Returns ------- Function """ - return Function(input_names=['in_func', 'in_roi', 'realignment', - 'identity_matrix'], - output_names=['out_func', 'out_roi'], - function=resample_func_roi, as_module=True) + return Function( + input_names=["in_func", "in_roi", "realignment", "identity_matrix"], + output_names=["out_func", "out_roi"], + function=resample_func_roi, + as_module=True, + ) @nodeblock( name="timeseries_extraction_AVG", config=["timeseries_extraction"], switch=["run"], - inputs=["space-template_desc-preproc_bold", - "space-template_desc-bold_mask"], + inputs=["space-template_desc-preproc_bold", "space-template_desc-bold_mask"], outputs=[ "space-template_desc-Mean_timeseries", "space-template_desc-ndmg_correlations", @@ -808,126 +806,157 @@ def resample_function() -> 'Function': ], ) def timeseries_extraction_AVG(wf, cfg, strat_pool, pipe_num, opt=None): - resample_functional_roi = pe.Node(resample_function(), - name='resample_functional_roi_' - f'{pipe_num}') - realignment = cfg.timeseries_extraction['realignment'] + resample_functional_roi = pe.Node( + resample_function(), name="resample_functional_roi_" f"{pipe_num}" + ) + realignment = cfg.timeseries_extraction["realignment"] resample_functional_roi.inputs.realignment = realignment - resample_functional_roi.inputs.identity_matrix = \ - cfg.registration_workflows['functional_registration'][ - 'func_registration_to_template']['FNIRT_pipelines']['identity_matrix'] + resample_functional_roi.inputs.identity_matrix = cfg.registration_workflows[ + "functional_registration" + ]["func_registration_to_template"]["FNIRT_pipelines"]["identity_matrix"] roi_dataflow = create_roi_mask_dataflow( - cfg.timeseries_extraction['tse_atlases']['Avg'], - f'roi_dataflow_{pipe_num}') + cfg.timeseries_extraction["tse_atlases"]["Avg"], f"roi_dataflow_{pipe_num}" + ) roi_dataflow.inputs.inputspec.set( - creds_path=cfg.pipeline_setup['input_creds_path'], - dl_dir=cfg.pipeline_setup['working_directory']['path'] + creds_path=cfg.pipeline_setup["input_creds_path"], + dl_dir=cfg.pipeline_setup["working_directory"]["path"], ) - roi_timeseries = get_roi_timeseries(f'roi_timeseries_{pipe_num}') - #roi_timeseries.inputs.inputspec.output_type = cfg.timeseries_extraction[ + roi_timeseries = get_roi_timeseries(f"roi_timeseries_{pipe_num}") + # roi_timeseries.inputs.inputspec.output_type = cfg.timeseries_extraction[ # 'roi_tse_outputs'] node, out = strat_pool.get_data("space-template_desc-preproc_bold") - wf.connect(node, out, resample_functional_roi, 'in_func') + wf.connect(node, out, resample_functional_roi, "in_func") - wf.connect(roi_dataflow, 'outputspec.out_file', - resample_functional_roi, 'in_roi') + wf.connect(roi_dataflow, "outputspec.out_file", resample_functional_roi, "in_roi") # connect it to the roi_timeseries # workflow.connect(roi_dataflow, 'outputspec.out_file', # roi_timeseries, 'input_roi.roi') - wf.connect(resample_functional_roi, 'out_roi', - roi_timeseries, 'input_roi.roi') - wf.connect(resample_functional_roi, 'out_func', - roi_timeseries, 'inputspec.rest') + wf.connect(resample_functional_roi, "out_roi", roi_timeseries, "input_roi.roi") + wf.connect(resample_functional_roi, "out_func", roi_timeseries, "inputspec.rest") # create the graphs: # - connectivity matrix matrix_outputs = {} - for cm_measure in cfg['timeseries_extraction', 'connectivity_matrix', - 'measure']: - for cm_tool in [tool for tool in cfg['timeseries_extraction', - 'connectivity_matrix', 'using'] if tool != 'ndmg']: + for cm_measure in cfg["timeseries_extraction", "connectivity_matrix", "measure"]: + for cm_tool in [ + tool + for tool in cfg["timeseries_extraction", "connectivity_matrix", "using"] + if tool != "ndmg" + ]: implementation = get_connectome_method(cm_measure, cm_tool) if implementation is NotImplemented: continue - if cm_tool == 'Nilearn': + if cm_tool == "Nilearn": timeseries_correlation = create_connectome_nilearn( - name=f'connectomeNilearn{cm_measure}_{pipe_num}' + name=f"connectomeNilearn{cm_measure}_{pipe_num}" ) elif cm_tool == "AFNI": timeseries_correlation = create_connectome_afni( - name=f'connectomeAfni{cm_measure}_{pipe_num}', + name=f"connectomeAfni{cm_measure}_{pipe_num}", method=cm_measure, - pipe_num=pipe_num + pipe_num=pipe_num, + ) + brain_mask_node, brain_mask_out = strat_pool.get_data( + ["space-template_desc-bold_mask"] ) - brain_mask_node, brain_mask_out = strat_pool.get_data([ - 'space-template_desc-bold_mask']) - if 'func_to_ROI' in realignment: + if "func_to_ROI" in realignment: resample_brain_mask_roi = pe.Node( - resample_function(), - name=f'resample_brain_mask_roi_{pipe_num}') + resample_function(), name=f"resample_brain_mask_roi_{pipe_num}" + ) resample_brain_mask_roi.inputs.realignment = realignment resample_brain_mask_roi.inputs.identity_matrix = ( - cfg.registration_workflows['functional_registration'][ - 'func_registration_to_template' - ]['FNIRT_pipelines']['identity_matrix']) - wf.connect([ - (brain_mask_node, resample_brain_mask_roi, [ - (brain_mask_out, 'in_func')]), - (roi_dataflow, resample_brain_mask_roi, [ - ('outputspec.out_file', 'in_roi')]), - (resample_brain_mask_roi, timeseries_correlation, [ - ('out_func', 'inputspec.mask')])]) + cfg.registration_workflows["functional_registration"][ + "func_registration_to_template" + ]["FNIRT_pipelines"]["identity_matrix"] + ) + wf.connect( + [ + ( + brain_mask_node, + resample_brain_mask_roi, + [(brain_mask_out, "in_func")], + ), + ( + roi_dataflow, + resample_brain_mask_roi, + [("outputspec.out_file", "in_roi")], + ), + ( + resample_brain_mask_roi, + timeseries_correlation, + [("out_func", "inputspec.mask")], + ), + ] + ) else: - wf.connect(brain_mask_node, brain_mask_out, - timeseries_correlation, 'inputspec.mask') + wf.connect( + brain_mask_node, + brain_mask_out, + timeseries_correlation, + "inputspec.mask", + ) timeseries_correlation.inputs.inputspec.method = cm_measure - wf.connect([ - (roi_dataflow, timeseries_correlation, [ - ('outputspec.out_name', 'inputspec.atlas_name')]), - (resample_functional_roi, timeseries_correlation, [ - ('out_roi', 'inputspec.in_rois'), - ('out_func', 'inputspec.in_file')])]) - - output_desc = ''.join(term.lower().capitalize() for term in [ - cm_measure, cm_tool]) - matrix_outputs[f'space-template_desc-{output_desc}_correlations' - ] = (timeseries_correlation, 'outputspec.out_file') + wf.connect( + [ + ( + roi_dataflow, + timeseries_correlation, + [("outputspec.out_name", "inputspec.atlas_name")], + ), + ( + resample_functional_roi, + timeseries_correlation, + [ + ("out_roi", "inputspec.in_rois"), + ("out_func", "inputspec.in_file"), + ], + ), + ] + ) + + output_desc = "".join( + term.lower().capitalize() for term in [cm_measure, cm_tool] + ) + matrix_outputs[f"space-template_desc-{output_desc}_correlations"] = ( + timeseries_correlation, + "outputspec.out_file", + ) outputs = { - 'space-template_desc-Mean_timeseries': ( - roi_timeseries, 'outputspec.roi_csv'), - 'atlas_name': (roi_dataflow, 'outputspec.out_name'), - **matrix_outputs + "space-template_desc-Mean_timeseries": (roi_timeseries, "outputspec.roi_csv"), + "atlas_name": (roi_dataflow, "outputspec.out_name"), + **matrix_outputs, } # - NDMG - if 'ndmg' in cfg['timeseries_extraction', 'connectivity_matrix', 'using']: + if "ndmg" in cfg["timeseries_extraction", "connectivity_matrix", "using"]: # pylint: disable=import-outside-toplevel from CPAC.utils.ndmg_utils import ndmg_create_graphs - ndmg_graph_imports = ['import os', - 'from CPAC.utils.ndmg_utils import graph'] - ndmg_graph = pe.Node(Function( - input_names=['ts', 'labels'], - output_names=['out_file'], - function=ndmg_create_graphs, - imports=ndmg_graph_imports, - as_module=True - ), name=f'ndmg_graphs_{pipe_num}', - mem_gb=0.664, - mem_x=(1928411764134803 / 302231454903657293676544, 'ts')) - - wf.connect(roi_timeseries, 'outputspec.roi_ts', ndmg_graph, 'ts') - wf.connect(roi_dataflow, 'outputspec.out_file', ndmg_graph, 'labels') - outputs['space-template_desc-ndmg_correlations' - ] = (ndmg_graph, 'out_file') + ndmg_graph_imports = ["import os", "from CPAC.utils.ndmg_utils import graph"] + ndmg_graph = pe.Node( + Function( + input_names=["ts", "labels"], + output_names=["out_file"], + function=ndmg_create_graphs, + imports=ndmg_graph_imports, + as_module=True, + ), + name=f"ndmg_graphs_{pipe_num}", + mem_gb=0.664, + mem_x=(1928411764134803 / 302231454903657293676544, "ts"), + ) + + wf.connect(roi_timeseries, "outputspec.roi_ts", ndmg_graph, "ts") + wf.connect(roi_dataflow, "outputspec.out_file", ndmg_graph, "labels") + outputs["space-template_desc-ndmg_correlations"] = (ndmg_graph, "out_file") return (wf, outputs) @@ -940,44 +969,43 @@ def timeseries_extraction_AVG(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["desc-Voxel_timeseries", "atlas_name"], ) def timeseries_extraction_Voxel(wf, cfg, strat_pool, pipe_num, opt=None): - - resample_functional_to_mask = pe.Node(resample_function(), - name='resample_functional_to_mask_' - f'{pipe_num}') + resample_functional_to_mask = pe.Node( + resample_function(), name="resample_functional_to_mask_" f"{pipe_num}" + ) resample_functional_to_mask.inputs.realignment = cfg.timeseries_extraction[ - 'realignment'] - resample_functional_to_mask.inputs.identity_matrix = \ - cfg.registration_workflows['functional_registration'][ - 'func_registration_to_template']['FNIRT_pipelines']['identity_matrix'] - - mask_dataflow = create_roi_mask_dataflow(cfg.timeseries_extraction[ - 'tse_atlases']['Voxel'], - f'mask_dataflow_{pipe_num}') - - voxel_timeseries = get_voxel_timeseries( - f'voxel_timeseries_{pipe_num}') - #voxel_timeseries.inputs.inputspec.output_type = cfg.timeseries_extraction[ - # 'roi_tse_outputs'] + "realignment" + ] + resample_functional_to_mask.inputs.identity_matrix = cfg.registration_workflows[ + "functional_registration" + ]["func_registration_to_template"]["FNIRT_pipelines"]["identity_matrix"] + + mask_dataflow = create_roi_mask_dataflow( + cfg.timeseries_extraction["tse_atlases"]["Voxel"], f"mask_dataflow_{pipe_num}" + ) + voxel_timeseries = get_voxel_timeseries(f"voxel_timeseries_{pipe_num}") + # voxel_timeseries.inputs.inputspec.output_type = cfg.timeseries_extraction[ + # 'roi_tse_outputs'] node, out = strat_pool.get_data("space-template_desc-preproc_bold") # resample the input functional file to mask - wf.connect(node, out, - resample_functional_to_mask, 'in_func') - wf.connect(mask_dataflow, 'outputspec.out_file', - resample_functional_to_mask, 'in_roi') + wf.connect(node, out, resample_functional_to_mask, "in_func") + wf.connect( + mask_dataflow, "outputspec.out_file", resample_functional_to_mask, "in_roi" + ) # connect it to the voxel_timeseries - wf.connect(resample_functional_to_mask, 'out_roi', - voxel_timeseries, 'input_mask.mask') - wf.connect(resample_functional_to_mask, 'out_func', - voxel_timeseries, 'inputspec.rest') + wf.connect( + resample_functional_to_mask, "out_roi", voxel_timeseries, "input_mask.mask" + ) + wf.connect( + resample_functional_to_mask, "out_func", voxel_timeseries, "inputspec.rest" + ) outputs = { - 'desc-Voxel_timeseries': - (voxel_timeseries, 'outputspec.mask_outputs'), - 'atlas_name': (mask_dataflow, 'outputspec.out_name') + "desc-Voxel_timeseries": (voxel_timeseries, "outputspec.mask_outputs"), + "atlas_name": (mask_dataflow, "outputspec.out_name"), } return (wf, outputs) @@ -987,67 +1015,84 @@ def timeseries_extraction_Voxel(wf, cfg, strat_pool, pipe_num, opt=None): name="spatial_regression", config=["timeseries_extraction"], switch=["run"], - inputs=["space-template_desc-preproc_bold", - ["space-template_desc-bold_mask", - "space-template_desc-brain_mask"]], + inputs=[ + "space-template_desc-preproc_bold", + ["space-template_desc-bold_mask", "space-template_desc-brain_mask"], + ], outputs=["desc-SpatReg_timeseries", "atlas_name"], ) def spatial_regression(wf, cfg, strat_pool, pipe_num, opt=None): - '''Performs spatial regression, extracting the spatial map timeseries of + """Performs spatial regression, extracting the spatial map timeseries of the given atlases. Note: this is a standalone function for when only spatial regression is selected for the given atlases - if dual regression is selected, that spatial regression is performed in the dual_regression function - ''' - + """ resample_spatial_map_to_native_space = pe.Node( interface=fsl.FLIRT(), - name=f'resample_spatial_map_to_native_space_{pipe_num}', + name=f"resample_spatial_map_to_native_space_{pipe_num}", mem_gb=3.4, - mem_x=(5381614225492473 / 1208925819614629174706176, 'in_file')) + mem_x=(5381614225492473 / 1208925819614629174706176, "in_file"), + ) resample_spatial_map_to_native_space.inputs.set( - interp='nearestneighbour', + interp="nearestneighbour", apply_xfm=True, - in_matrix_file=cfg.registration_workflows['functional_registration'][ - 'func_registration_to_template']['FNIRT_pipelines'][ - 'identity_matrix']) + in_matrix_file=cfg.registration_workflows["functional_registration"][ + "func_registration_to_template" + ]["FNIRT_pipelines"]["identity_matrix"], + ) spatial_map_dataflow = create_spatial_map_dataflow( - cfg.timeseries_extraction['tse_atlases']['SpatialReg'], - f'spatial_map_dataflow_{pipe_num}') + cfg.timeseries_extraction["tse_atlases"]["SpatialReg"], + f"spatial_map_dataflow_{pipe_num}", + ) spatial_map_dataflow.inputs.inputspec.set( - creds_path=cfg.pipeline_setup['input_creds_path'], - dl_dir=cfg.pipeline_setup['working_directory']['path']) + creds_path=cfg.pipeline_setup["input_creds_path"], + dl_dir=cfg.pipeline_setup["working_directory"]["path"], + ) spatial_map_timeseries = get_spatial_map_timeseries( - f'spatial_map_timeseries_{pipe_num}') + f"spatial_map_timeseries_{pipe_num}" + ) spatial_map_timeseries.inputs.inputspec.demean = True node, out = strat_pool.get_data("space-template_desc-preproc_bold") # resample the input functional file and functional mask # to spatial map - wf.connect(node, out, resample_spatial_map_to_native_space, 'reference') - wf.connect(spatial_map_dataflow, 'select_spatial_map.out_file', - resample_spatial_map_to_native_space, 'in_file') + wf.connect(node, out, resample_spatial_map_to_native_space, "reference") + wf.connect( + spatial_map_dataflow, + "select_spatial_map.out_file", + resample_spatial_map_to_native_space, + "in_file", + ) - wf.connect(node, out, spatial_map_timeseries, 'inputspec.subject_rest') + wf.connect(node, out, spatial_map_timeseries, "inputspec.subject_rest") # connect it to the spatial_map_timeseries - wf.connect(resample_spatial_map_to_native_space, 'out_file', - spatial_map_timeseries, 'inputspec.spatial_map') + wf.connect( + resample_spatial_map_to_native_space, + "out_file", + spatial_map_timeseries, + "inputspec.spatial_map", + ) - node, out = strat_pool.get_data(['space-template_desc-bold_mask', 'space-template_desc-brain_mask']) - wf.connect(node, out, spatial_map_timeseries, 'inputspec.subject_mask') + node, out = strat_pool.get_data( + ["space-template_desc-bold_mask", "space-template_desc-brain_mask"] + ) + wf.connect(node, out, spatial_map_timeseries, "inputspec.subject_mask") # 'atlas_name' will be an iterable and will carry through outputs = { - 'desc-SpatReg_timeseries': - (spatial_map_timeseries, 'outputspec.subject_timeseries'), - 'atlas_name': (spatial_map_dataflow, 'select_spatial_map.out_name') + "desc-SpatReg_timeseries": ( + spatial_map_timeseries, + "outputspec.subject_timeseries", + ), + "atlas_name": (spatial_map_dataflow, "select_spatial_map.out_name"), } return (wf, outputs) diff --git a/CPAC/unet/__init__.py b/CPAC/unet/__init__.py index a8e801042b..70826416c8 100644 --- a/CPAC/unet/__init__.py +++ b/CPAC/unet/__init__.py @@ -15,33 +15,44 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . from ._torch import torch # this import has to be first to install torch - -from .function import write_nifti, estimate_dice, extract_large_comp, \ - predict_volumes, MyParser - -from .model import weigths_init, Conv3dBlock, UpConv3dBlock, Conv2dBlock, \ - UpConv2dBlock, UNet3d, UNet2d, MultiSliceBcUNet, MultiSliceSsUNet, \ - MultiSliceModel - -from .dataset import VolumeDataset, BlockDataset +from .dataset import BlockDataset, VolumeDataset +from .function import ( + MyParser, + estimate_dice, + extract_large_comp, + predict_volumes, + write_nifti, +) +from .model import ( + Conv2dBlock, + Conv3dBlock, + MultiSliceBcUNet, + MultiSliceModel, + MultiSliceSsUNet, + UNet2d, + UNet3d, + UpConv2dBlock, + UpConv3dBlock, + weigths_init, +) __all__ = [ - 'write_nifti', - 'estimate_dice', - 'extract_large_comp', - 'predict_volumes', - 'MyParser', - 'weigths_init', - 'Conv3dBlock', - 'UpConv3dBlock', - 'Conv2dBlock', - 'UpConv2dBlock', - 'UNet3d', - 'UNet2d', - 'MultiSliceBcUNet', - 'MultiSliceSsUNet', - 'MultiSliceModel', - 'VolumeDataset', - 'BlockDataset', - 'torch' + "write_nifti", + "estimate_dice", + "extract_large_comp", + "predict_volumes", + "MyParser", + "weigths_init", + "Conv3dBlock", + "UpConv3dBlock", + "Conv2dBlock", + "UpConv2dBlock", + "UNet3d", + "UNet2d", + "MultiSliceBcUNet", + "MultiSliceSsUNet", + "MultiSliceModel", + "VolumeDataset", + "BlockDataset", + "torch", ] diff --git a/CPAC/unet/_torch.py b/CPAC/unet/_torch.py index a472ca8afa..6b9ec0ac61 100644 --- a/CPAC/unet/_torch.py +++ b/CPAC/unet/_torch.py @@ -14,7 +14,7 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -"""Dynamically install torch iff we're going to use it""" +"""Dynamically install torch iff we're going to use it.""" # pylint: disable=import-error,redefined-outer-name,ungrouped-imports,unused-import from importlib import invalidate_caches import os @@ -22,6 +22,7 @@ from subprocess import CalledProcessError import sys from typing import Optional + from CPAC.info import UNET_REQUIREMENTS from CPAC.utils.monitoring.custom_logging import log_subprocess @@ -29,7 +30,7 @@ def _custom_pip_install(env_var: Optional[str] = None) -> None: """ ``pip install --user torch``, in a custom ``--user`` directory if - one is provided, then ``import torch`` + one is provided, then ``import torch``. Parameters ---------- @@ -38,19 +39,21 @@ def _custom_pip_install(env_var: Optional[str] = None) -> None: """ if env_var is not None: if env_var not in os.environ: - raise FileNotFoundError(f'${env_var}') - site.USER_BASE = os.environ['PYTHONUSERBASE'] = os.path.join( - os.environ[env_var], '.local') - py_version = '.'.join(str(getattr(sys.version_info, attr)) for - attr in ['major', 'minor']) - pythonpath = f'{site.USER_BASE}/lib/python{py_version}/site-packages' + raise FileNotFoundError(f"${env_var}") + site.USER_BASE = os.environ["PYTHONUSERBASE"] = os.path.join( + os.environ[env_var], ".local" + ) + py_version = ".".join( + str(getattr(sys.version_info, attr)) for attr in ["major", "minor"] + ) + pythonpath = f"{site.USER_BASE}/lib/python{py_version}/site-packages" sys.path.append(pythonpath) - os.environ['PYTHONPATH'] = ':'.join([os.environ['PYTHONPATH'], - pythonpath]).replace('::', ':') + os.environ["PYTHONPATH"] = ":".join( + [os.environ["PYTHONPATH"], pythonpath] + ).replace("::", ":") invalidate_caches() - log_subprocess(['pip', 'install', '--user', *UNET_REQUIREMENTS]) + log_subprocess(["pip", "install", "--user", *UNET_REQUIREMENTS]) invalidate_caches() - import torch try: @@ -59,12 +62,22 @@ def _custom_pip_install(env_var: Optional[str] = None) -> None: torch = NotImplemented try: _custom_pip_install() # pip install in default user directory - except (CalledProcessError, FileNotFoundError, ImportError, - ModuleNotFoundError, OSError): + except ( + CalledProcessError, + FileNotFoundError, + ImportError, + ModuleNotFoundError, + OSError, + ): try: - _custom_pip_install('CPAC_WORKDIR') # pip install in $CPAC_WORKDIR - except (CalledProcessError, FileNotFoundError, ImportError, - ModuleNotFoundError, OSError): - _custom_pip_install('PWD') # pip install in $PWD + _custom_pip_install("CPAC_WORKDIR") # pip install in $CPAC_WORKDIR + except ( + CalledProcessError, + FileNotFoundError, + ImportError, + ModuleNotFoundError, + OSError, + ): + _custom_pip_install("PWD") # pip install in $PWD if torch is not NotImplemented: - __all__ = ['torch'] + __all__ = ["torch"] diff --git a/CPAC/unet/dataset.py b/CPAC/unet/dataset.py index 9adad8a62f..74ad9193f9 100755 --- a/CPAC/unet/dataset.py +++ b/CPAC/unet/dataset.py @@ -1,84 +1,79 @@ -import torch -import torch.utils.data as data -import torch.nn as nn -import scipy.io as io +import os +import sys + import numpy as np +import torch +from torch import nn +from torch.utils import data import nibabel as nib -import os, sys + class VolumeDataset(data.Dataset): - def __init__(self, - rimg_in=None, - cimg_in=None, - bmsk_in=None, - transform=None, - debug=True - ): + def __init__( + self, rimg_in=None, cimg_in=None, bmsk_in=None, transform=None, debug=True + ): super(VolumeDataset, self).__init__() # Raw Images - self.rimg_in=rimg_in + self.rimg_in = rimg_in if isinstance(rimg_in, type(None)): - self.rimg_dir=None - self.rimg_files=None + self.rimg_dir = None + self.rimg_files = None else: if isinstance(rimg_in, str) and os.path.isdir(rimg_in): - self.rimg_dir=rimg_in - self.rimg_files=os.listdir(rimg_in) + self.rimg_dir = rimg_in + self.rimg_files = os.listdir(rimg_in) self.rimg_files.sort() elif isinstance(rimg_in, str) and os.path.isfile(rimg_in): - rimg_dir, rimg_file=os.path.split(rimg_in) - self.rimg_dir=rimg_dir - self.rimg_files=[rimg_file] + rimg_dir, rimg_file = os.path.split(rimg_in) + self.rimg_dir = rimg_dir + self.rimg_files = [rimg_file] else: - print("Invalid rimg_in") sys.exit(1) # Corrected Images - self.cimg_in=cimg_in + self.cimg_in = cimg_in if isinstance(cimg_in, type(None)): - self.cimg_dir=None - self.cimg_files=None + self.cimg_dir = None + self.cimg_files = None else: if isinstance(str(cimg_in), str) and os.path.isdir(cimg_in): - self.cimg_dir=cimg_in - self.cimg_files=os.listdir(cimg_in) + self.cimg_dir = cimg_in + self.cimg_files = os.listdir(cimg_in) self.cimg_files.sort() elif isinstance(str(cimg_in), str) and os.path.isfile(cimg_in): - # if isinstance(cimg_in, str): - cimg_dir, cimg_file=os.path.split(cimg_in) - self.cimg_dir=cimg_dir - self.cimg_files=[cimg_file] + # if isinstance(cimg_in, str): + cimg_dir, cimg_file = os.path.split(cimg_in) + self.cimg_dir = cimg_dir + self.cimg_files = [cimg_file] else: # print(type(cimg_in)) # print(type(str(cimg_in))) # print(str(cimg_in)) - print("Invalid cimg_in") sys.exit(1) # Brain Masks - self.bmsk_in=bmsk_in + self.bmsk_in = bmsk_in if isinstance(bmsk_in, type(None)): - self.bmsk_dir=None - self.bmsk_files=None + self.bmsk_dir = None + self.bmsk_files = None else: if isinstance(bmsk_in, str) and os.path.isdir(bmsk_in): - self.bmsk_dir=bmsk_in - self.bmsk_files=os.listdir(bmsk_in) + self.bmsk_dir = bmsk_in + self.bmsk_files = os.listdir(bmsk_in) self.bmsk_files.sort() elif isinstance(bmsk_in, str) and os.path.isfile(bmsk_in): - bmsk_dir, bmsk_file=os.path.split(bmsk_in) - self.bmsk_dir=bmsk_dir - self.bmsk_files=[bmsk_file] + bmsk_dir, bmsk_file = os.path.split(bmsk_in) + self.bmsk_dir = bmsk_dir + self.bmsk_files = [bmsk_file] else: - print("Invalid bmsk_in") sys.exit(1) - self.cur_rimg_nii=None - self.cur_cimg_nii=None - self.cur_bmsk_nii=None + self.cur_rimg_nii = None + self.cur_cimg_nii = None + self.cur_bmsk_nii = None - self.debug=debug + self.debug = debug def getCurRimgNii(self): return self.cur_rimg_nii @@ -95,113 +90,116 @@ def __len__(self): def __getitem__(self, index): if self.debug: if isinstance(self.rimg_files, list): - print(self.rimg_files[index]) + pass if isinstance(self.cimg_files, list): - print(self.cimg_files[index]) + pass if isinstance(self.bmsk_files, list): - print(self.bmsk_files[index]) + pass - Out=list() + Out = [] if isinstance(self.rimg_files, list): - rimg_nii=nib.load(os.path.join(self.rimg_dir, self.rimg_files[index])) - rimg=np.array(rimg_nii.get_fdata(), dtype=np.float32) + rimg_nii = nib.load(os.path.join(self.rimg_dir, self.rimg_files[index])) + rimg = np.array(rimg_nii.get_fdata(), dtype=np.float32) # 0-1 Normalization - rimg=(rimg-rimg.min())/(rimg.max()-rimg.min()) - rimg=torch.from_numpy(rimg) + rimg = (rimg - rimg.min()) / (rimg.max() - rimg.min()) + rimg = torch.from_numpy(rimg) Out.append(rimg) - self.cur_rimg_nii=rimg_nii + self.cur_rimg_nii = rimg_nii if isinstance(self.cimg_files, list): - cimg_nii=nib.load(os.path.join(self.cimg_dir, self.cimg_files[index])) - cimg=np.array(cimg_nii.get_fdata(), dtype=np.float32) + cimg_nii = nib.load(os.path.join(self.cimg_dir, self.cimg_files[index])) + cimg = np.array(cimg_nii.get_fdata(), dtype=np.float32) # 0-1 Normalization - cimg=(cimg-cimg.min())/(cimg.max()-cimg.min()) - cimg=torch.from_numpy(cimg) + cimg = (cimg - cimg.min()) / (cimg.max() - cimg.min()) + cimg = torch.from_numpy(cimg) Out.append(cimg) - self.cur_cimg_nii=cimg_nii + self.cur_cimg_nii = cimg_nii if "rimg" in locals() and "cimg" in locals(): - bfld=cimg/rimg - bfld[np.isnan(bfld)]=1 - bfld[np.isinf(bfld)]=1 - bfld=torch.from_numpy(bfld) + bfld = cimg / rimg + bfld[np.isnan(bfld)] = 1 + bfld[np.isinf(bfld)] = 1 + bfld = torch.from_numpy(bfld) Out.append(bfld) - if isinstance(self.bmsk_files, list): - bmsk_nii=nib.load(os.path.join(self.bmsk_dir, self.bmsk_files[index])) - bmsk=np.array(bmsk_nii.get_fdata()>0, dtype=np.int64) - bmsk=torch.from_numpy(bmsk) + bmsk_nii = nib.load(os.path.join(self.bmsk_dir, self.bmsk_files[index])) + bmsk = np.array(bmsk_nii.get_fdata() > 0, dtype=np.int64) + bmsk = torch.from_numpy(bmsk) Out.append(bmsk) - self.cur_bmsk_nii=bmsk_nii + self.cur_bmsk_nii = bmsk_nii - if len(Out)==1: - Out=Out[0] + if len(Out) == 1: + Out = Out[0] else: - Out=tuple(Out) + Out = tuple(Out) return Out + class BlockDataset(data.Dataset): - def __init__(self, - rimg=None, - bfld=None, - bmsk=None, - num_slice=3, - rescale_dim=256): + def __init__(self, rimg=None, bfld=None, bmsk=None, num_slice=3, rescale_dim=256): super(BlockDataset, self).__init__() - - if isinstance(bmsk, torch.Tensor) and rimg.shape!=bmsk.shape: - print("Invalid shape of image") + + if isinstance(bmsk, torch.Tensor) and rimg.shape != bmsk.shape: return - raw_shape=rimg.data[0].shape - max_dim=torch.tensor(raw_shape).max() - rescale_factor=float(rescale_dim)/float(max_dim) + raw_shape = rimg.data[0].shape + max_dim = torch.tensor(raw_shape).max() + rescale_factor = float(rescale_dim) / float(max_dim) - uns_rimg=torch.unsqueeze(rimg, 0) - uns_rimg=nn.functional.interpolate(uns_rimg, scale_factor=rescale_factor, mode="trilinear", align_corners=False) - rimg=torch.squeeze(uns_rimg, 0) + uns_rimg = torch.unsqueeze(rimg, 0) + uns_rimg = nn.functional.interpolate( + uns_rimg, scale_factor=rescale_factor, mode="trilinear", align_corners=False + ) + rimg = torch.squeeze(uns_rimg, 0) if isinstance(bfld, torch.Tensor): - uns_bfld=torch.unsqueeze(bfld, 0) - uns_bfld=nn.functional.interpolate(uns_bfld, scale_factor=rescale_factor, mode="trilinear", align_corners=False) - bfld=torch.squeeze(uns_bfld, 0) + uns_bfld = torch.unsqueeze(bfld, 0) + uns_bfld = nn.functional.interpolate( + uns_bfld, + scale_factor=rescale_factor, + mode="trilinear", + align_corners=False, + ) + bfld = torch.squeeze(uns_bfld, 0) if isinstance(bmsk, torch.Tensor): - uns_bmsk=torch.unsqueeze(bmsk.float(), 0) - uns_bmsk=nn.functional.interpolate(uns_bmsk, scale_factor=rescale_factor, mode="nearest") - bmsk=torch.squeeze(uns_bmsk.long(), 0) - - rescale_shape=rimg.data[0].shape - slist0=list() - for i in range(rescale_shape[0]-num_slice+1): - slist0.append(range(i, i+num_slice)) - self.slist0=slist0 - - slist1=list() - for i in range(rescale_shape[1]-num_slice+1): - slist1.append(range(i, i+num_slice)) - self.slist1=slist1 - - slist2=list() - for i in range(rescale_shape[2]-num_slice+1): - slist2.append(range(i, i+num_slice)) - self.slist2=slist2 - - self.rimg=rimg - self.bfld=bfld - self.bmsk=bmsk - - self.batch_size=rimg.shape[0] - self.batch_len=len(self.slist0)+len(self.slist1)+len(self.slist2) - self.num_slice=num_slice - self.rescale_dim=rescale_dim - self.rescale_factor=rescale_factor - self.rescale_shape=rescale_shape - self.raw_shape=raw_shape - + uns_bmsk = torch.unsqueeze(bmsk.float(), 0) + uns_bmsk = nn.functional.interpolate( + uns_bmsk, scale_factor=rescale_factor, mode="nearest" + ) + bmsk = torch.squeeze(uns_bmsk.long(), 0) + + rescale_shape = rimg.data[0].shape + slist0 = [] + for i in range(rescale_shape[0] - num_slice + 1): + slist0.append(range(i, i + num_slice)) + self.slist0 = slist0 + + slist1 = [] + for i in range(rescale_shape[1] - num_slice + 1): + slist1.append(range(i, i + num_slice)) + self.slist1 = slist1 + + slist2 = [] + for i in range(rescale_shape[2] - num_slice + 1): + slist2.append(range(i, i + num_slice)) + self.slist2 = slist2 + + self.rimg = rimg + self.bfld = bfld + self.bmsk = bmsk + + self.batch_size = rimg.shape[0] + self.batch_len = len(self.slist0) + len(self.slist1) + len(self.slist2) + self.num_slice = num_slice + self.rescale_dim = rescale_dim + self.rescale_factor = rescale_factor + self.rescale_shape = rescale_shape + self.raw_shape = raw_shape + def get_rescale_factor(self): return self.rescale_factor @@ -215,87 +213,93 @@ def get_rescale_dim(self): return self.rescale_dim def get_one_directory(self, axis=0): - if axis==0: - ind=range(0, len(self.slist0)) - slist=self.slist0 - elif axis==1: - ind=range(len(self.slist0), len(self.slist0)+len(self.slist1)) - slist=self.slist1 - elif axis==2: - ind=range(len(self.slist0)+len(self.slist1), - len(self.slist0)+len(self.slist1)+len(self.slist2)) - slist=self.slist2 - - slice_weight=np.zeros(slist[-1][-1]+1) + if axis == 0: + ind = range(0, len(self.slist0)) + slist = self.slist0 + elif axis == 1: + ind = range(len(self.slist0), len(self.slist0) + len(self.slist1)) + slist = self.slist1 + elif axis == 2: + ind = range( + len(self.slist0) + len(self.slist1), + len(self.slist0) + len(self.slist1) + len(self.slist2), + ) + slist = self.slist2 + + slice_weight = np.zeros(slist[-1][-1] + 1) for l in slist: - slice_weight[l]+=1 - - slice_data=list() + slice_weight[l] += 1 + + slice_data = [] for i in ind: slice_data.append(self.__getitem__(i)) - + return slice_data, slist, slice_weight def __len__(self): - list_len=self.batch_size*self.batch_len - return list_len - + return self.batch_size * self.batch_len + def __getitem__(self, index): - bind=int(index/self.batch_len) - index=index%self.batch_len - if index0.5) - pr_bmsk_final=fill_holes(pr_bmsk_final) - if ed_iter>0: - pr_bmsk_final=erosion_dilation(pr_bmsk_final, iterations=ed_iter) - + pr_3_bmsk = torch.cat((pr_3_bmsk, torch.unsqueeze(pr_bmsk, 3)), dim=3) + + pr_bmsk = pr_3_bmsk.mean(dim=3) + + pr_bmsk = pr_bmsk.numpy() + pr_bmsk_final = extract_large_comp(pr_bmsk > 0.5) + pr_bmsk_final = fill_holes(pr_bmsk_final) + if ed_iter > 0: + pr_bmsk_final = erosion_dilation(pr_bmsk_final, iterations=ed_iter) + if isinstance(bmsk, torch.Tensor): - bmsk=bmsk.data[0].numpy() - dice=estimate_dice(bmsk, pr_bmsk_final) + bmsk = bmsk.data[0].numpy() + dice = estimate_dice(bmsk, pr_bmsk_final) if verbose: - print(dice) + pass - t1w_nii=volume_dataset.getCurCimgNii() - t1w_path=t1w_nii.get_filename() - t1w_dir, t1w_file=os.path.split(t1w_path) - t1w_name=os.path.splitext(t1w_file)[0] - t1w_name=os.path.splitext(t1w_name)[0] + t1w_nii = volume_dataset.getCurCimgNii() + t1w_path = t1w_nii.get_filename() + t1w_dir, t1w_file = os.path.split(t1w_path) + t1w_name = os.path.splitext(t1w_file)[0] + t1w_name = os.path.splitext(t1w_name)[0] if save_nii: - t1w_aff=t1w_nii.affine - t1w_shape=t1w_nii.shape + t1w_aff = t1w_nii.affine + t1w_shape = t1w_nii.shape if isinstance(nii_outdir, NoneType): nii_outdir = os.getcwd() - out_path=os.path.join(nii_outdir, t1w_name+"_"+suffix+".nii.gz") - write_nifti(np.array(pr_bmsk_final, dtype=np.float32), t1w_aff, t1w_shape, out_path) + out_path = os.path.join(nii_outdir, t1w_name + "_" + suffix + ".nii.gz") + write_nifti( + np.array(pr_bmsk_final, dtype=np.float32), t1w_aff, t1w_shape, out_path + ) if save_dice: - dice_dict[t1w_name]=dice + dice_dict[t1w_name] = dice if save_dice: return dice_dict - + # return output mask return out_path - diff --git a/CPAC/unet/model.py b/CPAC/unet/model.py index d4d8a3f9be..1644aae33e 100755 --- a/CPAC/unet/model.py +++ b/CPAC/unet/model.py @@ -1,299 +1,431 @@ import torch -import torch.nn as nn +from torch import nn -from torch.autograd import Variable def weigths_init(m): if isinstance(m, nn.Conv2d): nn.init.normal_(m.weight.data) nn.init.fill_(m.bias.data) -def Conv3dBlock(dim_in, dim_out, - kernel_size=3, stride=1, padding=1, - bias=True, use_bn=False): + +def Conv3dBlock( + dim_in, dim_out, kernel_size=3, stride=1, padding=1, bias=True, use_bn=False +): if use_bn: return nn.Sequential( - nn.Conv3d(dim_in, dim_out, kernel_size=kernel_size, stride=stride, padding=padding, bias=bias), + nn.Conv3d( + dim_in, + dim_out, + kernel_size=kernel_size, + stride=stride, + padding=padding, + bias=bias, + ), nn.BatchNorm3d(dim_out), nn.LeakyReLU(0.1), - nn.Conv3d(dim_out, dim_out, kernel_size=kernel_size, stride=stride, padding=padding, bias=bias), + nn.Conv3d( + dim_out, + dim_out, + kernel_size=kernel_size, + stride=stride, + padding=padding, + bias=bias, + ), nn.BatchNorm3d(dim_out), - nn.LeakyReLU(0.1) - ) + nn.LeakyReLU(0.1), + ) else: return nn.Sequential( - nn.Conv3d(dim_in, dim_out, kernel_size=kernel_size, stride=stride, padding=padding, bias=bias), + nn.Conv3d( + dim_in, + dim_out, + kernel_size=kernel_size, + stride=stride, + padding=padding, + bias=bias, + ), + nn.LeakyReLU(0.1), + nn.Conv3d( + dim_out, + dim_out, + kernel_size=kernel_size, + stride=stride, + padding=padding, + bias=bias, + ), nn.LeakyReLU(0.1), - nn.Conv3d(dim_out, dim_out, kernel_size=kernel_size, stride=stride, padding=padding, bias=bias), - nn.LeakyReLU(0.1) - ) - -def UpConv3dBlock(dim_in, dim_out, - kernel_size=4, stride=2, padding=1, - bias=False): - return nn.Sequential( - nn.ConvTranspose3d(dim_in, dim_out, kernel_size=kernel_size, stride=stride, padding=padding, bias=bias), - nn.LeakyReLU(0.1) ) -def Conv2dBlock(dim_in, dim_out, - kernel_size=3, stride=1, padding=1, - bias=True, use_bn=True): + +def UpConv3dBlock(dim_in, dim_out, kernel_size=4, stride=2, padding=1, bias=False): + return nn.Sequential( + nn.ConvTranspose3d( + dim_in, + dim_out, + kernel_size=kernel_size, + stride=stride, + padding=padding, + bias=bias, + ), + nn.LeakyReLU(0.1), + ) + + +def Conv2dBlock( + dim_in, dim_out, kernel_size=3, stride=1, padding=1, bias=True, use_bn=True +): if use_bn: return nn.Sequential( - nn.Conv2d(dim_in, dim_out, kernel_size=kernel_size, stride=stride, padding=padding, bias=bias), + nn.Conv2d( + dim_in, + dim_out, + kernel_size=kernel_size, + stride=stride, + padding=padding, + bias=bias, + ), nn.BatchNorm2d(dim_out), nn.LeakyReLU(0.1), - nn.Conv2d(dim_out, dim_out, kernel_size=kernel_size, stride=stride, padding=padding, bias=bias), + nn.Conv2d( + dim_out, + dim_out, + kernel_size=kernel_size, + stride=stride, + padding=padding, + bias=bias, + ), nn.BatchNorm2d(dim_out), - nn.LeakyReLU(0.1) - ) + nn.LeakyReLU(0.1), + ) else: return nn.Sequential( - nn.Conv2d(dim_in, dim_out, kernel_size=kernel_size, stride=stride, padding=padding, bias=bias), + nn.Conv2d( + dim_in, + dim_out, + kernel_size=kernel_size, + stride=stride, + padding=padding, + bias=bias, + ), + nn.LeakyReLU(0.1), + nn.Conv2d( + dim_out, + dim_out, + kernel_size=kernel_size, + stride=stride, + padding=padding, + bias=bias, + ), nn.LeakyReLU(0.1), - nn.Conv2d(dim_out, dim_out, kernel_size=kernel_size, stride=stride, padding=padding, bias=bias), - nn.LeakyReLU(0.1) - ) - -def UpConv2dBlock(dim_in, dim_out, - kernel_size=4, stride=2, padding=1, - bias=True): - return nn.Sequential( - nn.ConvTranspose2d(dim_in, dim_out, kernel_size=kernel_size, stride=stride, padding=padding, bias=bias), - nn.LeakyReLU(0.1) ) + +def UpConv2dBlock(dim_in, dim_out, kernel_size=4, stride=2, padding=1, bias=True): + return nn.Sequential( + nn.ConvTranspose2d( + dim_in, + dim_out, + kernel_size=kernel_size, + stride=stride, + padding=padding, + bias=bias, + ), + nn.LeakyReLU(0.1), + ) + + class UNet3d(nn.Module): - def __init__(self, - dim_in=1, num_conv_block=2, kernel_root=8, - use_bn=False): + def __init__(self, dim_in=1, num_conv_block=2, kernel_root=8, use_bn=False): super(UNet3d, self).__init__() - self.layers=dict() - self.num_conv_block=num_conv_block + self.layers = {} + self.num_conv_block = num_conv_block # Conv Layers for n in range(num_conv_block): - if n==0: - setattr(self, "conv%d" % (n+1), Conv3dBlock(dim_in, kernel_root, use_bn=use_bn)) + if n == 0: + setattr( + self, + "conv%d" % (n + 1), + Conv3dBlock(dim_in, kernel_root, use_bn=use_bn), + ) else: - setattr(self, "conv%d" % (n+1), Conv3dBlock(kernel_root*(2**(n-1)), kernel_root*(2**n), use_bn=use_bn)) + setattr( + self, + "conv%d" % (n + 1), + Conv3dBlock( + kernel_root * (2 ** (n - 1)), + kernel_root * (2**n), + use_bn=use_bn, + ), + ) # UpConv Layers - for n in range(num_conv_block-1): - i=num_conv_block-1-n - setattr(self, "upconv%dto%d" % (i+1, i), UpConv3dBlock(kernel_root*(2**i), kernel_root*(2**(i-1)))) - setattr(self, "conv%dm" % (i), Conv3dBlock(kernel_root*(2**i), kernel_root*(2**(i-1)))) + for n in range(num_conv_block - 1): + i = num_conv_block - 1 - n + setattr( + self, + "upconv%dto%d" % (i + 1, i), + UpConv3dBlock(kernel_root * (2**i), kernel_root * (2 ** (i - 1))), + ) + setattr( + self, + "conv%dm" % (i), + Conv3dBlock(kernel_root * (2**i), kernel_root * (2 ** (i - 1))), + ) setattr(self, "max_pool", nn.MaxPool3d(2)) setattr(self, "out_layer", nn.Conv3d(kernel_root, 2, 3, 1, 1)) - + # Weight Initialization - for m in self.modules(): - if isinstance(m, nn.Conv3d) or isinstance(m, nn.ConvTranspose3d): - m.weight.data.normal_(0, 0.02) - if m.bias is not None: - m.bias.data.zero_() - elif isinstance(m, nn.BatchNorm3d): + for m in self.modules(): + if isinstance(m, (nn.Conv3d, nn.ConvTranspose3d)): + m.weight.data.normal_(0, 0.02) + if m.bias is not None: + m.bias.data.zero_() + elif isinstance(m, nn.BatchNorm3d): m.weight.data.normal_(1.0, 0.02) def forward(self, x): - num_conv_block=self.num_conv_block - conv_out=dict() + num_conv_block = self.num_conv_block + conv_out = {} for n in range(num_conv_block): - if n==0: - conv_out["conv%d" % (n+1)]=getattr(self, "conv%d" % (n+1))(x) + if n == 0: + conv_out["conv%d" % (n + 1)] = getattr(self, "conv%d" % (n + 1))(x) else: - conv_out["conv%d" % (n+1)]=getattr(self, "conv%d" % (n+1))(self.max_pool(conv_out["conv%d" % n])) + conv_out["conv%d" % (n + 1)] = getattr(self, "conv%d" % (n + 1))( + self.max_pool(conv_out["conv%d" % n]) + ) - for n in range(num_conv_block-1): - i=num_conv_block-1-n - tmp=torch.cat( - ( - getattr(self, "upconv%dto%d" % (i+1, i))(conv_out["conv%d" % (i+1)]), - conv_out["conv%d" % (i)] + for n in range(num_conv_block - 1): + i = num_conv_block - 1 - n + tmp = torch.cat( + ( + getattr(self, "upconv%dto%d" % (i + 1, i))( + conv_out["conv%d" % (i + 1)] ), - 1 - ) - out=getattr(self, "conv%dm" % (i))(tmp) + conv_out["conv%d" % (i)], + ), + 1, + ) + out = getattr(self, "conv%dm" % (i))(tmp) - out=self.out_layer(out) + out = self.out_layer(out) if not self.training: - softmax_layer=nn.Softmax(dim=1) - out=softmax_layer(out) + softmax_layer = nn.Softmax(dim=1) + out = softmax_layer(out) return out + class UNet2d(nn.Module): - def __init__(self, - dim_in=6, num_conv_block=3, kernel_root=4, - use_bn=True): + def __init__(self, dim_in=6, num_conv_block=3, kernel_root=4, use_bn=True): super(UNet2d, self).__init__() - self.layers=dict() - self.num_conv_block=num_conv_block + self.layers = {} + self.num_conv_block = num_conv_block # Conv Layers for n in range(num_conv_block): - if n==0: - setattr(self, "conv%d" % (n+1), Conv2dBlock(dim_in, kernel_root, use_bn=use_bn)) + if n == 0: + setattr( + self, + "conv%d" % (n + 1), + Conv2dBlock(dim_in, kernel_root, use_bn=use_bn), + ) else: - setattr(self, "conv%d" % (n+1), Conv2dBlock(kernel_root*(2**(n-1)), kernel_root*(2**n), use_bn=use_bn)) + setattr( + self, + "conv%d" % (n + 1), + Conv2dBlock( + kernel_root * (2 ** (n - 1)), + kernel_root * (2**n), + use_bn=use_bn, + ), + ) # UpConv Layers - for n in range(num_conv_block-1): - i=num_conv_block-1-n - setattr(self, "upconv%dto%d" % (i+1, i), UpConv2dBlock(kernel_root*(2**i), kernel_root*(2**(i-1)))) - setattr(self, "conv%dm" % (i), Conv2dBlock(kernel_root*(2**i), kernel_root*(2**(i-1)))) + for n in range(num_conv_block - 1): + i = num_conv_block - 1 - n + setattr( + self, + "upconv%dto%d" % (i + 1, i), + UpConv2dBlock(kernel_root * (2**i), kernel_root * (2 ** (i - 1))), + ) + setattr( + self, + "conv%dm" % (i), + Conv2dBlock(kernel_root * (2**i), kernel_root * (2 ** (i - 1))), + ) setattr(self, "max_pool", nn.MaxPool2d(2)) - #setattr(self, "out_layer", nn.Sequential(nn.Conv2d(kernel_root, 2, 3, 1, 1), nn.Softmax2d())) + # setattr(self, "out_layer", nn.Sequential(nn.Conv2d(kernel_root, 2, 3, 1, 1), nn.Softmax2d())) setattr(self, "out_layer", nn.Conv2d(kernel_root, 2, 3, 1, 1)) - + # Weight Initialization self.apply(self.weights_init) def weights_init(self, m): - if isinstance(m, nn.Conv2d) or isinstance(m, nn.ConvTranspose2d): - m.weight.data.normal_(0, 0.02) - if m.bias is not None: - m.bias.data.zero_() - elif isinstance(m, nn.BatchNorm2d): + if isinstance(m, (nn.Conv2d, nn.ConvTranspose2d)): + m.weight.data.normal_(0, 0.02) + if m.bias is not None: + m.bias.data.zero_() + elif isinstance(m, nn.BatchNorm2d): m.weight.data.normal_(1.0, 0.02) def forward(self, x): - num_conv_block=self.num_conv_block - conv_out=dict() + num_conv_block = self.num_conv_block + conv_out = {} for n in range(num_conv_block): - if n==0: - conv_out["conv%d" % (n+1)]=getattr(self, "conv%d" % (n+1))(x) + if n == 0: + conv_out["conv%d" % (n + 1)] = getattr(self, "conv%d" % (n + 1))(x) else: - conv_out["conv%d" % (n+1)]=getattr(self, "conv%d" % (n+1))(self.max_pool(conv_out["conv%d" % n])) - - for n in range(num_conv_block-1): - i=num_conv_block-1-n - if n==0: - tmp=torch.cat( - ( - getattr(self, "upconv%dto%d" % (i+1, i))(conv_out["conv%d" % (i+1)]), - conv_out["conv%d" % (i)] + conv_out["conv%d" % (n + 1)] = getattr(self, "conv%d" % (n + 1))( + self.max_pool(conv_out["conv%d" % n]) + ) + + for n in range(num_conv_block - 1): + i = num_conv_block - 1 - n + if n == 0: + tmp = torch.cat( + ( + getattr(self, "upconv%dto%d" % (i + 1, i))( + conv_out["conv%d" % (i + 1)] ), - 1 - ) + conv_out["conv%d" % (i)], + ), + 1, + ) else: - tmp=torch.cat( - ( - getattr(self, "upconv%dto%d" % (i+1, i))(out),#(conv_out["conv%d" % (i+1)]), - conv_out["conv%d" % (i)] - ), - 1 - ) + tmp = torch.cat( + ( + getattr(self, "upconv%dto%d" % (i + 1, i))( + out + ), # (conv_out["conv%d" % (i+1)]), + conv_out["conv%d" % (i)], + ), + 1, + ) - out=getattr(self, "conv%dm" % (i))(tmp) + out = getattr(self, "conv%dm" % (i))(tmp) + + return self.out_layer(out) - out=self.out_layer(out) - return out class MultiSliceBcUNet(nn.Module): - def __init__(self, - num_slice=6, in_shape=256, - num_conv_block=4, kernel_root=16, - use_bn=True): + def __init__( + self, num_slice=6, in_shape=256, num_conv_block=4, kernel_root=16, use_bn=True + ): super(MultiSliceBcUNet, self).__init__() - + for i in range(num_slice): - setattr(self, "slice%d" % (i+1), + setattr( + self, + "slice%d" % (i + 1), nn.Sequential( - UNet2d(dim_in=num_slice, num_conv_block=num_conv_block, kernel_root=kernel_root, use_bn=use_bn), + UNet2d( + dim_in=num_slice, + num_conv_block=num_conv_block, + kernel_root=kernel_root, + use_bn=use_bn, + ), nn.Conv2d(2, 1, kernel_size=1, stride=1, padding=0), - nn.ReLU() - ) - ) + nn.ReLU(), + ), + ) - self.num_slice=num_slice + self.num_slice = num_slice def forward(self, x): for i in range(self.num_slice): - pho=getattr(self, "slice%d" % (i+1))(x) - if i==0: - out=pho + pho = getattr(self, "slice%d" % (i + 1))(x) + if i == 0: + out = pho else: - out=torch.cat( - ( - out, - pho - ), - 1 - ) + out = torch.cat((out, pho), 1) return out - + def freeze(self): for param in model.parameters(): - param.requires_grad=False - + param.requires_grad = False + def unfreeze(self): for param in model.parameters(): - param.requires_grad=True + param.requires_grad = True + class MultiSliceSsUNet(nn.Module): - def __init__(self, - num_slice=6, in_shape=256, - num_conv_block=5, kernel_root=16, - use_bn=True): + def __init__( + self, num_slice=6, in_shape=256, num_conv_block=5, kernel_root=16, use_bn=True + ): super(MultiSliceSsUNet, self).__init__() - + for i in range(num_slice): - setattr(self, "slice%d" % (i+1), - UNet2d(dim_in=num_slice, num_conv_block=num_conv_block, kernel_root=kernel_root, use_bn=use_bn) - ) + setattr( + self, + "slice%d" % (i + 1), + UNet2d( + dim_in=num_slice, + num_conv_block=num_conv_block, + kernel_root=kernel_root, + use_bn=use_bn, + ), + ) - self.num_slice=num_slice + self.num_slice = num_slice def forward(self, x): for i in range(self.num_slice): - pho=torch.unsqueeze(getattr(self, "slice%d" % (i+1))(x), 2) - if i==0: - out=pho + pho = torch.unsqueeze(getattr(self, "slice%d" % (i + 1))(x), 2) + if i == 0: + out = pho else: - out=torch.cat( - ( - out, - pho - ), - 2 - ) + out = torch.cat((out, pho), 2) return out - + def freeze(self): for param in model.parameters(): - param.requires_grad=False - + param.requires_grad = False + def unfreeze(self): for param in model.parameters(): - param.requires_grad=True + param.requires_grad = True + class MultiSliceModel(nn.Module): - def __init__(self, - num_slice=6, in_shape=256, - bc_num_conv_block=3, bc_kernel_root=8, - ss_num_conv_block=4, ss_kernel_root=8, - use_bn=True): + def __init__( + self, + num_slice=6, + in_shape=256, + bc_num_conv_block=3, + bc_kernel_root=8, + ss_num_conv_block=4, + ss_kernel_root=8, + use_bn=True, + ): super(MultiSliceModel, self).__init__() - - self.BcUNet=MultiSliceBcUNet(num_slice=num_slice, in_shape=in_shape, - num_conv_block=bc_num_conv_block, kernel_root=bc_kernel_root, - use_bn=use_bn) - self.SsUNet=MultiSliceSsUNet(num_slice=num_slice, in_shape=in_shape, - num_conv_block=ss_num_conv_block, kernel_root=ss_kernel_root, - use_bn=use_bn) - - def forward(self, x, model='forward_full'): - if model=="forward_bc_part": - b_field=self.BcUNet(x) - out=b_field - elif model=="forward_ss_part": - b_msk=self.SsUNet(x) - out=b_msk - elif model=="forward_full": - b_field=self.BcUNet(x) - x=x*b_field - - out=self.SsUNet(x) + + self.BcUNet = MultiSliceBcUNet( + num_slice=num_slice, + in_shape=in_shape, + num_conv_block=bc_num_conv_block, + kernel_root=bc_kernel_root, + use_bn=use_bn, + ) + self.SsUNet = MultiSliceSsUNet( + num_slice=num_slice, + in_shape=in_shape, + num_conv_block=ss_num_conv_block, + kernel_root=ss_kernel_root, + use_bn=use_bn, + ) + + def forward(self, x, model="forward_full"): + if model == "forward_bc_part": + b_field = self.BcUNet(x) + out = b_field + elif model == "forward_ss_part": + b_msk = self.SsUNet(x) + out = b_msk + elif model == "forward_full": + b_field = self.BcUNet(x) + x = x * b_field + + out = self.SsUNet(x) return out diff --git a/CPAC/unet/tests/test_torch.py b/CPAC/unet/tests/test_torch.py index bf9717e3e2..f1468e8d2f 100644 --- a/CPAC/unet/tests/test_torch.py +++ b/CPAC/unet/tests/test_torch.py @@ -14,14 +14,15 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -"""Test torch installation""" +"""Test torch installation.""" import os from unittest.mock import MagicMock, patch + import pytest -@pytest.mark.parametrize('readonly', [False, True]) -@pytest.mark.parametrize('workdir', [False, True]) +@pytest.mark.parametrize("readonly", [False, True]) +@pytest.mark.parametrize("workdir", [False, True]) def test_import_torch(monkeypatch, readonly, tmp_path, workdir): """ Test that ``torch`` successfully imports after being installed dynamically. @@ -32,31 +33,34 @@ def test_import_torch(monkeypatch, readonly, tmp_path, workdir): if readonly: # set PYTHONUSERBASE to a readonly directory os.chmod(tmp_path, 0o444) - monkeypatch.setenv('PYTHONUSERBASE', tmp_path) + monkeypatch.setenv("PYTHONUSERBASE", tmp_path) if workdir: - os.environ['CPAC_WORKDIR'] = str(tmp_path) + os.environ["CPAC_WORKDIR"] = str(tmp_path) else: - if 'CPAC_WORKDIR' in os.environ: - del os.environ['CPAC_WORKDIR'] + if "CPAC_WORKDIR" in os.environ: + del os.environ["CPAC_WORKDIR"] # pylint: disable=import-error,unused-import,wrong-import-order - from CPAC import unet - import torch -@pytest.mark.parametrize('error', [ImportError, ModuleNotFoundError, None]) +@pytest.mark.parametrize("error", [ImportError, ModuleNotFoundError, None]) def test_validate_unet(error): """Test that pipeline validation throws error if torch is not - installable""" + installable. + """ if error: import_module = MagicMock(side_effect=error()) - with patch('importlib.import_module', import_module): + with patch("importlib.import_module", import_module): with pytest.raises(OSError) as os_error: from CPAC.utils.configuration import Preconfiguration - monkey = Preconfiguration('monkey') + + monkey = Preconfiguration("monkey") assert "U-Net" in str(os_error) else: from CPAC.utils.configuration import Preconfiguration - monkey = Preconfiguration('monkey') - assert 'unet' in [using.lower() for using in monkey[ - 'anatomical_preproc', 'brain_extraction', 'using']] + + monkey = Preconfiguration("monkey") + assert "unet" in [ + using.lower() + for using in monkey["anatomical_preproc", "brain_extraction", "using"] + ] diff --git a/CPAC/utils/__init__.py b/CPAC/utils/__init__.py index c3d73c22b2..9ff85f2446 100644 --- a/CPAC/utils/__init__.py +++ b/CPAC/utils/__init__.py @@ -1,38 +1,5 @@ -from . import extract_data_multiscan -from . import create_fsl_model -from . import extract_parameters -from . import build_data_config -from .interfaces import function, masktool -from .extract_data import run +from .configuration import Configuration, check_pname, set_subject from .datatypes import ListFromItem -from .configuration import check_pname, Configuration, set_subject +from .interfaces import function -from .utils import ( - get_zscore, - get_fisher_zscore, - compute_fisher_z_score, - get_operand_string, - get_roi_num_list, - safe_shape, - extract_one_d, - extract_txt, - zscore, - correlation, - check, - check_random_state, - try_fetch_parameter, - get_scan_params, - get_tr, - check_tr, - find_files, - extract_output_mean, - create_output_mean_csv, - pick_wm, - check_command_path, - check_system_deps, - check_config_resources, - repickle, -) - -__all__ = ['check_pname', 'Configuration', 'function', 'ListFromItem', - 'set_subject'] +__all__ = ["check_pname", "Configuration", "function", "ListFromItem", "set_subject"] diff --git a/CPAC/utils/bids_utils.py b/CPAC/utils/bids_utils.py index 88aaaf6f9c..844a4cd6e8 100755 --- a/CPAC/utils/bids_utils.py +++ b/CPAC/utils/bids_utils.py @@ -29,41 +29,39 @@ def bids_decode_fname(file_path, dbg=False, raise_error=True): fname = os.path.basename(file_path) # first lets make sure that we know how to handle the file - if 'nii' not in fname.lower() and 'json' not in fname.lower(): - raise IOError("File (%s) does not appear to be" % fname + - "a nifti or json file") + if "nii" not in fname.lower() and "json" not in fname.lower(): + raise IOError( + "File (%s) does not appear to be" % fname + "a nifti or json file" + ) if dbg: - print("parsing %s" % file_path) + pass # first figure out if there is a site directory level, this isn't # specified in BIDS currently, but hopefully will be in the future - file_path_vals = os.path.dirname(file_path).split('/') - sub = [s for s in file_path_vals if 'sub-' in s] + file_path_vals = os.path.dirname(file_path).split("/") + sub = [s for s in file_path_vals if "sub-" in s] if dbg: - print("found subject %s in %s" % (sub, str(file_path_vals))) + pass if len(sub) > 1: - print("Odd that there is more than one subject directory" + - "in (%s), does the filename conform to" % file_path + - " BIDS format?") + pass if sub: sub_ndx = file_path_vals.index(sub[0]) if sub_ndx > 0 and file_path_vals[sub_ndx - 1]: if dbg: - print("setting site to %s" % (file_path_vals[sub_ndx - 1])) + pass f_dict["site"] = file_path_vals[sub_ndx - 1] else: f_dict["site"] = "none" elif file_path_vals[-1]: if dbg: - print("looking for subject id didn't pan out settling for last"+ - "subdir %s" % (str(file_path_vals[-1]))) + pass f_dict["site"] = file_path_vals[-1] else: f_dict["site"] = "none" - f_dict["site"] = re.sub(r'[\s\-\_]+', '', f_dict["site"]) + f_dict["site"] = re.sub(r"[\s\-\_]+", "", f_dict["site"]) fname = fname.split(".")[0] # convert the filename string into a dictionary to pull out the other @@ -78,28 +76,34 @@ def bids_decode_fname(file_path, dbg=False, raise_error=True): f_dict["scantype"] = key_val_pair.split(".")[0] if "scantype" not in f_dict: - msg = "Filename ({0}) does not appear to contain" \ - " scan type, does it conform to the BIDS format?".format(fname) + msg = ( + "Filename ({0}) does not appear to contain" + " scan type, does it conform to the BIDS format?".format(fname) + ) if raise_error: raise ValueError(msg) else: - print(msg) + pass elif not f_dict["scantype"]: - msg = "Filename ({0}) does not appear to contain" \ - " scan type, does it conform to the BIDS format?".format(fname) + msg = ( + "Filename ({0}) does not appear to contain" + " scan type, does it conform to the BIDS format?".format(fname) + ) if raise_error: raise ValueError(msg) else: - print(msg) + pass else: - if 'bold' in f_dict["scantype"] and not f_dict["task"]: - msg = "Filename ({0}) is a BOLD file, but " \ - "doesn't contain a task, does it conform to the" \ - " BIDS format?".format(fname) + if "bold" in f_dict["scantype"] and not f_dict["task"]: + msg = ( + "Filename ({0}) is a BOLD file, but " + "doesn't contain a task, does it conform to the" + " BIDS format?".format(fname) + ) if raise_error: raise ValueError(msg) else: - print(msg) + pass return f_dict @@ -124,8 +128,10 @@ def bids_entities_from_filename(filename): ['sub-0001', 'ses-NFB3', 'task-MSIT', 'bold'] """ return ( - filename.split('/')[-1] if '/' in filename else filename - ).split('.')[0].split('_') + (filename.split("/")[-1] if "/" in filename else filename) + .split(".")[0] + .split("_") + ) def bids_match_entities(file_list, entities, suffix): @@ -167,38 +173,48 @@ def bids_match_entities(file_list, entities, suffix): - task-PEER2 """ matches = [ - file for file in file_list if ( - f'_{entities}_' in '_'.join( - bids_entities_from_filename(file) - ) and bids_entities_from_filename(file)[-1] == suffix - ) or bids_entities_from_filename(file)[-1] != suffix + file + for file in file_list + if ( + f"_{entities}_" in "_".join(bids_entities_from_filename(file)) + and bids_entities_from_filename(file)[-1] == suffix + ) + or bids_entities_from_filename(file)[-1] != suffix ] if file_list and not matches: - pp_file_list = '\n'.join([f'- {file}' for file in file_list]) - error_message = ' '.join([ - 'No match found for provided', - 'entity' if len(entities.split('_')) == 1 else 'entities', - f'"{entities}" in\n{pp_file_list}' - ]) - partial_matches = [match.group() for match in [ - re.search(re.compile(f'[^_]*{entities}[^_]*'), file) for - file in file_list - ] if match is not None] + pp_file_list = "\n".join([f"- {file}" for file in file_list]) + error_message = " ".join( + [ + "No match found for provided", + "entity" if len(entities.split("_")) == 1 else "entities", + f'"{entities}" in\n{pp_file_list}', + ] + ) + partial_matches = [ + match.group() + for match in [ + re.search(re.compile(f"[^_]*{entities}[^_]*"), file) + for file in file_list + ] + if match is not None + ] if partial_matches: if len(partial_matches) == 1: error_message += f'\nPerhaps you meant "{partial_matches[0]}"?' else: - error_message = '\n'.join([ - error_message, - 'Perhaps you meant one of these?', - *[f'- {match}' for match in partial_matches] - ]) + error_message = "\n".join( + [ + error_message, + "Perhaps you meant one of these?", + *[f"- {match}" for match in partial_matches], + ] + ) raise LookupError(error_message) return matches def bids_remove_entity(name, key): - """Remove an entity from a BIDS string by key + """Remove an entity from a BIDS string by key. Parameters ---------- @@ -219,8 +235,11 @@ def bids_remove_entity(name, key): >>> bids_remove_entity('atlas-Yeo_space-MNI152NLin6_res-2x2x2', 'res') 'atlas-Yeo_space-MNI152NLin6' """ - return '_'.join(entity for entity in bids_entities_from_filename(name) - if not entity.startswith(f'{key.rstrip("-")}-')) + return "_".join( + entity + for entity in bids_entities_from_filename(name) + if not entity.startswith(f'{key.rstrip("-")}-') + ) def bids_retrieve_params(bids_config_dict, f_dict, dbg=False): @@ -249,23 +268,21 @@ def bids_retrieve_params(bids_config_dict, f_dict, dbg=False): t_dict = bids_config_dict # pointer to current dictionary # try to populate the configuration using information # already in the list - for level in ['scantype', 'site', 'sub', 'ses', 'task', 'acq', - 'rec', 'dir', 'run']: + for level in ["scantype", "site", "sub", "ses", "task", "acq", "rec", "dir", "run"]: if level in f_dict: key = "-".join([level, f_dict[level]]) else: key = "-".join([level, "none"]) if dbg: - print(key) + pass # if the key doesn't exist in the config dictionary, check to see if # the generic key exists and return that if key in t_dict: t_dict = t_dict[key] else: if dbg: - print("Couldn't find %s, so going with %s" % (key, - "-".join([level, "none"]))) + pass key = "-".join([level, "none"]) if key in t_dict: t_dict = t_dict[key] @@ -276,16 +293,16 @@ def bids_retrieve_params(bids_config_dict, f_dict, dbg=False): # sidecar files if dbg: - print(t_dict) + pass for key in t_dict.keys(): - if 'RepetitionTime' in key: + if "RepetitionTime" in key: params = t_dict break for k, v in params.items(): if isinstance(v, str): - params[k] = v.encode('ascii', errors='ignore') + params[k] = v.encode("ascii", errors="ignore") return params @@ -304,7 +321,6 @@ def bids_parse_sidecar(config_dict, dbg=False, raise_error=True): :return: a dictionary that maps parameters to components from BIDS filenames such as sub, sess, run, acq, and scan type """ - # we are going to build a large-scale data structure, consisting of many # levels of dictionaries to hold the data. bids_config_dict = {} @@ -312,14 +328,13 @@ def bids_parse_sidecar(config_dict, dbg=False, raise_error=True): # initialize 'default' entries, this essentially is a pointer traversal # of the dictionary t_dict = bids_config_dict - for level in ['scantype', 'site', 'sub', 'ses', 'task', - 'acq', 'rec', 'dir', 'run']: - key = '-'.join([level, 'none']) + for level in ["scantype", "site", "sub", "ses", "task", "acq", "rec", "dir", "run"]: + key = "-".join([level, "none"]) t_dict[key] = {} t_dict = t_dict[key] if dbg: - print(bids_config_dict) + pass # get the paths to the json yaml files in config_dict, the paths contain # the information needed to map the parameters from the jsons (the vals @@ -327,18 +342,14 @@ def bids_parse_sidecar(config_dict, dbg=False, raise_error=True): # by the number of path components, so that we can iterate from the outer # most path to inner-most, which will help us address the BIDS inheritance # principle - config_paths = sorted( - list(config_dict.keys()), - key=lambda p: len(p.split('/')) - ) + config_paths = sorted(config_dict.keys(), key=lambda p: len(p.split("/"))) if dbg: - print(config_paths) + pass for cp in config_paths: - if dbg: - print("processing %s" % (cp)) + pass # decode the filepath into its various components as defined by BIDS f_dict = bids_decode_fname(cp, raise_error=raise_error) @@ -365,11 +376,13 @@ def bids_parse_sidecar(config_dict, dbg=False, raise_error=True): try: bids_config.update(t_config) except ValueError: - err = "\n[!] Could not properly parse the AWS S3 path provided " \ - "- please double-check the bucket and the path.\n\nNote: " \ - "This could either be an issue with the path or the way " \ - "the data is organized in the directory. You can also " \ - "try providing a specific site sub-directory.\n\n" + err = ( + "\n[!] Could not properly parse the AWS S3 path provided " + "- please double-check the bucket and the path.\n\nNote: " + "This could either be an issue with the path or the way " + "the data is organized in the directory. You can also " + "try providing a specific site sub-directory.\n\n" + ) raise ValueError(err) # now put the configuration in the data structure, by first iterating @@ -379,8 +392,17 @@ def bids_parse_sidecar(config_dict, dbg=False, raise_error=True): # e.g. run-1, run-2, ... will all map to run-none if no jsons # explicitly define values for those runs t_dict = bids_config_dict # pointer to current dictionary - for level in ['scantype', 'site', 'sub', 'ses', 'task', 'acq', - 'rec', 'dir', 'run']: + for level in [ + "scantype", + "site", + "sub", + "ses", + "task", + "acq", + "rec", + "dir", + "run", + ]: if level in f_dict: key = "-".join([level, f_dict[level]]) else: @@ -393,7 +415,7 @@ def bids_parse_sidecar(config_dict, dbg=False, raise_error=True): t_dict.update(bids_config) - return(bids_config_dict) + return bids_config_dict def bids_shortest_entity(file_list): @@ -419,9 +441,7 @@ def bids_shortest_entity(file_list): ... ]) 's3://fake/data/sub-001_ses-001_bold.nii.gz' """ - entity_lists = [ - bids_entities_from_filename(filename) for filename in file_list - ] + entity_lists = [bids_entities_from_filename(filename) for filename in file_list] if not entity_lists: return None @@ -429,8 +449,9 @@ def bids_shortest_entity(file_list): shortest_len = min(len(entity_list) for entity_list in entity_lists) shortest_list = [ - file_list[i] for i in range(len(file_list)) if - len(entity_lists[i]) == shortest_len + file_list[i] + for i in range(len(file_list)) + if len(entity_lists[i]) == shortest_len ] return shortest_list[0] if len(shortest_list) == 1 else shortest_list @@ -439,15 +460,19 @@ def bids_shortest_entity(file_list): def gen_bids_outputs_sublist(base_path, paths_list, key_list, creds_path): import copy - func_keys = ["functional_to_anat_linear_xfm", "motion_params", - "movement_parameters", "motion_correct"] + func_keys = [ + "functional_to_anat_linear_xfm", + "motion_params", + "movement_parameters", + "motion_correct", + ] top_keys = list(set(key_list) - set(func_keys)) bot_keys = list(set(key_list).intersection(func_keys)) subjdict = {} - if not base_path.endswith('/'): - base_path = base_path + '/' + if not base_path.endswith("/"): + base_path = base_path + "/" # output directories are a bit different than standard BIDS, so # we handle things differently @@ -457,10 +482,10 @@ def gen_bids_outputs_sublist(base_path, paths_list, key_list, creds_path): # find the participant and session info which should be at # some level in the path - path_base = p.replace(base_path, '') + path_base = p.replace(base_path, "") - subj_info = path_base.split('/')[0] - resource = path_base.split('/')[1] + subj_info = path_base.split("/")[0] + resource = path_base.split("/")[1] if resource not in key_list: continue @@ -472,14 +497,13 @@ def gen_bids_outputs_sublist(base_path, paths_list, key_list, creds_path): subjdict[subj_info]["creds_path"] = creds_path if resource in func_keys: - run_info = path_base.split('/')[2] + run_info = path_base.split("/")[2] if "funcs" not in subjdict[subj_info]: subjdict[subj_info]["funcs"] = {} if run_info not in subjdict[subj_info]["funcs"]: - subjdict[subj_info]["funcs"][run_info] = {'run_info': run_info} + subjdict[subj_info]["funcs"][run_info] = {"run_info": run_info} if resource in subjdict[subj_info]["funcs"][run_info]: - print("warning resource %s already exists in subjdict ??" % - (resource)) + pass subjdict[subj_info]["funcs"][run_info][resource] = p else: subjdict[subj_info][resource] = p @@ -489,7 +513,6 @@ def gen_bids_outputs_sublist(base_path, paths_list, key_list, creds_path): missing = 0 for tkey in top_keys: if tkey not in subj_res: - print("%s not found for %s" % (tkey, subj_info)) missing += 1 break @@ -497,14 +520,9 @@ def gen_bids_outputs_sublist(base_path, paths_list, key_list, creds_path): for func_key, func_res in subj_res["funcs"].items(): for bkey in bot_keys: if bkey not in func_res: - print("%s not found for %s" % (bkey, - func_key)) missing += 1 break if missing == 0: - print("adding: %s, %s, %d" % (subj_info, - func_key, - len(sublist))) tdict = copy.deepcopy(subj_res) del tdict["funcs"] tdict.update(func_res) @@ -512,8 +530,15 @@ def gen_bids_outputs_sublist(base_path, paths_list, key_list, creds_path): return sublist -def bids_gen_cpac_sublist(bids_dir, paths_list, config_dict, creds_path, - dbg=False, raise_error=True, only_one_anat=True): +def bids_gen_cpac_sublist( + bids_dir, + paths_list, + config_dict, + creds_path, + dbg=False, + raise_error=True, + only_one_anat=True, +): """ Generates a CPAC formatted subject list from information contained in a BIDS formatted set of data. @@ -559,51 +584,43 @@ def bids_gen_cpac_sublist(bids_dir, paths_list, config_dict, creds_path, to be processed """ if dbg: - print("gen_bids_sublist called with:") - print(" bids_dir: {0}".format(bids_dir)) - print(" # paths: {0}".format(str(len(paths_list)))) - print(" config_dict: {0}".format( - "missing" if not config_dict else "found") - ) - print(" creds_path: {0}".format(creds_path)) + pass # if configuration information is not desired, config_dict will be empty, # otherwise parse the information in the sidecar json files into a dict # we can use to extract data for our nifti files if config_dict: - bids_config_dict = bids_parse_sidecar(config_dict, - raise_error=raise_error) + bids_config_dict = bids_parse_sidecar(config_dict, raise_error=raise_error) subdict = {} for p in paths_list: if bids_dir in p: str_list = p.split(bids_dir) val = str_list[0] - val = val.rsplit('/') + val = val.rsplit("/") val = val[0] else: - str_list = p.split('/') + str_list = p.split("/") val = str_list[0] - if 'sub-' not in val: + if "sub-" not in val: continue p = p.rstrip() f = os.path.basename(p) if f.endswith(".nii") or f.endswith(".nii.gz"): - f_dict = bids_decode_fname(p, raise_error=raise_error) if config_dict: - t_params = bids_retrieve_params(bids_config_dict, - f_dict) + t_params = bids_retrieve_params(bids_config_dict, f_dict) if not t_params: - print("Did not receive any parameters for %s," % (p) + - " is this a problem?") + pass - task_info = {"scan": os.path.join(bids_dir, p), - "scan_parameters": t_params.copy()} + task_info = { + "scan": os.path.join(bids_dir, p), + "scan_parameters": t_params.copy(), + } else: task_info = os.path.join(bids_dir, p) @@ -611,8 +628,9 @@ def bids_gen_cpac_sublist(bids_dir, paths_list, config_dict, creds_path, f_dict["ses"] = "1" if "sub" not in f_dict: - raise IOError("sub not found in %s," % (p) + - " perhaps it isn't in BIDS format?") + raise IOError( + "sub not found in %s," % (p) + " perhaps it isn't in BIDS format?" + ) if f_dict["sub"] not in subdict: subdict[f_dict["sub"]] = {} @@ -620,82 +638,80 @@ def bids_gen_cpac_sublist(bids_dir, paths_list, config_dict, creds_path, subjid = "-".join(["sub", f_dict["sub"]]) if f_dict["ses"] not in subdict[f_dict["sub"]]: - subdict[f_dict["sub"]][f_dict["ses"]] = \ - {"creds_path": creds_path, - "site_id": "-".join(["site", f_dict["site"]]), - "subject_id": subjid, - "unique_id": "-".join(["ses", f_dict["ses"]])} + subdict[f_dict["sub"]][f_dict["ses"]] = { + "creds_path": creds_path, + "site_id": "-".join(["site", f_dict["site"]]), + "subject_id": subjid, + "unique_id": "-".join(["ses", f_dict["ses"]]), + } if "T1w" in f_dict["scantype"] or "T2w" in f_dict["scantype"]: - if "lesion" in f_dict.keys() and "mask" in f_dict['lesion']: - if "lesion_mask" not in \ - subdict[f_dict["sub"]][f_dict["ses"]]: - subdict[f_dict["sub"]][f_dict["ses"]]["lesion_mask"] = \ - task_info["scan"] + if "lesion" in f_dict.keys() and "mask" in f_dict["lesion"]: + if "lesion_mask" not in subdict[f_dict["sub"]][f_dict["ses"]]: + subdict[f_dict["sub"]][f_dict["ses"]][ + "lesion_mask" + ] = task_info["scan"] else: - print("Lesion mask file (%s) already found" % - (subdict[f_dict["sub"]] - [f_dict["ses"]] - ["lesion_mask"]) + - " for (%s:%s) discarding %s" % - (f_dict["sub"], f_dict["ses"], p)) + pass # TODO deal with scan parameters anatomical if "anat" not in subdict[f_dict["sub"]][f_dict["ses"]]: subdict[f_dict["sub"]][f_dict["ses"]]["anat"] = {} - if f_dict["scantype"] not in subdict[f_dict["sub"]][ - f_dict["ses"] - ]["anat"]: + if ( + f_dict["scantype"] + not in subdict[f_dict["sub"]][f_dict["ses"]]["anat"] + ): if only_one_anat: subdict[f_dict["sub"]][f_dict["ses"]]["anat"][ f_dict["scantype"] ] = task_info["scan"] if config_dict else task_info else: subdict[f_dict["sub"]][f_dict["ses"]]["anat"][ - f_dict["scantype"]] = [] + f_dict["scantype"] + ] = [] if not only_one_anat: subdict[f_dict["sub"]][f_dict["ses"]]["anat"][ - f_dict["scantype"]].append( - task_info["scan"] if config_dict else task_info) + f_dict["scantype"] + ].append(task_info["scan"] if config_dict else task_info) if "bold" in f_dict["scantype"]: task_key = f_dict["task"] if "run" in f_dict: - task_key = "_".join([task_key, - "-".join(["run", f_dict["run"]])]) + task_key = "_".join([task_key, "-".join(["run", f_dict["run"]])]) if "acq" in f_dict: - task_key = "_".join([task_key, - "-".join(["acq", f_dict["acq"]])]) + task_key = "_".join([task_key, "-".join(["acq", f_dict["acq"]])]) if "func" not in subdict[f_dict["sub"]][f_dict["ses"]]: subdict[f_dict["sub"]][f_dict["ses"]]["func"] = {} - if task_key not in \ - subdict[f_dict["sub"]][f_dict["ses"]]["func"]: - + if task_key not in subdict[f_dict["sub"]][f_dict["ses"]]["func"]: if not isinstance(task_info, dict): task_info = {"scan": task_info} subdict[f_dict["sub"]][f_dict["ses"]]["func"][task_key] = task_info else: - print("Func file (%s)" % - subdict[f_dict["sub"]][f_dict["ses"]]["func"][task_key] + - " already found for ( % s: %s: % s) discarding % s" % ( - f_dict["sub"], - f_dict["ses"], - task_key, - p)) + pass if "phase" in f_dict["scantype"]: if "fmap" not in subdict[f_dict["sub"]][f_dict["ses"]]: subdict[f_dict["sub"]][f_dict["ses"]]["fmap"] = {} - if f_dict["scantype"] not in subdict[f_dict["sub"]][f_dict["ses"]]["fmap"]: - subdict[f_dict["sub"]][f_dict["ses"]]["fmap"][f_dict["scantype"]] = task_info + if ( + f_dict["scantype"] + not in subdict[f_dict["sub"]][f_dict["ses"]]["fmap"] + ): + subdict[f_dict["sub"]][f_dict["ses"]]["fmap"][ + f_dict["scantype"] + ] = task_info if "magnitude" in f_dict["scantype"]: if "fmap" not in subdict[f_dict["sub"]][f_dict["ses"]]: subdict[f_dict["sub"]][f_dict["ses"]]["fmap"] = {} - if f_dict["scantype"] not in subdict[f_dict["sub"]][f_dict["ses"]]["fmap"]: - subdict[f_dict["sub"]][f_dict["ses"]]["fmap"][f_dict["scantype"]] = task_info + if ( + f_dict["scantype"] + not in subdict[f_dict["sub"]][f_dict["ses"]]["fmap"] + ): + subdict[f_dict["sub"]][f_dict["ses"]]["fmap"][ + f_dict["scantype"] + ] = task_info if "epi" in f_dict["scantype"]: pe_dir = f_dict["dir"] @@ -703,12 +719,13 @@ def bids_gen_cpac_sublist(bids_dir, paths_list, config_dict, creds_path, if "fMRI" in f_dict["acq"]: if "fmap" not in subdict[f_dict["sub"]][f_dict["ses"]]: subdict[f_dict["sub"]][f_dict["ses"]]["fmap"] = {} - if "epi_{0}".format( - pe_dir - ) not in subdict[f_dict["sub"]][f_dict["ses"]]["fmap"]: - subdict[f_dict["sub"]][ - f_dict["ses"] - ]["fmap"]["epi_{0}".format(pe_dir)] = task_info + if ( + "epi_{0}".format(pe_dir) + not in subdict[f_dict["sub"]][f_dict["ses"]]["fmap"] + ): + subdict[f_dict["sub"]][f_dict["ses"]]["fmap"][ + "epi_{0}".format(pe_dir) + ] = task_info sublist = [] for ksub, sub in subdict.items(): @@ -717,102 +734,104 @@ def bids_gen_cpac_sublist(bids_dir, paths_list, config_dict, creds_path, sublist.append(ses) else: if "anat" not in ses: - print("%s %s %s is missing an anat" % ( - ses["site_id"] if 'none' not in ses["site_id"] else '', - ses["subject_id"], - ses["unique_id"] - )) + pass if "func" not in ses: - print("%s %s %s is missing an func" % ( - ses["site_id"] if 'none' not in ses["site_id"] else '', - ses["subject_id"], - ses["unique_id"] - )) + pass return sublist -def collect_bids_files_configs(bids_dir, aws_input_creds=''): +def collect_bids_files_configs(bids_dir, aws_input_creds=""): """ :param bids_dir: :param aws_input_creds: :return: """ - file_paths = [] config_dict = {} - suffixes = ['T1w', 'T2w', 'bold', 'epi', 'phasediff', 'phase1', - 'phase2', 'magnitude', 'magnitude1', 'magnitude2'] + suffixes = [ + "T1w", + "T2w", + "bold", + "epi", + "phasediff", + "phase1", + "phase2", + "magnitude", + "magnitude1", + "magnitude2", + ] if bids_dir.lower().startswith("s3://"): # s3 paths begin with s3://bucket/ - bucket_name = bids_dir.split('/')[2] - s3_prefix = '/'.join(bids_dir.split('/')[:3]) - prefix = bids_dir.replace(s3_prefix, '').lstrip('/') + bucket_name = bids_dir.split("/")[2] + s3_prefix = "/".join(bids_dir.split("/")[:3]) + prefix = bids_dir.replace(s3_prefix, "").lstrip("/") if aws_input_creds: if not os.path.isfile(aws_input_creds): - raise IOError("Could not find aws_input_creds (%s)" % - (aws_input_creds)) + raise IOError("Could not find aws_input_creds (%s)" % (aws_input_creds)) from indi_aws import fetch_creds - bucket = fetch_creds.return_bucket(aws_input_creds, bucket_name) - print(f"gathering files from S3 bucket ({bucket}) for {prefix}") + bucket = fetch_creds.return_bucket(aws_input_creds, bucket_name) for s3_obj in bucket.objects.filter(Prefix=prefix): for suf in suffixes: if suf in str(s3_obj.key): - if suf == 'epi' and 'acq-fMRI' not in s3_obj.key: + if suf == "epi" and "acq-fMRI" not in s3_obj.key: continue if str(s3_obj.key).endswith("json"): try: - config_dict[s3_obj.key.replace(prefix, "") - .lstrip('/')] = json.loads( - s3_obj.get()["Body"].read()) - except Exception as e: - print("Error retrieving %s (%s)" % - (s3_obj.key.replace(prefix, ""), - e.message)) + config_dict[ + s3_obj.key.replace(prefix, "").lstrip("/") + ] = json.loads(s3_obj.get()["Body"].read()) + except Exception: raise - elif 'nii' in str(s3_obj.key): - file_paths.append(str(s3_obj.key) - .replace(prefix,'').lstrip('/')) + elif "nii" in str(s3_obj.key): + file_paths.append( + str(s3_obj.key).replace(prefix, "").lstrip("/") + ) else: for root, dirs, files in os.walk(bids_dir, topdown=False, followlinks=True): if files: for f in files: for suf in suffixes: - if suf == 'epi' and 'acq-fMRI' not in f: + if suf == "epi" and "acq-fMRI" not in f: continue - if 'nii' in f and suf in f: - file_paths += [os.path.join(root, f) - .replace(bids_dir, '').lstrip('/')] - if f.endswith('json') and suf in f: + if "nii" in f and suf in f: + file_paths += [ + os.path.join(root, f).replace(bids_dir, "").lstrip("/") + ] + if f.endswith("json") and suf in f: try: config_dict.update( - {os.path.join(root.replace(bids_dir, '') - .lstrip('/'), f): - json.load( - open(os.path.join(root, f), 'r') - )}) + { + os.path.join( + root.replace(bids_dir, "").lstrip("/"), f + ): json.load(open(os.path.join(root, f), "r")) + } + ) except UnicodeDecodeError: - raise Exception("Could not decode {0}".format( - os.path.join(root, f))) + raise Exception( + "Could not decode {0}".format(os.path.join(root, f)) + ) if not file_paths and not config_dict: - raise IOError("Didn't find any files in {0}. Please verify that the " - "path is typed correctly, that you have read access to " - "the directory, and that it is not " - "empty.".format(bids_dir)) + raise IOError( + "Didn't find any files in {0}. Please verify that the " + "path is typed correctly, that you have read access to " + "the directory, and that it is not " + "empty.".format(bids_dir) + ) return file_paths, config_dict def camelCase(string: str) -> str: # pylint: disable=invalid-name - """Convert a hyphenated string to camelCase + """Convert a hyphenated string to camelCase. Parameters ---------- @@ -830,16 +849,16 @@ def camelCase(string: str) -> str: # pylint: disable=invalid-name >>> camelCase('mean-Pearson-Nilearn-aCompCor') 'meanPearsonNilearnACompCor' """ - pieces = string.split('-') + pieces = string.split("-") for i in range(1, len(pieces)): # don't change case of first piece if pieces[i]: # don't do anything to falsy pieces - pieces[i] = f'{pieces[i][0].upper()}{pieces[i][1:]}' - return ''.join(pieces) + pieces[i] = f"{pieces[i][0].upper()}{pieces[i][1:]}" + return "".join(pieces) def combine_multiple_entity_instances(bids_str: str) -> str: """Combines mutliple instances of a key in a BIDS string to a single - instance by camelCasing and concatenating the values + instance by camelCasing and concatenating the values. Parameters ---------- @@ -860,27 +879,26 @@ def combine_multiple_entity_instances(bids_str: str) -> str: ... 'run-1_framewise-displacement-power.1D') 'sub-1_ses-HBN_site-RU_task-rest_run-1_framewiseDisplacementPower.1D' """ - _entity_list = bids_str.split('_') + _entity_list = bids_str.split("_") entity_list = _entity_list[:-1] suffixes = [camelCase(_entity_list[-1])] entities = {} for entity in entity_list: - if '-' in entity: - key, value = entity.split('-', maxsplit=1) + if "-" in entity: + key, value = entity.split("-", maxsplit=1) if key not in entities: entities[key] = [] entities[key].append(value) for key, value in entities.items(): - entities[key] = camelCase('-'.join(value)) - if 'desc' in entities: # make 'desc' final entity + entities[key] = camelCase("-".join(value)) + if "desc" in entities: # make 'desc' final entity suffixes.insert(0, f'desc-{entities.pop("desc")}') - return '_'.join([f'{key}-{value}' for key, value in entities.items() - ] + suffixes) + return "_".join([f"{key}-{value}" for key, value in entities.items()] + suffixes) def insert_entity(resource, key, value): """Insert a `f'{key}-{value}'` BIDS entity before `desc-` if - present or before the suffix otherwise + present or before the suffix otherwise. Parameters ---------- @@ -901,60 +919,54 @@ def insert_entity(resource, key, value): >>> insert_entity('run-1_reg-default_bold', 'filt', 'notch4c0p31bw0p12') 'run-1_reg-default_filt-notch4c0p31bw0p12_bold' """ - entities = resource.split('_')[:-1] - suff = resource.split('_')[-1] + entities = resource.split("_")[:-1] + suff = resource.split("_")[-1] new_entities = [[], []] for entity in entities: - if entity.startswith('desc-'): + if entity.startswith("desc-"): new_entities[1].append(entity) else: new_entities[0].append(entity) - return '_'.join([*new_entities[0], f'{key}-{value}', *new_entities[1], - suff]) + return "_".join([*new_entities[0], f"{key}-{value}", *new_entities[1], suff]) def load_yaml_config(config_filename, aws_input_creds): - - if config_filename.lower().startswith('data:'): + if config_filename.lower().startswith("data:"): try: header, encoded = config_filename.split(",", 1) config_content = b64decode(encoded) - config_data = yaml.safe_load(config_content) - return config_data + return yaml.safe_load(config_content) except: - print("Error! Could not find load config from data URI") raise if config_filename.lower().startswith("s3://"): # s3 paths begin with s3://bucket/ - bucket_name = config_filename.split('/')[2] - s3_prefix = '/'.join(config_filename.split('/')[:3]) - prefix = config_filename.replace(s3_prefix, '').lstrip('/') + bucket_name = config_filename.split("/")[2] + s3_prefix = "/".join(config_filename.split("/")[:3]) + prefix = config_filename.replace(s3_prefix, "").lstrip("/") if aws_input_creds: if not os.path.isfile(aws_input_creds): - raise IOError("Could not find aws_input_creds (%s)" % - (aws_input_creds)) + raise IOError("Could not find aws_input_creds (%s)" % (aws_input_creds)) from indi_aws import fetch_creds + bucket = fetch_creds.return_bucket(aws_input_creds, bucket_name) - downloaded_config = '/tmp/' + os.path.basename(config_filename) + downloaded_config = "/tmp/" + os.path.basename(config_filename) bucket.download_file(prefix, downloaded_config) config_filename = downloaded_config config_filename = os.path.realpath(config_filename) try: - config_data = yaml.safe_load(open(config_filename, 'r')) - return config_data + return yaml.safe_load(open(config_filename, "r")) except IOError: - print("Error! Could not find config file {0}".format(config_filename)) raise def cl_strip_brackets(arg_list): """Removes '[' from before first and ']' from after final - arguments in a list of commandline arguments + arguments in a list of commandline arguments. Parameters ---------- @@ -973,14 +985,18 @@ def cl_strip_brackets(arg_list): >>> cl_strip_brackets('[ a b c ]'.split(' ')) ['a', 'b', 'c'] """ - arg_list[0] = arg_list[0].lstrip('[') - arg_list[-1] = arg_list[-1].rstrip(']') + arg_list[0] = arg_list[0].lstrip("[") + arg_list[-1] = arg_list[-1].rstrip("]") return [arg for arg in arg_list if arg] -def create_cpac_data_config(bids_dir, participant_labels=None, - aws_input_creds=None, skip_bids_validator=False, - only_one_anat=True): +def create_cpac_data_config( + bids_dir, + participant_labels=None, + aws_input_creds=None, + skip_bids_validator=False, + only_one_anat=True, +): """ Create a C-PAC data config YAML file from a BIDS directory. @@ -1003,23 +1019,19 @@ def create_cpac_data_config(bids_dir, participant_labels=None, ------- list """ - print("Parsing {0}..".format(bids_dir)) - - (file_paths, config) = collect_bids_files_configs(bids_dir, - aws_input_creds) + (file_paths, config) = collect_bids_files_configs(bids_dir, aws_input_creds) if participant_labels and file_paths: file_paths = [ - file_path for file_path in file_paths if any( + file_path + for file_path in file_paths + if any( participant_label in file_path for participant_label in participant_labels ) ] if not file_paths: - print("Did not find data for {0}".format( - ", ".join(participant_labels) - )) sys.exit(1) raise_error = not skip_bids_validator @@ -1030,20 +1042,18 @@ def create_cpac_data_config(bids_dir, participant_labels=None, config, aws_input_creds, raise_error=raise_error, - only_one_anat=only_one_anat + only_one_anat=only_one_anat, ) if not sub_list: - print("Did not find data in {0}".format(bids_dir)) sys.exit(1) return sub_list -def load_cpac_data_config(data_config_file, participant_labels, - aws_input_creds): +def load_cpac_data_config(data_config_file, participant_labels, aws_input_creds): """ - Loads the file as a check to make sure it is available and readable + Loads the file as a check to make sure it is available and readable. Parameters ---------- @@ -1061,33 +1071,25 @@ def load_cpac_data_config(data_config_file, participant_labels, sub_list = load_yaml_config(data_config_file, aws_input_creds) if participant_labels: - sub_list = [ d for d in sub_list if ( d["subject_id"] - if d["subject_id"].startswith('sub-') - else 'sub-' + d["subject_id"] - ) in participant_labels + if d["subject_id"].startswith("sub-") + else "sub-" + d["subject_id"] + ) + in participant_labels ] if not sub_list: - print("Did not find data for {0} in {1}".format( - ", ".join(participant_labels), - ( - data_config_file - if not data_config_file.startswith("data:") - else "data URI" - ) - )) sys.exit(1) return sub_list def res_in_filename(cfg, label): - """Specify resolution in filename + """Specify resolution in filename. Parameters ---------- @@ -1113,27 +1115,37 @@ def res_in_filename(cfg, label): ... 'sub-1_res-3mm_bold') 'sub-1_res-3mm_bold' """ - if '_res-' in label: + if "_res-" in label: # replace resolution text with actual resolution - resolution = label.split('_res-', 1)[1].split('_', 1)[0] + resolution = label.split("_res-", 1)[1].split("_", 1)[0] resolution = { - 'anat': cfg['registration_workflows', 'anatomical_registration', - 'resolution_for_anat'], - 'bold': cfg['registration_workflows', 'functional_registration', - 'func_registration_to_template', 'output_resolution', - 'func_preproc_outputs'], - 'derivative': cfg['registration_workflows', - 'functional_registration', - 'func_registration_to_template', - 'output_resolution', 'func_derivative_outputs'] + "anat": cfg[ + "registration_workflows", + "anatomical_registration", + "resolution_for_anat", + ], + "bold": cfg[ + "registration_workflows", + "functional_registration", + "func_registration_to_template", + "output_resolution", + "func_preproc_outputs", + ], + "derivative": cfg[ + "registration_workflows", + "functional_registration", + "func_registration_to_template", + "output_resolution", + "func_derivative_outputs", + ], }.get(resolution, resolution) - label = re.sub('_res-[A-Za-z0-9]*_', f'_res-{resolution}_', label) + label = re.sub("_res-[A-Za-z0-9]*_", f"_res-{resolution}_", label) return label def sub_list_filter_by_labels(sub_list, labels): """Function to filter a sub_list by provided BIDS labels for - specified suffixes + specified suffixes. Parameters ---------- @@ -1150,16 +1162,16 @@ def sub_list_filter_by_labels(sub_list, labels): ------- list """ - if labels.get('T1w'): - sub_list = _sub_list_filter_by_label(sub_list, 'T1w', labels['T1w']) - if labels.get('bold'): - labels['bold'] = cl_strip_brackets(labels['bold']) - sub_list = _sub_list_filter_by_label(sub_list, 'bold', labels['bold']) + if labels.get("T1w"): + sub_list = _sub_list_filter_by_label(sub_list, "T1w", labels["T1w"]) + if labels.get("bold"): + labels["bold"] = cl_strip_brackets(labels["bold"]) + sub_list = _sub_list_filter_by_label(sub_list, "bold", labels["bold"]) return sub_list def with_key(entity: str, key: str) -> str: - """Return a keyed BIDS entity + """Return a keyed BIDS entity. Parameters ---------- @@ -1178,13 +1190,13 @@ def with_key(entity: str, key: str) -> str: """ if not isinstance(entity, str): entity = str(entity) - if not entity.startswith(f'{key}-'): - entity = '-'.join((key, entity)) + if not entity.startswith(f"{key}-"): + entity = "-".join((key, entity)) return entity def without_key(entity: str, key: str) -> str: - """Return a BIDS entity value + """Return a BIDS entity value. Parameters ---------- @@ -1203,13 +1215,13 @@ def without_key(entity: str, key: str) -> str: """ if not isinstance(entity, str): entity = str(entity) - if entity.startswith(f'{key}-'): - entity = entity.replace(f'{key}-', '') + if entity.startswith(f"{key}-"): + entity = entity.replace(f"{key}-", "") return entity def _t1w_filter(anat, shortest_entity, label): - """Helper function to filter T1w paths + """Helper function to filter T1w paths. Parameters ---------- @@ -1228,10 +1240,10 @@ def _t1w_filter(anat, shortest_entity, label): if shortest_entity: anat = bids_shortest_entity(anat) else: - anat = bids_match_entities(anat, label, 'T1w') + anat = bids_match_entities(anat, label, "T1w") # pylint: disable=invalid-name try: - anat_T2 = bids_match_entities(anat, label, 'T2w') + anat_T2 = bids_match_entities(anat, label, "T2w") except LookupError: anat_T2 = None if anat_T2 is not None: @@ -1240,7 +1252,7 @@ def _t1w_filter(anat, shortest_entity, label): def _sub_anat_filter(anat, shortest_entity, label): - """Helper function to filter anat paths in sub_list + """Helper function to filter anat paths in sub_list. Parameters ---------- @@ -1256,10 +1268,8 @@ def _sub_anat_filter(anat, shortest_entity, label): same type as 'anat' parameter """ if isinstance(anat, dict): - if 'T1w' in anat: - anat['T1w'] = _t1w_filter(anat['T1w'], - shortest_entity, - label) + if "T1w" in anat: + anat["T1w"] = _t1w_filter(anat["T1w"], shortest_entity, label) return anat return _t1w_filter(anat, shortest_entity, label) @@ -1294,40 +1304,39 @@ def _sub_list_filter_by_label(sub_list, label_type, label): label_list.remove(label_type) else: shortest_entity = False - if label_type == 'T1w': - for sub in [sub for sub in sub_list if 'anat' in sub]: + if label_type == "T1w": + for sub in [sub for sub in sub_list if "anat" in sub]: try: - sub['anat'] = _sub_anat_filter(sub['anat'], - shortest_entity, - label_list[0] if not - shortest_entity else None) - if sub['anat']: + sub["anat"] = _sub_anat_filter( + sub["anat"], + shortest_entity, + label_list[0] if not shortest_entity else None, + ) + if sub["anat"]: new_sub_list.append(sub) except LookupError as lookup_error: warn(str(lookup_error)) - elif label_type == 'bold': - for sub in [sub for sub in sub_list if 'func' in sub]: + elif label_type == "bold": + for sub in [sub for sub in sub_list if "func" in sub]: try: - all_scans = [sub['func'][scan].get('scan') for - scan in sub['func']] + all_scans = [sub["func"][scan].get("scan") for scan in sub["func"]] new_func = {} for entities in label_list: - matched_scans = bids_match_entities(all_scans, entities, - label_type) + matched_scans = bids_match_entities(all_scans, entities, label_type) for scan in matched_scans: new_func = { **new_func, - **_match_functional_scan(sub['func'], scan) + **_match_functional_scan(sub["func"], scan), } if shortest_entity: new_func = { **new_func, **_match_functional_scan( - sub['func'], bids_shortest_entity(all_scans) - ) + sub["func"], bids_shortest_entity(all_scans) + ), } - sub['func'] = new_func + sub["func"] = new_func new_sub_list.append(sub) except LookupError as lookup_error: warn(str(lookup_error)) @@ -1335,10 +1344,10 @@ def _sub_list_filter_by_label(sub_list, label_type, label): def _match_functional_scan(sub_list_func_dict, scan_file_to_match): - """Function to subset a scan from a sub_list_func_dict by a scan filename + """Function to subset a scan from a sub_list_func_dict by a scan filename. Parameters - --------- + ---------- sub_list_func_dict : dict sub_list[sub]['func'] @@ -1363,7 +1372,7 @@ def _match_functional_scan(sub_list_func_dict, scan_file_to_match): True """ return { - entity: sub_list_func_dict[entity] for entity in - sub_list_func_dict if - sub_list_func_dict[entity].get('scan') == scan_file_to_match + entity: sub_list_func_dict[entity] + for entity in sub_list_func_dict + if sub_list_func_dict[entity].get("scan") == scan_file_to_match } diff --git a/CPAC/utils/build_data_config.py b/CPAC/utils/build_data_config.py index c21566c22e..8b4ebc21e8 100644 --- a/CPAC/utils/build_data_config.py +++ b/CPAC/utils/build_data_config.py @@ -1,7 +1,4 @@ - - def gather_file_paths(base_directory, verbose=False): - # this will go into core tools eventually # ideas: return number of paths, optionally instead @@ -22,16 +19,17 @@ def gather_file_paths(base_directory, verbose=False): path_list.append(fullpath) if verbose: - print("Number of paths: {0}".format(len(path_list))) + pass return path_list def pull_s3_sublist(data_folder, creds_path=None, keep_prefix=True): """Return a list of input data file paths that are available on an AWS S3 - bucket on the cloud.""" - + bucket on the cloud. + """ import os + from indi_aws import fetch_creds if creds_path: @@ -41,8 +39,6 @@ def pull_s3_sublist(data_folder, creds_path=None, keep_prefix=True): bucket_name = s3_path.split("/")[0] bucket_prefix = s3_path.split(bucket_name + "/")[1] - print("Pulling from {0} ...".format(data_folder)) - s3_list = [] bucket = fetch_creds.return_bucket(creds_path, bucket_name) @@ -60,22 +56,22 @@ def pull_s3_sublist(data_folder, creds_path=None, keep_prefix=True): else: s3_list.append(str(bk.key).replace(bucket_prefix, "")) - print("Finished pulling from S3. " \ - "{0} file paths found.".format(len(s3_list))) - if not s3_list: - err = "\n\n[!] No input data found matching your data settings in " \ - "the AWS S3 bucket provided:\n{0}\n\n".format(data_folder) + err = ( + "\n\n[!] No input data found matching your data settings in " + "the AWS S3 bucket provided:\n{0}\n\n".format(data_folder) + ) raise Exception(err) return s3_list -def get_file_list(base_directory, creds_path=None, write_txt=None, - write_pkl=None, write_info=False): +def get_file_list( + base_directory, creds_path=None, write_txt=None, write_pkl=None, write_info=False +): """Return a list of input and data file paths either stored locally or on - an AWS S3 bucket on the cloud.""" - + an AWS S3 bucket on the cloud. + """ import os if "s3://" in base_directory: @@ -87,8 +83,10 @@ def get_file_list(base_directory, creds_path=None, write_txt=None, file_list = gather_file_paths(base_directory) if len(file_list) == 0: - warn = "\n\n[!] No files were found in the base directory you " \ - "provided.\n\nDirectory: {0}\n\n".format(base_directory) + warn = ( + "\n\n[!] No files were found in the base directory you " + "provided.\n\nDirectory: {0}\n\n".format(base_directory) + ) raise Exception(warn) if write_txt: @@ -98,18 +96,15 @@ def get_file_list(base_directory, creds_path=None, write_txt=None, with open(write_txt, "wt") as f: for path in file_list: f.write("{0}\n".format(path)) - print("\nFilepath list text file written:\n" \ - "{0}".format(write_txt)) if write_pkl: import pickle + if ".pkl" not in write_pkl: write_pkl = "{0}.pkl".format(write_pkl) write_pkl = os.path.abspath(write_pkl) with open(write_pkl, "wb") as f: pickle.dump(list(file_list), f) - print("\nFilepath list pickle file written:\n" \ - "{0}".format(write_pkl)) if write_info: niftis = [] @@ -132,24 +127,17 @@ def get_file_list(base_directory, creds_path=None, write_txt=None, if "participants.tsv" in path: part_tsvs.append(path) - print("\nBase directory: {0}".format(base_directory)) - print("File paths found: {0}".format(len(file_list))) - print("..NIFTI files: {0}".format(len(niftis))) - print("..JSON files: {0}".format(len(jsons))) if jsons: - print("....{0} of which are scan parameter JSON files" \ - "".format(len(scan_jsons))) - print("..CSV files: {0}".format(len(csvs))) - print("..TSV files: {0}".format(len(tsvs))) + pass if tsvs: - print("....{0} of which are participants.tsv files" \ - "".format(len(part_tsvs))) + pass return file_list -def download_single_s3_path(s3_path, download_dir=None, creds_path=None, - overwrite=False): +def download_single_s3_path( + s3_path, download_dir=None, creds_path=None, overwrite=False +): """Download a single file from an AWS s3 bucket. :type s3_path: str @@ -161,11 +149,11 @@ def download_single_s3_path(s3_path, download_dir=None, creds_path=None, :rtype: str :return: The local filepath of the downloaded s3 file. """ - # TODO: really for core tools and/or utility import os - from indi_aws import fetch_creds, aws_utils + + from indi_aws import aws_utils, fetch_creds if "s3://" in s3_path: s3_prefix = s3_path.replace("s3://", "") @@ -186,12 +174,9 @@ def download_single_s3_path(s3_path, download_dir=None, creds_path=None, if os.path.isfile(local_dl): if overwrite: - print("\nS3 bucket file already downloaded! Overwriting..") aws_utils.s3_download(bucket, ([data_dir], [local_dl])) else: - print("\nS3 bucket file already downloaded! Skipping download.") - print("S3 file: %s" % s3_path) - print("Local file already exists: %s\n" % local_dl) + pass else: aws_utils.s3_download(bucket, ([data_dir], [local_dl])) @@ -199,8 +184,8 @@ def download_single_s3_path(s3_path, download_dir=None, creds_path=None, def pull_s3_sublist(data_folder, creds_path=None, keep_prefix=True): - import os + from indi_aws import fetch_creds if creds_path: @@ -210,8 +195,6 @@ def pull_s3_sublist(data_folder, creds_path=None, keep_prefix=True): bucket_name = s3_path.split("/")[0] bucket_prefix = s3_path.split(bucket_name + "/")[1] - print("Pulling from {0} ...".format(data_folder)) - s3_list = [] bucket = fetch_creds.return_bucket(creds_path, bucket_name) @@ -229,33 +212,32 @@ def pull_s3_sublist(data_folder, creds_path=None, keep_prefix=True): else: s3_list.append(str(bk.key).replace(bucket_prefix, "")) - print("Finished pulling from S3. " \ - "{0} file paths found.".format(len(s3_list))) - if not s3_list: - err = "\n\n[!] No input data found matching your data settings in " \ - "the AWS S3 bucket provided:\n{0}\n\n".format(data_folder) + err = ( + "\n\n[!] No input data found matching your data settings in " + "the AWS S3 bucket provided:\n{0}\n\n".format(data_folder) + ) raise Exception(err) return s3_list def generate_group_analysis_files(data_config_outdir, data_config_name): - """Create the group-level analysis inclusion list. - """ - + """Create the group-level analysis inclusion list.""" + import csv import os + from sets import Set - import csv import yaml data_config_path = os.path.join(data_config_outdir, data_config_name) try: - subjects_list = yaml.safe_load(open(data_config_path, 'r')) + subjects_list = yaml.safe_load(open(data_config_path, "r")) except: - err = "\n\n[!] Data configuration file couldn't be read!\nFile " \ - "path: {0}\n".format(data_config_path) + "\n\n[!] Data configuration file couldn't be read!\nFile " "path: {0}\n".format( + data_config_path + ) subject_scan_set = Set() subID_set = Set() @@ -266,41 +248,38 @@ def generate_group_analysis_files(data_config_outdir, data_config_name): try: for sub in subjects_list: - if sub['unique_id']: - subject_id = sub['subject_id'] + "_" + sub['unique_id'] + if sub["unique_id"]: + subject_id = sub["subject_id"] + "_" + sub["unique_id"] else: - subject_id = sub['subject_id'] + subject_id = sub["subject_id"] try: - for scan in sub['func']: + for scan in sub["func"]: subject_scan_set.add((subject_id, scan)) - subID_set.add(sub['subject_id']) - session_set.add(sub['unique_id']) + subID_set.add(sub["subject_id"]) + session_set.add(sub["unique_id"]) subject_set.add(subject_id) scan_set.add(scan) except KeyError: try: - for scan in sub['rest']: + for scan in sub["rest"]: subject_scan_set.add((subject_id, scan)) - subID_set.add(sub['subject_id']) - session_set.add(sub['unique_id']) + subID_set.add(sub["subject_id"]) + session_set.add(sub["unique_id"]) subject_set.add(subject_id) scan_set.add(scan) except KeyError: # one of the participants in the subject list has no # functional scans - subID_set.add(sub['subject_id']) - session_set.add(sub['unique_id']) + subID_set.add(sub["subject_id"]) + session_set.add(sub["unique_id"]) subject_set.add(subject_id) - except TypeError as e: - print('Subject list could not be populated!') - print('This is most likely due to a mis-formatting in your ' \ - 'inclusion and/or exclusion subjects txt file or your ' \ - 'anatomical and/or functional path templates.') - print('Error: %s' % e) - err_str = 'Check formatting of your anatomical/functional path ' \ - 'templates and inclusion/exclusion subjects text files' + except TypeError: + err_str = ( + "Check formatting of your anatomical/functional path " + "templates and inclusion/exclusion subjects text files" + ) raise TypeError(err_str) for item in subject_scan_set: @@ -321,52 +300,40 @@ def generate_group_analysis_files(data_config_outdir, data_config_name): data_list.append(list1) # generate the phenotypic file templates for group analysis - file_name = os.path.join(data_config_outdir, 'phenotypic_template_%s.csv' - % data_config_name) + file_name = os.path.join( + data_config_outdir, "phenotypic_template_%s.csv" % data_config_name + ) try: - f = open(file_name, 'wb') + f = open(file_name, "wb") except: - print('\n\nCPAC says: I couldn\'t save this file to your drive:\n') - print(file_name, '\n\n') - print('Make sure you have write access? Then come back. Don\'t ' \ - 'worry.. I\'ll wait.\n\n') raise IOError writer = csv.writer(f) - writer.writerow(['participant', 'EV1', '..']) + writer.writerow(["participant", "EV1", ".."]) for sub in sorted(subID_set): - writer.writerow([sub, '']) + writer.writerow([sub, ""]) f.close() - print("Template Phenotypic file for group analysis - %s" % file_name) - # generate the group analysis subject lists - file_name = os.path.join(data_config_outdir, - 'participant_list_group_analysis_%s.txt' - % data_config_name) + file_name = os.path.join( + data_config_outdir, "participant_list_group_analysis_%s.txt" % data_config_name + ) try: - with open(file_name, 'w') as f: + with open(file_name, "w") as f: for sub in sorted(subID_set): print(sub, file=f) except: - print('\n\nCPAC says: I couldn\'t save this file to your drive:\n') - print(file_name, '\n\n') - print('Make sure you have write access? Then come back. Don\'t ' \ - 'worry.. I\'ll wait.\n\n') raise IOError - print("Participant list required later for group analysis - %s\n\n" \ - % file_name) - def extract_scan_params_csv(scan_params_csv): """ Function to extract the site-based scan parameters from a csv file - and return a dictionary of their values + and return a dictionary of their values. Parameters ---------- @@ -379,45 +346,45 @@ def extract_scan_params_csv(scan_params_csv): a dictionary where site names are the keys and the scan parameters for that site are the values stored as a dictionary """ - # Import packages import csv # Init variables - csv_open = open(scan_params_csv, 'r') + csv_open = open(scan_params_csv, "r") site_dict = {} # Init csv dictionary reader reader = csv.DictReader(csv_open) - placeholders = ['None', 'NONE', 'none', 'All', 'ALL', 'all', '', ' '] + placeholders = ["None", "NONE", "none", "All", "ALL", "all", "", " "] - keys = {"TR (seconds)": "TR", - "TE (seconds)": "TE", - "Reference (slice no)": "reference", - "Acquisition (pattern)": "acquisition", - "FirstTR (start volume index)": "first_TR", - "LastTR (final volume index)": "last_TR"} + keys = { + "TR (seconds)": "TR", + "TE (seconds)": "TE", + "Reference (slice no)": "reference", + "Acquisition (pattern)": "acquisition", + "FirstTR (start volume index)": "first_TR", + "LastTR (final volume index)": "last_TR", + } # Iterate through the csv and pull in parameters for dict_row in reader: - - if dict_row['Site'] in placeholders: - site = 'All' + if dict_row["Site"] in placeholders: + site = "All" else: - site = dict_row['Site'] + site = dict_row["Site"] sub = "All" if "Participant" in dict_row.keys(): if dict_row["Participant"] not in placeholders: sub = dict_row["Participant"] - ses = 'All' - if 'Session' in dict_row.keys(): - if dict_row['Session'] not in placeholders: - ses = dict_row['Session'] + ses = "All" + if "Session" in dict_row.keys(): + if dict_row["Session"] not in placeholders: + ses = dict_row["Session"] - if ses != 'All': + if ses != "All": # for session-specific scan parameters if site not in site_dict.keys(): site_dict[site] = {} @@ -425,15 +392,14 @@ def extract_scan_params_csv(scan_params_csv): site_dict[site][sub] = {} site_dict[site][sub][ses] = { - keys[key]: val for key, val in dict_row.items( - ) if key != 'Site' and - key != 'Participant' and - key != 'Session' and key != 'Series' + keys[key]: val + for key, val in dict_row.items() + if key not in ("Site", "Participant", "Session", "Series") } # Assumes all other fields are formatted properly, but TR might # not be - #site_dict[site][sub][ses]['tr'] = \ + # site_dict[site][sub][ses]['tr'] = \ # site_dict[site][sub][ses].pop('tr (seconds)') elif sub != "All": @@ -444,15 +410,14 @@ def extract_scan_params_csv(scan_params_csv): site_dict[site][sub] = {} site_dict[site][sub][ses] = { - keys[key]: val for key, val in dict_row.items( - ) if key != 'Site' and - key != 'Participant' and - key != 'Session' and key != 'Series' + keys[key]: val + for key, val in dict_row.items() + if key not in ("Site", "Participant", "Session", "Series") } # Assumes all other fields are formatted properly, but TR might # not be - #site_dict[site][sub][ses]['tr'] = + # site_dict[site][sub][ses]['tr'] = # site_dict[site][sub][ses].pop('tr (seconds)') else: @@ -463,33 +428,33 @@ def extract_scan_params_csv(scan_params_csv): site_dict[site][sub] = {} site_dict[site][sub][ses] = { - keys[key]: val for key, val in dict_row.items( - ) if key != 'Site' and - key != 'Participant' and - key != 'Session' and key != 'Series' + keys[key]: val + for key, val in dict_row.items() + if key not in ("Site", "Participant", "Session", "Series") } # Assumes all other fields are formatted properly, but TR might # not be - #site_dict[site][sub][ses]['tr'] = \ + # site_dict[site][sub][ses]['tr'] = \ # site_dict[site][sub][ses].pop('tr (seconds)') return site_dict -def format_incl_excl_dct(incl_list, info_type='participants'): +def format_incl_excl_dct(incl_list, info_type="participants"): """Create either an inclusion or exclusion dictionary to determine which - input files to include or not include in the data configuration file.""" - + input files to include or not include in the data configuration file. + """ incl_dct = {} if isinstance(incl_list, str): - if '.txt' in incl_list: - with open(incl_list, 'r') as f: - incl_dct[info_type] = [x.rstrip("\n").replace(" ", "") for x in f.readlines() if x != ''] - elif ',' in incl_list: - incl_dct[info_type] = \ - [x.replace(" ", "") for x in incl_list.split(",")] + if ".txt" in incl_list: + with open(incl_list, "r") as f: + incl_dct[info_type] = [ + x.rstrip("\n").replace(" ", "") for x in f.readlines() if x != "" + ] + elif "," in incl_list: + incl_dct[info_type] = [x.replace(" ", "") for x in incl_list.split(",")] elif incl_list: # if there's only one item in the box, most common probably if "None" in incl_list or "none" in incl_list: @@ -501,10 +466,17 @@ def format_incl_excl_dct(incl_list, info_type='participants'): return incl_dct -def get_BIDS_data_dct(bids_base_dir, file_list=None, anat_scan=None, - aws_creds_path=None, freesurfer_dir=None, - brain_mask_template=None,inclusion_dct=None, - exclusion_dct=None, config_dir=None): +def get_BIDS_data_dct( + bids_base_dir, + file_list=None, + anat_scan=None, + aws_creds_path=None, + freesurfer_dir=None, + brain_mask_template=None, + inclusion_dct=None, + exclusion_dct=None, + config_dir=None, +): """Return a data dictionary mapping input file paths to participant, session, scan, and site IDs (where applicable) for a BIDS-formatted data directory. @@ -517,96 +489,109 @@ def get_BIDS_data_dct(bids_base_dir, file_list=None, anat_scan=None, derivatives. Thus, we allow users to modify what the expected BIDS layout is for their anatomical brain masks. """ - + import glob import os import re - import glob if not config_dir: config_dir = os.getcwd() - anat_sess = os.path.join(bids_base_dir, - "sub-{participant}/ses-{session}/anat/sub-" - "{participant}_ses-{session}_T1w.nii.gz") - anat = os.path.join(bids_base_dir, - "sub-{participant}/anat/sub-{participant}_T1w.nii.gz") + anat_sess = os.path.join( + bids_base_dir, + "sub-{participant}/ses-{session}/anat/sub-" + "{participant}_ses-{session}_T1w.nii.gz", + ) + anat = os.path.join( + bids_base_dir, "sub-{participant}/anat/sub-{participant}_T1w.nii.gz" + ) if anat_scan: - anat_sess = anat_sess.replace("_T1w", "_*T1w") #"_*_T1w") - anat = anat.replace("_T1w", "_*T1w") #"_*_T1w") - - func_sess = os.path.join(bids_base_dir, - "sub-{participant}" - "/ses-{session}/func/sub-" - "{participant}_ses-{session}_task-{scan}_" - "bold.nii.gz") - func = os.path.join(bids_base_dir, - "sub-{participant}/func/sub-{participant}_task-" - "{scan}_bold.nii.gz") - - fmap_phase_sess = os.path.join(bids_base_dir, - "sub-{participant}/ses-{session}/fmap/" - "sub-{participant}_ses-{session}*phase" - "diff.nii.gz") - fmap_phase = os.path.join(bids_base_dir, - "sub-{participant}/fmap/sub-{participant}" - "*phasediff.nii.gz") - - fmap_mag_sess = os.path.join(bids_base_dir, - "sub-{participant}/ses-{session}/fmap/" - "sub-{participant}_ses-{session}*" - "magnitud*.nii.gz") - - fmap_mag = os.path.join(bids_base_dir, - "sub-{participant}/fmap/sub-{participant}" - "*magnitud*.nii.gz") - - fmap_pedir_sess = os.path.join(bids_base_dir, - "sub-{participant}/ses-{session}/fmap/" - "sub-{participant}_ses-{session}/" - "*acq-fMR*_epi.nii.gz") - - fmap_pedir = os.path.join(bids_base_dir, - "sub-{participant}/fmap/sub-{participant}" - "*acq-fMR*_epi.nii.gz") + anat_sess = anat_sess.replace("_T1w", "_*T1w") # "_*_T1w") + anat = anat.replace("_T1w", "_*T1w") # "_*_T1w") + + func_sess = os.path.join( + bids_base_dir, + "sub-{participant}" + "/ses-{session}/func/sub-" + "{participant}_ses-{session}_task-{scan}_" + "bold.nii.gz", + ) + func = os.path.join( + bids_base_dir, + "sub-{participant}/func/sub-{participant}_task-" "{scan}_bold.nii.gz", + ) + + fmap_phase_sess = os.path.join( + bids_base_dir, + "sub-{participant}/ses-{session}/fmap/" + "sub-{participant}_ses-{session}*phase" + "diff.nii.gz", + ) + fmap_phase = os.path.join( + bids_base_dir, "sub-{participant}/fmap/sub-{participant}" "*phasediff.nii.gz" + ) + + fmap_mag_sess = os.path.join( + bids_base_dir, + "sub-{participant}/ses-{session}/fmap/" + "sub-{participant}_ses-{session}*" + "magnitud*.nii.gz", + ) + + fmap_mag = os.path.join( + bids_base_dir, "sub-{participant}/fmap/sub-{participant}" "*magnitud*.nii.gz" + ) + + fmap_pedir_sess = os.path.join( + bids_base_dir, + "sub-{participant}/ses-{session}/fmap/" + "sub-{participant}_ses-{session}/" + "*acq-fMR*_epi.nii.gz", + ) + + fmap_pedir = os.path.join( + bids_base_dir, "sub-{participant}/fmap/sub-{participant}" "*acq-fMR*_epi.nii.gz" + ) sess_glob = os.path.join(bids_base_dir, "sub-*/ses-*/*") - fmap_phase_scan_glob = os.path.join(bids_base_dir, - "sub-*fmap/" - "sub-*phasediff.nii.gz") + fmap_phase_scan_glob = os.path.join( + bids_base_dir, "sub-*fmap/" "sub-*phasediff.nii.gz" + ) - fmap_mag_scan_glob = os.path.join(bids_base_dir, - "sub-*fmap/" - "sub-*magnitud*.nii.gz") + fmap_mag_scan_glob = os.path.join( + bids_base_dir, "sub-*fmap/" "sub-*magnitud*.nii.gz" + ) - fmap_pedir_scan_glob = os.path.join(bids_base_dir, - "sub-*fmap/" - "sub-*_*acq-fMR*_epi.nii.gz") + os.path.join(bids_base_dir, "sub-*fmap/" "sub-*_*acq-fMR*_epi.nii.gz") part_tsv_glob = os.path.join(bids_base_dir, "*participants.tsv") - ''' + """ site_jsons_glob = os.path.join(bids_base_dir, "*bold.json") sub_jsons_glob = os.path.join(bids_base_dir, "*sub-*/*bold.json") ses_jsons_glob = os.path.join(bids_base_dir, "*ses-*bold.json") ses_scan_jsons_glob = os.path.join(bids_base_dir, "*ses-*task-*bold.json") scan_jsons_glob = os.path.join(bids_base_dir, "*task-*bold.json") - ''' + """ site_dir_glob = os.path.join(bids_base_dir, "*", "sub-*/*/*.nii*") - json_globs = [os.path.join(bids_base_dir, "sub-*/ses-*/func/*bold.json"), - os.path.join(bids_base_dir, "sub-*/ses-*/*bold.json"), - os.path.join(bids_base_dir, "sub-*/func/*bold.json"), - os.path.join(bids_base_dir, "sub-*/*bold.json"), - os.path.join(bids_base_dir, "*bold.json")] - - site_json_globs = [os.path.join(bids_base_dir, "*/sub-*/ses-*/func/*bold.json"), - os.path.join(bids_base_dir, "*/sub-*/ses-*/*bold.json"), - os.path.join(bids_base_dir, "*/sub-*/func/*bold.json"), - os.path.join(bids_base_dir, "*/sub-*/*bold.json"), - os.path.join(bids_base_dir, "*/*bold.json")] + json_globs = [ + os.path.join(bids_base_dir, "sub-*/ses-*/func/*bold.json"), + os.path.join(bids_base_dir, "sub-*/ses-*/*bold.json"), + os.path.join(bids_base_dir, "sub-*/func/*bold.json"), + os.path.join(bids_base_dir, "sub-*/*bold.json"), + os.path.join(bids_base_dir, "*bold.json"), + ] + + site_json_globs = [ + os.path.join(bids_base_dir, "*/sub-*/ses-*/func/*bold.json"), + os.path.join(bids_base_dir, "*/sub-*/ses-*/*bold.json"), + os.path.join(bids_base_dir, "*/sub-*/func/*bold.json"), + os.path.join(bids_base_dir, "*/sub-*/*bold.json"), + os.path.join(bids_base_dir, "*/*bold.json"), + ] ses = False site_dir = False @@ -619,8 +604,10 @@ def get_BIDS_data_dct(bids_base_dir, file_list=None, anat_scan=None, import fnmatch for filepath in file_list: - if fnmatch.fnmatch(filepath, site_dir_glob) and \ - "derivatives" not in filepath: + if ( + fnmatch.fnmatch(filepath, site_dir_glob) + and "derivatives" not in filepath + ): # check if there is a directory level encoding site ID, even # though that is not BIDS format site_dir = True @@ -632,24 +619,30 @@ def get_BIDS_data_dct(bids_base_dir, file_list=None, anat_scan=None, if fnmatch.fnmatch(filepath, fmap_phase_scan_glob): # check if there is a scan level for the fmap phase files - fmap_phase_sess = os.path.join(bids_base_dir, - "sub-{participant}/ses-{session}/fmap/" - "sub-{participant}_ses-{session}*phase" - "diff.nii.gz") - fmap_phase = os.path.join(bids_base_dir, - "sub-{participant}/fmap/sub-{participant}" - "*phasediff.nii.gz") + fmap_phase_sess = os.path.join( + bids_base_dir, + "sub-{participant}/ses-{session}/fmap/" + "sub-{participant}_ses-{session}*phase" + "diff.nii.gz", + ) + fmap_phase = os.path.join( + bids_base_dir, + "sub-{participant}/fmap/sub-{participant}" "*phasediff.nii.gz", + ) if fnmatch.fnmatch(filepath, fmap_mag_scan_glob): # check if there is a scan level for the fmap magnitude files - fmap_mag_sess = os.path.join(bids_base_dir, - "sub-{participant}/ses-{session}/fmap/" - "sub-{participant}_ses-{session}*magnitud*.nii.gz") - fmap_mag = os.path.join(bids_base_dir, - "sub-{participant}/fmap/sub-{participant}" - "*magnitud*.nii.gz") - - ''' + fmap_mag_sess = os.path.join( + bids_base_dir, + "sub-{participant}/ses-{session}/fmap/" + "sub-{participant}_ses-{session}*magnitud*.nii.gz", + ) + fmap_mag = os.path.join( + bids_base_dir, + "sub-{participant}/fmap/sub-{participant}" "*magnitud*.nii.gz", + ) + + """ if fnmatch.fnmatch(filepath, fmap_pedir_scan_glob): # check if there is a scan level for the fmap magnitude files fmap_pedir_sess = os.path.join(bids_base_dir, @@ -658,7 +651,7 @@ def get_BIDS_data_dct(bids_base_dir, file_list=None, anat_scan=None, fmap_pedir = os.path.join(bids_base_dir, "sub-{participant}/fmap/sub-{participant}" "task-{scan}_magnitud*.nii.gz") - ''' + """ if fnmatch.fnmatch(filepath, part_tsv_glob): # check if there is a participants.tsv file @@ -705,13 +698,9 @@ def get_BIDS_data_dct(bids_base_dir, file_list=None, anat_scan=None, import csv if part_tsv.startswith("s3://"): - print("\n\nFound a participants.tsv file in your BIDS data " \ - "set on the S3 bucket. Downloading..\n") - part_tsv = download_single_s3_path(part_tsv, config_dir, - aws_creds_path, overwrite=True) - - print("Checking participants.tsv file for site information:" \ - "\n{0}".format(part_tsv)) + part_tsv = download_single_s3_path( + part_tsv, config_dir, aws_creds_path, overwrite=True + ) with open(part_tsv, "r") as f: tsv = csv.DictReader(f) @@ -726,29 +715,29 @@ def get_BIDS_data_dct(bids_base_dir, file_list=None, anat_scan=None, if sites_dct: # check for duplicates sites = sites_dct.keys() - print("{0} sites found in the participant.tsv " \ - "file.".format(len(sites))) for site in sites: for other_site in sites: if site == other_site: continue dups = set(sites_dct[site]) & set(sites_dct[other_site]) if dups: - err = "\n\n[!] There are duplicate participant IDs " \ - "in different sites, as defined by your " \ - "participants.tsv file! Consider pre-fixing " \ - "the participant IDs with the site names.\n\n" \ - "Duplicates:\n" \ - "Sites: {0}, {1}\n" \ - "Duplicate IDs: {2}" \ - "\n\n".format(site, other_site, str(dups)) + err = ( + "\n\n[!] There are duplicate participant IDs " + "in different sites, as defined by your " + "participants.tsv file! Consider pre-fixing " + "the participant IDs with the site names.\n\n" + "Duplicates:\n" + "Sites: {0}, {1}\n" + "Duplicate IDs: {2}" + "\n\n".format(site, other_site, str(dups)) + ) raise Exception(err) # now invert for sub in sites_dct[site]: sites_subs_dct[sub] = site else: - print("No site information found in the participants.tsv file.") + pass if not sites_subs_dct: # if there was no participants.tsv file, (or no site column in the @@ -756,8 +745,7 @@ def get_BIDS_data_dct(bids_base_dir, file_list=None, anat_scan=None, # sites might be encoded if site_dir: sub_dir = os.path.join(bids_base_dir, "sub-{participant}") - new_dir = os.path.join(bids_base_dir, "{site}", - "sub-{participant}") + new_dir = os.path.join(bids_base_dir, "{site}", "sub-{participant}") if ses: anat_sess = anat_sess.replace(sub_dir, new_dir) func_sess = func_sess.replace(sub_dir, new_dir) @@ -777,7 +765,6 @@ def get_BIDS_data_dct(bids_base_dir, file_list=None, anat_scan=None, # example: /bids_dir/site01/sub-01/func/sub-01_task-rest_bold.json # instead of /bids_dir/sub-01/func/sub-01_task-rest_bold.json for json_file in site_jsons: - # get site ID site_id = json_file.replace("{0}/".format(bids_base_dir), "").split("/")[0] if "site-" in site_id: @@ -836,7 +823,6 @@ def get_BIDS_data_dct(bids_base_dir, file_list=None, anat_scan=None, elif jsons: for json_file in jsons: - # get other IDs, from the BIDS format tags, such as "sub-01" or # "task-rest". also handle things like "task-rest_run-1" ids = re.split("/|_", json_file) @@ -867,8 +853,7 @@ def get_BIDS_data_dct(bids_base_dir, file_list=None, anat_scan=None, # including any run- and acq- tags, since the rest of # CPAC encodes all of the task-, acq-, and run- tags # together for each series/scan - scan_id = json_filename.split("task-")[1].split("_bold")[ - 0] + scan_id = json_filename.split("task-")[1].split("_bold")[0] elif "_" in json_filename: # if this is an all-scan JSON file, but specific only to # the run or acquisition: create a tag that reads @@ -892,41 +877,48 @@ def get_BIDS_data_dct(bids_base_dir, file_list=None, anat_scan=None, if ses: # if there is a session level in the BIDS dataset - data_dct = get_nonBIDS_data(anat_sess, func_sess, file_list=file_list, - anat_scan=anat_scan, - scan_params_dct=scan_params_dct, - brain_mask_template=brain_mask_template, - fmap_phase_template=fmap_phase_sess, - fmap_mag_template=fmap_mag_sess, - fmap_pedir_template=fmap_pedir_sess, - freesurfer_dir=freesurfer_dir, - aws_creds_path=aws_creds_path, - inclusion_dct=inclusion_dct, - exclusion_dct=exclusion_dct, - sites_dct=sites_subs_dct) + data_dct = get_nonBIDS_data( + anat_sess, + func_sess, + file_list=file_list, + anat_scan=anat_scan, + scan_params_dct=scan_params_dct, + brain_mask_template=brain_mask_template, + fmap_phase_template=fmap_phase_sess, + fmap_mag_template=fmap_mag_sess, + fmap_pedir_template=fmap_pedir_sess, + freesurfer_dir=freesurfer_dir, + aws_creds_path=aws_creds_path, + inclusion_dct=inclusion_dct, + exclusion_dct=exclusion_dct, + sites_dct=sites_subs_dct, + ) else: # no session level - data_dct = get_nonBIDS_data(anat, func, file_list=file_list, - anat_scan=anat_scan, - scan_params_dct=scan_params_dct, - brain_mask_template=brain_mask_template, - fmap_phase_template=fmap_phase, - fmap_mag_template=fmap_mag, - fmap_pedir_template=fmap_pedir, - freesurfer_dir=freesurfer_dir, - aws_creds_path=aws_creds_path, - inclusion_dct=inclusion_dct, - exclusion_dct=exclusion_dct, - sites_dct=sites_subs_dct) + data_dct = get_nonBIDS_data( + anat, + func, + file_list=file_list, + anat_scan=anat_scan, + scan_params_dct=scan_params_dct, + brain_mask_template=brain_mask_template, + fmap_phase_template=fmap_phase, + fmap_mag_template=fmap_mag, + fmap_pedir_template=fmap_pedir, + freesurfer_dir=freesurfer_dir, + aws_creds_path=aws_creds_path, + inclusion_dct=inclusion_dct, + exclusion_dct=exclusion_dct, + sites_dct=sites_subs_dct, + ) return data_dct -def find_unique_scan_params(scan_params_dct, site_id, sub_id, ses_id, - scan_id): +def find_unique_scan_params(scan_params_dct, site_id, sub_id, ses_id, scan_id): """Return the scan parameters information stored in the provided scan - parameters dictionary for the IDs of a specific functional input scan.""" - + parameters dictionary for the IDs of a specific functional input scan. + """ scan_params = None if site_id not in scan_params_dct.keys(): @@ -934,7 +926,6 @@ def find_unique_scan_params(scan_params_dct, site_id, sub_id, ses_id, try: scan_params_dct[site_id] = {} except: - print(scan_params_dct) scan_params_dct = {site_id: {}} if sub_id not in scan_params_dct[site_id]: sub_id = "All" @@ -961,40 +952,49 @@ def find_unique_scan_params(scan_params_dct, site_id, sub_id, ses_id, scan_params = scan_params_dct[site_id][sub_id][ses_id][scan_id] except TypeError: # this ideally should never fire - err = "\n[!] The scan parameters dictionary supplied to the data " \ - "configuration builder is not in the proper format.\n\n The " \ - "key combination that caused this error:\n{0}, {1}, {2}, {3}" \ - "\n\n".format(site_id, sub_id, ses_id, scan_id) + err = ( + "\n[!] The scan parameters dictionary supplied to the data " + "configuration builder is not in the proper format.\n\n The " + "key combination that caused this error:\n{0}, {1}, {2}, {3}" + "\n\n".format(site_id, sub_id, ses_id, scan_id) + ) raise Exception(err) except KeyError: pass if not scan_params: - warn = "\n[!] No scan parameter information found in your scan " \ - "parameter configuration for the functional input file:\n" \ - "site: {0}, participant: {1}, session: {2}, series: {3}\n\n" \ - "".format(site_id, sub_id, ses_id, scan_id) - print(warn) + "\n[!] No scan parameter information found in your scan " "parameter configuration for the functional input file:\n" "site: {0}, participant: {1}, session: {2}, series: {3}\n\n" "".format( + site_id, sub_id, ses_id, scan_id + ) return scan_params -def update_data_dct(file_path, file_template, data_dct=None, data_type="anat", - anat_scan=None, sites_dct=None, scan_params_dct=None, - inclusion_dct=None, exclusion_dct=None, - aws_creds_path=None, verbose=True): +def update_data_dct( + file_path, + file_template, + data_dct=None, + data_type="anat", + anat_scan=None, + sites_dct=None, + scan_params_dct=None, + inclusion_dct=None, + exclusion_dct=None, + aws_creds_path=None, + verbose=True, +): """Return a data dictionary with a new file path parsed and added in, - keyed with its appropriate ID labels.""" - - import os + keyed with its appropriate ID labels. + """ import glob + import os from pathlib import Path if not data_dct: data_dct = {} # NIFTIs only - if '.nii' not in file_path and 'freesurfer' not in data_type: + if ".nii" not in file_path and "freesurfer" not in data_type: return data_dct if data_type == "anat": @@ -1016,8 +1016,11 @@ def update_data_dct(file_path, file_template, data_dct=None, data_type="anat", # one of the BIDS tags anat_scan_identifier = True else: - if "sub-" not in tag and "ses-" not in tag and \ - "T1w" not in tag: + if ( + "sub-" not in tag + and "ses-" not in tag + and "T1w" not in tag + ): bids_tags.append(tag) if anat_scan in tag: # the "anatomical_scan" substring provided was @@ -1044,7 +1047,7 @@ def update_data_dct(file_path, file_template, data_dct=None, data_type="anat", # file_template = # /path/to/{site}/sub-{participant}/ses-{session}/func/ # sub-{participant}_ses-{session}_task-{scan}_bold.nii.gz - site_parts = file_template.split('{site}') + site_parts = file_template.split("{site}") # Example, cont. # site_parts = @@ -1052,7 +1055,7 @@ def update_data_dct(file_path, file_template, data_dct=None, data_type="anat", partic_parts = [] for part in site_parts: - partic_parts = partic_parts + part.split('{participant}') + partic_parts = partic_parts + part.split("{participant}") # Example, cont. # partic_parts = @@ -1060,21 +1063,20 @@ def update_data_dct(file_path, file_template, data_dct=None, data_type="anat", ses_parts = [] for part in partic_parts: - ses_parts = ses_parts + part.split('{session}') + ses_parts = ses_parts + part.split("{session}") # Example, cont. # ses_parts = # ['/path/to/', '/sub-', '/ses-', '/func/sub-', '_ses-', # '_task-{scan}_bold.nii.gz'] - if data_type == "anat" or data_type == "brain_mask" \ - or data_type == "freesurfer_dir": + if data_type in ("anat", "brain_mask", "freesurfer_dir"): parts = ses_parts else: # if functional, or field map files parts = [] for part in ses_parts: - parts = parts + part.split('{scan}') + parts = parts + part.split("{scan}") # Example, cont. # parts = ['/path/to/', '/sub-', '/ses-', '/func/sub-', '_ses-', @@ -1083,7 +1085,7 @@ def update_data_dct(file_path, file_template, data_dct=None, data_type="anat", if "*" in file_template: wild_parts = [] for part in parts: - wild_parts = wild_parts + part.split('*') + wild_parts = wild_parts + part.split("*") parts = wild_parts new_template = file_template @@ -1100,7 +1102,7 @@ def update_data_dct(file_path, file_template, data_dct=None, data_type="anat", except IndexError: break label = new_template.split(part1, 1)[1] - if 'freesurfer' not in data_type: + if "freesurfer" not in data_type: label = label.split(part2, 1)[0] if label in path_dct.keys(): @@ -1114,7 +1116,7 @@ def update_data_dct(file_path, file_template, data_dct=None, data_type="anat", id = new_path.split(part1, 1)[1] id = id.split(part2, 1)[0] except: - print(f"Path split exception: {new_path} // {part1}, {part2}") + pass # example, ideally at this point, something like this: # template: /path/to/sub-{participant}/etc. @@ -1127,35 +1129,40 @@ def update_data_dct(file_path, file_template, data_dct=None, data_type="anat", skip = False else: if path_dct[label] != id: - warn = "\n\n[!] WARNING: While parsing your input data " \ - "files, a file path was found with conflicting " \ - "IDs for the same data level.\n\n" \ - "File path: {0}\n" \ - "Level: {1}\n" \ - "Conflicting IDs: {2}, {3}\n\n" \ - "Thus, we can't tell which {4} it belongs to, and " \ - "whether this file should be included or excluded! " \ - "Therefore, this file has not been added to the " \ - "data configuration.".format(file_path, label, - path_dct[label], id, - label.replace("{", "").replace("}", "")) - print(warn) + warn = ( + "\n\n[!] WARNING: While parsing your input data " + "files, a file path was found with conflicting " + "IDs for the same data level.\n\n" + "File path: {0}\n" + "Level: {1}\n" + "Conflicting IDs: {2}, {3}\n\n" + "Thus, we can't tell which {4} it belongs to, and " + "whether this file should be included or excluded! " + "Therefore, this file has not been added to the " + "data configuration.".format( + file_path, + label, + path_dct[label], + id, + label.replace("{", "").replace("}", ""), + ) + ) skip = True break - new_template = new_template.replace(part1, '', 1) - new_template = new_template.replace(label, '', 1) + new_template = new_template.replace(part1, "", 1) + new_template = new_template.replace(label, "", 1) - new_path = new_path.replace(part1, '', 1) - new_path = new_path.replace(id, '', 1) + new_path = new_path.replace(part1, "", 1) + new_path = new_path.replace(id, "", 1) if skip: return data_dct - sub_id = path_dct['{participant}'] + sub_id = path_dct["{participant}"] - if '{site}' in path_dct.keys(): - site_id = path_dct['{site}'] + if "{site}" in path_dct.keys(): + site_id = path_dct["{site}"] elif sites_dct: # mainly if we're pulling site info from a participants.tsv file # for a BIDS data set @@ -1164,80 +1171,84 @@ def update_data_dct(file_path, file_template, data_dct=None, data_type="anat", except KeyError: site_id = "site-1" else: - site_id = 'site-1' + site_id = "site-1" - if '{session}' in path_dct.keys(): - ses_id = path_dct['{session}'] + if "{session}" in path_dct.keys(): + ses_id = path_dct["{session}"] else: - ses_id = 'ses-1' + ses_id = "ses-1" - if data_type != "anat" and data_type != "brain_mask": - if '{scan}' in path_dct.keys(): - scan_id = path_dct['{scan}'] + if data_type not in ("anat", "brain_mask"): + if "{scan}" in path_dct.keys(): + scan_id = path_dct["{scan}"] else: if data_type == "func": - scan_id = 'func-1' + scan_id = "func-1" else: # field map files - keep these open as "None" so that they # can be applied to all scans, if there isn't one specified scan_id = None if inclusion_dct: - if 'sites' in inclusion_dct.keys(): - if site_id not in inclusion_dct['sites']: + if "sites" in inclusion_dct.keys(): + if site_id not in inclusion_dct["sites"]: return data_dct - if 'sessions' in inclusion_dct.keys(): - if ses_id not in inclusion_dct['sessions']: + if "sessions" in inclusion_dct.keys(): + if ses_id not in inclusion_dct["sessions"]: return data_dct - if 'participants' in inclusion_dct.keys(): - if all([ - sub_id not in inclusion_dct['participants'], - f"sub-{sub_id}" not in inclusion_dct['participants'], - sub_id.split("sub-")[-1] not in inclusion_dct['participants'] - ]): + if "participants" in inclusion_dct.keys(): + if all( + [ + sub_id not in inclusion_dct["participants"], + f"sub-{sub_id}" not in inclusion_dct["participants"], + sub_id.split("sub-")[-1] not in inclusion_dct["participants"], + ] + ): return data_dct if data_type != "anat": - if 'scans' in inclusion_dct.keys(): - if scan_id not in inclusion_dct['scans']: + if "scans" in inclusion_dct.keys(): + if scan_id not in inclusion_dct["scans"]: return data_dct if exclusion_dct: - if 'sites' in exclusion_dct.keys(): - if site_id in exclusion_dct['sites']: + if "sites" in exclusion_dct.keys(): + if site_id in exclusion_dct["sites"]: return data_dct - if 'sessions' in exclusion_dct.keys(): - if ses_id in exclusion_dct['sessions']: + if "sessions" in exclusion_dct.keys(): + if ses_id in exclusion_dct["sessions"]: return data_dct - if 'participants' in exclusion_dct.keys(): - if any([ - sub_id in exclusion_dct['participants'], - f"sub-{sub_id}" in exclusion_dct['participants'], - sub_id.split("sub-")[-1] in exclusion_dct['participants'] - ]): + if "participants" in exclusion_dct.keys(): + if any( + [ + sub_id in exclusion_dct["participants"], + f"sub-{sub_id}" in exclusion_dct["participants"], + sub_id.split("sub-")[-1] in exclusion_dct["participants"], + ] + ): return data_dct if data_type != "anat": - if 'scans' in exclusion_dct.keys(): - if scan_id in exclusion_dct['scans']: + if "scans" in exclusion_dct.keys(): + if scan_id in exclusion_dct["scans"]: return data_dct # start the data dictionary updating - if data_type == 'anat': + if data_type == "anat": if "*" in file_path: if "s3://" in file_path: - err = "\n\n[!] Cannot use wildcards (*) in AWS S3 bucket " \ - "(s3://) paths!" + err = ( + "\n\n[!] Cannot use wildcards (*) in AWS S3 bucket " + "(s3://) paths!" + ) raise Exception(err) - paths = glob.glob(file_path) - - temp_sub_dct = {'subject_id': sub_id, - 'unique_id': ses_id, - 'site': site_id, - 'anat': { - 'T1w': file_path, - 'freesurfer_dir': None - } - } + glob.glob(file_path) + + temp_sub_dct = { + "subject_id": sub_id, + "unique_id": ses_id, + "site": site_id, + "anat": {"T1w": file_path, "freesurfer_dir": None}, + } if aws_creds_path: - temp_sub_dct.update({ 'creds_path': str(aws_creds_path) }) + temp_sub_dct.update({"creds_path": str(aws_creds_path)}) if site_id not in data_dct.keys(): data_dct[site_id] = {} @@ -1247,89 +1258,78 @@ def update_data_dct(file_path, file_template, data_dct=None, data_type="anat", data_dct[site_id][sub_id][ses_id] = temp_sub_dct else: # doubt this ever happens, but just be safe - warn = "\n\n[!] WARNING: Multiple site-participant-session " \ - "entries found for anatomical scans in your input data " \ - "directory.\n\nDuplicate sets:\n\n{0}\n\n{1}\n\nOnly " \ - "adding the first one to the data configuration file." \ - "\n\n".format(str(data_dct[site_id][sub_id][ses_id]), - str(temp_sub_dct)) - print(warn) - - elif data_type == 'freesurfer_dir': + warn = ( + "\n\n[!] WARNING: Multiple site-participant-session " + "entries found for anatomical scans in your input data " + "directory.\n\nDuplicate sets:\n\n{0}\n\n{1}\n\nOnly " + "adding the first one to the data configuration file." + "\n\n".format(str(data_dct[site_id][sub_id][ses_id]), str(temp_sub_dct)) + ) + + elif data_type == "freesurfer_dir": if site_id not in data_dct.keys(): if verbose: - print("No anatomical entries found for freesurfer for " \ - "site {0}:" \ - "\n{1}\n".format(site_id, file_path)) + pass return data_dct if sub_id not in data_dct[site_id]: if verbose: - print("No anatomical found for freesurfer for participant " \ - "{0}:\n{1}\n".format(sub_id, file_path)) + pass return data_dct if ses_id not in data_dct[site_id][sub_id]: if verbose: - print("No anatomical found for freesurfer for session {0}:" \ - "\n{1}\n".format(ses_id, file_path)) + pass return data_dct - data_dct[site_id][sub_id][ses_id]['anat']['freesurfer_dir'] = file_path + data_dct[site_id][sub_id][ses_id]["anat"]["freesurfer_dir"] = file_path - elif data_type == 'brain_mask': + elif data_type == "brain_mask": if site_id not in data_dct.keys(): if verbose: - print("No anatomical entries found for brain mask for " \ - "site {0}:" \ - "\n{1}\n".format(site_id, file_path)) + pass return data_dct if sub_id not in data_dct[site_id]: if verbose: - print("No anatomical found for brain mask for participant " \ - "{0}:\n{1}\n".format(sub_id, file_path)) + pass return data_dct if ses_id not in data_dct[site_id][sub_id]: if verbose: - print("No anatomical found for brain mask for session {0}:" \ - "\n{1}\n".format(ses_id, file_path)) + pass return data_dct - data_dct[site_id][sub_id][ses_id]['brain_mask'] = file_path + data_dct[site_id][sub_id][ses_id]["brain_mask"] = file_path - elif data_type == 'func': + elif data_type == "func": temp_func_dct = {scan_id: {"scan": file_path, "scan_parameters": None}} _fp = Path(file_path) # scan parameters time scan_params = None if scan_params_dct: - scan_params = find_unique_scan_params(scan_params_dct, site_id, - sub_id, ses_id, scan_id) + scan_params = find_unique_scan_params( + scan_params_dct, site_id, sub_id, ses_id, scan_id + ) else: - scan_params = str(_fp.absolute()).replace(''.join(_fp.suffixes), '.json') + scan_params = str(_fp.absolute()).replace("".join(_fp.suffixes), ".json") if scan_params: - temp_func_dct[scan_id]['scan_parameters'] = scan_params + temp_func_dct[scan_id]["scan_parameters"] = scan_params if site_id not in data_dct.keys(): if verbose: - print("No anatomical entries found for functional for " \ - "site {0}:" \ - "\n{1}\n".format(site_id, file_path)) + pass return data_dct if sub_id not in data_dct[site_id]: if verbose: - print("No anatomical found for functional for participant " \ - "{0}:\n{1}\n".format(sub_id, file_path)) + pass return data_dct if ses_id not in data_dct[site_id][sub_id]: if verbose: - print("No anatomical found for functional for session {0}:" \ - "\n{1}\n".format(ses_id, file_path)) + pass return data_dct - if 'func' not in data_dct[site_id][sub_id][ses_id]: - data_dct[site_id][sub_id][ses_id]['func'] = temp_func_dct + if "func" not in data_dct[site_id][sub_id][ses_id]: + data_dct[site_id][sub_id][ses_id]["func"] = temp_func_dct else: - data_dct[site_id][sub_id][ses_id]['func'].update(temp_func_dct) + data_dct[site_id][sub_id][ses_id]["func"].update(temp_func_dct) else: # field map files! @@ -1352,18 +1352,16 @@ def update_data_dct(file_path, file_template, data_dct=None, data_type="anat", # how we use the function in "get_nonBIDS_data" below # but what if we want to use this manually as part of an API? try: - scan_ids = data_dct[site_id][sub_id][ses_id]['func'] + scan_ids = data_dct[site_id][sub_id][ses_id]["func"] for func_scan_id in scan_ids: if run_id and acq_id: if run_id and acq_id in func_scan_id: scan_id = func_scan_id elif run_id: - if run_id in func_scan_id and \ - "acq-" not in func_scan_id: + if run_id in func_scan_id and "acq-" not in func_scan_id: scan_id = func_scan_id elif acq_id: - if acq_id in func_scan_id and \ - "run-" not in func_scan_id: + if acq_id in func_scan_id and "run-" not in func_scan_id: scan_id = func_scan_id except KeyError: # TODO: error msg @@ -1373,75 +1371,91 @@ def update_data_dct(file_path, file_template, data_dct=None, data_type="anat", if site_id not in data_dct.keys(): if verbose: - print("No anatomical entries found for field map file for " \ - "site {0}:\n{1}\n".format(site_id, file_path)) + pass return data_dct if sub_id not in data_dct[site_id]: if verbose: - print("No anatomical found for field map file for " \ - "participant {0}:\n{1}\n".format(sub_id, file_path)) + pass return data_dct if ses_id not in data_dct[site_id][sub_id]: if verbose: for temp_ses in data_dct[site_id][sub_id]: - if 'anat' in data_dct[site_id][sub_id][temp_ses]: - warn = "Field map file found for session {0}, but " \ - "the anatomical scan chosen for this " \ - "participant-session is for session {1}, " \ - "so this field map file is being " \ - "skipped:\n{2}\n".format(ses_id, temp_ses, - file_path) - warn = "{0}\nIf you wish to use the anatomical " \ - "scan for session {1} for all participants " \ - "with this session instead, use the 'Which " \ - "Anatomical Scan?' option in the data " \ - "configuration builder (or populate the " \ - "'anatomical_scan' field in the data " \ - "settings file).\n".format(warn, ses_id) + if "anat" in data_dct[site_id][sub_id][temp_ses]: + warn = ( + "Field map file found for session {0}, but " + "the anatomical scan chosen for this " + "participant-session is for session {1}, " + "so this field map file is being " + "skipped:\n{2}\n".format(ses_id, temp_ses, file_path) + ) + warn = ( + "{0}\nIf you wish to use the anatomical " + "scan for session {1} for all participants " + "with this session instead, use the 'Which " + "Anatomical Scan?' option in the data " + "configuration builder (or populate the " + "'anatomical_scan' field in the data " + "settings file).\n".format(warn, ses_id) + ) break else: - warn = "No anatomical found for field map file for " \ - "session {0}:\n{1}\n".format(ses_id, file_path) - print(warn) + warn = ( + "No anatomical found for field map file for " + "session {0}:\n{1}\n".format(ses_id, file_path) + ) return data_dct - if 'fmap' not in data_dct[site_id][sub_id][ses_id]: + if "fmap" not in data_dct[site_id][sub_id][ses_id]: # would this ever fire? the way we're using this function now - data_dct[site_id][sub_id][ses_id]['fmap'] = {} - data_dct[site_id][sub_id][ses_id]['fmap'].update(temp_fmap_dct) + data_dct[site_id][sub_id][ses_id]["fmap"] = {} + data_dct[site_id][sub_id][ses_id]["fmap"].update(temp_fmap_dct) return data_dct -def get_nonBIDS_data(anat_template, func_template, file_list=None, - anat_scan=None, scan_params_dct=None, - brain_mask_template=None, fmap_phase_template=None, - fmap_mag_template=None, fmap_pedir_template=None, - freesurfer_dir=None, aws_creds_path=None, - inclusion_dct=None, exclusion_dct=None, - sites_dct=None, verbose=False): +def get_nonBIDS_data( + anat_template, + func_template, + file_list=None, + anat_scan=None, + scan_params_dct=None, + brain_mask_template=None, + fmap_phase_template=None, + fmap_mag_template=None, + fmap_pedir_template=None, + freesurfer_dir=None, + aws_creds_path=None, + inclusion_dct=None, + exclusion_dct=None, + sites_dct=None, + verbose=False, +): """Prepare a data dictionary for the data configuration file when given - file path templates describing the input data directories.""" - - import os - import glob + file path templates describing the input data directories. + """ import fnmatch + import glob + import os if not func_template: - func_template = '' + func_template = "" # should have the {participant} label at the very least - if '{participant}' not in anat_template: - err = "\n[!] The {participant} keyword is missing from your " \ - "anatomical path template.\n" + if "{participant}" not in anat_template: + err = ( + "\n[!] The {participant} keyword is missing from your " + "anatomical path template.\n" + ) raise Exception(err) - if len(func_template) > 0 and '{participant}' not in func_template: - err = "\n[!] The {participant} keyword is missing from your " \ - "functional path template.\n" + if len(func_template) > 0 and "{participant}" not in func_template: + err = ( + "\n[!] The {participant} keyword is missing from your " + "functional path template.\n" + ) raise Exception(err) - keywords = ['{site}', '{participant}', '{session}', '{scan}'] + keywords = ["{site}", "{participant}", "{session}", "{scan}"] # make globby templates, to use them to filter down the path_list into # only paths that will work with the templates @@ -1449,20 +1463,22 @@ def get_nonBIDS_data(anat_template, func_template, file_list=None, func_glob = func_template # backwards compatibility - if '{series}' in anat_glob: - anat_template = anat_template.replace('{series}', '{scan}') - anat_glob = anat_glob.replace('{series}', '{scan}') - - if '{series}' in func_glob: - func_template = func_template.replace('{series}', '{scan}') - func_glob = func_glob.replace('{series}', '{scan}') - - if '{scan}' in anat_glob: - err = "\n[!] CPAC does not support multiple anatomical scans. You " \ - "are seeing this message because you have a {scan} or " \ - "{series} keyword in your anatomical path template.\n\nSee " \ - "the help details for the 'Which Anatomical Scan?' " \ - "(anatomical_scan) setting for more information.\n\n" + if "{series}" in anat_glob: + anat_template = anat_template.replace("{series}", "{scan}") + anat_glob = anat_glob.replace("{series}", "{scan}") + + if "{series}" in func_glob: + func_template = func_template.replace("{series}", "{scan}") + func_glob = func_glob.replace("{series}", "{scan}") + + if "{scan}" in anat_glob: + err = ( + "\n[!] CPAC does not support multiple anatomical scans. You " + "are seeing this message because you have a {scan} or " + "{series} keyword in your anatomical path template.\n\nSee " + "the help details for the 'Which Anatomical Scan?' " + "(anatomical_scan) setting for more information.\n\n" + ) raise Exception(err) if anat_scan: @@ -1472,9 +1488,9 @@ def get_nonBIDS_data(anat_template, func_template, file_list=None, # replace the keywords with wildcards for keyword in keywords: if keyword in anat_glob: - anat_glob = anat_glob.replace(keyword, '*') + anat_glob = anat_glob.replace(keyword, "*") if keyword in func_glob: - func_glob = func_glob.replace(keyword, '*') + func_glob = func_glob.replace(keyword, "*") # presumably, the paths contained in each of these pools should be anat # and func files only, respectively, if the templates were set up properly @@ -1500,21 +1516,33 @@ def get_nonBIDS_data(anat_template, func_template, file_list=None, func_pool = func_pool + [x for x in func_local_pool if x not in func_pool] if not anat_pool: - err = "\n\n[!] No anatomical input file paths found given the data " \ - "settings provided.\n\nAnatomical file template being used: " \ - "{0}\n".format(anat_glob) + err = ( + "\n\n[!] No anatomical input file paths found given the data " + "settings provided.\n\nAnatomical file template being used: " + "{0}\n".format(anat_glob) + ) if anat_scan: - err = "{0}Anatomical scan identifier provided: {1}" \ - "\n\n".format(err, anat_scan) + err = "{0}Anatomical scan identifier provided: {1}" "\n\n".format( + err, anat_scan + ) raise Exception(err) # pull out the site/participant/etc. IDs from each path and connect them # for the anatomicals data_dct = {} for anat_path in anat_pool: - data_dct = update_data_dct(anat_path, anat_template, data_dct, "anat", - anat_scan, sites_dct, None, inclusion_dct, - exclusion_dct, aws_creds_path) + data_dct = update_data_dct( + anat_path, + anat_template, + data_dct, + "anat", + anat_scan, + sites_dct, + None, + inclusion_dct, + exclusion_dct, + aws_creds_path, + ) if not data_dct: # this fires if no anatomicals were found @@ -1522,52 +1550,70 @@ def get_nonBIDS_data(anat_template, func_template, file_list=None, possible_anats = [] all_tags = [] for anat_path in anat_pool: - if "T1w" in anat_path or "mprage" in anat_path or \ - "anat" in anat_path: + if "T1w" in anat_path or "mprage" in anat_path or "anat" in anat_path: possible_anats.append(anat_path) - all_tags += anat_path.replace('.nii', '').replace('.gz', '').split('_') + all_tags += anat_path.replace(".nii", "").replace(".gz", "").split("_") from collections import Counter + count = Counter(all_tags) all_tags = [k for k, v in count.items() if v > 1] tags = [] for tag in all_tags: - if '/' in tag or 'ses-' in tag: + if "/" in tag or "ses-" in tag: continue tags.append(tag) - err = "\n\n[!] No anatomical input files were found given the " \ - "data settings provided.\n\n" + err = ( + "\n\n[!] No anatomical input files were found given the " + "data settings provided.\n\n" + ) if possible_anats: - err = "{0}There are some file paths found in the directories " \ - "described in the data settings that may be anatomicals " \ - "that were missed. Here are a few examples:\n".format(err) + err = ( + "{0}There are some file paths found in the directories " + "described in the data settings that may be anatomicals " + "that were missed. Here are a few examples:\n".format(err) + ) for anat in possible_anats[0:5]: err = "{0}{1}\n".format(err, anat) - err = "{0}\nAnd here are some of the possible tags that were " \ - "found in the anatomical file paths that were grabbed:" \ - "\n".format(err) + err = ( + "{0}\nAnd here are some of the possible tags that were " + "found in the anatomical file paths that were grabbed:" + "\n".format(err) + ) for tag in tags[0:20]: err = "{0}{1}\n".format(err, tag) - err = "{0}\nCPAC only needs one anatomical scan defined for " \ - "each participant-session. If there are multiple " \ - "anatomical scans per participant-session, you can use " \ - "the 'Which Anatomical Scan?' (anatomical_scan) " \ - "parameter to choose which anatomical to " \ - "select.\n".format(err) - err = "{0}\nIf you are already using the 'anatomical_scan' " \ - "option in the data settings, check the setting to make " \ - "sure you are properly selecting which anatomical scan " \ - "to use for your analysis.\n\n".format(err) + err = ( + "{0}\nCPAC only needs one anatomical scan defined for " + "each participant-session. If there are multiple " + "anatomical scans per participant-session, you can use " + "the 'Which Anatomical Scan?' (anatomical_scan) " + "parameter to choose which anatomical to " + "select.\n".format(err) + ) + err = ( + "{0}\nIf you are already using the 'anatomical_scan' " + "option in the data settings, check the setting to make " + "sure you are properly selecting which anatomical scan " + "to use for your analysis.\n\n".format(err) + ) raise Exception(err) # now gather the functionals for func_path in func_pool: - data_dct = update_data_dct(func_path, func_template, data_dct, "func", - None, sites_dct, scan_params_dct, - inclusion_dct, exclusion_dct, - aws_creds_path) + data_dct = update_data_dct( + func_path, + func_template, + data_dct, + "func", + None, + sites_dct, + scan_params_dct, + inclusion_dct, + exclusion_dct, + aws_creds_path, + ) if freesurfer_dir: # make globby templates, to use them to filter down the path_list into @@ -1575,7 +1621,7 @@ def get_nonBIDS_data(anat_template, func_template, file_list=None, for keyword in keywords: if keyword in freesurfer_dir: - freesurfer_glob = freesurfer_dir.replace(keyword, '*') + freesurfer_glob = freesurfer_dir.replace(keyword, "*") # presumably, the paths contained in each of these pools should be # field map files only, if the templates were set up properly @@ -1587,15 +1633,21 @@ def get_nonBIDS_data(anat_template, func_template, file_list=None, freesurfer_pool.append(filepath) else: for dir in os.listdir(str(os.path.dirname(freesurfer_glob))): - freesurfer_pool.append(freesurfer_glob.replace('*', dir)) - + freesurfer_pool.append(freesurfer_glob.replace("*", dir)) for freesurfer_path in freesurfer_pool: - data_dct = update_data_dct(freesurfer_path, freesurfer_dir, - data_dct, "freesurfer_dir", None, - sites_dct, scan_params_dct, - inclusion_dct, exclusion_dct, - aws_creds_path) + data_dct = update_data_dct( + freesurfer_path, + freesurfer_dir, + data_dct, + "freesurfer_dir", + None, + sites_dct, + scan_params_dct, + inclusion_dct, + exclusion_dct, + aws_creds_path, + ) if brain_mask_template: # make globby templates, to use them to filter down the path_list into # only paths that will work with the templates @@ -1603,7 +1655,7 @@ def get_nonBIDS_data(anat_template, func_template, file_list=None, for keyword in keywords: if keyword in brain_mask_glob: - brain_mask_glob = brain_mask_glob.replace(keyword, '*') + brain_mask_glob = brain_mask_glob.replace(keyword, "*") # presumably, the paths contained in each of these pools should be # field map files only, if the templates were set up properly @@ -1617,11 +1669,18 @@ def get_nonBIDS_data(anat_template, func_template, file_list=None, brain_mask_pool = glob.glob(brain_mask_glob) for brain_mask in brain_mask_pool: - data_dct = update_data_dct(brain_mask, brain_mask_template, - data_dct, "brain_mask", None, - sites_dct, scan_params_dct, - inclusion_dct, exclusion_dct, - aws_creds_path) + data_dct = update_data_dct( + brain_mask, + brain_mask_template, + data_dct, + "brain_mask", + None, + sites_dct, + scan_params_dct, + inclusion_dct, + exclusion_dct, + aws_creds_path, + ) # do the same for the fieldmap files, if applicable if fmap_phase_template and fmap_mag_template: @@ -1633,18 +1692,18 @@ def get_nonBIDS_data(anat_template, func_template, file_list=None, fmap_mag_glob = fmap_mag_template # backwards compatibility - if '{series}' in fmap_phase_glob: - fmap_phase_template = fmap_phase_template.replace('{series}', '{scan}') - fmap_phase_glob = fmap_phase_glob.replace('{series}', '{scan}') - if '{series}' in fmap_mag_glob: - fmap_mag_template = fmap_mag_template.replace('{series}', '{scan}') - fmap_mag_glob = fmap_mag_glob.replace('{series}', '{scan}') + if "{series}" in fmap_phase_glob: + fmap_phase_template = fmap_phase_template.replace("{series}", "{scan}") + fmap_phase_glob = fmap_phase_glob.replace("{series}", "{scan}") + if "{series}" in fmap_mag_glob: + fmap_mag_template = fmap_mag_template.replace("{series}", "{scan}") + fmap_mag_glob = fmap_mag_glob.replace("{series}", "{scan}") for keyword in keywords: if keyword in fmap_phase_glob: - fmap_phase_glob = fmap_phase_glob.replace(keyword, '*') + fmap_phase_glob = fmap_phase_glob.replace(keyword, "*") if keyword in fmap_mag_glob: - fmap_mag_glob = fmap_mag_glob.replace(keyword, '*') + fmap_mag_glob = fmap_mag_glob.replace(keyword, "*") # presumably, the paths contained in each of these pools should be # field map files only, if the templates were set up properly @@ -1662,18 +1721,32 @@ def get_nonBIDS_data(anat_template, func_template, file_list=None, fmap_mag_pool = glob.glob(fmap_mag_glob) for fmap_phase in fmap_phase_pool: - data_dct = update_data_dct(fmap_phase, fmap_phase_template, - data_dct, "diff_phase", None, - sites_dct, scan_params_dct, - inclusion_dct, exclusion_dct, - aws_creds_path) + data_dct = update_data_dct( + fmap_phase, + fmap_phase_template, + data_dct, + "diff_phase", + None, + sites_dct, + scan_params_dct, + inclusion_dct, + exclusion_dct, + aws_creds_path, + ) for fmap_mag in fmap_mag_pool: - data_dct = update_data_dct(fmap_mag, fmap_mag_template, - data_dct, "diff_mag", None, - sites_dct, scan_params_dct, - inclusion_dct, exclusion_dct, - aws_creds_path) + data_dct = update_data_dct( + fmap_mag, + fmap_mag_template, + data_dct, + "diff_mag", + None, + sites_dct, + scan_params_dct, + inclusion_dct, + exclusion_dct, + aws_creds_path, + ) if fmap_pedir_template: # make globby templates, to use them to filter down the path_list into @@ -1682,7 +1755,7 @@ def get_nonBIDS_data(anat_template, func_template, file_list=None, for keyword in keywords: if keyword in fmap_pedir_glob: - fmap_pedir_glob = fmap_pedir_glob.replace(keyword, '*') + fmap_pedir_glob = fmap_pedir_glob.replace(keyword, "*") # presumably, the paths contained in each of these pools should be # field map files only, if the templates were set up properly @@ -1695,15 +1768,22 @@ def get_nonBIDS_data(anat_template, func_template, file_list=None, else: fmap_pedir_pool = glob.glob(fmap_pedir_glob) - #TODO: must now deal with phase encoding direction!!!! - #TODO: have to check scan params, first!!! + # TODO: must now deal with phase encoding direction!!!! + # TODO: have to check scan params, first!!! for fmap_pedir in fmap_pedir_pool: - data_dct = update_data_dct(fmap_pedir, fmap_pedir_template, - data_dct, "fmap_pedir", None, - sites_dct, scan_params_dct, - inclusion_dct, exclusion_dct, - aws_creds_path) + data_dct = update_data_dct( + fmap_pedir, + fmap_pedir_template, + data_dct, + "fmap_pedir", + None, + sites_dct, + scan_params_dct, + inclusion_dct, + exclusion_dct, + aws_creds_path, + ) return data_dct @@ -1712,208 +1792,226 @@ def util_copy_template(template_type=None): """Copy the data settings YAML file template to the current directory.""" import os import shutil + import pkg_resources as p + from CPAC.utils.configuration import preconfig_yaml template_type = "data_settings" if not template_type else template_type - settings_template = preconfig_yaml('default') if ( - template_type == "pipeline_config" - ) else p.resource_filename("CPAC", os.path.join( - "resources", "configs", f"{template_type}_template.yml")) + settings_template = ( + preconfig_yaml("default") + if (template_type == "pipeline_config") + else p.resource_filename( + "CPAC", + os.path.join("resources", "configs", f"{template_type}_template.yml"), + ) + ) settings_file = os.path.join(os.getcwd(), f"{template_type}.yml") try: if os.path.exists(settings_file): - settings_file = os.path.join(os.getcwd(), - f"{template_type}_1.yml") + settings_file = os.path.join(os.getcwd(), f"{template_type}_1.yml") while os.path.exists(settings_file): idx = int( - os.path.basename(settings_file).split("_")[2].replace( - ".yml", "")) - settings_file = os.path.join(os.getcwd(), - f"{template_type}_{idx + 1}.yml") + os.path.basename(settings_file).split("_")[2].replace(".yml", "") + ) + settings_file = os.path.join( + os.getcwd(), f"{template_type}_{idx + 1}.yml" + ) shutil.copy(settings_template, settings_file) except Exception as exception: - raise Exception(f"\n[!] Could not write the {template_type} file " - "template to the current directory.\n") from exception - - print(f"\nGenerated a default {template_type} YAML file for editing:\n" - f"{settings_file}\n\n") - if type == 'data_settings': - print("This file can be completed and entered into the C-PAC " - "command-line interface to generate a data configuration file " - "for individual-level analysis by running 'cpac utils " - "data_config build {data settings file}'.\nAdditionally, it can " - "also be loaded into the CPAC data configuration file builder " - "UI using the 'Load Preset' button.\n") - elif type == 'pipeline_config': - print("This file can be edited and then used in a C-PAC run by " - "running 'cpac run --pipe_config {pipeline config file} " - "{data config file}'.\n") + raise Exception( + f"\n[!] Could not write the {template_type} file " + "template to the current directory.\n" + ) from exception + + if type == "data_settings": + pass + elif type == "pipeline_config": + pass def run(data_settings_yml): """Generate and write out a CPAC data configuration (participant list) - YAML file.""" - + YAML file. + """ import os + import yaml - import CPAC - print("\nGenerating data configuration file..") + import CPAC - settings_dct = yaml.safe_load(open(data_settings_yml, 'r')) + settings_dct = yaml.safe_load(open(data_settings_yml, "r")) - if "awsCredentialsFile" not in settings_dct or \ - not settings_dct["awsCredentialsFile"]: + if ( + "awsCredentialsFile" not in settings_dct + or not settings_dct["awsCredentialsFile"] + ): settings_dct["awsCredentialsFile"] = None - elif "None" in settings_dct["awsCredentialsFile"] or \ - "none" in settings_dct["awsCredentialsFile"]: + elif ( + "None" in settings_dct["awsCredentialsFile"] + or "none" in settings_dct["awsCredentialsFile"] + ): settings_dct["awsCredentialsFile"] = None - if "anatomical_scan" not in settings_dct or \ - not settings_dct["anatomical_scan"]: + if "anatomical_scan" not in settings_dct or not settings_dct["anatomical_scan"]: settings_dct["anatomical_scan"] = None - elif "None" in settings_dct["anatomical_scan"] or \ - "none" in settings_dct["anatomical_scan"]: + elif ( + "None" in settings_dct["anatomical_scan"] + or "none" in settings_dct["anatomical_scan"] + ): settings_dct["anatomical_scan"] = None # inclusion lists - incl_dct = format_incl_excl_dct(settings_dct.get('siteList', None), 'sites') - incl_dct.update(format_incl_excl_dct(settings_dct.get('subjectList', None), - 'participants')) - incl_dct.update(format_incl_excl_dct(settings_dct.get('sessionList', None), - 'sessions')) - incl_dct.update(format_incl_excl_dct(settings_dct.get('scanList', None), 'scans')) + incl_dct = format_incl_excl_dct(settings_dct.get("siteList", None), "sites") + incl_dct.update( + format_incl_excl_dct(settings_dct.get("subjectList", None), "participants") + ) + incl_dct.update( + format_incl_excl_dct(settings_dct.get("sessionList", None), "sessions") + ) + incl_dct.update(format_incl_excl_dct(settings_dct.get("scanList", None), "scans")) # exclusion lists - excl_dct = format_incl_excl_dct(settings_dct.get('exclusionSiteList', None), - 'sites') - excl_dct.update(format_incl_excl_dct(settings_dct.get('exclusionSubjectList', None), - 'participants')) - excl_dct.update(format_incl_excl_dct(settings_dct.get('exclusionSessionList', None), - 'sessions')) - excl_dct.update(format_incl_excl_dct(settings_dct.get('exclusionScanList', None), - 'scans')) - - if 'bids' in settings_dct['dataFormat'].lower(): - - file_list = get_file_list(settings_dct["bidsBaseDir"], - creds_path=settings_dct["awsCredentialsFile"]) - - data_dct = get_BIDS_data_dct(settings_dct['bidsBaseDir'], - file_list=file_list, - brain_mask_template=settings_dct['brain_mask_template'], - anat_scan=settings_dct['anatomical_scan'], - aws_creds_path=settings_dct['awsCredentialsFile'], - inclusion_dct=incl_dct, - exclusion_dct=excl_dct, - config_dir=settings_dct["outputSubjectListLocation"]) - - elif 'custom' in settings_dct['dataFormat'].lower(): - + excl_dct = format_incl_excl_dct( + settings_dct.get("exclusionSiteList", None), "sites" + ) + excl_dct.update( + format_incl_excl_dct( + settings_dct.get("exclusionSubjectList", None), "participants" + ) + ) + excl_dct.update( + format_incl_excl_dct(settings_dct.get("exclusionSessionList", None), "sessions") + ) + excl_dct.update( + format_incl_excl_dct(settings_dct.get("exclusionScanList", None), "scans") + ) + + if "bids" in settings_dct["dataFormat"].lower(): + file_list = get_file_list( + settings_dct["bidsBaseDir"], creds_path=settings_dct["awsCredentialsFile"] + ) + + data_dct = get_BIDS_data_dct( + settings_dct["bidsBaseDir"], + file_list=file_list, + brain_mask_template=settings_dct["brain_mask_template"], + anat_scan=settings_dct["anatomical_scan"], + aws_creds_path=settings_dct["awsCredentialsFile"], + inclusion_dct=incl_dct, + exclusion_dct=excl_dct, + config_dir=settings_dct["outputSubjectListLocation"], + ) + + elif "custom" in settings_dct["dataFormat"].lower(): # keep as None if local data set (not on AWS S3 bucket) file_list = None base_dir = None if "s3://" in settings_dct["anatomicalTemplate"]: # hosted on AWS S3 bucket - if '{site}' in settings_dct["anatomicalTemplate"]: - base_dir = \ - settings_dct["anatomicalTemplate"].split('{site}')[0] - elif '{participant}' in settings_dct["anatomicalTemplate"]: - base_dir = \ - settings_dct["anatomicalTemplate"].split('{participant}')[0] + if "{site}" in settings_dct["anatomicalTemplate"]: + base_dir = settings_dct["anatomicalTemplate"].split("{site}")[0] + elif "{participant}" in settings_dct["anatomicalTemplate"]: + base_dir = settings_dct["anatomicalTemplate"].split("{participant}")[0] elif "s3://" in settings_dct["functionalTemplate"]: # hosted on AWS S3 bucket - if '{site}' in settings_dct["functionalTemplate"]: - base_dir = \ - settings_dct["functionalTemplate"].split('{site}')[0] - elif '{participant}' in settings_dct["functionalTemplate"]: - base_dir = \ - settings_dct["functionalTemplate"].split('{participant}')[0] + if "{site}" in settings_dct["functionalTemplate"]: + base_dir = settings_dct["functionalTemplate"].split("{site}")[0] + elif "{participant}" in settings_dct["functionalTemplate"]: + base_dir = settings_dct["functionalTemplate"].split("{participant}")[0] if base_dir: - file_list = pull_s3_sublist(base_dir, - settings_dct['awsCredentialsFile']) + file_list = pull_s3_sublist(base_dir, settings_dct["awsCredentialsFile"]) params_dct = None - if settings_dct['scanParametersCSV']: - if '.csv' in settings_dct['scanParametersCSV']: - params_dct = \ - extract_scan_params_csv(settings_dct['scanParametersCSV']) - - data_dct = get_nonBIDS_data(settings_dct['anatomicalTemplate'], - settings_dct['functionalTemplate'], - file_list=file_list, - anat_scan=settings_dct['anatomical_scan'], - scan_params_dct=params_dct, - brain_mask_template=settings_dct['brain_mask_template'], - fmap_phase_template=settings_dct['fieldMapPhase'], - fmap_mag_template=settings_dct['fieldMapMagnitude'], - freesurfer_dir=settings_dct['freesurfer_dir'], - aws_creds_path=settings_dct['awsCredentialsFile'], - inclusion_dct=incl_dct, - exclusion_dct=excl_dct) + if settings_dct["scanParametersCSV"]: + if ".csv" in settings_dct["scanParametersCSV"]: + params_dct = extract_scan_params_csv(settings_dct["scanParametersCSV"]) + + data_dct = get_nonBIDS_data( + settings_dct["anatomicalTemplate"], + settings_dct["functionalTemplate"], + file_list=file_list, + anat_scan=settings_dct["anatomical_scan"], + scan_params_dct=params_dct, + brain_mask_template=settings_dct["brain_mask_template"], + fmap_phase_template=settings_dct["fieldMapPhase"], + fmap_mag_template=settings_dct["fieldMapMagnitude"], + freesurfer_dir=settings_dct["freesurfer_dir"], + aws_creds_path=settings_dct["awsCredentialsFile"], + inclusion_dct=incl_dct, + exclusion_dct=excl_dct, + ) else: - err = "\n\n[!] You must select a data format- either 'BIDS' or " \ - "'Custom', in the 'dataFormat' field in the data settings " \ - "YAML file.\n\n" + err = ( + "\n\n[!] You must select a data format- either 'BIDS' or " + "'Custom', in the 'dataFormat' field in the data settings " + "YAML file.\n\n" + ) raise Exception(err) if len(data_dct) > 0: - - data_config_outfile = \ - os.path.join(settings_dct['outputSubjectListLocation'], - "data_config_{0}.yml" - "".format(settings_dct['subjectListName'])) - - group_list_outfile = \ - os.path.join(settings_dct['outputSubjectListLocation'], - "group_analysis_participants_{0}.txt" - "".format(settings_dct['subjectListName'])) + data_config_outfile = os.path.join( + settings_dct["outputSubjectListLocation"], + "data_config_{0}.yml" "".format(settings_dct["subjectListName"]), + ) + + group_list_outfile = os.path.join( + settings_dct["outputSubjectListLocation"], + "group_analysis_participants_{0}.txt" "".format( + settings_dct["subjectListName"] + ), + ) # put data_dct contents in an ordered list for the YAML dump data_list = [] group_list = [] - included = {'site': [], 'sub': []} + included = {"site": [], "sub": []} num_sess = num_scan = 0 for site in sorted(data_dct.keys()): for sub in sorted(data_dct[site]): for ses in sorted(data_dct[site][sub]): # if there are scans, get some numbers - included['site'].append(site) - included['sub'].append(sub) + included["site"].append(site) + included["sub"].append(sub) num_sess += 1 - if 'func' in data_dct[site][sub][ses]: - for scan in data_dct[site][sub][ses]['func']: + if "func" in data_dct[site][sub][ses]: + for scan in data_dct[site][sub][ses]["func"]: num_scan += 1 data_list.append(data_dct[site][sub][ses]) group_list.append("{0}_{1}".format(sub, ses)) # calculate numbers - num_sites = len(set(included['site'])) - num_subs = len(set(included['sub'])) + len(set(included["site"])) + len(set(included["sub"])) with open(data_config_outfile, "wt") as f: # Make sure YAML doesn't dump aliases (so it's more human # read-able) - f.write("# CPAC Data Configuration File\n# Version {0}" - "\n".format(CPAC.__version__)) - f.write("#\n# http://fcp-indi.github.io for more info.\n#\n" - "# Tip: This file can be edited manually with " - "a text editor for quick modifications.\n\n") + f.write( + "# CPAC Data Configuration File\n# Version {0}" "\n".format( + CPAC.__version__ + ) + ) + f.write( + "#\n# http://fcp-indi.github.io for more info.\n#\n" + "# Tip: This file can be edited manually with " + "a text editor for quick modifications.\n\n" + ) noalias_dumper = yaml.dumper.SafeDumper noalias_dumper.ignore_aliases = lambda self, data: True - f.write(yaml.dump(data_list, default_flow_style=False, - Dumper=noalias_dumper)) + f.write( + yaml.dump(data_list, default_flow_style=False, Dumper=noalias_dumper) + ) with open(group_list_outfile, "wt") as f: # write the inclusion list (mainly the group analysis sublist) @@ -1922,24 +2020,14 @@ def run(data_settings_yml): f.write("{0}\n".format(id)) if os.path.exists(data_config_outfile): - print("\nCPAC DATA SETTINGS file entered (use this preset file " \ - "to modify/regenerate the data configuration file):" \ - "\n{0}\n".format(data_settings_yml)) - print("Number of:") - print("...sites: {0}".format(num_sites)) - print("...participants: {0}".format(num_subs)) - print("...participant-sessions: {0}".format(num_sess)) - print("...functional scans: {0}".format(num_scan)) - print("\nCPAC DATA CONFIGURATION file created (use this for " \ - "individual-level analysis):" \ - "\n{0}\n".format(data_config_outfile)) + pass if os.path.exists(group_list_outfile): - print("Group-level analysis participant-session list text " \ - "file created (use this for group-level analysis):\n{0}" \ - "\n".format(group_list_outfile)) + pass else: - err = "\n\n[!] No anatomical input files were found given the data " \ - "settings provided.\n\n" + err = ( + "\n\n[!] No anatomical input files were found given the data " + "settings provided.\n\n" + ) raise Exception(err) diff --git a/CPAC/utils/configuration/__init__.py b/CPAC/utils/configuration/__init__.py index e1cc9f0f44..5a70b54d71 100644 --- a/CPAC/utils/configuration/__init__.py +++ b/CPAC/utils/configuration/__init__.py @@ -14,10 +14,22 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -"""C-PAC Configuration module""" -from .configuration import check_pname, Configuration, Preconfiguration, \ - preconfig_yaml, set_subject +"""C-PAC Configuration module.""" from . import configuration, diff +from .configuration import ( + Configuration, + Preconfiguration, + check_pname, + preconfig_yaml, + set_subject, +) -__all__ = ['check_pname', 'Configuration', 'configuration', 'diff', - 'Preconfiguration', 'preconfig_yaml', 'set_subject'] +__all__ = [ + "check_pname", + "Configuration", + "configuration", + "diff", + "Preconfiguration", + "preconfig_yaml", + "set_subject", +] diff --git a/CPAC/utils/configuration/configuration.py b/CPAC/utils/configuration/configuration.py index 72ff3b2363..d74d593302 100644 --- a/CPAC/utils/configuration/configuration.py +++ b/CPAC/utils/configuration/configuration.py @@ -1,4 +1,4 @@ -"""C-PAC Configuration class and related functions +"""C-PAC Configuration class and related functions. Copyright (C) 2022-2023 C-PAC Developers @@ -15,29 +15,31 @@ License for more details. You should have received a copy of the GNU Lesser General Public -License along with C-PAC. If not, see .""" +License along with C-PAC. If not, see . +""" import os import re from typing import Optional from warnings import warn -import pkg_resources as p + from click import BadParameter +import pkg_resources as p import yaml -from CPAC.utils.typing import ConfigKeyType, TUPLE -from .diff import dct_diff - -SPECIAL_REPLACEMENT_STRINGS = {r'${resolution_for_anat}', - r'${func_resolution}'} +from CPAC.utils.typing import TUPLE, ConfigKeyType +from .diff import dct_diff +SPECIAL_REPLACEMENT_STRINGS = {r"${resolution_for_anat}", r"${func_resolution}"} class ConfigurationDictUpdateConflation(SyntaxError): - """Custom exception to clarify similar methods""" + """Custom exception to clarify similar methods.""" + def __init__(self): self.msg = ( - '`Configuration().update` requires a key and a value. ' - 'Perhaps you meant `Configuration().dict().update`?') + "`Configuration().update` requires a key and a value. " + "Perhaps you meant `Configuration().dict().update`?" + ) super().__init__() @@ -99,6 +101,7 @@ class Configuration: >>> slack_420349_preconfig['pipeline_setup', 'pipeline_name'] 'slack_420349_preconfig' """ + def __init__(self, config_map=None): from CPAC.pipeline.schema import schema from CPAC.utils.utils import lookup_nested_value, update_nested_dict @@ -106,10 +109,10 @@ def __init__(self, config_map=None): if config_map is None: config_map = {} - base_config = config_map.pop('FROM', None) + base_config = config_map.pop("FROM", None) if base_config: - if base_config.lower() in ['default', 'default_pipeline']: - base_config = 'default' + if base_config.lower() in ["default", "default_pipeline"]: + base_config = "default" # import another config (specified with 'FROM' key) try: base_config = Preconfiguration(base_config) @@ -119,36 +122,37 @@ def __init__(self, config_map=None): else: # base everything on blank pipeline for unspecified keys config_map = update_nested_dict( - preconfig_yaml('blank', load=True), config_map) + preconfig_yaml("blank", load=True), config_map + ) config_map = self._nonestr_to_None(config_map) try: regressors = lookup_nested_value( config_map, - ['nuisance_corrections', '2-nuisance_regression', 'Regressors'] + ["nuisance_corrections", "2-nuisance_regression", "Regressors"], ) except KeyError: regressors = [] if isinstance(regressors, list): for i, regressor in enumerate(regressors): # set Regressor 'Name's if not provided - if 'Name' not in regressor: - regressor['Name'] = f'Regressor-{str(i + 1)}' + if "Name" not in regressor: + regressor["Name"] = f"Regressor-{i + 1!s}" # make Regressor 'Name's Nipype-friendly - regressor['Name'] = nipype_friendly_name(regressor['Name']) + regressor["Name"] = nipype_friendly_name(regressor["Name"]) config_map = schema(config_map) # remove 'FROM' before setting attributes now that it's imported - if 'FROM' in config_map: - del config_map['FROM'] + if "FROM" in config_map: + del config_map["FROM"] # set FSLDIR to the environment $FSLDIR if the user sets it to # 'FSLDIR' in the pipeline config file - _FSLDIR = config_map.get('FSLDIR') - if _FSLDIR and bool(re.match(r'^[\$\{]{0,2}?FSLDIR[\}]?$', _FSLDIR)): - config_map['FSLDIR'] = os.environ['FSLDIR'] + _FSLDIR = config_map.get("FSLDIR") + if _FSLDIR and bool(re.match(r"^[\$\{]{0,2}?FSLDIR[\}]?$", _FSLDIR)): + config_map["FSLDIR"] = os.environ["FSLDIR"] for key in config_map: # set attribute @@ -157,12 +161,10 @@ def __init__(self, config_map=None): self._update_attr() # set working directory as an environment variable - os.environ['CPAC_WORKDIR'] = self[ - 'pipeline_setup', 'working_directory', 'path'] + os.environ["CPAC_WORKDIR"] = self["pipeline_setup", "working_directory", "path"] def __str__(self): - return ('C-PAC Configuration ' - f"('{self['pipeline_setup', 'pipeline_name']}')") + return "C-PAC Configuration " f"('{self['pipeline_setup', 'pipeline_name']}')" def __repr__(self): # show Configuration as a dict when accessed directly @@ -177,21 +179,22 @@ def __copy__(self): def __getitem__(self, key): if isinstance(key, str): return getattr(self, key) - elif isinstance(key, tuple) or isinstance(key, list): + elif isinstance(key, (list, tuple)): return self.get_nested(self, key) else: self.key_type_error(key) + return None def __setitem__(self, key, value): if isinstance(key, str): setattr(self, key, value) - elif isinstance(key, tuple) or isinstance(key, list): + elif isinstance(key, (list, tuple)): self.set_nested(self, key, value) else: self.key_type_error(key) - def __sub__(self: 'Configuration', other: 'Configuration'): - '''Return the set difference between two Configurations + def __sub__(self: "Configuration", other: "Configuration"): + """Return the set difference between two Configurations. Examples -------- @@ -223,20 +226,20 @@ def __sub__(self: 'Configuration', other: 'Configuration'): 'cpac-default-pipeline' >>> diff.subtrahend['pipeline_setup']['pipeline_name'] 'cpac-default-pipeline' - ''' - return(dct_diff(self.dict(), other.dict())) + """ + return dct_diff(self.dict(), other.dict()) def dict(self): - '''Show contents of a C-PAC configuration as a dict''' + """Show contents of a C-PAC configuration as a dict.""" return {k: v for k, v in self.__dict__.items() if not callable(v)} def keys(self): - '''Show toplevel keys of a C-PAC configuration dict''' + """Show toplevel keys of a C-PAC configuration dict.""" return self.dict().keys() def _nonestr_to_None(self, d): - '''Recursive method to type convert 'None' to None in nested - config + """Recursive method to type convert 'None' to None in nested + config. Parameters ---------- @@ -248,8 +251,8 @@ def _nonestr_to_None(self, d): d : any same item, same type, but with 'none' strings converted to Nonetypes - ''' - if isinstance(d, str) and d.lower() == 'none': + """ + if isinstance(d, str) and d.lower() == "none": return None elif isinstance(d, list): return [self._nonestr_to_None(i) for i in d] @@ -264,15 +267,14 @@ def return_config_elements(self): # this returns a list of tuples # each tuple contains the name of the element in the yaml config file # and its value - attributes = [ + return [ (attr, getattr(self, attr)) for attr in dir(self) if not callable(attr) and not attr.startswith("__") ] - return attributes def sub_pattern(self, pattern, orig_key): - return orig_key.replace(pattern, self[pattern[2:-1].split('.')]) + return orig_key.replace(pattern, self[pattern[2:-1].split(".")]) def check_pattern(self, orig_key, tags=None): if tags is None: @@ -283,14 +285,11 @@ def check_pattern(self, orig_key, tags=None): return [self.check_pattern(item) for item in orig_key] if not isinstance(orig_key, str): return orig_key - template_pattern = r'\${.*}' + template_pattern = r"\${.*}" r = re.finditer(template_pattern, orig_key) for i in r: pattern = i.group(0) - if ( - isinstance(pattern, str) and len(pattern) and - pattern not in tags - ): + if isinstance(pattern, str) and len(pattern) and pattern not in tags: try: orig_key = self.sub_pattern(pattern, orig_key) except AttributeError as ae: @@ -301,29 +300,33 @@ def check_pattern(self, orig_key, tags=None): # method to find any pattern ($) in the configuration # and update the attributes with its pattern value def _update_attr(self): - def check_path(key): - if isinstance(key, str) and '/' in key: + if isinstance(key, str) and "/" in key: if not os.path.exists(key): - warn(f"Invalid path- {key}. Please check your " - "configuration file") + warn( + f"Invalid path- {key}. Please check your " "configuration file" + ) - attributes = [(attr, getattr(self, attr)) for attr in dir(self) - if not callable(attr) and not attr.startswith("__")] + attributes = [ + (attr, getattr(self, attr)) + for attr in dir(self) + if not callable(attr) and not attr.startswith("__") + ] - template_list = ['template_brain_only_for_anat', - 'template_skull_for_anat', - 'ref_mask', - 'template_brain_only_for_func', - 'template_skull_for_func', - 'template_symmetric_brain_only', - 'template_symmetric_skull', - 'dilated_symmetric_brain_mask'] + template_list = [ + "template_brain_only_for_anat", + "template_skull_for_anat", + "ref_mask", + "template_brain_only_for_func", + "template_skull_for_func", + "template_symmetric_brain_only", + "template_symmetric_skull", + "dilated_symmetric_brain_mask", + ] for attr_key, attr_value in attributes: - if attr_key in template_list: - new_key = self.check_pattern(attr_value, 'FSLDIR') + new_key = self.check_pattern(attr_value, "FSLDIR") else: new_key = self.check_pattern(attr_value) setattr(self, attr_key, new_key) @@ -356,9 +359,8 @@ def set_nested(self, d, keys, value): # pylint: disable=invalid-name d[keys[0]] = value return d - def _check_if_switch(self, key: ConfigKeyType, - error: bool = False) -> bool: - '''Check if a given entity is a switch + def _check_if_switch(self, key: ConfigKeyType, error: bool = False) -> bool: + """Check if a given entity is a switch. Parameters ---------- @@ -382,7 +384,7 @@ def _check_if_switch(self, key: ConfigKeyType, False >>> c._check_if_switch(['anatomical_preproc', 'run']) True - ''' + """ _maybe_switch = self[key] if isinstance(_maybe_switch, bool): return True @@ -391,12 +393,11 @@ def _check_if_switch(self, key: ConfigKeyType, if _answer: return _answer if error: - raise TypeError(f'`{key}` is not a switch in {str(self)}.') + raise TypeError(f"`{key}` is not a switch in {self!s}.") return False - def _switch_bool(self, key: ConfigKeyType, value: bool, exclusive: bool - ) -> bool: - '''Return True if the given key is set to the given value or + def _switch_bool(self, key: ConfigKeyType, value: bool, exclusive: bool) -> bool: + """Return True if the given key is set to the given value or False otherwise. Parameters @@ -416,17 +417,16 @@ def _switch_bool(self, key: ConfigKeyType, value: bool, exclusive: bool True if the given key is set to the given value or False otherwise. If exclusive is True, return False if the key is set to both True and False. - ''' - if not(exclusive and self.switch_is_on_off(key)): + """ + if not (exclusive and self.switch_is_on_off(key)): if isinstance(self[key], bool): return self[key] is value if isinstance(self[key], list): return value in self[key] return False - def switch_is_off(self, key: ConfigKeyType, exclusive: bool = False - ) -> bool: - '''Return True if the given key is set to 'off' OR 'on' and 'off' + def switch_is_off(self, key: ConfigKeyType, exclusive: bool = False) -> bool: + """Return True if the given key is set to 'off' OR 'on' and 'off' or False otherwise. Used for tracking forking. Parameters @@ -458,13 +458,12 @@ def switch_is_off(self, key: ConfigKeyType, exclusive: bool = False >>> c.switch_is_off(['nuisance_corrections', '2-nuisance_regression', ... 'run'], exclusive=True) False - ''' + """ self._check_if_switch(key, True) return self._switch_bool(key, False, exclusive) - def switch_is_on(self, key: ConfigKeyType, exclusive: bool = False - ) -> bool: - '''Return True if the given key is set to 'on' OR 'on' and 'off' + def switch_is_on(self, key: ConfigKeyType, exclusive: bool = False) -> bool: + """Return True if the given key is set to 'on' OR 'on' and 'off' or False otherwise. Used for tracking forking. Parameters @@ -496,12 +495,12 @@ def switch_is_on(self, key: ConfigKeyType, exclusive: bool = False >>> c.switch_is_on(['nuisance_corrections', '2-nuisance_regression', ... 'run'], exclusive=True) False - ''' + """ self._check_if_switch(key, True) return self._switch_bool(key, True, exclusive) def switch_is_on_off(self, key: ConfigKeyType) -> bool: - '''Return True if the given key is set to both 'on' and 'off' + """Return True if the given key is set to both 'on' and 'off' or False otherwise. Used for tracking forking. Parameters @@ -525,24 +524,28 @@ def switch_is_on_off(self, key: ConfigKeyType) -> bool: >>> c.switch_is_on_off(['nuisance_corrections', ... '2-nuisance_regression', 'run']) True - ''' + """ self._check_if_switch(key, True) if isinstance(self[key], list): return True in self[key] and False in self[key] return False def key_type_error(self, key): - raise KeyError(' '.join([ - 'Configuration key must be a string, list, or tuple;', - type(key).__name__, - f'`{str(key)}`', - 'was given.' - ])) + raise KeyError( + " ".join( + [ + "Configuration key must be a string, list, or tuple;", + type(key).__name__, + f"`{key!s}`", + "was given.", + ] + ) + ) def check_pname(p_name: str, pipe_config: Configuration) -> str: - '''Function to check / set `p_name`, the string representation of a - pipeline for use in filetrees + """Function to check / set `p_name`, the string representation of a + pipeline for use in filetrees. Parameters ---------- @@ -571,16 +574,16 @@ def check_pname(p_name: str, pipe_config: Configuration) -> str: >>> p_name = check_pname(None, Preconfiguration('default')) >>> p_name 'pipeline_cpac-default-pipeline' - ''' + """ if p_name is None: p_name = f'pipeline_{pipe_config["pipeline_setup", "pipeline_name"]}' - elif not p_name.startswith('pipeline_'): - p_name = f'pipeline_{p_name}' + elif not p_name.startswith("pipeline_"): + p_name = f"pipeline_{p_name}" return p_name def collect_key_list(config_dict): - '''Function to return a list of lists of keys for a nested dictionary + """Function to return a list of lists of keys for a nested dictionary. Parameters ---------- @@ -594,7 +597,7 @@ def collect_key_list(config_dict): -------- >>> collect_key_list({'test': {'nested': 1, 'dict': 2}}) [['test', 'nested'], ['test', 'dict']] - ''' + """ key_list = [] for key in config_dict: if isinstance(config_dict[key], dict): @@ -617,11 +620,11 @@ def configuration_from_file(config_file): ------- Configuration """ - with open(config_file, 'r', encoding='utf-8') as config: + with open(config_file, "r", encoding="utf-8") as config: return Configuration(yaml.safe_load(config)) -def preconfig_yaml(preconfig_name='default', load=False): +def preconfig_yaml(preconfig_name="default", load=False): """Get the path to a preconfigured pipeline's YAML file. Raises BadParameter if an invalid preconfig name is given. @@ -638,34 +641,39 @@ def preconfig_yaml(preconfig_name='default', load=False): path to YAML file or dict loaded from YAML """ from CPAC.pipeline import ALL_PIPELINE_CONFIGS, AVAILABLE_PIPELINE_CONFIGS + if preconfig_name not in ALL_PIPELINE_CONFIGS: raise BadParameter( f"The pre-configured pipeline name '{preconfig_name}' you " "provided is not one of the available pipelines.\n\nAvailable " - f"pipelines:\n{str(AVAILABLE_PIPELINE_CONFIGS)}\n", - param='preconfig') + f"pipelines:\n{AVAILABLE_PIPELINE_CONFIGS!s}\n", + param="preconfig", + ) if load: - with open(preconfig_yaml(preconfig_name), 'r', encoding='utf-8') as _f: + with open(preconfig_yaml(preconfig_name), "r", encoding="utf-8") as _f: return yaml.safe_load(_f) - return p.resource_filename("CPAC", os.path.join( - "resources", "configs", f"pipeline_config_{preconfig_name}.yml")) + return p.resource_filename( + "CPAC", + os.path.join("resources", "configs", f"pipeline_config_{preconfig_name}.yml"), + ) class Preconfiguration(Configuration): - """A preconfigured Configuration + """A preconfigured Configuration. Parameters ---------- preconfig : str The canonical name of the preconfig to load """ + def __init__(self, preconfig): super().__init__(config_map=preconfig_yaml(preconfig, True)) def set_from_ENV(conf): # pylint: disable=invalid-name - '''Function to replace strings like $VAR and ${VAR} with - environment variable values + """Function to replace strings like $VAR and ${VAR} with + environment variable values. Parameters ---------- @@ -685,7 +693,7 @@ def set_from_ENV(conf): # pylint: disable=invalid-name >>> set_from_ENV(['${SAMPLE_VALUE_SFE}', 'SAMPLE_VALUE_SFE']) ['/example/path', 'SAMPLE_VALUE_SFE'] >>> del os.environ['SAMPLE_VALUE_SFE'] - ''' + """ if isinstance(conf, list): return [set_from_ENV(item) for item in conf] if isinstance(conf, dict): @@ -694,23 +702,23 @@ def set_from_ENV(conf): # pylint: disable=invalid-name # set any specified environment variables # (only matching all-caps plus `-` and `_`) # like `${VAR}` - _pattern1 = r'\${[A-Z\-_]*}' + _pattern1 = r"\${[A-Z\-_]*}" # like `$VAR` - _pattern2 = r'\$[A-Z\-_]*(?=/|$)' + _pattern2 = r"\$[A-Z\-_]*(?=/|$)" # replace with environment variables if they exist for _pattern in [_pattern1, _pattern2]: _match = re.search(_pattern, conf) if _match: - _match = _match.group().lstrip('${').rstrip('}') - conf = re.sub( - _pattern, os.environ.get(_match, f'${_match}'), conf) + _match = _match.group().lstrip("${").rstrip("}") + conf = re.sub(_pattern, os.environ.get(_match, f"${_match}"), conf) return conf -def set_subject(sub_dict: dict, pipe_config: 'Configuration', - p_name: Optional[str] = None) -> TUPLE[str, str, str]: - '''Function to set pipeline name and log directory path for a given - sub_dict +def set_subject( + sub_dict: dict, pipe_config: "Configuration", p_name: Optional[str] = None +) -> TUPLE[str, str, str]: + """Function to set pipeline name and log directory path for a given + sub_dict. Parameters ---------- @@ -747,13 +755,14 @@ def set_subject(sub_dict: dict, pipe_config: 'Configuration', 'pipeline_cpac-blank-template' >>> log_dir.endswith(f'{p_name}/{subject_id}') True - ''' - subject_id = sub_dict['subject_id'] - if sub_dict.get('unique_id'): + """ + subject_id = sub_dict["subject_id"] + if sub_dict.get("unique_id"): subject_id += f'_{sub_dict["unique_id"]}' p_name = check_pname(p_name, pipe_config) - log_dir = os.path.join(pipe_config.pipeline_setup['log_directory']['path'], - p_name, subject_id) + log_dir = os.path.join( + pipe_config.pipeline_setup["log_directory"]["path"], p_name, subject_id + ) if not os.path.exists(log_dir): os.makedirs(os.path.join(log_dir)) return subject_id, p_name, log_dir @@ -761,7 +770,7 @@ def set_subject(sub_dict: dict, pipe_config: 'Configuration', def nipype_friendly_name(name: str) -> str: """Replace each sequence of non-alphanumeric characters with a single - underscore and remove any leading underscores + underscore and remove any leading underscores. Parameters ---------- @@ -771,4 +780,4 @@ def nipype_friendly_name(name: str) -> str: ------- str """ - return re.sub(r'[^a-zA-Z0-9]+', '_', name).lstrip('_') + return re.sub(r"[^a-zA-Z0-9]+", "_", name).lstrip("_") diff --git a/CPAC/utils/configuration/diff.py b/CPAC/utils/configuration/diff.py index 47fd3b5fc6..c27ad2ff34 100644 --- a/CPAC/utils/configuration/diff.py +++ b/CPAC/utils/configuration/diff.py @@ -14,12 +14,12 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -'''Tools for configuration differences''' +"""Tools for configuration differences.""" def dct_diff(dct1, dct2): - '''Function to compare 2 nested dicts, dropping values unspecified - in the second. Adapted from https://github.com/FCP-INDI/CPAC_regtest_pack/blob/9056ef63/cpac_pipe_diff.py#L31-L78 + """Function to compare 2 nested dicts, dropping values unspecified + in the second. Adapted from https://github.com/FCP-INDI/CPAC_regtest_pack/blob/9056ef63/cpac_pipe_diff.py#L31-L78. Parameters ---------- @@ -45,7 +45,7 @@ def dct_diff(dct1, dct2): >>> pipeline2 = Preconfiguration('fmriprep-options') >>> dct_diff(pipeline, pipeline2)['pipeline_setup']['pipeline_name'] ('cpac-default-pipeline', 'cpac_fmriprep-options') - ''' # pylint: disable=line-too-long # noqa: E501 + """ # pylint: disable=line-too-long dcts = [] for _d in [dct1, dct2]: if not isinstance(_d, dict): @@ -53,7 +53,7 @@ def dct_diff(dct1, dct2): _d = _d.dict() except AttributeError: # pylint: disable=raise-missing-from - raise TypeError(f'{_d} is not a dict.') + raise TypeError(f"{_d} is not a dict.") dcts.append(_d) dct1, dct2 = dcts # pylint: disable=unbalanced-tuple-unpacking del dcts @@ -61,7 +61,7 @@ def dct_diff(dct1, dct2): for key, dct1_value in dct1.items(): # handle parts of config where user-defined paths are keys dct2_value = dct2.get(key, {}) if isinstance(dct2, dict) else None - if key.endswith('_roi_paths') and isinstance(dct1_value, dict): + if key.endswith("_roi_paths") and isinstance(dct1_value, dict): paths1 = set(dct1_value.keys()) paths2 = set(dct2_value.keys() if dct2_value else {}) if paths1 != paths2: @@ -84,8 +84,8 @@ def dct_diff(dct1, dct2): def diff_dict(diff): - '''Method to return a dict of only changes given a nested dict - of ``(dict1_value, dict2_value)`` tuples + """Method to return a dict of only changes given a nested dict + of ``(dict1_value, dict2_value)`` tuples. Parameters ---------- @@ -105,7 +105,7 @@ def diff_dict(diff): ... 'using': DiffValue(['3dSkullStrip'], ... ['niworkflows-ants'])}}}}) {'anatomical_preproc': {'brain_extraction': {'extraction': {'run': False, 'using': ['niworkflows-ants']}}}} - ''' # noqa: E501 # pylint: disable=line-too-long + """ # pylint: disable=line-too-long if isinstance(diff, DiffValue): return diff.t_value if isinstance(diff, dict): @@ -122,8 +122,8 @@ def diff_dict(diff): class DiffDict(dict): - '''Class to semantically store a dictionary of set differences from - ``Configuration(S) - Configuration(T)`` + """Class to semantically store a dictionary of set differences from + ``Configuration(S) - Configuration(T)``. Attributes ---------- @@ -148,13 +148,14 @@ class DiffDict(dict): t_value : dict dictionary of differing values from ``Configuration(T)`` - ''' + """ + def __init__(self, *args, **kwargs): - '''Dictionary of difference ``Configuration(S) - Configuration(T)``. + """Dictionary of difference ``Configuration(S) - Configuration(T)``. Each value in a ~DiffDict should be either a ``DiffDict`` or a ~DiffValue. - ''' + """ super().__init__(*args, **kwargs) self.left = self.minuend = self.s_value = self._s_value() self.right = self.subtrahend = self.t_value = self._t_value() @@ -169,19 +170,21 @@ def _return_one_value(self, which_value): return return_dict def _s_value(self): - '''Get a dictionary of only the differing ``'S'`` values that differ - in ``S - T``''' - return self._return_one_value('s_value') + """Get a dictionary of only the differing ``'S'`` values that differ + in ``S - T``. + """ + return self._return_one_value("s_value") def _t_value(self): - '''Get a dictionary of only the differing ``'T'`` values that differ - in ``S - T``''' - return self._return_one_value('t_value') + """Get a dictionary of only the differing ``'T'`` values that differ + in ``S - T``. + """ + return self._return_one_value("t_value") class DiffValue: - '''Class to semantically store values of set difference from - ``Configuration(S) - Configuration(T)`` + """Class to semantically store values of set difference from + ``Configuration(S) - Configuration(T)``. Attributes ---------- @@ -202,9 +205,10 @@ class DiffValue: t_value : any value from ``Configuration(T)`` - ''' + """ + def __init__(self, s_value, t_value): - '''Different values from ``Configuration(S) - Configuration(T)`` + """Different values from ``Configuration(S) - Configuration(T)``. Parameters ---------- @@ -213,7 +217,7 @@ def __init__(self, s_value, t_value): t_value : any value from ``Configuration(T)`` - ''' + """ self.left = self.minuend = self.s_value = s_value self.right = self.subtrahend = self.t_value = t_value @@ -221,4 +225,4 @@ def __len__(self): return 2 # self.__repr__ should always be a 2-tuple def __repr__(self): - return str(tuple((self.s_value, self.t_value))) + return str((self.s_value, self.t_value)) diff --git a/CPAC/utils/configuration/yaml_template.py b/CPAC/utils/configuration/yaml_template.py index 8c43769ab0..0053de8605 100644 --- a/CPAC/utils/configuration/yaml_template.py +++ b/CPAC/utils/configuration/yaml_template.py @@ -16,24 +16,22 @@ # License along with C-PAC. If not, see . """Functions to create YAML configuration files from templates.""" from copy import deepcopy -import os -import re from datetime import datetime from hashlib import sha1 +import os +import re + from click import BadParameter import yaml -from CPAC.utils.configuration import Configuration, Preconfiguration, \ - preconfig_yaml -from CPAC.utils.utils import update_config_dict, update_pipeline_values_1_8, \ - YAML_BOOLS +from CPAC.utils.configuration import Configuration, Preconfiguration, preconfig_yaml +from CPAC.utils.utils import YAML_BOOLS, update_config_dict, update_pipeline_values_1_8 -YAML_LOOKUP = {yaml_str: key for key, value in YAML_BOOLS.items() for - yaml_str in value} +YAML_LOOKUP = {yaml_str: key for key, value in YAML_BOOLS.items() for yaml_str in value} -class YamlTemplate(): # pylint: disable=too-few-public-methods - """A class to link YAML comments to the contents of a YAML file +class YamlTemplate: # pylint: disable=too-few-public-methods + """A class to link YAML comments to the contents of a YAML file. Attributes ---------- @@ -50,6 +48,7 @@ class YamlTemplate(): # pylint: disable=too-few-public-methods original : str """ + def __init__(self, original_yaml, base_config=None): """ Parameters @@ -64,7 +63,7 @@ def __init__(self, original_yaml, base_config=None): except BadParameter: pass if os.path.exists(original_yaml): - with open(original_yaml, 'r', encoding='utf-8') as _f: + with open(original_yaml, "r", encoding="utf-8") as _f: original_yaml = _f.read() self.comments = {} self.template = original_yaml @@ -81,7 +80,7 @@ def __init__(self, original_yaml, base_config=None): def dump(self, new_dict, parents=None): """Dump a YAML file from a new dictionary with the comments from - the template dictionary + the template dictionary. Parameters ---------- @@ -93,41 +92,41 @@ def dump(self, new_dict, parents=None): ------- str """ - # SSOT FSLDIR try: # Get from current config - fsldir = self.get_nested(new_dict, - ['pipeline_setup', 'system_config', - 'FSLDIR']) + fsldir = self.get_nested( + new_dict, ["pipeline_setup", "system_config", "FSLDIR"] + ) except KeyError: # Get from imported base - fsldir = self.get_nested(self._dict, - ['pipeline_setup', 'system_config', - 'FSLDIR']) + fsldir = self.get_nested( + self._dict, ["pipeline_setup", "system_config", "FSLDIR"] + ) # Add YAML version directive to top of document and ensure # C-PAC version comment and 'FROM' are at the top of the YAML # output if parents is None: parents = [] - _dump = ['%YAML 1.1', '---'] - if 'pipeline_setup' not in new_dict: - new_dict['pipeline_setup'] = None + _dump = ["%YAML 1.1", "---"] + if "pipeline_setup" not in new_dict: + new_dict["pipeline_setup"] = None else: _dump = [] # Prepare for indentation line_level = len(parents) # Get a safely mutable copy of the dict - loop_dict = deepcopy(self.get_nested(new_dict, parents) if - parents else new_dict) + loop_dict = deepcopy( + self.get_nested(new_dict, parents) if parents else new_dict + ) # Grab special key to print first - import_from = loop_dict.pop('FROM', None) + import_from = loop_dict.pop("FROM", None) # Iterate through mutated dict for key in loop_dict: # List of progressively-indented key strings keys = [*parents, key] # Comments are stored in a flat dictionary with # '.'-delimited pseudonested keys - comment = self.comments.get('.'.join(keys)) + comment = self.comments.get(".".join(keys)) # This exception should only happen from mutations # introduced this function try: @@ -137,46 +136,47 @@ def dump(self, new_dict, parents=None): # Print comment if there's one above this key in the template if comment: - if key != 'pipeline_setup': - _dump += [''] # Add a blank line above the comment + if key != "pipeline_setup": + _dump += [""] # Add a blank line above the comment _dump += [indent(line_level, 0) + line for line in comment] # Print 'FROM' between preamble comment and rest of config # if applicable - if key == 'pipeline_setup' and import_from is not None: - _dump += [f'FROM: {import_from}', ''] + if key == "pipeline_setup" and import_from is not None: + _dump += [f"FROM: {import_from}", ""] # Apply indentation to key - indented_key = f'{indent(line_level, 0)}{key}:' + indented_key = f"{indent(line_level, 0)}{key}:" # Print YAML-formatted value if value is not None: # SSOT FSLDIR - if (isinstance(value, str) and fsldir in value and - key != 'FSLDIR'): - value = re.sub(r'\$*FSLDIR', '$FSLDIR', - value.replace(fsldir, '$FSLDIR')) + if isinstance(value, str) and fsldir in value and key != "FSLDIR": + value = re.sub( + r"\$*FSLDIR", "$FSLDIR", value.replace(fsldir, "$FSLDIR") + ) if isinstance(value, dict): _dump += [indented_key, self.dump(new_dict, keys)] elif isinstance(value, list): list_line = _format_list_items(value, line_level) - if '\n' in list_line: - _dump += [indented_key, *list_line.split('\n')] + if "\n" in list_line: + _dump += [indented_key, *list_line.split("\n")] else: - _dump += [f'{indented_key} {list_line}'] - elif isinstance(value, bool) or (isinstance(value, str) and - value.lower() in YAML_LOOKUP): + _dump += [f"{indented_key} {list_line}"] + elif isinstance(value, bool) or ( + isinstance(value, str) and value.lower() in YAML_LOOKUP + ): if isinstance(value, str): value = YAML_LOOKUP[value.lower()] - value = 'On' if value is True else 'Off' - _dump += [f'{indented_key} {value}'] + value = "On" if value is True else "Off" + _dump += [f"{indented_key} {value}"] else: - _dump += [f'{indented_key} {value}'] - elif key != 'pipeline_setup': + _dump += [f"{indented_key} {value}"] + elif key != "pipeline_setup": _dump += [indented_key] # Normalize line spacing and return YAML string - return re.sub('\n{3,}', '\n\n', '\n'.join(_dump)).rstrip() + '\n' + return re.sub("\n{3,}", "\n\n", "\n".join(_dump)).rstrip() + "\n" def _parse_comments(self): # Split YAML into lines - yaml_lines = self.template.split('\n') + yaml_lines = self.template.split("\n") # Initialize comment and key comment = [] key = [] @@ -186,27 +186,26 @@ def _parse_comments(self): # Remove indentation and trailing whitespace stripped_line = line.strip() # Collect a line of a comment - if stripped_line.startswith('#'): + if stripped_line.startswith("#"): comment.append(stripped_line) # If a line is not a comment line: - elif not any(stripped_line.startswith(seq) for - seq in ('%YAML', '---')): + elif not any(stripped_line.startswith(seq) for seq in ("%YAML", "---")): # If the line is a key - if ':' in stripped_line: + if ":" in stripped_line: # Set the key for the comments dictionary - line_key = stripped_line.split(':', 1)[0].strip() + line_key = stripped_line.split(":", 1)[0].strip() if line_level == 0: key = [line_key] else: key = [*key[:line_level], line_key] # Store the full list of comment lines - self.comments['.'.join(key)] = comment + self.comments[".".join(key)] = comment # Reset the comment variable to collect the next comment comment = [] def _count_indent(line): - '''Helper method to determine indentation level + """Helper method to determine indentation level. Parameters ---------- @@ -222,14 +221,17 @@ def _count_indent(line): 0 >>> _count_indent(' Four spaces') 2 - ''' + """ return (len(line) - len(line.lstrip())) // 2 -def create_yaml_from_template(d, # pylint: disable=invalid-name - template='default', import_from=None): +def create_yaml_from_template( + d, # pylint: disable=invalid-name + template="default", + import_from=None, +): """Save dictionary to a YAML file, keeping the structure - (such as first level comments and ordering) from the template + (such as first level comments and ordering) from the template. It may not be fully robust to YAML structures, but it works for C-PAC config files! @@ -286,14 +288,16 @@ def create_yaml_from_template(d, # pylint: disable=invalid-name d = Configuration(d) if not isinstance(d, Configuration) else d base_config = Preconfiguration(import_from) d = (d - base_config).left - d.update({'FROM': import_from}) + d.update({"FROM": import_from}) yaml_template = YamlTemplate(template, base_config) return yaml_template.dump(new_dict=d) -def _format_list_items(l, # noqa: E741 # pylint:disable=invalid-name - line_level): - '''Helper method to handle lists in the YAML +def _format_list_items( + l, # noqa: E741 # pylint:disable=invalid-name + line_level, +): + """Helper method to handle lists in the YAML. Parameters ---------- @@ -319,23 +323,27 @@ def _format_list_items(l, # noqa: E741 # pylint:disable=invalid-name - 3 - deep: - 4 - ''' + """ # keep short, simple lists in square brackets if all(isinstance(item, (str, bool, int, float)) for item in l): preformat = str([yaml_bool(item) for item in l]) if len(preformat) < 50: - return preformat.replace("'", '').replace('"', '') + return preformat.replace("'", "").replace('"', "") # list long or complex lists on lines with indented '-' lead-ins - return '\n'.join([ - f'{indent(line_level)}{li}' for li in yaml.dump( - yaml_bool(l), sort_keys=False - ).replace("'On'", 'On').replace("'Off'", 'Off').split('\n') - ]).rstrip() + return "\n".join( + [ + f"{indent(line_level)}{li}" + for li in yaml.dump(yaml_bool(l), sort_keys=False) + .replace("'On'", "On") + .replace("'Off'", "Off") + .split("\n") + ] + ).rstrip() def hash_data_config(sub_list): - '''Function to generate a short SHA1 hash from a data config - subject list of dicts + """Function to generate a short SHA1 hash from a data config + subject list of dicts. Parameters ---------- @@ -353,14 +361,19 @@ def hash_data_config(sub_list): {'site_id': 'site1', 'subject_id': 'sub1', 'unique_id': 'uid1'} >>> hash_data_config(sub_list) '6f49a278' - ''' - return sha1('_'.join([','.join([run.get(key, '') for run in sub_list]) for - key in ['site_id', 'subject_id', - 'unique_id']]).encode('utf-8')).hexdigest()[:8] + """ + return sha1( + "_".join( + [ + ",".join([run.get(key, "") for run in sub_list]) + for key in ["site_id", "subject_id", "unique_id"] + ] + ).encode("utf-8") + ).hexdigest()[:8] def indent(line_level, plus=2): - '''Function to return an indent string for a given level + """Function to return an indent string for a given level. Parameters ---------- @@ -371,12 +384,12 @@ def indent(line_level, plus=2): ------- str The string of spaces to use for indentation - ''' + """ return " " * (2 * line_level + plus) def yaml_bool(value): - '''Helper function to give On/Off value to bools + """Helper function to give On/Off value to bools. Parameters ---------- @@ -392,7 +405,7 @@ def yaml_bool(value): 'On' >>> yaml_bool([False, 'On', True]) ['Off', 'On', 'On'] - ''' + """ if isinstance(value, str): lookup_value = value.lower() if lookup_value in YAML_LOOKUP: @@ -401,17 +414,19 @@ def yaml_bool(value): return [yaml_bool(item) for item in value] elif isinstance(value, dict): # if 'Name' is a key, promote that item to the top - return {**({'Name': value['Name']} if 'Name' in value else {}), - **{k: yaml_bool(value[k]) for k in value if k != 'Name'}} + return { + **({"Name": value["Name"]} if "Name" in value else {}), + **{k: yaml_bool(value[k]) for k in value if k != "Name"}, + } if isinstance(value, bool): if value is True: - return 'On' - return 'Off' + return "On" + return "Off" return value def upgrade_pipeline_to_1_8(path): - '''Function to upgrade a C-PAC 1.7 pipeline config to C-PAC 1.8 + """Function to upgrade a C-PAC 1.7 pipeline config to C-PAC 1.8. Parameters ---------- @@ -428,31 +443,28 @@ def upgrade_pipeline_to_1_8(path): path upgraded file - ''' + """ # back up original config - now = datetime.isoformat(datetime.now()).replace(':', '_') - backup = f'{path}.{now}.bak' - print(f'Backing up {path} to {backup} and upgrading to C-PAC 1.8') - with open(path, 'r', encoding='utf-8') as _f: + now = datetime.isoformat(datetime.now()).replace(":", "_") + backup = f"{path}.{now}.bak" + with open(path, "r", encoding="utf-8") as _f: original = _f.read() - with open(backup, 'w', encoding='utf-8') as _f: + with open(backup, "w", encoding="utf-8") as _f: _f.write(original) # upgrade and overwrite orig_dict = yaml.safe_load(original) # set Regressor 'Name's if not provided - regressors = orig_dict.get('Regressors') + regressors = orig_dict.get("Regressors") if isinstance(regressors, list): for i, regressor in enumerate(regressors): - if 'Name' not in regressor: - regressor['Name'] = f'Regressor-{str(i + 1)}' - if 'pipelineName' in orig_dict and len(original.strip()): - middle_dict, leftovers_dict, _complete_dict = update_config_dict( - orig_dict) - with open(path, 'w', encoding='utf-8') as _f: - _f.write(create_yaml_from_template( - update_pipeline_values_1_8(middle_dict))) + if "Name" not in regressor: + regressor["Name"] = f"Regressor-{i + 1!s}" + if "pipelineName" in orig_dict and len(original.strip()): + middle_dict, leftovers_dict, _complete_dict = update_config_dict(orig_dict) + with open(path, "w", encoding="utf-8") as _f: + _f.write(create_yaml_from_template(update_pipeline_values_1_8(middle_dict))) if leftovers_dict: - with open(f'{path}.rem', 'w', encoding='utf-8') as _f: + with open(f"{path}.rem", "w", encoding="utf-8") as _f: _f.write(yaml.dump(leftovers_dict)) @@ -464,29 +476,35 @@ def update_a_preconfig(preconfig, import_from): import_from : str """ - import sys - print(f'Updating {preconfig} preconfig…', file=sys.stderr) - updated = create_yaml_from_template(Preconfiguration(preconfig), - import_from=import_from) - with open(preconfig_yaml(preconfig), 'w', encoding='utf-8') as _f: + updated = create_yaml_from_template( + Preconfiguration(preconfig), import_from=import_from + ) + with open(preconfig_yaml(preconfig), "w", encoding="utf-8") as _f: _f.write(updated) def update_all_preconfigs(): - """Update all other preconfigs with comments from default""" + """Update all other preconfigs with comments from default.""" from CPAC.pipeline import ALL_PIPELINE_CONFIGS - not_from_blank = ('anat-only', 'blank', 'default', 'fx-options', - 'nhp-macaque', 'preproc', 'rbc-options') - update_a_preconfig('blank', None) - for preconfig in ('anat-only', 'preproc'): - update_a_preconfig(preconfig, 'default') - for preconfig in ('fx-options', 'rbc-options'): - update_a_preconfig(preconfig, 'fmriprep-options') - update_a_preconfig('nhp-macaque', 'monkey') - for preconfig in (_ for _ in ALL_PIPELINE_CONFIGS if - _ not in not_from_blank): - update_a_preconfig(preconfig, 'blank') - - -if __name__ == '__main__': + + not_from_blank = ( + "anat-only", + "blank", + "default", + "fx-options", + "nhp-macaque", + "preproc", + "rbc-options", + ) + update_a_preconfig("blank", None) + for preconfig in ("anat-only", "preproc"): + update_a_preconfig(preconfig, "default") + for preconfig in ("fx-options", "rbc-options"): + update_a_preconfig(preconfig, "fmriprep-options") + update_a_preconfig("nhp-macaque", "monkey") + for preconfig in (_ for _ in ALL_PIPELINE_CONFIGS if _ not in not_from_blank): + update_a_preconfig(preconfig, "blank") + + +if __name__ == "__main__": update_all_preconfigs() diff --git a/CPAC/utils/create_flame_model_files.py b/CPAC/utils/create_flame_model_files.py index 320346acf5..d77ebaaf5e 100644 --- a/CPAC/utils/create_flame_model_files.py +++ b/CPAC/utils/create_flame_model_files.py @@ -1,16 +1,16 @@ - def create_dummy_string(length): ppstring = "" for i in range(0, length): - ppstring += '\t' + '%1.5e' %(1.0) - ppstring += '\n' + ppstring += "\t" + "%1.5e" % (1.0) + ppstring += "\n" return ppstring -def write_mat_file(design_matrix, output_dir, model_name, \ - depatsified_EV_names, current_output=None): - +def write_mat_file( + design_matrix, output_dir, model_name, depatsified_EV_names, current_output=None +): import os + import numpy as np dimx = None @@ -22,13 +22,12 @@ def write_mat_file(design_matrix, output_dir, model_name, \ else: dimx, dimy = design_matrix.shape - ppstring = '/PPheights' + ppstring = "/PPheights" for i in range(0, dimy): + ppstring += "\t" + "%1.5e" % (1.0) - ppstring += '\t' + '%1.5e' %(1.0) - - ppstring += '\n' + ppstring += "\n" filename = model_name + ".mat" @@ -36,30 +35,29 @@ def write_mat_file(design_matrix, output_dir, model_name, \ if not os.path.exists(output_dir): os.makedirs(output_dir) - with open(out_file, 'wt') as f: - - print('/NumWaves\t%d' %dimy, file=f) - print('/NumPoints\t%d' %dimx, file=f) + with open(out_file, "wt") as f: + print("/NumWaves\t%d" % dimy, file=f) + print("/NumPoints\t%d" % dimx, file=f) print(ppstring, file=f) # print labels for the columns - mainly for double-checking your model - col_string = '\n' + col_string = "\n" for col in depatsified_EV_names: - col_string = col_string + col + '\t' + col_string = col_string + col + "\t" - print(col_string, '\n', file=f) + print(col_string, "\n", file=f) - print('/Matrix', file=f) + print("/Matrix", file=f) - np.savetxt(f, design_matrix, fmt='%1.5e', delimiter='\t') + np.savetxt(f, design_matrix, fmt="%1.5e", delimiter="\t") return out_file def create_grp_file(design_matrix, grp_file_vector, output_dir, model_name): - import os + import numpy as np dimx = None @@ -71,86 +69,76 @@ def create_grp_file(design_matrix, grp_file_vector, output_dir, model_name): else: dimx, dimy = design_matrix.shape - filename = "grouping.grp" - grp_file_vector = [int(x) for x in grp_file_vector] out_file = os.path.join(output_dir, model_name + ".grp") with open(out_file, "wt") as f: - print('/NumWaves\t1', file=f) - print('/NumPoints\t%d\n' %dimx, file=f) - print('/Matrix', file=f) - np.savetxt(f, grp_file_vector, fmt='%d', delimiter='\t') + print("/NumWaves\t1", file=f) + print("/NumPoints\t%d\n" % dimx, file=f) + print("/Matrix", file=f) + np.savetxt(f, grp_file_vector, fmt="%d", delimiter="\t") return out_file -def create_con_file(con_vecs, con_names, col_names, model_name, - current_output, out_dir): - +def create_con_file( + con_vecs, con_names, col_names, model_name, current_output, out_dir +): import os out_file = os.path.join(out_dir, model_name) + ".con" - with open(out_file,'w+') as f: - + with open(out_file, "w+") as f: # write header num = 1 for key in con_names: - f.write("/ContrastName%s\t%s\n" %(num,key)) + f.write("/ContrastName%s\t%s\n" % (num, key)) num += 1 - f.write("/NumWaves\t%d\n" %len(col_names)) - f.write("/NumContrasts\t%d\n" %len(con_names)) - f.write("/PPheights%s" %create_dummy_string(len(con_vecs))) - f.write("/RequiredEffect%s" %create_dummy_string(len(con_vecs))) + f.write("/NumWaves\t%d\n" % len(col_names)) + f.write("/NumContrasts\t%d\n" % len(con_names)) + f.write("/PPheights%s" % create_dummy_string(len(con_vecs))) + f.write("/RequiredEffect%s" % create_dummy_string(len(con_vecs))) f.write("\n\n") # print labels for the columns - mainly for double-checking your # model - col_string = '\n' + col_string = "\n" for col in col_names: - col_string = col_string + col + '\t' - print(col_string, '\n', file=f) + col_string = col_string + col + "\t" + print(col_string, "\n", file=f) # write data f.write("/Matrix\n") for vector in con_vecs: for v in vector: - f.write("%1.5e\t" %v) + f.write("%1.5e\t" % v) f.write("\n") return out_file -def create_fts_file(ftest_list, con_names, model_name, - current_output, out_dir): - +def create_fts_file(ftest_list, con_names, model_name, current_output, out_dir): import os + import numpy as np try: + out_file = os.path.join(out_dir, model_name + ".fts") - print("\nFound f-tests in your model, writing f-tests file " \ - "(.fts)..\n") - - out_file = os.path.join(out_dir, model_name + '.fts') - - with open(out_file, 'w') as f: - - print('/NumWaves\t', len(con_names), file=f) - print('/NumContrasts\t', len(ftest_list), file=f) + with open(out_file, "w") as f: + print("/NumWaves\t", len(con_names), file=f) + print("/NumContrasts\t", len(ftest_list), file=f) # process each f-test ftst = [] for ftest_string in ftest_list: - ftest_vector = [] - + cons_in_ftest = ftest_string.split(",") for con in con_names: @@ -160,73 +148,75 @@ def create_fts_file(ftest_list, con_names, model_name, ftest_vector.append(0) ftst.append(ftest_vector) - + fts_n = np.array(ftst) # print labels for the columns - mainly for double-checking your # model - col_string = '\n' + col_string = "\n" for con in con_names: - col_string = col_string + con + '\t' - print(col_string, '\n', file=f) + col_string = col_string + con + "\t" + print(col_string, "\n", file=f) - print('/Matrix', file=f) + print("/Matrix", file=f) for i in range(fts_n.shape[0]): - print(' '.join(fts_n[i].astype('str')), file=f) + print(" ".join(fts_n[i].astype("str")), file=f) except Exception as e: + filepath = os.path.join( + out_dir, "model_files", current_output, model_name + ".fts" + ) - filepath = os.path.join(out_dir, "model_files", current_output, \ - model_name + '.fts') - - errmsg = "\n\n[!] CPAC says: Could not create .fts file for " \ - "FLAMEO or write it to disk.\nAttempted filepath: %s\n" \ - "Error details: %s\n\n" % (filepath, e) + errmsg = ( + "\n\n[!] CPAC says: Could not create .fts file for " + "FLAMEO or write it to disk.\nAttempted filepath: %s\n" + "Error details: %s\n\n" % (filepath, e) + ) raise Exception(errmsg) return out_file -def create_con_ftst_file(con_file, model_name, current_output, output_dir, - column_names, coding_scheme, group_sep): - - """ - Create the contrasts and fts file - """ - +def create_con_ftst_file( + con_file, + model_name, + current_output, + output_dir, + column_names, + coding_scheme, + group_sep, +): + """Create the contrasts and fts file.""" import os + import numpy as np - column_names = [x for x in list(column_names) if 'participant_id' not in x] + column_names = [x for x in list(column_names) if "participant_id" not in x] # Read the header of the contrasts file, which should contain the columns # of the design matrix and f-tests (if any) with open(con_file, "r") as f: evs = f.readline() - evs = evs.rstrip('\r\n').split(',') - + evs = evs.rstrip("\r\n").split(",") + if evs[0].strip().lower() != "contrasts": - print("Error: first cell in contrasts file should contain " \ - "'Contrasts' ") raise Exception # remove "Contrasts" label and replace it with "Intercept" - #evs[0] = "Intercept" + # evs[0] = "Intercept" # Count the number of f tests defined - count_ftests = len([ev for ev in evs if "f_test" in ev ]) + count_ftests = len([ev for ev in evs if "f_test" in ev]) # Whether any f tests are defined fTest = count_ftests > 0 # Now read the actual contrasts try: - contrasts_data = np.genfromtxt(con_file, names=True, delimiter=',', - dtype=None) + contrasts_data = np.genfromtxt(con_file, names=True, delimiter=",", dtype=None) except: - print("Error: Could not successfully read in contrast file: ",con_file) raise Exception lst = contrasts_data.tolist() @@ -239,7 +229,6 @@ def create_con_ftst_file(con_file, model_name, current_output, output_dir, if isinstance(lst, tuple): lst = [lst] - ftst = [] fts_columns = [] contrasts = [] contrast_names = [] @@ -255,10 +244,10 @@ def create_con_ftst_file(con_file, model_name, current_output, output_dir, # create a list of integers that is the vector for the contrast # ex. [0, 1, 1, 0, ..] - con_vector = list(contr)[1:(length-count_ftests)] + con_vector = list(contr)[1 : (length - count_ftests)] # fts_vector tells us which f-tests this contrast is a part of. - fts_vector = list(contr)[(length-count_ftests):length] + fts_vector = list(contr)[(length - count_ftests) : length] fts_columns.append(fts_vector) # add Intercept column @@ -290,132 +279,142 @@ def create_con_ftst_file(con_file, model_name, current_output, output_dir, ## f test itself contains enough contrasts, rather than whether ## there are in principle enough contrasts to form f tests. if len(contrast_names) < 2: - errmsg = "\n\n[!] CPAC says: Not enough contrasts for running " \ - "f-tests.\nTip: Do you have only one contrast in your " \ - "contrasts file? f-tests require more than one contrast.\n"\ - "Either remove the f-tests or include more contrasts.\n\n" + errmsg = ( + "\n\n[!] CPAC says: Not enough contrasts for running " + "f-tests.\nTip: Do you have only one contrast in your " + "contrasts file? f-tests require more than one contrast.\n" + "Either remove the f-tests or include more contrasts.\n\n" + ) raise Exception(errmsg) fts_n = fts_columns.T if len(column_names) != (num_EVs_in_con_file): - - err_string = "\n\n[!] CPAC says: The number of EVs in your model " \ - "design matrix (found in the %s.mat file) does not " \ - "match the number of EVs (columns) in your custom " \ - "contrasts matrix CSV file.\n\nCustom contrasts matrix "\ - "file: %s\n\nNumber of EVs in design matrix: %d\n" \ - "Number of EVs in contrasts file: %d\n\nThe column " \ - "labels in the design matrix should match those in " \ - "your contrasts .CSV file.\nColumn labels in design " \ - "matrix:\n%s" % (model_name, con_file, \ - len(column_names), num_EVs_in_con_file, \ - str(column_names)) - - #raise Exception(err_string) - print(err_string) + "\n\n[!] CPAC says: The number of EVs in your model " "design matrix (found in the %s.mat file) does not " "match the number of EVs (columns) in your custom " "contrasts matrix CSV file.\n\nCustom contrasts matrix " "file: %s\n\nNumber of EVs in design matrix: %d\n" "Number of EVs in contrasts file: %d\n\nThe column " "labels in the design matrix should match those in " "your contrasts .CSV file.\nColumn labels in design " "matrix:\n%s" % ( + model_name, + con_file, + len(column_names), + num_EVs_in_con_file, + str(column_names), + ) + + # raise Exception(err_string) return None, None for design_mat_col, con_csv_col in zip(column_names, evs[1:]): if con_csv_col not in design_mat_col: - errmsg = "\n\n[!] CPAC says: The names of the EVs in your " \ - "custom contrasts .csv file do not match the names or " \ - "order of the EVs in the design matrix. Please make " \ - "sure these are consistent.\nDesign matrix EV columns: "\ - "%s\nYour contrasts matrix columns: %s\n\n" \ - % (column_names, evs[1:]) - - print(errmsg) - return None, None + errmsg = ( + "\n\n[!] CPAC says: The names of the EVs in your " + "custom contrasts .csv file do not match the names or " + "order of the EVs in the design matrix. Please make " + "sure these are consistent.\nDesign matrix EV columns: " + "%s\nYour contrasts matrix columns: %s\n\n" % (column_names, evs[1:]) + ) - out_file = os.path.join(output_dir, model_name + '.con') + return None, None - with open(out_file,"wt") as f: + out_file = os.path.join(output_dir, model_name + ".con") + with open(out_file, "wt") as f: idx = 1 - pp_str = '/PPheights' - re_str = '/RequiredEffect' + pp_str = "/PPheights" + re_str = "/RequiredEffect" for name in contrast_names: - - print('/ContrastName%d' %idx, '\t', name, file=f) - pp_str += '\t%1.5e' %(1) - re_str += '\t%1.5e' %(1) + print("/ContrastName%d" % idx, "\t", name, file=f) + pp_str += "\t%1.5e" % (1) + re_str += "\t%1.5e" % (1) idx += 1 - print('/NumWaves\t', (contrasts.shape)[1], file=f) - print('/NumContrasts\t', (contrasts.shape)[0], file=f) + print("/NumWaves\t", (contrasts.shape)[1], file=f) + print("/NumContrasts\t", (contrasts.shape)[0], file=f) print(pp_str, file=f) - print(re_str + '\n', file=f) + print(re_str + "\n", file=f) # print labels for the columns - mainly for double-checking your model - col_string = '\n' + col_string = "\n" for ev in evs: if "contrast" not in ev and "Contrast" not in ev: - col_string = col_string + ev + '\t' - print(col_string, '\n', file=f) + col_string = col_string + ev + "\t" + print(col_string, "\n", file=f) - print('/Matrix', file=f) - - np.savetxt(f, contrasts, fmt='%1.5e', delimiter='\t') + print("/Matrix", file=f) + + np.savetxt(f, contrasts, fmt="%1.5e", delimiter="\t") ftest_out_file = None if fTest: + ftest_out_file = os.path.join(output_dir, model_name + ".fts") - print("\nFound f-tests in your model, writing f-tests file (.fts)..\n") - - ftest_out_file = os.path.join(output_dir, model_name + '.fts') - - with open(ftest_out_file,"wt") as f: - - print('/NumWaves\t', (contrasts.shape)[0], file=f) - print('/NumContrasts\t', count_ftests, file=f) + with open(ftest_out_file, "wt") as f: + print("/NumWaves\t", (contrasts.shape)[0], file=f) + print("/NumContrasts\t", count_ftests, file=f) # print labels for the columns - mainly for double-checking your # model - col_string = '\n' + col_string = "\n" for con in contrast_names: - col_string = col_string + con + '\t' - print(col_string, '\n', file=f) + col_string = col_string + con + "\t" + print(col_string, "\n", file=f) - print('/Matrix', file=f) + print("/Matrix", file=f) for i in range(fts_n.shape[0]): - print(' '.join(fts_n[i].astype('str')), file=f) + print(" ".join(fts_n[i].astype("str")), file=f) return out_file, ftest_out_file -def create_flame_model_files(design_matrix, col_names, contrasts_vectors, - contrast_names, custom_contrasts_csv, ftest_list, - group_sep, grouping_vector, coding_scheme, - model_name, output_measure, output_dir): - +def create_flame_model_files( + design_matrix, + col_names, + contrasts_vectors, + contrast_names, + custom_contrasts_csv, + ftest_list, + group_sep, + grouping_vector, + coding_scheme, + model_name, + output_measure, + output_dir, +): if contrasts_vectors: - con_file = create_con_file(contrasts_vectors, contrast_names, - col_names, model_name, output_measure, - output_dir) + con_file = create_con_file( + contrasts_vectors, + contrast_names, + col_names, + model_name, + output_measure, + output_dir, + ) if len(ftest_list) > 0: - fts_file = create_fts_file(ftest_list, contrast_names, - model_name, output_measure, output_dir) + fts_file = create_fts_file( + ftest_list, contrast_names, model_name, output_measure, output_dir + ) else: fts_file = None elif custom_contrasts_csv: - con_file, fts_file = create_con_ftst_file(custom_contrasts_csv, - model_name, output_measure, output_dir, col_names, coding_scheme, - group_sep) + con_file, fts_file = create_con_ftst_file( + custom_contrasts_csv, + model_name, + output_measure, + output_dir, + col_names, + coding_scheme, + group_sep, + ) if not con_file: # don't write out the rest of the files if this didn't work out return None, None, None, None - mat_file = write_mat_file(design_matrix, output_dir, model_name, - col_names, output_measure) + mat_file = write_mat_file( + design_matrix, output_dir, model_name, col_names, output_measure + ) - grp_file = create_grp_file(design_matrix, grouping_vector, output_dir, - model_name) + grp_file = create_grp_file(design_matrix, grouping_vector, output_dir, model_name) return mat_file, grp_file, con_file, fts_file - diff --git a/CPAC/utils/create_fsl_flame_preset.py b/CPAC/utils/create_fsl_flame_preset.py index 5bc447545c..1951a8aa86 100644 --- a/CPAC/utils/create_fsl_flame_preset.py +++ b/CPAC/utils/create_fsl_flame_preset.py @@ -1,34 +1,29 @@ - # TODO: create a function that can help easily map raw pheno files that do not # TODO: have the participant_session id that CPAC uses def read_group_list_text_file(group_list_text_file): """Read in the group-level analysis participant-session list text file.""" - with open(group_list_text_file, "r") as f: group_list = f.readlines() # each item here includes both participant and session, and this also will # become the main ID column in the written design matrix CSV - group_list = [str(x).rstrip("\n") for x in group_list if x != ""] - - return group_list + return [str(x).rstrip("\n") for x in group_list if x != ""] def read_pheno_csv_into_df(pheno_csv, id_label=None): """Read the phenotypic file CSV or TSV into a Pandas DataFrame.""" - import pandas as pd with open(pheno_csv, "r") as f: if id_label: - if '.tsv' in pheno_csv or '.TSV' in pheno_csv: + if ".tsv" in pheno_csv or ".TSV" in pheno_csv: pheno_df = pd.read_table(f, dtype={id_label: object}) else: pheno_df = pd.read_csv(f, dtype={id_label: object}) else: - if '.tsv' in pheno_csv or '.TSV' in pheno_csv: + if ".tsv" in pheno_csv or ".TSV" in pheno_csv: pheno_df = pd.read_table(f) else: pheno_df = pd.read_csv(f) @@ -38,7 +33,6 @@ def read_pheno_csv_into_df(pheno_csv, id_label=None): def write_group_list_text_file(group_list, out_file=None): """Write out the group-level analysis participant list as a text file.""" - import os # prevent duplicates - depending on how the design matrix is set up, we @@ -50,8 +44,7 @@ def write_group_list_text_file(group_list, out_file=None): new_group_list.append(sub_ses_id) if not out_file: - out_file = os.path.join(os.getcwd(), "group_analysis_participant_" - "list.txt") + out_file = os.path.join(os.getcwd(), "group_analysis_participant_" "list.txt") else: out_file = os.path.abspath(out_file) dir_path = out_file.split(os.path.basename(out_file))[0] @@ -63,15 +56,13 @@ def write_group_list_text_file(group_list, out_file=None): f.write("{0}\n".format(part_id)) if os.path.exists(out_file): - print("Group-level analysis participant list written:" \ - "\n{0}\n".format(out_file)) + pass return out_file def write_dataframe_to_csv(matrix_df, out_file=None): """Write out a matrix Pandas DataFrame into a CSV file.""" - import os if not out_file: @@ -83,7 +74,7 @@ def write_dataframe_to_csv(matrix_df, out_file=None): os.makedirs(dir_path) try: - matrix_df = matrix_df.drop(labels='participant_session_id', axis=1) + matrix_df = matrix_df.drop(labels="participant_session_id", axis=1) except ValueError: pass except KeyError: @@ -92,13 +83,13 @@ def write_dataframe_to_csv(matrix_df, out_file=None): matrix_df.to_csv(out_file, index=False) if os.path.exists(out_file): - print("CSV file written:\n{0}\n".format(out_file)) + pass def write_config_dct_to_yaml(config_dct, out_file=None): """Write out a configuration dictionary into a YAML file.""" - import os + import CPAC if not out_file: @@ -112,45 +103,85 @@ def write_config_dct_to_yaml(config_dct, out_file=None): if not out_file.endswith(".yml"): out_file = "{0}.yml".format(out_file) - field_order = ['pipeline_dir', 'participant_list', 'output_dir', 'work_dir', - 'log_dir', 'FSLDIR', 'run_fsl_feat', 'num_models_at_once', - 'model_name', 'preset', 'pheno_file', 'ev_selections', - 'participant_id_label', 'design_formula', 'mean_mask', - 'custom_roi_mask', 'derivative_list', 'coding_scheme', - 'group_sep', 'grouping_var', 'z_threshold', 'p_threshold', - 'sessions_list', 'series_list', 'contrasts', 'f_tests', - 'custom_contrasts', 'run_randomise', 'randomise_permutation', - 'randomise_thresh', 'randomise_demean', 'randomise_tfce'] + field_order = [ + "pipeline_dir", + "participant_list", + "output_dir", + "work_dir", + "log_dir", + "FSLDIR", + "run_fsl_feat", + "num_models_at_once", + "model_name", + "preset", + "pheno_file", + "ev_selections", + "participant_id_label", + "design_formula", + "mean_mask", + "custom_roi_mask", + "derivative_list", + "coding_scheme", + "group_sep", + "grouping_var", + "z_threshold", + "p_threshold", + "sessions_list", + "series_list", + "contrasts", + "f_tests", + "custom_contrasts", + "run_randomise", + "randomise_permutation", + "randomise_thresh", + "randomise_demean", + "randomise_tfce", + ] with open(out_file, "wt") as f: - f.write("# CPAC Group-Level Analysis Configuration File\n" - "# Version {0}\n".format(CPAC.__version__)) - f.write("#\n# http://fcp-indi.github.io for more info.\n#\n" - "# Tip: This file can be edited manually with " - "a text editor for quick modifications.\n\n\n") - f.write("# General Group-Level Analysis Settings\n" - "##############################################################" - "################\n\n") + f.write( + "# CPAC Group-Level Analysis Configuration File\n" "# Version {0}\n".format( + CPAC.__version__ + ) + ) + f.write( + "#\n# http://fcp-indi.github.io for more info.\n#\n" + "# Tip: This file can be edited manually with " + "a text editor for quick modifications.\n\n\n" + ) + f.write( + "# General Group-Level Analysis Settings\n" + "##############################################################" + "################\n\n" + ) for key in field_order: val = config_dct[key] f.write("{0}: {1}\n\n".format(key, val)) - if key == 'FSLDIR': - f.write("\n# FSL-FEAT\n########################################" - "######################################\n\n") - if key == 'custom_contrasts': - f.write("\n# FSL-Randomise\n###################################" - "###########################################\n\n") + if key == "FSLDIR": + f.write( + "\n# FSL-FEAT\n########################################" + "######################################\n\n" + ) + if key == "custom_contrasts": + f.write( + "\n# FSL-Randomise\n###################################" + "###########################################\n\n" + ) if os.path.exists(out_file): - print("Group-level analysis configuration YAML file written:\n" \ - "{0}\n".format(out_file)) + pass -def create_design_matrix_df(group_list, pheno_df=None, - ev_selections=None, pheno_sub_label=None, - pheno_ses_label=None, pheno_site_label=None, - ses_id=False): +def create_design_matrix_df( + group_list, + pheno_df=None, + ev_selections=None, + pheno_sub_label=None, + pheno_ses_label=None, + pheno_site_label=None, + ses_id=False, +): """Create the design matrix intended for group-level analysis via the FSL FLAME tool. @@ -161,27 +192,26 @@ def create_design_matrix_df(group_list, pheno_df=None, participant-session ID labels in the CPAC individual-level analysis output directory with the values listed in the phenotype file. """ - import pandas as pd - keep_cols = ['participant_id'] + keep_cols = ["participant_id"] if ses_id: # if the group_list is participant_session_id instead of participant_id - map_df = pd.DataFrame({'participant_session_id': group_list}) - keep_cols += ['participant_session_id'] + map_df = pd.DataFrame({"participant_session_id": group_list}) + keep_cols += ["participant_session_id"] part_ids = [] sess_ids = [] for part_ses in group_list: - part = part_ses.split('_')[0] - sess = part_ses.split('_')[1] + part = part_ses.split("_")[0] + sess = part_ses.split("_")[1] part_ids.append(part) sess_ids.append(sess) - map_df['participant_id'] = part_ids - map_df['session'] = sess_ids - map_df = map_df.sort_values(by=['session', 'participant_id']) + map_df["participant_id"] = part_ids + map_df["session"] = sess_ids + map_df = map_df.sort_values(by=["session", "participant_id"]) else: - map_df = pd.DataFrame({'participant_id': group_list}) + map_df = pd.DataFrame({"participant_id": group_list}) if pheno_df is None: # no phenotypic matrix provided; simpler design models @@ -195,8 +225,10 @@ def create_design_matrix_df(group_list, pheno_df=None, # the Patsy design formula rename_pheno_cols = {} for col_name in pheno_df.columns: - if ' ' in col_name or '-' in col_name: - rename_pheno_cols.update({col_name: col_name.replace(' ', '_').replace('-', '_')}) + if " " in col_name or "-" in col_name: + rename_pheno_cols.update( + {col_name: col_name.replace(" ", "_").replace("-", "_")} + ) pheno_df = pheno_df.rename(columns=rename_pheno_cols) # align the pheno's participant ID column with the group sublist text @@ -206,28 +238,25 @@ def create_design_matrix_df(group_list, pheno_df=None, raise Exception("there's a pheno file, but no pheno sub label") else: # rename the pheno sub label thingy - pheno_df = pheno_df.rename( - columns={pheno_sub_label: 'participant_id'}) + pheno_df = pheno_df.rename(columns={pheno_sub_label: "participant_id"}) if ev_selections: - ev_selections.insert(0, 'participant_id') - sort_by = ['participant_id'] + ev_selections.insert(0, "participant_id") + sort_by = ["participant_id"] if pheno_ses_label: # if sessions are important in the model, do this also - pheno_df = pheno_df.rename( - columns={pheno_ses_label: 'session_id'}) + pheno_df = pheno_df.rename(columns={pheno_ses_label: "session_id"}) if ev_selections: - ev_selections.append(1, 'session_id') + ev_selections.append(1, "session_id") # again, sort by session ID first in case of repeated # measures, where the sessions have to be all together first - sort_by.insert(0, 'session_id') + sort_by.insert(0, "session_id") if pheno_site_label: # and if sites are important as well, same here - pheno_df = pheno_df.rename( - columns={pheno_site_label: 'site_id'}) + pheno_df = pheno_df.rename(columns={pheno_site_label: "site_id"}) if ev_selections: - ev_selections.append(2, 'site_id') + ev_selections.append(2, "site_id") if ev_selections: # get specific covariates! @@ -236,42 +265,44 @@ def create_design_matrix_df(group_list, pheno_df=None, # check for inconsistency with leading zeroes # (sometimes, the sub_ids from individual will be something like # '0002601' and the phenotype will have '2601') - sublist_subs = map_df['participant_id'] - pheno_subs = list(pheno_df['participant_id']) + sublist_subs = map_df["participant_id"] + pheno_subs = list(pheno_df["participant_id"]) for index, row in pheno_df.iterrows(): - pheno_sub_id = str(row['participant_id']) + pheno_sub_id = str(row["participant_id"]) for sub_id in sublist_subs: - if str(sub_id).lstrip('0') == pheno_sub_id: - pheno_df.at[index, 'participant_id'] = sub_id + if str(sub_id).lstrip("0") == pheno_sub_id: + pheno_df.at[index, "participant_id"] = sub_id for sub in sublist_subs: if sub in pheno_subs: # okay, there's at least one match break else: - new_sublist_subs = [str(x).lstrip('0') for x in sublist_subs] + new_sublist_subs = [str(x).lstrip("0") for x in sublist_subs] for sub in new_sublist_subs: if sub in pheno_subs: # that's better - map_df['participant_id'] = new_sublist_subs + map_df["participant_id"] = new_sublist_subs break else: - raise Exception('the participant IDs in your group ' - 'analysis participant list and the ' - 'participant IDs in your phenotype file ' - 'do not match') + raise Exception( + "the participant IDs in your group " + "analysis participant list and the " + "participant IDs in your phenotype file " + "do not match" + ) # merge if pheno_ses_label: - design_df = pheno_df.merge(map_df, on=['participant_id']) + design_df = pheno_df.merge(map_df, on=["participant_id"]) else: - design_df = pheno_df.merge(map_df[['participant_id']], - on='participant_id') + design_df = pheno_df.merge( + map_df[["participant_id"]], on="participant_id" + ) design_df = design_df.sort_values(sort_by) - return design_df @@ -279,12 +310,12 @@ def create_contrasts_template_df(design_df, contrasts_dct_list=None): """Create the template Pandas DataFrame for the contrasts matrix CSV. The headers in the contrasts matrix needs to match the headers of the - design matrix.""" - + design matrix. + """ import pandas as pd contrast_cols = list(design_df.columns) - contrast_cols.remove('participant_id') + contrast_cols.remove("participant_id") if contrasts_dct_list: # if we are initializing the contrasts matrix with pre-set contrast @@ -296,9 +327,11 @@ def create_contrasts_template_df(design_df, contrasts_dct_list=None): if (len(contrast_dct) - 1) != len(contrast_cols): # it's -1 because of the "contrast" column in contrast_dct # TODO: message - raise Exception("number of columns in the contrast vector " - "does not match the number of covariate " - "columns in the design matrix") + raise Exception( + "number of columns in the contrast vector " + "does not match the number of covariate " + "columns in the design matrix" + ) else: # if default, start it up with a blank "template" contrast vector @@ -317,17 +350,20 @@ def create_contrasts_template_df(design_df, contrasts_dct_list=None): contrasts_df = pd.DataFrame(contrasts_dct_list) # order the columns properly - contrasts_df = contrasts_df[contrast_cols] + return contrasts_df[contrast_cols] - return contrasts_df - -def preset_single_group_avg(group_list, pheno_df=None, covariate=None, - pheno_sub_label=None, output_dir=None, - model_name="one_sample_T-test"): +def preset_single_group_avg( + group_list, + pheno_df=None, + covariate=None, + pheno_sub_label=None, + output_dir=None, + model_name="one_sample_T-test", +): """Set up the design matrix CSV for running a single group average - (one-sample T-test).""" - + (one-sample T-test). + """ import os if not output_dir: @@ -338,8 +374,8 @@ def preset_single_group_avg(group_list, pheno_df=None, covariate=None, # change spaces and dashes to underscores to prevent confusion with the # Patsy design formula if covariate: - covariate = covariate.lstrip(' ').rstrip(' ') - covariate = covariate.replace(' ', '_').replace('-', '_') + covariate = covariate.lstrip(" ").rstrip(" ") + covariate = covariate.replace(" ", "_").replace("-", "_") ev_selections = None if pheno_df is not None: @@ -347,9 +383,12 @@ def preset_single_group_avg(group_list, pheno_df=None, covariate=None, # if we're adding an additional covariate ev_selections = [covariate] - design_df = create_design_matrix_df(group_list, pheno_df, - ev_selections=ev_selections, - pheno_sub_label=pheno_sub_label) + design_df = create_design_matrix_df( + group_list, + pheno_df, + ev_selections=ev_selections, + pheno_sub_label=pheno_sub_label, + ) design_df["Group_Mean"] = 1 @@ -381,38 +420,45 @@ def preset_single_group_avg(group_list, pheno_df=None, covariate=None, contrasts_df = create_contrasts_template_df(design_df, contrasts) # create design and contrasts matrix file paths - design_mat_path = os.path.join(output_dir, model_name, - "design_matrix_{0}.csv".format(model_name)) + design_mat_path = os.path.join( + output_dir, model_name, "design_matrix_{0}.csv".format(model_name) + ) - contrasts_mat_path = os.path.join(output_dir, model_name, - "contrasts_matrix_{0}.csv" - "".format(model_name)) + contrasts_mat_path = os.path.join( + output_dir, model_name, "contrasts_matrix_{0}.csv" "".format(model_name) + ) # start group config yaml dictionary design_formula = "Group_Mean" if covariate: design_formula = "{0} + {1}".format(design_formula, covariate) - group_config = {"pheno_file": design_mat_path, - "ev_selections": {"demean": [str(covariate)], - "categorical": ["Group_Mean"]}, - "design_formula": design_formula, - "group_sep": "Off", - "grouping_var": None, - "sessions_list": [], - "series_list": [], - "custom_contrasts": contrasts_mat_path, - "model_name": model_name, - "output_dir": os.path.join(output_dir, model_name), - "work_dir": os.path.join(output_dir, model_name), - "log_dir": os.path.join(output_dir, model_name)} + group_config = { + "pheno_file": design_mat_path, + "ev_selections": {"demean": [str(covariate)], "categorical": ["Group_Mean"]}, + "design_formula": design_formula, + "group_sep": "Off", + "grouping_var": None, + "sessions_list": [], + "series_list": [], + "custom_contrasts": contrasts_mat_path, + "model_name": model_name, + "output_dir": os.path.join(output_dir, model_name), + "work_dir": os.path.join(output_dir, model_name), + "log_dir": os.path.join(output_dir, model_name), + } return design_df, contrasts_df, group_config -def preset_unpaired_two_group(group_list, pheno_df, groups, pheno_sub_label, - output_dir=None, - model_name="two_sample_unpaired_T-test"): +def preset_unpaired_two_group( + group_list, + pheno_df, + groups, + pheno_sub_label, + output_dir=None, + model_name="two_sample_unpaired_T-test", +): """Set up the design matrix and contrasts matrix for running an unpaired two-group difference (two-sample unpaired T-test). @@ -435,7 +481,6 @@ def preset_unpaired_two_group(group_list, pheno_df, groups, pheno_sub_label, pull it from the phenotype file, and this function will break it out into two columns using dummy-coding. """ - import os if not output_dir: @@ -448,8 +493,8 @@ def preset_unpaired_two_group(group_list, pheno_df, groups, pheno_sub_label, old_groups = groups groups = [] for group in old_groups: - group = group.lstrip(' ').rstrip(' ') - group = group.replace(' ', '_').replace('-', '_') + group = group.lstrip(" ").rstrip(" ") + group = group.replace(" ", "_").replace("-", "_") groups.append(group) # if the two groups are encoded in one categorical EV/column, then we will @@ -460,9 +505,12 @@ def preset_unpaired_two_group(group_list, pheno_df, groups, pheno_sub_label, for group in groups: ev_selections.append(group) - design_df = create_design_matrix_df(group_list, pheno_df, - ev_selections=ev_selections, - pheno_sub_label=pheno_sub_label) + design_df = create_design_matrix_df( + group_list, + pheno_df, + ev_selections=ev_selections, + pheno_sub_label=pheno_sub_label, + ) if len(groups) == 1: # we're going to split the one categorical EV into two @@ -476,25 +524,31 @@ def preset_unpaired_two_group(group_list, pheno_df, groups, pheno_sub_label, # Patsy design formula new_group_set = [] for group in group_set: - group = group.lstrip(' ').rstrip(' ') - group = group.replace(' ', '_').replace('-', '_') + group = group.lstrip(" ").rstrip(" ") + group = group.replace(" ", "_").replace("-", "_") new_group_set.append(group) # this preset is for an unpaired two-group difference- should only be # two groups encoded in this EV! if len(group_set) > 2: # TODO: message - raise Exception("more than two groups provided, but this is a" - "model for a two-group difference\n\ngroups " - "found in column:\n{0}".format(str(group_set))) + raise Exception( + "more than two groups provided, but this is a" + "model for a two-group difference\n\ngroups " + "found in column:\n{0}".format(str(group_set)) + ) elif len(group_set) == 0: - raise Exception("no groups were found - something went wrong " - "with reading the phenotype information") + raise Exception( + "no groups were found - something went wrong " + "with reading the phenotype information" + ) elif len(group_set) == 1: - raise Exception("only one group found in the column provided, " - "but this is a model for a two-group difference" - "\n\ngroups found in column:\n" - "{0}".format(str(group_set))) + raise Exception( + "only one group found in the column provided, " + "but this is a model for a two-group difference" + "\n\ngroups found in column:\n" + "{0}".format(str(group_set)) + ) # create the two new dummy-coded columns # column 1 @@ -503,8 +557,9 @@ def preset_unpaired_two_group(group_list, pheno_df, groups, pheno_sub_label, # create new column encoded in 0's design_df[new_name] = 0 # map the relevant values into 1's - design_df[new_name] = design_df[groups[0]].map({group_set[0]: 1, - group_set[1]: 0}) + design_df[new_name] = design_df[groups[0]].map( + {group_set[0]: 1, group_set[1]: 0} + ) # update groups list new_groups.append(new_name) @@ -514,8 +569,9 @@ def preset_unpaired_two_group(group_list, pheno_df, groups, pheno_sub_label, # create new column encoded in 0's design_df[new_name] = 0 # map the relevant values into 1's - design_df[new_name] = design_df[groups[0]].map({group_set[1]: 1, - group_set[0]: 0}) + design_df[new_name] = design_df[groups[0]].map( + {group_set[1]: 1, group_set[0]: 0} + ) # update groups list new_groups.append(new_name) @@ -548,36 +604,42 @@ def preset_unpaired_two_group(group_list, pheno_df, groups, pheno_sub_label, contrasts_df = create_contrasts_template_df(design_df, contrasts) # create design and contrasts matrix file paths - design_mat_path = os.path.join(output_dir, model_name, - "design_matrix_{0}.csv".format(model_name)) + design_mat_path = os.path.join( + output_dir, model_name, "design_matrix_{0}.csv".format(model_name) + ) - contrasts_mat_path = os.path.join(output_dir, model_name, - "contrasts_matrix_{0}.csv" - "".format(model_name)) + contrasts_mat_path = os.path.join( + output_dir, model_name, "contrasts_matrix_{0}.csv" "".format(model_name) + ) # start group config yaml dictionary design_formula = "{0} + {1}".format(groups[0], groups[1]) - group_config = {"pheno_file": design_mat_path, - "ev_selections": {"demean": [], - "categorical": str(groups)}, - "design_formula": design_formula, - "group_sep": "On", - "grouping_var": str(groups), - "sessions_list": [], - "series_list": [], - "custom_contrasts": contrasts_mat_path, - "model_name": model_name, - "output_dir": os.path.join(output_dir, model_name), - "work_dir": os.path.join(output_dir, model_name), - "log_dir": os.path.join(output_dir, model_name)} + group_config = { + "pheno_file": design_mat_path, + "ev_selections": {"demean": [], "categorical": str(groups)}, + "design_formula": design_formula, + "group_sep": "On", + "grouping_var": str(groups), + "sessions_list": [], + "series_list": [], + "custom_contrasts": contrasts_mat_path, + "model_name": model_name, + "output_dir": os.path.join(output_dir, model_name), + "work_dir": os.path.join(output_dir, model_name), + "log_dir": os.path.join(output_dir, model_name), + } return design_df, contrasts_df, group_config -def preset_paired_two_group(group_list, conditions, condition_type="session", - output_dir=None, - model_name="two_sample_unpaired_T-test"): +def preset_paired_two_group( + group_list, + conditions, + condition_type="session", + output_dir=None, + model_name="two_sample_unpaired_T-test", +): """Set up the design matrix and contrasts matrix for running an paired two-group difference (two-sample paired T-test). @@ -594,7 +656,6 @@ def preset_paired_two_group(group_list, conditions, condition_type="session", https://fsl.fmrib.ox.ac.uk/fsl/fslwiki/FEAT/UserGuide #Paired_Two-Group_Difference_.28Two-Sample_Paired_T-Test.29 """ - import os if not output_dir: @@ -663,19 +724,25 @@ def preset_paired_two_group(group_list, conditions, condition_type="session", # let's check to make sure it came out right # first half past_val = None - for val in condition_ev[0:(len(condition_ev) / 2) - 1]: + for val in condition_ev[0 : (len(condition_ev) / 2) - 1]: if past_val: if val != past_val: - raise Exception('Non-equal amount of participants for each ' - '{0}.\n'.format(condition_type)) + raise Exception( + "Non-equal amount of participants for each " "{0}.\n".format( + condition_type + ) + ) past_val = val # second half past_val = None - for val in condition_ev[(len(condition_ev) / 2):]: + for val in condition_ev[(len(condition_ev) / 2) :]: if past_val: if val != past_val: - raise Exception('Non-equal amount of participants for each ' - '{0}.\n'.format(condition_type)) + raise Exception( + "Non-equal amount of participants for each " "{0}.\n".format( + condition_type + ) + ) past_val = val design_df[condition_type] = condition_ev @@ -701,31 +768,33 @@ def preset_paired_two_group(group_list, conditions, condition_type="session", contrast_one.update({new_part_label: 0}) contrast_two.update({new_part_label: 0}) if new_part_label not in design_formula: - design_formula = "{0} + {1}".format(design_formula, - new_part_label) + design_formula = "{0} + {1}".format(design_formula, new_part_label) # finish the contrasts # should be something like # ses,sub,sub,sub, etc. # ses-1 - ses-2: 1, 0, 0, 0, 0... # ses-2 - ses-1: -1, 0, 0, 0, etc. - contrast_one.update({ - "Contrasts": "{0}_{1} - {2}_{3}".format(condition_type, - conditions[0], - condition_type, - conditions[1])}) - contrast_two.update({ - "Contrasts": "{0}_{1} - {2}_{3}".format(condition_type, - conditions[1], - condition_type, - conditions[0])}) + contrast_one.update( + { + "Contrasts": "{0}_{1} - {2}_{3}".format( + condition_type, conditions[0], condition_type, conditions[1] + ) + } + ) + contrast_two.update( + { + "Contrasts": "{0}_{1} - {2}_{3}".format( + condition_type, conditions[1], condition_type, conditions[0] + ) + } + ) contrast_one.update({condition_type: 1}) contrast_two.update({condition_type: -1}) try: - design_df = design_df.drop(labels=['participant_session_id'], - axis='columns') + design_df = design_df.drop(labels=["participant_session_id"], axis="columns") except KeyError: pass @@ -733,32 +802,40 @@ def preset_paired_two_group(group_list, conditions, condition_type="session", contrasts_df = create_contrasts_template_df(design_df, contrasts) # create design and contrasts matrix file paths - design_mat_path = os.path.join(output_dir, model_name, - "design_matrix_{0}.csv".format(model_name)) + design_mat_path = os.path.join( + output_dir, model_name, "design_matrix_{0}.csv".format(model_name) + ) - contrasts_mat_path = os.path.join(output_dir, model_name, - "contrasts_matrix_{0}.csv" - "".format(model_name)) + contrasts_mat_path = os.path.join( + output_dir, model_name, "contrasts_matrix_{0}.csv" "".format(model_name) + ) # start group config yaml dictionary - group_config.update({"pheno_file": design_mat_path, - "ev_selections": {"demean": [], - "categorical": []}, - "design_formula": design_formula, - "group_sep": "Off", - "grouping_var": None, - "custom_contrasts": contrasts_mat_path, - "model_name": model_name, - "output_dir": os.path.join(output_dir, model_name), - "work_dir": os.path.join(output_dir, model_name), - "log_dir": os.path.join(output_dir, model_name)}) + group_config.update( + { + "pheno_file": design_mat_path, + "ev_selections": {"demean": [], "categorical": []}, + "design_formula": design_formula, + "group_sep": "Off", + "grouping_var": None, + "custom_contrasts": contrasts_mat_path, + "model_name": model_name, + "output_dir": os.path.join(output_dir, model_name), + "work_dir": os.path.join(output_dir, model_name), + "log_dir": os.path.join(output_dir, model_name), + } + ) return design_df, contrasts_df, group_config -def preset_tripled_two_group(group_list, conditions, condition_type="Sessions", - output_dir=None, - model_name="tripled_T-test"): +def preset_tripled_two_group( + group_list, + conditions, + condition_type="Sessions", + output_dir=None, + model_name="tripled_T-test", +): """Set up the design matrix and contrasts matrix for running a tripled two-group difference ('tripled' T-test). @@ -775,7 +852,6 @@ def preset_tripled_two_group(group_list, conditions, condition_type="Sessions", https://fsl.fmrib.ox.ac.uk/fsl/fslwiki/FEAT/UserGuide #Tripled_Two-Group_Difference_.28.22Tripled.22_T-Test.29 """ - import os if not output_dir: @@ -783,8 +859,7 @@ def preset_tripled_two_group(group_list, conditions, condition_type="Sessions", if len(conditions) != 3: # TODO: msg - raise Exception('Three conditions are required for the tripled ' - 't-test.\n') + raise Exception("Three conditions are required for the tripled " "t-test.\n") sess_conditions = ["session", "Session", "sessions", "Sessions"] scan_conditions = ["scan", "scans", "series", "Series/Scans", "Series"] @@ -853,32 +928,36 @@ def preset_tripled_two_group(group_list, conditions, condition_type="Sessions", # let's check to make sure it came out right # first third - for val in condition_ev_one[0:(len(condition_ev_one) / 3) - 1]: + for val in condition_ev_one[0 : (len(condition_ev_one) / 3) - 1]: if val != 1: # TODO: msg raise Exception # second third - for val in condition_ev_one[(len(condition_ev_one) / 3):(len(condition_ev_one)/3)*2]: + for val in condition_ev_one[ + (len(condition_ev_one) / 3) : (len(condition_ev_one) / 3) * 2 + ]: if val != -1: # TODO: msg raise Exception # third... third - for val in condition_ev_one[((len(condition_ev_one)/3)*2 + 1):]: + for val in condition_ev_one[((len(condition_ev_one) / 3) * 2 + 1) :]: if val != 0: # TODO: msg raise Exception # first third - for val in condition_ev_two[0:(len(condition_ev_two) / 3) - 1]: + for val in condition_ev_two[0 : (len(condition_ev_two) / 3) - 1]: if val != 1: # TODO: msg raise Exception # second third - for val in condition_ev_two[(len(condition_ev_two) / 3):(len(condition_ev_two)/3)*2]: + for val in condition_ev_two[ + (len(condition_ev_two) / 3) : (len(condition_ev_two) / 3) * 2 + ]: if val != 0: # TODO: msg raise Exception # third... third - for val in condition_ev_two[((len(condition_ev_two)/3)*2 + 1):]: + for val in condition_ev_two[((len(condition_ev_two) / 3) * 2 + 1) :]: if val != -1: # TODO: msg raise Exception @@ -912,8 +991,7 @@ def preset_tripled_two_group(group_list, conditions, condition_type="Sessions", contrast_two.update({new_part_label: 0}) contrast_three.update({new_part_label: 0}) if new_part_label not in design_formula: - design_formula = "{0} + {1}".format(design_formula, - new_part_label) + design_formula = "{0} + {1}".format(design_formula, new_part_label) # finish the contrasts # should be something like @@ -921,30 +999,35 @@ def preset_tripled_two_group(group_list, conditions, condition_type="Sessions", # ses-1 - ses-2: 2, 1, 0, 0, 0... # ses-1 - ses-3: 1, 2, 0, 0, 0... # ses-2 - ses-3: -1, 1, 0, 0, 0, etc. - contrast_one.update({ - "Contrasts": "{0}_{1} - {2}_{3}".format(condition_type, - conditions[0], - condition_type, - conditions[1])}) - contrast_two.update({ - "Contrasts": "{0}_{1} - {2}_{3}".format(condition_type, - conditions[0], - condition_type, - conditions[2])}) - - contrast_three.update({ - "Contrasts": "{0}_{1} - {2}_{3}".format(condition_type, - conditions[1], - condition_type, - conditions[2])}) + contrast_one.update( + { + "Contrasts": "{0}_{1} - {2}_{3}".format( + condition_type, conditions[0], condition_type, conditions[1] + ) + } + ) + contrast_two.update( + { + "Contrasts": "{0}_{1} - {2}_{3}".format( + condition_type, conditions[0], condition_type, conditions[2] + ) + } + ) + + contrast_three.update( + { + "Contrasts": "{0}_{1} - {2}_{3}".format( + condition_type, conditions[1], condition_type, conditions[2] + ) + } + ) contrast_one.update({column_one: 2, column_two: 1}) contrast_two.update({column_one: 1, column_two: 2}) contrast_three.update({column_one: -1, column_two: 1}) try: - design_df = design_df.drop(labels=['participant_session_id'], - axis='columns') + design_df = design_df.drop(labels=["participant_session_id"], axis="columns") except KeyError: pass @@ -952,34 +1035,48 @@ def preset_tripled_two_group(group_list, conditions, condition_type="Sessions", contrasts_df = create_contrasts_template_df(design_df, contrasts) # create design and contrasts matrix file paths - design_mat_path = os.path.join(output_dir, model_name, - "design_matrix_{0}.csv".format(model_name)) + design_mat_path = os.path.join( + output_dir, model_name, "design_matrix_{0}.csv".format(model_name) + ) - contrasts_mat_path = os.path.join(output_dir, model_name, - "contrasts_matrix_{0}.csv" - "".format(model_name)) + contrasts_mat_path = os.path.join( + output_dir, model_name, "contrasts_matrix_{0}.csv" "".format(model_name) + ) # start group config yaml dictionary - group_config.update({"pheno_file": design_mat_path, - "ev_selections": {"demean": [], - "categorical": []}, - "design_formula": design_formula, - "group_sep": "Off", - "grouping_var": None, - "custom_contrasts": contrasts_mat_path, - "model_name": model_name, - "output_dir": os.path.join(output_dir, model_name), - "work_dir": os.path.join(output_dir, model_name), - "log_dir": os.path.join(output_dir, model_name)}) + group_config.update( + { + "pheno_file": design_mat_path, + "ev_selections": {"demean": [], "categorical": []}, + "design_formula": design_formula, + "group_sep": "Off", + "grouping_var": None, + "custom_contrasts": contrasts_mat_path, + "model_name": model_name, + "output_dir": os.path.join(output_dir, model_name), + "work_dir": os.path.join(output_dir, model_name), + "log_dir": os.path.join(output_dir, model_name), + } + ) return design_df, contrasts_df, group_config -def run(pipeline_dir, derivative_list, z_thresh, p_thresh, preset=None, - group_list_text_file=None, pheno_file=None, pheno_sub_label=None, - output_dir=None, model_name=None, covariate=None, condition_type=None, - run=False): - +def run( + pipeline_dir, + derivative_list, + z_thresh, + p_thresh, + preset=None, + group_list_text_file=None, + pheno_file=None, + pheno_sub_label=None, + output_dir=None, + model_name=None, + covariate=None, + condition_type=None, + run=False, +): # FSL FEAT presets: run regular group analysis with no changes to its # original flow- use the generated pheno as the pheno, use the # contrasts DF as a custom contrasts matrix, and auto-generate the @@ -989,21 +1086,32 @@ def run(pipeline_dir, derivative_list, z_thresh, p_thresh, preset=None, # or something import os + import pandas as pd import pkg_resources as p # make life easy - keys_csv = p.resource_filename('CPAC', 'resources/cpac_outputs.csv') + keys_csv = p.resource_filename("CPAC", "resources/cpac_outputs.csv") try: - keys = pd.read_csv(keys_csv) + pd.read_csv(keys_csv) except Exception as e: - err = "\n[!] Could not access or read the cpac_outputs.csv " \ - "resource file:\n{0}\n\nError details {1}\n".format(keys_csv, e) + err = ( + "\n[!] Could not access or read the cpac_outputs.csv " + "resource file:\n{0}\n\nError details {1}\n".format(keys_csv, e) + ) raise Exception(err) - if derivative_list == 'all': - derivative_list = ['alff', 'falff', 'reho', 'sca_roi', 'sca_tempreg', - 'vmhc', 'centrality', 'dr_tempreg'] + if derivative_list == "all": + derivative_list = [ + "alff", + "falff", + "reho", + "sca_roi", + "sca_tempreg", + "vmhc", + "centrality", + "dr_tempreg", + ] if pheno_file and not pheno_sub_label: # TODO: message @@ -1021,75 +1129,88 @@ def run(pipeline_dir, derivative_list, z_thresh, p_thresh, preset=None, if not group_list_text_file: from CPAC.pipeline.cpac_group_runner import grab_pipeline_dir_subs - if (preset == "paired_two" or preset == "tripled_two") and "Sessions" in condition_type: + + if (preset in ("paired_two", "tripled_two")) and "Sessions" in condition_type: group_list = grab_pipeline_dir_subs(pipeline_dir, True) else: group_list = grab_pipeline_dir_subs(pipeline_dir) - group_list_text_file = os.path.join(output_dir, model_name, - "group_participant_list_" - "{0}.txt".format(model_name)) + group_list_text_file = os.path.join( + output_dir, + model_name, + "group_participant_list_" "{0}.txt".format(model_name), + ) elif isinstance(group_list_text_file, list): group_list = group_list_text_file # write out a group analysis sublist text file so that it can be # linked in the group analysis config yaml - group_list_text_file = os.path.join(output_dir, model_name, - "group_participant_list_" - "{0}.txt".format(model_name)) + group_list_text_file = os.path.join( + output_dir, + model_name, + "group_participant_list_" "{0}.txt".format(model_name), + ) elif os.path.isfile(group_list_text_file): group_list = read_group_list_text_file(group_list_text_file) # write out a group analysis sublist text file so that it can be # linked in the group analysis config yaml - group_list_text_file = os.path.join(output_dir, model_name, - "group_participant_list_" - "{0}.txt".format(model_name)) + group_list_text_file = os.path.join( + output_dir, + model_name, + "group_participant_list_" "{0}.txt".format(model_name), + ) if len(group_list) == 0: - msg = "\n\n[!] C-PAC says: No participants found in the pipeline " \ - "directory you provided. Make sure the directory is the " \ - "individual-level pipeline directory that contains the sub-" \ - "directories labeled with the participant_session IDs.\n\n" \ - "Pipeline directory provided: {0}\n\n".format(pipeline_dir) + msg = ( + "\n\n[!] C-PAC says: No participants found in the pipeline " + "directory you provided. Make sure the directory is the " + "individual-level pipeline directory that contains the sub-" + "directories labeled with the participant_session IDs.\n\n" + "Pipeline directory provided: {0}\n\n".format(pipeline_dir) + ) raise Exception(msg) if not preset: # TODO: this pass - group_config = {"pipeline_dir": pipeline_dir, - "FSLDIR": "FSLDIR", - "run_fsl_feat": [1], - "num_models_at_once": 1, - "preset": preset, - "participant_list": group_list_text_file, - "participant_id_label": "participant_id", - "mean_mask": ["Group Mask"], - "custom_roi_mask": None, - "derivative_list": derivative_list, - "coding_scheme": ["Treatment"], - "z_threshold": [float(z_thresh)], - "p_threshold": [float(p_thresh)], - "contrasts": [], - "f_tests": [], - "run_randomise": [0], - "randomise_permutation": 500, - "randomise_thresh": 5, - "randomise_demean": True, - "randomise_tfce": True} + group_config = { + "pipeline_dir": pipeline_dir, + "FSLDIR": "FSLDIR", + "run_fsl_feat": [1], + "num_models_at_once": 1, + "preset": preset, + "participant_list": group_list_text_file, + "participant_id_label": "participant_id", + "mean_mask": ["Group Mask"], + "custom_roi_mask": None, + "derivative_list": derivative_list, + "coding_scheme": ["Treatment"], + "z_threshold": [float(z_thresh)], + "p_threshold": [float(p_thresh)], + "contrasts": [], + "f_tests": [], + "run_randomise": [0], + "randomise_permutation": 500, + "randomise_thresh": 5, + "randomise_demean": True, + "randomise_tfce": True, + } if preset == "single_grp": - design_df, contrasts_df, group_config_update = \ - preset_single_group_avg(group_list, pheno_df=None, covariate=None, - pheno_sub_label=None, - output_dir=output_dir, - model_name=model_name) + design_df, contrasts_df, group_config_update = preset_single_group_avg( + group_list, + pheno_df=None, + covariate=None, + pheno_sub_label=None, + output_dir=output_dir, + model_name=model_name, + ) group_config.update(group_config_update) elif preset == "single_grp_cov": - if not pheno_file: # TODO: message raise Exception("pheno file not provided") @@ -1100,16 +1221,18 @@ def run(pipeline_dir, derivative_list, z_thresh, p_thresh, preset=None, pheno_df = read_pheno_csv_into_df(pheno_file, pheno_sub_label) - design_df, contrasts_df, group_config_update = \ - preset_single_group_avg(group_list, pheno_df, covariate=covariate, - pheno_sub_label=pheno_sub_label, - output_dir=output_dir, - model_name=model_name) + design_df, contrasts_df, group_config_update = preset_single_group_avg( + group_list, + pheno_df, + covariate=covariate, + pheno_sub_label=pheno_sub_label, + output_dir=output_dir, + model_name=model_name, + ) group_config.update(group_config_update) elif preset == "unpaired_two": - if not pheno_file: # TODO: message raise Exception("pheno file not provided") @@ -1126,12 +1249,14 @@ def run(pipeline_dir, derivative_list, z_thresh, p_thresh, preset=None, pheno_df = read_pheno_csv_into_df(pheno_file, pheno_sub_label) # in this case, "covariate" gets sent in as a list of two covariates - design_df, contrasts_df, group_config_update = \ - preset_unpaired_two_group(group_list, pheno_df, - groups=covariate, - pheno_sub_label=pheno_sub_label, - output_dir=output_dir, - model_name=model_name) + design_df, contrasts_df, group_config_update = preset_unpaired_two_group( + group_list, + pheno_df, + groups=covariate, + pheno_sub_label=pheno_sub_label, + output_dir=output_dir, + model_name=model_name, + ) group_config.update(group_config_update) @@ -1149,8 +1274,10 @@ def run(pipeline_dir, derivative_list, z_thresh, p_thresh, preset=None, if not condition_type: # TODO: message - raise Exception("you didn't specify whether the two groups are " - "sessions or series/scans") + raise Exception( + "you didn't specify whether the two groups are " + "sessions or series/scans" + ) # we're assuming covariate (which in this case, is the two sessions, # or two scans) will be coming in as a string of either one covariate @@ -1162,12 +1289,13 @@ def run(pipeline_dir, derivative_list, z_thresh, p_thresh, preset=None, # it's already a list- keep it that way pass - design_df, contrasts_df, group_config_update = \ - preset_paired_two_group(group_list, - conditions=covariate, - condition_type=condition_type, - output_dir=output_dir, - model_name=model_name) + design_df, contrasts_df, group_config_update = preset_paired_two_group( + group_list, + conditions=covariate, + condition_type=condition_type, + output_dir=output_dir, + model_name=model_name, + ) group_config.update(group_config_update) @@ -1185,8 +1313,10 @@ def run(pipeline_dir, derivative_list, z_thresh, p_thresh, preset=None, if not condition_type: # TODO: message - raise Exception("you didn't specify whether the three groups are " - "sessions or series/scans") + raise Exception( + "you didn't specify whether the three groups are " + "sessions or series/scans" + ) # we're assuming covariate (which in this case, is the three sessions, # or three scans) will be coming in as a string of either one @@ -1199,12 +1329,13 @@ def run(pipeline_dir, derivative_list, z_thresh, p_thresh, preset=None, # it's already a list- keep it that way pass - design_df, contrasts_df, group_config_update = \ - preset_tripled_two_group(group_list, - conditions=covariate, - condition_type=condition_type, - output_dir=output_dir, - model_name=model_name) + design_df, contrasts_df, group_config_update = preset_tripled_two_group( + group_list, + conditions=covariate, + condition_type=condition_type, + output_dir=output_dir, + model_name=model_name, + ) group_config.update(group_config_update) @@ -1213,8 +1344,7 @@ def run(pipeline_dir, derivative_list, z_thresh, p_thresh, preset=None, raise Exception("not one of the valid presets") # write participant list text file - write_group_list_text_file(design_df["participant_id"], - group_list_text_file) + write_group_list_text_file(design_df["participant_id"], group_list_text_file) # write design matrix CSV write_dataframe_to_csv(design_df, group_config["pheno_file"]) @@ -1223,8 +1353,9 @@ def run(pipeline_dir, derivative_list, z_thresh, p_thresh, preset=None, write_dataframe_to_csv(contrasts_df, group_config["custom_contrasts"]) # write group-level analysis config YAML - out_config = os.path.join(output_dir, model_name, - "group_config_{0}.yml".format(model_name)) + out_config = os.path.join( + output_dir, model_name, "group_config_{0}.yml".format(model_name) + ) write_config_dct_to_yaml(group_config, out_config) if run: diff --git a/CPAC/utils/create_fsl_model.py b/CPAC/utils/create_fsl_model.py index 630436179b..35316e4ac7 100644 --- a/CPAC/utils/create_fsl_model.py +++ b/CPAC/utils/create_fsl_model.py @@ -1,54 +1,47 @@ - def load_pheno_file(pheno_file): - import os + import pandas as pd if not os.path.isfile(pheno_file): - err = "\n\n[!] CPAC says: The group-level analysis phenotype file "\ - "provided does not exist!\nPath provided: %s\n\n" \ - % pheno_file + err = ( + "\n\n[!] CPAC says: The group-level analysis phenotype file " + "provided does not exist!\nPath provided: %s\n\n" % pheno_file + ) raise Exception(err) - - with open(os.path.abspath(pheno_file),"r") as f: - - pheno_dataframe = pd.read_csv(f) - - - return pheno_dataframe + with open(os.path.abspath(pheno_file), "r") as f: + return pd.read_csv(f) def load_group_participant_list(group_participant_list_file): - import os - import pandas as pd + import pandas as pd if not os.path.isfile(group_participant_list_file): - err = "\n\n[!] CPAC says: The group-level analysis subject list "\ - "provided does not exist!\nPath provided: %s\n\n" \ - % group_subject_list + err = ( + "\n\n[!] CPAC says: The group-level analysis subject list " + "provided does not exist!\nPath provided: %s\n\n" % group_subject_list + ) raise Exception(err) - - with open(group_participant_list_file,"r") as f: - + with open(group_participant_list_file, "r") as f: group_subs_dataframe = pd.read_csv(f) - if "participant" not in ga_sublist.columns: - err = "\n\n[!] CPAC says: Your group-level analysis subject "\ - "list CSV is missing a 'participant' column.\n\n" + err = ( + "\n\n[!] CPAC says: Your group-level analysis subject " + "list CSV is missing a 'participant' column.\n\n" + ) raise Exception(err) - return group_subs_dataframe -def process_pheno_file(pheno_file_dataframe, group_subs_dataframe, \ - participant_id_label): - +def process_pheno_file( + pheno_file_dataframe, group_subs_dataframe, participant_id_label +): # drops participants from the phenotype file if they are not in the group # analysis participant list # also handles sessions and series appropriately for repeated measures @@ -64,27 +57,29 @@ def process_pheno_file(pheno_file_dataframe, group_subs_dataframe, \ # one of the rows from the phenotype file, with the # format of {header: value, header: value, ..} - import os import pandas as pd - if not isinstance(pheno_file_dataframe, pd.DataFrame): - err = "\n\n[!] CPAC says: The phenotype information input should " \ - "be a Python Pandas dataframe object.\n\n" + err = ( + "\n\n[!] CPAC says: The phenotype information input should " + "be a Python Pandas dataframe object.\n\n" + ) raise Exception(err) if not isinstance(group_subs_dataframe, pd.DataFrame): - err = "\n\n[!] CPAC says: The group analysis participant list input "\ - "should be a Python Pandas dataframe object.\n\n" + err = ( + "\n\n[!] CPAC says: The group analysis participant list input " + "should be a Python Pandas dataframe object.\n\n" + ) raise Exception(err) if not isinstance(participant_id_label, str): - err = "\n\n[!] CPAC says: The participant ID label input should be " \ - "a string.\n\n" - + err = ( + "\n\n[!] CPAC says: The participant ID label input should be " + "a string.\n\n" + ) pheno = pheno_file_dataframe - pheno_file_rows = [] df_rows = [] @@ -103,60 +98,63 @@ def process_pheno_file(pheno_file_dataframe, group_subs_dataframe, \ series = list(group_subs_dataframe.series) series = [str(i) for i in series] - # use an integer for iteration because we're not sure if there will be # sessions and/or series - for i in range(0,len(subjects)): - + for i in range(0, len(subjects)): full_id = [] subject = subjects[i] full_id.append(subject) if sessions and series: - session = sessions[i] scan = series[i] full_id.append(session) full_id.append(scan) try: - row = pheno[(pheno[participant_id_label] == subject) & \ - (pheno.session == session) & \ - (pheno.series == scan)] + row = pheno[ + (pheno[participant_id_label] == subject) + & (pheno.session == session) + & (pheno.series == scan) + ] except: - row = pheno[(pheno[participant_id_label] == int(subject)) & \ - (pheno.session == session) & \ - (pheno.series == scan)] + row = pheno[ + (pheno[participant_id_label] == int(subject)) + & (pheno.session == session) + & (pheno.series == scan) + ] elif sessions: - session = sessions[i] full_id.append(session) try: - row = pheno[(pheno[participant_id_label] == subject) & \ - (pheno.session == session)] + row = pheno[ + (pheno[participant_id_label] == subject) + & (pheno.session == session) + ] except: - row = pheno[(pheno[participant_id_label] == int(subject)) & \ - (pheno.session == session)] - + row = pheno[ + (pheno[participant_id_label] == int(subject)) + & (pheno.session == session) + ] elif series: - scan = series[i] full_id.append(scan) try: - row = pheno[(pheno[participant_id_label] == subject) & \ - (pheno.series == scan)] + row = pheno[ + (pheno[participant_id_label] == subject) & (pheno.series == scan) + ] except: - row = pheno[(pheno[participant_id_label] == int(subject)) & \ - (pheno.series == scan)] - + row = pheno[ + (pheno[participant_id_label] == int(subject)) + & (pheno.series == scan) + ] else: - full_id.append(subject) try: @@ -164,30 +162,24 @@ def process_pheno_file(pheno_file_dataframe, group_subs_dataframe, \ except: row = pheno[(pheno[participant_id_label] == int(subject))] - if len(row) > 1: - - err = "\n\n[!] CPAC says: Multiple phenotype entries were " \ - "found for these criteria:\n\n%s\n\nPlease ensure " \ - "your group analysis participant list and phenotype " \ - "file are configured correctly.\n\n" % str(full_id) + err = ( + "\n\n[!] CPAC says: Multiple phenotype entries were " + "found for these criteria:\n\n%s\n\nPlease ensure " + "your group analysis participant list and phenotype " + "file are configured correctly.\n\n" % str(full_id) + ) raise Exception(err) elif len(row) == 1: - df_rows.append(row) - new_pheno_df = pd.concat(df_rows) - pheno_file_rows = new_pheno_df.to_dict("records") - - - return pheno_file_rows + return new_pheno_df.to_dict("records") def create_pheno_dict(pheno_file_rows, ev_selections, participant_id_label): - # creates the phenotype data dictionary in a format Patsy requires, # and also demeans the continuous EVs marked for demeaning @@ -204,28 +196,22 @@ def create_pheno_dict(pheno_file_rows, ev_selections, participant_id_label): # and each entry being a list of values, IN ORDER # data is also in Patsy-acceptable format - import os - import csv import numpy as np pheno_data_dict = {} for line in pheno_file_rows: - for val in line.values(): - # if there are any blank values in the pheno row, skip this # row. if not, continue on with the "else" clause if val == "": break else: - for key in line.keys(): - # if there are blank entries because of an empty row in # the CSV (such as ",,,,,"), move on to the next entry - #if len(line[key]) == 0: + # if len(line[key]) == 0: # continue if key not in pheno_data_dict.keys(): @@ -236,19 +222,17 @@ def create_pheno_dict(pheno_file_rows, ev_selections, participant_id_label): # form Patsy can understand regarding categoricals: # example: { ADHD: ['adhd1', 'adhd1', 'adhd0'] } # instead of just [1, 1, 0], etc. - if 'categorical' in ev_selections.keys(): - if key in ev_selections['categorical']: + if "categorical" in ev_selections.keys(): + if key in ev_selections["categorical"]: pheno_data_dict[key].append(key + str(line[key])) - elif (key == subject_id_label) or (key == "session") or \ - (key == "series"): + elif key in (subject_id_label, "session", "series"): pheno_data_dict[key].append(line[key]) else: pheno_data_dict[key].append(float(line[key])) - elif (key == subject_id_label) or (key == "session") or \ - (key == "series"): + elif key in (subject_id_label, "session", "series"): pheno_data_dict[key].append(line[key]) else: @@ -257,11 +241,9 @@ def create_pheno_dict(pheno_file_rows, ev_selections, participant_id_label): # this needs to run after each list in each key has been fully # populated above for key in pheno_data_dict.keys(): - # demean the EVs marked for demeaning - if 'demean' in ev_selections.keys(): - if key in ev_selections['demean']: - + if "demean" in ev_selections.keys(): + if key in ev_selections["demean"]: new_demeaned_evs = [] mean_evs = 0.0 @@ -285,17 +267,14 @@ def create_pheno_dict(pheno_file_rows, ev_selections, participant_id_label): # converts non-categorical EV lists into NumPy arrays # so that Patsy may read them in properly - if 'categorical' in ev_selections.keys(): - if key not in ev_selections['categorical']: - + if "categorical" in ev_selections.keys(): + if key not in ev_selections["categorical"]: pheno_data_dict[key] = np.array(pheno_data_dict[key]) - return pheno_data_dict def get_measure_dict(param_file): - # load the CPAC-generated power parameters file and parse it # input @@ -309,29 +288,28 @@ def get_measure_dict(param_file): # ...} import os + import pandas as pd if not os.path.isfile(param_file): - err = "\n\n[!] CPAC says: You've included a motion parameter in " \ - "your group-level analysis model design formula, but " \ - "there is no motion parameters file available.\n\n" + err = ( + "\n\n[!] CPAC says: You've included a motion parameter in " + "your group-level analysis model design formula, but " + "there is no motion parameters file available.\n\n" + ) raise Exception(err) - with open(param_file,"r") as f: + with open(param_file, "r") as f: motion_params = pd.read_csv(f, index_col=False) - - measures = ['MeanFD_Power', 'MeanFD_Jenkinson', 'MeanDVARS'] + measures = ["MeanFD_Power", "MeanFD_Jenkinson", "MeanDVARS"] measure_dict = {} - for m in measures: - measure_map = {} if m in motion_params.columns: - part_ids = list(motion_params["Subject"]) part_ids = [str(i) for i in part_ids] @@ -341,27 +319,24 @@ def get_measure_dict(param_file): measure_vals = list(motion_params[m]) measure_vals = [float(i) for i in measure_vals] - for part_id, scan_id, measure_val in \ - zip(part_ids, scan_ids, measure_vals): - - measure_map[(part_id,scan_id)] = measure_val + for part_id, scan_id, measure_val in zip(part_ids, scan_ids, measure_vals): + measure_map[(part_id, scan_id)] = measure_val measure_dict[m] = measure_map - return measure_dict def get_custom_roi_info(roi_means_dict): - # check - if roi_means_dict == None: - err_string = "\n\n[!] CPAC says: The custom ROI means were not " \ - "calculated properly during the group analysis " \ - "model generation.\n\n" + if roi_means_dict is None: + err_string = ( + "\n\n[!] CPAC says: The custom ROI means were not " + "calculated properly during the group analysis " + "model generation.\n\n" + ) raise Exception(err_string) - roi_num = len(roi_means_dict.values()[0]) # this will be a dictionary matching ROI regressor header labels with @@ -371,55 +346,40 @@ def get_custom_roi_info(roi_means_dict): # split the roi_means_dict from { subID: [mean1,mean2,mean3,..], ..} # to three dictionaries of { subID: mean1, .. }, { subID: mean2, .. }, # and so on - for num in range(0,roi_num): - - label = "Custom_ROI_Mean_%d" % int(num+1) + for num in range(0, roi_num): + label = "Custom_ROI_Mean_%d" % int(num + 1) temp_roi_dict = {} for key in roi_means_dict.keys(): - - temp_roi_dict[key] = roi_means_dict[key][num-1] + temp_roi_dict[key] = roi_means_dict[key][num - 1] roi_dict_dict[label] = temp_roi_dict - return roi_dict_dict -def model_group_var_separately(grouping_var, formula, pheno_data_dict, \ - ev_selections, coding_scheme): - - if grouping_var == None or grouping_var not in formula: - print('\n\n[!] CPAC says: Model group variances separately is ' \ - 'enabled, but the grouping variable set is either set to ' \ - 'None, or was not included in the model as one of the ' \ - 'EVs.\n') - print('Design formula: ', formula) - print('Grouping variable: ', grouping_var, '\n\n') +def model_group_var_separately( + grouping_var, formula, pheno_data_dict, ev_selections, coding_scheme +): + if grouping_var is None or grouping_var not in formula: raise Exception - # do this a little early for the grouping variable so that it doesn't # get in the way of doing this for the other EVs once they have the # grouping variable in their names - if 'categorical' in ev_selections.keys(): - for EV_name in ev_selections['categorical']: - - if EV_name == grouping_var: - - if coding_scheme == 'Treatment': - formula = formula.replace(EV_name, 'C(' + EV_name + ')') - elif coding_scheme == 'Sum': - formula = formula.replace(EV_name, 'C(' + EV_name + \ - ', Sum)') - + if "categorical" in ev_selections.keys(): + for EV_name in ev_selections["categorical"]: + if EV_name == grouping_var: + if coding_scheme == "Treatment": + formula = formula.replace(EV_name, "C(" + EV_name + ")") + elif coding_scheme == "Sum": + formula = formula.replace(EV_name, "C(" + EV_name + ", Sum)") groupvar_levels = [] grouping_var_id_dict = {} idx = 0 for cat_ev_value in pheno_data_dict[grouping_var]: - # here, each "cat_ev_value" will be one of the Patsy-format values # of the categorical EV that the user has selected as the grouping # variable, i.e. "sex1, sex1, sex0, sex1", etc.. @@ -445,17 +405,14 @@ def model_group_var_separately(grouping_var, formula, pheno_data_dict, \ split_EVs = {} for key in pheno_data_dict.keys(): - # here, "key" is the name of each EV from the phenotype file, as # they are labeled in the phenotype file (not Patsy format) if (key in formula) and (key != grouping_var): - # for the formula edit new_key_string = "" for level in groupvar_levels: - # for the new split EV label groupvar_with_level = str(grouping_var) + str(level) new_key = key + "__" + groupvar_with_level @@ -472,15 +429,13 @@ def model_group_var_separately(grouping_var, formula, pheno_data_dict, \ if key in ev_selections["categorical"]: ev_selections["categorical"].append(new_key) - for val, groupvar_val in zip(pheno_data_dict[key], \ - pheno_data_dict[grouping_var]): - + for val, groupvar_val in zip( + pheno_data_dict[key], pheno_data_dict[grouping_var] + ): if groupvar_with_level == groupvar_val: - split_EVs[new_key].append(val) else: - split_EVs[new_key].append(0) del pheno_data_dict[key] @@ -499,16 +454,13 @@ def model_group_var_separately(grouping_var, formula, pheno_data_dict, \ # properly when generating the design matrix (this goes into the # .mat file) - if 'categorical' in ev_selections.keys(): - for EV_name in ev_selections['categorical']: - + if "categorical" in ev_selections.keys(): + for EV_name in ev_selections["categorical"]: if EV_name != grouping_var: - - if coding_scheme == 'Treatment': - formula = formula.replace(EV_name, 'C(' + EV_name + ')') - elif coding_scheme == 'Sum': - formula = formula.replace(EV_name, 'C(' + EV_name + \ - ', Sum)') + if coding_scheme == "Treatment": + formula = formula.replace(EV_name, "C(" + EV_name + ")") + elif coding_scheme == "Sum": + formula = formula.replace(EV_name, "C(" + EV_name + ", Sum)") # remove intercept when modeling group variances separately formula = formula + " - 1" @@ -517,41 +469,27 @@ def model_group_var_separately(grouping_var, formula, pheno_data_dict, \ def check_multicollinearity(matrix): - import numpy as np - print("\nChecking for multicollinearity in the model..") - U, s, V = np.linalg.svd(matrix) max_singular = np.max(s) min_singular = np.min(s) - print("Max singular: ", max_singular) - print("Min singular: ", min_singular) - print("Rank: ", np.linalg.matrix_rank(matrix), "\n") - if min_singular == 0: - - print('[!] CPAC warns: Detected multicollinearity in the ' \ - 'computed group-level analysis model. Please double-' \ - 'check your model design.\n\n') + pass else: - - condition_number = float(max_singular)/float(min_singular) - print("Condition number: %f\n\n" % condition_number) + condition_number = float(max_singular) / float(min_singular) if condition_number > 30: - - print('[!] CPAC warns: Detected multicollinearity in the ' \ - 'computed group-level analysis model. Please double-' \ - 'check your model design.\n\n') + pass -def write_mat_file(design_matrix, output_dir, model_name, \ - depatsified_EV_names, current_output=None): - +def write_mat_file( + design_matrix, output_dir, model_name, depatsified_EV_names, current_output=None +): import os + import numpy as np dimx = None @@ -563,15 +501,12 @@ def write_mat_file(design_matrix, output_dir, model_name, \ else: dimx, dimy = design_matrix.shape - - ppstring = '/PPheights' + ppstring = "/PPheights" for i in range(0, dimy): + ppstring += "\t" + "%1.5e" % (1.0) - ppstring += '\t' + '%1.5e' %(1.0) - - ppstring += '\n' - + ppstring += "\n" filename = model_name + ".mat" @@ -579,29 +514,29 @@ def write_mat_file(design_matrix, output_dir, model_name, \ if not os.path.exists(output_dir): os.makedirs(output_dir) - with open(out_file, 'wt') as f: - - print('/NumWaves\t%d' %dimy, file=f) - print('/NumPoints\t%d' %dimx, file=f) + with open(out_file, "wt") as f: + print("/NumWaves\t%d" % dimy, file=f) + print("/NumPoints\t%d" % dimx, file=f) print(ppstring, file=f) # print labels for the columns - mainly for double-checking your model - col_string = '\n' + col_string = "\n" for col in depatsified_EV_names: - col_string = col_string + col + '\t' - - print(col_string, '\n', file=f) + col_string = col_string + col + "\t" - print('/Matrix', file=f) + print(col_string, "\n", file=f) - np.savetxt(f, design_matrix, fmt='%1.5e', delimiter='\t') + print("/Matrix", file=f) + np.savetxt(f, design_matrix, fmt="%1.5e", delimiter="\t") -def create_grp_file(design_matrix, grouping_var_id_dict, output_dir, \ - model_name, current_output=None): +def create_grp_file( + design_matrix, grouping_var_id_dict, output_dir, model_name, current_output=None +): import os + import numpy as np dimx = None @@ -615,10 +550,9 @@ def create_grp_file(design_matrix, grouping_var_id_dict, output_dir, \ design_matrix_ones = np.ones(dimx) - if not (grouping_var_id_dict == None): + if grouping_var_id_dict is not None: i = 1 for key in sorted(grouping_var_id_dict.keys()): - for index in grouping_var_id_dict[key]: design_matrix_ones[index] = i @@ -631,20 +565,26 @@ def create_grp_file(design_matrix, grouping_var_id_dict, output_dir, \ os.makedirs(output_dir) with open(out_file, "wt") as f: - - print('/NumWaves\t1', file=f) - print('/NumPoints\t%d\n' %dimx, file=f) - print('/Matrix', file=f) - np.savetxt(f, design_matrix_ones, fmt='%d', delimiter='\t') - - -def create_design_matrix(pheno_file, ev_selections, formula, \ - subject_id_label, sub_list=None, \ - coding_scheme="Treatment", grouping_var=None, \ - new_regressor_dict=None, roi_means_dict=None, \ - output_dir=None, model_name="design", \ - current_output=None): - + print("/NumWaves\t1", file=f) + print("/NumPoints\t%d\n" % dimx, file=f) + print("/Matrix", file=f) + np.savetxt(f, design_matrix_ones, fmt="%d", delimiter="\t") + + +def create_design_matrix( + pheno_file, + ev_selections, + formula, + subject_id_label, + sub_list=None, + coding_scheme="Treatment", + grouping_var=None, + new_regressor_dict=None, + roi_means_dict=None, + output_dir=None, + model_name="design", + current_output=None, +): # this should allow the user to easily create a FLAMEO-formatted .mat file # and .grp file from the command line or from within CPAC @@ -703,32 +643,30 @@ def create_design_matrix(pheno_file, ev_selections, formula, \ # dmatrix: a Patsy object of the design matrix # depatsified_EV_names: a list of the column names of the design matrix - import os - import patsy + import numpy as np + import patsy # if running this script alone outside of CPAC - if output_dir == None: + if output_dir is None: output_dir = os.getcwd() - # let's process the phenotype file data and drop rows (participants) if # they are not listed in the participant list pheno_file_df = load_pheno_file(pheno_file) participant_list_df = load_group_participant_list(sub_list) - pheno_file_rows = process_pheno_file(pheno_file_df, participant_list_df, \ - subject_id_label) - + pheno_file_rows = process_pheno_file( + pheno_file_df, participant_list_df, subject_id_label + ) # get number of subjects that have the derivative for this current model # (basically, the amount of time points, which must be greater than the # number of EVs) num_subjects = len(participant_list_df) - # for repeated measures if "session" in participant_list_df.columns: ev_selections["categorical"].append("session") @@ -736,26 +674,21 @@ def create_design_matrix(pheno_file, ev_selections, formula, \ if "series" in participant_list_df.columns: ev_selections["categorical"].append("series") - # start adding additionally created EVs if new_regressor_dict: - for measure in new_regressor_dict.keys(): - - if (measure in formula): - + if measure in formula: measure_dict = new_regressor_dict[measure] for pheno_row_dict in pheno_file_rows: - participant_id = pheno_row_dict[subject_id_label] - if ("session" in pheno_row_dict.keys()) and \ - ("series" in pheno_row_dict.keys()): + if ("session" in pheno_row_dict.keys()) and ( + "series" in pheno_row_dict.keys() + ): session_id = pheno_row_dict["session"] series_id = pheno_row_dict["series"] - participant_tuple = \ - (participant_id, session_id, series_id) + participant_tuple = (participant_id, session_id, series_id) elif "session" in pheno_row_dict.keys(): session_id = pheno_row_dict["session"] @@ -766,22 +699,21 @@ def create_design_matrix(pheno_file, ev_selections, formula, \ participant_tuple = (participant_id, series_id) else: - participant_tuple = (participant_id) + participant_tuple = participant_id pheno_row_dict[measure] = measure_dict[participant_tuple] - ev_selections["demean"].append(measure) - if "Custom_ROI_Mean" in formula: - # include the means of the specified ROIs as regressors - if roi_means_dict == None: - err = "\n\n[!] You included 'Custom_ROI_Mean' in your model " \ - "design, but there are no mean of ROI values provided." \ - "\n\n" + if roi_means_dict is None: + err = ( + "\n\n[!] You included 'Custom_ROI_Mean' in your model " + "design, but there are no mean of ROI values provided." + "\n\n" + ) raise Exception(err) # roi_dict_dict is a dictionary of dictionaries, with each dictionary @@ -796,19 +728,17 @@ def create_design_matrix(pheno_file, ev_selections, formula, \ add_formula_string = "" for roi_column in roi_dict_dict.keys(): - roi_dict = roi_dict_dict[roi_column] for pheno_row_dict in pheno_file_rows: - participant_id = pheno_row_dict[subject_id_label] - if ("session" in pheno_row_dict.keys()) and \ - ("series" in pheno_row_dict.keys()): + if ("session" in pheno_row_dict.keys()) and ( + "series" in pheno_row_dict.keys() + ): session_id = pheno_row_dict["session"] series_id = pheno_row_dict["series"] - participant_tuple = \ - (participant_id, session_id, series_id) + participant_tuple = (participant_id, session_id, series_id) elif "session" in pheno_row_dict.keys(): session_id = pheno_row_dict["session"] @@ -819,11 +749,10 @@ def create_design_matrix(pheno_file, ev_selections, formula, \ participant_tuple = (participant_id, series_id) else: - participant_tuple = (participant_id) + participant_tuple = participant_id pheno_row_dict[roi_column] = roi_dict[participant_tuple] - ev_selections["demean"].append(roi_column) # create a string of all the new custom ROI regressor column names @@ -835,13 +764,10 @@ def create_design_matrix(pheno_file, ev_selections, formula, \ else: add_formula_string = add_formula_string + " + " + roi_column - # a regressor column of ROI means for each custom-specified ROI has # now been added to the model with appropriate column labels - formula = formula.replace("Custom_ROI_Mean",add_formula_string) - - + formula = formula.replace("Custom_ROI_Mean", add_formula_string) # return the data from the phenotype file processed properly for Patsy # and load it into 'pheno_data_dict' @@ -852,88 +778,68 @@ def create_design_matrix(pheno_file, ev_selections, formula, \ # - EVs to be demeaned are already demeaned # - numerical EVs (non-categorical) are in a list which # have been converted into a NumPy array - pheno_data_dict = create_pheno_dict(pheno_file_rows, ev_selections, \ - subject_id_label) - - + pheno_data_dict = create_pheno_dict( + pheno_file_rows, ev_selections, subject_id_label + ) # handle modeling group variances separately (if enabled), then edit the # formula to be in Patsy language - if grouping_var != None: - - pheno_data_dict, formula, grouping_var_id_dict = \ - model_group_var_separately(grouping_var, \ - formula, pheno_data_dict, \ - ev_selections, coding_scheme) + if grouping_var is not None: + pheno_data_dict, formula, grouping_var_id_dict = model_group_var_separately( + grouping_var, formula, pheno_data_dict, ev_selections, coding_scheme + ) else: - grouping_var_id_dict = None - if 'categorical' in ev_selections.keys(): - for EV_name in ev_selections['categorical']: - - if coding_scheme == 'Treatment': - formula = formula.replace(EV_name, 'C(' + EV_name + ')') - elif coding_scheme == 'Sum': - formula = formula.replace(EV_name, 'C(' + EV_name + \ - ', Sum)') - - + if "categorical" in ev_selections.keys(): + for EV_name in ev_selections["categorical"]: + if coding_scheme == "Treatment": + formula = formula.replace(EV_name, "C(" + EV_name + ")") + elif coding_scheme == "Sum": + formula = formula.replace(EV_name, "C(" + EV_name + ", Sum)") # create the Patsy design matrix! try: - - dmatrix = patsy.dmatrix(formula, pheno_data_dict, NA_action='raise') + dmatrix = patsy.dmatrix(formula, pheno_data_dict, NA_action="raise") except: - print('\n\n[!] CPAC says: Design matrix creation wasn\'t ' \ - 'successful - do the terms in your formula correctly ' \ - 'correspond to the EVs listed in your phenotype file?\n') - print('Phenotype file provided: ') - print(pheno_file, '\n') - print("Phenotypic data columns (regressors): ", list(pheno_data_dict.keys())) - print("Formula: %s\n\n" % formula) raise Exception - - # check the model for multicollinearity - Patsy takes care of this, but # just in case check_multicollinearity(np.array(dmatrix)) - # prepare for final stages design_matrix = np.array(dmatrix, dtype=np.float16) column_names = dmatrix.design_info.column_names - # check to make sure there are more time points than EVs! if len(column_names) >= num_subjects: - err = "\n\n[!] CPAC says: There are more EVs than there are " \ - "subjects currently included in the model for %s. There must " \ - "be more subjects than EVs in the design.\n\nNumber of " \ - "subjects: %d\nNumber of EVs: %d\n\nNote: An 'Intercept' " \ - "column gets added to the design as an EV, so there will be " \ - "one more EV than you may have specified in your design. In " \ - "addition, if you specified to model group variances " \ - "separately, an Intercept column will not be included, but " \ - "the amount of EVs can nearly double once they are split " \ - "along the grouping variable.\n\n" \ - "If the number of subjects is lower than the number of " \ - "subjects in your group analysis subject list, this may be " \ - "because not every subject in the subject list has an output " \ - "for %s in the individual-level analysis output directory.\n\n"\ - % (current_output, num_subjects, len(column_names), \ - current_output) + err = ( + "\n\n[!] CPAC says: There are more EVs than there are " + "subjects currently included in the model for %s. There must " + "be more subjects than EVs in the design.\n\nNumber of " + "subjects: %d\nNumber of EVs: %d\n\nNote: An 'Intercept' " + "column gets added to the design as an EV, so there will be " + "one more EV than you may have specified in your design. In " + "addition, if you specified to model group variances " + "separately, an Intercept column will not be included, but " + "the amount of EVs can nearly double once they are split " + "along the grouping variable.\n\n" + "If the number of subjects is lower than the number of " + "subjects in your group analysis subject list, this may be " + "because not every subject in the subject list has an output " + "for %s in the individual-level analysis output directory.\n\n" + % (current_output, num_subjects, len(column_names), current_output) + ) raise Exception(err) - # remove the header formatting Patsy creates for categorical variables # because we are going to use depatsified_EV_names in the "Available EVs # for Contrasts" list on the next page, and also to test user-made custom @@ -942,7 +848,6 @@ def create_design_matrix(pheno_file, ev_selections, formula, \ depatsified_EV_names = [] for column in column_names: - # if using Sum encoding, a column name may look like this: # C(adhd, Sum)[S.adhd0] @@ -951,38 +856,34 @@ def create_design_matrix(pheno_file, ev_selections, formula, \ column_string = column - string_for_removal = '' + string_for_removal = "" for char in column_string: - string_for_removal = string_for_removal + char - if char == '.': - column_string = column_string.replace(string_for_removal, '') - string_for_removal = '' + if char == ".": + column_string = column_string.replace(string_for_removal, "") + string_for_removal = "" - column_string = column_string.replace(']', '') + column_string = column_string.replace("]", "") depatsified_EV_names.append(column_string) - # write the .mat file finally - write_mat_file(design_matrix, output_dir, model_name, \ - depatsified_EV_names, current_output) - + write_mat_file( + design_matrix, output_dir, model_name, depatsified_EV_names, current_output + ) # write the .grp file also - create_grp_file(design_matrix, grouping_var_id_dict, output_dir, \ - model_name, current_output) - + create_grp_file( + design_matrix, grouping_var_id_dict, output_dir, model_name, current_output + ) # return the PATSY OBJECT of dmatrix, not the Numpy array "design_matrix" return dmatrix, depatsified_EV_names - def positive(dmat, a, coding, group_sep, grouping_var): - import numpy as np # this is also where the "Intercept" column gets introduced into @@ -991,8 +892,7 @@ def positive(dmat, a, coding, group_sep, grouping_var): evs = dmat.design_info.column_name_indexes con = np.zeros(dmat.shape[1]) - if group_sep == True: - + if group_sep is True: if "__" in a and grouping_var in a: ev_desc = a.split("__") @@ -1008,14 +908,13 @@ def positive(dmat, a, coding, group_sep, grouping_var): else: # it is a dropped term so make all other terms in that # category at -1 - term = a.split('[')[0] + term = a.split("[")[0] for ev in evs: if ev.startswith(term): - con[evs[ev]]= -1 + con[evs[ev]] = -1 elif len(a.split(grouping_var)) > 2: - # this is if the current parsed contrast is the actual # grouping variable, as the Patsified name will have the # variable's name string in it twice @@ -1027,25 +926,24 @@ def positive(dmat, a, coding, group_sep, grouping_var): else: # it is a dropped term so make all other terms in that # category at -1 - term = a.split('[')[0] + term = a.split("[")[0] for ev in evs: if ev.startswith(term): - con[evs[ev]]= -1 + con[evs[ev]] = -1 # else not modeling group variances separately else: - if a in evs: con[evs[a]] = 1 else: # it is a dropped term so make all other terms in that category # at -1 - term = a.split('[')[0] + term = a.split("[")[0] for ev in evs: if ev.startswith(term): - con[evs[ev]]= -1 + con[evs[ev]] = -1 if coding == "Treatment": # make Intercept 0 @@ -1060,80 +958,68 @@ def positive(dmat, a, coding, group_sep, grouping_var): def greater_than(dmat, a, b, coding, group_sep, grouping_var): c1 = positive(dmat, a, coding, group_sep, grouping_var) c2 = positive(dmat, b, coding, group_sep, grouping_var) - return c1-c2 + return c1 - c2 def negative(dmat, a, coding, group_sep, grouping_var): - con = 0-positive(dmat, a, coding, group_sep, grouping_var) - return con + return 0 - positive(dmat, a, coding, group_sep, grouping_var) def create_dummy_string(length): ppstring = "" for i in range(0, length): - ppstring += '\t' + '%1.5e' %(1.0) - ppstring += '\n' + ppstring += "\t" + "%1.5e" % (1.0) + ppstring += "\n" return ppstring def create_con_file(con_dict, col_names, file_name, current_output, out_dir): - import os - print("col names: ") - print(col_names) - - with open(os.path.join(out_dir, file_name) + ".con",'w+') as f: - + with open(os.path.join(out_dir, file_name) + ".con", "w+") as f: # write header num = 1 for key in con_dict: - f.write("/ContrastName%s\t%s\n" %(num,key)) + f.write("/ContrastName%s\t%s\n" % (num, key)) num += 1 - f.write("/NumWaves\t%d\n" %len(con_dict[key])) - f.write("/NumContrasts\t%d\n" %len(con_dict)) - f.write("/PPheights%s" %create_dummy_string(len(con_dict[key]))) - f.write("/RequiredEffect%s" %create_dummy_string(len(con_dict[key]))) + f.write("/NumWaves\t%d\n" % len(con_dict[key])) + f.write("/NumContrasts\t%d\n" % len(con_dict)) + f.write("/PPheights%s" % create_dummy_string(len(con_dict[key]))) + f.write("/RequiredEffect%s" % create_dummy_string(len(con_dict[key]))) f.write("\n\n") # print labels for the columns - mainly for double-checking your # model - col_string = '\n' + col_string = "\n" for col in col_names: - col_string = col_string + col + '\t' - print(col_string, '\n', file=f) + col_string = col_string + col + "\t" + print(col_string, "\n", file=f) # write data f.write("/Matrix\n") for key in con_dict: for v in con_dict[key]: - f.write("%1.5e\t" %v) + f.write("%1.5e\t" % v) f.write("\n") -def create_fts_file(ftest_list, con_dict, model_name, current_output, - out_dir): - +def create_fts_file(ftest_list, con_dict, model_name, current_output, out_dir): import os + import numpy as np try: - print("\nFound f-tests in your model, writing f-tests file " \ - "(.fts)..\n") - - with open(os.path.join(out_dir, model_name + '.fts'), 'w') as f: - - print('/NumWaves\t', len(con_dict), file=f) - print('/NumContrasts\t', len(ftest_list), file=f) + with open(os.path.join(out_dir, model_name + ".fts"), "w") as f: + print("/NumWaves\t", len(con_dict), file=f) + print("/NumContrasts\t", len(ftest_list), file=f) # process each f-test ftst = [] for ftest_string in ftest_list: - ftest_vector = [] cons_in_ftest = ftest_string.split(",") @@ -1150,41 +1036,48 @@ def create_fts_file(ftest_list, con_dict, model_name, current_output, # print labels for the columns - mainly for double-checking your # model - col_string = '\n' + col_string = "\n" for con in con_dict.keys(): - col_string = col_string + con + '\t' - print(col_string, '\n', file=f) + col_string = col_string + con + "\t" + print(col_string, "\n", file=f) - print('/Matrix', file=f) + print("/Matrix", file=f) for i in range(fts_n.shape[0]): - print(' '.join(fts_n[i].astype('str')), file=f) + print(" ".join(fts_n[i].astype("str")), file=f) except Exception as e: + filepath = os.path.join( + out_dir, "model_files", current_output, model_name + ".fts" + ) - filepath = os.path.join(out_dir, "model_files", current_output, \ - model_name + '.fts') - - errmsg = "\n\n[!] CPAC says: Could not create .fts file for " \ - "FLAMEO or write it to disk.\nAttempted filepath: %s\n" \ - "Error details: %s\n\n" % (filepath, e) + errmsg = ( + "\n\n[!] CPAC says: Could not create .fts file for " + "FLAMEO or write it to disk.\nAttempted filepath: %s\n" + "Error details: %s\n\n" % (filepath, e) + ) raise Exception(errmsg) -def create_con_ftst_file(con_file, model_name, current_output, output_dir, - column_names, coding_scheme, group_sep): - """ - Create the contrasts and fts file - """ - +def create_con_ftst_file( + con_file, + model_name, + current_output, + output_dir, + column_names, + coding_scheme, + group_sep, +): + """Create the contrasts and fts file.""" import os + import numpy as np with open(con_file, "r") as f: evs = f.readline() - evs = evs.rstrip('\r\n').split(',') + evs = evs.rstrip("\r\n").split(",") count_ftests = 0 # TODO: this needs to be re-visited, but I think this was originally added @@ -1193,11 +1086,9 @@ def create_con_ftst_file(con_file, model_name, current_output, output_dir, # TODO: would have the Intercept added to it? but what if it wasn't? # TODO: comment out for now... but test # remove "Contrasts" label and replace it with "Intercept" - #evs[0] = "Intercept" + # evs[0] = "Intercept" fTest = False - print("evs: ") - print(evs) for ev in evs: if "f_test" in ev: count_ftests += 1 @@ -1206,15 +1097,13 @@ def create_con_ftst_file(con_file, model_name, current_output, output_dir, fTest = True try: - data = np.genfromtxt(con_file, names=True, delimiter=',', dtype=None) + data = np.genfromtxt(con_file, names=True, delimiter=",", dtype=None) except: - print("Error: Could not successfully read in contrast file: ",con_file) raise Exception lst = data.tolist() - ftst = [] fts_columns = [] contrasts = [] contrast_names = [] @@ -1231,9 +1120,9 @@ def create_con_ftst_file(con_file, model_name, current_output, output_dir, # create a list of integers that is the vector for the contrast # ex. [0, 1, 1, 0, ..] - con_vector = list(tp)[1:(length-count_ftests)] + con_vector = list(tp)[1 : (length - count_ftests)] - fts_vector = list(tp)[(length-count_ftests):length] + fts_vector = list(tp)[(length - count_ftests) : length] fts_columns.append(fts_vector) # TODO: see note about Intercept above @@ -1257,93 +1146,100 @@ def create_con_ftst_file(con_file, model_name, current_output, output_dir, # if there are f-tests, create the array for them if fTest: if len(contrast_names) < 2: - errmsg = "\n\n[!] CPAC says: Not enough contrasts for running " \ - "f-tests.\nTip: Do you have only one contrast in your " \ - "contrasts file? f-tests require more than one contrast.\n"\ - "Either remove the f-tests or include more contrasts.\n\n" + errmsg = ( + "\n\n[!] CPAC says: Not enough contrasts for running " + "f-tests.\nTip: Do you have only one contrast in your " + "contrasts file? f-tests require more than one contrast.\n" + "Either remove the f-tests or include more contrasts.\n\n" + ) raise Exception(errmsg) fts_n = fts_columns.T if len(column_names) != (num_EVs_in_con_file): - err_string = "\n\n[!] CPAC says: The number of EVs in your model " \ - "design matrix (found in the %s.mat file) does not " \ - "match the number of EVs (columns) in your custom " \ - "contrasts matrix CSV file.\n\nCustom contrasts matrix "\ - "file: %s\n\nNumber of EVs in design matrix: %d\n" \ - "Number of EVs in contrasts file: %d\n\nThe column " \ - "labels in the design matrix should match those in " \ - "your contrasts .CSV file.\nColumn labels in design " \ - "matrix:\n%s" % (model_name, con_file, \ - len(column_names), num_EVs_in_con_file, \ - str(column_names)) + err_string = ( + "\n\n[!] CPAC says: The number of EVs in your model " + "design matrix (found in the %s.mat file) does not " + "match the number of EVs (columns) in your custom " + "contrasts matrix CSV file.\n\nCustom contrasts matrix " + "file: %s\n\nNumber of EVs in design matrix: %d\n" + "Number of EVs in contrasts file: %d\n\nThe column " + "labels in the design matrix should match those in " + "your contrasts .CSV file.\nColumn labels in design " + "matrix:\n%s" + % ( + model_name, + con_file, + len(column_names), + num_EVs_in_con_file, + str(column_names), + ) + ) raise Exception(err_string) for design_mat_col, con_csv_col in zip(column_names, evs[1:]): if con_csv_col not in design_mat_col: - errmsg = "\n\n[!] CPAC says: The names of the EVs in your " \ - "custom contrasts .csv file do not match the names or " \ - "order of the EVs in the design matrix. Please make " \ - "sure these are consistent.\nDesign matrix EV columns: "\ - "%s\nYour contrasts matrix columns: %s\n\n" \ - % (column_names, evs[1:]) + errmsg = ( + "\n\n[!] CPAC says: The names of the EVs in your " + "custom contrasts .csv file do not match the names or " + "order of the EVs in the design matrix. Please make " + "sure these are consistent.\nDesign matrix EV columns: " + "%s\nYour contrasts matrix columns: %s\n\n" % (column_names, evs[1:]) + ) raise Exception(errmsg) - out_dir = os.path.join(output_dir, model_name + '.con') + out_dir = os.path.join(output_dir, model_name + ".con") - with open(out_dir,"wt") as f: + with open(out_dir, "wt") as f: idx = 1 - pp_str = '/PPheights' - re_str = '/RequiredEffect' + pp_str = "/PPheights" + re_str = "/RequiredEffect" for name in contrast_names: - - print('/ContrastName%d' %idx, '\t', name, file=f) - pp_str += '\t%1.5e' %(1) - re_str += '\t%1.5e' %(1) + print("/ContrastName%d" % idx, "\t", name, file=f) + pp_str += "\t%1.5e" % (1) + re_str += "\t%1.5e" % (1) idx += 1 - print('/NumWaves\t', (contrasts.shape)[1], file=f) - print('/NumContrasts\t', (contrasts.shape)[0], file=f) + print("/NumWaves\t", (contrasts.shape)[1], file=f) + print("/NumContrasts\t", (contrasts.shape)[0], file=f) print(pp_str, file=f) - print(re_str + '\n', file=f) + print(re_str + "\n", file=f) # print labels for the columns - mainly for double-checking your model - col_string = '\n' + col_string = "\n" for ev in evs: - col_string = col_string + ev + '\t' - print(col_string, '\n', file=f) + col_string = col_string + ev + "\t" + print(col_string, "\n", file=f) - print('/Matrix', file=f) - np.savetxt(f, contrasts, fmt='%1.5e', delimiter='\t') + print("/Matrix", file=f) + np.savetxt(f, contrasts, fmt="%1.5e", delimiter="\t") if fTest: - print("\nFound f-tests in your model, writing f-tests file (.fts)..\n") + ftest_out_dir = os.path.join(output_dir, model_name + ".fts") - ftest_out_dir = os.path.join(output_dir, model_name + '.fts') - - with open(ftest_out_dir,"wt") as f: - print('/NumWaves\t', (contrasts.shape)[0], file=f) - print('/NumContrasts\t', count_ftests, file=f) + with open(ftest_out_dir, "wt") as f: + print("/NumWaves\t", (contrasts.shape)[0], file=f) + print("/NumContrasts\t", count_ftests, file=f) # print labels for the columns - mainly for double-checking your # model - col_string = '\n' + col_string = "\n" for con in contrast_names: - col_string = col_string + con + '\t' - print(col_string, '\n', file=f) + col_string = col_string + con + "\t" + print(col_string, "\n", file=f) - print('/Matrix', file=f) + print("/Matrix", file=f) for i in range(fts_n.shape[0]): - print(' '.join(fts_n[i].astype('str')), file=f) - + print(" ".join(fts_n[i].astype("str")), file=f) -def process_contrast(parsed_contrast, operator, ev_selections, group_sep, \ - grouping_var, coding_scheme): +def process_contrast( + parsed_contrast, operator, ev_selections, group_sep, grouping_var, coding_scheme +): # take the contrast strings and process them appropriately # extract the two separate contrasts (if there are two), and then # identify which are categorical - adapting the string if so @@ -1352,32 +1248,27 @@ def process_contrast(parsed_contrast, operator, ev_selections, group_sep, \ EVs_in_contrast = parsed_contrast.split(operator) - if '' in EVs_in_contrast: - EVs_in_contrast.remove('') - + if "" in EVs_in_contrast: + EVs_in_contrast.remove("") for EV in EVs_in_contrast: - skip = 0 # they need to be put back into Patsy formatted header titles # because the dmatrix gets passed into the function that writes # out the contrast matrix - if 'categorical' in ev_selections.keys(): - for cat_EV in ev_selections['categorical']: - + if "categorical" in ev_selections.keys(): + for cat_EV in ev_selections["categorical"]: # second half of this if clause is in case group variances # are being modeled separately, and we don't want the EV # that is the grouping variable (which is now present in # other EV names) to confound this operation - if group_sep == True: + if group_sep is True: gpvar = grouping_var else: gpvar = "..." - if (cat_EV in EV) and not (gpvar in EV and \ - "__" in EV): - + if (cat_EV in EV) and not (gpvar in EV and "__" in EV): # handle interactions if ":" in EV: temp_split_EV = EV.split(":") @@ -1387,56 +1278,57 @@ def process_contrast(parsed_contrast, operator, ev_selections, group_sep, \ else: current_EV = EV - if coding_scheme == 'Treatment': - cat_EV_contrast = EV.replace(EV, 'C(' + cat_EV + \ - ')[T.' + current_EV+\ - ']') + if coding_scheme == "Treatment": + cat_EV_contrast = EV.replace( + EV, "C(" + cat_EV + ")[T." + current_EV + "]" + ) - elif coding_scheme == 'Sum': - cat_EV_contrast = EV.replace(EV, 'C(' + cat_EV + \ - ', Sum)[S.' + \ - current_EV + ']') + elif coding_scheme == "Sum": + cat_EV_contrast = EV.replace( + EV, "C(" + cat_EV + ", Sum)[S." + current_EV + "]" + ) parsed_EVs_in_contrast.append(cat_EV_contrast) skip = 1 if skip == 0: - parsed_EVs_in_contrast.append(EV) # handle interactions if ":" in EV and len(parsed_EVs_in_contrast) == 2: - - parsed_EVs_in_contrast = [parsed_EVs_in_contrast[0] + ":" + \ - parsed_EVs_in_contrast[1]] + parsed_EVs_in_contrast = [ + parsed_EVs_in_contrast[0] + ":" + parsed_EVs_in_contrast[1] + ] if ":" in EV and len(parsed_EVs_in_contrast) == 3: - - parsed_EVs_in_contrast = [parsed_EVs_in_contrast[0], \ - parsed_EVs_in_contrast[1] + ":" + \ - parsed_EVs_in_contrast[2]] + parsed_EVs_in_contrast = [ + parsed_EVs_in_contrast[0], + parsed_EVs_in_contrast[1] + ":" + parsed_EVs_in_contrast[2], + ] return parsed_EVs_in_contrast -def run(group_config, current_output, param_file=None, \ - derivative_means_dict=None, roi_means_dict=None, \ - model_out_dir=None, CPAC_run=True): - +def run( + group_config, + current_output, + param_file=None, + derivative_means_dict=None, + roi_means_dict=None, + model_out_dir=None, + CPAC_run=True, +): import os - import csv - import numpy as np + # open the GROUP ANALYSIS FSL .YML CONFIG FILE, not the main pipeline # config .yml file! if CPAC_run: c = group_config else: try: - c = Configuration(yaml.safe_load(open(os.path.realpath(group_config), \ - 'r'))) + c = Configuration(yaml.safe_load(open(os.path.realpath(group_config), "r"))) except: - raise Exception("Error in reading %s configuration file" \ - % group_config) + raise Exception("Error in reading %s configuration file" % group_config) # pull in the gpa settings! ph = c.pheno_file @@ -1470,15 +1362,12 @@ def run(group_config, current_output, param_file=None, \ if not os.path.exists(output_dir): os.makedirs(output_dir) except: - print('\n\n[!] CPAC says: Could not successfully create the group ' \ - 'analysis output directory:\n', output_dir, '\n\nMake sure ' \ - 'you have write access in this file structure.\n\n\n') raise Exception measure_dict = {} # extract motion measures from CPAC-generated power params file - if param_file != None: + if param_file is not None: measure_dict = get_measure_dict(param_file) # combine the motion measures dictionary with the measure_mean @@ -1487,106 +1376,136 @@ def run(group_config, current_output, param_file=None, \ measure_dict["Measure_Mean"] = derivative_means_dict # create the .mat and .grp files for FLAME - design_matrix, regressor_names = create_design_matrix(ph, sublist, \ - ev_selections, formula, \ - subject_id_label, coding_scheme, \ - grouping_var, measure_dict, \ - roi_means_dict, model_out_dir, \ - model_name, current_output) + design_matrix, regressor_names = create_design_matrix( + ph, + sublist, + ev_selections, + formula, + subject_id_label, + coding_scheme, + grouping_var, + measure_dict, + roi_means_dict, + model_out_dir, + model_name, + current_output, + ) # Create contrasts_dict dictionary for the .con file generation later contrasts_list = contrasts contrasts_dict = {} for contrast in contrasts_list: - # each 'contrast' is a string the user input of the desired contrast # remove all spaces - parsed_contrast = contrast.replace(' ', '') + parsed_contrast = contrast.replace(" ", "") - EVs_in_contrast = [] parsed_EVs_in_contrast = [] - if '>' in parsed_contrast: - - parsed_EVs_in_contrast = \ - process_contrast(parsed_contrast, '>', ev_selections, \ - group_sep, grouping_var, coding_scheme) - - contrasts_dict[parsed_contrast] = \ - greater_than(design_matrix, parsed_EVs_in_contrast[0], \ - parsed_EVs_in_contrast[1], coding_scheme, \ - group_sep, grouping_var) - - - elif '<' in parsed_contrast: - - parsed_EVs_in_contrast = \ - process_contrast(parsed_contrast, '<', ev_selections, \ - group_sep, grouping_var, coding_scheme) - - contrasts_dict[parsed_contrast] = \ - greater_than(design_matrix, parsed_EVs_in_contrast[1], \ - parsed_EVs_in_contrast[0], coding_scheme, \ - group_sep, grouping_var) - + if ">" in parsed_contrast: + parsed_EVs_in_contrast = process_contrast( + parsed_contrast, + ">", + ev_selections, + group_sep, + grouping_var, + coding_scheme, + ) + + contrasts_dict[parsed_contrast] = greater_than( + design_matrix, + parsed_EVs_in_contrast[0], + parsed_EVs_in_contrast[1], + coding_scheme, + group_sep, + grouping_var, + ) + + elif "<" in parsed_contrast: + parsed_EVs_in_contrast = process_contrast( + parsed_contrast, + "<", + ev_selections, + group_sep, + grouping_var, + coding_scheme, + ) + + contrasts_dict[parsed_contrast] = greater_than( + design_matrix, + parsed_EVs_in_contrast[1], + parsed_EVs_in_contrast[0], + coding_scheme, + group_sep, + grouping_var, + ) else: - - contrast_string = parsed_contrast.replace('+',',+,') - contrast_string = contrast_string.replace('-',',-,') - contrast_items = contrast_string.split(',') - - if '' in contrast_items: - contrast_items.remove('') - - if '+' in contrast_items and len(contrast_items) == 2: - - parsed_EVs_in_contrast = \ - process_contrast(parsed_contrast, '+', ev_selections, \ - group_sep, grouping_var, coding_scheme) - - contrasts_dict[parsed_contrast] = \ - positive(design_matrix, parsed_EVs_in_contrast[0], \ - coding_scheme, group_sep, grouping_var) - - elif '-' in contrast_items and len(contrast_items) == 2: - - parsed_EVs_in_contrast = \ - process_contrast(parsed_contrast, '-', ev_selections, \ - group_sep, grouping_var, coding_scheme) - - contrasts_dict[parsed_contrast] = \ - negative(design_matrix, parsed_EVs_in_contrast[0], \ - coding_scheme, group_sep, grouping_var) + contrast_string = parsed_contrast.replace("+", ",+,") + contrast_string = contrast_string.replace("-", ",-,") + contrast_items = contrast_string.split(",") + + if "" in contrast_items: + contrast_items.remove("") + + if "+" in contrast_items and len(contrast_items) == 2: + parsed_EVs_in_contrast = process_contrast( + parsed_contrast, + "+", + ev_selections, + group_sep, + grouping_var, + coding_scheme, + ) + + contrasts_dict[parsed_contrast] = positive( + design_matrix, + parsed_EVs_in_contrast[0], + coding_scheme, + group_sep, + grouping_var, + ) + + elif "-" in contrast_items and len(contrast_items) == 2: + parsed_EVs_in_contrast = process_contrast( + parsed_contrast, + "-", + ev_selections, + group_sep, + grouping_var, + coding_scheme, + ) + + contrasts_dict[parsed_contrast] = negative( + design_matrix, + parsed_EVs_in_contrast[0], + coding_scheme, + group_sep, + grouping_var, + ) if len(contrast_items) > 2: - idx = 0 for item in contrast_items: - # they need to be put back into Patsy formatted header # titles because the dmatrix gets passed into the function # that writes out the contrast matrix - if 'categorical' in ev_selections.keys(): - for cat_EV in ev_selections['categorical']: - + if "categorical" in ev_selections.keys(): + for cat_EV in ev_selections["categorical"]: if cat_EV in item: + if coding_scheme == "Treatment": + item = item.replace( + item, "C(" + cat_EV + ")[T." + item + "]" + ) - if coding_scheme == 'Treatment': - item = item.replace(item, \ - 'C(' + cat_EV + ')[T.' + item + ']') - - elif coding_scheme == 'Sum': - item = item.replace(item, \ - 'C(' + cat_EV + ', Sum)[S.' + item + ']') - + elif coding_scheme == "Sum": + item = item.replace( + item, "C(" + cat_EV + ", Sum)[S." + item + "]" + ) if idx == 0: - - if item != '+' and item != '-': - + if item not in ("+", "-"): contrast_vector = positive(dmatrix, item) if parsed_contrast not in contrasts_dict.keys(): @@ -1595,26 +1514,29 @@ def run(group_config, current_output, param_file=None, \ contrasts_dict[parsed_contrast] += contrast_vector elif idx != 0: - - if item != '+' and item != '-': - - if contrast_items[idx-1] == '+': - - contrast_vector = positive(dmatrix, item, \ - coding_scheme, group_sep,\ - grouping_var) + if item not in ("+", "-"): + if contrast_items[idx - 1] == "+": + contrast_vector = positive( + dmatrix, + item, + coding_scheme, + group_sep, + grouping_var, + ) if parsed_contrast not in contrasts_dict.keys(): contrasts_dict[parsed_contrast] = contrast_vector else: contrasts_dict[parsed_contrast] += contrast_vector - - if contrast_items[idx-1] == '-': - - contrast_vector = negative(dmatrix, item, \ - coding_scheme, group_sep,\ - grouping_var) + if contrast_items[idx - 1] == "-": + contrast_vector = negative( + dmatrix, + item, + coding_scheme, + group_sep, + grouping_var, + ) if parsed_contrast not in contrasts_dict.keys(): contrasts_dict[parsed_contrast] = contrast_vector @@ -1624,24 +1546,27 @@ def run(group_config, current_output, param_file=None, \ idx += 1 # finally write out the .con file and .fts file (if f-tests) - if (custom_contrasts == None) or (custom_contrasts == '') or \ - ("None" in custom_contrasts): - - print("Writing contrasts file (.con) based on contrasts provided " \ - "using the group analysis model builder's contrasts editor..") - - create_con_file(contrasts_dict, regressor_names, model_name, \ - current_output, model_out_dir) + if ( + (custom_contrasts is None) + or (custom_contrasts == "") + or ("None" in custom_contrasts) + ): + create_con_file( + contrasts_dict, regressor_names, model_name, current_output, model_out_dir + ) if f_tests: - create_fts_file(f_tests, contrasts_dict, model_name, \ - current_output, model_out_dir) + create_fts_file( + f_tests, contrasts_dict, model_name, current_output, model_out_dir + ) else: - - print("\nWriting contrasts file (.con) based on contrasts provided " \ - "with a custom contrasts matrix CSV file..\n") - - create_con_ftst_file(custom_contrasts, model_name, current_output, \ - model_out_dir, regressor_names, \ - coding_scheme, group_sep) + create_con_ftst_file( + custom_contrasts, + model_name, + current_output, + model_out_dir, + regressor_names, + coding_scheme, + group_sep, + ) diff --git a/CPAC/utils/create_group_analysis_info_files.py b/CPAC/utils/create_group_analysis_info_files.py index 82df2fef31..56ce91e700 100644 --- a/CPAC/utils/create_group_analysis_info_files.py +++ b/CPAC/utils/create_group_analysis_info_files.py @@ -1,8 +1,6 @@ - - -def write_design_matrix_csv(patsy_dmatrix, participant_column, column_names, - outfile_path): - +def write_design_matrix_csv( + patsy_dmatrix, participant_column, column_names, outfile_path +): import pandas as pd try: @@ -11,22 +9,26 @@ def write_design_matrix_csv(patsy_dmatrix, participant_column, column_names, # variable please patsy_dmatrix.to_csv(outfile_path) except: - group_model_dataframe = pd.DataFrame(data=patsy_dmatrix, - index=participant_column, - columns=column_names) + group_model_dataframe = pd.DataFrame( + data=patsy_dmatrix, index=participant_column, columns=column_names + ) group_model_dataframe.to_csv(outfile_path) - #except Exception as e: + # except Exception as e: # err = "\n\n[!] Could not write the design matrix dataframe to the " \ # "CSV file: %s\n\nError details: %s\n\n" % (outfile_path, e) # raise Exception(err) -def write_blank_contrast_csv(contrasts_columns,contrast_out_path): + + +def write_blank_contrast_csv(contrasts_columns, contrast_out_path): import pandas as pd - data = [None]*len(contrasts_columns) - - contrast_csv_df = pd.DataFrame(columns=contrasts_columns) + + [None] * len(contrasts_columns) + + contrast_csv_df = pd.DataFrame(columns=contrasts_columns) #'contrast_vectors':contrast_vectors}) contrast_csv_df.to_csv(contrast_out_path) + + def write_custom_readme_file(): pass - diff --git a/CPAC/utils/datasource.py b/CPAC/utils/datasource.py index 5a550765df..a3f52e64a9 100644 --- a/CPAC/utils/datasource.py +++ b/CPAC/utils/datasource.py @@ -16,26 +16,27 @@ # License along with C-PAC. If not, see . import csv import json -import re from pathlib import Path +import re from typing import Union + from nipype import logging from nipype.interfaces import utility as util + from CPAC.pipeline import nipype_pipeline_engine as pe -from CPAC.resources.templates.lookup_table import format_identifier, \ - lookup_identifier +from CPAC.resources.templates.lookup_table import format_identifier, lookup_identifier from CPAC.utils import function from CPAC.utils.bids_utils import bids_remove_entity from CPAC.utils.interfaces.function import Function from CPAC.utils.typing import TUPLE from CPAC.utils.utils import get_scan_params -logger = logging.getLogger('nipype.workflow') +logger = logging.getLogger("nipype.workflow") def bidsier_prefix(unique_id): """ - Function to return a BIDSier prefix for a given unique_id + Function to return a BIDSier prefix for a given unique_id. Parameters ---------- @@ -56,13 +57,13 @@ def bidsier_prefix(unique_id): >>> bidsier_prefix('01_ses-1') 'sub-01_ses-1' """ - keys = ['sub', 'ses'] - components = unique_id.split('_') + keys = ["sub", "ses"] + components = unique_id.split("_") for i, component in enumerate(components): if i < len(keys): if not component.startswith(keys[i]): - components[i] = '-'.join([keys[i], component]) - return '_'.join(components) + components[i] = "-".join([keys[i], component]) + return "_".join(components) def get_rest(scan, rest_dict, resource="scan"): @@ -96,38 +97,37 @@ def get_map(map, map_dct): def select_model_files(model, ftest, model_name): - """ - Method to select model files - """ - - import os + """Method to select model files.""" import glob + import os - files = glob.glob(os.path.join(model, '*')) + files = glob.glob(os.path.join(model, "*")) if len(files) == 0: raise Exception("No files found inside directory %s" % model) - fts_file = '' + fts_file = "" for filename in files: - if (model_name + '.mat') in filename: + if (model_name + ".mat") in filename: mat_file = filename - elif (model_name + '.grp') in filename: + elif (model_name + ".grp") in filename: grp_file = filename - elif ((model_name + '.fts') in filename) and ftest: + elif ((model_name + ".fts") in filename) and ftest: fts_file = filename - elif (model_name + '.con') in filename: + elif (model_name + ".con") in filename: con_file = filename - if ftest == True and fts_file == '': - errmsg = "\n[!] CPAC says: You have f-tests included in your group " \ - "analysis model '%s', but no .fts files were found in the " \ - "output folder specified for group analysis: %s.\n\nThe " \ - ".fts file is automatically generated by CPAC, and if you " \ - "are seeing this error, it is because something went wrong " \ - "with the generation of this file, or it has been moved." \ - "\n\n" % (model_name, model) + if ftest is True and fts_file == "": + errmsg = ( + "\n[!] CPAC says: You have f-tests included in your group " + "analysis model '%s', but no .fts files were found in the " + "output folder specified for group analysis: %s.\n\nThe " + ".fts file is automatically generated by CPAC, and if you " + "are seeing this error, it is because something went wrong " + "with the generation of this file, or it has been moved." + "\n\n" % (model_name, model) + ) raise Exception(errmsg) @@ -136,223 +136,252 @@ def select_model_files(model, ftest, model_name): def check_func_scan(func_scan_dct, scan): """Run some checks on the functional timeseries-related files for a given - series/scan name or label.""" - + series/scan name or label. + """ scan_resources = func_scan_dct[scan] try: scan_resources.keys() except AttributeError: - err = "\n[!] The data configuration file you provided is " \ - "missing a level under the 'func:' key. CPAC versions " \ - "1.2 and later use data configurations with an " \ - "additional level of nesting.\n\nExample\nfunc:\n " \ - "rest01:\n scan: /path/to/rest01_func.nii.gz\n" \ - " scan parameters: /path/to/scan_params.json\n\n" \ - "See the User Guide for more information.\n\n" + err = ( + "\n[!] The data configuration file you provided is " + "missing a level under the 'func:' key. CPAC versions " + "1.2 and later use data configurations with an " + "additional level of nesting.\n\nExample\nfunc:\n " + "rest01:\n scan: /path/to/rest01_func.nii.gz\n" + " scan parameters: /path/to/scan_params.json\n\n" + "See the User Guide for more information.\n\n" + ) raise Exception(err) # actual 4D time series file if "scan" not in scan_resources.keys(): - err = "\n\n[!] The {0} scan is missing its actual time-series " \ - "scan file, which should be a filepath labeled with the " \ - "'scan' key.\n\n".format(scan) + err = ( + "\n\n[!] The {0} scan is missing its actual time-series " + "scan file, which should be a filepath labeled with the " + "'scan' key.\n\n".format(scan) + ) raise Exception(err) # Nipype restriction (may have changed) - if '.' in scan or '+' in scan or '*' in scan: - raise Exception('\n\n[!] Scan names cannot contain any special ' - 'characters (., +, *, etc.). Please update this ' - 'and try again.\n\nScan: {0}' - '\n\n'.format(scan)) + if "." in scan or "+" in scan or "*" in scan: + raise Exception( + "\n\n[!] Scan names cannot contain any special " + "characters (., +, *, etc.). Please update this " + "and try again.\n\nScan: {0}" + "\n\n".format(scan) + ) -def create_func_datasource(rest_dict, rpool, wf_name='func_datasource'): +def create_func_datasource(rest_dict, rpool, wf_name="func_datasource"): """Return the functional timeseries-related file paths for each series/scan, from the dictionary of functional files described in the data configuration (sublist) YAML file. Scan input (from inputnode) is an iterable. """ - from CPAC.pipeline import nipype_pipeline_engine as pe import nipype.interfaces.utility as util + from CPAC.pipeline import nipype_pipeline_engine as pe + wf = pe.Workflow(name=wf_name) - inputnode = pe.Node(util.IdentityInterface( - fields=['subject', 'scan', 'creds_path', 'dl_dir'], - mandatory_inputs=True), - name='inputnode') + inputnode = pe.Node( + util.IdentityInterface( + fields=["subject", "scan", "creds_path", "dl_dir"], mandatory_inputs=True + ), + name="inputnode", + ) - outputnode = pe.Node(util.IdentityInterface(fields=['subject', 'rest', - 'scan', 'scan_params', - 'phase_diff', - 'magnitude']), - name='outputspec') + outputnode = pe.Node( + util.IdentityInterface( + fields=["subject", "rest", "scan", "scan_params", "phase_diff", "magnitude"] + ), + name="outputspec", + ) # have this here for now because of the big change in the data - # configuration format + # configuration format # (Not necessary with ingress - format does not comply) - if not rpool.check_rpool('derivatives-dir'): - check_scan = pe.Node(function.Function(input_names=['func_scan_dct', - 'scan'], - output_names=[], - function=check_func_scan, - as_module=True), - name='check_func_scan') + if not rpool.check_rpool("derivatives-dir"): + check_scan = pe.Node( + function.Function( + input_names=["func_scan_dct", "scan"], + output_names=[], + function=check_func_scan, + as_module=True, + ), + name="check_func_scan", + ) check_scan.inputs.func_scan_dct = rest_dict - wf.connect(inputnode, 'scan', check_scan, 'scan') - + wf.connect(inputnode, "scan", check_scan, "scan") # get the functional scan itself - selectrest = pe.Node(function.Function(input_names=['scan', - 'rest_dict', - 'resource'], - output_names=['file_path'], - function=get_rest, - as_module=True), - name='selectrest') + selectrest = pe.Node( + function.Function( + input_names=["scan", "rest_dict", "resource"], + output_names=["file_path"], + function=get_rest, + as_module=True, + ), + name="selectrest", + ) selectrest.inputs.rest_dict = rest_dict selectrest.inputs.resource = "scan" - wf.connect(inputnode, 'scan', selectrest, 'scan') + wf.connect(inputnode, "scan", selectrest, "scan") # check to see if it's on an Amazon AWS S3 bucket, and download it, if it # is - otherwise, just return the local file path - check_s3_node = pe.Node(function.Function(input_names=['file_path', - 'creds_path', - 'dl_dir', - 'img_type'], - output_names=['local_path'], - function=check_for_s3, - as_module=True), - name='check_for_s3') - - wf.connect(selectrest, 'file_path', check_s3_node, 'file_path') - wf.connect(inputnode, 'creds_path', check_s3_node, 'creds_path') - wf.connect(inputnode, 'dl_dir', check_s3_node, 'dl_dir') - check_s3_node.inputs.img_type = 'func' - - wf.connect(inputnode, 'subject', outputnode, 'subject') - wf.connect(check_s3_node, 'local_path', outputnode, 'rest') - wf.connect(inputnode, 'scan', outputnode, 'scan') + check_s3_node = pe.Node( + function.Function( + input_names=["file_path", "creds_path", "dl_dir", "img_type"], + output_names=["local_path"], + function=check_for_s3, + as_module=True, + ), + name="check_for_s3", + ) + + wf.connect(selectrest, "file_path", check_s3_node, "file_path") + wf.connect(inputnode, "creds_path", check_s3_node, "creds_path") + wf.connect(inputnode, "dl_dir", check_s3_node, "dl_dir") + check_s3_node.inputs.img_type = "func" + + wf.connect(inputnode, "subject", outputnode, "subject") + wf.connect(check_s3_node, "local_path", outputnode, "rest") + wf.connect(inputnode, "scan", outputnode, "scan") # scan parameters CSV - select_scan_params = pe.Node(function.Function(input_names=['scan', - 'rest_dict', - 'resource'], - output_names=['file_path'], - function=get_rest, - as_module=True), - name='select_scan_params') + select_scan_params = pe.Node( + function.Function( + input_names=["scan", "rest_dict", "resource"], + output_names=["file_path"], + function=get_rest, + as_module=True, + ), + name="select_scan_params", + ) select_scan_params.inputs.rest_dict = rest_dict select_scan_params.inputs.resource = "scan_parameters" - wf.connect(inputnode, 'scan', select_scan_params, 'scan') + wf.connect(inputnode, "scan", select_scan_params, "scan") # if the scan parameters file is on AWS S3, download it - s3_scan_params = pe.Node(function.Function(input_names=['file_path', - 'creds_path', - 'dl_dir', - 'img_type'], - output_names=['local_path'], - function=check_for_s3, - as_module=True), - name='s3_scan_params') - - wf.connect(select_scan_params, 'file_path', s3_scan_params, 'file_path') - wf.connect(inputnode, 'creds_path', s3_scan_params, 'creds_path') - wf.connect(inputnode, 'dl_dir', s3_scan_params, 'dl_dir') - wf.connect(s3_scan_params, 'local_path', outputnode, 'scan_params') + s3_scan_params = pe.Node( + function.Function( + input_names=["file_path", "creds_path", "dl_dir", "img_type"], + output_names=["local_path"], + function=check_for_s3, + as_module=True, + ), + name="s3_scan_params", + ) + + wf.connect(select_scan_params, "file_path", s3_scan_params, "file_path") + wf.connect(inputnode, "creds_path", s3_scan_params, "creds_path") + wf.connect(inputnode, "dl_dir", s3_scan_params, "dl_dir") + wf.connect(s3_scan_params, "local_path", outputnode, "scan_params") return wf -def create_fmap_datasource(fmap_dct, wf_name='fmap_datasource'): +def create_fmap_datasource(fmap_dct, wf_name="fmap_datasource"): """Return the field map files, from the dictionary of functional files described in the data configuration (sublist) YAML file. """ + import nipype.interfaces.utility as util from CPAC.pipeline import nipype_pipeline_engine as pe - import nipype.interfaces.utility as util wf = pe.Workflow(name=wf_name) - inputnode = pe.Node(util.IdentityInterface( - fields=['subject', 'scan', 'creds_path', 'dl_dir'], - mandatory_inputs=True), - name='inputnode') - - outputnode = pe.Node(util.IdentityInterface(fields=['subject', 'rest', - 'scan', 'scan_params', - 'phase_diff', - 'magnitude']), - name='outputspec') - - selectrest = pe.Node(function.Function(input_names=['scan', - 'rest_dict', - 'resource'], - output_names=['file_path'], - function=get_rest, - as_module=True), - name='selectrest') + inputnode = pe.Node( + util.IdentityInterface( + fields=["subject", "scan", "creds_path", "dl_dir"], mandatory_inputs=True + ), + name="inputnode", + ) + + outputnode = pe.Node( + util.IdentityInterface( + fields=["subject", "rest", "scan", "scan_params", "phase_diff", "magnitude"] + ), + name="outputspec", + ) + + selectrest = pe.Node( + function.Function( + input_names=["scan", "rest_dict", "resource"], + output_names=["file_path"], + function=get_rest, + as_module=True, + ), + name="selectrest", + ) selectrest.inputs.rest_dict = fmap_dct selectrest.inputs.resource = "scan" - wf.connect(inputnode, 'scan', selectrest, 'scan') + wf.connect(inputnode, "scan", selectrest, "scan") # check to see if it's on an Amazon AWS S3 bucket, and download it, if it # is - otherwise, just return the local file path - check_s3_node = pe.Node(function.Function(input_names=['file_path', - 'creds_path', - 'dl_dir', - 'img_type'], - output_names=['local_path'], - function=check_for_s3, - as_module=True), - name='check_for_s3') - - wf.connect(selectrest, 'file_path', check_s3_node, 'file_path') - wf.connect(inputnode, 'creds_path', check_s3_node, 'creds_path') - wf.connect(inputnode, 'dl_dir', check_s3_node, 'dl_dir') - check_s3_node.inputs.img_type = 'other' - - wf.connect(inputnode, 'subject', outputnode, 'subject') - wf.connect(check_s3_node, 'local_path', outputnode, 'rest') - wf.connect(inputnode, 'scan', outputnode, 'scan') + check_s3_node = pe.Node( + function.Function( + input_names=["file_path", "creds_path", "dl_dir", "img_type"], + output_names=["local_path"], + function=check_for_s3, + as_module=True, + ), + name="check_for_s3", + ) + + wf.connect(selectrest, "file_path", check_s3_node, "file_path") + wf.connect(inputnode, "creds_path", check_s3_node, "creds_path") + wf.connect(inputnode, "dl_dir", check_s3_node, "dl_dir") + check_s3_node.inputs.img_type = "other" + + wf.connect(inputnode, "subject", outputnode, "subject") + wf.connect(check_s3_node, "local_path", outputnode, "rest") + wf.connect(inputnode, "scan", outputnode, "scan") # scan parameters CSV - select_scan_params = pe.Node(function.Function(input_names=['scan', - 'rest_dict', - 'resource'], - output_names=['file_path'], - function=get_rest, - as_module=True), - name='select_scan_params') + select_scan_params = pe.Node( + function.Function( + input_names=["scan", "rest_dict", "resource"], + output_names=["file_path"], + function=get_rest, + as_module=True, + ), + name="select_scan_params", + ) select_scan_params.inputs.rest_dict = fmap_dct select_scan_params.inputs.resource = "scan_parameters" - wf.connect(inputnode, 'scan', select_scan_params, 'scan') + wf.connect(inputnode, "scan", select_scan_params, "scan") # if the scan parameters file is on AWS S3, download it - s3_scan_params = pe.Node(function.Function(input_names=['file_path', - 'creds_path', - 'dl_dir', - 'img_type'], - output_names=['local_path'], - function=check_for_s3, - as_module=True), - name='s3_scan_params') - - wf.connect(select_scan_params, 'file_path', s3_scan_params, 'file_path') - wf.connect(inputnode, 'creds_path', s3_scan_params, 'creds_path') - wf.connect(inputnode, 'dl_dir', s3_scan_params, 'dl_dir') - wf.connect(s3_scan_params, 'local_path', outputnode, 'scan_params') + s3_scan_params = pe.Node( + function.Function( + input_names=["file_path", "creds_path", "dl_dir", "img_type"], + output_names=["local_path"], + function=check_for_s3, + as_module=True, + ), + name="s3_scan_params", + ) + + wf.connect(select_scan_params, "file_path", s3_scan_params, "file_path") + wf.connect(inputnode, "creds_path", s3_scan_params, "creds_path") + wf.connect(inputnode, "dl_dir", s3_scan_params, "dl_dir") + wf.connect(s3_scan_params, "local_path", outputnode, "scan_params") return wf def get_fmap_phasediff_metadata(data_config_scan_params): - if (not isinstance(data_config_scan_params, dict) and - ".json" in data_config_scan_params): - with open(data_config_scan_params, 'r', encoding='utf-8') as _f: + if ( + not isinstance(data_config_scan_params, dict) + and ".json" in data_config_scan_params + ): + with open(data_config_scan_params, "r", encoding="utf-8") as _f: data_config_scan_params = json.load(_f) echo_time = None @@ -360,21 +389,31 @@ def get_fmap_phasediff_metadata(data_config_scan_params): echo_time_two = None if "EchoTime" in data_config_scan_params: echo_time = data_config_scan_params.get("EchoTime") - elif "EchoTime1" in data_config_scan_params and "EchoTime2" \ - in data_config_scan_params: + elif ( + "EchoTime1" in data_config_scan_params + and "EchoTime2" in data_config_scan_params + ): echo_time_one = data_config_scan_params.get("EchoTime1") echo_time_two = data_config_scan_params.get("EchoTime2") dwell_time = data_config_scan_params.get("DwellTime") pe_direction = data_config_scan_params.get("PhaseEncodingDirection") total_readout = data_config_scan_params.get("TotalReadoutTime") - return (dwell_time, pe_direction, total_readout, echo_time, - echo_time_one, echo_time_two) + return ( + dwell_time, + pe_direction, + total_readout, + echo_time, + echo_time_one, + echo_time_two, + ) + -@Function.sig_imports(['from CPAC.utils.typing import TUPLE']) -def calc_delta_te_and_asym_ratio(effective_echo_spacing: float, - echo_times: list) -> TUPLE[float, float]: - """Calcluate ``deltaTE`` and ``ees_asym_ratio`` from given metadata +@Function.sig_imports(["from CPAC.utils.typing import TUPLE"]) +def calc_delta_te_and_asym_ratio( + effective_echo_spacing: float, echo_times: list +) -> TUPLE[float, float]: + """Calcluate ``deltaTE`` and ``ees_asym_ratio`` from given metadata. Parameters ---------- @@ -390,9 +429,11 @@ def calc_delta_te_and_asym_ratio(effective_echo_spacing: float, ees_asym_ratio : float """ if not isinstance(effective_echo_spacing, float): - raise LookupError('C-PAC could not find `EffectiveEchoSpacing` in ' - 'either fmap or func sidecar JSON, but that field ' - 'is required for PhaseDiff distortion correction.') + raise LookupError( + "C-PAC could not find `EffectiveEchoSpacing` in " + "either fmap or func sidecar JSON, but that field " + "is required for PhaseDiff distortion correction." + ) # convert into milliseconds if necessary # these values will/should never be more than 10ms @@ -401,7 +442,7 @@ def calc_delta_te_and_asym_ratio(effective_echo_spacing: float, echo_times[1] = echo_times[1] * 1000 deltaTE = abs(echo_times[0] - echo_times[1]) - ees_asym_ratio = (effective_echo_spacing / deltaTE) + ees_asym_ratio = effective_echo_spacing / deltaTE return deltaTE, ees_asym_ratio @@ -410,14 +451,21 @@ def gather_echo_times(echotime_1, echotime_2, echotime_3=None, echotime_4=None): echotime_list = list(filter(lambda item: item is not None, echotime_list)) echotime_list = list(set(echotime_list)) if len(echotime_list) != 2: - raise Exception("\n[!] Something went wrong with the field map echo " - "times - there should be two distinct values.\n\n" - f"Echo Times:\n{echotime_list}\n") + raise Exception( + "\n[!] Something went wrong with the field map echo " + "times - there should be two distinct values.\n\n" + f"Echo Times:\n{echotime_list}\n" + ) return echotime_list -def match_epi_fmaps(bold_pedir, epi_fmap_one, epi_fmap_params_one, - epi_fmap_two=None, epi_fmap_params_two=None): +def match_epi_fmaps( + bold_pedir, + epi_fmap_one, + epi_fmap_params_one, + epi_fmap_two=None, + epi_fmap_params_two=None, +): """Parse the field map files in the data configuration and determine which ones have the same and opposite phase-encoding directions as the BOLD scan in the current pipeline. @@ -438,7 +486,6 @@ def match_epi_fmaps(bold_pedir, epi_fmap_one, epi_fmap_params_one, 2. Check whether there are one or two EPI's in the field map data. 3. Grab the one or two EPI field maps. """ - fmap_dct = {epi_fmap_one: epi_fmap_params_one} if epi_fmap_two and epi_fmap_params_two: fmap_dct[epi_fmap_two] = epi_fmap_params_two @@ -449,7 +496,7 @@ def match_epi_fmaps(bold_pedir, epi_fmap_one, epi_fmap_params_one, for epi_scan in fmap_dct.keys(): scan_params = fmap_dct[epi_scan] if not isinstance(scan_params, dict) and ".json" in scan_params: - with open(scan_params, 'r') as f: + with open(scan_params, "r") as f: scan_params = json.load(f) if "PhaseEncodingDirection" in scan_params: epi_pedir = scan_params["PhaseEncodingDirection"] @@ -461,12 +508,20 @@ def match_epi_fmaps(bold_pedir, epi_fmap_one, epi_fmap_params_one, return (opposite_pe_epi, same_pe_epi) -def ingress_func_metadata(wf, cfg, rpool, sub_dict, subject_id, - input_creds_path, unique_id=None, num_strat=None): - name_suffix = '' +def ingress_func_metadata( + wf, + cfg, + rpool, + sub_dict, + subject_id, + input_creds_path, + unique_id=None, + num_strat=None, +): + name_suffix = "" for suffix_part in (unique_id, num_strat): if suffix_part is not None: - name_suffix += f'_{suffix_part}' + name_suffix += f"_{suffix_part}" # Grab field maps diff = False blip = False @@ -476,42 +531,58 @@ def ingress_func_metadata(wf, cfg, rpool, sub_dict, subject_id, second = False for key in sub_dict["fmap"]: gather_fmap = create_fmap_datasource( - sub_dict["fmap"], f"fmap_gather_{key}_{subject_id}") + sub_dict["fmap"], f"fmap_gather_{key}_{subject_id}" + ) gather_fmap.inputs.inputnode.set( - subject=subject_id, creds_path=input_creds_path, - dl_dir=cfg.pipeline_setup['working_directory']['path']) + subject=subject_id, + creds_path=input_creds_path, + dl_dir=cfg.pipeline_setup["working_directory"]["path"], + ) gather_fmap.inputs.inputnode.scan = key orig_key = key - if 'epi' in key and not second: - key = 'epi-1' + if "epi" in key and not second: + key = "epi-1" second = True - elif 'epi' in key and second: - key = 'epi-2' - - rpool.set_data(key, gather_fmap, 'outputspec.rest', {}, "", - "fmap_ingress") - rpool.set_data(f'{key}-scan-params', gather_fmap, - 'outputspec.scan_params', {}, "", - "fmap_params_ingress") + elif "epi" in key and second: + key = "epi-2" + + rpool.set_data(key, gather_fmap, "outputspec.rest", {}, "", "fmap_ingress") + rpool.set_data( + f"{key}-scan-params", + gather_fmap, + "outputspec.scan_params", + {}, + "", + "fmap_params_ingress", + ) fmap_rp_list.append(key) - get_fmap_metadata_imports = ['import json'] - get_fmap_metadata = pe.Node(Function( - input_names=['data_config_scan_params'], - output_names=['dwell_time', - 'pe_direction', - 'total_readout', - 'echo_time', - 'echo_time_one', - 'echo_time_two'], - function=get_fmap_phasediff_metadata, - imports=get_fmap_metadata_imports), - name=f'{key}_get_metadata{name_suffix}') - - wf.connect(gather_fmap, 'outputspec.scan_params', - get_fmap_metadata, 'data_config_scan_params') + get_fmap_metadata_imports = ["import json"] + get_fmap_metadata = pe.Node( + Function( + input_names=["data_config_scan_params"], + output_names=[ + "dwell_time", + "pe_direction", + "total_readout", + "echo_time", + "echo_time_one", + "echo_time_two", + ], + function=get_fmap_phasediff_metadata, + imports=get_fmap_metadata_imports, + ), + name=f"{key}_get_metadata{name_suffix}", + ) + + wf.connect( + gather_fmap, + "outputspec.scan_params", + get_fmap_metadata, + "data_config_scan_params", + ) if "phase" in key: # leave it open to all three options, in case there is a @@ -521,233 +592,302 @@ def ingress_func_metadata(wf, cfg, rpool, sub_dict, subject_id, # at least one of these rpool keys will have a None value, # which will be sorted out in gather_echo_times below - rpool.set_data(f'{key}-TE', get_fmap_metadata, 'echo_time', - {}, "", "fmap_TE_ingress") + rpool.set_data( + f"{key}-TE", + get_fmap_metadata, + "echo_time", + {}, + "", + "fmap_TE_ingress", + ) fmap_TE_list.append(f"{key}-TE") - rpool.set_data(f'{key}-TE1', - get_fmap_metadata, 'echo_time_one', - {}, "", "fmap_TE1_ingress") + rpool.set_data( + f"{key}-TE1", + get_fmap_metadata, + "echo_time_one", + {}, + "", + "fmap_TE1_ingress", + ) fmap_TE_list.append(f"{key}-TE1") - rpool.set_data(f'{key}-TE2', - get_fmap_metadata, 'echo_time_two', - {}, "", "fmap_TE2_ingress") + rpool.set_data( + f"{key}-TE2", + get_fmap_metadata, + "echo_time_two", + {}, + "", + "fmap_TE2_ingress", + ) fmap_TE_list.append(f"{key}-TE2") elif "magnitude" in key: - rpool.set_data(f'{key}-TE', get_fmap_metadata, 'echo_time', - {}, "", "fmap_TE_ingress") + rpool.set_data( + f"{key}-TE", + get_fmap_metadata, + "echo_time", + {}, + "", + "fmap_TE_ingress", + ) fmap_TE_list.append(f"{key}-TE") - rpool.set_data(f'{key}-dwell', get_fmap_metadata, - 'dwell_time', {}, "", "fmap_dwell_ingress") - rpool.set_data(f'{key}-pedir', get_fmap_metadata, - 'pe_direction', {}, "", "fmap_pedir_ingress") - rpool.set_data(f'{key}-total-readout', get_fmap_metadata, - 'total_readout', {}, "", "fmap_readout_ingress") + rpool.set_data( + f"{key}-dwell", + get_fmap_metadata, + "dwell_time", + {}, + "", + "fmap_dwell_ingress", + ) + rpool.set_data( + f"{key}-pedir", + get_fmap_metadata, + "pe_direction", + {}, + "", + "fmap_pedir_ingress", + ) + rpool.set_data( + f"{key}-total-readout", + get_fmap_metadata, + "total_readout", + {}, + "", + "fmap_readout_ingress", + ) - if 'phase' in key or 'mag' in key: + if "phase" in key or "mag" in key: diff = True - if re.match('epi_[AP]{2}', orig_key): + if re.match("epi_[AP]{2}", orig_key): blip = True if diff: - calc_delta_ratio = pe.Node(Function( - input_names=['effective_echo_spacing', - 'echo_times'], - output_names=['deltaTE', - 'ees_asym_ratio'], - function=calc_delta_te_and_asym_ratio, - imports=['from typing import Optional, Tuple']), - name=f'diff_distcor_calc_delta{name_suffix}') - - gather_echoes = pe.Node(Function( - input_names=['echotime_1', - 'echotime_2', - 'echotime_3', - 'echotime_4'], - output_names=['echotime_list'], - function=gather_echo_times), - name='fugue_gather_echo_times') + calc_delta_ratio = pe.Node( + Function( + input_names=["effective_echo_spacing", "echo_times"], + output_names=["deltaTE", "ees_asym_ratio"], + function=calc_delta_te_and_asym_ratio, + imports=["from typing import Optional, Tuple"], + ), + name=f"diff_distcor_calc_delta{name_suffix}", + ) + + gather_echoes = pe.Node( + Function( + input_names=[ + "echotime_1", + "echotime_2", + "echotime_3", + "echotime_4", + ], + output_names=["echotime_list"], + function=gather_echo_times, + ), + name="fugue_gather_echo_times", + ) for idx, fmap_file in enumerate(fmap_TE_list, start=1): try: node, out_file = rpool.get(fmap_file)[ - f"['{fmap_file}:fmap_TE_ingress']"]['data'] - wf.connect(node, out_file, gather_echoes, - f'echotime_{idx}') + f"['{fmap_file}:fmap_TE_ingress']" + ]["data"] + wf.connect(node, out_file, gather_echoes, f"echotime_{idx}") except KeyError: pass - wf.connect(gather_echoes, 'echotime_list', - calc_delta_ratio, 'echo_times') + wf.connect(gather_echoes, "echotime_list", calc_delta_ratio, "echo_times") # Add in nodes to get parameters from configuration file # a node which checks if scan_parameters are present for each scan - scan_params = pe.Node(Function( - input_names=['data_config_scan_params', - 'subject_id', - 'scan', - 'pipeconfig_tr', - 'pipeconfig_tpattern', - 'pipeconfig_start_indx', - 'pipeconfig_stop_indx'], - output_names=['tr', - 'tpattern', - 'template', - 'ref_slice', - 'start_indx', - 'stop_indx', - 'pe_direction', - 'effective_echo_spacing'], - function=get_scan_params, - imports=['from CPAC.utils.utils import check, try_fetch_parameter'] - ), name=f"bold_scan_params_{subject_id}{name_suffix}") + scan_params = pe.Node( + Function( + input_names=[ + "data_config_scan_params", + "subject_id", + "scan", + "pipeconfig_tr", + "pipeconfig_tpattern", + "pipeconfig_start_indx", + "pipeconfig_stop_indx", + ], + output_names=[ + "tr", + "tpattern", + "template", + "ref_slice", + "start_indx", + "stop_indx", + "pe_direction", + "effective_echo_spacing", + ], + function=get_scan_params, + imports=["from CPAC.utils.utils import check, try_fetch_parameter"], + ), + name=f"bold_scan_params_{subject_id}{name_suffix}", + ) scan_params.inputs.subject_id = subject_id scan_params.inputs.set( - pipeconfig_start_indx=cfg.functional_preproc['truncation'][ - 'start_tr'], - pipeconfig_stop_indx=cfg.functional_preproc['truncation']['stop_tr']) + pipeconfig_start_indx=cfg.functional_preproc["truncation"]["start_tr"], + pipeconfig_stop_indx=cfg.functional_preproc["truncation"]["stop_tr"], + ) - node, out = rpool.get('scan')["['scan:func_ingress']"]['data'] - wf.connect(node, out, scan_params, 'scan') + node, out = rpool.get("scan")["['scan:func_ingress']"]["data"] + wf.connect(node, out, scan_params, "scan") # Workaround for extracting metadata with ingress - if rpool.check_rpool('derivatives-dir'): - selectrest_json = pe.Node(function.Function(input_names=['scan', - 'rest_dict', - 'resource'], - output_names=['file_path'], - function=get_rest, - as_module=True), - name='selectrest_json') + if rpool.check_rpool("derivatives-dir"): + selectrest_json = pe.Node( + function.Function( + input_names=["scan", "rest_dict", "resource"], + output_names=["file_path"], + function=get_rest, + as_module=True, + ), + name="selectrest_json", + ) selectrest_json.inputs.rest_dict = sub_dict selectrest_json.inputs.resource = "scan_parameters" - wf.connect(node, out, selectrest_json, 'scan') - wf.connect(selectrest_json, 'file_path', scan_params, 'data_config_scan_params') - + wf.connect(node, out, selectrest_json, "scan") + wf.connect(selectrest_json, "file_path", scan_params, "data_config_scan_params") + else: # wire in the scan parameter workflow - node, out = rpool.get('scan-params')[ - "['scan-params:scan_params_ingress']"]['data'] - wf.connect(node, out, scan_params, 'data_config_scan_params') - - rpool.set_data('TR', scan_params, 'tr', {}, "", "func_metadata_ingress") - rpool.set_data('tpattern', scan_params, 'tpattern', {}, "", - "func_metadata_ingress") - rpool.set_data('template', scan_params, 'template', {}, "", - "func_metadata_ingress") - rpool.set_data('start-tr', scan_params, 'start_indx', {}, "", - "func_metadata_ingress") - rpool.set_data('stop-tr', scan_params, 'stop_indx', {}, "", - "func_metadata_ingress") - rpool.set_data('pe-direction', scan_params, 'pe_direction', {}, "", - "func_metadata_ingress") + node, out = rpool.get("scan-params")["['scan-params:scan_params_ingress']"][ + "data" + ] + wf.connect(node, out, scan_params, "data_config_scan_params") + + rpool.set_data("TR", scan_params, "tr", {}, "", "func_metadata_ingress") + rpool.set_data("tpattern", scan_params, "tpattern", {}, "", "func_metadata_ingress") + rpool.set_data("template", scan_params, "template", {}, "", "func_metadata_ingress") + rpool.set_data( + "start-tr", scan_params, "start_indx", {}, "", "func_metadata_ingress" + ) + rpool.set_data("stop-tr", scan_params, "stop_indx", {}, "", "func_metadata_ingress") + rpool.set_data( + "pe-direction", scan_params, "pe_direction", {}, "", "func_metadata_ingress" + ) if diff: # Connect EffectiveEchoSpacing from functional metadata - rpool.set_data('effectiveEchoSpacing', scan_params, - 'effective_echo_spacing', {}, '', - 'func_metadata_ingress') - node, out_file = rpool.get('effectiveEchoSpacing')[ - "['effectiveEchoSpacing:func_metadata_ingress']"]['data'] - wf.connect(node, out_file, calc_delta_ratio, 'effective_echo_spacing') - rpool.set_data('deltaTE', calc_delta_ratio, 'deltaTE', {}, '', - 'deltaTE_ingress') - rpool.set_data('ees-asym-ratio', calc_delta_ratio, - 'ees_asym_ratio', {}, '', - 'ees_asym_ratio_ingress') + rpool.set_data( + "effectiveEchoSpacing", + scan_params, + "effective_echo_spacing", + {}, + "", + "func_metadata_ingress", + ) + node, out_file = rpool.get("effectiveEchoSpacing")[ + "['effectiveEchoSpacing:func_metadata_ingress']" + ]["data"] + wf.connect(node, out_file, calc_delta_ratio, "effective_echo_spacing") + rpool.set_data( + "deltaTE", calc_delta_ratio, "deltaTE", {}, "", "deltaTE_ingress" + ) + rpool.set_data( + "ees-asym-ratio", + calc_delta_ratio, + "ees_asym_ratio", + {}, + "", + "ees_asym_ratio_ingress", + ) return wf, rpool, diff, blip, fmap_rp_list def create_general_datasource(wf_name): - from CPAC.pipeline import nipype_pipeline_engine as pe import nipype.interfaces.utility as util + from CPAC.pipeline import nipype_pipeline_engine as pe + wf = pe.Workflow(name=wf_name) - inputnode = pe.Node(util.IdentityInterface( - fields=['unique_id', 'data', 'scan', 'creds_path', - 'dl_dir'], - mandatory_inputs=True), - name='inputnode') - - check_s3_node = pe.Node(function.Function(input_names=['file_path', - 'creds_path', - 'dl_dir', - 'img_type'], - output_names=['local_path'], - function=check_for_s3, - as_module=True), - name='check_for_s3') + inputnode = pe.Node( + util.IdentityInterface( + fields=["unique_id", "data", "scan", "creds_path", "dl_dir"], + mandatory_inputs=True, + ), + name="inputnode", + ) + + check_s3_node = pe.Node( + function.Function( + input_names=["file_path", "creds_path", "dl_dir", "img_type"], + output_names=["local_path"], + function=check_for_s3, + as_module=True, + ), + name="check_for_s3", + ) check_s3_node.inputs.img_type = "other" - wf.connect(inputnode, 'data', check_s3_node, 'file_path') - wf.connect(inputnode, 'creds_path', check_s3_node, 'creds_path') - wf.connect(inputnode, 'dl_dir', check_s3_node, 'dl_dir') + wf.connect(inputnode, "data", check_s3_node, "file_path") + wf.connect(inputnode, "creds_path", check_s3_node, "creds_path") + wf.connect(inputnode, "dl_dir", check_s3_node, "dl_dir") - outputnode = pe.Node(util.IdentityInterface(fields=['unique_id', - 'data', - 'scan']), - name='outputspec') + outputnode = pe.Node( + util.IdentityInterface(fields=["unique_id", "data", "scan"]), name="outputspec" + ) - wf.connect(inputnode, 'unique_id', outputnode, 'unique_id') - wf.connect(inputnode, 'scan', outputnode, 'scan') - wf.connect(check_s3_node, 'local_path', outputnode, 'data') + wf.connect(inputnode, "unique_id", outputnode, "unique_id") + wf.connect(inputnode, "scan", outputnode, "scan") + wf.connect(check_s3_node, "local_path", outputnode, "data") return wf -def create_check_for_s3_node(name, file_path, img_type='other', - creds_path=None, dl_dir=None, map_node=False): +def create_check_for_s3_node( + name, file_path, img_type="other", creds_path=None, dl_dir=None, map_node=False +): if map_node: - check_s3_node = pe.MapNode(function.Function(input_names=['file_path', - 'creds_path', - 'dl_dir', - 'img_type'], - output_names=[ - 'local_path'], - function=check_for_s3, - as_module=True), - iterfield=['file_path'], - name='check_for_s3_%s' % name) + check_s3_node = pe.MapNode( + function.Function( + input_names=["file_path", "creds_path", "dl_dir", "img_type"], + output_names=["local_path"], + function=check_for_s3, + as_module=True, + ), + iterfield=["file_path"], + name="check_for_s3_%s" % name, + ) else: - check_s3_node = pe.Node(function.Function(input_names=['file_path', - 'creds_path', - 'dl_dir', - 'img_type'], - output_names=['local_path'], - function=check_for_s3, - as_module=True), - name='check_for_s3_%s' % name) + check_s3_node = pe.Node( + function.Function( + input_names=["file_path", "creds_path", "dl_dir", "img_type"], + output_names=["local_path"], + function=check_for_s3, + as_module=True, + ), + name="check_for_s3_%s" % name, + ) check_s3_node.inputs.set( - file_path=file_path, - creds_path=creds_path, - dl_dir=dl_dir, - img_type=img_type + file_path=file_path, creds_path=creds_path, dl_dir=dl_dir, img_type=img_type ) return check_s3_node # Check if passed-in file is on S3 -def check_for_s3(file_path, creds_path=None, dl_dir=None, img_type='other', - verbose=False): +def check_for_s3( + file_path, creds_path=None, dl_dir=None, img_type="other", verbose=False +): # Import packages import os - import nibabel as nib + import botocore.exceptions + import nibabel as nib from indi_aws import fetch_creds # Init variables - s3_str = 's3://' + s3_str = "s3://" if creds_path: - if "None" in creds_path or "none" in creds_path or \ - "null" in creds_path: + if "None" in creds_path or "none" in creds_path or "null" in creds_path: creds_path = None if dl_dir is None: @@ -761,18 +901,16 @@ def check_for_s3(file_path, creds_path=None, dl_dir=None, img_type='other', # TODO: remove this once scan parameter input as dictionary is phased out if isinstance(file_path, dict): # if this is a dictionary, just skip altogether - local_path = file_path - return local_path + return file_path if file_path.lower().startswith(s3_str): - - file_path = s3_str + file_path[len(s3_str):] + file_path = s3_str + file_path[len(s3_str) :] # Get bucket name and bucket object - bucket_name = file_path[len(s3_str):].split('/')[0] + bucket_name = file_path[len(s3_str) :].split("/")[0] # Extract relative key path from bucket and local path s3_prefix = s3_str + bucket_name - s3_key = file_path[len(s3_prefix) + 1:] + s3_key = file_path[len(s3_prefix) + 1 :] local_path = os.path.join(dl_dir, bucket_name, s3_key) # Get local directory and create folders if they dont exist @@ -781,35 +919,40 @@ def check_for_s3(file_path, creds_path=None, dl_dir=None, img_type='other', os.makedirs(local_dir, exist_ok=True) if os.path.exists(local_path): - print("{0} already exists- skipping download.".format(local_path)) + pass else: # Download file try: bucket = fetch_creds.return_bucket(creds_path, bucket_name) - print("Attempting to download from AWS S3: {0}".format( - file_path)) bucket.download_file(Key=s3_key, Filename=local_path) except botocore.exceptions.ClientError as exc: - error_code = int(exc.response['Error']['Code']) + error_code = int(exc.response["Error"]["Code"]) err_msg = str(exc) if error_code == 403: - err_msg = 'Access to bucket: "%s" is denied; using credentials ' \ - 'in subject list: "%s"; cannot access the file "%s"' \ - % (bucket_name, creds_path, file_path) + err_msg = ( + 'Access to bucket: "%s" is denied; using credentials ' + 'in subject list: "%s"; cannot access the file "%s"' + % (bucket_name, creds_path, file_path) + ) elif error_code == 404: - err_msg = 'File: {0} does not exist; check spelling and try ' \ - 'again'.format( - os.path.join(bucket_name, s3_key)) + err_msg = ( + "File: {0} does not exist; check spelling and try " + "again".format(os.path.join(bucket_name, s3_key)) + ) else: - err_msg = 'Unable to connect to bucket: "%s". Error message:\n%s' \ - % (bucket_name, exc) + err_msg = ( + 'Unable to connect to bucket: "%s". Error message:\n%s' + % (bucket_name, exc) + ) raise Exception(err_msg) except Exception as exc: - err_msg = 'Unable to connect to bucket: "%s". Error message:\n%s' \ - % (bucket_name, exc) + err_msg = 'Unable to connect to bucket: "%s". Error message:\n%s' % ( + bucket_name, + exc, + ) raise Exception(err_msg) # Otherwise just return what was passed in, resolving if a link @@ -821,52 +964,61 @@ def check_for_s3(file_path, creds_path=None, dl_dir=None, img_type='other', # alert users to 2020-07-20 Neuroparc atlas update (v0 to v1) ndmg_atlases = {} with open( - os.path.join( - os.path.dirname(os.path.dirname(__file__)), - 'resources/templates/ndmg_atlases.csv' - ) + os.path.join( + os.path.dirname(os.path.dirname(__file__)), + "resources/templates/ndmg_atlases.csv", + ) ) as ndmg_atlases_file: - ndmg_atlases['v0'], ndmg_atlases['v1'] = zip(*[( - f'/ndmg_atlases/label/Human/{atlas[0]}', - f'/ndmg_atlases/label/Human/{atlas[1]}' - ) for atlas in - csv.reader( - ndmg_atlases_file)]) - if local_path in ndmg_atlases['v0']: + ndmg_atlases["v0"], ndmg_atlases["v1"] = zip( + *[ + ( + f"/ndmg_atlases/label/Human/{atlas[0]}", + f"/ndmg_atlases/label/Human/{atlas[1]}", + ) + for atlas in csv.reader(ndmg_atlases_file) + ] + ) + if local_path in ndmg_atlases["v0"]: raise FileNotFoundError( - ''.join([ - 'Neuroparc atlas paths were updated on July 20, 2020. ' - 'C-PAC configuration files using Neuroparc v0 atlas paths ' - '(including C-PAC default and preconfigured pipeline ' - 'configurations from v1.6.2a and earlier) need to be ' - 'updated to use Neuroparc atlases. Your current ' - 'configuration includes the Neuroparc v0 path ' - f'{local_path} which needs to be updated to ', - ndmg_atlases['v1'][ndmg_atlases['v0'].index(local_path)], - '. For a full list such paths, see https://fcp-indi.' - 'github.io/docs/nightly/user/ndmg_atlases' - ]) + "".join( + [ + "Neuroparc atlas paths were updated on July 20, 2020. " + "C-PAC configuration files using Neuroparc v0 atlas paths " + "(including C-PAC default and preconfigured pipeline " + "configurations from v1.6.2a and earlier) need to be " + "updated to use Neuroparc atlases. Your current " + "configuration includes the Neuroparc v0 path " + f"{local_path} which needs to be updated to ", + ndmg_atlases["v1"][ndmg_atlases["v0"].index(local_path)], + ". For a full list such paths, see https://fcp-indi." + "github.io/docs/nightly/user/ndmg_atlases", + ] + ) ) else: - raise FileNotFoundError(f'File {local_path} does not exist!') + raise FileNotFoundError(f"File {local_path} does not exist!") if verbose: - print("Downloaded file:\n{0}\n".format(local_path)) + pass # Check image dimensionality - if local_path.endswith('.nii') or local_path.endswith('.nii.gz'): + if local_path.endswith(".nii") or local_path.endswith(".nii.gz"): img_nii = nib.load(local_path) - if img_type == 'anat': + if img_type == "anat": if len(img_nii.shape) != 3: - raise IOError('File: %s must be an anatomical image with 3 ' - 'dimensions but %d dimensions found!' - % (local_path, len(img_nii.shape))) - elif img_type == 'func': + raise IOError( + "File: %s must be an anatomical image with 3 " + "dimensions but %d dimensions found!" + % (local_path, len(img_nii.shape)) + ) + elif img_type == "func": if len(img_nii.shape) not in [3, 4]: - raise IOError('File: %s must be a functional image with 3 or ' - '4 dimensions but %d dimensions found!' - % (local_path, len(img_nii.shape))) + raise IOError( + "File: %s must be a functional image with 3 or " + "4 dimensions but %d dimensions found!" + % (local_path, len(img_nii.shape)) + ) return local_path @@ -875,41 +1027,38 @@ def gather_extraction_maps(c): ts_analysis_dict = {} sca_analysis_dict = {} - if hasattr(c, 'timeseries_extraction'): - - tsa_roi_dict = c.timeseries_extraction['tse_roi_paths'] + if hasattr(c, "timeseries_extraction"): + tsa_roi_dict = c.timeseries_extraction["tse_roi_paths"] # Timeseries and SCA config selections processing # flip the dictionary for roi_path in tsa_roi_dict.keys(): - ts_analysis_to_run = [ - x.strip() for x in tsa_roi_dict[roi_path].split(",") - ] + ts_analysis_to_run = [x.strip() for x in tsa_roi_dict[roi_path].split(",")] for analysis_type in ts_analysis_to_run: if analysis_type not in ts_analysis_dict.keys(): ts_analysis_dict[analysis_type] = [] ts_analysis_dict[analysis_type].append(roi_path) - if c.timeseries_extraction['run']: - + if c.timeseries_extraction["run"]: if not tsa_roi_dict: - err = "\n\n[!] CPAC says: Time Series Extraction is " \ - "set to run, but no ROI NIFTI file paths were " \ - "provided!\n\n" + err = ( + "\n\n[!] CPAC says: Time Series Extraction is " + "set to run, but no ROI NIFTI file paths were " + "provided!\n\n" + ) raise Exception(err) - if c.seed_based_correlation_analysis['run']: - + if c.seed_based_correlation_analysis["run"]: try: - sca_roi_dict = c.seed_based_correlation_analysis[ - 'sca_roi_paths' - ] + sca_roi_dict = c.seed_based_correlation_analysis["sca_roi_paths"] except KeyError: - err = "\n\n[!] CPAC says: Seed-based Correlation Analysis " \ - "is set to run, but no ROI NIFTI file paths were " \ - "provided!\n\n" + err = ( + "\n\n[!] CPAC says: Seed-based Correlation Analysis " + "is set to run, but no ROI NIFTI file paths were " + "provided!\n\n" + ) raise Exception(err) # flip the dictionary @@ -958,13 +1107,19 @@ def get_highest_local_res(template: Union[Path, str], tagname: str) -> Path: LookupError: Could not find template /cpac_templates/dne_T1w_2mm.nii.gz """ from CPAC.pipeline.schema import RESOLUTION_REGEX + if isinstance(template, str): template = Path(template) template_pattern = ( - RESOLUTION_REGEX.replace('^', '').replace('$', '').join([ - re.escape(_part) for _part in template.name.split(tagname, 1)])) - matching_templates = [file for file in template.parent.iterdir() if - re.match(template_pattern, file.name)] + RESOLUTION_REGEX.replace("^", "") + .replace("$", "") + .join([re.escape(_part) for _part in template.name.split(tagname, 1)]) + ) + matching_templates = [ + file + for file in template.parent.iterdir() + if re.match(template_pattern, file.name) + ] matching_templates.sort() try: return matching_templates[0] @@ -987,13 +1142,13 @@ def res_string_to_tuple(resolution): Tuple of floats, e.g. (3.438, 3.438, 3.4) """ if "x" in str(resolution): - return tuple( - float(i.replace('mm', '')) for i in resolution.split("x")) - return (float(resolution.replace('mm', '')),) * 3 + return tuple(float(i.replace("mm", "")) for i in resolution.split("x")) + return (float(resolution.replace("mm", "")),) * 3 def resolve_resolution(resolution, template, template_name, tag=None): from nipype.interfaces import afni + from CPAC.pipeline import nipype_pipeline_engine as pe from CPAC.utils.datasource import check_for_s3 @@ -1004,8 +1159,7 @@ def resolve_resolution(resolution, template, template_name, tag=None): tagname = "${" + tag + "}" try: if tagname is not None: - local_path = check_for_s3( - template.replace(tagname, str(resolution))) + local_path = check_for_s3(template.replace(tagname, str(resolution))) except (IOError, OSError): local_path = None @@ -1017,25 +1171,27 @@ def resolve_resolution(resolution, template, template_name, tag=None): if local_path is None: if tagname is not None: - if template.startswith('s3:'): - ref_template = template.replace(tagname, '1mm') + if template.startswith("s3:"): + ref_template = template.replace(tagname, "1mm") local_path = check_for_s3(ref_template) else: local_path = get_highest_local_res(template, tagname) - elif tagname is None and template.startswith('s3:'): + elif tagname is None and template.startswith("s3:"): local_path = check_for_s3(template) else: local_path = template - resample = pe.Node(interface=afni.Resample(), - name=template_name, - mem_gb=0, - mem_x=(0.0115, 'in_file', 't')) + resample = pe.Node( + interface=afni.Resample(), + name=template_name, + mem_gb=0, + mem_x=(0.0115, "in_file", "t"), + ) resample.inputs.voxel_size = res_string_to_tuple(resolution) - resample.inputs.outputtype = 'NIFTI_GZ' - resample.inputs.resample_mode = 'Cu' + resample.inputs.outputtype = "NIFTI_GZ" + resample.inputs.resample_mode = "Cu" resample.inputs.in_file = local_path - resample.base_dir = '.' + resample.base_dir = "." resampled_template = resample.run() local_path = resampled_template.outputs.out_file @@ -1043,77 +1199,82 @@ def resolve_resolution(resolution, template, template_name, tag=None): return local_path -def create_anat_datasource(wf_name='anat_datasource'): - from CPAC.pipeline import nipype_pipeline_engine as pe +def create_anat_datasource(wf_name="anat_datasource"): import nipype.interfaces.utility as util + from CPAC.pipeline import nipype_pipeline_engine as pe + wf = pe.Workflow(name=wf_name) - inputnode = pe.Node(util.IdentityInterface( - fields=['subject', 'anat', 'creds_path', - 'dl_dir', 'img_type'], - mandatory_inputs=True), - name='inputnode') - - check_s3_node = pe.Node(function.Function(input_names=['file_path', - 'creds_path', - 'dl_dir', - 'img_type'], - output_names=['local_path'], - function=check_for_s3, - as_module=True), - name='check_for_s3') - - wf.connect(inputnode, 'anat', check_s3_node, 'file_path') - wf.connect(inputnode, 'creds_path', check_s3_node, 'creds_path') - wf.connect(inputnode, 'dl_dir', check_s3_node, 'dl_dir') - wf.connect(inputnode, 'img_type', check_s3_node, 'img_type') - - outputnode = pe.Node(util.IdentityInterface(fields=['subject', - 'anat']), - name='outputspec') - - wf.connect(inputnode, 'subject', outputnode, 'subject') - wf.connect(check_s3_node, 'local_path', outputnode, 'anat') + inputnode = pe.Node( + util.IdentityInterface( + fields=["subject", "anat", "creds_path", "dl_dir", "img_type"], + mandatory_inputs=True, + ), + name="inputnode", + ) + + check_s3_node = pe.Node( + function.Function( + input_names=["file_path", "creds_path", "dl_dir", "img_type"], + output_names=["local_path"], + function=check_for_s3, + as_module=True, + ), + name="check_for_s3", + ) + + wf.connect(inputnode, "anat", check_s3_node, "file_path") + wf.connect(inputnode, "creds_path", check_s3_node, "creds_path") + wf.connect(inputnode, "dl_dir", check_s3_node, "dl_dir") + wf.connect(inputnode, "img_type", check_s3_node, "img_type") + + outputnode = pe.Node( + util.IdentityInterface(fields=["subject", "anat"]), name="outputspec" + ) + + wf.connect(inputnode, "subject", outputnode, "subject") + wf.connect(check_s3_node, "local_path", outputnode, "anat") # Return the workflow return wf -def create_roi_mask_dataflow(masks, wf_name='datasource_roi_mask'): +def create_roi_mask_dataflow(masks, wf_name="datasource_roi_mask"): import os mask_dict = {} for mask_file in masks: + mask_file = mask_file.rstrip("\r\n") - mask_file = mask_file.rstrip('\r\n') - - if mask_file.strip() == '' or mask_file.startswith('#'): + if mask_file.strip() == "" or mask_file.startswith("#"): continue name, desc = lookup_identifier(mask_file) - if name == 'template': + if name == "template": base_file = os.path.basename(mask_file) try: - valid_extensions = ['.nii', '.nii.gz'] + valid_extensions = [".nii", ".nii.gz"] - base_name = [ - base_file[:-len(ext)] + base_name = next( + base_file[: -len(ext)] for ext in valid_extensions if base_file.endswith(ext) - ][0] + ) - for key in ['res', 'space']: + for key in ["res", "space"]: base_name = bids_remove_entity(base_name, key) except IndexError: # pylint: disable=raise-missing-from - raise ValueError('Error in spatial_map_dataflow: File ' - f'extension of {base_file} not ".nii" or ' - '.nii.gz') + raise ValueError( + "Error in spatial_map_dataflow: File " + f'extension of {base_file} not ".nii" or ' + ".nii.gz" + ) except Exception as e: raise e @@ -1121,54 +1282,56 @@ def create_roi_mask_dataflow(masks, wf_name='datasource_roi_mask'): base_name = format_identifier(name, desc) if base_name in mask_dict: - raise ValueError('Duplicate templates/atlases not allowed: ' - f'{mask_file} {mask_dict[base_name]}') + raise ValueError( + "Duplicate templates/atlases not allowed: " + f"{mask_file} {mask_dict[base_name]}" + ) mask_dict[base_name] = mask_file wf = pe.Workflow(name=wf_name) - inputnode = pe.Node(util.IdentityInterface(fields=['mask', - 'mask_file', - 'creds_path', - 'dl_dir'], - mandatory_inputs=True), - name='inputspec') + inputnode = pe.Node( + util.IdentityInterface( + fields=["mask", "mask_file", "creds_path", "dl_dir"], mandatory_inputs=True + ), + name="inputspec", + ) - mask_keys, mask_values = \ - zip(*mask_dict.items()) + mask_keys, mask_values = zip(*mask_dict.items()) inputnode.synchronize = True inputnode.iterables = [ - ('mask', mask_keys), - ('mask_file', mask_values), + ("mask", mask_keys), + ("mask_file", mask_values), ] - check_s3_node = pe.Node(function.Function(input_names=['file_path', - 'creds_path', - 'dl_dir', - 'img_type'], - output_names=['local_path'], - function=check_for_s3, - as_module=True), - name='check_for_s3') + check_s3_node = pe.Node( + function.Function( + input_names=["file_path", "creds_path", "dl_dir", "img_type"], + output_names=["local_path"], + function=check_for_s3, + as_module=True, + ), + name="check_for_s3", + ) - wf.connect(inputnode, 'mask_file', check_s3_node, 'file_path') - wf.connect(inputnode, 'creds_path', check_s3_node, 'creds_path') - wf.connect(inputnode, 'dl_dir', check_s3_node, 'dl_dir') - check_s3_node.inputs.img_type = 'mask' + wf.connect(inputnode, "mask_file", check_s3_node, "file_path") + wf.connect(inputnode, "creds_path", check_s3_node, "creds_path") + wf.connect(inputnode, "dl_dir", check_s3_node, "dl_dir") + check_s3_node.inputs.img_type = "mask" - outputnode = pe.Node(util.IdentityInterface(fields=['out_file', - 'out_name']), - name='outputspec') + outputnode = pe.Node( + util.IdentityInterface(fields=["out_file", "out_name"]), name="outputspec" + ) - wf.connect(check_s3_node, 'local_path', outputnode, 'out_file') - wf.connect(inputnode, 'mask', outputnode, 'out_name') + wf.connect(check_s3_node, "local_path", outputnode, "out_file") + wf.connect(inputnode, "mask", outputnode, "out_name") return wf -def create_spatial_map_dataflow(spatial_maps, wf_name='datasource_maps'): +def create_spatial_map_dataflow(spatial_maps, wf_name="datasource_maps"): import os wf = pe.Workflow(name=wf_name) @@ -1176,162 +1339,173 @@ def create_spatial_map_dataflow(spatial_maps, wf_name='datasource_maps'): spatial_map_dict = {} for spatial_map_file in spatial_maps: - - spatial_map_file = spatial_map_file.rstrip('\r\n') + spatial_map_file = spatial_map_file.rstrip("\r\n") base_file = os.path.basename(spatial_map_file) try: - valid_extensions = ['.nii', '.nii.gz'] + valid_extensions = [".nii", ".nii.gz"] - base_name = [ - base_file[:-len(ext)] + base_name = next( + base_file[: -len(ext)] for ext in valid_extensions if base_file.endswith(ext) - ][0] + ) if base_name in spatial_map_dict: raise ValueError( - 'Files with same name not allowed: %s %s' % ( - spatial_map_file, - spatial_map_dict[base_name] - ) + "Files with same name not allowed: %s %s" + % (spatial_map_file, spatial_map_dict[base_name]) ) spatial_map_dict[base_name] = spatial_map_file - except IndexError as e: - raise Exception('Error in spatial_map_dataflow: ' - 'File extension not in .nii and .nii.gz') + except IndexError: + raise Exception( + "Error in spatial_map_dataflow: " + "File extension not in .nii and .nii.gz" + ) - inputnode = pe.Node(util.IdentityInterface(fields=['spatial_map', - 'spatial_map_file', - 'creds_path', - 'dl_dir'], - mandatory_inputs=True), - name='inputspec') + inputnode = pe.Node( + util.IdentityInterface( + fields=["spatial_map", "spatial_map_file", "creds_path", "dl_dir"], + mandatory_inputs=True, + ), + name="inputspec", + ) - spatial_map_keys, spatial_map_values = \ - zip(*spatial_map_dict.items()) + spatial_map_keys, spatial_map_values = zip(*spatial_map_dict.items()) inputnode.synchronize = True inputnode.iterables = [ - ('spatial_map', spatial_map_keys), - ('spatial_map_file', spatial_map_values), + ("spatial_map", spatial_map_keys), + ("spatial_map_file", spatial_map_values), ] - check_s3_node = pe.Node(function.Function(input_names=['file_path', - 'creds_path', - 'dl_dir', - 'img_type'], - output_names=['local_path'], - function=check_for_s3, - as_module=True), - name='check_for_s3') + check_s3_node = pe.Node( + function.Function( + input_names=["file_path", "creds_path", "dl_dir", "img_type"], + output_names=["local_path"], + function=check_for_s3, + as_module=True, + ), + name="check_for_s3", + ) - wf.connect(inputnode, 'spatial_map_file', check_s3_node, 'file_path') - wf.connect(inputnode, 'creds_path', check_s3_node, 'creds_path') - wf.connect(inputnode, 'dl_dir', check_s3_node, 'dl_dir') - check_s3_node.inputs.img_type = 'mask' + wf.connect(inputnode, "spatial_map_file", check_s3_node, "file_path") + wf.connect(inputnode, "creds_path", check_s3_node, "creds_path") + wf.connect(inputnode, "dl_dir", check_s3_node, "dl_dir") + check_s3_node.inputs.img_type = "mask" - select_spatial_map = pe.Node(util.IdentityInterface(fields=['out_file', - 'out_name'], - mandatory_inputs=True), - name='select_spatial_map') + select_spatial_map = pe.Node( + util.IdentityInterface(fields=["out_file", "out_name"], mandatory_inputs=True), + name="select_spatial_map", + ) - wf.connect(check_s3_node, 'local_path', select_spatial_map, 'out_file') - wf.connect(inputnode, 'spatial_map', select_spatial_map, 'out_name') + wf.connect(check_s3_node, "local_path", select_spatial_map, "out_file") + wf.connect(inputnode, "spatial_map", select_spatial_map, "out_name") return wf -def create_grp_analysis_dataflow(wf_name='gp_dataflow'): - from CPAC.pipeline import nipype_pipeline_engine as pe +def create_grp_analysis_dataflow(wf_name="gp_dataflow"): import nipype.interfaces.utility as util + + from CPAC.pipeline import nipype_pipeline_engine as pe from CPAC.utils.datasource import select_model_files wf = pe.Workflow(name=wf_name) - inputnode = pe.Node(util.IdentityInterface(fields=['ftest', - 'grp_model', - 'model_name'], - mandatory_inputs=True), - name='inputspec') - - selectmodel = pe.Node(function.Function(input_names=['model', - 'ftest', - 'model_name'], - output_names=['fts_file', - 'con_file', - 'grp_file', - 'mat_file'], - function=select_model_files, - as_module=True), - name='selectnode') - - wf.connect(inputnode, 'ftest', - selectmodel, 'ftest') - wf.connect(inputnode, 'grp_model', - selectmodel, 'model') - wf.connect(inputnode, 'model_name', selectmodel, 'model_name') - - outputnode = pe.Node(util.IdentityInterface(fields=['fts', - 'grp', - 'mat', - 'con'], - mandatory_inputs=True), - name='outputspec') - - wf.connect(selectmodel, 'mat_file', - outputnode, 'mat') - wf.connect(selectmodel, 'grp_file', - outputnode, 'grp') - wf.connect(selectmodel, 'fts_file', - outputnode, 'fts') - wf.connect(selectmodel, 'con_file', - outputnode, 'con') + inputnode = pe.Node( + util.IdentityInterface( + fields=["ftest", "grp_model", "model_name"], mandatory_inputs=True + ), + name="inputspec", + ) + + selectmodel = pe.Node( + function.Function( + input_names=["model", "ftest", "model_name"], + output_names=["fts_file", "con_file", "grp_file", "mat_file"], + function=select_model_files, + as_module=True, + ), + name="selectnode", + ) + + wf.connect(inputnode, "ftest", selectmodel, "ftest") + wf.connect(inputnode, "grp_model", selectmodel, "model") + wf.connect(inputnode, "model_name", selectmodel, "model_name") + + outputnode = pe.Node( + util.IdentityInterface( + fields=["fts", "grp", "mat", "con"], mandatory_inputs=True + ), + name="outputspec", + ) + + wf.connect(selectmodel, "mat_file", outputnode, "mat") + wf.connect(selectmodel, "grp_file", outputnode, "grp") + wf.connect(selectmodel, "fts_file", outputnode, "fts") + wf.connect(selectmodel, "con_file", outputnode, "con") return wf def resample_func_roi(in_func, in_roi, realignment, identity_matrix): import os - import nibabel as nb + + import nibabel as nib + from CPAC.utils.monitoring.custom_logging import log_subprocess # load func and ROI dimension - func_img = nb.load(in_func) + func_img = nib.load(in_func) func_shape = func_img.shape - roi_img = nb.load(in_roi) + roi_img = nib.load(in_roi) roi_shape = roi_img.shape # check if func size = ROI size, return func and ROI; else resample using flirt if roi_shape != func_shape: - # resample func to ROI: in_file = func, reference = ROI - if 'func_to_ROI' in realignment: + if "func_to_ROI" in realignment: in_file = in_func reference = in_roi - out_file = os.path.join(os.getcwd(), in_file[in_file.rindex( - '/') + 1:in_file.rindex('.nii')] + '_resampled.nii.gz') + out_file = os.path.join( + os.getcwd(), + in_file[in_file.rindex("/") + 1 : in_file.rindex(".nii")] + + "_resampled.nii.gz", + ) out_func = out_file out_roi = in_roi - interp = 'trilinear' + interp = "trilinear" # resample ROI to func: in_file = ROI, reference = func - elif 'ROI_to_func' in realignment: + elif "ROI_to_func" in realignment: in_file = in_roi reference = in_func - out_file = os.path.join(os.getcwd(), in_file[in_file.rindex( - '/') + 1:in_file.rindex('.nii')] + '_resampled.nii.gz') + out_file = os.path.join( + os.getcwd(), + in_file[in_file.rindex("/") + 1 : in_file.rindex(".nii")] + + "_resampled.nii.gz", + ) out_func = in_func out_roi = out_file - interp = 'nearestneighbour' - - cmd = ['flirt', '-in', in_file, - '-ref', reference, - '-out', out_file, - '-interp', interp, - '-applyxfm', '-init', identity_matrix] + interp = "nearestneighbour" + + cmd = [ + "flirt", + "-in", + in_file, + "-ref", + reference, + "-out", + out_file, + "-interp", + interp, + "-applyxfm", + "-init", + identity_matrix, + ] log_subprocess(cmd) else: diff --git a/CPAC/utils/datatypes.py b/CPAC/utils/datatypes.py index 9298d8a7cf..2b29d8741f 100644 --- a/CPAC/utils/datatypes.py +++ b/CPAC/utils/datatypes.py @@ -1,4 +1,6 @@ -"""Custom datatypes for C-PAC""" +"""Custom datatypes for C-PAC.""" + + class ItemFromList: # pylint: disable=too-few-public-methods """Coerce single-item lists into just the only item in the list. Returns item if item is not a list, set, or tuple. @@ -15,22 +17,26 @@ class ItemFromList: # pylint: disable=too-few-public-methods >>> ItemFromList('string') 'string' """ + def __new__(cls, list_of_one, msg=None): - """Initialize item from list""" + """Initialize item from list.""" from voluptuous import CoerceInvalid, Length, LengthInvalid + if not isinstance(list_of_one, (list, set, tuple)): return list_of_one try: Length(max=1)(list_of_one) except LengthInvalid as length_invalid: raise CoerceInvalid( - f'Cannot coerce list of length {len(list_of_one)} to item' - if msg is None else msg) from length_invalid + f"Cannot coerce list of length {len(list_of_one)} to item" + if msg is None + else msg + ) from length_invalid return list_of_one[0] class ListFromItem(list): - """Subclass of list to coerce non-lists into lists + """Subclass of list to coerce non-lists into lists. Examples -------- @@ -53,8 +59,9 @@ class ListFromItem(list): >>> ListFromItem(None) [] """ + def __init__(self, *args, **kwargs): - """Initialize ListFromItem""" + """Initialize ListFromItem.""" if len(args) == 1 and not isinstance(args[0], (list, tuple)): if args[0] is None: args = () diff --git a/CPAC/utils/docs.py b/CPAC/utils/docs.py index 57ff3db58c..859fd0c727 100644 --- a/CPAC/utils/docs.py +++ b/CPAC/utils/docs.py @@ -15,9 +15,9 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . """Utilties for documentation.""" -import ast from urllib import request from urllib.error import ContentTooShortError, HTTPError, URLError + from CPAC import __version__ from CPAC.utils import versioning @@ -41,41 +41,46 @@ def docstring_parameter(*args, **kwargs): >>> print(how_about_now.__doc__) How about { this }? """ + def dec(obj): if obj.__doc__ is None: - obj.__doc__ = '' + obj.__doc__ = "" obj.__doc__ = obj.__doc__.format(*args, **kwargs) return obj + return dec def _docs_url_prefix(): - """Function to determine the URL prefix for this version of C-PAC""" + """Function to determine the URL prefix for this version of C-PAC.""" + def _url(url_version): - return f'https://fcp-indi.github.io/docs/{url_version}' - url_version = f'v{__version__}' + return f"https://fcp-indi.github.io/docs/{url_version}" + + url_version = f"v{__version__}" try: request.urlopen( # pylint: disable=consider-using-with - _url(url_version)) + _url(url_version) + ) except (ContentTooShortError, HTTPError, URLError): - if 'dev' in url_version: - url_version = 'nightly' + if "dev" in url_version: + url_version = "nightly" else: - url_version = 'latest' + url_version = "latest" return _url(url_version) def version_report() -> str: - """A formatted block of versions included in CPAC's environment""" + """A formatted block of versions included in CPAC's environment.""" version_list = [] for pkg, version in versioning.REPORTED.items(): - version_list.append(f'{pkg}: {version}') - if pkg == 'Python': - version_list.append(' Python packages') - version_list.append(' ---------------') + version_list.append(f"{pkg}: {version}") + if pkg == "Python": + version_list.append(" Python packages") + version_list.append(" ---------------") for ppkg, pversion in versioning.PYTHON_PACKAGES.items(): - version_list.append(f' {ppkg}: {pversion}') - return '\n'.join(version_list) + version_list.append(f" {ppkg}: {pversion}") + return "\n".join(version_list) DOCS_URL_PREFIX = _docs_url_prefix() diff --git a/CPAC/utils/extract_data.py b/CPAC/utils/extract_data.py index dee70b4be2..6cfdcd7f88 100644 --- a/CPAC/utils/extract_data.py +++ b/CPAC/utils/extract_data.py @@ -1,17 +1,19 @@ -import sys -import os import glob -import string import logging +import os +import string +import sys + import yaml + def extract_data(c, param_map): """ Method to generate a CPAC input subject list python file. The method extracts anatomical and functional data for each site( if multiple site) and/or scan parameters for each site and put it into - a data structure read by python + a data structure read by python. Example: subjects_list =[ @@ -49,13 +51,13 @@ def extract_data(c, param_map): """ - #method to read each line of the file into list - #returns list + # method to read each line of the file into list + # returns list def get_list(arg): if isinstance(arg, list): ret_list = arg else: - ret_list = [fline.rstrip('\r\n') for fline in open(arg, 'r').readlines()] + ret_list = [fline.rstrip("\r\n") for fline in open(arg, "r").readlines()] return ret_list @@ -67,13 +69,14 @@ def get_list(arg): if c.subjectList is not None: subject_list = get_list(c.subjectList) - #check if Template is correct + # check if Template is correct def checkTemplate(template): - - if template.count('%s') != 2: - msg = "Please provide '%s' in the template" \ - "where your site and subjects are present"\ - "Please see examples" + if template.count("%s") != 2: + msg = ( + "Please provide '%s' in the template" + "where your site and subjects are present" + "Please see examples" + ) logging.exception(msg) raise Exception(msg) @@ -86,28 +89,36 @@ def checkTemplate(template): raise Exception(msg) def get_site_list(path): - base, relative = path.split('%s') - sites = os.listdir(base) - return sites + base, relative = path.split("%s") + return os.listdir(base) def check_length(scan_name, file_name): - if len(file_name) > 30: - msg = "filename- %s is too long."\ - "It should not be more than 30 characters."%(file_name) + msg = ( + "filename- %s is too long." + "It should not be more than 30 characters." % (file_name) + ) logging.exception(msg) raise Exception(msg) - if len(scan_name) - len(os.path.splitext(os.path.splitext(file_name)[0])[0])>= 40: - msg = "scan name %s is too long."\ - "It should not be more than 20 characters"\ - %(scan_name.replace("_"+os.path.splitext(os.path.splitext(file_name)[0])[0], '')) + if ( + len(scan_name) - len(os.path.splitext(os.path.splitext(file_name)[0])[0]) + >= 40 + ): + msg = ( + "scan name %s is too long." + "It should not be more than 20 characters" + % ( + scan_name.replace( + "_" + os.path.splitext(os.path.splitext(file_name)[0])[0], "" + ) + ) + ) logging.exception(msg) raise Exception(msg) def create_site_subject_mapping(base, relative): - - #mapping between site and subject + # mapping between site and subject site_subject_map = {} base_path_list = [] @@ -117,25 +128,24 @@ def create_site_subject_mapping(base, relative): site_list = get_site_list(base) for site in site_list: - paths = glob.glob(string.replace(base, '%s', site)) + paths = glob.glob(string.replace(base, "%s", site)) base_path_list.extend(paths) for path in paths: for sub in os.listdir(path): - #check if subject is present in subject_list + # check if subject is present in subject_list if subject_list: if sub in subject_list and sub not in exclusion_list: site_subject_map[sub] = site elif sub not in exclusion_list: - if sub not in '.DS_Store': + if sub not in ".DS_Store": site_subject_map[sub] = site return base_path_list, site_subject_map - #method to split the input template path - #into base, path before subject directory - #and relative, path after subject directory + # method to split the input template path + # into base, path before subject directory + # and relative, path after subject directory def getPath(template): - checkTemplate(template) base, relative = template.rsplit("%s", 1) base, subject_map = create_site_subject_mapping(base, relative) @@ -143,14 +153,17 @@ def getPath(template): relative = relative.lstrip("/") return base, relative, subject_map - #get anatomical base path and anatomical relative path + # get anatomical base path and anatomical relative path anat_base, anat_relative = getPath(c.anatomicalTemplate)[:2] - #get functional base path, functional relative path and site-subject map + # get functional base path, functional relative path and site-subject map func_base, func_relative, subject_map = getPath(c.functionalTemplate) if not anat_base: - msg = "Anatomical Data template incorrect. No such file or directory %s", anat_base + msg = ( + "Anatomical Data template incorrect. No such file or directory %s", + anat_base, + ) logging.exception(msg) raise Exception(msg) @@ -160,55 +173,67 @@ def getPath(template): raise Exception(msg) if len(anat_base) != len(func_base): - msg1 = "Some sites are missing, Please check your template"\ - , anat_base, "!=", func_base + msg1 = ( + "Some sites are missing, Please check your template", + anat_base, + "!=", + func_base, + ) logging.exception(msg1) - msg2 = " Base length Unequal. Some sites are missing."\ - "extract_data doesn't script support this.Please" \ - "Provide your own subjects_list file" + msg2 = ( + " Base length Unequal. Some sites are missing." + "extract_data doesn't script support this.Please" + "Provide your own subjects_list file" + ) logging.exception(msg2) raise Exception(msg2) - #calculate the length of relative paths(path after subject directory) - func_relative_len = len(func_relative.split('/')) - anat_relative_len = len(anat_relative.split('/')) + # calculate the length of relative paths(path after subject directory) + func_relative_len = len(func_relative.split("/")) + anat_relative_len = len(anat_relative.split("/")) def check_for_sessions(relative_path, path_length): - """ - Method to check if there are sessions present - - """ - #default + """Method to check if there are sessions present.""" + # default session_present = False - session_path = 'session_1' + session_path = "session_1" - #session present if path_length is equal to 3 + # session present if path_length is equal to 3 if path_length == 3: - relative_path_list = relative_path.split('/') + relative_path_list = relative_path.split("/") session_path = relative_path_list[0] relative_path = string.join(relative_path_list[1:], "/") session_present = True elif path_length > 3: - msg = "extract_data script currently doesn't support this directory structure."\ - "Please provide the subjects_list file to run CPAC."\ - "For more information refer to manual" + msg = ( + "extract_data script currently doesn't support this directory structure." + "Please provide the subjects_list file to run CPAC." + "For more information refer to manual" + ) logging.exception(msg) raise Exception(msg) return session_present, session_path, relative_path - func_session_present, func_session_path, func_relative = \ - check_for_sessions(func_relative, func_relative_len) + func_session_present, func_session_path, func_relative = check_for_sessions( + func_relative, func_relative_len + ) - anat_session_present, anat_session_path, anat_relative = \ - check_for_sessions(anat_relative, anat_relative_len) + anat_session_present, anat_session_path, anat_relative = check_for_sessions( + anat_relative, anat_relative_len + ) - f = open(os.path.join(c.outputSubjectListLocation, "CPAC_subject_list_%s.yml" % c.subjectListName), 'wb') + f = open( + os.path.join( + c.outputSubjectListLocation, "CPAC_subject_list_%s.yml" % c.subjectListName + ), + "wb", + ) def fetch_path(i, anat_sub, func_sub, session_id): """ Method to extract anatomical and functional - path for a session and print to file + path for a session and print to file. Parameters ---------- @@ -227,7 +252,6 @@ def fetch_path(i, anat_sub, func_sub, session_id): ------ Exception """ - try: def print_begin_of_file(sub, session_id): @@ -238,20 +262,52 @@ def print_begin_of_file(sub, session_id): def print_end_of_file(sub): if param_map is not None: try: - logging.debug("site for sub %s -> %s" %(sub, subject_map.get(sub))) - logging.debug("scan parameters for the above site %s"%param_map.get(subject_map.get(sub))) + logging.debug( + "site for sub %s -> %s" % (sub, subject_map.get(sub)) + ) + logging.debug( + "scan parameters for the above site %s" + % param_map.get(subject_map.get(sub)) + ) print(" scan_parameters:", file=f) - print(" tr: '" + param_map.get(subject_map.get(sub))[4] + "'", file=f) - print(" acquisition: '" + param_map.get(subject_map.get(sub))[0] + "'", file=f) - print(" reference: '" + param_map.get(subject_map.get(sub))[3] + "'", file=f) - print(" first_tr: '" + param_map.get(subject_map.get(sub))[1] + "'", file=f) - print(" last_tr: '" + param_map.get(subject_map.get(sub))[2] + "'", file=f) + print( + " tr: '" + + param_map.get(subject_map.get(sub))[4] + + "'", + file=f, + ) + print( + " acquisition: '" + + param_map.get(subject_map.get(sub))[0] + + "'", + file=f, + ) + print( + " reference: '" + + param_map.get(subject_map.get(sub))[3] + + "'", + file=f, + ) + print( + " first_tr: '" + + param_map.get(subject_map.get(sub))[1] + + "'", + file=f, + ) + print( + " last_tr: '" + + param_map.get(subject_map.get(sub))[2] + + "'", + file=f, + ) except: - msg = " No Parameter values for the %s site is defined in the scan"\ - " parameters csv file" %subject_map.get(sub) + msg = ( + " No Parameter values for the %s site is defined in the scan" + " parameters csv file" % subject_map.get(sub) + ) raise ValueError(msg) - #get anatomical file + # get anatomical file anat_base_path = os.path.join(anat_base[i], anat_sub) func_base_path = os.path.join(func_base[i], func_sub) @@ -266,28 +322,35 @@ def print_end_of_file(sub): print(" anat: '" + os.path.realpath(anat[0]) + "'", file=f) print(" rest: ", file=f) - #iterate for each rest session + # iterate for each rest session for iter in func: - #get scan_id - iterable = os.path.splitext(os.path.splitext(iter.replace(func_base_path, '').lstrip("/"))[0])[0] + # get scan_id + iterable = os.path.splitext( + os.path.splitext(iter.replace(func_base_path, "").lstrip("/"))[ + 0 + ] + )[0] iterable = iterable.replace("/", "_") check_length(iterable, os.path.basename(os.path.realpath(iter))) - print(" " + iterable + ": '" + os.path.realpath(iter) + "'", file=f) + print( + " " + iterable + ": '" + os.path.realpath(iter) + "'", + file=f, + ) print_end_of_file(anat_sub.split("/")[0]) else: - logging.debug("skipping subject %s"%anat_sub.split("/")[0]) + logging.debug("skipping subject %s" % anat_sub.split("/")[0]) except ValueError: - logging.exception(ValueError.message) raise except Exception as e: - - err_msg = 'Exception while felching anatomical and functional ' \ - 'paths: \n' + str(e) + err_msg = ( + "Exception while felching anatomical and functional " + "paths: \n" + str(e) + ) logging.exception(err_msg) raise Exception(err_msg) @@ -295,7 +358,7 @@ def print_end_of_file(sub): def walk(index, sub): """ Method which walks across each subject - path in the data site path + path in the data site path. Parameters ---------- @@ -309,11 +372,14 @@ def walk(index, sub): Exception """ try: - if func_session_present: - #if there are sessions + # if there are sessions if "*" in func_session_path: - session_list = glob.glob(os.path.join(func_base[index], os.path.join(sub, func_session_path))) + session_list = glob.glob( + os.path.join( + func_base[index], os.path.join(sub, func_session_path) + ) + ) else: session_list = [func_session_path] @@ -322,52 +388,57 @@ def walk(index, sub): session_id = os.path.basename(session) if anat_session_present: if func_session_path == anat_session_path: - fetch_path(index, os.path.join(sub, session_id), os.path.join(sub, session_id), session_id) + fetch_path( + index, + os.path.join(sub, session_id), + os.path.join(sub, session_id), + session_id, + ) else: - fetch_path(index, os.path.join(sub, anat_session_path), os.path.join(sub, session_id), session_id) + fetch_path( + index, + os.path.join(sub, anat_session_path), + os.path.join(sub, session_id), + session_id, + ) else: - fetch_path(index, sub, os.path.join(sub, session_id), session_id) + fetch_path( + index, sub, os.path.join(sub, session_id), session_id + ) else: logging.debug("Skipping subject %s", sub) else: logging.debug("No sessions") - session_id = '' + session_id = "" fetch_path(index, sub, sub, session_id) except Exception: - logging.exception(Exception.message) raise except: - - err_msg = 'Please make sessions are consistent across all ' \ - 'subjects.\n\n' + err_msg = "Please make sessions are consistent across all " "subjects.\n\n" logging.exception(err_msg) raise Exception(err_msg) - try: for i in range(len(anat_base)): for sub in os.listdir(anat_base[i]): - #check if subject is present in subject_list + # check if subject is present in subject_list if subject_list: if sub in subject_list and sub not in exclusion_list: logging.debug("extracting data for subject: %s", sub) walk(i, sub) - #check that subject is not in exclusion list - elif sub not in exclusion_list and sub not in '.DS_Store': + # check that subject is not in exclusion list + elif sub not in exclusion_list and sub not in ".DS_Store": logging.debug("extracting data for subject: %s", sub) walk(i, sub) - - name = os.path.join(c.outputSubjectListLocation, 'CPAC_subject_list.yml') - print("Extraction Successfully Completed...Input Subjects_list for CPAC - %s" % name) + os.path.join(c.outputSubjectListLocation, "CPAC_subject_list.yml") except Exception: - logging.exception(Exception.message) raise @@ -376,23 +447,23 @@ def walk(index, sub): def generate_supplementary_files(data_config_outdir, data_config_name): - """ Method to generate phenotypic template file - and subject list for group analysis + and subject list for group analysis. """ - + import csv import os + from sets import Set - import csv data_config_path = os.path.join(data_config_outdir, data_config_name) try: - subjects_list = yaml.safe_load(open(data_config_path, 'r')) + subjects_list = yaml.safe_load(open(data_config_path, "r")) except: - err = "\n\n[!] Data configuration file couldn't be read!\nFile " \ - "path: {0}\n".format(data_config_path) + "\n\n[!] Data configuration file couldn't be read!\nFile " "path: {0}\n".format( + data_config_path + ) subject_scan_set = Set() subID_set = Set() @@ -403,41 +474,38 @@ def generate_supplementary_files(data_config_outdir, data_config_name): try: for sub in subjects_list: - if sub['unique_id']: - subject_id = sub['subject_id'] + "_" + sub['unique_id'] + if sub["unique_id"]: + subject_id = sub["subject_id"] + "_" + sub["unique_id"] else: - subject_id = sub['subject_id'] + subject_id = sub["subject_id"] try: - for scan in sub['func']: + for scan in sub["func"]: subject_scan_set.add((subject_id, scan)) - subID_set.add(sub['subject_id']) - session_set.add(sub['unique_id']) + subID_set.add(sub["subject_id"]) + session_set.add(sub["unique_id"]) subject_set.add(subject_id) scan_set.add(scan) except KeyError: try: - for scan in sub['rest']: + for scan in sub["rest"]: subject_scan_set.add((subject_id, scan)) - subID_set.add(sub['subject_id']) - session_set.add(sub['unique_id']) + subID_set.add(sub["subject_id"]) + session_set.add(sub["unique_id"]) subject_set.add(subject_id) scan_set.add(scan) except KeyError: # one of the participants in the subject list has no # functional scans - subID_set.add(sub['subject_id']) - session_set.add(sub['unique_id']) + subID_set.add(sub["subject_id"]) + session_set.add(sub["unique_id"]) subject_set.add(subject_id) - except TypeError as e: - print('Subject list could not be populated!') - print('This is most likely due to a mis-formatting in your '\ - 'inclusion and/or exclusion subjects txt file or your '\ - 'anatomical and/or functional path templates.') - print('Error: %s' % e) - err_str = 'Check formatting of your anatomical/functional path '\ - 'templates and inclusion/exclusion subjects text files' + except TypeError: + err_str = ( + "Check formatting of your anatomical/functional path " + "templates and inclusion/exclusion subjects text files" + ) raise TypeError(err_str) for item in subject_scan_set: @@ -458,28 +526,23 @@ def generate_supplementary_files(data_config_outdir, data_config_name): data_list.append(list1) # generate the phenotypic file templates for group analysis - file_name = os.path.join(data_config_outdir, 'phenotypic_template_%s.csv' - % data_config_name) + file_name = os.path.join( + data_config_outdir, "phenotypic_template_%s.csv" % data_config_name + ) try: - f = open(file_name, 'wb') + f = open(file_name, "wb") except: - print('\n\nCPAC says: I couldn\'t save this file to your drive:\n') - print(file_name, '\n\n') - print('Make sure you have write access? Then come back. Don\'t ' \ - 'worry.. I\'ll wait.\n\n') raise IOError writer = csv.writer(f) - writer.writerow(['participant', 'EV1', '..']) + writer.writerow(["participant", "EV1", ".."]) for sub in sorted(subID_set): - writer.writerow([sub, '']) + writer.writerow([sub, ""]) f.close() - print("Template Phenotypic file for group analysis - %s" % file_name) - """ # generate the phenotypic file templates for repeated measures if (len(session_set) > 1) and (len(scan_set) > 1): @@ -566,24 +629,17 @@ def generate_supplementary_files(data_config_outdir, data_config_name): """ # generate the group analysis subject lists - file_name = os.path.join(data_config_outdir, - 'participant_list_group_analysis_%s.txt' - % data_config_name) + file_name = os.path.join( + data_config_outdir, "participant_list_group_analysis_%s.txt" % data_config_name + ) try: - with open(file_name, 'w') as f: + with open(file_name, "w") as f: for sub in sorted(subID_set): print(sub, file=f) except: - print('\n\nCPAC says: I couldn\'t save this file to your drive:\n') - print(file_name, '\n\n') - print('Make sure you have write access? Then come back. Don\'t ' \ - 'worry.. I\'ll wait.\n\n') raise IOError - print("Participant list required later for group analysis - %s\n\n" \ - % file_name) - def read_csv(csv_input): """ @@ -591,26 +647,26 @@ def read_csv(csv_input): 'Acquisition' 'Reference' 'Site' - 'TR (seconds)' + 'TR (seconds)'. """ - import csv from collections import defaultdict + import csv + try: reader = csv.DictReader(open(csv_input, "U")) dict_labels = defaultdict(list) for line in reader: - csv_dict = dict((k.lower(), v) for k, v in line.items()) - dict_labels[csv_dict.get('site')] = [ - csv_dict[key] for key in sorted(list( - csv_dict.keys() - )) if key != 'site' and key != 'scan' + csv_dict = {k.lower(): v for k, v in line.items()} + dict_labels[csv_dict.get("site")] = [ + csv_dict[key] + for key in sorted(csv_dict.keys()) + if key not in ("site", "scan") ] if len(dict_labels) < 1: - msg ="Scan Parameters File is either empty"\ - "or missing header" + msg = "Scan Parameters File is either empty" "or missing header" logging.exception(msg) raise Exception(msg) @@ -629,10 +685,12 @@ def read_csv(csv_input): """ Class to set dictionary keys as map attributes """ + + class Configuration(object): def __init__(self, config_map): for key in config_map: - if config_map[key] == 'None': + if config_map[key] == "None": config_map[key] = None setattr(self, key, config_map[key]) @@ -640,34 +698,35 @@ def __init__(self, config_map): def run(data_config): """ Run method takes data_config - file as the input argument + file as the input argument. """ root = logging.getLogger() if root.handlers: for handler in root.handlers: root.removeHandler(handler) - logging.basicConfig(filename=os.path.join(os.getcwd(), 'extract_data_logs.log'), filemode='w', level=logging.DEBUG,\ - format="%(levelname)s %(asctime)s %(lineno)d %(message)s") - - print("For any errors or messages check the log file - %s"\ - % os.path.join(os.getcwd(), 'extract_data_logs.log')) + logging.basicConfig( + filename=os.path.join(os.getcwd(), "extract_data_logs.log"), + filemode="w", + level=logging.DEBUG, + format="%(levelname)s %(asctime)s %(lineno)d %(message)s", + ) - c = Configuration(yaml.safe_load(open(os.path.realpath(data_config), 'r'))) + c = Configuration(yaml.safe_load(open(os.path.realpath(data_config), "r"))) if c.scanParametersCSV is not None: - s_param_map = read_csv(c.scanParametersCSV) + read_csv(c.scanParametersCSV) else: - logging.debug("no scan parameters csv included\n"\ - "make sure you turn off slice timing correction option\n"\ - "in CPAC configuration\n") - s_param_map = None + logging.debug( + "no scan parameters csv included\n" + "make sure you turn off slice timing correction option\n" + "in CPAC configuration\n" + ) generate_supplementary_files(c.outputSubjectListLocation, c.subjectListName) if __name__ == "__main__": if len(sys.argv) != 2: - print("Usage: python extract_data.py data_config.yml") sys.exit() else: run(sys.argv[1]) diff --git a/CPAC/utils/extract_data_multiscan.py b/CPAC/utils/extract_data_multiscan.py index 80687b9981..d24bc9a0a2 100644 --- a/CPAC/utils/extract_data_multiscan.py +++ b/CPAC/utils/extract_data_multiscan.py @@ -1,15 +1,17 @@ -import os import glob +import os import string + import yaml + def extract_data(c, param_map): """ Method to generate a CPAC input subject list python file. The method extracts anatomical functional data and scan parameters for each site( if multiple site) and for each scan - and put it into a data structure read by python + and put it into a data structure read by python. Note: ----- @@ -60,13 +62,13 @@ def extract_data(c, param_map): ] """ - #method to read each line of the file into list - #returns list + # method to read each line of the file into list + # returns list def get_list(arg): if isinstance(arg, list): ret_list = arg else: - ret_list = [fline.rstrip('\r\n') for fline in open(arg, 'r').readlines()] + ret_list = [fline.rstrip("\r\n") for fline in open(arg, "r").readlines()] return ret_list @@ -78,13 +80,14 @@ def get_list(arg): if c.subjectList is not None: subject_list = get_list(c.subjectList) - #check if Template is correct + # check if Template is correct def checkTemplate(template): - - if template.count('%s') != 2: - raise Exception("Please provide '%s' in the template" \ - "where your site and subjects are present"\ - "Please see examples") + if template.count("%s") != 2: + raise Exception( + "Please provide '%s' in the template" + "where your site and subjects are present" + "Please see examples" + ) filename, ext = os.path.splitext(os.path.basename(template)) ext = os.path.splitext(filename)[1] + ext @@ -93,28 +96,34 @@ def checkTemplate(template): raise Exception("Invalid file name", os.path.basename(template)) def get_site_list(path): - base = path.split('%s')[0] - sites = os.listdir(base) - return sites + base = path.split("%s")[0] + return os.listdir(base) def check_length(scan_name, file_name): - if len(file_name) > 30: - msg = "filename- %s is too long."\ - "It should not be more than 30 characters."%(file_name) + msg = ( + "filename- %s is too long." + "It should not be more than 30 characters." % (file_name) + ) raise Exception(msg) - if len(scan_name) - len(os.path.splitext(os.path.splitext(file_name)[0])[0])>= 20: - msg = "scan name %s is too long."\ - "It should not be more than 20 characters"\ - %(scan_name.replace("_"+os.path.splitext(os.path.splitext(file_name)[0])[0], '')) + if ( + len(scan_name) - len(os.path.splitext(os.path.splitext(file_name)[0])[0]) + >= 20 + ): + msg = ( + "scan name %s is too long." + "It should not be more than 20 characters" + % ( + scan_name.replace( + "_" + os.path.splitext(os.path.splitext(file_name)[0])[0], "" + ) + ) + ) raise Exception(msg) - - def create_site_subject_mapping(base, relative): - - #mapping between site and subject + # mapping between site and subject site_subject_map = {} base_path_list = [] @@ -124,25 +133,24 @@ def create_site_subject_mapping(base, relative): site_list = get_site_list(base) for site in site_list: - paths = glob.glob(string.replace(base, '%s', site)) + paths = glob.glob(string.replace(base, "%s", site)) base_path_list.extend(paths) for path in paths: for sub in os.listdir(path): - #check if subject is present in subject_list + # check if subject is present in subject_list if subject_list: if sub in subject_list and sub not in exclusion_list: site_subject_map[sub] = site elif sub not in exclusion_list: - if sub not in '.DS_Store': + if sub not in ".DS_Store": site_subject_map[sub] = site return base_path_list, site_subject_map - #method to split the input template path - #into base, path before subject directory - #and relative, path after subject directory + # method to split the input template path + # into base, path before subject directory + # and relative, path after subject directory def getPath(template): - checkTemplate(template) base, relative = template.rsplit("%s", 1) base, subject_map = create_site_subject_mapping(base, relative) @@ -150,71 +158,70 @@ def getPath(template): relative = relative.lstrip("/") return base, relative, subject_map - #get anatomical base path and anatomical relative path + # get anatomical base path and anatomical relative path anat_base, anat_relative = getPath(c.anatomicalTemplate)[:2] - #get functional base path, functional relative path and site-subject map + # get functional base path, functional relative path and site-subject map func_base, func_relative, subject_map = getPath(c.functionalTemplate) if not anat_base: - print("No such file or directory ", anat_base) raise Exception("Anatomical Data template incorrect") if not func_base: - print("No such file or directory", func_base) raise Exception("Functional Data template incorrect") if len(anat_base) != len(func_base): - print("Some sites are missing, Please check your"\ - "template", anat_base, "!=", func_base) - raise Exception(" Base length Unequal. Some sites are missing."\ - "extract_data doesn't script support this.Please" \ - "Provide your own subjects_list file") + raise Exception( + " Base length Unequal. Some sites are missing." + "extract_data doesn't script support this.Please" + "Provide your own subjects_list file" + ) - #calculate the length of relative paths(path after subject directory) - func_relative_len = len(func_relative.split('/')) - anat_relative_len = len(anat_relative.split('/')) + # calculate the length of relative paths(path after subject directory) + func_relative_len = len(func_relative.split("/")) + anat_relative_len = len(anat_relative.split("/")) def check_for_sessions(relative_path, path_length): - """ - Method to check if there are sessions present - - """ - #default + """Method to check if there are sessions present.""" + # default session_present = False - session_path = 'session_1' + session_path = "session_1" - #session present if path_length is equal to 3 + # session present if path_length is equal to 3 if path_length == 3: - relative_path_list = relative_path.split('/') + relative_path_list = relative_path.split("/") session_path = relative_path_list[0] relative_path = string.join(relative_path_list[1:], "/") session_present = True elif path_length > 3: - raise Exception("extract_data script currently doesn't support"\ - "this directory structure.Please provide the"\ - "subjects_list file to run CPAC." \ - "For more information refer to manual") + raise Exception( + "extract_data script currently doesn't support" + "this directory structure.Please provide the" + "subjects_list file to run CPAC." + "For more information refer to manual" + ) return session_present, session_path, relative_path -# if func_relative_len!= anat_relative_len: -# raise Exception(" extract_data script currently doesn't"\ -# "support different relative paths for"\ -# "Anatomical and functional files") + # if func_relative_len!= anat_relative_len: + # raise Exception(" extract_data script currently doesn't"\ + # "support different relative paths for"\ + # "Anatomical and functional files") - func_session_present, func_session_path, func_relative = \ - check_for_sessions(func_relative, func_relative_len) + func_session_present, func_session_path, func_relative = check_for_sessions( + func_relative, func_relative_len + ) - anat_session_present, anat_session_path, anat_relative = \ - check_for_sessions(anat_relative, anat_relative_len) + anat_session_present, anat_session_path, anat_relative = check_for_sessions( + anat_relative, anat_relative_len + ) - f = open(os.path.join(c.outputSubjectListLocation, "CPAC_subject_list.yml"), 'wb') + f = open(os.path.join(c.outputSubjectListLocation, "CPAC_subject_list.yml"), "wb") def fetch_path(i, anat_sub, func_sub, session_id): """ Method to extract anatomical and functional - path for a session and print to file + path for a session and print to file. Parameters ---------- @@ -233,7 +240,6 @@ def fetch_path(i, anat_sub, func_sub, session_id): ------ Exception """ - try: def print_begin_of_file(sub, session_id): @@ -243,17 +249,27 @@ def print_begin_of_file(sub, session_id): def print_end_of_file(sub, scan_list): if param_map is not None: + def print_scan_param(index): try: for scan in scan_list: - print(" " + scan[1] + ": '" + \ - param_map.get((subject_map.get(sub), scan[0]))[index] + "'", file=f) + print( + " " + + scan[1] + + ": '" + + param_map.get((subject_map.get(sub), scan[0]))[ + index + ] + + "'", + file=f, + ) except: - raise Exception(" No Parameter values for the %s site and %s scan is defined in the scan"\ - " parameters csv file" % (subject_map.get(sub), scan[0])) + raise Exception( + " No Parameter values for the %s site and %s scan is defined in the scan" + " parameters csv file" % (subject_map.get(sub), scan[0]) + ) - print("site for sub", sub, "->", subject_map.get(sub)) print(" scan_parameters: ", file=f) print(" tr:", file=f) print_scan_param(4) @@ -266,8 +282,7 @@ def print_scan_param(index): print(" last_tr:", file=f) print_scan_param(2) - - #get anatomical file + # get anatomical file anat_base_path = os.path.join(anat_base[i], anat_sub) func_base_path = os.path.join(func_base[i], func_sub) @@ -282,14 +297,18 @@ def print_scan_param(index): print(" anat: '" + anat[0] + "'", file=f) print(" rest: ", file=f) - #iterate for each rest session + # iterate for each rest session for iter in func: - #get scan_id - iterable = os.path.splitext(os.path.splitext(iter.replace(func_base_path,'').lstrip("/"))[0])[0] + # get scan_id + iterable = os.path.splitext( + os.path.splitext(iter.replace(func_base_path, "").lstrip("/"))[ + 0 + ] + )[0] scan_name = iterable.replace("/", "_") scan_list.append((os.path.dirname(iterable), scan_name)) check_length(scan_name, os.path.basename(iter)) - print(" " + scan_name + ": '" + iter + "'", file=f) + print(" " + scan_name + ": '" + iter + "'", file=f) print_end_of_file(anat_sub.split("/")[0], scan_list) except Exception: @@ -298,7 +317,7 @@ def print_scan_param(index): def walk(index, sub): """ Method which walks across each subject - path in the data site path + path in the data site path. Parameters ---------- @@ -312,11 +331,14 @@ def walk(index, sub): Exception """ try: - if func_session_present: - #if there are sessions + # if there are sessions if "*" in func_session_path: - session_list = glob.glob(os.path.join(func_base[index], os.path.join(sub, func_session_path))) + session_list = glob.glob( + os.path.join( + func_base[index], os.path.join(sub, func_session_path) + ) + ) else: session_list = [func_session_path] @@ -324,38 +346,44 @@ def walk(index, sub): session_id = os.path.basename(session) if anat_session_present: if func_session_path == anat_session_path: - fetch_path(index, os.path.join(sub, session_id), os.path.join(sub, session_id), session_id) + fetch_path( + index, + os.path.join(sub, session_id), + os.path.join(sub, session_id), + session_id, + ) else: - fetch_path(index, os.path.join(sub, anat_session_path), os.path.join(sub, session_id), session_id) + fetch_path( + index, + os.path.join(sub, anat_session_path), + os.path.join(sub, session_id), + session_id, + ) else: - fetch_path(index, sub, os.path.join(sub, session_id), session_id) + fetch_path( + index, sub, os.path.join(sub, session_id), session_id + ) else: - print("No sessions") - session_id = '' + session_id = "" fetch_path(index, sub, sub, session_id) except Exception: raise except: - print("Please make sessions are consistent across all subjects") raise try: for i in range(len(anat_base)): for sub in os.listdir(anat_base[i]): - #check if subject is present in subject_list + # check if subject is present in subject_list if subject_list: if sub in subject_list and sub not in exclusion_list: - print("extracting data for subject: ", sub) walk(i, sub) - #check that subject is not in exclusion list - elif sub not in exclusion_list and sub not in '.DS_Store': - print("extracting data for subject: ", sub) + # check that subject is not in exclusion list + elif sub not in exclusion_list and sub not in ".DS_Store": walk(i, sub) - - name = os.path.join(c.outputSubjectListLocation, 'CPAC_subject_list.yml') - print("Extraction Complete...Input Subjects_list for CPAC - %s" % name) + os.path.join(c.outputSubjectListLocation, "CPAC_subject_list.yml") except Exception: raise finally: @@ -365,12 +393,15 @@ def walk(index, sub): def generate_suplimentary_files(output_path): """ Method to generate phenotypic template file - and subject list for group analysis + and subject list for group analysis. """ - from sets import Set import csv - subjects_list = yaml.safe_load(open(os.path.join(output_path, 'CPAC_subject_list.yml'), 'r')) + from sets import Set + + subjects_list = yaml.safe_load( + open(os.path.join(output_path, "CPAC_subject_list.yml"), "r") + ) subject_scan_set = Set() subject_set = Set() @@ -378,13 +409,12 @@ def generate_suplimentary_files(output_path): data_list = [] for sub in subjects_list: - - if sub['unique_id']: - subject_id = sub['subject_id'] + "_" + sub['unique_id'] + if sub["unique_id"]: + subject_id = sub["subject_id"] + "_" + sub["unique_id"] else: - subject_id = sub['subject_id'] + subject_id = sub["subject_id"] - for scan in list(sub['rest']): + for scan in list(sub["rest"]): subject_scan_set.add((subject_id, scan)) subject_set.add(subject_id) scan_set.add(scan) @@ -406,35 +436,32 @@ def generate_suplimentary_files(output_path): data_list.append(list1) - #prepare data for phenotypic file + # prepare data for phenotypic file if len(scan_set) > 1: - list1 = ['subject_id/Scan'] + list1 = ["subject_id/Scan"] list1.extend(list(subject_set)) list1.extend(list(scan_set)) - file_name = os.path.join(output_path, 'phenotypic_template.csv') - f = open(file_name, 'wb') + file_name = os.path.join(output_path, "phenotypic_template.csv") + f = open(file_name, "wb") writer = csv.writer(f) if len(scan_set) > 1: writer.writerow(list1) writer.writerows(data_list) else: - writer.writerow(['subject_id']) + writer.writerow(["subject_id"]) for sub in subject_set: writer.writerow([sub]) f.close() - print("Template Phenotypic file for group analysis - %s" % file_name) - file_name = os.path.join(output_path, "subject_list_group_analysis.txt") - f = open(file_name, 'w') + f = open(file_name, "w") for sub in subject_set: print(sub, file=f) - print("Subject list required later for group analysis - %s" % file_name) f.close() @@ -444,37 +471,40 @@ def read_csv(csv_input): 'Acquisition' 'Reference' 'Site' - 'TR (seconds)' + 'TR (seconds)'. """ - - import csv from collections import defaultdict + import csv + try: reader = csv.DictReader(open(csv_input, "U")) dict_labels = defaultdict(list) for line in reader: - csv_dict = dict((k.lower(), v) for k, v in line.items()) - dict_labels[csv_dict.get('site'), csv_dict.get('scan')] = \ - [csv_dict[key] for key in sorted(csv_dict.keys()) \ - if key != 'site' and key != 'scan'] + csv_dict = {k.lower(): v for k, v in line.items()} + dict_labels[csv_dict.get("site"), csv_dict.get("scan")] = [ + csv_dict[key] + for key in sorted(csv_dict.keys()) + if key not in ("site", "scan") + ] if len(dict_labels) < 1: - raise Exception("Scan Parameters File is either empty"\ - "or missing header") + raise Exception("Scan Parameters File is either empty" "or missing header") except: - print("Error reading scan parameters csv") raise return dict_labels + """ Class to set dictionary keys as map attributes """ + + class Configuration(object): def __init__(self, config_map): for key in config_map: - if config_map[key] == 'None': + if config_map[key] == "None": config_map[key] = None setattr(self, key, config_map[key]) @@ -482,17 +512,13 @@ def __init__(self, config_map): def run(data_config): """ Run method takes data_config - file as the input argument + file as the input argument. """ - - c = Configuration(yaml.safe_load(open(os.path.realpath(data_config), 'r'))) + c = Configuration(yaml.safe_load(open(os.path.realpath(data_config), "r"))) if c.scanParametersCSV is not None: s_param_map = read_csv(c.scanParametersCSV) else: - print("no scan parameters csv included"\ - "make sure you turn off slice timing correction option"\ - "in CPAC configuration") s_param_map = None extract_data(c, s_param_map) diff --git a/CPAC/utils/extract_parameters.py b/CPAC/utils/extract_parameters.py index fe890e8e3c..9b7843ce45 100644 --- a/CPAC/utils/extract_parameters.py +++ b/CPAC/utils/extract_parameters.py @@ -1,69 +1,74 @@ def merge(output_dir, scan_name, threshold, motion_f, power_f, flag): """ Method to merge power parameters and motion - parameters file + parameters file. """ import os import re - if threshold == None: + if threshold is None: filename = scan_name + "_all_params.csv" filename = filename.lstrip("_") outfile = os.path.join(output_dir, filename) - threshold_val = 0.0 else: filename = scan_name + threshold + "_all_params.csv" filename = filename.lstrip("_") outfile = os.path.join(output_dir, filename) - threshold_val = float(re.sub(r"[a-zA-Z_]", '', threshold)) + float(re.sub(r"[a-zA-Z_]", "", threshold)) # Read in the motion and power parameters files try: - motion = open(motion_f, 'r').readlines() + motion = open(motion_f, "r").readlines() except Exception as e: - err_string = "\n\n[!] CPAC says: Could not read the motion " \ - "parameters file.\n\nFilepath: %s\n\nError details: %s" \ - "\n\n" % (motion_f, e) + err_string = ( + "\n\n[!] CPAC says: Could not read the motion " + "parameters file.\n\nFilepath: %s\n\nError details: %s" + "\n\n" % (motion_f, e) + ) raise Exception(err_string) try: - power = open(power_f, 'r').readlines() + power = open(power_f, "r").readlines() except Exception as e: - err_string = "\n\n[!] CPAC says: Could not read the power " \ - "parameters file.\n\nFilepath: %s\n\nError details: %s" \ - "\n\n" % (power_f, e) + err_string = ( + "\n\n[!] CPAC says: Could not read the power " + "parameters file.\n\nFilepath: %s\n\nError details: %s" + "\n\n" % (power_f, e) + ) raise Exception(err_string) # Write the combined motion and power parameters CSV file try: if flag: - f = open(outfile, 'w') - + f = open(outfile, "w") + m = motion[0].strip("\n") - p = ','.join(power[0].split(",")[1:]) + p = ",".join(power[0].split(",")[1:]) - f.write(m+p) + f.write(m + p) else: - f = open(outfile, 'a') - + f = open(outfile, "a") + m = motion[1] - p = ','.join(power[1].split(",")[2:]) + p = ",".join(power[1].split(",")[2:]) - f.write(m+p+"\n") + f.write(m + p + "\n") f.close() except Exception as e: - err_string = "\n\n[!] CPAC says: Could not create or open the motion "\ - "and power parameters CSV file. Ensure you have write " \ - "permissions for the directory it is writing to.\n\n" \ - "Attempted write path: %s\n\nError details: %s\n\n" \ - % (outfile, e) + err_string = ( + "\n\n[!] CPAC says: Could not create or open the motion " + "and power parameters CSV file. Ensure you have write " + "permissions for the directory it is writing to.\n\n" + "Attempted write path: %s\n\nError details: %s\n\n" % (outfile, e) + ) raise Exception(err_string) + def grab(output_dir, scrubbing): """ Method to grab all the motion parameters and power parameters file from each subject - for each pipeline and merge them + for each pipeline and merge them. Parameters ---------- @@ -73,21 +78,21 @@ def grab(output_dir, scrubbing): import glob import os import re - from sets import Set - pipelines = glob.glob(os.path.join(output_dir, 'pipeline*')) + from sets import Set + pipelines = glob.glob(os.path.join(output_dir, "pipeline*")) for p in pipelines: scan_list = [] threshold_list = [] - pattern1 = re.compile(r'(\w)*scan(\w)*(\d)*(\w)*[/]') - pattern2 = re.compile(r'(\w)*threshold_[-+]?([0-9]*\.[0-9]+|[0-9]+)') + pattern1 = re.compile(r"(\w)*scan(\w)*(\d)*(\w)*[/]") + pattern2 = re.compile(r"(\w)*threshold_[-+]?([0-9]*\.[0-9]+|[0-9]+)") - scans = glob.glob(os.path.join(p, '*/power_params/*/*')) + scans = glob.glob(os.path.join(p, "*/power_params/*/*")) - #get the unique scans and threshold value + # get the unique scans and threshold value for s in scans: val = re.search(pattern1, s) if val: @@ -103,44 +108,45 @@ def grab(output_dir, scrubbing): for scan in scan_list: for threshold in threshold_list: Flag = 1 - #merge files for each subject + # merge files for each subject for sub in os.listdir(p): sub = os.path.join(p, sub) - motion_file = os.path.join(sub, 'motion_params', scan, - 'motion_parameters.txt') - power_file = os.path.join(sub, 'power_params', scan, - threshold, 'pow_params.txt') - if os.path.exists(motion_file) and \ - os.path.exists(power_file): - merge(p, scan, threshold, - motion_file, power_file, Flag) + motion_file = os.path.join( + sub, "motion_params", scan, "motion_parameters.txt" + ) + power_file = os.path.join( + sub, "power_params", scan, threshold, "pow_params.txt" + ) + if os.path.exists(motion_file) and os.path.exists(power_file): + merge(p, scan, threshold, motion_file, power_file, Flag) Flag = 0 if 0 in scrubbing: for sub in os.listdir(p): sub = os.path.join(p, sub) - motion_file = os.path.join(sub, 'motion_params', scan, - 'motion_parameters.txt') - power_file = os.path.join(sub, 'power_params', scan, - 'pow_params.txt') - - if os.path.exists(motion_file) and \ - os.path.exists(power_file): + motion_file = os.path.join( + sub, "motion_params", scan, "motion_parameters.txt" + ) + power_file = os.path.join( + sub, "power_params", scan, "pow_params.txt" + ) + + if os.path.exists(motion_file) and os.path.exists(power_file): threshold = None - merge(p, scan, threshold, motion_file, - power_file, Flag) + merge(p, scan, threshold, motion_file, power_file, Flag) Flag = 0 - + return threshold + def run(output_path, scrubbing): - threshold = grab(output_path, scrubbing) - return threshold + return grab(output_path, scrubbing) -if __name__ == '__main__': + +if __name__ == "__main__": import sys - if (len(sys.argv) == 2): + + if len(sys.argv) == 2: grab(sys.argv[1], [0]) else: - print('Usage: python extract_parameters.py /path/to/output/dir') - + pass diff --git a/CPAC/utils/ga.py b/CPAC/utils/ga.py index d62b77751a..a59d979ab0 100644 --- a/CPAC/utils/ga.py +++ b/CPAC/utils/ga.py @@ -1,27 +1,27 @@ import configparser import os import os.path as op -import requests import tempfile import threading import traceback import uuid +import requests + from CPAC.info import __version__, ga_tracker -udir = op.expanduser('~') -if udir=='/': +udir = op.expanduser("~") +if udir == "/": udir = tempfile.mkdtemp() temp_dir = True -tracking_path = op.join(udir, '.cpac') +tracking_path = op.join(udir, ".cpac") def get_or_create_config(): if not op.exists(tracking_path): parser = configparser.ConfigParser() - parser.read_dict(dict(user=dict(uid=uuid.uuid1().hex, - track=True))) - with open(tracking_path, 'w+') as fhandle: + parser.read_dict({"user": {"uid": uuid.uuid1().hex, "track": True}}) + with open(tracking_path, "w+") as fhandle: parser.write(fhandle) else: parser = configparser.ConfigParser() @@ -31,17 +31,12 @@ def get_or_create_config(): def get_uid(): - if os.environ.get('CPAC_TRACKING', '').lower() not in [ - '', - '0', - 'false', - 'off' - ]: - return os.environ.get('CPAC_TRACKING') + if os.environ.get("CPAC_TRACKING", "").lower() not in ["", "0", "false", "off"]: + return os.environ.get("CPAC_TRACKING") parser = get_or_create_config() - if parser['user'].getboolean('track'): - return parser['user']['uid'] + if parser["user"].getboolean("track"): + return parser["user"]["uid"] return None @@ -49,32 +44,37 @@ def get_uid(): def do_it(data, timeout): try: headers = { - 'User-Agent': 'C-PAC/{} (https://fcp-indi.github.io)'.format( - __version__ - ) + "User-Agent": "C-PAC/{} (https://fcp-indi.github.io)".format(__version__) } - response = requests.post( - 'https://www.google-analytics.com/collect', + return requests.post( + "https://www.google-analytics.com/collect", data=data, timeout=timeout, - headers=headers + headers=headers, ) - return response except: return False if temp_dir: try: os.remove(tracking_path) os.rmdir(udir) - temp_dir = False except: - print("Unable to delete temporary tracking path.") + pass + return None -def track_event(category, action, uid=None, label=None, value=0, - software_version=None, timeout=2, thread=True): +def track_event( + category, + action, + uid=None, + label=None, + value=0, + software_version=None, + timeout=2, + thread=True, +): """ - Record an event with Google Analytics + Record an event with Google Analytics. Parameters ---------- @@ -97,7 +97,7 @@ def track_event(category, action, uid=None, label=None, value=0, event. After this duration has elapsed with no response (e.g., on a slow network connection), the tracking is dropped. """ - if os.environ.get('CPAC_TRACKING', '').lower() in ['0', 'false', 'off']: + if os.environ.get("CPAC_TRACKING", "").lower() in ["0", "false", "off"]: return if uid is None: @@ -109,7 +109,7 @@ def track_event(category, action, uid=None, label=None, value=0, this = "/CPAC/utils/ga.py" exec_stack = list(reversed(traceback.extract_stack())) assert exec_stack[0][0].endswith(this) - package_path = exec_stack[0][0][:-len(this)] + package_path = exec_stack[0][0][: -len(this)] # only CPAC paths are going to be recorded file_path = "" @@ -118,26 +118,25 @@ def track_event(category, action, uid=None, label=None, value=0, continue if not s[0].startswith(package_path): break - file_path = s[0][len(package_path):] + file_path = s[0][len(package_path) :] data = { - 'v': '1', # API version. - 'tid': ga_tracker, # GA tracking ID - 'dp': file_path, - 'cid': uid, # User unique ID, stored in `tracking_path` - 't': 'event', # Event hit type. - 'ec': category, # Event category. - 'ea': action, # Event action. - 'el': label, # Event label. - 'ev': value, # Event value, must be an integer - 'aid': "CPAC", - 'an': "CPAC", - 'av': __version__, - 'aip': 1, # anonymize IP by removing last octet, slightly worse - # geolocation + "v": "1", # API version. + "tid": ga_tracker, # GA tracking ID + "dp": file_path, + "cid": uid, # User unique ID, stored in `tracking_path` + "t": "event", # Event hit type. + "ec": category, # Event category. + "ea": action, # Event action. + "el": label, # Event label. + "ev": value, # Event value, must be an integer + "aid": "CPAC", + "an": "CPAC", + "av": __version__, + "aip": 1, # anonymize IP by removing last octet, slightly worse + # geolocation } - if thread: t = threading.Thread(target=do_it, args=(data, timeout)) t.start() @@ -146,29 +145,15 @@ def track_event(category, action, uid=None, label=None, value=0, def track_config(cpac_interface): - track_event( - 'config', - cpac_interface, - label=None, - value=None, - thread=False - ) + track_event("config", cpac_interface, label=None, value=None, thread=False) -def track_run(level='participant', participants=0): - if level in ['participant', 'group']: +def track_run(level="participant", participants=0): + if level in ["participant", "group"]: track_event( - 'run', - level, - label='participants', - value=participants, - thread=False + "run", level, label="participants", value=participants, thread=False ) else: track_event( - 'config', - 'test', - label='participants', - value=participants, - thread=False + "config", "test", label="participants", value=participants, thread=False ) diff --git a/CPAC/utils/interfaces/__init__.py b/CPAC/utils/interfaces/__init__.py index 9037cc4d34..3998a00d5d 100644 --- a/CPAC/utils/interfaces/__init__.py +++ b/CPAC/utils/interfaces/__init__.py @@ -1,13 +1,9 @@ -from . import function -from . import masktool -from . import pc -from . import brickstat -from . import datasink +from . import brickstat, datasink, function, masktool, pc __all__ = [ - 'function', - 'masktool', - 'pc', - 'brickstat', - 'datasink', -] \ No newline at end of file + "function", + "masktool", + "pc", + "brickstat", + "datasink", +] diff --git a/CPAC/utils/interfaces/afni.py b/CPAC/utils/interfaces/afni.py index f5b6472e69..41fec5a250 100644 --- a/CPAC/utils/interfaces/afni.py +++ b/CPAC/utils/interfaces/afni.py @@ -14,36 +14,40 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -"""AFNI Nipype interfaces with customized functionality""" +"""AFNI Nipype interfaces with customized functionality.""" # pylint: disable=too-few-public-methods -from nipype.interfaces.afni.preprocess import ECM as _ECM, \ - ECMInputSpec as _ECMInputSpec import semver import traits.api as traits +from nipype.interfaces.afni.preprocess import ECM as _ECM, ECMInputSpec as _ECMInputSpec + from CPAC.utils.versioning import REPORTED -AFNI_SEMVER = REPORTED.get('AFNI', '0.0.0').split(':')[0] +AFNI_SEMVER = REPORTED.get("AFNI", "0.0.0").split(":")[0] """Validated AFNI Semver string""" try: AFNI_SEMVER = str(semver.Version.parse(AFNI_SEMVER)) except ValueError: - _major, _minor, _patch = [int(part) for part in AFNI_SEMVER.split('.')] - AFNI_SEMVER = str(semver.Version.parse(f'{_major}.{_minor}.{_patch}')) + _major, _minor, _patch = [int(part) for part in AFNI_SEMVER.split(".")] + AFNI_SEMVER = str(semver.Version.parse(f"{_major}.{_minor}.{_patch}")) del _major, _minor, _patch -AFNI_GTE_21_1_1 = semver.compare(AFNI_SEMVER, '21.1.1') >= 0 +AFNI_GTE_21_1_1 = semver.compare(AFNI_SEMVER, "21.1.1") >= 0 """AFNI version >= 21.1.1?""" class ECMInputSpec(_ECMInputSpec): - """ECMInputSpec + 'do_binary' option""" - do_binary = traits.Bool(desc='whether to perform the ECM calculation on a ' - 'binarized version of the connectivity matrix', - argstr='-do_binary') + """ECMInputSpec + 'do_binary' option.""" + + do_binary = traits.Bool( + desc="whether to perform the ECM calculation on a " + "binarized version of the connectivity matrix", + argstr="-do_binary", + ) class ECM(_ECM): - """ECM + 'do_binary' option""" + """ECM + 'do_binary' option.""" + input_spec = ECMInputSpec -__all__ = ['AFNI_GTE_21_1_1', 'ECM'] +__all__ = ["AFNI_GTE_21_1_1", "ECM"] diff --git a/CPAC/utils/interfaces/ants.py b/CPAC/utils/interfaces/ants.py index 8eb5b2cf31..5fbd41c810 100644 --- a/CPAC/utils/interfaces/ants.py +++ b/CPAC/utils/interfaces/ants.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Nipype interfaces for ANTs commands +"""Nipype interfaces for ANTs commands. Some of this functionality is adapted from nipreps/niworkflows: - https://github.com/nipreps/niworkflows/blob/5a0f4bd3/niworkflows/interfaces/ants.py @@ -50,40 +50,55 @@ Modifications Copyright (C) 2022 C-PAC Developers This file is part of C-PAC. -""" # noqa: E501 # pylint: disable=line-too-long +""" # pylint: disable=line-too-long import os + from nipype.interfaces import base -from nipype.interfaces.ants.base import ANTSCommandInputSpec, ANTSCommand -from nipype.interfaces.base import traits, isdefined +from nipype.interfaces.ants.base import ANTSCommand, ANTSCommandInputSpec +from nipype.interfaces.base import isdefined, traits class ImageMathInputSpec(ANTSCommandInputSpec): - dimension = traits.Int(3, usedefault=True, position=1, argstr='%d', - desc='dimension of output image') - output_image = base.File(position=2, argstr='%s', name_source=['op1'], - name_template='%s_maths', desc='output image file', - keep_extension=True) - operation = base.Str(mandatory=True, position=3, argstr='%s', - desc='operations and intputs') - op1 = base.File(exists=True, mandatory=True, position=-2, argstr='%s', - desc='first operator') - op2 = traits.Either(base.File(exists=True), base.Str, position=-1, - argstr='%s', desc='second operator') + dimension = traits.Int( + 3, usedefault=True, position=1, argstr="%d", desc="dimension of output image" + ) + output_image = base.File( + position=2, + argstr="%s", + name_source=["op1"], + name_template="%s_maths", + desc="output image file", + keep_extension=True, + ) + operation = base.Str( + mandatory=True, position=3, argstr="%s", desc="operations and intputs" + ) + op1 = base.File( + exists=True, mandatory=True, position=-2, argstr="%s", desc="first operator" + ) + op2 = traits.Either( + base.File(exists=True), + base.Str, + position=-1, + argstr="%s", + desc="second operator", + ) class ImageMathOuputSpec(base.TraitedSpec): - output_image = base.File(exists=True, desc='output image file') + output_image = base.File(exists=True, desc="output image file") class ImageMath(ANTSCommand): """ - Operations over images + Operations over images. Example: -------- """ - _cmd = 'ImageMath' + + _cmd = "ImageMath" input_spec = ImageMathInputSpec output_spec = ImageMathOuputSpec @@ -92,29 +107,37 @@ class PrintHeaderInputSpec(ANTSCommandInputSpec): """InputSpec for ``PrintHeader``. See `PrintHeader: DESCRIPTION `_ for ``what_information`` values. - """ # noqa: E501 # pylint: disable=line-too-long - image = base.File(position=2, argstr='%s', name_source=['image'], - desc='image to read header from', exists=True, - mandatory=True) + """ # pylint: disable=line-too-long + + image = base.File( + position=2, + argstr="%s", + name_source=["image"], + desc="image to read header from", + exists=True, + mandatory=True, + ) - what_information = traits.Int(position=3, argstr='%i', - name='what_information', - desc='read what from header') + what_information = traits.Int( + position=3, argstr="%i", name="what_information", desc="read what from header" + ) class PrintHeaderOutputSpec(base.TraitedSpec): """OutputSpec for ``PrintHeader``.""" - header = traits.String(name='header') + + header = traits.String(name="header") class PrintHeader(ANTSCommand): """Print image header information.""" - _cmd = 'PrintHeader' + + _cmd = "PrintHeader" # pylint: disable=protected-access _gen_filename = base.StdOutCommandLine._gen_filename input_spec = PrintHeaderInputSpec output_spec = PrintHeaderOutputSpec - _terminal_output = 'stream' + _terminal_output = "stream" def aggregate_outputs(self, runtime=None, needed_outputs=None): outputs = super().aggregate_outputs(runtime, needed_outputs) @@ -127,34 +150,50 @@ def _list_outputs(self): class ResampleImageBySpacingInputSpec(ANTSCommandInputSpec): - dimension = traits.Int(3, usedefault=True, position=1, argstr='%d', - desc='dimension of output image') - input_image = base.File(exists=True, mandatory=True, position=2, argstr='%s', - desc='input image file') - output_image = base.File(position=3, argstr='%s', name_source=['input_image'], - name_template='%s_resampled', desc='output image file', - keep_extension=True) + dimension = traits.Int( + 3, usedefault=True, position=1, argstr="%d", desc="dimension of output image" + ) + input_image = base.File( + exists=True, mandatory=True, position=2, argstr="%s", desc="input image file" + ) + output_image = base.File( + position=3, + argstr="%s", + name_source=["input_image"], + name_template="%s_resampled", + desc="output image file", + keep_extension=True, + ) out_spacing = traits.Either( traits.List(traits.Float, minlen=2, maxlen=3), traits.Tuple(traits.Float, traits.Float, traits.Float), traits.Tuple(traits.Float, traits.Float), - position=4, argstr='%s', mandatory=True, desc='output spacing' + position=4, + argstr="%s", + mandatory=True, + desc="output spacing", + ) + apply_smoothing = traits.Bool( + False, argstr="%d", position=5, desc="smooth before resampling" + ) + addvox = traits.Int( + argstr="%d", + position=6, + requires=["apply_smoothing"], + desc="addvox pads each dimension by addvox", + ) + nn_interp = traits.Bool( + argstr="%d", desc="nn interpolation", position=-1, requires=["addvox"] ) - apply_smoothing = traits.Bool(False, argstr='%d', position=5, - desc='smooth before resampling') - addvox = traits.Int(argstr='%d', position=6, requires=['apply_smoothing'], - desc='addvox pads each dimension by addvox') - nn_interp = traits.Bool(argstr='%d', desc='nn interpolation', - position=-1, requires=['addvox']) class ResampleImageBySpacingOutputSpec(base.TraitedSpec): - output_image = traits.File(exists=True, desc='resampled file') + output_image = traits.File(exists=True, desc="resampled file") class ResampleImageBySpacing(ANTSCommand): """ - Resamples an image with a given spacing + Resamples an image with a given spacing. Example: -------- @@ -182,89 +221,118 @@ class ResampleImageBySpacing(ANTSCommand): 'ResampleImageBySpacing input.nii.gz output.nii.gz 4 4 4 1 2 0' """ - _cmd = 'ResampleImageBySpacing' + + _cmd = "ResampleImageBySpacing" input_spec = ResampleImageBySpacingInputSpec output_spec = ResampleImageBySpacingOutputSpec def _format_arg(self, name, trait_spec, value): - if name == 'out_spacing': + if name == "out_spacing": if len(value) != self.inputs.dimension: - raise ValueError('out_spacing dimensions should match dimension') + raise ValueError("out_spacing dimensions should match dimension") - value = ' '.join(['%d' % d for d in value]) + value = " ".join(["%d" % d for d in value]) - return super(ResampleImageBySpacing, self)._format_arg( - name, trait_spec, value) + return super(ResampleImageBySpacing, self)._format_arg(name, trait_spec, value) class SetDirectionByMatrixInputSpec(ANTSCommandInputSpec): """InputSpec for ``SetDirectionByMatrix``.""" - infile = base.File(position=2, argstr='%s', name_source=['infile'], - desc='image to copy header to', exists=True, - mandatory=True) - outfile = base.File(position=3, argstr='%s', - name_sources=['infile', 'outfile'], - desc='output image file', usedefault=True) - direction = traits.String(argstr='%s', position=4, - desc='dimensions, x-delimited') + + infile = base.File( + position=2, + argstr="%s", + name_source=["infile"], + desc="image to copy header to", + exists=True, + mandatory=True, + ) + outfile = base.File( + position=3, + argstr="%s", + name_sources=["infile", "outfile"], + desc="output image file", + usedefault=True, + ) + direction = traits.String(argstr="%s", position=4, desc="dimensions, x-delimited") class SetDirectionByMatrixOutputSpec(base.TraitedSpec): - """OutputSpec for ``SetDirectionByMatrix``""" - outfile = base.File(exists=True, desc='output image file') + """OutputSpec for ``SetDirectionByMatrix``.""" + + outfile = base.File(exists=True, desc="output image file") class SetDirectionByMatrix(ANTSCommand): """Set image header information from a matrix of dimensions.""" - _cmd = 'SetDirectionByMatrix' + + _cmd = "SetDirectionByMatrix" # pylint: disable=protected-access _gen_filename = base.StdOutCommandLine._gen_filename input_spec = SetDirectionByMatrixInputSpec output_spec = SetDirectionByMatrixOutputSpec def _format_arg(self, name, trait_spec, value): - if name == 'direction': - return value.replace('x', ' ') + if name == "direction": + return value.replace("x", " ") return super()._format_arg(name, trait_spec, value) def _list_outputs(self): - return {'outfile': self.inputs.outfile} + return {"outfile": self.inputs.outfile} class ThresholdImageInputSpec(ANTSCommandInputSpec): - dimension = traits.Int(3, usedefault=True, position=1, argstr='%d', - desc='dimension of output image') - input_image = base.File(exists=True, mandatory=True, position=2, argstr='%s', - desc='input image file') - output_image = base.File(position=3, argstr='%s', name_source=['input_image'], - name_template='%s_resampled', desc='output image file', - keep_extension=True) - - mode = traits.Enum('Otsu', 'Kmeans', argstr='%s', position=4, - requires=['num_thresholds'], xor=['th_low', 'th_high'], - desc='whether to run Otsu / Kmeans thresholding') - num_thresholds = traits.Int(position=5, argstr='%d', - desc='number of thresholds') - input_mask = base.File(exists=True, requires=['num_thresholds'], argstr='%s', - desc='input mask for Otsu, Kmeans') - - th_low = traits.Float(position=4, argstr='%f', xor=['mode'], - desc='lower threshold') - th_high = traits.Float(position=5, argstr='%f', xor=['mode'], - desc='upper threshold') - inside_value = traits.Float(1, position=6, argstr='%f', requires=['th_low'], - desc='inside value') - outside_value = traits.Float(0, position=7, argstr='%f', requires=['th_low'], - desc='outside value') + dimension = traits.Int( + 3, usedefault=True, position=1, argstr="%d", desc="dimension of output image" + ) + input_image = base.File( + exists=True, mandatory=True, position=2, argstr="%s", desc="input image file" + ) + output_image = base.File( + position=3, + argstr="%s", + name_source=["input_image"], + name_template="%s_resampled", + desc="output image file", + keep_extension=True, + ) + + mode = traits.Enum( + "Otsu", + "Kmeans", + argstr="%s", + position=4, + requires=["num_thresholds"], + xor=["th_low", "th_high"], + desc="whether to run Otsu / Kmeans thresholding", + ) + num_thresholds = traits.Int(position=5, argstr="%d", desc="number of thresholds") + input_mask = base.File( + exists=True, + requires=["num_thresholds"], + argstr="%s", + desc="input mask for Otsu, Kmeans", + ) + + th_low = traits.Float(position=4, argstr="%f", xor=["mode"], desc="lower threshold") + th_high = traits.Float( + position=5, argstr="%f", xor=["mode"], desc="upper threshold" + ) + inside_value = traits.Float( + 1, position=6, argstr="%f", requires=["th_low"], desc="inside value" + ) + outside_value = traits.Float( + 0, position=7, argstr="%f", requires=["th_low"], desc="outside value" + ) class ThresholdImageOutputSpec(base.TraitedSpec): - output_image = traits.File(exists=True, desc='resampled file') + output_image = traits.File(exists=True, desc="resampled file") class ThresholdImage(ANTSCommand): """ - Apply thresholds on images + Apply thresholds on images. Example: -------- @@ -286,108 +354,136 @@ class ThresholdImage(ANTSCommand): 'ThresholdImage input.nii.gz output.nii.gz Kmeans 4' """ - _cmd = 'ThresholdImage' + + _cmd = "ThresholdImage" input_spec = ThresholdImageInputSpec output_spec = ThresholdImageOutputSpec class AIInputSpec(ANTSCommandInputSpec): - dimension = traits.Int(3, usedefault=True, argstr='-d %d', - desc='dimension of output image') - verbose = traits.Bool(False, usedefault=True, argstr='-v %d', - desc='enable verbosity') + dimension = traits.Int( + 3, usedefault=True, argstr="-d %d", desc="dimension of output image" + ) + verbose = traits.Bool( + False, usedefault=True, argstr="-v %d", desc="enable verbosity" + ) fixed_image = traits.File( - exists=True, mandatory=True, - desc='Image to which the moving_image should be transformed') + exists=True, + mandatory=True, + desc="Image to which the moving_image should be transformed", + ) moving_image = traits.File( - exists=True, mandatory=True, - desc='Image that will be transformed to fixed_image') + exists=True, + mandatory=True, + desc="Image that will be transformed to fixed_image", + ) - fixed_image_mask = traits.File( - exists=True, argstr='-x %s', desc='fixed mage mask') + fixed_image_mask = traits.File(exists=True, argstr="-x %s", desc="fixed mage mask") moving_image_mask = traits.File( - exists=True, requires=['fixed_image_mask'], - desc='moving mage mask') + exists=True, requires=["fixed_image_mask"], desc="moving mage mask" + ) metric_trait = ( traits.Enum("Mattes", "GC", "MI"), traits.Int(32), - traits.Enum('Regular', 'Random', 'None'), - traits.Range(value=0.2, low=0.0, high=1.0) + traits.Enum("Regular", "Random", "None"), + traits.Range(value=0.2, low=0.0, high=1.0), + ) + metric = traits.Tuple( + *metric_trait, argstr="-m %s", mandatory=True, desc="the metric(s) to use." ) - metric = traits.Tuple(*metric_trait, argstr='-m %s', mandatory=True, - desc='the metric(s) to use.') transform = traits.Tuple( - traits.Enum('Affine', 'Rigid', 'Similarity'), + traits.Enum("Affine", "Rigid", "Similarity"), traits.Range(value=0.1, low=0.0, exclude_low=True), - argstr='-t %s[%f]', usedefault=True, - desc='Several transform options are available') + argstr="-t %s[%f]", + usedefault=True, + desc="Several transform options are available", + ) - principal_axes = traits.Bool(False, usedefault=True, argstr='-p %d', xor=['blobs'], - desc='align using principal axes') + principal_axes = traits.Bool( + False, + usedefault=True, + argstr="-p %d", + xor=["blobs"], + desc="align using principal axes", + ) search_factor = traits.Tuple( - traits.Float(20), traits.Range(value=0.12, low=0.0, high=1.0), - usedefault=True, argstr='-s [%f,%f]', desc='search factor') + traits.Float(20), + traits.Range(value=0.12, low=0.0, high=1.0), + usedefault=True, + argstr="-s [%f,%f]", + desc="search factor", + ) search_grid = traits.Either( - traits.Tuple(traits.Float, traits.Tuple(traits.Float, traits.Float, traits.Float)), + traits.Tuple( + traits.Float, traits.Tuple(traits.Float, traits.Float, traits.Float) + ), traits.Tuple(traits.Float, traits.Tuple(traits.Float, traits.Float)), - argstr='-g %s', desc='Translation search grid in mm') + argstr="-g %s", + desc="Translation search grid in mm", + ) convergence = traits.Tuple( traits.Range(low=1, high=10000, value=10), traits.Float(1e-6), traits.Range(low=1, high=100, value=10), - usedefault=True, argstr='-c [%d,%f,%d]', desc='convergence') + usedefault=True, + argstr="-c [%d,%f,%d]", + desc="convergence", + ) output_transform = traits.File( - 'initialization.mat', usedefault=True, argstr='-o %s', - desc='output file name') + "initialization.mat", usedefault=True, argstr="-o %s", desc="output file name" + ) class AIOuputSpec(base.TraitedSpec): - output_transform = traits.File(exists=True, desc='output file name') + output_transform = traits.File(exists=True, desc="output file name") class AI(ANTSCommand): """Replaces ``AffineInitializer``.""" - _cmd = 'antsAI' + _cmd = "antsAI" input_spec = AIInputSpec output_spec = AIOuputSpec - def _run_interface(self, runtime, correct_return_codes=(0, )): - runtime = super(AI, self)._run_interface( - runtime, correct_return_codes) - - setattr(self, '_output', { - 'output_transform': os.path.join( - runtime.cwd, - os.path.basename(self.inputs.output_transform)) - }) + def _run_interface(self, runtime, correct_return_codes=(0,)): + runtime = super(AI, self)._run_interface(runtime, correct_return_codes) + + setattr( + self, + "_output", + { + "output_transform": os.path.join( + runtime.cwd, os.path.basename(self.inputs.output_transform) + ) + }, + ) return runtime def _format_arg(self, opt, spec, val): - if opt == 'metric': - val = '%s[{fixed_image},{moving_image},%d,%s,%f]' % val + if opt == "metric": + val = "%s[{fixed_image},{moving_image},%d,%s,%f]" % val val = val.format( fixed_image=self.inputs.fixed_image, - moving_image=self.inputs.moving_image) + moving_image=self.inputs.moving_image, + ) return spec.argstr % val - if opt == 'search_grid': - val1 = 'x'.join(['%f' % v for v in val[1]]) - fmtval = '[%s]' % ','.join([str(val[0]), val1]) + if opt == "search_grid": + val1 = "x".join(["%f" % v for v in val[1]]) + fmtval = "[%s]" % ",".join([str(val[0]), val1]) return spec.argstr % fmtval - if opt == 'fixed_image_mask': + if opt == "fixed_image_mask": if isdefined(self.inputs.moving_image_mask): - return spec.argstr % ('[%s,%s]' % ( - val, self.inputs.moving_image_mask)) + return spec.argstr % ("[%s,%s]" % (val, self.inputs.moving_image_mask)) return super(AI, self)._format_arg(opt, spec, val) def _list_outputs(self): - return getattr(self, '_output') + return getattr(self, "_output") diff --git a/CPAC/utils/interfaces/brickstat.py b/CPAC/utils/interfaces/brickstat.py index 8e9a907871..513786bf88 100644 --- a/CPAC/utils/interfaces/brickstat.py +++ b/CPAC/utils/interfaces/brickstat.py @@ -1,61 +1,57 @@ import os -from builtins import str, bytes -import inspect - from nipype import logging -from nipype.interfaces.base import (CommandLineInputSpec, CommandLine, Directory, TraitedSpec, - traits, isdefined, File, InputMultiObject, InputMultiPath, - Undefined, Str) - -from nipype.utils.filemanip import (load_json, save_json, split_filename) - -from nipype.interfaces.afni.base import (AFNICommandBase, AFNICommand, AFNICommandInputSpec, - AFNICommandOutputSpec, AFNIPythonCommandInputSpec, - AFNIPythonCommand) -from nipype.interfaces.io import IOBase, add_traits -from nipype.utils.filemanip import ensure_list -from nipype.utils.functions import getsource, create_function_from_source +from nipype.interfaces.afni.base import ( + AFNICommandBase, +) +from nipype.interfaces.base import ( + CommandLineInputSpec, + File, + TraitedSpec, + traits, +) +from nipype.utils.filemanip import load_json, save_json -iflogger = logging.getLogger('nipype.interface') +iflogger = logging.getLogger("nipype.interface") class BrickStatInputSpec(CommandLineInputSpec): in_file = File( - desc='input file to 3dmaskave', - argstr='%s', + desc="input file to 3dmaskave", + argstr="%s", position=-1, mandatory=True, - exists=True) + exists=True, + ) mask = File( - desc='-mask dset = use dset as mask to include/exclude voxels', - argstr='-mask %s', + desc="-mask dset = use dset as mask to include/exclude voxels", + argstr="-mask %s", position=2, - exists=True) + exists=True, + ) min = traits.Bool( - desc='print the minimum value in dataset', argstr='-min', position=1) + desc="print the minimum value in dataset", argstr="-min", position=1 + ) slow = traits.Bool( - desc='read the whole dataset to find the min and max values', - argstr='-slow') - max = traits.Bool( - desc='print the maximum value in the dataset', argstr='-max') - mean = traits.Bool( - desc='print the mean value in the dataset', argstr='-mean') - sum = traits.Bool( - desc='print the sum of values in the dataset', argstr='-sum') - var = traits.Bool(desc='print the variance in the dataset', argstr='-var') + desc="read the whole dataset to find the min and max values", argstr="-slow" + ) + max = traits.Bool(desc="print the maximum value in the dataset", argstr="-max") + mean = traits.Bool(desc="print the mean value in the dataset", argstr="-mean") + sum = traits.Bool(desc="print the sum of values in the dataset", argstr="-sum") + var = traits.Bool(desc="print the variance in the dataset", argstr="-var") percentile = traits.Tuple( traits.Float, traits.Float, traits.Float, - desc='p0 ps p1 write the percentile values starting ' - 'at p0% and ending at p1% at a step of ps%. ' - 'only one sub-brick is accepted.', - argstr='-percentile %.3f %.3f %.3f') + desc="p0 ps p1 write the percentile values starting " + "at p0% and ending at p1% at a step of ps%. " + "only one sub-brick is accepted.", + argstr="-percentile %.3f %.3f %.3f", + ) class BrickStatOutputSpec(TraitedSpec): - min_val = traits.Float(desc='output') + min_val = traits.Float(desc="output") class BrickStat(AFNICommandBase): @@ -66,8 +62,7 @@ class BrickStat(AFNICommandBase): `_ Examples - ======== - + -------- >>> from nipype.interfaces import afni >>> brickstat = afni.BrickStat() >>> brickstat.inputs.in_file = 'functional.nii' # doctest: +SKIP @@ -78,24 +73,24 @@ class BrickStat(AFNICommandBase): >>> res = brickstat.run() # doctest: +SKIP """ - _cmd = '3dBrickStat' + + _cmd = "3dBrickStat" input_spec = BrickStatInputSpec output_spec = BrickStatOutputSpec def aggregate_outputs(self, runtime=None, needed_outputs=None): - outputs = self._outputs() - outfile = os.path.join(os.getcwd(), 'stat_result.json') + outfile = os.path.join(os.getcwd(), "stat_result.json") if runtime is None: try: - min_val = load_json(outfile)['stat'] + min_val = load_json(outfile)["stat"] except IOError: return self.run().outputs else: min_val = [] - for line in runtime.stdout.split('\n'): + for line in runtime.stdout.split("\n"): if line: values = line.split() if len(values) > 1: @@ -105,9 +100,9 @@ def aggregate_outputs(self, runtime=None, needed_outputs=None): if len(min_val) == 1: min_val = min_val[0] - save_json(outfile, dict(stat=min_val)) + save_json(outfile, {"stat": min_val}) if type(min_val) == list: min_val = min_val[-1] outputs.min_val = min_val - return outputs \ No newline at end of file + return outputs diff --git a/CPAC/utils/interfaces/datasink.py b/CPAC/utils/interfaces/datasink.py index eade448da9..e87bc2f4db 100644 --- a/CPAC/utils/interfaces/datasink.py +++ b/CPAC/utils/interfaces/datasink.py @@ -1,34 +1,23 @@ -from builtins import object, zip, filter, range, open, str - -import time -import glob -import fnmatch -import string -import json +from builtins import open, range, str import os -import os.path as op -import shutil import re -import copy -import tempfile -from os.path import join, dirname +import shutil from shutil import SameFileError -from warnings import warn -from nipype.interfaces.io import IOBase, DataSinkInputSpec, DataSinkOutputSpec, ProgressPercentage, copytree +import time from nipype import config, logging -from nipype.utils.misc import human_order_sorted, str2bool +from nipype.interfaces.base import Undefined, isdefined, traits +from nipype.interfaces.io import ( + DataSinkInputSpec, + DataSinkOutputSpec, + IOBase, + ProgressPercentage, + copytree, +) +from nipype.utils.filemanip import copyfile, ensure_list +from nipype.utils.misc import str2bool -from nipype.utils.filemanip import ( - copyfile, simplify_list, ensure_list, - get_related_files) - -from nipype.interfaces.base import (CommandLineInputSpec, CommandLine, Directory, TraitedSpec, - traits, isdefined, File, InputMultiObject, InputMultiPath, - Undefined, Str) - - -iflogger = logging.getLogger('nipype.interface') +iflogger = logging.getLogger("nipype.interface") RETRY = 5 @@ -36,10 +25,9 @@ def _get_head_bucket(s3_resource, bucket_name): - """ Try to get the header info of a bucket, in order to - check if it exists and its permissions + """Try to get the header info of a bucket, in order to + check if it exists and its permissions. """ - import botocore # Try fetch the bucket with the name argument @@ -50,22 +38,29 @@ def _get_head_bucket(s3_resource, bucket_name): return except botocore.exceptions.ClientError as exc: - error_code = int(exc.response['Error']['Code']) + error_code = int(exc.response["Error"]["Code"]) if error_code == 403: - err_msg = 'Access to bucket: %s is denied; check credentials'\ - % bucket_name + err_msg = ( + "Access to bucket: %s is denied; check credentials" % bucket_name + ) break elif error_code == 404: - err_msg = 'Bucket: %s does not exist; check spelling and try '\ - 'again' % bucket_name + err_msg = ( + "Bucket: %s does not exist; check spelling and try " + "again" % bucket_name + ) break else: - err_msg = 'Unable to connect to bucket: %s. Error message:\n%s'\ - % (bucket_name, exc) + err_msg = "Unable to connect to bucket: %s. Error message:\n%s" % ( + bucket_name, + exc, + ) except Exception as exc: - err_msg = 'Unable to connect to bucket: %s. Error message:\n%s'\ - % (bucket_name, exc) + err_msg = "Unable to connect to bucket: %s. Error message:\n%s" % ( + bucket_name, + exc, + ) time.sleep(RETRY_WAIT) @@ -74,47 +69,47 @@ def _get_head_bucket(s3_resource, bucket_name): class DataSink(IOBase): - """ Generic datasink module to store structured outputs - Primarily for use within a workflow. This interface allows arbitrary - creation of input attributes. The names of these attributes define the - directory structure to create for storage of the files or directories. - The attributes take the following form: - string[[.[@]]string[[.[@]]string]] ... - where parts between [] are optional. - An attribute such as contrasts.@con will create a 'contrasts' directory - to store the results linked to the attribute. If the @ is left out, such - as in 'contrasts.con', a subdirectory 'con' will be created under - 'contrasts'. - the general form of the output is:: - 'base_directory/container/parameterization/destloc/filename' - destloc = string[[.[@]]string[[.[@]]string]] and - filename comesfrom the input to the connect statement. - .. warning:: - This is not a thread-safe node because it can write to a common - shared location. It will not complain when it overwrites a file. - .. note:: - If both substitutions and regexp_substitutions are used, then - substitutions are applied first followed by regexp_substitutions. - This interface **cannot** be used in a MapNode as the inputs are - defined only when the connect statement is executed. - Examples - -------- - >>> ds = DataSink() - >>> ds.inputs.base_directory = 'results_dir' - >>> ds.inputs.container = 'subject' - >>> ds.inputs.structural = 'structural.nii' - >>> setattr(ds.inputs, 'contrasts.@con', ['cont1.nii', 'cont2.nii']) - >>> setattr(ds.inputs, 'contrasts.alt', ['cont1a.nii', 'cont2a.nii']) - >>> ds.run() # doctest: +SKIP - To use DataSink in a MapNode, its inputs have to be defined at the - time the interface is created. - >>> ds = DataSink(infields=['contasts.@con']) - >>> ds.inputs.base_directory = 'results_dir' - >>> ds.inputs.container = 'subject' - >>> ds.inputs.structural = 'structural.nii' - >>> setattr(ds.inputs, 'contrasts.@con', ['cont1.nii', 'cont2.nii']) - >>> setattr(ds.inputs, 'contrasts.alt', ['cont1a.nii', 'cont2a.nii']) - >>> ds.run() # doctest: +SKIP + """Generic datasink module to store structured outputs + Primarily for use within a workflow. This interface allows arbitrary + creation of input attributes. The names of these attributes define the + directory structure to create for storage of the files or directories. + The attributes take the following form: + string[[.[@]]string[[.[@]]string]] ... + where parts between [] are optional. + An attribute such as contrasts.@con will create a 'contrasts' directory + to store the results linked to the attribute. If the @ is left out, such + as in 'contrasts.con', a subdirectory 'con' will be created under + 'contrasts'. + the general form of the output is:: + 'base_directory/container/parameterization/destloc/filename' + destloc = string[[.[@]]string[[.[@]]string]] and + filename comesfrom the input to the connect statement. + .. warning:: + This is not a thread-safe node because it can write to a common + shared location. It will not complain when it overwrites a file. + .. note:: + If both substitutions and regexp_substitutions are used, then + substitutions are applied first followed by regexp_substitutions. + This interface **cannot** be used in a MapNode as the inputs are + defined only when the connect statement is executed. + Examples. + -------- + >>> ds = DataSink() + >>> ds.inputs.base_directory = 'results_dir' + >>> ds.inputs.container = 'subject' + >>> ds.inputs.structural = 'structural.nii' + >>> setattr(ds.inputs, 'contrasts.@con', ['cont1.nii', 'cont2.nii']) + >>> setattr(ds.inputs, 'contrasts.alt', ['cont1a.nii', 'cont2a.nii']) + >>> ds.run() # doctest: +SKIP + To use DataSink in a MapNode, its inputs have to be defined at the + time the interface is created. + >>> ds = DataSink(infields=['contasts.@con']) + >>> ds.inputs.base_directory = 'results_dir' + >>> ds.inputs.container = 'subject' + >>> ds.inputs.structural = 'structural.nii' + >>> setattr(ds.inputs, 'contrasts.@con', ['cont1.nii', 'cont2.nii']) + >>> setattr(ds.inputs, 'contrasts.alt', ['cont1a.nii', 'cont2a.nii']) + >>> ds.run() # doctest: +SKIP """ # Give obj .inputs and .outputs @@ -129,7 +124,6 @@ def __init__(self, infields=None, force_run=True, **kwargs): infields : list of str Indicates the input fields to be dynamically created """ - super(DataSink, self).__init__(**kwargs) undefined_traits = {} # used for mandatory inputs check @@ -152,10 +146,9 @@ def _get_dst(self, src): if self.inputs.parameterization: dst = path if isdefined(self.inputs.strip_dir): - dst = dst.replace(self.inputs.strip_dir, '') + dst = dst.replace(self.inputs.strip_dir, "") folders = [ - folder for folder in dst.split(os.path.sep) - if folder.startswith('_') + folder for folder in dst.split(os.path.sep) if folder.startswith("_") ] dst = os.path.sep.join(folders) if fname: @@ -177,27 +170,39 @@ def _substitute(self, pathstr): oldpathstr = pathstr pathstr = pathstr.replace(key, val) if pathstr != oldpathstr: - iflogger.debug('sub.str: %s -> %s using %r -> %r', - oldpathstr, pathstr, key, val) + iflogger.debug( + "sub.str: %s -> %s using %r -> %r", + oldpathstr, + pathstr, + key, + val, + ) if isdefined(self.inputs.regexp_substitutions): for key, val in self.inputs.regexp_substitutions: oldpathstr = pathstr pathstr, _ = re.subn(key, val, pathstr) if pathstr != oldpathstr: - iflogger.debug('sub.regexp: %s -> %s using %r -> %r', - oldpathstr, pathstr, key, val) + iflogger.debug( + "sub.regexp: %s -> %s using %r -> %r", + oldpathstr, + pathstr, + key, + val, + ) if pathstr_ != pathstr: - iflogger.info('sub: %s -> %s', pathstr_, pathstr) + iflogger.info("sub: %s -> %s", pathstr_, pathstr) return pathstr # Check for s3 in base directory def _check_s3_base_dir(self): - ''' + """ Method to see if the datasink's base directory specifies an S3 bucket path; if it does, it parses the path for the bucket - name in the form 's3://bucket_name/...' and returns it - Parameters + name in the form 's3://bucket_name/...' and returns it. + + Parameters. ---------- + Returns ------- s3_flag : boolean @@ -206,11 +211,10 @@ def _check_s3_base_dir(self): bucket_name : string name of the S3 bucket to connect to; if the base directory is not a valid S3 path, defaults to '' - ''' - + """ # Init variables - s3_str = 's3://' - bucket_name = '' + s3_str = "s3://" + bucket_name = "" base_directory = self.inputs.base_directory if not isdefined(base_directory): @@ -219,14 +223,14 @@ def _check_s3_base_dir(self): # Explicitly lower-case the "s3" if base_directory.lower().startswith(s3_str): - base_dir_sp = base_directory.split('/') + base_dir_sp = base_directory.split("/") base_dir_sp[0] = base_dir_sp[0].lower() - base_directory = '/'.join(base_dir_sp) + base_directory = "/".join(base_dir_sp) # Check if 's3://' in base dir if base_directory.startswith(s3_str): # Expects bucket name to be 's3://bucket_name/base_dir/..' - bucket_name = base_directory.split(s3_str)[1].split('/')[0] + bucket_name = base_directory.split(s3_str)[1].split("/")[0] s3_flag = True # Otherwise it's just a normal datasink else: @@ -237,21 +241,22 @@ def _check_s3_base_dir(self): # Function to return AWS secure environment variables def _return_aws_keys(self): - ''' + """ Method to return AWS access key id and secret access key using credentials found in a local file. - Parameters + + Parameters. ---------- self : nipype.interfaces.io.DataSink self for instance method + Returns ------- aws_access_key_id : string string of the AWS access key ID aws_secret_access_key : string string of the AWS secret access key - ''' - + """ # Import packages import os @@ -260,62 +265,61 @@ def _return_aws_keys(self): # Check if creds exist if creds_path and os.path.exists(creds_path): - with open(creds_path, 'r') as creds_in: + with open(creds_path, "r") as creds_in: # Grab csv rows row1 = creds_in.readline() row2 = creds_in.readline() # Are they root or user keys - if 'User Name' in row1: + if "User Name" in row1: # And split out for keys - aws_access_key_id = row2.split(',')[1] - aws_secret_access_key = row2.split(',')[2] - elif 'AWSAccessKeyId' in row1: + aws_access_key_id = row2.split(",")[1] + aws_secret_access_key = row2.split(",")[2] + elif "AWSAccessKeyId" in row1: # And split out for keys - aws_access_key_id = row1.split('=')[1] - aws_secret_access_key = row2.split('=')[1] + aws_access_key_id = row1.split("=")[1] + aws_secret_access_key = row2.split("=")[1] else: - err_msg = 'Credentials file not recognized, check file is correct' + err_msg = "Credentials file not recognized, check file is correct" raise Exception(err_msg) # Strip any carriage return/line feeds - aws_access_key_id = aws_access_key_id.replace('\r', '').replace( - '\n', '') - aws_secret_access_key = aws_secret_access_key.replace('\r', - '').replace( - '\n', '') + aws_access_key_id = aws_access_key_id.replace("\r", "").replace("\n", "") + aws_secret_access_key = aws_secret_access_key.replace("\r", "").replace( + "\n", "" + ) else: - aws_access_key_id = os.getenv('AWS_ACCESS_KEY_ID') - aws_secret_access_key = os.getenv('AWS_SECRET_ACCESS_KEY') + aws_access_key_id = os.getenv("AWS_ACCESS_KEY_ID") + aws_secret_access_key = os.getenv("AWS_SECRET_ACCESS_KEY") # Return keys return aws_access_key_id, aws_secret_access_key # Fetch bucket object def _fetch_bucket(self, bucket_name): - ''' + """ Method to return a bucket object which can be used to interact with an AWS S3 bucket using credentials found in a local file. - Parameters + + Parameters. ---------- self : nipype.interfaces.io.DataSink self for instance method bucket_name : string string corresponding to the name of the bucket on S3 + Returns ------- bucket : boto3.resources.factory.s3.Bucket boto3 s3 Bucket object which is used to interact with files in an S3 bucket on AWS - ''' - + """ # Import packages try: import boto3 import botocore - except ImportError as exc: - err_msg = 'Boto3 package is not installed - install boto3 and '\ - 'try again.' + except ImportError: + err_msg = "Boto3 package is not installed - install boto3 and " "try again." raise Exception(err_msg) # Init variables @@ -323,60 +327,57 @@ def _fetch_bucket(self, bucket_name): # Get AWS credentials try: - aws_access_key_id, aws_secret_access_key = \ - self._return_aws_keys() + aws_access_key_id, aws_secret_access_key = self._return_aws_keys() except Exception as exc: - err_msg = 'There was a problem extracting the AWS credentials '\ - 'from the credentials file provided: %s. Error:\n%s'\ - % (creds_path, exc) + err_msg = ( + "There was a problem extracting the AWS credentials " + "from the credentials file provided: %s. Error:\n%s" % (creds_path, exc) + ) raise Exception(err_msg) # Try and get AWS credentials if a creds_path is specified if aws_access_key_id and aws_secret_access_key: # Init connection - iflogger.info('Connecting to S3 bucket: %s with credentials...', - bucket_name) + iflogger.info( + "Connecting to S3 bucket: %s with credentials...", bucket_name + ) # Use individual session for each instance of DataSink # Better when datasinks are being used in multi-threading, see: # http://boto3.readthedocs.org/en/latest/guide/resources.html#multithreading session = boto3.session.Session( aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key) + aws_secret_access_key=aws_secret_access_key, + ) else: - iflogger.info('Connecting to S3 bucket: %s with IAM role...', - bucket_name) + iflogger.info("Connecting to S3 bucket: %s with IAM role...", bucket_name) # Lean on AWS environment / IAM role authentication and authorization session = boto3.session.Session() - s3_resource = session.resource('s3', use_ssl=True) + s3_resource = session.resource("s3", use_ssl=True) # And try fetch the bucket with the name argument try: _get_head_bucket(s3_resource, bucket_name) - except Exception as exc: - + except Exception: # Try to connect anonymously s3_resource.meta.client.meta.events.register( - 'choose-signer.s3.*', botocore.handlers.disable_signing) + "choose-signer.s3.*", botocore.handlers.disable_signing + ) - iflogger.info('Connecting to AWS: %s anonymously...', bucket_name) + iflogger.info("Connecting to AWS: %s anonymously...", bucket_name) _get_head_bucket(s3_resource, bucket_name) # Explicitly declare a secure SSL connection for bucket object - bucket = s3_resource.Bucket(bucket_name) + return s3_resource.Bucket(bucket_name) # Return the bucket - return bucket # Send up to S3 method def _upload_to_s3(self, bucket, src, dst): - ''' - Method to upload outputs to S3 bucket instead of on local disk - ''' - + """Method to upload outputs to S3 bucket instead of on local disk.""" # Import packages import hashlib import os @@ -384,12 +385,12 @@ def _upload_to_s3(self, bucket, src, dst): from botocore.exceptions import ClientError # Init variables - s3_str = 's3://' + s3_str = "s3://" s3_prefix = s3_str + bucket.name # Explicitly lower-case the "s3" - if dst[:len(s3_str)].lower() == s3_str: - dst = s3_str + dst[len(s3_str):] + if dst[: len(s3_str)].lower() == s3_str: + dst = s3_str + dst[len(s3_str) :] # If src is a directory, collect files (this assumes dst is a dir too) if os.path.isdir(src): @@ -397,10 +398,7 @@ def _upload_to_s3(self, bucket, src, dst): for root, dirs, files in os.walk(src): src_files.extend([os.path.join(root, fil) for fil in files]) # Make the dst files have the dst folder as base dir - dst_files = [ - os.path.join(dst, - src_f.split(src)[1]) for src_f in src_files - ] + dst_files = [os.path.join(dst, src_f.split(src)[1]) for src_f in src_files] else: src_files = [src] dst_files = [dst] @@ -409,7 +407,7 @@ def _upload_to_s3(self, bucket, src, dst): for src_idx, src_f in enumerate(src_files): # Get destination filename/keyname dst_f = dst_files[src_idx] - dst_k = dst_f.replace(s3_prefix, '').lstrip('/') + dst_k = dst_f.replace(s3_prefix, "").lstrip("/") # See if same file is already up there try: @@ -417,24 +415,24 @@ def _upload_to_s3(self, bucket, src, dst): dst_md5 = dst_obj.e_tag.strip('"') # See if same file is already there - src_read = open(src_f, 'rb').read() + src_read = open(src_f, "rb").read() src_md5 = hashlib.md5(src_read).hexdigest() # Move to next loop iteration if dst_md5 == src_md5: - iflogger.info('File %s already exists on S3, skipping...', - dst_f) + iflogger.info("File %s already exists on S3, skipping...", dst_f) continue else: - iflogger.info('Overwriting previous S3 file...') + iflogger.info("Overwriting previous S3 file...") except ClientError: - iflogger.info('New file to S3') + iflogger.info("New file to S3") # Copy file up to S3 (either encrypted or not) - iflogger.info('Uploading %s to S3 bucket, %s, as %s...', src_f, - bucket.name, dst_f) + iflogger.info( + "Uploading %s to S3 bucket, %s, as %s...", src_f, bucket.name, dst_f + ) if self.inputs.encrypt_bucket_keys: - extra_args = {'ServerSideEncryption': 'AES256'} + extra_args = {"ServerSideEncryption": "AES256"} else: extra_args = {} @@ -445,7 +443,7 @@ def _upload_to_s3(self, bucket, src, dst): src_f, dst_k, ExtraArgs=extra_args, - Callback=ProgressPercentage(src_f) + Callback=ProgressPercentage(src_f), ) break except Exception as exc: @@ -457,15 +455,12 @@ def _upload_to_s3(self, bucket, src, dst): # List outputs, main run routine def _list_outputs(self): - """Execute this module. - """ - + """Execute this module.""" # Init variables outputs = self.output_spec().get() out_files = [] # Use hardlink - use_hardlink = str2bool( - config.get('execution', 'try_hard_link_datasink')) + use_hardlink = str2bool(config.get("execution", "try_hard_link_datasink")) # Set local output directory if specified if isdefined(self.inputs.local_copy): @@ -474,7 +469,7 @@ def _list_outputs(self): outdir = self.inputs.base_directory # If base directory isn't given, assume current directory if not isdefined(outdir): - outdir = '.' + outdir = "." # Check if base directory reflects S3 bucket upload s3_flag, bucket_name = self._check_s3_base_dir() @@ -490,18 +485,21 @@ def _list_outputs(self): # If encountering an exception during bucket access, set output # base directory to a local folder except Exception as exc: - s3dir = '' + s3dir = "" if not isdefined(self.inputs.local_copy): local_out_exception = os.path.join( - os.path.expanduser('~'), - 's3_datasink_' + bucket_name) + os.path.expanduser("~"), "s3_datasink_" + bucket_name + ) outdir = local_out_exception # Log local copying directory iflogger.info( - 'Access to S3 failed! Storing outputs locally at: ' - '%s\nError: %s', outdir, exc) + "Access to S3 failed! Storing outputs locally at: " + "%s\nError: %s", + outdir, + exc, + ) else: - s3dir = '' + s3dir = "" # If container input is given, append that to outdir if isdefined(self.inputs.container): @@ -516,7 +514,7 @@ def _list_outputs(self): try: os.makedirs(outdir) except OSError as inst: - if 'File exists' in inst.strerror: + if "File exists" in inst.strerror: pass else: raise (inst) @@ -530,8 +528,8 @@ def _list_outputs(self): tempoutdir = outdir if s3_flag: s3tempoutdir = s3dir - for d in key.split('.'): - if d[0] == '@': + for d in key.split("."): + if d[0] == "@": continue tempoutdir = os.path.join(tempoutdir, d) if s3_flag: @@ -547,7 +545,7 @@ def _list_outputs(self): # Format src and dst files src = os.path.abspath(src) if not os.path.isfile(src): - src = os.path.join(src, '') + src = os.path.join(src, "") dst = self._get_dst(src) if s3_flag: s3dst = os.path.join(s3tempoutdir, dst) @@ -567,40 +565,36 @@ def _list_outputs(self): try: os.makedirs(path) except OSError as inst: - if 'File exists' in inst.strerror: + if "File exists" in inst.strerror: pass else: raise (inst) try: # If src == dst, it's already home - if (not os.path.exists(dst)) or ( - os.stat(src) != os.stat(dst) - ): + if (not os.path.exists(dst)) or (os.stat(src) != os.stat(dst)): # If src is a file, copy it to dst if os.path.isfile(src): - iflogger.debug(f'copyfile: {src} {dst}') + iflogger.debug(f"copyfile: {src} {dst}") copyfile( src, dst, copy=True, - hashmethod='content', - use_hardlink=use_hardlink) + hashmethod="content", + use_hardlink=use_hardlink, + ) # If src is a directory, copy # entire contents to dst dir elif os.path.isdir(src): - if ( - os.path.exists(dst) and - self.inputs.remove_dest_dir - ): - iflogger.debug('removing: %s', dst) + if os.path.exists(dst) and self.inputs.remove_dest_dir: + iflogger.debug("removing: %s", dst) shutil.rmtree(dst) - iflogger.debug('copydir: %s %s', src, dst) + iflogger.debug("copydir: %s %s", src, dst) copytree(src, dst) out_files.append(dst) except SameFileError: - iflogger.debug(f'copyfile (same file): {src} {dst}') + iflogger.debug(f"copyfile (same file): {src} {dst}") # Return outputs dictionary - outputs['out_file'] = out_files + outputs["out_file"] = out_files return outputs diff --git a/CPAC/utils/interfaces/fixes.py b/CPAC/utils/interfaces/fixes.py index d7f42e38ce..c2fc8089de 100644 --- a/CPAC/utils/interfaces/fixes.py +++ b/CPAC/utils/interfaces/fixes.py @@ -3,15 +3,15 @@ # vi: set ft=python sts=4 ts=4 sw=4 et: # This functionality is adapted from poldracklab/niworkflows: -# https://github.com/poldracklab/niworkflows/blob/master/niworkflows/interfaces/fixes.py +# https://github.com/poldracklab/niworkflows/blob/master/niworkflows/interfaces/fixes.py # https://fmriprep.readthedocs.io/ # https://poldracklab.stanford.edu/ # We are temporarily maintaining our own copy for more granular control. import os -from nipype.interfaces.ants.resampling import ApplyTransforms from nipype.interfaces.ants.registration import Registration +from nipype.interfaces.ants.resampling import ApplyTransforms from CPAC.info import __version__ from CPAC.utils.interfaces.utils import _copyxform @@ -21,18 +21,20 @@ class FixHeaderApplyTransforms(ApplyTransforms): """ A replacement for nipype.interfaces.ants.resampling.ApplyTransforms that fixes the resampled image header to match the xform of the reference - image + image. """ def _run_interface(self, runtime, correct_return_codes=(0,)): # Run normally runtime = super(FixHeaderApplyTransforms, self)._run_interface( - runtime, correct_return_codes) + runtime, correct_return_codes + ) - _copyxform(self.inputs.reference_image, - os.path.abspath(self._gen_filename('output_image')), - message='%s (niworkflows v%s)' % ( - self.__class__.__name__, __version__)) + _copyxform( + self.inputs.reference_image, + os.path.abspath(self._gen_filename("output_image")), + message="%s (niworkflows v%s)" % (self.__class__.__name__, __version__), + ) return runtime @@ -40,28 +42,31 @@ class FixHeaderRegistration(Registration): """ A replacement for nipype.interfaces.ants.registration.Registration that fixes the resampled image header to match the xform of the reference - image + image. """ def _run_interface(self, runtime, correct_return_codes=(0,)): # Run normally runtime = super(FixHeaderRegistration, self)._run_interface( - runtime, correct_return_codes) + runtime, correct_return_codes + ) # Forward transform out_file = self._get_outputfilenames(inverse=False) if out_file is not None and out_file: _copyxform( - self.inputs.fixed_image[0], os.path.abspath(out_file), - message='%s (niworkflows v%s)' % ( - self.__class__.__name__, __version__)) + self.inputs.fixed_image[0], + os.path.abspath(out_file), + message="%s (niworkflows v%s)" % (self.__class__.__name__, __version__), + ) # Inverse transform out_file = self._get_outputfilenames(inverse=True) if out_file is not None and out_file: _copyxform( - self.inputs.moving_image[0], os.path.abspath(out_file), - message='%s (niworkflows v%s)' % ( - self.__class__.__name__, __version__)) + self.inputs.moving_image[0], + os.path.abspath(out_file), + message="%s (niworkflows v%s)" % (self.__class__.__name__, __version__), + ) return runtime diff --git a/CPAC/utils/interfaces/fsl.py b/CPAC/utils/interfaces/fsl.py index b5deef5a6e..f08a45db0b 100644 --- a/CPAC/utils/interfaces/fsl.py +++ b/CPAC/utils/interfaces/fsl.py @@ -1,11 +1,13 @@ -"""FSL Nipype interfaces with customized functionality""" +"""FSL Nipype interfaces with customized functionality.""" import os + from nipype.interfaces.base import isdefined from nipype.interfaces.fsl.utils import Merge as fslMerge class Merge(fslMerge): - """Use relative paths for input files""" + """Use relative paths for input files.""" + def _format_arg(self, name, spec, value): if name == "tr": if self.inputs.dimension != "t": @@ -18,9 +20,7 @@ def _format_arg(self, name, spec, value): # Begin custom code # ----------------- if name == "in_files": - return ' '.join([ - os.path.relpath(in_file) for in_file in value - ]) + return " ".join([os.path.relpath(in_file) for in_file in value]) # --------------- # End custom code return super(Merge, self)._format_arg(name, spec, value) diff --git a/CPAC/utils/interfaces/function/__init__.py b/CPAC/utils/interfaces/function/__init__.py index 2c96c79fed..604012287f 100644 --- a/CPAC/utils/interfaces/function/__init__.py +++ b/CPAC/utils/interfaces/function/__init__.py @@ -1,5 +1,5 @@ -"""Function interface utilities for C-PAC""" +"""Function interface utilities for C-PAC.""" from .function import Function from .seg_preproc import pick_tissue_from_labels_file_interface -__all__ = ['Function', 'pick_tissue_from_labels_file_interface'] +__all__ = ["Function", "pick_tissue_from_labels_file_interface"] diff --git a/CPAC/utils/interfaces/function/function.py b/CPAC/utils/interfaces/function/function.py index 1c091a0c75..21178c2b2f 100644 --- a/CPAC/utils/interfaces/function/function.py +++ b/CPAC/utils/interfaces/function/function.py @@ -1,27 +1,31 @@ -from builtins import str, bytes +from builtins import bytes, str import inspect from typing import Callable, List from nipype import logging -from nipype.interfaces.base import (traits, DynamicTraitedSpec, Undefined, - isdefined, BaseInterfaceInputSpec) +from nipype.interfaces.base import ( + BaseInterfaceInputSpec, + DynamicTraitedSpec, + Undefined, + isdefined, + traits, +) from nipype.interfaces.io import IOBase, add_traits from nipype.utils.filemanip import ensure_list -from nipype.utils.functions import getsource, create_function_from_source +from nipype.utils.functions import create_function_from_source, getsource -iflogger = logging.getLogger('nipype.interface') +iflogger = logging.getLogger("nipype.interface") class FunctionInputSpec(DynamicTraitedSpec, BaseInterfaceInputSpec): - function_str = traits.Str(mandatory=True, desc='code for function') + function_str = traits.Str(mandatory=True, desc="code for function") class Function(IOBase): - """Runs arbitrary function as an interface + """Runs arbitrary function as an interface. Examples -------- - >>> func = 'def func(arg1, arg2=5): return arg1 + arg2' >>> fi = Function(input_names=['arg1', 'arg2'], output_names=['out']) >>> fi.inputs.function_str = func @@ -34,18 +38,19 @@ class Function(IOBase): input_spec = FunctionInputSpec output_spec = DynamicTraitedSpec - def __init__(self, - input_names=None, - output_names='out', - function=None, - imports=None, - as_module=False, - **inputs): + def __init__( + self, + input_names=None, + output_names="out", + function=None, + imports=None, + as_module=False, + **inputs, + ): """ Parameters ---------- - input_names : single str or list or None names corresponding to function inputs if ``None``, derive input names from function argument names @@ -63,46 +68,46 @@ def __init__(self, decorator, the imports given as a parameter here will take precedence over those from the decorator. """ - super().__init__(**inputs) if function: - if hasattr(function, 'ns_imports'): + if hasattr(function, "ns_imports"): # prepend the ns_imports from the decorator to # the paramater imports _ns_imports = [ - 'from CPAC.utils.interfaces.function import Function', - *function.ns_imports] - imports = _ns_imports if imports is None else [*_ns_imports, - *imports] + "from CPAC.utils.interfaces.function import Function", + *function.ns_imports, + ] + imports = _ns_imports if imports is None else [*_ns_imports, *imports] if as_module: module = inspect.getmodule(function).__name__ full_name = "%s.%s" % (module, function.__name__) self.inputs.function_str = full_name - elif hasattr(function, '__call__'): + elif hasattr(function, "__call__"): try: self.inputs.function_str = getsource(function) except IOError: - raise Exception('Interface Function does not accept ' - 'function objects defined interactively ' - 'in a python session') + raise Exception( + "Interface Function does not accept " + "function objects defined interactively " + "in a python session" + ) else: if input_names is None: fninfo = function.__code__ elif isinstance(function, (str, bytes)): self.inputs.function_str = function if input_names is None: - fninfo = create_function_from_source(function, - imports).__code__ + fninfo = create_function_from_source(function, imports).__code__ else: - raise Exception('Unknown type of function') + raise Exception("Unknown type of function") if input_names is None: - input_names = fninfo.co_varnames[:fninfo.co_argcount] + input_names = fninfo.co_varnames[: fninfo.co_argcount] self.as_module = as_module - self.inputs.on_trait_change(self._set_function_string, 'function_str') + self.inputs.on_trait_change(self._set_function_string, "function_str") self._input_names = ensure_list(input_names) self._output_names = ensure_list(output_names) - add_traits(self.inputs, [name for name in self._input_names]) + add_traits(self.inputs, list(self._input_names)) self.imports = imports self._out = {} for name in self._output_names: @@ -155,30 +160,30 @@ def sig_imports(imports: List[str]) -> Callable: >>> inspect.signature(calculate_FD_J) == inspect.signature(f) True """ + def _imports(func: Callable) -> Callable: - setattr(func, 'ns_imports', imports) + setattr(func, "ns_imports", imports) return func + return _imports def _set_function_string(self, obj, name, old, new): - if name == 'function_str': + if name == "function_str": if self.as_module: module = inspect.getmodule(new).__name__ full_name = "%s.%s" % (module, new.__name__) self.inputs.function_str = full_name - elif hasattr(new, '__call__'): + elif hasattr(new, "__call__"): function_source = getsource(new) fninfo = new.__code__ elif isinstance(new, (str, bytes)): function_source = new - fninfo = create_function_from_source(new, - self.imports).__code__ + fninfo = create_function_from_source(new, self.imports).__code__ self.inputs.trait_set( - trait_change_notify=False, **{ - '%s' % name: function_source - }) + trait_change_notify=False, **{"%s" % name: function_source} + ) # Update input traits - input_names = fninfo.co_varnames[:fninfo.co_argcount] + input_names = fninfo.co_varnames[: fninfo.co_argcount] new_names = set(input_names) - set(self._input_names) add_traits(self.inputs, list(new_names)) self._input_names.extend(new_names) @@ -193,18 +198,22 @@ def _add_output_traits(self, base): def _run_interface(self, runtime): # Create function handle - if self.as_module: - import importlib - pieces = self.inputs.function_str.split('.') - module = '.'.join(pieces[:-1]) - function = pieces[-1] - try: - function_handle = getattr(importlib.import_module(module), function) - except ImportError: - raise RuntimeError('Could not import module: %s' % self.inputs.function_str) - else: - function_handle = create_function_from_source(self.inputs.function_str, - self.imports) + if self.as_module: + import importlib + + pieces = self.inputs.function_str.split(".") + module = ".".join(pieces[:-1]) + function = pieces[-1] + try: + function_handle = getattr(importlib.import_module(module), function) + except ImportError: + raise RuntimeError( + "Could not import module: %s" % self.inputs.function_str + ) + else: + function_handle = create_function_from_source( + self.inputs.function_str, self.imports + ) # Get function args args = {} @@ -217,9 +226,8 @@ def _run_interface(self, runtime): if len(self._output_names) == 1: self._out[self._output_names[0]] = out else: - if isinstance(out, tuple) and \ - (len(out) != len(self._output_names)): - raise RuntimeError('Mismatch in number of expected outputs') + if isinstance(out, tuple) and (len(out) != len(self._output_names)): + raise RuntimeError("Mismatch in number of expected outputs") else: for idx, name in enumerate(self._output_names): diff --git a/CPAC/utils/interfaces/function/seg_preproc.py b/CPAC/utils/interfaces/function/seg_preproc.py index b713a6aa80..8dbed1c143 100644 --- a/CPAC/utils/interfaces/function/seg_preproc.py +++ b/CPAC/utils/interfaces/function/seg_preproc.py @@ -1,10 +1,10 @@ -"""Function interfaces for seg_preproc""" +"""Function interfaces for seg_preproc.""" from nipype.interfaces import utility as util def pick_tissue_from_labels_file_interface(input_names=None): """Function to create a Function interface for - CPAC.seg_preproc.utils.pick_tissue_from_labels_file + CPAC.seg_preproc.utils.pick_tissue_from_labels_file. Parameters ---------- @@ -18,9 +18,9 @@ def pick_tissue_from_labels_file_interface(input_names=None): from CPAC.seg_preproc.utils import pick_tissue_from_labels_file if input_names is None: - input_names = ['multiatlas_Labels', 'csf_label', 'gm_label', - 'wm_label'] + input_names = ["multiatlas_Labels", "csf_label", "gm_label", "wm_label"] return util.Function( input_names=input_names, - output_names=['csf_mask', 'gm_mask', 'wm_mask'], - function=pick_tissue_from_labels_file) + output_names=["csf_mask", "gm_mask", "wm_mask"], + function=pick_tissue_from_labels_file, + ) diff --git a/CPAC/utils/interfaces/masktool.py b/CPAC/utils/interfaces/masktool.py index a3686e4c64..5993cc3b59 100644 --- a/CPAC/utils/interfaces/masktool.py +++ b/CPAC/utils/interfaces/masktool.py @@ -1,85 +1,82 @@ -from builtins import str, bytes -import inspect - from nipype import logging -from nipype.interfaces.base import (CommandLineInputSpec, CommandLine, Directory, TraitedSpec, - traits, isdefined, File, InputMultiObject, InputMultiPath, - Undefined, Str) - -from nipype.interfaces.afni.base import (AFNICommandBase, AFNICommand, AFNICommandInputSpec, - AFNICommandOutputSpec, AFNIPythonCommandInputSpec, - AFNIPythonCommand) -from nipype.interfaces.io import IOBase, add_traits -from nipype.utils.filemanip import ensure_list -from nipype.utils.functions import getsource, create_function_from_source +from nipype.interfaces.afni.base import AFNICommand, AFNICommandInputSpec +from nipype.interfaces.base import File, InputMultiPath, Str, TraitedSpec, traits -iflogger = logging.getLogger('nipype.interface') +iflogger = logging.getLogger("nipype.interface") class MaskToolInputSpec(AFNICommandInputSpec): in_files = InputMultiPath( - File(desc='input file to 3dmerge', exists=True), - argstr='-input %s', + File(desc="input file to 3dmerge", exists=True), + argstr="-input %s", position=-1, mandatory=True, - copyfile=False) + copyfile=False, + ) out_file = File( - name_template='%s_mask', - desc='output image file name', - argstr='-prefix %s', - name_source='in_files') + name_template="%s_mask", + desc="output image file name", + argstr="-prefix %s", + name_source="in_files", + ) count = traits.Bool( - desc='Instead of created a binary 0/1 mask dataset, create one with ' - 'counts of voxel overlap, i.e., each voxel will contain the ' - 'number of masks that it is set in.', - argstr='-count', - position=2) + desc="Instead of created a binary 0/1 mask dataset, create one with " + "counts of voxel overlap, i.e., each voxel will contain the " + "number of masks that it is set in.", + argstr="-count", + position=2, + ) datum = traits.Enum( - 'byte', - 'short', - 'float', - argstr='-datum %s', - desc='specify data type for output. Valid types are \'byte\', ' - '\'short\' and \'float\'.') + "byte", + "short", + "float", + argstr="-datum %s", + desc="specify data type for output. Valid types are 'byte', " + "'short' and 'float'.", + ) dilate_inputs = Str( - desc='Use this option to dilate and/or erode datasets as they are ' - 'read. ex. \'5 -5\' to dilate and erode 5 times', - argstr='-dilate_inputs %s') + desc="Use this option to dilate and/or erode datasets as they are " + "read. ex. '5 -5' to dilate and erode 5 times", + argstr="-dilate_inputs %s", + ) dilate_results = Str( - desc='dilate and/or erode combined mask at the given levels.', - argstr='-dilate_results %s') + desc="dilate and/or erode combined mask at the given levels.", + argstr="-dilate_results %s", + ) frac = traits.Float( - desc='When combining masks (across datasets and sub-bricks), use ' - 'this option to restrict the result to a certain fraction of the ' - 'set of volumes', - argstr='-frac %s') - inter = traits.Bool( - desc='intersection, this means -frac 1.0', argstr='-inter') - union = traits.Bool(desc='union, this means -frac 0', argstr='-union') + desc="When combining masks (across datasets and sub-bricks), use " + "this option to restrict the result to a certain fraction of the " + "set of volumes", + argstr="-frac %s", + ) + inter = traits.Bool(desc="intersection, this means -frac 1.0", argstr="-inter") + union = traits.Bool(desc="union, this means -frac 0", argstr="-union") fill_holes = traits.Bool( - desc='This option can be used to fill holes in the resulting mask, ' - 'i.e. after all other processing has been done.', - argstr='-fill_holes') + desc="This option can be used to fill holes in the resulting mask, " + "i.e. after all other processing has been done.", + argstr="-fill_holes", + ) fill_dirs = Str( - desc='fill holes only in the given directions. This option is for use ' - 'with -fill holes. should be a single string that specifies ' - '1-3 of the axes using {x,y,z} labels (i.e. dataset axis order), ' - 'or using the labels in {R,L,A,P,I,S}.', - argstr='-fill_dirs %s', - requires=['fill_holes']) - verbose = traits.Int( - desc='specify verbosity level, for 0 to 3', argstr='-verb %s') + desc="fill holes only in the given directions. This option is for use " + "with -fill holes. should be a single string that specifies " + "1-3 of the axes using {x,y,z} labels (i.e. dataset axis order), " + "or using the labels in {R,L,A,P,I,S}.", + argstr="-fill_dirs %s", + requires=["fill_holes"], + ) + verbose = traits.Int(desc="specify verbosity level, for 0 to 3", argstr="-verb %s") class MaskToolOutputSpec(TraitedSpec): - out_file = File(desc='mask file', exists=True) + out_file = File(desc="mask file", exists=True) class MaskTool(AFNICommand): """3dmask_tool - for combining/dilating/eroding/filling masks For complete details, see the `3dmask_tool Documentation. - `_ - Examples + `_. + + Examples. ======== >>> from nipype.interfaces import afni >>> masktool = afni.MaskTool() @@ -90,6 +87,6 @@ class MaskTool(AFNICommand): >>> res = automask.run() # doctest: +SKIP """ - _cmd = '3dmask_tool' + _cmd = "3dmask_tool" input_spec = MaskToolInputSpec output_spec = MaskToolOutputSpec diff --git a/CPAC/utils/interfaces/netcorr.py b/CPAC/utils/interfaces/netcorr.py index 72fd89c6b6..c5b5bc52e7 100644 --- a/CPAC/utils/interfaces/netcorr.py +++ b/CPAC/utils/interfaces/netcorr.py @@ -1,11 +1,15 @@ -"""Interface for AFNI 3dNetCorr""" +"""Interface for AFNI 3dNetCorr.""" import glob import os import subprocess -from nipype.interfaces.afni.base import AFNICommand, AFNICommandInputSpec, \ - AFNICommandOutputSpec -from nipype.interfaces.base import File, isdefined + from traits.api import Bool, List +from nipype.interfaces.afni.base import ( + AFNICommand, + AFNICommandInputSpec, + AFNICommandOutputSpec, +) +from nipype.interfaces.base import File, isdefined class NetCorrInputSpec(AFNICommandInputSpec): @@ -30,9 +34,9 @@ class NetCorrInputSpec(AFNICommandInputSpec): ) automask_off = Bool( False, - desc='If you want to neither put in a mask ' - '*nor* have the automasking occur', - argstr='-automask_off', usedefault=True + desc="If you want to neither put in a mask " "*nor* have the automasking occur", + argstr="-automask_off", + usedefault=True, ) weight_ts = File( desc="input a 1D file WTS of weights that will be applied " @@ -57,9 +61,7 @@ class NetCorrInputSpec(AFNICommandInputSpec): "high Pearson-r value", argstr="-fish_z", ) - part_corr = Bool( - desc="output the partial correlation matrix", argstr="-part_corr" - ) + part_corr = Bool(desc="output the partial correlation matrix", argstr="-part_corr") ts_out = Bool( desc="switch to output the mean time series of the ROIs that " "have been used to generate the correlation matrices. " @@ -160,7 +162,7 @@ class NetCorrInputSpec(AFNICommandInputSpec): class NetCorrOutputSpec(AFNICommandOutputSpec): out_corr_matrix = File( desc="output correlation matrix between ROIs written to a text " - "file with .netcc suffix" + "file with .netcc suffix" ) out_corr_maps = List( File(), desc="output correlation maps in Pearson and/or Z-scores" @@ -189,7 +191,8 @@ class NetCorr(AFNICommand): '3dNetCorr -prefix sub0.tp1.ncorr -fish_z -inset functional.nii -in_rois maps.nii -mask mask.nii -ts_wb_Z -ts_wb_corr' >>> res = ncorr.run() # doctest: +SKIP - """ # noqa: E501 # pylint: disable=line-too-long + """ # pylint: disable=line-too-long + _cmd = "3dNetCorr" input_spec = NetCorrInputSpec output_spec = NetCorrOutputSpec @@ -204,22 +207,16 @@ def _list_outputs(self): # All outputs should be in the same directory as the prefix odir = os.path.dirname(os.path.abspath(prefix)) - outputs["out_corr_matrix"] = glob.glob( - os.path.join(odir, "*.netcc"))[0] + outputs["out_corr_matrix"] = glob.glob(os.path.join(odir, "*.netcc"))[0] - if ( - isdefined(self.inputs.ts_wb_corr) or - isdefined(self.inputs.ts_wb_Z) - ): + if isdefined(self.inputs.ts_wb_corr) or isdefined(self.inputs.ts_wb_Z): corrdir = os.path.join(odir, prefix + "_000_INDIV") - outputs["out_corr_maps"] = glob.glob( - os.path.join(corrdir, "*.nii.gz")) + outputs["out_corr_maps"] = glob.glob(os.path.join(corrdir, "*.nii.gz")) return outputs def strip_afni_output_header(in_file, out_file): - """Function to rewrite a file with all but the first 6 lines""" - subprocess.run(f'tail -n +7 {in_file} > {out_file}', shell=True, - check=True) + """Function to rewrite a file with all but the first 6 lines.""" + subprocess.run(f"tail -n +7 {in_file} > {out_file}", shell=True, check=True) return out_file diff --git a/CPAC/utils/interfaces/pc.py b/CPAC/utils/interfaces/pc.py index 20200f9a80..91deb20420 100644 --- a/CPAC/utils/interfaces/pc.py +++ b/CPAC/utils/interfaces/pc.py @@ -1,51 +1,45 @@ -from builtins import str, bytes -import inspect - from nipype import logging -from nipype.interfaces.base import (CommandLineInputSpec, CommandLine, Directory, TraitedSpec, - traits, isdefined, File, InputMultiObject, InputMultiPath, - Undefined, Str) - -from nipype.interfaces.afni.base import (AFNICommandBase, AFNICommand, AFNICommandInputSpec, - AFNICommandOutputSpec, AFNIPythonCommandInputSpec, - AFNIPythonCommand) -from nipype.interfaces.io import IOBase, add_traits -from nipype.utils.filemanip import ensure_list -from nipype.utils.functions import getsource, create_function_from_source +from nipype.interfaces.afni.base import AFNICommand, AFNICommandInputSpec +from nipype.interfaces.base import File, TraitedSpec, traits -iflogger = logging.getLogger('nipype.interface') +iflogger = logging.getLogger("nipype.interface") class PCInputSpec(AFNICommandInputSpec): in_file = File( - desc='input file to 3dpc', - argstr='%s', + desc="input file to 3dpc", + argstr="%s", position=-1, mandatory=True, exists=True, - copyfile=False) + copyfile=False, + ) out_file = File( - name_template='%s_pcs', - desc='output image file name', - argstr='-prefix %s', - name_source='in_file') - mask = File(desc='input mask', argstr='-mask %s', exists=True) + name_template="%s_pcs", + desc="output image file name", + argstr="-prefix %s", + name_source="in_file", + ) + mask = File(desc="input mask", argstr="-mask %s", exists=True) pcs_file = File( - name_template='%s_vec.1D', - desc='output image file name', - name_source='out_file', - keep_extension=True) + name_template="%s_vec.1D", + desc="output image file name", + name_source="out_file", + keep_extension=True, + ) pcs = traits.Int( - desc='number of components to save in the output;' - 'it cant be more than the number of input bricks', argstr='-pcsave %s') + desc="number of components to save in the output;" + "it cant be more than the number of input bricks", + argstr="-pcsave %s", + ) class PCOutputSpec(TraitedSpec): - out_file = File(desc='mask file', exists=True) - pcs_file = File(desc='pcs', exists=True) + out_file = File(desc="mask file", exists=True) + pcs_file = File(desc="pcs", exists=True) class PC(AFNICommand): - _cmd = '3dpc' + _cmd = "3dpc" input_spec = PCInputSpec output_spec = PCOutputSpec diff --git a/CPAC/utils/interfaces/tests/test_fsl.py b/CPAC/utils/interfaces/tests/test_fsl.py index 1f434dadf4..eb64dca4b0 100644 --- a/CPAC/utils/interfaces/tests/test_fsl.py +++ b/CPAC/utils/interfaces/tests/test_fsl.py @@ -15,53 +15,56 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . # pylint: disable=invalid-name -"""Tests for FSL interface""" +"""Tests for FSL interface.""" import os from pathlib import Path + from ..fsl import Merge def test_HO_ventricles_exists(): """Make sure we have this required file - Ref https://github.com/FCP-INDI/C-PAC/pull/1916#issuecomment-1579286459""" - assert (Path(os.environ["FSLDIR"]) / - 'data/atlases/HarvardOxford/' - 'HarvardOxford-lateral-ventricles-thr25-2mm.nii.gz').exists() + Ref https://github.com/FCP-INDI/C-PAC/pull/1916#issuecomment-1579286459. + """ + assert ( + Path(os.environ["FSLDIR"]) / "data/atlases/HarvardOxford/" + "HarvardOxford-lateral-ventricles-thr25-2mm.nii.gz" + ).exists() def test_Merge_inputs(): - input_map = dict( - args=dict( - argstr="%s", - ), - dimension=dict( - argstr="-%s", - mandatory=True, - position=0, - ), - environ=dict( - nohash=True, - usedefault=True, - ), - in_files=dict( - argstr="%s", - mandatory=True, - position=2, - ), - merged_file=dict( - argstr="%s", - extensions=None, - hash_files=False, - name_source="in_files", - name_template="%s_merged", - position=1, - ), - output_type=dict(), - tr=dict( - argstr="%.2f", - position=-1, - ), - ) + input_map = { + "args": { + "argstr": "%s", + }, + "dimension": { + "argstr": "-%s", + "mandatory": True, + "position": 0, + }, + "environ": { + "nohash": True, + "usedefault": True, + }, + "in_files": { + "argstr": "%s", + "mandatory": True, + "position": 2, + }, + "merged_file": { + "argstr": "%s", + "extensions": None, + "hash_files": False, + "name_source": "in_files", + "name_template": "%s_merged", + "position": 1, + }, + "output_type": {}, + "tr": { + "argstr": "%.2f", + "position": -1, + }, + } inputs = Merge.input_spec() for key, metadata in list(input_map.items()): @@ -70,11 +73,11 @@ def test_Merge_inputs(): def test_Merge_outputs(): - output_map = dict( - merged_file=dict( - extensions=None, - ), - ) + output_map = { + "merged_file": { + "extensions": None, + }, + } outputs = Merge.output_spec() for key, metadata in list(output_map.items()): diff --git a/CPAC/utils/interfaces/utils.py b/CPAC/utils/interfaces/utils.py index 6ec968bb8c..4c7e8c47fa 100644 --- a/CPAC/utils/interfaces/utils.py +++ b/CPAC/utils/interfaces/utils.py @@ -3,55 +3,48 @@ # vi: set ft=python sts=4 ts=4 sw=4 et: # This functionality is adapted from poldracklab/niworkflows: -# https://github.com/poldracklab/niworkflows/blob/master/niworkflows/interfaces/utils.py +# https://github.com/poldracklab/niworkflows/blob/master/niworkflows/interfaces/utils.py # https://fmriprep.readthedocs.io/ # https://poldracklab.stanford.edu/ # We are temporarily maintaining our own copy for more granular control. -""" -Utilities +"""Utilities.""" -""" - -import os -import re -import json +import copy import shutil + import numpy as np -import nibabel as nb -import nilearn.image as nli -# from textwrap import indent -from collections import OrderedDict +import nibabel as nib -import scipy.ndimage as nd +# from textwrap import indent from nipype import logging -from nipype.utils.filemanip import fname_presuffix -from nipype.interfaces.io import add_traits from nipype.interfaces.base import ( - traits, isdefined, File, InputMultiPath, - TraitedSpec, BaseInterfaceInputSpec, SimpleInterface, - DynamicTraitedSpec + BaseInterfaceInputSpec, + DynamicTraitedSpec, + File, + SimpleInterface, ) +from nipype.interfaces.io import add_traits +from nipype.utils.filemanip import fname_presuffix + from CPAC.info import __version__ -import copy -LOG = logging.getLogger('nipype.interface') +LOG = logging.getLogger("nipype.interface") class CopyXFormInputSpec(DynamicTraitedSpec, BaseInterfaceInputSpec): - hdr_file = File(exists=True, mandatory=True, desc='the file we get the header from') + hdr_file = File(exists=True, mandatory=True, desc="the file we get the header from") class CopyXForm(SimpleInterface): - """ - Copy the x-form matrices from `hdr_file` to `out_file`. - """ + """Copy the x-form matrices from `hdr_file` to `out_file`.""" + input_spec = CopyXFormInputSpec output_spec = DynamicTraitedSpec def __init__(self, fields=None, **inputs): - self._fields = fields or ['in_file'] + self._fields = fields or ["in_file"] if isinstance(self._fields, str): self._fields = [self._fields] @@ -65,10 +58,10 @@ def _outputs(self): base = super(CopyXForm, self)._outputs() if self._fields: fields = copy.copy(self._fields) - if 'in_file' in fields: - idx = fields.index('in_file') + if "in_file" in fields: + idx = fields.index("in_file") fields.pop(idx) - fields.insert(idx, 'out_file') + fields.insert(idx, "out_file") base = add_traits(base, fields) return base @@ -81,35 +74,40 @@ def _run_interface(self, runtime): in_files = [in_files] for in_file in in_files: out_name = fname_presuffix( - in_file, suffix='_xform', newpath=runtime.cwd) + in_file, suffix="_xform", newpath=runtime.cwd + ) # Copy and replace header shutil.copy(in_file, out_name) - _copyxform(self.inputs.hdr_file, out_name, - message='CopyXForm (niworkflows v%s)' % __version__) + _copyxform( + self.inputs.hdr_file, + out_name, + message="CopyXForm (niworkflows v%s)" % __version__, + ) self._results[f].append(out_name) # Flatten out one-element lists if len(self._results[f]) == 1: self._results[f] = self._results[f][0] - default = self._results.pop('in_file', None) + default = self._results.pop("in_file", None) if default: - self._results['out_file'] = default + self._results["out_file"] = default return runtime def _copyxform(ref_image, out_image, message=None): # Read in reference and output # Use mmap=False because we will be overwriting the output image - resampled = nb.load(out_image, mmap=False) - orig = nb.load(ref_image) + resampled = nib.load(out_image, mmap=False) + orig = nib.load(ref_image) if not np.allclose(orig.affine, resampled.affine): LOG.debug( - 'Affines of input and reference images do not match, ' - 'FMRIPREP will set the reference image headers. ' - 'Please, check that the x-form matrices of the input dataset' - 'are correct and manually verify the alignment of results.') + "Affines of input and reference images do not match, " + "FMRIPREP will set the reference image headers. " + "Please, check that the x-form matrices of the input dataset" + "are correct and manually verify the alignment of results." + ) # Copy xform infos qform, qform_code = orig.header.get_qform(coded=True) @@ -117,7 +115,7 @@ def _copyxform(ref_image, out_image, message=None): header = resampled.header.copy() header.set_qform(qform, int(qform_code)) header.set_sform(sform, int(sform_code)) - header['descrip'] = 'xform matrices modified by %s.' % (message or '(unknown)') + header["descrip"] = "xform matrices modified by %s." % (message or "(unknown)") newimg = resampled.__class__(resampled.get_fdata(), orig.affine, header) newimg.to_filename(out_image) diff --git a/CPAC/utils/misc.py b/CPAC/utils/misc.py index b0798918f3..ca034d5567 100644 --- a/CPAC/utils/misc.py +++ b/CPAC/utils/misc.py @@ -1,21 +1,19 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -""" -Miscellaneous utilities -""" +"""Miscellaneous utilities.""" # This functionality is adapted from poldracklab/niworkflows: -# https://github.com/poldracklab/niworkflows/blob/master/niworkflows/utils/misc.py +# https://github.com/poldracklab/niworkflows/blob/master/niworkflows/utils/misc.py # https://fmriprep.readthedocs.io/ # https://poldracklab.stanford.edu/ # We are temporarily maintaining our own copy for more granular control. -__all__ = ['get_template_specs'] +__all__ = ["get_template_specs"] def get_template_specs(in_template, template_spec=None, default_resolution=1): """ - Parse template specifications + Parse template specifications. >>> get_template_specs('MNI152NLin2009cAsym', {'suffix': 'T1w'})[1] # doctest: +SKIP {'resolution': 1} @@ -42,14 +40,16 @@ def get_template_specs(in_template, template_spec=None, default_resolution=1): # from templateflow.api import get as get_template # Massage spec (start creating if None) template_spec = template_spec or {} - template_spec['desc'] = template_spec.get('desc', None) - template_spec['atlas'] = template_spec.get('atlas', None) - template_spec['resolution'] = template_spec.pop( - 'res', template_spec.get('resolution', default_resolution)) + template_spec["desc"] = template_spec.get("desc", None) + template_spec["atlas"] = template_spec.get("atlas", None) + template_spec["resolution"] = template_spec.pop( + "res", template_spec.get("resolution", default_resolution) + ) - common_spec = {'resolution': template_spec['resolution']} - if 'cohort' in template_spec: - common_spec['cohort'] = template_spec['cohort'] + common_spec = {"resolution": template_spec["resolution"]} + if "cohort" in template_spec: + common_spec["cohort"] = template_spec["cohort"] -if __name__ == '__main__': + +if __name__ == "__main__": pass diff --git a/CPAC/utils/monitoring/__init__.py b/CPAC/utils/monitoring/__init__.py index d78c839771..d9b5f4b33a 100644 --- a/CPAC/utils/monitoring/__init__.py +++ b/CPAC/utils/monitoring/__init__.py @@ -1,14 +1,29 @@ -'''Module to customize Nipype's process monitoring for use in C-PAC +"""Module to customize Nipype's process monitoring for use in C-PAC. See https://fcp-indi.github.io/docs/developer/nodes for C-PAC-specific documentation. -See https://nipype.readthedocs.io/en/latest/api/generated/nipype.utils.profiler.html for Nipype's documentation.''' # noqa: E501 # pylint: disable=line-too-long +See https://nipype.readthedocs.io/en/latest/api/generated/nipype.utils.profiler.html for Nipype's documentation. +""" # pylint: disable=line-too-long from .config import LOGTAIL, WARNING_FREESURFER_OFF_WITH_DATA from .custom_logging import failed_to_start, getLogger, set_up_logger -from .monitoring import LoggingHTTPServer, LoggingRequestHandler, \ - log_nodes_cb, log_nodes_initial, monitor_server, \ - recurse_nodes +from .monitoring import ( + LoggingHTTPServer, + LoggingRequestHandler, + log_nodes_cb, + log_nodes_initial, + monitor_server, + recurse_nodes, +) -__all__ = ['failed_to_start', 'getLogger', 'LoggingHTTPServer', - 'LoggingRequestHandler', 'log_nodes_cb', 'log_nodes_initial', - 'LOGTAIL', 'monitor_server', 'recurse_nodes', 'set_up_logger', - 'WARNING_FREESURFER_OFF_WITH_DATA'] +__all__ = [ + "failed_to_start", + "getLogger", + "LoggingHTTPServer", + "LoggingRequestHandler", + "log_nodes_cb", + "log_nodes_initial", + "LOGTAIL", + "monitor_server", + "recurse_nodes", + "set_up_logger", + "WARNING_FREESURFER_OFF_WITH_DATA", +] diff --git a/CPAC/utils/monitoring/config.py b/CPAC/utils/monitoring/config.py index 75a345eb20..d709d16a5b 100644 --- a/CPAC/utils/monitoring/config.py +++ b/CPAC/utils/monitoring/config.py @@ -1,6 +1,7 @@ -"""Global variables for logging""" -LOGTAIL = {'warnings': []} +"""Global variables for logging.""" +LOGTAIL = {"warnings": []} MOCK_LOGGERS = {} WARNING_FREESURFER_OFF_WITH_DATA = ( - 'The provided data configuration includes \'freesurfer_dir\' but the ' - 'provided pipeline config is not configured to use FreeSurfer outputs.') + "The provided data configuration includes 'freesurfer_dir' but the " + "provided pipeline config is not configured to use FreeSurfer outputs." +) diff --git a/CPAC/utils/monitoring/custom_logging.py b/CPAC/utils/monitoring/custom_logging.py index 370eb80250..812dcbaf13 100644 --- a/CPAC/utils/monitoring/custom_logging.py +++ b/CPAC/utils/monitoring/custom_logging.py @@ -15,13 +15,16 @@ License for more details. You should have received a copy of the GNU Lesser General Public -License along with C-PAC. If not, see .""" +License along with C-PAC. If not, see . +""" import logging import os import subprocess from sys import exc_info as sys_exc_info from traceback import print_exception + from nipype import logging as nipype_logging + from CPAC.utils.docs import docstring_parameter from CPAC.utils.monitoring.config import MOCK_LOGGERS @@ -37,9 +40,8 @@ def failed_to_start(log_dir, exception): exception : Exception """ - logger = set_up_logger('failedToStart', 'failedToStart.log', 'error', - log_dir, True) - logger.exception('C-PAC failed to start') + logger = set_up_logger("failedToStart", "failedToStart.log", "error", log_dir, True) + logger.exception("C-PAC failed to start") logger.exception(exception) @@ -62,13 +64,13 @@ def getLogger(name): # pylint: disable=invalid-name def log_failed_subprocess(cpe): - """Pass STDERR from a subprocess to the interface's logger + """Pass STDERR from a subprocess to the interface's logger. Parameters ---------- cpe : subprocess.CalledProcessError """ - logger = getLogger('nipype.interface') + logger = getLogger("nipype.interface") logger.error("%s\nExit code %s", cpe.output, cpe.returncode) @@ -115,10 +117,11 @@ def log_subprocess(cmd, *args, raise_error=True, **kwargs): exit_code : int """ - logger = getLogger('nipype.interface') + logger = getLogger("nipype.interface") try: - output = subprocess.check_output(cmd, *args, stderr=subprocess.STDOUT, - universal_newlines=True, **kwargs) + output = subprocess.check_output( + cmd, *args, stderr=subprocess.STDOUT, universal_newlines=True, **kwargs + ) logger.info(output) except subprocess.CalledProcessError as cpe: log_failed_subprocess(cpe) @@ -131,6 +134,7 @@ def log_subprocess(cmd, *args, raise_error=True, **kwargs): # pylint: disable=too-few-public-methods class MockHandler: """Handler for MockLogger.""" + def __init__(self, filename): self.baseFilename = filename # pylint: disable=invalid-name @@ -138,13 +142,15 @@ def __init__(self, filename): # pylint: disable=too-few-public-methods class MockLogger: """Mock logging.Logger to provide the same API without keeping the - logger in memory.""" + logger in memory. + """ + def __init__(self, name, filename, level, log_dir): self.name = name self.level = level self.handlers = [MockHandler(os.path.join(log_dir, filename))] MOCK_LOGGERS[name] = self - for loglevel in ['debug', 'info', 'warning', 'error', 'critical']: + for loglevel in ["debug", "info", "warning", "error", "critical"]: # set up log methods for all built-in levels setattr(self, loglevel, self._factory_log(loglevel)) @@ -156,21 +162,29 @@ def exception(self, msg, *args, exc_info=True, **kwargs): def _factory_log(self, level): r"""Generate a log method like `self.log(message)` for a given - built-in level.""" + built-in level. + """ + @docstring_parameter(level=level) def _log(message, *items, exc_info=False): """Log a message if logging level >= {level}. See `Logging Levels `_ for a list of levels.""" - if self.level == 0 or self.level >= getattr(logging, level.upper(), - logging.NOTSET): - with open(self.handlers[0].baseFilename, 'a', - encoding='utf-8') as log_file: + if self.level == 0 or self.level >= getattr( + logging, level.upper(), logging.NOTSET + ): + with open( + self.handlers[0].baseFilename, "a", encoding="utf-8" + ) as log_file: if exc_info and isinstance(message, Exception): value, traceback = sys_exc_info()[1:] - print_exception(_lazy_sub(message, *items), - value=value, tb=traceback, - file=log_file) + print_exception( + _lazy_sub(message, *items), + value=value, + tb=traceback, + file=log_file, + ) else: print(_lazy_sub(message, *items), file=log_file) + return _log def delete(self): @@ -179,7 +193,7 @@ def delete(self): def _lazy_sub(message, *items): - """Given lazy-logging syntax, return string with substitutions + """Given lazy-logging syntax, return string with substitutions. Parameters ---------- @@ -206,9 +220,10 @@ def _lazy_sub(message, *items): return str([message, *items]) -def set_up_logger(name, filename=None, level=None, log_dir=None, mock=False, - overwrite_existing=False): - r"""Function to initialize a logger +def set_up_logger( + name, filename=None, level=None, log_dir=None, mock=False, overwrite_existing=False +): + r"""Function to initialize a logger. Parameters ---------- @@ -257,7 +272,7 @@ def set_up_logger(name, filename=None, level=None, log_dir=None, mock=False, False """ if filename is None: - filename = f'{name}.log' + filename = f"{name}.log" try: level = getattr(logging, level.upper()) except AttributeError: @@ -266,8 +281,8 @@ def set_up_logger(name, filename=None, level=None, log_dir=None, mock=False, log_dir = os.getcwd() filepath = os.path.join(log_dir, filename) if overwrite_existing and os.path.exists(filepath): - with open(filepath, 'w') as log_file: - log_file.write('') + with open(filepath, "w") as log_file: + log_file.write("") if mock: return MockLogger(name, filename, level, log_dir) logger = getLogger(name) diff --git a/CPAC/utils/monitoring/draw_gantt_chart.py b/CPAC/utils/monitoring/draw_gantt_chart.py index 6440aa30d0..6ebb3645cb 100644 --- a/CPAC/utils/monitoring/draw_gantt_chart.py +++ b/CPAC/utils/monitoring/draw_gantt_chart.py @@ -39,25 +39,23 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -'''Module to draw an html gantt chart from logfile produced by +"""Module to draw an html gantt chart from logfile produced by ``CPAC.utils.monitoring.log_nodes_cb()``. See https://nipype.readthedocs.io/en/latest/api/generated/nipype.utils.draw_gantt_chart.html -''' # noqa: E501 -import random - +""" from collections import OrderedDict from datetime import datetime +import random from warnings import warn -from nipype.utils.draw_gantt_chart import draw_lines, draw_resource_bar, \ - log_to_dict +from nipype.utils.draw_gantt_chart import draw_lines, draw_resource_bar, log_to_dict def create_event_dict(start_time, nodes_list): """ Function to generate a dictionary of event (start/finish) nodes - from the nodes list + from the nodes list. Parameters ---------- @@ -72,7 +70,6 @@ def create_event_dict(start_time, nodes_list): a dictionary where the key is the timedelta from the start of the pipeline execution to the value node it accompanies """ - # Import packages import copy @@ -100,8 +97,7 @@ def create_event_dict(start_time, nodes_list): # Populate dictionary if events.get(start_delta): - err_msg = "Event logged twice or events started at exact same " \ - "time!" + err_msg = "Event logged twice or events started at exact same " "time!" warn(str(KeyError(err_msg)), category=Warning) events[start_delta] = start_node events[finish_delta] = finish_node @@ -113,7 +109,7 @@ def create_event_dict(start_time, nodes_list): def calculate_resource_timeseries(events, resource): """ Given as event dictionary, calculate the resources used - as a timeseries + as a timeseries. Parameters ---------- @@ -159,17 +155,15 @@ def calculate_resource_timeseries(events, resource): time_series = pd.Series(data=list(res.values()), index=list(res.keys())) # Downsample where there is only value-diff ts_diff = time_series.diff() - time_series = time_series[ts_diff != 0] + return time_series[ts_diff != 0] # Return the new time series - return time_series -def draw_nodes(start, nodes_list, cores, minute_scale, space_between_minutes, - colors): +def draw_nodes(start, nodes_list, cores, minute_scale, space_between_minutes, colors): """ Function to return the html-string of the node drawings for the - gantt chart + gantt chart. Parameters ---------- @@ -204,8 +198,7 @@ def draw_nodes(start, nodes_list, cores, minute_scale, space_between_minutes, space_between_minutes = space_between_minutes / scale end_times = [ datetime( - start.year, start.month, start.day, start.hour, start.minute, - start.second + start.year, start.month, start.day, start.hour, start.minute, start.second ) for core in range(cores) ] @@ -220,8 +213,7 @@ def draw_nodes(start, nodes_list, cores, minute_scale, space_between_minutes, (node_start - start).total_seconds() / 60 ) * scale * space_between_minutes + 220 # Scale duration - scale_duration = (node["duration"] / 60) * scale \ - * space_between_minutes + scale_duration = (node["duration"] / 60) * scale * space_between_minutes if scale_duration < 5: scale_duration = 5 scale_duration -= 2 @@ -325,8 +317,7 @@ def generate_gantt_chart( # plugin_args={'n_procs':8, 'memory':12, 'status_callback': log_nodes_cb}) # generate_gantt_chart('callback.log', 8) - """ # noqa: E501 - + """ # add the html header html_string = """ @@ -383,7 +374,7 @@ def generate_gantt_chart(
- """ # noqa: E501 + """ close_header = """
@@ -392,7 +383,7 @@ def generate_gantt_chart(

Actual Resource

Failed Node

- """ # noqa: E501 + """ # Read in json-log to get list of node dicts nodes_list = log_to_dict(logfile) @@ -422,15 +413,12 @@ def generate_gantt_chart( # Summary strings of workflow at top html_string += ( - "

Start: " + start_node["start"].strftime("%Y-%m-%d %H:%M:%S") - + "

" + "

Start: " + start_node["start"].strftime("%Y-%m-%d %H:%M:%S") + "

" ) html_string += ( - "

Finish: " + last_node["finish"].strftime("%Y-%m-%d %H:%M:%S") - + "

" + "

Finish: " + last_node["finish"].strftime("%Y-%m-%d %H:%M:%S") + "

" ) - html_string += "

Duration: " + "{0:.2f}".format(duration / 60) \ - + " minutes

" + html_string += "

Duration: " + "{0:.2f}".format(duration / 60) + " minutes

" html_string += "

Nodes: " + str(len(nodes_list)) + "

" html_string += "

Cores: " + str(cores) + "

" html_string += close_header @@ -448,8 +436,7 @@ def generate_gantt_chart( ) # Get memory timeseries - estimated_mem_ts = calculate_resource_timeseries( - events, "estimated_memory_gb") + estimated_mem_ts = calculate_resource_timeseries(events, "estimated_memory_gb") runtime_mem_ts = calculate_resource_timeseries(events, "runtime_memory_gb") # Plot gantt chart resource_offset = 120 + 30 * cores @@ -475,10 +462,8 @@ def generate_gantt_chart( ) # Get threads timeseries - estimated_threads_ts = calculate_resource_timeseries( - events, "estimated_threads") - runtime_threads_ts = calculate_resource_timeseries( - events, "runtime_threads") + estimated_threads_ts = calculate_resource_timeseries(events, "estimated_threads") + runtime_threads_ts = calculate_resource_timeseries(events, "runtime_threads") # Plot gantt chart html_string += draw_resource_bar( start_node["start"], @@ -512,7 +497,7 @@ def generate_gantt_chart( def resource_overusage_report(cblog): - '''Function to parse through a callback log for memory and/or + """Function to parse through a callback log for memory and/or thread usage above estimates / limits. Parameters @@ -525,34 +510,47 @@ def resource_overusage_report(cblog): text_report: str excessive: dict - ''' + """ cb_dict_list = log_to_dict(cblog) - excessive = {node['id']: [ - node['runtime_memory_gb']if node.get('runtime_memory_gb', 0) - > node.get('estimated_memory_gb', 1) else None, - node['estimated_memory_gb'] if node.get('runtime_memory_gb', 0) - > node.get('estimated_memory_gb', 1) else None, - node['runtime_threads'] - 1 if node.get('runtime_threads', 0) - 1 - > node.get('num_threads', 1) else None, - node['num_threads'] if node.get('runtime_threads', 0) - 1 - > node.get('num_threads', 1) else None - ] for node in [node for node in cb_dict_list if ( - node.get('runtime_memory_gb', 0) > node.get('estimated_memory_gb', 1) - # or node.get('runtime_threads', 0) - 1 > node.get('num_threads', 1) - )]} - text_report = '' + excessive = { + node["id"]: [ + node["runtime_memory_gb"] + if node.get("runtime_memory_gb", 0) > node.get("estimated_memory_gb", 1) + else None, + node["estimated_memory_gb"] + if node.get("runtime_memory_gb", 0) > node.get("estimated_memory_gb", 1) + else None, + node["runtime_threads"] - 1 + if node.get("runtime_threads", 0) - 1 > node.get("num_threads", 1) + else None, + node["num_threads"] + if node.get("runtime_threads", 0) - 1 > node.get("num_threads", 1) + else None, + ] + for node in [ + node + for node in cb_dict_list + if ( + node.get("runtime_memory_gb", 0) > node.get("estimated_memory_gb", 1) + # or node.get('runtime_threads', 0) - 1 > node.get('num_threads', 1) + ) + ] + } + text_report = "" if excessive: - text_report += 'The following nodes used excessive resources:\n' - dotted_line = '-' * (len(text_report) - 1) + '\n' + text_report += "The following nodes used excessive resources:\n" + dotted_line = "-" * (len(text_report) - 1) + "\n" text_report += dotted_line for node in excessive: - node_id = '\n .'.join(node.split('.')) - text_report += f'\n{node_id}\n' + node_id = "\n .".join(node.split(".")) + text_report += f"\n{node_id}\n" if excessive[node][0]: - text_report += ' **memory_gb**\n' \ - ' runtime > estimated\n' \ - f' {excessive[node][0]} ' \ - f'> {excessive[node][1]}\n' + text_report += ( + " **memory_gb**\n" + " runtime > estimated\n" + f" {excessive[node][0]} " + f"> {excessive[node][1]}\n" + ) # JC: I'm not convinced 'runtime_threads' and 'threads' are # comparable in nipype ~1.5.1 # if excessive[node][2]: @@ -564,7 +562,7 @@ def resource_overusage_report(cblog): def resource_report(callback_log, num_cores, logger=None): - '''Function to attempt to warn any excessive resource usage and + """Function to attempt to warn any excessive resource usage and generate an interactive HTML chart. Parameters @@ -579,17 +577,19 @@ def resource_report(callback_log, num_cores, logger=None): Returns ------- None - ''' - e_msg = '' + """ + e_msg = "" try: txt_report = resource_overusage_report(callback_log)[0] if txt_report: - with open(callback_log + '.resource_overusage.txt', - 'w') as resource_overusage_file: + with open( + callback_log + ".resource_overusage.txt", "w" + ) as resource_overusage_file: resource_overusage_file.write(txt_report) except Exception as exception: # pylint: disable=broad-except - e_msg += f'Excessive usage report failed for {callback_log} ' \ - f'({str(exception)})\n' + e_msg += ( + f"Excessive usage report failed for {callback_log} " f"({exception!s})\n" + ) generate_gantt_chart(callback_log, num_cores) if e_msg: if logger is not None: @@ -599,7 +599,7 @@ def resource_report(callback_log, num_cores, logger=None): def _timing(nodes_list): - """Covert timestamps from strings to datetimes + """Covert timestamps from strings to datetimes. Parameters ---------- @@ -625,8 +625,11 @@ def _timing(nodes_list): True """ try: - return [_timing_timestamp(node) for node in nodes_list if - "start" in node and "finish" in node] + return [ + _timing_timestamp(node) + for node in nodes_list + if "start" in node and "finish" in node + ] except ValueError: # Drop any problematic nodes new_node_list = [] @@ -640,7 +643,7 @@ def _timing(nodes_list): def _timing_timestamp(node): - """Convert timestamps in a node from string to datetime + """Convert timestamps in a node from string to datetime. Parameters ---------- @@ -651,8 +654,14 @@ def _timing_timestamp(node): dict """ if node is None or node.items() is None: - raise ProcessLookupError('No logged nodes have timing information.') - return {k: (datetime.strptime(v, "%Y-%m-%dT%H:%M:%S.%f") if '.' in v else - datetime.fromisoformat(v)) if (k in {"start", "finish"} and - isinstance(v, str)) else - v for k, v in node.items()} + raise ProcessLookupError("No logged nodes have timing information.") + return { + k: ( + datetime.strptime(v, "%Y-%m-%dT%H:%M:%S.%f") + if "." in v + else datetime.fromisoformat(v) + ) + if (k in {"start", "finish"} and isinstance(v, str)) + else v + for k, v in node.items() + } diff --git a/CPAC/utils/monitoring/monitoring.py b/CPAC/utils/monitoring/monitoring.py index 00d8b34dfd..b6a554a6c0 100644 --- a/CPAC/utils/monitoring/monitoring.py +++ b/CPAC/utils/monitoring/monitoring.py @@ -1,11 +1,11 @@ import glob import json -import os import math -import networkx as nx +import os import socketserver import threading +import networkx as nx from traits.trait_base import Undefined from CPAC.pipeline import nipype_pipeline_engine as pe @@ -13,27 +13,27 @@ # Log initial information from all the nodes -def recurse_nodes(workflow, prefix=''): +def recurse_nodes(workflow, prefix=""): for node in nx.topological_sort(workflow._graph): if isinstance(node, pe.Workflow): - for subnode in recurse_nodes(node, prefix + workflow.name + '.'): + for subnode in recurse_nodes(node, prefix + workflow.name + "."): yield subnode else: yield { - "id": prefix + workflow.name + '.' + node.name, + "id": prefix + workflow.name + "." + node.name, "hash": node.inputs.get_hashval()[1], } def log_nodes_initial(workflow): - logger = getLogger('callback') + logger = getLogger("callback") for node in recurse_nodes(workflow): logger.debug(json.dumps(node)) def log_nodes_cb(node, status): """Function to record node run statistics to a log file as json - dictionaries + dictionaries. Parameters ---------- @@ -49,13 +49,12 @@ def log_nodes_cb(node, status): this function does not return any values, it logs the node status info to the callback logger """ - - if status != 'end': + if status != "end": return - import nipype.pipeline.engine.nodes as nodes + from nipype.pipeline.engine import nodes - logger = getLogger('callback') + logger = getLogger("callback") if isinstance(node, nodes.MapNode): return @@ -64,49 +63,42 @@ def log_nodes_cb(node, status): runtime = node.result.runtime except FileNotFoundError: runtime = {} - runtime_threads = getattr(runtime, 'cpu_percent', 'N/A') - if runtime_threads != 'N/A': - runtime_threads = math.ceil(runtime_threads/100) + runtime_threads = getattr(runtime, "cpu_percent", "N/A") + if runtime_threads != "N/A": + runtime_threads = math.ceil(runtime_threads / 100) status_dict = { - 'id': str(node), - 'hash': node.inputs.get_hashval()[1], - 'start': getattr(runtime, 'startTime', None), - 'finish': getattr(runtime, 'endTime', None), - 'runtime_threads': runtime_threads, - 'runtime_memory_gb': getattr(runtime, 'mem_peak_gb', 'N/A'), - 'estimated_memory_gb': node.mem_gb, - 'num_threads': node.n_procs, + "id": str(node), + "hash": node.inputs.get_hashval()[1], + "start": getattr(runtime, "startTime", None), + "finish": getattr(runtime, "endTime", None), + "runtime_threads": runtime_threads, + "runtime_memory_gb": getattr(runtime, "mem_peak_gb", "N/A"), + "estimated_memory_gb": node.mem_gb, + "num_threads": node.n_procs, } - if ( - hasattr(node, 'input_data_shape') and - node.input_data_shape is not Undefined - ): - status_dict['input_data_shape'] = node.input_data_shape + if hasattr(node, "input_data_shape") and node.input_data_shape is not Undefined: + status_dict["input_data_shape"] = node.input_data_shape - if status_dict['start'] is None or status_dict['finish'] is None: - status_dict['error'] = True + if status_dict["start"] is None or status_dict["finish"] is None: + status_dict["error"] = True logger.debug(json.dumps(status_dict)) class LoggingRequestHandler(socketserver.BaseRequestHandler): - def handle(self): - tree = {} logs = glob.glob( os.path.join( - self.server.logging_dir, - "pipeline_" + self.server.pipeline_name, - "*" + self.server.logging_dir, "pipeline_" + self.server.pipeline_name, "*" ) ) for log in logs: - subject = log.split('/')[-1] + subject = log.split("/")[-1] tree[subject] = {} callback_file = os.path.join(log, "callback.log") @@ -114,26 +106,20 @@ def handle(self): if not os.path.exists(callback_file): continue - with open(callback_file, 'rb') as lf: + with open(callback_file, "rb") as lf: for l in lf.readlines(): # noqa: E741 l = l.strip() # noqa: E741 try: node = json.loads(l) if node["id"] not in tree[subject]: - tree[subject][node["id"]] = { - "hash": node["hash"] - } + tree[subject][node["id"]] = {"hash": node["hash"]} if "start" in node and "finish" in node: - tree[subject][node["id"]]["start"] = node[ - "start"] - tree[subject][node["id"]]["finish"] = node[ - "finish"] + tree[subject][node["id"]]["start"] = node["start"] + tree[subject][node["id"]]["finish"] = node["finish"] else: if "start" in node and "finish" in node: - if tree[subject][node["id"]]["hash"] == node[ - "hash" - ]: + if tree[subject][node["id"]]["hash"] == node["hash"]: tree[subject][node["id"]]["cached"] = { "start": node["start"], "finish": node["finish"], @@ -141,24 +127,27 @@ def handle(self): # pipeline was changed, and we have a new hash else: - tree[subject][node["id"]]["start"] = node[ - "start"] - tree[subject][node["id"]]["finish"] = node[ - "finish"] + tree[subject][node["id"]]["start"] = node["start"] + tree[subject][node["id"]]["finish"] = node["finish"] except: break tree = {s: t for s, t in tree.items() if t} - headers = 'HTTP/1.1 200 OK\nConnection: close\n\n' + headers = "HTTP/1.1 200 OK\nConnection: close\n\n" self.request.sendall(headers + json.dumps(tree) + "\n") class LoggingHTTPServer(socketserver.ThreadingTCPServer, object): - - def __init__(self, pipeline_name, logging_dir='', host='', port=8080, - request=LoggingRequestHandler): + def __init__( + self, + pipeline_name, + logging_dir="", + host="", + port=8080, + request=LoggingRequestHandler, + ): super(LoggingHTTPServer, self).__init__((host, port), request) if not logging_dir: @@ -168,9 +157,10 @@ def __init__(self, pipeline_name, logging_dir='', host='', port=8080, self.pipeline_name = pipeline_name -def monitor_server(pipeline_name, logging_dir, host='0.0.0.0', port=8080): - httpd = LoggingHTTPServer(pipeline_name, logging_dir, host, port, - LoggingRequestHandler) +def monitor_server(pipeline_name, logging_dir, host="0.0.0.0", port=8080): + httpd = LoggingHTTPServer( + pipeline_name, logging_dir, host, port, LoggingRequestHandler + ) server_thread = threading.Thread(target=httpd.serve_forever) server_thread.isDaemon = True diff --git a/CPAC/utils/ndmg_utils.py b/CPAC/utils/ndmg_utils.py index b95d1ffa18..11ad7f002f 100644 --- a/CPAC/utils/ndmg_utils.py +++ b/CPAC/utils/ndmg_utils.py @@ -35,8 +35,9 @@ This file is part of C-PAC. """ import os -import nibabel as nb + import numpy as np +import nibabel as nib def ndmg_roi_timeseries(func_file, label_file): @@ -57,10 +58,10 @@ def ndmg_roi_timeseries(func_file, label_file): # Adapted from ndmg v0.1.1 # Copyright 2016 NeuroData (http://neurodata.io) """ - labeldata = nb.load(label_file).get_fdata() + labeldata = nib.load(label_file).get_fdata() # rois are all the nonzero unique values the parcellation can take rois = np.sort(np.unique(labeldata[labeldata > 0])) - funcdata = nb.load(func_file).get_fdata() + funcdata = nib.load(func_file).get_fdata() # initialize time series to [numrois]x[numtimepoints] roi_ts = np.zeros((len(rois), funcdata.shape[3])) @@ -71,9 +72,11 @@ def ndmg_roi_timeseries(func_file, label_file): try: roi_vts = funcdata[roibool, :] except IndexError as e: - err = '\n[!] Error: functional data and ROI mask may not be in ' \ - 'the same space or be the same size.\nDetails: ' \ - '{0}'.format(e) + err = ( + "\n[!] Error: functional data and ROI mask may not be in " + "the same space or be the same size.\nDetails: " + "{0}".format(e) + ) raise IndexError(err) # take the mean for the voxel timeseries, and ignore voxels with # no variance @@ -81,7 +84,7 @@ def ndmg_roi_timeseries(func_file, label_file): if ts.size != 0: roi_ts[idx, :] = ts - roits_file = os.path.join(os.getcwd(), 'timeseries.npz') + roits_file = os.path.join(os.getcwd(), "timeseries.npz") np.savez(roits_file, ts=roi_ts, rois=rois) return (roi_ts, rois, roits_file) @@ -104,13 +107,15 @@ def __init__(self, N, rois, attr=None, sens="dwi"): # Adapted from ndmg v0.1.1 # Copyright 2016 NeuroData (http://neurodata.io) """ - import numpy as np - import nibabel as nb from collections import defaultdict + + import numpy as np + import nibabel as nib + self.N = N self.edge_dict = defaultdict(int) - self.rois = nb.load(rois).get_fdata() + self.rois = nib.load(rois).get_fdata() n_ids = np.unique(self.rois) self.n_ids = n_ids[n_ids > 0] @@ -125,29 +130,29 @@ def make_graph(self, streamlines, attr=None): - Fiber streamlines either file or array in a dipy EuDX or compatible format. """ + from itertools import product import time - import numpy as np + import networkx as nx - from itertools import product + import numpy as np - self.g = nx.Graph(name="Generated by NeuroData's MRI Graphs (ndmg)", - version='0.1.1', - date=time.asctime(time.localtime()), - source="http://m2g.io", - region="brain", - sensor=self.modal, - ecount=0, - vcount=len(self.n_ids) - ) - print(self.g.graph) + self.g = nx.Graph( + name="Generated by NeuroData's MRI Graphs (ndmg)", + version="0.1.1", + date=time.asctime(time.localtime()), + source="http://m2g.io", + region="brain", + sensor=self.modal, + ecount=0, + vcount=len(self.n_ids), + ) [str(self.g.add_node(ids)) for ids in self.n_ids] nlines = np.shape(streamlines)[0] - print("# of Streamlines: " + str(nlines)) - print_id = np.max((int(nlines*0.05), 1)) # in case nlines*.05=0 + print_id = np.max((int(nlines * 0.05), 1)) # in case nlines*.05=0 for idx, streamline in enumerate(streamlines): if (idx % print_id) == 0: - print(idx) + pass points = np.round(streamline).astype(int) p = set() @@ -161,7 +166,7 @@ def make_graph(self, streamlines, attr=None): if loc: p.add(loc) - edges = set([tuple(sorted(x)) for x in product(p, p)]) + edges = {tuple(sorted(x)) for x in product(p, p)} for edge in edges: lst = tuple(sorted([str(node) for node in edge])) self.edge_dict[lst] += 1 @@ -174,12 +179,12 @@ def cor_graph(self, timeseries, attr=None): **Positional Arguments:** timeseries: -the timeseries file to extract correlation for. - dimensions are [numrois]x[numtimesteps] + dimensions are [numrois]x[numtimesteps]. """ import numpy as np - ts = timeseries[0] + + timeseries[0] rois = timeseries[1] - print("Estimating correlation matrix for {} ROIs...".format(self.N)) self.g = np.abs(np.corrcoef(timeseries)) # calculate pearson correlation self.g = np.nan_to_num(self.g).astype(object) self.n_ids = rois @@ -194,20 +199,16 @@ def cor_graph(self, timeseries, attr=None): return self.g def get_graph(self): - """ - Returns the graph object created - """ + """Returns the graph object created.""" try: return self.g except AttributeError: - print("Error: the graph has not yet been defined.") pass def as_matrix(self): - """ - Returns the graph as a matrix. - """ + """Returns the graph as a matrix.""" import networkx as nx + g = self.get_graph() return nx.to_numpy_matrix(g, nodelist=np.sort(g.nodes()).tolist()) @@ -216,34 +217,35 @@ def save_graph(self, graphname): Saves the graph to disk **Positional Arguments:** graphname: - - Filename for the graph + - Filename for the graph. """ - import numpy as np import networkx as nx - if self.modal == 'dwi': - self.g.graph['ecount'] = nx.number_of_edges(self.g) + import numpy as np + + if self.modal == "dwi": + self.g.graph["ecount"] = nx.number_of_edges(self.g) nx.write_weighted_edgelist(self.g, graphname, delimiter=",") - elif self.modal == 'func': - np.savetxt(graphname, self.g, comments='', delimiter=',', - header=','.join([str(n) for n in self.n_ids])) + elif self.modal == "func": + np.savetxt( + graphname, + self.g, + comments="", + delimiter=",", + header=",".join([str(n) for n in self.n_ids]), + ) else: raise ValueError("Unsupported Modality.") pass def summary(self): - """ - User friendly wrapping and display of graph properties - """ - import networkx as nx - print("\n Graph Summary:") - print(nx.info(self.g)) + """User friendly wrapping and display of graph properties.""" pass def ndmg_create_graphs(ts, labels): - out_file = os.path.join(os.getcwd(), 'measure-correlation.csv') + out_file = os.path.join(os.getcwd(), "measure-correlation.csv") connectome = graph(ts.shape[0], labels, sens="func") - conn = connectome.cor_graph(ts) + connectome.cor_graph(ts) connectome.save_graph(out_file) return out_file diff --git a/CPAC/utils/nifti_utils.py b/CPAC/utils/nifti_utils.py index 2b55b01ecc..cab42b51cb 100644 --- a/CPAC/utils/nifti_utils.py +++ b/CPAC/utils/nifti_utils.py @@ -1,4 +1,5 @@ import os + import numpy as np import six import nibabel as nib @@ -7,8 +8,9 @@ def nifti_image_input(image): """ Test if an input is a path or a nifti.image and the image loaded through - nibabel - Parameters + nibabel. + + Parameters. ---------- image : str or nibabel.nifti1.Nifti1Image path to the nifti file or the image already loaded through nibabel @@ -30,10 +32,12 @@ def nifti_image_input(image): raise TypeError("Image can be either a string or a nifti1.Nifti1Image") return img + def more_zeros_than_ones(image): """ Return True is there is more zeros than other values in a given nifti image. - Parameters + + Parameters. ---------- image : str or nibabel.nifti1.Nifti1Image path to the nifti file to be inverted or @@ -56,14 +60,14 @@ def more_zeros_than_ones(image): data = img.get_fdata() nb_zeros = len(np.where(data == 0)[0]) size = data.size - more_zeros = nb_zeros > size - nb_zeros - return more_zeros + return nb_zeros > size - nb_zeros def inverse_nifti_values(image): """ - Replace zeros by ones and non-zero values by 1 - Parameters + Replace zeros by ones and non-zero values by 1. + + Parameters. ---------- image : str or nibabel.nifti1.Nifti1Image path to the nifti file to be inverted or diff --git a/CPAC/utils/outputs.py b/CPAC/utils/outputs.py index 5aaf022a61..d814564ef1 100644 --- a/CPAC/utils/outputs.py +++ b/CPAC/utils/outputs.py @@ -1,17 +1,18 @@ -import pkg_resources as p import pandas as pd +import pkg_resources as p -class Outputs(): - +class Outputs: # Settle some things about the resource pool reference and the output directory - reference_csv = p.resource_filename('CPAC', 'resources/cpac_outputs.tsv') + reference_csv = p.resource_filename("CPAC", "resources/cpac_outputs.tsv") try: - reference = pd.read_csv(reference_csv, delimiter='\t', keep_default_na=False) + reference = pd.read_csv(reference_csv, delimiter="\t", keep_default_na=False) except Exception as e: - err = "\n[!] Could not access or read the cpac_outputs.tsv " \ - "resource file:\n{0}\n\nError details {1}\n".format(reference_csv, e) + err = ( + "\n[!] Could not access or read the cpac_outputs.tsv " + "resource file:\n{0}\n\nError details {1}\n".format(reference_csv, e) + ) raise Exception(err) # all outputs @@ -19,58 +20,64 @@ class Outputs(): # extra outputs that we don't write to the output directory, unless the # user selects to do so - debugging = list( - reference[reference['Optional: Debugging'] == 'Yes']['Resource'] - ) + debugging = list(reference[reference["Optional: Debugging"] == "Yes"]["Resource"]) # functional data that are 4D time series, instead of derivatives functional_timeseries = list( - reference[reference['4D Time Series'] == 'Yes']['Resource'] + reference[reference["4D Time Series"] == "Yes"]["Resource"] ) - anat = list(reference[reference['Sub-Directory'] == 'anat']['Resource']) - func = list(reference[reference['Sub-Directory'] == 'func']['Resource']) + anat = list(reference[reference["Sub-Directory"] == "anat"]["Resource"]) + func = list(reference[reference["Sub-Directory"] == "func"]["Resource"]) # outputs to send into smoothing, if smoothing is enabled, and # outputs to write out if the user selects to write non-smoothed outputs - _template_filter = reference['Space'] == 'template' - _epitemplate_filter = reference['Space'] == 'EPI template' - _symtemplate_filter = reference['Space'] == 'symmetric template' - _T1w_native_filter = reference['Space'] == 'T1w' - _bold_native_filter = reference['Space'] == 'functional' - _long_native_filter = reference['Space'] == 'longitudinal T1w' - _nonsmoothed_filter = reference['To Smooth'] == 'Yes' - _zstd_filter = reference['To z-std'] == 'Yes' - _corr_filter = reference['Type'] == 'correlation' - - all_template_filter = (_template_filter | _epitemplate_filter | - _symtemplate_filter) - all_native_filter = (_T1w_native_filter | _bold_native_filter | - _long_native_filter) - - native_nonsmooth = list(reference[all_native_filter & - _nonsmoothed_filter]['Resource']) - template_nonsmooth = list(reference[all_template_filter & - _nonsmoothed_filter]['Resource']) - - to_smooth = list(reference[_nonsmoothed_filter]['Resource']) - to_zstd = list(reference[_zstd_filter & ~_corr_filter]['Resource']) - to_fisherz = list(reference[_zstd_filter & _corr_filter]['Resource']) + _template_filter = reference["Space"] == "template" + _epitemplate_filter = reference["Space"] == "EPI template" + _symtemplate_filter = reference["Space"] == "symmetric template" + _T1w_native_filter = reference["Space"] == "T1w" + _bold_native_filter = reference["Space"] == "functional" + _long_native_filter = reference["Space"] == "longitudinal T1w" + _nonsmoothed_filter = reference["To Smooth"] == "Yes" + _zstd_filter = reference["To z-std"] == "Yes" + _corr_filter = reference["Type"] == "correlation" + + all_template_filter = _template_filter | _epitemplate_filter | _symtemplate_filter + all_native_filter = _T1w_native_filter | _bold_native_filter | _long_native_filter + + native_nonsmooth = list( + reference[all_native_filter & _nonsmoothed_filter]["Resource"] + ) + template_nonsmooth = list( + reference[all_template_filter & _nonsmoothed_filter]["Resource"] + ) + + to_smooth = list(reference[_nonsmoothed_filter]["Resource"]) + to_zstd = list(reference[_zstd_filter & ~_corr_filter]["Resource"]) + to_fisherz = list(reference[_zstd_filter & _corr_filter]["Resource"]) # don't write these, unless the user selects to write native-space outputs - native_smooth = list(reference[~all_template_filter & ~_nonsmoothed_filter]['Resource']) + native_smooth = list( + reference[~all_template_filter & ~_nonsmoothed_filter]["Resource"] + ) # ever used??? contains template-space, smoothed, both raw and z-scored - template_smooth = list(reference[all_template_filter & ~_nonsmoothed_filter]['Resource']) + template_smooth = list( + reference[all_template_filter & ~_nonsmoothed_filter]["Resource"] + ) - _bold_filter = reference['Type'] == 'bold' - _ts_filter = reference['4D Time Series'] == 'Yes' - bold_ts = list(reference[_bold_filter & _bold_native_filter & _ts_filter]['Resource']) + _bold_filter = reference["Type"] == "bold" + _ts_filter = reference["4D Time Series"] == "Yes" + bold_ts = list( + reference[_bold_filter & _bold_native_filter & _ts_filter]["Resource"] + ) # outputs to send into z-scoring, if z-scoring is enabled, and # outputs to write out if user selects to write non-z-scored outputs - native_raw = list(reference[all_native_filter & - (reference['To z-std'] == 'Yes')]['Resource']) + native_raw = list( + reference[all_native_filter & (reference["To z-std"] == "Yes")]["Resource"] + ) - template_raw = list(reference[all_template_filter & - (reference['To z-std'] == 'Yes')]['Resource']) + template_raw = list( + reference[all_template_filter & (reference["To z-std"] == "Yes")]["Resource"] + ) diff --git a/CPAC/utils/pytest.py b/CPAC/utils/pytest.py index 4ea9a46b0d..3579a53535 100644 --- a/CPAC/utils/pytest.py +++ b/CPAC/utils/pytest.py @@ -14,19 +14,22 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -"""Utilities for Pytest integration""" +"""Utilities for Pytest integration.""" try: import pytest + HAVE_PYTEST = True except ImportError: HAVE_PYTEST = False def skipif(condition, reason): - """Skip test if we have Pytest, ignore test entirely if not""" + """Skip test if we have Pytest, ignore test entirely if not.""" + def decorator(func): - """Skip test if we have Pytest""" + """Skip test if we have Pytest.""" if HAVE_PYTEST: return pytest.mark.skipif(condition, reason)(func) return func # return undecorated function + return decorator # return conditionally decorated function diff --git a/CPAC/utils/strategy.py b/CPAC/utils/strategy.py index b176b06073..4423a5cd4c 100644 --- a/CPAC/utils/strategy.py +++ b/CPAC/utils/strategy.py @@ -15,13 +15,15 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . import logging + import six + from CPAC.pipeline.engine import ResourcePool -logger = logging.getLogger('nipype.workflow') +logger = logging.getLogger("nipype.workflow") -class Strategy: +class Strategy: def __init__(self): self._resource_pool = ResourcePool({}) self.leaf_node = None @@ -45,33 +47,33 @@ def get_resource_pool(self): return self.resource_pool def get_nodes_names(self): - pieces = [n.split('_') for n in self.name] + pieces = [n.split("_") for n in self.name] assert all(p[-1].isdigit() for p in pieces) - return ['_'.join(p[:-1]) for p in pieces] + return ["_".join(p[:-1]) for p in pieces] def get_node_from_resource_pool(self, resource_key): try: return self.resource_pool[resource_key] except: - logger.error('No node for output: %s', resource_key) + logger.error("No node for output: %s", resource_key) raise @property def resource_pool(self): - '''Strategy's ResourcePool dict''' + """Strategy's ResourcePool dict.""" return self._resource_pool.get_entire_rpool() @property def rpool(self): - '''Strategy's ResourcePool''' + """Strategy's ResourcePool.""" return self._resource_pool def update_resource_pool(self, resources, override=False): for key, value in resources.items(): if key in self.resource_pool and not override: raise Exception( - 'Key %s already exists in resource pool, ' - 'replacing with %s ' % (key, value) + "Key %s already exists in resource pool, " + "replacing with %s " % (key, value) ) self.resource_pool[key] = value @@ -83,7 +85,7 @@ def __getitem__(self, resource_key): try: return self.resource_pool[resource_key] except: - logger.error('No node for output: %s', resource_key) + logger.error("No node for output: %s", resource_key) raise def __contains__(self, resource_key): @@ -101,11 +103,9 @@ def fork(self): @staticmethod def get_forking_points(strategies): - forking_points = [] for strat in strategies: - strat_node_names = set(strat.get_nodes_names()) strat_forking = [] @@ -121,7 +121,6 @@ def get_forking_points(strategies): @staticmethod def get_forking_labels(strategies): - fork_names = [] # fork_points is a list of lists, each list containing node names of @@ -129,81 +128,79 @@ def get_forking_labels(strategies): fork_points = Strategy.get_forking_points(strategies) for fork_point in fork_points: - fork_point.sort() fork_name = [] for fork in fork_point: - - fork_label = '' + fork_label = "" # TODO: reorganize labels - # registration - if 'anat_mni_ants_register' in fork: - fork_label = 'anat-ants' - if 'anat_mni_fnirt_register' in fork: - fork_label = 'anat-fnirt' - if 'anat_mni_flirt_register' in fork: - fork_label = 'anat-flirt' - if 'func_to_epi_ants' in fork: - fork_label = 'func-ants' - if 'func_to_epi_fsl' in fork: - fork_label = 'func-fsl' - # skullstripping - if 'func_preproc_afni' in fork: - fork_label = 'func-3dautomask' - if 'func_preproc_fsl' in fork: - fork_label = 'func-bet' - if 'func_preproc_fsl_afni' in fork: - fork_label = 'func-bet-3dautomask' + # registration + if "anat_mni_ants_register" in fork: + fork_label = "anat-ants" + if "anat_mni_fnirt_register" in fork: + fork_label = "anat-fnirt" + if "anat_mni_flirt_register" in fork: + fork_label = "anat-flirt" + if "func_to_epi_ants" in fork: + fork_label = "func-ants" + if "func_to_epi_fsl" in fork: + fork_label = "func-fsl" + # skullstripping + if "func_preproc_afni" in fork: + fork_label = "func-3dautomask" + if "func_preproc_fsl" in fork: + fork_label = "func-bet" + if "func_preproc_fsl_afni" in fork: + fork_label = "func-bet-3dautomask" # motion correction reference - if 'mean' in fork: - fork_label = 'mean' - if 'median' in fork: - fork_label = 'median' - if 'selected_volume' in fork: - fork_label = 'selected_volume' + if "mean" in fork: + fork_label = "mean" + if "median" in fork: + fork_label = "median" + if "selected_volume" in fork: + fork_label = "selected_volume" # motion correction - if 'mcflirt' in fork: - fork_label += '_func-mcflirt' - if '3dvolreg' in fork: - fork_label += '_func-3dvolreg' - if 'anat_refined' in fork: - fork_label = 'func-anat-refined' - if 'motion_filter' in fork: - fork_label = 'motion-filter' + if "mcflirt" in fork: + fork_label += "_func-mcflirt" + if "3dvolreg" in fork: + fork_label += "_func-3dvolreg" + if "anat_refined" in fork: + fork_label = "func-anat-refined" + if "motion_filter" in fork: + fork_label = "motion-filter" # distortion correction - if 'epi_distcorr' in fork: - fork_label = 'dist-corr' - if 'bbreg' in fork: - fork_label = 'bbreg' - - if 'aroma' in fork: - fork_label = 'aroma' - if 'nuisance' in fork: - fork_label = 'nuisance' - if 'frequency_filter' in fork: - fork_label = 'freq-filter' - - if 'gen_motion_stats_before_stc' in fork: - fork_label = 'motion_stats_before_stc' - if 'despike' in fork: - fork_label = 'despike' - if 'slice' in fork: - fork_label = 'slice' - if 'anat_preproc_afni' in fork: - fork_label = 'anat-afni' - if 'anat_preproc_bet' in fork: - fork_label = 'anat-bet' - if 'anat_preproc_ants' in fork: - fork_label = 'anat-ants' - if 'anat_preproc_unet' in fork: - fork_label = 'anat-unet' + if "epi_distcorr" in fork: + fork_label = "dist-corr" + if "bbreg" in fork: + fork_label = "bbreg" + + if "aroma" in fork: + fork_label = "aroma" + if "nuisance" in fork: + fork_label = "nuisance" + if "frequency_filter" in fork: + fork_label = "freq-filter" + + if "gen_motion_stats_before_stc" in fork: + fork_label = "motion_stats_before_stc" + if "despike" in fork: + fork_label = "despike" + if "slice" in fork: + fork_label = "slice" + if "anat_preproc_afni" in fork: + fork_label = "anat-afni" + if "anat_preproc_bet" in fork: + fork_label = "anat-bet" + if "anat_preproc_ants" in fork: + fork_label = "anat-ants" + if "anat_preproc_unet" in fork: + fork_label = "anat-unet" fork_name += [fork_label] - fork_names.append('_'.join(sorted(set(fork_name)))) + fork_names.append("_".join(sorted(set(fork_name)))) return dict(zip(strategies, fork_names)) diff --git a/CPAC/utils/symlinks.py b/CPAC/utils/symlinks.py index f464f02dc1..c9283394de 100644 --- a/CPAC/utils/symlinks.py +++ b/CPAC/utils/symlinks.py @@ -1,171 +1,173 @@ -import os -import errno from collections import defaultdict +import errno +import os output_renamings = { - 'anatomical_brain': 'anat', - 'anatomical_brain_mask': 'anat', - 'qc': 'qc', - 'anatomical_skull_leaf': 'anat', - 'anatomical_to_mni_linear_xfm': 'anat', - 'mni_to_anatomical_linear_xfm': 'anat', - 'mni_to_anatomical_nonlinear_xfm': 'anat', - 'anatomical_to_mni_nonlinear_xfm': 'anat', - 'anatomical_gm_mask': 'anat', - 'anatomical_csf_mask': 'anat', - 'anatomical_wm_mask': 'anat', - 'ants_initial_xfm': 'anat', - 'ants_rigid_xfm': 'anat', - 'ants_affine_xfm': 'anat', - 'mean_functional': 'func', - 'functional_preprocessed_mask': 'func', - 'functional_to_spatial_map': 'func', - 'functional_mask_to_spatial_map': 'func', - 'fmap_phase_diff': 'func', - 'fmap_magnitude': 'func', - 'functional_distortion_corrected': 'func', - 'despiked_fieldmap': 'func', - 'prepared_fieldmap_map': 'func', - 'fieldmap_mask': 'func', - 'slice_time_corrected': 'func', - 'slice_timing_corrected': 'func', - 'movement_parameters': 'parameters', - 'max_displacement': 'parameters', - 'xform_matrix': 'parameters', - 'output_means': 'parameters', - 'functional_preprocessed': 'func', - 'functional_brain_mask': 'func', - 'motion_correct': 'func', - 'motion_correct_smooth': 'func', - 'motion_correct_to_standard': 'func', - 'motion_correct_to_standard_smooth': 'func', - 'mean_functional_in_anat': 'func', - 'coordinate_transformation': 'func', - 'raw_functional': 'func', - 'selected_func_volume': 'func', - 'anatomical_wm_edge': 'registration', - 'anatomical_to_functional_xfm': 'registration', - 'inverse_anatomical_to_functional_xfm': 'registration', - 'functional_gm_mask': 'segmentation', - 'functional_wm_mask': 'segmentation', - 'functional_csf_mask': 'segmentation', - 'frame_wise_displacement_power': 'parameters', - 'frame_wise_displacement_jenkinson': 'parameters', - 'functional_nuisance_residuals': 'func', - 'functional_nuisance_regressors': 'func', - 'power_spectrum_distribution': 'alff', - 'functional_freq_filtered': 'func', - 'scrubbing_movement_parameters': 'parameters', - 'despiking_frames_included': 'parameters', - 'despiking_frames_excluded': 'parameters', - 'scrubbing_frames_included': 'parameters', - 'scrubbing_frames_excluded': 'parameters', - 'motion_params': 'parameters', - 'power_params': 'parameters', - 'scrubbed_preprocessed': 'func', - 'functional_to_standard': 'func', - 'functional_brain_mask_to_standard': 'func', - 'mean_functional_to_standard': 'func', - 'functional_to_anat_linear_xfm': 'registration', - 'functional_to_mni_linear_xfm': 'registration', - 'mni_to_functional_linear_xfm': 'registration', - 'ants_symmetric_initial_xfm': 'registration', - 'ants_symmetric_rigid_xfm': 'registration', - 'ants_symmetric_affine_xfm': 'registration', - 'anatomical_to_symmetric_mni_nonlinear_xfm': 'registration', - 'symmetric_mni_to_anatomical_nonlinear_xfm': 'registration', - 'symmetric_mni_to_anatomical_linear_xfm': 'registration', - 'anat_to_symmetric_mni_ants_composite_xfm': 'registration', - 'symmetric_anatomical_to_standard': 'registration', - 'anatomical_to_symmetric_mni_linear_xfm': 'registration', - 'anatomical_to_standard': 'anat', - 'leaf_node_to_standard': 'func', - 'vmhc_raw_score': 'vmhc', - 'vmhc_fisher_zstd': 'vmhc', - 'vmhc_fisher_zstd_zstat_map': 'vmhc', - 'alff': 'alff', - 'falff': 'alff', - 'alff_smooth': 'alff', - 'falff_smooth': 'alff', - 'alff_to_standard': 'alff', - 'falff_to_standard': 'alff', - 'alff_to_standard_smooth': 'alff', - 'falff_to_standard_smooth': 'alff', - 'alff_to_standard_zstd': 'alff', - 'falff_to_standard_zstd': 'alff', - 'alff_to_standard_smooth_zstd': 'alff', - 'falff_to_standard_smooth_zstd': 'alff', - 'alff_to_standard_zstd_smooth': 'alff', - 'falff_to_standard_zstd_smooth': 'alff', - 'reho': 'reho', - 'reho_smooth': 'reho', - 'reho_to_standard': 'reho', - 'reho_to_standard_smooth': 'reho', - 'reho_to_standard_zstd': 'reho', - 'reho_to_standard_smooth_zstd': 'reho', - 'reho_to_standard_zstd_smooth': 'reho', - 'voxel_timeseries': 'timeseries', - 'roi_timeseries': 'timeseries', - 'roi_timeseries_for_SCA': 'timeseries', - 'roi_timeseries_for_SCA_multreg': 'timeseries', - 'sca_roi_files': 'sca_roi', - 'sca_roi_files_smooth': 'sca_roi', - 'sca_roi_files_to_standard': 'sca_roi', - 'sca_roi_files_to_standard_smooth': 'sca_roi', - 'sca_roi_files_to_standard_fisher_zstd': 'sca_roi', - 'sca_roi_files_to_standard_smooth_fisher_zstd': 'sca_roi', - 'sca_roi_files_to_standard_fisher_zstd_smooth': 'sca_roi', - 'bbregister_registration': 'surface_registration', - 'left_hemisphere_surface': 'surface_registration', - 'right_hemisphere_surface': 'surface_registration', - 'vertices_timeseries': 'timeseries', - 'centrality': 'centrality', - 'centrality_smooth': 'centrality', - 'centrality_zstd': 'centrality', - 'centrality_smooth_zstd': 'centrality', - 'centrality_zstd_smooth': 'centrality', - 'centrality_graphs': 'centrality', - 'seg_probability_maps': 'anat', - 'seg_mixeltype': 'anat', - 'seg_partial_volume_map': 'anat', - 'seg_partial_volume_files': 'anat', - 'spatial_map_timeseries': 'timeseries', - 'spatial_map_timeseries_for_DR': 'timeseries', - 'dr_tempreg_maps_files': 'spatial_regression', - 'dr_tempreg_maps_files_smooth': 'spatial_regression', - 'dr_tempreg_maps_zstat_files': 'spatial_regression', - 'dr_tempreg_maps_zstat_files_smooth': 'spatial_regression', - 'dr_tempreg_maps_files_to_standard': 'spatial_regression', - 'dr_tempreg_maps_zstat_files_to_standard': 'spatial_regression', - 'dr_tempreg_maps_files_to_standard_smooth': 'spatial_regression', - 'dr_tempreg_maps_zstat_files_to_standard_smooth': 'spatial_regression', - 'sca_tempreg_maps_files': 'sca_roi', - 'sca_tempreg_maps_files_smooth': 'sca_roi', - 'sca_tempreg_maps_zstat_files': 'sca_roi', - 'sca_tempreg_maps_zstat_files_smooth': 'sca_roi', + "anatomical_brain": "anat", + "anatomical_brain_mask": "anat", + "qc": "qc", + "anatomical_skull_leaf": "anat", + "anatomical_to_mni_linear_xfm": "anat", + "mni_to_anatomical_linear_xfm": "anat", + "mni_to_anatomical_nonlinear_xfm": "anat", + "anatomical_to_mni_nonlinear_xfm": "anat", + "anatomical_gm_mask": "anat", + "anatomical_csf_mask": "anat", + "anatomical_wm_mask": "anat", + "ants_initial_xfm": "anat", + "ants_rigid_xfm": "anat", + "ants_affine_xfm": "anat", + "mean_functional": "func", + "functional_preprocessed_mask": "func", + "functional_to_spatial_map": "func", + "functional_mask_to_spatial_map": "func", + "fmap_phase_diff": "func", + "fmap_magnitude": "func", + "functional_distortion_corrected": "func", + "despiked_fieldmap": "func", + "prepared_fieldmap_map": "func", + "fieldmap_mask": "func", + "slice_time_corrected": "func", + "slice_timing_corrected": "func", + "movement_parameters": "parameters", + "max_displacement": "parameters", + "xform_matrix": "parameters", + "output_means": "parameters", + "functional_preprocessed": "func", + "functional_brain_mask": "func", + "motion_correct": "func", + "motion_correct_smooth": "func", + "motion_correct_to_standard": "func", + "motion_correct_to_standard_smooth": "func", + "mean_functional_in_anat": "func", + "coordinate_transformation": "func", + "raw_functional": "func", + "selected_func_volume": "func", + "anatomical_wm_edge": "registration", + "anatomical_to_functional_xfm": "registration", + "inverse_anatomical_to_functional_xfm": "registration", + "functional_gm_mask": "segmentation", + "functional_wm_mask": "segmentation", + "functional_csf_mask": "segmentation", + "frame_wise_displacement_power": "parameters", + "frame_wise_displacement_jenkinson": "parameters", + "functional_nuisance_residuals": "func", + "functional_nuisance_regressors": "func", + "power_spectrum_distribution": "alff", + "functional_freq_filtered": "func", + "scrubbing_movement_parameters": "parameters", + "despiking_frames_included": "parameters", + "despiking_frames_excluded": "parameters", + "scrubbing_frames_included": "parameters", + "scrubbing_frames_excluded": "parameters", + "motion_params": "parameters", + "power_params": "parameters", + "scrubbed_preprocessed": "func", + "functional_to_standard": "func", + "functional_brain_mask_to_standard": "func", + "mean_functional_to_standard": "func", + "functional_to_anat_linear_xfm": "registration", + "functional_to_mni_linear_xfm": "registration", + "mni_to_functional_linear_xfm": "registration", + "ants_symmetric_initial_xfm": "registration", + "ants_symmetric_rigid_xfm": "registration", + "ants_symmetric_affine_xfm": "registration", + "anatomical_to_symmetric_mni_nonlinear_xfm": "registration", + "symmetric_mni_to_anatomical_nonlinear_xfm": "registration", + "symmetric_mni_to_anatomical_linear_xfm": "registration", + "anat_to_symmetric_mni_ants_composite_xfm": "registration", + "symmetric_anatomical_to_standard": "registration", + "anatomical_to_symmetric_mni_linear_xfm": "registration", + "anatomical_to_standard": "anat", + "leaf_node_to_standard": "func", + "vmhc_raw_score": "vmhc", + "vmhc_fisher_zstd": "vmhc", + "vmhc_fisher_zstd_zstat_map": "vmhc", + "alff": "alff", + "falff": "alff", + "alff_smooth": "alff", + "falff_smooth": "alff", + "alff_to_standard": "alff", + "falff_to_standard": "alff", + "alff_to_standard_smooth": "alff", + "falff_to_standard_smooth": "alff", + "alff_to_standard_zstd": "alff", + "falff_to_standard_zstd": "alff", + "alff_to_standard_smooth_zstd": "alff", + "falff_to_standard_smooth_zstd": "alff", + "alff_to_standard_zstd_smooth": "alff", + "falff_to_standard_zstd_smooth": "alff", + "reho": "reho", + "reho_smooth": "reho", + "reho_to_standard": "reho", + "reho_to_standard_smooth": "reho", + "reho_to_standard_zstd": "reho", + "reho_to_standard_smooth_zstd": "reho", + "reho_to_standard_zstd_smooth": "reho", + "voxel_timeseries": "timeseries", + "roi_timeseries": "timeseries", + "roi_timeseries_for_SCA": "timeseries", + "roi_timeseries_for_SCA_multreg": "timeseries", + "sca_roi_files": "sca_roi", + "sca_roi_files_smooth": "sca_roi", + "sca_roi_files_to_standard": "sca_roi", + "sca_roi_files_to_standard_smooth": "sca_roi", + "sca_roi_files_to_standard_fisher_zstd": "sca_roi", + "sca_roi_files_to_standard_smooth_fisher_zstd": "sca_roi", + "sca_roi_files_to_standard_fisher_zstd_smooth": "sca_roi", + "bbregister_registration": "surface_registration", + "left_hemisphere_surface": "surface_registration", + "right_hemisphere_surface": "surface_registration", + "vertices_timeseries": "timeseries", + "centrality": "centrality", + "centrality_smooth": "centrality", + "centrality_zstd": "centrality", + "centrality_smooth_zstd": "centrality", + "centrality_zstd_smooth": "centrality", + "centrality_graphs": "centrality", + "seg_probability_maps": "anat", + "seg_mixeltype": "anat", + "seg_partial_volume_map": "anat", + "seg_partial_volume_files": "anat", + "spatial_map_timeseries": "timeseries", + "spatial_map_timeseries_for_DR": "timeseries", + "dr_tempreg_maps_files": "spatial_regression", + "dr_tempreg_maps_files_smooth": "spatial_regression", + "dr_tempreg_maps_zstat_files": "spatial_regression", + "dr_tempreg_maps_zstat_files_smooth": "spatial_regression", + "dr_tempreg_maps_files_to_standard": "spatial_regression", + "dr_tempreg_maps_zstat_files_to_standard": "spatial_regression", + "dr_tempreg_maps_files_to_standard_smooth": "spatial_regression", + "dr_tempreg_maps_zstat_files_to_standard_smooth": "spatial_regression", + "sca_tempreg_maps_files": "sca_roi", + "sca_tempreg_maps_files_smooth": "sca_roi", + "sca_tempreg_maps_zstat_files": "sca_roi", + "sca_tempreg_maps_zstat_files_smooth": "sca_roi", } fork_ids = { - 'compcor': 'nuis', - 'selector': 'nuis', + "compcor": "nuis", + "selector": "nuis", } -def group_files_in_strategies(output_dir, paths): +def group_files_in_strategies(output_dir, paths): strategies = defaultdict(set) for path in paths: - pieces = path.replace(output_dir, '').split(os.sep) - related_strategy = tuple([ - p for p in pieces if any( - p.startswith('_' + pattern) for pattern in fork_ids.keys() - ) - ]) + pieces = path.replace(output_dir, "").split(os.sep) + related_strategy = tuple( + [ + p + for p in pieces + if any(p.startswith("_" + pattern) for pattern in fork_ids.keys()) + ] + ) strategies[related_strategy].add(path) # Get every file that is not affected by a strategy and add it to every other strategy - strategies_keys = set(strategies.keys()) - set(tuple()) - for paths_without_strategy in strategies[tuple()]: + strategies_keys = set(strategies.keys()) - set() + for paths_without_strategy in strategies[()]: for strategy in strategies_keys: strategies[strategy].add(paths_without_strategy) @@ -178,22 +180,22 @@ def group_files_in_strategies(output_dir, paths): has_derived_strategy = False for specialized_strategy in strategies_keys: - if specialized_strategy not in strategies: continue # specialized_strategy is derived from strategy - if len(specialized_strategy) > len(strategy) and \ - strategy == specialized_strategy[0:len(strategy)]: - + if ( + len(specialized_strategy) > len(strategy) + and strategy == specialized_strategy[0 : len(strategy)] + ): strategies[specialized_strategy].update(strategy_files) has_derived_strategy = True if has_derived_strategy: del strategies[strategy] - if tuple() in strategies: - del strategies[tuple()] + if () in strategies: + del strategies[()] return strategies @@ -205,18 +207,12 @@ def compile_strategy_name(strategy): for id in fork_ids: if fork.startswith(id): fork_name = fork_ids[id] - name += [s.replace('_' + id, fork_name)] + name += [s.replace("_" + id, fork_name)] break - return '__'.join(name) + return "__".join(name) -def create_paths_to_symlinks( - output_dir, - pipeline_id, - subject_id, - paths -): - +def create_paths_to_symlinks(output_dir, pipeline_id, subject_id, paths): if len(paths) == 0: return {} @@ -226,29 +222,27 @@ def create_paths_to_symlinks( strategies = group_files_in_strategies(output_dir, paths) for strategy, strategy_paths in strategies.items(): - strategy_name = compile_strategy_name(strategy) paths_symlinks = [] for path in strategy_paths: - - path_parts = path.replace(output_dir, '').strip(os.sep).split(os.sep) + path_parts = path.replace(output_dir, "").strip(os.sep).split(os.sep) # If file is relative to a scan, use the tag # Otherwise, set to 'scan' - scan_name = [p for p in path_parts if p.startswith('_scan_')] + scan_name = [p for p in path_parts if p.startswith("_scan_")] if not scan_name: - scan_name = 'scan' + scan_name = "scan" else: scan_name = scan_name[0] path_parts = [p for p in path_parts if p != scan_name] - scan_name = scan_name.strip('_') + scan_name = scan_name.strip("_") # Treat QC differently - if path_parts[2] == 'qc': + if path_parts[2] == "qc": output_name = path_parts[3] - output_group = 'qc' + output_group = "qc" output_specs = path_parts[4:-1] else: output_name = path_parts[2] @@ -259,71 +253,74 @@ def create_paths_to_symlinks( # Remove strategies info from path, since it is included # on the outer path output_specs = [ - sp for sp in output_specs - if not any( - sp.startswith('_' + pattern) - for pattern in fork_ids.keys() - ) + sp + for sp in output_specs + if not any(sp.startswith("_" + pattern) for pattern in fork_ids.keys()) ] # Remove leading or trailing underscores - output_specs = [sp.strip('_') for sp in output_specs] + output_specs = [sp.strip("_") for sp in output_specs] # Get the file extension - output_file_ext = '' - output_file_parts = path_parts[-1].split('.') - if output_file_parts[-2:] == ['nii', 'gz']: - output_file_ext = '.nii.gz' + output_file_ext = "" + output_file_parts = path_parts[-1].split(".") + if output_file_parts[-2:] == ["nii", "gz"]: + output_file_ext = ".nii.gz" else: - output_file_ext = '.' + output_file_parts[-1] - - paths_symlinks.append({ - 'path': path, - 'scan': scan_name, - 'group': output_group, - 'specs': tuple(output_specs), - 'name': output_name, - 'file': path_parts[-1], - 'ext': output_file_ext, - }) + output_file_ext = "." + output_file_parts[-1] + + paths_symlinks.append( + { + "path": path, + "scan": scan_name, + "group": output_group, + "specs": tuple(output_specs), + "name": output_name, + "file": path_parts[-1], + "ext": output_file_ext, + } + ) for i, path_symlink in enumerate(paths_symlinks): + original_path = path_symlink["path"] - original_path = path_symlink['path'] - - remaining_paths_symlinks = paths_symlinks[0:i] + paths_symlinks[i+1:] + remaining_paths_symlinks = paths_symlinks[0:i] + paths_symlinks[i + 1 :] - group = path_symlink['group'] + group = path_symlink["group"] # Check if we need to keep the same basename or if we can give it # a better name, in order to avoid link collision # Outputs like from segmentation cant change its name since they # have several files (e.g. seg0, seg1, seg2) if any( - rps['group'] == group and - rps['scan'] == path_symlink['scan'] and - rps['name'] == path_symlink['name'] and - rps['specs'] == path_symlink['specs'] + rps["group"] == group + and rps["scan"] == path_symlink["scan"] + and rps["name"] == path_symlink["name"] + and rps["specs"] == path_symlink["specs"] for rps in remaining_paths_symlinks ): - symlink_basename = path_symlink['file'] + symlink_basename = path_symlink["file"] # if pretty name cant be used, append it to group name - group = os.path.join(group, path_symlink['name']) + group = os.path.join(group, path_symlink["name"]) else: - symlink_basename = path_symlink['name'] + path_symlink['ext'] - - sym_path = os.path.join(*[ - pipeline_id, - strategy_name, - path_symlink['scan'], - group, - ] + list(path_symlink['specs']) + [symlink_basename]) + symlink_basename = path_symlink["name"] + path_symlink["ext"] + + sym_path = os.path.join( + *[ + pipeline_id, + strategy_name, + path_symlink["scan"], + group, + *list(path_symlink["specs"]), + symlink_basename, + ] + ) symlinks[original_path] = sym_path values = list(symlinks.values()) - duplicates = set([x for x in values if values.count(x) > 1]) + duplicates = {x for x in values if values.count(x) > 1} if duplicates: raise Exception("Found duplicates: " + str(duplicates)) @@ -331,31 +328,19 @@ def create_paths_to_symlinks( def create_symlinks( - output_dir, - pipeline_id, - subject_id, - paths, - relative=True, - symlink_dir='sym_links' + output_dir, pipeline_id, subject_id, paths, relative=True, symlink_dir="sym_links" ): - original_cwd = os.getcwd() try: os.chdir(output_dir) - mapping = create_paths_to_symlinks( - output_dir, - pipeline_id, - subject_id, - paths - ) + mapping = create_paths_to_symlinks(output_dir, pipeline_id, subject_id, paths) for path, symlink in mapping.items(): - relpath = path if relpath.startswith(output_dir): - relpath = relpath[len(output_dir):].lstrip('/') + relpath = relpath[len(output_dir) :].lstrip("/") symlink = os.path.join(symlink_dir, symlink) @@ -365,9 +350,7 @@ def create_symlinks( pass if relative: - backtrack = os.path.join(*( - ['..'] * (len(symlink.split('/')) - 1) - )) + backtrack = os.path.join(*([".."] * (len(symlink.split("/")) - 1))) path = os.path.join(backtrack, relpath) try: diff --git a/CPAC/utils/test_init.py b/CPAC/utils/test_init.py index 0ba2423b40..d3d355269b 100644 --- a/CPAC/utils/test_init.py +++ b/CPAC/utils/test_init.py @@ -3,10 +3,10 @@ # Contributing authors (please append): # Daniel Clark # Jon Clucas -''' +""" This module contains functions that assist in initializing CPAC -tests resources -''' +tests resources. +""" from typing import Optional from nipype.interfaces.utility import IdentityInterface @@ -18,7 +18,7 @@ def create_dummy_node(name: str, fields: Optional[LIST[str]] = None): """ Create a dummy IdentityInterface Node source for resources upstream - in a graph from a section to be tested + in a graph from a section to be tested. Parameters ---------- @@ -34,15 +34,15 @@ def create_dummy_node(name: str, fields: Optional[LIST[str]] = None): Node """ if fields is None: - fields = ['resource'] + fields = ["resource"] return Node(IdentityInterface(fields=fields), name=name) # Return tests data config file def populate_template_config(config_type: str) -> str: - ''' + """ Function to read in a template config file from the - CPAC_RESOURCE_DIR and populate it with actual filepaths + CPAC_RESOURCE_DIR and populate it with actual filepaths. Parameters ---------- @@ -54,23 +54,22 @@ def populate_template_config(config_type: str) -> str: ------- config_test : string filepath to the newly written config file for testing - ''' - + """ # Import packages import os # Init variables resource_dir = return_resource_dir() - templates_dir = return_resource_subfolder('templates') - yamls = ['data_config', 'pipeline_config'] + templates_dir = return_resource_subfolder("templates") + yamls = ["data_config", "pipeline_config"] # Check config type and build path if config_type in yamls: - ext = '.yml' - out_name = 'configs' + ext = ".yml" + out_name = "configs" else: # Check if it's supported, otherwise raise an Exception - err_msg = 'config_type parameter: %s is unsupported' % config_type + err_msg = "config_type parameter: %s is unsupported" % config_type raise Exception(err_msg) # Get template and output paths @@ -79,12 +78,12 @@ def populate_template_config(config_type: str) -> str: output_path = os.path.join(output_dir, config_type + ext) # Open the files - tmp_f = open(template_path, 'r') - out_f = open(output_path, 'w') + tmp_f = open(template_path, "r") + out_f = open(output_path, "w") # Replace 'RESOURCE_DIR' string with actual directory for line in tmp_f: - out_f.write(line.replace('RESOURCE_DIR', resource_dir)) + out_f.write(line.replace("RESOURCE_DIR", resource_dir)) # Close file objects tmp_f.close() @@ -96,8 +95,8 @@ def populate_template_config(config_type: str) -> str: # Populate all of the template paths def populate_all_templates(): - ''' - Function to populate all of the template files + """ + Function to populate all of the template files. Parameters ---------- @@ -106,15 +105,21 @@ def populate_all_templates(): Returns ------- None - ''' - + """ # Import packages # Init variables outputs = [] - config_types = ['data_config', 'pipeline_config', 'centrality_spec', - 'map_spec', 'mask_spec', 'roi_spec', 'seed_spec', - 'spatial_maps_spec'] + config_types = [ + "data_config", + "pipeline_config", + "centrality_spec", + "map_spec", + "mask_spec", + "roi_spec", + "seed_spec", + "spatial_maps_spec", + ] # Populate all of the config templates with actual paths for config_type in config_types: @@ -123,17 +128,17 @@ def populate_all_templates(): # Check that they all returned a value if len(outputs) == len(config_types): - print('Successfully populated and saved templates!') + pass else: - err_msg = 'Something went wrong during template population' + err_msg = "Something went wrong during template population" raise Exception(err_msg) # Get the AWS credentials def return_aws_creds(): - ''' + """ Function to return the AWS credentials file given by the - CPAC_AWS_CREDS environment variable + CPAC_AWS_CREDS environment variable. Parameters ---------- @@ -144,28 +149,25 @@ def return_aws_creds(): aws_creds : string filepath to the AWS credentials with access key id and secret access key - ''' - + """ # Import packages import os # Init variables - creds_path = os.getenv('CPAC_AWS_CREDS') + creds_path = os.getenv("CPAC_AWS_CREDS") # Check if set if not creds_path: - err_msg = 'CPAC_AWS_CREDS environment variable not set!\n' \ - 'Set this to the filepath location of your AWS credentials.' - print(err_msg) - creds_path = input('Enter path to AWS credentials file: ') + creds_path = input("Enter path to AWS credentials file: ") + return None else: return creds_path # Get the default test bucket name def default_bucket_name(): - ''' - Function to return the default S3 bucket name used in test suite + """ + Function to return the default S3 bucket name used in test suite. Parameters ---------- @@ -175,20 +177,18 @@ def default_bucket_name(): ------- bucket_name : string default S3 bucket name for testing - ''' - + """ # Set default bucket name - bucket_name = 'fcp-indi' + return "fcp-indi" # Return bucket name - return bucket_name # Grab all nifti files within directory def return_all_niis(base_dir): - ''' + """ Function to walk through a base directory and all subsequent files - and return the filepaths of all nifti files found + and return the filepaths of all nifti files found. Parameters ---------- @@ -199,8 +199,7 @@ def return_all_niis(base_dir): ------- nii_list : list a list of filepath strings of the nifti files found in base_dir - ''' - + """ # Import packages import os @@ -210,8 +209,9 @@ def return_all_niis(base_dir): # Collect computed outputs for root, dirs, files in os.walk(base_dir): if files: - nii_list.extend([os.path.join(root, file) for file in files \ - if file.endswith('.nii.gz')]) + nii_list.extend( + [os.path.join(root, file) for file in files if file.endswith(".nii.gz")] + ) # Return the list of files return nii_list @@ -219,16 +219,15 @@ def return_all_niis(base_dir): # Download the CPAC resource dir from S3 def download_cpac_resources_from_s3(local_base): - ''' + """ Function to download the CPAC testing resources directory from - S3 + S3. Parameters ---------- local_base : string the local directory to save the 'cpac_resources' contents - ''' - + """ # Import packages import os @@ -236,8 +235,8 @@ def download_cpac_resources_from_s3(local_base): # Init variables bucket_name = default_bucket_name() - resource_folder = 'cpac_resources' - s3_prefix = os.path.join('data/test_resources', resource_folder) + resource_folder = "cpac_resources" + s3_prefix = os.path.join("data/test_resources", resource_folder) # Get bucket object bucket = fetch_creds.return_bucket(None, bucket_name) @@ -246,12 +245,13 @@ def download_cpac_resources_from_s3(local_base): for obj in bucket.objects.filter(Prefix=s3_prefix): bkey = obj.key # If the object is just a folder, move on to next object - if bkey.endswith('/'): + if bkey.endswith("/"): continue # Form local path from key - local_path = os.path.join(local_base, - bkey.split(resource_folder)[-1].lstrip('/')) + local_path = os.path.join( + local_base, bkey.split(resource_folder)[-1].lstrip("/") + ) # Make download directories local_dir = os.path.dirname(local_path) @@ -260,18 +260,18 @@ def download_cpac_resources_from_s3(local_base): # Download file if it doesn't exist if not os.path.exists(local_path): - bucket.download_file(bkey, local_path, - Callback=aws_utils.ProgressPercentage(obj)) + bucket.download_file( + bkey, local_path, Callback=aws_utils.ProgressPercentage(obj) + ) # Print done - print('CPAC resources folder in %s is complete!' % local_base) # Look for CPAC_RESOURCE_DIR to be in environment def return_resource_dir(): - ''' + """ Function to return the filepath of the CPAC_RESOURCE_DIR; note the - CPAC_RESOURCE_DIR environment variable must be set + CPAC_RESOURCE_DIR environment variable must be set. Parameters ---------- @@ -281,30 +281,27 @@ def return_resource_dir(): ------- resource_dir : string the file path on disk where the cpac resources folder is - ''' + """ # Import packages import os # Init variables - resource_dir = os.getenv('CPAC_RESOURCE_DIR') + resource_dir = os.getenv("CPAC_RESOURCE_DIR") # Check if set if not resource_dir: # Print notification of cpac resources directory - print_msg = 'CPAC_RESOURCE_DIR environment variable not set! Enter '\ - 'directory of the cpac_resources folder.\n\n*If the folder '\ - 'does not exist, it will be downloaded under the directory '\ - 'specified.' - print(print_msg) # Get user input - resource_dir = input('Enter C-PAC resources directory: ') + resource_dir = input("Enter C-PAC resources directory: ") # Check and download any new or missing resources from S3 copy try: download_cpac_resources_from_s3(resource_dir) except Exception as exc: - err_msg = 'There was a problem downloading the cpac_resources '\ - 'folder from S3.\nError: %s' % exc + err_msg = ( + "There was a problem downloading the cpac_resources " + "folder from S3.\nError: %s" % exc + ) raise Exception(err_msg) return resource_dir @@ -312,8 +309,8 @@ def return_resource_dir(): # Return any subfolder of the resource directory def return_resource_subfolder(subfolder): - ''' - Funnction to return subfolders of the CPAC_RESOURCE_DIR + """ + Funnction to return subfolders of the CPAC_RESOURCE_DIR. Parameters ---------- @@ -324,19 +321,17 @@ def return_resource_subfolder(subfolder): ------- resource_subfolder : string filepath to the resource subfolder - ''' - + """ # Import packages import os # Init variables resource_dir = return_resource_dir() - in_settings = ['configs', 'creds', 'resources', - 'subject_lists', 'templates'] + in_settings = ["configs", "creds", "resources", "subject_lists", "templates"] # Check if its a sub-subfolder if subfolder in in_settings: - resource_subfolder = os.path.join(resource_dir, 'settings', subfolder) + resource_subfolder = os.path.join(resource_dir, "settings", subfolder) else: resource_subfolder = os.path.join(resource_dir, subfolder) @@ -346,9 +341,9 @@ def return_resource_subfolder(subfolder): # Return test strategies obj file def return_strats_obj(): - ''' + """ Function to return the file path of the strategies obj file from - the CPAC_RESOURCE_DIR + the CPAC_RESOURCE_DIR. Parameters ---------- @@ -358,26 +353,24 @@ def return_strats_obj(): ------- strats_obj : string filepath to the strategies obj file - ''' - + """ # Import packages import os # Init variables - settings_dir = return_resource_subfolder('resources') + settings_dir = return_resource_subfolder("resources") # Get strategies obj - strats_obj = os.path.join(settings_dir, 'strategies_test.obj') + return os.path.join(settings_dir, "strategies_test.obj") # Return filepath - return strats_obj # Return tests subject list def return_subject_list(): - ''' + """ Function to return the file path of the subject list file from - the CPAC_RESOURCE_DIR + the CPAC_RESOURCE_DIR. Parameters ---------- @@ -387,26 +380,24 @@ def return_subject_list(): ------- subject_list : string filepath to the subject list yaml file - ''' - + """ # Import packages import os # Init variables - config_dir = return_resource_subfolder('subject_lists') + config_dir = return_resource_subfolder("subject_lists") # Get sublist - subject_list = os.path.join(config_dir, 'CPAC_subject_list_test.yml') + return os.path.join(config_dir, "CPAC_subject_list_test.yml") # Return filepath - return subject_list # Return the test subjects measure directories def return_subj_measure_dirs(measure): - ''' + """ Function to grab the base directories of the test subject's output - files for a given measure or workflow + files for a given measure or workflow. Parameters ---------- @@ -420,25 +411,23 @@ def return_subj_measure_dirs(measure): subj_measure_dirs : list a list of strings of the base directories for each instance of the desired measure folder within the test subjects outputs - ''' - + """ # Import packages import glob import os # Init variables test_subj = return_test_subj() - outputs_dir = return_resource_subfolder('output') + outputs_dir = return_resource_subfolder("output") # Root directories (cpac_resources/output/reg/subj_sess/scan/measure/..) - subj_measure_dirs = \ - glob.glob(os.path.join(outputs_dir, '*', '%s*' % test_subj, - '*', measure)) + subj_measure_dirs = glob.glob( + os.path.join(outputs_dir, "*", "%s*" % test_subj, "*", measure) + ) # Check to see if the directories exist if len(subj_measure_dirs) == 0: - err_msg = 'Unable to find any subject directories for the %s measure.' \ - % measure + err_msg = "Unable to find any subject directories for the %s measure." % measure raise Exception(err_msg) # Return base directories for test measures outputs @@ -447,9 +436,9 @@ def return_subj_measure_dirs(measure): # Get subject for individual tests def return_test_subj(): - ''' + """ Function to return the subject id; note the - CPAC_RESOURCE_DIR environment variable must be set + CPAC_RESOURCE_DIR environment variable must be set. Parameters ---------- @@ -459,33 +448,32 @@ def return_test_subj(): ------- resource_dir : string the file path on disk where the cpac resources folder is - ''' - + """ # Import packages import os # Init variables - test_subj = os.getenv('CPAC_TEST_SUBJ') + test_subj = os.getenv("CPAC_TEST_SUBJ") # Get cpac resource directory and get a list of subject folders - input_dir = return_resource_subfolder('input') - site_dir = os.path.join(input_dir, 'site_1') + input_dir = return_resource_subfolder("input") + site_dir = os.path.join(input_dir, "site_1") # Get list of subject directories subs = os.listdir(site_dir) # Check if set and exists if not test_subj: - info_msg = 'CPAC_TEST_SUBJ environment variable not set!' - print(info_msg) # Get user input - test_subj = input('Enter C-PAC benchmark test subject id: ') + test_subj = input("Enter C-PAC benchmark test subject id: ") # Check to make sure their input files exist if test_subj not in subs: - err_msg = 'Test subject %s is not in the cpac_resources subject ' \ - 'directory %s. Please specify different CPAC_TEST_SUBJ.' \ - %(test_subj, site_dir) + err_msg = ( + "Test subject %s is not in the cpac_resources subject " + "directory %s. Please specify different CPAC_TEST_SUBJ." + % (test_subj, site_dir) + ) raise Exception(err_msg) else: return test_subj @@ -493,9 +481,9 @@ def return_test_subj(): # Smooth nifti file def smooth_nii_file(self, nii_file, fwhm, mask_file=None): - ''' + """ Function to Gaussian smooth nifti files and optionally using a mask - on the smoothed data + on the smoothed data. Parameters ---------- @@ -510,11 +498,10 @@ def smooth_nii_file(self, nii_file, fwhm, mask_file=None): ------- smooth_arr : numpy.ndarray smoothed nifti image as a numpy array - ''' - + """ # Import packages - import nibabel as nib import numpy as np + import nibabel as nib import scipy.ndimage # Init variables @@ -526,20 +513,22 @@ def smooth_nii_file(self, nii_file, fwhm, mask_file=None): mask_arr = nib.load(mask_file).get_fdata() # Check the mask shape matches the raw nifti if mask_arr.shape != raw_arr.shape: - err_msg = 'Mask file has different dimensions than nifti.\n' \ - 'Check the paths are correct and try again.' + err_msg = ( + "Mask file has different dimensions than nifti.\n" + "Check the paths are correct and try again." + ) raise Exception(err_msg) # Calculate sigma for smoothing mm_res = np.abs(raw_nii.affine[0][0]) - sigma = fwhm/2.3548/mm_res + sigma = fwhm / 2.3548 / mm_res # Smooth input smooth_arr = scipy.ndimage.gaussian_filter(raw_arr, sigma, order=0) # And mask if using one (this writes it to a 1d array) if mask_arr: - smooth_out = smooth_arr[mask_arr.astype('bool')] + smooth_out = smooth_arr[mask_arr.astype("bool")] smooth_arr = np.zeros(mask_arr.shape, dtype=float) # Get mask coordinates and populate smoothed image @@ -554,13 +543,14 @@ def smooth_nii_file(self, nii_file, fwhm, mask_file=None): # Download test resource from S3 bucket def download_resource_from_s3(s3_url_path): - ''' - ''' + """ """ # Import packages import os import tempfile - import urllib.request, urllib.parse, urllib.error + import urllib.error + import urllib.parse + import urllib.request # Init variables temp_dir = tempfile.mkdtemp() @@ -577,7 +567,7 @@ def download_resource_from_s3(s3_url_path): # Setup log file def setup_test_logger(logger_name, log_file, level, to_screen=False): - ''' + """ Function to initialize and configure a logger that can write to file and (optionally) the screen. @@ -599,16 +589,16 @@ def setup_test_logger(logger_name, log_file, level, to_screen=False): logger : logging.Logger object Python logging.Logger object which is capable of logging run- time information about the program to file and/or screen - ''' - + """ # Import packages import logging + from CPAC.utils.monitoring.custom_logging import getLogger # Init logger, formatter, filehandler, streamhandler logger = getLogger(logger_name) logger.setLevel(level) - formatter = logging.Formatter('%(asctime)s : %(message)s') + formatter = logging.Formatter("%(asctime)s : %(message)s") # Write logs to file file_handler = logging.FileHandler(log_file) @@ -624,20 +614,22 @@ def setup_test_logger(logger_name, log_file, level, to_screen=False): # Return the logger return logger + def pearson_correlation(nii_1, nii_2): - import nibabel as nb import numpy as np + import nibabel as nib - data_1 = nb.load(nii_1).get_fdata() - data_2 = nb.load(nii_2).get_fdata() + data_1 = nib.load(nii_1).get_fdata() + data_2 = nib.load(nii_2).get_fdata() R = np.corrcoef(data_1.flatten(), data_2.flatten()) - return(R[0,1]) + return R[0, 1] + # Calculate concordance correlation coefficient def concordance(x, y): - ''' + """ Return the concordance correlation coefficient as defined by - Lin (1989) + Lin (1989). Parameters ---------- @@ -650,8 +642,7 @@ def concordance(x, y): ------- rho_c : numpy.float32 the concordance value as a float - ''' - + """ # Import packages import numpy as np @@ -659,16 +650,18 @@ def concordance(x, y): x_shape = np.shape(x) y_shape = np.shape(y) if len(x_shape) != 1 or len(y_shape) != 1: - err_msg = 'Inputs must be 1D lists or arrays.' + err_msg = "Inputs must be 1D lists or arrays." raise ValueError(err_msg) elif x_shape != y_shape: - err_msg = 'Length of the two inputs must be equal.\n'\ - 'Length of x: %d\nLength of y: %d' % (len(x), len(y)) + err_msg = ( + "Length of the two inputs must be equal.\n" + "Length of x: %d\nLength of y: %d" % (len(x), len(y)) + ) raise ValueError(err_msg) # Init variables - x_arr = np.array(x).astype('float64') - y_arr = np.array(y).astype('float64') + x_arr = np.array(x).astype("float64") + y_arr = np.array(y).astype("float64") # Get pearson correlation rho = np.corrcoef(x_arr, y_arr)[0][1] @@ -682,8 +675,8 @@ def concordance(x, y): mu_y = np.mean(y_arr) # Comput condordance - rho_c = (2*rho*sigma_x*sigma_y) /\ - (sigma_x**2 + sigma_y**2 + (mu_x-mu_y)**2) + return (2 * rho * sigma_x * sigma_y) / ( + sigma_x**2 + sigma_y**2 + (mu_x - mu_y) ** 2 + ) # Return variables - return rho_c diff --git a/CPAC/utils/test_mocks.py b/CPAC/utils/test_mocks.py index f2a0a6aafb..901b84659c 100644 --- a/CPAC/utils/test_mocks.py +++ b/CPAC/utils/test_mocks.py @@ -1,5 +1,7 @@ import os + from nipype.interfaces import utility as util + from CPAC.pipeline import nipype_pipeline_engine as pe from CPAC.utils.configuration import Configuration from CPAC.utils.datasource import resolve_resolution @@ -9,206 +11,234 @@ def file_node(path, file_node_num=0): input_node = pe.Node( - util.IdentityInterface(fields=['file']), name='file_node_{0}'.format( - file_node_num) + util.IdentityInterface(fields=["file"]), + name="file_node_{0}".format(file_node_num), ) input_node.inputs.file = path - return input_node, 'file' + return input_node, "file" -def configuration_strategy_mock(method='FSL'): - fsldir = os.environ.get('FSLDIR') +def configuration_strategy_mock(method="FSL"): + fsldir = os.environ.get("FSLDIR") # mock the config dictionary - c = Configuration({ - "pipeline_setup": { - "output_directory": { - "path": "/output/output/pipeline_analysis_nuisance/" - "sub-M10978008_ses-NFB3" - }, - "working_directory": { - "path": "/scratch/pipeline_tests" - }, - "system_config": { - "num_ants_threads": 4 - } - }, - "registration_workflows": { - "functional_registration": { - "EPI_registration": { - "FSL-FNIRT": { - "identity_matrix": f"{fsldir}/etc/flirtsch/" - "ident.mat", - "interpolation": "sinc" - } + c = Configuration( + { + "pipeline_setup": { + "output_directory": { + "path": "/output/output/pipeline_analysis_nuisance/" + "sub-M10978008_ses-NFB3" }, - "func_registration_to_template": { - "ANTs_pipelines": { - "interpolation": "LanczosWindowedSinc" + "working_directory": {"path": "/scratch/pipeline_tests"}, + "system_config": {"num_ants_threads": 4}, + }, + "registration_workflows": { + "functional_registration": { + "EPI_registration": { + "FSL-FNIRT": { + "identity_matrix": f"{fsldir}/etc/flirtsch/" "ident.mat", + "interpolation": "sinc", + } }, - "output_resolution": { - "func_preproc_outputs": "3mm", - "func_derivative_outputs": "3mm" + "func_registration_to_template": { + "ANTs_pipelines": {"interpolation": "LanczosWindowedSinc"}, + "output_resolution": { + "func_preproc_outputs": "3mm", + "func_derivative_outputs": "3mm", + }, + "target_template": { + "T1_template": { + "T1w_template_for_resample": f"{fsldir}/" + "data/standard/" + "MNI152_T1_1mm_brain." + "nii.gz", + "T1w_brain_template_funcreg": f"{fsldir}/" + "data/standard/" + "MNI152_T1_" + "${resolution_for_" + "func_preproc}_" + "brain.nii.gz", + "T1w_template_funcreg": f"{fsldir}/data/" + "standard/MNI152_T1_" + "${resolution_for_func_" + "preproc}.nii.gz", + } + }, }, - "target_template": { - "T1_template": { - "T1w_template_for_resample": f"{fsldir}/" - "data/standard/" - "MNI152_T1_1mm_brain." - "nii.gz", - "T1w_brain_template_funcreg": f"{fsldir}/" - "data/standard/" - "MNI152_T1_" - "${resolution_for_" - "func_preproc}_" - "brain.nii.gz", - "T1w_template_funcreg": f"{fsldir}/data/" - "standard/MNI152_T1_" - "${resolution_for_func_" - "preproc}.nii.gz" - } - } } - } - }, - "post_processing": { - "spatial_smoothing": { - "fwhm": [2, 3, 4] - } + }, + "post_processing": {"spatial_smoothing": {"fwhm": [2, 3, 4]}}, } - }) + ) - if method == 'ANTS': - c.update('regOption', 'ANTS') + if method == "ANTS": + c.update("regOption", "ANTS") else: - c.update('regOption', 'FSL') + c.update("regOption", "FSL") # mock the strategy strat = Strategy() resource_dict = { - "functional_nuisance_residuals": os.path.join( - c['pipeline_setup', 'output_directory', 'path'], - "motion_correct/_scan_test/" - "sub-M10978008_ses-NFB3_task-test_bold_calc_tshift_resample_" - "volreg.nii.gz"), - "mean_functional": os.path.join( - c['pipeline_setup', 'output_directory', 'path'], - "mean_functional/" - "sub-M10978008_ses-NFB3_task-test_bold_calc_tshift_resample_" - "volreg_calc_tstat.nii.gz"), - "functional_brain_mask": os.path.join( - c['pipeline_setup', 'output_directory', 'path'], - "functional_brain_mask/_scan_test/" - "sub-M10978008_ses-NFB3_task-test_bold_calc_tshift_resample_" - "volreg_mask.nii.gz"), - "motion_correct": os.path.join( - c['pipeline_setup', 'output_directory', 'path'], - "motion_correct/_scan_test/" - "sub-M10978008_ses-NFB3_task-test_bold_calc_tshift_resample_" - "volreg.nii.gz"), - "anatomical_brain": os.path.join( - c['pipeline_setup', 'output_directory', 'path'], - "anatomical_brain/" - "sub-M10978008_ses-NFB3_acq-ao_brain_resample.nii.gz"), - "ants_initial_xfm": os.path.join( - c['pipeline_setup', 'output_directory', 'path'], - "ants_initial_xfm/" - "transform0DerivedInitialMovingTranslation.mat"), - "ants_affine_xfm": os.path.join( - c['pipeline_setup', 'output_directory', 'path'], - "ants_affine_xfm/transform2Affine.mat"), - "ants_rigid_xfm": os.path.join( - c['pipeline_setup', 'output_directory', 'path'], - "ants_rigid_xfm/transform1Rigid.mat"), - "anatomical_to_mni_linear_xfm": os.path.join( - c['pipeline_setup', 'output_directory', 'path'], - "anatomical_to_mni_linear_xfm/" - "sub-M10978008_ses-NFB3_T1w_resample_calc_flirt.mat"), - "functional_to_anat_linear_xfm": os.path.join( - c['pipeline_setup', 'output_directory', 'path'], - "functional_to_anat_linear_xfm/_scan_test/" - "sub-M10978008_ses-NFB3_task-test_bold_calc_tshift_resample_" - "volreg_calc_tstat_flirt.mat"), - 'ants_symm_warp_field': os.path.join( - c['pipeline_setup', 'output_directory', 'path'], - "anatomical_to_symmetric_mni_nonlinear_xfm/" - "transform3Warp.nii.gz"), - 'ants_symm_affine_xfm': os.path.join( - c['pipeline_setup', 'output_directory', 'path'], - "ants_symmetric_affine_xfm/transform2Affine.mat"), - 'ants_symm_rigid_xfm': os.path.join( - c['pipeline_setup', 'output_directory', 'path'], - "ants_symmetric_rigid_xfm/transform1Rigid.mat"), - 'ants_symm_initial_xfm': os.path.join( - c['pipeline_setup', 'output_directory', 'path'], - "ants_symmetric_initial_xfm/" - "transform0DerivedInitialMovingTranslation.mat"), - "dr_tempreg_maps_files": [os.path.join( - '/scratch', - 'resting_preproc_sub-M10978008_ses-NFB3_cpac105', - 'temporal_dual_regression_0/_scan_test/' - '_selector_CSF-2mmE-M_aC-WM-2mmE-DPC5_G-M_M-SDB_P-2/' - '_spatial_map_PNAS_Smith09_rsn10_spatial_map_file_' - '..cpac_templates..PNAS_Smith09_rsn10.nii.gz/' - 'split_raw_volumes/temp_reg_map_000{0}.nii.gz'.format(n) - ) for n in range(10)] + "functional_nuisance_residuals": os.path.join( + c["pipeline_setup", "output_directory", "path"], + "motion_correct/_scan_test/" + "sub-M10978008_ses-NFB3_task-test_bold_calc_tshift_resample_" + "volreg.nii.gz", + ), + "mean_functional": os.path.join( + c["pipeline_setup", "output_directory", "path"], + "mean_functional/" + "sub-M10978008_ses-NFB3_task-test_bold_calc_tshift_resample_" + "volreg_calc_tstat.nii.gz", + ), + "functional_brain_mask": os.path.join( + c["pipeline_setup", "output_directory", "path"], + "functional_brain_mask/_scan_test/" + "sub-M10978008_ses-NFB3_task-test_bold_calc_tshift_resample_" + "volreg_mask.nii.gz", + ), + "motion_correct": os.path.join( + c["pipeline_setup", "output_directory", "path"], + "motion_correct/_scan_test/" + "sub-M10978008_ses-NFB3_task-test_bold_calc_tshift_resample_" + "volreg.nii.gz", + ), + "anatomical_brain": os.path.join( + c["pipeline_setup", "output_directory", "path"], + "anatomical_brain/" "sub-M10978008_ses-NFB3_acq-ao_brain_resample.nii.gz", + ), + "ants_initial_xfm": os.path.join( + c["pipeline_setup", "output_directory", "path"], + "ants_initial_xfm/" "transform0DerivedInitialMovingTranslation.mat", + ), + "ants_affine_xfm": os.path.join( + c["pipeline_setup", "output_directory", "path"], + "ants_affine_xfm/transform2Affine.mat", + ), + "ants_rigid_xfm": os.path.join( + c["pipeline_setup", "output_directory", "path"], + "ants_rigid_xfm/transform1Rigid.mat", + ), + "anatomical_to_mni_linear_xfm": os.path.join( + c["pipeline_setup", "output_directory", "path"], + "anatomical_to_mni_linear_xfm/" + "sub-M10978008_ses-NFB3_T1w_resample_calc_flirt.mat", + ), + "functional_to_anat_linear_xfm": os.path.join( + c["pipeline_setup", "output_directory", "path"], + "functional_to_anat_linear_xfm/_scan_test/" + "sub-M10978008_ses-NFB3_task-test_bold_calc_tshift_resample_" + "volreg_calc_tstat_flirt.mat", + ), + "ants_symm_warp_field": os.path.join( + c["pipeline_setup", "output_directory", "path"], + "anatomical_to_symmetric_mni_nonlinear_xfm/" "transform3Warp.nii.gz", + ), + "ants_symm_affine_xfm": os.path.join( + c["pipeline_setup", "output_directory", "path"], + "ants_symmetric_affine_xfm/transform2Affine.mat", + ), + "ants_symm_rigid_xfm": os.path.join( + c["pipeline_setup", "output_directory", "path"], + "ants_symmetric_rigid_xfm/transform1Rigid.mat", + ), + "ants_symm_initial_xfm": os.path.join( + c["pipeline_setup", "output_directory", "path"], + "ants_symmetric_initial_xfm/" + "transform0DerivedInitialMovingTranslation.mat", + ), + "dr_tempreg_maps_files": [ + os.path.join( + "/scratch", + "resting_preproc_sub-M10978008_ses-NFB3_cpac105", + "temporal_dual_regression_0/_scan_test/" + "_selector_CSF-2mmE-M_aC-WM-2mmE-DPC5_G-M_M-SDB_P-2/" + "_spatial_map_PNAS_Smith09_rsn10_spatial_map_file_" + "..cpac_templates..PNAS_Smith09_rsn10.nii.gz/" + "split_raw_volumes/temp_reg_map_000{0}.nii.gz".format(n), + ) + for n in range(10) + ], } - if method == 'ANTS': + if method == "ANTS": resource_dict["anatomical_to_mni_nonlinear_xfm"] = os.path.join( - c['pipeline_setup', 'output_directory', 'path'], - "anatomical_to_mni_nonlinear_xfm/transform3Warp.nii.gz") + c["pipeline_setup", "output_directory", "path"], + "anatomical_to_mni_nonlinear_xfm/transform3Warp.nii.gz", + ) else: resource_dict["anatomical_to_mni_nonlinear_xfm"] = os.path.join( - c['pipeline_setup', 'output_directory', 'path'], + c["pipeline_setup", "output_directory", "path"], "anatomical_to_mni_nonlinear_xfm/" - "sub-M10978008_ses-NFB3_T1w_resample_fieldwarp.nii.gz") + "sub-M10978008_ses-NFB3_T1w_resample_fieldwarp.nii.gz", + ) file_node_num = 0 for resource, filepath in resource_dict.items(): - strat.update_resource_pool({ - resource: file_node(filepath, file_node_num) - }) - strat.append_name(resource+'_0') + strat.update_resource_pool({resource: file_node(filepath, file_node_num)}) + strat.append_name(resource + "_0") file_node_num += 1 templates_for_resampling = [ - (c['registration_workflows', 'functional_registration', - 'func_registration_to_template', 'output_resolution', - 'func_preproc_outputs'], - c['registration_workflows', 'functional_registration', - 'func_registration_to_template', 'target_template', 'T1_template', - 'T1w_brain_template_funcreg'], - 'template_brain_for_func_preproc', - 'resolution_for_func_preproc'), - (c['registration_workflows', 'functional_registration', - 'func_registration_to_template', 'output_resolution', - 'func_preproc_outputs'], - c['registration_workflows', 'functional_registration', - 'func_registration_to_template', 'target_template', 'T1_template', - 'T1w_brain_template_funcreg'], - 'template_skull_for_func_preproc', - 'resolution_for_func_preproc') + ( + c[ + "registration_workflows", + "functional_registration", + "func_registration_to_template", + "output_resolution", + "func_preproc_outputs", + ], + c[ + "registration_workflows", + "functional_registration", + "func_registration_to_template", + "target_template", + "T1_template", + "T1w_brain_template_funcreg", + ], + "template_brain_for_func_preproc", + "resolution_for_func_preproc", + ), + ( + c[ + "registration_workflows", + "functional_registration", + "func_registration_to_template", + "output_resolution", + "func_preproc_outputs", + ], + c[ + "registration_workflows", + "functional_registration", + "func_registration_to_template", + "target_template", + "T1_template", + "T1w_brain_template_funcreg", + ], + "template_skull_for_func_preproc", + "resolution_for_func_preproc", + ), ] for resolution, template, template_name, tag in templates_for_resampling: - resampled_template = pe.Node(Function(input_names=[ - 'resolution', 'template', 'template_name', 'tag' - ], - output_names=[ - 'resampled_template' - ], - function=resolve_resolution, - as_module=True), - name='resampled_' + template_name) + resampled_template = pe.Node( + Function( + input_names=["resolution", "template", "template_name", "tag"], + output_names=["resampled_template"], + function=resolve_resolution, + as_module=True, + ), + name="resampled_" + template_name, + ) resampled_template.inputs.resolution = resolution resampled_template.inputs.template = template resampled_template.inputs.template_name = template_name resampled_template.inputs.tag = tag - strat.update_resource_pool({ - template_name: (resampled_template, 'resampled_template')}) - strat.append_name('resampled_template_0') + strat.update_resource_pool( + {template_name: (resampled_template, "resampled_template")} + ) + strat.append_name("resampled_template_0") return c, strat diff --git a/CPAC/utils/test_resources.py b/CPAC/utils/test_resources.py index 8b9410a6c2..88ff2b2354 100644 --- a/CPAC/utils/test_resources.py +++ b/CPAC/utils/test_resources.py @@ -1,12 +1,10 @@ - - def setup_test_wf(s3_prefix, paths_list, test_name, workdirs_to_keep=None): """Set up a basic template Nipype workflow for testing single nodes or small sub-workflows. """ - import os import shutil + from CPAC.pipeline import nipype_pipeline_engine as pe from CPAC.utils.datasource import check_for_s3 from CPAC.utils.interfaces.datasink import DataSink @@ -25,7 +23,6 @@ def setup_test_wf(s3_prefix, paths_list, test_name, workdirs_to_keep=None): for dirname in os.listdir(work_dir): if workdirs_to_keep: for keepdir in workdirs_to_keep: - print("{0} --- {1}\n".format(dirname, keepdir)) if keepdir in dirname: continue try: @@ -41,13 +38,13 @@ def setup_test_wf(s3_prefix, paths_list, test_name, workdirs_to_keep=None): wf = pe.Workflow(name=test_name) wf.base_dir = os.path.join(work_dir) - wf.config['execution'] = { - 'hash_method': 'timestamp', - 'crashdump_dir': os.path.abspath(test_dir) + wf.config["execution"] = { + "hash_method": "timestamp", + "crashdump_dir": os.path.abspath(test_dir), } - ds = pe.Node(DataSink(), name='sinker_{0}'.format(test_name)) + ds = pe.Node(DataSink(), name="sinker_{0}".format(test_name)) ds.inputs.base_directory = out_dir ds.inputs.parameterization = True - return (wf, ds, local_paths) \ No newline at end of file + return (wf, ds, local_paths) diff --git a/CPAC/utils/tests/_dataclasses.py b/CPAC/utils/tests/_dataclasses.py index b989a4ffd0..042dc5d939 100644 --- a/CPAC/utils/tests/_dataclasses.py +++ b/CPAC/utils/tests/_dataclasses.py @@ -1,14 +1,16 @@ -"""Dataclasses for pickled test data""" +"""Dataclasses for pickled test data.""" from dataclasses import dataclass -import nibabel as nib + import numpy as np +import nibabel as nib -__all__ = ['_FdjTestData', '_MockImageHeaderOnly'] +__all__ = ["_FdjTestData", "_MockImageHeaderOnly"] @dataclass class _FdjTestData: - """Class for storing test data for FD-J functions""" + """Class for storing test data for FD-J functions.""" + affine: np.ndarray img: nib.Nifti1Image rels_rms: np.ndarray diff --git a/CPAC/utils/tests/configs/__init__.py b/CPAC/utils/tests/configs/__init__.py index 573c0dc271..f1f27e830f 100644 --- a/CPAC/utils/tests/configs/__init__.py +++ b/CPAC/utils/tests/configs/__init__.py @@ -1,22 +1,29 @@ -"""Configs for testing""" +"""Configs for testing.""" from pathlib import Path + from pkg_resources import resource_filename import yaml _TEST_CONFIGS_PATH = Path(resource_filename("CPAC", "utils/tests/configs")) -with open(_TEST_CONFIGS_PATH / "neurostars_23786.yml", "r", encoding="utf-8" - ) as _f: +with open(_TEST_CONFIGS_PATH / "neurostars_23786.yml", "r", encoding="utf-8") as _f: # A loaded YAML file to test https://tinyurl.com/neurostars23786 NEUROSTARS_23786 = _f.read() -with open(_TEST_CONFIGS_PATH / "neurostars_24035.yml", "r", encoding="utf-8" - ) as _f: +with open(_TEST_CONFIGS_PATH / "neurostars_24035.yml", "r", encoding="utf-8") as _f: # A loaded YAML file to test https://tinyurl.com/neurostars24035 NEUROSTARS_24035 = _f.read() # A loaded YAML file to test https://tinyurl.com/cmicnlslack420349 SLACK_420349 = { - "filepath": yaml.dump({ - "FROM": str(_TEST_CONFIGS_PATH / "neurostars_23786.yml"), - "pipeline_setup": {"pipeline_name": "slack_420349_filepath"}}), - "preconfig": yaml.dump({"FROM": "preproc", - "pipeline_setup": {"pipeline_name": "slack_420349_preconfig"}})} + "filepath": yaml.dump( + { + "FROM": str(_TEST_CONFIGS_PATH / "neurostars_23786.yml"), + "pipeline_setup": {"pipeline_name": "slack_420349_filepath"}, + } + ), + "preconfig": yaml.dump( + { + "FROM": "preproc", + "pipeline_setup": {"pipeline_name": "slack_420349_preconfig"}, + } + ), +} __all__ = ["NEUROSTARS_23786", "NEUROSTARS_24035", "SLACK_420349"] diff --git a/CPAC/utils/tests/configs/neurostars_23786.yml b/CPAC/utils/tests/configs/neurostars_23786.yml index 3e02e0659b..ff7785ab50 100644 --- a/CPAC/utils/tests/configs/neurostars_23786.yml +++ b/CPAC/utils/tests/configs/neurostars_23786.yml @@ -1,4 +1,4 @@ seed_based_correlation_analysis: run: true sca_roi_paths: - /cpac_templates/PNAS_Smith09_rsn10.nii.gz: DualReg \ No newline at end of file + /cpac_templates/PNAS_Smith09_rsn10.nii.gz: DualReg diff --git a/CPAC/utils/tests/configs/neurostars_24035.yml b/CPAC/utils/tests/configs/neurostars_24035.yml index 8632dd1a67..4713fadefa 100644 --- a/CPAC/utils/tests/configs/neurostars_24035.yml +++ b/CPAC/utils/tests/configs/neurostars_24035.yml @@ -290,4 +290,4 @@ PyPEER: peer_scrub: False # Motion scrubbing threshold (PyPEER only) - scrub_thresh: 0.2 \ No newline at end of file + scrub_thresh: 0.2 diff --git a/CPAC/utils/tests/test_bids_utils.py b/CPAC/utils/tests/test_bids_utils.py index 0c6cabebbd..922d1bac8a 100644 --- a/CPAC/utils/tests/test_bids_utils.py +++ b/CPAC/utils/tests/test_bids_utils.py @@ -1,138 +1,145 @@ -"""Tests for bids_utils""" +"""Tests for bids_utils.""" import os + import pytest import yaml -from CPAC.utils.bids_utils import bids_gen_cpac_sublist, \ - cl_strip_brackets, \ - collect_bids_files_configs, \ - create_cpac_data_config, \ - load_cpac_data_config, \ - sub_list_filter_by_labels + +from CPAC.utils.bids_utils import ( + bids_gen_cpac_sublist, + cl_strip_brackets, + collect_bids_files_configs, + create_cpac_data_config, + load_cpac_data_config, + sub_list_filter_by_labels, +) def create_sample_bids_structure(root_dir): - """Function to create temporary synthetic BIDS data for testing parsing""" + """Function to create temporary synthetic BIDS data for testing parsing.""" + def _prefix_entities(paths, path): return f'sub-{paths[path]["sub"]}_ses-{paths[path]["ses"]}' - + suffixes = { - 'anat': ['T1w.nii.gz', 'acq-VNavNorm_T1w.nii.gz'], - 'fmap': [], - 'func': [ - f'task-{task}_run-{run}_bold.nii.gz' for - run in ['1', '2'] for task in ['rest'] - ] + "anat": ["T1w.nii.gz", "acq-VNavNorm_T1w.nii.gz"], + "fmap": [], + "func": [ + f"task-{task}_run-{run}_bold.nii.gz" + for run in ["1", "2"] + for task in ["rest"] + ], } paths = { os.path.join(root_dir, subpath): { - 'sub': sub, - 'ses': ses, - 'data_type': data_type - } for data_type in ['anat', 'fmap', 'func'] for - ses in ['1', '2'] for - sub in ['0001', '0002'] for subpath in [ - f'sub-{sub}/ses-{ses}/{data_type}' - ] + "sub": sub, + "ses": ses, + "data_type": data_type, + } + for data_type in ["anat", "fmap", "func"] + for ses in ["1", "2"] + for sub in ["0001", "0002"] + for subpath in [f"sub-{sub}/ses-{ses}/{data_type}"] } for path in paths: os.makedirs(path, exist_ok=True) - for suffix in suffixes[paths[path]['data_type']]: - open(os.path.join(path, '_'.join([ - _prefix_entities(paths, path), - suffix - ])), 'w') + for suffix in suffixes[paths[path]["data_type"]]: + open( + os.path.join(path, "_".join([_prefix_entities(paths, path), suffix])), + "w", + ) -@pytest.mark.parametrize('only_one_anat', [True, False]) +@pytest.mark.parametrize("only_one_anat", [True, False]) def test_create_cpac_data_config_only_one_anat(tmp_path, only_one_anat): """Function to test 'only_one_anat' parameter of - 'create_cpac_data_config' function""" + 'create_cpac_data_config' function. + """ create_sample_bids_structure(tmp_path) assert isinstance( - create_cpac_data_config( - str(tmp_path), only_one_anat=only_one_anat - )[0]['anat']['T1w'], str if only_one_anat else list) + create_cpac_data_config(str(tmp_path), only_one_anat=only_one_anat)[0]["anat"][ + "T1w" + ], + str if only_one_anat else list, + ) -@pytest.mark.skip(reason='needs local files not included in package') +@pytest.mark.skip(reason="needs local files not included in package") def test_gen_bids_sublist(bids_dir, test_yml, creds_path, dbg=False): - (img_files, config) = collect_bids_files_configs(bids_dir, creds_path) - print("Found %d config files for %d image files" % (len(config), - len(img_files))) - sublist = bids_gen_cpac_sublist(bids_dir, img_files, config, creds_path, - dbg) + sublist = bids_gen_cpac_sublist(bids_dir, img_files, config, creds_path, dbg) with open(test_yml, "w") as ofd: - yaml.dump(sublist, ofd, encoding='utf-8') + yaml.dump(sublist, ofd, encoding="utf-8") sublist = bids_gen_cpac_sublist(bids_dir, img_files, None, creds_path, dbg) test_yml = test_yml.replace(".yml", "_no_param.yml") with open(test_yml, "w") as ofd: - yaml.dump(sublist, ofd, encoding='utf-8') + yaml.dump(sublist, ofd, encoding="utf-8") assert sublist -@pytest.mark.parametrize('t1w_label', ['acq-HCP', 'acq-VNavNorm', 'T1w', None]) -@pytest.mark.parametrize('bold_label', [ - 'task-peer_run-1', - '[task-peer_run-1 task-peer_run-2]', - 'bold', - None]) -@pytest.mark.parametrize('participant_label', ['NDARAA504CRN', 'NDARAC462DZH', - None]) +@pytest.mark.parametrize("t1w_label", ["acq-HCP", "acq-VNavNorm", "T1w", None]) +@pytest.mark.parametrize( + "bold_label", ["task-peer_run-1", "[task-peer_run-1 task-peer_run-2]", "bold", None] +) +@pytest.mark.parametrize("participant_label", ["NDARAA504CRN", "NDARAC462DZH", None]) def test_sub_list_filter_by_labels(t1w_label, bold_label, participant_label): - """Tests for sub_list_filter_by_labels""" + """Tests for sub_list_filter_by_labels.""" if participant_label: participant_label = [ - 'sub-' + pt if not pt.startswith('sub-') else pt for - pt in participant_label + "sub-" + pt if not pt.startswith("sub-") else pt for pt in participant_label ] sub_list = load_cpac_data_config( - '/code/CPAC/pipeline/test/issue_1606_data_config.yml', - participant_label.split(' ') if - isinstance(participant_label, str) else None, - None) - bold_labels = bold_label.split(' ') if bold_label is not None else None + "/code/CPAC/pipeline/test/issue_1606_data_config.yml", + participant_label.split(" ") if isinstance(participant_label, str) else None, + None, + ) + bold_labels = bold_label.split(" ") if bold_label is not None else None sub_list = sub_list_filter_by_labels( - sub_list, { - 'T1w': t1w_label, - 'bold': bold_labels - } + sub_list, {"T1w": t1w_label, "bold": bold_labels} ) - print(sub_list) if t1w_label is not None: - if participant_label == 'NDARAA504CRN': - anat_sub_list = [sub.get('anat') for sub in sub_list] - assert any('T2w' in filepath for filepath in anat_sub_list) - if t1w_label != 'T1w': - assert 's3://fcp-indi/data/Projects/HBN/MRI/Site-CBIC/' \ - f'sub-NDARAA504CRN/anat/sub-NDARAA504CRN_{t1w_label}_' \ - 'T1w.nii.gz' in anat_sub_list + if participant_label == "NDARAA504CRN": + anat_sub_list = [sub.get("anat") for sub in sub_list] + assert any("T2w" in filepath for filepath in anat_sub_list) + if t1w_label != "T1w": + assert ( + "s3://fcp-indi/data/Projects/HBN/MRI/Site-CBIC/" + f"sub-NDARAA504CRN/anat/sub-NDARAA504CRN_{t1w_label}_" + "T1w.nii.gz" in anat_sub_list + ) else: - assert any('T1w' in filepath for filepath in anat_sub_list) + assert any("T1w" in filepath for filepath in anat_sub_list) else: assert 1 <= len(sub_list) <= 2 else: - assert 's3://fcp-indi/data/Projects/HBN/MRI/Site-CBIC/' \ - 'sub-NDARAA504CRN/anat/sub-NDARAA504CRN_acq-VNavNorm_' \ - 'T1w.nii.gz' not in [sub.get('anat') for sub in sub_list] + assert ( + "s3://fcp-indi/data/Projects/HBN/MRI/Site-CBIC/" + "sub-NDARAA504CRN/anat/sub-NDARAA504CRN_acq-VNavNorm_" + "T1w.nii.gz" not in [sub.get("anat") for sub in sub_list] + ) if bold_label is not None: - if participant_label == 'NDARAC462DZH': + if participant_label == "NDARAC462DZH": # all functional scans in data config - func_scans = [scan for scan in [ - sub.get('func').get(task, {}).get('scan') for task in [ - task for scan in [ - sub.get('func').keys() for sub in sub_list - ] for task in scan - ] for sub in sub_list - ] if scan] - if bold_label == 'bold': - assert any('bold' in filepath for filepath in func_scans) + func_scans = [ + scan + for scan in [ + sub.get("func").get(task, {}).get("scan") + for task in [ + task + for scan in [sub.get("func").keys() for sub in sub_list] + for task in scan + ] + for sub in sub_list + ] + if scan + ] + if bold_label == "bold": + assert any("bold" in filepath for filepath in func_scans) else: bold_labels = cl_strip_brackets(bold_labels) for label in bold_labels: @@ -141,8 +148,7 @@ def test_sub_list_filter_by_labels(t1w_label, bold_label, participant_label): assert 1 <= len(sub_list) <= 2 else: assert all( - len(sub.get('func')) in [0, len(bold_labels)] for - sub in sub_list + len(sub.get("func")) in [0, len(bold_labels)] for sub in sub_list ) else: - assert all(len(sub.get('func')) in [0, 5] for sub in sub_list) + assert all(len(sub.get("func")) in [0, 5] for sub in sub_list) diff --git a/CPAC/utils/tests/test_crash.py b/CPAC/utils/tests/test_crash.py index 9392fc3532..fefc8eb9cb 100644 --- a/CPAC/utils/tests/test_crash.py +++ b/CPAC/utils/tests/test_crash.py @@ -1,13 +1,17 @@ +from unittest import mock + import pytest from nipype.utils.filemanip import loadpkl -from unittest import mock -@pytest.mark.skip(reason='requires unincluded local file') +@pytest.mark.skip(reason="requires unincluded local file") def test_nipype_mock(): - def accept_all(object, name, value): return value - with mock.patch('nipype.interfaces.base.traits_extension.File.validate', side_effect=accept_all) as abc_urandom_function: - loadpkl('/home/anibalsolon/Downloads/crash-20190809-153710-victorpsanchez-nuisance_regression.b0.c0-d4597481-f3e5-43c2-a9b7-2b5e98c907d7.pklz') \ No newline at end of file + with mock.patch( + "nipype.interfaces.base.traits_extension.File.validate", side_effect=accept_all + ): + loadpkl( + "/home/anibalsolon/Downloads/crash-20190809-153710-victorpsanchez-nuisance_regression.b0.c0-d4597481-f3e5-43c2-a9b7-2b5e98c907d7.pklz" + ) diff --git a/CPAC/utils/tests/test_datasource.py b/CPAC/utils/tests/test_datasource.py index 5abbf29f33..9842310bb1 100644 --- a/CPAC/utils/tests/test_datasource.py +++ b/CPAC/utils/tests/test_datasource.py @@ -1,16 +1,15 @@ -import os import json -from CPAC.pipeline import nipype_pipeline_engine as pe -import nipype.interfaces.utility as util + import pytest +import nipype.interfaces.utility as util -from CPAC.utils.test_resources import setup_test_wf +from CPAC.pipeline import nipype_pipeline_engine as pe from CPAC.utils.datasource import match_epi_fmaps +from CPAC.utils.test_resources import setup_test_wf -@pytest.mark.skip(reason='needs refactoring') +@pytest.mark.skip(reason="needs refactoring") def test_match_epi_fmaps(): - # good data to use s3_prefix = "s3://fcp-indi/data/Projects/HBN/MRI/Site-CBIC/sub-NDARAB708LM5" s3_paths = [ @@ -18,11 +17,10 @@ def test_match_epi_fmaps(): "fmap/sub-NDARAB708LM5_dir-PA_acq-fMRI_epi.nii.gz", "fmap/sub-NDARAB708LM5_dir-PA_acq-fMRI_epi.json", "fmap/sub-NDARAB708LM5_dir-AP_acq-fMRI_epi.nii.gz", - "fmap/sub-NDARAB708LM5_dir-AP_acq-fMRI_epi.json" + "fmap/sub-NDARAB708LM5_dir-AP_acq-fMRI_epi.json", ] - wf, ds, local_paths = setup_test_wf(s3_prefix, s3_paths, - "test_match_epi_fmaps") + wf, ds, local_paths = setup_test_wf(s3_prefix, s3_paths, "test_match_epi_fmaps") opposite_pe_json = local_paths["fmap/sub-NDARAB708LM5_dir-PA_acq-fMRI_epi.json"] same_pe_json = local_paths["fmap/sub-NDARAB708LM5_dir-AP_acq-fMRI_epi.json"] @@ -38,22 +36,26 @@ def test_match_epi_fmaps(): func_params = json.load(f) bold_pedir = func_params["PhaseEncodingDirection"] - fmap_paths_dct = {"epi_PA": - {"scan": local_paths["fmap/sub-NDARAB708LM5_dir-PA_acq-fMRI_epi.nii.gz"], - "scan_parameters": opposite_pe_params}, - "epi_AP": - {"scan": local_paths["fmap/sub-NDARAB708LM5_dir-AP_acq-fMRI_epi.nii.gz"], - "scan_parameters": same_pe_params} - } + fmap_paths_dct = { + "epi_PA": { + "scan": local_paths["fmap/sub-NDARAB708LM5_dir-PA_acq-fMRI_epi.nii.gz"], + "scan_parameters": opposite_pe_params, + }, + "epi_AP": { + "scan": local_paths["fmap/sub-NDARAB708LM5_dir-AP_acq-fMRI_epi.nii.gz"], + "scan_parameters": same_pe_params, + }, + } - match_fmaps = \ - pe.Node(util.Function(input_names=['fmap_dct', - 'bold_pedir'], - output_names=['opposite_pe_epi', - 'same_pe_epi'], - function=match_epi_fmaps, - as_module=True), - name='match_epi_fmaps') + match_fmaps = pe.Node( + util.Function( + input_names=["fmap_dct", "bold_pedir"], + output_names=["opposite_pe_epi", "same_pe_epi"], + function=match_epi_fmaps, + as_module=True, + ), + name="match_epi_fmaps", + ) match_fmaps.inputs.fmap_dct = fmap_paths_dct match_fmaps.inputs.bold_pedir = bold_pedir @@ -61,7 +63,7 @@ def test_match_epi_fmaps(): ds.inputs.opposite_pe_json = opposite_pe_json ds.inputs.same_pe_json = same_pe_json - wf.connect(match_fmaps, 'opposite_pe_epi', ds, 'should_be_dir-PA') - wf.connect(match_fmaps, 'same_pe_epi', ds, 'should_be_dir-AP') + wf.connect(match_fmaps, "opposite_pe_epi", ds, "should_be_dir-PA") + wf.connect(match_fmaps, "same_pe_epi", ds, "should_be_dir-AP") wf.run() diff --git a/CPAC/utils/tests/test_function.py b/CPAC/utils/tests/test_function.py index aac14b4f17..13d59037a8 100644 --- a/CPAC/utils/tests/test_function.py +++ b/CPAC/utils/tests/test_function.py @@ -1,52 +1,45 @@ from CPAC.pipeline import nipype_pipeline_engine as pe -from CPAC.utils.interfaces.function import Function from CPAC.utils.datasource import get_rest - +from CPAC.utils.interfaces.function import Function rest_dict = { - 'rest_acq-1_run-1': { - 'scan': 's3://fcp-indi/sub-1019436_ses-1_task-rest_bold.nii.gz' + "rest_acq-1_run-1": { + "scan": "s3://fcp-indi/sub-1019436_ses-1_task-rest_bold.nii.gz" } } -resource = 'scan' -scan = 'rest_acq-1_run-1' +resource = "scan" +scan = "rest_acq-1_run-1" def test_function(): - - f = pe.Node(Function(input_names=['scan', - 'rest_dict', - 'resource'], - output_names=['file_path'], - function=get_rest, - as_module=True), - name='get_rest') - - f.inputs.set( - resource=resource, - rest_dict=rest_dict, - scan=scan + f = pe.Node( + Function( + input_names=["scan", "rest_dict", "resource"], + output_names=["file_path"], + function=get_rest, + as_module=True, + ), + name="get_rest", ) + f.inputs.set(resource=resource, rest_dict=rest_dict, scan=scan) + results = f.run() - assert rest_dict['rest_acq-1_run-1']['scan'] == results.outputs.file_path + assert rest_dict["rest_acq-1_run-1"]["scan"] == results.outputs.file_path def test_function_str(): - - f = pe.Node(Function(input_names=['scan', - 'rest_dict', - 'resource'], - output_names=['file_path'], - function=get_rest), - name='get_rest') - - f.inputs.set( - resource=resource, - rest_dict=rest_dict, - scan=scan + f = pe.Node( + Function( + input_names=["scan", "rest_dict", "resource"], + output_names=["file_path"], + function=get_rest, + ), + name="get_rest", ) + f.inputs.set(resource=resource, rest_dict=rest_dict, scan=scan) + results = f.run() - assert rest_dict['rest_acq-1_run-1']['scan'] == results.outputs.file_path + assert rest_dict["rest_acq-1_run-1"]["scan"] == results.outputs.file_path diff --git a/CPAC/utils/tests/test_s3.py b/CPAC/utils/tests/test_s3.py index 74b77a8b14..1c7b0a9233 100644 --- a/CPAC/utils/tests/test_s3.py +++ b/CPAC/utils/tests/test_s3.py @@ -1,38 +1,37 @@ - def test_check_s3(): - import os + from CPAC.utils.datasource import check_for_s3 data = check_for_s3( - file_path='s3://fcp-indi/resources/cpac/resources/rois_2mm.nii.gz', + file_path="s3://fcp-indi/resources/cpac/resources/rois_2mm.nii.gz", creds_path=None, - dl_dir='/tmp', - img_type='anat' + dl_dir="/tmp", + img_type="anat", ) assert os.path.isfile(data) def test_check_s3_node(): - import os + from CPAC.utils.datasource import create_check_for_s3_node node = create_check_for_s3_node( - 'image', - file_path='s3://fcp-indi/resources/cpac/resources/mask-thr50-3mm.nii.gz', + "image", + file_path="s3://fcp-indi/resources/cpac/resources/mask-thr50-3mm.nii.gz", creds_path=None, - dl_dir='/tmp', - img_type='anat' + dl_dir="/tmp", + img_type="anat", ) res = node.run() assert os.path.isfile(res.outputs.local_path) node = create_check_for_s3_node( - 'image', - file_path='s3://fcp-indi/resources/cpac/resources/mask-thr50-3mm.nii.gz' + "image", + file_path="s3://fcp-indi/resources/cpac/resources/mask-thr50-3mm.nii.gz", ) res = node.run() diff --git a/CPAC/utils/tests/test_symlinks.py b/CPAC/utils/tests/test_symlinks.py index 19ffa347e4..72def9a2c8 100644 --- a/CPAC/utils/tests/test_symlinks.py +++ b/CPAC/utils/tests/test_symlinks.py @@ -1,42 +1,31 @@ import os import tempfile + import pkg_resources as p from CPAC.utils.symlinks import create_symlinks - -mocked_outputs = \ - p.resource_filename( - "CPAC", - os.path.join( - 'utils', - 'tests', - 'test_symlinks-outputs.txt' - ) - ) +mocked_outputs = p.resource_filename( + "CPAC", os.path.join("utils", "tests", "test_symlinks-outputs.txt") +) def test_symlinks(): - - temp_dir = tempfile.mkdtemp(suffix='test_symlinks') + temp_dir = tempfile.mkdtemp(suffix="test_symlinks") paths = [] - with open(mocked_outputs, 'r') as f: + with open(mocked_outputs, "r") as f: for path in f.readlines(): path = path.strip() if path: paths += [path] create_symlinks( - temp_dir, - 'sym_links', - 'pipeline_benchmark-FNIRT', '1019436_1', paths + temp_dir, "sym_links", "pipeline_benchmark-FNIRT", "1019436_1", paths ) - print("Links created at", temp_dir) - # TODO test the generated links # Normal resource case # Several resources within same key case - # QC case \ No newline at end of file + # QC case diff --git a/CPAC/utils/tests/test_trimmer.py b/CPAC/utils/tests/test_trimmer.py index 199189bb5e..1d1f7361f7 100644 --- a/CPAC/utils/tests/test_trimmer.py +++ b/CPAC/utils/tests/test_trimmer.py @@ -1,99 +1,81 @@ -import pytest -import shutil +from copy import copy import tempfile + +import pytest import yaml -from copy import copy -from unittest import mock def accept_all(object, name, value): return value -@pytest.mark.skip(reason='needs refactored') +@pytest.mark.skip(reason="needs refactored") def test_trimmer(): + import os + + import pkg_resources as p - from CPAC.utils.trimmer import the_trimmer, is_datasink, expand_workflow, compute_datasink_dirs from CPAC.pipeline.cpac_pipeline import build_workflow from CPAC.utils.configuration import Configuration - - import os - import pkg_resources as p - - pipe_config = \ - p.resource_filename( - "CPAC", - os.path.join( - "resources", - "configs", - "pipeline_config_template.yml" - ) - ) - - data_config = \ - p.resource_filename( - "CPAC", - os.path.join( - "resources", - "configs", - "data_config_S3-BIDS-ABIDE.yml" - ) - ) - - data_config = yaml.safe_load(open(data_config, 'r')) + from CPAC.utils.trimmer import ( + compute_datasink_dirs, + expand_workflow, + is_datasink, + the_trimmer, + ) + + pipe_config = p.resource_filename( + "CPAC", os.path.join("resources", "configs", "pipeline_config_template.yml") + ) + + data_config = p.resource_filename( + "CPAC", os.path.join("resources", "configs", "data_config_S3-BIDS-ABIDE.yml") + ) + + data_config = yaml.safe_load(open(data_config, "r")) sub_dict = data_config[0] - - c = Configuration(yaml.safe_load(open(pipe_config, 'r'))) + + c = Configuration(yaml.safe_load(open(pipe_config, "r"))) temp_dir = tempfile.mkdtemp() c.logDirectory = temp_dir c.workingDirectory = temp_dir c.outputDirectory = temp_dir c.crashLogDirectory = temp_dir - - # Disable functional, let only the anatomical workflow c_anatomical = copy(c) c_anatomical.runFunctional = [0] - wf, _, _ = build_workflow(sub_dict['subject_id'], sub_dict, c_anatomical) + wf, _, _ = build_workflow(sub_dict["subject_id"], sub_dict, c_anatomical) # Create fake files to trick THE TRIMMER exec_graph = expand_workflow(wf) - datasinks = [ - n for n in exec_graph.nodes() - if is_datasink(n) - ] + datasinks = [n for n in exec_graph.nodes() if is_datasink(n)] anat_derivatives = {} for datasink in datasinks: paths = compute_datasink_dirs(exec_graph, datasink) anat_derivatives.update(paths) for (node, derivative), path in paths.items(): os.makedirs(path) - open(os.path.join(path, '%s.txt' % derivative), 'a').close() - - + open(os.path.join(path, "%s.txt" % derivative), "a").close() # Enable functional, so the workflow should only run this # and enable trimming c_functional = copy(c) c_functional.runFunctional = [1] - wf, _, _ = build_workflow(sub_dict['subject_id'], sub_dict, c_functional) + wf, _, _ = build_workflow(sub_dict["subject_id"], sub_dict, c_functional) exec_wf, _ = the_trimmer(wf) exec_graph = exec_wf._graph - datasinks = [ - n for n in exec_graph.nodes() - if is_datasink(n) - ] + datasinks = [n for n in exec_graph.nodes() if is_datasink(n)] func_derivatives = {} for datasink in datasinks: paths = compute_datasink_dirs(exec_graph, datasink) func_derivatives.update(paths) - - # Assert that the functional pipeline remove all the anatomical nodes, # as they were already computed - assert set(func_derivatives.keys()).intersection(set(anat_derivatives.keys())) == set() + assert ( + set(func_derivatives.keys()).intersection(set(anat_derivatives.keys())) == set() + ) diff --git a/CPAC/utils/tests/test_utils.py b/CPAC/utils/tests/test_utils.py index 4cb6440e94..21032c9736 100644 --- a/CPAC/utils/tests/test_utils.py +++ b/CPAC/utils/tests/test_utils.py @@ -1,25 +1,30 @@ -"""Tests of CPAC utility functions""" +"""Tests of CPAC utility functions.""" import multiprocessing from unittest import mock + import pytest + from CPAC.func_preproc import get_motion_ref from CPAC.pipeline.nodeblock import NodeBlockFunction from CPAC.utils.configuration import Configuration from CPAC.utils.monitoring.custom_logging import log_subprocess -from CPAC.utils.utils import check_config_resources, check_system_deps, \ - try_fetch_parameter +from CPAC.utils.utils import ( + check_config_resources, + check_system_deps, + try_fetch_parameter, +) scan_params_bids = { - 'RepetitionTime': 2.0, - 'ScanOptions': 'FS', - 'SliceAcquisitionOrder': 'Interleaved Ascending', + "RepetitionTime": 2.0, + "ScanOptions": "FS", + "SliceAcquisitionOrder": "Interleaved Ascending", } scan_params_cpac = { - 'tr': 2.5, - 'acquisition': 'seq+z', - 'reference': '24', - 'first_tr': '', - 'last_tr': '', + "tr": 2.5, + "acquisition": "seq+z", + "reference": "24", + "first_tr": "", + "last_tr": "", } @@ -50,40 +55,42 @@ def _installation_check(command: str, flag: str) -> None: def test_check_config_resources(): """Test check_config_resources function.""" - with mock.patch.object(multiprocessing, 'cpu_count', return_value=2), \ - pytest.raises(SystemError) as system_error: - check_config_resources(Configuration({'pipeline_setup': { - 'system_config': {'max_cores_per_participant': 10}}})) + with mock.patch.object(multiprocessing, "cpu_count", return_value=2), pytest.raises( + SystemError + ) as system_error: + check_config_resources( + Configuration( + {"pipeline_setup": {"system_config": {"max_cores_per_participant": 10}}} + ) + ) error_string = str(system_error.value) - assert 'threads running in parallel (10)' in error_string - assert 'threads available (2)' in error_string + assert "threads running in parallel (10)" in error_string + assert "threads available (2)" in error_string def test_function(): - TR = try_fetch_parameter(scan_params_bids, '0001', 'scan', - ['TR', 'RepetitionTime']) + TR = try_fetch_parameter(scan_params_bids, "0001", "scan", ["TR", "RepetitionTime"]) assert TR == 2.0 - TR = try_fetch_parameter(scan_params_cpac, '0001', 'scan', - ['TR', 'RepetitionTime']) + TR = try_fetch_parameter(scan_params_cpac, "0001", "scan", ["TR", "RepetitionTime"]) assert TR == 2.5 @pytest.mark.parametrize("executable", ["Xvfb"]) def test_executable(executable): - """Make sure executable is installed""" + """Make sure executable is installed.""" _installation_check(executable, "-help") def test_NodeBlock_option_SSOT(): # pylint: disable=invalid-name - '''Test using NodeBlock dictionaries for SSOT for options''' + """Test using NodeBlock dictionaries for SSOT for options.""" assert isinstance(get_motion_ref, NodeBlockFunction) with pytest.raises(ValueError) as value_error: - get_motion_ref(None, None, None, None, opt='chaos') + get_motion_ref(None, None, None, None, opt="chaos") error_message = str(value_error.value).rstrip() for opt in get_motion_ref.option_val: assert f"'{opt}'" in error_message - assert error_message.endswith('Tool input: \'chaos\'') + assert error_message.endswith("Tool input: 'chaos'") def test_system_deps(): diff --git a/CPAC/utils/tests/test_yaml.py b/CPAC/utils/tests/test_yaml.py index df6112640b..60e11441c4 100644 --- a/CPAC/utils/tests/test_yaml.py +++ b/CPAC/utils/tests/test_yaml.py @@ -1,4 +1,4 @@ -'''Tests for C-PAC YAML +"""Tests for C-PAC YAML. Copyright (C) 2022 C-PAC Developers @@ -15,66 +15,70 @@ License for more details. You should have received a copy of the GNU Lesser General Public -License along with C-PAC. If not, see .''' +License along with C-PAC. If not, see . +""" import os import tempfile -import pytest +import pytest import yaml -from CPAC.utils.configuration import Configuration, Preconfiguration, \ - preconfig_yaml +from CPAC.utils.configuration import Configuration, Preconfiguration, preconfig_yaml from CPAC.utils.configuration.yaml_template import create_yaml_from_template from .configs import NEUROSTARS_23786, NEUROSTARS_24035 -@pytest.mark.parametrize('config', ['blank', 'default', NEUROSTARS_23786, - NEUROSTARS_24035]) +@pytest.mark.parametrize( + "config", ["blank", "default", NEUROSTARS_23786, NEUROSTARS_24035] +) def test_sca_config(config): - '''Test SCA config parsing''' - if '\n' in config: + """Test SCA config parsing.""" + if "\n" in config: participant_config = Configuration(yaml.safe_load(config)) - assert participant_config['seed_based_correlation_analysis', - 'run'] is True + assert participant_config["seed_based_correlation_analysis", "run"] is True else: participant_config = Preconfiguration(config) - assert participant_config['seed_based_correlation_analysis', - 'run'] is False + assert participant_config["seed_based_correlation_analysis", "run"] is False def test_yaml_template(): - '''Test YAML pipeline template''' + """Test YAML pipeline template.""" # Create temporary file - config_file = tempfile.mkstemp(suffix='test_yaml_template')[1] + config_file = tempfile.mkstemp(suffix="test_yaml_template")[1] # Create a new YAML configuration file based on the default pipeline # YAML file. - pipeline_file = preconfig_yaml('default') - config = preconfig_yaml('default', load=True) + pipeline_file = preconfig_yaml("default") + config = preconfig_yaml("default", load=True) new_config = create_yaml_from_template(config, pipeline_file) - with open(config_file, 'w', encoding='utf-8') as f: + with open(config_file, "w", encoding="utf-8") as f: f.write(new_config) # Verify that the output has preserved blank lines, comments - with open(config_file, 'r', encoding='utf-8') as f: + with open(config_file, "r", encoding="utf-8") as f: lines = f.readlines() # Delete temporary file os.remove(config_file) # Assert first line declares YAML version - assert lines[0].startswith('%YAML') - assert lines[1] == '---\n' + assert lines[0].startswith("%YAML") + assert lines[1] == "---\n" # Assert first lines starts with a comment - assert lines[2].startswith('# ') + assert lines[2].startswith("# ") # Assert that regressors configuration written in block style - assert (line for line in lines if line.strip() == 'Regressors:') - assert (line for line in lines if line.strip() == '- Bandpass:') + assert (line for line in lines if line.strip() == "Regressors:") + assert (line for line in lines if line.strip() == "- Bandpass:") # Assert that short lists of values are written in flow style # (e.g., "GM_label: [3, 42]") - assert (line for line in lines if - '[' in line and ',' in line and ']' in line and - not line.lstrip().startswith('#')) + assert ( + line + for line in lines + if "[" in line + and "," in line + and "]" in line + and not line.lstrip().startswith("#") + ) diff --git a/CPAC/utils/trimmer.py b/CPAC/utils/trimmer.py index 01f76c966d..82863ac09a 100644 --- a/CPAC/utils/trimmer.py +++ b/CPAC/utils/trimmer.py @@ -1,29 +1,30 @@ # -*- coding: utf-8 -*- -import glob from copy import deepcopy -from CPAC.pipeline import nipype_pipeline_engine as pe -from nipype.interfaces.utility import Function -from nipype.pipeline.engine.utils import generate_expanded_graph -import networkx as nx +import glob +import networkx as nx +from nipype.pipeline.engine.utils import generate_expanded_graph from indi_aws import fetch_creds from CPAC.utils.datasource import ( create_check_for_s3_node, ) + def expand_workflow(wf): return generate_expanded_graph(deepcopy(wf._create_flat_graph())) + def is_datasink(n): - return type(n).__name__ == 'Node' and type(n.interface).__name__ == 'DataSink' + return type(n).__name__ == "Node" and type(n.interface).__name__ == "DataSink" + def compute_datasink_dirs(graph, datasink, output_dir=None, container=None): directories = {} for inp in graph.in_edges(datasink): src, _ = inp - for edge in graph.get_edge_data(*inp)['connect']: + for edge in graph.get_edge_data(*inp)["connect"]: _, derivative_name = edge datasink_output_dir = datasink.interface.inputs.base_directory @@ -37,25 +38,27 @@ def compute_datasink_dirs(graph, datasink, output_dir=None, container=None): # Look if there is an output in this datasink directory iterables = datasink.parameterization - path = '/'.join(['', derivative_name] + iterables) + path = "/".join(["", derivative_name, *iterables]) path = datasink.interface._substitute(path)[1:] - path = '/'.join([datasink_output_dir, datasink_container, path]) + path = "/".join([datasink_output_dir, datasink_container, path]) directories[(src, derivative_name)] = path return directories + def list_files(path, s3_creds_path=None): - if path.startswith('s3://'): - pieces = path[5:].split('/') - bucket_name, path = pieces[0], '/'.join(pieces[1:]) + if path.startswith("s3://"): + pieces = path[5:].split("/") + bucket_name, path = pieces[0], "/".join(pieces[1:]) bucket = fetch_creds.return_bucket(s3_creds_path, bucket_name) return [ - 's3://%s/%s' % (bucket, obj['Key']) + "s3://%s/%s" % (bucket, obj["Key"]) for obj in bucket.objects.filter(Prefix=path) ] else: - return list(glob.glob(path + '/*')) + return list(glob.glob(path + "/*")) + def the_trimmer(wf, output_dir=None, container=None, s3_creds_path=None): """ @@ -100,7 +103,7 @@ def the_trimmer(wf, output_dir=None, container=None, s3_creds_path=None): another branch, for [node4], that is not cached. After trimming, the remaining workflow is: - [node1] + [node1] ↳ [node4] → [datasink to file2.txt ❌] 2) The node has several outputs, and an uncached branch down the @@ -115,7 +118,7 @@ def the_trimmer(wf, output_dir=None, container=None, s3_creds_path=None): is cached, we will reexecute the [registration] again to get the transforms. After trimming, the remaining workflow is: - [registration] + [registration] ↳(transforms)→ [apply transforms] → [datasink to func_warped.nii.gz ❌] [functional] ↗ @@ -137,57 +140,49 @@ def the_trimmer(wf, output_dir=None, container=None, s3_creds_path=None): container : Path The subdirectory from the output_dir in which the output are stored. If not provided, value is inferred from the DataSink nodes. - + s3_creds_path : Path Path to S3 credentials, in case output_dir is in a S3 bucket. - + Returns ------- wf_new : Workflow Prunned workflow (replacement_mapping, deletions): (Dict, List) - + replacement_mapping contains the nodes replaces with input nodes, pointing to files from the output_dir deletions contains the nodes removed from the workflow, as they do not need to be executed - - """ + """ # Expand graph, to flatten out sub-workflows and iterables execgraph = expand_workflow(wf) replacements = {} deletions = [] - + # Check out for datasinks (i.e. the ones who throws things at the output dir) - datasinks = [ - n for n in execgraph.nodes() - if is_datasink(n) - ] + datasinks = [n for n in execgraph.nodes() if is_datasink(n)] for datasink in datasinks: - - for (src, derivative_name), path in \ - compute_datasink_dirs(execgraph, - datasink, - output_dir=output_dir, - container=container).items(): - + for (src, derivative_name), path in compute_datasink_dirs( + execgraph, datasink, output_dir=output_dir, container=container + ).items(): files = list_files(path, s3_creds_path=s3_creds_path) if len(files) == 1: # Ignore multi-file nodes if src not in replacements: replacements[src] = {} replacements[src][src_field] = files[0] - - # if the replacements have all the fields from the datasink, datasink + + # if the replacements have all the fields from the datasink, datasink # can be deleted (we do not want to output again the same file :)) if all( any( field in replacements.get(src, {}) - for field, _ in execgraph.get_edge_data(src, dst)['connect'] + for field, _ in execgraph.get_edge_data(src, dst)["connect"] ) for src, dst in execgraph.in_edges(datasink) ): @@ -199,11 +194,11 @@ def the_trimmer(wf, output_dir=None, container=None, s3_creds_path=None): for edge in execgraph.out_edges(node): if any( src_field not in cached_fields - for src_field, _ in execgraph.get_edge_data(*edge)['connect'] + for src_field, _ in execgraph.get_edge_data(*edge)["connect"] ): del replacements[node] break - + # Delete them! It also removes the edges, and recursively delete nodes # before rationalizing about replacements for node in reversed(nx.topological_sort(execgraph)): @@ -211,53 +206,51 @@ def the_trimmer(wf, output_dir=None, container=None, s3_creds_path=None): execgraph.remove_node(node) if is_datasink(node): continue - + if len(execgraph.out_edges(node)) == 0: deletions += [node] if node in replacements: del replacements[node] execgraph.remove_node(node) - - # And now we replace the cached with a data input node, from the + + # And now we replace the cached with a data input node, from the # output directory. replacement_mapping = {} for replacement, cached_files in replacements.items(): out_edges_data = execgraph.edge[replacement] - + # Get this cached node, and replace all the out-connections # from this node with a data input node out_edges = execgraph.successors(replacement) if out_edges: for to_node in out_edges: - for from_field, to_field in out_edges_data[to_node]['connect']: - + for from_field, to_field in out_edges_data[to_node]["connect"]: # Reuse the data input node for this field if replacement not in replacement_mapping: replacement_mapping[replacement] = {} if from_field not in replacement_mapping[replacement]: - new_node = create_check_for_s3_node( - name='%s_%s_triminput' % (replacement.name, from_field), + name="%s_%s_triminput" % (replacement.name, from_field), file_path=cached_files[from_field], - img_type='other', + img_type="other", creds_path=s3_creds_path, - dl_dir=None + dl_dir=None, ) new_node._hierarchy = deepcopy(replacement._hierarchy) execgraph.add_node(new_node) replacement_mapping[replacement][from_field] = new_node - + # Connect the new data input node to the node # it was connected execgraph.add_edge( replacement_mapping[replacement][from_field], to_node, - connect=[('local_path', to_field)] + connect=[("local_path", to_field)], ) execgraph.remove_node(replacement) - + # Second round of backtrack deletion, affected by replacements for node in reversed(nx.topological_sort(execgraph)): if is_datasink(node): @@ -268,9 +261,9 @@ def the_trimmer(wf, output_dir=None, container=None, s3_creds_path=None): if node in replacements: del replacements[node] execgraph.remove_node(node) - - wf_new = wf.clone(wf.name + '_trimmed') + + wf_new = wf.clone(wf.name + "_trimmed") wf_new.name = wf.name wf_new._graph = execgraph - + return wf_new, (replacement_mapping, deletions) diff --git a/CPAC/utils/typing.py b/CPAC/utils/typing.py index a838a7c76b..171ed22d04 100644 --- a/CPAC/utils/typing.py +++ b/CPAC/utils/typing.py @@ -15,7 +15,7 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . """ -Helpers and aliases for handling typing in main and variant Python versions +Helpers and aliases for handling typing in main and variant Python versions. Once all variants (see {DOCS_URL_PREFIX}/user/versions#variants) run Python ≥ 3.10, these global variables can be replaced with the @@ -23,31 +23,36 @@ """ import sys from typing import Union + from CPAC.utils.docs import DOCS_URL_PREFIX # Set the version-specific documentation URL in the module docstring: -__doc__ = __doc__.replace(r'{DOCS_URL_PREFIX}', DOCS_URL_PREFIX) +__doc__ = __doc__.replace(r"{DOCS_URL_PREFIX}", DOCS_URL_PREFIX) if sys.version_info >= (3, 8): from typing import Literal + LITERAL = Literal else: from typing_extensions import Literal + LITERAL = Literal if sys.version_info >= (3, 9): from collections.abc import Iterable + LIST = list else: from typing import Iterable, List + LIST = List if sys.version_info >= (3, 10): LIST_OR_STR = LIST[str] | str # pylint: disable=invalid-name TUPLE = tuple else: from typing import Tuple + LIST_OR_STR = Union[LIST[str], str] # pylint: disable=invalid-name TUPLE = Tuple ITERABLE = Iterable ConfigKeyType = Union[str, LIST[str]] -__all__ = ['ConfigKeyType', 'ITERABLE', 'LIST', 'LIST_OR_STR', 'LITERAL', - 'TUPLE'] +__all__ = ["ConfigKeyType", "ITERABLE", "LIST", "LIST_OR_STR", "LITERAL", "TUPLE"] diff --git a/CPAC/utils/utils.py b/CPAC/utils/utils.py index 2144ea3f83..911c51e55c 100644 --- a/CPAC/utils/utils.py +++ b/CPAC/utils/utils.py @@ -14,31 +14,35 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -import os -import sys import collections.abc +from copy import deepcopy import fnmatch import gzip +from itertools import repeat import json import numbers +import os import pickle -import numpy as np -import yaml -from copy import deepcopy -from itertools import repeat +import numpy as np from voluptuous.error import Invalid +import yaml -CONFIGS_DIR = os.path.abspath(os.path.join( - __file__, *repeat(os.path.pardir, 2), 'resources/configs/')) -with open(os.path.join(CONFIGS_DIR, '1.7-1.8-nesting-mappings.yml'), 'r', - encoding='utf-8') as _f: +CONFIGS_DIR = os.path.abspath( + os.path.join(__file__, *repeat(os.path.pardir, 2), "resources/configs/") +) +with open( + os.path.join(CONFIGS_DIR, "1.7-1.8-nesting-mappings.yml"), "r", encoding="utf-8" +) as _f: NESTED_CONFIG_MAPPING = yaml.safe_load(_f) -with open(os.path.join(CONFIGS_DIR, '1.7-1.8-deprecations.yml'), 'r', - encoding='utf-8') as _f: +with open( + os.path.join(CONFIGS_DIR, "1.7-1.8-deprecations.yml"), "r", encoding="utf-8" +) as _f: NESTED_CONFIG_DEPRECATIONS = yaml.safe_load(_f) -YAML_BOOLS = {True: ('on', 't', 'true', 'y', 'yes'), - False: ('f', 'false', 'n', 'no', 'off')} +YAML_BOOLS = { + True: ("on", "t", "true", "y", "yes"), + False: ("f", "false", "n", "no", "off"), +} def get_last_prov_entry(prov): @@ -49,49 +53,47 @@ def get_last_prov_entry(prov): def check_prov_for_regtool(prov): last_entry = get_last_prov_entry(prov) - last_node = last_entry.split(':')[1] - if 'ants' in last_node.lower(): - return 'ants' - elif 'fsl' in last_node.lower(): - return 'fsl' + last_node = last_entry.split(":")[1] + if "ants" in last_node.lower(): + return "ants" + elif "fsl" in last_node.lower(): + return "fsl" else: # go further back in case we're checking against a concatenated # downstream xfm like bold-to-template (and prov is the provenance of # that downstream xfm) - if 'from-T1w_to-template_mode-image_xfm:' in str(prov): - splitprov = str(prov).split( - 'from-T1w_to-template_mode-image_xfm:') + if "from-T1w_to-template_mode-image_xfm:" in str(prov): + splitprov = str(prov).split("from-T1w_to-template_mode-image_xfm:") node_name = splitprov[1].split("']")[0] - if 'ANTs' in node_name: - return 'ants' - elif 'FSL' in node_name: - return 'fsl' - elif 'from-bold_to-template_mode-image_xfm:' in str(prov): - splitprov = str(prov).split( - 'from-bold_to-template_mode-image_xfm:') + if "ANTs" in node_name: + return "ants" + elif "FSL" in node_name: + return "fsl" + return None + elif "from-bold_to-template_mode-image_xfm:" in str(prov): + splitprov = str(prov).split("from-bold_to-template_mode-image_xfm:") node_name = splitprov[1].split("']")[0] - if 'ANTs' in node_name: - return 'ants' - elif 'FSL' in node_name: - return 'fsl' + if "ANTs" in node_name: + return "ants" + elif "FSL" in node_name: + return "fsl" else: return None - elif 'from-T1w_to-symtemplate_mode-image_xfm:' in str(prov): - splitprov = str(prov).split( - 'from-T1w_to-symtemplate_mode-image_xfm:') + elif "from-T1w_to-symtemplate_mode-image_xfm:" in str(prov): + splitprov = str(prov).split("from-T1w_to-symtemplate_mode-image_xfm:") node_name = splitprov[1].split("']")[0] - if 'ANTs' in node_name: - return 'ants' - elif 'FSL' in node_name: - return 'fsl' - elif 'from-bold_to-symtemplate_mode-image_xfm:' in str(prov): - splitprov = str(prov).split( - 'from-bold_to-symtemplate_mode-image_xfm:') + if "ANTs" in node_name: + return "ants" + elif "FSL" in node_name: + return "fsl" + return None + elif "from-bold_to-symtemplate_mode-image_xfm:" in str(prov): + splitprov = str(prov).split("from-bold_to-symtemplate_mode-image_xfm:") node_name = splitprov[1].split("']")[0] - if 'ANTs' in node_name: - return 'ants' - elif 'FSL' in node_name: - return 'fsl' + if "ANTs" in node_name: + return "ants" + elif "FSL" in node_name: + return "fsl" else: return None else: @@ -100,56 +102,60 @@ def check_prov_for_regtool(prov): def check_prov_for_motion_tool(prov): last_entry = get_last_prov_entry(prov) - last_node = last_entry.split(':')[1] - if '3dvolreg' in last_node.lower(): - return '3dvolreg' - elif 'mcflirt' in last_node.lower(): - return 'mcflirt' + last_node = last_entry.split(":")[1] + if "3dvolreg" in last_node.lower(): + return "3dvolreg" + elif "mcflirt" in last_node.lower(): + return "mcflirt" else: # check entire prov - if '3dvolreg' in str(prov): - return '3dvolreg' - elif 'mcflirt' in str(prov): - return 'mcflirt' + if "3dvolreg" in str(prov): + return "3dvolreg" + elif "mcflirt" in str(prov): + return "mcflirt" else: return None - - - def get_flag(in_flag): return in_flag -def get_flag_wf(wf_name='get_flag'): - from CPAC.pipeline import nipype_pipeline_engine as pe +def get_flag_wf(wf_name="get_flag"): import nipype.interfaces.utility as util + from CPAC.pipeline import nipype_pipeline_engine as pe + wf = pe.Workflow(name=wf_name) - input_node = pe.Node(util.IdentityInterface(fields=['in_flag']), - name='inputspec') + input_node = pe.Node(util.IdentityInterface(fields=["in_flag"]), name="inputspec") - get_flag = pe.Node(util.Function(input_names=['in_flag'], - function=get_flag), - name='get_flag') + get_flag = pe.Node( + util.Function(input_names=["in_flag"], function=get_flag), name="get_flag" + ) - wf.connect(input_node, 'in_flag', get_flag, 'in_flag') + wf.connect(input_node, "in_flag", get_flag, "in_flag") def read_json(json_file): try: - with open(json_file, 'r') as f: + with open(json_file, "r") as f: json_dct = json.load(f) except json.decoder.JSONDecodeError as err: - raise Exception(f'\n\n{err}\n\nJSON file: {json_file}\n') + raise Exception(f"\n\n{err}\n\nJSON file: {json_file}\n") return json_dct -def create_id_string(cfg, unique_id, resource, scan_id=None, - template_desc=None, atlas_id=None, fwhm=None, - subdir=None): +def create_id_string( + cfg, + unique_id, + resource, + scan_id=None, + template_desc=None, + atlas_id=None, + fwhm=None, + subdir=None, +): """Create the unique key-value identifier string for BIDS-Derivatives compliant file names. @@ -165,68 +171,67 @@ def create_id_string(cfg, unique_id, resource, scan_id=None, 'sub-1_ses-1_task-rest_atlas-Yeo7_res-3mm_desc-Mean1_timeseries' """ import re - from CPAC.utils.bids_utils import combine_multiple_entity_instances, \ - res_in_filename + + from CPAC.utils.bids_utils import combine_multiple_entity_instances, res_in_filename + if atlas_id: - if '_desc-' in atlas_id: - atlas, desc = atlas_id.split('_desc-') - if not re.match(r'.*[0-9]$', atlas) and re.match(r'[a-z].*', desc): - atlas_id = f'{atlas}{desc[0].upper()}{desc[1:]}' + if "_desc-" in atlas_id: + atlas, desc = atlas_id.split("_desc-") + if not re.match(r".*[0-9]$", atlas) and re.match(r"[a-z].*", desc): + atlas_id = f"{atlas}{desc[0].upper()}{desc[1:]}" else: - atlas_id = atlas_id.replace('_desc-', '') - resource = f'atlas-{atlas_id}_{resource}' - - part_id = unique_id.split('_')[0] - ses_id = unique_id.split('_')[1] - if 'sub-' not in part_id: - part_id = f'sub-{part_id}' - if 'ses-' not in ses_id: - ses_id = f'ses-{ses_id}' + atlas_id = atlas_id.replace("_desc-", "") + resource = f"atlas-{atlas_id}_{resource}" + + part_id = unique_id.split("_")[0] + ses_id = unique_id.split("_")[1] + if "sub-" not in part_id: + part_id = f"sub-{part_id}" + if "ses-" not in ses_id: + ses_id = f"ses-{ses_id}" if scan_id: - out_filename = f'{part_id}_{ses_id}_task-{scan_id}_{resource}' + out_filename = f"{part_id}_{ses_id}_task-{scan_id}_{resource}" else: - out_filename = f'{part_id}_{ses_id}_{resource}' + out_filename = f"{part_id}_{ses_id}_{resource}" - template_tag = template_desc.split(' -')[0] if template_desc else '*' - for prefix in ['space-', 'from-', 'to-']: - for bidstag in out_filename.split('_'): - if prefix in bidstag and 'template' in bidstag: - out_filename = out_filename.replace( - bidstag, f'{prefix}{template_tag}') + template_tag = template_desc.split(" -")[0] if template_desc else "*" + for prefix in ["space-", "from-", "to-"]: + for bidstag in out_filename.split("_"): + if prefix in bidstag and "template" in bidstag: + out_filename = out_filename.replace(bidstag, f"{prefix}{template_tag}") if fwhm: - for tag in resource.split('_'): - if 'desc-' in tag and '-sm' in tag: - newtag = tag.replace('-sm', f'-sm{fwhm}') + for tag in resource.split("_"): + if "desc-" in tag and "-sm" in tag: + newtag = tag.replace("-sm", f"-sm{fwhm}") out_filename = out_filename.replace(tag, newtag) break else: - raise Exception('\n[!] FWHM provided but no desc-sm?\n') + raise Exception("\n[!] FWHM provided but no desc-sm?\n") # drop space- entities from from native-space filenames - if subdir == 'anat': - out_filename = out_filename.replace('_space-T1w_', '_') - if subdir == 'func': - out_filename = out_filename.replace('_space-bold_', '_') - return combine_multiple_entity_instances( - res_in_filename(cfg, out_filename)) + if subdir == "anat": + out_filename = out_filename.replace("_space-T1w_", "_") + if subdir == "func": + out_filename = out_filename.replace("_space-bold_", "_") + return combine_multiple_entity_instances(res_in_filename(cfg, out_filename)) def write_output_json(json_data, filename, indent=3, basedir=None): if not basedir: basedir = os.getcwd() - if '.json' not in filename: - filename = f'{filename}.json' + if ".json" not in filename: + filename = f"{filename}.json" json_file = os.path.join(basedir, filename) json_data = json.dumps(json_data, indent=indent, sort_keys=True) - with open(json_file, 'wt') as f: + with open(json_file, "wt") as f: f.write(json_data) return json_file -def get_zscore(map_node=False, wf_name='z_score'): +def get_zscore(map_node=False, wf_name="z_score"): """ - Workflow to calculate z-scores + Workflow to calculate z-scores. Parameters ---------- @@ -278,124 +283,134 @@ def get_zscore(map_node=False, wf_name='z_score'): >>> wf.inputs.inputspec.input_file = '/home/data/graph_working_dir/calculate_centrality/degree_centrality_binarize.nii.gz' >>> wf.inputs.inputspec.mask_file = '/home/data/graphs/GraphGeneration/new_mask_3m.nii.gz' >>> wf.run() # doctest: +SKIP - """ # noqa: E501 # pylint: disable=line-too-long + """ # pylint: disable=line-too-long # pylint: disable=import-outside-toplevel,redefined-outer-name,reimported - from CPAC.pipeline import nipype_pipeline_engine as pe + from nipype.interfaces import fsl import nipype.interfaces.utility as util - import nipype.interfaces.fsl as fsl + + from CPAC.pipeline import nipype_pipeline_engine as pe wflow = pe.Workflow(name=wf_name) - inputNode = pe.Node(util.IdentityInterface(fields=['input_file', - 'mask_file']), - name='inputspec') + inputNode = pe.Node( + util.IdentityInterface(fields=["input_file", "mask_file"]), name="inputspec" + ) - outputNode = pe.Node(util.IdentityInterface(fields=['z_score_img']), - name='outputspec') + outputNode = pe.Node( + util.IdentityInterface(fields=["z_score_img"]), name="outputspec" + ) if map_node: - mean = pe.MapNode(interface=fsl.ImageStats(), - name='mean', - iterfield=['in_file']) + mean = pe.MapNode( + interface=fsl.ImageStats(), name="mean", iterfield=["in_file"] + ) - standard_deviation = pe.MapNode(interface=fsl.ImageStats(), - name='standard_deviation', - iterfield=['in_file']) + standard_deviation = pe.MapNode( + interface=fsl.ImageStats(), name="standard_deviation", iterfield=["in_file"] + ) - op_string = pe.MapNode(util.Function(input_names=['mean', 'std_dev'], - output_names=['op_string'], - function=get_operand_string), - name='op_string', - iterfield=['mean', 'std_dev']) + op_string = pe.MapNode( + util.Function( + input_names=["mean", "std_dev"], + output_names=["op_string"], + function=get_operand_string, + ), + name="op_string", + iterfield=["mean", "std_dev"], + ) - z_score = pe.MapNode(interface=fsl.MultiImageMaths(), - name='z_score', - iterfield=['in_file', 'op_string']) + z_score = pe.MapNode( + interface=fsl.MultiImageMaths(), + name="z_score", + iterfield=["in_file", "op_string"], + ) else: - mean = pe.Node(interface=fsl.ImageStats(), name='mean') + mean = pe.Node(interface=fsl.ImageStats(), name="mean") - standard_deviation = pe.Node(interface=fsl.ImageStats(), - name='standard_deviation') + standard_deviation = pe.Node( + interface=fsl.ImageStats(), name="standard_deviation" + ) - op_string = pe.Node(util.Function(input_names=['mean', 'std_dev'], - output_names=['op_string'], - function=get_operand_string), - name='op_string') + op_string = pe.Node( + util.Function( + input_names=["mean", "std_dev"], + output_names=["op_string"], + function=get_operand_string, + ), + name="op_string", + ) - z_score = pe.Node(interface=fsl.MultiImageMaths(), name='z_score') + z_score = pe.Node(interface=fsl.MultiImageMaths(), name="z_score") # calculate the mean - mean.inputs.op_string = '-k %s -m' - wflow.connect(inputNode, 'input_file', mean, 'in_file') - wflow.connect(inputNode, 'mask_file', mean, 'mask_file') + mean.inputs.op_string = "-k %s -m" + wflow.connect(inputNode, "input_file", mean, "in_file") + wflow.connect(inputNode, "mask_file", mean, "mask_file") # calculate the standard deviation - standard_deviation.inputs.op_string = '-k %s -s' - wflow.connect(inputNode, 'input_file', standard_deviation, 'in_file') - wflow.connect(inputNode, 'mask_file', standard_deviation, 'mask_file') + standard_deviation.inputs.op_string = "-k %s -s" + wflow.connect(inputNode, "input_file", standard_deviation, "in_file") + wflow.connect(inputNode, "mask_file", standard_deviation, "mask_file") # calculate the z-score - wflow.connect(mean, 'out_stat', op_string, 'mean') - wflow.connect(standard_deviation, 'out_stat', op_string, 'std_dev') + wflow.connect(mean, "out_stat", op_string, "mean") + wflow.connect(standard_deviation, "out_stat", op_string, "std_dev") # z_score.inputs.out_file = input_name + '_zstd.nii.gz' - wflow.connect(op_string, 'op_string', z_score, 'op_string') - wflow.connect(inputNode, 'input_file', z_score, 'in_file') - wflow.connect(inputNode, 'mask_file', z_score, 'operand_files') + wflow.connect(op_string, "op_string", z_score, "op_string") + wflow.connect(inputNode, "input_file", z_score, "in_file") + wflow.connect(inputNode, "mask_file", z_score, "operand_files") - wflow.connect(z_score, 'out_file', outputNode, 'z_score_img') + wflow.connect(z_score, "out_file", outputNode, "z_score_img") return wflow -def get_fisher_zscore(input_name, map_node=False, wf_name='fisher_z_score'): - """ - Runs the compute_fisher_z_score function as part of a one-node workflow. - """ +def get_fisher_zscore(input_name, map_node=False, wf_name="fisher_z_score"): + """Runs the compute_fisher_z_score function as part of a one-node workflow.""" + import nipype.interfaces.utility as util from CPAC.pipeline import nipype_pipeline_engine as pe - import nipype.interfaces.utility as util - import nipype.interfaces.fsl as fsl wflow = pe.Workflow(name=wf_name) - inputNode = pe.Node(util.IdentityInterface(fields=['correlation_file', - 'timeseries_one_d']), - name='inputspec') + inputNode = pe.Node( + util.IdentityInterface(fields=["correlation_file", "timeseries_one_d"]), + name="inputspec", + ) outputNode = pe.Node( - util.IdentityInterface(fields=['fisher_z_score_img']), - name='outputspec') + util.IdentityInterface(fields=["fisher_z_score_img"]), name="outputspec" + ) if map_node: # node to separate out fisher_z_score = pe.MapNode( - util.Function(input_names=['correlation_file', - 'timeseries_one_d', - 'input_name'], - output_names=['out_file'], - function=compute_fisher_z_score), - name='fisher_z_score', - iterfield=['correlation_file']) + util.Function( + input_names=["correlation_file", "timeseries_one_d", "input_name"], + output_names=["out_file"], + function=compute_fisher_z_score, + ), + name="fisher_z_score", + iterfield=["correlation_file"], + ) else: fisher_z_score = pe.Node( - util.Function(input_names=['correlation_file', - 'timeseries_one_d', - 'input_name'], - output_names=['out_file'], - function=compute_fisher_z_score), - name='fisher_z_score') + util.Function( + input_names=["correlation_file", "timeseries_one_d", "input_name"], + output_names=["out_file"], + function=compute_fisher_z_score, + ), + name="fisher_z_score", + ) fisher_z_score.inputs.input_name = input_name - wflow.connect(inputNode, 'correlation_file', - fisher_z_score, 'correlation_file') - wflow.connect(inputNode, 'timeseries_one_d', - fisher_z_score, 'timeseries_one_d') - wflow.connect(fisher_z_score, 'out_file', - outputNode, 'fisher_z_score_img') + wflow.connect(inputNode, "correlation_file", fisher_z_score, "correlation_file") + wflow.connect(inputNode, "timeseries_one_d", fisher_z_score, "timeseries_one_d") + wflow.connect(fisher_z_score, "out_file", outputNode, "fisher_z_score_img") return wflow @@ -405,35 +420,33 @@ def compute_fisher_z_score(correlation_file, timeseries_one_d, input_name): Computes the fisher z transform of the input correlation map If the correlation map contains data for multiple ROIs then the function returns z score for each ROI as a seperate nifti - file + file. Parameters ---------- - correlation_file : string Input correlations file Returns ------- - out_file : list (nifti files) list of z_scores for mask or ROI """ + import os - import nibabel as nb import numpy as np - import os + import nibabel as nib if isinstance(timeseries_one_d, str): - if '.1D' in timeseries_one_d or '.csv' in timeseries_one_d: - timeseries_file = timeseries_one_d + if ".1D" in timeseries_one_d or ".csv" in timeseries_one_d: + pass else: for timeseries in timeseries_one_d: - if '.1D' in timeseries or '.csv' in timeseries: - timeseries_file = timeseries + if ".1D" in timeseries or ".csv" in timeseries: + pass # get the specific roi number filename = correlation_file.split("/")[-1] @@ -441,7 +454,7 @@ def compute_fisher_z_score(correlation_file, timeseries_one_d, input_name): if ".gz" in filename: filename = filename.replace(".gz", "") - corr_img = nb.load(correlation_file) + corr_img = nib.load(correlation_file) corr_data = corr_img.get_fdata() hdr = corr_img.header @@ -449,10 +462,9 @@ def compute_fisher_z_score(correlation_file, timeseries_one_d, input_name): # calculate the Fisher r-to-z transformation corr_data = np.log((1 + corr_data) / (1 - corr_data)) / 2.0 - z_score_img = nb.Nifti1Image(corr_data, header=hdr, - affine=corr_img.affine) + z_score_img = nib.Nifti1Image(corr_data, header=hdr, affine=corr_img.affine) - out_file = os.path.join(os.getcwd(), filename + '_fisher_zstd.nii.gz') + out_file = os.path.join(os.getcwd(), filename + "_fisher_zstd.nii.gz") z_score_img.to_filename(out_file) @@ -461,7 +473,7 @@ def compute_fisher_z_score(correlation_file, timeseries_one_d, input_name): def get_operand_string(mean, std_dev): """ - Method to get operand string for Fsl Maths + Method to get operand string for Fsl Maths. Parameters ---------- @@ -471,14 +483,12 @@ def get_operand_string(mean, std_dev): path to img containing standard deviation Returns - ------ + ------- op_string : string operand string """ - str1 = "-sub %f -div %f" % (float(mean), float(std_dev)) - op_string = str1 + " -mas %s" - return op_string + return str1 + " -mas %s" def get_roi_num_list(timeseries_file, prefix=None): @@ -486,11 +496,13 @@ def get_roi_num_list(timeseries_file, prefix=None): with open(timeseries_file, "r") as f: roi_file_lines = f.read().splitlines() - roi_err = "\n\n[!] The output of 3dROIstats, used in extracting the " \ - "timeseries, is either empty, or not in the expected " \ - "format.\n\nROI output file: {0}\n\nIf there are no rows " \ - "in the output file, double-check your ROI/mask selection." \ - "\n\n".format(str(timeseries_file)) + roi_err = ( + "\n\n[!] The output of 3dROIstats, used in extracting the " + "timeseries, is either empty, or not in the expected " + "format.\n\nROI output file: {0}\n\nIf there are no rows " + "in the output file, double-check your ROI/mask selection." + "\n\n".format(str(timeseries_file)) + ) for line in roi_file_lines: if "Mean_" in line: @@ -501,7 +513,8 @@ def get_roi_num_list(timeseries_file, prefix=None): # rename labels roi_list = [ x.replace("Mean", "ROI").replace(" ", "").replace("#", "") - for x in roi_list] + for x in roi_list + ] except: raise Exception(roi_err) break @@ -537,42 +550,46 @@ def safe_shape(*vol_data): first_vol_shape = vol_data[0].shape[:3] for vol in vol_data[1:]: - same_volume &= (first_vol_shape == vol.shape[:3]) + same_volume &= first_vol_shape == vol.shape[:3] return same_volume def extract_one_d(list_timeseries): if isinstance(list_timeseries, str): - if '.1D' in list_timeseries or '.csv' in list_timeseries: + if ".1D" in list_timeseries or ".csv" in list_timeseries: return list_timeseries for timeseries in list_timeseries: - if '.1D' in timeseries or '.csv' in timeseries: + if ".1D" in timeseries or ".csv" in timeseries: return timeseries - raise Exception("Unable to retrieve roi timeseries 1D or csv" - " file. Files found:" + list_timeseries) + raise Exception( + "Unable to retrieve roi timeseries 1D or csv" + " file. Files found:" + list_timeseries + ) def extract_txt(list_timeseries): """ Method to extract txt file containing - roi timeseries required for dual regression + roi timeseries required for dual regression. """ if isinstance(list_timeseries, str): - if list_timeseries.endswith('.txt'): + if list_timeseries.endswith(".txt"): return list_timeseries out_file = None for timeseries in list_timeseries: - if timeseries.endswith('.txt'): + if timeseries.endswith(".txt"): out_file = timeseries if not out_file: - raise Exception("Unable to retrieve roi timeseries txt" - " file required for dual regression." - " Existing files are:%s" % (list_timeseries)) + raise Exception( + "Unable to retrieve roi timeseries txt" + " file required for dual regression." + " Existing files are:%s" % (list_timeseries) + ) return out_file @@ -585,8 +602,7 @@ def zscore(data, axis): return data -def correlation(matrix1, matrix2, - match_rows=False, z_scored=False, symmetric=False): +def correlation(matrix1, matrix2, match_rows=False, z_scored=False, symmetric=False): d1 = matrix1.shape[-1] d2 = matrix2.shape[-1] @@ -603,7 +619,7 @@ def correlation(matrix1, matrix2, matrix2 = zscore(matrix2, matrix2.ndim - 1) if match_rows: - return np.einsum('...i,...i', matrix1, matrix2) / var + return np.einsum("...i,...i", matrix1, matrix2) / var if matrix1.ndim >= matrix2.ndim: r = np.dot(matrix1, matrix2.T) / var @@ -621,8 +637,11 @@ def correlation(matrix1, matrix2, def check(params_dct, subject_id, scan_id, val_to_check, throw_exception): if val_to_check not in params_dct: if throw_exception: - raise Exception("Missing Value for {0} for participant " - "{1}".format(val_to_check, subject_id)) + raise Exception( + "Missing Value for {0} for participant " "{1}".format( + val_to_check, subject_id + ) + ) return None if isinstance(params_dct[val_to_check], dict): @@ -630,16 +649,22 @@ def check(params_dct, subject_id, scan_id, val_to_check, throw_exception): else: ret_val = params_dct[val_to_check] - if ret_val == 'None': + if ret_val == "None": if throw_exception: - raise Exception("'None' Parameter Value for {0} for participant " - "{1}".format(val_to_check, subject_id)) + raise Exception( + "'None' Parameter Value for {0} for participant " "{1}".format( + val_to_check, subject_id + ) + ) else: ret_val = None - if ret_val == '' and throw_exception: - raise Exception("Missing Value for {0} for participant " - "{1}".format(val_to_check, subject_id)) + if ret_val == "" and throw_exception: + raise Exception( + "Missing Value for {0} for participant " "{1}".format( + val_to_check, subject_id + ) + ) return ret_val @@ -647,7 +672,7 @@ def check(params_dct, subject_id, scan_id, val_to_check, throw_exception): def check_random_state(seed): """ Turn seed into a np.random.RandomState instance - Code from scikit-learn (https://github.com/scikit-learn/scikit-learn) + Code from scikit-learn (https://github.com/scikit-learn/scikit-learn). Parameters ---------- @@ -663,18 +688,15 @@ def check_random_state(seed): return np.random.RandomState(seed) if isinstance(seed, np.random.RandomState): return seed - raise ValueError('%r cannot be used to seed a numpy.random.RandomState' - ' instance' % seed) + raise ValueError( + "%r cannot be used to seed a numpy.random.RandomState" " instance" % seed + ) def try_fetch_parameter(scan_parameters, subject, scan, keys): - scan_parameters = dict( - (k.lower(), v) - for k, v in scan_parameters.items() - ) + scan_parameters = {k.lower(): v for k, v in scan_parameters.items()} for key in keys: - key = key.lower() if key not in scan_parameters: @@ -686,7 +708,7 @@ def try_fetch_parameter(scan_parameters, subject, scan, keys): value = scan_parameters[key] # Explicit none value - if value == 'None': + if value == "None": return None if value is not None: @@ -694,8 +716,13 @@ def try_fetch_parameter(scan_parameters, subject, scan, keys): return None -def get_scan_params(subject_id, scan, pipeconfig_start_indx, - pipeconfig_stop_indx, data_config_scan_params=None): +def get_scan_params( + subject_id, + scan, + pipeconfig_start_indx, + pipeconfig_stop_indx, + data_config_scan_params=None, +): """ Method to extract slice timing correction parameters and scan parameters. @@ -728,23 +755,21 @@ def get_scan_params(subject_id, scan, pipeconfig_start_indx, pe_direction : str effective_echo_spacing : float """ - - import os import json + import os import warnings - check2 = lambda val: val if val == None or val == '' or \ - isinstance(val, str) else int(val) + def check2(val): + return val if val is None or val == "" or isinstance(val, str) else int(val) # initialize vars to empty - TR = '' - TE = None - pattern = '' - ref_slice = '' - first_tr = '' - last_tr = '' - unit = 's' - pe_direction = '' + TR = "" + pattern = "" + ref_slice = "" + first_tr = "" + last_tr = "" + unit = "s" + pe_direction = "" effective_echo_spacing = None template = None @@ -754,9 +779,11 @@ def get_scan_params(subject_id, scan, pipeconfig_start_indx, if data_config_scan_params: if ".json" in data_config_scan_params: if not os.path.exists(data_config_scan_params): - err = "\n[!] WARNING: Scan parameters JSON file listed in " \ - "your data configuration file does not exist:\n{0}" \ - "\n\n".format(data_config_scan_params) + err = ( + "\n[!] WARNING: Scan parameters JSON file listed in " + "your data configuration file does not exist:\n{0}" + "\n\n".format(data_config_scan_params) + ) raise Exception(err) with open(data_config_scan_params, "r") as f: @@ -767,52 +794,46 @@ def get_scan_params(subject_id, scan, pipeconfig_start_indx, # standard # TODO: better handling of errant key values!!! if "RepetitionTime" in params_dct.keys(): - TR = float(check(params_dct, subject_id, scan, - "RepetitionTime", False)) + TR = float(check(params_dct, subject_id, scan, "RepetitionTime", False)) if "SliceTiming" in params_dct.keys(): - pattern = str(check(params_dct, subject_id, scan, - "SliceTiming", False)) + pattern = str(check(params_dct, subject_id, scan, "SliceTiming", False)) elif "SliceAcquisitionOrder" in params_dct.keys(): - pattern = str(check(params_dct, subject_id, scan, - "SliceAcquisitionOrder", False)) + pattern = str( + check(params_dct, subject_id, scan, "SliceAcquisitionOrder", False) + ) if "PhaseEncodingDirection" in params_dct.keys(): - pe_direction = str(check(params_dct, subject_id, scan, - "PhaseEncodingDirection", False)) + pe_direction = str( + check(params_dct, subject_id, scan, "PhaseEncodingDirection", False) + ) try: "EffectiveEchoSpacing" in params_dct.keys() effective_echo_spacing = float( - check(params_dct, subject_id, scan, - "EffectiveEchoSpacing", False)) + check(params_dct, subject_id, scan, "EffectiveEchoSpacing", False) + ) except TypeError: pass - elif len(data_config_scan_params) > 0 and \ - isinstance(data_config_scan_params, dict): - + elif len(data_config_scan_params) > 0 and isinstance( + data_config_scan_params, dict + ): params_dct = data_config_scan_params # TODO: better handling of errant key values!!! # TODO: use schema validator to deal with it # get details from the configuration - try: + try: TR = float( try_fetch_parameter( - params_dct, - subject_id, - scan, - ['TR', 'RepetitionTime'] + params_dct, subject_id, scan, ["TR", "RepetitionTime"] ) ) except TypeError: TR = None - try: + try: template = str( try_fetch_parameter( - params_dct, - subject_id, - scan, - ['Template', 'template'] + params_dct, subject_id, scan, ["Template", "template"] ) ) except TypeError: @@ -823,67 +844,81 @@ def get_scan_params(subject_id, scan, pipeconfig_start_indx, params_dct, subject_id, scan, - ['acquisition', 'SliceTiming', 'SliceAcquisitionOrder'] + ["acquisition", "SliceTiming", "SliceAcquisitionOrder"], ) ) - ref_slice = check(params_dct, subject_id, scan, 'reference', - False) + ref_slice = check(params_dct, subject_id, scan, "reference", False) if ref_slice: ref_slice = int(ref_slice) - first_tr = check(params_dct, subject_id, scan, 'first_TR', False) + first_tr = check(params_dct, subject_id, scan, "first_TR", False) if first_tr: first_tr = check2(first_tr) - last_tr = check(params_dct, subject_id, scan, 'last_TR', False) + last_tr = check(params_dct, subject_id, scan, "last_TR", False) if last_tr: last_tr = check2(last_tr) - pe_direction = check(params_dct, subject_id, scan, - 'PhaseEncodingDirection', False) + pe_direction = check( + params_dct, subject_id, scan, "PhaseEncodingDirection", False + ) try: effective_echo_spacing = float( - try_fetch_parameter(params_dct, subject_id, scan, - ["EffectiveEchoSpacing"])) + try_fetch_parameter( + params_dct, subject_id, scan, ["EffectiveEchoSpacing"] + ) + ) except TypeError: pass else: - err = "\n\n[!] Could not read the format of the scan parameters " \ - "information included in the data configuration file for " \ - f"the participant {subject_id}.\n\n" + err = ( + "\n\n[!] Could not read the format of the scan parameters " + "information included in the data configuration file for " + f"the participant {subject_id}.\n\n" + ) raise Exception(err) - if first_tr == '' or first_tr is None: + if first_tr == "" or first_tr is None: first_tr = pipeconfig_start_indx - if last_tr == '' or last_tr is None: + if last_tr == "" or last_tr is None: last_tr = pipeconfig_stop_indx - unit = 's' + unit = "s" - if 'None' in pattern or 'none' in pattern: + if "None" in pattern or "none" in pattern: pattern = None - ''' + """ if not pattern: if pipeconfig_tpattern: if "Use NIFTI Header" in pipeconfig_tpattern: pattern = '' else: pattern = pipeconfig_tpattern - ''' + """ # pattern can be one of a few keywords, a filename, or blank which # indicates that the images header information should be used tpattern_file = None - valid_patterns = ['alt+z', 'altplus', 'alt+z2', 'alt-z', 'altminus', - 'alt-z2', 'seq+z', 'seqplus', 'seq-z', 'seqminus'] - if pattern and pattern != '' and pattern not in valid_patterns: - - if isinstance(pattern, list) or \ - ("[" in pattern and "]" in pattern and "," in pattern): + valid_patterns = [ + "alt+z", + "altplus", + "alt+z2", + "alt-z", + "altminus", + "alt-z2", + "seq+z", + "seqplus", + "seq-z", + "seqminus", + ] + if pattern and pattern != "" and pattern not in valid_patterns: + if isinstance(pattern, list) or ( + "[" in pattern and "]" in pattern and "," in pattern + ): # if we got the slice timing as a list, from a BIDS-format scan # parameters JSON file @@ -899,32 +934,40 @@ def get_scan_params(subject_id, scan, pipeconfig_start_indx, for time in slice_timings: f.write("{0}\n".format(time).replace(" ", "")) except: - err = "\n[!] Could not write the slice timing file meant as " \ - "an input for AFNI 3dTshift (slice timing correction):" \ - "\n{0}\n\n".format(tpattern_file) + err = ( + "\n[!] Could not write the slice timing file meant as " + "an input for AFNI 3dTshift (slice timing correction):" + "\n{0}\n\n".format(tpattern_file) + ) raise Exception(err) elif ".txt" in pattern and not os.path.exists(pattern): # if the user provided an acquisition pattern text file for # 3dTshift - raise Exception("Invalid Pattern file path {0}, Please provide " - "the correct path".format(pattern)) + raise Exception( + "Invalid Pattern file path {0}, Please provide " + "the correct path".format(pattern) + ) elif ".txt" in pattern: with open(pattern, "r") as f: lines = f.readlines() if len(lines) < 2: - raise Exception('Invalid slice timing file format. The file ' - 'should contain only one value per row. Use ' - 'new line char as delimiter') + raise Exception( + "Invalid slice timing file format. The file " + "should contain only one value per row. Use " + "new line char as delimiter" + ) tpattern_file = pattern - slice_timings = [float(l.rstrip('\r\n')) for l in lines] + slice_timings = [float(l.rstrip("\r\n")) for l in lines] else: # this only happens if there is a non-path string set in the data # config dictionary for acquisition pattern (like "alt+z"), except # the pattern is not listed in that list - err = "\n[!] The slice timing acquisition pattern provided is " \ - "not supported by AFNI 3dTshift:\n" \ - "{0}\n".format(str(pattern)) + err = ( + "\n[!] The slice timing acquisition pattern provided is " + "not supported by AFNI 3dTshift:\n" + "{0}\n".format(str(pattern)) + ) raise Exception(err) pattern = tpattern_file @@ -935,19 +978,19 @@ def get_scan_params(subject_id, scan, pipeconfig_start_indx, # checking if the unit of TR and slice timing match or not # if slice timing in ms convert TR to ms as well if TR and max_slice_offset > TR: - warnings.warn("TR is in seconds and slice timings are in " - "milliseconds. Converting TR into milliseconds") + warnings.warn( + "TR is in seconds and slice timings are in " + "milliseconds. Converting TR into milliseconds" + ) TR = TR * 1000 - print("New TR value {0} ms".format(TR)) - unit = 'ms' + unit = "ms" else: # check to see, if TR is in milliseconds, convert it into seconds if TR and TR > 10: - warnings.warn('TR is in milliseconds, Converting it into seconds') + warnings.warn("TR is in milliseconds, Converting it into seconds") TR = TR / 1000.0 - print("New TR value {0} s".format(TR)) - unit = 's' + unit = "s" # swap back in if TR: @@ -959,21 +1002,22 @@ def get_scan_params(subject_id, scan, pipeconfig_start_indx, start_indx = first_tr stop_indx = last_tr - return (tr if tr else None, - tpattern if tpattern else None, - template if template else None, - ref_slice, - start_indx, - stop_indx, - pe_direction, - effective_echo_spacing) + return ( + tr if tr else None, + tpattern if tpattern else None, + template if template else None, + ref_slice, + start_indx, + stop_indx, + pe_direction, + effective_echo_spacing, + ) def get_tr(tr): - """ - Method to return TR in seconds - """ + """Method to return TR in seconds.""" import re + if tr: tr = re.search(r"\d+.\d+", str(tr)).group(0) tr = float(tr) @@ -988,6 +1032,7 @@ def check_tr(tr, in_file): # imageData would have to be the image data from the funcFlow workflow, # funcFlow outputspec.subject import nibabel as nib + img = nib.load(in_file) # get header from image data, then extract TR information, TR is fourth @@ -1001,17 +1046,20 @@ def check_tr(tr, in_file): # from either convert_tr or header_tr using afni 3drefit, then append to # func_to_mni if header_tr != tr: - if tr != None and tr != "": + if tr is not None and tr != "": TR = tr else: TR = header_tr import warnings - warnings.warn('Warning: The TR information does not match between ' - 'the config and subject list files.') + + warnings.warn( + "Warning: The TR information does not match between " + "the config and subject list files." + ) return TR - + def add_afni_prefix(tpattern): if ".txt" in tpattern: @@ -1020,18 +1068,16 @@ def add_afni_prefix(tpattern): def write_to_log(workflow, log_dir, index, inputs, scan_id): - """ - Method to write into log file the status of the workflow run. - """ - + """Method to write into log file the status of the workflow run.""" + import datetime import os import time - import datetime - from CPAC import __version__ from nipype import logging - iflogger = logging.getLogger('nipype.interface') + from CPAC import __version__ + + iflogger = logging.getLogger("nipype.interface") version = __version__ subject_id = os.path.basename(log_dir) @@ -1041,20 +1087,19 @@ def write_to_log(workflow, log_dir, index, inputs, scan_id): strategy = "" ts = time.time() - stamp = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S') + stamp = datetime.datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M:%S") try: - if workflow != 'DONE': - wf_path = \ - os.path.dirname((os.getcwd()).split(workflow)[1]).strip("/") + if workflow != "DONE": + wf_path = os.path.dirname((os.getcwd()).split(workflow)[1]).strip("/") if wf_path and wf_path != "": - if '/' in wf_path: - scan_id, strategy = wf_path.split('/', 1) - scan_id = scan_id.strip('_') + if "/" in wf_path: + scan_id, strategy = wf_path.split("/", 1) + scan_id = scan_id.strip("_") strategy = strategy.replace("/", "") else: - scan_id = wf_path.strip('_') + scan_id = wf_path.strip("_") file_path = os.path.join(log_dir, scan_id, workflow) @@ -1062,25 +1107,26 @@ def write_to_log(workflow, log_dir, index, inputs, scan_id): os.makedirs(file_path) except Exception: iflogger.info( - "filepath already exist, filepath- {0}, " - "curr_dir - {1}".format(file_path, os.getcwd())) + "filepath already exist, filepath- {0}, " "curr_dir - {1}".format( + file_path, os.getcwd() + ) + ) else: file_path = os.path.join(log_dir, scan_id) except Exception: - print("ERROR in write log") raise try: os.makedirs(file_path) except Exception: iflogger.info( - "filepath already exist, " - "filepath: {0}, " - "curr_dir: {1}".format(file_path, os.getcwd()) + "filepath already exist, " "filepath: {0}, " "curr_dir: {1}".format( + file_path, os.getcwd() + ) ) - out_file = os.path.join(file_path, 'log_{0}.yml'.format(strategy)) + out_file = os.path.join(file_path, "log_{0}.yml".format(strategy)) iflogger.info("CPAC custom log:") @@ -1097,8 +1143,7 @@ def write_to_log(workflow, log_dir, index, inputs, scan_id): "strategy: {4}, " "workflow: {5}, " "status: COMPLETED".format( - str(version), str(stamp), subject_id, - scan_id, strategy, workflow + str(version), str(stamp), subject_id, scan_id, strategy, workflow ) ) else: @@ -1111,12 +1156,11 @@ def write_to_log(workflow, log_dir, index, inputs, scan_id): "strategy: {4}, " "workflow: {5}, " "status: ERROR".format( - str(version), str(stamp), subject_id, - scan_id, strategy, workflow + str(version), str(stamp), subject_id, scan_id, strategy, workflow ) ) - with open(out_file, 'w') as f: + with open(out_file, "w") as f: f.write("version: {0}\n".format(str(version))) f.write("timestamp: {0}\n".format(str(stamp))) f.write("pipeline_index: {0}\n".format(index)) @@ -1130,52 +1174,50 @@ def write_to_log(workflow, log_dir, index, inputs, scan_id): def create_log(wf_name="log", scan_id=None): - """ - Workflow to create log - """ + """Workflow to create log.""" + import nipype.interfaces.utility as util from CPAC.pipeline import nipype_pipeline_engine as pe - import nipype.interfaces.utility as util - import CPAC.utils.interfaces.function as function + from CPAC.utils.interfaces import function wf = pe.Workflow(name=wf_name) - input_node = pe.Node(util.IdentityInterface(fields=['workflow', - 'log_dir', - 'index', - 'inputs']), - name='inputspec') - - output_node = pe.Node(util.IdentityInterface(fields=['out_file']), - name='outputspec') - - write_log = pe.Node(function.Function(input_names=['workflow', - 'log_dir', - 'index', - 'inputs', - 'scan_id'], - output_names=['out_file'], - function=write_to_log, - as_module=True), - name='write_log') + input_node = pe.Node( + util.IdentityInterface(fields=["workflow", "log_dir", "index", "inputs"]), + name="inputspec", + ) - write_log.inputs.scan_id = scan_id + output_node = pe.Node( + util.IdentityInterface(fields=["out_file"]), name="outputspec" + ) - wf.connect([ - ( - input_node, write_log, [ - ('workflow', 'workflow'), - ('log_dir', 'log_dir'), - ('index', 'index'), - ('inputs', 'inputs') - ] + write_log = pe.Node( + function.Function( + input_names=["workflow", "log_dir", "index", "inputs", "scan_id"], + output_names=["out_file"], + function=write_to_log, + as_module=True, ), - ( - write_log, output_node, [ - ('out_file', 'out_file') - ] - ) - ]) + name="write_log", + ) + + write_log.inputs.scan_id = scan_id + + wf.connect( + [ + ( + input_node, + write_log, + [ + ("workflow", "workflow"), + ("log_dir", "log_dir"), + ("index", "index"), + ("inputs", "inputs"), + ], + ), + (write_log, output_node, [("out_file", "out_file")]), + ] + ) return wf @@ -1194,21 +1236,19 @@ def find_files(directory, pattern): def extract_output_mean(in_file, output_name): - ''' + """ function takes 'in_file', which should be an intermediary 1D file from individual-level analysis, containing the mean of the output across - all voxels + all voxels. it then parses this value and writes it to a .csv file named output_means.csv located in the subject's output directory - ''' - + """ if os.path.exists(in_file): - - with open(in_file, 'r') as f: + with open(in_file, "r") as f: line = f.readline() - line = line.split('[')[0].strip(' ') + line = line.split("[")[0].strip(" ") # get filename of input maskave 1D file filename = in_file.split("/")[-1] @@ -1216,10 +1256,7 @@ def extract_output_mean(in_file, output_name): split_fullpath = in_file.split("/") - if "_mask_" in in_file and ( - "sca_roi" in in_file or "sca_tempreg" in in_file - ): - + if "_mask_" in in_file and ("sca_roi" in in_file or "sca_tempreg" in in_file): for dirname in split_fullpath: if "_mask_" in dirname: maskname = dirname @@ -1229,11 +1266,9 @@ def extract_output_mean(in_file, output_name): if ".1D" in filename: filename = filename.replace(".1D", "") - resource_name = "{0}_{1}_{2}".format(output_name, maskname, - filename) + resource_name = "{0}_{1}_{2}".format(output_name, maskname, filename) elif "_spatial_map_" in in_file and "dr_tempreg" in in_file: - for dirname in split_fullpath: if "_spatial_map_" in dirname: mapname = dirname @@ -1243,11 +1278,9 @@ def extract_output_mean(in_file, output_name): if ".1D" in filename: filename = filename.replace(".1D", "") - resource_name = "{0}_{1}_{2}".format(output_name, mapname, - filename) + resource_name = "{0}_{1}_{2}".format(output_name, mapname, filename) elif "_mask_" in in_file and "centrality" in in_file: - for dirname in split_fullpath: if "_mask_" in dirname: maskname = dirname @@ -1257,94 +1290,84 @@ def extract_output_mean(in_file, output_name): if ".1D" in filename: filename = filename.replace(".1D", "") - resource_name = "{0}_{1}_{2}".format(output_name, maskname, - filename) + resource_name = "{0}_{1}_{2}".format(output_name, maskname, filename) else: resource_name = output_name - output_means_file = os.path.join(os.getcwd(), - 'mean_{0}.txt'.format(resource_name)) + output_means_file = os.path.join( + os.getcwd(), "mean_{0}.txt".format(resource_name) + ) - with open(output_means_file, 'w') as f: + with open(output_means_file, "w") as f: f.write(line) return output_means_file def create_output_mean_csv(subject_dir): - ''' + """ this function finds all of the mean_{output}.txt files in the subject's output directory, collects the data and organizes them into one .csv - file in the subject directory - ''' - + file in the subject directory. + """ import os - import csv output_vals = {} - subID = subject_dir.split('/')[len(subject_dir.split('/')) - 1] - means_dir = os.path.join(subject_dir, 'output_means') + subID = subject_dir.split("/")[len(subject_dir.split("/")) - 1] + means_dir = os.path.join(subject_dir, "output_means") # extract the mean values for root, _, files in os.walk(means_dir): - for filename in files: - - if 'mean_' in filename: - - output = filename.replace('mean_', '') - output = output.replace('.txt', '') + if "mean_" in filename: + output = filename.replace("mean_", "") + output = output.replace(".txt", "") filepath = os.path.join(root, filename) if os.path.exists(filepath): try: - mean_file = open(filepath, 'rU') + mean_file = open(filepath, "rU") val = mean_file.readline() - val = val.strip('\n') + val = val.strip("\n") except: - print('\n\n[!] CPAC says: Could not open the output ' - 'mean text file.\n') - print('Path: ', filepath, '\n\n') raise Exception else: - print('\n\n[!] CPAC says: Could not find the output mean ' - 'text file.\n') - print('Path not found: ', filepath, '\n\n') raise Exception output_vals[output] = val # now take the extracted mean values and write them into the .csv file! - csv_file_path = os.path.join(subject_dir, 'output_means_%s.csv' % subID) - with open(csv_file_path, 'wt') as csv_file: - + csv_file_path = os.path.join(subject_dir, "output_means_%s.csv" % subID) + with open(csv_file_path, "wt") as csv_file: output_items = list(output_vals.items()) - deriv_string = ','.join(v for v, _ in output_items) - val_string = ','.join(v for _, v in output_items) + deriv_string = ",".join(v for v, _ in output_items) + val_string = ",".join(v for _, v in output_items) - csv_file.write(deriv_string + '\n') - csv_file.write(val_string + '\n') + csv_file.write(deriv_string + "\n") + csv_file.write(val_string + "\n") def check_command_path(path): import os + return os.system("%s >/dev/null 2>&1" % path) != 32512 -def check_system_deps(check_ants=False, - check_ica_aroma=False, - check_centrality_degree=False, - check_centrality_lfcd=False): - ''' +def check_system_deps( + check_ants=False, + check_ica_aroma=False, + check_centrality_degree=False, + check_centrality_lfcd=False, +): + """ Function to check system for neuroimaging tools AFNI, C3D, FSL, - and (optionally) ANTs - ''' - + and (optionally) ANTs. + """ missing_install = [] # Check AFNI @@ -1379,116 +1402,134 @@ def check_system_deps(check_ants=False, missing_string = "" for string in missing_install: missing_string = missing_string + string + "\n" - err = "\n\n[!] CPAC says: It appears the following software " \ - "packages are not installed or configured properly:\n\n%s\n" \ - % missing_string + err = ( + "\n\n[!] CPAC says: It appears the following software " + "packages are not installed or configured properly:\n\n%s\n" + % missing_string + ) raise Exception(err) # Check pipeline config againts computer resources def check_config_resources(c): # Import packages - import psutil from multiprocessing import cpu_count + import psutil + # Init variables sys_virt_mem = psutil.virtual_memory() num_cores = cpu_count() # Check for pipeline memory for subject - if c.pipeline_setup['system_config'][ - 'maximum_memory_per_participant'] is None: + if c.pipeline_setup["system_config"]["maximum_memory_per_participant"] is None: # Get system memory and numSubsAtOnce - sys_mem_gb = sys_virt_mem.total / (1024.0 ** 3) - sub_mem_gb = sys_mem_gb / c.pipeline_setup['system_config'][ - 'num_participants_at_once'] + sys_mem_gb = sys_virt_mem.total / (1024.0**3) + sub_mem_gb = ( + sys_mem_gb / c.pipeline_setup["system_config"]["num_participants_at_once"] + ) else: - sub_mem_gb = c.pipeline_setup['system_config'][ - 'maximum_memory_per_participant'] + sub_mem_gb = c.pipeline_setup["system_config"]["maximum_memory_per_participant"] # If centrality is enabled, check to mem_sub >= mem_centrality - if c.network_centrality['run']: - if sub_mem_gb < c.network_centrality['memory_allocation']: - err_msg = 'Memory allocated for subject: %d needs to be greater ' \ - 'than the memory allocated for centrality: %d. Fix ' \ - 'and try again.' % (c.pipeline_setup[ - 'system_config']['maximum_memory_per_participant'], - c.network_centrality[ - 'memory_allocation']) + if c.network_centrality["run"]: + if sub_mem_gb < c.network_centrality["memory_allocation"]: + err_msg = ( + "Memory allocated for subject: %d needs to be greater " + "than the memory allocated for centrality: %d. Fix " + "and try again." + % ( + c.pipeline_setup["system_config"]["maximum_memory_per_participant"], + c.network_centrality["memory_allocation"], + ) + ) raise Exception(err_msg) # Check for pipeline threads # Check if user specified cores - if c.pipeline_setup['system_config']['max_cores_per_participant']: - total_user_cores = c.pipeline_setup['system_config'][ - 'num_participants_at_once'] * \ - c.pipeline_setup['system_config'][ - 'max_cores_per_participant'] + if c.pipeline_setup["system_config"]["max_cores_per_participant"]: + total_user_cores = ( + c.pipeline_setup["system_config"]["num_participants_at_once"] + * c.pipeline_setup["system_config"]["max_cores_per_participant"] + ) if total_user_cores > num_cores: - raise SystemError('Configuration specifies more threads running ' - 'in parallel (%d) than number of threads ' - 'available (%d). Change this and try again' % - (total_user_cores, num_cores)) - num_cores_per_sub = c.pipeline_setup['system_config'][ - 'max_cores_per_participant'] + raise SystemError( + "Configuration specifies more threads running " + "in parallel (%d) than number of threads " + "available (%d). Change this and try again" + % (total_user_cores, num_cores) + ) + num_cores_per_sub = c.pipeline_setup["system_config"][ + "max_cores_per_participant" + ] else: - num_cores_per_sub = num_cores / c.pipeline_setup['system_config'][ - 'num_participants_at_once'] + num_cores_per_sub = ( + num_cores / c.pipeline_setup["system_config"]["num_participants_at_once"] + ) # Now check ANTS - if 'ANTS' in c.registration_workflows['anatomical_registration'][ - 'registration']['using']: - if c.pipeline_setup['system_config']['num_ants_threads'] is None: + if ( + "ANTS" + in c.registration_workflows["anatomical_registration"]["registration"]["using"] + ): + if c.pipeline_setup["system_config"]["num_ants_threads"] is None: num_ants_cores = num_cores_per_sub - elif c.pipeline_setup['system_config']['num_ants_threads'] > \ - c.pipeline_setup['system_config'][ - 'max_cores_per_participant']: - err_msg = 'Number of threads for ANTS: %d is greater than the ' \ - 'number of threads per subject: %d. Change this and ' \ - 'try again.' % ( - c.pipeline_setup['system_config']['num_ants_threads'], - c.pipeline_setup['system_config'][ - 'max_cores_per_participant']) + elif ( + c.pipeline_setup["system_config"]["num_ants_threads"] + > c.pipeline_setup["system_config"]["max_cores_per_participant"] + ): + err_msg = ( + "Number of threads for ANTS: %d is greater than the " + "number of threads per subject: %d. Change this and " + "try again." + % ( + c.pipeline_setup["system_config"]["num_ants_threads"], + c.pipeline_setup["system_config"]["max_cores_per_participant"], + ) + ) raise Exception(err_msg) else: - num_ants_cores = c.pipeline_setup['system_config'][ - 'num_ants_threads'] + num_ants_cores = c.pipeline_setup["system_config"]["num_ants_threads"] else: num_ants_cores = 1 # Now check OMP - if c.pipeline_setup['system_config']['num_OMP_threads'] is None: + if c.pipeline_setup["system_config"]["num_OMP_threads"] is None: num_omp_cores = 1 - elif c.pipeline_setup['system_config']['num_OMP_threads'] > \ - c.pipeline_setup['system_config']['max_cores_per_participant']: - err_msg = 'Number of threads for OMP: %d is greater than the ' \ - 'number of threads per subject: %d. Change this and ' \ - 'try again.' % (c.pipeline_setup['system_config'][ - 'num_OMP_threads'], - c.pipeline_setup['system_config'][ - 'max_cores_per_participant']) + elif ( + c.pipeline_setup["system_config"]["num_OMP_threads"] + > c.pipeline_setup["system_config"]["max_cores_per_participant"] + ): + err_msg = ( + "Number of threads for OMP: %d is greater than the " + "number of threads per subject: %d. Change this and " + "try again." + % ( + c.pipeline_setup["system_config"]["num_OMP_threads"], + c.pipeline_setup["system_config"]["max_cores_per_participant"], + ) + ) raise Exception(err_msg) else: - num_omp_cores = c.pipeline_setup['system_config']['num_OMP_threads'] + num_omp_cores = c.pipeline_setup["system_config"]["num_OMP_threads"] # Return memory and cores return sub_mem_gb, num_cores_per_sub, num_ants_cores, num_omp_cores def _check_nested_types(d, keys): - '''Helper function to check types for *_nested_value functions''' + """Helper function to check types for *_nested_value functions.""" if not isinstance(d, dict): - raise TypeError(f'Expected dict, got {type(d).__name__}: {str(d)}') + raise TypeError(f"Expected dict, got {type(d).__name__}: {d!s}") if not isinstance(keys, list) and not isinstance(keys, tuple): - raise TypeError( - f'Expected list, got {type(keys).__name__}: {str(keys)}') + raise TypeError(f"Expected list, got {type(keys).__name__}: {keys!s}") def delete_nested_value(d, keys): - '''Helper function to delete nested values + """Helper function to delete nested values. Parameters - --------- + ---------- d: dict keys: list or tuple @@ -1503,7 +1544,7 @@ def delete_nested_value(d, keys): ... {'nested': {'key1': 'value', 'key2': 'value'}}, ... ['nested', 'key1']) {'nested': {'key2': 'value'}} - ''' + """ _check_nested_types(d, keys) if len(keys) == 1: del d[keys[0]] @@ -1515,9 +1556,7 @@ def delete_nested_value(d, keys): def ordereddict_to_dict(value): - ''' - this function convert ordereddict into regular dict - ''' + """This function convert ordereddict into regular dict.""" for k, v in value.items(): if isinstance(v, dict): value[k] = ordereddict_to_dict(v) @@ -1543,39 +1582,25 @@ def repickle(directory): if fn.endswith(".pkl"): if _pickle2(p): try: - with open(p, 'rb') as fp: - f = pickle.load(fp, encoding='latin1') - with open(p, 'wb') as fp: + with open(p, "rb") as fp: + f = pickle.load(fp, encoding="latin1") + with open(p, "wb") as fp: pickle.dump(f, fp) - print( - f"Converted pickle {fn} from a Python 2 pickle to " - "a Python 3 pickle." - ) - except Exception as e: - print( - f"Could not convert Python 2 pickle {p} " - f"because {e}\n" - ) + except Exception: + pass else: - print(f"Pickle {fn} is a Python 3 pickle.") + pass elif fn.endswith(".pklz"): if _pickle2(p, True): try: - with gzip.open(p, 'rb') as fp: - f = pickle.load(fp, encoding='latin1') - with gzip.open(p, 'wb') as fp: + with gzip.open(p, "rb") as fp: + f = pickle.load(fp, encoding="latin1") + with gzip.open(p, "wb") as fp: pickle.dump(f, fp) - print( - f"Converted pickle {fn} from a Python 2 pickle to " - "a Python 3 pickle." - ) - except Exception as e: - print( - f"Could not convert Python 2 pickle {p} " - f"because {e}\n" - ) + except Exception: + pass else: - print(f"Pickle {fn} is a Python 3 pickle.") + pass def _pickle2(p, z=False): @@ -1597,32 +1622,26 @@ def _pickle2(p, z=False): True if p is a Python 2 pickle """ if z: - with gzip.open(p, 'rb') as fp: + with gzip.open(p, "rb") as fp: try: pickle.load(fp) except UnicodeDecodeError: return True - except Exception as e: - print( - f"Pickle {p} may be a Python 3 pickle, but raised " - f"exception {e}" - ) + except Exception: + pass else: - with open(p, 'rb') as fp: + with open(p, "rb") as fp: try: pickle.load(fp) except UnicodeDecodeError: return True - except Exception as e: - print( - f"Pickle {p} may be a Python 3 pickle, but raised " - f"exception {e}" - ) + except Exception: + pass return False def _changes_1_8_0_to_1_8_1(config_dict): - ''' + """ Examples -------- Starting with 1.8.0 @@ -1672,55 +1691,84 @@ def _changes_1_8_0_to_1_8_1(config_dict): [1, 2] >>> updated_apb['WM_label'] [3, 4] - ''' - for key_sequence in { - ('anatomical_preproc', 'non_local_means_filtering'), - ('anatomical_preproc', 'n4_bias_field_correction') - }: + """ + for key_sequence in ( + ("anatomical_preproc", "non_local_means_filtering"), + ("anatomical_preproc", "n4_bias_field_correction"), + ): config_dict = _now_runswitch(config_dict, key_sequence) - for combiners in { - (( - ('segmentation', 'tissue_segmentation', 'ANTs_Prior_Based', - 'CSF_label'), - ), ('segmentation', 'tissue_segmentation', 'ANTs_Prior_Based', - 'CSF_label')), - (( - ('segmentation', 'tissue_segmentation', 'ANTs_Prior_Based', - 'left_GM_label'), - ('segmentation', 'tissue_segmentation', 'ANTs_Prior_Based', - 'right_GM_label') - ), ('segmentation', 'tissue_segmentation', 'ANTs_Prior_Based', - 'GM_label')), - (( - ('segmentation', 'tissue_segmentation', 'ANTs_Prior_Based', - 'left_WM_label'), - ('segmentation', 'tissue_segmentation', 'ANTs_Prior_Based', - 'right_WM_label') - ), ('segmentation', 'tissue_segmentation', 'ANTs_Prior_Based', - 'WM_label')) - }: + for combiners in ( + ( + (("segmentation", "tissue_segmentation", "ANTs_Prior_Based", "CSF_label"),), + ("segmentation", "tissue_segmentation", "ANTs_Prior_Based", "CSF_label"), + ), + ( + ( + ( + "segmentation", + "tissue_segmentation", + "ANTs_Prior_Based", + "left_GM_label", + ), + ( + "segmentation", + "tissue_segmentation", + "ANTs_Prior_Based", + "right_GM_label", + ), + ), + ("segmentation", "tissue_segmentation", "ANTs_Prior_Based", "GM_label"), + ), + ( + ( + ( + "segmentation", + "tissue_segmentation", + "ANTs_Prior_Based", + "left_WM_label", + ), + ( + "segmentation", + "tissue_segmentation", + "ANTs_Prior_Based", + "right_WM_label", + ), + ), + ("segmentation", "tissue_segmentation", "ANTs_Prior_Based", "WM_label"), + ), + ): config_dict = _combine_labels(config_dict, *combiners) try: calculate_motion_first = lookup_nested_value( config_dict, - ['functional_preproc', 'motion_estimates_and_correction', - 'calculate_motion_first'] + [ + "functional_preproc", + "motion_estimates_and_correction", + "calculate_motion_first", + ], ) except KeyError: calculate_motion_first = None if calculate_motion_first is not None: - del config_dict['functional_preproc'][ - 'motion_estimates_and_correction']['calculate_motion_first'] - config_dict = set_nested_value(config_dict, [ - 'functional_preproc', 'motion_estimates_and_correction', - 'motion_estimates', 'calculate_motion_first' - ], calculate_motion_first) + del config_dict["functional_preproc"]["motion_estimates_and_correction"][ + "calculate_motion_first" + ] + config_dict = set_nested_value( + config_dict, + [ + "functional_preproc", + "motion_estimates_and_correction", + "motion_estimates", + "calculate_motion_first", + ], + calculate_motion_first, + ) return config_dict def _combine_labels(config_dict, list_to_combine, new_key): - ''' + """ Helper function to combine formerly separate keys into a combined key. @@ -1735,7 +1783,7 @@ def _combine_labels(config_dict, list_to_combine, new_key): Returns ------- updated_config_dict: dict - ''' + """ new_value = [] any_old_values = False for _to_combine in list_to_combine: @@ -1771,7 +1819,6 @@ def concat_list(in_list1=None, in_list2=None): out_list : list a list of file paths """ - if in_list1 is not None: if not isinstance(in_list1, list): in_list1 = [in_list1] @@ -1784,14 +1831,15 @@ def concat_list(in_list1=None, in_list2=None): else: in_list2 = [] - out_list = in_list1 + in_list2 + return in_list1 + in_list2 - return out_list - -def list_item_replace(l, # noqa: E741 # pylint: disable=invalid-name - old, new): - '''Function to replace an item in a list +def list_item_replace( + l, # noqa: E741 # pylint: disable=invalid-name + old, + new, +): + """Function to replace an item in a list. Parameters ---------- @@ -1814,7 +1862,7 @@ def list_item_replace(l, # noqa: E741 # pylint: disable=invalid-name ['3dSkullStrip', 'FSL'] >>> list_item_replace(['AFNI', 'FSL'], 'FSL', 'BET') ['AFNI', 'BET'] - ''' + """ if isinstance(l, list) and old in l: l[l.index(old)] = new elif isinstance(l, str): @@ -1823,10 +1871,10 @@ def list_item_replace(l, # noqa: E741 # pylint: disable=invalid-name def lookup_nested_value(d, keys): - '''Helper method to look up nested values + """Helper method to look up nested values. Parameters - --------- + ---------- d: dict keys: list or tuple @@ -1840,13 +1888,13 @@ def lookup_nested_value(d, keys): True >>> lookup_nested_value({'nested': {'None': None}}, ['nested', 'None']) '' - ''' + """ if not isinstance(d, dict): return d if len(keys) == 1: value = d[keys[0]] if value is None: - return '' + return "" return value else: try: @@ -1857,7 +1905,7 @@ def lookup_nested_value(d, keys): def _now_runswitch(config_dict, key_sequence): - ''' + """ Helper function to convert a formerly forkable value to a runswitch. @@ -1870,19 +1918,18 @@ def _now_runswitch(config_dict, key_sequence): Returns ------- updated_config_dict: dict - ''' + """ try: old_forkable = lookup_nested_value(config_dict, key_sequence) except KeyError: return config_dict if isinstance(old_forkable, (bool, list)): - return set_nested_value( - config_dict, key_sequence, {'run': old_forkable}) + return set_nested_value(config_dict, key_sequence, {"run": old_forkable}) return config_dict def _remove_somethings(value, things_to_remove): - '''Helper function to remove instances of any in a given set of + """Helper function to remove instances of any in a given set of values from a list. Parameters @@ -1894,7 +1941,7 @@ def _remove_somethings(value, things_to_remove): Returns ------- list - ''' + """ if isinstance(value, list): for thing in things_to_remove: while thing in value: @@ -1903,7 +1950,7 @@ def _remove_somethings(value, things_to_remove): def remove_False(d, k): - '''Function to remove "Off" and False from a list at a given nested key. + """Function to remove "Off" and False from a list at a given nested key. Parameters ---------- @@ -1920,13 +1967,13 @@ def remove_False(d, k): -------- >>> remove_False({'a': {'b': [1, False, 2, "Off", 3]}}, ['a', 'b']) {'a': {'b': [1, 2, 3]}} - ''' - value = _remove_somethings(lookup_nested_value(d, k), {False, 'Off'}) + """ + value = _remove_somethings(lookup_nested_value(d, k), {False, "Off"}) return set_nested_value(d, k, value) def remove_None(d, k): - '''Function to remove "None" and None from a list at a given nested key. + """Function to remove "None" and None from a list at a given nested key. Parameters ---------- @@ -1943,13 +1990,13 @@ def remove_None(d, k): -------- >>> remove_None({'a': {'b': [1, None, 2, "None", 3]}}, ['a', 'b']) {'a': {'b': [1, 2, 3]}} - ''' - value = _remove_somethings(lookup_nested_value(d, k), {None, 'None'}) + """ + value = _remove_somethings(lookup_nested_value(d, k), {None, "None"}) return set_nested_value(d, k, value) def replace_in_strings(d, replacements=None): - '''Helper function to recursively replace substrings. + """Helper function to recursively replace substrings. Parameters ---------- @@ -1970,10 +2017,9 @@ def replace_in_strings(d, replacements=None): -------- >>> replace_in_strings({'key': 'test${resolution_for_func_preproc}'}) {'key': 'test${func_resolution}'} - ''' + """ if replacements is None: - replacements = [(r'${resolution_for_func_preproc}', - r'${func_resolution}')] + replacements = [(r"${resolution_for_func_preproc}", r"${func_resolution}")] if isinstance(d, dict): return {k: replace_in_strings(d[k], replacements) for k in d} if isinstance(d, list): @@ -1985,10 +2031,10 @@ def replace_in_strings(d, replacements=None): def set_nested_value(d, keys, value): - '''Helper method to set nested values + """Helper method to set nested values. Parameters - --------- + ---------- d: dict keys: list or tuple value: any @@ -2002,22 +2048,19 @@ def set_nested_value(d, keys, value): -------- >>> set_nested_value({}, ['nested', 'keys'], 'value') {'nested': {'keys': 'value'}} - ''' + """ _check_nested_types(d, keys) if len(keys) == 1: d.update({keys[0]: value}) return d if not len(keys): # pylint: disable=len-as-condition return d - new_d = { - keys[0]: set_nested_value(d.get(keys[0], {}), keys[1:], value) - } - d = update_nested_dict(d, new_d) - return d + new_d = {keys[0]: set_nested_value(d.get(keys[0], {}), keys[1:], value)} + return update_nested_dict(d, new_d) def update_config_dict(old_dict): - '''Function to convert an old config dict to a new config dict + """Function to convert an old config dict to a new config dict. Parameters ---------- @@ -2044,11 +2087,12 @@ def update_config_dict(old_dict): {'2': None} >>> c {'pipeline_setup': {'pipeline_name': 'example-pipeline'}, '2': None} - ''' + """ + def _append_to_list(current_value, new_value): - '''Helper function to add new_value to the current_value list + """Helper function to add new_value to the current_value list or create a list if one does not exist. Skips falsy elements - in new_value + in new_value. Parameters ---------- @@ -2072,7 +2116,7 @@ def _append_to_list(current_value, new_value): [1, 2] >>> _append_to_list([1], [None, 2]) [1, 2] - ''' + """ if not isinstance(current_value, list): if current_value is not None: current_value = [current_value] @@ -2082,18 +2126,15 @@ def _append_to_list(current_value, new_value): current_value = [v for v in current_value if v is not None] if isinstance(new_value, list): for i in new_value: - if i and i not in current_value and i != 'Off': + if i and i not in current_value and i != "Off": current_value.append(i) - elif ( - new_value and new_value not in current_value and - new_value != 'Off' - ): + elif new_value and new_value not in current_value and new_value != "Off": current_value.append(new_value) return current_value def _bool_to_str(old_value, value_if_true): - '''Helper function to convert a True or a list containing a - True to a given string + """Helper function to convert a True or a list containing a + True to a given string. Parameters ---------- @@ -2119,7 +2160,7 @@ def _bool_to_str(old_value, value_if_true): >>> _bool_to_str([0, None, False], 'test_str') >>> _bool_to_str([0, None, False, 1], 'test_str') 'test_str' - ''' + """ if isinstance(old_value, list): if any(bool(i) for i in old_value): return value_if_true @@ -2128,7 +2169,7 @@ def _bool_to_str(old_value, value_if_true): return None def _get_old_values(old_dict, new_dict, key): - '''Helper function to get old and current values of a special key + """Helper function to get old and current values of a special key being updated. Parameters @@ -2148,11 +2189,9 @@ def _get_old_values(old_dict, new_dict, key): old_value : any current_value : any - ''' + """ old_value = old_dict.pop(key) - current_value = lookup_nested_value( - new_dict, NESTED_CONFIG_MAPPING[key] - ) + current_value = lookup_nested_value(new_dict, NESTED_CONFIG_MAPPING[key]) return old_dict, new_dict, old_value, current_value new_dict = {} @@ -2160,140 +2199,152 @@ def _get_old_values(old_dict, new_dict, key): if key in NESTED_CONFIG_MAPPING: # handle special cases special_cases = { - 'acpc_run_preprocessing', - 'acpc_template_brain', - 'ANTs_prior_based_segmentation', - 'func_reg_input', - 'runRegisterFuncToTemplate', - 'runRegisterFuncToEPI', - 'fsl_linear_reg_only', - 'functional_registration', - 'template_for_resample', - 'fnirtConfig', - 'run_smoothing', - 'runZScoring', - 'run_longitudinal' + "acpc_run_preprocessing", + "acpc_template_brain", + "ANTs_prior_based_segmentation", + "func_reg_input", + "runRegisterFuncToTemplate", + "runRegisterFuncToEPI", + "fsl_linear_reg_only", + "functional_registration", + "template_for_resample", + "fnirtConfig", + "run_smoothing", + "runZScoring", + "run_longitudinal", } if key in special_cases: try: - ( - old_dict, new_dict, old_value, current_value - ) = _get_old_values(old_dict, new_dict, key) + (old_dict, new_dict, old_value, current_value) = _get_old_values( + old_dict, new_dict, key + ) except KeyError: continue # longitudinal_template_generation.run - if key == 'run_longitudinal': - if 'anat' in old_value or 'func' in old_value: + if key == "run_longitudinal": + if "anat" in old_value or "func" in old_value: current_value = True else: current_value = False # anatomical_preproc.acpc_alignment.run_before_preproc - if key == 'acpc_run_preprocessing': - current_value = True if old_value.lower( - ) == 'before' else False if old_value.lower( - ) == 'after' else None + if key == "acpc_run_preprocessing": + current_value = ( + True + if old_value.lower() == "before" + else False + if old_value.lower() == "after" + else None + ) # anatomical_preproc.acpc_alignment.acpc_target - if key == 'acpc_template_brain': - if current_value in {'None', None, ''}: + if key == "acpc_template_brain": + if current_value in {"None", None, ""}: new_dict = set_nested_value( new_dict, - ['anatomical_preproc', 'acpc_alignment', - 'acpc_target'], - 'whole-head' + ["anatomical_preproc", "acpc_alignment", "acpc_target"], + "whole-head", ) # segmentation.tissue_segmentation.using - elif key == 'ANTs_prior_based_segmentation': - new_value = _bool_to_str(old_value, 'ANTs_Prior_Based') - if new_value == 'ANTs_Prior_Based': + elif key == "ANTs_prior_based_segmentation": + new_value = _bool_to_str(old_value, "ANTs_Prior_Based") + if new_value == "ANTs_Prior_Based": new_dict = set_nested_value( new_dict, - NESTED_CONFIG_MAPPING[key][:-1] + - [new_value, 'run'], - old_value + NESTED_CONFIG_MAPPING[key][:-1] + [new_value, "run"], + old_value, ) # registration_workflows.functional_registration. # coregistration.func_input_prep.input - elif key == 'func_reg_input': - new_value = _replace_in_value_list(old_value, (' ', '_')) - current_value = _replace_in_value_list( - current_value, (' ', '_')) + elif key == "func_reg_input": + new_value = _replace_in_value_list(old_value, (" ", "_")) + current_value = _replace_in_value_list(current_value, (" ", "_")) # registration_workflows.functional_registration. # func_registration_to_template.target_template.using - elif key in { - 'runRegisterFuncToTemplate', 'runRegisterFuncToEPI' - }: - current_value = _replace_in_value_list( - current_value, (' ', '_')) - if key == 'runRegisterFuncToTemplate': + elif key in {"runRegisterFuncToTemplate", "runRegisterFuncToEPI"}: + current_value = _replace_in_value_list(current_value, (" ", "_")) + if key == "runRegisterFuncToTemplate": current_value = [ - v for v in current_value if v not in { - 'Off', 'False', False - } + v for v in current_value if v not in {"Off", "False", False} ] new_value = [] new_dict = set_nested_value( new_dict, - ['registration_workflows', - 'functional_registration', - 'func_registration_to_template', 'run'], - bool(current_value) + [ + "registration_workflows", + "functional_registration", + "func_registration_to_template", + "run", + ], + bool(current_value), ) - if key == 'runRegisterFuncToEPI': - new_value = _bool_to_str(old_value, 'EPI_template') + if key == "runRegisterFuncToEPI": + new_value = _bool_to_str(old_value, "EPI_template") # registration_workflows.anatomical_registration.registration. # using - elif key == 'fsl_linear_reg_only': - new_value = _bool_to_str(old_value, 'FSL-linear') + elif key == "fsl_linear_reg_only": + new_value = _bool_to_str(old_value, "FSL-linear") # registration_workflows.functional_registration. # func_registration_to_template.target_template. # EPI_template.EPI_template_for_resample - elif key == 'template_for_resample': + elif key == "template_for_resample": new_dict = set_nested_value( new_dict, - ['registration_workflows', 'functional_registration', - 'func_registration_to_template', 'target_template', - 'EPI_template', 'EPI_template_for_resample'], - current_value + [ + "registration_workflows", + "functional_registration", + "func_registration_to_template", + "target_template", + "EPI_template", + "EPI_template_for_resample", + ], + current_value, ) # registration_workflows.functional_registration. # EPI_registration.FSL-FNIRT.fnirt_config - elif key == 'fnirtConfig': + elif key == "fnirtConfig": current_value = old_value new_dict = set_nested_value( new_dict, - ['registration_workflows', 'functional_registration', - 'EPI_registration', 'FSL-FNIRT', 'fnirt_config'], - current_value + [ + "registration_workflows", + "functional_registration", + "EPI_registration", + "FSL-FNIRT", + "fnirt_config", + ], + current_value, ) # post_processing.spatial_smoothing.output - elif key == 'run_smoothing': - new_value = [_bool_to_str(old_value, 'smoothed')] + elif key == "run_smoothing": + new_value = [_bool_to_str(old_value, "smoothed")] if any(not bool(value) for value in old_value): - new_value.append('nonsmoothed') + new_value.append("nonsmoothed") current_value = new_value # post_processing.z-scoring.output - elif key == 'runZScoring': - new_value = [_bool_to_str(old_value, 'z-scored')] + elif key == "runZScoring": + new_value = [_bool_to_str(old_value, "z-scored")] if any(not bool(value) for value in old_value): - new_value.append('raw') + new_value.append("raw") current_value = new_value # make sure list values are cast as lists if key not in { # if key not in non-list-valued keys - 'acpc_run_preprocessing', 'acpc_template_brain', - 'functional_registration', 'template_for_resample', - 'fnirtConfig', 'run_longitudinal' + "acpc_run_preprocessing", + "acpc_template_brain", + "functional_registration", + "template_for_resample", + "fnirtConfig", + "run_longitudinal", }: current_value = _append_to_list(current_value, new_value) @@ -2301,11 +2352,12 @@ def _get_old_values(old_dict, new_dict, key): else: current_value = old_dict.pop(key) - if current_value == 'None': + if current_value == "None": current_value = None new_dict = set_nested_value( - new_dict, NESTED_CONFIG_MAPPING[key], current_value) + new_dict, NESTED_CONFIG_MAPPING[key], current_value + ) elif key in NESTED_CONFIG_DEPRECATIONS: old_dict.pop(key) return new_dict, old_dict, update_nested_dict(new_dict.copy(), old_dict) @@ -2408,29 +2460,27 @@ def update_nested_dict(d_base, d_update, fully_specified=False): ... '/cpac_templates/aal_mask_pad.nii.gz': 'Voxel' ... }, 'realignment': 'ROI_to_func'}}) True - """ # noqa: E501 # pylint: disable=line-too-long - + """ # pylint: disable=line-too-long # short-circuit if d_update has `*_roi_paths` and # `roi_paths_fully_specified` children if fully_specified: return d_update d_new = {} if d_base is None else deepcopy(d_base) for k, v in d_update.items(): - if k.endswith('_roi_paths'): - fully_specified = d_update.get('roi_paths_fully_specified', True) + if k.endswith("_roi_paths"): + fully_specified = d_update.get("roi_paths_fully_specified", True) else: fully_specified = False - if k != 'roi_paths_fully_specified': + if k != "roi_paths_fully_specified": if isinstance(v, collections.abc.Mapping): - d_new[k] = update_nested_dict(d_new.get(k, {}), v, - fully_specified) + d_new[k] = update_nested_dict(d_new.get(k, {}), v, fully_specified) else: d_new[k] = v return d_new def update_pipeline_values_1_8(d_old): - '''Function to update pipeline config values that changed from + """Function to update pipeline config values that changed from C-PAC 1.7 to 1.8. Parameters @@ -2450,64 +2500,69 @@ def update_pipeline_values_1_8(d_old): >>> update_pipeline_values_1_8({'segmentation': {'tissue_segmentation': { ... 'using': ['FSL-FAST Thresholding']}}}) {'segmentation': {'tissue_segmentation': {'using': ['FSL-FAST'], 'FSL-FAST': {'thresholding': {'use': 'Auto'}}}}} - ''' # noqa: E501 # pylint: disable=line-too-long - from CPAC.pipeline.schema import valid_options \ - # pylint: disable=import-outside-toplevel + """ # pylint: disable=line-too-long + from CPAC.pipeline.schema import valid_options + # pylint: disable=import-outside-toplevel d = replace_in_strings(d_old.copy()) d = _replace_changed_values( d, - ['anatomical_preproc', 'brain_extraction', 'using'], - [('AFNI', '3dSkullStrip'), ('FSL', 'BET'), ('unet', 'UNet')] + ["anatomical_preproc", "brain_extraction", "using"], + [("AFNI", "3dSkullStrip"), ("FSL", "BET"), ("unet", "UNet")], ) d = _replace_changed_values( d, - ['functional_preproc', 'func_masking', 'using'], - [('3dAutoMask', 'AFNI'), ('BET', 'FSL')] + ["functional_preproc", "func_masking", "using"], + [("3dAutoMask", "AFNI"), ("BET", "FSL")], ) try: - seg_use_threshold = lookup_nested_value(d, [ - 'segmentation', 'tissue_segmentation', 'using']) + seg_use_threshold = lookup_nested_value( + d, ["segmentation", "tissue_segmentation", "using"] + ) except KeyError: seg_use_threshold = [] if not isinstance(seg_use_threshold, list): seg_use_threshold = [seg_use_threshold] - if 'FSL-FAST Thresholding' in seg_use_threshold: - if 'using' in d['segmentation'].get( - 'tissue_segmentation', {} - ): - d['segmentation'][ - 'tissue_segmentation' - ]['using'].append('FSL-FAST') + if "FSL-FAST Thresholding" in seg_use_threshold: + if "using" in d["segmentation"].get("tissue_segmentation", {}): + d["segmentation"]["tissue_segmentation"]["using"].append("FSL-FAST") else: - d = set_nested_value(d, [ - 'segmentation', 'tissue_segmentation', - 'using'], ['FSL-FAST']) - seg_use_threshold.remove('FSL-FAST Thresholding') - if 'Customized Thresholding' in seg_use_threshold: - seg_use_threshold.remove('Customized Thresholding') - d = set_nested_value(d, [ - 'segmentation', 'tissue_segmentation', - 'FSL-FAST', 'thresholding', 'use'], 'Custom') + d = set_nested_value( + d, ["segmentation", "tissue_segmentation", "using"], ["FSL-FAST"] + ) + seg_use_threshold.remove("FSL-FAST Thresholding") + if "Customized Thresholding" in seg_use_threshold: + seg_use_threshold.remove("Customized Thresholding") + d = set_nested_value( + d, + ["segmentation", "tissue_segmentation", "FSL-FAST", "thresholding", "use"], + "Custom", + ) else: - d = set_nested_value(d, [ - 'segmentation', 'tissue_segmentation', - 'FSL-FAST', 'thresholding', 'use'], 'Auto') + d = set_nested_value( + d, + ["segmentation", "tissue_segmentation", "FSL-FAST", "thresholding", "use"], + "Auto", + ) - for centr in ['degree_centrality', 'eigenvector_centrality', - 'local_functional_connectivity_density']: - centr_keys = ['network_centrality', centr, 'weight_options'] + for centr in [ + "degree_centrality", + "eigenvector_centrality", + "local_functional_connectivity_density", + ]: + centr_keys = ["network_centrality", centr, "weight_options"] try: centr_value = lookup_nested_value(d, centr_keys) if any(isinstance(v, bool) for v in centr_value): for i in range(2): if centr_value[i] is True: - centr_value[i] = valid_options['centrality'][ - 'weight_options'][i] + centr_value[i] = valid_options["centrality"]["weight_options"][ + i + ] while False in centr_value: centr_value.remove(False) d = set_nested_value(d, centr_keys, centr_value) @@ -2515,17 +2570,20 @@ def update_pipeline_values_1_8(d_old): continue seg_template_key = [ - 'segmentation', 'tissue_segmentation', - 'Template_Based', 'template_for_segmentation'] + "segmentation", + "tissue_segmentation", + "Template_Based", + "template_for_segmentation", + ] try: seg_template = lookup_nested_value(d, seg_template_key) for replacement in [ - ('EPI_template', valid_options['segmentation']['template'][0]), - ('T1_template', valid_options['segmentation']['template'][1]) + ("EPI_template", valid_options["segmentation"]["template"][0]), + ("T1_template", valid_options["segmentation"]["template"][1]), ]: seg_template = list_item_replace(seg_template, *replacement) - while 'Off' in seg_template: - seg_template.remove('Off') + while "Off" in seg_template: + seg_template.remove("Off") while False in seg_template: seg_template.remove(False) d = set_nested_value(d, seg_template_key, seg_template) @@ -2533,31 +2591,31 @@ def update_pipeline_values_1_8(d_old): except KeyError: pass - distcor_key = ['functional_preproc', 'distortion_correction', 'using'] + distcor_key = ["functional_preproc", "distortion_correction", "using"] try: lookup_nested_value(d, distcor_key) d = remove_None(d, distcor_key) except KeyError: pass - if 'functional_registration' in d and isinstance( - d['functional_registration'], dict + if "functional_registration" in d and isinstance( + d["functional_registration"], dict ): - if '1-coregistration' in d['functional_registration']: - coreg = d['functional_registration'].pop('1-coregistration') + if "1-coregistration" in d["functional_registration"]: + coreg = d["functional_registration"].pop("1-coregistration") d = set_nested_value( - d, ['registration_workflows', 'functional_registration', - 'coregistration'], - coreg + d, + ["registration_workflows", "functional_registration", "coregistration"], + coreg, ) - if not(bool(d['functional_registration'])): - d.pop('functional_registration') + if not (bool(d["functional_registration"])): + d.pop("functional_registration") return update_values_from_list(d) def update_values_from_list(d_old, last_exception=None): - '''Function to convert 1-length lists of an expected type to + """Function to convert 1-length lists of an expected type to single items of that type, or to convert singletons of an expected list of a type into lists thereof. Also handles some type conversions against the schema. @@ -2582,7 +2640,7 @@ def update_values_from_list(d_old, last_exception=None): >>> update_values_from_list({'nuisance_corrections': { ... '1-ICA-AROMA': {'run': [False]}}}) {'nuisance_corrections': {'1-ICA-AROMA': {'run': [False]}}} - ''' + """ from CPAC.pipeline.schema import schema d = d_old.copy() @@ -2591,39 +2649,40 @@ def update_values_from_list(d_old, last_exception=None): schema(d) except Invalid as e: if ( - last_exception and last_exception.path == e.path and - last_exception.msg == e.msg + last_exception + and last_exception.path == e.path + and last_exception.msg == e.msg ): raise e observed = lookup_nested_value(d, e.path) - if observed == 'None': - return update_values_from_list( - set_nested_value(d, e.path, None), e) + if observed == "None": + return update_values_from_list(set_nested_value(d, e.path, None), e) - expected = e.msg.split('expected')[-1].strip( - ) if 'expected' in e.msg else 'unknown' + expected = ( + e.msg.split("expected")[-1].strip() if "expected" in e.msg else "unknown" + ) - if ( - expected != 'bool' and isinstance(observed, list) and - len(observed) == 1 - ): + if expected != "bool" and isinstance(observed, list) and len(observed) == 1: try: return update_values_from_list( - set_nested_value(d, e.path, observed[0]), e) + set_nested_value(d, e.path, observed[0]), e + ) except TypeError: raise e - if expected == 'bool': + if expected == "bool": if isinstance(observed, int): # pylint: disable=no-else-return return update_values_from_list( - set_nested_value(d, e.path, bool(observed)), e) + set_nested_value(d, e.path, bool(observed)), e + ) elif isinstance(observed, list): if len(observed) == 0: # pylint: disable=no-else-return - return update_values_from_list(set_nested_value( - d, e.path, False), e) + return update_values_from_list( + set_nested_value(d, e.path, False), e + ) else: # maintain a list if list expected - list_expected = (e.path[-1] == 0) + list_expected = e.path[-1] == 0 e_path = e.path[:-1] if list_expected else e.path if len(observed) == 1: # pylint: disable=no-else-return if isinstance(observed[0], int): @@ -2632,31 +2691,37 @@ def update_values_from_list(d_old, last_exception=None): value = True elif observed[0].lower() in YAML_BOOLS[False]: value = False - return update_values_from_list(set_nested_value( - d, e_path, [value] if list_expected else value), e) + return update_values_from_list( + set_nested_value( + d, e_path, [value] if list_expected else value + ), + e, + ) else: - return update_values_from_list(set_nested_value( - d, e_path, [bool(value) for value in observed]), e) + return update_values_from_list( + set_nested_value( + d, e_path, [bool(value) for value in observed] + ), + e, + ) elif observed.lower() in YAML_BOOLS[True]: - return update_values_from_list( - set_nested_value(d, e.path, True), e) + return update_values_from_list(set_nested_value(d, e.path, True), e) elif observed.lower() in YAML_BOOLS[False]: - return update_values_from_list( - set_nested_value(d, e.path, False), e) + return update_values_from_list(set_nested_value(d, e.path, False), e) else: return update_values_from_list( - set_nested_value(d, e_path, observed[0]), e) + set_nested_value(d, e_path, observed[0]), e + ) - elif expected == 'a list': - return update_values_from_list( - set_nested_value(d, e.path, [observed]), e) + elif expected == "a list": + return update_values_from_list(set_nested_value(d, e.path, [observed]), e) else: raise e return d def _replace_changed_values(d, nested_key, replacement_list): - '''Helper function to replace values changed from C-PAC 1.7 to C-PAC 1.8. + """Helper function to replace values changed from C-PAC 1.7 to C-PAC 1.8. Parameters ---------- @@ -2679,14 +2744,13 @@ def _replace_changed_values(d, nested_key, replacement_list): >>> d = {'test': {'this': ['function']}} >>> _replace_changed_values(d, ['test', 'this'], [('function', 'success')]) {'test': {'this': ['success']}} - ''' + """ try: current_value = lookup_nested_value(d, nested_key) except KeyError: return d if isinstance(current_value, list): - current_value = _replace_in_value_list( - current_value, replacement_list) + current_value = _replace_in_value_list(current_value, replacement_list) else: for replacement in replacement_list: current_value = list_item_replace(current_value, *replacement) @@ -2694,7 +2758,7 @@ def _replace_changed_values(d, nested_key, replacement_list): def _replace_in_value_list(current_value, replacement_tuple): - '''Helper function to make character replacements in + """Helper function to make character replacements in `current_value` and drop falsy values. Parameters @@ -2720,7 +2784,7 @@ def _replace_in_value_list(current_value, replacement_tuple): >>> _replace_in_value_list(current_value, [ ... ('AFNI', '3dSkullStrip'), ('FSL', 'BET')]) ['3dSkullStrip', 'BET'] - ''' + """ if isinstance(replacement_tuple, list): for rt in replacement_tuple: current_value = _replace_in_value_list(current_value, rt) @@ -2728,6 +2792,7 @@ def _replace_in_value_list(current_value, replacement_tuple): if not isinstance(current_value, list): current_value = [current_value] return [ - v.replace(*replacement_tuple) for v in current_value - if bool(v) and v not in {'None', 'Off', ''} + v.replace(*replacement_tuple) + for v in current_value + if bool(v) and v not in {"None", "Off", ""} ] diff --git a/CPAC/utils/versioning/__init__.py b/CPAC/utils/versioning/__init__.py index e06a9eab4d..8706789574 100644 --- a/CPAC/utils/versioning/__init__.py +++ b/CPAC/utils/versioning/__init__.py @@ -14,7 +14,7 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -"""Gather and report versions""" -from CPAC.utils.versioning.dependencies import PYTHON_PACKAGES, REPORTED, \ - REQUIREMENTS -__all__ = ['PYTHON_PACKAGES', 'REPORTED', 'REQUIREMENTS'] +"""Gather and report versions.""" +from CPAC.utils.versioning.dependencies import PYTHON_PACKAGES, REPORTED, REQUIREMENTS + +__all__ = ["PYTHON_PACKAGES", "REPORTED", "REQUIREMENTS"] diff --git a/CPAC/utils/versioning/dependencies.py b/CPAC/utils/versioning/dependencies.py index 115160336a..48237651fc 100644 --- a/CPAC/utils/versioning/dependencies.py +++ b/CPAC/utils/versioning/dependencies.py @@ -14,21 +14,22 @@ # You should have received a copy of the GNU Lesser General Public # License along with C-PAC. If not, see . -"""Gather and report dependency versions alphabetically""" +"""Gather and report dependency versions alphabetically.""" try: from importlib.metadata import distributions except ModuleNotFoundError: from importlib_metadata import distributions from pathlib import Path -from subprocess import PIPE, Popen, STDOUT +from subprocess import PIPE, STDOUT, Popen import sys -__all__ = ['PYTHON_PACKAGES', 'REPORTED', 'REQUIREMENTS'] +__all__ = ["PYTHON_PACKAGES", "REPORTED", "REQUIREMENTS"] -def cli_version(command, dependency=None, in_result=True, delimiter=' ', - formatting=None): - """Collect a version from a CLI +def cli_version( + command, dependency=None, in_result=True, delimiter=" ", formatting=None +): + """Collect a version from a CLI. Parameters ---------- @@ -52,7 +53,7 @@ def cli_version(command, dependency=None, in_result=True, delimiter=' ', {software: version} """ with Popen(command, stdout=PIPE, stderr=STDOUT, shell=True) as _command: - _version = _command.stdout.read().decode('utf-8') + _version = _command.stdout.read().decode("utf-8") _command_poll = _command.poll() if _command_poll is None or int(_command_poll) == 127: # handle missing command @@ -65,38 +66,47 @@ def cli_version(command, dependency=None, in_result=True, delimiter=' ', def first_line(stdout): - """Return first line of stdout""" - if '\n' in stdout: - return stdout.split('\n', 1)[0] + """Return first line of stdout.""" + if "\n" in stdout: + return stdout.split("\n", 1)[0] return stdout -def last_line(stdout : str) -> str: - """Return final line of stdout""" - if '\n' in stdout: - return stdout.rstrip().split('\n')[-1] +def last_line(stdout: str) -> str: + """Return final line of stdout.""" + if "\n" in stdout: + return stdout.rstrip().split("\n")[-1] return stdout def _version_sort(_version_item): - """Key to report by case-insensitive dependecy name""" + """Key to report by case-insensitive dependecy name.""" return _version_item[0].lower() -PYTHON_PACKAGES = dict(sorted({ - getattr(d, 'name', d.metadata['Name']): d.version for d in - list(distributions())}.items(), - key=_version_sort)) +PYTHON_PACKAGES = dict( + sorted( + { + getattr(d, "name", d.metadata["Name"]): d.version + for d in list(distributions()) + }.items(), + key=_version_sort, + ) +) def requirements() -> dict: - """Create a dictionary from requirements.txt""" + """Create a dictionary from requirements.txt.""" import CPAC - delimiters = ['==', ' @ ', '>='] + + delimiters = ["==", " @ ", ">="] reqs = {} try: - with open(Path(CPAC.__path__[0]).parent.joinpath('requirements.txt'), - 'r', encoding='utf8') as _req: + with open( + Path(CPAC.__path__[0]).parent.joinpath("requirements.txt"), + "r", + encoding="utf8", + ) as _req: for line in _req.readlines(): for delimiter in delimiters: if delimiter in line: @@ -105,7 +115,8 @@ def requirements() -> dict: continue except FileNotFoundError: from requests.structures import CaseInsensitiveDict - _reqs = {_req: '' for _req in CPAC.info.REQUIREMENTS} + + _reqs = {_req: "" for _req in CPAC.info.REQUIREMENTS} for _req in _reqs: _delimited = False for delimiter in delimiters: @@ -114,14 +125,22 @@ def requirements() -> dict: reqs[_package] = _version _delimited = True if not _delimited: - reqs[_req] = CaseInsensitiveDict(PYTHON_PACKAGES).get(_req, '') + reqs[_req] = CaseInsensitiveDict(PYTHON_PACKAGES).get(_req, "") return reqs -REPORTED = dict(sorted({ - **cli_version('ldd --version', formatting=first_line), - 'Python': sys.version.replace('\n', ' ').replace(' ', ' '), - **cli_version('3dECM -help', delimiter='_', - formatting=lambda _: last_line(_).split('{')[-1].rstrip('}')) -}.items(), key=_version_sort)) +REPORTED = dict( + sorted( + { + **cli_version("ldd --version", formatting=first_line), + "Python": sys.version.replace("\n", " ").replace(" ", " "), + **cli_version( + "3dECM -help", + delimiter="_", + formatting=lambda _: last_line(_).split("{")[-1].rstrip("}"), + ), + }.items(), + key=_version_sort, + ) +) REQUIREMENTS = requirements() diff --git a/CPAC/utils/workflow_serialization.py b/CPAC/utils/workflow_serialization.py index 40a3f2919e..2109cdd653 100644 --- a/CPAC/utils/workflow_serialization.py +++ b/CPAC/utils/workflow_serialization.py @@ -4,8 +4,7 @@ def cpac_flowdump_serializer( - flowdump_serializer: Callable[[object], object], - obj: object + flowdump_serializer: Callable[[object], object], obj: object ) -> object: """ Custom flowdump serializer that removes `json_data` fields @@ -13,11 +12,11 @@ def cpac_flowdump_serializer( for every node (and increase file size dramatically). """ if isinstance(obj, dict): - if 'json_data' in obj: + if "json_data" in obj: obj_clone = obj.copy() - obj_clone['json_data'] = '[truncated]' + obj_clone["json_data"] = "[truncated]" obj = obj_clone return flowdump_serializer(obj) if isinstance(obj, Configuration): - return '[C-PAC config]' + return "[C-PAC config]" return flowdump_serializer(obj) diff --git a/CPAC/vmhc/__init__.py b/CPAC/vmhc/__init__.py index 5e6c8fb330..51d5ef918e 100644 --- a/CPAC/vmhc/__init__.py +++ b/CPAC/vmhc/__init__.py @@ -1,7 +1,3 @@ +from .utils import get_img_nvols, get_operand_expression -from .utils import get_img_nvols, \ - get_operand_expression - - -__all__ = ['get_img_nvols', \ - 'get_operand_expression'] +__all__ = ["get_img_nvols", "get_operand_expression"] diff --git a/CPAC/vmhc/tests/test_vmhc.py b/CPAC/vmhc/tests/test_vmhc.py index d497fc2de4..bcdc109d9f 100644 --- a/CPAC/vmhc/tests/test_vmhc.py +++ b/CPAC/vmhc/tests/test_vmhc.py @@ -1,31 +1,35 @@ -from CPAC.vmhc.vmhc import vmhc as create_vmhc -from CPAC.utils.test_mocks import configuration_strategy_mock -from CPAC.pipeline import nipype_pipeline_engine as pe import os + import pytest +from CPAC.pipeline import nipype_pipeline_engine as pe +from CPAC.utils.test_mocks import configuration_strategy_mock +from CPAC.vmhc.vmhc import vmhc as create_vmhc + @pytest.mark.skip(reason="test needs refactoring") def test_vmhc_ants(): - - test_name = 'test_vmhc_ants' + test_name = "test_vmhc_ants" # get the config and strat for the mock - pipeline_config, strat = configuration_strategy_mock(method='ANTS') + pipeline_config, strat = configuration_strategy_mock(method="ANTS") num_strat = 0 workflow = pe.Workflow(name=test_name) workflow.base_dir = pipeline_config.workingDirectory - workflow.config['execution'] = { - 'hash_method': 'timestamp', - 'crashdump_dir': os.path.abspath(pipeline_config.crashLogDirectory) + workflow.config["execution"] = { + "hash_method": "timestamp", + "crashdump_dir": os.path.abspath(pipeline_config.crashLogDirectory), } - nodes = strat.get_nodes_names() - - print('nodes {0}'.format(nodes)) + strat.get_nodes_names() - workflow, strat = create_vmhc(workflow, num_strat, strat, pipeline_config, - output_name='vmhc_{0}'.format(num_strat)) + workflow, strat = create_vmhc( + workflow, + num_strat, + strat, + pipeline_config, + output_name="vmhc_{0}".format(num_strat), + ) workflow.run() diff --git a/CPAC/vmhc/utils.py b/CPAC/vmhc/utils.py index 2926057a51..2688d45c0d 100644 --- a/CPAC/vmhc/utils.py +++ b/CPAC/vmhc/utils.py @@ -1,23 +1,19 @@ def get_img_nvols(in_files): - """ - Calculates the number of volumes in the given nifti image + Calculates the number of volumes in the given nifti image. Parameters ---------- - in_files : string (nifti file) Returns ------- - out : int number of volumes of input nifti file """ - - out = None from nibabel import load + img = load(in_files) hdr = img.header nvols = None @@ -25,30 +21,21 @@ def get_img_nvols(in_files): nvols = int(hdr.get_data_shape()[3]) else: nvols = 1 - out = nvols - - return out + return nvols def get_operand_expression(nvols): - """ - Generates operand string + Generates operand string. Parameters ---------- - nvols : int Returns ------- - expr : string """ - - expr = None vol = int(nvols) - expr = ('a*sqrt(%d-3)' % vol) - - return expr + return "a*sqrt(%d-3)" % vol diff --git a/CPAC/vmhc/vmhc.py b/CPAC/vmhc/vmhc.py index 1d6deee976..3c547a8e2f 100644 --- a/CPAC/vmhc/vmhc.py +++ b/CPAC/vmhc/vmhc.py @@ -1,17 +1,13 @@ -import os +from nipype.interfaces import fsl +from nipype.interfaces.afni import preprocess + +from CPAC.image_utils import spatial_smoothing from CPAC.pipeline import nipype_pipeline_engine as pe from CPAC.pipeline.nodeblock import nodeblock -import nipype.algorithms.rapidart as ra -import nipype.interfaces.fsl as fsl -import nipype.interfaces.io as nio -import nipype.interfaces.utility as util -from .utils import * -from CPAC.vmhc import * -from nipype.interfaces.afni import preprocess from CPAC.registration.registration import apply_transform -from CPAC.image_utils import spatial_smoothing - from CPAC.utils.utils import check_prov_for_regtool +from CPAC.vmhc import * +from .utils import * @nodeblock( @@ -26,24 +22,22 @@ outputs=["desc-sm_bold", "fwhm"], ) def smooth_func_vmhc(wf, cfg, strat_pool, pipe_num, opt=None): - fwhm = cfg.post_processing['spatial_smoothing']['fwhm'] + fwhm = cfg.post_processing["spatial_smoothing"]["fwhm"] - smooth = spatial_smoothing(f'smooth_symmetric_{pipe_num}', - fwhm, opt=opt) + smooth = spatial_smoothing(f"smooth_symmetric_{pipe_num}", fwhm, opt=opt) - node, out = strat_pool.get_data(["desc-cleaned_bold", - "desc-brain_bold", - "desc-preproc_bold", - "bold"]) - wf.connect(node, out, smooth, 'inputspec.in_file') + node, out = strat_pool.get_data( + ["desc-cleaned_bold", "desc-brain_bold", "desc-preproc_bold", "bold"] + ) + wf.connect(node, out, smooth, "inputspec.in_file") node, out = strat_pool.get_data("space-bold_desc-brain_mask") - wf.connect(node, out, smooth, 'inputspec.mask') + wf.connect(node, out, smooth, "inputspec.mask") # 'fwhm' output for iterable outputs = { - "desc-sm_bold": (smooth, 'outputspec.out_file'), - "fwhm": (smooth, 'fwhm_input.fwhm') + "desc-sm_bold": (smooth, "outputspec.out_file"), + "fwhm": (smooth, "fwhm_input.fwhm"), } return (wf, outputs) @@ -66,48 +60,50 @@ def smooth_func_vmhc(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["space-symtemplate_desc-sm_bold"], ) def warp_timeseries_to_sym_template(wf, cfg, strat_pool, pipe_num, opt=None): - - xfm_prov = strat_pool.get_cpac_provenance( - 'from-bold_to-symtemplate_mode-image_xfm') + xfm_prov = strat_pool.get_cpac_provenance("from-bold_to-symtemplate_mode-image_xfm") reg_tool = check_prov_for_regtool(xfm_prov) - num_cpus = cfg.pipeline_setup['system_config'][ - 'max_cores_per_participant'] + num_cpus = cfg.pipeline_setup["system_config"]["max_cores_per_participant"] - num_ants_cores = cfg.pipeline_setup['system_config']['num_ants_threads'] + num_ants_cores = cfg.pipeline_setup["system_config"]["num_ants_threads"] - apply_xfm = apply_transform(f'warp_ts_to_sym_template_{pipe_num}', - reg_tool, time_series=True, num_cpus=num_cpus, - num_ants_cores=num_ants_cores) + apply_xfm = apply_transform( + f"warp_ts_to_sym_template_{pipe_num}", + reg_tool, + time_series=True, + num_cpus=num_cpus, + num_ants_cores=num_ants_cores, + ) - if reg_tool == 'ants': + if reg_tool == "ants": apply_xfm.inputs.inputspec.interpolation = cfg.registration_workflows[ - 'functional_registration']['func_registration_to_template'][ - 'ANTs_pipelines']['interpolation'] - elif reg_tool == 'fsl': + "functional_registration" + ]["func_registration_to_template"]["ANTs_pipelines"]["interpolation"] + elif reg_tool == "fsl": apply_xfm.inputs.inputspec.interpolation = cfg.registration_workflows[ - 'functional_registration']['func_registration_to_template'][ - 'FNIRT_pipelines']['interpolation'] + "functional_registration" + ]["func_registration_to_template"]["FNIRT_pipelines"]["interpolation"] # smoothed BOLD - connect, resource = strat_pool.get_data(["desc-cleaned-sm_bold", - "desc-brain-sm_bold", - "desc-preproc-sm_bold", - "desc-sm_bold"], - report_fetched=True) + connect, resource = strat_pool.get_data( + [ + "desc-cleaned-sm_bold", + "desc-brain-sm_bold", + "desc-preproc-sm_bold", + "desc-sm_bold", + ], + report_fetched=True, + ) node, out = connect - wf.connect(node, out, apply_xfm, 'inputspec.input_image') + wf.connect(node, out, apply_xfm, "inputspec.input_image") node, out = strat_pool.get_data("T1w-brain-template-symmetric") - wf.connect(node, out, apply_xfm, 'inputspec.reference') + wf.connect(node, out, apply_xfm, "inputspec.reference") node, out = strat_pool.get_data("from-bold_to-symtemplate_mode-image_xfm") - wf.connect(node, out, apply_xfm, 'inputspec.transform') + wf.connect(node, out, apply_xfm, "inputspec.transform") - outputs = { - f'space-symtemplate_{resource}': - (apply_xfm, 'outputspec.output_image') - } + outputs = {f"space-symtemplate_{resource}": (apply_xfm, "outputspec.output_image")} return (wf, outputs) @@ -127,43 +123,45 @@ def warp_timeseries_to_sym_template(wf, cfg, strat_pool, pipe_num, opt=None): outputs=["vmhc"], ) def vmhc(wf, cfg, strat_pool, pipe_num, opt=None): - '''Compute Voxel-Mirrored Homotopic Connectivity. + """Compute Voxel-Mirrored Homotopic Connectivity. VMHC is the map of brain functional homotopy, the high degree of synchrony in spontaneous activity between geometrically corresponding interhemispheric (i.e., homotopic) regions. - ''' - + """ # write out a swapped version of the file # copy and L/R swap file - copy_and_L_R_swap = pe.Node(interface=fsl.SwapDimensions(), - name=f'copy_and_L_R_swap_{pipe_num}', - mem_gb=3.0) + copy_and_L_R_swap = pe.Node( + interface=fsl.SwapDimensions(), name=f"copy_and_L_R_swap_{pipe_num}", mem_gb=3.0 + ) - copy_and_L_R_swap.inputs.new_dims = ('-x', 'y', 'z') + copy_and_L_R_swap.inputs.new_dims = ("-x", "y", "z") - node, out = strat_pool.get_data(["space-symtemplate_desc-cleaned-sm_bold", - "space-symtemplate_desc-brain-sm_bold", - "space-symtemplate_desc-preproc-sm_bold", - "space-symtemplate_desc-sm_bold"]) - wf.connect(node, out, copy_and_L_R_swap, 'in_file') + node, out = strat_pool.get_data( + [ + "space-symtemplate_desc-cleaned-sm_bold", + "space-symtemplate_desc-brain-sm_bold", + "space-symtemplate_desc-preproc-sm_bold", + "space-symtemplate_desc-sm_bold", + ] + ) + wf.connect(node, out, copy_and_L_R_swap, "in_file") # calculate correlation between original and swapped images - pearson_correlation = pe.Node(interface=preprocess.TCorrelate(), - name=f'pearson_correlation_{pipe_num}', - mem_gb=3.0) + pearson_correlation = pe.Node( + interface=preprocess.TCorrelate(), + name=f"pearson_correlation_{pipe_num}", + mem_gb=3.0, + ) pearson_correlation.inputs.pearson = True pearson_correlation.inputs.polort = -1 - pearson_correlation.inputs.outputtype = 'NIFTI_GZ' + pearson_correlation.inputs.outputtype = "NIFTI_GZ" - wf.connect(node, out, pearson_correlation, 'xset') + wf.connect(node, out, pearson_correlation, "xset") - wf.connect(copy_and_L_R_swap, 'out_file', - pearson_correlation, 'yset') + wf.connect(copy_and_L_R_swap, "out_file", pearson_correlation, "yset") - outputs = { - 'vmhc': (pearson_correlation, 'out_file') - } + outputs = {"vmhc": (pearson_correlation, "out_file")} return (wf, outputs) diff --git a/dev/ami_data/setup.sh b/dev/ami_data/setup.sh index 88f812fde6..2a6422402e 100644 --- a/dev/ami_data/setup.sh +++ b/dev/ami_data/setup.sh @@ -55,4 +55,4 @@ until [ `aws ec2 --region=${REGION} describe-images --filters "Name=image-id,Val done aws ec2 --region=${REGION} create-tags --resources ${INSTANCE_ID} --tags Key=ami,Value=done --output json -echo "Image build!" \ No newline at end of file +echo "Image build!" diff --git a/dev/ami_data/setup_cpac.sh b/dev/ami_data/setup_cpac.sh index a7aff980da..fb5a19952b 100644 --- a/dev/ami_data/setup_cpac.sh +++ b/dev/ami_data/setup_cpac.sh @@ -3,7 +3,7 @@ set -e wget -O /etc/apt/sources.list.d/neurodebian.sources.list http://neuro.debian.net/lists/bionic.us-ca.full -for i in {1..5}; do +for i in {1..5}; do apt-key adv --recv-keys --keyserver hkp://pool.sks-keyservers.net:80 0xA5D32F012649A5A9 && break || sleep 5; done @@ -29,7 +29,7 @@ cat < /etc/lightdm/lightdm.conf [Seat:*] pam-service=lightdm pam-autologin-service=lightdm-autologin -autologin-user=ubuntu +autologin-user=ubuntu autologin-user-timeout=0 session-wrapper=/etc/X11/Xsession greeter-session=lightdm-greeter @@ -126,7 +126,7 @@ wget -q -O /tmp/libpng12.deb http://mirrors.kernel.org/ubuntu/pool/main/libp/lib && dpkg -i /tmp/libpng12.deb \ && rm /tmp/libpng12.deb -libs_path=/usr/lib/x86_64-linux-gnu +libs_path=/usr/lib/x86_64-linux-gnu if [ -f $libs_path/libgsl.so.19 ]; then \ ln $libs_path/libgsl.so.19 $libs_path/libgsl.so.0; \ fi diff --git a/dev/build_docker.sh b/dev/build_docker.sh index d345e6c3b3..b74f93c370 100755 --- a/dev/build_docker.sh +++ b/dev/build_docker.sh @@ -11,4 +11,4 @@ VERSION=$1 docker build -t fcpindi/c-pac:latest . docker tag fcpindi/c-pac:latest fcpindi/c-pac:v$VERSION # docker push fcpindi/c-pac:latest -# docker push fcpindi/c-pac:v$VERSION \ No newline at end of file +# docker push fcpindi/c-pac:v$VERSION diff --git a/dev/bump_version.sh b/dev/bump_version.sh index ccf755fdf1..c0d22ed60e 100755 --- a/dev/bump_version.sh +++ b/dev/bump_version.sh @@ -26,4 +26,4 @@ rename_patterns () { } export -f rename_patterns -find -regex ".*\.ya?ml$" -exec bash -c "rename_patterns "${VERSION}" \"\$0\"" {} \; \ No newline at end of file +find -regex ".*\.ya?ml$" -exec bash -c "rename_patterns "${VERSION}" \"\$0\"" {} \; diff --git a/dev/circleci_data/data_settings_bids_examples_ds051_default_BIDS.yml b/dev/circleci_data/data_settings_bids_examples_ds051_default_BIDS.yml index 4a09acd864..5449692350 100644 --- a/dev/circleci_data/data_settings_bids_examples_ds051_default_BIDS.yml +++ b/dev/circleci_data/data_settings_bids_examples_ds051_default_BIDS.yml @@ -20,26 +20,26 @@ bidsBaseDir: ./bids-examples/ds051 # File Path Template for Anatomical Files # Custom Data Format only. -# +# # Place tags for the appropriate data directory levels with the tags {site}, {participant}, and {session}. Only {participant} is required. -# +# # Examples: # /data/{site}/{participant}/{session}/anat/mprage.nii.gz # /data/{site}/{participant}/anat.nii.gz -# +# # See the User Guide for more detailed instructions. anatomicalTemplate: None # File Path Template for Functional Files # Custom Data Format only. -# +# # Place tags for the appropriate data directory levels with the tags {site}, {participant}, {session}, and {series}. Only {participant} is required. -# +# # Examples: # /data/{site}/{participant}/{session}/func/{series}_bold.nii.gz # /data/{site}/{participant}/{series}/func.nii.gz -# +# # See the User Guide for more detailed instructions. functionalTemplate: None @@ -124,66 +124,64 @@ fieldMapMagnitude: None # Include only a sub-set of the participants present in the folders defined above. -# +# # List participants in this box (ex: sub101, sub102) or provide the path to a text file with one participant ID on each line. -# +# # If 'None' is specified, CPAC will include all participants. subjectList: None # Exclude a sub-set of the participants present in the folders defined above. -# +# # List participants in this box (ex: sub101, sub102) or provide the path to a text file with one participant ID on each line. -# +# # If 'None' is specified, CPAC will not exclude any participants. exclusionSubjectList: None # Include only a sub-set of the sites present in the folders defined above. -# +# # List sites in this box (ex: NYU, UCLA) or provide the path to a text file with one site name on each line. -# +# # If 'None' is specified, CPAC will include all sites. siteList: None # Exclude a sub-set of the sites present in the folders defined above. -# +# # List sites in this box (ex: NYU, UCLA) or provide the path to a text file with one site name on each line. -# +# # If 'None' is specified, CPAC will include all sites. exclusionSiteList: None # Include only a sub-set of the sessions present in the folders defined above. -# +# # List sessions in this box (ex: session-1, session-2) or provide the path to a text file with one session name on each line. -# +# # If 'None' is specified, CPAC will include all sessions. sessionList: None # Exclude a sub-set of the sessions present in the folders defined above. -# +# # List sessions in this box (ex: session-1, session-2) or provide the path to a text file with one session name on each line. -# +# # If 'None' is specified, CPAC will include all sessions. exclusionSessionList: None # Include only a sub-set of the series present in the folders defined above. -# +# # List series in this box (ex: func-1, func-2) or provide the path to a text file with one series name on each line. -# +# # If 'None' is specified, CPAC will include all series. scanList: None # Exclude a sub-set of the series present in the folders defined above. -# +# # List series in this box (ex: func-1, func-2) or provide the path to a text file with one series name on each line. -# +# # If 'None' is specified, CPAC will include all series. exclusionScanList: None - - diff --git a/dev/circleci_data/generate_run_command.py b/dev/circleci_data/generate_run_command.py index 892f883b1a..ef3153a45c 100644 --- a/dev/circleci_data/generate_run_command.py +++ b/dev/circleci_data/generate_run_command.py @@ -1,12 +1,11 @@ import os import random -import stat -import yaml - from warnings import warn +import yaml + -def get_random_subject(species='human'): +def get_random_subject(species="human"): """ Function to get a random data config file and subject for a given species. @@ -23,16 +22,15 @@ def get_random_subject(species='human'): participant_ndx: int """ - if species == 'human': + if species == "human": data_config_file = ( - 'CPAC/resources/configs/test_configs/' - 'data-test_4-projects_5-subjects.yml' + "CPAC/resources/configs/test_configs/" "data-test_4-projects_5-subjects.yml" ) else: raise NotImplementedError( - f'Data configurations not yet set for random test of {species}' + f"Data configurations not yet set for random test of {species}" ) - with open(data_config_file, 'r') as data_config: + with open(data_config_file, "r") as data_config: subject_list = yaml.safe_load(data_config) return (data_config_file, random.randrange(len(subject_list))) @@ -52,66 +50,68 @@ def get_random_test_run_command(): """ # collect preconfigs all_configs = { - 'default', + "default", *{ - config[16:-4] for config in os.listdir( - 'CPAC/resources/configs' - ) if config.startswith('pipeline_config') - } + config[16:-4] + for config in os.listdir("CPAC/resources/configs") + if config.startswith("pipeline_config") + }, } # choose a random preconfig random_config = random.choice(list(all_configs)) - config_string = '' if ( - random_config == 'default' - ) else f'--preconfig {random_config}' + config_string = ( + "" if (random_config == "default") else f"--preconfig {random_config}" + ) # determine appropriate species - if random_config in {'nhp-macaque', 'monkey'}: - data_species = 'nhp' - elif random_config == 'rodent': - data_species = 'rodent' + if random_config in {"nhp-macaque", "monkey"}: + data_species = "nhp" + elif random_config == "rodent": + data_species = "rodent" else: - data_species = 'human' + data_species = "human" try: data_config_file, participant_ndx = get_random_subject(data_species) - command = ' '.join([ - 'python -m coverage run /code/dev/docker_data/run.py', - '/home/circleci/project', - '/home/circleci/project/outputs participant', - f'--save_working_dir --data_config_file {data_config_file}', - f'{config_string} --n_cpus 1 --mem_gb 12'.lstrip(), - f'--participant_ndx {participant_ndx}' - ]) + command = " ".join( + [ + "python -m coverage run /code/dev/docker_data/run.py", + "/home/circleci/project", + "/home/circleci/project/outputs participant", + f"--save_working_dir --data_config_file {data_config_file}", + f"{config_string} --n_cpus 1 --mem_gb 12".lstrip(), + f"--participant_ndx {participant_ndx}", + ] + ) except NotImplementedError as nie: # pass error along to user as warning, but don't fail warn(nie, Warning) - command = ( - f'echo "{nie}, which we need for \'--preconfig {random_config}\'"' - ) + command = f"echo \"{nie}, which we need for '--preconfig {random_config}'\"" return command -if __name__ == '__main__': - fp = os.path.join(os.path.dirname(__file__), 'run_command.sh') +if __name__ == "__main__": + fp = os.path.join(os.path.dirname(__file__), "run_command.sh") run_string = get_random_test_run_command() - with open(fp, 'w') as run_command: - run_command.write('\n'.join([ - '#!/bin/bash\n', - '# install testing requirements', - 'pip install -r /code/dev/circleci_data/requirements.txt\n\n' - ])) - if run_string[:4] != 'echo': - run_command.write('\n'.join([ - '# run one participant with coverage', - run_string, - '' - ])) + with open(fp, "w") as run_command: + run_command.write( + "\n".join( + [ + "#!/bin/bash\n", + "# install testing requirements", + "pip install -r /code/dev/circleci_data/requirements.txt\n\n", + ] + ) + ) + if run_string[:4] != "echo": + run_command.write( + "\n".join(["# run one participant with coverage", run_string, ""]) + ) else: - with open('run_warning.py', 'w') as warning_script: - warning_script.write(f'print({run_string[5:]})') - run_command.write('python -m coverage run run_warning.py') + with open("run_warning.py", "w") as warning_script: + warning_script.write(f"print({run_string[5:]})") + run_command.write("python -m coverage run run_warning.py") # ↓ chmod +x ↓ - os.chmod(fp, os.stat(fp).st_mode | 0o0111) \ No newline at end of file + os.chmod(fp, os.stat(fp).st_mode | 0o0111) diff --git a/dev/circleci_data/pipe-test_ci.yml b/dev/circleci_data/pipe-test_ci.yml index 19823076be..3ae1e4103f 100644 --- a/dev/circleci_data/pipe-test_ci.yml +++ b/dev/circleci_data/pipe-test_ci.yml @@ -10,12 +10,12 @@ FROM: default -pipeline_setup: +pipeline_setup: # Name for this pipeline configuration - useful for identification. pipeline_name: pipe-test_all - working_directory: + working_directory: # Directory where C-PAC should store temporary and intermediate files. # - This directory must be saved if you wish to re-run your pipeline from where you left off (if not completed). @@ -27,22 +27,22 @@ pipeline_setup: # - This can be written to '/tmp' if you do not intend to save your working directory. path: ./cpac_runs/work - log_directory: + log_directory: path: ./cpac_runs/default/log - crash_log_directory: + crash_log_directory: # Directory where CPAC should write crash logs. path: ./cpac_runs/crash - system_config: + system_config: # Select Off if you intend to run CPAC on a single machine. # If set to On, CPAC will attempt to submit jobs through the job scheduler / resource manager selected below. - on_grid: + on_grid: - SGE: + SGE: # SGE Parallel Environment to use when running CPAC. # Only applies when you are running on a grid or compute cluster using SGE. @@ -57,7 +57,7 @@ pipeline_setup: # 'Number of Participants to Run Simultaneously' is as much RAM you can safely allocate. maximum_memory_per_participant: 3 - Amazon-AWS: + Amazon-AWS: # Enable server-side 256-AES encryption on data to the S3 bucket s3_encryption: On @@ -69,9 +69,9 @@ FSLDIR: FSLDIR # PREPROCESSING # ------------- -anatomical_preproc: +anatomical_preproc: - registration_workflow: + registration_workflow: # Template to be used during registration. # It is not necessary to change this path unless you intend to use a non-standard template. @@ -81,26 +81,26 @@ anatomical_preproc: # It is not necessary to change this path unless you intend to use a non-standard template. template_skull_for_anat: $FSLDIR/data/standard/MNI152_T1_${anatomical_preproc.registration_workflow.resolution_for_anat}.nii.gz - registration: + registration: - FSL-FNIRT: + FSL-FNIRT: # Configuration file to be used by FSL to set FNIRT parameters. # It is not necessary to change this path unless you intend to use custom FNIRT parameters or a non-standard template. ref_mask: $FSLDIR/data/standard/MNI152_T1_${anatomical_preproc.registration_workflow.resolution_for_anat}_brain_mask_dil.nii.gz -nuisance_corrections: +nuisance_corrections: - 1-ICA-AROMA: + 1-ICA-AROMA: # this is a fork point # run: [On, Off] - this will run both and fork the pipeline run: [Off] - 2-nuisance_regression: + 2-nuisance_regression: # Select which nuisance signal corrections to apply - Regressors: + Regressors: - Bandpass: bottom_frequency: 0.01 top_frequency: 0.1 @@ -148,22 +148,22 @@ nuisance_corrections: - WhiteMatter - CerebrospinalFluid -functional_registration: +functional_registration: - 1-coregistration: + 1-coregistration: - boundary_based_registration: + boundary_based_registration: # Standard FSL 5.0 Scheduler used for Boundary Based Registration. # It is not necessary to change this path unless you intend to use non-standard MNI registration. bbr_schedule: $FSLDIR/etc/flirtsch/bbr.sch - 2-func_registration_to_template: + 2-func_registration_to_template: - target_template: + target_template: # option parameters - T1_template: + T1_template: # Standard Skull Stripped Template. Used as a reference image for functional registration. # This can be different than the template used as the reference/fixed for T1-to-template registration. @@ -173,7 +173,7 @@ functional_registration: # This can be different than the template used as the reference/fixed for T1-to-template registration. template_skull: $FSLDIR/data/standard/MNI152_T1_${functional_registration.2-func_registration_to_template.output_resolution.func_preproc_outputs}.nii.gz - FNIRT_pipelines: + FNIRT_pipelines: # Identity matrix used during FSL-based resampling of functional-space data throughout the pipeline. # It is not necessary to change this path unless you intend to use a different template. @@ -181,37 +181,37 @@ functional_registration: # OUTPUTS AND DERIVATIVES # ----------------------- -post_processing: +post_processing: - spatial_smoothing: + spatial_smoothing: # Smooth the derivative outputs. run: [On] - z-scoring: + z-scoring: # z-score standardize the derivatives. This may be needed for group-level analysis. run: [On] -timeseries_extraction: +timeseries_extraction: # Enter paths to region-of-interest (ROI) NIFTI files (.nii or .nii.gz) to be used for time-series extraction, and then select which types of analyses to run. # Denote which analyses to run for each ROI path by listing the names below. For example, if you wish to run Avg and SpatialReg, you would enter: '/path/to/ROI.nii.gz': Avg, SpatialReg # available analyses: # /path/to/atlas.nii.gz: Avg, Voxel, SpatialReg - tse_roi_paths: + tse_roi_paths: -seed_based_correlation_analysis: +seed_based_correlation_analysis: # Enter paths to region-of-interest (ROI) NIFTI files (.nii or .nii.gz) to be used for time-series extraction, and then select which types of analyses to run. # Denote which analyses to run for each ROI path by listing the names below. For example, if you wish to run Avg and MultReg, you would enter: '/path/to/ROI.nii.gz': Avg, MultReg # available analyses: # /path/to/atlas.nii.gz: Avg, DualReg, MultReg - sca_roi_paths: + sca_roi_paths: -voxel_mirrored_homotopic_connectivity: +voxel_mirrored_homotopic_connectivity: - symmetric_registration: + symmetric_registration: # Included as part of the 'Image Resource Files' package available on the Install page of the User Guide. # It is not necessary to change this path unless you intend to use a non-standard symmetric template. @@ -225,7 +225,7 @@ voxel_mirrored_homotopic_connectivity: # It is not necessary to change this path unless you intend to use a non-standard symmetric template. dilated_symmetric_brain_mask: $FSLDIR/data/standard/MNI152_T1_${anatomical_preproc.registration_workflow.resolution_for_anat}_brain_mask_symmetric_dil.nii.gz -network_centrality: +network_centrality: # Maximum amount of RAM (in GB) to be used when calculating Degree Centrality. # Calculating Eigenvector Centrality will require additional memory based on the size of the mask or number of ROI nodes. @@ -234,7 +234,7 @@ network_centrality: # Full path to a NIFTI file describing the mask. Centrality will be calculated for all voxels within the mask. template_specification_file: s3://fcp-indi/resources/cpac/resources/mask-thr50-3mm.nii.gz - eigenvector_centrality: + eigenvector_centrality: # Enable/Disable eigenvector centrality by selecting the connectivity weights # weight_options: ['Binarized', 'Weighted'] @@ -242,7 +242,7 @@ network_centrality: # weight_options: [] weight_options: [Binarized, Weighted] - local_functional_connectivity_density: + local_functional_connectivity_density: # Select the type of threshold used when creating the lFCD adjacency matrix. # options: @@ -257,7 +257,7 @@ network_centrality: # PACKAGE INTEGRATIONS # -------------------- -PyPEER: +PyPEER: # Template-space eye mask - eye_mask_path: $FSLDIR/data/standard/MNI152_T1_${functional_registration.2-func_registration_to_template.output_resolution.func_preproc_outputs}_eye_mask.nii.gz \ No newline at end of file + eye_mask_path: $FSLDIR/data/standard/MNI152_T1_${functional_registration.2-func_registration_to_template.output_resolution.func_preproc_outputs}_eye_mask.nii.gz diff --git a/dev/circleci_data/python_2_pickle.pkl b/dev/circleci_data/python_2_pickle.pkl index 149df4dd98..58f8bd0d02 100644 --- a/dev/circleci_data/python_2_pickle.pkl +++ b/dev/circleci_data/python_2_pickle.pkl @@ -92,4 +92,4 @@ sS'scanList' p46 S'None' p47 -sa. \ No newline at end of file +sa. diff --git a/dev/circleci_data/requirements.txt b/dev/circleci_data/requirements.txt index da690102d4..b59c3413be 100644 --- a/dev/circleci_data/requirements.txt +++ b/dev/circleci_data/requirements.txt @@ -6,4 +6,4 @@ pytest_click pytest-timeout pyyaml semver -spython >= 0.0.81 \ No newline at end of file +spython >= 0.0.81 diff --git a/dev/circleci_data/test_external_utils.py b/dev/circleci_data/test_external_utils.py index 567d4dff9f..e142ad161e 100644 --- a/dev/circleci_data/test_external_utils.py +++ b/dev/circleci_data/test_external_utils.py @@ -1,7 +1,6 @@ import os -import sys - from pathlib import Path +import sys import click import pytest @@ -9,58 +8,60 @@ CPAC_DIR = str(Path(__file__).parent.parent.parent) sys.path.append(CPAC_DIR) -DATA_DIR = os.path.join(CPAC_DIR, 'dev', 'circleci_data') +DATA_DIR = os.path.join(CPAC_DIR, "dev", "circleci_data") + +from CPAC.__main__ import utils as CPAC_main_utils -from CPAC.__main__ import utils as CPAC_main_utils \ - # noqa: E402 # pylint: disable=wrong-import-position +# pylint: disable=wrong-import-position def _click_backport(command, key): - """Switch back to underscores for older versions of click""" - return _resolve_alias(command, - key.replace('-', '_').replace('opt_out', 'opt-out')) + """Switch back to underscores for older versions of click.""" + return _resolve_alias(command, key.replace("-", "_").replace("opt_out", "opt-out")) def _compare_version(left: str, right: str) -> int: - """Handle required verbosity after ``semver.compare`` deprecation""" - return semver.Version.compare(semver.Version.parse(left), - semver.Version.parse(right)) + """Handle required verbosity after ``semver.compare`` deprecation.""" + return semver.Version.compare( + semver.Version.parse(left), semver.Version.parse(right) + ) try: - _BACKPORT_CLICK = _compare_version(click.__version__, '7.0.0') < 0 + _BACKPORT_CLICK = _compare_version(click.__version__, "7.0.0") < 0 except ValueError: try: - _BACKPORT_CLICK = _compare_version(f'{click.__version__}.0', - '7.0.0') < 0 + _BACKPORT_CLICK = _compare_version(f"{click.__version__}.0", "7.0.0") < 0 except ValueError: _BACKPORT_CLICK = True def _resolve_alias(command, key): - """Resolve alias if possible""" - return command.resolve_alias(key) if hasattr(command, - 'resolve_alias') else key + """Resolve alias if possible.""" + return command.resolve_alias(key) if hasattr(command, "resolve_alias") else key -@pytest.mark.parametrize('multiword_connector', ['-', '_']) +@pytest.mark.parametrize("multiword_connector", ["-", "_"]) def test_build_data_config(cli_runner, multiword_connector): """Test CLI ``utils data-config new-settings-template`` and - ``utils data_config new_settings_template``""" - if multiword_connector == '-' and _BACKPORT_CLICK: + ``utils data_config new_settings_template``. + """ + if multiword_connector == "-" and _BACKPORT_CLICK: return os.chdir(DATA_DIR) test_yaml = os.path.join(DATA_DIR, "data_settings.yml") _delete_test_yaml(test_yaml) - if multiword_connector == '_': + if multiword_connector == "_": data_config = CPAC_main_utils.commands[ - _click_backport(CPAC_main_utils, 'data-config')] + _click_backport(CPAC_main_utils, "data-config") + ] result = cli_runner.invoke( - data_config.commands[ - _click_backport(data_config, 'new-settings-template')]) + data_config.commands[_click_backport(data_config, "new-settings-template")] + ) else: - result = cli_runner.invoke(CPAC_main_utils.commands[ - 'data-config'].commands['new-settings-template']) + result = cli_runner.invoke( + CPAC_main_utils.commands["data-config"].commands["new-settings-template"] + ) assert result.exit_code == 0 assert result.output.startswith( @@ -73,27 +74,23 @@ def test_build_data_config(cli_runner, multiword_connector): def test_new_settings_template(cli_runner): os.chdir(CPAC_DIR) - example_dir = os.path.join(CPAC_DIR, 'bids-examples') + example_dir = os.path.join(CPAC_DIR, "bids-examples") if not os.path.exists(example_dir): from git import Repo + Repo.clone_from( - "https://github.com/bids-standard/bids-examples.git", - example_dir + "https://github.com/bids-standard/bids-examples.git", example_dir ) result = cli_runner.invoke( CPAC_main_utils.commands[ - _click_backport(CPAC_main_utils, 'data-config') - ].commands['build'], - [os.path.join( - DATA_DIR, - "data_settings_bids_examples_ds051_default_BIDS.yml" - )] + _click_backport(CPAC_main_utils, "data-config") + ].commands["build"], + [os.path.join(DATA_DIR, "data_settings_bids_examples_ds051_default_BIDS.yml")], ) participant_yaml = os.path.join(DATA_DIR, "data_config_ds051.yml") - group_yaml = os.path.join(DATA_DIR, - "group_analysis_participants_ds051.txt") + group_yaml = os.path.join(DATA_DIR, "group_analysis_participants_ds051.txt") assert result.exit_code == 0 assert result.output.startswith("\nGenerating data configuration file..") @@ -104,39 +101,33 @@ def test_new_settings_template(cli_runner): def test_repickle(cli_runner): - fn = 'python_2_pickle.pkl' + fn = "python_2_pickle.pkl" pickle_path = os.path.join(DATA_DIR, fn) - backups = [_Backup(pickle_path), _Backup(f'{pickle_path}z')] + backups = [_Backup(pickle_path), _Backup(f"{pickle_path}z")] - result = cli_runner.invoke( - CPAC_main_utils.commands['repickle'], - [DATA_DIR] - ) + result = cli_runner.invoke(CPAC_main_utils.commands["repickle"], [DATA_DIR]) assert result.exit_code == 0 assert ( - f'Converted pickle {fn} from a Python 2 pickle to a Python 3 ' - 'pickle.' in result.output + f"Converted pickle {fn} from a Python 2 pickle to a Python 3 " + "pickle." in result.output ) - result = cli_runner.invoke( - CPAC_main_utils.commands['repickle'], - [DATA_DIR] - ) + result = cli_runner.invoke(CPAC_main_utils.commands["repickle"], [DATA_DIR]) assert result.exit_code == 0 - assert f'Pickle {fn} is a Python 3 pickle.' in result.output + assert f"Pickle {fn} is a Python 3 pickle." in result.output [backup.restore() for backup in backups] -class _Backup(): +class _Backup: def __init__(self, filepath): self.path = filepath - with open(self.path, 'rb') as r: + with open(self.path, "rb") as r: self.data = r.read() def restore(self): - with open(self.path, 'wb') as w: + with open(self.path, "wb") as w: w.write(self.data) diff --git a/dev/circleci_data/test_in_image.sh b/dev/circleci_data/test_in_image.sh index 1afca588e4..b62de84994 100755 --- a/dev/circleci_data/test_in_image.sh +++ b/dev/circleci_data/test_in_image.sh @@ -10,4 +10,4 @@ echo "$?" > test-results/exitcode echo "coverage saved to ${COVERAGE_FILE}" -exit $(cat test-results/exitcode) \ No newline at end of file +exit $(cat test-results/exitcode) diff --git a/dev/circleci_data/test_install.py b/dev/circleci_data/test_install.py index 135c02998b..4c1e7b68ff 100644 --- a/dev/circleci_data/test_install.py +++ b/dev/circleci_data/test_install.py @@ -1,25 +1,25 @@ import os - from pathlib import Path + from spython.main import Client def test_AFNI_libraries(): - SINGULARITY_IMAGE_PATH = '/home/circleci/project/C-PAC-CI.simg' + SINGULARITY_IMAGE_PATH = "/home/circleci/project/C-PAC-CI.simg" if not os.path.exists(SINGULARITY_IMAGE_PATH): try: - SINGULARITY_IMAGE_PATH = [d for d in os.listdir( - str(Path(__file__).parent.parent.parent) - ) if (d.endswith('.simg') or d.endswith('.sif'))][0] + SINGULARITY_IMAGE_PATH = next( + d + for d in os.listdir(str(Path(__file__).parent.parent.parent)) + if (d.endswith(".simg") or d.endswith(".sif")) + ) except: raise Exception("Singularity image not in expected location.") if os.path.exists(SINGULARITY_IMAGE_PATH): afni_libraries = Client.execute( Client.instance(SINGULARITY_IMAGE_PATH), - ['./dev/circleci_data/AFNI_libraries.sh'] + ["./dev/circleci_data/AFNI_libraries.sh"], + ) + assert "not found" not in afni_libraries, "\n".join( + [line.strip() for line in afni_libraries.split("\n") if "not found" in line] ) - assert "not found" not in afni_libraries, '\n'.join([ - line.strip() for line in afni_libraries.split( - '\n' - ) if "not found" in line - ]) \ No newline at end of file diff --git a/dev/create_ami.sh b/dev/create_ami.sh index 27b4a4f732..bbb6b0dfbb 100755 --- a/dev/create_ami.sh +++ b/dev/create_ami.sh @@ -161,8 +161,8 @@ else fi if (( $ELAPSED > 14400 )); then # 4 hours echo "It should not take that much (more then 4h). Please take a look." - read + read fi sleep 5 done -fi \ No newline at end of file +fi diff --git a/dev/docker_data/checksum/AFNI.23.3.09.sha384 b/dev/docker_data/checksum/AFNI.23.3.09.sha384 index 0d2747c171..eb2633422f 100644 --- a/dev/docker_data/checksum/AFNI.23.3.09.sha384 +++ b/dev/docker_data/checksum/AFNI.23.3.09.sha384 @@ -1 +1 @@ -dcc1416af72c90636ab4b5d94d28b7c5bd64c7f1c95bbe7b38b011e7e1759a8b90de095f1ee4716e5cfad4949fe892d0 afni-AFNI_23.3.09.tar.gz \ No newline at end of file +dcc1416af72c90636ab4b5d94d28b7c5bd64c7f1c95bbe7b38b011e7e1759a8b90de095f1ee4716e5cfad4949fe892d0 afni-AFNI_23.3.09.tar.gz diff --git a/dev/docker_data/checksum/ANTs.2.4.3.sha384 b/dev/docker_data/checksum/ANTs.2.4.3.sha384 index d8d1b0bb87..8b38cc4888 100644 --- a/dev/docker_data/checksum/ANTs.2.4.3.sha384 +++ b/dev/docker_data/checksum/ANTs.2.4.3.sha384 @@ -1,2 +1,2 @@ 16f55cd622ff9fa99b368fc6ad860dfadaf371ac4671c1c775f03c00c25cd544ae53cc351545d4bd32aa51ae3aead09c /tmp/ANTs.zip -f42da7425908cdeef288a78545ae6cff5dd2bfdfbe7c6bd07e16706db86c91f6e398000bd1fd08589c4ef0ca4504fcb8 /tmp/Oasis.zip \ No newline at end of file +f42da7425908cdeef288a78545ae6cff5dd2bfdfbe7c6bd07e16706db86c91f6e398000bd1fd08589c4ef0ca4504fcb8 /tmp/Oasis.zip diff --git a/dev/docker_data/checksum/FSL.data.sha384 b/dev/docker_data/checksum/FSL.data.sha384 index 39a552cebd..c5c5be25d1 100644 --- a/dev/docker_data/checksum/FSL.data.sha384 +++ b/dev/docker_data/checksum/FSL.data.sha384 @@ -1 +1 @@ -327b022a81f54fda915c530a632ec5501bd14325fb8f10d5be14be186f92866c5851d90b1a648efb0cd177d33733d108 /tmp/cpac_resources.tar.gz \ No newline at end of file +327b022a81f54fda915c530a632ec5501bd14325fb8f10d5be14be186f92866c5851d90b1a648efb0cd177d33733d108 /tmp/cpac_resources.tar.gz diff --git a/dev/docker_data/checksum/ICA-AROMA.0.4.4.sha384 b/dev/docker_data/checksum/ICA-AROMA.0.4.4.sha384 index 0045192c35..21468e94d3 100644 --- a/dev/docker_data/checksum/ICA-AROMA.0.4.4.sha384 +++ b/dev/docker_data/checksum/ICA-AROMA.0.4.4.sha384 @@ -1 +1 @@ -64d182286c70a7a0abbd02d011adff9c4e13924db8e69248e1f9f15c4c7a45414d5b61bd687ccbc2ab1c1357e9649dc9 /tmp/ICA-AROMA.tar.gz \ No newline at end of file +64d182286c70a7a0abbd02d011adff9c4e13924db8e69248e1f9f15c4c7a45414d5b61bd687ccbc2ab1c1357e9649dc9 /tmp/ICA-AROMA.tar.gz diff --git a/dev/docker_data/checksum/Python3.10-bionic.sha384 b/dev/docker_data/checksum/Python3.10-bionic.sha384 index de2893406b..1322abc9aa 100644 --- a/dev/docker_data/checksum/Python3.10-bionic.sha384 +++ b/dev/docker_data/checksum/Python3.10-bionic.sha384 @@ -1 +1 @@ -5bcd43506af6431d79b3b9368328bb30bb4992b886885e066962a286912dd79ac5fca20a0ce9f620b7fa1c3f9c94ee67 /etc/apt/trusted.gpg.d/github_git-lfs-archive-keyring.gpg \ No newline at end of file +5bcd43506af6431d79b3b9368328bb30bb4992b886885e066962a286912dd79ac5fca20a0ce9f620b7fa1c3f9c94ee67 /etc/apt/trusted.gpg.d/github_git-lfs-archive-keyring.gpg diff --git a/dev/docker_data/checksum/c3d.1.0.0.sha384 b/dev/docker_data/checksum/c3d.1.0.0.sha384 index 52cb6c112c..0c8646e360 100644 --- a/dev/docker_data/checksum/c3d.1.0.0.sha384 +++ b/dev/docker_data/checksum/c3d.1.0.0.sha384 @@ -1 +1 @@ -206bde9eb1038093344dee002b78ffff5acdea2f9177935481129921f6940ea5aa57fd5035031e99aec50b797c32eb1f /tmp/c3d.tar.gz \ No newline at end of file +206bde9eb1038093344dee002b78ffff5acdea2f9177935481129921f6940ea5aa57fd5035031e99aec50b797c32eb1f /tmp/c3d.tar.gz diff --git a/dev/docker_data/checksum/connectome-workbench.1.5.0.sha384 b/dev/docker_data/checksum/connectome-workbench.1.5.0.sha384 index a73ef2054a..ffeb4d699c 100644 --- a/dev/docker_data/checksum/connectome-workbench.1.5.0.sha384 +++ b/dev/docker_data/checksum/connectome-workbench.1.5.0.sha384 @@ -1 +1 @@ -28c7939716db61e11dc04002b01476ae6ab3e1c49245481545a04f67870c93da5e4e49a757e527e826a497a25c79c317 /opt/workbench.zip \ No newline at end of file +28c7939716db61e11dc04002b01476ae6ab3e1c49245481545a04f67870c93da5e4e49a757e527e826a497a25c79c317 /opt/workbench.zip diff --git a/dev/docker_data/checksum/msm.2.0.sha384 b/dev/docker_data/checksum/msm.2.0.sha384 index 26a0105d21..495dcf8315 100644 --- a/dev/docker_data/checksum/msm.2.0.sha384 +++ b/dev/docker_data/checksum/msm.2.0.sha384 @@ -1 +1 @@ -fbde9f06173b5a82bee4bb280df18564f8735dcc84c54054fb570ba5603b2e7226389252fb8daf3c185e6801f6c342ca msm.tgz \ No newline at end of file +fbde9f06173b5a82bee4bb280df18564f8735dcc84c54054fb570ba5603b2e7226389252fb8daf3c185e6801f6c342ca msm.tgz diff --git a/dev/docker_data/fsl/6.0.6.5/fsl-release.yml b/dev/docker_data/fsl/6.0.6.5/fsl-release.yml index b207a4245e..605b683b55 100644 --- a/dev/docker_data/fsl/6.0.6.5/fsl-release.yml +++ b/dev/docker_data/fsl/6.0.6.5/fsl-release.yml @@ -147,4 +147,4 @@ dependencies: - setuptools 68.0.0 - urllib3 1.26.15 - wheel 0.40.0 - - zipp 3.16.0 \ No newline at end of file + - zipp 3.16.0 diff --git a/dev/docker_data/fsl/6.0.6.5/manifest.json b/dev/docker_data/fsl/6.0.6.5/manifest.json index 121d4bbe18..e486cfe2c8 100644 --- a/dev/docker_data/fsl/6.0.6.5/manifest.json +++ b/dev/docker_data/fsl/6.0.6.5/manifest.json @@ -1,14 +1,14 @@ { - "installer" : { - "version" : "3.5.3", - "url" : "https://git.fmrib.ox.ac.uk/fsl/conda/installer/-/raw/3.5.3/fsl/installer/fslinstaller.py?inline=false", - "sha256" : "da59a5c94ec05c813ecf5e1b6da2ef44cc2cf7bf78243f517f77dee1cf06647f" + "installer": { + "version": "3.5.3", + "url": "https://git.fmrib.ox.ac.uk/fsl/conda/installer/-/raw/3.5.3/fsl/installer/fslinstaller.py?inline=false", + "sha256": "da59a5c94ec05c813ecf5e1b6da2ef44cc2cf7bf78243f517f77dee1cf06647f" }, - "miniconda" : { - "linux-64" : { - "url" : "https://github.com/conda-forge/miniforge/releases/download/23.1.0-4/Mambaforge-23.1.0-4-Linux-x86_64.sh", - "sha256" : "6ca38e02be99c410644c283bac74601f296dd10995ce1c8d345af995a39b5916", - "output" : "194" + "miniconda": { + "linux-64": { + "url": "https://github.com/conda-forge/miniforge/releases/download/23.1.0-4/Mambaforge-23.1.0-4-Linux-x86_64.sh", + "sha256": "6ca38e02be99c410644c283bac74601f296dd10995ce1c8d345af995a39b5916", + "output": "194" } }, "versions": { diff --git a/dev/docker_data/required_afni_pkgs.txt b/dev/docker_data/required_afni_pkgs.txt index b87e81669a..4aa745c906 100644 --- a/dev/docker_data/required_afni_pkgs.txt +++ b/dev/docker_data/required_afni_pkgs.txt @@ -123,4 +123,4 @@ linux_openmp_64/plug_vol2surf.so linux_openmp_64/plug_volreg.so linux_openmp_64/plug_wavelets.so linux_openmp_64/plug_zeropad.so -linux_openmp_64/R_io.so \ No newline at end of file +linux_openmp_64/R_io.so diff --git a/dev/docker_data/required_freesurfer_pkgs.txt b/dev/docker_data/required_freesurfer_pkgs.txt index c5958eb919..e0f6bfe859 100644 --- a/dev/docker_data/required_freesurfer_pkgs.txt +++ b/dev/docker_data/required_freesurfer_pkgs.txt @@ -6820,4 +6820,4 @@ /opt/freesurfer/lib/resource /opt/freesurfer/._lib /opt/freesurfer/._subjects -/opt/freesurfer/.DS_Store \ No newline at end of file +/opt/freesurfer/.DS_Store diff --git a/dev/docker_data/run-with-freesurfer.sh b/dev/docker_data/run-with-freesurfer.sh index 440c6a47bb..b1551b4512 100755 --- a/dev/docker_data/run-with-freesurfer.sh +++ b/dev/docker_data/run-with-freesurfer.sh @@ -9,4 +9,4 @@ # You should have received a copy of the GNU Lesser General Public License along with C-PAC. If not, see . source $FREESURFER_HOME/SetUpFreeSurfer.sh -/code/run.py "$@" \ No newline at end of file +/code/run.py "$@" diff --git a/dev/docker_data/run.py b/dev/docker_data/run.py index 344f7206c3..a1b21a097b 100755 --- a/dev/docker_data/run.py +++ b/dev/docker_data/run.py @@ -18,44 +18,51 @@ import argparse import datetime import os +import shutil import subprocess import sys import time -import shutil from warnings import simplefilter -from nipype import logging + import yaml -from CPAC import license_notice, __version__ +from nipype import logging + +from CPAC import __version__, license_notice from CPAC.pipeline import AVAILABLE_PIPELINE_CONFIGS from CPAC.pipeline.random_state import set_up_random_state from CPAC.pipeline.schema import str_to_bool1_1 -from CPAC.utils.bids_utils import cl_strip_brackets, \ - create_cpac_data_config, \ - load_cpac_data_config, \ - load_yaml_config, \ - sub_list_filter_by_labels +from CPAC.utils.bids_utils import ( + cl_strip_brackets, + create_cpac_data_config, + load_cpac_data_config, + load_yaml_config, + sub_list_filter_by_labels, +) from CPAC.utils.configuration import Configuration, preconfig_yaml, set_subject +from CPAC.utils.configuration.yaml_template import ( + create_yaml_from_template, + hash_data_config, + upgrade_pipeline_to_1_8, +) from CPAC.utils.docs import DOCS_URL_PREFIX from CPAC.utils.monitoring import failed_to_start, log_nodes_cb -from CPAC.utils.configuration.yaml_template import create_yaml_from_template, \ - hash_data_config, \ - upgrade_pipeline_to_1_8 from CPAC.utils.utils import update_nested_dict -simplefilter(action='ignore', category=FutureWarning) -logger = logging.getLogger('nipype.workflow') + +simplefilter(action="ignore", category=FutureWarning) +logger = logging.getLogger("nipype.workflow") DEFAULT_TMP_DIR = "/tmp" def run(command, env=None): if env is None: env = {} - process = subprocess.Popen(command, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - shell=True, env=env) + process = subprocess.Popen( + command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True, env=env + ) while True: line = process.stdout.readline() line = line.decode()[:-1] - if line == '' and process.poll() is not None: + if line == "" and process.poll() is not None: break @@ -71,21 +78,19 @@ def parse_yaml(value): def resolve_aws_credential(source): - if source == "env": from urllib.request import urlopen + aws_creds_address = "169.254.170.2{}".format( os.environ["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"] ) aws_creds = urlopen(aws_creds_address).read() - aws_input_creds = "/tmp/aws_input_creds_%d.csv" % int( - round(time.time() * 1000) - ) + aws_input_creds = "/tmp/aws_input_creds_%d.csv" % int(round(time.time() * 1000)) with open(aws_input_creds) as ofd: for key, vname in [ ("AccessKeyId", "AWSAcessKeyId"), - ("SecretAccessKey", "AWSSecretKey") + ("SecretAccessKey", "AWSSecretKey"), ]: ofd.write("{0}={1}".format(vname, aws_creds[key])) @@ -94,296 +99,380 @@ def resolve_aws_credential(source): if os.path.isfile(source): return source else: - raise IOError( - "Could not find aws credentials {0}" - .format(source) - ) + raise IOError("Could not find aws credentials {0}".format(source)) def run_main(): - """Run this function if not importing as a script""" - parser = argparse.ArgumentParser(description='C-PAC Pipeline Runner. ' + - license_notice) - parser.add_argument('bids_dir', - help='The directory with the input dataset ' - 'formatted according to the BIDS standard. ' - 'Use the format s3://bucket/path/to/bidsdir to ' - 'read data directly from an S3 bucket. This may ' - 'require AWS S3 credentials specified via the ' - '--aws_input_creds option.') - parser.add_argument('output_dir', - help='The directory where the output files should be ' - 'stored. If you are running group level analysis ' - 'this folder should be prepopulated with the ' - 'results of the participant level analysis. Use ' - 'the format s3://bucket/path/to/bidsdir to ' - 'write data directly to an S3 bucket. This may ' - 'require AWS S3 credentials specified via the ' - '--aws_output_creds option.') - parser.add_argument('analysis_level', - help='Level of the analysis that will be performed. ' - 'Multiple participant level analyses can be run ' - 'independently (in parallel) using the same ' - 'output_dir. test_config will run through the ' - 'entire configuration process but will not ' - 'execute the pipeline.', - choices=['participant', 'group', 'test_config', 'cli'], - type=lambda choice: choice.replace('-', '_').lower()) - - parser.add_argument('--pipeline-file', '--pipeline_file', - help='Path for the pipeline configuration file to ' - 'use. Use the format s3://bucket/path/to/' - 'pipeline_file to read data directly from an ' - 'S3 bucket. This may require AWS S3 credentials ' - 'specified via the --aws_input_creds option.', - default=preconfig_yaml('default')) - parser.add_argument('--group-file', '--group_file', - help='Path for the group analysis configuration file ' - 'to use. Use the format s3://bucket/path/to/' - 'pipeline_file to read data directly from an S3 ' - 'bucket. This may require AWS S3 credentials ' - 'specified via the --aws_input_creds option. ' - 'The output directory needs to refer to the ' - 'output of a preprocessing individual pipeline.', - default=None) - parser.add_argument('--data-config-file', '--data_config_file', - help='Yaml file containing the location of the data ' - 'that is to be processed. This file is not ' - 'necessary if the data in bids_dir is organized ' - 'according to the BIDS format. This enables ' - 'support for legacy data organization and cloud ' - 'based storage. A bids_dir must still be ' - 'specified when using this option, but its ' - 'value will be ignored. Use the format s3://' - 'bucket/path/to/data_config_file to read data ' - 'directly from an S3 bucket. This may require ' - 'AWS S3 credentials specified via the ' - '--aws_input_creds option.', - default=None) - - parser.add_argument('--preconfig', - help='Name of the preconfigured pipeline to run. ' - 'Available preconfigured pipelines: ' + - str(AVAILABLE_PIPELINE_CONFIGS) + '. See ' - f'{DOCS_URL_PREFIX}/user/pipelines/preconfig ' - 'for more information about the preconfigured ' - 'pipelines.', - default=None) - if [_ for _ in ['--pipeline-override', - '--pipeline_override']if _ in sys.argv]: # secret option - parser.add_argument('--pipeline-override', '--pipeline_override', - type=parse_yaml, action='append', - help='Override specific options from the ' - 'pipeline configuration. E.g.: ' - '"{\'pipeline_setup\': {\'system_config\': ' - '{\'maximum_memory_per_participant\': 1}}}"') - - parser.add_argument('--aws-input-creds', '--aws_input_creds', - help='Credentials for reading from S3. If not ' - 'provided and s3 paths are specified in the ' - 'data config we will try to access the bucket ' - 'anonymously use the string "env" to indicate ' - 'that input credentials should read from the ' - 'environment. (E.g. when using AWS iam roles).', - default=None) - parser.add_argument('--aws-output-creds', '--aws_output_creds', - help='Credentials for writing to S3. If not provided ' - 'and s3 paths are specified in the output ' - 'directory we will try to access the bucket ' - 'anonymously use the string "env" to indicate ' - 'that output credentials should read from the ' - 'environment. (E.g. when using AWS iam roles).', - default=None) + """Run this function if not importing as a script.""" + parser = argparse.ArgumentParser( + description="C-PAC Pipeline Runner. " + license_notice + ) + parser.add_argument( + "bids_dir", + help="The directory with the input dataset " + "formatted according to the BIDS standard. " + "Use the format s3://bucket/path/to/bidsdir to " + "read data directly from an S3 bucket. This may " + "require AWS S3 credentials specified via the " + "--aws_input_creds option.", + ) + parser.add_argument( + "output_dir", + help="The directory where the output files should be " + "stored. If you are running group level analysis " + "this folder should be prepopulated with the " + "results of the participant level analysis. Use " + "the format s3://bucket/path/to/bidsdir to " + "write data directly to an S3 bucket. This may " + "require AWS S3 credentials specified via the " + "--aws_output_creds option.", + ) + parser.add_argument( + "analysis_level", + help="Level of the analysis that will be performed. " + "Multiple participant level analyses can be run " + "independently (in parallel) using the same " + "output_dir. test_config will run through the " + "entire configuration process but will not " + "execute the pipeline.", + choices=["participant", "group", "test_config", "cli"], + type=lambda choice: choice.replace("-", "_").lower(), + ) + + parser.add_argument( + "--pipeline-file", + "--pipeline_file", + help="Path for the pipeline configuration file to " + "use. Use the format s3://bucket/path/to/" + "pipeline_file to read data directly from an " + "S3 bucket. This may require AWS S3 credentials " + "specified via the --aws_input_creds option.", + default=preconfig_yaml("default"), + ) + parser.add_argument( + "--group-file", + "--group_file", + help="Path for the group analysis configuration file " + "to use. Use the format s3://bucket/path/to/" + "pipeline_file to read data directly from an S3 " + "bucket. This may require AWS S3 credentials " + "specified via the --aws_input_creds option. " + "The output directory needs to refer to the " + "output of a preprocessing individual pipeline.", + default=None, + ) + parser.add_argument( + "--data-config-file", + "--data_config_file", + help="Yaml file containing the location of the data " + "that is to be processed. This file is not " + "necessary if the data in bids_dir is organized " + "according to the BIDS format. This enables " + "support for legacy data organization and cloud " + "based storage. A bids_dir must still be " + "specified when using this option, but its " + "value will be ignored. Use the format s3://" + "bucket/path/to/data_config_file to read data " + "directly from an S3 bucket. This may require " + "AWS S3 credentials specified via the " + "--aws_input_creds option.", + default=None, + ) + + parser.add_argument( + "--preconfig", + help="Name of the preconfigured pipeline to run. " + "Available preconfigured pipelines: " + + str(AVAILABLE_PIPELINE_CONFIGS) + + ". See " + f"{DOCS_URL_PREFIX}/user/pipelines/preconfig " + "for more information about the preconfigured " + "pipelines.", + default=None, + ) + if [ + _ for _ in ["--pipeline-override", "--pipeline_override"] if _ in sys.argv + ]: # secret option + parser.add_argument( + "--pipeline-override", + "--pipeline_override", + type=parse_yaml, + action="append", + help="Override specific options from the " + "pipeline configuration. E.g.: " + "\"{'pipeline_setup': {'system_config': " + "{'maximum_memory_per_participant': 1}}}\"", + ) + + parser.add_argument( + "--aws-input-creds", + "--aws_input_creds", + help="Credentials for reading from S3. If not " + "provided and s3 paths are specified in the " + "data config we will try to access the bucket " + 'anonymously use the string "env" to indicate ' + "that input credentials should read from the " + "environment. (E.g. when using AWS iam roles).", + default=None, + ) + parser.add_argument( + "--aws-output-creds", + "--aws_output_creds", + help="Credentials for writing to S3. If not provided " + "and s3 paths are specified in the output " + "directory we will try to access the bucket " + 'anonymously use the string "env" to indicate ' + "that output credentials should read from the " + "environment. (E.g. when using AWS iam roles).", + default=None, + ) # TODO: restore for <--n_cpus> once we remove # from config file # - parser.add_argument('--n-cpus', '--n_cpus', type=int, default=0, - help='Number of execution resources per participant ' - 'available for the pipeline. This flag takes ' - 'precidence over max_cores_per_participant in ' - 'the pipeline configuration file.') - parser.add_argument('--mem-mb', '--mem_mb', type=float, - help='Amount of RAM available per participant in ' - 'megabytes. Included for compatibility with ' - 'BIDS-Apps standard, but mem_gb is preferred. ' - 'This flag takes precedence over ' - 'maximum_memory_per_participant in the pipeline ' - 'configuration file.') - parser.add_argument('--mem-gb', '--mem_gb', type=float, - help='Amount of RAM available per participant in ' - 'gigabytes. If this is specified along with ' - 'mem_mb, this flag will take precedence. This ' - 'flag also takes precedence over ' - 'maximum_memory_per_participant in the pipeline ' - 'configuration file.') - parser.add_argument('--runtime-usage', '--runtime_usage', type=str, - help='Path to a callback.log from a prior run of the ' - 'same pipeline configuration (including any ' - 'resource-management parameters that will be ' - "applied in this run, like 'n_cpus' and " - "'num_ants_threads'). This log will be used to " - 'override per-node memory estimates with ' - 'observed values plus a buffer.') - parser.add_argument('--runtime-buffer', '--runtime_buffer', type=float, - help='Buffer to add to per-node memory estimates if ' - '--runtime_usage is specified. This number is a ' - 'percentage of the observed memory usage.') - parser.add_argument('--num-ants-threads', '--num_ants_threads', type=int, - default=0, - help='The number of cores to allocate to ANTS-' - 'based anatomical registration per ' - 'participant. Multiple cores can greatly ' - 'speed up this preprocessing step. This ' - 'number cannot be greater than the number of ' - 'cores per participant.') - parser.add_argument('--random-seed', '--random_seed', type=str, - help='Random seed used to fix the state of execution. ' - 'If unset, each process uses its own default. If ' - 'set, a `random.log` file will be generated ' - 'logging the random state used by each process. ' - 'If set to a positive integer (up to 2147483647' - '), that integer will be used to seed each ' - 'process. If set to \'random\', a random seed ' - 'will be generated and recorded for each ' - 'process.') - parser.add_argument('--save-working-dir', '--save_working_dir', nargs='?', - help='Save the contents of the working directory.', - default=False) - parser.add_argument('--fail-fast', '--fail_fast', type=str.title, - help='Stop worklow execution on first crash?') - parser.add_argument('--participant-label', '--participant_label', - help='The label of the participant that should be ' - 'analyzed. The label corresponds to ' - 'sub- from the BIDS spec ' - '(so it does not include "sub-"). If this ' - 'parameter is not provided all participants ' - 'should be analyzed. Multiple participants ' - 'can be specified with a space separated ' - 'list.', - nargs="+") - parser.add_argument('--participant-ndx', '--participant_ndx', - help='The index of the participant that should be ' - 'analyzed. This corresponds to the index of ' - 'the participant in the data config file. ' - 'This was added to make it easier to ' - 'accommodate SGE array jobs. Only a single ' - 'participant will be analyzed. Can be used ' - 'with participant label, in which case it is ' - 'the index into the list that follows the ' - 'participant_label flag. Use the value "-1" ' - 'to indicate that the participant index ' - 'should be read from the ' - 'AWS_BATCH_JOB_ARRAY_INDEX environment ' - 'variable.', - default=None, type=int) - - parser.add_argument('--T1w-label', '--T1w_label', - help='C-PAC only runs one T1w per participant-' - 'session at a time, at this time. Use this ' - 'flag to specify any BIDS entity (e.g., "acq-' - 'VNavNorm") or sequence of BIDS entities (' - 'e.g., "acq-VNavNorm_run-1") to specify ' - 'which of multiple T1w files to use. Specify ' - '"--T1w_label T1w" to choose the T1w file ' - 'with the fewest BIDS entities (i.e., the ' - 'final option of [*_acq-VNavNorm_T1w.nii.gz, ' - '*_acq-HCP_T1w.nii.gz, *_T1w.nii.gz"]). ' - 'C-PAC will choose the first T1w it finds if ' - 'the user does not provide this flag, or ' - 'if multiple T1w files match the --T1w_label ' - 'provided.\nIf multiple T2w files are present ' - 'and a comparable filter is possible, T2w ' - 'files will be filtered as well. If no T2w files ' - 'match this --T1w_label, T2w files will be ' - 'processed as if no --T1w_label were provided.') - parser.add_argument('--bold-label', '--bold_label', - help='To include a specified subset of available ' - 'BOLD files, use this flag to specify any ' - 'BIDS entity (e.g., "task-rest") or sequence ' - 'of BIDS entities (e.g. "task-rest_run-1"). ' - 'To specify the bold file with the fewest ' - 'BIDS entities in the file name, specify ' - '"--bold_label bold". Multiple `--bold_' - 'label`s can be specified with a space-' - 'separated list. If multiple `--bold_label`s ' - 'are provided (e.g., "--bold_label task-rest_' - 'run-1 task-rest_run-2", each scan that ' - 'includes all BIDS entities specified in any ' - 'of the provided `--bold_label`s will be ' - 'analyzed. If this parameter is not provided ' - 'all BOLD scans should be analyzed.', - nargs="+") - - parser.add_argument('-v', '--version', action='version', - version=f'C-PAC BIDS-App version {__version__}') - parser.add_argument('--bids-validator-config', '--bids_validator_config', - help='JSON file specifying configuration of ' - 'bids-validator: See https://github.com/bids-' - 'standard/bids-validator for more info.') - parser.add_argument('--skip-bids-validator', '--skip_bids_validator', - help='Skips bids validation.', - action='store_true') - - parser.add_argument('--anat-only', '--anat_only', - help='run only the anatomical preprocessing', - action='store_true') - - parser.add_argument('--user_defined', type=str, - help='Arbitrary user defined string that will be ' - 'included in every output sidecar file.') - - parser.add_argument('--tracking-opt-out', '--tracking_opt-out', - action='store_true', - help='Disable usage tracking. Only the number of ' - 'participants on the analysis is tracked.', - default=False) - - parser.add_argument('--monitoring', - help='Enable monitoring server on port 8080. You ' - 'need to bind the port using the Docker ' - 'flag "-p".', - action='store_true') + parser.add_argument( + "--n-cpus", + "--n_cpus", + type=int, + default=0, + help="Number of execution resources per participant " + "available for the pipeline. This flag takes " + "precidence over max_cores_per_participant in " + "the pipeline configuration file.", + ) + parser.add_argument( + "--mem-mb", + "--mem_mb", + type=float, + help="Amount of RAM available per participant in " + "megabytes. Included for compatibility with " + "BIDS-Apps standard, but mem_gb is preferred. " + "This flag takes precedence over " + "maximum_memory_per_participant in the pipeline " + "configuration file.", + ) + parser.add_argument( + "--mem-gb", + "--mem_gb", + type=float, + help="Amount of RAM available per participant in " + "gigabytes. If this is specified along with " + "mem_mb, this flag will take precedence. This " + "flag also takes precedence over " + "maximum_memory_per_participant in the pipeline " + "configuration file.", + ) + parser.add_argument( + "--runtime-usage", + "--runtime_usage", + type=str, + help="Path to a callback.log from a prior run of the " + "same pipeline configuration (including any " + "resource-management parameters that will be " + "applied in this run, like 'n_cpus' and " + "'num_ants_threads'). This log will be used to " + "override per-node memory estimates with " + "observed values plus a buffer.", + ) + parser.add_argument( + "--runtime-buffer", + "--runtime_buffer", + type=float, + help="Buffer to add to per-node memory estimates if " + "--runtime_usage is specified. This number is a " + "percentage of the observed memory usage.", + ) + parser.add_argument( + "--num-ants-threads", + "--num_ants_threads", + type=int, + default=0, + help="The number of cores to allocate to ANTS-" + "based anatomical registration per " + "participant. Multiple cores can greatly " + "speed up this preprocessing step. This " + "number cannot be greater than the number of " + "cores per participant.", + ) + parser.add_argument( + "--random-seed", + "--random_seed", + type=str, + help="Random seed used to fix the state of execution. " + "If unset, each process uses its own default. If " + "set, a `random.log` file will be generated " + "logging the random state used by each process. " + "If set to a positive integer (up to 2147483647" + "), that integer will be used to seed each " + "process. If set to 'random', a random seed " + "will be generated and recorded for each " + "process.", + ) + parser.add_argument( + "--save-working-dir", + "--save_working_dir", + nargs="?", + help="Save the contents of the working directory.", + default=False, + ) + parser.add_argument( + "--fail-fast", + "--fail_fast", + type=str.title, + help="Stop worklow execution on first crash?", + ) + parser.add_argument( + "--participant-label", + "--participant_label", + help="The label of the participant that should be " + "analyzed. The label corresponds to " + "sub- from the BIDS spec " + '(so it does not include "sub-"). If this ' + "parameter is not provided all participants " + "should be analyzed. Multiple participants " + "can be specified with a space separated " + "list.", + nargs="+", + ) + parser.add_argument( + "--participant-ndx", + "--participant_ndx", + help="The index of the participant that should be " + "analyzed. This corresponds to the index of " + "the participant in the data config file. " + "This was added to make it easier to " + "accommodate SGE array jobs. Only a single " + "participant will be analyzed. Can be used " + "with participant label, in which case it is " + "the index into the list that follows the " + 'participant_label flag. Use the value "-1" ' + "to indicate that the participant index " + "should be read from the " + "AWS_BATCH_JOB_ARRAY_INDEX environment " + "variable.", + default=None, + type=int, + ) + + parser.add_argument( + "--T1w-label", + "--T1w_label", + help="C-PAC only runs one T1w per participant-" + "session at a time, at this time. Use this " + 'flag to specify any BIDS entity (e.g., "acq-' + 'VNavNorm") or sequence of BIDS entities (' + 'e.g., "acq-VNavNorm_run-1") to specify ' + "which of multiple T1w files to use. Specify " + '"--T1w_label T1w" to choose the T1w file ' + "with the fewest BIDS entities (i.e., the " + "final option of [*_acq-VNavNorm_T1w.nii.gz, " + '*_acq-HCP_T1w.nii.gz, *_T1w.nii.gz"]). ' + "C-PAC will choose the first T1w it finds if " + "the user does not provide this flag, or " + "if multiple T1w files match the --T1w_label " + "provided.\nIf multiple T2w files are present " + "and a comparable filter is possible, T2w " + "files will be filtered as well. If no T2w files " + "match this --T1w_label, T2w files will be " + "processed as if no --T1w_label were provided.", + ) + parser.add_argument( + "--bold-label", + "--bold_label", + help="To include a specified subset of available " + "BOLD files, use this flag to specify any " + 'BIDS entity (e.g., "task-rest") or sequence ' + 'of BIDS entities (e.g. "task-rest_run-1"). ' + "To specify the bold file with the fewest " + "BIDS entities in the file name, specify " + '"--bold_label bold". Multiple `--bold_' + "label`s can be specified with a space-" + "separated list. If multiple `--bold_label`s " + 'are provided (e.g., "--bold_label task-rest_' + 'run-1 task-rest_run-2", each scan that ' + "includes all BIDS entities specified in any " + "of the provided `--bold_label`s will be " + "analyzed. If this parameter is not provided " + "all BOLD scans should be analyzed.", + nargs="+", + ) + + parser.add_argument( + "-v", + "--version", + action="version", + version=f"C-PAC BIDS-App version {__version__}", + ) + parser.add_argument( + "--bids-validator-config", + "--bids_validator_config", + help="JSON file specifying configuration of " + "bids-validator: See https://github.com/bids-" + "standard/bids-validator for more info.", + ) + parser.add_argument( + "--skip-bids-validator", + "--skip_bids_validator", + help="Skips bids validation.", + action="store_true", + ) + + parser.add_argument( + "--anat-only", + "--anat_only", + help="run only the anatomical preprocessing", + action="store_true", + ) + + parser.add_argument( + "--user_defined", + type=str, + help="Arbitrary user defined string that will be " + "included in every output sidecar file.", + ) + + parser.add_argument( + "--tracking-opt-out", + "--tracking_opt-out", + action="store_true", + help="Disable usage tracking. Only the number of " + "participants on the analysis is tracked.", + default=False, + ) + + parser.add_argument( + "--monitoring", + help="Enable monitoring server on port 8080. You " + "need to bind the port using the Docker " + 'flag "-p".', + action="store_true", + ) # get the command line arguments args = parser.parse_args( - sys.argv[ - 1:( - sys.argv.index('--') - if '--' in sys.argv - else len(sys.argv) - ) - ] + sys.argv[1 : (sys.argv.index("--") if "--" in sys.argv else len(sys.argv))] ) bids_dir_is_s3 = args.bids_dir.lower().startswith("s3://") - bids_dir = args.bids_dir if bids_dir_is_s3 else os.path.realpath( - args.bids_dir) + bids_dir = args.bids_dir if bids_dir_is_s3 else os.path.realpath(args.bids_dir) output_dir_is_s3 = args.output_dir.lower().startswith("s3://") - output_dir = args.output_dir if output_dir_is_s3 else os.path.realpath( - args.output_dir) + output_dir = ( + args.output_dir if output_dir_is_s3 else os.path.realpath(args.output_dir) + ) exitcode = 0 if args.analysis_level == "cli": from CPAC.__main__ import main - main.main(args=sys.argv[sys.argv.index('--') + 1:]) + + main.main(args=sys.argv[sys.argv.index("--") + 1 :]) sys.exit(0) elif args.analysis_level == "group": if not args.group_file or not os.path.exists(args.group_file): - - print() - print("No group analysis configuration file was supplied.") - print() - import pkg_resources as p - args.group_file = \ - p.resource_filename( - "CPAC", - os.path.join( - "resources", - "configs", - "group_config_template.yml" - ) - ) + + args.group_file = p.resource_filename( + "CPAC", + os.path.join("resources", "configs", "group_config_template.yml"), + ) output_group = os.path.join(output_dir, "group_config.yml") @@ -394,69 +483,48 @@ def run_main(): if not os.path.exists(output_group): shutil.copyfile(args.group_file, output_group) except (Exception, IOError): - print("Could not create group analysis configuration file.") - print("Please refer to the C-PAC documentation for group " - "analysis setup.") - print() + pass else: - print( - "Please refer to the output directory for a template of " - "the file and, after customizing to your analysis, add " - "the flag" - "\n\n" - " --group_file {0}" - "\n\n" - "to your `docker run` command" - "\n" - .format(output_group) - ) + pass sys.exit(1) else: import CPAC.pipeline.cpac_group_runner as cgr - print("Starting group level analysis of data in {0} using " - "{1}".format(bids_dir, args.group_file)) + cgr.run(args.group_file) sys.exit(0) elif args.analysis_level in ["test_config", "participant"]: - # check to make sure that the input directory exists if ( - not args.data_config_file and - not bids_dir_is_s3 and - not os.path.exists(bids_dir) + not args.data_config_file + and not bids_dir_is_s3 + and not os.path.exists(bids_dir) ): - - print(f"Error! Could not find {bids_dir}") sys.exit(1) # check to make sure that the output directory exists if not output_dir_is_s3 and not os.path.exists(output_dir): - try: os.makedirs(output_dir) except Exception: - print(f"Error! Could not find/create output dir {output_dir}") sys.exit(1) # validate input dir (if skip_bids_validator is not set) if not args.data_config_file: - print() if args.bids_validator_config: - print("Running BIDS validator") - run("bids-validator --config {config} {bids_dir}".format( - config=args.bids_validator_config, - bids_dir=bids_dir - )) + run( + "bids-validator --config {config} {bids_dir}".format( + config=args.bids_validator_config, bids_dir=bids_dir + ) + ) elif args.skip_bids_validator: - print('Skipping bids-validator...') + pass elif bids_dir_is_s3: - print('Skipping bids-validator for S3 datasets...') + pass else: - print("Running BIDS validator") run(f"bids-validator {bids_dir}") if args.preconfig: @@ -469,282 +537,266 @@ def run_main(): else: c = load_yaml_config(args.pipeline_file, args.aws_input_creds) - if 'pipeline_setup' not in c: - _url = (f'{DOCS_URL_PREFIX}/user/pipelines/' - '1.7-1.8-nesting-mappings') + if "pipeline_setup" not in c: + _url = f"{DOCS_URL_PREFIX}/user/pipelines/" "1.7-1.8-nesting-mappings" - logger.warning('\nC-PAC changed its pipeline configuration ' - 'format in v1.8.0.\nSee %s for details.\n', _url) + logger.warning( + "\nC-PAC changed its pipeline configuration " + "format in v1.8.0.\nSee %s for details.\n", + _url, + ) updated_config = os.path.join( - output_dir, - 'updated_config', - os.path.basename(args.pipeline_file) + output_dir, "updated_config", os.path.basename(args.pipeline_file) ) - os.makedirs( - os.path.join(output_dir, 'updated_config'), exist_ok=True) + os.makedirs(os.path.join(output_dir, "updated_config"), exist_ok=True) - open(updated_config, 'w').write(yaml.dump(c)) + open(updated_config, "w").write(yaml.dump(c)) upgrade_pipeline_to_1_8(updated_config) c = load_yaml_config(updated_config, args.aws_input_creds) overrides = {} - if hasattr(args, 'pipeline_override') and args.pipeline_override: - overrides = { - k: v for d in args.pipeline_override for k, v in d.items()} + if hasattr(args, "pipeline_override") and args.pipeline_override: + overrides = {k: v for d in args.pipeline_override for k, v in d.items()} c = update_nested_dict(c, overrides) if args.anat_only: - c = update_nested_dict(c, {'FROM': 'anat-only'}) + c = update_nested_dict(c, {"FROM": "anat-only"}) if args.user_defined: - c['pipeline_setup']['output_directory']['user_defined'] = args.user_defined + c["pipeline_setup"]["output_directory"]["user_defined"] = args.user_defined c = Configuration(c) # get the aws_input_credentials, if any are specified if args.aws_input_creds: - c['awsCredentialsFile'] = resolve_aws_credential( - args.aws_input_creds) + c["awsCredentialsFile"] = resolve_aws_credential(args.aws_input_creds) if args.aws_output_creds: - c['pipeline_setup']['Amazon-AWS'][ - 'aws_output_bucket_credentials' - ] = resolve_aws_credential( - args.aws_output_creds - ) + c["pipeline_setup"]["Amazon-AWS"][ + "aws_output_bucket_credentials" + ] = resolve_aws_credential(args.aws_output_creds) - c['pipeline_setup']['output_directory']['path'] = os.path.join( - output_dir, "output") + c["pipeline_setup"]["output_directory"]["path"] = os.path.join( + output_dir, "output" + ) if not output_dir_is_s3: - c['pipeline_setup']['log_directory']['path'] = os.path.join( - output_dir, "log") + c["pipeline_setup"]["log_directory"]["path"] = os.path.join( + output_dir, "log" + ) else: - c['pipeline_setup']['log_directory']['path'] = os.path.join( - DEFAULT_TMP_DIR, "log") + c["pipeline_setup"]["log_directory"]["path"] = os.path.join( + DEFAULT_TMP_DIR, "log" + ) if args.mem_gb: - c['pipeline_setup']['system_config'][ - 'maximum_memory_per_participant'] = float(args.mem_gb) + c["pipeline_setup"]["system_config"][ + "maximum_memory_per_participant" + ] = float(args.mem_gb) elif args.mem_mb: - c['pipeline_setup']['system_config'][ - 'maximum_memory_per_participant'] = float(args.mem_mb) / 1024.0 + c["pipeline_setup"]["system_config"]["maximum_memory_per_participant"] = ( + float(args.mem_mb) / 1024.0 + ) else: try: - c['pipeline_setup', 'system_config', - 'maximum_memory_per_participant'] = float( - c['pipeline_setup', 'system_config', - 'maximum_memory_per_participant']) + c[ + "pipeline_setup", "system_config", "maximum_memory_per_participant" + ] = float( + c[ + "pipeline_setup", + "system_config", + "maximum_memory_per_participant", + ] + ) except KeyError: - c['pipeline_setup', 'system_config', - 'maximum_memory_per_participant'] = 6.0 + c[ + "pipeline_setup", "system_config", "maximum_memory_per_participant" + ] = 6.0 # Preference: n_cpus if given, override if present, else from config if # present, else n_cpus=3 if int(args.n_cpus) == 0: try: - args.n_cpus = c['pipeline_setup', 'system_config', - 'max_cores_per_participant'] + args.n_cpus = c[ + "pipeline_setup", "system_config", "max_cores_per_participant" + ] except KeyError: args.n_cpus = 3 - c['pipeline_setup', 'system_config', - 'max_cores_per_participant'] = int(args.n_cpus) + c["pipeline_setup", "system_config", "max_cores_per_participant"] = int( + args.n_cpus + ) - c['pipeline_setup']['system_config']['num_participants_at_once'] = int( - c['pipeline_setup']['system_config'].get( - 'num_participants_at_once', 1)) + c["pipeline_setup"]["system_config"]["num_participants_at_once"] = int( + c["pipeline_setup"]["system_config"].get("num_participants_at_once", 1) + ) # Reduce cores per participant if cores times participants is more than # available CPUS. n_cpus is a hard upper limit. if ( - c['pipeline_setup']['system_config']['max_cores_per_participant'] * - c['pipeline_setup']['system_config']['num_participants_at_once'] + c["pipeline_setup"]["system_config"]["max_cores_per_participant"] + * c["pipeline_setup"]["system_config"]["num_participants_at_once"] ) > int(args.n_cpus): - c['pipeline_setup']['system_config'][ - 'max_cores_per_participant' - ] = int(args.n_cpus) // c['pipeline_setup']['system_config'][ - 'num_participants_at_once' - ] - if c['pipeline_setup']['system_config'][ - 'max_cores_per_participant' - ] == 0: - c['pipeline_setup']['system_config'][ - 'max_cores_per_participant'] = args.n_cpus - c['pipeline_setup']['system_config'][ - 'num_participants_at_once'] = 1 + c["pipeline_setup"]["system_config"]["max_cores_per_participant"] = ( + int(args.n_cpus) + // c["pipeline_setup"]["system_config"]["num_participants_at_once"] + ) + if c["pipeline_setup"]["system_config"]["max_cores_per_participant"] == 0: + c["pipeline_setup"]["system_config"][ + "max_cores_per_participant" + ] = args.n_cpus + c["pipeline_setup"]["system_config"]["num_participants_at_once"] = 1 if int(args.num_ants_threads) == 0: try: - args.num_ants_threads = c['pipeline_setup', 'system_config', - 'num_ants_threads'] + args.num_ants_threads = c[ + "pipeline_setup", "system_config", "num_ants_threads" + ] except KeyError: args.num_ants_threads = 3 - c['pipeline_setup', 'system_config', 'num_ants_threads'] = int( - args.num_ants_threads) + c["pipeline_setup", "system_config", "num_ants_threads"] = int( + args.num_ants_threads + ) - c['pipeline_setup']['system_config']['num_ants_threads'] = min( - c['pipeline_setup']['system_config']['max_cores_per_participant'], - int(c['pipeline_setup']['system_config']['num_ants_threads']) + c["pipeline_setup"]["system_config"]["num_ants_threads"] = min( + c["pipeline_setup"]["system_config"]["max_cores_per_participant"], + int(c["pipeline_setup"]["system_config"]["num_ants_threads"]), ) if args.random_seed: - c['pipeline_setup']['system_config']['random_seed'] = \ - args.random_seed + c["pipeline_setup"]["system_config"]["random_seed"] = args.random_seed - if c['pipeline_setup']['system_config']['random_seed'] is not None: - c['pipeline_setup']['system_config']['random_seed'] = \ - set_up_random_state(c['pipeline_setup']['system_config'][ - 'random_seed']) + if c["pipeline_setup"]["system_config"]["random_seed"] is not None: + c["pipeline_setup"]["system_config"]["random_seed"] = set_up_random_state( + c["pipeline_setup"]["system_config"]["random_seed"] + ) if args.runtime_usage is not None: - c['pipeline_setup']['system_config']['observed_usage'][ - 'callback_log'] = args.runtime_usage + c["pipeline_setup"]["system_config"]["observed_usage"][ + "callback_log" + ] = args.runtime_usage if args.runtime_buffer is not None: - c['pipeline_setup']['system_config']['observed_usage'][ - 'buffer'] = args.runtime_buffer + c["pipeline_setup"]["system_config"]["observed_usage"][ + "buffer" + ] = args.runtime_buffer if args.save_working_dir is not False: - c['pipeline_setup']['working_directory'][ - 'remove_working_dir'] = False + c["pipeline_setup"]["working_directory"]["remove_working_dir"] = False if isinstance(args.save_working_dir, str): - c['pipeline_setup']['working_directory']['path'] = \ - os.path.abspath(args.save_working_dir) + c["pipeline_setup"]["working_directory"]["path"] = os.path.abspath( + args.save_working_dir + ) elif not output_dir_is_s3: - c['pipeline_setup']['working_directory']['path'] = \ - os.path.join(output_dir, "working") + c["pipeline_setup"]["working_directory"]["path"] = os.path.join( + output_dir, "working" + ) else: - logger.warning('Cannot write working directory to S3 bucket. ' - 'Either change the output directory to something ' - 'local or turn off the --save_working_dir flag') + logger.warning( + "Cannot write working directory to S3 bucket. " + "Either change the output directory to something " + "local or turn off the --save_working_dir flag" + ) if args.fail_fast is not None: - c['pipeline_setup', 'system_config', - 'fail_fast'] = str_to_bool1_1(args.fail_fast) + c["pipeline_setup", "system_config", "fail_fast"] = str_to_bool1_1( + args.fail_fast + ) - if c['pipeline_setup']['output_directory']['quality_control'][ - 'generate_xcpqc_files']: - c['functional_preproc']['motion_estimates_and_correction'][ - 'motion_estimates']['calculate_motion_first'] = True - c['functional_preproc']['motion_estimates_and_correction'][ - 'motion_estimates']['calculate_motion_after'] = True + if c["pipeline_setup"]["output_directory"]["quality_control"][ + "generate_xcpqc_files" + ]: + c["functional_preproc"]["motion_estimates_and_correction"][ + "motion_estimates" + ]["calculate_motion_first"] = True + c["functional_preproc"]["motion_estimates_and_correction"][ + "motion_estimates" + ]["calculate_motion_after"] = True if args.participant_label: - print( - "#### Running C-PAC for {0}" - .format(", ".join(args.participant_label)) - ) + pass else: - print("#### Running C-PAC") - - print("Number of participants to run in parallel: {0}" - .format(c['pipeline_setup']['system_config'][ - 'num_participants_at_once'])) + pass if not args.data_config_file: - print("Input directory: {0}".format(bids_dir)) - - print("Output directory: {0}".format( - c['pipeline_setup']['output_directory']['path'])) - print("Working directory: {0}".format( - c['pipeline_setup']['working_directory']['path'])) - print("Log directory: {0}".format( - c['pipeline_setup']['log_directory']['path'])) - print("Remove working directory: {0}".format( - c['pipeline_setup']['working_directory']['remove_working_dir'])) - print("Available memory: {0} (GB)".format( - c['pipeline_setup']['system_config'][ - 'maximum_memory_per_participant'])) - print("Available threads: {0}".format( - c['pipeline_setup']['system_config']['max_cores_per_participant'])) - print("Number of threads for ANTs: {0}".format( - c['pipeline_setup']['system_config']['num_ants_threads'])) + pass # create a timestamp for writing config files # pylint: disable=invalid-name - st = datetime.datetime.now().strftime('%Y-%m-%dT%H-%M-%SZ') + st = datetime.datetime.now().strftime("%Y-%m-%dT%H-%M-%SZ") if args.participant_label: args.participant_label = cl_strip_brackets(args.participant_label) args.participant_label = [ - 'sub-' + pt if not pt.startswith('sub-') else pt + "sub-" + pt if not pt.startswith("sub-") else pt for pt in args.participant_label ] # otherwise we move on to conforming the data configuration if not args.data_config_file: - sub_list = create_cpac_data_config(bids_dir, - args.participant_label, - args.aws_input_creds, - args.skip_bids_validator, - only_one_anat=False) + sub_list = create_cpac_data_config( + bids_dir, + args.participant_label, + args.aws_input_creds, + args.skip_bids_validator, + only_one_anat=False, + ) else: - sub_list = load_cpac_data_config(args.data_config_file, - args.participant_label, - args.aws_input_creds) - prefilter = list(sub_list) - sub_list = sub_list_filter_by_labels(sub_list, - {'T1w': args.T1w_label, - 'bold': args.bold_label}) + sub_list = load_cpac_data_config( + args.data_config_file, args.participant_label, args.aws_input_creds + ) + list(sub_list) + sub_list = sub_list_filter_by_labels( + sub_list, {"T1w": args.T1w_label, "bold": args.bold_label} + ) # C-PAC only handles single anatomical images (for now) # so we take just the first as a string if we have a list for i, sub in enumerate(sub_list): - if isinstance(sub.get('anat'), dict): - for anat_key in sub['anat']: - if( - isinstance(sub['anat'][anat_key], list) and - len(sub['anat'][anat_key]) + if isinstance(sub.get("anat"), dict): + for anat_key in sub["anat"]: + if isinstance(sub["anat"][anat_key], list) and len( + sub["anat"][anat_key] ): - sub_list[i]['anat'][ - anat_key] = sub['anat'][anat_key][0] - if isinstance(sub.get('anat'), list) and len(sub['anat']): - sub_list[i]['anat'] = sub['anat'][0] + sub_list[i]["anat"][anat_key] = sub["anat"][anat_key][0] + if isinstance(sub.get("anat"), list) and len(sub["anat"]): + sub_list[i]["anat"] = sub["anat"][0] if args.participant_ndx is not None: - participant_ndx = int(args.participant_ndx) if participant_ndx == -1: - args.participant_ndx = os.environ['AWS_BATCH_JOB_ARRAY_INDEX'] + args.participant_ndx = os.environ["AWS_BATCH_JOB_ARRAY_INDEX"] if 0 <= participant_ndx < len(sub_list): - print('Processing data for participant {0} ({1})'.format( - args.participant_ndx, - sub_list[participant_ndx]["subject_id"] - )) sub_list = [sub_list[participant_ndx]] data_hash = hash_data_config(sub_list) - data_config_file = (f"cpac_data_config_{data_hash}_idx-" - f"{args.participant_ndx}_{st}.yml") + data_config_file = ( + f"cpac_data_config_{data_hash}_idx-" + f"{args.participant_ndx}_{st}.yml" + ) else: - print("Participant ndx {0} is out of bounds [0, {1})".format( - participant_ndx, - str(len(sub_list)) - )) sys.exit(1) else: data_hash = hash_data_config(sub_list) - data_config_file = (f"cpac_data_config_{data_hash}_{st}.yml") + data_config_file = f"cpac_data_config_{data_hash}_{st}.yml" sublogdirs = [set_subject(sub, c)[2] for sub in sub_list] # write out the data configuration file data_config_file = os.path.join(sublogdirs[0], data_config_file) - with open(data_config_file, 'w', encoding='utf-8') as _f: + with open(data_config_file, "w", encoding="utf-8") as _f: noalias_dumper = yaml.dumper.SafeDumper noalias_dumper.ignore_aliases = lambda self, data: True - yaml.dump(sub_list, _f, default_flow_style=False, - Dumper=noalias_dumper) + yaml.dump(sub_list, _f, default_flow_style=False, Dumper=noalias_dumper) # update and write out pipeline config file pipeline_config_file = os.path.join( - sublogdirs[0], f"cpac_pipeline_config_{data_hash}_{st}.yml") - with open(pipeline_config_file, 'w', encoding='utf-8') as _f: + sublogdirs[0], f"cpac_pipeline_config_{data_hash}_{st}.yml" + ) + with open(pipeline_config_file, "w", encoding="utf-8") as _f: _f.write(create_yaml_from_template(c)) - minimized_config = f'{pipeline_config_file[:-4]}_min.yml' - with open(minimized_config, 'w', encoding='utf-8') as _f: - _f.write(create_yaml_from_template(c, import_from='blank')) - for config_file in (data_config_file, pipeline_config_file, - minimized_config): + minimized_config = f"{pipeline_config_file[:-4]}_min.yml" + with open(minimized_config, "w", encoding="utf-8") as _f: + _f.write(create_yaml_from_template(c, import_from="blank")) + for config_file in (data_config_file, pipeline_config_file, minimized_config): os.chmod(config_file, 0o444) # Make config files readonly if len(sublogdirs) > 1: @@ -752,56 +804,67 @@ def run_main(): # file, an identical copy of the data and pipeline config # will be included in the log directory for each run for sublogdir in sublogdirs[1:]: - for config_file in (data_config_file, pipeline_config_file, - minimized_config): + for config_file in ( + data_config_file, + pipeline_config_file, + minimized_config, + ): try: - os.link(config_file, config_file.replace( - sublogdirs[0], sublogdir)) + os.link( + config_file, config_file.replace(sublogdirs[0], sublogdir) + ) except FileExistsError: pass if args.analysis_level in ["participant", "test_config"]: # build pipeline easy way - from CPAC.utils.monitoring import monitor_server import CPAC.pipeline.cpac_runner + from CPAC.utils.monitoring import monitor_server monitoring = None if args.monitoring: try: monitoring = monitor_server( - c['pipeline_setup']['pipeline_name'], - c['pipeline_setup']['log_directory']['path'] + c["pipeline_setup"]["pipeline_name"], + c["pipeline_setup"]["log_directory"]["path"], ) except: pass plugin_args = { - 'n_procs': int(c['pipeline_setup']['system_config'][ - 'max_cores_per_participant']), - 'memory_gb': int(c['pipeline_setup']['system_config'][ - 'maximum_memory_per_participant']), - 'raise_insufficient': c['pipeline_setup']['system_config'][ - 'raise_insufficient'], - 'status_callback': log_nodes_cb + "n_procs": int( + c["pipeline_setup"]["system_config"]["max_cores_per_participant"] + ), + "memory_gb": int( + c["pipeline_setup"]["system_config"][ + "maximum_memory_per_participant" + ] + ), + "raise_insufficient": c["pipeline_setup"]["system_config"][ + "raise_insufficient" + ], + "status_callback": log_nodes_cb, } - if c['pipeline_setup']['system_config']['observed_usage'][ - 'callback_log'] is not None: - plugin_args['runtime'] = { - 'usage': c['pipeline_setup']['system_config'][ - 'observed_usage']['callback_log'], - 'buffer': c['pipeline_setup']['system_config'][ - 'observed_usage']['buffer']} - - print("Starting participant level processing") + if ( + c["pipeline_setup"]["system_config"]["observed_usage"]["callback_log"] + is not None + ): + plugin_args["runtime"] = { + "usage": c["pipeline_setup"]["system_config"]["observed_usage"][ + "callback_log" + ], + "buffer": c["pipeline_setup"]["system_config"]["observed_usage"][ + "buffer" + ], + } + exitcode = CPAC.pipeline.cpac_runner.run( data_config_file, pipeline_config_file, - plugin='MultiProc' if plugin_args[ - 'n_procs' - ] > 1 else 'Linear', + plugin="MultiProc" if plugin_args["n_procs"] > 1 else "Linear", plugin_args=plugin_args, tracking=not args.tracking_opt_out, - test_config=args.analysis_level == "test_config" + test_config=args.analysis_level == "test_config", ) if monitoring: @@ -810,24 +873,26 @@ def run_main(): if args.analysis_level == "test_config": if exitcode == 0: logger.info( - '\nPipeline and data configuration files should' - ' have been written to %s and %s respectively.\n', - pipeline_config_file, data_config_file) + "\nPipeline and data configuration files should" + " have been written to %s and %s respectively.\n", + pipeline_config_file, + data_config_file, + ) # wait to import `LOGTAIL` here so it has any runtime updates from CPAC.utils.monitoring import LOGTAIL - for warning in LOGTAIL['warnings']: - logger.warning('%s\n', warning.rstrip()) + + for warning in LOGTAIL["warnings"]: + logger.warning("%s\n", warning.rstrip()) sys.exit(exitcode) -if __name__ == '__main__': +if __name__ == "__main__": try: run_main() except Exception as exception: # if we hit an exception before the pipeline starts to build but # we're still able to create a logfile, log the error in the file - failed_to_start(sys.argv[2] if len(sys.argv) > 2 else os.getcwd(), - exception) + failed_to_start(sys.argv[2] if len(sys.argv) > 2 else os.getcwd(), exception) raise exception diff --git a/dev/docker_data/unpinned_requirements.txt b/dev/docker_data/unpinned_requirements.txt index 36deea071f..1d14e14e2f 100644 --- a/dev/docker_data/unpinned_requirements.txt +++ b/dev/docker_data/unpinned_requirements.txt @@ -30,4 +30,4 @@ sdcflows torch torchvision traits -voluptuous \ No newline at end of file +voluptuous diff --git a/dev/generate_singularity.sh b/dev/generate_singularity.sh index 15c804c225..0ee7c5fd5d 100755 --- a/dev/generate_singularity.sh +++ b/dev/generate_singularity.sh @@ -9,4 +9,4 @@ docker run \ singularityware/docker2singularity \ ${1} -find ${OUT} -name "*.img" \ No newline at end of file +find ${OUT} -name "*.img" diff --git a/dev/rc_tests/README.md b/dev/rc_tests/README.md index 8b1bb769e2..604c20c736 100644 --- a/dev/rc_tests/README.md +++ b/dev/rc_tests/README.md @@ -18,4 +18,4 @@ - [ ] MDMR - [ ] ISC / ISFC - [ ] BASC - - [ ] QPP \ No newline at end of file + - [ ] QPP diff --git a/scripts/cpac b/scripts/cpac index 7f4d63b4e5..4bc78b7dee 100644 --- a/scripts/cpac +++ b/scripts/cpac @@ -2,4 +2,4 @@ from CPAC.__main__ import main -main() \ No newline at end of file +main() diff --git a/setup.py b/setup.py index 4a1fd59694..3567525911 100755 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -"""Copyright (C) 2022 C-PAC Developers +"""Copyright (C) 2022 C-PAC Developers. This file is part of C-PAC. @@ -13,53 +13,55 @@ License for more details. You should have received a copy of the GNU Lesser General Public -License along with C-PAC. If not, see .""" +License along with C-PAC. If not, see . +""" import os -def configuration(parent_package='', top_path=None): +def configuration(parent_package="", top_path=None): from numpy.distutils.misc_util import Configuration config = Configuration(None, parent_package, top_path) - config.set_options(ignore_setup_xxx_py=True, - assume_default_configuration=True, - delegate_options_to_subpackages=True, - quiet=True) + config.set_options( + ignore_setup_xxx_py=True, + assume_default_configuration=True, + delegate_options_to_subpackages=True, + quiet=True, + ) - config.get_version('CPAC/__init__.py') + config.get_version("CPAC/__init__.py") - for root, _, _ in list(os.walk('CPAC')): - if os.path.isfile(os.path.join(root, '__init__.py')): + for root, _, _ in list(os.walk("CPAC")): + if os.path.isfile(os.path.join(root, "__init__.py")): config.add_subpackage(root) - config.add_data_dir('CPAC/resources') + config.add_data_dir("CPAC/resources") - config.add_define_macros([ - "NPY_NO_DEPRECATED_API", - "NPY_1_7_API_VERSION" - ]) + config.add_define_macros(["NPY_NO_DEPRECATED_API", "NPY_1_7_API_VERSION"]) return config def main(**extra_args): - from numpy.distutils.core import setup from glob import glob + + from numpy.distutils.core import setup + from CPAC.info import ( - NAME, - MAINTAINER, - MAINTAINER_EMAIL, + AUTHOR, + AUTHOR_EMAIL, + CLASSIFIERS, DESCRIPTION, - LONG_DESCRIPTION, - URL, DOWNLOAD_URL, LICENSE, - CLASSIFIERS, - AUTHOR, - AUTHOR_EMAIL, + LONG_DESCRIPTION, + MAINTAINER, + MAINTAINER_EMAIL, + NAME, PLATFORMS, - VERSION, REQUIREMENTS, + URL, + VERSION, ) setup( @@ -78,21 +80,17 @@ def main(**extra_args): version=VERSION, install_requires=REQUIREMENTS, configuration=configuration, - scripts=glob('scripts/*'), - entry_points={ - 'console_scripts': [ - 'cpac = CPAC.__main__:main' - ] - }, + scripts=glob("scripts/*"), + entry_points={"console_scripts": ["cpac = CPAC.__main__:main"]}, package_data={ - 'CPAC': [ - 'test_data/*', - 'test/templates/*', - 'qc/colors/*.txt', - 'qc/data/index.html', + "CPAC": [ + "test_data/*", + "test/templates/*", + "qc/colors/*.txt", + "qc/data/index.html", ] }, - **extra_args + **extra_args, )