✅ Fix tests broken by #2058

.github/Dockerfiles/C-PAC.develop-ABCD-HCP-bionic.Dockerfile

@@ -1,3 +1,13 @@
+#!/bin/bash
+# Copyright (C) 2022-2024  C-PAC Developers
+
+# This file is part of C-PAC.
+
+# C-PAC is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
+
+# C-PAC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public License along with C-PAC. If not, see <https://www.gnu.org/licenses/>.
 FROM ghcr.io/fcp-indi/c-pac/stage-base:abcd-hcp-v1.8.7.dev1
 LABEL org.opencontainers.image.description "Full C-PAC image with software dependencies version-matched to [ABCD-HCP BIDS fMRI Pipeline](https://github.com/DCAN-Labs/abcd-hcp-pipeline/blob/e480a8f99534f1b05f37bf44c64827384b69b383/Dockerfile)"
 LABEL org.opencontainers.image.source https://github.com/FCP-INDI/C-PAC
@@ -12,9 +22,9 @@ COPY dev/docker_data /code/docker_data
 RUN rm -Rf /code/docker_data/checksum && \
     mv /code/docker_data/* /code && \
     rm -Rf /code/docker_data && \
-    chmod +x /code/run.py && \
-    chmod +x /code/run-with-freesurfer.sh
-ENTRYPOINT ["/code/run-with-freesurfer.sh"]
+    chmod +x /code/CPAC/_entrypoints/run.py && \
+    chmod +x /code/CPAC/_entrypoints/run-with-freesurfer.sh
+ENTRYPOINT ["/code/CPAC/_entrypoints/run-with-freesurfer.sh"]
 
 # Link libraries for Singularity images
 RUN ldconfig \

.github/Dockerfiles/C-PAC.develop-fMRIPrep-LTS-xenial.Dockerfile

@@ -1,3 +1,13 @@
+#!/bin/bash
+# Copyright (C) 2021-2024  C-PAC Developers
+
+# This file is part of C-PAC.
+
+# C-PAC is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
+
+# C-PAC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public License along with C-PAC. If not, see <https://www.gnu.org/licenses/>.
 FROM ghcr.io/fcp-indi/c-pac/stage-base:fmriprep-lts-v1.8.7.dev1
 LABEL org.opencontainers.image.description "Full C-PAC image with software dependencies version-matched to [fMRIPrep LTS](https://reproducibility.stanford.edu/fmriprep-lts#long-term-support-lts)"
 LABEL org.opencontainers.image.source https://github.com/FCP-INDI/C-PAC
@@ -12,9 +22,9 @@ COPY dev/docker_data /code/docker_data
 RUN rm -Rf /code/docker_data/checksum && \
     mv /code/docker_data/* /code && \
     rm -Rf /code/docker_data && \
-    chmod +x /code/run.py && \
-    chmod +x /code/run-with-freesurfer.sh
-ENTRYPOINT ["/code/run-with-freesurfer.sh"]
+    chmod +x /code/CPAC/_entrypoints/run.py && \
+    chmod +x /code/CPAC/_entrypoints/run-with-freesurfer.sh
+ENTRYPOINT ["/code/CPAC/_entrypoints/run-with-freesurfer.sh"]
 
 # link libraries & clean up
 RUN sed -i 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen \

.github/Dockerfiles/C-PAC.develop-jammy.Dockerfile

@@ -1,4 +1,4 @@
-# Copyright (C) 2022-2023  C-PAC Developers
+# Copyright (C) 2022-2024  C-PAC Developers
 
 # This file is part of C-PAC.
 
@@ -28,9 +28,9 @@ COPY dev/docker_data /code/docker_data
 RUN rm -Rf /code/docker_data/checksum && \
     mv /code/docker_data/* /code && \
     rm -Rf /code/docker_data && \
-    chmod +x /code/run.py && \
-    chmod +x /code/run-with-freesurfer.sh
-ENTRYPOINT ["/code/run-with-freesurfer.sh"]
+    chmod +x /code/CPAC/_entrypoints/run.py && \
+    chmod +x /code/CPAC/_entrypoints/run-with-freesurfer.sh
+ENTRYPOINT ["/code/CPAC/_entrypoints/run-with-freesurfer.sh"]
 
 # link libraries & clean up
 # link libraries & clean up

.github/Dockerfiles/C-PAC.develop-lite-jammy.Dockerfile

@@ -1,4 +1,4 @@
-# Copyright (C) 2022-2023  C-PAC Developers
+# Copyright (C) 2022-2024  C-PAC Developers
 
 # This file is part of C-PAC.
 
@@ -29,9 +29,9 @@ COPY dev/docker_data /code/docker_data
 RUN rm -Rf /code/docker_data/checksum && \
     mv /code/docker_data/* /code && \
     rm -Rf /code/docker_data && \
-    chmod +x /code/run.py && \
-    rm -Rf /code/run-with-freesurfer.sh
-ENTRYPOINT ["/code/run.py"]
+    chmod +x /code/CPAC/_entrypoints/run.py && \
+    rm -Rf /code/CPAC/_entrypoints/run-with-freesurfer.sh
+ENTRYPOINT ["/code/CPAC/_entrypoints/run.py"]
 
 # link libraries & clean up
 # link libraries & clean up

.github/scripts/autoversioning.sh

@@ -18,7 +18,14 @@
 # License along with C-PAC. If not, see <https://www.gnu.org/licenses/>.
 
 # Update version comment strings
-cd CPAC
+function wait_for_git_lock() {
+    while [ -f "./.git/index.lock" ]; do
+        echo "Waiting for the git lock file to be removed..."
+        sleep 1
+    done
+}
+
+cd CPAC || exit 1
 VERSION=$(python -c "from info import __version__; print(('.'.join(('.'.join(__version__[::-1].split('-')[1].split('.')[1:])[::-1], __version__.split('-')[1])) if '-' in __version__ else __version__).split('+', 1)[0])")
 cd ..
 echo "v${VERSION}" > version
@@ -30,7 +37,7 @@ else
     # Linux and others
     find ./CPAC/resources/configs -name "*.yml" -exec sed -i'' -r "${_SED_COMMAND}" {} \;
 fi
-git add version
+wait_for_git_lock && git add version
 VERSIONS=( `git show $(git log --pretty=format:'%h' -n 2 version | tail -n 1):version` `cat version` )
 export PATTERN="(declare|typeset) -a"
 if [[ "$(declare -p VERSIONS)" =~ $PATTERN ]]
@@ -52,11 +59,15 @@ then
   done
   unset IFS
 fi
-git add CPAC/resources/configs .github/Dockerfiles
+wait_for_git_lock && git add CPAC/resources/configs .github/Dockerfiles
 
 # Overwrite top-level Dockerfiles with the CI Dockerfiles
 cp .github/Dockerfiles/C-PAC.develop-jammy.Dockerfile Dockerfile
 cp .github/Dockerfiles/C-PAC.develop-ABCD-HCP-bionic.Dockerfile variant-ABCD-HCP.Dockerfile
 cp .github/Dockerfiles/C-PAC.develop-fMRIPrep-LTS-xenial.Dockerfile variant-fMRIPrep-LTS.Dockerfile
 cp .github/Dockerfiles/C-PAC.develop-lite-jammy.Dockerfile variant-lite.Dockerfile
-git add *Dockerfile
+while [ -f "/Users/jon.clucas/CNL/GitHub/C-PAC/.git/index.lock" ]; do
+    echo "Waiting for the lock file to be removed..."
+    sleep 1
+done
+wait_for_git_lock && git add *Dockerfile

.github/scripts/get_package_id.py

@@ -1,3 +1,19 @@
+# Copyright (C) 2021-2024  C-PAC Developers
+
+# This file is part of C-PAC.
+
+# C-PAC is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.
+
+# C-PAC is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
+# License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with C-PAC. If not, see <https://www.gnu.org/licenses/>.
 """Get Package ID.
 
 Script to get GHCR ID string for a given owner + image tag
@@ -11,7 +27,7 @@
 
 
 def get_packages(owner, tag, api_token=None):
-    """Function to collect GHCR packages for a given owner & tag.
+    """Collect GHCR packages for a given owner & tag.
 
     Parameters
     ----------
@@ -32,7 +48,7 @@ def get_packages(owner, tag, api_token=None):
         api_token = os.environ.get("GITHUB_TOKEN", "")
 
     def fetch(url):
-        """Method to make API call and return response, given a URL.
+        """Make API call and return response, given a URL.
 
         Parameters
         ----------
@@ -82,7 +98,7 @@ def fetch(url):
 
 
 def id_from_tag(owner, image, tag, api_token=None):
-    """Function to return a package ID given an image version tag.
+    """Return a package ID given an image version tag.
 
     Parameters
     ----------
@@ -106,12 +122,12 @@ def id_from_tag(owner, image, tag, api_token=None):
     ]
     if len(versions):
         return versions[0]
-    else:
-        raise LookupError(f"Image not found: ghcr.io/{owner}/{image}:{tag}")
+    msg = f"Image not found: ghcr.io/{owner}/{image}:{tag}"
+    raise LookupError(msg)
 
 
 if __name__ == "__main__":
-    if len(sys.argv) == 4:
-        pass
+    if len(sys.argv) == 4:  # noqa: PLR2004
+        print(id_from_tag(*sys.argv[1:]))  # noqa: T201
     else:
-        pass
+        print(__doc__)  # noqa: T201

.github/scripts/get_pr_base_shas.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2022-2023  C-PAC Developers
+# Copyright (C) 2022-2024  C-PAC Developers
 
 # This file is part of C-PAC.
 
@@ -15,3 +15,18 @@
 # You should have received a copy of the GNU Lesser General Public
 # License along with C-PAC. If not, see <https://www.gnu.org/licenses/>.
 """Get base SHAs for open PRs."""
+import os
+
+from github import Github
+
+print(  # noqa: T201
+    " ".join(
+        [
+            pr.base.sha
+            for pr in Github(os.environ.get("GITHUB_TOKEN"))
+            .get_repo(os.environ.get("GITHUB_REPOSITORY"))
+            .get_commit(os.environ.get("GITHUB_SHA"))
+            .get_pulls()
+        ]
+    )
+)

.pre-commit-config.yaml

@@ -29,10 +29,12 @@ repos:
     hooks:
     - id: check-case-conflict
     - id: end-of-file-fixer
+      exclude: '.*\.?sv|.*\.pkl(z)?'
     - id: mixed-line-ending
       args:
       - --fix=lf
     - id: trailing-whitespace
+      exclude: '.*\.tsv'
     - id: check-json
     - id: pretty-format-json
       args:

.ruff.toml

@@ -1,10 +1,11 @@
 extend-exclude = ["dev/docker_data/get-pip_23.0.1.py"]
-extend-select = ["A", "C4", "D", "G", "I", "ICN", "NPY", "PL", "RET", "RSE", "RUF", "Q", "T20", "W"]
+extend-select = ["A", "C4", "D", "EM", "F541", "G", "I", "ICN", "NPY", "PL", "RET", "RSE", "RUF", "Q", "T20", "UP032", "W"]
+# proposed rules to add next release cylce: ["B904", "LOG007", "TRY002", "TRY201", "TRY400", "TRY401"]
 # variants still use 3.7
 target-version = "py37"
 
-[per-file-ignores]
-"CPAC/func_preproc/func_preproc.py" = ["E402"]
+[lint]
+external = ["T20"]  # Don't autoremove 'noqa` comments for these rules
 
 [lint.flake8-import-conventions.extend-aliases]
 "CPAC.pipeline.cpac_group_runner" = "cgr"
@@ -19,7 +20,7 @@ combine-as-imports = true
 force-sort-within-sections = true
 known-first-party = ["CPAC"]
 no-lines-before = ["collab", "other-first-party", "local-folder"]
-order-by-type = true
+order-by-type = false
 section-order = ["future", "standard-library", "third-party", "collab", "other-first-party", "first-party", "local-folder"]
 
 [lint.isort.sections]
@@ -39,3 +40,8 @@ max-statements = 100
 [lint.pyupgrade]
 # Until variants Python ≥ 3.10
 keep-runtime-typing = true
+
+[per-file-ignores]
+"CPAC/func_preproc/func_preproc.py" = ["E402"]
+"CPAC/utils/sklearn.py" = ["RUF003"]
+"CPAC/utils/utils.py" = ["T201"]  # until `repickle` is removed

CPAC/__main__.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# Copyright (C) 2018-2022  C-PAC Developers
+# Copyright (C) 2018-2024  C-PAC Developers
 
 # This file is part of C-PAC.
 
@@ -15,12 +15,20 @@
 
 # You should have received a copy of the GNU Lesser General Public
 # License along with C-PAC. If not, see <https://www.gnu.org/licenses/>.
+from logging import basicConfig, INFO
 import os
+from typing import Union
 
 import click
 from click_aliases import ClickAliasedGroup
 import pkg_resources as p
 
+from CPAC.utils.docs import version_report
+from CPAC.utils.monitoring.custom_logging import getLogger
+
+logger = getLogger("CPAC")
+basicConfig(format="%(message)s", level=INFO)
+
 # CLI tree
 #
 # cpac
@@ -55,6 +63,13 @@ def main():
 @main.command()
 def version():
     """Display environment version information."""
+    import CPAC
+
+    logger.info(
+        "Environment\n===========\n%s\nC-PAC version: %s",
+        version_report(),
+        CPAC.__version__,
+    )
 
 
 @main.command()
@@ -550,7 +565,7 @@ def test():
 @test.command(aliases=["run_suite"])
 @click.option("--list", "-l", "show_list", is_flag=True)
 @click.option("--filter", "-f", "pipeline_filter", default="")
-def run_suite(show_list=False, pipeline_filter=""):
+def run_suite(show_list: Union[bool, str] = False, pipeline_filter=""):
     from CPAC.pipeline import cpac_runner
 
     test_config_dir = p.resource_filename(
@@ -579,7 +594,7 @@ def run_suite(show_list=False, pipeline_filter=""):
     )
 
     if show_list:
-        pass
+        show_list = "\nAvailables pipelines:"
 
     no_params = False
     for config_file in os.listdir(test_config_dir):
@@ -588,6 +603,7 @@ def run_suite(show_list=False, pipeline_filter=""):
                 continue
 
             if show_list:
+                show_list += f"\n- {config_file[len('pipe-test_'):]}"
                 continue
 
             pipe = os.path.join(test_config_dir, config_file)
@@ -604,7 +620,7 @@ def run_suite(show_list=False, pipeline_filter=""):
             cpac_runner.run(data, pipe)
 
     if show_list:
-        pass
+        logger.info("%s\n", show_list)
 
 
 @test.group(cls=ClickAliasedGroup)

CPAC/_entrypoints/__init__.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+# Copyright (C) 2018-2024  C-PAC Developers
+
+# This file is part of C-PAC.
+
+# C-PAC is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.
+
+# C-PAC is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
+# License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with C-PAC. If not, see <https://www.gnu.org/licenses/>.
+"""Entrypoints for C-PAC containers."""
+__all__ = []

dev/docker_data/run-with-freesurfer.sh → CPAC/_entrypoints/run-with-freesurfer.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (C) 2022  C-PAC Developers
+# Copyright (C) 2022-2024  C-PAC Developers
 
 # This file is part of C-PAC.
 
@@ -9,4 +9,4 @@
 
 # You should have received a copy of the GNU Lesser General Public License along with C-PAC. If not, see <https://www.gnu.org/licenses/>.
 source $FREESURFER_HOME/SetUpFreeSurfer.sh
-/code/run.py "$@"
+/code/CPAC/_entrypoints/run.py "$@"