From 71962aabd8148d8fd077dbeb747f64d1d018d262 Mon Sep 17 00:00:00 2001 From: kopardev Date: Thu, 28 Mar 2024 17:36:06 -0400 Subject: [PATCH 1/7] chore: adding GHAs --- .github/workflows/add_reponame_labels.yml | 10 ++++++++++ .github/workflows/auto-add-user-project.yml | 16 ++++++++++++++++ .github/workflows/docs.yml | 20 ++++++++++++++++++++ 3 files changed, 46 insertions(+) create mode 100644 .github/workflows/add_reponame_labels.yml create mode 100644 .github/workflows/auto-add-user-project.yml create mode 100644 .github/workflows/docs.yml diff --git a/.github/workflows/add_reponame_labels.yml b/.github/workflows/add_reponame_labels.yml new file mode 100644 index 0000000..a6d4acb --- /dev/null +++ b/.github/workflows/add_reponame_labels.yml @@ -0,0 +1,10 @@ +name: Auto add reponame as label to all issues and PRs + +on: + issues: + pull_request: + +jobs: + add_label: + uses: CCBR/.github/.github/workflows/add_reponame_issue_label.yml@v0.2.0 + secrets: inherit diff --git a/.github/workflows/auto-add-user-project.yml b/.github/workflows/auto-add-user-project.yml new file mode 100644 index 0000000..7fdaa6e --- /dev/null +++ b/.github/workflows/auto-add-user-project.yml @@ -0,0 +1,16 @@ +name: Add to personal projects + +on: + issues: + types: + - assigned + pull_request: + types: + - assigned + +jobs: + add-to-project: + uses: CCBR/.github/.github/workflows/auto-add-user-project.yml@6af5593b1ad6d7ee2b7f4c23b351902d4baaacd6 + with: + username: ${{ github.event.assignee.login }} + secrets: inherit diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..534fa92 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,20 @@ +name: docs +on: + workflow_dispatch: + push: + branches: + - main + paths: + - "docs/**" + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: 3.11 + - run: pip install --upgrade pip + - run: pip install -r docs/requirements.txt + - run: mkdocs gh-deploy --force From d41d9714dfd80a5289940eb0206cb2ec26b5f472 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Fri, 12 Apr 2024 22:55:15 -0400 Subject: [PATCH 2/7] fix: parkit package layout resolves #8 --- .gitignore | 2 +- pyproject.toml | 14 +++++--------- {parkit_pkg => src/parkit}/README.md | 0 {parkit_pkg => src/parkit}/__init__.py | 2 +- {parkit_pkg => src/parkit}/__main__.py | 10 +++++----- {parkit_pkg => src/parkit}/parkit_folder2hpcdme.py | 2 +- .../parkit}/parkit_tarball2hpcdme.py | 2 +- {parkit_pkg => src/parkit}/resources/argparse.bash | 0 .../parkit}/resources/user-projects.csv | 0 .../parkit}/scripts/parkit_folder2hpcdme | 0 .../parkit}/scripts/parkit_tarball2hpcdme | 0 {parkit_pkg => src/parkit}/src/VERSION | 0 {parkit_pkg => src/parkit}/src/VersionCheck.py | 0 {parkit_pkg => src/parkit}/src/__init__.py | 0 .../parkit}/src/createemptycollection.py | 2 +- {parkit_pkg => src/parkit}/src/createmetadata.py | 2 +- {parkit_pkg => src/parkit}/src/createtar.py | 2 +- {parkit_pkg => src/parkit}/src/deposittar.py | 2 +- {parkit_pkg => src/parkit}/src/utils.py | 0 .../parkit}/update_collection_metadata.py | 2 +- 20 files changed, 19 insertions(+), 23 deletions(-) rename {parkit_pkg => src/parkit}/README.md (100%) rename {parkit_pkg => src/parkit}/__init__.py (96%) rename {parkit_pkg => src/parkit}/__main__.py (92%) rename {parkit_pkg => src/parkit}/parkit_folder2hpcdme.py (85%) rename {parkit_pkg => src/parkit}/parkit_tarball2hpcdme.py (85%) rename {parkit_pkg => src/parkit}/resources/argparse.bash (100%) rename {parkit_pkg => src/parkit}/resources/user-projects.csv (100%) rename {parkit_pkg => src/parkit}/scripts/parkit_folder2hpcdme (100%) rename {parkit_pkg => src/parkit}/scripts/parkit_tarball2hpcdme (100%) rename {parkit_pkg => src/parkit}/src/VERSION (100%) rename {parkit_pkg => src/parkit}/src/VersionCheck.py (100%) rename {parkit_pkg => src/parkit}/src/__init__.py (100%) rename {parkit_pkg => src/parkit}/src/createemptycollection.py (98%) rename {parkit_pkg => src/parkit}/src/createmetadata.py (98%) rename {parkit_pkg => src/parkit}/src/createtar.py (98%) rename {parkit_pkg => src/parkit}/src/deposittar.py (96%) rename {parkit_pkg => src/parkit}/src/utils.py (100%) rename {parkit_pkg => src/parkit}/update_collection_metadata.py (98%) diff --git a/.gitignore b/.gitignore index 6321bd9..ff5c58d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ **/site/* **/dist/* **/*.egg_info/* -**/parkit_pkg.egg-info/* +**/parkit.egg-info/* **/.koparde* **/*.pyc **/.prettierrc diff --git a/pyproject.toml b/pyproject.toml index f748e5e..8cd3033 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,12 +6,8 @@ requires = ["setuptools>=61.0", "wheel", "setuptools-scm"] build-backend = "setuptools.build_meta" -# flat-layout fix .. because you have other directory than "parkit" in your main project folder -[tool.setuptools] -py-modules = ["parkit_pkg"] - [project] -name = "parkit_pkg" +name = "parkit" version = "2.0.0" description = "Parkit" authors = [ @@ -34,7 +30,7 @@ dependencies = ["pandas"] # Issues = "https://github.com/pypa/sampleproject/issues" [project.scripts] -parkit = "parkit_pkg.__main__:main" -parkit_folder2hpcdme = "parkit_pkg.parkit_folder2hpcdme:main" -parkit_tarball2hpcdme = "parkit_pkg.parkit_tarball2hpcdme:main" -update_collection_metadata = "parkit_pkg.update_collection_metadata:main" +parkit = "parkit.__main__:main" +parkit_folder2hpcdme = "parkit.parkit_folder2hpcdme:main" +parkit_tarball2hpcdme = "parkit.parkit_tarball2hpcdme:main" +update_collection_metadata = "parkit.update_collection_metadata:main" diff --git a/parkit_pkg/README.md b/src/parkit/README.md similarity index 100% rename from parkit_pkg/README.md rename to src/parkit/README.md diff --git a/parkit_pkg/__init__.py b/src/parkit/__init__.py similarity index 96% rename from parkit_pkg/__init__.py rename to src/parkit/__init__.py index 8fb9ea8..a13bd07 100644 --- a/parkit_pkg/__init__.py +++ b/src/parkit/__init__.py @@ -1,5 +1,5 @@ import os -from parkit_pkg.src.utils import * +from parkit.src.utils import * hpc = "" if which("scontrol") != None: diff --git a/parkit_pkg/__main__.py b/src/parkit/__main__.py similarity index 92% rename from parkit_pkg/__main__.py rename to src/parkit/__main__.py index ca65c52..a752b3a 100755 --- a/parkit_pkg/__main__.py +++ b/src/parkit/__main__.py @@ -1,11 +1,11 @@ #!/usr/bin/env python3 import argparse -from parkit_pkg.src.createtar import createtar, tarprep -from parkit_pkg.src.createmetadata import createmetadata -from parkit_pkg.src.createemptycollection import createemptycollection -from parkit_pkg.src.deposittar import deposittocollection -from parkit_pkg.src.VersionCheck import __version__ +from parkit.src.createtar import createtar, tarprep +from parkit.src.createmetadata import createmetadata +from parkit.src.createemptycollection import createemptycollection +from parkit.src.deposittar import deposittocollection +from parkit.src.VersionCheck import __version__ def main(): diff --git a/parkit_pkg/parkit_folder2hpcdme.py b/src/parkit/parkit_folder2hpcdme.py similarity index 85% rename from parkit_pkg/parkit_folder2hpcdme.py rename to src/parkit/parkit_folder2hpcdme.py index 6a9ee74..32292da 100644 --- a/parkit_pkg/parkit_folder2hpcdme.py +++ b/src/parkit/parkit_folder2hpcdme.py @@ -8,7 +8,7 @@ def main(): p = Path(__file__).absolute() pp = str(p.parent) - # script_path = 'parkit_pkg/scripts/parkit_folder2hpcdme' + # script_path = 'parkit/scripts/parkit_folder2hpcdme' script_path = os.path.join(pp, "scripts", "parkit_folder2hpcdme") # Pass all arguments to the bash script diff --git a/parkit_pkg/parkit_tarball2hpcdme.py b/src/parkit/parkit_tarball2hpcdme.py similarity index 85% rename from parkit_pkg/parkit_tarball2hpcdme.py rename to src/parkit/parkit_tarball2hpcdme.py index 0401172..adcff56 100644 --- a/parkit_pkg/parkit_tarball2hpcdme.py +++ b/src/parkit/parkit_tarball2hpcdme.py @@ -8,7 +8,7 @@ def main(): p = Path(__file__).absolute() pp = str(p.parent) - # script_path = 'parkit_pkg/scripts/parkit_tarball2hpcdme' + # script_path = 'parkit/scripts/parkit_tarball2hpcdme' script_path = os.path.join(pp, "scripts", "parkit_tarball2hpcdme") # Pass all arguments to the bash script diff --git a/parkit_pkg/resources/argparse.bash b/src/parkit/resources/argparse.bash similarity index 100% rename from parkit_pkg/resources/argparse.bash rename to src/parkit/resources/argparse.bash diff --git a/parkit_pkg/resources/user-projects.csv b/src/parkit/resources/user-projects.csv similarity index 100% rename from parkit_pkg/resources/user-projects.csv rename to src/parkit/resources/user-projects.csv diff --git a/parkit_pkg/scripts/parkit_folder2hpcdme b/src/parkit/scripts/parkit_folder2hpcdme similarity index 100% rename from parkit_pkg/scripts/parkit_folder2hpcdme rename to src/parkit/scripts/parkit_folder2hpcdme diff --git a/parkit_pkg/scripts/parkit_tarball2hpcdme b/src/parkit/scripts/parkit_tarball2hpcdme similarity index 100% rename from parkit_pkg/scripts/parkit_tarball2hpcdme rename to src/parkit/scripts/parkit_tarball2hpcdme diff --git a/parkit_pkg/src/VERSION b/src/parkit/src/VERSION similarity index 100% rename from parkit_pkg/src/VERSION rename to src/parkit/src/VERSION diff --git a/parkit_pkg/src/VersionCheck.py b/src/parkit/src/VersionCheck.py similarity index 100% rename from parkit_pkg/src/VersionCheck.py rename to src/parkit/src/VersionCheck.py diff --git a/parkit_pkg/src/__init__.py b/src/parkit/src/__init__.py similarity index 100% rename from parkit_pkg/src/__init__.py rename to src/parkit/src/__init__.py diff --git a/parkit_pkg/src/createemptycollection.py b/src/parkit/src/createemptycollection.py similarity index 98% rename from parkit_pkg/src/createemptycollection.py rename to src/parkit/src/createemptycollection.py index f0a0b4e..0b47ad1 100644 --- a/parkit_pkg/src/createemptycollection.py +++ b/src/parkit/src/createemptycollection.py @@ -2,7 +2,7 @@ import os from uuid import uuid4 from datetime import datetime -from parkit_pkg.src.utils import * +from parkit.src.utils import * def createemptycollection(collectionpath, projectdesc="", projecttitle=""): diff --git a/parkit_pkg/src/createmetadata.py b/src/parkit/src/createmetadata.py similarity index 98% rename from parkit_pkg/src/createmetadata.py rename to src/parkit/src/createmetadata.py index 467c00a..25a9a55 100644 --- a/parkit_pkg/src/createmetadata.py +++ b/src/parkit/src/createmetadata.py @@ -1,7 +1,7 @@ from pathlib import Path import json from uuid import uuid4 -from parkit_pkg.src.utils import * +from parkit.src.utils import * def createmetadata(infile, collectionpath): diff --git a/parkit_pkg/src/createtar.py b/src/parkit/src/createtar.py similarity index 98% rename from parkit_pkg/src/createtar.py rename to src/parkit/src/createtar.py index cfe80f7..3e98412 100644 --- a/parkit_pkg/src/createtar.py +++ b/src/parkit/src/createtar.py @@ -1,4 +1,4 @@ -from parkit_pkg.src.utils import * +from parkit.src.utils import * from pathlib import Path diff --git a/parkit_pkg/src/deposittar.py b/src/parkit/src/deposittar.py similarity index 96% rename from parkit_pkg/src/deposittar.py rename to src/parkit/src/deposittar.py index 18ac846..eb3db16 100644 --- a/parkit_pkg/src/deposittar.py +++ b/src/parkit/src/deposittar.py @@ -1,5 +1,5 @@ from pathlib import Path -from parkit_pkg.src.utils import * +from parkit.src.utils import * def deposittocollection(tar, collectionpath): diff --git a/parkit_pkg/src/utils.py b/src/parkit/src/utils.py similarity index 100% rename from parkit_pkg/src/utils.py rename to src/parkit/src/utils.py diff --git a/parkit_pkg/update_collection_metadata.py b/src/parkit/update_collection_metadata.py similarity index 98% rename from parkit_pkg/update_collection_metadata.py rename to src/parkit/update_collection_metadata.py index e0a662c..b80652a 100755 --- a/parkit_pkg/update_collection_metadata.py +++ b/src/parkit/update_collection_metadata.py @@ -7,7 +7,7 @@ from uuid import uuid4 # sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) -from parkit_pkg.src.utils import * +from parkit.src.utils import * def in_att_list(att_list, att): From daa4ece245f95496c12f5383bb9dfbc440fefa16 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Mon, 15 Apr 2024 09:27:23 -0400 Subject: [PATCH 3/7] fix: move readme to root dir --- README.md | 134 ++++++++++++++++++++++++++++++++++++++++++- src/parkit/README.md | 133 ------------------------------------------ 2 files changed, 133 insertions(+), 134 deletions(-) mode change 120000 => 100644 README.md delete mode 100644 src/parkit/README.md diff --git a/README.md b/README.md deleted file mode 120000 index 5a104e3..0000000 --- a/README.md +++ /dev/null @@ -1 +0,0 @@ -parkit_pkg/README.md \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..2e3367f --- /dev/null +++ b/README.md @@ -0,0 +1,133 @@ +## parkit :parking: :blue_car: + +**Park** an **arc**hived project tool**kit**! + +> DISCLAIMERS: +> +> - works only on [BIOWULF](https://hpc.nih.gov/) or HELIX +> - moves files to [HPC-DME](https://hpcdmeweb.nci.nih.gov/login) + +### Prerequisites: + +- On helix or biowulf you can get access to `parkit` by loading the appropriate conda env + +```bash +%> . "/data/CCBR_Pipeliner/db/PipeDB/Conda/etc/profile.d/conda.sh" +%> conda activate parkit +``` + +If not on helix or biowulf then you will have to **clone** the repo and **pip install** it. + +- [HPC_DME_APIs](https://github.com/CBIIT/HPC_DME_APIs) package needs to be cloned and set up correctly. Run `dm_generate_token` to successfully generate a token prior to running `parkit`. + +- **HPC_DM_UTILS** environmental variable should be preset before calling `parkit`. It also needs to be passed as an argument to `parkit_folder2hpcdme` and `parkit_tarball2hpcdme` end-to-end workflows. + +### Usage: + +```bash +%> parkit --help +usage: parkit [-h] {createtar,createmetadata,createemptycollection,deposittar} ... + +parkit subcommands to park data in HPCDME + +positional arguments: + {createtar,createmetadata,createemptycollection,deposittar} + Subcommand to run + createtar create tarball(and its filelist) from a project folder. + createmetadata create the metadata.json file required for a tarball (and its filelist) + createemptycollection + creates empty project and analysis collections + deposittar deposit tarball(and filelist) into vault + +options: + -h, --help show this help message and exit +``` + +### Example: + +- Say you want to archive `/data/CCBR/projects/CCBR-12345` folder to `/CCBR_Archive/GRIDFTP/CCBR-12345` collection on HPC-DME +- you can run the following commands sequentially to do this: + +```bash +# create the tarball +%> parkit createtar --folder /data/CCBR/projects/ccbr_12345 + +# create an empty collection on HPC-DME +%> parkit createemptycollection --dest /CCBR_Archive/GRIDFTP/CCBR-12345 --projectdesc "testing" --projecttitle "test project 1" + +# create required metadata +%> parkit createmetadata --tarball /data/CCBR/projects/ccbr_12345.tar --dest /CCBR_Archive/GRIDFTP/CCBR-12345 + +# deposit the tar into HPC-DME +%> parkit deposittar --tarball /data/CCBR/projects/ccbr_12345.tar --dest /CCBR_Archive/GRIDFTP/CCBR-12345 + +# bunch of extra files are created in the process +%> ls /data/CCBR/projects/ccbr_12345.tar* +/data/CCBR/projects/ccbr_12345.tar /data/CCBR/projects/ccbr_12345.tar.filelist.md5 /data/CCBR/projects/ccbr_12345.tar.md5 +/data/CCBR/projects/ccbr_12345.tar.filelist /data/CCBR/projects/ccbr_12345.tar.filelist.metadata.json /data/CCBR/projects/ccbr_12345.tar.metadata.json + +# these extra files can now be deleted +%> rm -f /data/CCBR/projects/ccbr_12345.tar* + +# you can also deleted the recently parked project folder +%> rm -rf /data/CCBR/projects/ccbr_12345 + +# test results with +%> dm_get_collection /CCBR_Archive/GRIDFTP/CCBR-12345 +# Done! +``` + +We also have end-to-end slurm-supported folder-to-hpcdme and tarball-to-hpcdme workflows: + +- `parkit_folder2hpcdme` +- `parkit_tarball2hpcdme` + +```bash +%> parkit_folder2hpcdme --help +usage: parkit_folder2hpcdme [-h] [--restartfrom RESTARTFROM] [--executor EXECUTOR] [--folder FOLDER] [--dest DEST] + [--projectdesc PROJECTDESC] [--projecttitle PROJECTTITLE] [--cleanup] --hpcdmutilspath HPCDMUTILSPATH + [--version] + +End-to-end parkit: Folder 2 HPCDME + +options: + -h, --help show this help message and exit + --restartfrom RESTARTFROM + if restarting then restart from this step. Options are: createemptycollection, createmetadata, deposittar + --executor EXECUTOR slurm or local + --folder FOLDER project folder to archive + --dest DEST vault collection path (Analysis goes under here!) + --projectdesc PROJECTDESC + project description + --projecttitle PROJECTTITLE + project title + --cleanup post transfer step to delete local files + --hpcdmutilspath HPCDMUTILSPATH + what should be the value of env var HPC_DM_UTILS + --version print version +``` + +```bash +parkit_tarball2hpcdme --help +usage: parkit_tarball2hpcdme [-h] [--restartfrom RESTARTFROM] [--executor EXECUTOR] [--tarball TARBALL] [--dest DEST] + [--projectdesc PROJECTDESC] [--projecttitle PROJECTTITLE] [--cleanup] --hpcdmutilspath HPCDMUTILSPATH + [--version] + +End-to-end parkit: Tarball 2 HPCDME + +options: + -h, --help show this help message and exit + --restartfrom RESTARTFROM + if restarting then restart from this step. Options are: createemptycollection, createmetadata, deposittar + --executor EXECUTOR slurm or local + --tarball TARBALL project tarball to archive + --dest DEST vault collection path (Analysis goes under here!) + --projectdesc PROJECTDESC + project description + --projecttitle PROJECTTITLE + project title + --cleanup post transfer step to delete local files + --hpcdmutilspath HPCDMUTILSPATH + what should be the value of env var HPC_DM_UTILS + --version print version +``` diff --git a/src/parkit/README.md b/src/parkit/README.md deleted file mode 100644 index 2e3367f..0000000 --- a/src/parkit/README.md +++ /dev/null @@ -1,133 +0,0 @@ -## parkit :parking: :blue_car: - -**Park** an **arc**hived project tool**kit**! - -> DISCLAIMERS: -> -> - works only on [BIOWULF](https://hpc.nih.gov/) or HELIX -> - moves files to [HPC-DME](https://hpcdmeweb.nci.nih.gov/login) - -### Prerequisites: - -- On helix or biowulf you can get access to `parkit` by loading the appropriate conda env - -```bash -%> . "/data/CCBR_Pipeliner/db/PipeDB/Conda/etc/profile.d/conda.sh" -%> conda activate parkit -``` - -If not on helix or biowulf then you will have to **clone** the repo and **pip install** it. - -- [HPC_DME_APIs](https://github.com/CBIIT/HPC_DME_APIs) package needs to be cloned and set up correctly. Run `dm_generate_token` to successfully generate a token prior to running `parkit`. - -- **HPC_DM_UTILS** environmental variable should be preset before calling `parkit`. It also needs to be passed as an argument to `parkit_folder2hpcdme` and `parkit_tarball2hpcdme` end-to-end workflows. - -### Usage: - -```bash -%> parkit --help -usage: parkit [-h] {createtar,createmetadata,createemptycollection,deposittar} ... - -parkit subcommands to park data in HPCDME - -positional arguments: - {createtar,createmetadata,createemptycollection,deposittar} - Subcommand to run - createtar create tarball(and its filelist) from a project folder. - createmetadata create the metadata.json file required for a tarball (and its filelist) - createemptycollection - creates empty project and analysis collections - deposittar deposit tarball(and filelist) into vault - -options: - -h, --help show this help message and exit -``` - -### Example: - -- Say you want to archive `/data/CCBR/projects/CCBR-12345` folder to `/CCBR_Archive/GRIDFTP/CCBR-12345` collection on HPC-DME -- you can run the following commands sequentially to do this: - -```bash -# create the tarball -%> parkit createtar --folder /data/CCBR/projects/ccbr_12345 - -# create an empty collection on HPC-DME -%> parkit createemptycollection --dest /CCBR_Archive/GRIDFTP/CCBR-12345 --projectdesc "testing" --projecttitle "test project 1" - -# create required metadata -%> parkit createmetadata --tarball /data/CCBR/projects/ccbr_12345.tar --dest /CCBR_Archive/GRIDFTP/CCBR-12345 - -# deposit the tar into HPC-DME -%> parkit deposittar --tarball /data/CCBR/projects/ccbr_12345.tar --dest /CCBR_Archive/GRIDFTP/CCBR-12345 - -# bunch of extra files are created in the process -%> ls /data/CCBR/projects/ccbr_12345.tar* -/data/CCBR/projects/ccbr_12345.tar /data/CCBR/projects/ccbr_12345.tar.filelist.md5 /data/CCBR/projects/ccbr_12345.tar.md5 -/data/CCBR/projects/ccbr_12345.tar.filelist /data/CCBR/projects/ccbr_12345.tar.filelist.metadata.json /data/CCBR/projects/ccbr_12345.tar.metadata.json - -# these extra files can now be deleted -%> rm -f /data/CCBR/projects/ccbr_12345.tar* - -# you can also deleted the recently parked project folder -%> rm -rf /data/CCBR/projects/ccbr_12345 - -# test results with -%> dm_get_collection /CCBR_Archive/GRIDFTP/CCBR-12345 -# Done! -``` - -We also have end-to-end slurm-supported folder-to-hpcdme and tarball-to-hpcdme workflows: - -- `parkit_folder2hpcdme` -- `parkit_tarball2hpcdme` - -```bash -%> parkit_folder2hpcdme --help -usage: parkit_folder2hpcdme [-h] [--restartfrom RESTARTFROM] [--executor EXECUTOR] [--folder FOLDER] [--dest DEST] - [--projectdesc PROJECTDESC] [--projecttitle PROJECTTITLE] [--cleanup] --hpcdmutilspath HPCDMUTILSPATH - [--version] - -End-to-end parkit: Folder 2 HPCDME - -options: - -h, --help show this help message and exit - --restartfrom RESTARTFROM - if restarting then restart from this step. Options are: createemptycollection, createmetadata, deposittar - --executor EXECUTOR slurm or local - --folder FOLDER project folder to archive - --dest DEST vault collection path (Analysis goes under here!) - --projectdesc PROJECTDESC - project description - --projecttitle PROJECTTITLE - project title - --cleanup post transfer step to delete local files - --hpcdmutilspath HPCDMUTILSPATH - what should be the value of env var HPC_DM_UTILS - --version print version -``` - -```bash -parkit_tarball2hpcdme --help -usage: parkit_tarball2hpcdme [-h] [--restartfrom RESTARTFROM] [--executor EXECUTOR] [--tarball TARBALL] [--dest DEST] - [--projectdesc PROJECTDESC] [--projecttitle PROJECTTITLE] [--cleanup] --hpcdmutilspath HPCDMUTILSPATH - [--version] - -End-to-end parkit: Tarball 2 HPCDME - -options: - -h, --help show this help message and exit - --restartfrom RESTARTFROM - if restarting then restart from this step. Options are: createemptycollection, createmetadata, deposittar - --executor EXECUTOR slurm or local - --tarball TARBALL project tarball to archive - --dest DEST vault collection path (Analysis goes under here!) - --projectdesc PROJECTDESC - project description - --projecttitle PROJECTTITLE - project title - --cleanup post transfer step to delete local files - --hpcdmutilspath HPCDMUTILSPATH - what should be the value of env var HPC_DM_UTILS - --version print version -``` From 51ac8fd5607cb65ca061380b7890006ce791df0c Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Mon, 15 Apr 2024 09:55:18 -0400 Subject: [PATCH 4/7] test: setup testing infra --- .github/workflows/test.yml | 33 +++++++++++++++++++++++++++++++++ pyproject.toml | 18 +++++++++++++++--- tests/test_cli.py | 9 +++++++++ 3 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/test.yml create mode 100644 tests/test_cli.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..1805e2a --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,33 @@ +name: test + +on: + push: + branches: + - main + - develop + pull_request: + branches: + - main + - develop + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 2 + strategy: + matrix: + python-version: ["3.11"] + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: "pip" + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools + pip install .[dev,test] + - name: Test + run: | + python -m pytest diff --git a/pyproject.toml b/pyproject.toml index 8cd3033..5af1d3f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ build-backend = "setuptools.build_meta" [project] name = "parkit" -version = "2.0.0" +dynamic = ['version','readme'] description = "Parkit" authors = [ { name="kopardev", email="vishal.koparde@nih.gov" }, @@ -17,13 +17,21 @@ license = {file = "LICENSE.md"} readme = "README.md" requires-python = ">=3.11" # keywords = [] -dependencies = ["pandas"] - # classifiers = [ # "Programming Language :: Python :: 3", # "License :: OSI Approved :: MIT License", # "Operating System :: OS Independent", # ] +dependencies = ["pandas"] + +[project.optional-dependencies] +dev = [ + "black >= 22.0.0", + "pre-commit" +] +test = [ + "pytest" +] # [project.urls] # Homepage = "https://github.com/pypa/sampleproject" @@ -34,3 +42,7 @@ parkit = "parkit.__main__:main" parkit_folder2hpcdme = "parkit.parkit_folder2hpcdme:main" parkit_tarball2hpcdme = "parkit.parkit_tarball2hpcdme:main" update_collection_metadata = "parkit.update_collection_metadata:main" + +[tool.setuptools.dynamic] +version = {file = "VERSION"} +readme = {file = "README.md"} diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..ab3a8d0 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,9 @@ +import pytest +import subprocess + + +def test_help(): + output = subprocess.run( + "parkit --help", capture_output=True, shell=True, text=True + ).stdout + assert "ERROR:HPC_DM_UTILS in unset!" in output From 56539dd8a319c94b1945ec04a424340c3a980d7c Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Mon, 15 Apr 2024 09:58:03 -0400 Subject: [PATCH 5/7] refactor: make readme & version dynamic --- pyproject.toml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8cd3033..6e381d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,17 +8,15 @@ build-backend = "setuptools.build_meta" [project] name = "parkit" -version = "2.0.0" +dynamic = ['version','readme'] description = "Parkit" authors = [ { name="kopardev", email="vishal.koparde@nih.gov" }, ] license = {file = "LICENSE.md"} -readme = "README.md" requires-python = ">=3.11" # keywords = [] dependencies = ["pandas"] - # classifiers = [ # "Programming Language :: Python :: 3", # "License :: OSI Approved :: MIT License", @@ -34,3 +32,7 @@ parkit = "parkit.__main__:main" parkit_folder2hpcdme = "parkit.parkit_folder2hpcdme:main" parkit_tarball2hpcdme = "parkit.parkit_tarball2hpcdme:main" update_collection_metadata = "parkit.update_collection_metadata:main" + +[tool.setuptools.dynamic] +version = {file = "src/parkit/src/VERSION"} +readme = {file = "README.md"} From 4379732fa79e6382739f3e92e99c8267c450affe Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Mon, 15 Apr 2024 09:58:31 -0400 Subject: [PATCH 6/7] chore: ignore build dir --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ff5c58d..41f184a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +build/ **/site/* **/dist/* **/*.egg_info/* From 195437c940612501eb7a7abb8fe2f08b1c2b46fd Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Mon, 15 Apr 2024 10:03:56 -0400 Subject: [PATCH 7/7] fix: only run labels action when issue or PR is opened Otherwise, it runs every time new commits are pushed to a branch with an open PR. This version of the workflow has this correction: https://github.com/CCBR/.github/blob/58cae128e4dc13c05186a2f187c6114653ebc843/.github/workflows/reponame_labels.yaml --- .github/workflows/add_reponame_labels.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/add_reponame_labels.yml b/.github/workflows/add_reponame_labels.yml index a6d4acb..4d5c25e 100644 --- a/.github/workflows/add_reponame_labels.yml +++ b/.github/workflows/add_reponame_labels.yml @@ -2,7 +2,11 @@ name: Auto add reponame as label to all issues and PRs on: issues: + types: + - opened pull_request: + types: + - opened jobs: add_label: