Merge pull request #9 from CCBR/iss-8

fix: parkit package layout

.gitignore

@@ -1,7 +1,8 @@
+build/
 **/site/*
 **/dist/*
 **/*.egg_info/*
-**/parkit_pkg.egg-info/*
+**/parkit.egg-info/*
 **/.koparde*
 **/*.pyc
 **/.prettierrc

README.md

@@ -0,0 +1,133 @@
+## parkit :parking: :blue_car:
+
+**Park** an **arc**hived project tool**kit**!
+
+> DISCLAIMERS:
+>
+> - works only on [BIOWULF](https://hpc.nih.gov/) or HELIX
+> - moves files to [HPC-DME](https://hpcdmeweb.nci.nih.gov/login)
+
+### Prerequisites:
+
+- On helix or biowulf you can get access to `parkit` by loading the appropriate conda env
+
+```bash
+%> . "/data/CCBR_Pipeliner/db/PipeDB/Conda/etc/profile.d/conda.sh"
+%> conda activate parkit
+```
+
+If not on helix or biowulf then you will have to **clone** the repo and **pip install** it.
+
+- [HPC_DME_APIs](https://github.com/CBIIT/HPC_DME_APIs) package needs to be cloned and set up correctly. Run `dm_generate_token` to successfully generate a token prior to running `parkit`.
+
+- **HPC_DM_UTILS** environmental variable should be preset before calling `parkit`. It also needs to be passed as an argument to `parkit_folder2hpcdme` and `parkit_tarball2hpcdme` end-to-end workflows.
+
+### Usage:
+
+```bash
+%> parkit --help
+usage: parkit [-h] {createtar,createmetadata,createemptycollection,deposittar} ...
+
+parkit subcommands to park data in HPCDME
+
+positional arguments:
+  {createtar,createmetadata,createemptycollection,deposittar}
+                        Subcommand to run
+    createtar           create tarball(and its filelist) from a project folder.
+    createmetadata      create the metadata.json file required for a tarball (and its filelist)
+    createemptycollection
+                        creates empty project and analysis collections
+    deposittar          deposit tarball(and filelist) into vault
+
+options:
+  -h, --help            show this help message and exit
+```
+
+### Example:
+
+- Say you want to archive `/data/CCBR/projects/CCBR-12345` folder to `/CCBR_Archive/GRIDFTP/CCBR-12345` collection on HPC-DME
+- you can run the following commands sequentially to do this:
+
+```bash
+# create the tarball
+%> parkit createtar --folder /data/CCBR/projects/ccbr_12345
+
+# create an empty collection on HPC-DME
+%> parkit createemptycollection --dest /CCBR_Archive/GRIDFTP/CCBR-12345 --projectdesc "testing" --projecttitle "test project 1"
+
+# create required metadata
+%> parkit createmetadata --tarball /data/CCBR/projects/ccbr_12345.tar --dest /CCBR_Archive/GRIDFTP/CCBR-12345
+
+# deposit the tar into HPC-DME
+%> parkit deposittar --tarball /data/CCBR/projects/ccbr_12345.tar --dest /CCBR_Archive/GRIDFTP/CCBR-12345
+
+# bunch of extra files are created in the process
+%> ls /data/CCBR/projects/ccbr_12345.tar*
+/data/CCBR/projects/ccbr_12345.tar           /data/CCBR/projects/ccbr_12345.tar.filelist.md5            /data/CCBR/projects/ccbr_12345.tar.md5
+/data/CCBR/projects/ccbr_12345.tar.filelist  /data/CCBR/projects/ccbr_12345.tar.filelist.metadata.json  /data/CCBR/projects/ccbr_12345.tar.metadata.json
+
+# these extra files can now be deleted
+%> rm -f /data/CCBR/projects/ccbr_12345.tar*
+
+# you can also deleted the recently parked project folder
+%> rm -rf /data/CCBR/projects/ccbr_12345
+
+# test results with
+%> dm_get_collection /CCBR_Archive/GRIDFTP/CCBR-12345
+# Done!
+```
+
+We also have end-to-end slurm-supported folder-to-hpcdme and tarball-to-hpcdme workflows:
+
+- `parkit_folder2hpcdme`
+- `parkit_tarball2hpcdme`
+
+```bash
+%> parkit_folder2hpcdme --help
+usage: parkit_folder2hpcdme [-h] [--restartfrom RESTARTFROM] [--executor EXECUTOR] [--folder FOLDER] [--dest DEST]
+                            [--projectdesc PROJECTDESC] [--projecttitle PROJECTTITLE] [--cleanup] --hpcdmutilspath HPCDMUTILSPATH
+                            [--version]
+
+End-to-end parkit: Folder 2 HPCDME
+
+options:
+  -h, --help            show this help message and exit
+  --restartfrom RESTARTFROM
+                        if restarting then restart from this step. Options are: createemptycollection, createmetadata, deposittar
+  --executor EXECUTOR   slurm or local
+  --folder FOLDER       project folder to archive
+  --dest DEST           vault collection path (Analysis goes under here!)
+  --projectdesc PROJECTDESC
+                        project description
+  --projecttitle PROJECTTITLE
+                        project title
+  --cleanup             post transfer step to delete local files
+  --hpcdmutilspath HPCDMUTILSPATH
+                        what should be the value of env var HPC_DM_UTILS
+  --version             print version
+```
+
+```bash
+parkit_tarball2hpcdme --help
+usage: parkit_tarball2hpcdme [-h] [--restartfrom RESTARTFROM] [--executor EXECUTOR] [--tarball TARBALL] [--dest DEST]
+                             [--projectdesc PROJECTDESC] [--projecttitle PROJECTTITLE] [--cleanup] --hpcdmutilspath HPCDMUTILSPATH
+                             [--version]
+
+End-to-end parkit: Tarball 2 HPCDME
+
+options:
+  -h, --help            show this help message and exit
+  --restartfrom RESTARTFROM
+                        if restarting then restart from this step. Options are: createemptycollection, createmetadata, deposittar
+  --executor EXECUTOR   slurm or local
+  --tarball TARBALL     project tarball to archive
+  --dest DEST           vault collection path (Analysis goes under here!)
+  --projectdesc PROJECTDESC
+                        project description
+  --projecttitle PROJECTTITLE
+                        project title
+  --cleanup             post transfer step to delete local files
+  --hpcdmutilspath HPCDMUTILSPATH
+                        what should be the value of env var HPC_DM_UTILS
+  --version             print version
+```

pyproject.toml

@@ -6,23 +6,17 @@
 requires = ["setuptools>=61.0", "wheel", "setuptools-scm"]
 build-backend = "setuptools.build_meta"
 
-# flat-layout fix .. because you have other directory than "parkit" in your main project folder
-[tool.setuptools]
-py-modules = ["parkit_pkg"]
-
 [project]
-name = "parkit_pkg"
-version = "2.0.0"
+name = "parkit"
+dynamic = ['version','readme']
 description = "Parkit"
 authors = [
     { name="kopardev", email="vishal.koparde@nih.gov" },
 ]
 license = {file = "LICENSE.md"}
-readme = "README.md"
 requires-python = ">=3.11"
 # keywords = []
 dependencies = ["pandas"]
-
 # classifiers = [
 #     "Programming Language :: Python :: 3",
 #     "License :: OSI Approved :: MIT License",
@@ -34,7 +28,11 @@ dependencies = ["pandas"]
 # Issues = "https://github.com/pypa/sampleproject/issues"
 
 [project.scripts]
-parkit = "parkit_pkg.__main__:main"
-parkit_folder2hpcdme = "parkit_pkg.parkit_folder2hpcdme:main"
-parkit_tarball2hpcdme = "parkit_pkg.parkit_tarball2hpcdme:main"
-update_collection_metadata = "parkit_pkg.update_collection_metadata:main"
+parkit = "parkit.__main__:main"
+parkit_folder2hpcdme = "parkit.parkit_folder2hpcdme:main"
+parkit_tarball2hpcdme = "parkit.parkit_tarball2hpcdme:main"
+update_collection_metadata = "parkit.update_collection_metadata:main"
+
+[tool.setuptools.dynamic]
+version = {file = "src/parkit/src/VERSION"}
+readme = {file = "README.md"}

parkit_pkg/__init__.py → src/parkit/__init__.py

@@ -1,5 +1,5 @@
 import os
-from parkit_pkg.src.utils import *
+from parkit.src.utils import *
 
 hpc = ""
 if which("scontrol") != None:

parkit_pkg/__main__.py → src/parkit/__main__.py

@@ -1,11 +1,11 @@
 #!/usr/bin/env python3
 
 import argparse
-from parkit_pkg.src.createtar import createtar, tarprep
-from parkit_pkg.src.createmetadata import createmetadata
-from parkit_pkg.src.createemptycollection import createemptycollection
-from parkit_pkg.src.deposittar import deposittocollection
-from parkit_pkg.src.VersionCheck import __version__
+from parkit.src.createtar import createtar, tarprep
+from parkit.src.createmetadata import createmetadata
+from parkit.src.createemptycollection import createemptycollection
+from parkit.src.deposittar import deposittocollection
+from parkit.src.VersionCheck import __version__
 
 
 def main():

parkit_pkg/parkit_folder2hpcdme.py → src/parkit/parkit_folder2hpcdme.py

@@ -8,7 +8,7 @@ def main():
     p = Path(__file__).absolute()
     pp = str(p.parent)
 
-    # script_path = 'parkit_pkg/scripts/parkit_folder2hpcdme'
+    # script_path = 'parkit/scripts/parkit_folder2hpcdme'
     script_path = os.path.join(pp, "scripts", "parkit_folder2hpcdme")
 
     # Pass all arguments to the bash script

parkit_pkg/parkit_tarball2hpcdme.py → src/parkit/parkit_tarball2hpcdme.py

@@ -8,7 +8,7 @@ def main():
     p = Path(__file__).absolute()
     pp = str(p.parent)
 
-    # script_path = 'parkit_pkg/scripts/parkit_tarball2hpcdme'
+    # script_path = 'parkit/scripts/parkit_tarball2hpcdme'
     script_path = os.path.join(pp, "scripts", "parkit_tarball2hpcdme")
 
     # Pass all arguments to the bash script

parkit_pkg/src/createemptycollection.py → src/parkit/src/createemptycollection.py

@@ -2,7 +2,7 @@
 import os
 from uuid import uuid4
 from datetime import datetime
-from parkit_pkg.src.utils import *
+from parkit.src.utils import *
 
 
 def createemptycollection(collectionpath, projectdesc="", projecttitle=""):

parkit_pkg/src/createmetadata.py → src/parkit/src/createmetadata.py

@@ -1,7 +1,7 @@
 from pathlib import Path
 import json
 from uuid import uuid4
-from parkit_pkg.src.utils import *
+from parkit.src.utils import *
 
 
 def createmetadata(infile, collectionpath):

parkit_pkg/src/createtar.py → src/parkit/src/createtar.py

@@ -1,4 +1,4 @@
-from parkit_pkg.src.utils import *
+from parkit.src.utils import *
 from pathlib import Path
 
 

parkit_pkg/src/deposittar.py → src/parkit/src/deposittar.py

@@ -1,5 +1,5 @@
 from pathlib import Path
-from parkit_pkg.src.utils import *
+from parkit.src.utils import *
 
 
 def deposittocollection(tar, collectionpath):