add more steps to the ingest update jenkins script

monarch-initiative · Jun 13, 2024 · 324a582 · 324a582
1 parent 2608764
commit 324a582
Showing 1 changed file with 49 additions and 0 deletions.
diff --git a/Jenkinsfile-redo-solr b/Jenkinsfile-redo-solr
@@ -12,6 +12,16 @@ pipeline {
                 script {
                     properties([
                         parameters([
+                            stringParam(
+                                defaultValue: 'latest',
+                                description: 'monarch-kg-dev release',
+                                name: 'RELEASE'
+                            ),
+                            booleanParam(
+                                defaultValue: true,
+                                description: 'Run KGX Graph Summary',
+                                name: 'KGX_GRAPH_SUMMARY'
+                            ),
                             booleanParam(
                                 defaultValue: true,
                                 description: 'Re-run denormalization step',
@@ -32,6 +42,16 @@ pipeline {
                                 description: 'Make TSV Exports',
                                 name: 'EXPORT'
                             ),
+                            booleanParam(
+                                defaultValue: false,
+                                description: 'Make JSONL Export',
+                                name: 'JSONL'
+                            ),
+                            booleanParam(
+                                defaultValue: false,
+                                description: 'Run KGX Transforms',
+                                name: 'KGX_TRANSFORMS'
+                            ),
                             booleanParam(
                                 defaultValue: false,
                                 description: 'Upload to bucket',
@@ -73,6 +93,16 @@ pipeline {
                 '''
             }
         }
+        stage('kgx-graph-summary') {
+            when {
+                expression {
+                   return params.KGX_GRAPH_SUMMARY
+                }
+            }
+            steps {
+                sh 'poetry run kgx graph-summary -i tsv -c "tar.gz" --node-facet-properties provided_by --edge-facet-properties provided_by output/monarch-kg.tar.gz -o output/merged_graph_stats.yaml'
+            }
+        }
         stage('denormalize') {
             when {
                 expression {
@@ -113,6 +143,25 @@ pipeline {
                 sh 'poetry run ingest export'
             }
         }
+        stage("jsonl") {
+            when {
+                expression {
+                   return params.JSONL
+               }
+               steps {
+                   sh 'poetry run ingest jsonl'
+               }
+        }
+        stage('kgx-transforms'){
+            when {
+               expression {
+                   return params.KGX_TRANSFORMS
+                   }
+            }
+            steps {
+                sh './scripts/kgx_transforms.sh'
+            }
+        }
         stage('upload files') {
             when {
                 expression {