Adding munged container start script that pauses sigterm handling

naps-product-sa · Nov 7, 2023 · e7db767 · e7db767
1 parent 3a5c769
commit e7db767
Show file tree

Hide file tree

Showing 4 changed files with 25 additions and 4 deletions.
diff --git a/slurm/images/munge/Dockerfile b/slurm/images/munge/Dockerfile
@@ -52,7 +52,8 @@ RUN useradd user1 && \
 
 RUN mkdir /scripts
 ADD fix-munge-paths.sh /scripts
+ADD munged-container.sh /scripts
 RUN chmod +x /scripts/*.sh
 
 WORKDIR /
-CMD ['/sbin/munged', '-F']
+CMD ['/bin/bash', '/scripts/munged-container.sh']
diff --git a/slurm/images/munge/munged-container.sh b/slurm/images/munge/munged-container.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+_term() { 
+  echo "Caught SIGTERM, waiting 10 seconds"
+  sleep 10
+  kill -TERM "$child" 2>/dev/null
+}
+
+trap _term SIGTERM
+
+/sbin/munged -F &
+
+child=$!
+wait "$child"
diff --git a/slurm/manifests/kustomization.yaml b/slurm/manifests/kustomization.yaml
@@ -20,5 +20,6 @@ configMapGenerator:
 - name: slurm-conf
   files:
     - configs/slurm.conf
+    - configs/munge-init
   options:
     disableNameSuffixHash: true
diff --git a/slurm/manifests/statefulset-compute.yaml b/slurm/manifests/statefulset-compute.yaml
@@ -68,6 +68,14 @@ spec:
         - -Z
         # - --conf
         # - RealMemory=4000
+        lifecycle:
+          preStop:
+            exec:
+              command:
+              - /bin/bash
+              - -c
+              - |
+                scontrol update nodename=$HOSTNAME state=down reason=shutdown
         securityContext:
           runAsUser: 0
           runAsGroup: 0
@@ -81,9 +89,6 @@ spec:
           mountPath: /home
       - name: munge
         image: ghcr.io/naps-product-sa/openshift-batch/munge:latest
-        command:
-        - /sbin/munged
-        - -F
         securityContext:
           runAsUser: 500
           runAsGroup: 500