dist: add workaround to capture coredump during system reboot or shut…

…down

We found that systemd-coredump does not correctly capturing coredump
during system reboot or shutdown.
As a workaround of this issue, set coredump file path to
kernel.core_pattern during system reboot or shutdown.
It will save core to /var/lib/scylla/shutdown-coredump/.

Fixes scylladb/scylla-machine-image#436

dist/common/scripts/scylla_kill

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# When shutdown is being called, stop using systemd-coredump handler
+# since it does not work correctly
+st_jobs=$(systemctl list-jobs --no-pager shutdown.target)
+if [[ ! $st_jobs =~ (^No jobs .*\.$) ]]; then
+    sysctl -w kernel.core_pattern=/var/lib/scylla/coredump-shutdown/core.%e.%P.%u.%g.%s.%t.%c.%h
+fi
+# Shutdown Scylla
+pkill scylla

dist/common/scripts/scylla_raid_setup

@@ -217,7 +217,7 @@ WantedBy=multi-user.target
     gid = grp.getgrnam('scylla').gr_gid
     os.chown(root, uid, gid)
 
-    for d in ['coredump', 'data', 'commitlog', 'hints', 'view_hints', 'saved_caches']:
+    for d in ['coredump', 'coredump-shutdown', 'data', 'commitlog', 'hints', 'view_hints', 'saved_caches']:
         dpath = '{}/{}'.format(root, d)
         os.makedirs(dpath, exist_ok=True)
         os.chown(dpath, uid, gid)

dist/common/systemd/scylla-server.service

@@ -16,6 +16,7 @@ EnvironmentFile=/etc/sysconfig/scylla-server
 EnvironmentFile=/etc/scylla.d/*.conf
 ExecStartPre=/opt/scylladb/scripts/scylla_prepare
 ExecStart=/usr/bin/scylla $SCYLLA_ARGS $SEASTAR_IO $DEV_MODE $CPUSET $MEM_CONF
+ExecStop=/opt/scylladb/scripts/scylla_kill
 ExecStopPost=/opt/scylladb/scripts/scylla_stop
 TimeoutStartSec=1y
 TimeoutStopSec=900

dist/debian/debian/scylla-server.install

@@ -25,6 +25,7 @@ var/lib/scylla/commitlog
 var/lib/scylla/hints
 var/lib/scylla/view_hints
 var/lib/scylla/coredump
+var/lib/scylla/coredump-shutdown
 var/lib/scylla-housekeeping
 etc/systemd/system/scylla-server.service.d/dependencies.conf
 etc/systemd/system/scylla-server.service.d/sysconfdir.conf

dist/redhat/scylla.spec

@@ -144,6 +144,7 @@ rm -rf $RPM_BUILD_ROOT
 %attr(0755,scylla,scylla) %dir %{_sharedstatedir}/scylla/hints
 %attr(0755,scylla,scylla) %dir %{_sharedstatedir}/scylla/view_hints
 %attr(0755,scylla,scylla) %dir %{_sharedstatedir}/scylla/coredump
+%attr(0755,scylla,scylla) %dir %{_sharedstatedir}/scylla/coredump-shutdown
 %attr(0755,scylla,scylla) %dir %{_sharedstatedir}/scylla-housekeeping
 %ghost /etc/systemd/system/scylla-helper.slice.d/
 %ghost /etc/systemd/system/scylla-helper.slice.d/memory.conf

install.sh

@@ -416,6 +416,7 @@ install -m755 -d "$rdata"/commitlog
 install -m755 -d "$rdata"/hints
 install -m755 -d "$rdata"/view_hints
 install -m755 -d "$rdata"/coredump
+install -m755 -d "$rdata"/coredump-shutdown
 install -m755 -d "$rprefix"/swagger-ui
 cp -r swagger-ui/dist "$rprefix"/swagger-ui
 install -d -m755 -d "$rprefix"/api