From 6593397d7f21b248b997189710d79b4338dcbe0c Mon Sep 17 00:00:00 2001
From: Thomas Bonnin <233326+TBonnin@users.noreply.github.com>
Date: Wed, 15 Jan 2025 12:43:34 -0500
Subject: [PATCH] fix(jobs): closing logic

- close the app if db healthCheck fails 5 times in a row (not after the
  first failure)
- when closing first close the webserver and then destroy the db client
---
 packages/jobs/lib/app.ts | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/packages/jobs/lib/app.ts b/packages/jobs/lib/app.ts
index 357cedccd8b..05e3448b101 100644
--- a/packages/jobs/lib/app.ts
+++ b/packages/jobs/lib/app.ts
@@ -22,13 +22,22 @@ try {
 
     // We are using a setTimeout because we don't want overlapping setInterval if the DB is down
     let healthCheck: NodeJS.Timeout | undefined;
+    let healthCheckFailures = 0;
     const check = async () => {
+        const MAX_FAILURES = 5;
+        const TIMEOUT = 1000;
         try {
-            await db.knex.raw('SELECT 1').timeout(1000);
-            healthCheck = setTimeout(check, 1000);
+            await db.knex.raw('SELECT 1').timeout(TIMEOUT);
+            healthCheckFailures = 0;
+            healthCheck = setTimeout(check, TIMEOUT);
         } catch (err) {
-            logger.error('HealthCheck failed...', err);
-            void close();
+            healthCheckFailures += 1;
+            logger.error(`HealthCheck failed (${healthCheckFailures} times)...`, err);
+            if (healthCheckFailures > MAX_FAILURES) {
+                void close();
+            } else {
+                healthCheck = setTimeout(check, TIMEOUT);
+            }
         }
     };
     void check();
@@ -39,8 +48,8 @@ try {
         processor.stop();
         otlp.stop();
         await runnersFleet.stop();
-        await db.knex.destroy();
-        srv.close(() => {
+        srv.close(async () => {
+            await db.knex.destroy();
             process.exit();
         });
     });