fix(jobs): closing logic

- close the app if db healthCheck fails 5 times in a row (not after the
  first failure)
- when closing first close the webserver and then destroy the db client

packages/jobs/lib/app.ts

@@ -22,13 +22,22 @@ try {
 
     // We are using a setTimeout because we don't want overlapping setInterval if the DB is down
     let healthCheck: NodeJS.Timeout | undefined;
+    let healthCheckFailures = 0;
     const check = async () => {
+        const MAX_FAILURES = 5;
+        const TIMEOUT = 1000;
         try {
-            await db.knex.raw('SELECT 1').timeout(1000);
-            healthCheck = setTimeout(check, 1000);
+            await db.knex.raw('SELECT 1').timeout(TIMEOUT);
+            healthCheckFailures = 0;
+            healthCheck = setTimeout(check, TIMEOUT);
         } catch (err) {
-            logger.error('HealthCheck failed...', err);
-            void close();
+            healthCheckFailures += 1;
+            logger.error(`HealthCheck failed (${healthCheckFailures} times)...`, err);
+            if (healthCheckFailures > MAX_FAILURES) {
+                void close();
+            } else {
+                healthCheck = setTimeout(check, TIMEOUT);
+            }
         }
     };
     void check();
@@ -39,8 +48,8 @@ try {
         processor.stop();
         otlp.stop();
         await runnersFleet.stop();
-        await db.knex.destroy();
-        srv.close(() => {
+        srv.close(async () => {
+            await db.knex.destroy();
             process.exit();
         });
     });