fix(runner): tweak runner idling retry logic

Currently the runner keeps notifying `jobs` that it is idle every 10s
However 10s might not be enough for jobs/fleet to terminate the runner
and following request to /idle fails with an error.
This commit increase the timeout when /idle was called successfully to
give enough time to jobs/fleet to terminate the runner
In theory we could also simply stop the loop once /idle has been successful

packages/runner/lib/monitor.ts

@@ -93,9 +93,10 @@ export class RunnerMonitor {
         }
     }
 
-    private checkIdle(): NodeJS.Timeout | null {
+    private checkIdle(timeoutMs: number = 10000): NodeJS.Timeout | null {
         // eslint-disable-next-line @typescript-eslint/no-misused-promises
-        return setInterval(async () => {
+        return setTimeout(async () => {
+            let nextTimeout = timeoutMs;
             if (this.idleMaxDurationMs > 0 && this.tracked.size == 0) {
                 const idleTimeMs = Date.now() - this.lastIdleTrackingDate;
                 if (idleTimeMs > this.idleMaxDurationMs) {
@@ -105,7 +106,11 @@ export class RunnerMonitor {
                         const res = await idle();
                         if (res.isErr()) {
                             logger.error(`Failed to idle runner`, res.error);
+                            nextTimeout = timeoutMs; // Reset to default on error
                         }
+                        // Increase the timeout to 2 minutes after a successful idle
+                        // to give enough time to fleet to terminate the runner
+                        nextTimeout = 120_000;
                     } else {
                         // TODO: DEPRECATE legacy /idle endpoint
                         await httpFetch({
@@ -120,7 +125,8 @@ export class RunnerMonitor {
                     this.lastIdleTrackingDate = Date.now();
                 }
             }
-        }, 10000);
+            this.checkIdle(nextTimeout);
+        }, timeoutMs);
     }
 }