Merge branch 'master' into francismaina/ext-362-slack-document-access…

…-requirements-and-setup-guide
NangoHQ · Jan 17, 2025 · d17e7a4 · d17e7a4
2 parents aedecdb + 2f43687
commit d17e7a4
Show file tree

Hide file tree

Showing 8 changed files with 43 additions and 49 deletions.
diff --git a/docs-v2/integrations/all/emarsys-oauth/form.png b/docs-v2/integrations/all/emarsys-oauth/form.png
diff --git a/packages/jobs/lib/routes/tasks/putTask.ts b/packages/jobs/lib/routes/tasks/putTask.ts
@@ -68,7 +68,7 @@ const nangoPropsSchema = z
                 pre_built: z.boolean(),
                 is_public: z.boolean(),
                 input: z.string().nullable(),
-                sync_type: z.enum(['full', 'incremental']).nullable(),
+                sync_type: z.enum(['full', 'incremental', 'FULL', 'INCREMENTAL']).nullable(),
                 metadata: z.record(z.string(), z.any())
             })
             .passthrough(),

diff --git a/packages/jobs/lib/runner/render.ts b/packages/jobs/lib/runner/render.ts
@@ -6,7 +6,7 @@ import { envs } from '../env.js';
 import { getPersistAPIUrl, getProvidersUrl, getRedisUrl } from '@nangohq/shared';
 import type { AxiosResponse } from 'axios';
 import { isAxiosError } from 'axios';
-import type { RateLimiterAbstract } from 'rate-limiter-flexible';
+import type { IRateLimiterRedisOptions, RateLimiterAbstract } from 'rate-limiter-flexible';
 import { RateLimiterRedis, RateLimiterMemory } from 'rate-limiter-flexible';
 import { createClient } from 'redis';
 
@@ -172,13 +172,13 @@ class CombinedThrottler {
 }
 
 const serviceCreationThrottler = await (async () => {
-    const minuteThrottlerOpts = {
+    const minuteThrottlerOpts: Omit<IRateLimiterRedisOptions, 'storeClient'> = {
         keyPrefix: 'minute',
         points: envs.RENDER_SERVICE_CREATION_MAX_PER_MINUTE || 50,
         duration: 60,
         blockDuration: 0
     };
-    const hourThrottlerOpts = {
+    const hourThrottlerOpts: Omit<IRateLimiterRedisOptions, 'storeClient'> = {
         keyPrefix: 'hour',
         points: envs.RENDER_SERVICE_CREATION_MAX_PER_HOUR || 700,
         duration: 3600,

diff --git a/packages/runner/lib/monitor.ts b/packages/runner/lib/monitor.ts
@@ -93,9 +93,10 @@ export class RunnerMonitor {
         }
     }
 
-    private checkIdle(): NodeJS.Timeout | null {
+    private checkIdle(timeoutMs: number = 10000): NodeJS.Timeout | null {
         // eslint-disable-next-line @typescript-eslint/no-misused-promises
-        return setInterval(async () => {
+        return setTimeout(async () => {
+            let nextTimeout = timeoutMs;
             if (this.idleMaxDurationMs > 0 && this.tracked.size == 0) {
                 const idleTimeMs = Date.now() - this.lastIdleTrackingDate;
                 if (idleTimeMs > this.idleMaxDurationMs) {
@@ -105,7 +106,11 @@ export class RunnerMonitor {
                         const res = await idle();
                         if (res.isErr()) {
                             logger.error(`Failed to idle runner`, res.error);
+                            nextTimeout = timeoutMs; // Reset to default on error
                         }
+                        // Increase the timeout to 2 minutes after a successful idle
+                        // to give enough time to fleet to terminate the runner
+                        nextTimeout = 120_000;
                     } else {
                         // TODO: DEPRECATE legacy /idle endpoint
                         await httpFetch({
@@ -120,7 +125,8 @@ export class RunnerMonitor {
                     this.lastIdleTrackingDate = Date.now();
                 }
             }
-        }, 10000);
+            this.checkIdle(nextTimeout);
+        }, timeoutMs);
     }
 }
 

diff --git a/packages/shared/lib/services/proxy.service.ts b/packages/shared/lib/services/proxy.service.ts
@@ -76,52 +76,39 @@ class ProxyService {
         let token;
         switch (connection.credentials.type) {
             case 'OAUTH2':
-                {
-                    const credentials = connection.credentials;
-                    token = credentials.access_token;
-                }
+            case 'APP': {
+                const credentials = connection.credentials;
+                token = credentials.access_token;
                 break;
-            case 'OAUTH1': {
-                const error = new Error('OAuth1 is not supported yet in the proxy.');
-                const nangoError = new NangoError('pass_through_error', error);
-                return { success: false, error: nangoError, response: null, logs };
             }
-            case 'BASIC':
-                token = connection.credentials;
-                break;
-            case 'API_KEY':
-                token = connection.credentials;
-                break;
-            case 'APP':
-                {
-                    const credentials = connection.credentials;
-                    token = credentials.access_token;
-                }
-                break;
             case 'OAUTH2_CC':
-                {
-                    const credentials = connection.credentials;
-                    token = credentials.token;
-                }
-                break;
+            case 'TWO_STEP':
             case 'TABLEAU':
-                {
-                    const credentials = connection.credentials;
-                    token = credentials.token;
-                }
-                break;
             case 'JWT':
-                {
-                    const credentials = connection.credentials;
-                    token = credentials.token;
-                }
+            case 'SIGNATURE': {
+                const credentials = connection.credentials;
+                token = credentials.token;
                 break;
-            case 'SIGNATURE':
-                {
-                    const credentials = connection.credentials;
-                    token = credentials.token;
-                }
+            }
+            case 'BASIC':
+            case 'API_KEY':
+                token = connection.credentials;
                 break;
+            case 'OAUTH1': {
+                const error = new Error('OAuth1 is not supported yet in the proxy.');
+                const nangoError = new NangoError('pass_through_error', error);
+                return { success: false, error: nangoError, response: null, logs };
+            }
+            case 'APP_STORE':
+            case 'CUSTOM':
+            case 'TBA':
+            case undefined:
+            case 'BILL': {
+                break;
+            }
+            default: {
+                throw new Error(`Unhandled connection.credentials for: ${(connection.credentials as any).type}`);
+            }
         }
 
         const provider = getProvider(providerName);
@@ -470,6 +457,7 @@ class ProxyService {
                         case 'API_KEY':
                         case 'OAUTH2_CC':
                         case 'TABLEAU':
+                        case 'TWO_STEP':
                         case 'JWT':
                             if (value.includes('connectionConfig')) {
                                 value = value.replace(/connectionConfig\./g, '');

diff --git a/packages/shared/providers.yaml b/packages/shared/providers.yaml
@@ -2325,7 +2325,7 @@ emarsys:
             pattern: '^[a-zA-Z0-9-]+$'
 
 emarsys-oauth:
-    display_name: Emarsys (Ouath)
+    display_name: Emarsys (OAuth)
     categories:
         - marketing
     auth_mode: OAUTH2_CC

diff --git a/packages/types/lib/nangoYaml/index.ts b/packages/types/lib/nangoYaml/index.ts
@@ -1,7 +1,7 @@
 import type { OnEventType } from '../scripts/on-events/api';
 
 export type HTTP_METHOD = 'GET' | 'POST' | 'PATCH' | 'PUT' | 'DELETE';
-export type SyncTypeLiteral = 'incremental' | 'full';
+export type SyncTypeLiteral = 'incremental' | 'full' | 'FULL' | 'INCREMENTAL'; // TODO: There has been some mixed type in the DB, fix casing
 export type ScriptFileType = 'actions' | 'syncs' | 'on-events' | 'post-connection-scripts'; // post-connection-scripts is deprecated
 export type ScriptTypeLiteral = 'action' | 'sync' | 'on-event';
 

diff --git a/packages/webapp/src/pages/Connection/components/SyncRow.tsx b/packages/webapp/src/pages/Connection/components/SyncRow.tsx
@@ -30,7 +30,7 @@ export const SyncRow: React.FC<{ sync: SyncResponse; connection: ApiConnectionFu
 
     const [showPauseStartLoader, setShowPauseStartLoader] = useState(false);
     const [showInterruptLoader, setShowInterruptLoader] = useState(false);
-    const [triggerMode, setTriggerMode] = useState<'incremental' | 'full'>(sync.sync_type === 'full' ? 'full' : 'incremental');
+    const [triggerMode, setTriggerMode] = useState<'incremental' | 'full'>(sync.sync_type.toLocaleLowerCase() === 'full' ? 'full' : 'incremental');
     const [deleteRecords, setDeleteRecords] = useState(false);
     const [modalSpinner, setModalShowSpinner] = useState(false);
     const [openConfirm, setOpenConfirm] = useState(false);