fix(log): handle large payload

packages/server/lib/controllers/sync.controller.ts

@@ -31,7 +31,7 @@ import {
 import type { LogContext } from '@nangohq/logs';
 import { defaultOperationExpiration, logContextGetter } from '@nangohq/logs';
 import type { LastAction } from '@nangohq/records';
-import { getHeaders, isHosted } from '@nangohq/utils';
+import { getHeaders, isHosted, truncateJson } from '@nangohq/utils';
 import { records as recordsService } from '@nangohq/records';
 import type { RequestLocals } from '../utils/express.js';
 import { getOrchestrator } from '../utils/utils.js';
@@ -303,7 +303,7 @@ class SyncController {
                     integration: { id: provider.id!, name: connection.provider_config_key, provider: provider.provider },
                     connection: { id: connection.id!, name: connection.connection_id },
                     syncConfig: { id: syncConfig.id!, name: syncConfig.sync_name },
-                    meta: { input }
+                    meta: truncateJson({ input })
                 }
             );
 

packages/shared/lib/sdk/sync.ts

@@ -9,12 +9,20 @@ import axios, { AxiosError } from 'axios';
 import { getPersistAPIUrl } from '../utils/utils.js';
 import type { IntegrationWithCreds } from '@nangohq/node';
 import type { UserProvidedProxyConfiguration } from '../models/Proxy.js';
-import { getLogger, httpRetryStrategy, metrics, retryWithBackoff } from '@nangohq/utils';
+import {
+    getLogger,
+    httpRetryStrategy,
+    metrics,
+    retryWithBackoff,
+    MAX_LOG_PAYLOAD,
+    stringifyAndTruncateValue,
+    stringifyObject,
+    truncateJsonString
+} from '@nangohq/utils';
 import type { SyncConfig } from '../models/Sync.js';
 import type { RunnerFlags } from '../services/sync/run.utils.js';
 import { validateData } from './dataValidation.js';
 import { NangoError } from '../utils/error.js';
-import { stringifyAndTruncateLog, stringifyObject } from './utils.js';
 import type { DBTeam, MessageRowInsert } from '@nangohq/types';
 
 const logger = getLogger('SDK');
@@ -720,7 +728,7 @@ export class NangoAction {
             type: 'log',
             level: oldLevelToNewLevel[level],
             source: 'user',
-            message: stringifyAndTruncateLog(message, 99_000),
+            message: stringifyAndTruncateValue(message),
             meta,
             createdAt: new Date().toISOString(),
             environmentId: this.environmentId
@@ -817,17 +825,23 @@ export class NangoAction {
         try {
             response = await retryWithBackoff(
                 async () => {
+                    let data = stringifyObject({ activityLogId: this.activityLogId, log });
+
+                    // We try to keep log object under an acceptable size, before reaching network
+                    // The idea is to always log something instead of silently crashing without overloading persist
+                    if (data.length > MAX_LOG_PAYLOAD) {
+                        log.message += ` ... (truncated, payload was too large)`;
+                        data = truncateJsonString(stringifyObject({ activityLogId: this.activityLogId, log }), MAX_LOG_PAYLOAD);
+                    }
+
                     return await this.persistApi({
                         method: 'POST',
                         url: `/environment/${this.environmentId}/log`,
                         headers: {
                             Authorization: `Bearer ${this.nango.secretKey}`,
                             'Content-Type': 'application/json'
                         },
-                        data: stringifyObject({
-                            activityLogId: this.activityLogId,
-                            log
-                        })
+                        data
                     });
                 },
                 { retry: httpRetryStrategy }

packages/shared/package.json

@@ -35,7 +35,6 @@
         "dayjs-plugin-utc": "^0.1.2",
         "dd-trace": "5.21.0",
         "exponential-backoff": "^3.1.1",
-        "fast-safe-stringify": "2.1.1",
         "form-data": "4.0.0",
         "js-yaml": "^4.1.0",
         "jsonwebtoken": "^9.0.2",

packages/shared/tsconfig.json

@@ -33,8 +33,5 @@
             "path": "../orchestrator"
         }
     ],
-    "include": [
-        "lib/**/*",
-        "../utils/lib/vitest.d.ts",
-    ]
+    "include": ["lib/**/*", "../utils/lib/vitest.d.ts"]
 }

packages/utils/lib/index.ts

@@ -4,6 +4,7 @@ export * from './environment/parse.js';
 export * from './errors.js';
 export * from './logger.js';
 export * from './id.js';
+export * from './json.js';
 export * from './result.js';
 export * from './encryption.js';
 export * as metrics from './telemetry/metrics.js';

packages/utils/lib/json.ts

@@ -0,0 +1,60 @@
+import safeStringify from 'fast-safe-stringify';
+import truncateJsonPkg from 'truncate-json';
+
+export const MAX_LOG_PAYLOAD = 99_000; // in  bytes
+
+/**
+ * Safely stringify an object (mostly handle circular ref and known problematic keys)
+ */
+export function stringifyObject(value: any): string {
+    return safeStringify.stableStringify(
+        value,
+        (key, value) => {
+            if (value instanceof Buffer || key === '_sessionCache') {
+                return '[Buffer]';
+            }
+            if (key === 'Authorization') {
+                return '[Redacted]';
+            }
+            return value;
+        },
+        undefined,
+        { depthLimit: 10, edgesLimit: 20 }
+    );
+}
+
+/**
+ * Stringify and truncate unknown value
+ */
+export function stringifyAndTruncateValue(value: any, maxSize: number = MAX_LOG_PAYLOAD): string {
+    if (value === null) {
+        return 'null';
+    }
+    if (value === undefined) {
+        return 'undefined';
+    }
+
+    let msg = typeof value === 'string' ? value : truncateJsonString(stringifyObject(value), maxSize);
+
+    if (msg && msg.length > maxSize) {
+        msg = `${msg.substring(0, maxSize)}... (truncated)`;
+    }
+
+    return msg;
+}
+
+/**
+ * Truncate a JSON
+ * Will entirely remove properties that are too big
+ */
+export function truncateJson(value: Record<string, any>, maxSize: number = MAX_LOG_PAYLOAD) {
+    return JSON.parse(truncateJsonPkg(JSON.stringify(value), maxSize).jsonString);
+}
+
+/**
+ * Truncate a JSON as string
+ * Will entirely remove properties that are too big
+ */
+export function truncateJsonString(value: string, maxSize: number = MAX_LOG_PAYLOAD): string {
+    return truncateJsonPkg(value, maxSize).jsonString;
+}

packages/shared/lib/sdk/utils.unit.test.ts → packages/utils/lib/json.unit.test.ts

@@ -1,52 +1,52 @@
 import { describe, expect, it } from 'vitest';
-import { stringifyAndTruncateLog } from './utils.js';
+import { stringifyAndTruncateValue } from './json.js';
 
-describe('stringifyAndTruncateLog', () => {
+describe('stringifyAndTruncateValue', () => {
     it('should not break a small string', () => {
-        const str = stringifyAndTruncateLog('hello');
+        const str = stringifyAndTruncateValue('hello');
         expect(str).toStrictEqual('hello');
     });
 
     it('should limit a string', () => {
-        const str = stringifyAndTruncateLog('hello', 2);
+        const str = stringifyAndTruncateValue('hello', 2);
         expect(str).toStrictEqual('he... (truncated)');
     });
 
     it('should limit an object', () => {
-        const str = stringifyAndTruncateLog({ foo: 'bar' }, 10);
-        expect(str).toStrictEqual('{"foo":"ba... (truncated)');
+        const str = stringifyAndTruncateValue({ foo: 'bar' }, 10);
+        expect(str).toStrictEqual('{}');
     });
 
     it('should handle undefined', () => {
-        const str = stringifyAndTruncateLog(undefined);
+        const str = stringifyAndTruncateValue(undefined);
         expect(str).toStrictEqual('undefined');
     });
 
     it('should handle null', () => {
-        const str = stringifyAndTruncateLog(null);
+        const str = stringifyAndTruncateValue(null);
         expect(str).toStrictEqual('null');
     });
 
     it('should handle object', () => {
-        const str = stringifyAndTruncateLog({ foo: 1 });
+        const str = stringifyAndTruncateValue({ foo: 1 });
         expect(str).toStrictEqual('{"foo":1}');
     });
 
     it('should handle array', () => {
-        const str = stringifyAndTruncateLog([{ foo: 1 }, 2]);
+        const str = stringifyAndTruncateValue([{ foo: 1 }, 2]);
         expect(str).toStrictEqual('[{"foo":1},2]');
     });
 
     it('should handle circular', () => {
         const obj: Record<string, any> = { foo: 'bar' };
         obj['circular'] = obj;
-        const str = stringifyAndTruncateLog(obj);
+        const str = stringifyAndTruncateValue(obj);
         expect(str).toStrictEqual('{"circular":"[Circular]","foo":"bar"}');
     });
 
     it('should redac known keys', () => {
         const obj: Record<string, any> = { Authorization: 'super secret key' };
-        const str = stringifyAndTruncateLog(obj);
+        const str = stringifyAndTruncateValue(obj);
         expect(str).toStrictEqual('{"Authorization":"[Redacted]"}');
     });
 });

packages/utils/package.json

@@ -19,10 +19,12 @@
         "axios": "^1.7.4",
         "dd-trace": "5.21.0",
         "exponential-backoff": "3.1.1",
+        "fast-safe-stringify": "2.1.1",
         "http-proxy-agent": "7.0.2",
         "https-proxy-agent": "7.0.4",
         "nanoid": "5.0.7",
         "serialize-error": "11.0.3",
+        "truncate-json": "3.0.0",
         "winston": "3.13.0",
         "zod": "3.23.8"
     },