From 759e3ededaa4114f1c0c4153908943b4facf70f4 Mon Sep 17 00:00:00 2001 From: Thomas Bonnin <233326+TBonnin@users.noreply.github.com> Date: Thu, 28 Mar 2024 15:27:28 -0400 Subject: [PATCH] add records package (records migration) --- Dockerfile | 1 + package-lock.json | 1093 ++++++++++++++++- package.json | 1 + packages/records/lib/constants.ts | 1 + packages/records/lib/db/client.ts | 4 + packages/records/lib/db/config.ts | 30 + packages/records/lib/db/migrate.ts | 21 + .../20240327151027_create_records_table.ts | 95 ++ packages/records/lib/env.ts | 16 + packages/records/lib/helpers/format.ts | 59 + packages/records/lib/helpers/uniqueKey.ts | 43 + .../lib/helpers/uniqueKey.unit.test.ts | 31 + packages/records/lib/index.ts | 2 + .../lib/models/records.integration.test.ts | 175 +++ packages/records/lib/models/records.ts | 424 +++++++ packages/records/lib/types.ts | 60 + packages/records/lib/utils/encryption.ts | 42 + packages/records/lib/utils/logger.ts | 3 + packages/records/lib/vitest.d.ts | 14 + packages/records/package.json | 31 + packages/records/tsconfig.json | 16 + packages/server/package.json | 1 + packages/server/tsconfig.json | 3 + .../shared/lib/utils/encryption.manager.ts | 8 +- tests/setup.ts | 6 + tsconfig.build.json | 3 + 26 files changed, 2177 insertions(+), 6 deletions(-) create mode 100644 packages/records/lib/constants.ts create mode 100644 packages/records/lib/db/client.ts create mode 100644 packages/records/lib/db/config.ts create mode 100644 packages/records/lib/db/migrate.ts create mode 100644 packages/records/lib/db/migrations/20240327151027_create_records_table.ts create mode 100644 packages/records/lib/env.ts create mode 100644 packages/records/lib/helpers/format.ts create mode 100644 packages/records/lib/helpers/uniqueKey.ts create mode 100644 packages/records/lib/helpers/uniqueKey.unit.test.ts create mode 100644 packages/records/lib/index.ts create mode 100644 packages/records/lib/models/records.integration.test.ts create mode 100644 packages/records/lib/models/records.ts create mode 100644 packages/records/lib/types.ts create mode 100644 packages/records/lib/utils/encryption.ts create mode 100644 packages/records/lib/utils/logger.ts create mode 100644 packages/records/lib/vitest.d.ts create mode 100644 packages/records/package.json create mode 100644 packages/records/tsconfig.json diff --git a/Dockerfile b/Dockerfile index 39752d17bbc..cbe40e1d3dc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,6 +18,7 @@ COPY packages/shared/package.json ./packages/shared/package.json COPY packages/webapp/package.json ./packages/webapp/package.json COPY packages/data-ingestion/package.json ./packages/data-ingestion/package.json COPY packages/logs/package.json ./packages/logs/package.json +COPY packages/records/package.json ./packages/records/package.json COPY package*.json ./ # Install every dependencies diff --git a/package-lock.json b/package-lock.json index 8933221ff6a..1dea455bba0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "packages/frontend", "packages/logs", "packages/node-client", + "packages/records", "packages/server", "packages/runner", "packages/persist", @@ -6742,6 +6743,10 @@ "resolved": "packages/persist", "link": true }, + "node_modules/@nangohq/nango-records": { + "resolved": "packages/records", + "link": true + }, "node_modules/@nangohq/nango-runner": { "resolved": "packages/runner", "link": true @@ -6754,6 +6759,10 @@ "resolved": "packages/node-client", "link": true }, + "node_modules/@nangohq/records": { + "resolved": "packages/records", + "link": true + }, "node_modules/@nangohq/server": { "resolved": "packages/server", "link": true @@ -13024,8 +13033,9 @@ } }, "node_modules/dayjs": { - "version": "1.11.7", - "license": "MIT" + "version": "1.11.10", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.10.tgz", + "integrity": "sha512-vjAczensTgRcqDERK0SR2XMwsF/tSvnvlv6VcF2GIhg6Sx4yOIt/irsr1RDJsKiIyBzJDpCoXiWWq28MqH2cnQ==" }, "node_modules/dayjs-plugin-utc": { "version": "0.1.2", @@ -32490,6 +32500,1084 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "packages/records": { + "name": "@nangohq/nango-records", + "version": "1.0.0", + "dependencies": { + "@nangohq/shared": "file:../shared", + "@nangohq/utils": "file:../utils", + "dayjs": "1.11.10", + "knex": "3.1.0", + "md5": "2.3.0", + "pg": "8.11.3", + "uuid": "9.0.1", + "zod": "3.22.4" + }, + "devDependencies": { + "vitest": "1.4.0" + } + }, + "packages/records/node_modules/@esbuild/aix-ppc64": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.20.2.tgz", + "integrity": "sha512-D+EBOJHXdNZcLJRBkhENNG8Wji2kgc9AZ9KiPr1JuZjsNtyHzrsfLRrY0tk2H2aoFu6RANO1y1iPPUCDYWkb5g==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=12" + } + }, + "packages/records/node_modules/@esbuild/android-arm": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.20.2.tgz", + "integrity": "sha512-t98Ra6pw2VaDhqNWO2Oph2LXbz/EJcnLmKLGBJwEwXX/JAN83Fym1rU8l0JUWK6HkIbWONCSSatf4sf2NBRx/w==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "packages/records/node_modules/@esbuild/android-arm64": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.20.2.tgz", + "integrity": "sha512-mRzjLacRtl/tWU0SvD8lUEwb61yP9cqQo6noDZP/O8VkwafSYwZ4yWy24kan8jE/IMERpYncRt2dw438LP3Xmg==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "packages/records/node_modules/@esbuild/android-x64": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.20.2.tgz", + "integrity": "sha512-btzExgV+/lMGDDa194CcUQm53ncxzeBrWJcncOBxuC6ndBkKxnHdFJn86mCIgTELsooUmwUm9FkhSp5HYu00Rg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "packages/records/node_modules/@esbuild/darwin-arm64": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.20.2.tgz", + "integrity": "sha512-4J6IRT+10J3aJH3l1yzEg9y3wkTDgDk7TSDFX+wKFiWjqWp/iCfLIYzGyasx9l0SAFPT1HwSCR+0w/h1ES/MjA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "packages/records/node_modules/@esbuild/darwin-x64": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.20.2.tgz", + "integrity": "sha512-tBcXp9KNphnNH0dfhv8KYkZhjc+H3XBkF5DKtswJblV7KlT9EI2+jeA8DgBjp908WEuYll6pF+UStUCfEpdysA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "packages/records/node_modules/@esbuild/freebsd-arm64": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.20.2.tgz", + "integrity": "sha512-d3qI41G4SuLiCGCFGUrKsSeTXyWG6yem1KcGZVS+3FYlYhtNoNgYrWcvkOoaqMhwXSMrZRl69ArHsGJ9mYdbbw==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "packages/records/node_modules/@esbuild/freebsd-x64": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.20.2.tgz", + "integrity": "sha512-d+DipyvHRuqEeM5zDivKV1KuXn9WeRX6vqSqIDgwIfPQtwMP4jaDsQsDncjTDDsExT4lR/91OLjRo8bmC1e+Cw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "packages/records/node_modules/@esbuild/linux-arm": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.20.2.tgz", + "integrity": "sha512-VhLPeR8HTMPccbuWWcEUD1Az68TqaTYyj6nfE4QByZIQEQVWBB8vup8PpR7y1QHL3CpcF6xd5WVBU/+SBEvGTg==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "packages/records/node_modules/@esbuild/linux-arm64": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.20.2.tgz", + "integrity": "sha512-9pb6rBjGvTFNira2FLIWqDk/uaf42sSyLE8j1rnUpuzsODBq7FvpwHYZxQ/It/8b+QOS1RYfqgGFNLRI+qlq2A==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "packages/records/node_modules/@esbuild/linux-ia32": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.20.2.tgz", + "integrity": "sha512-o10utieEkNPFDZFQm9CoP7Tvb33UutoJqg3qKf1PWVeeJhJw0Q347PxMvBgVVFgouYLGIhFYG0UGdBumROyiig==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "packages/records/node_modules/@esbuild/linux-loong64": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.20.2.tgz", + "integrity": "sha512-PR7sp6R/UC4CFVomVINKJ80pMFlfDfMQMYynX7t1tNTeivQ6XdX5r2XovMmha/VjR1YN/HgHWsVcTRIMkymrgQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "packages/records/node_modules/@esbuild/linux-mips64el": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.20.2.tgz", + "integrity": "sha512-4BlTqeutE/KnOiTG5Y6Sb/Hw6hsBOZapOVF6njAESHInhlQAghVVZL1ZpIctBOoTFbQyGW+LsVYZ8lSSB3wkjA==", + "cpu": [ + "mips64el" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "packages/records/node_modules/@esbuild/linux-ppc64": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.20.2.tgz", + "integrity": "sha512-rD3KsaDprDcfajSKdn25ooz5J5/fWBylaaXkuotBDGnMnDP1Uv5DLAN/45qfnf3JDYyJv/ytGHQaziHUdyzaAg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "packages/records/node_modules/@esbuild/linux-riscv64": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.20.2.tgz", + "integrity": "sha512-snwmBKacKmwTMmhLlz/3aH1Q9T8v45bKYGE3j26TsaOVtjIag4wLfWSiZykXzXuE1kbCE+zJRmwp+ZbIHinnVg==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "packages/records/node_modules/@esbuild/linux-s390x": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.20.2.tgz", + "integrity": "sha512-wcWISOobRWNm3cezm5HOZcYz1sKoHLd8VL1dl309DiixxVFoFe/o8HnwuIwn6sXre88Nwj+VwZUvJf4AFxkyrQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "packages/records/node_modules/@esbuild/linux-x64": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.20.2.tgz", + "integrity": "sha512-1MdwI6OOTsfQfek8sLwgyjOXAu+wKhLEoaOLTjbijk6E2WONYpH9ZU2mNtR+lZ2B4uwr+usqGuVfFT9tMtGvGw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "packages/records/node_modules/@esbuild/netbsd-x64": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.20.2.tgz", + "integrity": "sha512-K8/DhBxcVQkzYc43yJXDSyjlFeHQJBiowJ0uVL6Tor3jGQfSGHNNJcWxNbOI8v5k82prYqzPuwkzHt3J1T1iZQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=12" + } + }, + "packages/records/node_modules/@esbuild/openbsd-x64": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.20.2.tgz", + "integrity": "sha512-eMpKlV0SThJmmJgiVyN9jTPJ2VBPquf6Kt/nAoo6DgHAoN57K15ZghiHaMvqjCye/uU4X5u3YSMgVBI1h3vKrQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=12" + } + }, + "packages/records/node_modules/@esbuild/sunos-x64": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.20.2.tgz", + "integrity": "sha512-2UyFtRC6cXLyejf/YEld4Hajo7UHILetzE1vsRcGL3earZEW77JxrFjH4Ez2qaTiEfMgAXxfAZCm1fvM/G/o8w==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=12" + } + }, + "packages/records/node_modules/@esbuild/win32-arm64": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.20.2.tgz", + "integrity": "sha512-GRibxoawM9ZCnDxnP3usoUDO9vUkpAxIIZ6GQI+IlVmr5kP3zUq+l17xELTHMWTWzjxa2guPNyrpq1GWmPvcGQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "packages/records/node_modules/@esbuild/win32-ia32": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.20.2.tgz", + "integrity": "sha512-HfLOfn9YWmkSKRQqovpnITazdtquEW8/SoHW7pWpuEeguaZI4QnCRW6b+oZTztdBnZOS2hqJ6im/D5cPzBTTlQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "packages/records/node_modules/@esbuild/win32-x64": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.20.2.tgz", + "integrity": "sha512-N49X4lJX27+l9jbLKSqZ6bKNjzQvHaT8IIFUy+YIqmXQdjYCToGWwOItDrfby14c78aDd5NHQl29xingXfCdLQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "packages/records/node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.13.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.13.2.tgz", + "integrity": "sha512-3XFIDKWMFZrMnao1mJhnOT1h2g0169Os848NhhmGweEcfJ4rCi+3yMCOLG4zA61rbJdkcrM/DjVZm9Hg5p5w7g==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ] + }, + "packages/records/node_modules/@rollup/rollup-android-arm64": { + "version": "4.13.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.13.2.tgz", + "integrity": "sha512-GdxxXbAuM7Y/YQM9/TwwP+L0omeE/lJAR1J+olu36c3LqqZEBdsIWeQ91KBe6nxwOnb06Xh7JS2U5ooWU5/LgQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ] + }, + "packages/records/node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.13.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.13.2.tgz", + "integrity": "sha512-mCMlpzlBgOTdaFs83I4XRr8wNPveJiJX1RLfv4hggyIVhfB5mJfN4P8Z6yKh+oE4Luz+qq1P3kVdWrCKcMYrrA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ] + }, + "packages/records/node_modules/@rollup/rollup-darwin-x64": { + "version": "4.13.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.13.2.tgz", + "integrity": "sha512-yUoEvnH0FBef/NbB1u6d3HNGyruAKnN74LrPAfDQL3O32e3k3OSfLrPgSJmgb3PJrBZWfPyt6m4ZhAFa2nZp2A==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ] + }, + "packages/records/node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.13.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.13.2.tgz", + "integrity": "sha512-GYbLs5ErswU/Xs7aGXqzc3RrdEjKdmoCrgzhJWyFL0r5fL3qd1NPcDKDowDnmcoSiGJeU68/Vy+OMUluRxPiLQ==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "packages/records/node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.13.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.13.2.tgz", + "integrity": "sha512-L1+D8/wqGnKQIlh4Zre9i4R4b4noxzH5DDciyahX4oOz62CphY7WDWqJoQ66zNR4oScLNOqQJfNSIAe/6TPUmQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "packages/records/node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.13.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.13.2.tgz", + "integrity": "sha512-tK5eoKFkXdz6vjfkSTCupUzCo40xueTOiOO6PeEIadlNBkadH1wNOH8ILCPIl8by/Gmb5AGAeQOFeLev7iZDOA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "packages/records/node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.13.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.13.2.tgz", + "integrity": "sha512-C3GSKvMtdudHCN5HdmAMSRYR2kkhgdOfye4w0xzyii7lebVr4riCgmM6lRiSCnJn2w1Xz7ZZzHKuLrjx5620kw==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "packages/records/node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.13.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.13.2.tgz", + "integrity": "sha512-xXMLUAMzrtsvh3cZ448vbXqlUa7ZL8z0MwHp63K2IIID2+DeP5iWIT6g1SN7hg1VxPzqx0xZdiDM9l4n9LRU1A==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "packages/records/node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.13.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.13.2.tgz", + "integrity": "sha512-M/JYAWickafUijWPai4ehrjzVPKRCyDb1SLuO+ZyPfoXgeCEAlgPkNXewFZx0zcnoIe3ay4UjXIMdXQXOZXWqA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "packages/records/node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.13.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.13.2.tgz", + "integrity": "sha512-2YWwoVg9KRkIKaXSh0mz3NmfurpmYoBBTAXA9qt7VXk0Xy12PoOP40EFuau+ajgALbbhi4uTj3tSG3tVseCjuA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ] + }, + "packages/records/node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.13.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.13.2.tgz", + "integrity": "sha512-2FSsE9aQ6OWD20E498NYKEQLneShWes0NGMPQwxWOdws35qQXH+FplabOSP5zEe1pVjurSDOGEVCE2agFwSEsw==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ] + }, + "packages/records/node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.13.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.13.2.tgz", + "integrity": "sha512-7h7J2nokcdPePdKykd8wtc8QqqkqxIrUz7MHj6aNr8waBRU//NLDVnNjQnqQO6fqtjrtCdftpbTuOKAyrAQETQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ] + }, + "packages/records/node_modules/@vitest/expect": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-1.4.0.tgz", + "integrity": "sha512-Jths0sWCJZ8BxjKe+p+eKsoqev1/T8lYcrjavEaz8auEJ4jAVY0GwW3JKmdVU4mmNPLPHixh4GNXP7GFtAiDHA==", + "dev": true, + "dependencies": { + "@vitest/spy": "1.4.0", + "@vitest/utils": "1.4.0", + "chai": "^4.3.10" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "packages/records/node_modules/@vitest/runner": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-1.4.0.tgz", + "integrity": "sha512-EDYVSmesqlQ4RD2VvWo3hQgTJ7ZrFQ2VSJdfiJiArkCerDAGeyF1i6dHkmySqk573jLp6d/cfqCN+7wUB5tLgg==", + "dev": true, + "dependencies": { + "@vitest/utils": "1.4.0", + "p-limit": "^5.0.0", + "pathe": "^1.1.1" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "packages/records/node_modules/@vitest/snapshot": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-1.4.0.tgz", + "integrity": "sha512-saAFnt5pPIA5qDGxOHxJ/XxhMFKkUSBJmVt5VgDsAqPTX6JP326r5C/c9UuCMPoXNzuudTPsYDZCoJ5ilpqG2A==", + "dev": true, + "dependencies": { + "magic-string": "^0.30.5", + "pathe": "^1.1.1", + "pretty-format": "^29.7.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "packages/records/node_modules/@vitest/spy": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-1.4.0.tgz", + "integrity": "sha512-Ywau/Qs1DzM/8Uc+yA77CwSegizMlcgTJuYGAi0jujOteJOUf1ujunHThYo243KG9nAyWT3L9ifPYZ5+As/+6Q==", + "dev": true, + "dependencies": { + "tinyspy": "^2.2.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "packages/records/node_modules/@vitest/utils": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-1.4.0.tgz", + "integrity": "sha512-mx3Yd1/6e2Vt/PUC98DcqTirtfxUyAZ32uK82r8rZzbtBeBo+nqgnjx/LvqQdWsrvNtm14VmurNgcf4nqY5gJg==", + "dev": true, + "dependencies": { + "diff-sequences": "^29.6.3", + "estree-walker": "^3.0.3", + "loupe": "^2.3.7", + "pretty-format": "^29.7.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "packages/records/node_modules/esbuild": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.20.2.tgz", + "integrity": "sha512-WdOOppmUNU+IbZ0PaDiTst80zjnrOkyJNHoKupIcVyU8Lvla3Ugx94VzkQ32Ijqd7UhHJy75gNWDMUekcrSJ6g==", + "dev": true, + "hasInstallScript": true, + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=12" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.20.2", + "@esbuild/android-arm": "0.20.2", + "@esbuild/android-arm64": "0.20.2", + "@esbuild/android-x64": "0.20.2", + "@esbuild/darwin-arm64": "0.20.2", + "@esbuild/darwin-x64": "0.20.2", + "@esbuild/freebsd-arm64": "0.20.2", + "@esbuild/freebsd-x64": "0.20.2", + "@esbuild/linux-arm": "0.20.2", + "@esbuild/linux-arm64": "0.20.2", + "@esbuild/linux-ia32": "0.20.2", + "@esbuild/linux-loong64": "0.20.2", + "@esbuild/linux-mips64el": "0.20.2", + "@esbuild/linux-ppc64": "0.20.2", + "@esbuild/linux-riscv64": "0.20.2", + "@esbuild/linux-s390x": "0.20.2", + "@esbuild/linux-x64": "0.20.2", + "@esbuild/netbsd-x64": "0.20.2", + "@esbuild/openbsd-x64": "0.20.2", + "@esbuild/sunos-x64": "0.20.2", + "@esbuild/win32-arm64": "0.20.2", + "@esbuild/win32-ia32": "0.20.2", + "@esbuild/win32-x64": "0.20.2" + } + }, + "packages/records/node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "dependencies": { + "@types/estree": "^1.0.0" + } + }, + "packages/records/node_modules/execa": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/execa/-/execa-8.0.1.tgz", + "integrity": "sha512-VyhnebXciFV2DESc+p6B+y0LjSm0krU4OgJN44qFAhBY0TJ+1V61tYD2+wHusZ6F9n5K+vl8k0sTy7PEfV4qpg==", + "dev": true, + "dependencies": { + "cross-spawn": "^7.0.3", + "get-stream": "^8.0.1", + "human-signals": "^5.0.0", + "is-stream": "^3.0.0", + "merge-stream": "^2.0.0", + "npm-run-path": "^5.1.0", + "onetime": "^6.0.0", + "signal-exit": "^4.1.0", + "strip-final-newline": "^3.0.0" + }, + "engines": { + "node": ">=16.17" + }, + "funding": { + "url": "https://github.com/sindresorhus/execa?sponsor=1" + } + }, + "packages/records/node_modules/get-stream": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-8.0.1.tgz", + "integrity": "sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA==", + "dev": true, + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "packages/records/node_modules/human-signals": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-5.0.0.tgz", + "integrity": "sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ==", + "dev": true, + "engines": { + "node": ">=16.17.0" + } + }, + "packages/records/node_modules/is-stream": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-3.0.0.tgz", + "integrity": "sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA==", + "dev": true, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "packages/records/node_modules/js-tokens": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-9.0.0.tgz", + "integrity": "sha512-WriZw1luRMlmV3LGJaR6QOJjWwgLUTf89OwT2lUOyjX2dJGBwgmIkbcz+7WFZjrZM635JOIR517++e/67CP9dQ==", + "dev": true + }, + "packages/records/node_modules/local-pkg": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/local-pkg/-/local-pkg-0.5.0.tgz", + "integrity": "sha512-ok6z3qlYyCDS4ZEU27HaU6x/xZa9Whf8jD4ptH5UZTQYZVYeb9bnZ3ojVhiJNLiXK1Hfc0GNbLXcmZ5plLDDBg==", + "dev": true, + "dependencies": { + "mlly": "^1.4.2", + "pkg-types": "^1.0.3" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "packages/records/node_modules/mimic-fn": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-4.0.0.tgz", + "integrity": "sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "packages/records/node_modules/npm-run-path": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-5.3.0.tgz", + "integrity": "sha512-ppwTtiJZq0O/ai0z7yfudtBpWIoxM8yE6nHi1X47eFR2EWORqfbu6CnPlNsjeN683eT0qG6H/Pyf9fCcvjnnnQ==", + "dev": true, + "dependencies": { + "path-key": "^4.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "packages/records/node_modules/onetime": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-6.0.0.tgz", + "integrity": "sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ==", + "dev": true, + "dependencies": { + "mimic-fn": "^4.0.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "packages/records/node_modules/p-limit": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-5.0.0.tgz", + "integrity": "sha512-/Eaoq+QyLSiXQ4lyYV23f14mZRQcXnxfHrN0vCai+ak9G0pp9iEQukIIZq5NccEvwRB8PUnZT0KsOoDCINS1qQ==", + "dev": true, + "dependencies": { + "yocto-queue": "^1.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "packages/records/node_modules/path-key": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-4.0.0.tgz", + "integrity": "sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "packages/records/node_modules/postcss": { + "version": "8.4.38", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.38.tgz", + "integrity": "sha512-Wglpdk03BSfXkHoQa3b/oulrotAkwrlLDRSOb9D0bN86FdRyE9lppSp33aHNPgBa0JKCoB+drFLZkQoRRYae5A==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "dependencies": { + "nanoid": "^3.3.7", + "picocolors": "^1.0.0", + "source-map-js": "^1.2.0" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "packages/records/node_modules/rollup": { + "version": "4.13.2", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.13.2.tgz", + "integrity": "sha512-MIlLgsdMprDBXC+4hsPgzWUasLO9CE4zOkj/u6j+Z6j5A4zRY+CtiXAdJyPtgCsc42g658Aeh1DlrdVEJhsL2g==", + "dev": true, + "dependencies": { + "@types/estree": "1.0.5" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@rollup/rollup-android-arm-eabi": "4.13.2", + "@rollup/rollup-android-arm64": "4.13.2", + "@rollup/rollup-darwin-arm64": "4.13.2", + "@rollup/rollup-darwin-x64": "4.13.2", + "@rollup/rollup-linux-arm-gnueabihf": "4.13.2", + "@rollup/rollup-linux-arm64-gnu": "4.13.2", + "@rollup/rollup-linux-arm64-musl": "4.13.2", + "@rollup/rollup-linux-powerpc64le-gnu": "4.13.2", + "@rollup/rollup-linux-riscv64-gnu": "4.13.2", + "@rollup/rollup-linux-s390x-gnu": "4.13.2", + "@rollup/rollup-linux-x64-gnu": "4.13.2", + "@rollup/rollup-linux-x64-musl": "4.13.2", + "@rollup/rollup-win32-arm64-msvc": "4.13.2", + "@rollup/rollup-win32-ia32-msvc": "4.13.2", + "@rollup/rollup-win32-x64-msvc": "4.13.2", + "fsevents": "~2.3.2" + } + }, + "packages/records/node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "dev": true, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "packages/records/node_modules/strip-final-newline": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-3.0.0.tgz", + "integrity": "sha512-dOESqjYr96iWYylGObzd39EuNTa5VJxyvVAEm5Jnh7KGo75V43Hk1odPQkNDyXNmUR6k+gEiDVXnjB8HJ3crXw==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "packages/records/node_modules/strip-literal": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/strip-literal/-/strip-literal-2.1.0.tgz", + "integrity": "sha512-Op+UycaUt/8FbN/Z2TWPBLge3jWrP3xj10f3fnYxf052bKuS3EKs1ZQcVGjnEMdsNVAM+plXRdmjrZ/KgG3Skw==", + "dev": true, + "dependencies": { + "js-tokens": "^9.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "packages/records/node_modules/tinypool": { + "version": "0.8.3", + "resolved": "https://registry.npmjs.org/tinypool/-/tinypool-0.8.3.tgz", + "integrity": "sha512-Ud7uepAklqRH1bvwy22ynrliC7Dljz7Tm8M/0RBUW+YRa4YHhZ6e4PpgE+fu1zr/WqB1kbeuVrdfeuyIBpy4tw==", + "dev": true, + "engines": { + "node": ">=14.0.0" + } + }, + "packages/records/node_modules/vite": { + "version": "5.2.7", + "resolved": "https://registry.npmjs.org/vite/-/vite-5.2.7.tgz", + "integrity": "sha512-k14PWOKLI6pMaSzAuGtT+Cf0YmIx12z9YGon39onaJNy8DLBfBJrzg9FQEmkAM5lpHBZs9wksWAsyF/HkpEwJA==", + "dev": true, + "dependencies": { + "esbuild": "^0.20.1", + "postcss": "^8.4.38", + "rollup": "^4.13.0" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^18.0.0 || >=20.0.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^18.0.0 || >=20.0.0", + "less": "*", + "lightningcss": "^1.21.0", + "sass": "*", + "stylus": "*", + "sugarss": "*", + "terser": "^5.4.0" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + } + } + }, + "packages/records/node_modules/vite-node": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-1.4.0.tgz", + "integrity": "sha512-VZDAseqjrHgNd4Kh8icYHWzTKSCZMhia7GyHfhtzLW33fZlG9SwsB6CEhgyVOWkJfJ2pFLrp/Gj1FSfAiqH9Lw==", + "dev": true, + "dependencies": { + "cac": "^6.7.14", + "debug": "^4.3.4", + "pathe": "^1.1.1", + "picocolors": "^1.0.0", + "vite": "^5.0.0" + }, + "bin": { + "vite-node": "vite-node.mjs" + }, + "engines": { + "node": "^18.0.0 || >=20.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "packages/records/node_modules/vitest": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-1.4.0.tgz", + "integrity": "sha512-gujzn0g7fmwf83/WzrDTnncZt2UiXP41mHuFYFrdwaLRVQ6JYQEiME2IfEjU3vcFL3VKa75XhI3lFgn+hfVsQw==", + "dev": true, + "dependencies": { + "@vitest/expect": "1.4.0", + "@vitest/runner": "1.4.0", + "@vitest/snapshot": "1.4.0", + "@vitest/spy": "1.4.0", + "@vitest/utils": "1.4.0", + "acorn-walk": "^8.3.2", + "chai": "^4.3.10", + "debug": "^4.3.4", + "execa": "^8.0.1", + "local-pkg": "^0.5.0", + "magic-string": "^0.30.5", + "pathe": "^1.1.1", + "picocolors": "^1.0.0", + "std-env": "^3.5.0", + "strip-literal": "^2.0.0", + "tinybench": "^2.5.1", + "tinypool": "^0.8.2", + "vite": "^5.0.0", + "vite-node": "1.4.0", + "why-is-node-running": "^2.2.2" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^18.0.0 || >=20.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@types/node": "^18.0.0 || >=20.0.0", + "@vitest/browser": "1.4.0", + "@vitest/ui": "1.4.0", + "happy-dom": "*", + "jsdom": "*" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + } + } + }, + "packages/records/node_modules/yocto-queue": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.0.0.tgz", + "integrity": "sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==", + "dev": true, + "engines": { + "node": ">=12.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "packages/runner": { "name": "@nangohq/nango-runner", "version": "1.0.0", @@ -33420,6 +34508,7 @@ "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { "@hapi/boom": "^10.0.1", + "@nangohq/records": "file:../records", "@nangohq/shared": "^0.39.15", "@nangohq/utils": "file:../utils", "@workos-inc/node": "^6.2.0", diff --git a/package.json b/package.json index d17a006d688..e91ab3f456f 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "packages/frontend", "packages/logs", "packages/node-client", + "packages/records", "packages/server", "packages/runner", "packages/persist", diff --git a/packages/records/lib/constants.ts b/packages/records/lib/constants.ts new file mode 100644 index 00000000000..bc12fe51607 --- /dev/null +++ b/packages/records/lib/constants.ts @@ -0,0 +1 @@ +export const RECORDS_TABLE = 'records'; diff --git a/packages/records/lib/db/client.ts b/packages/records/lib/db/client.ts new file mode 100644 index 00000000000..1493c777780 --- /dev/null +++ b/packages/records/lib/db/client.ts @@ -0,0 +1,4 @@ +import knex from 'knex'; +import { config } from './config.js'; + +export const db = knex(config); diff --git a/packages/records/lib/db/config.ts b/packages/records/lib/db/config.ts new file mode 100644 index 00000000000..60a5d1ab4a4 --- /dev/null +++ b/packages/records/lib/db/config.ts @@ -0,0 +1,30 @@ +import { envs } from '../env.js'; +import type { Knex } from 'knex'; + +export const schema = 'nango_records'; +const runningMigrationOnly = process.argv.some((v) => v === 'migrate:latest'); +const isJS = !runningMigrationOnly; + +const config: Knex.Config = { + client: 'postgres', + connection: { + host: envs.RECORDS_DB_HOST, + port: envs.RECORDS_DB_PORT, + user: envs.RECORDS_DB_USER, + password: envs.RECORDS_DB_PASSWORD, + database: envs.RECORDS_DB_NAME, + ssl: envs.RECORDS_DB_SSL === 'true' ? { rejectUnauthorized: true } : false, + statement_timeout: 60000 + }, + searchPath: schema, + pool: { min: 2, max: 20 }, + migrations: { + extension: isJS ? 'js' : 'ts', + directory: 'migrations', + tableName: 'migrations', + loadExtensions: [isJS ? '.js' : '.ts'], + schemaName: schema + } +}; + +export { config }; diff --git a/packages/records/lib/db/migrate.ts b/packages/records/lib/db/migrate.ts new file mode 100644 index 00000000000..76104e90caa --- /dev/null +++ b/packages/records/lib/db/migrate.ts @@ -0,0 +1,21 @@ +import { logger } from '../utils/logger.js'; +import { db } from './client.js'; +import { schema, config } from './config.js'; +import { dirname } from '../env.js'; +import path from 'node:path'; + +export async function migrate(): Promise { + logger.info('[records] migration'); + const dir = path.join(dirname, 'records/dist/db/migrations'); + await db.raw(`CREATE SCHEMA IF NOT EXISTS ${schema}`); + + const [, pendingMigrations] = (await db.migrate.list({ ...config.migrations, directory: dir })) as [unknown, string[]]; + + if (pendingMigrations.length === 0) { + logger.info('[records] nothing to do'); + return; + } + + await db.migrate.latest({ ...config.migrations, directory: dir }); + logger.info('[records] migrations completed.'); +} diff --git a/packages/records/lib/db/migrations/20240327151027_create_records_table.ts b/packages/records/lib/db/migrations/20240327151027_create_records_table.ts new file mode 100644 index 00000000000..edb6f7a39ce --- /dev/null +++ b/packages/records/lib/db/migrations/20240327151027_create_records_table.ts @@ -0,0 +1,95 @@ +import type { Knex } from 'knex'; + +const TABLE = 'records'; +const PARTITION_COUNT = 256; + +function partitionTable(i: number): string { + return `${TABLE}_p${i}`; +} + +export async function up(knex: Knex): Promise { + await knex.transaction(async (trx) => { + // TABLE + await trx.raw(` + CREATE TABLE "${TABLE}" ( + id uuid NOT NULL, + external_id character varying(255) NOT NULL, + json jsonb, + data_hash character varying(255) NOT NULL, + connection_id integer NOT NULL, + model character varying(255) NOT NULL, + created_at timestamp with time zone NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at timestamp with time zone NOT NULL DEFAULT CURRENT_TIMESTAMP, + deleted_at timestamp with time zone, + sync_id uuid, + sync_job_id integer, + CONSTRAINT ${TABLE}_connection_id_external_id_model UNIQUE (connection_id, external_id, model) + ) PARTITION BY HASH (connection_id, model) + `); + for (let i = 0; i < PARTITION_COUNT; i++) { + await trx.raw(` + CREATE TABLE "${partitionTable(i)}" PARTITION OF "${TABLE}" + FOR VALUES WITH (MODULUS ${PARTITION_COUNT}, REMAINDER ${i}); + `); + } + // TRIGGERS + await trx.raw(` + CREATE OR REPLACE FUNCTION ${TABLE}_undelete() + RETURNS TRIGGER AS $$ + BEGIN + IF OLD.deleted_at IS NOT NULL AND NEW.deleted_at IS NULL THEN + NEW.created_at = NOW(); + NEW.updated_at = NOW(); + END IF; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + `); + await trx.raw(` + CREATE TRIGGER ${TABLE}_undelete_trigger + BEFORE UPDATE ON ${TABLE} + FOR EACH ROW + EXECUTE FUNCTION ${TABLE}_undelete(); + `); + await knex.raw(` + CREATE OR REPLACE FUNCTION ${TABLE}_updated_at() + RETURNS TRIGGER AS $$ + BEGIN + IF OLD.data_hash IS DISTINCT FROM NEW.data_hash THEN + NEW.updated_at = NOW(); + END IF; + + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + `); + await knex.raw(` + CREATE TRIGGER ${TABLE}_updated_at_trigger + BEFORE UPDATE ON ${TABLE} + FOR EACH ROW + EXECUTE FUNCTION ${TABLE}_updated_at(); + `); + // INDEXES + await knex.schema.alterTable(TABLE, function (table) { + table.index(['connection_id', 'model', 'external_id']); + table.index(['connection_id', 'model', 'updated_at', 'id']); + table.index('sync_id'); + table.index('sync_job_id'); + table.unique(['connection_id', 'model', 'id']); + }); + }); +} + +export async function down(knex: Knex): Promise { + // TABLE + // INDEXES are dropped automatically + await knex.raw(`DROP TABLE IF EXISTS "${TABLE}"`); + for (let i = 0; i < PARTITION_COUNT; i++) { + await knex.raw(`DROP TABLE IF EXISTS "${partitionTable(i)}"`); + } + // TRIGGERS + await knex.raw(`DROP TRIGGER IF EXISTS ${TABLE}_undelete_trigger ON ${TABLE};`); + await knex.raw(`DROP FUNCTION IF EXISTS ${TABLE}_undelete();`); + await knex.raw(`DROP TRIGGER IF EXISTS ${TABLE}_updated_at_trigger ON ${TABLE};`); + await knex.raw(`DROP FUNCTION IF EXISTS ${TABLE}_updated_at();`); +} diff --git a/packages/records/lib/env.ts b/packages/records/lib/env.ts new file mode 100644 index 00000000000..9776dddf5a1 --- /dev/null +++ b/packages/records/lib/env.ts @@ -0,0 +1,16 @@ +import { z } from 'zod'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const schema = z.object({ + RECORDS_DB_HOST: z.string().optional().default('localhost'), + RECORDS_DB_PORT: z.coerce.number().optional().default(5432), + RECORDS_DB_USER: z.string().optional().default('nango'), + RECORDS_DB_NAME: z.string().optional().default('nango'), + RECORDS_DB_PASSWORD: z.string().optional().default('nango'), + RECORDS_DB_SSL: z.enum(['true', 'false']).optional().default('false') +}); +export const envs = schema.parse(process.env); + +export const filename = fileURLToPath(import.meta.url); +export const dirname = path.dirname(path.join(filename, '../../')); diff --git a/packages/records/lib/helpers/format.ts b/packages/records/lib/helpers/format.ts new file mode 100644 index 00000000000..2bf183ff51c --- /dev/null +++ b/packages/records/lib/helpers/format.ts @@ -0,0 +1,59 @@ +import md5 from 'md5'; +import * as uuid from 'uuid'; +import dayjs from 'dayjs'; +import utc from 'dayjs/plugin/utc.js'; +import type { FormattedRecord, UnencryptedRecordData } from '../types.js'; +import { NangoError, resultErr, resultOk } from '@nangohq/shared'; +import type { Result } from '@nangohq/shared'; + +dayjs.extend(utc); + +export const formatRecords = ( + data: UnencryptedRecordData[], + connection_id: number, + model: string, + syncId: string, + sync_job_id: number, + softDelete = false +): Result => { + // hashing unique composite key (connection, model, external_id) + // to generate stable record ids across script executions + const stableId = (unencryptedData: UnencryptedRecordData): string => { + const namespace = uuid.v5(`${connection_id}${model}`, uuid.NIL); + return uuid.v5(`${connection_id}${model}${unencryptedData.id}`, namespace); + }; + const formattedRecords: FormattedRecord[] = []; + for (const datum of data) { + const data_hash = md5(JSON.stringify(datum)); + + if (!datum) { + break; + } + + if (!datum['id']) { + const error = new NangoError('missing_id_field', model); + return resultErr(error); + } + + const formattedRecord: FormattedRecord = { + id: stableId(datum), + json: datum, + external_id: datum['id'], + data_hash, + model, + connection_id, + sync_id: syncId, + sync_job_id + }; + + if (softDelete) { + const deletedAt = datum['deletedAt']; + formattedRecord.updated_at = new Date(); + formattedRecord.deleted_at = deletedAt ? dayjs(deletedAt as string).toDate() : new Date(); + } else { + formattedRecord.deleted_at = null; + } + formattedRecords.push(formattedRecord); + } + return resultOk(formattedRecords); +}; diff --git a/packages/records/lib/helpers/uniqueKey.ts b/packages/records/lib/helpers/uniqueKey.ts new file mode 100644 index 00000000000..1d36d1541a4 --- /dev/null +++ b/packages/records/lib/helpers/uniqueKey.ts @@ -0,0 +1,43 @@ +import type { FormattedRecord } from '../types.js'; + +export function getUniqueId(record: FormattedRecord): string { + return record.external_id; +} + +export function verifyUniqueKeysAreUnique(records: FormattedRecord[]): { nonUniqueKeys: Set } { + const idMap = new Set(); + const nonUniqueKeys = new Set(); + + for (const record of records) { + const id = getUniqueId(record); + if (idMap.has(id)) { + nonUniqueKeys.add(id); + } else { + idMap.add(id); + } + } + + return { nonUniqueKeys }; +} + +export function removeDuplicateKey(records: FormattedRecord[]): { records: FormattedRecord[]; nonUniqueKeys: string[] } { + // TODO: + // for (const nonUniqueKey of nonUniqueKeys) { + // await createActivityLogMessage({ + // level: 'error', + // environment_id, + // activity_log_id: activityLogId, + // content: `There was a duplicate key found: ${nonUniqueKey}. This record will be ignore in relation to the model ${model}.`, + // timestamp: Date.now() + // }); + // } + + const { nonUniqueKeys } = verifyUniqueKeysAreUnique(records); + const seen = new Set(); + const recordsWithoutDuplicates = records.filter((record) => { + const key = getUniqueId(record); + return seen.has(key) ? false : seen.add(key); + }); + + return { records: recordsWithoutDuplicates, nonUniqueKeys: Array.from(nonUniqueKeys) }; +} diff --git a/packages/records/lib/helpers/uniqueKey.unit.test.ts b/packages/records/lib/helpers/uniqueKey.unit.test.ts new file mode 100644 index 00000000000..d593f93d0a8 --- /dev/null +++ b/packages/records/lib/helpers/uniqueKey.unit.test.ts @@ -0,0 +1,31 @@ +import { expect, describe, it } from 'vitest'; +import { removeDuplicateKey } from './uniqueKey.js'; +import type { FormattedRecord } from '../types.js'; + +describe('removeDuplicateKey', () => { + it('Should remove duplicates', () => { + const duplicateRecords = [ + { external_id: '1', name: 'John Doe' }, + { external_id: '1', name: 'John Doe' }, + { external_id: '2', name: 'Jane Doe' }, + { external_id: '2', name: 'Jane Doe' }, + { external_id: '3', name: 'John Doe' }, + { external_id: '3', name: 'John Doe' }, + { external_id: '4', name: 'Mike Doe' }, + { external_id: '5', name: 'Mark Doe' } + ]; + + const expected = [ + { external_id: '1', name: 'John Doe' }, + { external_id: '2', name: 'Jane Doe' }, + { external_id: '3', name: 'John Doe' }, + { external_id: '4', name: 'Mike Doe' }, + { external_id: '5', name: 'Mark Doe' } + ]; + + const { records, nonUniqueKeys } = removeDuplicateKey(duplicateRecords as unknown as FormattedRecord[]); + + expect(records).toEqual(expected); + expect(nonUniqueKeys).toEqual(['1', '2', '3']); + }); +}); diff --git a/packages/records/lib/index.ts b/packages/records/lib/index.ts new file mode 100644 index 00000000000..14e54fde979 --- /dev/null +++ b/packages/records/lib/index.ts @@ -0,0 +1,2 @@ +export * from './db/migrate.js'; +export * as Records from './models/records.js'; diff --git a/packages/records/lib/models/records.integration.test.ts b/packages/records/lib/models/records.integration.test.ts new file mode 100644 index 00000000000..7864c56555d --- /dev/null +++ b/packages/records/lib/models/records.integration.test.ts @@ -0,0 +1,175 @@ +import { expect, describe, it, beforeAll, afterEach } from 'vitest'; +import dayjs from 'dayjs'; +import * as uuid from 'uuid'; +import { migrate } from '../db/migrate.js'; +import { RECORDS_TABLE } from '../constants.js'; +import { db } from '../db/client.js'; +import * as Records from '../models/records.js'; +import { formatRecords } from '../helpers/format.js'; +import type { UnencryptedRecordData } from '../types.js'; +import type { UpsertSummary } from '../types.js'; +import { isErr } from '@nangohq/shared'; + +describe('Records service', () => { + beforeAll(async () => { + await migrate(); + }); + + afterEach(async () => { + await db(RECORDS_TABLE).truncate(); + }); + + it('Should write records', async () => { + const connectionId = 1; + const model = 'my-model'; + const syncId = '00000000-0000-0000-0000-000000000000'; + const records = [ + { id: '1', name: 'John Doe' }, + { id: '1', name: 'John Doe' }, + { id: '2', name: 'Jane Doe' }, + { id: '3', name: 'Max Doe' }, + { id: '4', name: 'Mike Doe' } + ]; + const res = await upsertRecords(records, connectionId, model, syncId, 1); + expect(res).toStrictEqual({ addedKeys: ['1', '2', '3', '4'], updatedKeys: [], deletedKeys: [], nonUniqueKeys: ['1'] }); + + const newRecords = [{ id: '2', name: 'Jane Moe' }]; + const updateRes = await updateRecords(newRecords, connectionId, model, syncId, 2); + expect(updateRes).toStrictEqual({ addedKeys: [], updatedKeys: ['2'], deletedKeys: [], nonUniqueKeys: [] }); + }); + + it('Should retrieve records', async () => { + const n = 10; + const { connectionId, model } = await upsertNRecords(n); + const response = await Records.getRecords({ connectionId, model }); + if (isErr(response)) { + throw new Error('Response is undefined'); + } + const { records, next_cursor } = response.res; + expect(records.length).toBe(n); + expect(records[0]?.['_nango_metadata']).toMatchObject({ + first_seen_at: expect.toBeIsoDateTimezone(), + last_modified_at: expect.toBeIsoDateTimezone(), + last_action: 'ADDED', + deleted_at: null, + cursor: expect.stringMatching(/^[A-Za-z0-9+/]+={0,2}$/) // base64 encoded string + }); + expect(next_cursor).toBe(null); // no next page + }); + it('Should paginate the records to retrieve all records', async () => { + const numOfRecords = 3000; + const limit = 100; + const { connectionId, model } = await upsertNRecords(numOfRecords); + + let cursor: string | undefined | null = null; + const allFetchedRecords = []; + do { + const response = await Records.getRecords({ + connectionId, + model, + limit, + ...(cursor && { cursor }) + }); + + if (isErr(response) || !response.res) { + throw new Error('Fail to fetch records'); + } + + const { records, next_cursor } = response.res; + + allFetchedRecords.push(...records); + + cursor = next_cursor; + + expect(records).not.toBe(undefined); + expect(records?.length).toBeLessThanOrEqual(limit); + } while (cursor); + + for (let i = 1; i < allFetchedRecords.length; i++) { + const currentRecordDate = dayjs(allFetchedRecords[i]?._nango_metadata.first_seen_at); + const previousRecordDate = dayjs(allFetchedRecords[i - 1]?._nango_metadata.first_seen_at); + + expect(currentRecordDate.isAfter(previousRecordDate) || currentRecordDate.isSame(previousRecordDate)).toBe(true); + } + expect(allFetchedRecords.length).toBe(numOfRecords); + }); + + it('Should be able to retrieve 20K records in under 5s with a cursor', async () => { + const numOfRecords = 20000; + const limit = 1000; + const { connectionId, model } = await upsertNRecords(numOfRecords); + + let cursor: string | undefined | null = null; + let allRecordsLength = 0; + + const startTime = Date.now(); + do { + const response = await Records.getRecords({ + connectionId, + model, + limit, + ...(cursor && { cursor }) + }); + + if (isErr(response) || !response.res) { + throw new Error('Error fetching records'); + } + + const { records, next_cursor } = response.res; + + allRecordsLength += records.length; + + cursor = next_cursor; + + expect(records).not.toBe(undefined); + expect(records?.length).toBeLessThanOrEqual(limit); + } while (cursor); + + const endTime = Date.now(); + + const runTime = endTime - startTime; + expect(runTime).toBeLessThan(5000); + + expect(allRecordsLength).toBe(numOfRecords); + }); +}); + +async function upsertNRecords(n: number): Promise<{ connectionId: number; model: string; syncId: string; syncJobId: number; result: UpsertSummary }> { + const records = Array.from({ length: n }, (_, i) => ({ id: `${i}`, name: `record ${i}` })); + const connectionId = Math.floor(Math.random() * 1000); + const model = 'model-' + Math.random().toString(36).substring(0, 4); + const syncId = uuid.v4(); + const syncJobId = Math.floor(Math.random() * 1000); + const result = await upsertRecords(records, connectionId, model, '00000000-0000-0000-0000-000000000000', 1); + return { + connectionId, + model, + syncId, + syncJobId, + result + }; +} + +async function upsertRecords(records: UnencryptedRecordData[], connectionId: number, model: string, syncId: string, syncJobId: number): Promise { + const formatRes = formatRecords(records, connectionId, model, syncId, syncJobId); + if (isErr(formatRes)) { + throw new Error(`Failed to format records: ${formatRes.err.message}`); + } + const updateRes = await Records.upsert(formatRes.res, connectionId, model); + if (isErr(updateRes)) { + throw new Error(`Failed to update records: ${updateRes.err.message}`); + } + return updateRes.res; +} + +async function updateRecords(records: UnencryptedRecordData[], connectionId: number, model: string, syncId: string, syncJobId: number) { + const formatRes = formatRecords(records, connectionId, model, syncId, syncJobId); + if (isErr(formatRes)) { + throw new Error(`Failed to format records: ${formatRes.err.message}`); + } + const updateRes = await Records.update(formatRes.res, connectionId, model); + if (isErr(updateRes)) { + throw new Error(`Failed to update records: ${updateRes.err.message}`); + } + return updateRes.res; +} diff --git a/packages/records/lib/models/records.ts b/packages/records/lib/models/records.ts new file mode 100644 index 00000000000..2cfb5cf99f3 --- /dev/null +++ b/packages/records/lib/models/records.ts @@ -0,0 +1,424 @@ +import dayjs from 'dayjs'; +import utc from 'dayjs/plugin/utc.js'; +import { db } from '../db/client.js'; +import type { + FormattedRecord, + FormattedRecordWithMetadata, + GetRecordsResponse, + LastAction, + ReturnedRecord, + UnencryptedRecord, + UpsertSummary +} from '../types.js'; +import { decryptRecordData, decryptRecords, encryptDataRecords } from '../utils/encryption.js'; +import { RECORDS_TABLE } from '../constants.js'; +import { removeDuplicateKey, getUniqueId } from '../helpers/uniqueKey.js'; +import { logger } from '../utils/logger.js'; +import { resultErr, resultOk } from '@nangohq/shared'; +import type { Result } from '@nangohq/shared'; + +dayjs.extend(utc); + +const BATCH_SIZE = 1000; + +export async function getRecords({ + connectionId, + model, + modifiedAfter, + limit, + filter, + cursor +}: { + connectionId: number; + model: string; + modifiedAfter?: string; + limit?: number | string; + filter?: LastAction; + cursor?: string; +}): Promise> { + try { + if (!model) { + const error = new Error('missing_model'); + return resultErr(error); + } + + let query = db + .from(RECORDS_TABLE) + .timeout(60000) // timeout after 1 minute + .where({ + connection_id: connectionId, + model + }) + .orderBy([ + { column: 'updated_at', order: 'asc' }, + { column: 'id', order: 'asc' } + ]); + + if (cursor) { + const decodedCursorValue = Buffer.from(cursor, 'base64').toString('ascii'); + const [cursorSort, cursorId] = decodedCursorValue.split('||'); + + if (!cursorSort || !cursorId) { + const error = new Error('invalid_cursor_value'); + return resultErr(error); + } + + query = query.where( + (builder) => + void builder + .where('updated_at', '>', cursorSort) + .orWhere((builder) => void builder.where('updated_at', '=', cursorSort).andWhere('id', '>', cursorId)) + ); + } + + if (limit) { + if (isNaN(Number(limit))) { + const error = new Error('invalid_limit'); + return resultErr(error); + } + query = query.limit(Number(limit) + 1); + } else { + query = query.limit(101); + } + + if (modifiedAfter) { + const time = dayjs(modifiedAfter); + + if (!time.isValid()) { + const error = new Error('invalid_timestamp'); + return resultErr(error); + } + + const formattedDelta = time.toISOString(); + + query = query.andWhere('updated_at', '>=', formattedDelta); + } + + if (filter) { + const formattedFilter = filter.toUpperCase(); + switch (true) { + case formattedFilter.includes('ADDED') && formattedFilter.includes('UPDATED'): + query = query.andWhere('deleted_at', null).andWhere(function () { + void this.where('created_at', '=', db.raw('updated_at')).orWhere('created_at', '!=', db.raw('updated_at')); + }); + break; + case formattedFilter.includes('UPDATED') && formattedFilter.includes('DELETED'): + query = query.andWhere(function () { + void this.where('deleted_at', null).andWhere('created_at', '!=', db.raw('updated_at')); + }); + break; + case formattedFilter.includes('ADDED') && formattedFilter.includes('DELETED'): + query = query.andWhere(function () { + void this.where('deleted_at', null).andWhere('created_at', '=', db.raw('updated_at')); + }); + break; + case formattedFilter === 'ADDED': + query = query.andWhere('deleted_at', null).andWhere('created_at', '=', db.raw('updated_at')); + break; + case formattedFilter === 'UPDATED': + query = query.andWhere('deleted_at', null).andWhere('created_at', '!=', db.raw('updated_at')); + break; + case formattedFilter === 'DELETED': + query = query.andWhereNot({ deleted_at: null }); + break; + } + } + + const rawResults: FormattedRecordWithMetadata[] = await query.select( + // PostgreSQL stores timestamp with microseconds precision + // however, javascript date only supports milliseconds precision + // we therefore convert timestamp to string (using to_json()) in order to avoid precision loss + db.raw(` + id, + json, + to_json(created_at) as first_seen_at, + to_json(updated_at) as last_modified_at, + to_json(deleted_at) as deleted_at, + CASE + WHEN deleted_at IS NOT NULL THEN 'DELETED' + WHEN created_at = updated_at THEN 'ADDED' + ELSE 'UPDATED' + END as last_action + `) + ); + + if (rawResults.length === 0) { + return resultOk({ records: [], next_cursor: null }); + } + + const results = rawResults.map((item) => { + const decryptedData = decryptRecordData(item); + const encodedCursor = Buffer.from(`${item.last_modified_at}||${item.id}`).toString('base64'); + return { + ...decryptedData, + _nango_metadata: { + first_seen_at: item.first_seen_at, + last_modified_at: item.last_modified_at, + last_action: item.last_action, + deleted_at: item.deleted_at, + cursor: encodedCursor + } + } as ReturnedRecord; + }); + + if (results.length > Number(limit || 100)) { + results.pop(); + rawResults.pop(); + + const cursorRawElement = rawResults[rawResults.length - 1]; + if (cursorRawElement) { + const encodedCursorValue = Buffer.from(`${cursorRawElement.last_modified_at}||${cursorRawElement.id}`).toString('base64'); + return resultOk({ records: results, next_cursor: encodedCursorValue }); + } + } + return resultOk({ records: results, next_cursor: null }); + } catch (_error) { + // TODO: telemetry doesn't belong in models + // await telemetry.log(LogTypes.SYNC_GET_RECORDS_QUERY_TIMEOUT, errorMessage, LogActionEnum.SYNC, { + // environmentId: String(environmentId), + // connectionId: String(connectionId), + // modifiedAfter: String(modifiedAfter), + // model, + // error: JSON.stringify(e) + // }); + const e = new Error(`List records error for model ${model}`); + return resultErr(e); + } +} + +export async function upsert(records: FormattedRecord[], connectionId: number, model: string, softDelete = false): Promise> { + const { records: recordsWithoutDuplicates, nonUniqueKeys } = removeDuplicateKey(records); + + if (!recordsWithoutDuplicates || recordsWithoutDuplicates.length === 0) { + return resultErr( + `There are no records to upsert because there were no records that were not duplicates to insert, but there were ${records.length} records received for the "${model}" model.` + ); + } + + const addedKeys: string[] = []; + const updatedKeys: string[] = []; + try { + const externalIds: { external_id: string }[] = []; + await db.transaction(async (trx) => { + for (let i = 0; i < recordsWithoutDuplicates.length; i += BATCH_SIZE) { + const chunk = recordsWithoutDuplicates.slice(i, i + BATCH_SIZE); + addedKeys.push(...(await getAddedKeys(chunk, connectionId, model))); + updatedKeys.push(...(await getUpdatedKeys(chunk, connectionId, model))); + const encryptedRecords = encryptDataRecords(chunk); + const insertedIds = await trx + .from(RECORDS_TABLE) + .insert(encryptedRecords) + .onConflict(['connection_id', 'external_id', 'model']) + .merge() + .returning('external_id'); + externalIds.push(...insertedIds); + } + }); + + if (softDelete) { + return resultOk({ + deletedKeys: externalIds.map(({ external_id }) => external_id), + addedKeys: [], + updatedKeys: [], + nonUniqueKeys + }); + } + + return resultOk({ + addedKeys, + updatedKeys, + deletedKeys: [], + nonUniqueKeys + }); + } catch (error: any) { + let errorMessage = `Failed to upsert records to table ${RECORDS_TABLE}.\n`; + errorMessage += `Model: ${model}, Nango Connection ID: ${connectionId}.\n`; + errorMessage += `Attempted to insert/update/delete: ${recordsWithoutDuplicates.length} records\n`; + + if ('code' in error) { + const errorCode = (error as { code: string }).code; + errorMessage += `Error code: ${errorCode}.\n`; + let errorDetail = ''; + switch (errorCode) { + case '22001': { + errorDetail = "String length exceeds the column's maximum length (string_data_right_truncation)"; + break; + } + } + if (errorDetail) errorMessage += `Info: ${errorDetail}.\n`; + } + + logger.error(`${errorMessage}${error}`); + + return resultErr(errorMessage); + } +} + +export async function update(records: FormattedRecord[], connectionId: number, model: string): Promise> { + const { records: recordsWithoutDuplicates, nonUniqueKeys } = removeDuplicateKey(records); + + if (!recordsWithoutDuplicates || recordsWithoutDuplicates.length === 0) { + return resultErr( + `There are no records to upsert because there were no records that were not duplicates to insert, but there were ${records.length} records received for the "${model}" model.` + ); + } + + try { + const updatedKeys: string[] = []; + await db.transaction(async (trx) => { + for (let i = 0; i < recordsWithoutDuplicates.length; i += BATCH_SIZE) { + const chunk = recordsWithoutDuplicates.slice(i, i + BATCH_SIZE); + + updatedKeys.push(...(await getUpdatedKeys(chunk, connectionId, model))); + + const recordsToUpdate: FormattedRecord[] = []; + const rawOldRecords = await getRecordsByExternalIds(updatedKeys, connectionId, model); + for (const rawOldRecord of rawOldRecords) { + if (!rawOldRecord) { + continue; + } + + const { record: oldRecord, ...oldRecordRest } = rawOldRecord; + + const record = records.find((record) => record.external_id === oldRecord.id); + + const newRecord: FormattedRecord = { + ...oldRecordRest, + json: { + ...oldRecord, + ...record?.json + }, + updated_at: new Date() + }; + recordsToUpdate.push(newRecord); + } + const encryptedRecords = encryptDataRecords(recordsToUpdate); + await trx.from(RECORDS_TABLE).insert(encryptedRecords).onConflict(['connection_id', 'external_id', 'model']).merge(); + } + }); + + return resultOk({ + addedKeys: [], + updatedKeys, + deletedKeys: [], + nonUniqueKeys + }); + } catch (error: any) { + let errorMessage = `Failed to update records to table ${RECORDS_TABLE}.\n`; + errorMessage += `Model: ${model}, Nango Connection ID: ${connectionId}.\n`; + errorMessage += `Attempted to update: ${recordsWithoutDuplicates.length} records\n`; + + if ('code' in error) errorMessage += `Error code: ${(error as { code: string }).code}.\n`; + if ('detail' in error) errorMessage += `Detail: ${(error as { detail: string }).detail}.\n`; + if ('message' in error) errorMessage += `Error Message: ${(error as { message: string }).message}`; + + return resultErr(errorMessage); + } +} + +export async function deleteRecordsBySyncId({ syncId, limit = 5000 }: { syncId: string; limit?: number }): Promise<{ totalDeletedRecords: number }> { + let totalDeletedRecords = 0; + let deletedRecords = 0; + do { + deletedRecords = await db + .from(RECORDS_TABLE) + .whereIn('id', function (sub) { + void sub.select('id').from(RECORDS_TABLE).where({ sync_id: syncId }).limit(limit); + }) + .del(); + totalDeletedRecords += deletedRecords; + } while (deletedRecords >= limit); + + return { totalDeletedRecords }; +} + +// Mark all non-deleted records that don't belong to currentGeneration as deleted +// returns the ids of records being deleted +export async function markNonCurrentGenerationRecordsAsDeleted(connectionId: number, model: string, syncId: string, generation: number): Promise { + const now = db.fn.now(6); + return (await db + .from(RECORDS_TABLE) + .where({ + connection_id: connectionId, + model, + sync_id: syncId, + deleted_at: null + }) + .whereNot({ + sync_job_id: generation + }) + .update({ + deleted_at: now, + updated_at: now, + sync_job_id: generation + }) + .returning('id')) as unknown as string[]; +} + +/** + * Compute Added Keys + * @desc for any incoming payload use the provided unique to check against the rows + * in the database and return the keys that are not in the database + * + */ +async function getAddedKeys(response: FormattedRecord[], connectionId: number, model: string): Promise { + const keys: string[] = response.map((record: FormattedRecord) => getUniqueId(record)); + + const knownKeys: string[] = (await db + .from(RECORDS_TABLE) + .where({ + connection_id: connectionId, + model, + deleted_at: null + }) + .whereIn('external_id', keys) + .pluck('external_id')) as unknown as string[]; + + return keys?.filter((key: string) => !knownKeys.includes(key)); +} + +/** + * Get Updated Keys + * @desc generate an array of the keys that exist in the database and also in + * the incoming payload that will be used to update the database. + * Compare using the data_hash key + * + */ +async function getUpdatedKeys(response: FormattedRecord[], connectionId: number, model: string): Promise { + const keys: string[] = response.map((record: FormattedRecord) => getUniqueId(record)); + const keysWithHash: [string, string][] = response.map((record: FormattedRecord) => [getUniqueId(record), record.data_hash]); + + const rowsToUpdate = (await db + .from(RECORDS_TABLE) + .pluck('external_id') + .where({ + connection_id: connectionId, + model + }) + .whereIn('external_id', keys) + .whereNotIn(['external_id', 'data_hash'], keysWithHash)) as unknown as string[]; + + return rowsToUpdate; +} + +async function getRecordsByExternalIds(external_ids: string[], connection_id: number, model: string): Promise { + const encryptedRecords = await db + .from(RECORDS_TABLE) + .where({ + connection_id, + model + }) + .whereIn('external_id', external_ids); + + if (!encryptedRecords) { + return []; + } + + const result = decryptRecords(encryptedRecords); + + if (!result || result.length === 0) { + return []; + } + + return result; +} diff --git a/packages/records/lib/types.ts b/packages/records/lib/types.ts new file mode 100644 index 00000000000..b7142d2d6c2 --- /dev/null +++ b/packages/records/lib/types.ts @@ -0,0 +1,60 @@ +type RecordValue = string | number | boolean | null | undefined | object | Record; + +export interface RecordInput { + id: string; + [index: string]: RecordValue; +} + +export interface FormattedRecord { + id: string; + external_id: string; + json: RecordData; + data_hash: string; + connection_id: number; + model: string; + sync_id: string; + sync_job_id: number; + created_at?: Date; + updated_at?: Date; + deleted_at?: Date | null; +} + +export type FormattedRecordWithMetadata = FormattedRecord & RecordMetadata; + +export type UnencryptedRecord = FormattedRecord & { record: UnencryptedRecordData }; + +export interface EncryptedRecordData { + iv: string; + authTag: string; + encryptedValue: string; +} + +export type RecordData = UnencryptedRecordData | EncryptedRecordData; + +export type UnencryptedRecordData = Record & { id: string }; + +export type ReturnedRecord = { + _nango_metadata: RecordMetadata; +} & Record & { id: string | number }; + +export type LastAction = 'ADDED' | 'UPDATED' | 'DELETED' | 'added' | 'updated' | 'deleted'; + +interface RecordMetadata { + first_seen_at: string; + last_modified_at: string; + last_action: LastAction; + deleted_at: string | null; + cursor: string; +} + +export interface GetRecordsResponse { + records: ReturnedRecord[]; + next_cursor?: string | null; +} + +export interface UpsertSummary { + addedKeys: string[]; + updatedKeys: string[]; + deletedKeys?: string[]; + nonUniqueKeys: string[]; +} diff --git a/packages/records/lib/utils/encryption.ts b/packages/records/lib/utils/encryption.ts new file mode 100644 index 00000000000..aa52c995341 --- /dev/null +++ b/packages/records/lib/utils/encryption.ts @@ -0,0 +1,42 @@ +import type { EncryptedRecordData, FormattedRecord, UnencryptedRecord, UnencryptedRecordData } from '../types'; +import { encryptionManager } from '@nangohq/shared'; + +function isEncrypted(data: UnencryptedRecordData | EncryptedRecordData): data is EncryptedRecordData { + return 'encryptedValue' in data; +} + +export function decryptRecordData(record: FormattedRecord): UnencryptedRecordData { + const { json } = record; + if (isEncrypted(json)) { + const { encryptedValue, iv, authTag } = json; + const decryptedString = encryptionManager.decrypt(encryptedValue, iv, authTag); + return JSON.parse(decryptedString) as UnencryptedRecordData; + } + return json; +} + +export function decryptRecords(records: FormattedRecord[]): UnencryptedRecord[] { + const decryptedRecords: UnencryptedRecord[] = []; + for (const record of records) { + decryptedRecords.push({ + ...record, + record: decryptRecordData(record) + }); + } + return decryptedRecords; +} + +export function encryptDataRecords(records: FormattedRecord[]): FormattedRecord[] { + if (!encryptionManager.shouldEncrypt()) { + return records; + } + + const encryptedDataRecords: FormattedRecord[] = Object.assign([], records); + + for (const record of encryptedDataRecords) { + const [encryptedValue, iv, authTag] = encryptionManager.encrypt(JSON.stringify(record.json)); + record.json = { encryptedValue, iv, authTag }; + } + + return encryptedDataRecords; +} diff --git a/packages/records/lib/utils/logger.ts b/packages/records/lib/utils/logger.ts new file mode 100644 index 00000000000..e12924c1195 --- /dev/null +++ b/packages/records/lib/utils/logger.ts @@ -0,0 +1,3 @@ +import { getLogger } from '@nangohq/utils/dist/logger.js'; + +export const logger = getLogger('records'); diff --git a/packages/records/lib/vitest.d.ts b/packages/records/lib/vitest.d.ts new file mode 100644 index 00000000000..b07ff6fed5f --- /dev/null +++ b/packages/records/lib/vitest.d.ts @@ -0,0 +1,14 @@ +export * from 'vitest'; + +/* eslint-disable @typescript-eslint/no-empty-interface */ +export interface CustomMatchers { + toBeIsoDate: () => TR; + toBeIsoDateTimezone: () => TR; + toBeUUID: () => TR; +} + +declare module 'vitest' { + export interface Assertion extends CustomMatchers {} + export interface AsymmetricMatchersContaining extends CustomMatchers {} + export interface ExpectStatic extends CustomMatchers {} +} diff --git a/packages/records/package.json b/packages/records/package.json new file mode 100644 index 00000000000..53a4decdfbe --- /dev/null +++ b/packages/records/package.json @@ -0,0 +1,31 @@ +{ + "name": "@nangohq/nango-records", + "version": "1.0.0", + "type": "module", + "main": "dist/index.js", + "scripts": { + "dev:migration:create": "npm run knex -- migrate:make", + "dev:migration:run": "npm run knex -- migrate:latest", + "knex": "tsx ../../node_modules/knex/bin/cli.js --knexfile lib/db/knexfile.ts", + "prod:migration:run": "knex --knexfile dist/db/knexfile.js migrate:latest" + }, + "keywords": [], + "repository": { + "type": "git", + "url": "git+https://github.com/NangoHQ/nango.git", + "directory": "packages/records" + }, + "dependencies": { + "@nangohq/shared": "file:../shared", + "@nangohq/utils": "file:../utils", + "dayjs": "1.11.10", + "knex": "3.1.0", + "md5": "2.3.0", + "pg": "8.11.3", + "uuid": "9.0.1", + "zod": "3.22.4" + }, + "devDependencies": { + "vitest": "1.4.0" + } +} diff --git a/packages/records/tsconfig.json b/packages/records/tsconfig.json new file mode 100644 index 00000000000..ec18af8262b --- /dev/null +++ b/packages/records/tsconfig.json @@ -0,0 +1,16 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "rootDir": "lib", + "outDir": "dist" + }, + "references": [ + { + "path": "../shared" + }, + { + "path": "../utils" + } + ], + "include": ["lib/**/*"] +} diff --git a/packages/server/package.json b/packages/server/package.json index 8029d8eb12e..c94b6ac6762 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -25,6 +25,7 @@ "@hapi/boom": "^10.0.1", "@nangohq/shared": "^0.39.15", "@nangohq/utils": "file:../utils", + "@nangohq/records": "file:../records", "@workos-inc/node": "^6.2.0", "axios": "^1.3.4", "body-parser": "1.20.2", diff --git a/packages/server/tsconfig.json b/packages/server/tsconfig.json index e16701dc927..f1166b1bdd3 100644 --- a/packages/server/tsconfig.json +++ b/packages/server/tsconfig.json @@ -13,6 +13,9 @@ }, { "path": "../utils" + }, + { + "path": "../records" } ], "include": ["lib/**/*", "../shared/lib/express.d.ts"] diff --git a/packages/shared/lib/utils/encryption.manager.ts b/packages/shared/lib/utils/encryption.manager.ts index 5204b1040a3..d29f74b8596 100644 --- a/packages/shared/lib/utils/encryption.manager.ts +++ b/packages/shared/lib/utils/encryption.manager.ts @@ -35,11 +35,11 @@ class EncryptionManager { } } - private shouldEncrypt(): boolean { + public shouldEncrypt(): boolean { return Boolean((this?.key as string) && (this.key as string).length > 0); } - private encrypt(str: string): [string, string | null, string | null] { + public encrypt(str: string): [string, string, string] { const iv = crypto.randomBytes(12); const cipher = crypto.createCipheriv(this.algo, Buffer.from(this.key!, this.encoding), iv); let enc = cipher.update(str, 'utf8', this.encoding); @@ -47,7 +47,7 @@ class EncryptionManager { return [enc, iv.toString(this.encoding), cipher.getAuthTag().toString(this.encoding)]; } - private decrypt(enc: string, iv: string, authTag: string): string { + public decrypt(enc: string, iv: string, authTag: string): string { const decipher = crypto.createDecipheriv(this.algo, Buffer.from(this.key!, this.encoding), Buffer.from(iv, this.encoding)); decipher.setAuthTag(Buffer.from(authTag, this.encoding)); let str = decipher.update(enc, this.encoding, 'utf8'); @@ -200,7 +200,7 @@ class EncryptionManager { if (config.custom) { const [encryptedValue, iv, authTag] = this.encrypt(JSON.stringify(config.custom)); - encryptedConfig.custom = { encryptedValue, iv: iv as string, authTag: authTag as string }; + encryptedConfig.custom = { encryptedValue, iv: iv, authTag: authTag }; } return encryptedConfig; diff --git a/tests/setup.ts b/tests/setup.ts index 4ee5f8a0c91..cf89b6d8684 100644 --- a/tests/setup.ts +++ b/tests/setup.ts @@ -43,6 +43,12 @@ async function setupPostgres() { process.env['NANGO_DB_NAME'] = dbName; process.env['NANGO_DB_MIGRATION_FOLDER'] = './packages/shared/lib/db/migrations'; process.env['TELEMETRY'] = 'false'; + + process.env['RECORDS_DB_HOST'] = process.env['NANGO_DB_HOST']; + process.env['RECORDS_DB_PORT'] = process.env['NANGO_DB_PORT']; + process.env['RECORDS_DB_NAME'] = process.env['NANGO_DB_NAME']; + process.env['RECORDS_DB_USER'] = process.env['NANGO_DB_USER']; + process.env['RECORDS_DB_PASSWORD'] = process.env['NANGO_DB_PASSWORD']; } export async function setup() { diff --git a/tsconfig.build.json b/tsconfig.build.json index 6d171a543c9..bc6e5e10fe7 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -22,6 +22,9 @@ { "path": "packages/persist" }, + { + "path": "packages/records" + }, { "path": "packages/runner" },