From 40a524ddb784debf9ad89e6ab0a0c3b300c54444 Mon Sep 17 00:00:00 2001 From: Andrey Sobolev Date: Mon, 30 Sep 2024 12:00:48 +0700 Subject: [PATCH] UBERF-8310: Optimize backup service (#6763) Signed-off-by: Andrey Sobolev --- server/backup-service/src/config.ts | 3 ++ server/backup/src/backup.ts | 23 ++++++++++++-- server/backup/src/service.ts | 48 ++++++++--------------------- 3 files changed, 37 insertions(+), 37 deletions(-) diff --git a/server/backup-service/src/config.ts b/server/backup-service/src/config.ts index c8c75a628af..522719a6306 100644 --- a/server/backup-service/src/config.ts +++ b/server/backup-service/src/config.ts @@ -21,6 +21,7 @@ interface Config extends Omit { Secret: string Interval: number // Timeout in seconds + CoolDown: number Timeout: number // Timeout in seconds BucketName: string Storage: string // A bucket storage config @@ -37,6 +38,7 @@ const envMap: { [key in keyof Config]: string } = { Secret: 'SECRET', BucketName: 'BUCKET_NAME', Interval: 'INTERVAL', + CoolDown: 'COOL_DOWN', Timeout: 'TIMEOUT', MongoURL: 'MONGO_URL', SkipWorkspaces: 'SKIP_WORKSPACES', @@ -62,6 +64,7 @@ const config: Config = (() => { ServiceID: process.env[envMap.ServiceID] ?? 'backup-service', Interval: parseInt(process.env[envMap.Interval] ?? '3600'), Timeout: parseInt(process.env[envMap.Timeout] ?? '3600'), + CoolDown: parseInt(process.env[envMap.CoolDown] ?? '300'), MongoURL: process.env[envMap.MongoURL], SkipWorkspaces: process.env[envMap.SkipWorkspaces] ?? '', WorkspaceStorage: process.env[envMap.WorkspaceStorage], diff --git a/server/backup/src/backup.ts b/server/backup/src/backup.ts index 39a67c01bf7..c0c07aa7ae0 100644 --- a/server/backup/src/backup.ts +++ b/server/backup/src/backup.ts @@ -1121,9 +1121,26 @@ export async function backup ( } result.result = true + const sizeFile = 'backup.size.gz' + + let sizeInfo: Record = {} + + if (await storage.exists(sizeFile)) { + sizeInfo = JSON.parse(gunzipSync(await storage.loadFile(sizeFile)).toString()) + } + let processed = 0 + const addFileSize = async (file: string | undefined | null): Promise => { - if (file != null && (await storage.exists(file))) { - const fileSize = await storage.stat(file) + if (file != null) { + const sz = sizeInfo[file] + const fileSize = sz ?? (await storage.stat(file)) + if (sz === undefined) { + sizeInfo[file] = fileSize + processed++ + if (processed % 10 === 0) { + ctx.info('Calculate size processed', { processed, size: Math.round(result.backupSize / (1024 * 1024)) }) + } + } result.backupSize += fileSize } } @@ -1142,6 +1159,8 @@ export async function backup ( } await addFileSize(infoFile) + await storage.writeFile(sizeFile, gzipSync(JSON.stringify(sizeInfo, undefined, 2), { level: defaultLevel })) + return result } catch (err: any) { ctx.error('backup error', { err, workspace: workspaceId.name }) diff --git a/server/backup/src/service.ts b/server/backup/src/service.ts index 2355d70a2f9..20355f0bb65 100644 --- a/server/backup/src/service.ts +++ b/server/backup/src/service.ts @@ -47,6 +47,8 @@ export interface BackupConfig { Token: string Interval: number // Timeout in seconds + + CoolDown: number // Cooldown in seconds Timeout: number // Timeout in seconds BucketName: string SkipWorkspaces: string @@ -67,15 +69,10 @@ class BackupWorker { ) {} canceled = false - interval: any - async close (): Promise { this.canceled = true - clearTimeout(this.interval) } - backupPromise: Promise | undefined - printStats ( ctx: MeasureContext, stats: { failedWorkspaces: BaseWorkspaceInfo[], processed: number, skipped: number } @@ -91,27 +88,14 @@ class BackupWorker { ) } - async triggerBackup (ctx: MeasureContext): Promise { - const { failedWorkspaces } = await this.backup(ctx) - if (failedWorkspaces.length > 0) { - ctx.info('Failed to backup workspaces, Retry failed workspaces once.', { failed: failedWorkspaces.length }) - this.printStats(ctx, await this.doBackup(ctx, failedWorkspaces)) - } - } - async schedule (ctx: MeasureContext): Promise { - console.log('schedule timeout for', this.config.Interval, ' seconds') - this.interval = setTimeout( - () => { - if (this.backupPromise !== undefined) { - void this.backupPromise.then(() => { - void this.triggerBackup(ctx) - }) - } - void this.triggerBackup(ctx) - }, - 5 * 60 * 1000 - ) // Re-check every 5 minutes. + console.log('schedule backup with interval', this.config.Interval, 'seconds') + while (!this.canceled) { + const res = await this.backup(ctx) + this.printStats(ctx, res) + console.log('cool down', this.config.CoolDown, 'seconds') + await new Promise((resolve) => setTimeout(resolve, this.config.CoolDown * 1000)) + } } async backup ( @@ -171,11 +155,7 @@ class BackupWorker { index, total: workspaces.length }) - const childLogger = rootCtx.logger.childLogger?.(ws.workspace, { - workspace: ws.workspace, - enableConsole: 'true' - }) - const ctx = rootCtx.newChild(ws.workspace, { workspace: ws.workspace }, {}, childLogger) + const ctx = rootCtx.newChild(ws.workspace, { workspace: ws.workspace }) let pipeline: Pipeline | undefined try { const storage = await createStorageBackupStorage( @@ -245,6 +225,8 @@ class BackupWorker { } rootCtx.warn('BACKUP STATS', { workspace: ws.workspace, + workspaceUrl: ws.workspaceUrl, + workspaceName: ws.workspaceName, index, ...backupInfo, time: Math.round((Date.now() - st) / 1000), @@ -259,7 +241,6 @@ class BackupWorker { } catch (err: any) { rootCtx.error('\n\nFAILED to BACKUP', { workspace: ws.workspace, err }) failedWorkspaces.push(ws) - await childLogger?.close() } finally { if (pipeline !== undefined) { await pipeline.close() @@ -351,9 +332,6 @@ export function backupService ( void backupWorker.close() } - void backupWorker.backup(ctx).then((res) => { - backupWorker.printStats(ctx, res) - void backupWorker.schedule(ctx) - }) + void backupWorker.schedule(ctx) return shutdown }