fix: fix downloading all resources on backend (#122)

cieslarmichal · Sep 6, 2024 · 365431a · 365431a
2 parents 6eee94b + 46f0dbd
commit 365431a
Show file tree

Hide file tree

Showing 5 changed files with 137 additions and 4 deletions.
diff --git a/apps/backend/src/core/application.ts b/apps/backend/src/core/application.ts
@@ -1,3 +1,8 @@
+import { existsSync, type Stats } from 'fs';
+import { stat, rm, readdir } from 'fs/promises';
+import { schedule } from 'node-cron';
+import { join } from 'path';
+
 import { UserRole } from '@common/contracts';
 
 import { ApplicationHttpController } from './api/httpControllers/applicationHttpController/applicationHttpController.js';
@@ -125,8 +130,50 @@ export class Application {
 
     await this.createAdminUser(container);
 
+    this.createCleanupCronJob();
+
     const server = new HttpServer(container);
 
     await server.start();
   }
+
+  private static createCleanupCronJob(): void {
+    const directoryPath = '/tmp/buckets';
+
+    schedule('*/30 * * * *', async () => {
+      if (!existsSync(directoryPath)) {
+        return;
+      }
+
+      const directories = await readdir(directoryPath);
+
+      await Promise.all(
+        directories.map(async (file) => {
+          const subdirPath = join(directoryPath, file);
+
+          let subdirStats: Stats;
+
+          try {
+            subdirStats = await stat(subdirPath);
+          } catch {
+            console.error({ subdirPath }, 'Error reading directory, skipping.');
+
+            return;
+          }
+
+          const now = new Date().getTime();
+
+          const subdirCreationTime = new Date(subdirStats.birthtime).getTime();
+
+          const diffInMinutes = (now - subdirCreationTime) / (1000 * 60);
+
+          if (diffInMinutes > 30) {
+            await rm(subdirPath, { recursive: true });
+
+            console.log({ subdirPath }, 'Directory deleted.');
+          }
+        }),
+      );
+    });
+  }
 }
diff --git a/...lication/queryHandlers/downloadResourcesQueryHandler/downloadResourcesQueryHandlerImpl.ts b/...lication/queryHandlers/downloadResourcesQueryHandler/downloadResourcesQueryHandlerImpl.ts
@@ -1,4 +1,5 @@
 import archiver from 'archiver';
+import { createWriteStream, mkdirSync } from 'node:fs';
 
 import {
   type DownloadResourcesQueryHandler,
@@ -7,6 +8,7 @@ import {
 } from './downloadResourcesQueryHandler.js';
 import { OperationNotValidError } from '../../../../../common/errors/common/operationNotValidError.js';
 import { type LoggerService } from '../../../../../libs/logger/services/loggerService/loggerService.js';
+import { type UuidService } from '../../../../../libs/uuid/services/uuidService/uuidService.js';
 import { ForbiddenAccessError } from '../../../../authModule/application/errors/forbiddenAccessError.js';
 import { type FindUserBucketsQueryHandler } from '../../../../userModule/application/queryHandlers/findUserBucketsQueryHandler/findUserBucketsQueryHandler.js';
 import { type ResourceBlobService } from '../../../domain/services/resourceBlobService/resourceBlobService.js';
@@ -16,6 +18,7 @@ export class DownloadResourcesQueryHandlerImpl implements DownloadResourcesQuery
     private readonly resourceBlobSerice: ResourceBlobService,
     private readonly loggerService: LoggerService,
     private readonly findUserBucketsQueryHandler: FindUserBucketsQueryHandler,
+    private readonly uuidService: UuidService,
   ) {}
 
   public async execute(payload: DownloadResourcesQueryHandlerPayload): Promise<DownloadResourcesQueryHandlerResult> {
@@ -32,14 +35,19 @@ export class DownloadResourcesQueryHandlerImpl implements DownloadResourcesQuery
       });
     }
 
+    const blobsIds = await this.resourceBlobSerice.getResourcesIds({ bucketName });
+
+    const tempDir = `/tmp/buckets/${this.uuidService.generateUuid()}`;
+
+    mkdirSync(tempDir, { recursive: true });
+
     this.loggerService.debug({
       message: 'Downloading Resources...',
       userId,
       bucketName,
+      tempDir,
     });
 
-    const blobsIds = await this.resourceBlobSerice.getResourcesIds({ bucketName });
-
     if (!blobsIds.length) {
       this.loggerService.error({
         message: 'Resources not found.',
@@ -65,6 +73,24 @@ export class DownloadResourcesQueryHandlerImpl implements DownloadResourcesQuery
 
     const archive = archiver('zip', { zlib: { level: 9 } });
 
+    archive.on('error', (error) => {
+      this.loggerService.error({
+        message: 'Error while creating archive.',
+        userId,
+        bucketName,
+        error,
+      });
+
+      throw error;
+    });
+
+    archive.on('progress', (progress) => {
+      this.loggerService.debug({
+        message: 'Archive in progress...',
+        progress: progress.entries,
+      });
+    });
+
     let archivedResourcesCount = 0;
 
     for (const blobId of blobsIds) {
@@ -74,19 +100,45 @@ export class DownloadResourcesQueryHandlerImpl implements DownloadResourcesQuery
           resourceId: blobId,
         });
 
-        archive.append(blobData, { name });
+        const tempFilePath = `${tempDir}/${blobId}`;
+
+        const writeStream = createWriteStream(tempFilePath);
+
+        await new Promise<void>((resolve, reject) => {
+          blobData.pipe(writeStream);
+
+          blobData.on('end', resolve);
+
+          blobData.on('error', reject);
+        });
+
+        this.loggerService.debug({
+          message: 'Resource downloaded.',
+          bucketName,
+          resourceId: blobId,
+          tempFilePath,
+        });
+
+        archive.file(tempFilePath, { name });
 
         archivedResourcesCount++;
       }
     }
 
+    this.loggerService.debug({
+      message: 'Finalizing archive...',
+      userId,
+      bucketName,
+    });
+
     archive.finalize();
 
     this.loggerService.debug({
       message: 'Resources downloaded.',
       userId,
       bucketName,
       count: archivedResourcesCount,
+      tempDir,
     });
 
     return { resourcesData: archive };

diff --git a/apps/backend/src/modules/resourceModule/resourceModule.ts b/apps/backend/src/modules/resourceModule/resourceModule.ts
@@ -76,6 +76,7 @@ export class ResourceModule implements DependencyInjectionModule {
           container.get<ResourceBlobService>(symbols.resourceBlobService),
           container.get<LoggerService>(coreSymbols.loggerService),
           container.get<FindUserBucketsQueryHandler>(userSymbols.findUserBucketsQueryHandler),
+          container.get<UuidService>(coreSymbols.uuidService),
         ),
     );
 

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -12,6 +12,7 @@
   },
   "devDependencies": {
     "@types/node": "20.14.12",
+    "@types/node-cron": "3.0.11",
     "@typescript-eslint/eslint-plugin": "7.8.0",
     "@typescript-eslint/parser": "7.8.0",
     "eslint": "8.56.0",
@@ -42,6 +43,7 @@
   ],
   "packageManager": "npm@10.8.2",
   "dependencies": {
+    "node-cron": "3.0.3",
     "sharp": "0.33.4"
   }
 }