Merge pull request #52 from snyk-tech-services/fix/log-current-batches

feat: log batches to file for progress
snyk · Jun 16, 2020 · 925aa23 · 925aa23
2 parents d89fe18 + f04414c
commit 925aa23
Show file tree

Hide file tree

Showing 5 changed files with 51 additions and 14 deletions.
diff --git a/src/common.ts b/src/common.ts
@@ -5,3 +5,4 @@ export const FAILED_PROJECTS_LOG_NAME = 'failed-projects.log';
 export const FAILED_POLLS_LOG_NAME = 'failed-polls.log';
 export const IMPORT_JOBS_LOG_NAME = 'import-jobs.log';
 export const IMPORTED_PROJECTS_LOG_NAME = 'imported-projects.log';
+export const IMPORTED_BATCHES_LOG_NAME = 'imported-batches.log';
diff --git a/src/log-imported-batch.ts b/src/log-imported-batch.ts
@@ -0,0 +1,15 @@
+import * as fs from 'fs';
+import { getLoggingPath } from './lib/get-logging-path';
+import { IMPORTED_BATCHES_LOG_NAME } from './common';
+
+export async function logImportedBatch(
+  message: string,
+  loggingPath: string = getLoggingPath(),
+): Promise<void> {
+  try {
+    const dateNow = new Date(Date.now());
+    fs.appendFileSync(`${loggingPath}/${IMPORTED_BATCHES_LOG_NAME}`, `${dateNow.toUTCString()} - ${message}\n`);
+  } catch (e) {
+    // do nothing
+  }
+}
diff --git a/src/scripts/import-projects.ts b/src/scripts/import-projects.ts
@@ -1,13 +1,11 @@
 import * as debugLib from 'debug';
 import * as path from 'path';
 import { loadFile } from '../load-file';
-import {
-  importTargets,
-  pollImportUrls,
-} from '../lib';
+import { importTargets, pollImportUrls } from '../lib';
 import { Project, ImportTarget } from '../lib/types';
 import { getLoggingPath } from '../lib/get-logging-path';
 import { getConcurrentImportsNumber } from '../lib/get-concurrent-imports-number';
+import { logImportedBatch } from '../log-imported-batch';
 
 const debug = debugLib('snyk:import-projects-script');
 
@@ -28,16 +26,21 @@ async function filterOutImportedTargets(
   targets.forEach((targetItem) => {
     const { orgId, integrationId, target } = targetItem;
     try {
-      const data = `${orgId}:${integrationId}:${Object.values(target).join(':')}`;
+      const data = `${orgId}:${integrationId}:${Object.values(target).join(
+        ':',
+      )}`;
       const targetRegExp = regexForTarget(data);
       const match = logFile.match(targetRegExp);
       if (!match) {
         filterOutImportedTargets.push(targetItem);
       } else {
-        debug('Dropped previously imported target: ', JSON.stringify(targetItem));
+        debug(
+          'Dropped previously imported target: ',
+          JSON.stringify(targetItem),
+        );
       }
     } catch (e) {
-      debug('failed to process target', JSON.stringify(targetItem))
+      debug('failed to process target', JSON.stringify(targetItem));
     }
   });
 
@@ -64,7 +67,11 @@ export async function ImportProjects(
     return [];
   }
   const skippedTargets = targets.length - filteredTargets.length;
-  debug(`Skipped previously imported ${skippedTargets}/${targets.length} targets`)
+  if (skippedTargets > 0) {
+    debug(
+      `Skipped previously imported ${skippedTargets}/${targets.length} targets`,
+    );
+  }
   for (
     let targetIndex = 0;
     targetIndex < filteredTargets.length;
@@ -74,11 +81,16 @@ export async function ImportProjects(
       targetIndex,
       targetIndex + concurrentTargets,
     );
-    const currentTargets = skippedTargets + targetIndex;
-    debug(
-      `Importing batch ${currentTargets} - ${currentTargets +
-        concurrentTargets} out of ${filteredTargets.length}`,
-    );
+    const currentTargets = skippedTargets + targetIndex + 1;
+    let currentBatchEnd = currentTargets + concurrentTargets - 1;
+    if (currentBatchEnd > filteredTargets.length) {
+      currentBatchEnd = currentTargets;
+    }
+    const batchProgressMessages = `Importing batch ${currentTargets} - ${currentBatchEnd} out of ${
+      filteredTargets.length
+    } ${skippedTargets > 0 ? `(skipped ${skippedTargets})` : ''}`;
+    debug(batchProgressMessages);
+    logImportedBatch(batchProgressMessages);
     const pollingUrlsAndContext = await importTargets(batch, loggingPath);
     projects.push(...(await pollImportUrls(pollingUrlsAndContext)));
   }

diff --git a/test/generate-log-file-names.ts b/test/generate-log-file-names.ts
@@ -6,6 +6,7 @@ import {
   FAILED_PROJECTS_LOG_NAME,
   IMPORT_JOBS_LOG_NAME,
   IMPORTED_PROJECTS_LOG_NAME,
+  IMPORTED_BATCHES_LOG_NAME,
 } from '../src/common';
 
 export function generateLogsPaths(
@@ -17,19 +18,22 @@ export function generateLogsPaths(
   failedProjectsLogPath: string;
   importJobIdsLogsPath: string;
   importedProjectsLogPath: string;
+  importedBatchesLogPath: string;
 } {
   process.env.SNYK_LOG_PATH = logPath;
-  const importLogPath = path.resolve(logPath, `${IMPORT_LOG_NAME}`);
+  const importLogPath = path.resolve(logPath, IMPORT_LOG_NAME);
   const failedImportLogPath = path.resolve(logPath, `${orgId}.${FAILED_LOG_NAME}`);
   const failedProjectsLogPath = path.resolve(logPath, `${orgId}.${FAILED_PROJECTS_LOG_NAME}`);
   const importJobIdsLogsPath = path.resolve(logPath, `${orgId}.${IMPORT_JOBS_LOG_NAME}`);
   const importedProjectsLogPath = path.resolve(logPath, `${orgId}.${IMPORTED_PROJECTS_LOG_NAME}`);
+  const importedBatchesLogPath = path.resolve(logPath, IMPORTED_BATCHES_LOG_NAME);
 
   return {
     importLogPath,
     failedImportLogPath,
     failedProjectsLogPath,
     importJobIdsLogsPath,
     importedProjectsLogPath,
+    importedBatchesLogPath,
   };
 }
diff --git a/test/scripts/import-projects.test.ts b/test/scripts/import-projects.test.ts
@@ -141,6 +141,11 @@ describe('Skips & logs issues', () => {
     );
     const logFile = fs.readFileSync(logFiles.importLogPath, 'utf8');
     expect(logFile).not.toBeNull();
+    const batchesLogFile = fs.readFileSync(
+      logFiles.importedBatchesLogPath,
+      'utf8',
+    );
+    expect(batchesLogFile).not.toBeNull();
     const failedProjectsLog = fs.readFileSync(
       logFiles.failedProjectsLogPath,
       'utf-8',