Log improvements

tools/spec-gen-sdk/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Release
 
+## 2024-12-19 - 0.1.1
+
+- Added saveFilterLog function to save the filtered log
+- Introduced loggerWaitToFinish function to ensure log transports complete
+
 ## 2024-12-17 - 0.1.0
 
 - Initial Release

tools/spec-gen-sdk/package.json

@@ -5,7 +5,7 @@
     "email": "azsdkteam@microsoft.com",
     "url": "https://github.com/Azure/azure-sdk-tools"
   },
-  "version": "0.1.0",
+  "version": "0.1.1",
   "description": "A TypeScript implementation of the API specification to SDK tool",
   "tags": [
     "spec-gen-sdk"

tools/spec-gen-sdk/src/automation/entrypoint.ts

@@ -13,10 +13,11 @@ import {
   loggerDevOpsTransport,
   loggerFileTransport,
   loggerTestTransport,
+  loggerWaitToFinish,
   sdkAutoLogLevels
 } from './logging';
 import path from 'path';
-import { generateReport } from './reportStatus';
+import { generateReport, saveFilteredLog } from './reportStatus';
 import { SpecConfig, SdkRepoConfig, getSpecConfig, specConfigPath } from '../types/SpecConfig';
 import { getSwaggerToSdkConfig, SwaggerToSdkConfig } from '../types/SwaggerToSdkConfig';
 
@@ -37,13 +38,14 @@ interface SdkAutoOptions {
   headRepoHttpsUrl?: string;
   headBranch?: string;
   runEnv: 'local' | 'azureDevOps' | 'test';
+  version: string;
 }
 
 export type SdkAutoContext = {
   config: SdkAutoOptions;
   logger: winston.Logger;
   fullLogFileName: string;
-  filterLogFileName: string;
+  filteredLogFileName: string;
   specRepoConfig: SpecConfig;
   sdkRepoConfig: SdkRepoConfig;
   swaggerToSdkConfig: SwaggerToSdkConfig
@@ -65,12 +67,12 @@ export const getSdkAutoContext = async (options: SdkAutoOptions): Promise<SdkAut
   }
 
   const fullLogFileName = path.join(options.workingFolder, 'full.log');
-  const filterLogFileName = path.join(options.workingFolder, 'filter.log');
+  const filteredLogFileName = path.join(options.workingFolder, 'filtered.log');
   if (fs.existsSync(fullLogFileName)) {
     fs.rmSync(fullLogFileName);
   }
-  if (fs.existsSync(filterLogFileName)) {
-    fs.rmSync(filterLogFileName);
+  if (fs.existsSync(filteredLogFileName)) {
+    fs.rmSync(filteredLogFileName);
   }
   logger.add(loggerFileTransport(fullLogFileName));
   logger.info(`Log to ${fullLogFileName}`);
@@ -87,7 +89,7 @@ export const getSdkAutoContext = async (options: SdkAutoOptions): Promise<SdkAut
     config: options,
     logger,
     fullLogFileName,
-    filterLogFileName,
+    filteredLogFileName,
     specRepoConfig,
     sdkRepoConfig,
     swaggerToSdkConfig,
@@ -114,8 +116,10 @@ export const sdkAutoMain = async (options: SdkAutoOptions) => {
     }
   }
   if (workflowContext) {
-    await generateReport(workflowContext);
+    generateReport(workflowContext);
+    saveFilteredLog(workflowContext);
   }
+  await loggerWaitToFinish(sdkContext.logger);
   return workflowContext?.status;
 };
 

tools/spec-gen-sdk/src/automation/logging.ts

@@ -1,6 +1,7 @@
 import * as winston from 'winston';
 import { default as Transport } from 'winston-transport';
 import { SDKAutomationState } from './sdkAutomationState';
+import { setTimeout } from 'timers/promises';
 
 export const sdkAutoLogLevels = {
   levels: {
@@ -130,3 +131,15 @@ export const loggerFileTransport = (fileName: string) => {
     ),
   });
 };
+
+export const loggerWaitToFinish = async (logger: winston.Logger) => {
+  logger.info('Wait for logger transports to complete');
+  for (const transport of logger.transports) {
+    if (transport instanceof winston.transports.File) {
+      if (transport.end) {
+          transport.end();
+          await setTimeout(2000);
+        }
+    }
+  }
+};

tools/spec-gen-sdk/src/automation/reportStatus.ts

@@ -17,14 +17,6 @@ const commentLimit = 60;
 
 export const generateReport = (context: WorkflowContext) => {
   context.logger.log('section', 'Generate report');
-  /*
-  const captureTransport = new CommentCaptureTransport({
-    extraLevelFilter: ['error', 'warn'],
-    level: 'debug',
-    output: context.messages
-  });*/
-  //context.logger.add(captureTransport);
-
   let executionReport: ExecutionReport;
   const packageReports: PackageReport[] = [];
 
@@ -77,7 +69,7 @@ export const generateReport = (context: WorkflowContext) => {
     packages: packageReports,
     executionResult: context.status,
     fullLogPath: context.fullLogFileName,
-    filteredLogPath: context.filterLogFileName,
+    filteredLogPath: context.filteredLogFileName,
     sdkArtifactFolder: context.sdkArtifactFolder,
     sdkApiViewArtifactFolder: context.sdkApiViewArtifactFolder
   };
@@ -92,9 +84,62 @@ export const generateReport = (context: WorkflowContext) => {
   }
 
   context.logger.log('endsection', 'Generate report');
-  //context.logger.remove(captureTransport);
 }
 
+export const saveFilteredLog = async (context: WorkflowContext) => {
+  context.logger.log('section', 'Save filtered log');
+  let hasBreakingChange = false;
+  let isBetaMgmtSdk = true;
+  let isDataPlane = true;
+  let showLiteInstallInstruction = false;
+  let hasSuppressions = false
+  let hasAbsentSuppressions = false;
+  if (context.pendingPackages.length > 0) {
+    setSdkAutoStatus(context, 'failed');
+    setFailureType(context, FailureType.PipelineFrameworkFailed);
+    context.logger.error(`GenerationError: The following packages are still pending.`);
+    for (const pkg of context.pendingPackages) {
+      context.logger.error(`\t${pkg.name}`);
+      context.handledPackages.push(pkg);
+    }
+  }
+
+  for (const pkg of context.handledPackages) {
+    setSdkAutoStatus(context, pkg.status);
+    hasBreakingChange = hasBreakingChange || Boolean(pkg.hasBreakingChange);
+    isBetaMgmtSdk = isBetaMgmtSdk && Boolean(pkg.isBetaMgmtSdk);
+    isDataPlane = isDataPlane && Boolean(pkg.isDataPlane);
+    hasSuppressions = hasSuppressions || Boolean(pkg.presentSuppressionLines.length > 0);
+    hasAbsentSuppressions = hasAbsentSuppressions || Boolean(pkg.absentSuppressionLines.length > 0);
+    showLiteInstallInstruction = showLiteInstallInstruction || !!pkg.liteInstallationInstruction;
+  }
+
+  const extra = { hasBreakingChange, showLiteInstallInstruction };
+  let commentBody = renderHandlebarTemplate(commentDetailView, context, extra);
+  const statusMap = {
+    pending: 'Error',
+    inProgress: 'Error',
+    failed: 'Error',
+    warning: 'Warning',
+    succeeded: 'Info'
+  } as const;
+  const type = statusMap[context.status];
+  const filteredResultData = [
+    {
+      type: 'Markdown',
+      mode: 'replace',
+      level: type,
+      message: commentBody,
+      time: new Date()
+    } as MessageRecord
+  ].concat(context.extraResultRecords);
+
+  context.logger.info(`Writing filtered log to ${context.filteredLogFileName}`);
+  const content = JSON.stringify(filteredResultData);
+  fs.writeFileSync(context.filteredLogFileName, content);
+  context.logger.log('endsection', 'Save filtered log status');
+};
+
 export const sdkAutoReportStatus = async (context: WorkflowContext) => {
   context.logger.log('section', 'Report status');
 

tools/spec-gen-sdk/src/automation/workflow.ts

@@ -230,7 +230,7 @@ export const workflowValidateSdkConfig = async (context: SdkAutoContext) => {
     context.logger.info(`SDK to generate:${context.config.sdkName}`);
   }
   else {
-    throw new Error(`No SDKs are enabled for generation. Please check the configuration in the realted tspconfig.yaml or readme.md`);
+    throw new Error(`No SDKs are enabled for generation. Please check the configuration in the related tspconfig.yaml or readme.md`);
   }
   context.logger.log('endsection', 'Validate SDK configuration');
 };
@@ -614,7 +614,7 @@ const workflowCallGenerateScript = async (
   const statusContext = { status: 'succeeded' as SDKAutomationState };
   let generateOutput: GenerateOutput | undefined = undefined;
   const generateInput: GenerateInput = {
-    specFolder: path.relative(context.config.workingFolder, context.specFolder),
+    specFolder: path.relative(context.sdkFolder, context.specFolder),
     headSha: context.config.specCommitSha,
     repoHttpsUrl: context.config.specRepoHttpsUrl ?? "",
     changedFiles,

tools/spec-gen-sdk/src/automation/workflowPackage.ts

@@ -6,20 +6,13 @@ import { getInstallInstructionScriptOutput } from '../types/InstallInstructionSc
 import { PackageData } from '../types/PackageData';
 import { deleteTmpJsonFile, readTmpJsonFile, writeTmpJsonFile } from '../utils/fsUtils';
 import { isLineMatch, runSdkAutoCustomScript, setSdkAutoStatus } from '../utils/runScript';
-import { CommentCaptureTransport } from './logging';
 import {
   WorkflowContext
 } from './workflow';
 import { mkdirpSync } from 'fs-extra';
 import { getLanguageByRepoName } from './entrypoint';
 
 export const workflowPkgMain = async (context: WorkflowContext, pkg: PackageData) => {
-  const captureTransport = new CommentCaptureTransport({
-    extraLevelFilter: ['error', 'warn'],
-    level: 'debug',
-    output: pkg.messages
-  });
-  context.logger.add(captureTransport);
   context.logger.log('section', `Handle package ${pkg.name}`);
   context.logger.info(`Package log to a new logFile`);
 
@@ -31,8 +24,6 @@ export const workflowPkgMain = async (context: WorkflowContext, pkg: PackageData
   await workflowPkgCallInstallInstructionScript(context, pkg);
 
   setSdkAutoStatus(pkg, 'succeeded');
-
-  context.logger.remove(captureTransport);
   context.logger.log('endsection', `Handle package ${pkg.name}`);
 };
 
@@ -66,19 +57,12 @@ const workflowPkgCallChangelogScript = async (context: WorkflowContext, pkg: Pac
     }
   } else {
     context.logger.log('section', 'Call ChangelogScript');
-
-    const captureTransport = new CommentCaptureTransport({
-      extraLevelFilter: ['cmdout', 'cmderr'],
-      output: pkg.changelogs
-    });
-    context.logger.add(captureTransport);
     const result = await runSdkAutoCustomScript(context, runOptions, {
       cwd: context.sdkFolder,
       fallbackName: 'Changelog',
       argList: [pkg.relativeFolderPath, ...pkg.extraRelativeFolderPaths],
       statusContext: pkg
     });
-    context.logger.remove(captureTransport);
 
     setSdkAutoStatus(pkg, result);
     if (result !== 'failed') {

tools/spec-gen-sdk/src/cli/cli.ts

@@ -27,6 +27,7 @@ export type SpecGenSdkCliConfig = {
   specRepoHttpsUrl: string;
   headRepoHttpsUrl?: string;
   headBranch?: string;
+  version: string;
 };
 
 const initCliConfig = (argv) : SpecGenSdkCliConfig => {
@@ -45,6 +46,7 @@ const initCliConfig = (argv) : SpecGenSdkCliConfig => {
     specRepoHttpsUrl: argv.specRepoHttpsUrl,
     headRepoHttpsUrl: argv.headRepoHttpsUrl,
     headBranch: argv.headBranch,
+    version: packageJson.version
   };
 };
 
@@ -73,7 +75,8 @@ const generateSdk = async (config: SpecGenSdkCliConfig) => {
       headBranch: config.headBranch,
       isTriggeredByPipeline: config.isTriggeredByPipeline,
       runEnv: config.isTriggeredByPipeline ? 'azureDevOps' : 'local',
-      branchPrefix: 'sdkAuto'
+      branchPrefix: 'sdkAuto',
+      version: config.version
     });
   } catch (e) {
     console.error(e.message);
@@ -173,7 +176,7 @@ yargs(hideBin(process.argv))
           description: "The branch of the head repository of the specification pull request",
         },
         'api-version': {
-          alias: "v",
+          alias: "apiv",
           type: "string",
           description: "The version of the API spec to be used to generate the SDK",
         }

tools/spec-gen-sdk/src/templates/commentDetailNew.handlebars

@@ -3,7 +3,7 @@
   <li>
     <div>
 {{renderStatus status}}{{#if failureType}}{{failureType}}{{else}}{{renderStatusName status}}{{/if}}
-{{#if useMergedRoutine}}Release -{{/if}} in generating from {{specCommitSha}}. {{renderSDKNameMapping config.sdkName}} Automation {{version}} </div>
+ in generating from {{config.specCommitSha}}. spec-gen-sdk {{config.version}} </div>
     {{#if messages}}
       {{renderMessagesUnifiedPipeline messages status}}
     {{/if}}
@@ -13,7 +13,7 @@
     <div>
       <span>{{renderStatus status}}<b>{{name}}</b>
         {{#if generationPullRequestUrl}}
-        [<a href="{{generationPullRequestUrl}}">{{#if ../useMergedRoutine}}Release{{else}}Preview{{/if}} {{renderSDKNameMapping ../config.sdkName}} Changes</a>]
+        [<a href="{{generationPullRequestUrl}}">Preview {{config.sdkName}} Changes</a>]
         {{/if}}
         {{#if (shouldRender hasBreakingChange isBetaMgmtSdk)}}
         <b>Breaking Change Detected</b>