diff --git a/tools/spec-gen-sdk/CHANGELOG.md b/tools/spec-gen-sdk/CHANGELOG.md index d27568382d9..d9a5b3e7ac5 100644 --- a/tools/spec-gen-sdk/CHANGELOG.md +++ b/tools/spec-gen-sdk/CHANGELOG.md @@ -1,5 +1,10 @@ # Release +## 2024-12-19 - 0.1.1 + +- Added saveFilterLog function to save the filtered log +- Introduced loggerWaitToFinish function to ensure log transports complete + ## 2024-12-17 - 0.1.0 - Initial Release diff --git a/tools/spec-gen-sdk/package.json b/tools/spec-gen-sdk/package.json index 542ebdef4eb..02d0f7b968f 100644 --- a/tools/spec-gen-sdk/package.json +++ b/tools/spec-gen-sdk/package.json @@ -5,7 +5,7 @@ "email": "azsdkteam@microsoft.com", "url": "https://github.com/Azure/azure-sdk-tools" }, - "version": "0.1.0", + "version": "0.1.1", "description": "A TypeScript implementation of the API specification to SDK tool", "tags": [ "spec-gen-sdk" diff --git a/tools/spec-gen-sdk/src/automation/entrypoint.ts b/tools/spec-gen-sdk/src/automation/entrypoint.ts index eaab0548911..7faad3e0a94 100644 --- a/tools/spec-gen-sdk/src/automation/entrypoint.ts +++ b/tools/spec-gen-sdk/src/automation/entrypoint.ts @@ -13,10 +13,11 @@ import { loggerDevOpsTransport, loggerFileTransport, loggerTestTransport, + loggerWaitToFinish, sdkAutoLogLevels } from './logging'; import path from 'path'; -import { generateReport } from './reportStatus'; +import { generateReport, saveFilteredLog } from './reportStatus'; import { SpecConfig, SdkRepoConfig, getSpecConfig, specConfigPath } from '../types/SpecConfig'; import { getSwaggerToSdkConfig, SwaggerToSdkConfig } from '../types/SwaggerToSdkConfig'; @@ -37,13 +38,14 @@ interface SdkAutoOptions { headRepoHttpsUrl?: string; headBranch?: string; runEnv: 'local' | 'azureDevOps' | 'test'; + version: string; } export type SdkAutoContext = { config: SdkAutoOptions; logger: winston.Logger; fullLogFileName: string; - filterLogFileName: string; + filteredLogFileName: string; specRepoConfig: SpecConfig; sdkRepoConfig: SdkRepoConfig; swaggerToSdkConfig: SwaggerToSdkConfig @@ -65,12 +67,12 @@ export const getSdkAutoContext = async (options: SdkAutoOptions): Promise { } } if (workflowContext) { - await generateReport(workflowContext); + generateReport(workflowContext); + saveFilteredLog(workflowContext); } + await loggerWaitToFinish(sdkContext.logger); return workflowContext?.status; }; diff --git a/tools/spec-gen-sdk/src/automation/logging.ts b/tools/spec-gen-sdk/src/automation/logging.ts index d516b651009..ecd271182ad 100644 --- a/tools/spec-gen-sdk/src/automation/logging.ts +++ b/tools/spec-gen-sdk/src/automation/logging.ts @@ -1,6 +1,7 @@ import * as winston from 'winston'; import { default as Transport } from 'winston-transport'; import { SDKAutomationState } from './sdkAutomationState'; +import { setTimeout } from 'timers/promises'; export const sdkAutoLogLevels = { levels: { @@ -130,3 +131,15 @@ export const loggerFileTransport = (fileName: string) => { ), }); }; + +export const loggerWaitToFinish = async (logger: winston.Logger) => { + logger.info('Wait for logger transports to complete'); + for (const transport of logger.transports) { + if (transport instanceof winston.transports.File) { + if (transport.end) { + transport.end(); + await setTimeout(2000); + } + } + } +}; diff --git a/tools/spec-gen-sdk/src/automation/reportStatus.ts b/tools/spec-gen-sdk/src/automation/reportStatus.ts index c71eb49030b..74918cb1704 100644 --- a/tools/spec-gen-sdk/src/automation/reportStatus.ts +++ b/tools/spec-gen-sdk/src/automation/reportStatus.ts @@ -17,14 +17,6 @@ const commentLimit = 60; export const generateReport = (context: WorkflowContext) => { context.logger.log('section', 'Generate report'); - /* - const captureTransport = new CommentCaptureTransport({ - extraLevelFilter: ['error', 'warn'], - level: 'debug', - output: context.messages - });*/ - //context.logger.add(captureTransport); - let executionReport: ExecutionReport; const packageReports: PackageReport[] = []; @@ -77,7 +69,7 @@ export const generateReport = (context: WorkflowContext) => { packages: packageReports, executionResult: context.status, fullLogPath: context.fullLogFileName, - filteredLogPath: context.filterLogFileName, + filteredLogPath: context.filteredLogFileName, sdkArtifactFolder: context.sdkArtifactFolder, sdkApiViewArtifactFolder: context.sdkApiViewArtifactFolder }; @@ -92,9 +84,62 @@ export const generateReport = (context: WorkflowContext) => { } context.logger.log('endsection', 'Generate report'); - //context.logger.remove(captureTransport); } +export const saveFilteredLog = async (context: WorkflowContext) => { + context.logger.log('section', 'Save filtered log'); + let hasBreakingChange = false; + let isBetaMgmtSdk = true; + let isDataPlane = true; + let showLiteInstallInstruction = false; + let hasSuppressions = false + let hasAbsentSuppressions = false; + if (context.pendingPackages.length > 0) { + setSdkAutoStatus(context, 'failed'); + setFailureType(context, FailureType.PipelineFrameworkFailed); + context.logger.error(`GenerationError: The following packages are still pending.`); + for (const pkg of context.pendingPackages) { + context.logger.error(`\t${pkg.name}`); + context.handledPackages.push(pkg); + } + } + + for (const pkg of context.handledPackages) { + setSdkAutoStatus(context, pkg.status); + hasBreakingChange = hasBreakingChange || Boolean(pkg.hasBreakingChange); + isBetaMgmtSdk = isBetaMgmtSdk && Boolean(pkg.isBetaMgmtSdk); + isDataPlane = isDataPlane && Boolean(pkg.isDataPlane); + hasSuppressions = hasSuppressions || Boolean(pkg.presentSuppressionLines.length > 0); + hasAbsentSuppressions = hasAbsentSuppressions || Boolean(pkg.absentSuppressionLines.length > 0); + showLiteInstallInstruction = showLiteInstallInstruction || !!pkg.liteInstallationInstruction; + } + + const extra = { hasBreakingChange, showLiteInstallInstruction }; + let commentBody = renderHandlebarTemplate(commentDetailView, context, extra); + const statusMap = { + pending: 'Error', + inProgress: 'Error', + failed: 'Error', + warning: 'Warning', + succeeded: 'Info' + } as const; + const type = statusMap[context.status]; + const filteredResultData = [ + { + type: 'Markdown', + mode: 'replace', + level: type, + message: commentBody, + time: new Date() + } as MessageRecord + ].concat(context.extraResultRecords); + + context.logger.info(`Writing filtered log to ${context.filteredLogFileName}`); + const content = JSON.stringify(filteredResultData); + fs.writeFileSync(context.filteredLogFileName, content); + context.logger.log('endsection', 'Save filtered log status'); +}; + export const sdkAutoReportStatus = async (context: WorkflowContext) => { context.logger.log('section', 'Report status'); diff --git a/tools/spec-gen-sdk/src/automation/workflow.ts b/tools/spec-gen-sdk/src/automation/workflow.ts index 90687490270..ce213a5fdb5 100644 --- a/tools/spec-gen-sdk/src/automation/workflow.ts +++ b/tools/spec-gen-sdk/src/automation/workflow.ts @@ -230,7 +230,7 @@ export const workflowValidateSdkConfig = async (context: SdkAutoContext) => { context.logger.info(`SDK to generate:${context.config.sdkName}`); } else { - throw new Error(`No SDKs are enabled for generation. Please check the configuration in the realted tspconfig.yaml or readme.md`); + throw new Error(`No SDKs are enabled for generation. Please check the configuration in the related tspconfig.yaml or readme.md`); } context.logger.log('endsection', 'Validate SDK configuration'); }; @@ -614,7 +614,7 @@ const workflowCallGenerateScript = async ( const statusContext = { status: 'succeeded' as SDKAutomationState }; let generateOutput: GenerateOutput | undefined = undefined; const generateInput: GenerateInput = { - specFolder: path.relative(context.config.workingFolder, context.specFolder), + specFolder: path.relative(context.sdkFolder, context.specFolder), headSha: context.config.specCommitSha, repoHttpsUrl: context.config.specRepoHttpsUrl ?? "", changedFiles, diff --git a/tools/spec-gen-sdk/src/automation/workflowPackage.ts b/tools/spec-gen-sdk/src/automation/workflowPackage.ts index a2f2085c7da..2bf66b667cb 100644 --- a/tools/spec-gen-sdk/src/automation/workflowPackage.ts +++ b/tools/spec-gen-sdk/src/automation/workflowPackage.ts @@ -6,7 +6,6 @@ import { getInstallInstructionScriptOutput } from '../types/InstallInstructionSc import { PackageData } from '../types/PackageData'; import { deleteTmpJsonFile, readTmpJsonFile, writeTmpJsonFile } from '../utils/fsUtils'; import { isLineMatch, runSdkAutoCustomScript, setSdkAutoStatus } from '../utils/runScript'; -import { CommentCaptureTransport } from './logging'; import { WorkflowContext } from './workflow'; @@ -14,12 +13,6 @@ import { mkdirpSync } from 'fs-extra'; import { getLanguageByRepoName } from './entrypoint'; export const workflowPkgMain = async (context: WorkflowContext, pkg: PackageData) => { - const captureTransport = new CommentCaptureTransport({ - extraLevelFilter: ['error', 'warn'], - level: 'debug', - output: pkg.messages - }); - context.logger.add(captureTransport); context.logger.log('section', `Handle package ${pkg.name}`); context.logger.info(`Package log to a new logFile`); @@ -31,8 +24,6 @@ export const workflowPkgMain = async (context: WorkflowContext, pkg: PackageData await workflowPkgCallInstallInstructionScript(context, pkg); setSdkAutoStatus(pkg, 'succeeded'); - - context.logger.remove(captureTransport); context.logger.log('endsection', `Handle package ${pkg.name}`); }; @@ -66,19 +57,12 @@ const workflowPkgCallChangelogScript = async (context: WorkflowContext, pkg: Pac } } else { context.logger.log('section', 'Call ChangelogScript'); - - const captureTransport = new CommentCaptureTransport({ - extraLevelFilter: ['cmdout', 'cmderr'], - output: pkg.changelogs - }); - context.logger.add(captureTransport); const result = await runSdkAutoCustomScript(context, runOptions, { cwd: context.sdkFolder, fallbackName: 'Changelog', argList: [pkg.relativeFolderPath, ...pkg.extraRelativeFolderPaths], statusContext: pkg }); - context.logger.remove(captureTransport); setSdkAutoStatus(pkg, result); if (result !== 'failed') { diff --git a/tools/spec-gen-sdk/src/cli/cli.ts b/tools/spec-gen-sdk/src/cli/cli.ts index db84dbfb9c2..d9f19b39344 100644 --- a/tools/spec-gen-sdk/src/cli/cli.ts +++ b/tools/spec-gen-sdk/src/cli/cli.ts @@ -27,6 +27,7 @@ export type SpecGenSdkCliConfig = { specRepoHttpsUrl: string; headRepoHttpsUrl?: string; headBranch?: string; + version: string; }; const initCliConfig = (argv) : SpecGenSdkCliConfig => { @@ -45,6 +46,7 @@ const initCliConfig = (argv) : SpecGenSdkCliConfig => { specRepoHttpsUrl: argv.specRepoHttpsUrl, headRepoHttpsUrl: argv.headRepoHttpsUrl, headBranch: argv.headBranch, + version: packageJson.version }; }; @@ -73,7 +75,8 @@ const generateSdk = async (config: SpecGenSdkCliConfig) => { headBranch: config.headBranch, isTriggeredByPipeline: config.isTriggeredByPipeline, runEnv: config.isTriggeredByPipeline ? 'azureDevOps' : 'local', - branchPrefix: 'sdkAuto' + branchPrefix: 'sdkAuto', + version: config.version }); } catch (e) { console.error(e.message); @@ -173,7 +176,7 @@ yargs(hideBin(process.argv)) description: "The branch of the head repository of the specification pull request", }, 'api-version': { - alias: "v", + alias: "apiv", type: "string", description: "The version of the API spec to be used to generate the SDK", } diff --git a/tools/spec-gen-sdk/src/templates/commentDetail.handlebars b/tools/spec-gen-sdk/src/templates/commentDetail.handlebars deleted file mode 100644 index 8beb80a64ac..00000000000 --- a/tools/spec-gen-sdk/src/templates/commentDetail.handlebars +++ /dev/null @@ -1,44 +0,0 @@ -{{#each generation.sdkRepositories}} - -{{/each}} diff --git a/tools/spec-gen-sdk/src/templates/commentDetailNew.handlebars b/tools/spec-gen-sdk/src/templates/commentDetailNew.handlebars index 957f0a12ffe..e1127b4b8a2 100644 --- a/tools/spec-gen-sdk/src/templates/commentDetailNew.handlebars +++ b/tools/spec-gen-sdk/src/templates/commentDetailNew.handlebars @@ -3,7 +3,7 @@
  • {{renderStatus status}}{{#if failureType}}{{failureType}}{{else}}{{renderStatusName status}}{{/if}} -{{#if useMergedRoutine}}Release -{{/if}} in generating from {{specCommitSha}}. {{renderSDKNameMapping config.sdkName}} Automation {{version}}
    + in generating from {{config.specCommitSha}}. spec-gen-sdk {{config.version}} {{#if messages}} {{renderMessagesUnifiedPipeline messages status}} {{/if}} @@ -13,7 +13,7 @@
    {{renderStatus status}}{{name}} {{#if generationPullRequestUrl}} - [{{#if ../useMergedRoutine}}Release{{else}}Preview{{/if}} {{renderSDKNameMapping ../config.sdkName}} Changes] + [Preview {{config.sdkName}} Changes] {{/if}} {{#if (shouldRender hasBreakingChange isBetaMgmtSdk)}} Breaking Change Detected diff --git a/tools/spec-gen-sdk/src/templates/commentSubtitle.handlebars b/tools/spec-gen-sdk/src/templates/commentSubtitle.handlebars deleted file mode 100644 index 51b5936d893..00000000000 --- a/tools/spec-gen-sdk/src/templates/commentSubtitle.handlebars +++ /dev/null @@ -1,23 +0,0 @@ -{{#each generation.sdkRepositories}} -
    • {{#if ../merged}}Release - {{/if}}{{status}} - {{#if ../generation.logsBlobUrl}} [Logs] - {{/if}}
      • - -{{#if isShowInstruction}} - {{#each changedPackages}} -
    • Installation instruction for service {{name}} [Preview Changes]
      - 
      {{liteInstallationInstruction}}
      • - {{/each}} -{{/if}} -
    - -{{#if hasBreakingChange}} - - -```diff -- Breaking Change detected in SDK -``` - - -{{/if}} -{{/each}} diff --git a/tools/spec-gen-sdk/src/templates/generationView.handlebars b/tools/spec-gen-sdk/src/templates/generationView.handlebars deleted file mode 100644 index bc3d29cad6e..00000000000 --- a/tools/spec-gen-sdk/src/templates/generationView.handlebars +++ /dev/null @@ -1,86 +0,0 @@ -

    {{renderSDKTitleMapping sdkRepoName}}{{#if merged}} - Release{{/if}}

    - -
    -{{#if generation.message}} -{{generation.message}} -{{else}} -{{#unless generation.sdkRepositories.length}} -{{renderStatus 'pending'}} Pending... -{{/unless}} -{{/if}} -
    - -{{#if isShowInstruction}} -{{#each generation.sdkRepositories}} - {{#each changedPackages}} -
      -
    • Installation instruction for service {{name}} [Preview Changes]
      - 
      {{liteInstallationInstruction}}
      • -
    - {{/each}} -{{/each}} -{{/if}} - -{{#each generation.sdkRepositories}} -{{#if hasBreakingChange}} -

    - -```diff -- Breaking Change detected in SDK -``` - -

    -{{/if}} -
    - - {{renderStatus status}} {{status}} - {{#if ../generation.logsBlobUrl}} - [Logs] [Expand Details] - {{/if}} - - -
      -
    • -
      - {{renderStatus status}} Generate from {{../headCommit}} with merge commit {{../mergeCommit}}. {{renderSDKNameMapping ../sdkRepoName}} Automation {{../version}} -
      - {{#if messages}} - {{renderMessages messages}} - {{/if}} -
      • - {{#each changedPackages}} -
    • -
      - {{renderStatus status}} {{name}} - {{#if logsBlobUrl}} - [View full logs]  - {{/if}} - {{#if generationPullRequestUrl}} - [{{#if ../../merged}}Release{{else}}Preview{{/if}} {{renderSDKNameMapping ../../sdkRepoName}} Changes] - {{/if}} - {{#if hasBreakingChange}} - Breaking Change Detected - {{/if}} - -
      - {{#if (isPublicRelease this)}} - - {{/if}} - {{#if messages}} - {{renderMessages messages}} - {{/if}} - {{#unless messages}}{{#unless artifactBlobUrls}} -
      No Artifact Generated.
      - {{/unless}}{{/unless}} -
      • - {{/each}} -
    - -
    -{{/each}}