diff --git a/.github/workflows/SDK-Suppressions-Label.yaml b/.github/workflows/SDK-Suppressions-Label.yaml new file mode 100644 index 000000000000..20db6347056f --- /dev/null +++ b/.github/workflows/SDK-Suppressions-Label.yaml @@ -0,0 +1,104 @@ +name: SDK Suppressions + +on: + pull_request: + branches: + - main + - RPSaaSMaster + - release* + +jobs: + process-sdk-suppressions-labels: + name: Sdk Suppressions + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + # Required since "HEAD^" is passed to Get-ChangedFiles + fetch-depth: 2 + + - name: Setup Node and run `npm ci` + uses: ./.github/actions/setup-node-npm-ci + + - name: Get GitHub PullRequest Changed Files + shell: pwsh + id: get-changedFiles + run: | + . eng/scripts/ChangedFiles-Functions.ps1 + $changedFiles = @(Get-ChangedFiles) + echo "PR Changed files: $changedFiles" + Add-Content -Path $env:GITHUB_OUTPUT -Value "changedFiles=$changedFiles" + + - name: Get GitHub PullRequest Context + uses: actions/github-script@v7 + id: fetch-pullRequest-context + with: + script: | + const pr = context.payload.pull_request; + if (!pr) { + throw new Error("This workflow must run in the context of a pull request."); + } + console.log("This action trigger by ", context.eventName); + core.setOutput("prLabels", pr.labels.map(label => label.name)); + result-encoding: string + + - name: Run Get suppressions label script + id: run-suppressions-script + env: + OUTPUT_FILE: "output.json" + GITHUB_PULL_REQUEST_CHANGE_FILES: ${{ steps.get-changedFiles.outputs.changedFiles }} + GITHUB_PULL_REQUEST_LABELS: ${{ steps.fetch-pullRequest-context.outputs.prLabels }} + run: | + node eng/tools/sdk-suppressions/cmd/sdk-suppressions-label.js HEAD^ HEAD "$GITHUB_PULL_REQUEST_CHANGE_FILES" "$GITHUB_PULL_REQUEST_LABELS" + + OUTPUT=$(cat $OUTPUT_FILE) + echo "Script output labels: $OUTPUT" + + labelsToAdd=$(echo "$OUTPUT" | sed -n 's/.*"labelsToAdd":\[\([^]]*\)\].*/\1/p' | tr -d '" ') + labelsToRemove=$(echo "$OUTPUT" | sed -n 's/.*"labelsToRemove":\[\([^]]*\)\].*/\1/p' | tr -d '" ') + + for label in $(echo $labelsToAdd | tr ',' '\n'); do + echo "Label to add: $label" + echo "$label=true" >> $GITHUB_OUTPUT + done + + for label in $(echo $labelsToRemove | tr ',' '\n'); do + echo "Label to remove: $label" + echo "$label=false" >> $GITHUB_OUTPUT + done + + # No Action or Add/Remove label ​​according to step run-suppressions-script output + # e.g. + # If the output of the step does not include the BreakingChange-Go-Sdk-Suppression, no action will be taken. + # If the step's output is "BreakingChange-Go-Sdk-Suppression='true'", the label "BreakingChange-Go-Sdk-Suppression" will be applied to the PR. + # If the step's output is "BreakingChange-Go-Sdk-Suppression='false'", the label "BreakingChange-Go-Sdk-Suppression" will be removed from the PR. + - uses: ./.github/actions/add-label-artifact + name: Upload artifact with results-go + if: ${{ steps.run-suppressions-script.outputs.BreakingChange-Go-Sdk-Suppression }} + with: + name: "BreakingChange-Go-Sdk-Suppression" + value: "${{ steps.run-suppressions-script.outputs.BreakingChange-Go-Sdk-Suppression == 'true' }}" + + - uses: ./.github/actions/add-label-artifact + name: Upload artifact with results java + if: ${{ steps.run-suppressions-script.outputs.BreakingChange-Java-Sdk-Suppression }} + with: + name: "BreakingChange-Java-Sdk-Suppression" + value: "${{ steps.run-suppressions-script.outputs.BreakingChange-Java-Sdk-Suppression == 'true' }}" + + - uses: ./.github/actions/add-label-artifact + name: Upload artifact with results js + if: ${{ steps.run-suppressions-script.outputs.BreakingChange-JavaScript-Sdk-Suppression }} + with: + name: "BreakingChange-JavaScript-Sdk-Suppression" + value: "${{ steps.run-suppressions-script.outputs.BreakingChange-JavaScript-Sdk-Suppression == 'true' }}" + + - uses: ./.github/actions/add-label-artifact + name: Upload artifact with results python + if: ${{ steps.run-suppressions-script.outputs.BreakingChange-Python-Sdk-Suppression }} + with: + name: "BreakingChange-Python-Sdk-Suppression" + value: "${{ steps.run-suppressions-script.outputs.BreakingChange-Python-Sdk-Suppression == 'true' }}" diff --git a/.github/workflows/update-labels.yaml b/.github/workflows/update-labels.yaml index d1663f57ae8c..5fd9c8d151d7 100644 --- a/.github/workflows/update-labels.yaml +++ b/.github/workflows/update-labels.yaml @@ -6,7 +6,7 @@ on: # types: [labeled, unlabeled] # If an upstream workflow if completed, get only the artifacts from that workflow, and update labels workflow_run: - workflows: ["TypeSpec Requirement"] + workflows: ["TypeSpec Requirement", "SDK Suppressions"] types: [completed] workflow_dispatch: inputs: diff --git a/eng/tools/package.json b/eng/tools/package.json index 8dcd0c6a9d2d..1385766802eb 100644 --- a/eng/tools/package.json +++ b/eng/tools/package.json @@ -5,7 +5,8 @@ "@azure-tools/suppressions": "file:suppressions", "@azure-tools/tsp-client-tests": "file:tsp-client-tests", "@azure-tools/typespec-requirement": "file:typespec-requirement", - "@azure-tools/typespec-validation": "file:typespec-validation" + "@azure-tools/typespec-validation": "file:typespec-validation", + "@azure-tools/sdk-suppressions": "file:sdk-suppressions" }, "scripts": { "build": "tsc --build", diff --git a/eng/tools/sdk-suppressions/cmd/sdk-suppressions-label.js b/eng/tools/sdk-suppressions/cmd/sdk-suppressions-label.js new file mode 100755 index 000000000000..e2a37e0b5491 --- /dev/null +++ b/eng/tools/sdk-suppressions/cmd/sdk-suppressions-label.js @@ -0,0 +1,5 @@ +#!/usr/bin/env node + +import { main } from "../dist/src/index.js"; + +await main(); diff --git a/eng/tools/sdk-suppressions/package.json b/eng/tools/sdk-suppressions/package.json new file mode 100644 index 000000000000..4bb3985cc706 --- /dev/null +++ b/eng/tools/sdk-suppressions/package.json @@ -0,0 +1,30 @@ +{ + "name": "@azure-tools/sdk-suppressions", + "private": true, + "type": "module", + "main": "dist/src/index.js", + "version": "1.0.0", + "bin": { + "get-sdk-suppressions-label": "cmd/sdk-suppressions-label.js" + }, + "scripts": { + "build": "tsc --build", + "test": "vitest", + "test:ci": "vitest run --coverage --reporter=verbose" + }, + "engines": { + "node": ">= 18.0.0" + }, + "dependencies": { + "ajv": "^8.17.1", + "lodash": "^4.17.20", + "yaml": "^2.4.2" + }, + "devDependencies": { + "@types/lodash": "^4.14.161", + "@types/node": "^18.19.31", + "@vitest/coverage-v8": "^2.0.4", + "typescript": "~5.6.2", + "vitest": "^2.0.4" + } +} diff --git a/eng/tools/sdk-suppressions/src/common.ts b/eng/tools/sdk-suppressions/src/common.ts new file mode 100644 index 000000000000..b13781e4e20c --- /dev/null +++ b/eng/tools/sdk-suppressions/src/common.ts @@ -0,0 +1,63 @@ +import { parse as yamlParse } from "yaml"; + +import { exec } from "child_process"; +import { promisify } from "util"; + +/** + * @param yamlContent + * @returns {result: string | object | undefined | null, message: string} + * special return + * if the content is empty, return {result: null, message: string + * if the file parse error, return {result: undefined, message: string + */ +export function parseYamlContent(yamlContent: string, path: string): { + result: string | object | undefined | null; + message: string; +}{ + let content = undefined; + // if yaml file is not a valid yaml, catch error and return undefined + try { + content = yamlParse(yamlContent); + } catch (error) { + console.error(`The file parsing failed in the ${path}. Details: ${error}`); + return { + result: content, + message: `The file parsing failed in the ${path}. Details: ${error}` + };; + } + + // if yaml file is empty, run yaml.safeload success but get undefined + // to identify whether it is empty return null to distinguish. + if (!content) { + console.info(`The file in the ${path} has been successfully parsed, but it is an empty file.`) + return { + result: null, + message: `The file in the ${path} has been successfully parsed, but it is an empty file.` + };; + } + + return { + result: content, + message: 'The file has been successfully parsed.' + }; + +} + +// Promisify the exec function +const execAsync = promisify(exec); + +export async function runGitCommand(command: string): Promise { + try { + const { stdout, stderr } = await execAsync(command); + + if (stderr) { + console.error("Error Output:", stderr); + // throw new Error(stderr); + } + + return stdout.trim(); + } catch (error:any) { + console.error("Error details:", error.stderr || error); + throw error; + } +} diff --git a/eng/tools/sdk-suppressions/src/index.ts b/eng/tools/sdk-suppressions/src/index.ts new file mode 100644 index 000000000000..5100ec2104d8 --- /dev/null +++ b/eng/tools/sdk-suppressions/src/index.ts @@ -0,0 +1,39 @@ + +import { exit } from "process"; +import { updateSdkSuppressionsLabels } from "./updateSdkSuppressionsLabel.js"; + +function getArgsError(args: string[]): string { + return ( + "Get args lengths: " + args.length + "\n" + + "Details: " + args.join(', ') + "\n" + + "Usage: node eng/tools/sdk-suppressions/cmd/sdk-suppressions-label.js baseCommitHash headCommitHash changeFiles prLabels\n" + + "Returns: {labelsToAdd: [label1, label2],labelsToRemove: [lable3, label4]}\n" + + "Parameters:\n" + + " baseCommitHash: The base commit hash. Example: HEAD^ \n" + + " headCommitHash: The head commit hash. Example: HEAD \n" + + " changeFiles: The changed files. Example: 'specification/workloads/Workloads.Operations.Management/sdk-suppressions.yaml specification/workloads/Workloads.Operations.Management/main.tsp'\n" + + " prLabels: The PR has added labels. Example: '['BreakingChange-Go-Sdk-Suppression', 'BreakingChange-Python-Sdk-Suppression']'\n" + ); +} + +export async function main() { + const args: string[] = process.argv.slice(2); + if (args.length === 4) { + const baseCommitHash: string = args[0]; + const headCommitHash: string = args[1]; + const changeFiles: string = args[2]; + const lables: string = args[3]; + const outputFile = process.env.OUTPUT_FILE as string; + const changedLabels: {labelsToAdd: String[], labelsToRemove: String[]} = await updateSdkSuppressionsLabels(lables, changeFiles, baseCommitHash, headCommitHash, outputFile); + console.log(JSON.stringify(changedLabels)); + exit(0); + } else { + console.error(getArgsError(args)); + exit(1); + } + +} + +export { updateSdkSuppressionsLabels }; + + diff --git a/eng/tools/sdk-suppressions/src/sdk.ts b/eng/tools/sdk-suppressions/src/sdk.ts new file mode 100644 index 000000000000..8d1e3ec2452d --- /dev/null +++ b/eng/tools/sdk-suppressions/src/sdk.ts @@ -0,0 +1,59 @@ +/** + * This file is the single source of truth for the labels used by the SDK generation tooling + * in the Azure/azure-rest-api-specs and Azure/azure-rest-api-specs-pr repositories. + * + * For additional context, see: + * - https://gist.github.com/raych1/353949d19371b69fb82a10dd70032a51 + * - https://github.com/Azure/azure-sdk-tools/issues/6327 + * - https://microsoftapc-my.sharepoint.com/:w:/g/personal/raychen_microsoft_com/EbOAA9SkhQhGlgxtf7mc0kUB-25bFue0EFbXKXS3TFLTQA + */ +export type SdkName = + | "azure-sdk-for-go" + | "azure-sdk-for-java" + | "azure-sdk-for-js" + | "azure-sdk-for-net" + | "azure-sdk-for-python" + +export const sdkLabels: { + [sdkName in SdkName]: { + breakingChange: string | undefined; + breakingChangeApproved: string | undefined; + breakingChangeSuppression: string | undefined; + breakingChangeSuppressionApproved: string | undefined; + }; +} = { + "azure-sdk-for-go": { + breakingChange: "BreakingChange-Go-Sdk", + breakingChangeApproved: "BreakingChange-Go-Sdk-Approved", + breakingChangeSuppression: "BreakingChange-Go-Sdk-Suppression", + breakingChangeSuppressionApproved: + "BreakingChange-Go-Sdk-Suppression-Approved", + }, + "azure-sdk-for-java": { + breakingChange: "BreakingChange-Java-Sdk", + breakingChangeApproved: "BreakingChange-Java-Sdk-Approved", + breakingChangeSuppression: "BreakingChange-Java-Sdk-Suppression", + breakingChangeSuppressionApproved: + "BreakingChange-Java-Sdk-Suppression-Approved" + }, + "azure-sdk-for-js": { + breakingChange: "BreakingChange-JavaScript-Sdk", + breakingChangeApproved: "BreakingChange-JavaScript-Sdk-Approved", + breakingChangeSuppression: "BreakingChange-JavaScript-Sdk-Suppression", + breakingChangeSuppressionApproved: + "BreakingChange-JavaScript-Sdk-Suppression-Approved" + }, + "azure-sdk-for-net": { + breakingChange: undefined, + breakingChangeApproved: undefined, + breakingChangeSuppression: undefined, + breakingChangeSuppressionApproved: undefined + }, + "azure-sdk-for-python": { + breakingChange: "BreakingChange-Python-Sdk", + breakingChangeApproved: "BreakingChange-Python-Sdk-Approved", + breakingChangeSuppression: "BreakingChange-Python-Sdk-Suppression", + breakingChangeSuppressionApproved: + "BreakingChange-Python-Sdk-Suppression-Approved" + } +}; diff --git a/eng/tools/sdk-suppressions/src/sdkSuppressions.ts b/eng/tools/sdk-suppressions/src/sdkSuppressions.ts new file mode 100644 index 000000000000..3a379bba78d2 --- /dev/null +++ b/eng/tools/sdk-suppressions/src/sdkSuppressions.ts @@ -0,0 +1,85 @@ +/** + * This file contains types for the contents of the SDK suppressions file, sdk-suppressions.yml. + * For details, see: + * - https://microsoftapc-my.sharepoint.com/:w:/g/personal/raychen_microsoft_com/EbOAA9SkhQhGlgxtf7mc0kUB-25bFue0EFbXKXS3TFLTQA + */ + +import { Ajv } from "ajv"; +import { SdkName, sdkLabels } from "./sdk.js"; + +export const sdkSuppressionsFileName = "sdk-suppressions.yaml"; + +export type SdkSuppressionsYml = { + suppressions: SdkSuppressionsSection; +}; + +export type SdkSuppressionsSection = { + [sdkName in SdkName]?: SdkPackageSuppressionsEntry[]; +}; + +export type SdkPackageSuppressionsEntry = { + package: string; + "breaking-changes": string[]; +}; + +function exitWithError(error: string): never { + console.error("Error:", error); + process.exit(1); +} + +export function validateSdkSuppressionsFile( + suppressionContent: string | object | undefined | null, +): { + result: boolean; + message: string; +} { + if (suppressionContent === null) { + exitWithError("This suppression file is a empty file"); + } + + if (!suppressionContent) { + exitWithError("This suppression file is not a valid yaml. Refer to https://aka.ms/azsdk/sdk-suppression for more information."); + } + + const suppressionFileSchema = { + type: "object", + properties: { + suppressions: { + type: "object", + propertyNames: { + enum: Object.keys(sdkLabels), + }, + patternProperties: { + "^.*$": { + type: "array", + items: { + type: "object", + properties: { + package: { type: "string" }, + "breaking-changes": { type: "array", items: { type: "string" } }, + }, + required: ["package", "breaking-changes"], + additionalProperties: false, + }, + }, + }, + }, + }, + required: ["suppressions"], + additionalProperties: false, + }; + + const suppressionAjv = new Ajv({ allErrors: true }); + const suppressionAjvCompile = suppressionAjv.compile(suppressionFileSchema); + + const isValid = suppressionAjvCompile(suppressionContent); + + if (isValid) { + return { + result: true, + message: "This suppression file is a valid yaml.", + }; + } else { + exitWithError("This suppression file is a valid yaml but the schema is wrong: " + suppressionAjv.errorsText(suppressionAjvCompile.errors, { separator: "\n" })); + } +} diff --git a/eng/tools/sdk-suppressions/src/updateSdkSuppressionsLabel.ts b/eng/tools/sdk-suppressions/src/updateSdkSuppressionsLabel.ts new file mode 100644 index 000000000000..da9997d6a69f --- /dev/null +++ b/eng/tools/sdk-suppressions/src/updateSdkSuppressionsLabel.ts @@ -0,0 +1,337 @@ +import _ from "lodash"; +import { writeFileSync } from "fs"; +import { sdkLabels, SdkName } from "./sdk.js"; +import { + SdkSuppressionsYml, + SdkSuppressionsSection, + sdkSuppressionsFileName, + SdkPackageSuppressionsEntry, + validateSdkSuppressionsFile, +} from "./sdkSuppressions.js"; +import { parseYamlContent, runGitCommand } from "./common.js"; + +/** + * + * @param prChangeFiles + * @param baseCommitHash + * @param headCommitHash + * @returns SdkName list + * This part compares the suppression files of the head branch and the base branch. + * To get the SDK, we need to identify which package name, SDK name, or breaking changes are different and apply the SDK suppression label accordingly in the next step. + * change details can see at getSdkNamesWithChangedSuppressions function + * on the other hand that the sdkName list will return an empty array if it does not have a suppression file or if the file is blank. + */ +export async function getSdkSuppressionsSdkNames( + prChangeFiles: string, + baseCommitHash: string, + headCommitHash: string +): Promise { + console.log(`Will compare base commit: ${baseCommitHash} and head commit: ${headCommitHash} to get different SDK.`); + const filesChangedPaths = prChangeFiles.split(" "); + console.log(`The pr origin changed files: ${filesChangedPaths.join(", ")}`); + let suppressionFileList = filterSuppressionList(filesChangedPaths); + console.log(`Will compare sdk-suppression.yaml files: ${suppressionFileList.join(", ")}`); + let sdkNameList: SdkName[] = []; + if (suppressionFileList.length > 0) { + for (const suppressionFile of suppressionFileList) { + let baseSuppressionContent = await getSdkSuppressionsFileContent(baseCommitHash, suppressionFile); + const headSuppressionContent = await getSdkSuppressionsFileContent(headCommitHash, suppressionFile); + + // if the head suppression file is present but anything is wrong like schema error with it return + const validateSdkSuppressionsFileResult = + validateSdkSuppressionsFile(headSuppressionContent).result; + if (!validateSdkSuppressionsFileResult) { + return []; + } + // if base suppression file does not exist, set it to an empty object but has correct schema + if (!baseSuppressionContent) { + baseSuppressionContent = { suppressions: {} }; + } + + console.log( + `updateSdkSuppressionsLabels: Will compare base suppressions content:\n ` + + `${JSON.stringify(baseSuppressionContent)}\n ` + + `and head suppressions content:\n ` + + `${JSON.stringify(headSuppressionContent)} to get different SDK.`, + ); + + sdkNameList = getSdkNamesWithChangedSuppressions( + headSuppressionContent as SdkSuppressionsYml, + baseSuppressionContent as SdkSuppressionsYml, + ); + } + } + + return [...new Set(sdkNameList)]; +} + +export async function getSdkSuppressionsFileContent( + ref: string, + path: string, +): Promise { + try { + const suppressionFileContent = await runGitCommand(`git show ${ref}:${path}`); + console.log(`Found content in ${ref}#${path}`); + return parseYamlContent(suppressionFileContent, path).result; + } catch (error) { + console.log(`Not found content in ${ref}#${path}, Error: ${error}`); + return null; + } +} + +function getSdksWithSuppressionsDefined(suppressions: SdkSuppressionsSection): SdkName[] { + return _.keys(suppressions) as SdkName[]; +} + +/** + * + * @param headSuppressionFile + * @param baseSuppressionFile + * @returns SdkName[] + * + * Analyze the suppression files across three dimensions: language, package, and breaking-change. Finally, determine the outermost sdkName. + */ + +export function getSdkNamesWithChangedSuppressions( + headSuppressionFile: SdkSuppressionsYml, + baseSuppressionFile: SdkSuppressionsYml, +): SdkName[] { + let sdkNamesWithChangedSuppressions: SdkName[] = []; + + const headSdkSuppressionsSection: SdkSuppressionsSection = headSuppressionFile.suppressions; + const baseSdkSuppressionsSection: SdkSuppressionsSection = baseSuppressionFile.suppressions; + + const headSdksWithSuppressions: SdkName[] = getSdksWithSuppressionsDefined( + headSdkSuppressionsSection, + ); + const baseSdksWithSuppressions: SdkName[] = getSdksWithSuppressionsDefined( + baseSdkSuppressionsSection, + ); + + if (headSdksWithSuppressions.length === 0) { + if (baseSdksWithSuppressions.length > 0) { + sdkNamesWithChangedSuppressions = [ + ...sdkNamesWithChangedSuppressions, + ...baseSdksWithSuppressions, + ]; + } + } + + // 1. If modify Sdk in SdkSuppressionsSection, add SdkName to sdkNamesWithChangedSuppressions + const differentSdkNamesWithChangedSuppressions = _.xorWith( + headSdksWithSuppressions, + baseSdksWithSuppressions, + _.isEqual, + ); + if (differentSdkNamesWithChangedSuppressions.length > 0) { + sdkNamesWithChangedSuppressions = [ + ...sdkNamesWithChangedSuppressions, + ...differentSdkNamesWithChangedSuppressions, + ]; + } + + // 2. If modify SdkPackageSuppressionsEntry in SdkSuppressionsSection include package name and breaking changes + // add SdkName to sdkNamesWithChangedSuppressions + const similarSdkNamesWithChangedSuppressions = _.intersectionWith( + headSdksWithSuppressions, + baseSdksWithSuppressions, + ); + similarSdkNamesWithChangedSuppressions.forEach((sdkName: SdkName) => { + const headSdkPackageSuppressionsEntry = headSdkSuppressionsSection[ + sdkName + ] as SdkPackageSuppressionsEntry[]; + const baseSdkPackageSuppressionsEntry = baseSdkSuppressionsSection[ + sdkName + ] as SdkPackageSuppressionsEntry[]; + // Determine whether packageName has changed + const differentPackageNamesWithChangedSuppressions = _.xorWith( + headSdkPackageSuppressionsEntry.map((entry) => entry.package), + baseSdkPackageSuppressionsEntry.map((entry) => entry.package), + _.isEqual, + ); + if (differentPackageNamesWithChangedSuppressions.length > 0) { + sdkNamesWithChangedSuppressions = [...sdkNamesWithChangedSuppressions, sdkName]; + return; + } + // Determine whether breaking-changes has changed + headSdkPackageSuppressionsEntry.forEach((headEntry) => { + const baseEntry = baseSdkPackageSuppressionsEntry.find( + (entry) => entry.package === headEntry.package, + ); + if (!baseEntry) { + sdkNamesWithChangedSuppressions = [...sdkNamesWithChangedSuppressions, sdkName]; + return; + } + if (!_.isEqual(headEntry["breaking-changes"].sort(), baseEntry["breaking-changes"].sort())) { + sdkNamesWithChangedSuppressions = [...sdkNamesWithChangedSuppressions, sdkName]; + return; + } + }); + }); + + return [...new Set(sdkNamesWithChangedSuppressions)]; +} + +/** + * + * @param prLabels + * @param prChangeFiles + * @param baseCommitHash + * @param headCommitHash + * @param outputFile + * @returns { labelsToAdd: String[]; labelsToRemove: String[] } + * This code performs two key functions: + * First, it retrieves the corresponding SDKNames based on the differences between the two sdk-suppression files. + * Second, it compares the SDKNames obtained in the previous step with the existing PR labels and processes the PR labels accordingly. + */ +export async function updateSdkSuppressionsLabels( + prLabels: string, + prChangeFiles: string, + baseCommitHash: string, + headCommitHash: string, + outputFile?: string, +): Promise<{ labelsToAdd: String[]; labelsToRemove: String[] }> { + try { + const status = await runGitCommand("git status"); + console.log("Git status:", status); + } catch (err) { + console.error("Error running git command:", err); + } + + const sdkNames = await getSdkSuppressionsSdkNames(prChangeFiles, baseCommitHash, headCommitHash); + + console.log( + `updateSdkSuppressionsLabels: Get the required suppressions label based on compared SDK List ${sdkNames.join(", ")}`, + ); + + const presentLabels = JSON.parse(prLabels) as string[]; + console.log(`updateSdkSuppressionsLabels: Present labels: ${presentLabels.join(", ")}`); + + const result = processLabels(presentLabels, sdkNames); + + if(outputFile){ + writeFileSync(outputFile, JSON.stringify(result)); + console.log(`😊 JSON output saved to ${outputFile}`); + } + + return result; +} + +/** + * + * @param presentLabels + * @param sdkNames + * @returns {labelsToAdd: String[], labelsToRemove: String[]} + * + * Based on the various sdknames and existing labels, process the suppression label of PR. + * + * Add logic: If the breakingChangeSuppression label corresponding to an SDK in sdkNames is not in the current presentLabels list, + * add the label to labelsToAdd. + * Remove logic: If a label is in presentLabels and the corresponding breakingChangeSuppression is not in sdkNames + * and there is no corresponding breakingChangeSuppressionApproved label, then the label is deleted. + * Otherwise, the label is not deleted. + */ +export function processLabels(presentLabels: string[], sdkNames: string[]): { labelsToAdd: String[]; labelsToRemove: String[] } { + // The sdkNames indicates whether any suppression files have been modified. If it is empty + // then check if the suppression label was previously applied and remove it if so. Otherwise, no action is needed. + let addSdkSuppressionsLabels: string[] = []; + let removeSdkSuppressionsLabels: string[] = []; + sdkNames.forEach((sdkName) => { + const sdk = sdkLabels[sdkName as SdkName]; + const breakingChangeSuppression = sdk.breakingChangeSuppression; + // If breakingChangeSuppression is not in the existing labels, add it to labelsToAdd + if ( + breakingChangeSuppression && + !presentLabels.includes(breakingChangeSuppression) + ) { + addSdkSuppressionsLabels.push(breakingChangeSuppression); + } + }); + + presentLabels.forEach(label => { + // Check if it is a suppression label + const suppressionLabelExists = Object.values(sdkLabels).some(sdk => { + return sdk.breakingChangeSuppression === label; + }); + + // If it is a suppression label + if (suppressionLabelExists) { + // Check if there is a corresponding approved label + const hasApprovedLabel = Object.values(sdkLabels).some(sdk => { + return sdk.breakingChangeSuppression === label && sdk.breakingChangeSuppressionApproved && presentLabels.includes(sdk.breakingChangeSuppressionApproved); + }); + // If there is no corresponding approved label and there is no suppression label in sdkNames, delete it. + if (!hasApprovedLabel && !sdkNames.some(sdkName => sdkLabels[sdkName as SdkName].breakingChangeSuppression === label)) { + removeSdkSuppressionsLabels.push(label); + } + } + }); + + return { + labelsToAdd: addSdkSuppressionsLabels, + labelsToRemove: removeSdkSuppressionsLabels, + }; +} + +/** + * + * @param filesChangedPaths + * @returns string[] + * check suppressionFileList is swagger suppression or tsp suppression + * if the change includes both swagger suppression and tsp suppression, only handle the tsp suppression + * others keep swagger suppression + * + * filter data-plane for swagger suppression and tsp suppression for each service + */ +export function filterSuppressionList(filesChangedPaths: string[]): string[] { + let initialSuppressionFiles = filesChangedPaths.filter((suppressionFile) => + suppressionFile.split("/").includes(sdkSuppressionsFileName), + ); + let tspSuppressionFileList = initialSuppressionFiles.filter((suppressionFile) => + suppressionFile.split("/").some((suppressionFile) => suppressionFile.endsWith(".Management")), + ); + let swaggerSuppressionFileList = initialSuppressionFiles.filter((suppressionFile) => + suppressionFile.split("/").includes("resource-manager"), + ); + + let filterSuppressionFileList = [...tspSuppressionFileList, ...swaggerSuppressionFileList]; + + const groupedSuppressionFileList = filterSuppressionFileList.reduce( + (acc: { [key: string]: string[] }, path) => { + const key = path.split("/")[1]; + if (!acc[key]) { + acc[key] = []; + } + acc[key].push(path); + + return acc; + }, + {}, + ); + + let suppressionFileList: string[] = []; + for (const serviceName in groupedSuppressionFileList) { + if (groupedSuppressionFileList.hasOwnProperty(serviceName)) { + let serviceSuppressionList = groupedSuppressionFileList[serviceName]; + if ( + serviceSuppressionList.some((suppressionFile) => + suppressionFile + .split("/") + .some((suppressionFile) => suppressionFile.endsWith(".Management")), + ) + ) { + suppressionFileList = suppressionFileList.concat( + serviceSuppressionList.filter((suppressionFile) => + suppressionFile + .split("/") + .some((suppressionFile) => suppressionFile.endsWith(".Management")), + ), + ); + } else { + suppressionFileList = suppressionFileList.concat(serviceSuppressionList); + } + } + } + + return suppressionFileList; +} diff --git a/eng/tools/sdk-suppressions/test/updateSdkSuppressionsLabel.test.ts b/eng/tools/sdk-suppressions/test/updateSdkSuppressionsLabel.test.ts new file mode 100644 index 000000000000..e03c3b570012 --- /dev/null +++ b/eng/tools/sdk-suppressions/test/updateSdkSuppressionsLabel.test.ts @@ -0,0 +1,184 @@ +import { vi, expect, test } from "vitest"; +import { filterSuppressionList, getSdkNamesWithChangedSuppressions, processLabels } from "../src/updateSdkSuppressionsLabel.js"; +import { validateSdkSuppressionsFile } from "../src/sdkSuppressions.js"; + +vi.mock("process", () => ({ + exit: vi.fn(), +})); + +test("test filterSuppressionList for only resource-manager files", () => { + const changeFiles = [ + "specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/datafactory.json", + "specification/datafactory/resource-manager/sdk-suppressions.yaml" + ]; + const suppressionsFiles: String[] = filterSuppressionList(changeFiles); + expect(suppressionsFiles).toEqual(["specification/datafactory/resource-manager/sdk-suppressions.yaml"]); +}); + +test("test filterSuppressionList for both tsp files and resource-manager files", () => { + const changeFiles = [ + "specification/workloads/Workloads.Operations.Management/main.tsp", + "specification/workloads/Workloads.Operations.Management/sdk-suppressions.yaml", + "specification/workloads/resource-manager/Microsoft.Workloads/operations/preview/2023-10-01-preview/operations.json", + "specification/workloads/resource-manager/Microsoft.Workloads/operations/preview/2024-02-01-preview/operations.json", + "specification/workloads/resource-manager/Microsoft.Workloads/operations/preview/2023-12-01-preview/operations.json", + "specification/workloads/resource-manager/Microsoft.Workloads/operations/stable/2024-09-01/operations.json", + "specification/workloads/resource-manager/sdk-suppressions.yaml" + ]; + const suppressionsFiles: String[] = filterSuppressionList(changeFiles); + expect(suppressionsFiles).toEqual(["specification/workloads/Workloads.Operations.Management/sdk-suppressions.yaml"]); +}); + +test("test validateSdkSuppressionsFile for sdk-suppression file", () => { + const suppressionContent = { + "suppressions": { + "azure-sdk-for-go": [ + { + "package": "sdk/resourcemanager/appcontainers/armappcontainers", + "breaking-changes": [ + "Field `EndTime`, `StartTime`, `Status`, `Template` of struct `JobExecution` has been removed" + ] + } + ], + "azure-sdk-for-python": [ + { + "package": "azure-mgmt-appcontainers", + "breaking-changes": [ + "Model BillingMeter no longer has parameter system_data" + ] + } + ] + } + }; + + const validateResult = validateSdkSuppressionsFile(suppressionContent); + expect(validateResult).toEqual({ result: true, message: 'This suppression file is a valid yaml.' }); +}); + +test("test validateSdkSuppressionsFile for empty file", () => { + const consoleSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + const mockProcessExit = vi.spyOn(process, "exit").mockImplementation(() => { + throw new Error("process.exit called"); // Prevent actual exit + }); + + expect(() => validateSdkSuppressionsFile(null)).toThrow("process.exit called"); + expect(consoleSpy).toHaveBeenCalledWith("Error:", "This suppression file is a empty file"); + expect(mockProcessExit).toHaveBeenCalledWith(1); + + consoleSpy.mockRestore(); + mockProcessExit.mockRestore(); +}); + +test("test validateSdkSuppressionsFile for undefined file", () => { + const consoleSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + const mockProcessExit = vi.spyOn(process, "exit").mockImplementation(() => { + throw new Error("process.exit called"); // Prevent actual exit + }); + + expect(() => validateSdkSuppressionsFile(undefined)).toThrow("process.exit called"); + expect(consoleSpy).toHaveBeenCalledWith("Error:", "This suppression file is not a valid yaml. Refer to https://aka.ms/azsdk/sdk-suppression for more information."); + expect(mockProcessExit).toHaveBeenCalledWith(1); + + consoleSpy.mockRestore(); + mockProcessExit.mockRestore(); +}); + +test("test validateSdkSuppressionsFile for error structor file", () => { + const consoleSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + const mockProcessExit = vi.spyOn(process, "exit").mockImplementation(() => { + throw new Error("process.exit called"); // Prevent actual exit + }); + + const suppressionContent = { + "suppressions": { + "azure-sdk-for-go": [ + { + "package": "sdk/resourcemanager/appcontainers/armappcontainers" + } + ], + "azure-sdk-for-python": [ + { + "package": "azure-mgmt-appcontainers", + "breaking-changes": [ + "Model BillingMeter no longer has parameter system_data" + ] + } + ] + } + }; + + expect(() => validateSdkSuppressionsFile(suppressionContent)).toThrow("process.exit called"); + expect(consoleSpy).toHaveBeenCalledWith("Error:", "This suppression file is a valid yaml but the schema is wrong: data/suppressions/azure-sdk-for-go/0 must have required property 'breaking-changes'"); + expect(mockProcessExit).toHaveBeenCalledWith(1); + + consoleSpy.mockRestore(); + mockProcessExit.mockRestore(); +}); + +test("test getSdkNamesWithChangedSuppressions", () => { + const headCont = { + "suppressions": { + "azure-sdk-for-python": [ + { + "package": "azure-mgmt-appcontainers", + "breaking-changes": [ + "Model BillingMeter no longer has parameter system_data AAA" + ] + } + ], + "azure-sdk-for-go": [ + { + "package": "sdk/resourcemanager/appcontainers/armappcontainers", + "breaking-changes": [ + "Field `EndTime`, `StartTime`, `Status`, `Template` of struct `JobExecution` has been removed" + ] + } + ] + } + }; + const baseCont = { + "suppressions": { + "azure-sdk-for-python": [ + { + "package": "azure-mgmt-appcontainers", + "breaking-changes": [ + "Model BillingMeter no longer has parameter system_data" + ] + } + ], + "azure-sdk-for-go": [ + { + "package": "sdk/resourcemanager/appcontainers/armappcontainers", + "breaking-changes": [ + "Field `EndTime`, `StartTime`, `Status`, `Template` of struct `JobExecution` has been removed" + ] + } + ] + } + }; + + const sdkNames = getSdkNamesWithChangedSuppressions(headCont, baseCont); + expect(sdkNames).toEqual(["azure-sdk-for-python"]); +}); + +test("test processLabels will add new label when has sdkNames", () => { + const sdkNames: string[] = ["azure-sdk-for-go", "azure-sdk-for-js"]; + const presentLabels: string[] = ["aa", "BreakingChange-Go-Sdk-Suppression"]; + const result = processLabels(presentLabels, sdkNames); + expect(result).toEqual({ labelsToAdd: ["BreakingChange-JavaScript-Sdk-Suppression"], labelsToRemove: [] }); + +}); + +test("test processLabels will remove old label when has the sdkNames not exist", () => { + const sdkNames: string[] = ["azure-sdk-for-js"]; + const presentLabels: string[] = ["aa", "BreakingChange-Go-Sdk-Suppression"]; + const result = processLabels(presentLabels, sdkNames); + expect(result).toEqual({ labelsToAdd: ["BreakingChange-JavaScript-Sdk-Suppression"], labelsToRemove: ["BreakingChange-Go-Sdk-Suppression"] }); + }); + +test("test processLabels will not remove old label when has the sdkNames not exist & has corresponding suppression approved", () => { + const sdkNames: string[] = ["azure-sdk-for-go"]; + const presentLabels: string[] = ["aa", "BreakingChange-Go-Sdk-Suppression", "BreakingChange-JavaScript-Sdk-Suppression", "BreakingChange-JavaScript-Sdk-Suppression-Approved"]; + const result = processLabels(presentLabels, sdkNames); + expect(result).toEqual({ labelsToAdd: [], labelsToRemove: [] }); +}); diff --git a/eng/tools/sdk-suppressions/tsconfig.json b/eng/tools/sdk-suppressions/tsconfig.json new file mode 100644 index 000000000000..eae537921c52 --- /dev/null +++ b/eng/tools/sdk-suppressions/tsconfig.json @@ -0,0 +1,6 @@ +{ + "extends": "../tsconfig.json", + "compilerOptions": { + "outDir": "./dist" + } +} diff --git a/eng/tools/tsconfig.json b/eng/tools/tsconfig.json index ffa89e56a6d8..eca3d4cdace7 100644 --- a/eng/tools/tsconfig.json +++ b/eng/tools/tsconfig.json @@ -15,6 +15,7 @@ { "path": "./suppressions" }, { "path": "./tsp-client-tests" }, { "path": "./typespec-requirement" }, - { "path": "./typespec-validation" } + { "path": "./typespec-validation" }, + { "path": "./sdk-suppressions" } ] } diff --git a/package-lock.json b/package-lock.json index 1316151cf1d0..aad5134b06db 100644 --- a/package-lock.json +++ b/package-lock.json @@ -39,6 +39,7 @@ "dev": true, "hasInstallScript": true, "devDependencies": { + "@azure-tools/sdk-suppressions": "file:sdk-suppressions", "@azure-tools/specs-model": "file:specs-model", "@azure-tools/suppressions": "file:suppressions", "@azure-tools/tsp-client-tests": "file:tsp-client-tests", @@ -275,6 +276,29 @@ } } }, + "eng/tools/sdk-suppressions": { + "name": "@azure-tools/sdk-suppressions", + "version": "1.0.0", + "dev": true, + "dependencies": { + "ajv": "^8.17.1", + "lodash": "^4.17.20", + "yaml": "^2.4.2" + }, + "bin": { + "get-sdk-suppressions-label": "cmd/sdk-suppressions-label.js" + }, + "devDependencies": { + "@types/lodash": "^4.14.161", + "@types/node": "^18.19.31", + "@vitest/coverage-v8": "^2.0.4", + "typescript": "~5.6.2", + "vitest": "^2.0.4" + }, + "engines": { + "node": ">= 18.0.0" + } + }, "eng/tools/specs-model": { "name": "@azure-tools/specs-model", "dev": true, @@ -911,6 +935,10 @@ "node": ">=18.0.0" } }, + "node_modules/@azure-tools/sdk-suppressions": { + "resolved": "eng/tools/sdk-suppressions", + "link": true + }, "node_modules/@azure-tools/specs-model": { "resolved": "eng/tools/specs-model", "link": true @@ -3375,6 +3403,13 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/lodash": { + "version": "4.17.13", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.13.tgz", + "integrity": "sha512-lfx+dftrEZcdBPczf9d0Qv0x+j/rfNCMuC6OcfXmO8gkfeNAY88PgKUbvG56whcN23gc27yenwF6oJZXGFpYxg==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/node": { "version": "10.17.60", "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.60.tgz",