From d9d1e85d54827d4171e6555ba8a2302e58dfba8d Mon Sep 17 00:00:00 2001 From: Marc Paine Date: Thu, 14 Dec 2023 16:55:39 -0800 Subject: [PATCH 01/21] Update the fluent assertions version to improve test failure output. Include a failing tests to verify. --- test/EndToEnd/ProjectBuildTests.cs | 2 +- .../Assertions/CommandResultAssertions.cs | 6 +++--- test/core-sdk-tasks.Tests/core-sdk-tasks.Tests.csproj | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/EndToEnd/ProjectBuildTests.cs b/test/EndToEnd/ProjectBuildTests.cs index 01925eb5fd95..0e5b8bb4b11a 100644 --- a/test/EndToEnd/ProjectBuildTests.cs +++ b/test/EndToEnd/ProjectBuildTests.cs @@ -82,7 +82,7 @@ public void ItCanRunAnAppUsingTheWebSdk() var runCommand = new RunCommand() .WithWorkingDirectory(projectDirectory) .ExecuteWithCapturedOutput() - .Should().Pass().And.HaveStdOutContaining("Hello, World!"); + .Should().Pass().And.HaveStdOutContaining("Hello, World!ds"); } [WindowsOnlyTheory] diff --git a/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/CommandResultAssertions.cs b/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/CommandResultAssertions.cs index 7301d9a814a9..c140ec1cd9ca 100644 --- a/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/CommandResultAssertions.cs +++ b/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/CommandResultAssertions.cs @@ -42,7 +42,7 @@ public AndConstraint Fail() public AndConstraint HaveStdOut() { Execute.Assertion.ForCondition(!string.IsNullOrEmpty(_commandResult.StdOut)) - .FailWith(AppendDiagnosticsTo("Command did not output anything to stdout")); + .FailWith(AppendDiagnosticsTo($"Command did not output anything to stdout")); return new AndConstraint(this); } @@ -97,7 +97,7 @@ public AndConstraint HaveStdOutMatching(string pattern, public AndConstraint HaveStdErr() { Execute.Assertion.ForCondition(!string.IsNullOrEmpty(_commandResult.StdErr)) - .FailWith(AppendDiagnosticsTo("Command did not output anything to stderr.")); + .FailWith(AppendDiagnosticsTo($"Command did not output anything to stderr.")); return new AndConstraint(this); } @@ -132,7 +132,7 @@ public AndConstraint NotHaveStdOut() public AndConstraint NotHaveStdErr() { Execute.Assertion.ForCondition(string.IsNullOrEmpty(_commandResult.StdErr)) - .FailWith(AppendDiagnosticsTo("Expected command to not output to stderr but it was not:")); + .FailWith(AppendDiagnosticsTo($"Expected command to not output to stderr but it was not:")); return new AndConstraint(this); } diff --git a/test/core-sdk-tasks.Tests/core-sdk-tasks.Tests.csproj b/test/core-sdk-tasks.Tests/core-sdk-tasks.Tests.csproj index 47b6ce51724b..0e751c8abc19 100644 --- a/test/core-sdk-tasks.Tests/core-sdk-tasks.Tests.csproj +++ b/test/core-sdk-tasks.Tests/core-sdk-tasks.Tests.csproj @@ -5,7 +5,7 @@ - + From f4311e2723cafabaa3678fa510e51ffe9ad203fd Mon Sep 17 00:00:00 2001 From: Marc Paine Date: Fri, 15 Dec 2023 09:43:15 -0800 Subject: [PATCH 02/21] Add FailWithPreformatted which saves the failure message before it gets reformated by fluent --- .../Assertions/AssertionScopeExtensions.cs | 20 ++++++++++++ .../Assertions/CommandResultAssertions.cs | 32 +++++++++---------- 2 files changed, 36 insertions(+), 16 deletions(-) create mode 100644 test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/AssertionScopeExtensions.cs diff --git a/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/AssertionScopeExtensions.cs b/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/AssertionScopeExtensions.cs new file mode 100644 index 000000000000..4256b1ef78ab --- /dev/null +++ b/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/AssertionScopeExtensions.cs @@ -0,0 +1,20 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using FluentAssertions.Execution; + +namespace Microsoft.DotNet.Tools.Test.Utilities +{ + public static class AssertionScopeExtensions + { + public static Continuation FailWithPreformatted(this AssertionScope assertionScope, string message) + { + if (!assertionScope.Succeeded) + { + assertionScope.AddFailure(message); + } + + return new Continuation(assertionScope, assertionScope.Succeeded); + } + } +} \ No newline at end of file diff --git a/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/CommandResultAssertions.cs b/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/CommandResultAssertions.cs index c140ec1cd9ca..e6bf83b15881 100644 --- a/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/CommandResultAssertions.cs +++ b/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/CommandResultAssertions.cs @@ -21,35 +21,35 @@ public CommandResultAssertions(CommandResult commandResult) public AndConstraint ExitWith(int expectedExitCode) { Execute.Assertion.ForCondition(_commandResult.ExitCode == expectedExitCode) - .FailWith(AppendDiagnosticsTo($"Expected command to exit with {expectedExitCode} but it did not.")); + .FailWithPreformatted(AppendDiagnosticsTo($"Expected command to exit with {expectedExitCode} but it did not.")); return new AndConstraint(this); } public AndConstraint Pass() { Execute.Assertion.ForCondition(_commandResult.ExitCode == 0) - .FailWith(AppendDiagnosticsTo($"Expected command to pass but it did not.")); + .FailWithPreformatted(AppendDiagnosticsTo($"Expected command to pass but it did not.")); return new AndConstraint(this); } public AndConstraint Fail() { Execute.Assertion.ForCondition(_commandResult.ExitCode != 0) - .FailWith(AppendDiagnosticsTo($"Expected command to fail but it did not.")); + .FailWithPreformatted(AppendDiagnosticsTo($"Expected command to fail but it did not.")); return new AndConstraint(this); } public AndConstraint HaveStdOut() { Execute.Assertion.ForCondition(!string.IsNullOrEmpty(_commandResult.StdOut)) - .FailWith(AppendDiagnosticsTo($"Command did not output anything to stdout")); + .FailWithPreformatted(AppendDiagnosticsTo($"Command did not output anything to stdout")); return new AndConstraint(this); } public AndConstraint HaveStdOut(string expectedOutput) { Execute.Assertion.ForCondition(_commandResult.StdOut.Equals(expectedOutput, StringComparison.Ordinal)) - .FailWith(AppendDiagnosticsTo($"Command did not output with Expected Output. Expected: {expectedOutput}")); + .FailWithPreformatted(AppendDiagnosticsTo($"Command did not output with Expected Output. Expected: {expectedOutput}")); return new AndConstraint(this); } @@ -57,7 +57,7 @@ public AndConstraint HaveStdOutContaining(string patter { Execute.Assertion .ForCondition(_commandResult.StdOut.Contains(pattern)) - .FailWith(AppendDiagnosticsTo($"The command output did not contain expected result: {pattern}{Environment.NewLine}")); + .FailWithPreformatted(AppendDiagnosticsTo($"The command output did not contain expected result: {pattern}{Environment.NewLine}")); return new AndConstraint(this); } @@ -68,7 +68,7 @@ public AndConstraint HaveStdOutContainingIgnoreSpaces(s Execute.Assertion .ForCondition(commandResultNoSpaces.Contains(pattern)) - .FailWith(AppendDiagnosticsTo($"The command output did not contain expected result: {pattern}{Environment.NewLine}")); + .FailWithPreformatted(AppendDiagnosticsTo($"The command output did not contain expected result: {pattern}{Environment.NewLine}")); return new AndConstraint(this); } @@ -76,63 +76,63 @@ public AndConstraint HaveStdOutContainingIgnoreSpaces(s public AndConstraint HaveStdOutContainingIgnoreCase(string pattern) { Execute.Assertion.ForCondition(_commandResult.StdOut.IndexOf(pattern, StringComparison.OrdinalIgnoreCase) >= 0) - .FailWith(AppendDiagnosticsTo($"The command output did not contain expected result (ignoring case): {pattern}{Environment.NewLine}")); + .FailWithPreformatted(AppendDiagnosticsTo($"The command output did not contain expected result (ignoring case): {pattern}{Environment.NewLine}")); return new AndConstraint(this); } public AndConstraint NotHaveStdOutContaining(string pattern) { Execute.Assertion.ForCondition(!_commandResult.StdOut.Contains(pattern)) - .FailWith(AppendDiagnosticsTo($"The command output contained a result it should not have contained: {pattern}{Environment.NewLine}")); + .FailWithPreformatted(AppendDiagnosticsTo($"The command output contained a result it should not have contained: {pattern}{Environment.NewLine}")); return new AndConstraint(this); } public AndConstraint HaveStdOutMatching(string pattern, RegexOptions options = RegexOptions.None) { Execute.Assertion.ForCondition(Regex.Match(_commandResult.StdOut, pattern, options).Success) - .FailWith(AppendDiagnosticsTo($"Matching the command output failed. Pattern: {pattern}{Environment.NewLine}")); + .FailWithPreformatted(AppendDiagnosticsTo($"Matching the command output failed. Pattern: {pattern}{Environment.NewLine}")); return new AndConstraint(this); } public AndConstraint HaveStdErr() { Execute.Assertion.ForCondition(!string.IsNullOrEmpty(_commandResult.StdErr)) - .FailWith(AppendDiagnosticsTo($"Command did not output anything to stderr.")); + .FailWithPreformatted(AppendDiagnosticsTo($"Command did not output anything to stderr.")); return new AndConstraint(this); } public AndConstraint HaveStdErrContaining(string pattern) { Execute.Assertion.ForCondition(_commandResult.StdErr.Contains(pattern)) - .FailWith(AppendDiagnosticsTo($"The command error output did not contain expected result: {pattern}{Environment.NewLine}")); + .FailWithPreformatted(AppendDiagnosticsTo($"The command error output did not contain expected result: {pattern}{Environment.NewLine}")); return new AndConstraint(this); } public AndConstraint NotHaveStdErrContaining(string pattern) { Execute.Assertion.ForCondition(!_commandResult.StdErr.Contains(pattern)) - .FailWith(AppendDiagnosticsTo($"The command error output contained a result it should not have contained: {pattern}{Environment.NewLine}")); + .FailWithPreformatted(AppendDiagnosticsTo($"The command error output contained a result it should not have contained: {pattern}{Environment.NewLine}")); return new AndConstraint(this); } public AndConstraint HaveStdErrMatching(string pattern, RegexOptions options = RegexOptions.None) { Execute.Assertion.ForCondition(Regex.Match(_commandResult.StdErr, pattern, options).Success) - .FailWith(AppendDiagnosticsTo($"Matching the command error output failed. Pattern: {pattern}{Environment.NewLine}")); + .FailWithPreformatted(AppendDiagnosticsTo($"Matching the command error output failed. Pattern: {pattern}{Environment.NewLine}")); return new AndConstraint(this); } public AndConstraint NotHaveStdOut() { Execute.Assertion.ForCondition(string.IsNullOrEmpty(_commandResult.StdOut)) - .FailWith(AppendDiagnosticsTo($"Expected command to not output to stdout but it was not:")); + .FailWithPreformatted(AppendDiagnosticsTo($"Expected command to not output to stdout but it was not:")); return new AndConstraint(this); } public AndConstraint NotHaveStdErr() { Execute.Assertion.ForCondition(string.IsNullOrEmpty(_commandResult.StdErr)) - .FailWith(AppendDiagnosticsTo($"Expected command to not output to stderr but it was not:")); + .FailWithPreformatted(AppendDiagnosticsTo($"Expected command to not output to stderr but it was not:")); return new AndConstraint(this); } From 636f714882fe44351e43d32d0b6326c0eaf32b64 Mon Sep 17 00:00:00 2001 From: Marc Paine Date: Tue, 2 Jan 2024 15:31:52 -0800 Subject: [PATCH 03/21] Update to call FailWithPreformatted --- .../Assertions/DirectoryInfoAssertions.cs | 26 +++++++++---------- .../Assertions/FileInfoAssertions.cs | 4 +-- .../Assertions/StringAssertionsExtensions.cs | 4 +-- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/DirectoryInfoAssertions.cs b/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/DirectoryInfoAssertions.cs index adb83caf28c2..6e658d848e47 100644 --- a/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/DirectoryInfoAssertions.cs +++ b/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/DirectoryInfoAssertions.cs @@ -26,7 +26,7 @@ public DirectoryInfoAssertions(DirectoryInfo dir) public AndConstraint Exist() { Execute.Assertion.ForCondition(_dirInfo.Exists) - .FailWith("Expected directory {0} does not exist.", _dirInfo.FullName); + .FailWithPreformatted("Expected directory {0} does not exist.", _dirInfo.FullName); return new AndConstraint(this); } @@ -37,7 +37,7 @@ public AndConstraint HaveFile(string expectedFile, stri Execute.Assertion .ForCondition(file != null) .BecauseOf(because, reasonArgs) - .FailWith($"Expected File {expectedFile} cannot be found in directory {_dirInfo.FullName}."); + .FailWithPreformatted($"Expected File {expectedFile} cannot be found in directory {_dirInfo.FullName}."); return new AndConstraint(this); } @@ -57,7 +57,7 @@ public AndConstraint HaveTextFile( Execute.Assertion .ForCondition(contents.Equals(expectedContents)) .BecauseOf(because, reasonArgs) - .FailWith($"Expected file {expectedFile} to contain \n\n{expectedContents}\n\nbut it contains\n\n{contents}\n"); + .FailWithPreformatted($"Expected file {expectedFile} to contain \n\n{expectedContents}\n\nbut it contains\n\n{contents}\n"); return new AndConstraint(this); } @@ -71,7 +71,7 @@ public AndConstraint NotHaveFile( Execute.Assertion .ForCondition(file == null) .BecauseOf(because, reasonArgs) - .FailWith("File {0} should not be found in directory {1}.", expectedFile, _dirInfo.FullName); + .FailWithPreformatted("File {0} should not be found in directory {1}.", expectedFile, _dirInfo.FullName); return new AndConstraint(this); } @@ -109,7 +109,7 @@ public AndConstraint HaveFilesMatching( Execute.Assertion .ForCondition(matchingFileExists == true) .BecauseOf(because, reasonArgs) - .FailWith("Expected directory {0} to contain files matching {1}, but no matching file exists.", + .FailWithPreformatted("Expected directory {0} to contain files matching {1}, but no matching file exists.", _dirInfo.FullName, expectedFilesSearchPattern); return new AndConstraint(this); @@ -132,7 +132,7 @@ public AndConstraint NotHaveFilesMatching(string expect { var matchingFileCount = _dirInfo.EnumerateFiles(expectedFilesSearchPattern, searchOption).Count(); Execute.Assertion.ForCondition(matchingFileCount == 0) - .FailWith("Found {0} files that should not exist in directory {1}. No file matching {2} should exist.", + .FailWithPreformatted("Found {0} files that should not exist in directory {1}. No file matching {2} should exist.", matchingFileCount, _dirInfo.FullName, expectedFilesSearchPattern); return new AndConstraint(this); } @@ -141,7 +141,7 @@ public AndConstraint HaveDirectory(string expectedDir) { var dir = _dirInfo.EnumerateDirectories(expectedDir, SearchOption.TopDirectoryOnly).SingleOrDefault(); Execute.Assertion.ForCondition(dir != null) - .FailWith("Expected directory {0} cannot be found inside directory {1}.", expectedDir, _dirInfo.FullName); + .FailWithPreformatted("Expected directory {0} cannot be found inside directory {1}.", expectedDir, _dirInfo.FullName); return new AndConstraint(new DirectoryInfoAssertions(dir)); } @@ -160,7 +160,7 @@ public AndConstraint NotHaveDirectory(string unexpected { var dir = _dirInfo.EnumerateDirectories(unexpectedDir, SearchOption.TopDirectoryOnly).SingleOrDefault(); Execute.Assertion.ForCondition(dir == null) - .FailWith("Directory {0} should not be found in directory {1}.", unexpectedDir, _dirInfo.FullName); + .FailWithPreformatted("Directory {0} should not be found in directory {1}.", unexpectedDir, _dirInfo.FullName); return new AndConstraint(new DirectoryInfoAssertions(dir)); } @@ -183,10 +183,10 @@ public AndConstraint OnlyHaveFiles(IEnumerable var nl = Environment.NewLine; Execute.Assertion.ForCondition(!missingFiles.Any()) - .FailWith($"Following files cannot be found inside directory {_dirInfo.FullName} {nl} {string.Join(nl, missingFiles)}"); + .FailWithPreformatted($"Following files cannot be found inside directory {_dirInfo.FullName} {nl} {string.Join(nl, missingFiles)}"); Execute.Assertion.ForCondition(!extraFiles.Any()) - .FailWith($"Following extra files are found inside directory {_dirInfo.FullName} {nl} {string.Join(nl, extraFiles)}"); + .FailWithPreformatted($"Following extra files are found inside directory {_dirInfo.FullName} {nl} {string.Join(nl, extraFiles)}"); return new AndConstraint(this); } @@ -194,7 +194,7 @@ public AndConstraint OnlyHaveFiles(IEnumerable public AndConstraint BeEmpty() { Execute.Assertion.ForCondition(!_dirInfo.EnumerateFileSystemInfos().Any()) - .FailWith($"The directory {_dirInfo.FullName} is not empty."); + .FailWithPreformatted($"The directory {_dirInfo.FullName} is not empty."); return new AndConstraint(this); } @@ -202,7 +202,7 @@ public AndConstraint BeEmpty() public AndConstraint NotBeEmpty() { Execute.Assertion.ForCondition(_dirInfo.EnumerateFileSystemInfos().Any()) - .FailWith($"The directory {_dirInfo.FullName} is empty."); + .FailWithPreformatted($"The directory {_dirInfo.FullName} is empty."); return new AndConstraint(this); } @@ -212,7 +212,7 @@ public AndConstraint NotExist(string because = "", para Execute.Assertion .ForCondition(_dirInfo.Exists == false) .BecauseOf(because, reasonArgs) - .FailWith($"Expected directory {_dirInfo.FullName} to not exist, but it does."); + .FailWithPreformatted($"Expected directory {_dirInfo.FullName} to not exist, but it does."); return new AndConstraint(this); } diff --git a/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/FileInfoAssertions.cs b/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/FileInfoAssertions.cs index dca27f0c52c4..f5d1d84a2d61 100644 --- a/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/FileInfoAssertions.cs +++ b/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/FileInfoAssertions.cs @@ -28,7 +28,7 @@ public AndConstraint Exist(string because = "", params objec Execute.Assertion .ForCondition(_fileInfo.Exists) .BecauseOf(because, reasonArgs) - .FailWith($"Expected File {_fileInfo.FullName} to exist, but it does not."); + .FailWithPreformatted($"Expected File {_fileInfo.FullName} to exist, but it does not."); return new AndConstraint(this); } @@ -37,7 +37,7 @@ public AndConstraint NotExist(string because = "", params ob Execute.Assertion .ForCondition(!_fileInfo.Exists) .BecauseOf(because, reasonArgs) - .FailWith($"Expected File {_fileInfo.FullName} to not exist, but it does."); + .FailWithPreformatted($"Expected File {_fileInfo.FullName} to not exist, but it does."); return new AndConstraint(this); } } diff --git a/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/StringAssertionsExtensions.cs b/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/StringAssertionsExtensions.cs index e0483c39a4b0..799809879013 100644 --- a/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/StringAssertionsExtensions.cs +++ b/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/StringAssertionsExtensions.cs @@ -20,7 +20,7 @@ public static AndConstraint BeVisuallyEquivalentTo(this String Execute.Assertion .ForCondition(NormalizeLineEndings(assertions.Subject) == NormalizeLineEndings(expected)) .BecauseOf(because, becauseArgs) - .FailWith($"String \"{assertions.Subject}\" is not visually equivalent to expected string \"{expected}\"."); + .FailWithPreformatted($"String \"{assertions.Subject}\" is not visually equivalent to expected string \"{expected}\"."); return new AndConstraint(assertions); } @@ -30,7 +30,7 @@ public static AndConstraint ContainVisuallySameFragment(this S Execute.Assertion .ForCondition(NormalizeLineEndings(assertions.Subject).Contains(NormalizeLineEndings(expected))) .BecauseOf(because, becauseArgs) - .FailWith($"String \"{assertions.Subject}\" does not contain visually same fragment string \"{expected}\"."); + .FailWithPreformatted($"String \"{assertions.Subject}\" does not contain visually same fragment string \"{expected}\"."); return new AndConstraint(assertions); } From 7ad11327958c321d112bf521780e4a5fa76c3320 Mon Sep 17 00:00:00 2001 From: Marc Paine Date: Tue, 9 Jan 2024 16:12:12 -0800 Subject: [PATCH 04/21] Switch to addpreformattedfailure --- .../Assertions/AssertionScopeExtensions.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/AssertionScopeExtensions.cs b/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/AssertionScopeExtensions.cs index 4256b1ef78ab..54a567eb3865 100644 --- a/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/AssertionScopeExtensions.cs +++ b/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/AssertionScopeExtensions.cs @@ -11,10 +11,10 @@ public static Continuation FailWithPreformatted(this AssertionScope assertionSco { if (!assertionScope.Succeeded) { - assertionScope.AddFailure(message); + assertionScope.AddPreFormattedFailure(message); } return new Continuation(assertionScope, assertionScope.Succeeded); } } -} \ No newline at end of file +} From c9b14717bcd029fad2dfbfbc4b40eeb887ffa8ca Mon Sep 17 00:00:00 2001 From: Marc Paine Date: Thu, 11 Jan 2024 15:42:20 -0800 Subject: [PATCH 05/21] Looks like i missed a fluent assertions update in the test utilities csproj so trying that to see if it is enough to solve the output issue. --- test/EndToEnd/ValidateInsertedManifests.cs | 2 +- .../Assertions/AssertionScopeExtensions.cs | 20 ------------ .../Assertions/CommandResultAssertions.cs | 32 +++++++++---------- .../Assertions/DirectoryInfoAssertions.cs | 30 ++++++++--------- .../Assertions/FileInfoAssertions.cs | 4 +-- .../Assertions/StringAssertionsExtensions.cs | 4 +-- ...rosoft.DotNet.Tools.Tests.Utilities.csproj | 2 +- 7 files changed, 37 insertions(+), 57 deletions(-) delete mode 100644 test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/AssertionScopeExtensions.cs diff --git a/test/EndToEnd/ValidateInsertedManifests.cs b/test/EndToEnd/ValidateInsertedManifests.cs index cdede126c997..0b2668afc476 100644 --- a/test/EndToEnd/ValidateInsertedManifests.cs +++ b/test/EndToEnd/ValidateInsertedManifests.cs @@ -34,7 +34,7 @@ public void ManifestReaderCanReadManifests() File.Exists(manifestFile).Should().BeTrue(); using var fileStream = new FileStream(manifestFile, FileMode.Open, FileAccess.Read); Action readManifest = () => WorkloadManifestReader.ReadWorkloadManifest(manifestId, fileStream, manifestFile); - readManifest.ShouldNotThrow("manifestId:" + manifestId + " manifestFile:" + manifestFile + "is invalid"); + readManifest.Should().NotThrow("manifestId:" + manifestId + " manifestFile:" + manifestFile + "is invalid"); } } diff --git a/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/AssertionScopeExtensions.cs b/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/AssertionScopeExtensions.cs deleted file mode 100644 index 54a567eb3865..000000000000 --- a/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/AssertionScopeExtensions.cs +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (c) .NET Foundation and contributors. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using FluentAssertions.Execution; - -namespace Microsoft.DotNet.Tools.Test.Utilities -{ - public static class AssertionScopeExtensions - { - public static Continuation FailWithPreformatted(this AssertionScope assertionScope, string message) - { - if (!assertionScope.Succeeded) - { - assertionScope.AddPreFormattedFailure(message); - } - - return new Continuation(assertionScope, assertionScope.Succeeded); - } - } -} diff --git a/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/CommandResultAssertions.cs b/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/CommandResultAssertions.cs index e6bf83b15881..c140ec1cd9ca 100644 --- a/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/CommandResultAssertions.cs +++ b/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/CommandResultAssertions.cs @@ -21,35 +21,35 @@ public CommandResultAssertions(CommandResult commandResult) public AndConstraint ExitWith(int expectedExitCode) { Execute.Assertion.ForCondition(_commandResult.ExitCode == expectedExitCode) - .FailWithPreformatted(AppendDiagnosticsTo($"Expected command to exit with {expectedExitCode} but it did not.")); + .FailWith(AppendDiagnosticsTo($"Expected command to exit with {expectedExitCode} but it did not.")); return new AndConstraint(this); } public AndConstraint Pass() { Execute.Assertion.ForCondition(_commandResult.ExitCode == 0) - .FailWithPreformatted(AppendDiagnosticsTo($"Expected command to pass but it did not.")); + .FailWith(AppendDiagnosticsTo($"Expected command to pass but it did not.")); return new AndConstraint(this); } public AndConstraint Fail() { Execute.Assertion.ForCondition(_commandResult.ExitCode != 0) - .FailWithPreformatted(AppendDiagnosticsTo($"Expected command to fail but it did not.")); + .FailWith(AppendDiagnosticsTo($"Expected command to fail but it did not.")); return new AndConstraint(this); } public AndConstraint HaveStdOut() { Execute.Assertion.ForCondition(!string.IsNullOrEmpty(_commandResult.StdOut)) - .FailWithPreformatted(AppendDiagnosticsTo($"Command did not output anything to stdout")); + .FailWith(AppendDiagnosticsTo($"Command did not output anything to stdout")); return new AndConstraint(this); } public AndConstraint HaveStdOut(string expectedOutput) { Execute.Assertion.ForCondition(_commandResult.StdOut.Equals(expectedOutput, StringComparison.Ordinal)) - .FailWithPreformatted(AppendDiagnosticsTo($"Command did not output with Expected Output. Expected: {expectedOutput}")); + .FailWith(AppendDiagnosticsTo($"Command did not output with Expected Output. Expected: {expectedOutput}")); return new AndConstraint(this); } @@ -57,7 +57,7 @@ public AndConstraint HaveStdOutContaining(string patter { Execute.Assertion .ForCondition(_commandResult.StdOut.Contains(pattern)) - .FailWithPreformatted(AppendDiagnosticsTo($"The command output did not contain expected result: {pattern}{Environment.NewLine}")); + .FailWith(AppendDiagnosticsTo($"The command output did not contain expected result: {pattern}{Environment.NewLine}")); return new AndConstraint(this); } @@ -68,7 +68,7 @@ public AndConstraint HaveStdOutContainingIgnoreSpaces(s Execute.Assertion .ForCondition(commandResultNoSpaces.Contains(pattern)) - .FailWithPreformatted(AppendDiagnosticsTo($"The command output did not contain expected result: {pattern}{Environment.NewLine}")); + .FailWith(AppendDiagnosticsTo($"The command output did not contain expected result: {pattern}{Environment.NewLine}")); return new AndConstraint(this); } @@ -76,63 +76,63 @@ public AndConstraint HaveStdOutContainingIgnoreSpaces(s public AndConstraint HaveStdOutContainingIgnoreCase(string pattern) { Execute.Assertion.ForCondition(_commandResult.StdOut.IndexOf(pattern, StringComparison.OrdinalIgnoreCase) >= 0) - .FailWithPreformatted(AppendDiagnosticsTo($"The command output did not contain expected result (ignoring case): {pattern}{Environment.NewLine}")); + .FailWith(AppendDiagnosticsTo($"The command output did not contain expected result (ignoring case): {pattern}{Environment.NewLine}")); return new AndConstraint(this); } public AndConstraint NotHaveStdOutContaining(string pattern) { Execute.Assertion.ForCondition(!_commandResult.StdOut.Contains(pattern)) - .FailWithPreformatted(AppendDiagnosticsTo($"The command output contained a result it should not have contained: {pattern}{Environment.NewLine}")); + .FailWith(AppendDiagnosticsTo($"The command output contained a result it should not have contained: {pattern}{Environment.NewLine}")); return new AndConstraint(this); } public AndConstraint HaveStdOutMatching(string pattern, RegexOptions options = RegexOptions.None) { Execute.Assertion.ForCondition(Regex.Match(_commandResult.StdOut, pattern, options).Success) - .FailWithPreformatted(AppendDiagnosticsTo($"Matching the command output failed. Pattern: {pattern}{Environment.NewLine}")); + .FailWith(AppendDiagnosticsTo($"Matching the command output failed. Pattern: {pattern}{Environment.NewLine}")); return new AndConstraint(this); } public AndConstraint HaveStdErr() { Execute.Assertion.ForCondition(!string.IsNullOrEmpty(_commandResult.StdErr)) - .FailWithPreformatted(AppendDiagnosticsTo($"Command did not output anything to stderr.")); + .FailWith(AppendDiagnosticsTo($"Command did not output anything to stderr.")); return new AndConstraint(this); } public AndConstraint HaveStdErrContaining(string pattern) { Execute.Assertion.ForCondition(_commandResult.StdErr.Contains(pattern)) - .FailWithPreformatted(AppendDiagnosticsTo($"The command error output did not contain expected result: {pattern}{Environment.NewLine}")); + .FailWith(AppendDiagnosticsTo($"The command error output did not contain expected result: {pattern}{Environment.NewLine}")); return new AndConstraint(this); } public AndConstraint NotHaveStdErrContaining(string pattern) { Execute.Assertion.ForCondition(!_commandResult.StdErr.Contains(pattern)) - .FailWithPreformatted(AppendDiagnosticsTo($"The command error output contained a result it should not have contained: {pattern}{Environment.NewLine}")); + .FailWith(AppendDiagnosticsTo($"The command error output contained a result it should not have contained: {pattern}{Environment.NewLine}")); return new AndConstraint(this); } public AndConstraint HaveStdErrMatching(string pattern, RegexOptions options = RegexOptions.None) { Execute.Assertion.ForCondition(Regex.Match(_commandResult.StdErr, pattern, options).Success) - .FailWithPreformatted(AppendDiagnosticsTo($"Matching the command error output failed. Pattern: {pattern}{Environment.NewLine}")); + .FailWith(AppendDiagnosticsTo($"Matching the command error output failed. Pattern: {pattern}{Environment.NewLine}")); return new AndConstraint(this); } public AndConstraint NotHaveStdOut() { Execute.Assertion.ForCondition(string.IsNullOrEmpty(_commandResult.StdOut)) - .FailWithPreformatted(AppendDiagnosticsTo($"Expected command to not output to stdout but it was not:")); + .FailWith(AppendDiagnosticsTo($"Expected command to not output to stdout but it was not:")); return new AndConstraint(this); } public AndConstraint NotHaveStdErr() { Execute.Assertion.ForCondition(string.IsNullOrEmpty(_commandResult.StdErr)) - .FailWithPreformatted(AppendDiagnosticsTo($"Expected command to not output to stderr but it was not:")); + .FailWith(AppendDiagnosticsTo($"Expected command to not output to stderr but it was not:")); return new AndConstraint(this); } diff --git a/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/DirectoryInfoAssertions.cs b/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/DirectoryInfoAssertions.cs index 6e658d848e47..74b45b8dbc5e 100644 --- a/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/DirectoryInfoAssertions.cs +++ b/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/DirectoryInfoAssertions.cs @@ -26,7 +26,7 @@ public DirectoryInfoAssertions(DirectoryInfo dir) public AndConstraint Exist() { Execute.Assertion.ForCondition(_dirInfo.Exists) - .FailWithPreformatted("Expected directory {0} does not exist.", _dirInfo.FullName); + .FailWith(String.Format("Expected directory {0} does not exist.", _dirInfo.FullName)); return new AndConstraint(this); } @@ -37,7 +37,7 @@ public AndConstraint HaveFile(string expectedFile, stri Execute.Assertion .ForCondition(file != null) .BecauseOf(because, reasonArgs) - .FailWithPreformatted($"Expected File {expectedFile} cannot be found in directory {_dirInfo.FullName}."); + .FailWith($"Expected File {expectedFile} cannot be found in directory {_dirInfo.FullName}."); return new AndConstraint(this); } @@ -57,7 +57,7 @@ public AndConstraint HaveTextFile( Execute.Assertion .ForCondition(contents.Equals(expectedContents)) .BecauseOf(because, reasonArgs) - .FailWithPreformatted($"Expected file {expectedFile} to contain \n\n{expectedContents}\n\nbut it contains\n\n{contents}\n"); + .FailWith($"Expected file {expectedFile} to contain \n\n{expectedContents}\n\nbut it contains\n\n{contents}\n"); return new AndConstraint(this); } @@ -71,7 +71,7 @@ public AndConstraint NotHaveFile( Execute.Assertion .ForCondition(file == null) .BecauseOf(because, reasonArgs) - .FailWithPreformatted("File {0} should not be found in directory {1}.", expectedFile, _dirInfo.FullName); + .FailWith(String.Format("File {0} should not be found in directory {1}.", expectedFile, _dirInfo.FullName)); return new AndConstraint(this); } @@ -109,8 +109,8 @@ public AndConstraint HaveFilesMatching( Execute.Assertion .ForCondition(matchingFileExists == true) .BecauseOf(because, reasonArgs) - .FailWithPreformatted("Expected directory {0} to contain files matching {1}, but no matching file exists.", - _dirInfo.FullName, expectedFilesSearchPattern); + .FailWith(String.Format("Expected directory {0} to contain files matching {1}, but no matching file exists.", + _dirInfo.FullName, expectedFilesSearchPattern)); return new AndConstraint(this); } @@ -132,8 +132,8 @@ public AndConstraint NotHaveFilesMatching(string expect { var matchingFileCount = _dirInfo.EnumerateFiles(expectedFilesSearchPattern, searchOption).Count(); Execute.Assertion.ForCondition(matchingFileCount == 0) - .FailWithPreformatted("Found {0} files that should not exist in directory {1}. No file matching {2} should exist.", - matchingFileCount, _dirInfo.FullName, expectedFilesSearchPattern); + .FailWith(String.Format("Found {0} files that should not exist in directory {1}. No file matching {2} should exist.", + matchingFileCount, _dirInfo.FullName, expectedFilesSearchPattern)); return new AndConstraint(this); } @@ -141,7 +141,7 @@ public AndConstraint HaveDirectory(string expectedDir) { var dir = _dirInfo.EnumerateDirectories(expectedDir, SearchOption.TopDirectoryOnly).SingleOrDefault(); Execute.Assertion.ForCondition(dir != null) - .FailWithPreformatted("Expected directory {0} cannot be found inside directory {1}.", expectedDir, _dirInfo.FullName); + .FailWith(String.Format("Expected directory {0} cannot be found inside directory {1}.", expectedDir, _dirInfo.FullName)); return new AndConstraint(new DirectoryInfoAssertions(dir)); } @@ -160,7 +160,7 @@ public AndConstraint NotHaveDirectory(string unexpected { var dir = _dirInfo.EnumerateDirectories(unexpectedDir, SearchOption.TopDirectoryOnly).SingleOrDefault(); Execute.Assertion.ForCondition(dir == null) - .FailWithPreformatted("Directory {0} should not be found in directory {1}.", unexpectedDir, _dirInfo.FullName); + .FailWith(String.Format("Directory {0} should not be found in directory {1}.", unexpectedDir, _dirInfo.FullName)); return new AndConstraint(new DirectoryInfoAssertions(dir)); } @@ -183,10 +183,10 @@ public AndConstraint OnlyHaveFiles(IEnumerable var nl = Environment.NewLine; Execute.Assertion.ForCondition(!missingFiles.Any()) - .FailWithPreformatted($"Following files cannot be found inside directory {_dirInfo.FullName} {nl} {string.Join(nl, missingFiles)}"); + .FailWith($"Following files cannot be found inside directory {_dirInfo.FullName} {nl} {string.Join(nl, missingFiles)}"); Execute.Assertion.ForCondition(!extraFiles.Any()) - .FailWithPreformatted($"Following extra files are found inside directory {_dirInfo.FullName} {nl} {string.Join(nl, extraFiles)}"); + .FailWith($"Following extra files are found inside directory {_dirInfo.FullName} {nl} {string.Join(nl, extraFiles)}"); return new AndConstraint(this); } @@ -194,7 +194,7 @@ public AndConstraint OnlyHaveFiles(IEnumerable public AndConstraint BeEmpty() { Execute.Assertion.ForCondition(!_dirInfo.EnumerateFileSystemInfos().Any()) - .FailWithPreformatted($"The directory {_dirInfo.FullName} is not empty."); + .FailWith($"The directory {_dirInfo.FullName} is not empty."); return new AndConstraint(this); } @@ -202,7 +202,7 @@ public AndConstraint BeEmpty() public AndConstraint NotBeEmpty() { Execute.Assertion.ForCondition(_dirInfo.EnumerateFileSystemInfos().Any()) - .FailWithPreformatted($"The directory {_dirInfo.FullName} is empty."); + .FailWith($"The directory {_dirInfo.FullName} is empty."); return new AndConstraint(this); } @@ -212,7 +212,7 @@ public AndConstraint NotExist(string because = "", para Execute.Assertion .ForCondition(_dirInfo.Exists == false) .BecauseOf(because, reasonArgs) - .FailWithPreformatted($"Expected directory {_dirInfo.FullName} to not exist, but it does."); + .FailWith($"Expected directory {_dirInfo.FullName} to not exist, but it does."); return new AndConstraint(this); } diff --git a/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/FileInfoAssertions.cs b/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/FileInfoAssertions.cs index f5d1d84a2d61..dca27f0c52c4 100644 --- a/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/FileInfoAssertions.cs +++ b/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/FileInfoAssertions.cs @@ -28,7 +28,7 @@ public AndConstraint Exist(string because = "", params objec Execute.Assertion .ForCondition(_fileInfo.Exists) .BecauseOf(because, reasonArgs) - .FailWithPreformatted($"Expected File {_fileInfo.FullName} to exist, but it does not."); + .FailWith($"Expected File {_fileInfo.FullName} to exist, but it does not."); return new AndConstraint(this); } @@ -37,7 +37,7 @@ public AndConstraint NotExist(string because = "", params ob Execute.Assertion .ForCondition(!_fileInfo.Exists) .BecauseOf(because, reasonArgs) - .FailWithPreformatted($"Expected File {_fileInfo.FullName} to not exist, but it does."); + .FailWith($"Expected File {_fileInfo.FullName} to not exist, but it does."); return new AndConstraint(this); } } diff --git a/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/StringAssertionsExtensions.cs b/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/StringAssertionsExtensions.cs index 799809879013..e0483c39a4b0 100644 --- a/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/StringAssertionsExtensions.cs +++ b/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/StringAssertionsExtensions.cs @@ -20,7 +20,7 @@ public static AndConstraint BeVisuallyEquivalentTo(this String Execute.Assertion .ForCondition(NormalizeLineEndings(assertions.Subject) == NormalizeLineEndings(expected)) .BecauseOf(because, becauseArgs) - .FailWithPreformatted($"String \"{assertions.Subject}\" is not visually equivalent to expected string \"{expected}\"."); + .FailWith($"String \"{assertions.Subject}\" is not visually equivalent to expected string \"{expected}\"."); return new AndConstraint(assertions); } @@ -30,7 +30,7 @@ public static AndConstraint ContainVisuallySameFragment(this S Execute.Assertion .ForCondition(NormalizeLineEndings(assertions.Subject).Contains(NormalizeLineEndings(expected))) .BecauseOf(because, becauseArgs) - .FailWithPreformatted($"String \"{assertions.Subject}\" does not contain visually same fragment string \"{expected}\"."); + .FailWith($"String \"{assertions.Subject}\" does not contain visually same fragment string \"{expected}\"."); return new AndConstraint(assertions); } diff --git a/test/Microsoft.DotNet.Tools.Tests.Utilities/Microsoft.DotNet.Tools.Tests.Utilities.csproj b/test/Microsoft.DotNet.Tools.Tests.Utilities/Microsoft.DotNet.Tools.Tests.Utilities.csproj index 27da1cfb8f57..485ec5fad717 100644 --- a/test/Microsoft.DotNet.Tools.Tests.Utilities/Microsoft.DotNet.Tools.Tests.Utilities.csproj +++ b/test/Microsoft.DotNet.Tools.Tests.Utilities/Microsoft.DotNet.Tools.Tests.Utilities.csproj @@ -4,7 +4,7 @@ - + From ec45808e4418026611f41cd433856c072bacbc7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20K=C3=B6plinger?= Date: Mon, 22 Apr 2024 14:35:06 +0200 Subject: [PATCH 06/21] Use VSTest runner for tests --- Directory.Build.props | 2 ++ eng/build-pr.yml | 4 ++-- eng/build.yml | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index 39b30a121778..0d470e4a5bfc 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -31,6 +31,8 @@ This is a prevalent problem in building the precomputed assembly reference cache. Limiting the assemblies resolved to those outside the culture folders would work, but that is a nontrivial problem. --> NU5125;NU5105;NU1701;MSB3243;MSB3247 + + true diff --git a/eng/build-pr.yml b/eng/build-pr.yml index 831d96781f0a..e9a14cdf7e80 100644 --- a/eng/build-pr.yml +++ b/eng/build-pr.yml @@ -233,8 +233,8 @@ jobs: - task: PublishTestResults@2 displayName: Publish Test Results inputs: - testRunner: XUnit - testResultsFiles: 'artifacts/TestResults/${{ parameters.buildConfiguration }}/*.xml' + testRunner: VSTest + testResultsFiles: 'artifacts/TestResults/${{ parameters.buildConfiguration }}/*.trx' testRunTitle: '$(_AgentOSName)_$(Agent.JobName)' platform: '$(BuildPlatform)' configuration: '${{ parameters.buildConfiguration }}' diff --git a/eng/build.yml b/eng/build.yml index 4f947ea8f7ea..d901233c41d0 100644 --- a/eng/build.yml +++ b/eng/build.yml @@ -239,8 +239,8 @@ jobs: - task: PublishTestResults@2 displayName: Publish Test Results inputs: - testRunner: XUnit - testResultsFiles: 'artifacts/TestResults/${{ parameters.buildConfiguration }}/*.xml' + testRunner: VSTest + testResultsFiles: 'artifacts/TestResults/${{ parameters.buildConfiguration }}/*.trx' testRunTitle: '$(_AgentOSName)_$(Agent.JobName)' platform: '$(BuildPlatform)' configuration: '${{ parameters.buildConfiguration }}' From 68acf81475b787e427697c1534ad8a21858fc560 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20K=C3=B6plinger?= Date: Thu, 20 Jun 2024 14:31:27 +0200 Subject: [PATCH 07/21] Remove intentional error --- test/EndToEnd/ProjectBuildTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/EndToEnd/ProjectBuildTests.cs b/test/EndToEnd/ProjectBuildTests.cs index a3e4fa4e2dee..fa7350c1652d 100644 --- a/test/EndToEnd/ProjectBuildTests.cs +++ b/test/EndToEnd/ProjectBuildTests.cs @@ -82,7 +82,7 @@ public void ItCanRunAnAppUsingTheWebSdk() var runCommand = new RunCommand() .WithWorkingDirectory(projectDirectory) .ExecuteWithCapturedOutput() - .Should().Pass().And.HaveStdOutContaining("Hello, World!ds"); + .Should().Pass().And.HaveStdOutContaining("Hello, World!"); } [WindowsOnlyTheory] From dcebdfb923e296f44aa300fc62f8b92d957a018f Mon Sep 17 00:00:00 2001 From: Farhad Alizada Date: Fri, 21 Jun 2024 11:43:19 +0200 Subject: [PATCH 08/21] Onboard new inter-branch merge flow [Workflow] --- .github/workflows/inter-branch-merge-flow.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/workflows/inter-branch-merge-flow.yml diff --git a/.github/workflows/inter-branch-merge-flow.yml b/.github/workflows/inter-branch-merge-flow.yml new file mode 100644 index 000000000000..7672d36811f9 --- /dev/null +++ b/.github/workflows/inter-branch-merge-flow.yml @@ -0,0 +1,13 @@ +name: Inter-branch merge workflow +on: + push: + branches: + - release/** + +permissions: + contents: write + pull-requests: write + +jobs: + Merge: + uses: dotnet/arcade/.github/workflows/inter-branch-merge-base.yml@main \ No newline at end of file From 5d670e214d26ca23b9ac037e7e07dcc35572d1f1 Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Tue, 2 Jul 2024 09:38:35 -0700 Subject: [PATCH 09/21] Update arcade --- eng/Version.Details.xml | 12 +++--- eng/Versions.props | 2 +- .../templates-official/job/source-build.yml | 8 ++++ .../job/source-index-stage1.yml | 35 ++++++++++----- .../templates-official/jobs/source-build.yml | 8 ++++ .../post-build/common-variables.yml | 1 - .../steps/enable-internal-runtimes.yml | 28 ++++++++++++ .../steps/get-delegation-sas.yml | 43 +++++++++++++++++++ .../steps/get-federated-access-token.yml | 28 ++++++++++++ eng/common/templates/job/source-build.yml | 8 ++++ .../templates/job/source-index-stage1.yml | 35 ++++++++++----- eng/common/templates/jobs/source-build.yml | 8 ++++ .../templates/post-build/common-variables.yml | 1 - .../steps/enable-internal-runtimes.yml | 28 ++++++++++++ .../templates/steps/get-delegation-sas.yml | 43 +++++++++++++++++++ .../steps/get-federated-access-token.yml | 28 ++++++++++++ global.json | 4 +- 17 files changed, 289 insertions(+), 31 deletions(-) create mode 100644 eng/common/templates-official/steps/enable-internal-runtimes.yml create mode 100644 eng/common/templates-official/steps/get-delegation-sas.yml create mode 100644 eng/common/templates-official/steps/get-federated-access-token.yml create mode 100644 eng/common/templates/steps/enable-internal-runtimes.yml create mode 100644 eng/common/templates/steps/get-delegation-sas.yml create mode 100644 eng/common/templates/steps/get-federated-access-token.yml diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 36042503161b..3a6323aa2631 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -197,19 +197,19 @@ - + https://github.com/dotnet/arcade - 2eab07c3d7b78219d10099b19fafeef2ecae1779 + 1844d819e6f33f1106083c5066ea01e0310eefa3 - + https://github.com/dotnet/arcade - 2eab07c3d7b78219d10099b19fafeef2ecae1779 + 1844d819e6f33f1106083c5066ea01e0310eefa3 - + https://github.com/dotnet/arcade - 2eab07c3d7b78219d10099b19fafeef2ecae1779 + 1844d819e6f33f1106083c5066ea01e0310eefa3 https://github.com/dotnet/source-build-reference-packages diff --git a/eng/Versions.props b/eng/Versions.props index e1c4b987b4f2..2c099a32dce9 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -19,7 +19,7 @@ - 6.0.0-beta.24266.4 + 6.0.0-beta.24326.2 diff --git a/eng/common/templates-official/job/source-build.yml b/eng/common/templates-official/job/source-build.yml index f5fa09f41511..a6f7efa77685 100644 --- a/eng/common/templates-official/job/source-build.yml +++ b/eng/common/templates-official/job/source-build.yml @@ -31,6 +31,12 @@ parameters: # container and pool. platform: {} + # If set to true and running on a non-public project, + # Internal blob storage locations will be enabled. + # This is not enabled by default because many repositories do not need internal sources + # and do not need to have the required service connections approved in the pipeline. + enableInternalSources: false + jobs: - job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} displayName: Source-Build (${{ parameters.platform.name }}) @@ -59,6 +65,8 @@ jobs: clean: all steps: + - ${{ if eq(parameters.enableInternalSources, true) }}: + - template: /eng/common/templates-official/steps/enable-internal-runtimes.yml - template: /eng/common/templates-official/steps/source-build.yml parameters: platform: ${{ parameters.platform }} diff --git a/eng/common/templates-official/job/source-index-stage1.yml b/eng/common/templates-official/job/source-index-stage1.yml index f04ad04c2b18..f6f0c15bfba4 100644 --- a/eng/common/templates-official/job/source-index-stage1.yml +++ b/eng/common/templates-official/job/source-index-stage1.yml @@ -1,6 +1,7 @@ parameters: runAsPublic: false - sourceIndexPackageVersion: 1.0.1-20240320.1 + sourceIndexUploadPackageVersion: 2.0.0-20240502.12 + sourceIndexProcessBinlogPackageVersion: 1.0.1-20240129.2 sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" preSteps: [] @@ -17,14 +18,14 @@ jobs: dependsOn: ${{ parameters.dependsOn }} condition: ${{ parameters.condition }} variables: - - name: SourceIndexPackageVersion - value: ${{ parameters.sourceIndexPackageVersion }} + - name: SourceIndexUploadPackageVersion + value: ${{ parameters.sourceIndexUploadPackageVersion }} + - name: SourceIndexProcessBinlogPackageVersion + value: ${{ parameters.sourceIndexProcessBinlogPackageVersion }} - name: SourceIndexPackageSource value: ${{ parameters.sourceIndexPackageSource }} - name: BinlogPath value: ${{ parameters.binlogPath }} - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: source-dot-net stage1 variables pool: ${{ parameters.pool }} steps: @@ -40,8 +41,8 @@ jobs: workingDirectory: $(Agent.TempDirectory) - script: | - $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools - $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(sourceIndexProcessBinlogPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(sourceIndexUploadPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools displayName: Download Tools # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. workingDirectory: $(Agent.TempDirectory) @@ -53,7 +54,21 @@ jobs: displayName: Process Binlog into indexable sln - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) + - task: AzureCLI@2 + displayName: Get stage 1 auth token + inputs: + azureSubscription: 'SourceDotNet Stage1 Publish' + addSpnToEnvironment: true + scriptType: 'ps' + scriptLocation: 'inlineScript' + inlineScript: | + echo "##vso[task.setvariable variable=ARM_CLIENT_ID;issecret=true]$env:servicePrincipalId" + echo "##vso[task.setvariable variable=ARM_ID_TOKEN;issecret=true]$env:idToken" + echo "##vso[task.setvariable variable=ARM_TENANT_ID;issecret=true]$env:tenantId" + + - script: | + az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN) + displayName: "Login to Azure" + + - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1 displayName: Upload stage1 artifacts to source index - env: - BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) diff --git a/eng/common/templates-official/jobs/source-build.yml b/eng/common/templates-official/jobs/source-build.yml index b9a1f67b9a9a..9f60019cb01d 100644 --- a/eng/common/templates-official/jobs/source-build.yml +++ b/eng/common/templates-official/jobs/source-build.yml @@ -21,6 +21,12 @@ parameters: # one job runs on 'defaultManagedPlatform'. platforms: [] + # If set to true and running on a non-public project, + # Internal nuget and blob storage locations will be enabled. + # This is not enabled by default because many repositories do not need internal sources + # and do not need to have the required service connections approved in the pipeline. + enableInternalSources: false + jobs: - ${{ if ne(parameters.allCompletedJobId, '') }}: @@ -38,9 +44,11 @@ jobs: parameters: jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ platform }} + enableInternalSources: ${{ parameters.enableInternalSources }} - ${{ if eq(length(parameters.platforms), 0) }}: - template: /eng/common/templates-official/job/source-build.yml parameters: jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ parameters.defaultManagedPlatform }} + enableInternalSources: ${{ parameters.enableInternalSources }} diff --git a/eng/common/templates-official/post-build/common-variables.yml b/eng/common/templates-official/post-build/common-variables.yml index fae340f4d20d..5404cc7ecbef 100644 --- a/eng/common/templates-official/post-build/common-variables.yml +++ b/eng/common/templates-official/post-build/common-variables.yml @@ -2,7 +2,6 @@ variables: - group: AzureDevOps-Artifact-Feeds-Pats - group: DotNet-Blob-Feed - group: DotNet-DotNetCli-Storage - - group: DotNet-MSRC-Storage - group: Publish-Build-Assets # Whether the build is internal or not diff --git a/eng/common/templates-official/steps/enable-internal-runtimes.yml b/eng/common/templates-official/steps/enable-internal-runtimes.yml new file mode 100644 index 000000000000..93a8394a666b --- /dev/null +++ b/eng/common/templates-official/steps/enable-internal-runtimes.yml @@ -0,0 +1,28 @@ +# Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64' +# variable with the base64-encoded SAS token, by default + +parameters: +- name: federatedServiceConnection + type: string + default: 'dotnetbuilds-internal-read' +- name: outputVariableName + type: string + default: 'dotnetbuilds-internal-container-read-token-base64' +- name: expiryInHours + type: number + default: 1 +- name: base64Encode + type: boolean + default: true + +steps: +- ${{ if ne(variables['System.TeamProject'], 'public') }}: + - template: /eng/common/templates-official/steps/get-delegation-sas.yml + parameters: + federatedServiceConnection: ${{ parameters.federatedServiceConnection }} + outputVariableName: ${{ parameters.outputVariableName }} + expiryInHours: ${{ parameters.expiryInHours }} + base64Encode: ${{ parameters.base64Encode }} + storageAccount: dotnetbuilds + container: internal + permissions: rl diff --git a/eng/common/templates-official/steps/get-delegation-sas.yml b/eng/common/templates-official/steps/get-delegation-sas.yml new file mode 100644 index 000000000000..c0e8f91317f0 --- /dev/null +++ b/eng/common/templates-official/steps/get-delegation-sas.yml @@ -0,0 +1,43 @@ +parameters: +- name: federatedServiceConnection + type: string +- name: outputVariableName + type: string +- name: expiryInHours + type: number + default: 1 +- name: base64Encode + type: boolean + default: false +- name: storageAccount + type: string +- name: container + type: string +- name: permissions + type: string + default: 'rl' + +steps: +- task: AzureCLI@2 + displayName: 'Generate delegation SAS Token for ${{ parameters.storageAccount }}/${{ parameters.container }}' + inputs: + azureSubscription: ${{ parameters.federatedServiceConnection }} + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + # Calculate the expiration of the SAS token and convert to UTC + $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ") + + $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv + + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to generate SAS token." + exit 1 + } + + if ('${{ parameters.base64Encode }}' -eq 'true') { + $sas = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($sas)) + } + + Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$sas" diff --git a/eng/common/templates-official/steps/get-federated-access-token.yml b/eng/common/templates-official/steps/get-federated-access-token.yml new file mode 100644 index 000000000000..e3786cef6dfd --- /dev/null +++ b/eng/common/templates-official/steps/get-federated-access-token.yml @@ -0,0 +1,28 @@ +parameters: +- name: federatedServiceConnection + type: string +- name: outputVariableName + type: string +# Resource to get a token for. Common values include: +# - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps +# - 'https://storage.azure.com/' for storage +# Defaults to Azure DevOps +- name: resource + type: string + default: '499b84ac-1321-427f-aa17-267ca6975798' + +steps: +- task: AzureCLI@2 + displayName: 'Getting federated access token for feeds' + inputs: + azureSubscription: ${{ parameters.federatedServiceConnection }} + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + $accessToken = az account get-access-token --query accessToken --resource ${{ parameters.resource }} --output tsv + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to get access token for resource '${{ parameters.resource }}'" + exit 1 + } + Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$accessToken" diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml index b6137f44ada1..8009ccb954ac 100644 --- a/eng/common/templates/job/source-build.yml +++ b/eng/common/templates/job/source-build.yml @@ -31,6 +31,12 @@ parameters: # container and pool. platform: {} + # If set to true and running on a non-public project, + # Internal blob storage locations will be enabled. + # This is not enabled by default because many repositories do not need internal sources + # and do not need to have the required service connections approved in the pipeline. + enableInternalSources: false + jobs: - job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} displayName: Source-Build (${{ parameters.platform.name }}) @@ -58,6 +64,8 @@ jobs: clean: all steps: + - ${{ if eq(parameters.enableInternalSources, true) }}: + - template: /eng/common/templates/steps/enable-internal-runtimes.yml - template: /eng/common/templates/steps/source-build.yml parameters: platform: ${{ parameters.platform }} diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml index b710698eb4d4..47f36e98bc16 100644 --- a/eng/common/templates/job/source-index-stage1.yml +++ b/eng/common/templates/job/source-index-stage1.yml @@ -1,6 +1,7 @@ parameters: runAsPublic: false - sourceIndexPackageVersion: 1.0.1-20240320.1 + sourceIndexUploadPackageVersion: 2.0.0-20240502.12 + sourceIndexProcessBinlogPackageVersion: 1.0.1-20240129.2 sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" preSteps: [] @@ -15,14 +16,14 @@ jobs: dependsOn: ${{ parameters.dependsOn }} condition: ${{ parameters.condition }} variables: - - name: SourceIndexPackageVersion - value: ${{ parameters.sourceIndexPackageVersion }} + - name: SourceIndexUploadPackageVersion + value: ${{ parameters.sourceIndexUploadPackageVersion }} + - name: SourceIndexProcessBinlogPackageVersion + value: ${{ parameters.sourceIndexProcessBinlogPackageVersion }} - name: SourceIndexPackageSource value: ${{ parameters.sourceIndexPackageSource }} - name: BinlogPath value: ${{ parameters.binlogPath }} - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: source-dot-net stage1 variables pool: ${{ parameters.pool }} steps: @@ -38,8 +39,8 @@ jobs: workingDirectory: $(Agent.TempDirectory) - script: | - $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools - $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(sourceIndexProcessBinlogPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(sourceIndexUploadPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools displayName: Download Tools # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. workingDirectory: $(Agent.TempDirectory) @@ -51,7 +52,21 @@ jobs: displayName: Process Binlog into indexable sln - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) + - task: AzureCLI@2 + displayName: Get stage 1 auth token + inputs: + azureSubscription: 'SourceDotNet Stage1 Publish' + addSpnToEnvironment: true + scriptType: 'ps' + scriptLocation: 'inlineScript' + inlineScript: | + echo "##vso[task.setvariable variable=ARM_CLIENT_ID;issecret=true]$env:servicePrincipalId" + echo "##vso[task.setvariable variable=ARM_ID_TOKEN;issecret=true]$env:idToken" + echo "##vso[task.setvariable variable=ARM_TENANT_ID;issecret=true]$env:tenantId" + + - script: | + az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN) + displayName: "Login to Azure" + + - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1 displayName: Upload stage1 artifacts to source index - env: - BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) diff --git a/eng/common/templates/jobs/source-build.yml b/eng/common/templates/jobs/source-build.yml index 7c240e654472..e3fbd9f38e10 100644 --- a/eng/common/templates/jobs/source-build.yml +++ b/eng/common/templates/jobs/source-build.yml @@ -21,6 +21,12 @@ parameters: # one job runs on 'defaultManagedPlatform'. platforms: [] + # If set to true and running on a non-public project, + # Internal nuget and blob storage locations will be enabled. + # This is not enabled by default because many repositories do not need internal sources + # and do not need to have the required service connections approved in the pipeline. + enableInternalSources: false + jobs: - ${{ if ne(parameters.allCompletedJobId, '') }}: @@ -38,9 +44,11 @@ jobs: parameters: jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ platform }} + enableInternalSources: ${{ parameters.enableInternalSources }} - ${{ if eq(length(parameters.platforms), 0) }}: - template: /eng/common/templates/job/source-build.yml parameters: jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ parameters.defaultManagedPlatform }} + enableInternalSources: ${{ parameters.enableInternalSources }} diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml index fae340f4d20d..5404cc7ecbef 100644 --- a/eng/common/templates/post-build/common-variables.yml +++ b/eng/common/templates/post-build/common-variables.yml @@ -2,7 +2,6 @@ variables: - group: AzureDevOps-Artifact-Feeds-Pats - group: DotNet-Blob-Feed - group: DotNet-DotNetCli-Storage - - group: DotNet-MSRC-Storage - group: Publish-Build-Assets # Whether the build is internal or not diff --git a/eng/common/templates/steps/enable-internal-runtimes.yml b/eng/common/templates/steps/enable-internal-runtimes.yml new file mode 100644 index 000000000000..54dc9416c519 --- /dev/null +++ b/eng/common/templates/steps/enable-internal-runtimes.yml @@ -0,0 +1,28 @@ +# Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64' +# variable with the base64-encoded SAS token, by default + +parameters: +- name: federatedServiceConnection + type: string + default: 'dotnetbuilds-internal-read' +- name: outputVariableName + type: string + default: 'dotnetbuilds-internal-container-read-token-base64' +- name: expiryInHours + type: number + default: 1 +- name: base64Encode + type: boolean + default: true + +steps: +- ${{ if ne(variables['System.TeamProject'], 'public') }}: + - template: /eng/common/templates/steps/get-delegation-sas.yml + parameters: + federatedServiceConnection: ${{ parameters.federatedServiceConnection }} + outputVariableName: ${{ parameters.outputVariableName }} + expiryInHours: ${{ parameters.expiryInHours }} + base64Encode: ${{ parameters.base64Encode }} + storageAccount: dotnetbuilds + container: internal + permissions: rl diff --git a/eng/common/templates/steps/get-delegation-sas.yml b/eng/common/templates/steps/get-delegation-sas.yml new file mode 100644 index 000000000000..c0e8f91317f0 --- /dev/null +++ b/eng/common/templates/steps/get-delegation-sas.yml @@ -0,0 +1,43 @@ +parameters: +- name: federatedServiceConnection + type: string +- name: outputVariableName + type: string +- name: expiryInHours + type: number + default: 1 +- name: base64Encode + type: boolean + default: false +- name: storageAccount + type: string +- name: container + type: string +- name: permissions + type: string + default: 'rl' + +steps: +- task: AzureCLI@2 + displayName: 'Generate delegation SAS Token for ${{ parameters.storageAccount }}/${{ parameters.container }}' + inputs: + azureSubscription: ${{ parameters.federatedServiceConnection }} + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + # Calculate the expiration of the SAS token and convert to UTC + $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ") + + $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv + + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to generate SAS token." + exit 1 + } + + if ('${{ parameters.base64Encode }}' -eq 'true') { + $sas = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($sas)) + } + + Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$sas" diff --git a/eng/common/templates/steps/get-federated-access-token.yml b/eng/common/templates/steps/get-federated-access-token.yml new file mode 100644 index 000000000000..c8c49cc0e8f0 --- /dev/null +++ b/eng/common/templates/steps/get-federated-access-token.yml @@ -0,0 +1,28 @@ +parameters: +- name: federatedServiceConnection + type: string +- name: outputVariableName + type: string +# Resource to get a token for. Common values include: +# - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps +# - 'https://storage.azure.com/' for storage +# Defaults to Azure DevOps +- name: resource + type: string + default: '499b84ac-1321-427f-aa17-267ca6975798' + +steps: +- task: AzureCLI@2 + displayName: 'Getting federated access token for feeds' + inputs: + azureSubscription: ${{ parameters.federatedServiceConnection }} + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + $accessToken = az account get-access-token --query accessToken --resource ${{ parameters.resource }} --output tsv + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to get access token for resource '${{ parameters.resource }}'" + exit 1 + } + Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$accessToken" \ No newline at end of file diff --git a/global.json b/global.json index f624a99df800..750534f91754 100644 --- a/global.json +++ b/global.json @@ -11,7 +11,7 @@ "cmake": "3.16.4" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.24266.4", - "Microsoft.DotNet.CMake.Sdk": "6.0.0-beta.24266.4" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.24326.2", + "Microsoft.DotNet.CMake.Sdk": "6.0.0-beta.24326.2" } } From 8eb3ff9b97626b2526ec417a06ee9246772e5260 Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Tue, 2 Jul 2024 09:41:57 -0700 Subject: [PATCH 10/21] Use delegation SAS for internal builds --- .vsts-ci.yml | 2 -- .vsts-pr.yml | 1 - eng/build-pr.yml | 2 +- eng/build.yml | 1 + 4 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 1461605be723..9dd53a10ebe6 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -19,7 +19,6 @@ variables: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - name: Codeql.Enabled value: true - - group: DotNet-DotNetCli-Storage - group: DotNet-Installer-SDLValidation-Params - name: _PublishUsingPipelines value: true @@ -36,7 +35,6 @@ variables: - name: _InternalRuntimeDownloadArgs value: '' - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - group: DotNetBuilds storage account read tokens - name: _InternalRuntimeDownloadArgs value: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) diff --git a/.vsts-pr.yml b/.vsts-pr.yml index 4fb55d65ec64..961efdb90e8a 100644 --- a/.vsts-pr.yml +++ b/.vsts-pr.yml @@ -34,7 +34,6 @@ variables: - name: _InternalRuntimeDownloadArgs value: '' - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - group: DotNetBuilds storage account read tokens - name: _InternalRuntimeDownloadArgs value: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) diff --git a/eng/build-pr.yml b/eng/build-pr.yml index f05d66fa690d..653774872f18 100644 --- a/eng/build-pr.yml +++ b/eng/build-pr.yml @@ -66,6 +66,7 @@ phases: steps: - checkout: self clean: true + - template: /eng/common/templates/steps/enable-internal-runtimes.yml - ${{ if eq(parameters.agentOs, 'Windows_NT') }}: - ${{ if ne(variables['System.TeamProject'], 'public') }}: - task: PowerShell@2 @@ -96,7 +97,6 @@ phases: arguments: $(Build.SourcesDirectory)/NuGet.config $Token env: Token: $(dn-bot-dnceng-artifact-feeds-rw) - - ${{ if eq(parameters.agentOs, 'Linux') }}: - script: ./build.sh $(_TestArg) $(_PackArg) diff --git a/eng/build.yml b/eng/build.yml index 2ce873c3d4d8..84d86839b0b8 100644 --- a/eng/build.yml +++ b/eng/build.yml @@ -66,6 +66,7 @@ phases: steps: - checkout: self clean: true + - template: /eng/common/templates/steps/enable-internal-runtimes.yml - ${{ if eq(parameters.agentOs, 'Windows_NT') }}: - ${{ if ne(variables['System.TeamProject'], 'public') }}: - task: PowerShell@2 From 9ee18250ae139546bb336a80c0406155fce0208e Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Tue, 2 Jul 2024 09:53:08 -0700 Subject: [PATCH 11/21] Enable internal sources for source build --- .vsts-ci.yml | 2 ++ .vsts-pr.yml | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 9dd53a10ebe6..6176d5031035 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -417,6 +417,8 @@ extends: _TestArg: '' - template: /eng/common/templates-official/jobs/source-build.yml@self + parameters: + enableInternalSources: true - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball.yml@self diff --git a/.vsts-pr.yml b/.vsts-pr.yml index 961efdb90e8a..2a4266e8a91f 100644 --- a/.vsts-pr.yml +++ b/.vsts-pr.yml @@ -17,7 +17,6 @@ variables: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - name: Codeql.Enabled value: true - - group: DotNet-DotNetCli-Storage - group: DotNet-Installer-SDLValidation-Params - name: _PublishUsingPipelines value: true @@ -370,6 +369,8 @@ stages: _TestArg: '' - template: /eng/common/templates/jobs/source-build.yml + parameters: + enableInternalSources: true - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball-pr.yml From bbd77dd234848ab6f9f63ffcb42597a04fe704b6 Mon Sep 17 00:00:00 2001 From: Sean Reeser Date: Tue, 2 Jul 2024 10:48:25 -0700 Subject: [PATCH 12/21] Update branding to 6.0.133 --- eng/Versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props index e1c4b987b4f2..2b4caf1b4384 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -8,7 +8,7 @@ 6 0 1 - 32 + 33 $(VersionMajor).$(VersionMinor).$(VersionSDKMinor)$(VersionFeature) servicing $(VersionMajor).$(VersionMinor) From 5777cef58607c401d9864fa6956b8208092ba0a7 Mon Sep 17 00:00:00 2001 From: Sean Reeser Date: Tue, 2 Jul 2024 10:48:32 -0700 Subject: [PATCH 13/21] Update branding to 6.0.425 --- eng/Versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props index 976db20b32aa..267e282eea50 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -8,7 +8,7 @@ 6 0 4 - 24 + 25 $(VersionMajor).$(VersionMinor).$(VersionSDKMinor)$(VersionFeature) servicing $(VersionMajor).$(VersionMinor) From c6c1747abb0538456e691831430d2cca397d79a4 Mon Sep 17 00:00:00 2001 From: Matt Mitchell Date: Tue, 2 Jul 2024 15:13:30 -0700 Subject: [PATCH 14/21] [release/6.0.4xx] Remove EOL centos 7 from PRs/CI (#19949) --- .vsts-ci.yml | 7 ------- .vsts-pr.yml | 7 ------- 2 files changed, 14 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 071768c91a79..35964b6cab7e 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -213,13 +213,6 @@ extends: _RuntimeIdentifier: '' _BuildArchitecture: 'x64' _TestArg: $(_NonWindowsTestArg) - Build_CentOS_7_1_Debug_x64: - _BuildConfig: Debug - _DockerParameter: '--docker centos' - _LinuxPortable: '' - _RuntimeIdentifier: '' - _BuildArchitecture: 'x64' - _TestArg: $(_NonWindowsTestArg) Build_Debian_11_Debug_x64: _BuildConfig: Debug _DockerParameter: '--docker debian' diff --git a/.vsts-pr.yml b/.vsts-pr.yml index 42ca67ae988e..b1590392b32e 100644 --- a/.vsts-pr.yml +++ b/.vsts-pr.yml @@ -169,13 +169,6 @@ stages: _RuntimeIdentifier: '' _BuildArchitecture: 'x64' _TestArg: $(_NonWindowsTestArg) - Build_CentOS_7_1_Debug_x64: - _BuildConfig: Debug - _DockerParameter: '--docker centos' - _LinuxPortable: '' - _RuntimeIdentifier: '' - _BuildArchitecture: 'x64' - _TestArg: $(_NonWindowsTestArg) Build_Debian_11_Debug_x64: _BuildConfig: Debug _DockerParameter: '--docker debian' From 12d6c8e5f7e694c3ddbd67bb953f46088c056e9a Mon Sep 17 00:00:00 2001 From: Matt Mitchell Date: Tue, 2 Jul 2024 16:11:06 -0700 Subject: [PATCH 15/21] [release/6.0.1xx] Remove EOL centos 7 from PRs (#19948) --- .vsts-ci.yml | 7 ------- .vsts-pr.yml | 7 ------- 2 files changed, 14 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 1461605be723..36ff81d8bee0 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -213,13 +213,6 @@ extends: _RuntimeIdentifier: '' _BuildArchitecture: 'x64' _TestArg: $(_NonWindowsTestArg) - Build_CentOS_7_1_Debug_x64: - _BuildConfig: Debug - _DockerParameter: '--docker centos' - _LinuxPortable: '' - _RuntimeIdentifier: '' - _BuildArchitecture: 'x64' - _TestArg: $(_NonWindowsTestArg) Build_Debian_11_Debug_x64: _BuildConfig: Debug _DockerParameter: '--docker debian' diff --git a/.vsts-pr.yml b/.vsts-pr.yml index 4fb55d65ec64..229ab9fc075f 100644 --- a/.vsts-pr.yml +++ b/.vsts-pr.yml @@ -169,13 +169,6 @@ stages: _RuntimeIdentifier: '' _BuildArchitecture: 'x64' _TestArg: $(_NonWindowsTestArg) - Build_CentOS_7_1_Debug_x64: - _BuildConfig: Debug - _DockerParameter: '--docker centos' - _LinuxPortable: '' - _RuntimeIdentifier: '' - _BuildArchitecture: 'x64' - _TestArg: $(_NonWindowsTestArg) Build_Debian_11_Debug_x64: _BuildConfig: Debug _DockerParameter: '--docker debian' From 4b96a39553df9c8cadc77c887b433d835b4c1bc5 Mon Sep 17 00:00:00 2001 From: Farhad Alizada Date: Wed, 3 Jul 2024 09:18:43 +0200 Subject: [PATCH 16/21] Target the configuration file branch to release/8.0.4xx --- .github/workflows/inter-branch-merge-flow.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/inter-branch-merge-flow.yml b/.github/workflows/inter-branch-merge-flow.yml index 7672d36811f9..89e1a6530479 100644 --- a/.github/workflows/inter-branch-merge-flow.yml +++ b/.github/workflows/inter-branch-merge-flow.yml @@ -10,4 +10,6 @@ permissions: jobs: Merge: - uses: dotnet/arcade/.github/workflows/inter-branch-merge-base.yml@main \ No newline at end of file + uses: dotnet/arcade/.github/workflows/inter-branch-merge-base.yml@main + with: + configuration_file_branch: 'release/8.0.4xx' \ No newline at end of file From 37e3da1524d81095daa9a99907f09459518eaf3a Mon Sep 17 00:00:00 2001 From: Matt Mitchell Date: Wed, 3 Jul 2024 08:13:23 -0700 Subject: [PATCH 17/21] [release/6.0.4xx] Switch to dSAS for internal runtimes (#19939) --- .vsts-ci.yml | 4 +- .vsts-pr.yml | 4 +- eng/Version.Details.xml | 12 +++--- eng/Versions.props | 2 +- eng/build-pr.yml | 2 +- eng/build.yml | 1 + .../templates-official/job/source-build.yml | 8 ++++ .../job/source-index-stage1.yml | 35 ++++++++++----- .../templates-official/jobs/source-build.yml | 8 ++++ .../post-build/common-variables.yml | 1 - .../steps/enable-internal-runtimes.yml | 28 ++++++++++++ .../steps/get-delegation-sas.yml | 43 +++++++++++++++++++ .../steps/get-federated-access-token.yml | 28 ++++++++++++ eng/common/templates/job/source-build.yml | 8 ++++ .../templates/job/source-index-stage1.yml | 35 ++++++++++----- eng/common/templates/jobs/source-build.yml | 8 ++++ .../templates/post-build/common-variables.yml | 1 - .../steps/enable-internal-runtimes.yml | 28 ++++++++++++ .../templates/steps/get-delegation-sas.yml | 43 +++++++++++++++++++ .../steps/get-federated-access-token.yml | 28 ++++++++++++ global.json | 4 +- 21 files changed, 295 insertions(+), 36 deletions(-) create mode 100644 eng/common/templates-official/steps/enable-internal-runtimes.yml create mode 100644 eng/common/templates-official/steps/get-delegation-sas.yml create mode 100644 eng/common/templates-official/steps/get-federated-access-token.yml create mode 100644 eng/common/templates/steps/enable-internal-runtimes.yml create mode 100644 eng/common/templates/steps/get-delegation-sas.yml create mode 100644 eng/common/templates/steps/get-federated-access-token.yml diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 35964b6cab7e..52847992578e 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -19,7 +19,6 @@ variables: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - name: Codeql.Enabled value: true - - group: DotNet-DotNetCli-Storage - group: DotNet-Installer-SDLValidation-Params - name: _PublishUsingPipelines value: true @@ -36,7 +35,6 @@ variables: - name: _InternalRuntimeDownloadArgs value: '' - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - group: DotNetBuilds storage account read tokens - name: _InternalRuntimeDownloadArgs value: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) @@ -412,6 +410,8 @@ extends: _TestArg: '' - template: /eng/common/templates-official/jobs/source-build.yml@self + parameters: + enableInternalSources: true - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball.yml@self diff --git a/.vsts-pr.yml b/.vsts-pr.yml index b1590392b32e..daac9010733d 100644 --- a/.vsts-pr.yml +++ b/.vsts-pr.yml @@ -17,7 +17,6 @@ variables: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - name: Codeql.Enabled value: true - - group: DotNet-DotNetCli-Storage - group: DotNet-Installer-SDLValidation-Params - name: _PublishUsingPipelines value: true @@ -34,7 +33,6 @@ variables: - name: _InternalRuntimeDownloadArgs value: '' - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - group: DotNetBuilds storage account read tokens - name: _InternalRuntimeDownloadArgs value: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) @@ -364,6 +362,8 @@ stages: _TestArg: '' - template: /eng/common/templates/jobs/source-build.yml + parameters: + enableInternalSources: true - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball-pr.yml diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 512605343b09..3775c4ec5840 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -197,19 +197,19 @@ - + https://github.com/dotnet/arcade - 2eab07c3d7b78219d10099b19fafeef2ecae1779 + 1844d819e6f33f1106083c5066ea01e0310eefa3 - + https://github.com/dotnet/arcade - 2eab07c3d7b78219d10099b19fafeef2ecae1779 + 1844d819e6f33f1106083c5066ea01e0310eefa3 - + https://github.com/dotnet/arcade - 2eab07c3d7b78219d10099b19fafeef2ecae1779 + 1844d819e6f33f1106083c5066ea01e0310eefa3 https://github.com/dotnet/source-build-reference-packages diff --git a/eng/Versions.props b/eng/Versions.props index 267e282eea50..d09290b7a0b7 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -19,7 +19,7 @@ - 6.0.0-beta.24266.4 + 6.0.0-beta.24326.2 diff --git a/eng/build-pr.yml b/eng/build-pr.yml index f05d66fa690d..653774872f18 100644 --- a/eng/build-pr.yml +++ b/eng/build-pr.yml @@ -66,6 +66,7 @@ phases: steps: - checkout: self clean: true + - template: /eng/common/templates/steps/enable-internal-runtimes.yml - ${{ if eq(parameters.agentOs, 'Windows_NT') }}: - ${{ if ne(variables['System.TeamProject'], 'public') }}: - task: PowerShell@2 @@ -96,7 +97,6 @@ phases: arguments: $(Build.SourcesDirectory)/NuGet.config $Token env: Token: $(dn-bot-dnceng-artifact-feeds-rw) - - ${{ if eq(parameters.agentOs, 'Linux') }}: - script: ./build.sh $(_TestArg) $(_PackArg) diff --git a/eng/build.yml b/eng/build.yml index 2ce873c3d4d8..84d86839b0b8 100644 --- a/eng/build.yml +++ b/eng/build.yml @@ -66,6 +66,7 @@ phases: steps: - checkout: self clean: true + - template: /eng/common/templates/steps/enable-internal-runtimes.yml - ${{ if eq(parameters.agentOs, 'Windows_NT') }}: - ${{ if ne(variables['System.TeamProject'], 'public') }}: - task: PowerShell@2 diff --git a/eng/common/templates-official/job/source-build.yml b/eng/common/templates-official/job/source-build.yml index f5fa09f41511..a6f7efa77685 100644 --- a/eng/common/templates-official/job/source-build.yml +++ b/eng/common/templates-official/job/source-build.yml @@ -31,6 +31,12 @@ parameters: # container and pool. platform: {} + # If set to true and running on a non-public project, + # Internal blob storage locations will be enabled. + # This is not enabled by default because many repositories do not need internal sources + # and do not need to have the required service connections approved in the pipeline. + enableInternalSources: false + jobs: - job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} displayName: Source-Build (${{ parameters.platform.name }}) @@ -59,6 +65,8 @@ jobs: clean: all steps: + - ${{ if eq(parameters.enableInternalSources, true) }}: + - template: /eng/common/templates-official/steps/enable-internal-runtimes.yml - template: /eng/common/templates-official/steps/source-build.yml parameters: platform: ${{ parameters.platform }} diff --git a/eng/common/templates-official/job/source-index-stage1.yml b/eng/common/templates-official/job/source-index-stage1.yml index f04ad04c2b18..f6f0c15bfba4 100644 --- a/eng/common/templates-official/job/source-index-stage1.yml +++ b/eng/common/templates-official/job/source-index-stage1.yml @@ -1,6 +1,7 @@ parameters: runAsPublic: false - sourceIndexPackageVersion: 1.0.1-20240320.1 + sourceIndexUploadPackageVersion: 2.0.0-20240502.12 + sourceIndexProcessBinlogPackageVersion: 1.0.1-20240129.2 sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" preSteps: [] @@ -17,14 +18,14 @@ jobs: dependsOn: ${{ parameters.dependsOn }} condition: ${{ parameters.condition }} variables: - - name: SourceIndexPackageVersion - value: ${{ parameters.sourceIndexPackageVersion }} + - name: SourceIndexUploadPackageVersion + value: ${{ parameters.sourceIndexUploadPackageVersion }} + - name: SourceIndexProcessBinlogPackageVersion + value: ${{ parameters.sourceIndexProcessBinlogPackageVersion }} - name: SourceIndexPackageSource value: ${{ parameters.sourceIndexPackageSource }} - name: BinlogPath value: ${{ parameters.binlogPath }} - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: source-dot-net stage1 variables pool: ${{ parameters.pool }} steps: @@ -40,8 +41,8 @@ jobs: workingDirectory: $(Agent.TempDirectory) - script: | - $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools - $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(sourceIndexProcessBinlogPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(sourceIndexUploadPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools displayName: Download Tools # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. workingDirectory: $(Agent.TempDirectory) @@ -53,7 +54,21 @@ jobs: displayName: Process Binlog into indexable sln - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) + - task: AzureCLI@2 + displayName: Get stage 1 auth token + inputs: + azureSubscription: 'SourceDotNet Stage1 Publish' + addSpnToEnvironment: true + scriptType: 'ps' + scriptLocation: 'inlineScript' + inlineScript: | + echo "##vso[task.setvariable variable=ARM_CLIENT_ID;issecret=true]$env:servicePrincipalId" + echo "##vso[task.setvariable variable=ARM_ID_TOKEN;issecret=true]$env:idToken" + echo "##vso[task.setvariable variable=ARM_TENANT_ID;issecret=true]$env:tenantId" + + - script: | + az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN) + displayName: "Login to Azure" + + - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1 displayName: Upload stage1 artifacts to source index - env: - BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) diff --git a/eng/common/templates-official/jobs/source-build.yml b/eng/common/templates-official/jobs/source-build.yml index b9a1f67b9a9a..9f60019cb01d 100644 --- a/eng/common/templates-official/jobs/source-build.yml +++ b/eng/common/templates-official/jobs/source-build.yml @@ -21,6 +21,12 @@ parameters: # one job runs on 'defaultManagedPlatform'. platforms: [] + # If set to true and running on a non-public project, + # Internal nuget and blob storage locations will be enabled. + # This is not enabled by default because many repositories do not need internal sources + # and do not need to have the required service connections approved in the pipeline. + enableInternalSources: false + jobs: - ${{ if ne(parameters.allCompletedJobId, '') }}: @@ -38,9 +44,11 @@ jobs: parameters: jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ platform }} + enableInternalSources: ${{ parameters.enableInternalSources }} - ${{ if eq(length(parameters.platforms), 0) }}: - template: /eng/common/templates-official/job/source-build.yml parameters: jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ parameters.defaultManagedPlatform }} + enableInternalSources: ${{ parameters.enableInternalSources }} diff --git a/eng/common/templates-official/post-build/common-variables.yml b/eng/common/templates-official/post-build/common-variables.yml index fae340f4d20d..5404cc7ecbef 100644 --- a/eng/common/templates-official/post-build/common-variables.yml +++ b/eng/common/templates-official/post-build/common-variables.yml @@ -2,7 +2,6 @@ variables: - group: AzureDevOps-Artifact-Feeds-Pats - group: DotNet-Blob-Feed - group: DotNet-DotNetCli-Storage - - group: DotNet-MSRC-Storage - group: Publish-Build-Assets # Whether the build is internal or not diff --git a/eng/common/templates-official/steps/enable-internal-runtimes.yml b/eng/common/templates-official/steps/enable-internal-runtimes.yml new file mode 100644 index 000000000000..93a8394a666b --- /dev/null +++ b/eng/common/templates-official/steps/enable-internal-runtimes.yml @@ -0,0 +1,28 @@ +# Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64' +# variable with the base64-encoded SAS token, by default + +parameters: +- name: federatedServiceConnection + type: string + default: 'dotnetbuilds-internal-read' +- name: outputVariableName + type: string + default: 'dotnetbuilds-internal-container-read-token-base64' +- name: expiryInHours + type: number + default: 1 +- name: base64Encode + type: boolean + default: true + +steps: +- ${{ if ne(variables['System.TeamProject'], 'public') }}: + - template: /eng/common/templates-official/steps/get-delegation-sas.yml + parameters: + federatedServiceConnection: ${{ parameters.federatedServiceConnection }} + outputVariableName: ${{ parameters.outputVariableName }} + expiryInHours: ${{ parameters.expiryInHours }} + base64Encode: ${{ parameters.base64Encode }} + storageAccount: dotnetbuilds + container: internal + permissions: rl diff --git a/eng/common/templates-official/steps/get-delegation-sas.yml b/eng/common/templates-official/steps/get-delegation-sas.yml new file mode 100644 index 000000000000..c0e8f91317f0 --- /dev/null +++ b/eng/common/templates-official/steps/get-delegation-sas.yml @@ -0,0 +1,43 @@ +parameters: +- name: federatedServiceConnection + type: string +- name: outputVariableName + type: string +- name: expiryInHours + type: number + default: 1 +- name: base64Encode + type: boolean + default: false +- name: storageAccount + type: string +- name: container + type: string +- name: permissions + type: string + default: 'rl' + +steps: +- task: AzureCLI@2 + displayName: 'Generate delegation SAS Token for ${{ parameters.storageAccount }}/${{ parameters.container }}' + inputs: + azureSubscription: ${{ parameters.federatedServiceConnection }} + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + # Calculate the expiration of the SAS token and convert to UTC + $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ") + + $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv + + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to generate SAS token." + exit 1 + } + + if ('${{ parameters.base64Encode }}' -eq 'true') { + $sas = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($sas)) + } + + Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$sas" diff --git a/eng/common/templates-official/steps/get-federated-access-token.yml b/eng/common/templates-official/steps/get-federated-access-token.yml new file mode 100644 index 000000000000..e3786cef6dfd --- /dev/null +++ b/eng/common/templates-official/steps/get-federated-access-token.yml @@ -0,0 +1,28 @@ +parameters: +- name: federatedServiceConnection + type: string +- name: outputVariableName + type: string +# Resource to get a token for. Common values include: +# - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps +# - 'https://storage.azure.com/' for storage +# Defaults to Azure DevOps +- name: resource + type: string + default: '499b84ac-1321-427f-aa17-267ca6975798' + +steps: +- task: AzureCLI@2 + displayName: 'Getting federated access token for feeds' + inputs: + azureSubscription: ${{ parameters.federatedServiceConnection }} + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + $accessToken = az account get-access-token --query accessToken --resource ${{ parameters.resource }} --output tsv + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to get access token for resource '${{ parameters.resource }}'" + exit 1 + } + Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$accessToken" diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml index b6137f44ada1..8009ccb954ac 100644 --- a/eng/common/templates/job/source-build.yml +++ b/eng/common/templates/job/source-build.yml @@ -31,6 +31,12 @@ parameters: # container and pool. platform: {} + # If set to true and running on a non-public project, + # Internal blob storage locations will be enabled. + # This is not enabled by default because many repositories do not need internal sources + # and do not need to have the required service connections approved in the pipeline. + enableInternalSources: false + jobs: - job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} displayName: Source-Build (${{ parameters.platform.name }}) @@ -58,6 +64,8 @@ jobs: clean: all steps: + - ${{ if eq(parameters.enableInternalSources, true) }}: + - template: /eng/common/templates/steps/enable-internal-runtimes.yml - template: /eng/common/templates/steps/source-build.yml parameters: platform: ${{ parameters.platform }} diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml index b710698eb4d4..47f36e98bc16 100644 --- a/eng/common/templates/job/source-index-stage1.yml +++ b/eng/common/templates/job/source-index-stage1.yml @@ -1,6 +1,7 @@ parameters: runAsPublic: false - sourceIndexPackageVersion: 1.0.1-20240320.1 + sourceIndexUploadPackageVersion: 2.0.0-20240502.12 + sourceIndexProcessBinlogPackageVersion: 1.0.1-20240129.2 sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" preSteps: [] @@ -15,14 +16,14 @@ jobs: dependsOn: ${{ parameters.dependsOn }} condition: ${{ parameters.condition }} variables: - - name: SourceIndexPackageVersion - value: ${{ parameters.sourceIndexPackageVersion }} + - name: SourceIndexUploadPackageVersion + value: ${{ parameters.sourceIndexUploadPackageVersion }} + - name: SourceIndexProcessBinlogPackageVersion + value: ${{ parameters.sourceIndexProcessBinlogPackageVersion }} - name: SourceIndexPackageSource value: ${{ parameters.sourceIndexPackageSource }} - name: BinlogPath value: ${{ parameters.binlogPath }} - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: source-dot-net stage1 variables pool: ${{ parameters.pool }} steps: @@ -38,8 +39,8 @@ jobs: workingDirectory: $(Agent.TempDirectory) - script: | - $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools - $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(sourceIndexProcessBinlogPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(sourceIndexUploadPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools displayName: Download Tools # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. workingDirectory: $(Agent.TempDirectory) @@ -51,7 +52,21 @@ jobs: displayName: Process Binlog into indexable sln - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) + - task: AzureCLI@2 + displayName: Get stage 1 auth token + inputs: + azureSubscription: 'SourceDotNet Stage1 Publish' + addSpnToEnvironment: true + scriptType: 'ps' + scriptLocation: 'inlineScript' + inlineScript: | + echo "##vso[task.setvariable variable=ARM_CLIENT_ID;issecret=true]$env:servicePrincipalId" + echo "##vso[task.setvariable variable=ARM_ID_TOKEN;issecret=true]$env:idToken" + echo "##vso[task.setvariable variable=ARM_TENANT_ID;issecret=true]$env:tenantId" + + - script: | + az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN) + displayName: "Login to Azure" + + - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1 displayName: Upload stage1 artifacts to source index - env: - BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) diff --git a/eng/common/templates/jobs/source-build.yml b/eng/common/templates/jobs/source-build.yml index 7c240e654472..e3fbd9f38e10 100644 --- a/eng/common/templates/jobs/source-build.yml +++ b/eng/common/templates/jobs/source-build.yml @@ -21,6 +21,12 @@ parameters: # one job runs on 'defaultManagedPlatform'. platforms: [] + # If set to true and running on a non-public project, + # Internal nuget and blob storage locations will be enabled. + # This is not enabled by default because many repositories do not need internal sources + # and do not need to have the required service connections approved in the pipeline. + enableInternalSources: false + jobs: - ${{ if ne(parameters.allCompletedJobId, '') }}: @@ -38,9 +44,11 @@ jobs: parameters: jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ platform }} + enableInternalSources: ${{ parameters.enableInternalSources }} - ${{ if eq(length(parameters.platforms), 0) }}: - template: /eng/common/templates/job/source-build.yml parameters: jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ parameters.defaultManagedPlatform }} + enableInternalSources: ${{ parameters.enableInternalSources }} diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml index fae340f4d20d..5404cc7ecbef 100644 --- a/eng/common/templates/post-build/common-variables.yml +++ b/eng/common/templates/post-build/common-variables.yml @@ -2,7 +2,6 @@ variables: - group: AzureDevOps-Artifact-Feeds-Pats - group: DotNet-Blob-Feed - group: DotNet-DotNetCli-Storage - - group: DotNet-MSRC-Storage - group: Publish-Build-Assets # Whether the build is internal or not diff --git a/eng/common/templates/steps/enable-internal-runtimes.yml b/eng/common/templates/steps/enable-internal-runtimes.yml new file mode 100644 index 000000000000..54dc9416c519 --- /dev/null +++ b/eng/common/templates/steps/enable-internal-runtimes.yml @@ -0,0 +1,28 @@ +# Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64' +# variable with the base64-encoded SAS token, by default + +parameters: +- name: federatedServiceConnection + type: string + default: 'dotnetbuilds-internal-read' +- name: outputVariableName + type: string + default: 'dotnetbuilds-internal-container-read-token-base64' +- name: expiryInHours + type: number + default: 1 +- name: base64Encode + type: boolean + default: true + +steps: +- ${{ if ne(variables['System.TeamProject'], 'public') }}: + - template: /eng/common/templates/steps/get-delegation-sas.yml + parameters: + federatedServiceConnection: ${{ parameters.federatedServiceConnection }} + outputVariableName: ${{ parameters.outputVariableName }} + expiryInHours: ${{ parameters.expiryInHours }} + base64Encode: ${{ parameters.base64Encode }} + storageAccount: dotnetbuilds + container: internal + permissions: rl diff --git a/eng/common/templates/steps/get-delegation-sas.yml b/eng/common/templates/steps/get-delegation-sas.yml new file mode 100644 index 000000000000..c0e8f91317f0 --- /dev/null +++ b/eng/common/templates/steps/get-delegation-sas.yml @@ -0,0 +1,43 @@ +parameters: +- name: federatedServiceConnection + type: string +- name: outputVariableName + type: string +- name: expiryInHours + type: number + default: 1 +- name: base64Encode + type: boolean + default: false +- name: storageAccount + type: string +- name: container + type: string +- name: permissions + type: string + default: 'rl' + +steps: +- task: AzureCLI@2 + displayName: 'Generate delegation SAS Token for ${{ parameters.storageAccount }}/${{ parameters.container }}' + inputs: + azureSubscription: ${{ parameters.federatedServiceConnection }} + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + # Calculate the expiration of the SAS token and convert to UTC + $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ") + + $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv + + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to generate SAS token." + exit 1 + } + + if ('${{ parameters.base64Encode }}' -eq 'true') { + $sas = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($sas)) + } + + Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$sas" diff --git a/eng/common/templates/steps/get-federated-access-token.yml b/eng/common/templates/steps/get-federated-access-token.yml new file mode 100644 index 000000000000..c8c49cc0e8f0 --- /dev/null +++ b/eng/common/templates/steps/get-federated-access-token.yml @@ -0,0 +1,28 @@ +parameters: +- name: federatedServiceConnection + type: string +- name: outputVariableName + type: string +# Resource to get a token for. Common values include: +# - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps +# - 'https://storage.azure.com/' for storage +# Defaults to Azure DevOps +- name: resource + type: string + default: '499b84ac-1321-427f-aa17-267ca6975798' + +steps: +- task: AzureCLI@2 + displayName: 'Getting federated access token for feeds' + inputs: + azureSubscription: ${{ parameters.federatedServiceConnection }} + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + $accessToken = az account get-access-token --query accessToken --resource ${{ parameters.resource }} --output tsv + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to get access token for resource '${{ parameters.resource }}'" + exit 1 + } + Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$accessToken" \ No newline at end of file diff --git a/global.json b/global.json index f624a99df800..750534f91754 100644 --- a/global.json +++ b/global.json @@ -11,7 +11,7 @@ "cmake": "3.16.4" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.24266.4", - "Microsoft.DotNet.CMake.Sdk": "6.0.0-beta.24266.4" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.24326.2", + "Microsoft.DotNet.CMake.Sdk": "6.0.0-beta.24326.2" } } From 502a6d149a4b822dafd022d43a93b6a71a028655 Mon Sep 17 00:00:00 2001 From: Matt Mitchell Date: Wed, 3 Jul 2024 08:48:22 -0700 Subject: [PATCH 18/21] [release/8.0.1xx] Do not run TestSdkRpm target (#19950) --- src/redist/targets/GenerateRPMs.targets | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/redist/targets/GenerateRPMs.targets b/src/redist/targets/GenerateRPMs.targets index a82f068fcb12..b5d03d5a90c1 100644 --- a/src/redist/targets/GenerateRPMs.targets +++ b/src/redist/targets/GenerateRPMs.targets @@ -12,8 +12,10 @@ DependsOnTargets="GetCurrentRuntimeInformation; GenerateRpmsInner" /> + From 34a414a79dbed589edd7e524dfb906ac85115dc2 Mon Sep 17 00:00:00 2001 From: Matt Mitchell Date: Wed, 3 Jul 2024 08:48:50 -0700 Subject: [PATCH 19/21] [release/8.0.1xx] Switch to dSAS for internal runtimes (#19945) --- .vsts-ci.yml | 4 +- .vsts-pr.yml | 4 +- eng/Version.Details.xml | 12 +++--- eng/Versions.props | 2 +- eng/build-pr.yml | 1 + eng/build.yml | 1 + eng/common/post-build/publish-using-darc.ps1 | 15 +++---- .../job/publish-build-assets.yml | 12 +++--- .../templates-official/job/source-build.yml | 8 ++++ .../job/source-index-stage1.yml | 16 +++---- .../templates-official/jobs/source-build.yml | 8 ++++ .../post-build/post-build.yml | 8 ++-- .../steps/enable-internal-runtimes.yml | 28 ++++++++++++ .../steps/get-delegation-sas.yml | 43 +++++++++++++++++++ .../steps/get-federated-access-token.yml | 28 ++++++++++++ .../templates/job/publish-build-assets.yml | 12 +++--- eng/common/templates/job/source-build.yml | 8 ++++ .../templates/job/source-index-stage1.yml | 11 ++--- eng/common/templates/jobs/source-build.yml | 8 ++++ .../templates/post-build/post-build.yml | 8 ++-- .../post-build/setup-maestro-vars.yml | 28 ++++++------ .../steps/enable-internal-runtimes.yml | 28 ++++++++++++ .../templates/steps/get-delegation-sas.yml | 43 +++++++++++++++++++ .../steps/get-federated-access-token.yml | 28 ++++++++++++ global.json | 4 +- 25 files changed, 301 insertions(+), 67 deletions(-) create mode 100644 eng/common/templates-official/steps/enable-internal-runtimes.yml create mode 100644 eng/common/templates-official/steps/get-delegation-sas.yml create mode 100644 eng/common/templates-official/steps/get-federated-access-token.yml create mode 100644 eng/common/templates/steps/enable-internal-runtimes.yml create mode 100644 eng/common/templates/steps/get-delegation-sas.yml create mode 100644 eng/common/templates/steps/get-federated-access-token.yml diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 55787da54f6c..cd29cedc9092 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -21,14 +21,12 @@ variables: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - name: Codeql.Enabled value: true - - group: DotNet-DotNetCli-Storage - group: DotNet-Installer-SDLValidation-Params - name: _PublishUsingPipelines value: true - name: _InternalRuntimeDownloadArgs value: '' - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - group: DotNetBuilds storage account read tokens - name: _InternalRuntimeDownloadArgs value: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) @@ -334,6 +332,8 @@ extends: # Source Build - template: /eng/common/templates-official/jobs/source-build.yml@self + parameters: + enableInternalSources: true - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - stage: Publish diff --git a/.vsts-pr.yml b/.vsts-pr.yml index ad4c8979503f..4176f4fb1ce5 100644 --- a/.vsts-pr.yml +++ b/.vsts-pr.yml @@ -19,7 +19,6 @@ variables: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - name: Codeql.Enabled value: true - - group: DotNet-DotNetCli-Storage - group: DotNet-Installer-SDLValidation-Params - name: _PublishUsingPipelines value: true @@ -28,7 +27,6 @@ variables: value: '' - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - group: DotNetBuilds storage account read tokens - name: _InternalRuntimeDownloadArgs value: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) @@ -345,6 +343,8 @@ stages: runTests: false - template: /eng/common/templates/jobs/source-build.yml + parameters: + enableInternalSources: true - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - stage: Publish diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index dca855301a4f..cebc01e95e5a 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -219,18 +219,18 @@ - + https://github.com/dotnet/arcade - e6f70c7dd528f05cd28cec2a179d58c22e91d9ac + 8b879da4e449c48d99f3f642fc429379a64e8fe8 - + https://github.com/dotnet/arcade - e6f70c7dd528f05cd28cec2a179d58c22e91d9ac + 8b879da4e449c48d99f3f642fc429379a64e8fe8 - + https://github.com/dotnet/arcade - e6f70c7dd528f05cd28cec2a179d58c22e91d9ac + 8b879da4e449c48d99f3f642fc429379a64e8fe8 https://github.com/dotnet/arcade-services diff --git a/eng/Versions.props b/eng/Versions.props index c0c8666a89da..4cddaab3ecad 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -40,7 +40,7 @@ - 8.0.0-beta.24266.3 + 8.0.0-beta.24352.1 diff --git a/eng/build-pr.yml b/eng/build-pr.yml index 831d96781f0a..d86f37aabdbb 100644 --- a/eng/build-pr.yml +++ b/eng/build-pr.yml @@ -173,6 +173,7 @@ jobs: steps: - checkout: self clean: true + - template: /eng/common/templates/steps/enable-internal-runtimes.yml - ${{ if eq(parameters.agentOs, 'Windows_NT') }}: - ${{ if and(not(parameters.isBuiltFromVmr), ne(variables['System.TeamProject'], 'public')) }}: - task: PowerShell@2 diff --git a/eng/build.yml b/eng/build.yml index 4f947ea8f7ea..23b1c7148061 100644 --- a/eng/build.yml +++ b/eng/build.yml @@ -179,6 +179,7 @@ jobs: steps: - checkout: self clean: true + - template: /eng/common/templates/steps/enable-internal-runtimes.yml - ${{ if eq(parameters.agentOs, 'Windows_NT') }}: - ${{ if and(not(parameters.isBuiltFromVmr), ne(variables['System.TeamProject'], 'public')) }}: - task: PowerShell@2 diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1 index 5a3a32ea8d75..238945cb5ab4 100644 --- a/eng/common/post-build/publish-using-darc.ps1 +++ b/eng/common/post-build/publish-using-darc.ps1 @@ -2,7 +2,6 @@ param( [Parameter(Mandatory=$true)][int] $BuildId, [Parameter(Mandatory=$true)][int] $PublishingInfraVersion, [Parameter(Mandatory=$true)][string] $AzdoToken, - [Parameter(Mandatory=$true)][string] $MaestroToken, [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro.dot.net', [Parameter(Mandatory=$true)][string] $WaitPublishingFinish, [Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters, @@ -31,13 +30,13 @@ try { } & $darc add-build-to-channel ` - --id $buildId ` - --publishing-infra-version $PublishingInfraVersion ` - --default-channels ` - --source-branch main ` - --azdev-pat $AzdoToken ` - --bar-uri $MaestroApiEndPoint ` - --password $MaestroToken ` + --id $buildId ` + --publishing-infra-version $PublishingInfraVersion ` + --default-channels ` + --source-branch main ` + --azdev-pat "$AzdoToken" ` + --bar-uri "$MaestroApiEndPoint" ` + --ci ` @optionalParams if ($LastExitCode -ne 0) { diff --git a/eng/common/templates-official/job/publish-build-assets.yml b/eng/common/templates-official/job/publish-build-assets.yml index 589ac80a18b7..d01739c12857 100644 --- a/eng/common/templates-official/job/publish-build-assets.yml +++ b/eng/common/templates-official/job/publish-build-assets.yml @@ -76,13 +76,16 @@ jobs: - task: NuGetAuthenticate@1 - - task: PowerShell@2 + - task: AzureCLI@2 displayName: Publish Build Assets inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1 + arguments: > + -task PublishBuildAssets -restore -msbuildEngine dotnet /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' - /p:BuildAssetRegistryToken=$(MaestroAccessToken) /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} /p:OfficialBuildId=$(Build.BuildNumber) @@ -144,7 +147,6 @@ jobs: arguments: -BuildId $(BARBuildId) -PublishingInfraVersion 3 -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' - -MaestroToken '$(MaestroApiAccessToken)' -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates-official/job/source-build.yml b/eng/common/templates-official/job/source-build.yml index f193dfbe2366..f983033bb028 100644 --- a/eng/common/templates-official/job/source-build.yml +++ b/eng/common/templates-official/job/source-build.yml @@ -31,6 +31,12 @@ parameters: # container and pool. platform: {} + # If set to true and running on a non-public project, + # Internal blob storage locations will be enabled. + # This is not enabled by default because many repositories do not need internal sources + # and do not need to have the required service connections approved in the pipeline. + enableInternalSources: false + jobs: - job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} displayName: Source-Build (${{ parameters.platform.name }}) @@ -62,6 +68,8 @@ jobs: clean: all steps: + - ${{ if eq(parameters.enableInternalSources, true) }}: + - template: /eng/common/templates-official/steps/enable-internal-runtimes.yml - template: /eng/common/templates-official/steps/source-build.yml parameters: platform: ${{ parameters.platform }} diff --git a/eng/common/templates-official/job/source-index-stage1.yml b/eng/common/templates-official/job/source-index-stage1.yml index 43ee0c202fc7..60dfb6b2d1c0 100644 --- a/eng/common/templates-official/job/source-index-stage1.yml +++ b/eng/common/templates-official/job/source-index-stage1.yml @@ -23,7 +23,7 @@ jobs: value: ${{ parameters.sourceIndexPackageSource }} - name: BinlogPath value: ${{ parameters.binlogPath }} - - template: /eng/common/templates/variables/pool-providers.yml + - template: /eng/common/templates-official/variables/pool-providers.yml ${{ if ne(parameters.pool, '') }}: pool: ${{ parameters.pool }} @@ -34,7 +34,8 @@ jobs: demands: ImageOverride -equals windows.vs2019.amd64.open ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2019.amd64 + image: windows.vs2022.amd64 + os: windows steps: - ${{ each preStep in parameters.preSteps }}: @@ -70,16 +71,13 @@ jobs: scriptType: 'ps' scriptLocation: 'inlineScript' inlineScript: | - echo "##vso[task.setvariable variable=ARM_CLIENT_ID]$env:servicePrincipalId" - echo "##vso[task.setvariable variable=ARM_ID_TOKEN]$env:idToken" - echo "##vso[task.setvariable variable=ARM_TENANT_ID]$env:tenantId" + echo "##vso[task.setvariable variable=ARM_CLIENT_ID;issecret=true]$env:servicePrincipalId" + echo "##vso[task.setvariable variable=ARM_ID_TOKEN;issecret=true]$env:idToken" + echo "##vso[task.setvariable variable=ARM_TENANT_ID;issecret=true]$env:tenantId" - script: | - echo "Client ID: $(ARM_CLIENT_ID)" - echo "ID Token: $(ARM_ID_TOKEN)" - echo "Tenant ID: $(ARM_TENANT_ID)" az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN) displayName: "Login to Azure" - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1 - displayName: Upload stage1 artifacts to source index \ No newline at end of file + displayName: Upload stage1 artifacts to source index diff --git a/eng/common/templates-official/jobs/source-build.yml b/eng/common/templates-official/jobs/source-build.yml index 08e5db9bb116..5cf6a269c0b6 100644 --- a/eng/common/templates-official/jobs/source-build.yml +++ b/eng/common/templates-official/jobs/source-build.yml @@ -21,6 +21,12 @@ parameters: # one job runs on 'defaultManagedPlatform'. platforms: [] + # If set to true and running on a non-public project, + # Internal nuget and blob storage locations will be enabled. + # This is not enabled by default because many repositories do not need internal sources + # and do not need to have the required service connections approved in the pipeline. + enableInternalSources: false + jobs: - ${{ if ne(parameters.allCompletedJobId, '') }}: @@ -38,9 +44,11 @@ jobs: parameters: jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ platform }} + enableInternalSources: ${{ parameters.enableInternalSources }} - ${{ if eq(length(parameters.platforms), 0) }}: - template: /eng/common/templates-official/job/source-build.yml parameters: jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ parameters.defaultManagedPlatform }} + enableInternalSources: ${{ parameters.enableInternalSources }} diff --git a/eng/common/templates-official/post-build/post-build.yml b/eng/common/templates-official/post-build/post-build.yml index da1f40958b45..0dfa387e7b78 100644 --- a/eng/common/templates-official/post-build/post-build.yml +++ b/eng/common/templates-official/post-build/post-build.yml @@ -272,14 +272,16 @@ stages: - task: NuGetAuthenticate@1 - - task: PowerShell@2 + - task: AzureCLI@2 displayName: Publish Using Darc inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 arguments: -BuildId $(BARBuildId) -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' - -MaestroToken '$(MaestroApiAccessToken)' -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates-official/steps/enable-internal-runtimes.yml b/eng/common/templates-official/steps/enable-internal-runtimes.yml new file mode 100644 index 000000000000..93a8394a666b --- /dev/null +++ b/eng/common/templates-official/steps/enable-internal-runtimes.yml @@ -0,0 +1,28 @@ +# Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64' +# variable with the base64-encoded SAS token, by default + +parameters: +- name: federatedServiceConnection + type: string + default: 'dotnetbuilds-internal-read' +- name: outputVariableName + type: string + default: 'dotnetbuilds-internal-container-read-token-base64' +- name: expiryInHours + type: number + default: 1 +- name: base64Encode + type: boolean + default: true + +steps: +- ${{ if ne(variables['System.TeamProject'], 'public') }}: + - template: /eng/common/templates-official/steps/get-delegation-sas.yml + parameters: + federatedServiceConnection: ${{ parameters.federatedServiceConnection }} + outputVariableName: ${{ parameters.outputVariableName }} + expiryInHours: ${{ parameters.expiryInHours }} + base64Encode: ${{ parameters.base64Encode }} + storageAccount: dotnetbuilds + container: internal + permissions: rl diff --git a/eng/common/templates-official/steps/get-delegation-sas.yml b/eng/common/templates-official/steps/get-delegation-sas.yml new file mode 100644 index 000000000000..c0e8f91317f0 --- /dev/null +++ b/eng/common/templates-official/steps/get-delegation-sas.yml @@ -0,0 +1,43 @@ +parameters: +- name: federatedServiceConnection + type: string +- name: outputVariableName + type: string +- name: expiryInHours + type: number + default: 1 +- name: base64Encode + type: boolean + default: false +- name: storageAccount + type: string +- name: container + type: string +- name: permissions + type: string + default: 'rl' + +steps: +- task: AzureCLI@2 + displayName: 'Generate delegation SAS Token for ${{ parameters.storageAccount }}/${{ parameters.container }}' + inputs: + azureSubscription: ${{ parameters.federatedServiceConnection }} + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + # Calculate the expiration of the SAS token and convert to UTC + $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ") + + $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv + + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to generate SAS token." + exit 1 + } + + if ('${{ parameters.base64Encode }}' -eq 'true') { + $sas = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($sas)) + } + + Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$sas" diff --git a/eng/common/templates-official/steps/get-federated-access-token.yml b/eng/common/templates-official/steps/get-federated-access-token.yml new file mode 100644 index 000000000000..e3786cef6dfd --- /dev/null +++ b/eng/common/templates-official/steps/get-federated-access-token.yml @@ -0,0 +1,28 @@ +parameters: +- name: federatedServiceConnection + type: string +- name: outputVariableName + type: string +# Resource to get a token for. Common values include: +# - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps +# - 'https://storage.azure.com/' for storage +# Defaults to Azure DevOps +- name: resource + type: string + default: '499b84ac-1321-427f-aa17-267ca6975798' + +steps: +- task: AzureCLI@2 + displayName: 'Getting federated access token for feeds' + inputs: + azureSubscription: ${{ parameters.federatedServiceConnection }} + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + $accessToken = az account get-access-token --query accessToken --resource ${{ parameters.resource }} --output tsv + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to get access token for resource '${{ parameters.resource }}'" + exit 1 + } + Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$accessToken" diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index 8ec0151def21..9fd69fa7c9b7 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -74,13 +74,16 @@ jobs: - task: NuGetAuthenticate@1 - - task: PowerShell@2 + - task: AzureCLI@2 displayName: Publish Build Assets inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1 + arguments: > + -task PublishBuildAssets -restore -msbuildEngine dotnet /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' - /p:BuildAssetRegistryToken=$(MaestroAccessToken) /p:MaestroApiEndpoint=https://maestro.dot.net /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} /p:OfficialBuildId=$(Build.BuildNumber) @@ -140,7 +143,6 @@ jobs: arguments: -BuildId $(BARBuildId) -PublishingInfraVersion 3 -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' - -MaestroToken '$(MaestroApiAccessToken)' -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml index 8a3deef2b727..c0ff472b697b 100644 --- a/eng/common/templates/job/source-build.yml +++ b/eng/common/templates/job/source-build.yml @@ -31,6 +31,12 @@ parameters: # container and pool. platform: {} + # If set to true and running on a non-public project, + # Internal blob storage locations will be enabled. + # This is not enabled by default because many repositories do not need internal sources + # and do not need to have the required service connections approved in the pipeline. + enableInternalSources: false + jobs: - job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} displayName: Source-Build (${{ parameters.platform.name }}) @@ -61,6 +67,8 @@ jobs: clean: all steps: + - ${{ if eq(parameters.enableInternalSources, true) }}: + - template: /eng/common/templates/steps/enable-internal-runtimes.yml - template: /eng/common/templates/steps/source-build.yml parameters: platform: ${{ parameters.platform }} diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml index 43ee0c202fc7..0b6bb89dc78a 100644 --- a/eng/common/templates/job/source-index-stage1.yml +++ b/eng/common/templates/job/source-index-stage1.yml @@ -70,16 +70,13 @@ jobs: scriptType: 'ps' scriptLocation: 'inlineScript' inlineScript: | - echo "##vso[task.setvariable variable=ARM_CLIENT_ID]$env:servicePrincipalId" - echo "##vso[task.setvariable variable=ARM_ID_TOKEN]$env:idToken" - echo "##vso[task.setvariable variable=ARM_TENANT_ID]$env:tenantId" + echo "##vso[task.setvariable variable=ARM_CLIENT_ID;issecret=true]$env:servicePrincipalId" + echo "##vso[task.setvariable variable=ARM_ID_TOKEN;issecret=true]$env:idToken" + echo "##vso[task.setvariable variable=ARM_TENANT_ID;issecret=true]$env:tenantId" - script: | - echo "Client ID: $(ARM_CLIENT_ID)" - echo "ID Token: $(ARM_ID_TOKEN)" - echo "Tenant ID: $(ARM_TENANT_ID)" az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN) displayName: "Login to Azure" - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1 - displayName: Upload stage1 artifacts to source index \ No newline at end of file + displayName: Upload stage1 artifacts to source index diff --git a/eng/common/templates/jobs/source-build.yml b/eng/common/templates/jobs/source-build.yml index a15b07eb51d9..5f46bfa895c1 100644 --- a/eng/common/templates/jobs/source-build.yml +++ b/eng/common/templates/jobs/source-build.yml @@ -21,6 +21,12 @@ parameters: # one job runs on 'defaultManagedPlatform'. platforms: [] + # If set to true and running on a non-public project, + # Internal nuget and blob storage locations will be enabled. + # This is not enabled by default because many repositories do not need internal sources + # and do not need to have the required service connections approved in the pipeline. + enableInternalSources: false + jobs: - ${{ if ne(parameters.allCompletedJobId, '') }}: @@ -38,9 +44,11 @@ jobs: parameters: jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ platform }} + enableInternalSources: ${{ parameters.enableInternalSources }} - ${{ if eq(length(parameters.platforms), 0) }}: - template: /eng/common/templates/job/source-build.yml parameters: jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ parameters.defaultManagedPlatform }} + enableInternalSources: ${{ parameters.enableInternalSources }} diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index aba44a25a338..2db4933468fd 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -268,14 +268,16 @@ stages: - task: NuGetAuthenticate@1 - - task: PowerShell@2 + - task: AzureCLI@2 displayName: Publish Using Darc inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 arguments: -BuildId $(BARBuildId) -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' - -MaestroToken '$(MaestroApiAccessToken)' -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates/post-build/setup-maestro-vars.yml b/eng/common/templates/post-build/setup-maestro-vars.yml index 0c87f149a4ad..64b9abc68504 100644 --- a/eng/common/templates/post-build/setup-maestro-vars.yml +++ b/eng/common/templates/post-build/setup-maestro-vars.yml @@ -11,13 +11,14 @@ steps: artifactName: ReleaseConfigs checkDownloadedFiles: true - - task: PowerShell@2 + - task: AzureCLI@2 name: setReleaseVars displayName: Set Release Configs Vars inputs: - targetType: inline - pwsh: true - script: | + azureSubscription: "Darc: Maestro Production" + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | try { if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') { $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt @@ -31,15 +32,16 @@ steps: $AzureDevOpsBuildId = $Env:Build_BuildId } else { - $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" + . $(Build.SourcesDirectory)\eng\common\tools.ps1 + $darc = Get-Darc + $buildInfo = & $darc get-build ` + --id ${{ parameters.BARBuildId }} ` + --extended ` + --output-format json ` + --ci ` + | convertFrom-Json - $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' - $apiHeaders.Add('Accept', 'application/json') - $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") - - $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } - - $BarId = $Env:BARBuildId + $BarId = ${{ parameters.BARBuildId }} $Channels = $Env:PromoteToMaestroChannels -split "," $Channels = $Channels -join "][" $Channels = "[$Channels]" @@ -65,6 +67,4 @@ steps: exit 1 } env: - MAESTRO_API_TOKEN: $(MaestroApiAccessToken) - BARBuildId: ${{ parameters.BARBuildId }} PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} diff --git a/eng/common/templates/steps/enable-internal-runtimes.yml b/eng/common/templates/steps/enable-internal-runtimes.yml new file mode 100644 index 000000000000..54dc9416c519 --- /dev/null +++ b/eng/common/templates/steps/enable-internal-runtimes.yml @@ -0,0 +1,28 @@ +# Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64' +# variable with the base64-encoded SAS token, by default + +parameters: +- name: federatedServiceConnection + type: string + default: 'dotnetbuilds-internal-read' +- name: outputVariableName + type: string + default: 'dotnetbuilds-internal-container-read-token-base64' +- name: expiryInHours + type: number + default: 1 +- name: base64Encode + type: boolean + default: true + +steps: +- ${{ if ne(variables['System.TeamProject'], 'public') }}: + - template: /eng/common/templates/steps/get-delegation-sas.yml + parameters: + federatedServiceConnection: ${{ parameters.federatedServiceConnection }} + outputVariableName: ${{ parameters.outputVariableName }} + expiryInHours: ${{ parameters.expiryInHours }} + base64Encode: ${{ parameters.base64Encode }} + storageAccount: dotnetbuilds + container: internal + permissions: rl diff --git a/eng/common/templates/steps/get-delegation-sas.yml b/eng/common/templates/steps/get-delegation-sas.yml new file mode 100644 index 000000000000..c0e8f91317f0 --- /dev/null +++ b/eng/common/templates/steps/get-delegation-sas.yml @@ -0,0 +1,43 @@ +parameters: +- name: federatedServiceConnection + type: string +- name: outputVariableName + type: string +- name: expiryInHours + type: number + default: 1 +- name: base64Encode + type: boolean + default: false +- name: storageAccount + type: string +- name: container + type: string +- name: permissions + type: string + default: 'rl' + +steps: +- task: AzureCLI@2 + displayName: 'Generate delegation SAS Token for ${{ parameters.storageAccount }}/${{ parameters.container }}' + inputs: + azureSubscription: ${{ parameters.federatedServiceConnection }} + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + # Calculate the expiration of the SAS token and convert to UTC + $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ") + + $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv + + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to generate SAS token." + exit 1 + } + + if ('${{ parameters.base64Encode }}' -eq 'true') { + $sas = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($sas)) + } + + Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$sas" diff --git a/eng/common/templates/steps/get-federated-access-token.yml b/eng/common/templates/steps/get-federated-access-token.yml new file mode 100644 index 000000000000..c8c49cc0e8f0 --- /dev/null +++ b/eng/common/templates/steps/get-federated-access-token.yml @@ -0,0 +1,28 @@ +parameters: +- name: federatedServiceConnection + type: string +- name: outputVariableName + type: string +# Resource to get a token for. Common values include: +# - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps +# - 'https://storage.azure.com/' for storage +# Defaults to Azure DevOps +- name: resource + type: string + default: '499b84ac-1321-427f-aa17-267ca6975798' + +steps: +- task: AzureCLI@2 + displayName: 'Getting federated access token for feeds' + inputs: + azureSubscription: ${{ parameters.federatedServiceConnection }} + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + $accessToken = az account get-access-token --query accessToken --resource ${{ parameters.resource }} --output tsv + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to get access token for resource '${{ parameters.resource }}'" + exit 1 + } + Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$accessToken" \ No newline at end of file diff --git a/global.json b/global.json index 50fd7a407e5e..7ce95a54adc7 100644 --- a/global.json +++ b/global.json @@ -11,7 +11,7 @@ "cmake": "3.21.0" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24266.3", - "Microsoft.DotNet.CMake.Sdk": "8.0.0-beta.24266.3" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24352.1", + "Microsoft.DotNet.CMake.Sdk": "8.0.0-beta.24352.1" } } From cc62208676c7dd727e4b0fdbcd7aae58dbd575b9 Mon Sep 17 00:00:00 2001 From: Marc Paine Date: Wed, 3 Jul 2024 15:22:39 -0700 Subject: [PATCH 20/21] Respond to PR feedback in moving the UseVSTestRunner into the test DBPs file and removing the unnecessary $ in the command result assertion file --- Directory.Build.props | 2 -- test/Directory.Build.props | 1 + .../Assertions/CommandResultAssertions.cs | 12 ++++++------ 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index 0d470e4a5bfc..39b30a121778 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -31,8 +31,6 @@ This is a prevalent problem in building the precomputed assembly reference cache. Limiting the assemblies resolved to those outside the culture folders would work, but that is a nontrivial problem. --> NU5125;NU5105;NU1701;MSB3243;MSB3247 - - true diff --git a/test/Directory.Build.props b/test/Directory.Build.props index 26d486322e94..8369b6680a26 100644 --- a/test/Directory.Build.props +++ b/test/Directory.Build.props @@ -3,5 +3,6 @@ true + true diff --git a/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/CommandResultAssertions.cs b/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/CommandResultAssertions.cs index c140ec1cd9ca..2c883c4186d7 100644 --- a/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/CommandResultAssertions.cs +++ b/test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/CommandResultAssertions.cs @@ -28,21 +28,21 @@ public AndConstraint ExitWith(int expectedExitCode) public AndConstraint Pass() { Execute.Assertion.ForCondition(_commandResult.ExitCode == 0) - .FailWith(AppendDiagnosticsTo($"Expected command to pass but it did not.")); + .FailWith(AppendDiagnosticsTo("Expected command to pass but it did not.")); return new AndConstraint(this); } public AndConstraint Fail() { Execute.Assertion.ForCondition(_commandResult.ExitCode != 0) - .FailWith(AppendDiagnosticsTo($"Expected command to fail but it did not.")); + .FailWith(AppendDiagnosticsTo("Expected command to fail but it did not.")); return new AndConstraint(this); } public AndConstraint HaveStdOut() { Execute.Assertion.ForCondition(!string.IsNullOrEmpty(_commandResult.StdOut)) - .FailWith(AppendDiagnosticsTo($"Command did not output anything to stdout")); + .FailWith(AppendDiagnosticsTo("Command did not output anything to stdout")); return new AndConstraint(this); } @@ -97,7 +97,7 @@ public AndConstraint HaveStdOutMatching(string pattern, public AndConstraint HaveStdErr() { Execute.Assertion.ForCondition(!string.IsNullOrEmpty(_commandResult.StdErr)) - .FailWith(AppendDiagnosticsTo($"Command did not output anything to stderr.")); + .FailWith(AppendDiagnosticsTo("Command did not output anything to stderr.")); return new AndConstraint(this); } @@ -125,14 +125,14 @@ public AndConstraint HaveStdErrMatching(string pattern, public AndConstraint NotHaveStdOut() { Execute.Assertion.ForCondition(string.IsNullOrEmpty(_commandResult.StdOut)) - .FailWith(AppendDiagnosticsTo($"Expected command to not output to stdout but it was not:")); + .FailWith(AppendDiagnosticsTo("Expected command to not output to stdout but it was not:")); return new AndConstraint(this); } public AndConstraint NotHaveStdErr() { Execute.Assertion.ForCondition(string.IsNullOrEmpty(_commandResult.StdErr)) - .FailWith(AppendDiagnosticsTo($"Expected command to not output to stderr but it was not:")); + .FailWith(AppendDiagnosticsTo("Expected command to not output to stderr but it was not:")); return new AndConstraint(this); } From 89aca7c07eb32bf3ab6571cd3e14ab0a89496005 Mon Sep 17 00:00:00 2001 From: Jason Zhai Date: Wed, 3 Jul 2024 20:08:39 -0700 Subject: [PATCH 21/21] Undo excess changes --- test/EndToEnd/ValidateInsertedManifests.cs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/test/EndToEnd/ValidateInsertedManifests.cs b/test/EndToEnd/ValidateInsertedManifests.cs index f5fa4c59b54f..64e3a3b5d9f3 100644 --- a/test/EndToEnd/ValidateInsertedManifests.cs +++ b/test/EndToEnd/ValidateInsertedManifests.cs @@ -31,10 +31,13 @@ public void ManifestReaderCanReadManifests() string manifestFile = manifestDir.GetFile("WorkloadManifest.json").FullName; - File.Exists(manifestFile).Should().BeTrue(); - using var fileStream = new FileStream(manifestFile, FileMode.Open, FileAccess.Read); - Action readManifest = () => WorkloadManifestReader.ReadWorkloadManifest(manifestId, fileStream, manifestFile); - readManifest.Should().NotThrow("manifestId:" + manifestId + " manifestFile:" + manifestFile + "is invalid"); + if (!string.Equals(manifestId, "workloadsets")) + { + new FileInfo(manifestFile).Exists.Should().BeTrue(); + using var fileStream = new FileStream(manifestFile, FileMode.Open, FileAccess.Read); + Action readManifest = () => WorkloadManifestReader.ReadWorkloadManifest(manifestId, fileStream, manifestFile); + readManifest.Should().NotThrow("manifestId:" + manifestId + " manifestFile:" + manifestFile + "is invalid"); + } } }