diff --git a/eng/ci/templates/official/jobs/build-core-tools-host.yml b/eng/ci/templates/official/jobs/build-core-tools-host.yml index 3ceb3bad9..7d03bbbb6 100644 --- a/eng/ci/templates/official/jobs/build-core-tools-host.yml +++ b/eng/ci/templates/official/jobs/build-core-tools-host.yml @@ -35,89 +35,138 @@ jobs: arguments: -c Release -r win-arm64 -o $(Build.SourcesDirectory)/pkg_output/windows/win-arm64 --self-contained workingDirectory: $(Build.SourcesDirectory)/host/src/CoreToolsHost - - template: ci/sign-files.yml@eng - parameters: - displayName: 'Authenticode signing (dll) (win-arm64)' - folderPath: '$(Build.SourcesDirectory)/pkg_output/windows/win-arm64' - pattern: '*.dll, *.exe' - signType: inline - inlineOperation: | - [ - { - "KeyCode": "CP-230012", - "OperationCode": "SigntoolSign", - "Parameters": { - "OpusName": "Microsoft", - "OpusInfo": "http://www.microsoft.com", - "FileDigest": "/fd \"SHA256\"", - "PageHash": "/NPH", - "TimeStamp": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" - }, - "ToolName": "sign", - "ToolVersion": "1.0" - }, - { - "KeyCode": "CP-230012", - "OperationCode": "SigntoolVerify", - "Parameters": {}, - "ToolName": "sign", - "ToolVersion": "1.0" - } - ] - - - template: ci/sign-files.yml@eng - parameters: - displayName: 'Authenticode signing (dll) (win-x64)' - folderPath: '$(Build.SourcesDirectory)/pkg_output/windows/win-x64' - pattern: '*.dll, *.exe' - signType: inline - inlineOperation: | - [ - { - "KeyCode": "CP-230012", - "OperationCode": "SigntoolSign", - "Parameters": { - "OpusName": "Microsoft", - "OpusInfo": "http://www.microsoft.com", - "FileDigest": "/fd \"SHA256\"", - "PageHash": "/NPH", - "TimeStamp": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" - }, - "ToolName": "sign", - "ToolVersion": "1.0" - }, - { - "KeyCode": "CP-230012", - "OperationCode": "SigntoolVerify", - "Parameters": {}, - "ToolName": "sign", - "ToolVersion": "1.0" - } - ] - - task: CopyFiles@2 displayName: Copy files (win-x64) inputs: SourceFolder: $(Build.SourcesDirectory)/pkg_output/windows/win-x64 - # Publish output will include many other files. We only need func.exe & nethost.dll + # Publish output will include many other files. We only need func & nethost.dll Contents: | func.exe nethost.dll - TargetFolder: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackagesWindows/win-x64 + TargetFolder: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackages/win-x64 - task: CopyFiles@2 displayName: Copy files (win-arm64) inputs: SourceFolder: $(Build.SourcesDirectory)/pkg_output/windows/win-arm64 - # Publish output will include many other files. We only need func.exe & nethost.dll + # Publish output will include many other files. We only need func & nethost.dll Contents: | func.exe nethost.dll - TargetFolder: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackagesWindows/win-arm64 - + TargetFolder: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackages/win-arm64 + + templateContext: + outputParentDirectory: $(Build.ArtifactStagingDirectory) + outputs: + - output: pipelineArtifact + path: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackages + artifact: drop-coretools-host-windows + +- job: BuildCoreToolsHostLinux + displayName: '[Linux] Build CoreToolsHost' + pool: + name: 1es-pool-azfunc + image: 1es-ubuntu-22.04 + os: linux + + steps: + - task: UseDotNet@2 + inputs: + version: 9.x + includePreviewVersions: true + displayName: Install .NET 9 + - task: UseDotNet@2 + inputs: + version: 6.x + displayName: Install .NET 6 + + - task: DotnetCoreCLI@2 + displayName: Dotnet Publish (linux-x64) + inputs: + command: publish + publishWebProjects: false + zipAfterPublish: false + arguments: -c Release -r linux-x64 -o $(Build.SourcesDirectory)/pkg_output/linux/linux-x64 --self-contained + workingDirectory: $(Build.SourcesDirectory)/host/src/CoreToolsHost + + - task: CopyFiles@2 + displayName: Copy files (linux-x64) + inputs: + SourceFolder: $(Build.SourcesDirectory)/pkg_output/linux/linux-x64 + # Publish output will include many other files. We only need func & nethost.dll + Contents: | + func + libnethost.so + TargetFolder: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackages/linux-x64 + + templateContext: + outputParentDirectory: $(Build.ArtifactStagingDirectory) + outputs: + - output: pipelineArtifact + path: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackages + artifact: drop-coretools-host-linux + +- job: BuildCoreToolsHostMac + displayName: '[Mac] Build CoreToolsHost' + pool: + name: Azure Pipelines + image: 'macOS-latest' + os: macOS + + steps: + - task: UseDotNet@2 + inputs: + version: 9.x + includePreviewVersions: true + displayName: Install .NET 9 + - task: UseDotNet@2 + inputs: + version: 6.x + displayName: Install .NET 6 + + - task: DotnetCoreCLI@2 + displayName: Dotnet Publish (osx-x64) + inputs: + command: publish + publishWebProjects: false + zipAfterPublish: false + arguments: -c Release -r osx-x64 -o $(Build.SourcesDirectory)/pkg_output/osx/osx-x64 --self-contained + workingDirectory: $(Build.SourcesDirectory)/host/src/CoreToolsHost + + - task: DotnetCoreCLI@2 + displayName: Dotnet Publish (osx-arm64) + inputs: + command: publish + publishWebProjects: false + zipAfterPublish: false + arguments: -c Release -r osx-arm64 -o $(Build.SourcesDirectory)/pkg_output/osx/osx-arm64 --self-contained + workingDirectory: $(Build.SourcesDirectory)/host/src/CoreToolsHost + + - task: CopyFiles@2 + displayName: Copy files (osx-x64) + inputs: + SourceFolder: $(Build.SourcesDirectory)/pkg_output/osx/osx-x64 + # Publish output will include many other files. We only need func & nethost.dll + Contents: | + func + libnethost.dylib + libnethost.a + TargetFolder: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackages/osx-x64 + + - task: CopyFiles@2 + displayName: Copy files (osx-arm64) + inputs: + SourceFolder: $(Build.SourcesDirectory)/pkg_output/osx/osx-arm64 + # Publish output will include many other files. We only need func & nethost.dll + Contents: | + func + libnethost.dylib + libnethost.a + TargetFolder: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackages/osx-arm64 + templateContext: outputParentDirectory: $(Build.ArtifactStagingDirectory) outputs: - output: pipelineArtifact - path: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackagesWindows - artifact: drop-coretools-host-windows \ No newline at end of file + path: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackages + artifact: drop-coretools-host-mac \ No newline at end of file diff --git a/src/Azure.Functions.ArtifactAssembler/ArtifactAssembler.cs b/src/Azure.Functions.ArtifactAssembler/ArtifactAssembler.cs index b510107ad..0a699daf8 100644 --- a/src/Azure.Functions.ArtifactAssembler/ArtifactAssembler.cs +++ b/src/Azure.Functions.ArtifactAssembler/ArtifactAssembler.cs @@ -11,10 +11,13 @@ internal sealed class ArtifactAssembler /// The artifacts for which we want to pack a custom host with it. /// This dictionary contains the artifact name and the corresponding runtime identifier value. /// - private readonly Dictionary _customHostArtifacts = new() + private readonly Dictionary _visualStudioArtifacts = new() { { "Azure.Functions.Cli.min.win-x64", "win-x64" }, - { "Azure.Functions.Cli.min.win-arm64", "win-arm64" } + { "Azure.Functions.Cli.min.win-arm64", "win-arm64" }, + { "Azure.Functions.Cli.linux-x64", "linux-x64" }, + { "Azure.Functions.Cli.osx-x64", "osx-x64" }, + { "Azure.Functions.Cli.osx-arm64", "osx-arm64" } }; /// @@ -38,7 +41,9 @@ internal sealed class ArtifactAssembler private readonly string _outOfProcArtifactDirectoryName; private readonly string _inProc6ArtifactName; private readonly string _inProc8ArtifactName; - private readonly string _coreToolsHostArtifactName; + private readonly string _coreToolsHostWindowsArtifactName; + private readonly string _coreToolsHostLinuxArtifactName; + private readonly string _coreToolsHostMacArtifactName; private readonly string _outOfProcArtifactName; private readonly string _rootWorkingDirectory; private readonly string _stagingDirectory; @@ -56,7 +61,9 @@ internal ArtifactAssembler(string rootWorkingDirectory) _inProc6ArtifactName = GetRequiredEnvironmentVariable(EnvironmentVariables.InProc6ArtifactName); _inProc8ArtifactName = GetRequiredEnvironmentVariable(EnvironmentVariables.InProc8ArtifactName); - _coreToolsHostArtifactName = GetRequiredEnvironmentVariable(EnvironmentVariables.CoreToolsHostArtifactName); + _coreToolsHostWindowsArtifactName = GetRequiredEnvironmentVariable(EnvironmentVariables.CoreToolsHostWindowsArtifactName); + _coreToolsHostLinuxArtifactName = GetRequiredEnvironmentVariable(EnvironmentVariables.CoreToolsHostLinuxArtifactName); + _coreToolsHostMacArtifactName = GetRequiredEnvironmentVariable(EnvironmentVariables.CoreToolsHostOsxArtifactName); _outOfProcArtifactName = GetRequiredEnvironmentVariable(EnvironmentVariables.OutOfProcArtifactName); _rootWorkingDirectory = rootWorkingDirectory; @@ -66,8 +73,8 @@ internal ArtifactAssembler(string rootWorkingDirectory) internal async Task AssembleArtifactsAsync() { await ExtractDownloadedArtifactsAsync(); - await CreateVisualStudioCoreToolsAsync(); await CreateCliCoreToolsAsync(); + await CreateVisualStudioCoreToolsAsync(); } private static string GetRequiredEnvironmentVariable(string variableName) @@ -80,28 +87,34 @@ private async Task ExtractDownloadedArtifactsAsync() { var inProcArtifactDownloadDir = Path.Combine(_rootWorkingDirectory, _inProcArtifactDirectoryName); var coreToolsHostArtifactDownloadDir = Path.Combine(_rootWorkingDirectory, _coreToolsHostArtifactDirectoryName); - var outOfProcArtifactDownloadDir = Path.Combine(_rootWorkingDirectory, _outOfProcArtifactDirectoryName); var inProc6ArtifactDirPath = Path.Combine(inProcArtifactDownloadDir, _inProc6ArtifactName); var inProc8ArtifactDirPath = Path.Combine(inProcArtifactDownloadDir, _inProc8ArtifactName); - var coreToolsHostArtifactDirPath = Path.Combine(coreToolsHostArtifactDownloadDir, _coreToolsHostArtifactName); - var outOfProcArtifactDirPath = Path.Combine(outOfProcArtifactDownloadDir, _outOfProcArtifactName); EnsureArtifactDirectoryExist(inProc6ArtifactDirPath); EnsureArtifactDirectoryExist(inProc8ArtifactDirPath); - EnsureArtifactDirectoryExist(coreToolsHostArtifactDirPath); + + var coreToolsHostWindowsArtifactDirPath = Path.Combine(coreToolsHostArtifactDownloadDir, _coreToolsHostWindowsArtifactName); + var coreToolsHostLinuxArtifactDirPath = Path.Combine(coreToolsHostArtifactDownloadDir, _coreToolsHostLinuxArtifactName); + var coreToolsHostMacArtifactDirPath = Path.Combine(coreToolsHostArtifactDownloadDir, _coreToolsHostMacArtifactName); + EnsureArtifactDirectoryExist(coreToolsHostWindowsArtifactDirPath); + EnsureArtifactDirectoryExist(coreToolsHostLinuxArtifactDirPath); + EnsureArtifactDirectoryExist(coreToolsHostMacArtifactDirPath); + + _coreToolsHostExtractedRootDir = await MoveArtifactsToStagingDirectoryAndExtractIfNeeded(coreToolsHostWindowsArtifactDirPath, Path.Combine(_stagingDirectory, Constants.CoreToolsHostDirectoryName)); + await MoveArtifactsToStagingDirectoryAndExtractIfNeeded(coreToolsHostLinuxArtifactDirPath, Path.Combine(_stagingDirectory, Constants.CoreToolsHostDirectoryName)); + await MoveArtifactsToStagingDirectoryAndExtractIfNeeded(coreToolsHostMacArtifactDirPath, Path.Combine(_stagingDirectory, Constants.CoreToolsHostDirectoryName)); + Directory.Delete(coreToolsHostArtifactDownloadDir, true); + var outOfProcArtifactDownloadDir = Path.Combine(_rootWorkingDirectory, _outOfProcArtifactDirectoryName); + var outOfProcArtifactDirPath = Path.Combine(outOfProcArtifactDownloadDir, _outOfProcArtifactName); EnsureArtifactDirectoryExist(outOfProcArtifactDirPath); + _outOfProcExtractedRootDir = await MoveArtifactsToStagingDirectoryAndExtractIfNeeded(outOfProcArtifactDirPath, Path.Combine(_stagingDirectory, Constants.OutOfProcDirectoryName)); + Directory.Delete(outOfProcArtifactDownloadDir, true); _inProc6ExtractedRootDir = await MoveArtifactsToStagingDirectoryAndExtractIfNeeded(inProc6ArtifactDirPath, Path.Combine(_stagingDirectory, Constants.InProc6DirectoryName)); _inProc8ExtractedRootDir = await MoveArtifactsToStagingDirectoryAndExtractIfNeeded(inProc8ArtifactDirPath, Path.Combine(_stagingDirectory, Constants.InProc8DirectoryName)); Directory.Delete(inProcArtifactDownloadDir, true); - - _coreToolsHostExtractedRootDir = await MoveArtifactsToStagingDirectoryAndExtractIfNeeded(coreToolsHostArtifactDirPath, Path.Combine(_stagingDirectory, Constants.CoreToolsHostDirectoryName)); - Directory.Delete(coreToolsHostArtifactDownloadDir, true); - - _outOfProcExtractedRootDir = await MoveArtifactsToStagingDirectoryAndExtractIfNeeded(outOfProcArtifactDirPath, Path.Combine(_stagingDirectory, Constants.OutOfProcDirectoryName)); - Directory.Delete(outOfProcArtifactDownloadDir, true); } private static void EnsureArtifactDirectoryExist(string directoryExist) @@ -132,6 +145,14 @@ private static async Task MoveArtifactsToStagingDirectoryAndExtractIfNee { await Task.Run(() => FileUtilities.CopyDirectory(artifactZipPath, destinationDirectory)); await ExtractZipFilesInDirectoryAsync(artifactZipPath, destinationDirectory); + + // Delete additional files that are not needed + var filesToBeDeleted = Directory.GetFiles(destinationDirectory); + Console.WriteLine($"{filesToBeDeleted.Length} files found in {destinationDirectory}"); + foreach (var file in filesToBeDeleted) + { + File.Delete(file); + } return destinationDirectory; } @@ -156,7 +177,7 @@ private async Task CreateVisualStudioCoreToolsAsync() var customHostTargetArtifactDir = Path.Combine(_stagingDirectory, Constants.VisualStudioOutputArtifactDirectoryName); Directory.CreateDirectory(customHostTargetArtifactDir); - var packTasks = _customHostArtifacts.Keys.Select(async artifactName => + foreach (string artifactName in _visualStudioArtifacts.Keys) { var inProc8ArtifactDirPath = Directory.EnumerateDirectories(_inProc8ExtractedRootDir) .FirstOrDefault(dir => dir.Contains(artifactName)); @@ -172,20 +193,19 @@ private async Task CreateVisualStudioCoreToolsAsync() Directory.CreateDirectory(consolidatedArtifactDirPath); // Copy in-proc8 files - var inProc8CopyTask = Task.Run(() => FileUtilities.CopyDirectory(inProc8ArtifactDirPath, Path.Combine(consolidatedArtifactDirPath, Constants.InProc8DirectoryName))); + await Task.Run(() => FileUtilities.CopyDirectory(inProc8ArtifactDirPath, Path.Combine(consolidatedArtifactDirPath, Constants.InProc8DirectoryName))); // Copy in-proc6 files var inProc6ArtifactDirPath = Path.Combine(_inProc6ExtractedRootDir, artifactDirName); EnsureArtifactDirectoryExist(inProc6ArtifactDirPath); - var inProc6CopyTask = Task.Run(() => FileUtilities.CopyDirectory(inProc6ArtifactDirPath, Path.Combine(consolidatedArtifactDirPath, Constants.InProc6DirectoryName))); + await Task.Run(() => FileUtilities.CopyDirectory(inProc6ArtifactDirPath, Path.Combine(consolidatedArtifactDirPath, Constants.InProc6DirectoryName))); // Copy core-tools-host files var rid = GetRuntimeIdentifierForArtifactName(artifactName); var coreToolsHostArtifactDirPath = Path.Combine(_coreToolsHostExtractedRootDir, rid); EnsureArtifactDirectoryExist(coreToolsHostArtifactDirPath); - var coreToolsHostCopyTask = Task.Run(() => FileUtilities.CopyDirectory(coreToolsHostArtifactDirPath, consolidatedArtifactDirPath)); - - await Task.WhenAll(inProc8CopyTask, inProc6CopyTask, coreToolsHostCopyTask); + await Task.Run(() => FileUtilities.CopyDirectory(coreToolsHostArtifactDirPath, consolidatedArtifactDirPath)); + Directory.Delete(coreToolsHostArtifactDirPath, true); // consolidatedArtifactDirPath now contains custom core-tools host, in-proc6 and in-proc8 sub directories. Create a zip file. var zipPath = Path.Combine(customHostTargetArtifactDir, $"{consolidatedArtifactDirName}.zip"); @@ -193,9 +213,8 @@ private async Task CreateVisualStudioCoreToolsAsync() Console.WriteLine($"Successfully created target runtime zip at: {zipPath}"); Directory.Delete(consolidatedArtifactDirPath, true); - }); + } - await Task.WhenAll(packTasks); Console.WriteLine("Finished assembling Visual Studio Core Tools artifacts"); } @@ -213,6 +232,7 @@ private async Task CreateCliCoreToolsAsync() foreach (var artifactName in _cliArtifacts) { + Console.WriteLine($"Starting to assemble {artifactName}"); // If we are running this for the first time, extract the directory path and out of proc version if (String.IsNullOrEmpty(outOfProcArtifactDirPath)) { @@ -239,6 +259,7 @@ private async Task CreateCliCoreToolsAsync() // If we are currently on the minified version of the artifacts, we do not want the inproc6/inproc8 subfolders if (artifactName.Contains("min.win")) { + Console.WriteLine($"Finished assembling {artifactName}"); continue; } @@ -261,7 +282,6 @@ private async Task CreateCliCoreToolsAsync() // Copy in-proc8 files and delete old directory await Task.Run(() => FileUtilities.CopyDirectory(newInProc8ArtifactDirPath, Path.Combine(consolidatedArtifactDirPath, Constants.InProc8DirectoryName))); - Directory.Delete(newInProc8ArtifactDirPath, true); // Rename inproc6 directory to have the same version as the out-of-proc artifact before copying var inProcArtifactName = Path.GetFileName(inProc8ArtifactDirPath); @@ -271,13 +291,11 @@ private async Task CreateCliCoreToolsAsync() // Copy in-proc6 files and delete old directory await Task.Run(() => FileUtilities.CopyDirectory(newInProc6ArtifactDirPath, Path.Combine(consolidatedArtifactDirPath, Constants.InProc6DirectoryName))); - Directory.Delete(newInProc6ArtifactDirPath, true); + + Console.WriteLine($"Finished assembling {artifactName}"); } // Delete the extracted directories - Directory.Delete(_inProc6ExtractedRootDir, true); - Directory.Delete(_inProc8ExtractedRootDir, true); - Directory.Delete(_coreToolsHostExtractedRootDir, true); Directory.Delete(_outOfProcExtractedRootDir, true); Console.WriteLine("Finished assembling CLI Core Tools artifacts"); @@ -306,18 +324,18 @@ private string RenameInProcDirectory(string oldArtifactDirPath, string newVersio } var artifactName = match.Groups[1]; - var newDirectoryName = $"{artifactName}{newVersion}"; + var newDirectoryName = $"{artifactName}{newVersion}"; // Rename (move) the directory Directory.Move(oldArtifactDirPath, newDirectoryName); - + return newDirectoryName; } private string GetRuntimeIdentifierForArtifactName(string artifactName) { - if (_customHostArtifacts.TryGetValue(artifactName, out var rid)) + if (_visualStudioArtifacts.TryGetValue(artifactName, out var rid)) { return rid; } @@ -346,5 +364,4 @@ private static async Task ExtractZipFilesInDirectoryAsync(string zipSourceDir, s } } } -} - +} \ No newline at end of file diff --git a/src/Azure.Functions.ArtifactAssembler/EnvironmentVariables.cs b/src/Azure.Functions.ArtifactAssembler/EnvironmentVariables.cs index 2074b6012..079b3ac30 100644 --- a/src/Azure.Functions.ArtifactAssembler/EnvironmentVariables.cs +++ b/src/Azure.Functions.ArtifactAssembler/EnvironmentVariables.cs @@ -10,8 +10,9 @@ internal static class EnvironmentVariables internal const string OutOfProcArtifactAlias = "OUT_OF_PROC_ARTIFACT_ALIAS"; internal const string InProc6ArtifactName = "IN_PROC6_ARTIFACT_NAME"; internal const string InProc8ArtifactName = "IN_PROC8_ARTIFACT_NAME"; - internal const string CoreToolsHostArtifactName = "CORETOOLS_HOST_ARTIFACT_NAME"; + internal const string CoreToolsHostWindowsArtifactName = "CORETOOLS_HOST_WINDOWS_ARTIFACT_NAME"; + internal const string CoreToolsHostLinuxArtifactName = "CORETOOLS_HOST_LINUX_ARTIFACT_NAME"; + internal const string CoreToolsHostOsxArtifactName = "CORETOOLS_HOST_OSX_ARTIFACT_NAME"; internal const string OutOfProcArtifactName = "OUT_OF_PROC_ARTIFACT_NAME"; } -} - +} \ No newline at end of file