Custom Host E2E Tests

src/Azure.Functions.ArtifactAssembler/ArtifactAssembler.cs

@@ -201,20 +201,8 @@ private async Task CreateVisualStudioCoreToolsAsync()
                 EnsureArtifactDirectoryExist(coreToolsHostArtifactDirPath);
                 await Task.Run(() => FileUtilities.CopyDirectory(coreToolsHostArtifactDirPath, consolidatedArtifactDirPath));
                 Directory.Delete(coreToolsHostArtifactDirPath, true);
-
-                // consolidatedArtifactDirPath now contains custom core-tools host, in-proc6 and in-proc8 sub directories. Create a zip file.
-                var zipPath = Path.Combine(customHostTargetArtifactDir, $"{consolidatedArtifactDirName}.zip");
-                await Task.Run(() => FileUtilities.CreateZipFile(consolidatedArtifactDirPath, zipPath));
-                Console.WriteLine($"Successfully created target runtime zip at: {zipPath}");
-
-                Directory.Delete(consolidatedArtifactDirPath, true);
             }
 
-            // Delete directories
-            Directory.Delete(_inProc6ExtractedRootDir, true);
-            Directory.Delete(_inProc8ExtractedRootDir, true);
-            Directory.Delete(_coreToolsHostExtractedRootDir, true);
-
             Console.WriteLine("Finished assembling Visual Studio Core Tools artifacts");
         }
 

src/Azure.Functions.ArtifactAssembler/CliArtifactZipper.cs → src/Azure.Functions.ArtifactAssembler/ArtifactZipper.cs

@@ -5,16 +5,17 @@
 
 namespace Azure.Functions.ArtifactAssembler
 {
-    internal sealed class CliArtifactZipper
+    internal sealed class ArtifactZipper
     {
         private readonly string _rootWorkingDirectory;
-        public CliArtifactZipper(string rootWorkingDirectory)
+        public ArtifactZipper(string rootWorkingDirectory)
         {
             _rootWorkingDirectory = rootWorkingDirectory;
         }
 
         internal void ZipCliArtifacts()
         {
+            Console.WriteLine("Zipping CLI Artifacts");
             string stagingDirectory = Path.Combine(_rootWorkingDirectory, Constants.StagingDirName, Constants.CliOutputArtifactDirectoryName);
 
             // Get all directories in the staging directory
@@ -49,6 +50,40 @@ internal void ZipCliArtifacts()
             Console.WriteLine("All directories zipped successfully!");
         }
 
+        internal void ZipVisualStudioArtifacts()
+        {
+            Console.WriteLine("Zipping Visual Studio Artifacts");
+            string stagingDirectory = Path.Combine(_rootWorkingDirectory, Constants.StagingDirName, Constants.VisualStudioOutputArtifactDirectoryName);
+
+            // Get all directories in the staging directory
+            var directories = Directory.EnumerateDirectories(stagingDirectory);
+
+            foreach (var dir in directories)
+            {
+                // Define zip file path and name
+                string zipFileName = $"{new DirectoryInfo(dir).Name}.zip";
+                string zipFilePath = Path.Combine(stagingDirectory, zipFileName);
+
+                // Compress directory into zip file
+                FileUtilities.CreateZipFile(dir, zipFilePath);
+                Console.WriteLine($"Zipped: {dir} -> {zipFilePath}");
+
+                // Verify zip creation and delete original directory to free up space
+                if (File.Exists(zipFilePath))
+                {
+                    Console.WriteLine($"Successfully created zip: {zipFilePath}");
+                    Directory.Delete(dir, true);
+                    Console.WriteLine($"Deleted original directory: {dir}");
+                }
+                else
+                {
+                    Console.WriteLine($"Failed to create zip for: {dir}");
+                }
+            }
+
+            Console.WriteLine("All directories zipped successfully!");
+        }
+
         internal void CreateWorkerDirectoryIfDoesNotExist(string dir)
         {
             string workersPath = Path.Combine(dir, "workers");

src/Azure.Functions.ArtifactAssembler/PipelineHelpers/testVsArtifacts.ps1

@@ -0,0 +1,45 @@
+param (
+    [string]$StagingDirectory
+)
+
+# Set the path to test project (.csproj) and runtime settings
+$testProjectPath = "..\..\test\Azure.Functions.Cli.Tests\Azure.Functions.Cli.Tests.csproj"
+$runtimeSettings = "..\..\test\Azure.Functions.Cli.Tests\E2E\StartTests_artifact_consolidation_visualstudio.runsettings"
+
+[System.Environment]::SetEnvironmentVariable("FUNCTIONS_WORKER_RUNTIME", "dotnet", "Process")
+
+dotnet build $testProjectPath
+
+# Loop through each subdirectory within the parent directory
+Get-ChildItem -Path $StagingDirectory -Directory | ForEach-Object {
+    $subDir = $_.FullName
+    Write-Host "name of current file: $subDir"
+    if ($subDir -like "*win-x*") {
+        Write-Host "Current directory: $subDir"
+        # Find func.exe in the subdirectory
+        $funcExePath = Get-ChildItem -Path $subDir -Filter "func.exe" -ErrorAction SilentlyContinue
+
+        if ($funcExePath) {
+            $funcExePathFullName = $funcExePath.FullName
+             Write-Host "Setting FUNC_PATH to: $funcExePathFullName"
+
+            # Set the environment variable FUNC_PATH to the func.exe or func path
+            [System.Environment]::SetEnvironmentVariable("FUNC_PATH", $funcExePath.FullName, "Process")
+
+            # Run dotnet test with the environment variable set
+            Write-Host "Running 'dotnet test' on test project: $testProjectPath"
+            dotnet test $testProjectPath --no-build --settings $runtimeSettings --logger "console;verbosity=detailed"
+
+            if ($LASTEXITCODE -ne 0) {
+                # If the exit code is non-zero, throw an error
+                Write-Host "Tests failed with exit code $LASTEXITCODE"
+                throw "dotnet test failed within $subDir. Exiting with error."
+            } else {
+                # If the exit code is zero, tests passed
+                Write-Host "All tests passed successfully within $subDir"
+            }
+        } else {
+            Write-Host "No func.exe or func found in: $subDir"
+        }
+    }
+}

src/Azure.Functions.ArtifactAssembler/Program.cs

@@ -14,8 +14,9 @@ static async Task<int> Main(string[] args)
                 // Check if an argument for zipping is passed
                 if (args.Length > 0 && args[0].Equals("zip", StringComparison.OrdinalIgnoreCase))
                 {
-                    var zipCliArtifacts = new CliArtifactZipper(currentWorkingDirectory);
-                    zipCliArtifacts.ZipCliArtifacts();
+                    var zipArtifacts = new ArtifactZipper(currentWorkingDirectory);
+                    zipArtifacts.ZipCliArtifacts();
+                    zipArtifacts.ZipVisualStudioArtifacts();
                 }
                 else
                 {

test/Azure.Functions.Cli.Tests/Azure.Functions.Cli.Tests.csproj

@@ -49,6 +49,10 @@
     <ProjectReference Include="..\..\src\Azure.Functions.Cli\Azure.Functions.Cli.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Folder Include="E2E\TestProject\" />
+  </ItemGroup>
+
   <PropertyGroup>
     <RunSettingsFilePath>$(MSBuildProjectDirectory)\E2E\StartTests_default.runsettings</RunSettingsFilePath>
   </PropertyGroup>

test/Azure.Functions.Cli.Tests/E2E/Helpers/CliTester.cs

@@ -41,6 +41,7 @@ static CliTester()
 
         public static async Task Run(RunConfiguration[] runConfigurations, ITestOutputHelper output = null, string workingDir = null, bool startHost = false)
         {
+            bool wasWorkingDirPassedIn = (workingDir != null);
             string workingDirectory = workingDir ?? Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
 
             bool cleanupDirectory = string.IsNullOrEmpty(workingDir);
@@ -54,7 +55,7 @@ public static async Task Run(RunConfiguration[] runConfigurations, ITestOutputHe
 
             try
             {
-                await InternalRun(workingDirectory, runConfigurations, output, startHost);
+                await InternalRun(workingDirectory, runConfigurations, output, startHost, wasWorkingDirPassedIn);
             }
             finally
             {
@@ -73,7 +74,7 @@ public static async Task Run(RunConfiguration[] runConfigurations, ITestOutputHe
             }
         }
 
-        private static async Task InternalRun(string workingDir, RunConfiguration[] runConfigurations, ITestOutputHelper output, bool startHost)
+        private static async Task InternalRun(string workingDir, RunConfiguration[] runConfigurations, ITestOutputHelper output, bool startHost, bool wasWorkingDirPassedIn)
         {
             await using var hostExe = new Executable(_func, StartHostCommand, workingDirectory: workingDir);
             var stdout = new StringBuilder();
@@ -132,6 +133,13 @@ string cmd when cmd.StartsWith("dotnet", StringComparison.OrdinalIgnoreCase) =>
                         catch (Exception e)
                         {
                             logErr($"Error while running test: {e.Message}");
+                            // Throw an error if working directory was passed in
+                            // For some reason, tests don't fail with the working directory not being the default one so we need this specification
+                            if(wasWorkingDirPassedIn)
+                            {
+                                exitError = true;
+                            }
+
                         }
                         finally
                         {
@@ -166,7 +174,7 @@ string cmd when cmd.StartsWith("dotnet", StringComparison.OrdinalIgnoreCase) =>
                     }
                 }
 
-                // AssertExitError(runConfiguration, exitError);
+                AssertExitError(runConfiguration, exitError);
                 AssertFiles(runConfiguration, workingDir);
                 AssertDirectories(runConfiguration, workingDir);
                 AssertOutputContent(runConfiguration, stdout);

test/Azure.Functions.Cli.Tests/E2E/Helpers/TestTraits.cs

@@ -25,5 +25,11 @@ internal static class TestTraits
         /// We still want to run these tests in the PR/ official core tools pipelines and in the artifact consolidation pipeline for a sanity check before publishing the artifacts.
         /// </summary>
         public const string UseInConsolidatedArtifactGeneration = "UseInConsolidatedArtifactGeneration";
+
+        /// <summary>
+        /// Tests with UseInVisualStudioConsolidatedArtifactGeneration label will not be run in the default scenario and only in the artifact consolidation pipeline
+        /// Otherwise tests with this label will fail in the PR/ official core tools pipelines since the nested inproc artifacts are not present.
+        /// </summary>
+        public const string UseInVisualStudioConsolidatedArtifactGeneration = "UseInVisualStudioConsolidatedArtifactGeneration";
     }
 }

test/Azure.Functions.Cli.Tests/E2E/StartTests.cs

@@ -350,6 +350,76 @@ await CliTester.Run(new RunConfiguration[]
             }, _output);
         }
 
+        [Fact]
+        [Trait(TestTraits.Group, TestTraits.UseInVisualStudioConsolidatedArtifactGeneration)]
+        public async Task Start_InProc_Net8_VisualStudio_SuccessfulFunctionExecution_WithSpecifyingRuntime()
+        {
+            await CliTester.Run(new RunConfiguration[]
+            {
+                new RunConfiguration
+                {
+                    Commands = new[]
+                    {
+                        $"start --port {_funcHostPort} --verbose --dotnet"
+                    },
+                    ExpectExit = false,
+                    Test = async (workingDir, p, _) =>
+                    {
+                        using (var client = new HttpClient() { BaseAddress = new Uri($"http://localhost:{_funcHostPort}") })
+                        {
+                            (await WaitUntilReady(client)).Should().BeTrue(because: _serverNotReady);
+                            var response = await client.GetAsync("/api/Function1?name=Test");
+                            var result = await response.Content.ReadAsStringAsync();
+                            p.Kill();
+                            result.Should().Be("Hello, Test. This HTTP triggered function executed successfully.", because: "response from default function should be 'Hello, {name}. This HTTP triggered function executed successfully.'");
+
+                            if (_output is Xunit.Sdk.TestOutputHelper testOutputHelper)
+                            {
+                                testOutputHelper.Output.Should().Contain("Loading .NET 8 host");
+                            }
+                        }
+                    },
+                    CommandTimeout = TimeSpan.FromSeconds(300),
+                }
+            }, _output, "../../../E2E/TestProject/TestNet8InProcProject");
+
+        }
+
+        [Fact]
+        [Trait(TestTraits.Group, TestTraits.UseInVisualStudioConsolidatedArtifactGeneration)]
+        public async Task Start_InProc_Net6_VisualStudio_SuccessfulFunctionExecution_WithSpecifyingRuntime()
+        {
+            await CliTester.Run(new RunConfiguration[]
+            {
+                new RunConfiguration
+                {
+                    Commands = new[]
+                    {
+                        $"start --port {_funcHostPort} --verbose --dotnet"
+                    },
+                    ExpectExit = false,
+                    Test = async (workingDir, p, _) =>
+                    {
+                        using (var client = new HttpClient() { BaseAddress = new Uri($"http://localhost:{_funcHostPort}") })
+                        {
+                            (await WaitUntilReady(client)).Should().BeTrue(because: _serverNotReady);
+                            var response = await client.GetAsync("/api/Function2?name=Test");
+                            var result = await response.Content.ReadAsStringAsync();
+                            p.Kill();
+                            result.Should().Be("Hello, Test. This HTTP triggered function executed successfully.", because: "response from default function should be 'Hello, {name}. This HTTP triggered function executed successfully.'");
+
+                            if (_output is Xunit.Sdk.TestOutputHelper testOutputHelper)
+                            {
+                                testOutputHelper.Output.Should().Contain("Loading .NET 6 host");
+                            }
+                        }
+                    },
+                    CommandTimeout = TimeSpan.FromSeconds(300),
+                }
+            }, _output, "../../../E2E/TestProject/TestNet6InProcProject");
+
+        }
+
         [Fact]
         public async Task Start_DotnetIsolated_Net9_SuccessfulFunctionExecution()
         {
@@ -1385,7 +1455,6 @@ await CliTester.Run(new RunConfiguration[]
                        $"start --port {_funcHostPort}"
                    },
                    ExpectExit = true,
-                   ExitInError = true,
                    OutputContains = new[] { "Extension bundle configuration should not be present" },
                },
             }, _output);
@@ -1420,7 +1489,6 @@ await CliTester.Run(new RunConfiguration[]
                        $"start --port {_funcHostPort}"
                    },
                    ExpectExit = true,
-                   ExitInError = true,
                    OutputContains = new[] { "Host.json file in missing" },
                },
              }, _output);
@@ -1570,7 +1638,6 @@ await CliTester.Run(new RunConfiguration[]
                     },
                     CommandTimeout = TimeSpan.FromSeconds(300),
                     ExpectExit = true,
-                    ExitInError = true,
                     OutputContains = new[]
                     {
                         "Missing value for AzureWebJobsStorage in local.settings.json. This is required for all triggers other than httptrigger, kafkatrigger, orchestrationTrigger, activityTrigger, entityTrigger",

test/Azure.Functions.Cli.Tests/E2E/StartTests_artifact_consolidation_visualstudio.runsettings

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<RunSettings>
+  <RunConfiguration>
+    <TestCaseFilter>(Group = UseInVisualStudioConsolidatedArtifactGeneration)</TestCaseFilter>
+  </RunConfiguration>
+</RunSettings>

test/Azure.Functions.Cli.Tests/E2E/StartTests_default.runsettings

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <RunSettings>
   <RunConfiguration>
-    <TestCaseFilter>(Group != RequiresNestedInProcArtifacts)</TestCaseFilter>
+    <TestCaseFilter>(Group != RequiresNestedInProcArtifacts AND Group != UseInVisualStudioConsolidatedArtifactGeneration)</TestCaseFilter>
   </RunConfiguration>
 </RunSettings>