From d521837e80c85241b987e1b6adb417603f0f1aff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Leh=C3=B3czky?= Date: Mon, 30 Dec 2024 01:58:12 +0100 Subject: [PATCH] Changing browser logging to BiDi --- .../Tests/ErrorHandlingTests.cs | 8 +- Lombiq.Tests.UI.Samples/Tests/MonkeyTests.cs | 13 ++- .../Extensions/LoggingWebDriverExtensions.cs | 21 ----- .../Extensions/SeleniumLogEntryExtensions.cs | 8 +- .../VerificationUITestContextExtensions.cs | 6 +- .../OrchardCoreUITestExecutorConfiguration.cs | 16 ++-- Lombiq.Tests.UI/Services/UITestContext.cs | 93 ++++++++----------- .../Services/UITestExecutionSession.cs | 4 +- Lombiq.Tests.UI/Services/WebDriverFactory.cs | 1 + 9 files changed, 65 insertions(+), 105 deletions(-) delete mode 100644 Lombiq.Tests.UI/Extensions/LoggingWebDriverExtensions.cs diff --git a/Lombiq.Tests.UI.Samples/Tests/ErrorHandlingTests.cs b/Lombiq.Tests.UI.Samples/Tests/ErrorHandlingTests.cs index dd0df24c5..bf9f62e22 100644 --- a/Lombiq.Tests.UI.Samples/Tests/ErrorHandlingTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/ErrorHandlingTests.cs @@ -60,7 +60,7 @@ public Task ClientSideErrorOnLoadedPageShouldHaltTest() => catch (PageChangeAssertionException) { // Remove browser logs to have a clean slate. - context.ClearHistoricBrowserLog(); + context.ClearCumulativeBrowserLog(); } }); @@ -89,11 +89,9 @@ public Task BrowserLogsShouldPersist() => WriteConsoleLog(); WriteConsoleLog(); - await context.UpdateHistoricBrowserLogAsync(); - context - .HistoricBrowserLog - .Count(entry => entry.Message.Contains(testLog)) + .CumulativeBrowserLog + .Count(entry => entry.Text.Contains(testLog)) .ShouldBe(6); }); diff --git a/Lombiq.Tests.UI.Samples/Tests/MonkeyTests.cs b/Lombiq.Tests.UI.Samples/Tests/MonkeyTests.cs index 86bbcffc2..a58b6929a 100644 --- a/Lombiq.Tests.UI.Samples/Tests/MonkeyTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/MonkeyTests.cs @@ -3,14 +3,13 @@ using Lombiq.Tests.UI.MonkeyTesting; using Lombiq.Tests.UI.MonkeyTesting.UrlFilters; using Lombiq.Tests.UI.Services; -using OpenQA.Selenium; +using OpenQA.Selenium.BiDi.Modules.Log; using Shouldly; using System; using System.Linq; using System.Threading.Tasks; using Xunit; using Xunit.Abstractions; -using LogLevel = OpenQA.Selenium.LogLevel; namespace Lombiq.Tests.UI.Samples.Tests; @@ -96,13 +95,13 @@ public Task TestAdminBackgroundTasksAsMonkeyRecursivelyShouldWorkWithAdminUser() configuration => configuration.AssertBrowserLog = (logEntries) => logEntries .Where(logEntry => !logEntry - .Message + .Text .Contains("An invalid form control with name='LockTimeout' is not focusable.") && !logEntry - .Message + .Text .Contains("An invalid form control with name='LockExpiration' is not focusable.") && !logEntry.IsNotFoundLogEntry("/favicon.ico") - && logEntry.Level != LogLevel.Info) + && logEntry.Level != Level.Info) .ShouldBeEmpty()); // Monkey testing has its own configuration too. Check out the docs of the options too. @@ -112,11 +111,11 @@ private static MonkeyTestingOptions CreateMonkeyTestingOptions() => PageTestTime = TimeSpan.FromSeconds(5), }; - private static bool IsValidAdminBrowserLogEntry(LogEntry logEntry) => + private static bool IsValidAdminBrowserLogEntry(Entry logEntry) => OrchardCoreUITestExecutorConfiguration.IsValidBrowserLogEntry(logEntry) && // Requests to /api/graphql without further parameters will fail with HTTP 400, but that's OK, since some // parameters are required. - !logEntry.Message.ContainsOrdinalIgnoreCase("/api/graphql - Failed to load resource: the server responded with a status of 400"); + !logEntry.Text.ContainsOrdinalIgnoreCase("/api/graphql - Failed to load resource: the server responded with a status of 400"); } // END OF TRAINING SECTION: Monkey tests. diff --git a/Lombiq.Tests.UI/Extensions/LoggingWebDriverExtensions.cs b/Lombiq.Tests.UI/Extensions/LoggingWebDriverExtensions.cs deleted file mode 100644 index 3224cb16e..000000000 --- a/Lombiq.Tests.UI/Extensions/LoggingWebDriverExtensions.cs +++ /dev/null @@ -1,21 +0,0 @@ -using OpenQA.Selenium; -using OpenQA.Selenium.Firefox; -using System; -using System.Collections.Generic; - -namespace Lombiq.Tests.UI.Extensions; - -public static class LoggingWebDriverExtensions -{ - /// - /// Retrieves the console logs from the browser. This log will contain all the log messages since the start of the - /// session, not just the ones for the current page. NOTE that once you call this the log will be emptied and only - /// subsequent entries will be included in it. Supports Chrome only. - /// - public static IEnumerable GetAndEmptyBrowserLog(this IWebDriver driver) => - driver is FirefoxDriver - ? throw new NotSupportedException( - "Firefox doesn't support getting the browser logs this way, and it will never support it, see " + - "https://github.com/mozilla/geckodriver/issues/284. You can access") - : driver.Manage().Logs.GetLog(LogType.Browser); -} diff --git a/Lombiq.Tests.UI/Extensions/SeleniumLogEntryExtensions.cs b/Lombiq.Tests.UI/Extensions/SeleniumLogEntryExtensions.cs index 69495f1f5..4a3e15b4d 100644 --- a/Lombiq.Tests.UI/Extensions/SeleniumLogEntryExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/SeleniumLogEntryExtensions.cs @@ -1,4 +1,4 @@ -using OpenQA.Selenium; +using OpenQA.Selenium.BiDi.Modules.Log; using System; using System.Collections.Generic; @@ -6,10 +6,10 @@ namespace Lombiq.Tests.UI.Extensions; public static class SeleniumLogEntryExtensions { - public static string ToFormattedString(this IEnumerable logEntries) => + public static string ToFormattedString(this IEnumerable logEntries) => string.Join(Environment.NewLine, logEntries); - public static bool IsNotFoundLogEntry(this LogEntry logEntry, string url) => - logEntry.Message.ContainsOrdinalIgnoreCase( + public static bool IsNotFoundLogEntry(this Entry logEntry, string url) => + logEntry.Text.ContainsOrdinalIgnoreCase( @$"{url} - Failed to load resource: the server responded with a status of 404"); } diff --git a/Lombiq.Tests.UI/Extensions/VerificationUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/VerificationUITestContextExtensions.cs index ab2a509a3..8c0c2fdef 100644 --- a/Lombiq.Tests.UI/Extensions/VerificationUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/VerificationUITestContextExtensions.cs @@ -20,8 +20,6 @@ public static async Task AssertLogsAsync(this UITestContext context) var configuration = context.Configuration; var testOutputHelper = configuration.TestOutputHelper; - if (context.IsBrowserRunning) await context.UpdateHistoricBrowserLogAsync(); - try { await configuration.AssertAppLogsAsync.InvokeFuncAsync(context.Application); @@ -38,12 +36,12 @@ public static async Task AssertLogsAsync(this UITestContext context) { try { - configuration.AssertBrowserLog?.Invoke(context.HistoricBrowserLog); + configuration.AssertBrowserLog?.Invoke(context.CumulativeBrowserLog); } catch (Exception) { testOutputHelper.WriteLine("Browser logs: " + Environment.NewLine); - testOutputHelper.WriteLine(context.HistoricBrowserLog.ToFormattedString()); + testOutputHelper.WriteLine(context.CumulativeBrowserLog.ToFormattedString()); throw; } diff --git a/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs b/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs index 54cff1c0a..8604d70ff 100644 --- a/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs +++ b/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs @@ -2,7 +2,7 @@ using Lombiq.Tests.UI.Helpers; using Lombiq.Tests.UI.SecurityScanning; using Lombiq.Tests.UI.Services.GitHub; -using OpenQA.Selenium; +using OpenQA.Selenium.BiDi.Modules.Log; using Shouldly; using System; using System.Collections.Generic; @@ -36,20 +36,20 @@ public class OrchardCoreUITestExecutorConfiguration public static readonly Func AssertAppLogsCanContainCacheFolderErrorsAsync = app => app.LogsShouldNotContainAsync(AppLogAssertionHelper.NotMediaCacheEntriesPredicate); - public static readonly Action> AssertBrowserLogIsEmpty = + public static readonly Action> AssertBrowserLogIsEmpty = logEntries => logEntries.ShouldNotContain( logEntry => IsValidBrowserLogEntry(logEntry), logEntries.Where(IsValidBrowserLogEntry).ToFormattedString()); - public static readonly Func IsValidBrowserLogEntry = - logEntry => - logEntry.Level >= LogLevel.Warning && + public static readonly Func IsValidBrowserLogEntry = + entry => + entry.Level >= Level.Warn && // HTML imports are somehow used by Selenium or something but this deprecation notice is always there for // every page. - !logEntry.Message.ContainsOrdinalIgnoreCase("HTML Imports is deprecated") && + !entry.Text.ContainsOrdinalIgnoreCase("HTML Imports is deprecated") && // The 404 is because of how browsers automatically request /favicon.ico even if a favicon is declared to be // under a different URL. - !logEntry.IsNotFoundLogEntry("/favicon.ico"); + !entry.IsNotFoundLogEntry("/favicon.ico"); /// /// Gets the global events available during UI test execution. @@ -108,7 +108,7 @@ public class OrchardCoreUITestExecutorConfiguration : Environment.ProcessorCount; public Func AssertAppLogsAsync { get; set; } = AssertAppLogsCanContainCacheFolderErrorsAsync; - public Action> AssertBrowserLog { get; set; } = AssertBrowserLogIsEmpty; + public Action> AssertBrowserLog { get; set; } = AssertBrowserLogIsEmpty; public ITestOutputHelper TestOutputHelper { get; set; } /// diff --git a/Lombiq.Tests.UI/Services/UITestContext.cs b/Lombiq.Tests.UI/Services/UITestContext.cs index b4cabc373..49131c077 100644 --- a/Lombiq.Tests.UI/Services/UITestContext.cs +++ b/Lombiq.Tests.UI/Services/UITestContext.cs @@ -4,6 +4,8 @@ using Lombiq.Tests.UI.Models; using Lombiq.Tests.UI.SecurityScanning; using OpenQA.Selenium; +using OpenQA.Selenium.BiDi; +using OpenQA.Selenium.BiDi.Modules.Log; using OrchardCore.Environment.Shell; using System; using System.Collections.Generic; @@ -15,7 +17,7 @@ namespace Lombiq.Tests.UI.Services; public class UITestContext { - private readonly List _historicBrowserLog = []; + private readonly List _cumulativeBrowserLog = []; /// /// Gets the globally unique ID of this context. You can use this ID to refer to the current text execution in @@ -103,9 +105,9 @@ public class UITestContext public ZapManager ZapManager { get; } /// - /// Gets a cumulative log of browser console entries. + /// Gets a cumulative log of browser console entries, containing all entries since the start of the test. /// - public IReadOnlyList HistoricBrowserLog => _historicBrowserLog; + public IReadOnlyList CumulativeBrowserLog => _cumulativeBrowserLog; /// /// Gets a dictionary storing some custom contextual data. @@ -149,7 +151,7 @@ public class UITestContext // This is a central context object, we need the data to be passed in the constructor. #pragma warning disable S107 // Methods should not have too many parameters - public UITestContext( + private UITestContext( string id, UITestManifest testManifest, OrchardCoreUITestExecutorConfiguration configuration, @@ -173,64 +175,18 @@ public UITestContext( } /// - /// Updates with current console entries from the browser. + /// Clears accumulated browser log messages from . /// - public Task> UpdateHistoricBrowserLogAsync() - { - var windowHandles = Driver.WindowHandles; - - if (windowHandles.Count > 1) - { - var currentWindowHandle = Driver.CurrentWindowHandle; - - foreach (var windowHandle in windowHandles) - { - // Not using the logging SwitchTo() deliberately as this is not part of what the test does. - Driver.SwitchTo().Window(windowHandle); - _historicBrowserLog.AddRange(Driver.GetAndEmptyBrowserLog()); - } - - try - { - Driver.SwitchTo().Window(currentWindowHandle); - } - catch (NoSuchWindowException) - { - // This can happen in rare instances if the current window/tab was just closed. - Driver.SwitchTo().Window(Driver.WindowHandles[^1]); - } - } - else - { - _historicBrowserLog.AddRange(Driver.GetAndEmptyBrowserLog()); - } - - return Task.FromResult>(_historicBrowserLog); - } - - /// - /// Clears accumulated historic browser log messages from . - /// - public void ClearHistoricBrowserLog() => _historicBrowserLog.Clear(); - - /// - /// Run an assertion on the browser logs of the current tab with the delegate configured in . This doesn't use . - /// - public Task AssertCurrentBrowserLogAsync() - { - Configuration.AssertBrowserLog?.Invoke(Scope.Driver.GetAndEmptyBrowserLog()); - return Task.CompletedTask; - } + public void ClearCumulativeBrowserLog() => _cumulativeBrowserLog.Clear(); /// - /// Clears the application and historic browser logs. + /// Clears the application and cumulative browser logs. /// /// Optional cancellation token for reading the application logs. public async Task ClearLogsAsync(CancellationToken cancellationToken = default) { foreach (var log in await Application.GetLogsAsync(cancellationToken)) await log.RemoveAsync(); - ClearHistoricBrowserLog(); + ClearCumulativeBrowserLog(); } /// @@ -305,6 +261,35 @@ public void SwitchCurrentTenant(string tenantName, Uri baseUri) Scope.BaseUri = baseUri; } + // This is a central context object, we need the data to be passed in the constructor. +#pragma warning disable S107 // Methods should not have too many parameters + public static async Task CreateAsync( + string id, + UITestManifest testManifest, + OrchardCoreUITestExecutorConfiguration configuration, + IWebApplicationInstance application, + AtataScope scope, + Uri testStartUri, + RunningContextContainer runningContextContainer, + ZapManager zapManager) +#pragma warning restore S107 // Methods should not have too many parameters + { + var context = new UITestContext( + id, + testManifest, + configuration, + application, + scope, + testStartUri, + runningContextContainer, + zapManager); + + var biDi = await scope.Driver.AsBiDiAsync(); + await biDi.Log.OnEntryAddedAsync(context._cumulativeBrowserLog.Add); + + return context; + } + /// /// Returns the subdirectory described by inside the current test instance's /// directory. diff --git a/Lombiq.Tests.UI/Services/UITestExecutionSession.cs b/Lombiq.Tests.UI/Services/UITestExecutionSession.cs index 568dd8f55..ef271c67b 100644 --- a/Lombiq.Tests.UI/Services/UITestExecutionSession.cs +++ b/Lombiq.Tests.UI/Services/UITestExecutionSession.cs @@ -650,7 +650,7 @@ Task UITestingBeforeAppStartHandlerAsync(OrchardCoreAppStartContext context, Ins var atataScope = await AtataFactory.StartAtataScopeAsync(contextId, _testOutputHelper, appBaseUri, _configuration); - return new UITestContext( + return await UITestContext.CreateAsync( contextId, _testManifest, _configuration, @@ -814,7 +814,7 @@ private async Task CaptureBrowserUsingDumpsAsync(string debugInformationPath) await File.WriteAllLinesAsync( browserLogPath, - (await _context.UpdateHistoricBrowserLogAsync()).Select(message => message.ToString())); + _context.CumulativeBrowserLog.Select(entry => entry.ToString())); if (_configuration.ReportTeamCityMetadata) { diff --git a/Lombiq.Tests.UI/Services/WebDriverFactory.cs b/Lombiq.Tests.UI/Services/WebDriverFactory.cs index d76022f87..f424bfd8c 100644 --- a/Lombiq.Tests.UI/Services/WebDriverFactory.cs +++ b/Lombiq.Tests.UI/Services/WebDriverFactory.cs @@ -148,6 +148,7 @@ private static TDriverOptions SetCommonOptions(this TDriverOptio driverOptions.AcceptInsecureCertificates = true; driverOptions.UnhandledPromptBehavior = UnhandledPromptBehavior.Ignore; driverOptions.PageLoadStrategy = PageLoadStrategy.Normal; + driverOptions.UseWebSocketUrl = true; return driverOptions; }