Merge pull request #2476 from GuillaumeGomez/gui-tests

Add base for GUI tests

.github/workflows/main.yml

@@ -3,6 +3,9 @@ on:
   pull_request:
   merge_group:
 
+env:
+  BROWSER_UI_TEST_VERSION: '0.18.2'
+
 jobs:
   test:
     runs-on: ${{ matrix.os }}
@@ -70,6 +73,22 @@ jobs:
       run: rustup update stable && rustup default stable && rustup component add rustfmt
     - run: cargo fmt --check
 
+  gui:
+    name: GUI tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Rust
+        run: bash ci/install-rust.sh stable x86_64-unknown-linux-gnu
+      - name: Install npm
+        uses: actions/setup-node@v3
+        with:
+          node-version: 20
+      - name: Install browser-ui-test
+        run: npm install browser-ui-test@"${BROWSER_UI_TEST_VERSION}"
+      - name: Build and run tests (+ GUI)
+        run: cargo test --locked --target x86_64-unknown-linux-gnu --test gui
+
   # The success job is here to consolidate the total success/failure state of
   # all other jobs. This job is then included in the GitHub branch protection
   # rule which prevents merges unless all other jobs are passing. This makes

.gitignore

@@ -16,3 +16,8 @@ test_book/book/
 # Ignore Vim temporary and swap files.
 *.sw?
 *~
+
+# GUI tests
+node_modules
+package-lock.json
+package.json

CONTRIBUTING.md

@@ -138,8 +138,23 @@ We generally strive to keep mdBook compatible with a relatively recent browser o
 That is, supporting Chrome, Safari, Firefox, Edge on Windows, macOS, Linux, iOS, and Android.
 If possible, do your best to avoid breaking older browser releases.
 
-Any change to the HTML or styling is encouraged to manually check on as many browsers and platforms that you can.
-Unfortunately at this time we don't have any automated UI or browser testing, so your assistance in testing is appreciated.
+GUI tests are checked with the GUI testsuite. To run it, you need to install `npm` first. Then run:
+
+```
+cargo test --test gui
+```
+
+The first time, it'll fail and ask you to install the `browser-ui-test` package. Install it then re-run the tests.
+
+If you want to disable the headless mode, use the `DISABLE_HEADLESS_TEST=1` environment variable:
+
+```
+cargo test --test gui -- --disable-headless-test
+```
+
+The GUI tests are in the directory `tests/gui` in text files with the `.goml` extension. These tests are run
+using a `node.js` framework called `browser-ui-test`. You can find documentation for this language on its
+[repository](https://github.com/GuillaumeGomez/browser-UI-test/blob/master/goml-script.md).
 
 ## Updating highlight.js
 

Cargo.toml

@@ -82,3 +82,10 @@ name = "remove-emphasis"
 path = "examples/remove-emphasis/test.rs"
 crate-type = ["lib"]
 test = true
+
+[[test]]
+harness = false
+test = false
+name = "gui"
+path = "tests/gui/runner.rs"
+crate-type = ["bin"]

tests/gui/runner.rs

@@ -0,0 +1,88 @@
+use std::env::current_dir;
+use std::fs::{read_to_string, remove_dir_all};
+use std::process::Command;
+
+fn get_available_browser_ui_test_version_inner(global: bool) -> Option<String> {
+    let mut command = Command::new("npm");
+    command
+        .arg("list")
+        .arg("--parseable")
+        .arg("--long")
+        .arg("--depth=0");
+    if global {
+        command.arg("--global");
+    }
+    let stdout = command.output().expect("`npm` command not found").stdout;
+    let lines = String::from_utf8_lossy(&stdout);
+    lines
+        .lines()
+        .find_map(|l| l.split(':').nth(1)?.strip_prefix("browser-ui-test@"))
+        .map(std::borrow::ToOwned::to_owned)
+}
+
+fn get_available_browser_ui_test_version() -> Option<String> {
+    get_available_browser_ui_test_version_inner(false)
+        .or_else(|| get_available_browser_ui_test_version_inner(true))
+}
+
+fn expected_browser_ui_test_version() -> String {
+    let content = read_to_string(".github/workflows/main.yml")
+        .expect("failed to read `.github/workflows/main.yml`");
+    for line in content.lines() {
+        let line = line.trim();
+        if let Some(version) = line.strip_prefix("BROWSER_UI_TEST_VERSION:") {
+            return version.trim().replace('\'', "");
+        }
+    }
+    panic!("failed to retrieved `browser-ui-test` version");
+}
+
+fn main() {
+    let browser_ui_test_version = expected_browser_ui_test_version();
+    match get_available_browser_ui_test_version() {
+        Some(version) => {
+            if version != browser_ui_test_version {
+                eprintln!(
+                    "⚠️ Installed version of browser-ui-test (`{version}`) is different than the \
+                     one used in the CI (`{browser_ui_test_version}`) You can install this version \
+                     using `npm update browser-ui-test` or by using `npm install browser-ui-test\
+                     @{browser_ui_test_version}`",
+                );
+            }
+        }
+        None => {
+            panic!(
+                "`browser-ui-test` is not installed. You can install this package using `npm \
+                 update browser-ui-test` or by using `npm install browser-ui-test\
+                 @{browser_ui_test_version}`",
+            );
+        }
+    }
+
+    let current_dir = current_dir().expect("failed to retrieve current directory");
+    let test_book = current_dir.join("test_book");
+
+    // Result doesn't matter.
+    let _ = remove_dir_all(test_book.join("book"));
+
+    let mut cmd = Command::new("cargo");
+    cmd.arg("run").arg("build").arg(&test_book);
+    // Then we run the GUI tests on it.
+    assert!(cmd.status().is_ok_and(|status| status.success()));
+
+    let book_dir = format!("file://{}", current_dir.join("test_book/book/").display());
+
+    let mut command = Command::new("npx");
+    command
+        .arg("browser-ui-test")
+        .arg("--no-sandbox")
+        .args(["--variable", "DOC_PATH", book_dir.as_str()])
+        .args(["--test-folder", "tests/gui"]);
+    if std::env::args().any(|arg| arg == "--disable-headless-test") {
+        command.arg("--no-headless");
+    }
+
+    // Then we run the GUI tests on it.
+    let status = command.status().expect("failed to get command output");
+    assert!(status.success(), "{status:?}");
+}

tests/gui/sidebar.goml

@@ -0,0 +1,59 @@
+// This GUI test checks sidebar hide/show and also its behaviour on smaller
+// width.
+
+// We disable the requests checks because `searchindex.json` will always fail
+// locally.
+fail-on-request-error: false
+go-to: |DOC_PATH| + "index.html"
+set-window-size: (1100, 600)
+// Need to reload for the new size to be taken account by the JS.
+reload:
+
+store-value: (content_indent, 308)
+
+define-function: (
+    "hide-sidebar",
+    [],
+    block {
+        // The content should be "moved" to the right because of the sidebar.
+        assert-css: ("#sidebar", {"transform": "none"})
+        assert-position: ("#page-wrapper", {"x": |content_indent|})
+
+        // We now hide the sidebar.
+        click: "#sidebar-toggle"
+        wait-for: "body.sidebar-hidden"
+        // `transform` is 0.3s so we need to wait a bit (0.5s) to ensure the animation is done.
+        wait-for: 5000
+        assert-css-false: ("#sidebar", {"transform": "none"})
+        // The page content should now be on the left.
+        assert-position: ("#page-wrapper", {"x": 0})
+    },
+)
+
+define-function: (
+    "show-sidebar",
+    [],
+    block {
+        // The page content should be on the left and the sidebar "moved out".
+        assert-css: ("#sidebar", {"transform": "matrix(1, 0, 0, 1, -308, 0)"})
+        assert-position: ("#page-wrapper", {"x": 0})
+
+        // We expand the sidebar.
+        click: "#sidebar-toggle"
+        wait-for: "body.sidebar-visible"
+        // `transform` is 0.3s so we need to wait a bit (0.5s) to ensure the animation is done.
+        wait-for: 5000
+        assert-css-false: ("#sidebar", {"transform": "matrix(1, 0, 0, 1, -308, 0)"})
+        // The page content should be moved to the right.
+        assert-position: ("#page-wrapper", {"x": |content_indent|})
+    },
+)
+
+call-function: ("hide-sidebar", {})
+call-function: ("show-sidebar", {})
+
+// We now test on smaller width to ensure that the sidebar is collapsed by default.
+set-window-size: (900, 600)
+reload:
+call-function: ("show-sidebar", {})
+call-function: ("hide-sidebar", {})