From f8d6820137357fd04c55a69ed1145a2df18295f4 Mon Sep 17 00:00:00 2001 From: Kevin Garton <71028750+ddl-kgarton@users.noreply.github.com> Date: Thu, 4 Jan 2024 14:36:42 -0600 Subject: [PATCH] QE-12356 Allow multiple screenshots (#417) Previously, each step could be associated with only one screenshot. Now, a step can take as many screenshots as it feels like. QE-12356 --- CHANGELOG.md | 3 + features/browser/images.feature | 12 ++-- features/browser/mht.feature | 2 +- features/cli/report_basics.feature | 44 +++++++++++---- features/cli/secrets.feature | 4 +- pyproject.toml | 2 +- src/cucu/environment.py | 59 ++----------------- src/cucu/helpers.py | 8 ++- src/cucu/reporter/html.py | 36 +++++++++--- src/cucu/reporter/templates/scenario.html | 6 +- src/cucu/utils.py | 69 +++++++++++++++++++++++ 11 files changed, 159 insertions(+), 86 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc7ab962..c4c54f31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project closely adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 0.183.0 +- Change - Added ability to take multiple screenshots per step + ## 0.182.0 Chore - Update docker README for ARM64 based CPU Chore - Add seleniarm_hub bash file diff --git a/features/browser/images.feature b/features/browser/images.feature index 46cf2b28..e3d0fe23 100644 --- a/features/browser/images.feature +++ b/features/browser/images.feature @@ -8,10 +8,10 @@ Feature: Images And I open a browser at the url "http://{HOST_ADDRESS}:{PORT}/index.html" When I click the link "Multiple scenarios with browser steps" And I click the link "Open our test buttons page" - Then I should not see the image with the alt text "Given I start a webserver at directory "data/www" and save the port to the variable "PORT"" - And I should not see the image with the alt text "And I open a browser at the url "http://{HOST_ADDRESS}:{PORT}/buttons.html"" - And I should not see the image with the alt text "Then I should see the button "button with child"" + Then I should not see the image with the alt text "After I start a webserver at directory "data/www" and save the port to the variable "PORT"" + And I should not see the image with the alt text "After I open a browser at the url "http://{HOST_ADDRESS}:{PORT}/buttons.html"" + And I should not see the image with the alt text "After I should see the button "button with child"" When I click the button "I should see the button "button with child"" - Then I should not see the image with the alt text "Given I open a browser at the url "http://{HOST_ADDRESS}:{PORT}/buttons.html"" - And I should not see the image with the alt text "And I open a browser at the url "http://{HOST_ADDRESS}:{PORT}/buttons.html"" - And I should see the image with the alt text "Then I should see the button "button with child"" + Then I should not see the image with the alt text "After I open a browser at the url "http://{HOST_ADDRESS}:{PORT}/buttons.html"" + And I should not see the image with the alt text "After I open a browser at the url "http://{HOST_ADDRESS}:{PORT}/buttons.html"" + And I should see the image with the alt text "After I should see the button "button with child"" diff --git a/features/browser/mht.feature b/features/browser/mht.feature index 1f90a877..5920d5d8 100644 --- a/features/browser/mht.feature +++ b/features/browser/mht.feature @@ -1,7 +1,7 @@ Feature: Download MHT archives As a user of web-based testing I want to download MHT archives - So that I can bebug DOM-based failures more easily + So that I can debug DOM-based failures more easily Scenario: Download an MHT file during a test Given I skip this scenario if the current browser is not "chrome" diff --git a/features/cli/report_basics.feature b/features/cli/report_basics.feature index f93a9812..4acc0c9f 100644 --- a/features/cli/report_basics.feature +++ b/features/cli/report_basics.feature @@ -26,7 +26,7 @@ Feature: Report basics When I run the command "ls "{SCENARIO_RESULTS_DIR}/"" and save stdout to "STDOUT", stderr to "STDERR" and expect exit code "0" Then I should see "{STDOUT}" matches the following """ - [\s\S]*\.png + (\d\d\d\d - [\s\S]*) [\s\S]* """ @@ -50,7 +50,7 @@ Feature: Report basics * # Can see image for step with secret in name When I click the button "I should see the text "\{MY_SECRET\}"" - Then I should see the image with the alt text "Then I should see the text "\{MY_SECRET\}"" + Then I should see the image with the alt text "After I should see the text "\{MY_SECRET\}"" * # Cannot see secrets in the exception message When I click the button "Then I click the button "\{MY_SECRET\}"" @@ -66,19 +66,19 @@ Feature: Report basics # Verify HTML report When I click the link "Multiple scenarios with browser steps" And I click the link "Open our test checkboxes page" - And I should not see the image with the alt text "Given I start a webserver at directory \"data/www\" and save the port to the variable \"PORT\"" - And I should not see the image with the alt text "And I open a browser at the url \"http://{HOST_ADDRESS}:{PORT}/checkboxes.html\"" - And I should not see the image with the alt text "Then I should see the checkbox \"checkbox with inner label\"" + And I should not see the image with the alt text "After I start a webserver at directory \"data/www\" and save the port to the variable \"PORT\"" + And I should not see the image with the alt text "After I open a browser at the url \"http://{HOST_ADDRESS}:{PORT}/checkboxes.html\"" + And I should not see the image with the alt text "After I should see the checkbox \"checkbox with inner label\"" Then I click the button "Then I should see the checkbox \"checkbox with inner label\"" - And I should see the image with the alt text "Then I should see the checkbox \"checkbox with inner label\"" - And I should not see the image with the alt text "Given I start a webserver at directory \"data/www\" and save the port to the variable \"PORT\"" - And I should not see the image with the alt text "And I open a browser at the url \"http://{HOST_ADDRESS}:{PORT}/checkboxes.html"" + And I should see the image with the alt text "After I should see the checkbox \"checkbox with inner label\"" + And I should not see the image with the alt text "After I start a webserver at directory \"data/www\" and save the port to the variable \"PORT\"" + And I should not see the image with the alt text "After I open a browser at the url \"http://{HOST_ADDRESS}:{PORT}/checkboxes.html"" When I save the current url to the variable "CURRENT_URL" And I click the link "Index" Then I navigate to the url "{CURRENT_URL}" - And I wait to see the image with the alt text "Then I should see the checkbox \"checkbox with inner label\"" - And I should not see the image with the alt text "Given I start a webserver at directory \"data/www\" and save the port to the variable \"PORT\"" - And I should not see the image with the alt text "And I open a browser at the url \"http://{HOST_ADDRESS}:{PORT}/checkboxes.html\"" + And I wait to see the image with the alt text "After I should see the checkbox \"checkbox with inner label\"" + And I should not see the image with the alt text "After I start a webserver at directory \"data/www\" and save the port to the variable \"PORT\"" + And I should not see the image with the alt text "After I open a browser at the url \"http://{HOST_ADDRESS}:{PORT}/checkboxes.html\"" Scenario: User can run a feature with mixed results and has all results reported correctly without skips Given I run the command "cucu run data/features/feature_with_mixed_results.feature --results {CUCU_RESULTS_DIR}/mixed-results" and expect exit code "1" @@ -279,6 +279,26 @@ Feature: Report basics And I open a browser at the url "http://{HOST_ADDRESS}:{PORT}/index.html" Then I should see the text "No data available in table" + Scenario: User can run and generate reports that perform highlighting + Given I create a file at "{CUCU_RESULTS_DIR}/highlights/environment.py" with the following: + """ + from cucu.environment import * + """ + And I create a file at "{CUCU_RESULTS_DIR}/highlights/steps/__init__.py" with the following: + """ + from cucu.steps import * + """ + And I start a webserver at directory "data/www" and save the port to the variable "PORT" + And I create a file at "{CUCU_RESULTS_DIR}/highlights/text.feature" with the following: + """ + Feature: run a test with highlighting + + Scenario: See text that should be highlit + When I open a browser at the url "http://{HOST_ADDRESS}:{PORT}/text.html" + Then I should see the text "just some text in a label" + """ + And I run the command "cucu run {CUCU_RESULTS_DIR}/highlights --results {CUCU_RESULTS_DIR}/empty_features_results --env CUCU_INJECT_ELEMENT_BORDER='True' --generate-report --report {CUCU_RESULTS_DIR}/highlights_report" and save stdout to "STDOUT", stderr to "STDERR" and expect exit code "0" + @report-only-failures Scenario: User can generate a report with only failures Given I run the command "cucu run data/features --tags @passing,@failing --report-only-failures --results {CUCU_RESULTS_DIR}/report_only_failures --generate-report --report {CUCU_RESULTS_DIR}/report_only_failures_report" and expect exit code "1" @@ -295,4 +315,4 @@ Feature: Report basics | .* | Just a scenario that opens a web page | 3 | failed | .* | When I click the button "Just a scenario that opens a web page" And I wait to click the button "show images" - And I should see the image with the alt text "And I should see the text \"inexistent\"" + And I should see the image with the alt text "After I should see the text \"inexistent\"" diff --git a/features/cli/secrets.feature b/features/cli/secrets.feature index f22f130c..6c078ddd 100644 --- a/features/cli/secrets.feature +++ b/features/cli/secrets.feature @@ -144,7 +144,7 @@ Feature: Secrets Then I pass the secret "\{MY_SECRET\}" to a substep """ When I run the command "cucu run {CUCU_RESULTS_DIR}/substeps_without_variable_passthru --secrets MY_SECRET --results={CUCU_RESULTS_DIR}/substeps_without_variable_passthru_results" and save stdout to "STDOUT", stderr to "STDERR" and expect exit code "0" - Then I should see a file at "{CUCU_RESULTS_DIR}/substeps_without_variable_passthru_results/Feature that spills the beans/This scenario prints some secrets to the logs/3 - I echo "************".png" + Then I should see a file at "{CUCU_RESULTS_DIR}/substeps_without_variable_passthru_results/Feature that spills the beans/This scenario prints some secrets to the logs/0003 - I echo "************"/0000 - After I echo "************".png" Scenario: User gets expected behavior when using variable_passthru=True Given I create a file at "{CUCU_RESULTS_DIR}/substeps_with_variable_passthru/environment.py" with the following: @@ -178,4 +178,4 @@ Feature: Secrets Then I pass the secret "\{MY_SECRET\}" to a substep """ When I run the command "cucu run {CUCU_RESULTS_DIR}/substeps_with_variable_passthru --secrets MY_SECRET --results={CUCU_RESULTS_DIR}/substeps_with_variable_passthru_results" and save stdout to "STDOUT", stderr to "STDERR" and expect exit code "0" - Then I should see a file at "{CUCU_RESULTS_DIR}/substeps_with_variable_passthru_results/Feature that spills the beans/This scenario prints some secrets to the logs/3 - I echo "\{MY_SECRET\}".png" + Then I should see a file at "{CUCU_RESULTS_DIR}/substeps_with_variable_passthru_results/Feature that spills the beans/This scenario prints some secrets to the logs/0003 - I echo "\{MY_SECRET\}"/0000 - After I echo "\{MY_SECRET\}".png" diff --git a/pyproject.toml b/pyproject.toml index e6d1b6f5..182a995d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "cucu" -version = "0.182.0" +version = "0.183.0" license = "MIT" description = "Easy BDD web testing" authors = ["Domino Data Lab "] diff --git a/src/cucu/environment.py b/src/cucu/environment.py index 9ea40ed3..bef19620 100644 --- a/src/cucu/environment.py +++ b/src/cucu/environment.py @@ -2,7 +2,6 @@ import hashlib import json import os -import shutil import sys import time from functools import partial @@ -10,6 +9,7 @@ from cucu import config, init_scenario_hook_variables, logger from cucu.config import CONFIG from cucu.page_checks import init_page_checks +from cucu.utils import take_screenshot CONFIG.define( "FEATURE_RESULTS_DIR", @@ -31,30 +31,6 @@ init_page_checks() -def generate_image_filename(step_index, step_name): - """ - generate .png image file name that meets these criteria: - - hides secrets - - escaped - - filename does not exceed 255 chars (OS limitation) - - uniqueness comes from step number - """ - max_filename = 255 - len(".png") - escaped_step_name = CONFIG.hide_secrets(step_name).replace("/", "_") - filename = f"{step_index} - {escaped_step_name}" - - if len(filename) > max_filename: - ellipsis = "..." - # save the last chars as the ending often important - end_count = 100 - front_count = max_filename - (len(ellipsis) + end_count) - filename = ( - filename[:front_count] + ellipsis + filename[-1 * end_count :] - ) - - return f"{filename}.png" - - def check_browser_initialized(ctx): """ check browser session initialized otherwise throw an exception indicating @@ -218,6 +194,8 @@ def before_step(ctx, step): ctx.current_step.has_substeps = False ctx.start_time = time.monotonic() + CONFIG["__STEP_SCREENSHOT_COUNT"] = 0 + # run before all step hooks for hook in CONFIG["__CUCU_BEFORE_STEP_HOOKS"]: hook(ctx) @@ -240,40 +218,13 @@ def after_step(ctx, step): # and this step has no substeps as in the reporting the substeps that # may actually do something on the browser take their own screenshots if ctx.browser is not None and ctx.current_step.has_substeps is False: - filepath = os.path.join( - ctx.scenario_dir, generate_image_filename(ctx.step_index, step.name) - ) - - # If we've marked an element as the one we're interacting with, - # inject a border to highlight that element - if ( - not CONFIG["CUCU_INJECT_ELEMENT_BORDER"] - or not CONFIG["__PERTINENT_ELEMENT"] - ): - ctx.browser.screenshot(filepath) - else: - border_style = "solid magenta 4px" - border_radius = "4px" - highlighter = ( - f'arguments[0].style["border"] = "{border_style}";' - f'arguments[0].style["border-radius"] = "{border_radius}";' - ) - ctx.browser.execute(highlighter, CONFIG["__PERTINENT_ELEMENT"]) - ctx.browser.screenshot(filepath) - clear_highlight = ( - 'arguments[0].style["border"] = "";' - 'arguments[0].style["border-radius"] = "";' - ) - ctx.browser.execute(clear_highlight, CONFIG["__PERTINENT_ELEMENT"]) - CONFIG["__PERTINENT_ELEMENT"] = None - - if CONFIG["CUCU_MONITOR_PNG"] is not None: - shutil.copyfile(filepath, CONFIG["CUCU_MONITOR_PNG"]) + take_screenshot(ctx, step.name, label=f"After {step.name}") # if the step has substeps from using `run_steps` then we already moved # the step index in the run_steps method and shouldn't do it here if not step.has_substeps: ctx.step_index += 1 + CONFIG["__STEP_SCREENSHOT_COUNT"] = 0 if CONFIG.bool("CUCU_IPDB_ON_FAILURE") and step.status == "failed": ctx._runner.stop_capture() diff --git a/src/cucu/helpers.py b/src/cucu/helpers.py index a4bb27a6..36d8e814 100644 --- a/src/cucu/helpers.py +++ b/src/cucu/helpers.py @@ -4,6 +4,7 @@ from cucu import logger, retry, run_steps from cucu.config import CONFIG +from cucu.utils import take_screenshot class step(object): @@ -120,7 +121,12 @@ def base_should_see_the(ctx, thing, name, index=0): if element is None: raise RuntimeError(f'unable to find the {prefix}{thing} "{name}"') logger.debug(f'Success: saw {prefix}{thing} "{name}"') - CONFIG["__PERTINENT_ELEMENT"] = element + take_screenshot( + ctx, + ctx.current_step.name, + label=f"saw {thing} {name}", + highlight_element=element, + ) @step(f'I should immediately see the {thing} "{{name}}"') def should_immediately_see_the(ctx, thing, name): diff --git a/src/cucu/reporter/html.py b/src/cucu/reporter/html.py index 13cfdc8b..5c0cc1bb 100644 --- a/src/cucu/reporter/html.py +++ b/src/cucu/reporter/html.py @@ -14,7 +14,7 @@ from cucu import format_gherkin_table, logger from cucu.ansi_parser import parse_log_to_html from cucu.config import CONFIG -from cucu.environment import generate_image_filename +from cucu.utils import get_step_image_dir def escape(data): @@ -184,16 +184,38 @@ def generate(results, basepath, only_failures=False): if show_status: print("s", end="", flush=True) total_steps += 1 - image_filename = generate_image_filename( - step_index, step["name"] - ) - image_filepath = os.path.join(scenario_filepath, image_filename) + image_dir = get_step_image_dir(step_index, step["name"]) + image_dirpath = os.path.join(scenario_filepath, image_dir) if step["name"].startswith("#"): step["heading_level"] = "h4" - if os.path.exists(image_filepath): - step["image"] = urllib.parse.quote(image_filename) + if os.path.exists(image_dirpath): + _, _, image_names = next(os.walk(image_dirpath)) + images = [] + for image_name in image_names: + words = image_name.split("-", 1) + index = words[0].strip() + try: + # Images with label should have a name in the form: + # 0000 - This is the image label.png + label, _ = os.path.splitext(words[1].strip()) + except IndexError: + # Images with no label should instead look like: + # 0000.png + # so we default to the step name in this case. + label = step["name"] + + images.append( + { + "src": urllib.parse.quote( + os.path.join(image_dir, image_name) + ), + "index": index, + "label": label, + } + ) + step["images"] = sorted(images, key=lambda x: x["index"]) if "result" in step: if step["result"]["status"] in ["failed", "passed"]: diff --git a/src/cucu/reporter/templates/scenario.html b/src/cucu/reporter/templates/scenario.html index ff93e5ef..b1d6afad 100644 --- a/src/cucu/reporter/templates/scenario.html +++ b/src/cucu/reporter/templates/scenario.html @@ -134,11 +134,13 @@ 
{{ escape("\n".join(step['result']['stdout'])) }}
{% endif %} - {% if step['image'] is defined %} + {% if step['images'] is defined %} {% if step['result']['stdout'] %}
{% endif %} - {{ step_name }} + {% for image in step['images'] %} + {{ image["label"] }} + {% endfor %} {% endif %} {% if step['result']['error_message'] is defined %} diff --git a/src/cucu/utils.py b/src/cucu/utils.py index 0ab6560d..374e8cb0 100644 --- a/src/cucu/utils.py +++ b/src/cucu/utils.py @@ -3,7 +3,9 @@ the src/cucu/__init__.py """ import logging +import os import pkgutil +import shutil from tabulate import DataRow, TableFormat, tabulate from tenacity import ( @@ -150,3 +152,70 @@ def text_in_current_frame(browser: Browser) -> str: browser.execute("window.jqCucu = jQuery.noConflict(true);") text = browser.execute('return jqCucu("body").children(":visible").text();') return text + + +def ellipsize_filename(raw_filename): + max_filename = 255 + if len(raw_filename) < max_filename: + return raw_filename + + ellipsis = "..." + # save the last chars, as the ending is often important + end_count = 100 + front_count = max_filename - (len(ellipsis) + end_count) + ellipsized_filename = ( + raw_filename[:front_count] + ellipsis + raw_filename[-1 * end_count :] + ) + + return ellipsized_filename + + +def get_step_image_dir(step_index, step_name): + """ + generate .png image file name that meets these criteria: + - hides secrets + - escaped + - filename does not exceed 255 chars (OS limitation) + - uniqueness comes from step number + """ + escaped_step_name = CONFIG.hide_secrets(step_name).replace("/", "_") + unabridged_dirname = f"{step_index:0>4} - {escaped_step_name}" + dirname = ellipsize_filename(unabridged_dirname) + + return dirname + + +def take_screenshot(ctx, step_name, label="", highlight_element=None): + screenshot_dir = os.path.join( + ctx.scenario_dir, get_step_image_dir(ctx.step_index, step_name) + ) + if not os.path.exists(screenshot_dir): + os.mkdir(screenshot_dir) + + if len(label) > 0: + label = f" - {CONFIG.hide_secrets(label).replace('/', '_')}" + filename = f"{CONFIG['__STEP_SCREENSHOT_COUNT']:0>4}{label}.png" + filename = ellipsize_filename(filename) + filepath = os.path.join(screenshot_dir, filename) + + if not CONFIG["CUCU_INJECT_ELEMENT_BORDER"] or not highlight_element: + ctx.browser.screenshot(filepath) + else: + border_style = "solid magenta 4px" + border_radius = "4px" + highlighter = ( + f'arguments[0].style["border"] = "{border_style}";' + f'arguments[0].style["border-radius"] = "{border_radius}";' + ) + ctx.browser.execute(highlighter, highlight_element) + ctx.browser.screenshot(filepath) + clear_highlight = ( + 'arguments[0].style["border"] = "";' + 'arguments[0].style["border-radius"] = "";' + ) + ctx.browser.execute(clear_highlight, highlight_element) + + if CONFIG["CUCU_MONITOR_PNG"] is not None: + shutil.copyfile(filepath, CONFIG["CUCU_MONITOR_PNG"]) + + CONFIG["__STEP_SCREENSHOT_COUNT"] += 1