add error message to junit and json (#406)

- Add - error message to json and junit - Change - print expected and found tables on fail
dominodatalab · Dec 1, 2023 · eb00189 · eb00189
1 parent fcea3ef
commit eb00189
Show file tree

Hide file tree

Showing 10 changed files with 97 additions and 43 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,8 +5,16 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project closely adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 0.178.0
+- Add - error message to json and junit
+- Change - print expected and found tables on fail
+- Chore - bump pebble
+
 ## 0.177.0
 - Fix - text regex steps mess up the fuzzy find js code
+- Chore - bump dependencies
+- Chore - update pip in CI pre-commit
+- Chore - group dependabot updates
 
 ## 0.176.0
 - Add - preliminary border injection behind feature flag

diff --git a/data/features/feature_with_failing_scenario_with_table.feature b/data/features/feature_with_failing_scenario_with_table.feature
@@ -0,0 +1,8 @@
+@failing
+Feature: Feature with failing to find a table
+
+  Scenario: Scenario that opens a page and fails to find a table
+    Given I start a webserver at directory "data/www" and save the port to the variable "PORT"
+      And I open a browser at the url "http://{HOST_ADDRESS}:{PORT}/tables.html"
+     Then I should see a table that is the following:
+        | nope | this | is | not | it |
diff --git a/data/features/steps/cucu_steps.py b/data/features/steps/cucu_steps.py
@@ -60,7 +60,7 @@ def i_wait_to_fail(_):
     def fail():
         raise RuntimeError("step fails on purpose after a while")
 
-    retry(fail)
+    retry(fail)()
 
 
 @step('I use a step with "{nth:nth}" usage')

diff --git a/features/cli/report_basics.feature b/features/cli/report_basics.feature
@@ -283,6 +283,7 @@ Feature: Report basics
      Then I should see a table that matches the following:
        | Started at | Feature                                | Total | Passed | Failed | Skipped | Status | Duration |
        | .*         | Feature with failing scenario          | 1     | 0      | 1      | 0       | failed | .*       |
+       | .*         | Feature with failing to find a table   | 1     | 0      | 1      | 0       | failed | .*       |
        | .*         | Feature with failing scenario with web | 1     | 0      | 1      | 0       | failed | .*       |
      When I click the button "Feature with failing scenario with web"
      Then I should see a table that matches the following:

diff --git a/features/cli/run_outputs.feature b/features/cli/run_outputs.feature
@@ -217,43 +217,30 @@ Feature: Run outputs
       """
 
   Scenario: User gets expected table representation in console output
-    Given I create a file at "{CUCU_RESULTS_DIR}/tables_in_output/environment.py" with the following:
-      """
-      from cucu.environment import *
-      """
-      And I create a file at "{CUCU_RESULTS_DIR}/tables_in_output/steps/__init__.py" with the following:
-      """
-      from cucu.steps import *
-      """
-      And I create a file at "{CUCU_RESULTS_DIR}/tables_in_output/tables_in_output.feature" with the following:
-      """
-      Feature: Feature with failing to find a table
-
-        Scenario: Scenario that opens a page and fails to find a table
-          Given I start a webserver at directory "data/www" and save the port to the variable "PORT"
-            And I open a browser at the url "http://\{HOST_ADDRESS\}:\{PORT\}/tables.html"
-           Then I should see a table that is the following:
-             | nope | this | is | not | it |
-      """
-     Then I run the command "cucu run {CUCU_RESULTS_DIR}/tables_in_output/tables_in_output.feature --results {CUCU_RESULTS_DIR}/tables_in_output_results" and save stdout to "STDOUT" and expect exit code "1"
+    Given I run the command "cucu run data/features/feature_with_failing_scenario_with_table.feature --results {CUCU_RESULTS_DIR}/tables_in_output_results" and save stdout to "STDOUT" and expect exit code "1"
       And I should see "{STDOUT}" contains the following
       """
+      RuntimeError: unable to find desired table
+      expected:
+        | nope | this | is | not | it |
+
+      found:
       "1st" table:
-      | Name   | City          | Country       |
-      | Alfred | Berlin        | Germany       |
-      | Joe    | San Francisco | United States |
-      | Maria  | Cancun        | Mexico        |
+        | Name   | City          | Country       |
+        | Alfred | Berlin        | Germany       |
+        | Joe    | San Francisco | United States |
+        | Maria  | Cancun        | Mexico        |
       "2nd" table:
-      | Name   | Age |
-      | Alfred | 31  |
-      | Joe    | 35  |
-      | Maria  | 33  |
+        | Name   | Age |
+        | Alfred | 31  |
+        | Joe    | 35  |
+        | Maria  | 33  |
       "3rd" table:
-      | Name (last name optional) | Age (in years) |
-      | Alfred                    | 31             |
-      | Maria Lopez               | 33             |
+        | Name (last name optional) | Age (in years) |
+        | Alfred                    | 31             |
+        | Maria Lopez               | 33             |
       "4th" table:
-      | User Title   | Yearly Salary |
-      | Alfred White | 120,000       |
-      | Maria Lopez  | 133,000       |
+        | User Title   | Yearly Salary |
+        | Alfred White | 120,000       |
+        | Maria Lopez  | 133,000       |
       """
diff --git a/features/cli/run_with_junit.feature b/features/cli/run_with_junit.feature
@@ -11,6 +11,22 @@ Feature: Run with JUnit
       And I should not see a file at "{CUCU_RESULTS_DIR}/run_with_custom_junit_results/Echo.xml"
       And I should see a file at "{CUCU_RESULTS_DIR}/junit_files/Echo.xml"
 
+  Scenario: User can see the error message for tables in the JUnit XML results
+    Given I run the command "cucu run data/features/feature_with_failing_scenario_with_table.feature --results {CUCU_RESULTS_DIR}/tables_in_output_results" and save stdout to "STDOUT" and expect exit code "1"
+     Then I should see the file at "{CUCU_RESULTS_DIR}/tables_in_output_results/Feature with failing to find a table.xml" contains the following:
+      """
+      RuntimeError: unable to find desired table
+      expected:
+        | nope | this | is | not | it |
+
+      found:
+      "1st" table:
+        | Name   | City          | Country       |
+        | Alfred | Berlin        | Germany       |
+        | Joe    | San Francisco | United States |
+        | Maria  | Cancun        | Mexico        |
+      """
+
   Scenario: User can choose to include or exclude stacktraces from the JUnit XML results
     Given I run the command "cucu run data/features/feature_with_failing_scenario.feature --results {CUCU_RESULTS_DIR}/junit_without_stacktraces_results" and expect exit code "1"
      Then I should see the file at "{CUCU_RESULTS_DIR}/junit_without_stacktraces_results/Feature with failing scenario.xml" does not contain the following:

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cucu"
-version = "0.177.0"
+version = "0.178.0"
 license = "MIT"
 description = "Easy BDD web testing"
 authors = ["Domino Data Lab <open-source@dominodatalab.com>"]

diff --git a/src/cucu/formatter/json.py b/src/cucu/formatter/json.py
@@ -12,6 +12,7 @@
 import six
 from behave.formatter.base import Formatter
 from behave.model_core import Status
+from tenacity import RetryError
 
 from cucu import behave_tweaks
 from cucu.config import CONFIG
@@ -202,9 +203,24 @@ def result(self, step):
             error_message = step.error_message
             if self.split_text_into_lines:
                 error_message = error_message.splitlines()
+
             result_element = steps[step_index]["result"]
             result_element["error_message"] = error_message
 
+            if error := step.exception:
+                if isinstance(error, RetryError):
+                    error = error.last_attempt.exception()
+
+                if len(error.args) > 0 and isinstance(error.args[0], str):
+                    error_lines = error.args[0].splitlines()
+                    error_lines[
+                        0
+                    ] = f"{error.__class__.__name__}: {error_lines[0]}"
+                else:
+                    error_lines = [repr(error)]
+
+                result_element["exception"] = error_lines
+
     def embedding(self, mime_type, data):
         # nothing to do, but we need to implement the method
         pass

diff --git a/src/cucu/formatter/junit.py b/src/cucu/formatter/junit.py
@@ -8,6 +8,7 @@
 from behave.formatter.base import Formatter
 from behave.model_core import Status
 from bs4.formatter import XMLFormatter
+from tenacity import RetryError
 
 from cucu.config import CONFIG
 
@@ -77,6 +78,19 @@ def update_scenario(self):
                     f"{self.current_step.keyword} {self.current_step.name}"
                 ]
 
+                if error := self.current_step.exception:
+                    if isinstance(error, RetryError):
+                        error = error.last_attempt.exception()
+
+                    if len(error.args) > 0 and isinstance(error.args[0], str):
+                        error_lines = error.args[0].splitlines()
+                        error_lines[
+                            0
+                        ] = f"{error.__class__.__name__}: {error_lines[0]}"
+                    else:
+                        error_lines = [repr(error)]
+                    failures += error_lines
+
                 if CONFIG["CUCU_JUNIT_WITH_STACKTRACE"] == "true":
                     failures += traceback.format_tb(
                         self.current_scenario_traceback

diff --git a/src/cucu/steps/table_steps.py b/src/cucu/steps/table_steps.py
@@ -116,15 +116,19 @@ def check_table_contains_matching_rows_in_table(table, expected_table):
     return False
 
 
-def report_unable_to_find_table(tables):
+def report_unable_to_find_table(expected_table, found_tables):
     stream = StringIO()
     stream.write("\n")
-    for index, table in enumerate(tables):
+    for index, table in enumerate(found_tables):
         print_index = helpers.nth_to_ordinal(index) or '"1st" '
-        stream.write(f"{print_index}table:\n{format_gherkin_table(table)}\n")
+        stream.write(
+            f"{print_index}table:\n{format_gherkin_table(table, [], '  ')}\n"
+        )
 
     stream.seek(0)
-    raise RuntimeError(f"unable to find desired table, found: {stream.read()}")
+    raise RuntimeError(
+        f"unable to find desired table\nexpected:\n{format_gherkin_table(expected_table, [], '  ')}\n\nfound:{stream.read()}"
+    )
 
 
 def find_table(ctx, assert_func, nth=None):
@@ -143,18 +147,18 @@ def find_table(ctx, assert_func, nth=None):
         RuntimeError when the desired table was not found
     """
     expected = behave_table_to_array(ctx.table)
-    tables = find_tables(ctx)
+    found_tables = find_tables(ctx)
 
     if nth is not None:
-        if assert_func(tables[nth], expected):
+        if assert_func(found_tables[nth], expected):
             return
 
     else:
-        for table in tables:
+        for table in found_tables:
             if assert_func(table, expected):
                 return
 
-    report_unable_to_find_table(tables)
+    report_unable_to_find_table(expected, found_tables)
 
 
 def do_not_find_table(ctx, assert_func, nth=None):