Run the report step upon failures as well (#3466)

In order to report at least partial test results, let's execute
the `report` step also when the test run is interrupted in the
middle of the test execution (for example when the guest becomes
unresponsive).

docs/releases.rst

@@ -28,6 +28,12 @@ a string to a list of strings, to better accommodate multiple notes.
 The ``Node`` alias for the ``Core`` class has been dropped as it
 has been deprecated a long time ago.
 
+Previously when the test run was interrupted in the middle of the
+test execution the :ref:`/spec/plans/report` step would be skipped
+and no results would be reported. Now the report step is performed
+always so that users can access results of those tests which were
+successfully executed.
+
 
 tmt-1.40.0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

tests/report/interrupted/data/.fmf/version

@@ -0,0 +1 @@
+1

tests/report/interrupted/data/main.fmf

@@ -0,0 +1,23 @@
+/plan:
+    discover:
+        how: fmf
+    provision:
+        how: virtual
+    execute:
+        how: tmt
+    report:
+        how: html
+
+/test:
+    /good:
+        order: 0
+        test: exit 0
+    /bad:
+        order: 1
+        test: exit 1
+    /weird:
+        order: 2
+        test: rm -f /bin/bash
+    /missed:
+        order: 3
+        test: /bin/true

tests/report/interrupted/main.fmf

@@ -0,0 +1,8 @@
+summary: Make sure that report is performed on error
+description:
+    If the run is interrupted in the middle of the test execution,
+    the report step should still be performed in order to provide
+    at least partial test results to the user.
+tag+:
+  - provision-only
+  - provision-virtual

tests/report/interrupted/test.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+. /usr/share/beakerlib/beakerlib.sh || exit 1
+
+rlJournalStart
+    rlPhaseStartSetup
+        rlRun "run=\$(mktemp -d)" 0 "Create a run directory"
+        rlRun "pushd data"
+    rlPhaseEnd
+
+    rlPhaseStartTest
+        rlRun -s "tmt run -vvv --id $run" 2
+
+        # Tests before the breakage are executed as expected
+        rlAssertGrep "pass /test/good" $rlRun_LOG
+        rlAssertGrep "fail /test/bad" $rlRun_LOG
+
+        # Report is generated
+        rlAssertGrep "^\s*report\s*$" $rlRun_LOG
+        rlAssertGrep "how: html" $rlRun_LOG
+        rlAssertGrep "output: /.*/plan/report/default-0/index.html" $rlRun_LOG
+        rlAssertGrep "summary: 1 test passed and 1 test failed" $rlRun_LOG
+    rlPhaseEnd
+
+    rlPhaseStartCleanup
+        rlRun "popd"
+        rlGetTestState || rlFileSubmit "$run/log.txt"
+        rlRun "rm -r $run" 0 "Remove the run directory"
+    rlPhaseEnd
+rlJournalEnd

tmt/base.py

@@ -2446,10 +2446,13 @@ def go(self) -> None:
                 # Source the plan environment file after prepare and execute step
                 if isinstance(step, (tmt.steps.prepare.Prepare, tmt.steps.execute.Execute)):
                     self._source_plan_environment_file()
-        # Make sure we run 'finish' step always if enabled
+        # Make sure we run 'report' and 'finish' steps always if enabled
         finally:
-            if not abort and self.finish.enabled:
-                self.finish.go()
+            if not abort:
+                if self.report.enabled and self.report.status() != "done":
+                    self.report.go()
+                if self.finish.enabled:
+                    self.finish.go()
 
     def _export(
             self,