Add switch to log results to JSON file

The --consolidated-results switch causes results like tests per process, time taken, and total failures
to be logged to a JSON file at the path supplied.

lib/parallel_tests/cli.rb

@@ -41,46 +41,49 @@ def execute_in_parallel(items, num_processes, options)
       Tempfile.open 'parallel_tests-lock' do |lock|
         ParallelTests.with_pid_file do
           simulate_output_for_ci options[:serialize_stdout] do
-            Parallel.map(items, :in_threads => num_processes) do |item|
-              result = yield(item)
-              reprint_output(result, lock.path) if options[:serialize_stdout]
-              result
-            end
+          Parallel.map(items, :in_threads => num_processes) do |item|
+            result = yield(item)
+            reprint_output(result, lock.path) if options[:serialize_stdout]
+            result
           end
         end
       end
     end
+    end
 
     def run_tests_in_parallel(num_processes, options)
       test_results = nil
+      consolidated_results = {}
 
       run_tests_proc = -> {
         groups = @runner.tests_in_groups(options[:files], num_processes, options)
-        groups.reject! &:empty?
+        groups.reject!(&:empty?)
 
         test_results = if options[:only_group]
           groups_to_run = options[:only_group].collect{|i| groups[i - 1]}.compact
-          report_number_of_tests(groups_to_run) unless options[:quiet]
+          report_number_of_tests(groups_to_run, consolidated_results) unless options[:quiet]
           execute_in_parallel(groups_to_run, groups_to_run.size, options) do |group|
             run_tests(group, groups_to_run.index(group), 1, options)
           end
         else
-          report_number_of_tests(groups) unless options[:quiet]
+          report_number_of_tests(groups, consolidated_results) unless options[:quiet]
 
           execute_in_parallel(groups, groups.size, options) do |group|
             run_tests(group, groups.index(group), num_processes, options)
           end
         end
 
-        report_results(test_results, options) unless options[:quiet]
+        report_results(test_results, consolidated_results, options) unless options[:quiet]
       }
 
       if options[:quiet]
         run_tests_proc.call
       else
-        report_time_taken(&run_tests_proc)
+        report_time_taken(consolidated_results, &run_tests_proc)
       end
 
+      write_consolidated_results(consolidated_results, options)
+
       abort final_fail_message if any_test_failed?(test_results)
     end
 
@@ -112,10 +115,12 @@ def lock(lockfile)
       end
     end
 
-    def report_results(test_results, options)
+    def report_results(test_results, consolidated_results, options)
       results = @runner.find_results(test_results.map { |result| result[:stdout] }*"")
+      @runner.summarize_results(results, consolidated_results)
+
       puts ""
-      puts @runner.summarize_results(results)
+      puts consolidated_results[:summary][:message]
 
       report_failure_rerun_commmand(test_results, options)
     end
@@ -135,12 +140,27 @@ def report_failure_rerun_commmand(test_results, options)
       end
     end
 
-    def report_number_of_tests(groups)
+    def report_number_of_tests(groups, consolidated_results)
       name = @runner.test_file_name
       num_processes = groups.size
       num_tests = groups.map(&:size).inject(0, :+)
       tests_per_process = (num_processes == 0 ? 0 : num_tests / num_processes)
-      puts "#{num_processes} processes for #{num_tests} #{name}s, ~ #{tests_per_process} #{name}s per process"
+      message = "#{num_processes} processes for #{num_tests} #{name}s, ~ #{tests_per_process} #{name}s per process"
+      consolidated_results[:tests] = {
+        message: message,
+        tests: num_tests,
+        processes: num_processes,
+        tests_per_process: tests_per_process
+      }
+      puts message
+    end
+
+    def write_consolidated_results(consolidated_results, options)
+      return if options[:consolidated_results].nil? || options[:consolidated_results].empty?
+
+      File.write(options[:consolidated_results], consolidated_results.to_json)
+
+      puts "Wrote consolidated results to #{options[:consolidated_results]}"
     end
 
     #exit with correct status code so rake parallel:test && echo 123 works
@@ -218,8 +238,9 @@ def parse_options!(argv)
         opts.on("--allowed-missing [INT]", Integer, "Allowed percentage of missing runtimes (default = 50)") { |percent| options[:allowed_missing_percent] = percent }
         opts.on("--unknown-runtime [FLOAT]", Float, "Use given number as unknown runtime (otherwise use average time)") { |time| options[:unknown_runtime] = time }
         opts.on("--first-is-1", "Use \"1\" as TEST_ENV_NUMBER to not reuse the default test environment") { options[:first_is_1] = true }
-        opts.on("--verbose", "Print more output (mutually exclusive with quiet)") { options[:verbose] = true }
+        opts.on("--verbose", "Print more output") { options[:verbose] = true }
         opts.on("--quiet", "Print tests output only (mutually exclusive with verbose)") { options[:quiet] = true }
+        opts.on("--consolidated-results [PATH]", "Location to write consolidated test results") { |path| options[:consolidated_results] = path }
         opts.on("-v", "--version", "Show Version") { puts ParallelTests::VERSION; exit }
         opts.on("-h", "--help", "Show this.") { puts opts; exit }
       end.parse!(argv)
@@ -299,9 +320,14 @@ def execute_shell_command_in_parallel(command, num_processes, options)
       abort if results.any? { |r| r[:exit_status] != 0 }
     end
 
-    def report_time_taken
+    def report_time_taken(consolidated_results)
       seconds = ParallelTests.delta { yield }.to_i
-      puts "\nTook #{seconds} seconds#{detailed_duration(seconds)}"
+      message = "Took #{seconds} seconds#{detailed_duration(seconds)}"
+      consolidated_results[:time_taken] = {
+        seconds: seconds,
+        message: message
+      }
+      puts "\n#{message}"
     end
 
     def detailed_duration(seconds)
@@ -332,11 +358,11 @@ def simulate_output_for_ci(simulate)
       if simulate
         progress_indicator = Thread.new do
           interval = Float(ENV.fetch('PARALLEL_TEST_HEARTBEAT_INTERVAL', 60))
-          loop do
-            sleep interval
-            print '.'
-          end
+        loop do
+          sleep interval
+          print '.'
         end
+      end
         test_results = yield
         progress_indicator.exit
         test_results

lib/parallel_tests/cucumber/runner.rb

@@ -15,7 +15,7 @@ def line_is_result?(line)
           super || line =~ SCENARIO_REGEX || line =~ SCENARIOS_RESULTS_BOUNDARY_REGEX
         end
 
-        def summarize_results(results)
+        def summarize_results(results, consolidated_results)
           output = []
 
           scenario_groups = results.slice_before(SCENARIOS_RESULTS_BOUNDARY_REGEX).group_by(&:first)
@@ -28,7 +28,11 @@ def summarize_results(results)
 
           output << super
 
-          output.join("\n\n")
+          consolidated_results[:summary] = {
+            message: output.join("\n\n")
+          }
+
+          consolidated_results[:summary][:message]
         end
 
         def command_with_seed(cmd, seed)

lib/parallel_tests/gherkin/runner.rb

@@ -43,10 +43,10 @@ def line_is_result?(line)
         # cucumber has 2 result lines per test run, that cannot be added
         # 1 scenario (1 failed)
         # 1 step (1 failed)
-        def summarize_results(results)
+        def summarize_results(results, consolidated_results)
           sort_order = %w[scenario step failed flaky undefined skipped pending passed]
 
-          %w[scenario step].map do |group|
+          message = %w[scenario step].map do |group|
             group_results = results.grep(/^\d+ #{group}/)
             next if group_results.empty?
 
@@ -58,6 +58,12 @@ def summarize_results(results)
             end
             "#{sums[0]} (#{sums[1..-1].join(", ")})"
           end.compact.join("\n")
+
+          consolidated_results[:summary] = {
+            message: message
+          }
+
+          consolidated_results[:summary][:message]
         end
 
         def cucumber_opts(given)

lib/parallel_tests/test/runner.rb

@@ -116,9 +116,15 @@ def test_env_number(process_number, options={})
           end
         end
 
-        def summarize_results(results)
+        def summarize_results(results, consolidated_results)
           sums = sum_up_results(results)
-          sums.sort.map{|word, number|  "#{number} #{word}#{'s' if number != 1}" }.join(', ')
+          message = sums.sort.map{|word, number|  "#{number} #{word}#{'s' if number != 1}" }.join(', ')
+
+          consolidated_results[:summary] = sums.merge({
+            message: message
+          })
+
+          consolidated_results[:summary][:message]
         end
 
         # remove old seed and add new seed

spec/parallel_tests/cucumber/runner_spec.rb

@@ -20,7 +20,7 @@ def call(*args)
           "Failing Scenarios:", "cucumber features/failure:3", "cucumber features/failure:4",
           "Failing Scenarios:", "cucumber features/failure:5", "cucumber features/failure:6"
         ]
-        expect(call(results)).to eq("Failing Scenarios:\ncucumber features/failure:1\ncucumber features/failure:2\ncucumber features/failure:3\ncucumber features/failure:4\ncucumber features/failure:5\ncucumber features/failure:6\n\n")
+        expect(call(results, {})).to eq("Failing Scenarios:\ncucumber features/failure:1\ncucumber features/failure:2\ncucumber features/failure:3\ncucumber features/failure:4\ncucumber features/failure:5\ncucumber features/failure:6\n\n")
       end
 
       it "collates flaky scenarios separately" do
@@ -30,7 +30,7 @@ def call(*args)
           "Failing Scenarios:", "cucumber features/failure:5", "cucumber features/failure:6",
           "Flaky Scenarios:", "cucumber features/failure:7", "cucumber features/failure:8",
         ]
-        expect(call(results)).to eq("Failing Scenarios:\ncucumber features/failure:1\ncucumber features/failure:2\ncucumber features/failure:5\ncucumber features/failure:6\n\nFlaky Scenarios:\ncucumber features/failure:3\ncucumber features/failure:4\ncucumber features/failure:7\ncucumber features/failure:8\n\n")
+        expect(call(results, {})).to eq("Failing Scenarios:\ncucumber features/failure:1\ncucumber features/failure:2\ncucumber features/failure:5\ncucumber features/failure:6\n\nFlaky Scenarios:\ncucumber features/failure:3\ncucumber features/failure:4\ncucumber features/failure:7\ncucumber features/failure:8\n\n")
       end
     end
   end

spec/parallel_tests/gherkin/runner_behaviour.rb

@@ -179,27 +179,27 @@ def call(*args)
         "4 scenarios (4 passed)", "40 steps (40 passed)",
         "1 scenario (1 passed)", "1 step (1 passed)"
       ]
-      expect(call(results)).to eq("12 scenarios (2 failed, 1 flaky, 9 passed)\n74 steps (3 failed, 2 skipped, 69 passed)")
+      expect(call(results, {})).to eq("12 scenarios (2 failed, 1 flaky, 9 passed)\n74 steps (3 failed, 2 skipped, 69 passed)")
     end
 
     it "adds same results with plurals" do
       results = [
         "1 scenario (1 passed)", "2 steps (2 passed)",
         "2 scenarios (2 passed)", "7 steps (7 passed)"
       ]
-      expect(call(results)).to eq("3 scenarios (3 passed)\n9 steps (9 passed)")
+      expect(call(results, {})).to eq("3 scenarios (3 passed)\n9 steps (9 passed)")
     end
 
     it "adds non-similar results" do
       results = [
         "1 scenario (1 passed)", "1 step (1 passed)",
         "2 scenarios (1 failed, 1 pending)", "2 steps (1 failed, 1 pending)"
       ]
-      expect(call(results)).to eq("3 scenarios (1 failed, 1 pending, 1 passed)\n3 steps (1 failed, 1 pending, 1 passed)")
+      expect(call(results, {})).to eq("3 scenarios (1 failed, 1 pending, 1 passed)\n3 steps (1 failed, 1 pending, 1 passed)")
     end
 
     it "does not pluralize 1" do
-      expect(call(["1 scenario (1 passed)", "1 step (1 passed)"])).to eq("1 scenario (1 passed)\n1 step (1 passed)")
+      expect(call(["1 scenario (1 passed)", "1 step (1 passed)"], {})).to eq("1 scenario (1 passed)\n1 step (1 passed)")
     end
   end
 

spec/parallel_tests/test/runner_spec.rb

@@ -317,23 +317,23 @@ def call(*args)
     end
 
     it "adds results" do
-      expect(call(['1 foo 3 bar','2 foo 5 bar'])).to eq('8 bars, 3 foos')
+      expect(call(['1 foo 3 bar','2 foo 5 bar'], {})).to eq('8 bars, 3 foos')
     end
 
     it "adds results with braces" do
-      expect(call(['1 foo(s) 3 bar(s)','2 foo 5 bar'])).to eq('8 bars, 3 foos')
+      expect(call(['1 foo(s) 3 bar(s)','2 foo 5 bar'], {})).to eq('8 bars, 3 foos')
     end
 
     it "adds same results with plurals" do
-      expect(call(['1 foo 3 bar','2 foos 5 bar'])).to eq('8 bars, 3 foos')
+      expect(call(['1 foo 3 bar','2 foos 5 bar'], {})).to eq('8 bars, 3 foos')
     end
 
     it "adds non-similar results" do
-      expect(call(['1 xxx 2 yyy','1 xxx 2 zzz'])).to eq('2 xxxs, 2 yyys, 2 zzzs')
+      expect(call(['1 xxx 2 yyy','1 xxx 2 zzz'], {})).to eq('2 xxxs, 2 yyys, 2 zzzs')
     end
 
     it "does not pluralize 1" do
-      expect(call(['1 xxx 2 yyy'])).to eq('1 xxx, 2 yyys')
+      expect(call(['1 xxx 2 yyy'], {})).to eq('1 xxx, 2 yyys')
     end
   end