From 6c1bfca4549ec8439bb1510680ea79386c2ebfa8 Mon Sep 17 00:00:00 2001 From: Dylan Ratcliffe Date: Sat, 16 Mar 2019 23:47:37 +0000 Subject: [PATCH 1/3] Initial add of parallel code --- lib/onceover/rspec/formatters.rb | 70 +++++++++++++++++++------------- lib/onceover/runner.rb | 14 +++---- lib/onceover/testconfig.rb | 2 +- 3 files changed, 48 insertions(+), 38 deletions(-) diff --git a/lib/onceover/rspec/formatters.rb b/lib/onceover/rspec/formatters.rb index 5c18f442..3d26a650 100644 --- a/lib/onceover/rspec/formatters.rb +++ b/lib/onceover/rspec/formatters.rb @@ -189,34 +189,48 @@ def longest_group end -# class OnceoverFormatterParallel < OnceoverFormatter -# require 'yaml' - -# def example_group_started notification -# # Do nothing -# end - -# def example_passed notification -# @output << green('P') -# end - -# def example_failed notification -# @output << red('F') -# end - -# def example_pending notification -# @output << yellow('?') -# end - -# def dump_failures -# # TODO: This should write to a file and then get picked up and formatted by onceover itself -# # might need to use a module for the formatting -# require 'pry' -# binding.pry -# RSpec.configuration.onceover_tempdir -# end - -# end +class OnceoverFormatterParallel < OnceoverFormatter + require 'yaml' + + def example_group_started notification + # Do nothing + end + + def example_passed notification + @output << green('P') + end + + def example_failed notification + @output << red('F') + end + + def example_pending notification + @output << yellow('?') + end + + def dump_failures + # Create a random string + require 'securerandom' + random_string = SecureRandom.hex + + # Ensure that the folder exists + FileUtils.mkdir_p "#{RSpec.configuration.onceover_tempdir}/parallel" + + # Dump the notification to a unique file + File.open("#{RSpec.configuration.onceover_tempdir}/parallel/results-#{random_string}", "w") do |file| + file.write(pets.to_yaml) + end + end + + def self.read_results(directory) + # Read all yaml files + require 'pry' + binding.pry + # Delete them from the disk + + # Merge data + end +end class FailureCollector RSpec::Core::Formatters.register self, :dump_failures diff --git a/lib/onceover/runner.rb b/lib/onceover/runner.rb index 03492ff2..a89c5ca8 100644 --- a/lib/onceover/runner.rb +++ b/lib/onceover/runner.rb @@ -84,15 +84,11 @@ def run_spec! logger.debug "Running #{@command_prefix}rake spec_standalone from #{@repo.tempdir}" result = Backticks::Runner.new(interactive:true).run(@command_prefix.strip.split, 'rake', 'spec_standalone').join end - # TODO: Refactor this to be much nicer - if @config.formatters.include? 'FailureCollector' - puts '----------- Summary of failures -----------' - if File.exist?("#{@repo.tempdir}/failures.out") and ! File.zero?("#{@repo.tempdir}/failures.out") - logger.debug "Reading failures from #{@repo.tempdir}/failures.out" - puts File.read("#{@repo.tempdir}/failures.out") - else - puts 'No failures detected' - end + + # Print a summary if we were running ion parallel + if @config.formatters.include? 'OnceoverFormatterParallel' + require 'onceover/rspec/formatters' + results = OnceoverFormatterParallel.read_results("#{repo.tempdir}/parallel") end # Finally exit and preserve the exit code diff --git a/lib/onceover/testconfig.rb b/lib/onceover/testconfig.rb index 8ba28f6f..f2a0b55f 100644 --- a/lib/onceover/testconfig.rb +++ b/lib/onceover/testconfig.rb @@ -53,7 +53,7 @@ def initialize(file, opts = {}) # Set dynamic defaults for format if opts[:format] == [:defaults] - @formatters = opts[:parallel] ? ['documentation', 'FailureCollector'] : ['OnceoverFormatter'] + @formatters = opts[:parallel] ? ['OnceoverFormatterParallel'] : ['OnceoverFormatter'] else @formatters = opts[:format] end From 67aa34458c2a645b7297fedd8fde55dcc30c29ed Mon Sep 17 00:00:00 2001 From: Dylan Ratcliffe Date: Sun, 17 Mar 2019 15:44:14 +0000 Subject: [PATCH 2/3] Properly implemented parallel test output --- features/run.feature | 5 +++ lib/onceover/rspec/formatters.rb | 60 ++++++++++++++++++++++++-------- lib/onceover/runner.rb | 3 +- 3 files changed, 52 insertions(+), 16 deletions(-) diff --git a/features/run.feature b/features/run.feature index 7a335a30..7a812ef5 100644 --- a/features/run.feature +++ b/features/run.feature @@ -12,6 +12,11 @@ Feature: Run rspec and acceptance test suites When I run onceover command "run spec" Then I should not see any errors + Scenario: Run correct spec tests in parallel + Given initialized control repo "basic" + When I run onceover command "run spec --parallel" + Then I should not see any errors + Scenario: Using regexes to define tests Given initialized control repo "caching" When I run onceover command "run spec" diff --git a/lib/onceover/rspec/formatters.rb b/lib/onceover/rspec/formatters.rb index 3d26a650..5d4018db 100644 --- a/lib/onceover/rspec/formatters.rb +++ b/lib/onceover/rspec/formatters.rb @@ -1,3 +1,4 @@ +require 'rspec' require 'pathname' class OnceoverFormatter @@ -53,6 +54,21 @@ def example_pending notification def dump_failures notification require 'onceover/controlrepo' + failures = extract_failures(notification) + + # Put some spacing before the results + @output << "\n\n\n" + + failures.each do |_name, role| + @output << Onceover::Controlrepo.evaluate_template('error_summary.yaml.erb', binding) + end + + @output << "\n" + end + + # This method takes a notification and formats it into a hash that can be + # printed easily + def extract_failures notification # Group by role grouped = notification.failed_examples.group_by { |e| e.metadata[:example_group][:parent_example_group][:description]} @@ -61,17 +77,15 @@ def dump_failures notification grouped[role] = failures.uniq { |f| f.metadata[:execution_result].exception.to_s } end - # Put some spacing before the results - @output << "\n\n\n" - + # Extract the errors and remove all RSpec objects grouped.each do |role, failures| - role = { + grouped[role] = { name: role, errors: failures.map { |f| parse_errors(f.metadata[:execution_result].exception.to_s)}.flatten, } - - @output << Onceover::Controlrepo.evaluate_template('error_summary.yaml.erb', binding) end + + grouped end def parse_errors(raw_error) @@ -140,8 +154,6 @@ def calculate_relative_source(file) file.relative_path_from(tempdir + environmentpath + "production").to_s end - private - # Below are defined the styles for the output def class_name(text) RSpec::Core::Formatters::ConsoleCodes.wrap(text, :bold) @@ -192,6 +204,9 @@ def longest_group class OnceoverFormatterParallel < OnceoverFormatter require 'yaml' + RSpec::Core::Formatters.register self, :example_group_started, + :example_passed, :example_failed, :example_pending, :dump_failures + def example_group_started notification # Do nothing end @@ -208,7 +223,7 @@ def example_pending notification @output << yellow('?') end - def dump_failures + def dump_failures notification # Create a random string require 'securerandom' random_string = SecureRandom.hex @@ -217,18 +232,33 @@ def dump_failures FileUtils.mkdir_p "#{RSpec.configuration.onceover_tempdir}/parallel" # Dump the notification to a unique file - File.open("#{RSpec.configuration.onceover_tempdir}/parallel/results-#{random_string}", "w") do |file| - file.write(pets.to_yaml) + File.open("#{RSpec.configuration.onceover_tempdir}/parallel/results-#{random_string}.yaml", "w") do |file| + file.write(extract_failures(notification).to_yaml) end end - def self.read_results(directory) + def output_results(directory) + require 'rspec/core/example' # Read all yaml files - require 'pry' - binding.pry - # Delete them from the disk + results = {} + files = Dir["#{directory}/*.yaml"] # Merge data + errors = files.reduce({}) do |errs, file| + # Read all files and merge them + errs.merge(YAML.load(File.read(file))) # rubocop:disable Security/YAMLLoad + end + + # Delete files from the disk + files.each { |f| File.delete(f) } + + @output << "\n\n\n" + + # Output errors + errors.each do |_name, role| + @output << Onceover::Controlrepo.evaluate_template('error_summary.yaml.erb', binding) + end + @output << "\n" end end diff --git a/lib/onceover/runner.rb b/lib/onceover/runner.rb index a89c5ca8..30e4cbe3 100644 --- a/lib/onceover/runner.rb +++ b/lib/onceover/runner.rb @@ -88,7 +88,8 @@ def run_spec! # Print a summary if we were running ion parallel if @config.formatters.include? 'OnceoverFormatterParallel' require 'onceover/rspec/formatters' - results = OnceoverFormatterParallel.read_results("#{repo.tempdir}/parallel") + formatter = OnceoverFormatterParallel.new(STDOUT) + formatter.output_results("#{repo.tempdir}/parallel") end # Finally exit and preserve the exit code From 05615da92805cecfa9f64ba0f6e2221de31d4ab1 Mon Sep 17 00:00:00 2001 From: Dylan Ratcliffe Date: Sun, 17 Mar 2019 16:03:14 +0000 Subject: [PATCH 3/3] Added flushing as it seems that when running in parallel it takes lonher to flush STDOUT to the terminal --- lib/onceover/rspec/formatters.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/onceover/rspec/formatters.rb b/lib/onceover/rspec/formatters.rb index 5d4018db..ad852179 100644 --- a/lib/onceover/rspec/formatters.rb +++ b/lib/onceover/rspec/formatters.rb @@ -213,14 +213,17 @@ def example_group_started notification def example_passed notification @output << green('P') + @output.flush end def example_failed notification @output << red('F') + @output.flush end def example_pending notification @output << yellow('?') + @output.flush end def dump_failures notification