diff --git a/src/Machine.Specifications.Runner.Console.Specs/ProgramSpecs.cs b/src/Machine.Specifications.Runner.Console.Specs/ProgramSpecs.cs index b7e24444..00482024 100644 --- a/src/Machine.Specifications.Runner.Console.Specs/ProgramSpecs.cs +++ b/src/Machine.Specifications.Runner.Console.Specs/ProgramSpecs.cs @@ -162,7 +162,7 @@ public class when_a_specification_fails_and_silent_is_set : FailingSpecs console.Lines.ShouldContain(l => l.Contains("hi scott, love you, miss you.")); It should_separate_failures_from_the_rest_of_the_test_run = () => - console.Output.ShouldMatchRegex(String.Format("\\S{0}{0}Failures:{0}{0}\\S", Regex.Escape(Environment.NewLine))); + console.Output.ShouldMatchRegex(String.Format("{0}Failures:{0}{0}\\S", Regex.Escape(Environment.NewLine))); } [Subject("Console runner")] @@ -189,7 +189,7 @@ public class when_a_specification_fails_and_progress_is_set : FailingSpecs console.Lines.ShouldContain(l => l.Contains("hi scott, love you, miss you.")); It should_separate_failures_from_the_rest_of_the_test_run = () => - console.Output.ShouldMatchRegex(String.Format("\\S{0}{0}{0}Failures:{0}{0}\\S", Regex.Escape(Environment.NewLine))); + console.Output.ShouldMatchRegex(String.Format("{0}Failures:{0}{0}\\S", Regex.Escape(Environment.NewLine))); } [Subject("Console runner")] diff --git a/src/Machine.Specifications.Specs/CompileContext.cs b/src/Machine.Specifications.Specs/CompileContext.cs index 9734df87..f4d76c19 100644 --- a/src/Machine.Specifications.Specs/CompileContext.cs +++ b/src/Machine.Specifications.Specs/CompileContext.cs @@ -34,7 +34,7 @@ public string Compile(string code) .Emit(filename); if (!result.Success) - throw new InvalidOperationException(); + throw new InvalidOperationException(result.Diagnostics[0].GetMessage()); #else var parameters = new CompilerParameters { diff --git a/src/Machine.Specifications.Specs/Fixtures/LargeFixture.cs b/src/Machine.Specifications.Specs/Fixtures/LargeFixture.cs new file mode 100644 index 00000000..c96a991a --- /dev/null +++ b/src/Machine.Specifications.Specs/Fixtures/LargeFixture.cs @@ -0,0 +1,59 @@ +using System.Text; + +namespace Machine.Specifications.Specs.Fixtures +{ + public class LargeFixture + { + public static string CreateCode(int specCount) + { + var sb = new StringBuilder(); + + sb.AppendLine(@" +using System; +using System.Diagnostics; +using System.Reflection; +using System.Threading.Tasks; +using Machine.Specifications; + +namespace Example.Large +{ + public class when_there_are_many_contexts + { + public static bool Created = false; + + public when_there_are_many_contexts() + { + Created = true; + } + + It spec = () => {}; + } + + public static class OtherTests + { + public static bool Created = false; + } + +"); + + for (var i = 1; i <= specCount; i++) + { + sb.AppendLine($@" + public class when_there_are_many_contexts_{i} + {{ + public when_there_are_many_contexts_{i}() + {{ + OtherTests.Created = true; + }} + + It spec = () => {{}}; + }}"); + } + + sb.AppendLine(@" +}"); + + return sb.ToString(); + } + } +} diff --git a/src/Machine.Specifications.Specs/Runner/SpecificationRunnerSpecs.cs b/src/Machine.Specifications.Specs/Runner/SpecificationRunnerSpecs.cs index 4cfcc438..4078282c 100644 --- a/src/Machine.Specifications.Specs/Runner/SpecificationRunnerSpecs.cs +++ b/src/Machine.Specifications.Specs/Runner/SpecificationRunnerSpecs.cs @@ -1,4 +1,6 @@ using System; +using System.Diagnostics; +using System.Linq; using System.Reflection; using Machine.Specifications.Runner; using Machine.Specifications.Runner.Impl; @@ -691,6 +693,62 @@ public class when_running_a_context_inside_a_static_class_that_is_nested_in_a_no testListener.LastResult.Passed.ShouldBeTrue(); } + [Subject("Specification Runner")] + public class when_running_a_single_spec_out_of_a_large_number_of_specifications : RunnerSpecs + { + static Type when_a_context_has_many_specifications; + static Type filtered_out_spec; + static TimeSpan elapsed { get; set; } + + Establish context = () => + { + using (var compiler = new CompileContext()) + { + var assemblyPath = compiler.Compile(LargeFixture.CreateCode(10000)); + var assembly = Assembly.LoadFile(assemblyPath); + + when_a_context_has_many_specifications = assembly.GetType("Example.Large.when_there_are_many_contexts"); + filtered_out_spec = assembly.GetType("Example.Large.OtherTests"); + } + }; + + Because of = () => + { + var runner = new DefaultRunner(testListener, new RunOptions( + Enumerable.Empty(), + Enumerable.Empty(), + new[] {when_a_context_has_many_specifications.FullName}) + ); + + var sw = Stopwatch.StartNew(); + runner.RunAssembly(when_a_context_has_many_specifications.Assembly); + sw.Stop(); + elapsed = sw.Elapsed; + }; + + It should_run_the_single_specification = () => + { + testListener.SpecCount.ShouldEqual(1); + }; + + It should_run_in_a_reasonable_period_of_time = () => + { + elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(1)); + }; + + It should_have_created_the_test_instance = () => + { + var fieldInfo = when_a_context_has_many_specifications.GetField("Created"); + ((bool) fieldInfo.GetValue(null)).ShouldBeTrue(); + }; + + It should_have_not_have_created_any_of_the_filtered_out_tests = () => + { + var fieldInfo = filtered_out_spec.GetField("Created"); + ((bool) fieldInfo.GetValue(null)).ShouldBeFalse(); + }; + } + public class RandomRunnerSpecs : RunnerSpecs { static CompileContext compiler; diff --git a/src/Machine.Specifications/Explorers/AssemblyExplorer.cs b/src/Machine.Specifications/Explorers/AssemblyExplorer.cs index 9a100875..5f6a5432 100644 --- a/src/Machine.Specifications/Explorers/AssemblyExplorer.cs +++ b/src/Machine.Specifications/Explorers/AssemblyExplorer.cs @@ -4,6 +4,7 @@ using System.Reflection; using Machine.Specifications.Factories; using Machine.Specifications.Model; +using Machine.Specifications.Runner; using Machine.Specifications.Sdk; using Machine.Specifications.Utility; @@ -18,37 +19,73 @@ public AssemblyExplorer() _contextFactory = new ContextFactory(); } + public Context FindContexts(Type type, RunOptions options = null) + { + var types = new[] {type}; + + return types + .Where(IsContext) + .FilterBy(options) + .Select(CreateContextFrom) + .FirstOrDefault(); + } + + public Context FindContexts(FieldInfo info, RunOptions options = null) + { + var types = new[] {info.DeclaringType}; + + return types + .Where(IsContext) + .FilterBy(options) + .Select(t => CreateContextFrom(t, info)) + .FirstOrDefault(); + } + public IEnumerable FindContextsIn(Assembly assembly) { - return EnumerateContextsIn(assembly).Select(CreateContextFrom); + return FindContextsIn(assembly, options: null); + } + + public IEnumerable FindContextsIn(Assembly assembly, RunOptions options) + { + return EnumerateContextsIn(assembly) + .FilterBy(options) + .OrderBy(t => t.Namespace) + .Select(CreateContextFrom); } public IEnumerable FindContextsIn(Assembly assembly, string targetNamespace) + { + return FindContextsIn(assembly, targetNamespace, options: null); + } + + public IEnumerable FindContextsIn(Assembly assembly, string targetNamespace, RunOptions options) { return EnumerateContextsIn(assembly) - .Where(x => x.Namespace == targetNamespace) - .Select(CreateContextFrom); + .Where(x => x.Namespace == targetNamespace) + .FilterBy(options) + .Select(CreateContextFrom); } public IEnumerable FindAssemblyWideContextCleanupsIn(Assembly assembly) { return assembly.GetExportedTypes() - .Where(x => x.GetInterfaces().Contains(typeof(ICleanupAfterEveryContextInAssembly))) - .Select(x => (ICleanupAfterEveryContextInAssembly)Activator.CreateInstance(x)); + .Where(x => x.GetInterfaces().Contains(typeof(ICleanupAfterEveryContextInAssembly))) + .Select(x => (ICleanupAfterEveryContextInAssembly) Activator.CreateInstance(x)); } public IEnumerable FindSpecificationSupplementsIn(Assembly assembly) { return assembly.GetExportedTypes() - .Where(x => x.GetInterfaces().Contains(typeof(ISupplementSpecificationResults))) - .Select(x => (ISupplementSpecificationResults)Activator.CreateInstance(x)); + .Where(x => x.GetInterfaces().Contains(typeof(ISupplementSpecificationResults))) + .Select(x => (ISupplementSpecificationResults) Activator.CreateInstance(x)); } public IEnumerable FindAssemblyContextsIn(Assembly assembly) { return assembly.GetExportedTypes() - .Where(x => x.GetTypeInfo().IsClass && !x.GetTypeInfo().IsAbstract && x.GetInterfaces().Contains(typeof(IAssemblyContext))) - .Select(x => (IAssemblyContext)Activator.CreateInstance(x)); + .Where(x => x.GetTypeInfo().IsClass && !x.GetTypeInfo().IsAbstract && x.GetInterfaces().Contains(typeof(IAssemblyContext))) + .Select(x => (IAssemblyContext) Activator.CreateInstance(x)); } Context CreateContextFrom(Type type) @@ -76,30 +113,52 @@ static bool HasSpecificationMembers(Type type) static IEnumerable EnumerateContextsIn(Assembly assembly) { return assembly - .GetTypes() - .Where(IsContext) - .OrderBy(t => t.Namespace); + .GetTypes() + .Where(IsContext); } + } - public Context FindContexts(Type type) + public static class FilteringExtensions + { + public static IEnumerable FilterBy(this IEnumerable types, RunOptions options) { - if (IsContext(type)) + if (options == null) { - return CreateContextFrom(type); + return types; } - return null; - } + var filteredTypes = types; - public Context FindContexts(FieldInfo info) - { - Type type = info.DeclaringType; - if (IsContext(type)) + var restrictToTypes = new HashSet(options.Filters, StringComparer.OrdinalIgnoreCase); + + if (restrictToTypes.Any()) + { + filteredTypes = filteredTypes.Where(x => restrictToTypes.Contains(x.FullName)); + } + + var includeTags = new HashSet(options.IncludeTags.Select(tag => new Tag(tag))); + var excludeTags = new HashSet(options.ExcludeTags.Select(tag => new Tag(tag))); + + if (includeTags.Any() || excludeTags.Any()) { - return CreateContextFrom(type, info); + var extractor = new AttributeTagExtractor(); + + var filteredTypesWithTags = filteredTypes.Select(type => (Type: type, Tags: extractor.ExtractTags(type))); + + if (includeTags.Any()) + { + filteredTypesWithTags = filteredTypesWithTags.Where(x => x.Tags.Intersect(includeTags).Any()); + } + + if (excludeTags.Any()) + { + filteredTypesWithTags = filteredTypesWithTags.Where(x => !x.Tags.Intersect(excludeTags).Any()); + } + + filteredTypes = filteredTypesWithTags.Select(x => x.Type); } - return null; + return filteredTypes; } } } diff --git a/src/Machine.Specifications/Runner/Impl/AssemblyRunner.cs b/src/Machine.Specifications/Runner/Impl/AssemblyRunner.cs index e3700137..8c807bd2 100644 --- a/src/Machine.Specifications/Runner/Impl/AssemblyRunner.cs +++ b/src/Machine.Specifications/Runner/Impl/AssemblyRunner.cs @@ -44,18 +44,17 @@ public void Run(Assembly assembly, IEnumerable contexts) try { - hasExecutableSpecifications = contexts.Any(x => x.HasExecutableSpecifications); - var globalCleanups = _explorer.FindAssemblyWideContextCleanupsIn(assembly).ToList(); var specificationSupplements = _explorer.FindSpecificationSupplementsIn(assembly).ToList(); - if (hasExecutableSpecifications) - { - _assemblyStart(assembly); - } - foreach (var context in contexts) { + if (!hasExecutableSpecifications) + { + _assemblyStart(assembly); + hasExecutableSpecifications = true; + } + RunContext(context, globalCleanups, specificationSupplements); } } diff --git a/src/Machine.Specifications/Runner/Impl/DefaultRunner.cs b/src/Machine.Specifications/Runner/Impl/DefaultRunner.cs index e722297b..bf96c872 100644 --- a/src/Machine.Specifications/Runner/Impl/DefaultRunner.cs +++ b/src/Machine.Specifications/Runner/Impl/DefaultRunner.cs @@ -1,15 +1,15 @@ using System; using System.Collections.Generic; -using System.IO; using System.Linq; using System.Reflection; -using System.Xml.Linq; -using System.Xml.XPath; using Machine.Specifications.Explorers; using Machine.Specifications.Model; using Machine.Specifications.Utility; #if !NETSTANDARD +using System.IO; +using System.Xml.Linq; +using System.Xml.XPath; using System.Runtime.Remoting; using System.Runtime.Remoting.Messaging; using System.Security; @@ -66,7 +66,7 @@ public DefaultRunner(ISpecificationRunListener listener, RunOptions options, boo public void RunAssembly(Assembly assembly) { - var contexts = _explorer.FindContextsIn(assembly); + var contexts = _explorer.FindContextsIn(assembly, _options); var map = CreateMap(assembly, contexts); StartRun(map); @@ -76,14 +76,14 @@ public void RunAssemblies(IEnumerable assemblies) { var map = new Dictionary>(); - assemblies.Each(assembly => map.Add(assembly, _explorer.FindContextsIn(assembly))); + assemblies.Each(assembly => map.Add(assembly, _explorer.FindContextsIn(assembly, _options))); StartRun(map); } public void RunNamespace(Assembly assembly, string targetNamespace) { - var contexts = _explorer.FindContextsIn(assembly, targetNamespace); + var contexts = _explorer.FindContextsIn(assembly, targetNamespace, _options); StartRun(CreateMap(assembly, contexts)); } @@ -102,7 +102,7 @@ public void RunMember(Assembly assembly, MemberInfo member) public void RunType(Assembly assembly, Type type, IEnumerable specs) { - Context context = _explorer.FindContexts(type); + Context context = _explorer.FindContexts(type, _options); IEnumerable specsToRun = context.Specifications.Where(s => specs.Contains(s.FieldInfo.Name)); context.Filter(specsToRun); @@ -112,15 +112,15 @@ public void RunType(Assembly assembly, Type type, IEnumerable specs) void RunField(MemberInfo member, Assembly assembly) { var fieldInfo = (FieldInfo)member; - var context = _explorer.FindContexts(fieldInfo); + var context = _explorer.FindContexts(fieldInfo, _options); StartRun(CreateMap(assembly, new[] { context })); } void RunClass(MemberInfo member, Assembly assembly) { - Type type = member.AsType(); - var context = _explorer.FindContexts(type); + var type = member.AsType(); + var context = _explorer.FindContexts(type, _options); if (context == null) { @@ -144,12 +144,8 @@ void StartRun(IDictionary> contextMap) _runStart.Invoke(); } - foreach (var pair in contextMap) + foreach (var (assembly, contexts) in contextMap) { - var assembly = pair.Key; - // TODO: move this filtering to a more sensible place - var contexts = pair.Value.FilteredBy(_options); - _assemblyRunner.Run(assembly, contexts); } @@ -234,37 +230,4 @@ public IMessageCtrl AsyncProcessMessage(IMessage msg, IMessageSink replySink) #endif } - - public static class TagFilteringExtensions - { - public static IEnumerable FilteredBy(this IEnumerable contexts, RunOptions options) - { - if (options == null) - throw new ArgumentNullException("options"); - - var results = contexts; - - if (options.Filters.Any()) - { - var includeFilters = options.Filters; - - results = results.Where(x => includeFilters.Any(filter => StringComparer.OrdinalIgnoreCase.Equals(filter, x.Type.FullName))); - } - - if (options.IncludeTags.Any()) - { - var tags = options.IncludeTags.Select(tag => new Tag(tag)); - - results = results.Where(x => x.Tags.Intersect(tags).Any()); - } - - if (options.ExcludeTags.Any()) - { - var tags = options.ExcludeTags.Select(tag => new Tag(tag)); - results = results.Where(x => !x.Tags.Intersect(tags).Any()); - } - - return results; - } - } } diff --git a/src/Machine.Specifications/Utility/KeyValuePairExtensions.cs b/src/Machine.Specifications/Utility/KeyValuePairExtensions.cs new file mode 100644 index 00000000..4f402b64 --- /dev/null +++ b/src/Machine.Specifications/Utility/KeyValuePairExtensions.cs @@ -0,0 +1,16 @@ +using System.Collections.Generic; + +namespace Machine.Specifications.Runner.Impl +{ + internal static class KeyValuePairExtensions + { + public static void Deconstruct( + this KeyValuePair kvp, + out TKey key, + out TValue value) + { + key = kvp.Key; + value = kvp.Value; + } + } +}