Performance Improvement - Perform test filtering earlier in process (#…

…433) * Add test to run one out of 10000 specs * Prevent multiple enumeration of Contexts * Move filtering to before Context is created * Simplify FindContexts to use same pattern as FindContextsIn * Use IEnumerable extension method * Display compile diagnostic when compilation fails * Make parameter optional * Extend test to track instantiations * Update src/Machine.Specifications/Utility/KeyValuePairExtensions.cs Co-authored-by: Robert Coltheart <[email protected]> * Update src/Machine.Specifications/Explorers/AssemblyExplorer.cs Co-authored-by: Robert Coltheart <[email protected]> * Update src/Machine.Specifications/Explorers/AssemblyExplorer.cs Co-authored-by: Robert Coltheart <[email protected]> Co-authored-by: Robert Coltheart <[email protected]>
machine · Apr 24, 2021 · 7c910d6 · 7c910d6
1 parent c4eb55a
commit 7c910d6
Show file tree

Hide file tree

Showing 8 changed files with 235 additions and 81 deletions.
diff --git a/src/Machine.Specifications.Runner.Console.Specs/ProgramSpecs.cs b/src/Machine.Specifications.Runner.Console.Specs/ProgramSpecs.cs
@@ -162,7 +162,7 @@ public class when_a_specification_fails_and_silent_is_set : FailingSpecs
             console.Lines.ShouldContain(l => l.Contains("hi scott, love you, miss you."));
 
         It should_separate_failures_from_the_rest_of_the_test_run = () =>
-            console.Output.ShouldMatchRegex(String.Format("\\S{0}{0}Failures:{0}{0}\\S", Regex.Escape(Environment.NewLine)));
+            console.Output.ShouldMatchRegex(String.Format("{0}Failures:{0}{0}\\S", Regex.Escape(Environment.NewLine)));
     }
 
     [Subject("Console runner")]
@@ -189,7 +189,7 @@ public class when_a_specification_fails_and_progress_is_set : FailingSpecs
             console.Lines.ShouldContain(l => l.Contains("hi scott, love you, miss you."));
 
         It should_separate_failures_from_the_rest_of_the_test_run = () =>
-            console.Output.ShouldMatchRegex(String.Format("\\S{0}{0}{0}Failures:{0}{0}\\S", Regex.Escape(Environment.NewLine)));
+            console.Output.ShouldMatchRegex(String.Format("{0}Failures:{0}{0}\\S", Regex.Escape(Environment.NewLine)));
     }
 
     [Subject("Console runner")]

diff --git a/src/Machine.Specifications.Specs/CompileContext.cs b/src/Machine.Specifications.Specs/CompileContext.cs
@@ -34,7 +34,7 @@ public string Compile(string code)
                 .Emit(filename);
 
             if (!result.Success)
-                throw new InvalidOperationException();
+                throw new InvalidOperationException(result.Diagnostics[0].GetMessage());
 #else
             var parameters = new CompilerParameters
             {

diff --git a/src/Machine.Specifications.Specs/Fixtures/LargeFixture.cs b/src/Machine.Specifications.Specs/Fixtures/LargeFixture.cs
@@ -0,0 +1,59 @@
+using System.Text;
+
+namespace Machine.Specifications.Specs.Fixtures
+{
+    public class LargeFixture
+    {
+        public static string CreateCode(int specCount)
+        {
+            var sb = new StringBuilder();
+
+            sb.AppendLine(@"
+using System;
+using System.Diagnostics;
+using System.Reflection;
+using System.Threading.Tasks;
+using Machine.Specifications;
+
+namespace Example.Large
+{
+    public class when_there_are_many_contexts
+    {
+        public static bool Created = false;
+
+        public when_there_are_many_contexts()
+        {
+            Created = true;
+        }
+
+        It spec = () => {};
+    }
+
+    public static class OtherTests
+    {
+        public static bool Created = false;
+    }
+
+");
+
+            for (var i = 1; i <= specCount; i++)
+            {
+                sb.AppendLine($@"
+    public class when_there_are_many_contexts_{i}
+    {{
+        public when_there_are_many_contexts_{i}()
+        {{
+            OtherTests.Created = true;
+        }}
+
+        It spec = () => {{}};
+    }}");
+            }
+
+            sb.AppendLine(@"
+}");
+
+            return sb.ToString();
+        }
+    }
+}
diff --git a/src/Machine.Specifications.Specs/Runner/SpecificationRunnerSpecs.cs b/src/Machine.Specifications.Specs/Runner/SpecificationRunnerSpecs.cs
@@ -1,4 +1,6 @@
 using System;
+using System.Diagnostics;
+using System.Linq;
 using System.Reflection;
 using Machine.Specifications.Runner;
 using Machine.Specifications.Runner.Impl;
@@ -691,6 +693,62 @@ public class when_running_a_context_inside_a_static_class_that_is_nested_in_a_no
             testListener.LastResult.Passed.ShouldBeTrue();
     }
 
+    [Subject("Specification Runner")]
+    public class when_running_a_single_spec_out_of_a_large_number_of_specifications : RunnerSpecs
+    {
+        static Type when_a_context_has_many_specifications;
+        static Type filtered_out_spec;
+        static TimeSpan elapsed { get; set; }
+
+        Establish context = () =>
+        {
+            using (var compiler = new CompileContext())
+            {
+                var assemblyPath = compiler.Compile(LargeFixture.CreateCode(10000));
+                var assembly = Assembly.LoadFile(assemblyPath);
+
+                when_a_context_has_many_specifications = assembly.GetType("Example.Large.when_there_are_many_contexts");
+                filtered_out_spec = assembly.GetType("Example.Large.OtherTests");
+            }
+        };
+
+        Because of = () =>
+        {
+            var runner = new DefaultRunner(testListener, new RunOptions(
+                Enumerable.Empty<string>(),
+                Enumerable.Empty<string>(),
+                new[] {when_a_context_has_many_specifications.FullName})
+            );
+
+            var sw = Stopwatch.StartNew();
+            runner.RunAssembly(when_a_context_has_many_specifications.Assembly);
+            sw.Stop();
+            elapsed = sw.Elapsed;
+        };
+
+        It should_run_the_single_specification = () =>
+        {
+            testListener.SpecCount.ShouldEqual(1);
+        };
+
+        It should_run_in_a_reasonable_period_of_time = () =>
+        {
+            elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(1));
+        };
+
+        It should_have_created_the_test_instance = () =>
+        {
+            var fieldInfo = when_a_context_has_many_specifications.GetField("Created");
+            ((bool) fieldInfo.GetValue(null)).ShouldBeTrue();
+        };
+
+        It should_have_not_have_created_any_of_the_filtered_out_tests = () =>
+        {
+            var fieldInfo = filtered_out_spec.GetField("Created");
+            ((bool) fieldInfo.GetValue(null)).ShouldBeFalse();
+        };
+    }
+
     public class RandomRunnerSpecs : RunnerSpecs
     {
         static CompileContext compiler;

diff --git a/src/Machine.Specifications/Explorers/AssemblyExplorer.cs b/src/Machine.Specifications/Explorers/AssemblyExplorer.cs
@@ -4,6 +4,7 @@
 using System.Reflection;
 using Machine.Specifications.Factories;
 using Machine.Specifications.Model;
+using Machine.Specifications.Runner;
 using Machine.Specifications.Sdk;
 using Machine.Specifications.Utility;
 
@@ -18,37 +19,73 @@ public AssemblyExplorer()
             _contextFactory = new ContextFactory();
         }
 
+        public Context FindContexts(Type type, RunOptions options = null)
+        {
+            var types = new[] {type};
+
+            return types
+                .Where(IsContext)
+                .FilterBy(options)
+                .Select(CreateContextFrom)
+                .FirstOrDefault();
+        }
+
+        public Context FindContexts(FieldInfo info, RunOptions options = null)
+        {
+            var types = new[] {info.DeclaringType};
+
+            return types
+                .Where(IsContext)
+                .FilterBy(options)
+                .Select(t => CreateContextFrom(t, info))
+                .FirstOrDefault();
+        }
+
         public IEnumerable<Context> FindContextsIn(Assembly assembly)
         {
-            return EnumerateContextsIn(assembly).Select(CreateContextFrom);
+            return FindContextsIn(assembly, options: null);
+        }
+
+        public IEnumerable<Context> FindContextsIn(Assembly assembly, RunOptions options)
+        {
+            return EnumerateContextsIn(assembly)
+                .FilterBy(options)
+                .OrderBy(t => t.Namespace)
+                .Select(CreateContextFrom);
         }
 
         public IEnumerable<Context> FindContextsIn(Assembly assembly, string targetNamespace)
+        {
+            return FindContextsIn(assembly, targetNamespace, options: null);
+        }
+
+        public IEnumerable<Context> FindContextsIn(Assembly assembly, string targetNamespace, RunOptions options)
         {
             return EnumerateContextsIn(assembly)
-              .Where(x => x.Namespace == targetNamespace)
-              .Select(CreateContextFrom);
+                .Where(x => x.Namespace == targetNamespace)
+                .FilterBy(options)
+                .Select(CreateContextFrom);
         }
 
         public IEnumerable<ICleanupAfterEveryContextInAssembly> FindAssemblyWideContextCleanupsIn(Assembly assembly)
         {
             return assembly.GetExportedTypes()
-              .Where(x => x.GetInterfaces().Contains(typeof(ICleanupAfterEveryContextInAssembly)))
-              .Select(x => (ICleanupAfterEveryContextInAssembly)Activator.CreateInstance(x));
+                .Where(x => x.GetInterfaces().Contains(typeof(ICleanupAfterEveryContextInAssembly)))
+                .Select(x => (ICleanupAfterEveryContextInAssembly) Activator.CreateInstance(x));
         }
 
         public IEnumerable<ISupplementSpecificationResults> FindSpecificationSupplementsIn(Assembly assembly)
         {
             return assembly.GetExportedTypes()
-              .Where(x => x.GetInterfaces().Contains(typeof(ISupplementSpecificationResults)))
-              .Select(x => (ISupplementSpecificationResults)Activator.CreateInstance(x));
+                .Where(x => x.GetInterfaces().Contains(typeof(ISupplementSpecificationResults)))
+                .Select(x => (ISupplementSpecificationResults) Activator.CreateInstance(x));
         }
 
         public IEnumerable<IAssemblyContext> FindAssemblyContextsIn(Assembly assembly)
         {
             return assembly.GetExportedTypes()
-              .Where(x => x.GetTypeInfo().IsClass && !x.GetTypeInfo().IsAbstract && x.GetInterfaces().Contains(typeof(IAssemblyContext)))
-              .Select(x => (IAssemblyContext)Activator.CreateInstance(x));
+                .Where(x => x.GetTypeInfo().IsClass && !x.GetTypeInfo().IsAbstract && x.GetInterfaces().Contains(typeof(IAssemblyContext)))
+                .Select(x => (IAssemblyContext) Activator.CreateInstance(x));
         }
 
         Context CreateContextFrom(Type type)
@@ -76,30 +113,52 @@ static bool HasSpecificationMembers(Type type)
         static IEnumerable<Type> EnumerateContextsIn(Assembly assembly)
         {
             return assembly
-              .GetTypes()
-              .Where(IsContext)
-              .OrderBy(t => t.Namespace);
+                .GetTypes()
+                .Where(IsContext);
         }
+    }
 
-        public Context FindContexts(Type type)
+    public static class FilteringExtensions
+    {
+        public static IEnumerable<Type> FilterBy(this IEnumerable<Type> types, RunOptions options)
         {
-            if (IsContext(type))
+            if (options == null)
             {
-                return CreateContextFrom(type);
+                return types;
             }
 
-            return null;
-        }
+            var filteredTypes = types;
 
-        public Context FindContexts(FieldInfo info)
-        {
-            Type type = info.DeclaringType;
-            if (IsContext(type))
+            var restrictToTypes = new HashSet<string>(options.Filters, StringComparer.OrdinalIgnoreCase);
+
+            if (restrictToTypes.Any())
+            {
+                filteredTypes = filteredTypes.Where(x => restrictToTypes.Contains(x.FullName));
+            }
+
+            var includeTags = new HashSet<Tag>(options.IncludeTags.Select(tag => new Tag(tag)));
+            var excludeTags = new HashSet<Tag>(options.ExcludeTags.Select(tag => new Tag(tag)));
+
+            if (includeTags.Any() || excludeTags.Any())
             {
-                return CreateContextFrom(type, info);
+                var extractor = new AttributeTagExtractor();
+
+                var filteredTypesWithTags = filteredTypes.Select(type => (Type: type, Tags: extractor.ExtractTags(type)));
+
+                if (includeTags.Any())
+                {
+                    filteredTypesWithTags = filteredTypesWithTags.Where(x => x.Tags.Intersect(includeTags).Any());
+                }
+
+                if (excludeTags.Any())
+                {
+                    filteredTypesWithTags = filteredTypesWithTags.Where(x => !x.Tags.Intersect(excludeTags).Any());
+                }
+
+                filteredTypes = filteredTypesWithTags.Select(x => x.Type);
             }
 
-            return null;
+            return filteredTypes;
         }
     }
 }
diff --git a/src/Machine.Specifications/Runner/Impl/AssemblyRunner.cs b/src/Machine.Specifications/Runner/Impl/AssemblyRunner.cs
@@ -44,18 +44,17 @@ public void Run(Assembly assembly, IEnumerable<Context> contexts)
 
             try
             {
-                hasExecutableSpecifications = contexts.Any(x => x.HasExecutableSpecifications);
-
                 var globalCleanups = _explorer.FindAssemblyWideContextCleanupsIn(assembly).ToList();
                 var specificationSupplements = _explorer.FindSpecificationSupplementsIn(assembly).ToList();
 
-                if (hasExecutableSpecifications)
-                {
-                    _assemblyStart(assembly);
-                }
-
                 foreach (var context in contexts)
                 {
+                    if (!hasExecutableSpecifications)
+                    {
+                        _assemblyStart(assembly);
+                        hasExecutableSpecifications = true;
+                    }
+
                     RunContext(context, globalCleanups, specificationSupplements);
                 }
             }