benchmarks updated

MoonStorm · Feb 16, 2022 · 7907e11 · 7907e11
1 parent 0ceac05
commit 7907e11
Show file tree

Hide file tree

Showing 5 changed files with 118 additions and 101 deletions.
diff --git a/Dapper.FastCrud.Benchmarks/BenchmarkSteps.cs b/Dapper.FastCrud.Benchmarks/BenchmarkSteps.cs
@@ -1,39 +1,50 @@
-using Dapper.FastCrud.Tests.Contexts;
-using System;
-using System.Diagnostics;
-using System.IO;
-using System.Text.RegularExpressions;
-using TechTalk.SpecFlow;
-
-namespace Dapper.FastCrud.Benchmarks
+namespace Dapper.FastCrud.Benchmarks
 {
     using Dapper.FastCrud.Tests.Common;
+    using Dapper.FastCrud.Tests.Contexts;
+    using NUnit.Framework;
+    using System;
+    using System.Collections;
+    using System.Collections.Concurrent;
+    using System.Diagnostics;
+    using System.IO;
+    using System.Reflection;
+    using System.Text.RegularExpressions;
+    using TechTalk.SpecFlow;
 
     [Binding]
     public sealed class BenchmarkSteps
     {
         private readonly DatabaseTestContext _testContext;
+        private static readonly Regex _benchmarkHeaderRegex = new Regex($@"(?<=#+\s*?Automatic Benchmark Report)[^{Environment.NewLine}]*", RegexOptions.Singleline| RegexOptions.Compiled);
+        private static readonly Regex _newEntryInsertRegex = new Regex($@"(?<=\|\s*\<a name=""new_entry_marker""\/\>\s*\|\s*){Environment.NewLine}", RegexOptions.Singleline|RegexOptions.Compiled);
 
         public BenchmarkSteps(DatabaseTestContext testContext)
         {
             _testContext = testContext;
         }
 
+        [BeforeFeature]
+        public static void BenchmarksSetup()
+        {
+            Process.GetCurrentProcess().PriorityClass = ProcessPriorityClass.High;
+            Assert.That(Process.GetCurrentProcess().PriorityClass, Is.EqualTo(ProcessPriorityClass.High));
+        }
+
         [When(@"I report the stopwatch value for (.*) finished processing (.*) operations of type (.*)")]
         public void WhenIReportTheStopwatchValueFor(string ormType, int opCount, string operation)
         {
-            Trace.WriteLine($"Stopwatch reported: {_testContext.Stopwatch.Elapsed.TotalMilliseconds:0,0.00} milliseconds for {ormType}");
+            var elapsedTime = _testContext.Stopwatch.Elapsed;
+            Trace.WriteLine($"Stopwatch reported: {elapsedTime.TotalMilliseconds:0,0.00} milliseconds for {ormType}");
 
-#if !DEBUG
+ #if !DEBUG
             // automatically update the docs
             var docsPath = Path.Combine(typeof(BenchmarkSteps).Assembly.GetDirectory(), "../../../../README.MD");
             var docsContents = File.ReadAllText(docsPath);
 
             var reportTitle = $"| {ormType} | {operation} | {opCount} |";
-            var report = $"{reportTitle} {_testContext.Stopwatch.Elapsed.TotalMilliseconds:0,0.00} | {_testContext.Stopwatch.Elapsed.TotalMilliseconds * 1000 / opCount:0,0.00} |{Environment.NewLine}";
+            var report = $"{reportTitle} {elapsedTime.TotalMilliseconds:0,0.00} | {elapsedTime.TotalMilliseconds * 1000 / opCount:0,0.00} |{Environment.NewLine}";
 
-            var benchmarkHeaderRegex = new Regex($@"(?<=#+\s*?Automatic Benchmark Report)[^{Environment.NewLine}]*", RegexOptions.Singleline);
-            var emptySpaceInsertRegex = new Regex($@"(?<=#+\s*?Automatic Benchmark Report(.*?{Environment.NewLine}){{3,3}})\s*?", RegexOptions.Singleline);
             var reportReplaceRegex = new Regex($@"{reportTitle.Replace("|", @"\|")}.*?{Environment.NewLine}", RegexOptions.Singleline);
 
             if (reportReplaceRegex.Match(docsContents).Success)
@@ -42,14 +53,13 @@ public void WhenIReportTheStopwatchValueFor(string ormType, int opCount, string
             }
             else
             {
-                docsContents = emptySpaceInsertRegex.Replace(docsContents, report, 1);
+                docsContents = _newEntryInsertRegex.Replace(docsContents, $@"{Environment.NewLine}{report}", 1);
             }
 
-            docsContents = benchmarkHeaderRegex.Replace(docsContents, $" (Last Run: {DateTime.Now:D})", 1);
+            docsContents = _benchmarkHeaderRegex.Replace(docsContents, $" (Last Run: {DateTime.Now:D})", 1);
 
             File.WriteAllText(docsPath, docsContents);
 #endif
         }
-
     }
 }
diff --git a/Dapper.FastCrud.Benchmarks/Benchmarks.feature b/Dapper.FastCrud.Benchmarks/Benchmarks.feature
@@ -1,6 +1,6 @@
 Feature: Benchmarks
 
-Scenario Outline: Insert Benchmark
+Scenario Outline: Insert benchmark
 	Given I have initialized a <database type> database
 	When I start the stopwatch
 	And I insert <entity count> <entity type> using <orm>
@@ -9,12 +9,14 @@ Scenario Outline: Insert Benchmark
 	Then I should have <entity count> <entity type> in the database
 	And I cleanup the <database type> database
 	Examples: 
-	| database type     | entity type        | entity count | orm               |
-	| Benchmark LocalDb | benchmark entities | 30000        | Simple Crud       |
-	| Benchmark LocalDb | benchmark entities | 30000        | Dapper Extensions |
-	| Benchmark LocalDb | benchmark entities | 30000        | Fast Crud         |
-	| Benchmark LocalDb | benchmark entities | 30000        | Dapper            |
-	| Benchmark LocalDb | benchmark entities | 30000        | Entity Framework  |
+	| database type     | entity type        | entity count | orm                               |
+	| Benchmark LocalDb | benchmark entities | 10000        | Simple Crud                       |
+	| Benchmark LocalDb | benchmark entities | 10000        | Dapper Extensions                 |
+	| Benchmark LocalDb | benchmark entities | 10000        | Fast Crud                         |
+	| Benchmark LocalDb | benchmark entities | 10000        | Dapper                            |
+	| Benchmark LocalDb | benchmark entities | 10000        | Entity Framework - single op/call |
+	# unrealistic
+	# | Benchmark LocalDb | benchmark entities | 10000        | Entity Framework - batch          |
 
 #Scenario Outline: Batch Select No Filter No Warmup Benchmark
 #	Given I have initialized a <database type> database
@@ -29,35 +31,32 @@ Scenario Outline: Insert Benchmark
 #	And I cleanup the <database type> database
 #	Examples: 
 #	| database type | entity type        | entity count | orm         |
-#	| LocalDb   | benchmark entities | 20000        | Simple Crud       |
-#	| LocalDb   | benchmark entities | 20000        | Dapper Extensions |
-#	| LocalDb   | benchmark entities | 20000        | Fast Crud         |
-#	| LocalDb   | benchmark entities | 20000        | Dapper            |
+#	| LocalDb   | benchmark entities | 100000        | Simple Crud       |
+#	| LocalDb   | benchmark entities | 100000        | Dapper Extensions |
+#	| LocalDb   | benchmark entities | 100000        | Fast Crud         |
+#	| LocalDb   | benchmark entities | 100000        | Dapper            |
 
-Scenario Outline: Batch Select No Filter
+Scenario Outline: Select all
 	Given I have initialized a <database type> database
 	When I insert <entity count> <entity type> entities using ADO .NET
 	And I refresh the database connection
 	And I start the stopwatch
-	And I select all the <entity type> entities using <orm>
-	And I clear all the queried entities
-	And I select all the <entity type> entities using <orm>
-	And I clear all the queried entities
-	And I select all the <entity type> entities using <orm>
+	And I select all the <entity type> entities using <orm> <op count> times
 	And I stop the stopwatch
-	And I report the stopwatch value for <orm> finished processing 3 operations of type select all
+	And I report the stopwatch value for <orm> finished processing <op count> operations of type select all
 	Then I should have queried <entity count> <entity type> entities
-	Then the queried <entity type> entities should be the same as the inserted ones
+	# already tested for consistency (this takes a long time)
+	#And the queried <entity type> entities should be shallowly the same as the inserted ones
 	And I cleanup the <database type> database
 	Examples: 
-	| database type     | entity type | entity count | orm               |
-	| Benchmark LocalDb | benchmark   | 30000        | Simple Crud       |
-	| Benchmark LocalDb | benchmark   | 30000        | Dapper Extensions |
-	| Benchmark LocalDb | benchmark   | 30000        | Fast Crud         |
-	| Benchmark LocalDb | benchmark   | 30000        | Dapper            |
-	| Benchmark LocalDb | benchmark   | 30000        | Entity Framework  |
+	| database type     | entity type | entity count | op count | orm               |
+	| Benchmark LocalDb | benchmark   | 10           | 10000    | Simple Crud       |
+	| Benchmark LocalDb | benchmark   | 10           | 10000    | Dapper Extensions |
+	| Benchmark LocalDb | benchmark   | 10           | 10000    | Fast Crud         |
+	| Benchmark LocalDb | benchmark   | 10           | 10000    | Dapper            |
+	| Benchmark LocalDb | benchmark   | 10           | 10000    | Entity Framework  |
 
-Scenario Outline: Single Delete Benchmark
+Scenario Outline: Delete by id
 	Given I have initialized a <database type> database
 	When I insert <entity count> <entity type> entities using ADO .NET
 	And I refresh the database connection
@@ -68,14 +67,16 @@ Scenario Outline: Single Delete Benchmark
 	Then I should have 0 <entity type> entities in the database
 	And I cleanup the <database type> database
 	Examples: 
-	| database type     | entity type | entity count | orm               |
-	| Benchmark LocalDb | benchmark   | 30000        | Simple Crud       |
-	| Benchmark LocalDb | benchmark   | 30000        | Dapper Extensions |
-	| Benchmark LocalDb | benchmark   | 30000        | Fast Crud         |
-	| Benchmark LocalDb | benchmark   | 30000        | Dapper            |
-	| Benchmark LocalDb | benchmark   | 30000        | Entity Framework  |
+	| database type     | entity type | entity count | orm                               |
+	| Benchmark LocalDb | benchmark   | 10000        | Simple Crud                       |
+	| Benchmark LocalDb | benchmark   | 10000        | Dapper Extensions                 |
+	| Benchmark LocalDb | benchmark   | 10000        | Fast Crud                         |
+	| Benchmark LocalDb | benchmark   | 10000        | Dapper                            |
+	| Benchmark LocalDb | benchmark   | 10000        | Entity Framework - single op/call |
+	# unrealistic
+	#| Benchmark LocalDb | benchmark   | 10000        | Entity Framework - batch          |
 
-Scenario Outline: Single Select Id Filter Benchmark
+Scenario Outline: Select by id
 	Given I have initialized a <database type> database
 	When I insert <entity count> <entity type> entities using ADO .NET
 	And I refresh the database connection
@@ -84,31 +85,36 @@ Scenario Outline: Single Select Id Filter Benchmark
 	And I stop the stopwatch
 	And I report the stopwatch value for <orm> finished processing <entity count> operations of type select by id
 	Then I should have queried <entity count> <entity type> entities
-	Then the queried <entity type> entities should be the same as the inserted ones
-	Then I cleanup the <database type> database
+	# already tested for consistency (this takes a long time)
+	#And the queried <entity type> entities should be shallowly the same as the inserted ones
+	And I cleanup the <database type> database
 	Examples: 
 	| database type     | entity type | entity count | orm               |
-	| Benchmark LocalDb | benchmark   | 30000        | Simple Crud       |
-	| Benchmark LocalDb | benchmark   | 30000        | Dapper Extensions |
-	| Benchmark LocalDb | benchmark   | 30000        | Fast Crud         |
-	| Benchmark LocalDb | benchmark   | 30000        | Dapper            |
-	| Benchmark LocalDb | benchmark   | 30000        | Entity Framework  |
+	| Benchmark LocalDb | benchmark   | 10000        | Simple Crud       |
+	| Benchmark LocalDb | benchmark   | 10000        | Dapper Extensions |
+	| Benchmark LocalDb | benchmark   | 10000        | Fast Crud         |
+	| Benchmark LocalDb | benchmark   | 10000        | Dapper            |
+	| Benchmark LocalDb | benchmark   | 10000        | Entity Framework  |
 
-Scenario Outline: Single Update Benchmark
+Scenario Outline: Update by id
 	Given I have initialized a <database type> database
-	When I insert <entity count> <entity type> entities using ADO .NET
+	When I insert <entity count> <entity type> entities using <orm>
 	And I refresh the database connection
 	And I start the stopwatch
 	And I update all the <entity type> entities that I previously inserted using <orm>
 	And I stop the stopwatch
 	And I report the stopwatch value for <orm> finished processing <entity count> operations of type update
-	And I select all the <entity type> entities using Dapper
-	Then the queried <entity type> entities should be the same as the updated ones
+	And I select all the <entity type> entities using Dapper 1 times
+	Then I should have queried <entity count> <entity type> entities
+	# already tested for consistency (this takes a long time)
+	#Then the queried <entity type> entities should be shallowly the same as the updated ones
 	Then I cleanup the <database type> database
 	Examples: 
-	| database type     | entity type | entity count | orm               |
-	| Benchmark LocalDb | benchmark   | 30000        | Simple Crud       |
-	| Benchmark LocalDb | benchmark   | 30000        | Dapper Extensions |
-	| Benchmark LocalDb | benchmark   | 30000        | Fast Crud         |
-	| Benchmark LocalDb | benchmark   | 30000        | Dapper            |
-	| Benchmark LocalDb | benchmark   | 30000        | Entity Framework  |
+	| database type     | entity type | entity count | orm                               |
+	| Benchmark LocalDb | benchmark   | 10000        | Simple Crud                       |
+	| Benchmark LocalDb | benchmark   | 10000        | Dapper Extensions                 |
+	| Benchmark LocalDb | benchmark   | 10000        | Fast Crud                         |
+	| Benchmark LocalDb | benchmark   | 10000        | Dapper                            |
+	| Benchmark LocalDb | benchmark   | 10000        | Entity Framework - single op/call |
+	#unrealistic
+	# | Benchmark LocalDb | benchmark   | 10000        | Entity Framework - batch         |
diff --git a/Dapper.FastCrud.Benchmarks/Dapper.FastCrud.Benchmarks.csproj b/Dapper.FastCrud.Benchmarks/Dapper.FastCrud.Benchmarks.csproj
@@ -30,6 +30,7 @@
     <PackageReference Include="Microsoft.Windows.Compatibility" Version="3.1.2" />
     <PackageReference Include="MySql.Data" Version="8.0.27" />
     <PackageReference Include="Npgsql" Version="6.0.2" />
+    <PackageReference Include="NUnit3TestAdapter" Version="4.2.1" />
     <PackageReference Include="SpecFlow.NUnit.Runners" Version="3.9.40" />
     <PackageReference Include="SpecFlow.Tools.MsBuild.Generation" Version="3.9.40" />
     <PackageReference Include="System.Data.SqlClient" Version="4.8.3" />

diff --git a/README.md b/README.md
@@ -33,7 +33,7 @@ Type safety, clean code, less mistakes, more peace of mind, while still being cl
 - The following mapping styles are supported:
   - Database first (limited to SQL Server)
   - Code first, using model data annotations (preferred)
-  - Fluent validation for POCO objects
+  - Fluent registration for POCO objects
   - Semi-POCO using metadata objects
 - Extensibility points are also provided.
 
@@ -88,52 +88,52 @@ Type safety, clean code, less mistakes, more peace of mind, while still being cl
 
 
 #### Speed
-Most of us love Dapper for its speed. 
-Let's have a look at how ``Fast Crud`` performs against other similar libraries out there:  
+Let's have a look at some of the most popular ORMs out there in terms of speed:  
 
 - ``Dapper.SimpleCRUD v2.3.0``
 - ``DapperExtensions v1.6.3 ``
 - ``Entity Framework Core v6.0.2`` 
 
-##### Automatic Benchmark Report (Last Run: Sunday, January 03, 2016)
+##### Automatic Benchmark Report (Last Run: Wednesday, February 16, 2022)
 
 |  Library   |  Operation | Op Count |Time (ms) | Time/op (μs) |
 |------------|------------|----------|----------|--------------|
+| <a name="new_entry_marker"/> |
 ||||||
-| Dapper | insert | 30000 | 10,016.88 | 333.90 |
-| Fast Crud | insert | 30000 | 10,431.64 | 347.72 |
-| Dapper Extensions | insert | 30000 | 13,272.40 | 442.41 |
-| Simple Crud | insert | 30000 | 19,954.31 | 665.14 |
-| Entity Framework | insert | 30000 | 43,636.47 | 1,454.55 |
+| Dapper | insert | 10000 | 2,974.40 | 297.44 |
+| Fast Crud | insert | 10000 | 3,172.67 | 317.27 |
+| Dapper Extensions | insert | 10000 | 3,529.16 | 352.92 |
+| Simple Crud | insert | 10000 | 3,110.40 | 311.04 |
+| Entity Framework - single op/call | insert | 10000 | 119,784.63 | 11,978.46 |
 ||||||
-| Dapper | update | 30000 | 6,439.43 | 214.65 |
-| Fast Crud | update | 30000 | 6,668.78 | 222.29 |
-| Dapper Extensions | update | 30000 | 8,803.26 | 293.44 |
-| Simple Crud | update | 30000 | 10,204.28 | 340.14 |
-| Entity Framework | update | 30000 | 39,954.88 | 1,331.83 |
+| Dapper | update | 10000 | 3,319.82 | 331.98 |
+| Fast Crud | update | 10000 | 3,276.99 | 327.70 |
+| Dapper Extensions | update | 10000 | 3,657.30 | 365.73 |
+| Simple Crud | update | 10000 | 3,150.76 | 315.08 |
+| Entity Framework - single op/call | update | 10000 | 235,678.77 | 23,567.88 |
 ||||||
-| Dapper | delete | 30000 | 8,312.94 | 277.10 |
-| Fast Crud | delete | 30000 | 8,693.02 | 289.77 |
-| Dapper Extensions | delete | 30000 | 10,804.55 | 360.15 |
-| Simple Crud | delete | 30000 | 13,642.98 | 454.77 |
-| Entity Framework | delete | 30000 | 35,973.40 | 1,199.11 |
+| Dapper | delete | 10000 | 2,869.64 | 286.96 |
+| Fast Crud | delete | 10000 | 2,916.99 | 291.70 |
+| Dapper Extensions | delete | 10000 | 3,106.24 | 310.62 |
+| Simple Crud | delete | 10000 | 3,120.13 | 312.01 |
+| Entity Framework - single op/call | delete | 10000 | 7,326.03 | 732.60 |
 ||||||
-| Dapper | select by id | 30000 | 6,010.58 | 200.35 |
-| Fast Crud | select by id | 30000 | 6,172.08 | 205.74 |
-| Dapper Extensions | select by id | 30000 | 6,355.57 | 211.85 |
-| Simple Crud | select by id | 30000 | 12,912.19 | 430.41 |
-| Entity Framework | select by id | 30000 | 21,126.72 | 704.22 |
+| Dapper | select by id | 10000 | 1,499.67 | 149.97 |
+| Fast Crud | select by id | 10000 | 1,861.50 | 186.15 |
+| Dapper Extensions | select by id | 10000 | 2,026.22 | 202.62 |
+| Simple Crud | select by id | 10000 | 2,113.70 | 211.37 |
+| Entity Framework | select by id | 10000 | 3,665.71 | 366.57 |
 ||||||
-| Dapper | select all | 3 | 217.06 | 72,353.87 |
-| Fast Crud | select all | 3 | 262.41 | 87,468.83 |
-| Dapper Extensions | select all | 3 | 291.22 | 97,073.00 |
-| Simple Crud | select all | 3 | 814.65 | 271,549.57 |
-| Entity Framework | select all | 3 | 5,431.27 | 1,810,423.27 |
-
-Dapper is used as reference only, for the purpose of observing the overhead of the automatic SQL generation compared to verbatim  constructs. The database is re-created at every run, data file is pre-allocated, and the statistics are turned off.
-The tests are following the same steps and are running on the same number and size of records.
-
-Environment details: Windows 7, i7 3930K @3.2GHz, 16GB DDR3-1600, SATA600 SSD  
+| Dapper | select all | 10000 | 1,449.41 | 144.94 |
+| Fast Crud | select all | 10000 | 1,748.86 | 174.89 |
+| Dapper Extensions | select all | 10000 | 1,762.69 | 176.27 |
+| Simple Crud | select all | 10000 | 2,281.44 | 228.14 |
+| Entity Framework | select all | 10000 | 6,233.53 | 623.35 |
+
+Dapper is included for reference. All the libs involved get their own internal cache cleared before each run, the benchmark database is re-created, data file gets pre-allocated, and the statistics are turned off.
+The tests are following the same steps and are running in the same environment on the same number and size of records.
+We're happy to see that most light ORMs have matured enough over the years and caught up with ``FastCrud`` and even exceeded its speed in some cases. 
+The really heavy ones are still trailing far behind.
 
 You can find more details about how to run the benchmarks yourself in the wiki.
 

diff --git a/RunBenchmarks.ps1 b/RunBenchmarks.ps1
@@ -1,6 +1,6 @@
 $ErrorActionPreference = "Stop"
 
-& dotnet test "$PSScriptRoot\Dapper.FastCRUD.Benchmarks\bin\Release\net6.0\Dapper.FastCrud.Benchmarks.dll"
+dotnet test "$PSScriptRoot\Dapper.FastCRUD.Benchmarks\bin\Release\net6.0\Dapper.FastCrud.Benchmarks.dll" -l "console;verbosity=normal"
 if ($LASTEXITCODE -ne 0){
     throw "tests failed"
 }