Introduce new performance benchmark runners

BUILDGUIDE.md

@@ -451,20 +451,24 @@ $ sqlcmd -S localhost -U sa -P password
 1> quit
 ```
 
-The default `runnerconfig.json` expects a database named `sqlclient-perf-db`,
-but you may change the config to use any existing database.  All tables in
-the database will be dropped when running the benchmarks.
+The default `runnerconfig.jsonc` expects a database named `sqlclient-perf-db`,
+but you may change the config to use any existing database.  The benchmarks
+create and drop their own tables (typically prefixed with `perf_`) in this
+database; other existing tables are left untouched.
 
 ### Configure Runner
 
-Configure the benchmarks by editing the `runnerconfig.json` file directly in the
+Configure the benchmarks by editing the `runnerconfig.jsonc` file directly in the
 `PerformanceTests` directory with an appropriate connection string and benchmark
 settings:
 
 ```json
 {
   "ConnectionString": "Server=tcp:localhost; Integrated Security=true; Initial Catalog=sqlclient-perf-db;",
   "UseManagedSniOnWindows": false,
+  "UseOptimizedAsyncBehaviour": true,
+  "WaitForProfiler": false,
+  "UseNativeMemoryAndETWProfiler": false,
   "Benchmarks":
   {
     "SqlConnectionRunnerConfig":
@@ -484,36 +488,51 @@ settings:
 Individual benchmarks may be enabled or disabled, and each has several
 benchmarking options for fine tuning.
 
-After making edits to `runnerconfig.json` you must perform a build which will
+The top-level flags control global runner behavior:
+
+| Flag | Description |
+| --- | --- |
+| `UseManagedSniOnWindows` | Enables the managed SNI implementation on Windows instead of native SNI. |
+| `UseOptimizedAsyncBehaviour` | Enables packet multiplexing and other async optimizations in SqlClient. |
+| `WaitForProfiler` | Pauses at startup and prints the process ID so you can attach an external profiler (e.g. `dotnet-trace`) before benchmarks run. |
+| `UseNativeMemoryAndETWProfiler` | Attaches the `NativeMemoryProfiler` and `EtwProfiler` BenchmarkDotNet diagnosers. Windows only; has no effect on other OSes. |
+
+Some benchmarks (e.g. `DataTypeReaderRunner`, `DataTypeReaderAsyncRunner`) also
+read per-type test values from `datatypes.json` in the `PerformanceTests`
+directory. Like `runnerconfig.jsonc`, this file's location can be overridden
+with the `DATATYPES_CONFIG` environment variable.
+
+After making edits to `runnerconfig.jsonc` you must perform a build which will
 copy the file into the `artifacts` directory alongside the benchmark DLL.  By
-default, the benchmarks look for `runnerconfig.json` in the same directory as
+default, the benchmarks look for `runnerconfig.jsonc` in the same directory as
 the DLL.
 
 Optionally, to avoid polluting your git workspace and requiring a build after
-each config change, copy `runnerconfig.json` to a new file, make your edits
+each config change, copy `runnerconfig.jsonc` to a new file, make your edits
 there, and then specify the new file with the RUNNER_CONFIG environment
-variable.
+variable. The same approach works for `datatypes.json` via the
+`DATATYPES_CONFIG` environment variable.
 
 PowerShell:
 
 ```pwsh
-> copy runnerconfig.json $HOME\.configs\runnerconfig.json
+> copy runnerconfig.jsonc $HOME\.configs\runnerconfig.jsonc
 
-# Make edits to $HOME\.configs\runnerconfig.json
+# Make edits to $HOME\.configs\runnerconfig.jsonc
 
 # You must set the RUNNER_CONFIG environment variable for the current shell.
-> $env:RUNNER_CONFIG="${HOME}\.configs\runnerconfig.json"
+> $env:RUNNER_CONFIG="${HOME}\.configs\runnerconfig.jsonc"
 ```
 
 Bash:
 
 ```bash
-$ cp runnerconfig.json ~/.configs/runnerconfig.json
+$ cp runnerconfig.jsonc ~/.configs/runnerconfig.jsonc
 
-# Make edits to ~/.configs/runnerconfig.json
+# Make edits to ~/.configs/runnerconfig.jsonc
 
 # Optionally export RUNNER_CONFIG.
-$ export RUNNER_CONFIG=~/.configs/runnerconfig.json
+$ export RUNNER_CONFIG=~/.configs/runnerconfig.jsonc
 ```
 
 ### Run Benchmarks
@@ -533,5 +552,5 @@ Bash:
 # copy prepared by the build.
 $ dotnet run -c Release -f net9.0
 
-$ RUNNER_CONFIG=~/.configs/runnerconfig.json dotnet run -c Release -f net9.0
+$ RUNNER_CONFIG=~/.configs/runnerconfig.jsonc dotnet run -c Release -f net9.0
 ```

src/Microsoft.Data.SqlClient/tests/Directory.Packages.props

@@ -14,6 +14,7 @@
   <!-- SqlClient test dependencies. -->
   <ItemGroup>
     <PackageVersion Include="BenchmarkDotNet" Version="0.15.8" />
+    <PackageVersion Include="BenchmarkDotNet.Diagnostics.Windows" Version="0.15.8" />
     <PackageVersion Include="Microsoft.Extensions.Hosting" Version="9.0.13" />
     <PackageVersion Include="Microsoft.SqlServer.SqlManagementObjects" Version="181.15.0" />
     <PackageVersion Include="Microsoft.SqlServer.Types" Version="170.1000.7" />

src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/AsyncLargeDataReadRunner.cs

@@ -0,0 +1,119 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Data;
+using System.Threading.Tasks;
+using BenchmarkDotNet.Attributes;
+
+namespace Microsoft.Data.SqlClient.PerformanceTests
+{
+    /// <summary>
+    /// Benchmarks for sync vs async reading of large VARBINARY(MAX) values.
+    /// Reproduces issues #593 and #1562.
+    /// </summary>
+    public class AsyncLargeDataReadRunner : BaseRunner
+    {
+        private string _tableName;
+        private string _connectionString;
+
+        /// <summary>
+        /// Size of the data to read in bytes.
+        /// </summary>
+        [Params(1_048_576, 5_242_880, 10_485_760, 20_971_520)]
+        public int DataSizeBytes { get; set; }
+
+        /// <summary>
+        /// Size of the client-side read buffer used to drain the VARBINARY(MAX) column.
+        /// Kept small (8 KB) and large (1 MB) to observe whether buffer size relative to
+        /// the payload materially changes throughput.
+        /// </summary>
+        [Params(8_192, 1_048_576)]
+        public int ReadBufferBytes { get; set; }
+
+        [GlobalSetup]
+        public void Setup()
+        {
+            _connectionString = s_config.ConnectionString;
+            string machineHash = ((uint)Environment.MachineName.GetHashCode()).ToString("x8");
+            _tableName = $"[perf_AsyncLargeData_{machineHash}_{Guid.NewGuid():N}]";
+
+            using var conn = new SqlConnection(_connectionString);
+            conn.Open();
+
+            using var createCmd = new SqlCommand(
+                $"CREATE TABLE {_tableName} (Id INT IDENTITY PRIMARY KEY, Data VARBINARY(MAX))", conn);
+            createCmd.ExecuteNonQuery();
+
+            // Generate the payload entirely server-side via CRYPT_GEN_RANDOM so we don't
+            // allocate a multi-megabyte byte[] on the client and don't ship the payload
+            // over the wire just to seed the benchmark.
+            using var insertCmd = new SqlCommand(
+                $@"INSERT INTO {_tableName} (Data)
+                   SELECT SUBSTRING(
+                       CONVERT(
+                           varbinary(max),
+                           REPLICATE(
+                               CONVERT(varchar(max), CRYPT_GEN_RANDOM(8000), 2),
+                               (@dataSizeBytes + 7999) / 8000
+                           ),
+                           2
+                       ),
+                       1,
+                       @dataSizeBytes
+                   );", conn);
+            insertCmd.Parameters.Add("@dataSizeBytes", SqlDbType.Int).Value = DataSizeBytes;
+            insertCmd.ExecuteNonQuery();
+        }
+
+        [GlobalCleanup]
+        public void Cleanup()
+        {
+            using var conn = new SqlConnection(_connectionString);
+            conn.Open();
+            using var cmd = new SqlCommand($"DROP TABLE IF EXISTS {_tableName}", conn);
+            cmd.ExecuteNonQuery();
+            SqlConnection.ClearAllPools();
+        }
+
+        [Benchmark]
+        public void ReadLargeDataSync()
+        {
+            using var conn = new SqlConnection(_connectionString);
+            conn.Open();
+            using var cmd = new SqlCommand($"SELECT Data FROM {_tableName}", conn);
+            using var reader = cmd.ExecuteReader(CommandBehavior.SequentialAccess);
+            while (reader.Read())
+            {
+                byte[] buffer = new byte[ReadBufferBytes];
+                long offset = 0;
+                long bytesRead;
+                do
+                {
+                    bytesRead = reader.GetBytes(0, offset, buffer, 0, buffer.Length);
+                    offset += bytesRead;
+                } while (bytesRead > 0);
+            }
+        }
+
+        [Benchmark]
+        public async Task ReadLargeDataAsync()
+        {
+            using var conn = new SqlConnection(_connectionString);
+            await conn.OpenAsync();
+            using var cmd = new SqlCommand($"SELECT Data FROM {_tableName}", conn);
+            using var reader = await cmd.ExecuteReaderAsync(CommandBehavior.SequentialAccess);
+            while (await reader.ReadAsync())
+            {
+                using var stream = reader.GetStream(0);
+                byte[] buffer = new byte[ReadBufferBytes];
+                int bytesRead;
+                do
+                {
+                    bytesRead = await stream.ReadAsync(buffer, 0, buffer.Length);
+                } while (bytesRead > 0);
+            }
+        }
+    }
+}

src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/BeginTransactionRunner.cs

@@ -0,0 +1,86 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Threading.Tasks;
+using BenchmarkDotNet.Attributes;
+
+namespace Microsoft.Data.SqlClient.PerformanceTests
+{
+    /// <summary>
+    /// Benchmarks measuring BeginTransaction round-trip latency.
+    /// Reproduces issue #1554.
+    /// </summary>
+    public class BeginTransactionRunner : BaseRunner
+    {
+        private SqlConnection _connection;
+        private string _tableName;
+
+        [GlobalSetup]
+        public void Setup()
+        {
+            _connection = new SqlConnection(s_config.ConnectionString);
+            _connection.Open();
+
+            string machineHash = ((uint)Environment.MachineName.GetHashCode()).ToString("x8");
+            _tableName = $"[perf_TxnBench_{machineHash}_{Guid.NewGuid():N}]";
+            using var cmd = new SqlCommand(
+                $"CREATE TABLE {_tableName} (Id INT IDENTITY PRIMARY KEY, Value INT)", _connection);
+            cmd.ExecuteNonQuery();
+        }
+
+        [GlobalCleanup]
+        public void Cleanup()
+        {
+            using var cmd = new SqlCommand($"DROP TABLE IF EXISTS {_tableName}", _connection);
+            cmd.ExecuteNonQuery();
+            _connection.Close();
+            _connection.Dispose();
+            SqlConnection.ClearAllPools();
+        }
+
+        // Truncate the table between iterations so every iteration inserts into an empty
+        // table. Without this, the table grows across iterations (and across benchmarks,
+        // since they share the same connection/table), which would skew the measured
+        // BeginTransaction overhead as row count increases.
+        [IterationCleanup]
+        public void IterationCleanup()
+        {
+            using var cmd = new SqlCommand($"TRUNCATE TABLE {_tableName}", _connection);
+            cmd.ExecuteNonQuery();
+        }
+
+        [Benchmark]
+        public void WithTransaction()
+        {
+            using var txn = _connection.BeginTransaction();
+            using var cmd = new SqlCommand($"INSERT INTO {_tableName} (Value) VALUES (1)", _connection, txn);
+            cmd.ExecuteNonQuery();
+            txn.Commit();
+        }
+
+        [Benchmark]
+        public void WithoutTransaction()
+        {
+            using var cmd = new SqlCommand($"INSERT INTO {_tableName} (Value) VALUES (1)", _connection);
+            cmd.ExecuteNonQuery();
+        }
+
+        [Benchmark]
+        public async Task WithTransactionAsync()
+        {
+            using var txn = (SqlTransaction)await _connection.BeginTransactionAsync();
+            using var cmd = new SqlCommand($"INSERT INTO {_tableName} (Value) VALUES (1)", _connection, txn);
+            await cmd.ExecuteNonQueryAsync();
+            await txn.CommitAsync();
+        }
+
+        [Benchmark]
+        public async Task WithoutTransactionAsync()
+        {
+            using var cmd = new SqlCommand($"INSERT INTO {_tableName} (Value) VALUES (1)", _connection);
+            await cmd.ExecuteNonQueryAsync();
+        }
+    }
+}