Use complete native readiness and stable mount identities

Share Aspire health and bootstrap completion readiness across the workload client, native controls and resource observation. Compare full mount source/target/type identities independently of Docker list order while retaining actual-change guards.

Exercise MongoDB replica readiness and million-document OpenSearch controls; retain bounded rejection types. Keep the full workflow group and gates, and run the same native controls in an independent contracts group.

Verified: canonical full Debug solution build with -warnaserror (21 projects, zero warnings/errors), full format verification, governance and actionlint. Local million-document OpenSearch controls passed at one/three nodes, and the original MongoDB three-node readiness failure passed after correction. Linux sustained OpenSearch n3 HTTP 429 remains unresolved pending exact rejection evidence; no retry or server-setting change.

Author: KSemenenko

.github/workflows/benchmarks.yml

@@ -5,13 +5,13 @@ on:
   workflow_dispatch:
     inputs:
       mode:
-        description: Run harness contracts or the full database suite
+        description: Run harness/native controls or the full database suite
         type: choice
         options: [contracts, full]
         default: full
 permissions: {contents: read, actions: read}
 concurrency:
-  group: keyload-benchmarks-${{ github.ref }}
+  group: keyload-benchmarks-${{ github.ref }}${{ github.event_name == 'workflow_dispatch' && inputs.mode == 'contracts' && '-contracts' || '' }}
   cancel-in-progress: false
 env:
   NUGET_PACKAGES: ${{ github.workspace }}/artifacts/benchmark-runtime/packages
@@ -71,7 +71,6 @@ jobs:
           if-no-files-found: error
           retention-days: 90
       - name: Check Docker
-        if: (github.event_name == 'push' || inputs.mode == 'full')
         run: docker version
       - name: Resolve the latest released KeyLoad server image to one digest
         id: keyload-image
@@ -111,8 +110,7 @@ jobs:
           compression-level: 0
           retention-days: 90
       - name: Test external database bulk operations and joined cleanup
-        if: (github.event_name == 'push' || inputs.mode == 'full')
-        run: dotnet benchmarks/KeyLoad.ComparisonHost/bin/Release/net10.0/KeyLoad.ComparisonHost.dll --output Detailed --maximum-parallel-tests 1 --minimum-expected-tests 5 --treenode-filter '/*/*/NativeDatabaseTests/*' --results-directory "$GITHUB_WORKSPACE/benchmarks/KeyLoad.ComparisonHost/bin/Release/net10.0/TestResults/NativeAdapters" --report-trx --timeout 20m
+        run: dotnet benchmarks/KeyLoad.ComparisonHost/bin/Release/net10.0/KeyLoad.ComparisonHost.dll --output Detailed --maximum-parallel-tests 1 --minimum-expected-tests 6 --treenode-filter '/*/*/NativeDatabaseTests/*' --results-directory "$GITHUB_WORKSPACE/benchmarks/KeyLoad.ComparisonHost/bin/Release/net10.0/TestResults/NativeAdapters" --report-trx --timeout 20m
       - name: Test bounded terminal execution diagnostics
         run: dotnet benchmarks/KeyLoad.ComparisonHost/bin/Release/net10.0/KeyLoad.ComparisonHost.dll --output Detailed --minimum-expected-tests 1 --treenode-filter '/*/*/ComparisonHostCaptureTests/*' --results-directory "$GITHUB_WORKSPACE/benchmarks/KeyLoad.ComparisonHost/bin/Release/net10.0/TestResults/TerminalDiagnostics" --report-trx
       - name: Save original preparation diagnostics

benchmarks/KeyLoad.ComparisonHost/DatabaseAdapters/Cases/NativeDatabaseTests.cs

@@ -7,14 +7,18 @@ namespace KeyLoad.ComparisonHost.BenchmarkComparisons;
 internal sealed class NativeDatabaseTests
 {
     internal const string Redis = "Redis", OpenSearch = "OpenSearch", HelixDb = "HelixDB", MongoDb = "MongoDB";
-    private const int Empty = 0, Clients = 16, First = 0, Updated = 1, Deleted = 2, Single = 1;
-    private const int RedisRecords = 800_000, OpenSearchRecords = 80_000;
+    private const int Empty = 0, Clients = 16, First = 0, Updated = 1, Deleted = 2, Single = 1, ReplicaNodes = 3;
+    private const int RedisRecords = 800_000, OpenSearchRecords = 1_000_000;
     private const string BodyMismatch = "NativeDatabaseBulkReadbackMismatch";
 
     [Test, Arguments(OpenSearch, 1), Arguments(OpenSearch, 3), Arguments(Redis, 3), Arguments(HelixDb, 1), Arguments(MongoDb, 1)]
     public Task NativeBulkPreservesBodiesAndIndependentClients(string target, int nodes)
         => AdditionalDatabaseResources.RunAsync(target, VerifyAsync, TestContext.Current!.Execution.CancellationToken, nodes);
 
+    [Test]
+    public Task MongoReplicaSetReadinessPrecedesIndependentBulkClients()
+        => AdditionalDatabaseResources.RunAsync(MongoDb, VerifyAsync, TestContext.Current!.Execution.CancellationToken, ReplicaNodes);
+
     private static async Task VerifyAsync(IComparisonTarget target, CancellationToken token)
     {
         var documents = (IDocumentComparisonTarget)target;

benchmarks/KeyLoad.ComparisonHost/DatabaseAdapters/Resources/AdditionalDatabaseResources.cs

@@ -6,7 +6,6 @@
 using KeyLoad.ComparisonHost.BenchmarkOrchestration;
 using KeyLoad.Comparisons;
 using KeyLoad.Comparisons.Targets;
-using KeyLoad.LoadTests.AppHost.BenchmarkComparisons;
 using ManagedCode.Communication.CQRS;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
@@ -66,12 +65,8 @@ or BenchmarkAdditionalTargets.MySql or BenchmarkAdditionalTargets.ClickHouse or
     private static async Task RunNativeAsync(DistributedApplication application, string target,
         Func<IComparisonTarget, CancellationToken, Task> verify, NativeDatabaseResourceObservation observations, CancellationToken token)
     {
-        var settings = application.Services.GetRequiredService<LoadTestClientSettings>();
         await application.StartAsync(token);
-        foreach (var resource in settings.Resources)
-        {
-            await application.ResourceNotifications.WaitForResourceHealthyAsync(resource, WaitBehavior.StopOnResourceUnavailable, token);
-        }
+        var settings = await BenchmarkSessionClientRuntime.WaitForReadinessAsync(application, token);
         await observations.VerifyReadyAsync(token);
         var values = await settings.ResolveAsync(token);
         using var configuration = new ConfigurationManager();

benchmarks/KeyLoad.ComparisonHost/DatabaseAdapters/Resources/NativeDatabaseResourceObservation.cs

@@ -31,7 +31,7 @@ internal NativeDatabaseResourceObservation(DistributedApplication app, string ro
             .OfType<ContainerResource>().Where(resource => settings.Resources.Contains(resource.Name))];
         collector = app.Services.GetRequiredService<ScaleServerResourceEvidenceCollector>();
         observation = collector.StartAsync(resources,
-            ready => ScaleServerResourceReadiness.WaitAsync(app, resources, ready), token);
+            ready => BenchmarkSessionClientRuntime.WaitForReadinessAsync(app, ready), token);
     }
 
     internal async Task VerifyReadyAsync(CancellationToken token)
@@ -56,6 +56,7 @@ internal async Task VerifyReadyAsync(CancellationToken token)
             await Assert.That(actual.Length).IsGreaterThan(0);
             await Assert.That(actual.Length).IsLessThanOrEqualTo(budget.Settings.MaxMounts);
             await Assert.That(actual.Distinct(StringComparer.Ordinal).Count()).IsEqualTo(actual.Length);
+            await VerifyMountIdentityAsync(descriptor);
             ScaleServerMount[] mounts = [];
             if (OperatingSystem.IsLinux())
             {
@@ -66,6 +67,25 @@ internal async Task VerifyReadyAsync(CancellationToken token)
         }
     }
 
+    private static async Task VerifyMountIdentityAsync(string descriptor)
+    {
+        const string Changed = "-changed";
+        const int First = 0, Fields = 3;
+        var rows = descriptor.Split(MountSeparator, StringSplitOptions.RemoveEmptyEntries);
+        var identity = ScaleServerMountReader.NormalizeDescriptor(descriptor);
+        var reordered = string.Join(MountSeparator, rows.Reverse());
+        await Assert.That(ScaleServerMountReader.NormalizeDescriptor(reordered)).IsEqualTo(identity);
+        for (var field = First; field < Fields; field++)
+        {
+            var changed = rows.ToArray();
+            var values = changed[First].Split(MountFieldSeparator);
+            values[field] += Changed;
+            changed[First] = string.Join(MountFieldSeparator, values);
+            await Assert.That(ScaleServerMountReader.NormalizeDescriptor(string.Join(MountSeparator, changed)))
+                .IsNotEqualTo(identity);
+        }
+    }
+
     private async Task CaptureHostMountsAsync(CancellationToken token)
     {
         const string MountInfoFile = "host-mountinfo.txt";

benchmarks/KeyLoad.ComparisonHost/Orchestration/Execution/BenchmarkSessionClientRuntime.cs

@@ -15,8 +15,17 @@ internal static class BenchmarkSessionClientRuntime
 
     internal static async Task<int> RunAsync(DistributedApplication application, string evidence, CancellationToken token)
     {
-        var settings = application.Services.GetRequiredService<LoadTestClientSettings>();
         await application.StartAsync(token);
+        var settings = await WaitForReadinessAsync(application, token);
+        var environment = (await settings.ResolveAsync(token)).ToDictionary(StringComparer.Ordinal);
+        var policy = application.Services.GetRequiredService<IOptions<TestExecutionOptions>>();
+        return await IsolatedClientExecution.RunAsync(application, settings.Resources,
+            lifetime => BenchmarkSessionClientProcess.RunAsync(environment, evidence, policy, lifetime), token);
+    }
+
+    internal static async Task<LoadTestClientSettings> WaitForReadinessAsync(DistributedApplication application, CancellationToken token)
+    {
+        var settings = application.Services.GetRequiredService<LoadTestClientSettings>();
         foreach (var resource in settings.Resources)
         {
             await application.ResourceNotifications.WaitForResourceHealthyAsync(resource, WaitBehavior.StopOnResourceUnavailable, token);
@@ -30,9 +39,6 @@ internal static async Task<int> RunAsync(DistributedApplication application, str
                 throw new InvalidOperationException(TheSelectedDatabaseBootstrapProcessFailedMessage);
             }
         }
-        var environment = (await settings.ResolveAsync(token)).ToDictionary(StringComparer.Ordinal);
-        var policy = application.Services.GetRequiredService<IOptions<TestExecutionOptions>>();
-        return await IsolatedClientExecution.RunAsync(application, settings.Resources,
-            lifetime => BenchmarkSessionClientProcess.RunAsync(environment, evidence, policy, lifetime), token);
+        return settings;
     }
 }

benchmarks/KeyLoad.ComparisonHost/Orchestration/Fixtures/BenchmarkSessionAspireFixture.cs

@@ -69,7 +69,7 @@ await IsolatedReportAssertions.VerifyModelAsync(
                 item.Target == selected.Target && item.NodeCounts.Contains(selected.NodeCount)))
             {
                 work.Observation = work.Collector.StartAsync(containers,
-                    readinessToken => ScaleServerResourceReadiness.WaitAsync(application, containers, readinessToken), token);
+                    readinessToken => BenchmarkSessionClientRuntime.WaitForReadinessAsync(application, readinessToken), token);
             }
             else
             {

benchmarks/KeyLoad.ComparisonHost/Orchestration/Lifecycle/ScaleServerResourceReadiness.cs

@@ -2,8 +2,7 @@ namespace KeyLoad.Comparisons.Targets;
 
 public sealed partial class OpenSearchTarget
 {
-    // Sixteen clients retain concurrent coordinating, primary and replica work. Keep each request near 1 MiB
-    // so the selected small-heap topology admits the whole cohort without indexing-pressure rejections.
+    // Bound each bulk request independently of the total corpus and the number of active clients.
     private const int BulkRequestDocuments = 1_000;
     /// <inheritdoc/>
     public int IngestionBatchRecords => BulkRequestDocuments;

benchmarks/KeyLoad.Comparisons/Documents/Adapters/OpenSearchTarget.cs

@@ -52,7 +52,13 @@ private static async Task<JsonDocument> SendAsync(HttpClient client, HttpMethod
 
         if (mutation is null && !response.IsSuccessStatusCode)
         {
-            throw new ComparisonFailureException(HttpFailure + (int)response.StatusCode);
+            var failure = HttpFailure + (int)response.StatusCode;
+            if (policy is not null)
+            {
+                using var error = await ReadJsonAsync(response.Content, policy, cancellationToken);
+                failure += ReadFailureKind(error.RootElement);
+            }
+            throw new ComparisonFailureException(failure);
         }
         var json = await ReadJsonAsync(response.Content, policy, cancellationToken);
         try
@@ -70,6 +76,21 @@ private static async Task<JsonDocument> SendAsync(HttpClient client, HttpMethod
         }
     }
 
+    private static string ReadFailureKind(JsonElement response)
+    {
+        const string FailureKindSeparator = ".";
+        const char TypeWordSeparator = '_';
+        const int MaximumTypeLength = 128;
+        if (response.ValueKind == JsonValueKind.Object && response.TryGetProperty(OpenSearchNames.Error, out var error)
+            && error.ValueKind == JsonValueKind.Object && error.TryGetProperty(OpenSearchNames.ErrorType, out var kind)
+            && kind.ValueKind == JsonValueKind.String && kind.GetString() is { Length: > 0 and <= MaximumTypeLength } text
+            && text.All(character => char.IsAsciiLetterOrDigit(character) || character == TypeWordSeparator))
+        {
+            return FailureKindSeparator + text;
+        }
+        return string.Empty;
+    }
+
     private static async Task<JsonDocument> ReadJsonAsync(HttpContent content, ComparisonExecutionOptions? policy, CancellationToken cancellationToken)
     {
         if (policy is not null)

benchmarks/KeyLoad.Comparisons/Targets/OpenSearch/Transport/OpenSearchHttp.cs

@@ -12,6 +12,10 @@ internal static class ScaleServerMountReader
     private const int First = 0, Source = 0, Target = 1, MountType = 2, MountFields = 3;
     private const int StatFields = 4, BlockSize = 1, Blocks = 2, AvailableBlocks = 3;
 
+    internal static string NormalizeDescriptor(string text)
+        => string.Join(MountSeparator, text.Split(MountSeparator, StringSplitOptions.RemoveEmptyEntries)
+            .Order(StringComparer.Ordinal));
+
     internal static async Task<ScaleServerMount[]> ReadAsync(string text, string[] expectedTargets,
         ScaleServerResourceSampleBudget budget, CancellationToken token)
     {