From 8a6eeccb6d3d6e1c3f9a7c366732354e7f8c0fb1 Mon Sep 17 00:00:00 2001 From: Alex Williams-Ferreira Date: Mon, 9 Nov 2026 13:20:51 -0800 Subject: [PATCH 1/2] Fix ASP.NET server lifecycle and Bombardier port propagation Cancel and await each owned server lifetime before replacement so intentional shutdown cannot trigger stale retries. Preserve process diagnostics and standard workload failure handling for Kestrel and Orchard, and resolve Bombardier URLs from the configured server port. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../ProfileTests.cs | 36 ++ .../AspNetOrchardServerExecutorTests.cs | 22 +- .../AspNetBench/AspNetServerExecutorTests.cs | 20 +- .../AspNetBench/AspNetServerLifecycleTests.cs | 387 ++++++++++++++++++ .../ASPNET/AspNetOrchardServerExecutor.cs | 161 ++------ .../ASPNET/AspNetServerExecutor.cs | 155 ++----- .../ASPNET/AspNetServerProcess.cs | 196 +++++++++ .../profiles/PERF-WEB-ASPNET-BOMBARDIER.json | 3 +- .../aspnetbench/aspnetbench-profiles.md | 2 +- 9 files changed, 701 insertions(+), 281 deletions(-) create mode 100644 src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetServerLifecycleTests.cs create mode 100644 src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerProcess.cs diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/ProfileTests.cs b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/ProfileTests.cs index ceef931fbb..b6aaa55b4a 100644 --- a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/ProfileTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/ProfileTests.cs @@ -4,6 +4,7 @@ namespace VirtualClient.Actions { using System; + using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -24,6 +25,41 @@ public void SetupFixture() ComponentTypeCache.Instance.LoadComponentTypes(MockFixture.TestAssemblyDirectory); } + [Test] + public void AspNetBombardierProfilePreservesDefaultFailurePolicy() + { + this.mockFixture.Setup(PlatformID.Win32NT); + using (ProfileExecutor executor = TestProfileResources.CreateProfileExecutor( + "PERF-WEB-ASPNET-BOMBARDIER.json", + this.mockFixture.Dependencies)) + { + WorkloadAssert.ParameterReferencesInlined(executor.Profile); + Assert.That(executor.Profile.Actions[0].Parameters.ContainsKey("FailFast"), Is.False); + Assert.That(executor.Profile.Actions[1].Parameters["ServerPort"].ToString(), Is.EqualTo("9876")); + } + } + + [Test] + [TestCase("9876")] + [TestCase("9800")] + public async Task AspNetBombardierProfileResolvesTheConfiguredPortForBothActions(string port) + { + this.mockFixture.Setup(PlatformID.Win32NT); + using (ProfileExecutor executor = TestProfileResources.CreateProfileExecutor( + "PERF-WEB-ASPNET-BOMBARDIER.json", + this.mockFixture.Dependencies, + parameterOverrides: new Dictionary { ["ServerPort"] = port })) + { + var serverParameters = executor.Profile.Actions[0].Parameters; + var clientParameters = executor.Profile.Actions[1].Parameters; + Assert.That(clientParameters.ContainsKey("ServerPort"), Is.True); + await ProfileExpressionEvaluator.Instance.EvaluateAsync(this.mockFixture.Dependencies, clientParameters, CancellationToken.None); + Assert.That(serverParameters["ServerPort"].ToString(), Is.EqualTo(port)); + Assert.That(clientParameters["CommandArguments"].ToString(), Does.Contain($":{port}/json")); + Assert.That(clientParameters["CommandArguments"].ToString(), Does.Not.Contain("{ServerPort}")); + } + } + [Test] [TestCase("EXAMPLE-WORKLOAD.json")] public async Task ParametersAndMetadataArePassedToTheVirtualClientComponentOnExecution(string profile) diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetOrchardServerExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetOrchardServerExecutorTests.cs index c26b7606b3..1eb5087b2e 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetOrchardServerExecutorTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetOrchardServerExecutorTests.cs @@ -63,14 +63,12 @@ public async Task AspNetOrchardServerExecutorRunsTheExpectedWorkloadCommandInLin await executor.ExecuteAsync(CancellationToken.None); this.mockFixture.Tracking.AssertCommandsExecuted(true, - "pkill OrchardCore", - "fuser -n tcp -k 5014", $"{Regex.Escape(this.mockDotNetPackage.Path)}/dotnet publish -c Release --sc -f net9\\.0 {Regex.Escape(this.mockOrchardCorePackage.Path)}/src/OrchardCore\\.Cms\\.Web/OrchardCore\\.Cms\\.Web\\.csproj", - $"nohup {Regex.Escape(this.mockOrchardCorePackage.Path)}/src/OrchardCore\\.Cms\\.Web/bin/Release/net9\\.0/linux-x64/publish/OrchardCore\\.Cms\\.Web --urls http://\\*:5014" + $"{Regex.Escape(this.mockOrchardCorePackage.Path)}/src/OrchardCore\\.Cms\\.Web/bin/Release/net9\\.0/linux-x64/publish/OrchardCore\\.Cms\\.Web --urls http://\\*:5014" ); - this.mockFixture.Tracking.AssertCommandExecutedTimes("pkill", 1); - this.mockFixture.Tracking.AssertCommandExecutedTimes("fuser", 1); + this.mockFixture.Tracking.AssertCommandExecutedTimes("pkill", 0); + this.mockFixture.Tracking.AssertCommandExecutedTimes("fuser", 0); } } @@ -86,13 +84,12 @@ public async Task AspNetOrchardServerExecutorRunsTheExpectedWorkloadCommandInWin await executor.ExecuteAsync(CancellationToken.None); this.mockFixture.Tracking.AssertCommandsExecuted(true, - "taskkill /F /IM OrchardCore\\.Cms\\.Web\\.exe", $"{Regex.Escape(this.mockDotNetPackage.Path)}\\\\dotnet\\.exe publish -c Release --sc -f net9\\.0 {Regex.Escape(this.mockOrchardCorePackage.Path)}\\\\src\\\\OrchardCore\\.Cms\\.Web\\\\OrchardCore\\.Cms\\.Web\\.csproj", $"{Regex.Escape(this.mockOrchardCorePackage.Path)}\\\\src\\\\OrchardCore\\.Cms\\.Web\\\\bin\\\\Release\\\\net9\\.0\\\\win-x64\\\\publish\\\\OrchardCore\\.Cms\\.Web\\.exe --urls http://\\*:5014" ); // On Windows the Linux-only 'pkill'/'fuser'/'nohup' commands must NOT be used. - this.mockFixture.Tracking.AssertCommandExecutedTimes("taskkill", 1); + this.mockFixture.Tracking.AssertCommandExecutedTimes("taskkill", 0); this.mockFixture.Tracking.AssertCommandExecutedTimes("pkill", 0); this.mockFixture.Tracking.AssertCommandExecutedTimes("fuser", 0); this.mockFixture.Tracking.AssertCommandExecutedTimes("nohup", 0); @@ -159,10 +156,7 @@ public async Task AspNetOrchardServerExecutorExecutesWithCoreAffinityOnLinux() await executor.ExecuteAsync(CancellationToken.None); // Verify numactl was used with correct core affinity - // Actual format: /bin/bash -c "numactl -C 0-3 nohup ..." this.mockFixture.Tracking.AssertCommandsExecuted(true, - "pkill OrchardCore", - "fuser -n tcp -k 5014", $"{Regex.Escape(this.mockDotNetPackage.Path)}/dotnet publish -c Release --sc -f net9\\.0 {Regex.Escape(this.mockOrchardCorePackage.Path)}/src/OrchardCore\\.Cms\\.Web/OrchardCore\\.Cms\\.Web\\.csproj", "/bin/bash -c \\\"numactl -C 0-3 .*" ); @@ -240,6 +234,14 @@ private void SetupDefaultMockBehaviors(PlatformID platform) this.mockFixture.Directory.Setup(f => f.Exists(It.IsAny())) .Returns(true); this.mockFixture.FileSystem.SetupGet(fs => fs.File).Returns(this.mockFixture.File.Object); + this.mockFixture.Layout = new EnvironmentLayout(new[] + { + new ClientInstance(Environment.MachineName, "127.0.0.1", ClientRole.Client) + }); + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, directory) => + arguments?.Contains("--urls") == true + ? new AspNetServerLifecycleTests.ServerProcess { StartInfo = new System.Diagnostics.ProcessStartInfo(command, arguments) } + : new InMemoryProcess { StartInfo = new System.Diagnostics.ProcessStartInfo(command, arguments ?? string.Empty) }; this.mockFixture.Parameters = new Dictionary() { diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetServerExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetServerExecutorTests.cs index 8b1dd59770..b0d15c070e 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetServerExecutorTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetServerExecutorTests.cs @@ -72,14 +72,12 @@ public async Task AspNetServerExecutorRunsTheExpectedWorkloadCommandInLinux() await executor.ExecuteAsync(CancellationToken.None); this.mockFixture.Tracking.AssertCommandsExecuted(true, - "pkill dotnet", - "fuser -n tcp -k 9876", $"{Regex.Escape(this.mockDotNetPackage.Path)}/dotnet build -c Release -p:BenchmarksTargetFramework=net8\\.0", $"{Regex.Escape(this.mockDotNetPackage.Path)}/dotnet {Regex.Escape(this.mockAspNetBenchPackage.Path)}/src/Benchmarks/bin/Release/net8\\.0/Benchmarks\\.dll --nonInteractive true --scenarios json --urls http://\\*:9876 --server Kestrel --kestrelTransport Sockets --protocol http --header \\\"Accept: application/json,text/html;q=0\\.9,application/xhtml\\+xml;q=0\\.9,application/xml;q=0\\.8,\\*/\\*;q=0\\.7\\\" --header \\\"Connection: keep-alive\\\"" ); - this.mockFixture.Tracking.AssertCommandExecutedTimes("pkill", 1); - this.mockFixture.Tracking.AssertCommandExecutedTimes("fuser", 1); + this.mockFixture.Tracking.AssertCommandExecutedTimes("pkill", 0); + this.mockFixture.Tracking.AssertCommandExecutedTimes("fuser", 0); } } @@ -101,13 +99,11 @@ public async Task AspNetServerExecutorRunsTheExpectedWorkloadCommandInWindows() await executor.ExecuteAsync(CancellationToken.None); this.mockFixture.Tracking.AssertCommandsExecuted(true, - "taskkill /F /IM dotnet\\.exe", $"{Regex.Escape(this.mockDotNetPackage.Path)}\\\\dotnet\\.exe build -c Release -p:BenchmarksTargetFramework=net8\\.0", $"{Regex.Escape(this.mockDotNetPackage.Path)}\\\\dotnet\\.exe {Regex.Escape(this.mockAspNetBenchPackage.Path)}\\\\src\\\\Benchmarks\\\\bin\\\\Release\\\\net8\\.0\\\\Benchmarks\\.dll --nonInteractive true --scenarios json --urls http://\\*:9876 --server Kestrel --kestrelTransport Sockets --protocol http --header \\\"Accept: application/json,text/html;q=0\\.9,application/xhtml\\+xml;q=0\\.9,application/xml;q=0\\.8,\\*/\\*;q=0\\.7\\\" --header \\\"Connection: keep-alive\\\"" ); - // On Windows the Linux-only 'pkill'/'fuser' commands must NOT be used; 'taskkill' is used instead. - this.mockFixture.Tracking.AssertCommandExecutedTimes("taskkill", 1); + this.mockFixture.Tracking.AssertCommandExecutedTimes("taskkill", 0); this.mockFixture.Tracking.AssertCommandExecutedTimes("pkill", 0); this.mockFixture.Tracking.AssertCommandExecutedTimes("fuser", 0); } @@ -181,8 +177,6 @@ public async Task AspNetServerExecutorExecutesWithCoreAffinityOnLinux() // Verify numactl was used with correct core affinity // The actual command format is: /bin/bash -c "numactl -C 0-7 /path/to/dotnet /path/to/dll ..." this.mockFixture.Tracking.AssertCommandsExecuted(true, - "pkill dotnet", - "fuser -n tcp -k 9876", $"{Regex.Escape(this.mockDotNetPackage.Path)}/dotnet build -c Release -p:BenchmarksTargetFramework=net8\\.0", "/bin/bash -c \\\"numactl -C 0-7 .*" ); @@ -275,6 +269,14 @@ private void SetupDefaultMockBehaviors(PlatformID platform) this.mockFixture.Directory.Setup(f => f.Exists(It.IsAny())) .Returns(true); this.mockFixture.FileSystem.SetupGet(fs => fs.File).Returns(this.mockFixture.File.Object); + this.mockFixture.Layout = new EnvironmentLayout(new[] + { + new ClientInstance(Environment.MachineName, "127.0.0.1", ClientRole.Client) + }); + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, directory) => + arguments?.Contains("Benchmarks.dll") == true + ? new AspNetServerLifecycleTests.ServerProcess { StartInfo = new ProcessStartInfo(command, arguments) } + : new InMemoryProcess { StartInfo = new ProcessStartInfo(command, arguments ?? string.Empty) }; this.mockFixture.Parameters = new Dictionary() { diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetServerLifecycleTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetServerLifecycleTests.cs new file mode 100644 index 0000000000..f0dd3ea2cd --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetServerLifecycleTests.cs @@ -0,0 +1,387 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.Linq; + using System.Net; + using System.Net.Sockets; + using System.Threading; + using System.Threading.Tasks; + using Moq; + using Newtonsoft.Json.Linq; + using NUnit.Framework; + using Polly; + using VirtualClient.Common; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + + [TestFixture] + [Category("Unit")] + public class AspNetServerLifecycleTests + { + [TestCase(PlatformID.Win32NT)] + [TestCase(PlatformID.Unix)] + public async Task ReplacingServerStopsPreviousInstanceWithoutRestartingIt(PlatformID platform) + { + MockFixture fixture = CreateFixture(platform); + List servers = new List(); + TaskCompletionSource retryStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + fixture.ProcessManager.OnCreateProcess = (command, arguments, directory) => + { + if ((command == "taskkill" || command == "pkill") && servers.Count == 1) + { + servers[0].Stop(1); + retryStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)).GetAwaiter().GetResult(); + } + + if (arguments?.Contains("Benchmarks.dll") == true) + { + ServerProcess server = new ServerProcess(); + server.StartInfo = new ProcessStartInfo(command, arguments); + server.StandardOutput.Append("Now listening on: http://localhost:9876"); + server.StandardError.Append("Server diagnostic output"); + server.OnStart = () => + { + if (servers.Count == 2) + { + retryStarted.TrySetResult(true); + } + + return true; + }; + servers.Add(server); + return server; + } + + return new InMemoryProcess { StartInfo = new ProcessStartInfo(command, arguments ?? string.Empty) }; + }; + + using (CancellationTokenSource cancellation = new CancellationTokenSource(TimeSpan.FromSeconds(15))) + using (TestServer executor = new TestServer(fixture, retry: true)) + { + try + { + await executor.ExecuteAsync(cancellation.Token); + await executor.ExecuteAsync(cancellation.Token); + + Assert.That(servers.Count, Is.EqualTo(2), "Replacement must create one new server, not restart the previous instance."); + Assert.That(servers[0].HasExited, Is.True); + Assert.That(servers[1].HasExited, Is.False); + Assert.That(fixture.Logger.MessagesLogged($"{executor.TypeName}.KillServerInstances").Count(), Is.EqualTo(2)); + var processLogs = fixture.Logger.MessagesLogged($"{executor.TypeName}.AspNetBenchmarks.ProcessDetails").ToArray(); + Assert.That(processLogs.Length, Is.EqualTo(1), "Replacing the server must capture its process output."); + EventContext processContext = (EventContext)processLogs[0].Item3; + JObject processDetails = JObject.FromObject(processContext.Properties["process"]); + Assert.That(processDetails.Value("standardOutput"), Does.Contain("Now listening on:")); + Assert.That(processDetails.Value("standardError"), Is.EqualTo("Server diagnostic output")); + Assert.That(fixture.ProcessManager.Commands.Any(command => + command.StartsWith("taskkill") || command.StartsWith("pkill") || command.StartsWith("fuser")), Is.False); + } + finally + { + cancellation.Cancel(); + foreach (ServerProcess server in servers) + { + server.Stop(0); + } + } + + Assert.That(fixture.Logger.MessagesLogged("TestServer.AspNetBenchmarks.ProcessDetails").Count(), Is.EqualTo(2), + "Disposal must capture the final server's process output exactly once."); + } + } + + [Test] + public async Task ReplacingServerCancelsAPendingRetryDelay() + { + MockFixture fixture = CreateFixture(PlatformID.Win32NT); + List servers = new List(); + TaskCompletionSource retryPending = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + fixture.ProcessManager.OnCreateProcess = (command, arguments, directory) => + { + if (arguments?.Contains("Benchmarks.dll") == true) + { + ServerProcess server = new ServerProcess { StartInfo = new ProcessStartInfo(command, arguments) }; + servers.Add(server); + return server; + } + + return new InMemoryProcess { StartInfo = new ProcessStartInfo(command, arguments ?? string.Empty) }; + }; + + using (CancellationTokenSource cancellation = new CancellationTokenSource(TimeSpan.FromSeconds(10))) + using (TestServer executor = new TestServer(fixture)) + { + executor.SetRetryPolicy(Policy.Handle(exception => !(exception is OperationCanceledException)) + .WaitAndRetryAsync(1, attempt => TimeSpan.FromSeconds(10), (exception, delay) => retryPending.TrySetResult(true))); + await executor.ExecuteAsync(cancellation.Token); + servers[0].Stop(1); + await retryPending.Task.WaitAsync(cancellation.Token); + Assert.That(fixture.Logger.MessagesLogged($"{executor.TypeName}.StartServerInstanceError").Count(), Is.EqualTo(1), + "A failed server attempt must be logged even when a retry is pending."); + await executor.ExecuteAsync(cancellation.Token).WaitAsync(TimeSpan.FromSeconds(2)); + + Assert.That(servers.Count, Is.EqualTo(2)); + Assert.That(servers[0].HasExited, Is.True); + cancellation.Cancel(); + } + } + + [Test] + public void AnOccupiedPortDoesNotMakeAFailedServerAppearReady() + { + MockFixture fixture = CreateFixture(PlatformID.Win32NT); + using (TcpListener listener = new TcpListener(IPAddress.Loopback, 0)) + { + listener.Start(); + fixture.Parameters["ServerPort"] = ((IPEndPoint)listener.LocalEndpoint).Port.ToString(); + fixture.ProcessManager.OnCreateProcess = (command, arguments, directory) => + new InMemoryProcess + { + StartInfo = new ProcessStartInfo(command, arguments ?? string.Empty), + ExitCode = arguments?.Contains("Benchmarks.dll") == true ? 1 : 0 + }; + + using (TestServer executor = new TestServer(fixture, realReadiness: true)) + { + Assert.ThrowsAsync(() => executor.ExecuteAsync(CancellationToken.None)); + } + } + } + + [Test] + public void AServerFailureWhileSavingReadinessIsNotReportedAsSuccess() + { + MockFixture fixture = CreateFixture(PlatformID.Win32NT); + ServerProcess server = null; + fixture.ProcessManager.OnCreateProcess = (command, arguments, directory) => + { + if (arguments?.Contains("Benchmarks.dll") == true) + { + server = new ServerProcess { StartInfo = new ProcessStartInfo(command, arguments) }; + return server; + } + + return new InMemoryProcess { StartInfo = new ProcessStartInfo(command, arguments ?? string.Empty) }; + }; + + using (TestServer executor = new TestServer(fixture)) + { + fixture.ApiClient.OnUpdateState(nameof(State)).Returns(async () => + { + server.Stop(1); + await executor.GetServerTask().ContinueWith( + task => Assert.That(task.Exception.InnerException, Is.InstanceOf()), + TaskScheduler.Default); + return fixture.CreateHttpResponse(HttpStatusCode.OK); + }); + + Assert.ThrowsAsync(() => executor.ExecuteAsync(CancellationToken.None)); + } + } + + [Test] + public void ANativeProcessStartFailureUsesTheStandardWorkloadErrorClassification() + { + MockFixture fixture = CreateFixture(PlatformID.Unix); + fixture.ProcessManager.OnCreateProcess = (command, arguments, directory) => + new InMemoryProcess + { + StartInfo = new ProcessStartInfo(command, arguments ?? string.Empty), + OnStart = arguments?.Contains("Benchmarks.dll") == true + ? () => throw new System.ComponentModel.Win32Exception(2, "No such file or directory") + : () => true + }; + + using (TestServer executor = new TestServer(fixture)) + { + WorkloadException failure = Assert.ThrowsAsync(() => executor.ExecuteAsync(CancellationToken.None)); + Assert.That(failure.Reason, Is.EqualTo(ErrorReason.WorkloadFailed)); + Assert.That(failure.InnerException, Is.InstanceOf()); + Assert.That(failure.Message, Does.Contain("Benchmarks.dll")); + var outcomes = fixture.Logger.MessagesLogged($"{executor.TypeName}.SucceededOrFailed").ToArray(); + Assert.That(outcomes.Length, Is.EqualTo(1)); + Assert.That(((EventContext)outcomes[0].Item3).Properties["metricName"], Is.EqualTo("Failed")); + } + } + + [Test] + public void AnUnexpectedCancellationIsReportedAsAWorkloadFailure() + { + MockFixture fixture = CreateFixture(PlatformID.Win32NT); + fixture.ProcessManager.OnCreateProcess = (command, arguments, directory) => + arguments?.Contains("Benchmarks.dll") == true + ? new ServerProcess { StartInfo = new ProcessStartInfo(command, arguments), ThrowOnWait = true } + : new InMemoryProcess { StartInfo = new ProcessStartInfo(command, arguments ?? string.Empty) }; + + using (TestServer executor = new TestServer(fixture)) + { + Assert.ThrowsAsync(() => executor.ExecuteAsync(CancellationToken.None)); + } + } + + [Test] + public void MultiRoleServerPropagatesBackgroundProcessFailure() + { + MockFixture fixture = CreateFixture(PlatformID.Win32NT); + fixture.Layout = new EnvironmentLayout(new[] + { + new ClientInstance(Environment.MachineName, "127.0.0.1", ClientRole.Server), + new ClientInstance("client", "127.0.0.2", ClientRole.Client) + }); + fixture.ProcessManager.OnCreateProcess = (command, arguments, directory) => + new InMemoryProcess + { + StartInfo = new ProcessStartInfo(command, arguments ?? string.Empty), + ExitCode = arguments?.Contains("Benchmarks.dll") == true ? 1 : 0 + }; + + using (TestServer executor = new TestServer(fixture)) + { + Assert.ThrowsAsync(() => executor.ExecuteAsync(CancellationToken.None)); + } + } + + [Test] + public async Task ABackgroundFailureIsNotDiscardedByTheNextIteration() + { + MockFixture fixture = CreateFixture(PlatformID.Win32NT); + ServerProcess firstServer = null; + TaskCompletionSource exited = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + fixture.ProcessManager.OnCreateProcess = (command, arguments, directory) => + { + if (arguments?.Contains("Benchmarks.dll") == true) + { + ServerProcess server = new ServerProcess { StartInfo = new ProcessStartInfo(command, arguments) }; + if (firstServer == null) + { + firstServer = server; + server.OnDispose = () => exited.TrySetResult(true); + } + + return server; + } + + return new InMemoryProcess { StartInfo = new ProcessStartInfo(command, arguments ?? string.Empty) }; + }; + + using (CancellationTokenSource cancellation = new CancellationTokenSource(TimeSpan.FromSeconds(10))) + using (TestServer executor = new TestServer(fixture)) + { + await executor.ExecuteAsync(cancellation.Token); + firstServer.Stop(1); + await exited.Task.WaitAsync(cancellation.Token); + await executor.GetServerTask().ContinueWith( + task => Assert.That(task.Exception.InnerException, Is.InstanceOf()), + TaskScheduler.Default); + Assert.That(fixture.Logger.MessagesLogged($"{executor.TypeName}.SucceededOrFailed").Count(), Is.EqualTo(1), + "A background error must not emit a second outcome for a completed action."); + Assert.ThrowsAsync(() => executor.ExecuteAsync(cancellation.Token)); + + EventContext[] outcomes = fixture.Logger.MessagesLogged($"{executor.TypeName}.SucceededOrFailed") + .Select(entry => (EventContext)entry.Item3).ToArray(); + Assert.That(outcomes.Length, Is.EqualTo(2), "Each action execution must emit exactly one outcome."); + Assert.That(outcomes.Select(outcome => outcome.Properties["metricName"]), Is.EqualTo(new[] { "Succeeded", "Failed" })); + EventContext failure = outcomes[1]; + Assert.That(failure.Properties["metricValue"], Is.EqualTo(1)); + Assert.That(failure.Properties["toolName"], Is.EqualTo(executor.TypeName)); + Assert.That(failure.Properties["scenarioName"], Is.EqualTo(executor.Scenario)); + Assert.That(failure.Properties["scenarioStartTime"], Is.EqualTo(executor.StartTime)); + Assert.That(failure.Properties["scenarioEndTime"], Is.InRange(executor.StartTime, executor.EndTime)); + Assert.That(failure.Properties["metricUnit"], Is.EqualTo(string.Empty)); + Assert.That(failure.Properties["metricRelativity"], Is.EqualTo(MetricRelativity.LowerIsBetter)); + Assert.That(failure.Properties["metricVerbosity"], Is.EqualTo(1)); + cancellation.Cancel(); + } + } + + private static MockFixture CreateFixture(PlatformID platform) + { + MockFixture fixture = new MockFixture(); + fixture.Setup(platform); + fixture.Layout = new EnvironmentLayout(new[] + { + new ClientInstance(Environment.MachineName, "127.0.0.1", ClientRole.Client) + }); + fixture.Parameters["PackageName"] = "aspnetbenchmarks"; + fixture.Parameters["TargetFramework"] = "net8.0"; + fixture.Parameters["ServerPort"] = "9876"; + fixture.PackageManager.OnGetPackage("aspnetbenchmarks") + .ReturnsAsync(new DependencyPath("aspnetbenchmarks", fixture.GetPackagePath("aspnetbenchmarks"))); + fixture.PackageManager.OnGetPackage("dotnetsdk") + .ReturnsAsync(new DependencyPath("dotnetsdk", fixture.GetPackagePath("dotnetsdk"))); + fixture.ApiClient.OnUpdateState(nameof(State)) + .ReturnsAsync(fixture.CreateHttpResponse(HttpStatusCode.OK)); + return fixture; + } + + internal class ServerProcess : InMemoryProcess, IProcessProxy + { + private readonly TaskCompletionSource exit = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + public ServerProcess() + { + this.OnHasExited = () => this.exit.Task.IsCompleted; + this.OnKill = () => this.Stop(1); + } + + public void Stop(int exitCode) + { + this.ExitCode = exitCode; + this.exit.TrySetResult(true); + } + + public bool ThrowOnWait { get; set; } + + Task IProcessProxy.WaitForExitAsync(CancellationToken cancellationToken, TimeSpan? timeout) + { + if (this.ThrowOnWait) + { + this.ThrowOnWait = false; + return Task.FromException(new OperationCanceledException()); + } + + return this.exit.Task.WaitAsync(cancellationToken); + } + } + + private class TestServer : AspNetServerExecutor + { + private readonly bool realReadiness; + + public TestServer(MockFixture fixture, bool retry = false, bool realReadiness = false) + : base(fixture.Dependencies, fixture.Parameters) + { + this.realReadiness = realReadiness; + this.ServerRetryPolicy = retry + ? Policy.Handle(exception => !(exception is OperationCanceledException)).RetryAsync(1) + : Policy.NoOpAsync(); + } + + protected override Task WaitForPortReadyAsync(EventContext context, CancellationToken cancellationToken) + { + return this.realReadiness + ? base.WaitForPortReadyAsync(context, cancellationToken) + : Task.CompletedTask; + } + + public void SetRetryPolicy(IAsyncPolicy policy) + { + this.ServerRetryPolicy = policy; + } + + public Task GetServerTask() + { + object process = typeof(AspNetServerExecutor).GetField( + "serverProcess", + System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic).GetValue(this); + return (Task)process.GetType().GetProperty("Completion").GetValue(process); + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetOrchardServerExecutor.cs b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetOrchardServerExecutor.cs index b43323a80c..d57c7c10ad 100644 --- a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetOrchardServerExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetOrchardServerExecutor.cs @@ -29,7 +29,7 @@ namespace VirtualClient.Actions [SupportedPlatforms("linux-arm64,linux-x64,win-arm64,win-x64")] public class AspNetOrchardServerExecutor : VirtualClientMultiRoleComponent { - private Task serverProcess; + private readonly AspNetServerProcess serverProcess; private bool disposed; private IFileSystem fileSystem; private IPackageManager packageManager; @@ -52,6 +52,7 @@ public AspNetOrchardServerExecutor(IServiceCollection dependencies, IDictionary< this.packageManager = this.systemManagement.PackageManager; this.stateManager = this.systemManagement.StateManager; this.fileSystem = this.systemManagement.FileSystem; + this.serverProcess = new AspNetServerProcess(this, this.systemManagement, "Orchard"); this.ServerRetryPolicy = Policy.Handle(exc => !(exc is OperationCanceledException)) .WaitAndRetryAsync(3, (retries) => TimeSpan.FromSeconds(retries)); } @@ -146,14 +147,15 @@ protected override void Dispose(bool disposing) { try { - this.KillServerInstancesAsync(EventContext.None, CancellationToken.None) - .GetAwaiter().GetResult(); + this.serverProcess.StopAsync(EventContext.None).GetAwaiter().GetResult(); } - catch + catch (Exception exception) { - // Best-effort cleanup during dispose. + this.Logger.LogMessage($"{this.TypeName}.CleanupError", LogLevel.Error, EventContext.None.AddError(exception)); } + base.Dispose(disposing); + this.disposed = true; } } @@ -262,38 +264,28 @@ protected override Task ExecuteAsync(EventContext telemetryContext, Cancellation await this.ServerApi.PollForHeartbeatAsync(TimeSpan.FromMinutes(5), cancellationToken); await this.DeleteStateAsync(telemetryContext, cancellationToken); - await this.KillServerInstancesAsync(telemetryContext, cancellationToken); + await this.serverProcess.StopAsync(telemetryContext); await this.BuildAspNetOrchardAsync(telemetryContext, cancellationToken); this.StartServerInstances(telemetryContext, cancellationToken); await this.WaitForPortReadyAsync(telemetryContext, cancellationToken); + await this.serverProcess.EnsureRunningAsync(); await this.SaveStateAsync(telemetryContext, cancellationToken); this.SetServerOnline(true); + await this.serverProcess.EnsureRunningAsync(); - if (!this.IsMultiRoleLayout()) - { - // In single-VM mode, clear cleanup tasks to prevent the base class - // CleanupAsync from killing the server process. The server must stay - // alive for the subsequent client action. It will be killed by - // KillServerInstancesAsync on the next iteration or by Dispose. - this.CleanupTasks.Clear(); - } - else + if (this.IsMultiRoleLayout()) { using (BackgroundOperations profiling = BackgroundOperations.BeginProfiling(this, cancellationToken)) { - await Task.WhenAny(this.serverProcess); - if (cancellationToken.IsCancellationRequested) - { - await Task.WhenAll(this.serverProcess); - } + await this.serverProcess.Completion; } } } catch { this.SetServerOnline(false); - await this.KillServerInstancesAsync(telemetryContext, cancellationToken); + await this.serverProcess.StopAsync(telemetryContext); throw; } }); @@ -314,11 +306,13 @@ protected virtual async Task WaitForPortReadyAsync(EventContext telemetryContext while (DateTime.UtcNow < deadline && !cancellationToken.IsCancellationRequested) { + await this.serverProcess.EnsureRunningAsync(); try { using (TcpClient client = new TcpClient()) { - await client.ConnectAsync(IPAddress.Loopback, port).ConfigureAwait(false); + await client.ConnectAsync(IPAddress.Loopback, port, cancellationToken).ConfigureAwait(false); + await this.serverProcess.EnsureRunningAsync(); this.Logger.LogTraceMessage($"{this.TypeName}: Server is accepting connections on port {port}."); return; } @@ -351,27 +345,6 @@ private Task DeleteStateAsync(EventContext telemetryContext, CancellationToken c }); } - private async Task KillServerInstancesAsync(EventContext telemetryContext, CancellationToken cancellationToken) - { - this.Logger.LogTraceMessage($"{this.TypeName}.KillServerInstances"); - - // 'pkill'/'fuser' exist only on Unix; on Windows they fail to start ("The system cannot - // find the file specified"), which crashes the workload. On Windows the equivalent - // 'taskkill' is used to terminate the self-contained OrchardCore server (which also - // releases the TCP port it was holding). - if (this.Platform == PlatformID.Unix) - { - await this.ExecuteCommandAsync("pkill", "OrchardCore", this.aspnetOrchardDirectory, telemetryContext, cancellationToken); - await this.ExecuteCommandAsync("fuser", $"-n tcp -k {this.ServerPort}", this.aspnetOrchardDirectory, telemetryContext, cancellationToken); - } - else - { - await this.ExecuteCommandAsync("taskkill", "/F /IM OrchardCore.Cms.Web.exe", this.aspnetOrchardDirectory, telemetryContext, cancellationToken); - } - - await this.WaitAsync(TimeSpan.FromSeconds(3), cancellationToken); - } - private void StartServerInstances(EventContext telemetryContext, CancellationToken cancellationToken) { EventContext relatedContext = telemetryContext.Clone(); @@ -382,17 +355,12 @@ private void StartServerInstances(EventContext telemetryContext, CancellationTok { string workingDirectory = this.aspnetOrchardDirectory; - // 'nohup' (run a process immune to hangups, detached) exists only on Unix. On - // Windows it fails to start ("The system cannot find the file specified"), so the - // self-contained OrchardCore server executable is launched directly instead. string serverExecutable = this.Combine( this.aspnetOrchardPublishPath, this.Platform == PlatformID.Unix ? "OrchardCore.Cms.Web" : "OrchardCore.Cms.Web.exe"); - string command = this.Platform == PlatformID.Unix ? "nohup" : serverExecutable; - string commandArguments = this.Platform == PlatformID.Unix - ? $"{serverExecutable} --urls http://*:{this.ServerPort}" - : $"--urls http://*:{this.ServerPort}"; + string command = serverExecutable; + string commandArguments = $"--urls http://*:{this.ServerPort}"; relatedContext.AddContext("command", command); relatedContext.AddContext("commandArguments", commandArguments); @@ -400,93 +368,22 @@ private void StartServerInstances(EventContext telemetryContext, CancellationTok relatedContext.AddContext("bindToCores", this.BindToCores); relatedContext.AddContext("coreAffinity", this.CoreAffinity); - this.serverProcess = this.StartServerInstanceAsync(command, commandArguments, workingDirectory, relatedContext, cancellationToken); - } - catch (OperationCanceledException) - { - // Expected whenever certain operations (e.g. Task.Delay) are cancelled. - } - }); - } - - private Task StartServerInstanceAsync(string command, string commandArguments, string workingDirectory, EventContext telemetryContext, CancellationToken cancellationToken) - { - return (this.ServerRetryPolicy ?? Policy.NoOpAsync()).ExecuteAsync(async () => - { - try - { - IProcessProxy process = null; - ProcessAffinityConfiguration affinityConfig = null; - - if (this.BindToCores && !string.IsNullOrWhiteSpace(this.CoreAffinity)) - { - affinityConfig = ProcessAffinityConfiguration.Create( - this.Platform, - this.CoreAffinity); - - telemetryContext.AddContext("affinityMask", affinityConfig.ToString()); - - if (this.Platform == PlatformID.Win32NT) - { - process = this.systemManagement.ProcessManager.CreateProcess( - command, - commandArguments, - workingDirectory); - } - else - { - string fullCommandLine = $"{command} {commandArguments}"; - - LinuxProcessAffinityConfiguration linuxConfig = (LinuxProcessAffinityConfiguration)affinityConfig; - string wrappedCommand = linuxConfig.GetCommandWithAffinity(null, fullCommandLine); - - process = this.systemManagement.ProcessManager.CreateProcess( - "/bin/bash", - $"-c {wrappedCommand}", - workingDirectory); - } - } - else - { - process = await this.ExecuteCommandAsync(command, commandArguments, workingDirectory, telemetryContext, cancellationToken); - } - - using (process) - { - if (affinityConfig != null) - { - if (this.Platform == PlatformID.Win32NT) - { - process.Start(); - process.ApplyAffinity((WindowsProcessAffinityConfiguration)affinityConfig); - await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false); - } - else - { - await process.StartAndWaitAsync(cancellationToken).ConfigureAwait(false); - } - } - - if (!cancellationToken.IsCancellationRequested) - { - await this.LogProcessDetailsAsync(process, telemetryContext, "Orchard"); - process.ThrowIfWorkloadFailed(successCodes: new int[] { 0 }); - } - } + ProcessAffinityConfiguration affinity = this.BindToCores + ? ProcessAffinityConfiguration.Create(this.Platform, this.CoreAffinity) + : null; + this.serverProcess.Start( + command, + commandArguments, + workingDirectory, + affinity, + this.ServerRetryPolicy, + relatedContext, + cancellationToken); } catch (OperationCanceledException) { // Expected whenever certain operations (e.g. Task.Delay) are cancelled. } - catch (Exception exc) - { - this.Logger.LogMessage( - $"{this.TypeName}.StartServerInstanceError", - LogLevel.Error, - telemetryContext.Clone().AddError(exc)); - - throw; - } }); } diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerExecutor.cs b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerExecutor.cs index 6334e7b8d4..dfc79281f5 100644 --- a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerExecutor.cs @@ -29,7 +29,7 @@ namespace VirtualClient.Actions [SupportedPlatforms("linux-arm64,linux-x64,win-arm64,win-x64")] public class AspNetServerExecutor : VirtualClientMultiRoleComponent { - private Task serverProcess; + private readonly AspNetServerProcess serverProcess; private bool disposed; private IFileSystem fileSystem; private IPackageManager packageManager; @@ -52,6 +52,7 @@ public AspNetServerExecutor(IServiceCollection dependencies, IDictionary(exc => !(exc is OperationCanceledException)) .WaitAndRetryAsync(3, (retries) => TimeSpan.FromSeconds(retries)); } @@ -157,16 +158,17 @@ protected override void Dispose(bool disposing) { try { - this.KillServerInstancesAsync(EventContext.None, CancellationToken.None) - .GetAwaiter().GetResult(); + this.serverProcess.StopAsync(EventContext.None).GetAwaiter().GetResult(); } - catch + catch (Exception exception) { - // Best-effort cleanup during dispose. + this.Logger.LogMessage($"{this.TypeName}.CleanupError", LogLevel.Error, EventContext.None.AddError(exception)); } this.disposed = true; } + + base.Dispose(disposing); } } @@ -252,39 +254,28 @@ protected override Task ExecuteAsync(EventContext telemetryContext, Cancellation await this.ServerApi.PollForHeartbeatAsync(TimeSpan.FromMinutes(5), cancellationToken); await this.DeleteStateAsync(telemetryContext, cancellationToken); - await this.KillServerInstancesAsync(telemetryContext, cancellationToken); + await this.serverProcess.StopAsync(telemetryContext); await this.BuildAspNetBenchAsync(telemetryContext, cancellationToken); this.StartServerInstances(telemetryContext, cancellationToken); await this.WaitForPortReadyAsync(telemetryContext, cancellationToken); + await this.serverProcess.EnsureRunningAsync(); await this.SaveStateAsync(telemetryContext, cancellationToken); this.SetServerOnline(true); + await this.serverProcess.EnsureRunningAsync(); - if (!this.IsMultiRoleLayout()) - { - // In single-VM mode, clear cleanup tasks to prevent the base class - // CleanupAsync from killing the server process. The server must stay - // alive for the subsequent client action. It will be killed by - // KillServerInstancesAsync on the next iteration or by Dispose. - this.CleanupTasks.Clear(); - } - else + if (this.IsMultiRoleLayout()) { using (BackgroundOperations profiling = BackgroundOperations.BeginProfiling(this, cancellationToken)) { - await Task.WhenAny(this.serverProcess); - - if (cancellationToken.IsCancellationRequested) - { - await Task.WhenAll(this.serverProcess); - } + await this.serverProcess.Completion; } } } catch { this.SetServerOnline(false); - await this.KillServerInstancesAsync(telemetryContext, cancellationToken); + await this.serverProcess.StopAsync(telemetryContext); throw; } }); @@ -325,11 +316,13 @@ protected virtual async Task WaitForPortReadyAsync(EventContext telemetryContext while (DateTime.UtcNow < deadline && !cancellationToken.IsCancellationRequested) { + await this.serverProcess.EnsureRunningAsync(); try { using (TcpClient client = new TcpClient()) { - await client.ConnectAsync(IPAddress.Loopback, port).ConfigureAwait(false); + await client.ConnectAsync(IPAddress.Loopback, port, cancellationToken).ConfigureAwait(false); + await this.serverProcess.EnsureRunningAsync(); this.Logger.LogTraceMessage($"{this.TypeName}: Server is accepting connections on port {port}."); return; } @@ -362,29 +355,6 @@ private Task DeleteStateAsync(EventContext telemetryContext, CancellationToken c }); } - private async Task KillServerInstancesAsync(EventContext telemetryContext, CancellationToken cancellationToken) - { - this.Logger.LogTraceMessage($"{this.TypeName}.KillServerInstances"); - - // The ASP.NET (Kestrel) benchmark server is hosted by the .NET runtime (dotnet). Both the - // server process(es) and the TCP port they hold must be released using platform-native - // commands. 'pkill'/'fuser' exist only on Unix; on Windows they fail to start - // ("The system cannot find the file specified"), which crashes the workload. On Windows the - // equivalent 'taskkill' is used instead (killing the process also releases its port). - if (this.Platform == PlatformID.Unix) - { - await this.ExecuteCommandAsync("pkill", "dotnet", this.aspnetBenchDirectory, telemetryContext, cancellationToken); - - await this.ExecuteCommandAsync("fuser", $"-n tcp -k {this.ServerPort}", this.aspnetBenchDirectory, telemetryContext, cancellationToken); - } - else - { - await this.ExecuteCommandAsync("taskkill", "/F /IM dotnet.exe", this.aspnetBenchDirectory, telemetryContext, cancellationToken); - } - - await this.WaitAsync(TimeSpan.FromSeconds(3), cancellationToken); - } - private void StartServerInstances(EventContext telemetryContext, CancellationToken cancellationToken) { EventContext relatedContext = telemetryContext.Clone(); @@ -404,7 +374,17 @@ private void StartServerInstances(EventContext telemetryContext, CancellationTok relatedContext.AddContext("bindToCores", this.BindToCores); relatedContext.AddContext("coreAffinity", this.CoreAffinity); - this.serverProcess = this.StartServerInstanceAsync(this.dotnetExePath, commandArguments, workingDirectory, relatedContext, cancellationToken); + ProcessAffinityConfiguration affinity = this.BindToCores + ? ProcessAffinityConfiguration.Create(this.Platform, this.CoreAffinity) + : null; + this.serverProcess.Start( + this.dotnetExePath, + commandArguments, + workingDirectory, + affinity, + this.ServerRetryPolicy, + relatedContext, + cancellationToken); } catch (OperationCanceledException) { @@ -413,87 +393,6 @@ private void StartServerInstances(EventContext telemetryContext, CancellationTok }); } - private Task StartServerInstanceAsync(string command, string commandArguments, string workingDirectory, EventContext telemetryContext, CancellationToken cancellationToken) - { - return (this.ServerRetryPolicy ?? Policy.NoOpAsync()).ExecuteAsync(async () => - { - try - { - IProcessProxy process = null; - ProcessAffinityConfiguration affinityConfig = null; - - if (this.BindToCores && !string.IsNullOrWhiteSpace(this.CoreAffinity)) - { - affinityConfig = ProcessAffinityConfiguration.Create( - this.Platform, - this.CoreAffinity); - - telemetryContext.AddContext("affinityMask", affinityConfig.ToString()); - - if (this.Platform == PlatformID.Win32NT) - { - process = this.systemManagement.ProcessManager.CreateProcess( - command, - commandArguments, - workingDirectory); - } - else - { - string fullCommandLine = $"{command} {commandArguments}"; - - LinuxProcessAffinityConfiguration linuxConfig = (LinuxProcessAffinityConfiguration)affinityConfig; - string wrappedCommand = linuxConfig.GetCommandWithAffinity(null, fullCommandLine); - - process = this.systemManagement.ProcessManager.CreateProcess( - "/bin/bash", - $"-c {wrappedCommand}", - workingDirectory); - } - } - else - { - process = await this.ExecuteCommandAsync(command, commandArguments, workingDirectory, telemetryContext, cancellationToken); - } - - using (process) - { - if (affinityConfig != null) - { - if (this.Platform == PlatformID.Win32NT) - { - process.Start(); - process.ApplyAffinity((WindowsProcessAffinityConfiguration)affinityConfig); - await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false); - } - else - { - await process.StartAndWaitAsync(cancellationToken).ConfigureAwait(false); - } - } - - if (!cancellationToken.IsCancellationRequested) - { - await this.LogProcessDetailsAsync(process, telemetryContext, "AspNetBenchmarks"); - process.ThrowIfWorkloadFailed(successCodes: new int[] { 0 }); - } - } - } - catch (OperationCanceledException) - { - // Expected whenever certain operations (e.g. Task.Delay) are cancelled. - } - catch (Exception exc) - { - this.Logger.LogMessage( - $"{this.TypeName}.StartServerInstanceError", - LogLevel.Error, - telemetryContext.Clone().AddError(exc)); - - throw; - } - }); - } - private Task SaveStateAsync(EventContext telemetryContext, CancellationToken cancellationToken) { EventContext relatedContext = telemetryContext.Clone(); diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerProcess.cs b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerProcess.cs new file mode 100644 index 0000000000..1ca9c69bc5 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerProcess.cs @@ -0,0 +1,196 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.Logging; + using Polly; + using VirtualClient.Common; + using VirtualClient.Common.Contracts; + using VirtualClient.Common.ProcessAffinity; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + + internal sealed class AspNetServerProcess + { + private readonly VirtualClientComponent component; + private readonly ISystemManagement systemManagement; + private readonly string processLogName; + private CancellationTokenSource cancellation; + + public AspNetServerProcess(VirtualClientComponent component, ISystemManagement systemManagement, string processLogName) + { + this.component = component; + this.systemManagement = systemManagement; + this.processLogName = processLogName; + } + + public Task Completion { get; private set; } + + public void Start( + string command, + string arguments, + string directory, + ProcessAffinityConfiguration affinity, + IAsyncPolicy retryPolicy, + EventContext context, + CancellationToken cancellationToken) + { + if (this.Completion != null) + { + throw new InvalidOperationException("The previous ASP.NET server must be stopped before starting another instance."); + } + + cancellationToken.ThrowIfCancellationRequested(); + this.cancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + if (affinity != null) + { + context.AddContext("affinityMask", affinity.ToString()); + } + + this.Completion = this.RunAsync(command, arguments, directory, affinity, retryPolicy, context, this.cancellation.Token); + } + + public async Task EnsureRunningAsync() + { + if (this.Completion.IsCompleted) + { + await this.Completion.ConfigureAwait(false); + } + } + + public async Task StopAsync(EventContext context) + { + this.component.Logger.LogTraceMessage($"{this.component.TypeName}.KillServerInstances", context); + if (this.cancellation != null) + { + try + { + await this.cancellation.CancelAsync().ConfigureAwait(false); + await this.Completion.ConfigureAwait(false); + } + catch (OperationCanceledException) when (this.cancellation.IsCancellationRequested) + { + } + finally + { + this.cancellation.Dispose(); + this.cancellation = null; + this.Completion = null; + } + } + } + + private async Task RunAsync( + string command, + string arguments, + string directory, + ProcessAffinityConfiguration affinity, + IAsyncPolicy retryPolicy, + EventContext context, + CancellationToken cancellationToken) + { + try + { + await (retryPolicy ?? Policy.NoOpAsync()).ExecuteAsync( + token => this.RunInstanceAsync(command, arguments, directory, affinity, context, token), + cancellationToken).ConfigureAwait(false); + } + catch (Exception exception) when (!(exception is OperationCanceledException && cancellationToken.IsCancellationRequested)) + { + this.component.SetServerOnline(false); + this.component.Logger.LogMessage( + $"{this.component.TypeName}.ServerFailed", + LogLevel.Error, + context.Clone().AddError(exception)); + + throw; + } + } + + private async Task RunInstanceAsync( + string command, + string arguments, + string directory, + ProcessAffinityConfiguration affinity, + EventContext context, + CancellationToken cancellationToken) + { + try + { + cancellationToken.ThrowIfCancellationRequested(); + IProcessProxy process; + if (affinity is LinuxProcessAffinityConfiguration linuxAffinity) + { + string fullCommand = linuxAffinity.GetCommandWithAffinity(null, $"{command} {arguments}"); + process = this.systemManagement.ProcessManager.CreateProcess("/bin/bash", $"-c {fullCommand}", directory); + } + else + { + process = this.systemManagement.ProcessManager.CreateProcess(command, arguments, directory); + } + + using (process) + { + this.component.Logger.LogTraceMessage($"Executing: {command} {SensitiveData.ObscureSecrets(arguments)}", context); + cancellationToken.ThrowIfCancellationRequested(); + try + { + if (!process.Start()) + { + throw new WorkloadException("The ASP.NET server process did not start.", ErrorReason.WorkloadFailed); + } + } + catch (System.ComponentModel.Win32Exception exception) + { + throw new WorkloadException( + $"The ASP.NET server process could not start (command={SensitiveData.ObscureSecrets(process.FullCommand())}).", + exception, + ErrorReason.WorkloadFailed); + } + + try + { + if (affinity is WindowsProcessAffinityConfiguration windowsAffinity) + { + process.ApplyAffinity(windowsAffinity); + } + + await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false); + } + finally + { + process.SafeKill(this.component.Logger, TimeSpan.FromSeconds(30)); + if (!process.HasExited) + { + throw new WorkloadException("The ASP.NET server process could not be stopped.", ErrorReason.WorkloadFailed); + } + + await this.component.LogProcessDetailsAsync(process, context, this.processLogName).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + process.ThrowIfWorkloadFailed(); + throw new WorkloadException("The ASP.NET server exited unexpectedly.", ErrorReason.WorkloadFailed); + } + } + catch (Exception exception) when (!(exception is OperationCanceledException && cancellationToken.IsCancellationRequested)) + { + this.component.Logger.LogMessage( + $"{this.component.TypeName}.StartServerInstanceError", + LogLevel.Error, + context.Clone().AddError(exception)); + + if (exception is OperationCanceledException) + { + throw new WorkloadException("The ASP.NET server was cancelled unexpectedly.", exception, ErrorReason.WorkloadFailed); + } + + throw; + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-BOMBARDIER.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-BOMBARDIER.json index a3ccce1208..004f8f1f9f 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-BOMBARDIER.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-BOMBARDIER.json @@ -28,7 +28,8 @@ "Role": "Client", "Scenario": "ExecuteJsonSerializationBenchmark", "PackageName": "bombardier", - "CommandArguments": "--duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://{ServerIp}:9876/json --print r --format json", + "ServerPort": "$.Parameters.ServerPort", + "CommandArguments": "--duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://{ServerIp}:{ServerPort}/json --print r --format json", "WarmUp": false, "Timeout": "00:05:00" } diff --git a/website/docs/workloads/aspnetbench/aspnetbench-profiles.md b/website/docs/workloads/aspnetbench/aspnetbench-profiles.md index 49926b0513..30815edfe4 100644 --- a/website/docs/workloads/aspnetbench/aspnetbench-profiles.md +++ b/website/docs/workloads/aspnetbench/aspnetbench-profiles.md @@ -173,7 +173,7 @@ with 256 concurrent connections. Uses .NET 8 SDK. |---------------------------|-------------------------------------------------------------------|---------------| | DotNetVersion | The version of the .NET SDK to download and install. | 8.0.204 | | TargetFramework | The .NET target framework to run. | net8.0 | - | ServerPort | The port the Kestrel server listens on. | 9876 | + | ServerPort | The port used by both Kestrel and Bombardier. | 9876 | * **Profile Runtimes** See the 'Metadata' section of the profile for estimated runtimes. From f9039f98b2f7f7e979deaa0f57f27862c9af2d86 Mon Sep 17 00:00:00 2001 From: Alex Williams-Ferreira Date: Mon, 9 Nov 2026 14:10:28 -0800 Subject: [PATCH 2/2] Document ASP.NET server lifecycle ownership Clarify serialized lifecycle calls, completion versus readiness, and cancellation-independent shutdown. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../ASPNET/AspNetServerProcess.cs | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerProcess.cs b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerProcess.cs index 1ca9c69bc5..ecae30af0a 100644 --- a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerProcess.cs +++ b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerProcess.cs @@ -14,6 +14,12 @@ namespace VirtualClient.Actions using VirtualClient.Common.Telemetry; using VirtualClient.Contracts; + /// + /// Owns an ASP.NET server's retry task, cancellation, and process cleanup. + /// + /// + /// Callers must serialize lifecycle operations and await StopAsync before starting a replacement. + /// internal sealed class AspNetServerProcess { private readonly VirtualClientComponent component; @@ -28,8 +34,14 @@ public AspNetServerProcess(VirtualClientComponent component, ISystemManagement s this.processLogName = processLogName; } + /// + /// Gets the current server lifetime task, or null when no lifetime is tracked. Does not indicate readiness. + /// public Task Completion { get; private set; } + /// + /// Starts a server lifetime linked to the caller's cancellation token, including retries. + /// public void Start( string command, string arguments, @@ -54,6 +66,9 @@ public void Start( this.Completion = this.RunAsync(command, arguments, directory, affinity, retryPolicy, context, this.cancellation.Token); } + /// + /// Propagates a completed lifetime's failure without waiting for a running server to exit. + /// public async Task EnsureRunningAsync() { if (this.Completion.IsCompleted) @@ -62,6 +77,12 @@ public async Task EnsureRunningAsync() } } + /// + /// Cancels retries and awaits owned-process cleanup. Unexpected lifetime failures still propagate. + /// + /// + /// Cleanup has no caller cancellation token so an already-cancelled run cannot interrupt shutdown. + /// public async Task StopAsync(EventContext context) { this.component.Logger.LogTraceMessage($"{this.component.TypeName}.KillServerInstances", context);