diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/ProfileTests.cs b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/ProfileTests.cs index ceef931fbb..b6aaa55b4a 100644 --- a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/ProfileTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/ProfileTests.cs @@ -4,6 +4,7 @@ namespace VirtualClient.Actions { using System; + using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -24,6 +25,41 @@ public void SetupFixture() ComponentTypeCache.Instance.LoadComponentTypes(MockFixture.TestAssemblyDirectory); } + [Test] + public void AspNetBombardierProfilePreservesDefaultFailurePolicy() + { + this.mockFixture.Setup(PlatformID.Win32NT); + using (ProfileExecutor executor = TestProfileResources.CreateProfileExecutor( + "PERF-WEB-ASPNET-BOMBARDIER.json", + this.mockFixture.Dependencies)) + { + WorkloadAssert.ParameterReferencesInlined(executor.Profile); + Assert.That(executor.Profile.Actions[0].Parameters.ContainsKey("FailFast"), Is.False); + Assert.That(executor.Profile.Actions[1].Parameters["ServerPort"].ToString(), Is.EqualTo("9876")); + } + } + + [Test] + [TestCase("9876")] + [TestCase("9800")] + public async Task AspNetBombardierProfileResolvesTheConfiguredPortForBothActions(string port) + { + this.mockFixture.Setup(PlatformID.Win32NT); + using (ProfileExecutor executor = TestProfileResources.CreateProfileExecutor( + "PERF-WEB-ASPNET-BOMBARDIER.json", + this.mockFixture.Dependencies, + parameterOverrides: new Dictionary { ["ServerPort"] = port })) + { + var serverParameters = executor.Profile.Actions[0].Parameters; + var clientParameters = executor.Profile.Actions[1].Parameters; + Assert.That(clientParameters.ContainsKey("ServerPort"), Is.True); + await ProfileExpressionEvaluator.Instance.EvaluateAsync(this.mockFixture.Dependencies, clientParameters, CancellationToken.None); + Assert.That(serverParameters["ServerPort"].ToString(), Is.EqualTo(port)); + Assert.That(clientParameters["CommandArguments"].ToString(), Does.Contain($":{port}/json")); + Assert.That(clientParameters["CommandArguments"].ToString(), Does.Not.Contain("{ServerPort}")); + } + } + [Test] [TestCase("EXAMPLE-WORKLOAD.json")] public async Task ParametersAndMetadataArePassedToTheVirtualClientComponentOnExecution(string profile) diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetOrchardServerExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetOrchardServerExecutorTests.cs index c26b7606b3..1eb5087b2e 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetOrchardServerExecutorTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetOrchardServerExecutorTests.cs @@ -63,14 +63,12 @@ public async Task AspNetOrchardServerExecutorRunsTheExpectedWorkloadCommandInLin await executor.ExecuteAsync(CancellationToken.None); this.mockFixture.Tracking.AssertCommandsExecuted(true, - "pkill OrchardCore", - "fuser -n tcp -k 5014", $"{Regex.Escape(this.mockDotNetPackage.Path)}/dotnet publish -c Release --sc -f net9\\.0 {Regex.Escape(this.mockOrchardCorePackage.Path)}/src/OrchardCore\\.Cms\\.Web/OrchardCore\\.Cms\\.Web\\.csproj", - $"nohup {Regex.Escape(this.mockOrchardCorePackage.Path)}/src/OrchardCore\\.Cms\\.Web/bin/Release/net9\\.0/linux-x64/publish/OrchardCore\\.Cms\\.Web --urls http://\\*:5014" + $"{Regex.Escape(this.mockOrchardCorePackage.Path)}/src/OrchardCore\\.Cms\\.Web/bin/Release/net9\\.0/linux-x64/publish/OrchardCore\\.Cms\\.Web --urls http://\\*:5014" ); - this.mockFixture.Tracking.AssertCommandExecutedTimes("pkill", 1); - this.mockFixture.Tracking.AssertCommandExecutedTimes("fuser", 1); + this.mockFixture.Tracking.AssertCommandExecutedTimes("pkill", 0); + this.mockFixture.Tracking.AssertCommandExecutedTimes("fuser", 0); } } @@ -86,13 +84,12 @@ public async Task AspNetOrchardServerExecutorRunsTheExpectedWorkloadCommandInWin await executor.ExecuteAsync(CancellationToken.None); this.mockFixture.Tracking.AssertCommandsExecuted(true, - "taskkill /F /IM OrchardCore\\.Cms\\.Web\\.exe", $"{Regex.Escape(this.mockDotNetPackage.Path)}\\\\dotnet\\.exe publish -c Release --sc -f net9\\.0 {Regex.Escape(this.mockOrchardCorePackage.Path)}\\\\src\\\\OrchardCore\\.Cms\\.Web\\\\OrchardCore\\.Cms\\.Web\\.csproj", $"{Regex.Escape(this.mockOrchardCorePackage.Path)}\\\\src\\\\OrchardCore\\.Cms\\.Web\\\\bin\\\\Release\\\\net9\\.0\\\\win-x64\\\\publish\\\\OrchardCore\\.Cms\\.Web\\.exe --urls http://\\*:5014" ); // On Windows the Linux-only 'pkill'/'fuser'/'nohup' commands must NOT be used. - this.mockFixture.Tracking.AssertCommandExecutedTimes("taskkill", 1); + this.mockFixture.Tracking.AssertCommandExecutedTimes("taskkill", 0); this.mockFixture.Tracking.AssertCommandExecutedTimes("pkill", 0); this.mockFixture.Tracking.AssertCommandExecutedTimes("fuser", 0); this.mockFixture.Tracking.AssertCommandExecutedTimes("nohup", 0); @@ -159,10 +156,7 @@ public async Task AspNetOrchardServerExecutorExecutesWithCoreAffinityOnLinux() await executor.ExecuteAsync(CancellationToken.None); // Verify numactl was used with correct core affinity - // Actual format: /bin/bash -c "numactl -C 0-3 nohup ..." this.mockFixture.Tracking.AssertCommandsExecuted(true, - "pkill OrchardCore", - "fuser -n tcp -k 5014", $"{Regex.Escape(this.mockDotNetPackage.Path)}/dotnet publish -c Release --sc -f net9\\.0 {Regex.Escape(this.mockOrchardCorePackage.Path)}/src/OrchardCore\\.Cms\\.Web/OrchardCore\\.Cms\\.Web\\.csproj", "/bin/bash -c \\\"numactl -C 0-3 .*" ); @@ -240,6 +234,14 @@ private void SetupDefaultMockBehaviors(PlatformID platform) this.mockFixture.Directory.Setup(f => f.Exists(It.IsAny())) .Returns(true); this.mockFixture.FileSystem.SetupGet(fs => fs.File).Returns(this.mockFixture.File.Object); + this.mockFixture.Layout = new EnvironmentLayout(new[] + { + new ClientInstance(Environment.MachineName, "127.0.0.1", ClientRole.Client) + }); + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, directory) => + arguments?.Contains("--urls") == true + ? new AspNetServerLifecycleTests.ServerProcess { StartInfo = new System.Diagnostics.ProcessStartInfo(command, arguments) } + : new InMemoryProcess { StartInfo = new System.Diagnostics.ProcessStartInfo(command, arguments ?? string.Empty) }; this.mockFixture.Parameters = new Dictionary() { diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetServerExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetServerExecutorTests.cs index 8b1dd59770..b0d15c070e 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetServerExecutorTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetServerExecutorTests.cs @@ -72,14 +72,12 @@ public async Task AspNetServerExecutorRunsTheExpectedWorkloadCommandInLinux() await executor.ExecuteAsync(CancellationToken.None); this.mockFixture.Tracking.AssertCommandsExecuted(true, - "pkill dotnet", - "fuser -n tcp -k 9876", $"{Regex.Escape(this.mockDotNetPackage.Path)}/dotnet build -c Release -p:BenchmarksTargetFramework=net8\\.0", $"{Regex.Escape(this.mockDotNetPackage.Path)}/dotnet {Regex.Escape(this.mockAspNetBenchPackage.Path)}/src/Benchmarks/bin/Release/net8\\.0/Benchmarks\\.dll --nonInteractive true --scenarios json --urls http://\\*:9876 --server Kestrel --kestrelTransport Sockets --protocol http --header \\\"Accept: application/json,text/html;q=0\\.9,application/xhtml\\+xml;q=0\\.9,application/xml;q=0\\.8,\\*/\\*;q=0\\.7\\\" --header \\\"Connection: keep-alive\\\"" ); - this.mockFixture.Tracking.AssertCommandExecutedTimes("pkill", 1); - this.mockFixture.Tracking.AssertCommandExecutedTimes("fuser", 1); + this.mockFixture.Tracking.AssertCommandExecutedTimes("pkill", 0); + this.mockFixture.Tracking.AssertCommandExecutedTimes("fuser", 0); } } @@ -101,13 +99,11 @@ public async Task AspNetServerExecutorRunsTheExpectedWorkloadCommandInWindows() await executor.ExecuteAsync(CancellationToken.None); this.mockFixture.Tracking.AssertCommandsExecuted(true, - "taskkill /F /IM dotnet\\.exe", $"{Regex.Escape(this.mockDotNetPackage.Path)}\\\\dotnet\\.exe build -c Release -p:BenchmarksTargetFramework=net8\\.0", $"{Regex.Escape(this.mockDotNetPackage.Path)}\\\\dotnet\\.exe {Regex.Escape(this.mockAspNetBenchPackage.Path)}\\\\src\\\\Benchmarks\\\\bin\\\\Release\\\\net8\\.0\\\\Benchmarks\\.dll --nonInteractive true --scenarios json --urls http://\\*:9876 --server Kestrel --kestrelTransport Sockets --protocol http --header \\\"Accept: application/json,text/html;q=0\\.9,application/xhtml\\+xml;q=0\\.9,application/xml;q=0\\.8,\\*/\\*;q=0\\.7\\\" --header \\\"Connection: keep-alive\\\"" ); - // On Windows the Linux-only 'pkill'/'fuser' commands must NOT be used; 'taskkill' is used instead. - this.mockFixture.Tracking.AssertCommandExecutedTimes("taskkill", 1); + this.mockFixture.Tracking.AssertCommandExecutedTimes("taskkill", 0); this.mockFixture.Tracking.AssertCommandExecutedTimes("pkill", 0); this.mockFixture.Tracking.AssertCommandExecutedTimes("fuser", 0); } @@ -181,8 +177,6 @@ public async Task AspNetServerExecutorExecutesWithCoreAffinityOnLinux() // Verify numactl was used with correct core affinity // The actual command format is: /bin/bash -c "numactl -C 0-7 /path/to/dotnet /path/to/dll ..." this.mockFixture.Tracking.AssertCommandsExecuted(true, - "pkill dotnet", - "fuser -n tcp -k 9876", $"{Regex.Escape(this.mockDotNetPackage.Path)}/dotnet build -c Release -p:BenchmarksTargetFramework=net8\\.0", "/bin/bash -c \\\"numactl -C 0-7 .*" ); @@ -275,6 +269,14 @@ private void SetupDefaultMockBehaviors(PlatformID platform) this.mockFixture.Directory.Setup(f => f.Exists(It.IsAny())) .Returns(true); this.mockFixture.FileSystem.SetupGet(fs => fs.File).Returns(this.mockFixture.File.Object); + this.mockFixture.Layout = new EnvironmentLayout(new[] + { + new ClientInstance(Environment.MachineName, "127.0.0.1", ClientRole.Client) + }); + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, directory) => + arguments?.Contains("Benchmarks.dll") == true + ? new AspNetServerLifecycleTests.ServerProcess { StartInfo = new ProcessStartInfo(command, arguments) } + : new InMemoryProcess { StartInfo = new ProcessStartInfo(command, arguments ?? string.Empty) }; this.mockFixture.Parameters = new Dictionary() { diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetServerLifecycleTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetServerLifecycleTests.cs new file mode 100644 index 0000000000..f0dd3ea2cd --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetServerLifecycleTests.cs @@ -0,0 +1,387 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.Linq; + using System.Net; + using System.Net.Sockets; + using System.Threading; + using System.Threading.Tasks; + using Moq; + using Newtonsoft.Json.Linq; + using NUnit.Framework; + using Polly; + using VirtualClient.Common; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + + [TestFixture] + [Category("Unit")] + public class AspNetServerLifecycleTests + { + [TestCase(PlatformID.Win32NT)] + [TestCase(PlatformID.Unix)] + public async Task ReplacingServerStopsPreviousInstanceWithoutRestartingIt(PlatformID platform) + { + MockFixture fixture = CreateFixture(platform); + List servers = new List(); + TaskCompletionSource retryStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + fixture.ProcessManager.OnCreateProcess = (command, arguments, directory) => + { + if ((command == "taskkill" || command == "pkill") && servers.Count == 1) + { + servers[0].Stop(1); + retryStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)).GetAwaiter().GetResult(); + } + + if (arguments?.Contains("Benchmarks.dll") == true) + { + ServerProcess server = new ServerProcess(); + server.StartInfo = new ProcessStartInfo(command, arguments); + server.StandardOutput.Append("Now listening on: http://localhost:9876"); + server.StandardError.Append("Server diagnostic output"); + server.OnStart = () => + { + if (servers.Count == 2) + { + retryStarted.TrySetResult(true); + } + + return true; + }; + servers.Add(server); + return server; + } + + return new InMemoryProcess { StartInfo = new ProcessStartInfo(command, arguments ?? string.Empty) }; + }; + + using (CancellationTokenSource cancellation = new CancellationTokenSource(TimeSpan.FromSeconds(15))) + using (TestServer executor = new TestServer(fixture, retry: true)) + { + try + { + await executor.ExecuteAsync(cancellation.Token); + await executor.ExecuteAsync(cancellation.Token); + + Assert.That(servers.Count, Is.EqualTo(2), "Replacement must create one new server, not restart the previous instance."); + Assert.That(servers[0].HasExited, Is.True); + Assert.That(servers[1].HasExited, Is.False); + Assert.That(fixture.Logger.MessagesLogged($"{executor.TypeName}.KillServerInstances").Count(), Is.EqualTo(2)); + var processLogs = fixture.Logger.MessagesLogged($"{executor.TypeName}.AspNetBenchmarks.ProcessDetails").ToArray(); + Assert.That(processLogs.Length, Is.EqualTo(1), "Replacing the server must capture its process output."); + EventContext processContext = (EventContext)processLogs[0].Item3; + JObject processDetails = JObject.FromObject(processContext.Properties["process"]); + Assert.That(processDetails.Value("standardOutput"), Does.Contain("Now listening on:")); + Assert.That(processDetails.Value("standardError"), Is.EqualTo("Server diagnostic output")); + Assert.That(fixture.ProcessManager.Commands.Any(command => + command.StartsWith("taskkill") || command.StartsWith("pkill") || command.StartsWith("fuser")), Is.False); + } + finally + { + cancellation.Cancel(); + foreach (ServerProcess server in servers) + { + server.Stop(0); + } + } + + Assert.That(fixture.Logger.MessagesLogged("TestServer.AspNetBenchmarks.ProcessDetails").Count(), Is.EqualTo(2), + "Disposal must capture the final server's process output exactly once."); + } + } + + [Test] + public async Task ReplacingServerCancelsAPendingRetryDelay() + { + MockFixture fixture = CreateFixture(PlatformID.Win32NT); + List servers = new List(); + TaskCompletionSource retryPending = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + fixture.ProcessManager.OnCreateProcess = (command, arguments, directory) => + { + if (arguments?.Contains("Benchmarks.dll") == true) + { + ServerProcess server = new ServerProcess { StartInfo = new ProcessStartInfo(command, arguments) }; + servers.Add(server); + return server; + } + + return new InMemoryProcess { StartInfo = new ProcessStartInfo(command, arguments ?? string.Empty) }; + }; + + using (CancellationTokenSource cancellation = new CancellationTokenSource(TimeSpan.FromSeconds(10))) + using (TestServer executor = new TestServer(fixture)) + { + executor.SetRetryPolicy(Policy.Handle(exception => !(exception is OperationCanceledException)) + .WaitAndRetryAsync(1, attempt => TimeSpan.FromSeconds(10), (exception, delay) => retryPending.TrySetResult(true))); + await executor.ExecuteAsync(cancellation.Token); + servers[0].Stop(1); + await retryPending.Task.WaitAsync(cancellation.Token); + Assert.That(fixture.Logger.MessagesLogged($"{executor.TypeName}.StartServerInstanceError").Count(), Is.EqualTo(1), + "A failed server attempt must be logged even when a retry is pending."); + await executor.ExecuteAsync(cancellation.Token).WaitAsync(TimeSpan.FromSeconds(2)); + + Assert.That(servers.Count, Is.EqualTo(2)); + Assert.That(servers[0].HasExited, Is.True); + cancellation.Cancel(); + } + } + + [Test] + public void AnOccupiedPortDoesNotMakeAFailedServerAppearReady() + { + MockFixture fixture = CreateFixture(PlatformID.Win32NT); + using (TcpListener listener = new TcpListener(IPAddress.Loopback, 0)) + { + listener.Start(); + fixture.Parameters["ServerPort"] = ((IPEndPoint)listener.LocalEndpoint).Port.ToString(); + fixture.ProcessManager.OnCreateProcess = (command, arguments, directory) => + new InMemoryProcess + { + StartInfo = new ProcessStartInfo(command, arguments ?? string.Empty), + ExitCode = arguments?.Contains("Benchmarks.dll") == true ? 1 : 0 + }; + + using (TestServer executor = new TestServer(fixture, realReadiness: true)) + { + Assert.ThrowsAsync(() => executor.ExecuteAsync(CancellationToken.None)); + } + } + } + + [Test] + public void AServerFailureWhileSavingReadinessIsNotReportedAsSuccess() + { + MockFixture fixture = CreateFixture(PlatformID.Win32NT); + ServerProcess server = null; + fixture.ProcessManager.OnCreateProcess = (command, arguments, directory) => + { + if (arguments?.Contains("Benchmarks.dll") == true) + { + server = new ServerProcess { StartInfo = new ProcessStartInfo(command, arguments) }; + return server; + } + + return new InMemoryProcess { StartInfo = new ProcessStartInfo(command, arguments ?? string.Empty) }; + }; + + using (TestServer executor = new TestServer(fixture)) + { + fixture.ApiClient.OnUpdateState(nameof(State)).Returns(async () => + { + server.Stop(1); + await executor.GetServerTask().ContinueWith( + task => Assert.That(task.Exception.InnerException, Is.InstanceOf()), + TaskScheduler.Default); + return fixture.CreateHttpResponse(HttpStatusCode.OK); + }); + + Assert.ThrowsAsync(() => executor.ExecuteAsync(CancellationToken.None)); + } + } + + [Test] + public void ANativeProcessStartFailureUsesTheStandardWorkloadErrorClassification() + { + MockFixture fixture = CreateFixture(PlatformID.Unix); + fixture.ProcessManager.OnCreateProcess = (command, arguments, directory) => + new InMemoryProcess + { + StartInfo = new ProcessStartInfo(command, arguments ?? string.Empty), + OnStart = arguments?.Contains("Benchmarks.dll") == true + ? () => throw new System.ComponentModel.Win32Exception(2, "No such file or directory") + : () => true + }; + + using (TestServer executor = new TestServer(fixture)) + { + WorkloadException failure = Assert.ThrowsAsync(() => executor.ExecuteAsync(CancellationToken.None)); + Assert.That(failure.Reason, Is.EqualTo(ErrorReason.WorkloadFailed)); + Assert.That(failure.InnerException, Is.InstanceOf()); + Assert.That(failure.Message, Does.Contain("Benchmarks.dll")); + var outcomes = fixture.Logger.MessagesLogged($"{executor.TypeName}.SucceededOrFailed").ToArray(); + Assert.That(outcomes.Length, Is.EqualTo(1)); + Assert.That(((EventContext)outcomes[0].Item3).Properties["metricName"], Is.EqualTo("Failed")); + } + } + + [Test] + public void AnUnexpectedCancellationIsReportedAsAWorkloadFailure() + { + MockFixture fixture = CreateFixture(PlatformID.Win32NT); + fixture.ProcessManager.OnCreateProcess = (command, arguments, directory) => + arguments?.Contains("Benchmarks.dll") == true + ? new ServerProcess { StartInfo = new ProcessStartInfo(command, arguments), ThrowOnWait = true } + : new InMemoryProcess { StartInfo = new ProcessStartInfo(command, arguments ?? string.Empty) }; + + using (TestServer executor = new TestServer(fixture)) + { + Assert.ThrowsAsync(() => executor.ExecuteAsync(CancellationToken.None)); + } + } + + [Test] + public void MultiRoleServerPropagatesBackgroundProcessFailure() + { + MockFixture fixture = CreateFixture(PlatformID.Win32NT); + fixture.Layout = new EnvironmentLayout(new[] + { + new ClientInstance(Environment.MachineName, "127.0.0.1", ClientRole.Server), + new ClientInstance("client", "127.0.0.2", ClientRole.Client) + }); + fixture.ProcessManager.OnCreateProcess = (command, arguments, directory) => + new InMemoryProcess + { + StartInfo = new ProcessStartInfo(command, arguments ?? string.Empty), + ExitCode = arguments?.Contains("Benchmarks.dll") == true ? 1 : 0 + }; + + using (TestServer executor = new TestServer(fixture)) + { + Assert.ThrowsAsync(() => executor.ExecuteAsync(CancellationToken.None)); + } + } + + [Test] + public async Task ABackgroundFailureIsNotDiscardedByTheNextIteration() + { + MockFixture fixture = CreateFixture(PlatformID.Win32NT); + ServerProcess firstServer = null; + TaskCompletionSource exited = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + fixture.ProcessManager.OnCreateProcess = (command, arguments, directory) => + { + if (arguments?.Contains("Benchmarks.dll") == true) + { + ServerProcess server = new ServerProcess { StartInfo = new ProcessStartInfo(command, arguments) }; + if (firstServer == null) + { + firstServer = server; + server.OnDispose = () => exited.TrySetResult(true); + } + + return server; + } + + return new InMemoryProcess { StartInfo = new ProcessStartInfo(command, arguments ?? string.Empty) }; + }; + + using (CancellationTokenSource cancellation = new CancellationTokenSource(TimeSpan.FromSeconds(10))) + using (TestServer executor = new TestServer(fixture)) + { + await executor.ExecuteAsync(cancellation.Token); + firstServer.Stop(1); + await exited.Task.WaitAsync(cancellation.Token); + await executor.GetServerTask().ContinueWith( + task => Assert.That(task.Exception.InnerException, Is.InstanceOf()), + TaskScheduler.Default); + Assert.That(fixture.Logger.MessagesLogged($"{executor.TypeName}.SucceededOrFailed").Count(), Is.EqualTo(1), + "A background error must not emit a second outcome for a completed action."); + Assert.ThrowsAsync(() => executor.ExecuteAsync(cancellation.Token)); + + EventContext[] outcomes = fixture.Logger.MessagesLogged($"{executor.TypeName}.SucceededOrFailed") + .Select(entry => (EventContext)entry.Item3).ToArray(); + Assert.That(outcomes.Length, Is.EqualTo(2), "Each action execution must emit exactly one outcome."); + Assert.That(outcomes.Select(outcome => outcome.Properties["metricName"]), Is.EqualTo(new[] { "Succeeded", "Failed" })); + EventContext failure = outcomes[1]; + Assert.That(failure.Properties["metricValue"], Is.EqualTo(1)); + Assert.That(failure.Properties["toolName"], Is.EqualTo(executor.TypeName)); + Assert.That(failure.Properties["scenarioName"], Is.EqualTo(executor.Scenario)); + Assert.That(failure.Properties["scenarioStartTime"], Is.EqualTo(executor.StartTime)); + Assert.That(failure.Properties["scenarioEndTime"], Is.InRange(executor.StartTime, executor.EndTime)); + Assert.That(failure.Properties["metricUnit"], Is.EqualTo(string.Empty)); + Assert.That(failure.Properties["metricRelativity"], Is.EqualTo(MetricRelativity.LowerIsBetter)); + Assert.That(failure.Properties["metricVerbosity"], Is.EqualTo(1)); + cancellation.Cancel(); + } + } + + private static MockFixture CreateFixture(PlatformID platform) + { + MockFixture fixture = new MockFixture(); + fixture.Setup(platform); + fixture.Layout = new EnvironmentLayout(new[] + { + new ClientInstance(Environment.MachineName, "127.0.0.1", ClientRole.Client) + }); + fixture.Parameters["PackageName"] = "aspnetbenchmarks"; + fixture.Parameters["TargetFramework"] = "net8.0"; + fixture.Parameters["ServerPort"] = "9876"; + fixture.PackageManager.OnGetPackage("aspnetbenchmarks") + .ReturnsAsync(new DependencyPath("aspnetbenchmarks", fixture.GetPackagePath("aspnetbenchmarks"))); + fixture.PackageManager.OnGetPackage("dotnetsdk") + .ReturnsAsync(new DependencyPath("dotnetsdk", fixture.GetPackagePath("dotnetsdk"))); + fixture.ApiClient.OnUpdateState(nameof(State)) + .ReturnsAsync(fixture.CreateHttpResponse(HttpStatusCode.OK)); + return fixture; + } + + internal class ServerProcess : InMemoryProcess, IProcessProxy + { + private readonly TaskCompletionSource exit = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + public ServerProcess() + { + this.OnHasExited = () => this.exit.Task.IsCompleted; + this.OnKill = () => this.Stop(1); + } + + public void Stop(int exitCode) + { + this.ExitCode = exitCode; + this.exit.TrySetResult(true); + } + + public bool ThrowOnWait { get; set; } + + Task IProcessProxy.WaitForExitAsync(CancellationToken cancellationToken, TimeSpan? timeout) + { + if (this.ThrowOnWait) + { + this.ThrowOnWait = false; + return Task.FromException(new OperationCanceledException()); + } + + return this.exit.Task.WaitAsync(cancellationToken); + } + } + + private class TestServer : AspNetServerExecutor + { + private readonly bool realReadiness; + + public TestServer(MockFixture fixture, bool retry = false, bool realReadiness = false) + : base(fixture.Dependencies, fixture.Parameters) + { + this.realReadiness = realReadiness; + this.ServerRetryPolicy = retry + ? Policy.Handle(exception => !(exception is OperationCanceledException)).RetryAsync(1) + : Policy.NoOpAsync(); + } + + protected override Task WaitForPortReadyAsync(EventContext context, CancellationToken cancellationToken) + { + return this.realReadiness + ? base.WaitForPortReadyAsync(context, cancellationToken) + : Task.CompletedTask; + } + + public void SetRetryPolicy(IAsyncPolicy policy) + { + this.ServerRetryPolicy = policy; + } + + public Task GetServerTask() + { + object process = typeof(AspNetServerExecutor).GetField( + "serverProcess", + System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic).GetValue(this); + return (Task)process.GetType().GetProperty("Completion").GetValue(process); + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetOrchardServerExecutor.cs b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetOrchardServerExecutor.cs index b43323a80c..d57c7c10ad 100644 --- a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetOrchardServerExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetOrchardServerExecutor.cs @@ -29,7 +29,7 @@ namespace VirtualClient.Actions [SupportedPlatforms("linux-arm64,linux-x64,win-arm64,win-x64")] public class AspNetOrchardServerExecutor : VirtualClientMultiRoleComponent { - private Task serverProcess; + private readonly AspNetServerProcess serverProcess; private bool disposed; private IFileSystem fileSystem; private IPackageManager packageManager; @@ -52,6 +52,7 @@ public AspNetOrchardServerExecutor(IServiceCollection dependencies, IDictionary< this.packageManager = this.systemManagement.PackageManager; this.stateManager = this.systemManagement.StateManager; this.fileSystem = this.systemManagement.FileSystem; + this.serverProcess = new AspNetServerProcess(this, this.systemManagement, "Orchard"); this.ServerRetryPolicy = Policy.Handle(exc => !(exc is OperationCanceledException)) .WaitAndRetryAsync(3, (retries) => TimeSpan.FromSeconds(retries)); } @@ -146,14 +147,15 @@ protected override void Dispose(bool disposing) { try { - this.KillServerInstancesAsync(EventContext.None, CancellationToken.None) - .GetAwaiter().GetResult(); + this.serverProcess.StopAsync(EventContext.None).GetAwaiter().GetResult(); } - catch + catch (Exception exception) { - // Best-effort cleanup during dispose. + this.Logger.LogMessage($"{this.TypeName}.CleanupError", LogLevel.Error, EventContext.None.AddError(exception)); } + base.Dispose(disposing); + this.disposed = true; } } @@ -262,38 +264,28 @@ protected override Task ExecuteAsync(EventContext telemetryContext, Cancellation await this.ServerApi.PollForHeartbeatAsync(TimeSpan.FromMinutes(5), cancellationToken); await this.DeleteStateAsync(telemetryContext, cancellationToken); - await this.KillServerInstancesAsync(telemetryContext, cancellationToken); + await this.serverProcess.StopAsync(telemetryContext); await this.BuildAspNetOrchardAsync(telemetryContext, cancellationToken); this.StartServerInstances(telemetryContext, cancellationToken); await this.WaitForPortReadyAsync(telemetryContext, cancellationToken); + await this.serverProcess.EnsureRunningAsync(); await this.SaveStateAsync(telemetryContext, cancellationToken); this.SetServerOnline(true); + await this.serverProcess.EnsureRunningAsync(); - if (!this.IsMultiRoleLayout()) - { - // In single-VM mode, clear cleanup tasks to prevent the base class - // CleanupAsync from killing the server process. The server must stay - // alive for the subsequent client action. It will be killed by - // KillServerInstancesAsync on the next iteration or by Dispose. - this.CleanupTasks.Clear(); - } - else + if (this.IsMultiRoleLayout()) { using (BackgroundOperations profiling = BackgroundOperations.BeginProfiling(this, cancellationToken)) { - await Task.WhenAny(this.serverProcess); - if (cancellationToken.IsCancellationRequested) - { - await Task.WhenAll(this.serverProcess); - } + await this.serverProcess.Completion; } } } catch { this.SetServerOnline(false); - await this.KillServerInstancesAsync(telemetryContext, cancellationToken); + await this.serverProcess.StopAsync(telemetryContext); throw; } }); @@ -314,11 +306,13 @@ protected virtual async Task WaitForPortReadyAsync(EventContext telemetryContext while (DateTime.UtcNow < deadline && !cancellationToken.IsCancellationRequested) { + await this.serverProcess.EnsureRunningAsync(); try { using (TcpClient client = new TcpClient()) { - await client.ConnectAsync(IPAddress.Loopback, port).ConfigureAwait(false); + await client.ConnectAsync(IPAddress.Loopback, port, cancellationToken).ConfigureAwait(false); + await this.serverProcess.EnsureRunningAsync(); this.Logger.LogTraceMessage($"{this.TypeName}: Server is accepting connections on port {port}."); return; } @@ -351,27 +345,6 @@ private Task DeleteStateAsync(EventContext telemetryContext, CancellationToken c }); } - private async Task KillServerInstancesAsync(EventContext telemetryContext, CancellationToken cancellationToken) - { - this.Logger.LogTraceMessage($"{this.TypeName}.KillServerInstances"); - - // 'pkill'/'fuser' exist only on Unix; on Windows they fail to start ("The system cannot - // find the file specified"), which crashes the workload. On Windows the equivalent - // 'taskkill' is used to terminate the self-contained OrchardCore server (which also - // releases the TCP port it was holding). - if (this.Platform == PlatformID.Unix) - { - await this.ExecuteCommandAsync("pkill", "OrchardCore", this.aspnetOrchardDirectory, telemetryContext, cancellationToken); - await this.ExecuteCommandAsync("fuser", $"-n tcp -k {this.ServerPort}", this.aspnetOrchardDirectory, telemetryContext, cancellationToken); - } - else - { - await this.ExecuteCommandAsync("taskkill", "/F /IM OrchardCore.Cms.Web.exe", this.aspnetOrchardDirectory, telemetryContext, cancellationToken); - } - - await this.WaitAsync(TimeSpan.FromSeconds(3), cancellationToken); - } - private void StartServerInstances(EventContext telemetryContext, CancellationToken cancellationToken) { EventContext relatedContext = telemetryContext.Clone(); @@ -382,17 +355,12 @@ private void StartServerInstances(EventContext telemetryContext, CancellationTok { string workingDirectory = this.aspnetOrchardDirectory; - // 'nohup' (run a process immune to hangups, detached) exists only on Unix. On - // Windows it fails to start ("The system cannot find the file specified"), so the - // self-contained OrchardCore server executable is launched directly instead. string serverExecutable = this.Combine( this.aspnetOrchardPublishPath, this.Platform == PlatformID.Unix ? "OrchardCore.Cms.Web" : "OrchardCore.Cms.Web.exe"); - string command = this.Platform == PlatformID.Unix ? "nohup" : serverExecutable; - string commandArguments = this.Platform == PlatformID.Unix - ? $"{serverExecutable} --urls http://*:{this.ServerPort}" - : $"--urls http://*:{this.ServerPort}"; + string command = serverExecutable; + string commandArguments = $"--urls http://*:{this.ServerPort}"; relatedContext.AddContext("command", command); relatedContext.AddContext("commandArguments", commandArguments); @@ -400,93 +368,22 @@ private void StartServerInstances(EventContext telemetryContext, CancellationTok relatedContext.AddContext("bindToCores", this.BindToCores); relatedContext.AddContext("coreAffinity", this.CoreAffinity); - this.serverProcess = this.StartServerInstanceAsync(command, commandArguments, workingDirectory, relatedContext, cancellationToken); - } - catch (OperationCanceledException) - { - // Expected whenever certain operations (e.g. Task.Delay) are cancelled. - } - }); - } - - private Task StartServerInstanceAsync(string command, string commandArguments, string workingDirectory, EventContext telemetryContext, CancellationToken cancellationToken) - { - return (this.ServerRetryPolicy ?? Policy.NoOpAsync()).ExecuteAsync(async () => - { - try - { - IProcessProxy process = null; - ProcessAffinityConfiguration affinityConfig = null; - - if (this.BindToCores && !string.IsNullOrWhiteSpace(this.CoreAffinity)) - { - affinityConfig = ProcessAffinityConfiguration.Create( - this.Platform, - this.CoreAffinity); - - telemetryContext.AddContext("affinityMask", affinityConfig.ToString()); - - if (this.Platform == PlatformID.Win32NT) - { - process = this.systemManagement.ProcessManager.CreateProcess( - command, - commandArguments, - workingDirectory); - } - else - { - string fullCommandLine = $"{command} {commandArguments}"; - - LinuxProcessAffinityConfiguration linuxConfig = (LinuxProcessAffinityConfiguration)affinityConfig; - string wrappedCommand = linuxConfig.GetCommandWithAffinity(null, fullCommandLine); - - process = this.systemManagement.ProcessManager.CreateProcess( - "/bin/bash", - $"-c {wrappedCommand}", - workingDirectory); - } - } - else - { - process = await this.ExecuteCommandAsync(command, commandArguments, workingDirectory, telemetryContext, cancellationToken); - } - - using (process) - { - if (affinityConfig != null) - { - if (this.Platform == PlatformID.Win32NT) - { - process.Start(); - process.ApplyAffinity((WindowsProcessAffinityConfiguration)affinityConfig); - await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false); - } - else - { - await process.StartAndWaitAsync(cancellationToken).ConfigureAwait(false); - } - } - - if (!cancellationToken.IsCancellationRequested) - { - await this.LogProcessDetailsAsync(process, telemetryContext, "Orchard"); - process.ThrowIfWorkloadFailed(successCodes: new int[] { 0 }); - } - } + ProcessAffinityConfiguration affinity = this.BindToCores + ? ProcessAffinityConfiguration.Create(this.Platform, this.CoreAffinity) + : null; + this.serverProcess.Start( + command, + commandArguments, + workingDirectory, + affinity, + this.ServerRetryPolicy, + relatedContext, + cancellationToken); } catch (OperationCanceledException) { // Expected whenever certain operations (e.g. Task.Delay) are cancelled. } - catch (Exception exc) - { - this.Logger.LogMessage( - $"{this.TypeName}.StartServerInstanceError", - LogLevel.Error, - telemetryContext.Clone().AddError(exc)); - - throw; - } }); } diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerExecutor.cs b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerExecutor.cs index 6334e7b8d4..dfc79281f5 100644 --- a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerExecutor.cs @@ -29,7 +29,7 @@ namespace VirtualClient.Actions [SupportedPlatforms("linux-arm64,linux-x64,win-arm64,win-x64")] public class AspNetServerExecutor : VirtualClientMultiRoleComponent { - private Task serverProcess; + private readonly AspNetServerProcess serverProcess; private bool disposed; private IFileSystem fileSystem; private IPackageManager packageManager; @@ -52,6 +52,7 @@ public AspNetServerExecutor(IServiceCollection dependencies, IDictionary(exc => !(exc is OperationCanceledException)) .WaitAndRetryAsync(3, (retries) => TimeSpan.FromSeconds(retries)); } @@ -157,16 +158,17 @@ protected override void Dispose(bool disposing) { try { - this.KillServerInstancesAsync(EventContext.None, CancellationToken.None) - .GetAwaiter().GetResult(); + this.serverProcess.StopAsync(EventContext.None).GetAwaiter().GetResult(); } - catch + catch (Exception exception) { - // Best-effort cleanup during dispose. + this.Logger.LogMessage($"{this.TypeName}.CleanupError", LogLevel.Error, EventContext.None.AddError(exception)); } this.disposed = true; } + + base.Dispose(disposing); } } @@ -252,39 +254,28 @@ protected override Task ExecuteAsync(EventContext telemetryContext, Cancellation await this.ServerApi.PollForHeartbeatAsync(TimeSpan.FromMinutes(5), cancellationToken); await this.DeleteStateAsync(telemetryContext, cancellationToken); - await this.KillServerInstancesAsync(telemetryContext, cancellationToken); + await this.serverProcess.StopAsync(telemetryContext); await this.BuildAspNetBenchAsync(telemetryContext, cancellationToken); this.StartServerInstances(telemetryContext, cancellationToken); await this.WaitForPortReadyAsync(telemetryContext, cancellationToken); + await this.serverProcess.EnsureRunningAsync(); await this.SaveStateAsync(telemetryContext, cancellationToken); this.SetServerOnline(true); + await this.serverProcess.EnsureRunningAsync(); - if (!this.IsMultiRoleLayout()) - { - // In single-VM mode, clear cleanup tasks to prevent the base class - // CleanupAsync from killing the server process. The server must stay - // alive for the subsequent client action. It will be killed by - // KillServerInstancesAsync on the next iteration or by Dispose. - this.CleanupTasks.Clear(); - } - else + if (this.IsMultiRoleLayout()) { using (BackgroundOperations profiling = BackgroundOperations.BeginProfiling(this, cancellationToken)) { - await Task.WhenAny(this.serverProcess); - - if (cancellationToken.IsCancellationRequested) - { - await Task.WhenAll(this.serverProcess); - } + await this.serverProcess.Completion; } } } catch { this.SetServerOnline(false); - await this.KillServerInstancesAsync(telemetryContext, cancellationToken); + await this.serverProcess.StopAsync(telemetryContext); throw; } }); @@ -325,11 +316,13 @@ protected virtual async Task WaitForPortReadyAsync(EventContext telemetryContext while (DateTime.UtcNow < deadline && !cancellationToken.IsCancellationRequested) { + await this.serverProcess.EnsureRunningAsync(); try { using (TcpClient client = new TcpClient()) { - await client.ConnectAsync(IPAddress.Loopback, port).ConfigureAwait(false); + await client.ConnectAsync(IPAddress.Loopback, port, cancellationToken).ConfigureAwait(false); + await this.serverProcess.EnsureRunningAsync(); this.Logger.LogTraceMessage($"{this.TypeName}: Server is accepting connections on port {port}."); return; } @@ -362,29 +355,6 @@ private Task DeleteStateAsync(EventContext telemetryContext, CancellationToken c }); } - private async Task KillServerInstancesAsync(EventContext telemetryContext, CancellationToken cancellationToken) - { - this.Logger.LogTraceMessage($"{this.TypeName}.KillServerInstances"); - - // The ASP.NET (Kestrel) benchmark server is hosted by the .NET runtime (dotnet). Both the - // server process(es) and the TCP port they hold must be released using platform-native - // commands. 'pkill'/'fuser' exist only on Unix; on Windows they fail to start - // ("The system cannot find the file specified"), which crashes the workload. On Windows the - // equivalent 'taskkill' is used instead (killing the process also releases its port). - if (this.Platform == PlatformID.Unix) - { - await this.ExecuteCommandAsync("pkill", "dotnet", this.aspnetBenchDirectory, telemetryContext, cancellationToken); - - await this.ExecuteCommandAsync("fuser", $"-n tcp -k {this.ServerPort}", this.aspnetBenchDirectory, telemetryContext, cancellationToken); - } - else - { - await this.ExecuteCommandAsync("taskkill", "/F /IM dotnet.exe", this.aspnetBenchDirectory, telemetryContext, cancellationToken); - } - - await this.WaitAsync(TimeSpan.FromSeconds(3), cancellationToken); - } - private void StartServerInstances(EventContext telemetryContext, CancellationToken cancellationToken) { EventContext relatedContext = telemetryContext.Clone(); @@ -404,7 +374,17 @@ private void StartServerInstances(EventContext telemetryContext, CancellationTok relatedContext.AddContext("bindToCores", this.BindToCores); relatedContext.AddContext("coreAffinity", this.CoreAffinity); - this.serverProcess = this.StartServerInstanceAsync(this.dotnetExePath, commandArguments, workingDirectory, relatedContext, cancellationToken); + ProcessAffinityConfiguration affinity = this.BindToCores + ? ProcessAffinityConfiguration.Create(this.Platform, this.CoreAffinity) + : null; + this.serverProcess.Start( + this.dotnetExePath, + commandArguments, + workingDirectory, + affinity, + this.ServerRetryPolicy, + relatedContext, + cancellationToken); } catch (OperationCanceledException) { @@ -413,87 +393,6 @@ private void StartServerInstances(EventContext telemetryContext, CancellationTok }); } - private Task StartServerInstanceAsync(string command, string commandArguments, string workingDirectory, EventContext telemetryContext, CancellationToken cancellationToken) - { - return (this.ServerRetryPolicy ?? Policy.NoOpAsync()).ExecuteAsync(async () => - { - try - { - IProcessProxy process = null; - ProcessAffinityConfiguration affinityConfig = null; - - if (this.BindToCores && !string.IsNullOrWhiteSpace(this.CoreAffinity)) - { - affinityConfig = ProcessAffinityConfiguration.Create( - this.Platform, - this.CoreAffinity); - - telemetryContext.AddContext("affinityMask", affinityConfig.ToString()); - - if (this.Platform == PlatformID.Win32NT) - { - process = this.systemManagement.ProcessManager.CreateProcess( - command, - commandArguments, - workingDirectory); - } - else - { - string fullCommandLine = $"{command} {commandArguments}"; - - LinuxProcessAffinityConfiguration linuxConfig = (LinuxProcessAffinityConfiguration)affinityConfig; - string wrappedCommand = linuxConfig.GetCommandWithAffinity(null, fullCommandLine); - - process = this.systemManagement.ProcessManager.CreateProcess( - "/bin/bash", - $"-c {wrappedCommand}", - workingDirectory); - } - } - else - { - process = await this.ExecuteCommandAsync(command, commandArguments, workingDirectory, telemetryContext, cancellationToken); - } - - using (process) - { - if (affinityConfig != null) - { - if (this.Platform == PlatformID.Win32NT) - { - process.Start(); - process.ApplyAffinity((WindowsProcessAffinityConfiguration)affinityConfig); - await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false); - } - else - { - await process.StartAndWaitAsync(cancellationToken).ConfigureAwait(false); - } - } - - if (!cancellationToken.IsCancellationRequested) - { - await this.LogProcessDetailsAsync(process, telemetryContext, "AspNetBenchmarks"); - process.ThrowIfWorkloadFailed(successCodes: new int[] { 0 }); - } - } - } - catch (OperationCanceledException) - { - // Expected whenever certain operations (e.g. Task.Delay) are cancelled. - } - catch (Exception exc) - { - this.Logger.LogMessage( - $"{this.TypeName}.StartServerInstanceError", - LogLevel.Error, - telemetryContext.Clone().AddError(exc)); - - throw; - } - }); - } - private Task SaveStateAsync(EventContext telemetryContext, CancellationToken cancellationToken) { EventContext relatedContext = telemetryContext.Clone(); diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerProcess.cs b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerProcess.cs new file mode 100644 index 0000000000..ecae30af0a --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerProcess.cs @@ -0,0 +1,217 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.Logging; + using Polly; + using VirtualClient.Common; + using VirtualClient.Common.Contracts; + using VirtualClient.Common.ProcessAffinity; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + + /// + /// Owns an ASP.NET server's retry task, cancellation, and process cleanup. + /// + /// + /// Callers must serialize lifecycle operations and await StopAsync before starting a replacement. + /// + internal sealed class AspNetServerProcess + { + private readonly VirtualClientComponent component; + private readonly ISystemManagement systemManagement; + private readonly string processLogName; + private CancellationTokenSource cancellation; + + public AspNetServerProcess(VirtualClientComponent component, ISystemManagement systemManagement, string processLogName) + { + this.component = component; + this.systemManagement = systemManagement; + this.processLogName = processLogName; + } + + /// + /// Gets the current server lifetime task, or null when no lifetime is tracked. Does not indicate readiness. + /// + public Task Completion { get; private set; } + + /// + /// Starts a server lifetime linked to the caller's cancellation token, including retries. + /// + public void Start( + string command, + string arguments, + string directory, + ProcessAffinityConfiguration affinity, + IAsyncPolicy retryPolicy, + EventContext context, + CancellationToken cancellationToken) + { + if (this.Completion != null) + { + throw new InvalidOperationException("The previous ASP.NET server must be stopped before starting another instance."); + } + + cancellationToken.ThrowIfCancellationRequested(); + this.cancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + if (affinity != null) + { + context.AddContext("affinityMask", affinity.ToString()); + } + + this.Completion = this.RunAsync(command, arguments, directory, affinity, retryPolicy, context, this.cancellation.Token); + } + + /// + /// Propagates a completed lifetime's failure without waiting for a running server to exit. + /// + public async Task EnsureRunningAsync() + { + if (this.Completion.IsCompleted) + { + await this.Completion.ConfigureAwait(false); + } + } + + /// + /// Cancels retries and awaits owned-process cleanup. Unexpected lifetime failures still propagate. + /// + /// + /// Cleanup has no caller cancellation token so an already-cancelled run cannot interrupt shutdown. + /// + public async Task StopAsync(EventContext context) + { + this.component.Logger.LogTraceMessage($"{this.component.TypeName}.KillServerInstances", context); + if (this.cancellation != null) + { + try + { + await this.cancellation.CancelAsync().ConfigureAwait(false); + await this.Completion.ConfigureAwait(false); + } + catch (OperationCanceledException) when (this.cancellation.IsCancellationRequested) + { + } + finally + { + this.cancellation.Dispose(); + this.cancellation = null; + this.Completion = null; + } + } + } + + private async Task RunAsync( + string command, + string arguments, + string directory, + ProcessAffinityConfiguration affinity, + IAsyncPolicy retryPolicy, + EventContext context, + CancellationToken cancellationToken) + { + try + { + await (retryPolicy ?? Policy.NoOpAsync()).ExecuteAsync( + token => this.RunInstanceAsync(command, arguments, directory, affinity, context, token), + cancellationToken).ConfigureAwait(false); + } + catch (Exception exception) when (!(exception is OperationCanceledException && cancellationToken.IsCancellationRequested)) + { + this.component.SetServerOnline(false); + this.component.Logger.LogMessage( + $"{this.component.TypeName}.ServerFailed", + LogLevel.Error, + context.Clone().AddError(exception)); + + throw; + } + } + + private async Task RunInstanceAsync( + string command, + string arguments, + string directory, + ProcessAffinityConfiguration affinity, + EventContext context, + CancellationToken cancellationToken) + { + try + { + cancellationToken.ThrowIfCancellationRequested(); + IProcessProxy process; + if (affinity is LinuxProcessAffinityConfiguration linuxAffinity) + { + string fullCommand = linuxAffinity.GetCommandWithAffinity(null, $"{command} {arguments}"); + process = this.systemManagement.ProcessManager.CreateProcess("/bin/bash", $"-c {fullCommand}", directory); + } + else + { + process = this.systemManagement.ProcessManager.CreateProcess(command, arguments, directory); + } + + using (process) + { + this.component.Logger.LogTraceMessage($"Executing: {command} {SensitiveData.ObscureSecrets(arguments)}", context); + cancellationToken.ThrowIfCancellationRequested(); + try + { + if (!process.Start()) + { + throw new WorkloadException("The ASP.NET server process did not start.", ErrorReason.WorkloadFailed); + } + } + catch (System.ComponentModel.Win32Exception exception) + { + throw new WorkloadException( + $"The ASP.NET server process could not start (command={SensitiveData.ObscureSecrets(process.FullCommand())}).", + exception, + ErrorReason.WorkloadFailed); + } + + try + { + if (affinity is WindowsProcessAffinityConfiguration windowsAffinity) + { + process.ApplyAffinity(windowsAffinity); + } + + await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false); + } + finally + { + process.SafeKill(this.component.Logger, TimeSpan.FromSeconds(30)); + if (!process.HasExited) + { + throw new WorkloadException("The ASP.NET server process could not be stopped.", ErrorReason.WorkloadFailed); + } + + await this.component.LogProcessDetailsAsync(process, context, this.processLogName).ConfigureAwait(false); + } + + cancellationToken.ThrowIfCancellationRequested(); + process.ThrowIfWorkloadFailed(); + throw new WorkloadException("The ASP.NET server exited unexpectedly.", ErrorReason.WorkloadFailed); + } + } + catch (Exception exception) when (!(exception is OperationCanceledException && cancellationToken.IsCancellationRequested)) + { + this.component.Logger.LogMessage( + $"{this.component.TypeName}.StartServerInstanceError", + LogLevel.Error, + context.Clone().AddError(exception)); + + if (exception is OperationCanceledException) + { + throw new WorkloadException("The ASP.NET server was cancelled unexpectedly.", exception, ErrorReason.WorkloadFailed); + } + + throw; + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-BOMBARDIER.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-BOMBARDIER.json index a3ccce1208..004f8f1f9f 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-BOMBARDIER.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-ASPNET-BOMBARDIER.json @@ -28,7 +28,8 @@ "Role": "Client", "Scenario": "ExecuteJsonSerializationBenchmark", "PackageName": "bombardier", - "CommandArguments": "--duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://{ServerIp}:9876/json --print r --format json", + "ServerPort": "$.Parameters.ServerPort", + "CommandArguments": "--duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://{ServerIp}:{ServerPort}/json --print r --format json", "WarmUp": false, "Timeout": "00:05:00" } diff --git a/website/docs/workloads/aspnetbench/aspnetbench-profiles.md b/website/docs/workloads/aspnetbench/aspnetbench-profiles.md index 49926b0513..30815edfe4 100644 --- a/website/docs/workloads/aspnetbench/aspnetbench-profiles.md +++ b/website/docs/workloads/aspnetbench/aspnetbench-profiles.md @@ -173,7 +173,7 @@ with 256 concurrent connections. Uses .NET 8 SDK. |---------------------------|-------------------------------------------------------------------|---------------| | DotNetVersion | The version of the .NET SDK to download and install. | 8.0.204 | | TargetFramework | The .NET target framework to run. | net8.0 | - | ServerPort | The port the Kestrel server listens on. | 9876 | + | ServerPort | The port used by both Kestrel and Bombardier. | 9876 | * **Profile Runtimes** See the 'Metadata' section of the profile for estimated runtimes.