From 70c721a90ed108db9337a1fff40521a82eb8dcf5 Mon Sep 17 00:00:00 2001 From: "Calvin A. Allen" Date: Thu, 3 Sep 2026 12:39:20 -0400 Subject: [PATCH] feat(tools): add Test Explorer run, debug and stats tools Adds seven MCP tools over Test Explorer: test_run_all, test_debug_all, test_run, test_debug, test_cancel, test_status and test_stats. The extension cannot reference Microsoft.VisualStudio.TestWindow.Interfaces.dll at compile time. It is absent from the Microsoft.VisualStudio.SDK metapackage, nuget.org carries nothing newer than 11.0.61030 (2012), and $(DevEnvDir) is undefined under dotnet build, which is how this repo builds locally and in CI. TestExplorerInterop therefore resolves ITestExplorerStatsService and IOperationState by MEF contract name and reads them reflectively. The surface needed is one bool, one struct of four ints and one event, and it is identical in VS 2022 17.14 and VS 2026 18.0. Run tools start the run and return immediately, matching the non-blocking contract the build tools were corrected to in #96. Completion is observed by subscribing to IOperationState rather than by blocking, so the UI thread is never held; test_status reports the collapsed run state and current counts. test_stats distinguishes "Test Explorer not initialized" from "no tests", so an uninitialized window cannot be misread as a solution with zero tests. The *InContext commands act on the caret, so test_run and test_debug resolve the requested class or method through the workspace symbol search, open the file and position the caret before issuing the command. Test Explorer exposes no documented cancel command, so test_cancel probes candidate names and uses the first the running Visual Studio recognises. Commands are probed with Command.IsAvailable before execution, because ExecuteCommand throws when a command is disabled, which for Test Explorer is the normal state before discovery finishes. --- CLAUDE.md | 10 + README.md | 12 + .../Program.cs | 3 +- .../RpcClient.cs | 10 +- .../Tools/TestTools.cs | 93 +++++ .../Models/TestModels.cs | 64 ++++ .../RpcContracts.cs | 8 + .../TestExplorerInteropTests.cs | 266 +++++++++++++++ .../TestTargetSelectionTests.cs | 107 ++++++ .../TestWindowStandIns.cs | 55 +++ .../Services/IVisualStudioService.cs | 8 + .../Services/RpcServer.cs | 8 + .../Services/TestExplorerInterop.cs | 321 ++++++++++++++++++ .../Services/VisualStudioService.cs | 220 ++++++++++++ 14 files changed, 1183 insertions(+), 2 deletions(-) create mode 100644 src/CodingWithCalvin.MCPServer.Server/Tools/TestTools.cs create mode 100644 src/CodingWithCalvin.MCPServer.Shared/Models/TestModels.cs create mode 100644 src/CodingWithCalvin.MCPServer.Tests/TestExplorerInteropTests.cs create mode 100644 src/CodingWithCalvin.MCPServer.Tests/TestTargetSelectionTests.cs create mode 100644 src/CodingWithCalvin.MCPServer.Tests/TestWindowStandIns.cs create mode 100644 src/CodingWithCalvin.MCPServer/Services/TestExplorerInterop.cs diff --git a/CLAUDE.md b/CLAUDE.md index 6cb09ad..b6dbb20 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -124,6 +124,7 @@ dotnet build src/CodingWithCalvin.VSMCP/CodingWithCalvin.VSMCP.csproj - `Services/VisualStudioService.cs` - VS API wrapper for all VS operations - `Services/RpcServer.cs` - Named pipe JSON-RPC server - `Services/ServerProcessManager.cs` - Server process lifecycle management +- `Services/TestExplorerInterop.cs` - Reflection bridge to Test Explorer (see the file's remarks for why it cannot be a compile-time reference) - `Server/Tools/*.cs` - MCP tool definitions ## MCP Tools Available @@ -156,6 +157,15 @@ dotnet build src/CodingWithCalvin.VSMCP/CodingWithCalvin.VSMCP.csproj - `build_cancel` - Cancel build - `build_status` - Get build status +### Test Tools +- `test_run_all` - Run every test in the solution +- `test_debug_all` - Debug every test in the solution +- `test_run` - Run tests in a specific class or method +- `test_debug` - Debug tests in a specific class or method +- `test_cancel` - Cancel the test run in progress +- `test_status` - Get run state plus current counts +- `test_stats` - Get passed/failed/skipped/not-run counts + ## Technology Stack - .NET Framework 4.8 (VSIX) diff --git a/README.md b/README.md index 09bf98c..1acf91f 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,18 @@ | `output_read` | Read content from an Output window pane | | `output_write` | Write a message to an Output window pane | +### ๐Ÿงช Test Tools + +| Tool | Description | +|------|-------------| +| `test_cancel` | Cancel the test run in progress | +| `test_debug` | Debug the tests in a single class or method | +| `test_debug_all` | Debug every test in the solution | +| `test_run` | Run the tests in a single class or method | +| `test_run_all` | Run every test in the solution | +| `test_stats` | Get passed, failed, skipped, and not-run counts | +| `test_status` | Get the run state plus current counts | + ### ๐ŸชŸ Window Tools | Tool | Description | diff --git a/src/CodingWithCalvin.MCPServer.Server/Program.cs b/src/CodingWithCalvin.MCPServer.Server/Program.cs index fd48782..d410590 100644 --- a/src/CodingWithCalvin.MCPServer.Server/Program.cs +++ b/src/CodingWithCalvin.MCPServer.Server/Program.cs @@ -101,7 +101,8 @@ static async Task RunServerAsync(string pipeName, string host, int port, string .WithTools() .WithTools() .WithTools() - .WithTools(); + .WithTools() + .WithTools(); var app = builder.Build(); diff --git a/src/CodingWithCalvin.MCPServer.Server/RpcClient.cs b/src/CodingWithCalvin.MCPServer.Server/RpcClient.cs index 8cdf111..2cfcc61 100644 --- a/src/CodingWithCalvin.MCPServer.Server/RpcClient.cs +++ b/src/CodingWithCalvin.MCPServer.Server/RpcClient.cs @@ -65,7 +65,7 @@ public Task> GetAvailableToolsAsync() } var tools = new List(); - var toolTypes = new[] { typeof(Tools.SolutionTools), typeof(Tools.DocumentTools), typeof(Tools.BuildTools), typeof(Tools.NavigationTools), typeof(Tools.DebuggerTools), typeof(Tools.DiagnosticsTools), typeof(Tools.WindowTools) }; + var toolTypes = new[] { typeof(Tools.SolutionTools), typeof(Tools.DocumentTools), typeof(Tools.BuildTools), typeof(Tools.NavigationTools), typeof(Tools.DebuggerTools), typeof(Tools.DiagnosticsTools), typeof(Tools.WindowTools), typeof(Tools.TestTools) }; foreach (var toolType in toolTypes) { @@ -170,6 +170,14 @@ public Task WriteOutputPaneAsync(string paneIdentifier, string message, bo => Proxy.WriteOutputPaneAsync(paneIdentifier, message, activate); public Task> GetOutputPanesAsync() => Proxy.GetOutputPanesAsync(); + public Task RunAllTestsAsync() => Proxy.RunAllTestsAsync(); + public Task DebugAllTestsAsync() => Proxy.DebugAllTestsAsync(); + public Task RunTestsInContextAsync(string target, bool debug) + => Proxy.RunTestsInContextAsync(target, debug); + public Task CancelTestRunAsync() => Proxy.CancelTestRunAsync(); + public Task GetTestRunStatusAsync() => Proxy.GetTestRunStatusAsync(); + public Task GetTestStatsAsync() => Proxy.GetTestStatsAsync(); + public Task> GetWindowsAsync() => Proxy.GetWindowsAsync(); public Task ActivateWindowAsync(string caption) => Proxy.ActivateWindowAsync(caption); public Task ShowToolWindowAsync(string name) => Proxy.ShowToolWindowAsync(name); diff --git a/src/CodingWithCalvin.MCPServer.Server/Tools/TestTools.cs b/src/CodingWithCalvin.MCPServer.Server/Tools/TestTools.cs new file mode 100644 index 0000000..0372d09 --- /dev/null +++ b/src/CodingWithCalvin.MCPServer.Server/Tools/TestTools.cs @@ -0,0 +1,93 @@ +using System.ComponentModel; +using System.Text.Json; +using System.Threading.Tasks; +using ModelContextProtocol.Server; + +namespace CodingWithCalvin.MCPServer.Server.Tools; + +[McpServerToolType] +public class TestTools +{ + private const string StatsUnavailableMessage = + "Test Explorer has not been initialized in this Visual Studio session, so no counts are " + + "available. Open Test Explorer (window_show with 'TestExplorer') or start a run first. " + + "Note this is not the same as the solution having no tests."; + + private readonly RpcClient _rpcClient; + private readonly JsonSerializerOptions _jsonOptions; + + public TestTools(RpcClient rpcClient) + { + _rpcClient = rpcClient; + _jsonOptions = new JsonSerializerOptions { WriteIndented = true }; + } + + [McpServerTool(Name = "test_run_all", Destructive = false)] + [Description("Run every test in the solution via Test Explorer. The run starts asynchronously and this returns immediately; poll test_status to observe completion. Requires a solution with discovered tests.")] + public async Task RunAllTestsAsync() + { + var started = await _rpcClient.RunAllTestsAsync(); + return started + ? "Test run started. Poll test_status for progress." + : "Failed to start the test run. Test Explorer may still be discovering tests, or the solution has none."; + } + + [McpServerTool(Name = "test_debug_all", Destructive = false)] + [Description("Debug every test in the solution via Test Explorer, stopping on any breakpoints that are set. The run starts asynchronously and this returns immediately; poll test_status to observe completion.")] + public async Task DebugAllTestsAsync() + { + var started = await _rpcClient.DebugAllTestsAsync(); + return started + ? "Test debug run started. Poll test_status for progress." + : "Failed to start the test debug run. Test Explorer may still be discovering tests, or the solution has none."; + } + + [McpServerTool(Name = "test_run", Destructive = false)] + [Description("Run the tests in a single test class or test method. Accepts a simple name ('CalculatorTests', 'Add_ReturnsSum') or a fully qualified one ('MyApp.Tests.CalculatorTests.Add_ReturnsSum'); a fully qualified name is matched first and avoids ambiguity. The run starts asynchronously; poll test_status to observe completion.")] + public async Task RunTestAsync( + [Description("Test class or test method name to run. Fully qualified names are preferred when several types share a simple name.")] string target) + { + var result = await _rpcClient.RunTestsInContextAsync(target, debug: false); + return JsonSerializer.Serialize(result, _jsonOptions); + } + + [McpServerTool(Name = "test_debug", Destructive = false)] + [Description("Debug the tests in a single test class or test method, stopping on any breakpoints that are set. Accepts a simple or fully qualified name; a fully qualified name is matched first and avoids ambiguity. The run starts asynchronously; poll test_status to observe completion.")] + public async Task DebugTestAsync( + [Description("Test class or test method name to debug. Fully qualified names are preferred when several types share a simple name.")] string target) + { + var result = await _rpcClient.RunTestsInContextAsync(target, debug: true); + return JsonSerializer.Serialize(result, _jsonOptions); + } + + [McpServerTool(Name = "test_cancel", Destructive = false, Idempotent = true)] + [Description("Cancel the test run that is currently in progress.")] + public async Task CancelTestRunAsync() + { + var cancelled = await _rpcClient.CancelTestRunAsync(); + return cancelled + ? "Test run cancelled" + : "No test run is currently in progress"; + } + + [McpServerTool(Name = "test_status", ReadOnly = true)] + [Description("Get the Test Explorer run state plus current counts. State is one of 'NoRunObserved', 'Discovering', 'Running', 'Canceling', 'Completed', or 'Canceled'. Use this to poll for completion after test_run_all, test_run, or their debug equivalents. State is only tracked for runs started after this extension first contacted Test Explorer.")] + public async Task GetTestStatusAsync() + { + var status = await _rpcClient.GetTestRunStatusAsync(); + return JsonSerializer.Serialize(status, _jsonOptions); + } + + [McpServerTool(Name = "test_stats", ReadOnly = true)] + [Description("Get the passed, failed, skipped and not-run test counts from Test Explorer. Returns counts only - Visual Studio exposes no public API for the names of the individual failed or not-run tests.")] + public async Task GetTestStatsAsync() + { + var stats = await _rpcClient.GetTestStatsAsync(); + if (!stats.Available) + { + return StatsUnavailableMessage; + } + + return JsonSerializer.Serialize(stats, _jsonOptions); + } +} diff --git a/src/CodingWithCalvin.MCPServer.Shared/Models/TestModels.cs b/src/CodingWithCalvin.MCPServer.Shared/Models/TestModels.cs new file mode 100644 index 0000000..0e5f31e --- /dev/null +++ b/src/CodingWithCalvin.MCPServer.Shared/Models/TestModels.cs @@ -0,0 +1,64 @@ +namespace CodingWithCalvin.MCPServer.Shared.Models; + +/// +/// Aggregate test counts as reported by the Test Explorer window. +/// +public class TestStats +{ + /// + /// False when Test Explorer has not been initialized in this Visual Studio session. + /// The counts are meaningless when this is false, and must not be read as "no tests". + /// + public bool Available { get; set; } + + public int Passed { get; set; } + public int Failed { get; set; } + public int Skipped { get; set; } + public int NotRun { get; set; } +} + +/// +/// Test Explorer run state, plus the counts as of the most recent state change. +/// +public class TestRunStatus +{ + /// + /// One of "NoRunObserved", "Discovering", "Running", "Canceling", "Completed", or "Canceled". + /// + public string State { get; set; } = string.Empty; + + /// + /// True while discovery or execution is still in flight. + /// + public bool IsRunning { get; set; } + + /// + /// Raw Test Explorer operation name behind , retained for diagnostics. + /// Null until a state change has been observed. + /// + public string? LastOperation { get; set; } + + public TestStats Stats { get; set; } = new(); +} + +/// +/// Outcome of asking Test Explorer to start a run for a specific class or method. +/// +public class TestTargetResult +{ + public bool Started { get; set; } + + /// + /// Fully qualified name of the symbol the caret was placed on, when one was resolved. + /// + public string? ResolvedTarget { get; set; } + + public string? FilePath { get; set; } + public int Line { get; set; } + + /// + /// Populated when is false, or when the resolved target was + /// ambiguous and the first match was used. + /// + public string? Message { get; set; } +} diff --git a/src/CodingWithCalvin.MCPServer.Shared/RpcContracts.cs b/src/CodingWithCalvin.MCPServer.Shared/RpcContracts.cs index f21a96e..497b678 100644 --- a/src/CodingWithCalvin.MCPServer.Shared/RpcContracts.cs +++ b/src/CodingWithCalvin.MCPServer.Shared/RpcContracts.cs @@ -73,6 +73,14 @@ public interface IVisualStudioRpc Task WriteOutputPaneAsync(string paneIdentifier, string message, bool activate = false); Task> GetOutputPanesAsync(); + // Test Explorer tools + Task RunAllTestsAsync(); + Task DebugAllTestsAsync(); + Task RunTestsInContextAsync(string target, bool debug); + Task CancelTestRunAsync(); + Task GetTestRunStatusAsync(); + Task GetTestStatsAsync(); + // Window management tools Task> GetWindowsAsync(); Task ActivateWindowAsync(string caption); diff --git a/src/CodingWithCalvin.MCPServer.Tests/TestExplorerInteropTests.cs b/src/CodingWithCalvin.MCPServer.Tests/TestExplorerInteropTests.cs new file mode 100644 index 0000000..8eb6bf7 --- /dev/null +++ b/src/CodingWithCalvin.MCPServer.Tests/TestExplorerInteropTests.cs @@ -0,0 +1,266 @@ +using System; +using System.ComponentModel.Composition; +using System.ComponentModel.Composition.Hosting; +using System.ComponentModel.Composition.Primitives; +using CodingWithCalvin.MCPServer.Services; +using Microsoft.VisualStudio.ComponentModelHost; +using Microsoft.VisualStudio.TestWindow.Extensibility; +using Xunit; + +namespace CodingWithCalvin.MCPServer.Tests; + +/// +/// Covers the reflection bridge to Test Explorer. +/// +/// +/// Microsoft.VisualStudio.TestWindow.Interfaces.dll cannot be referenced at build time, so +/// resolves its services by MEF contract name and reads them +/// reflectively. These tests stand up a real MEF container holding stand-in parts declared in +/// the genuine namespace and shaped like the real ones, which exercises the contract-name +/// lookup, the interface discovery, the property reads and the explicitly implemented event +/// subscription rather than just the pure helpers. +/// +public class TestExplorerInteropTests +{ + [Fact] + public void GetStats_ReportsUnavailable_WhenComponentModelMissing() + { + var interop = new TestExplorerInterop(() => null); + + var stats = interop.GetStats(); + + Assert.False(stats.Available); + } + + [Fact] + public void GetStats_ReportsUnavailable_WhenTestExplorerNotRunning() + { + using var host = new StubHost(); + host.Stats.IsTestExplorerStatsServiceRunning = false; + host.Stats.Counts = new TestExplorerStats { PassedTestCount = 7 }; + + var stats = new TestExplorerInterop(host.Accessor).GetStats(); + + // Counts must not leak through as zeros that read like "solution has no tests". + Assert.False(stats.Available); + Assert.Equal(0, stats.Passed); + } + + [Fact] + public void GetStats_ReadsCounts_WhenTestExplorerRunning() + { + using var host = new StubHost(); + host.Stats.IsTestExplorerStatsServiceRunning = true; + host.Stats.Counts = new TestExplorerStats + { + PassedTestCount = 12, + FailedTestCount = 3, + SkippedTestCount = 1, + NotRunTestCount = 4 + }; + + var stats = new TestExplorerInterop(host.Accessor).GetStats(); + + Assert.True(stats.Available); + Assert.Equal(12, stats.Passed); + Assert.Equal(3, stats.Failed); + Assert.Equal(1, stats.Skipped); + Assert.Equal(4, stats.NotRun); + } + + [Fact] + public void GetRunStatus_ReportsNoRunObserved_BeforeAnyStateChange() + { + using var host = new StubHost(); + + var status = new TestExplorerInterop(host.Accessor).GetRunStatus(); + + Assert.Equal("NoRunObserved", status.State); + Assert.False(status.IsRunning); + Assert.Null(status.LastOperation); + } + + [Fact] + public void GetRunStatus_TracksExecutionThroughToCompletion() + { + using var host = new StubHost(); + var interop = new TestExplorerInterop(host.Accessor); + interop.EnsureTracking(); + + host.Operations.Raise(TestOperationStates.TestExecutionStarted); + var running = interop.GetRunStatus(); + + host.Operations.Raise(TestOperationStates.TestExecutionFinished); + var finished = interop.GetRunStatus(); + + Assert.Equal("Running", running.State); + Assert.True(running.IsRunning); + + Assert.Equal("Completed", finished.State); + Assert.False(finished.IsRunning); + Assert.Equal("TestExecutionFinished", finished.LastOperation); + } + + [Fact] + public void GetRunStatus_ReportsCanceled_AfterCancellation() + { + using var host = new StubHost(); + var interop = new TestExplorerInterop(host.Accessor); + interop.EnsureTracking(); + + host.Operations.Raise(TestOperationStates.TestExecutionStarted); + host.Operations.Raise(TestOperationStates.TestExecutionCancelAndFinished); + + Assert.Equal("Canceled", interop.GetRunStatus().State); + } + + [Fact] + public void GetRunStatus_KeepsRunOutcome_WhenDiscoveryRunsAfterwards() + { + using var host = new StubHost(); + var interop = new TestExplorerInterop(host.Accessor); + interop.EnsureTracking(); + + host.Operations.Raise(TestOperationStates.TestExecutionFinished); + host.Operations.Raise(TestOperationStates.DiscoveryFinished); + + // A discovery pass triggered by editing code must not erase the last run's outcome. + Assert.Equal("Completed", interop.GetRunStatus().State); + } + + [Fact] + public void EnsureTracking_SubscribesOnlyOnce() + { + using var host = new StubHost(); + var interop = new TestExplorerInterop(host.Accessor); + + interop.EnsureTracking(); + interop.EnsureTracking(); + interop.EnsureTracking(); + + Assert.Equal(1, host.Operations.SubscriberCount); + } + + [Fact] + public void EnsureTracking_DoesNotThrow_WhenComponentModelMissing() + { + var interop = new TestExplorerInterop(() => null); + + interop.EnsureTracking(); + + Assert.Equal("NoRunObserved", interop.GetRunStatus().State); + } + + [Theory] + [InlineData(null, null, "NoRunObserved")] + [InlineData("DiscoveryStarted", null, "Discovering")] + [InlineData("DiscoveryStarting", null, "Discovering")] + [InlineData("DiscoveryFinished", null, "NoRunObserved")] + [InlineData("TestExecutionStarted", "TestExecutionStarted", "Running")] + [InlineData("TestExecutionStarting", "TestExecutionStarting", "Running")] + [InlineData("TestExecutionCanceling", "TestExecutionCanceling", "Canceling")] + [InlineData("TestExecutionFinished", "TestExecutionFinished", "Completed")] + [InlineData("TestExecutionCancelAndFinished", "TestExecutionCancelAndFinished", "Canceled")] + [InlineData("DiscoveryStarted", "TestExecutionFinished", "Discovering")] + [InlineData("DiscoveryFinished", "TestExecutionFinished", "Completed")] + public void DeriveState_CollapsesOperationNames(string? last, string? lastExecution, string expected) + { + Assert.Equal(expected, TestExplorerInterop.DeriveState(last, lastExecution)); + } + + /// + /// A MEF container holding the stand-in Test Explorer parts, exposed through the + /// shape that consumes. + /// + private sealed class StubHost : IDisposable + { + private readonly CompositionContainer _container; + private readonly StubComponentModel _componentModel; + + internal StubHost() + { + _container = new CompositionContainer( + new TypeCatalog(typeof(StubStatsService), typeof(StubOperationState))); + _componentModel = new StubComponentModel(_container); + + Stats = (StubStatsService)_container.GetExportedValue(); + Operations = (StubOperationState)_container.GetExportedValue(); + } + + internal StubStatsService Stats { get; } + + internal StubOperationState Operations { get; } + + internal Func Accessor => () => _componentModel; + + public void Dispose() => _container.Dispose(); + } + + private sealed class StubComponentModel : IComponentModel + { + internal StubComponentModel(ExportProvider exportProvider) + { + DefaultExportProvider = exportProvider; + } + + public ExportProvider DefaultExportProvider { get; } + + public ComposablePartCatalog DefaultCatalog => throw new NotSupportedException(); + + public ICompositionService DefaultCompositionService => throw new NotSupportedException(); + +#pragma warning disable CS0618, CS0672 // Obsolete member is part of the interface being stubbed. + public ComposablePartCatalog GetCatalog(string catalogName) => throw new NotSupportedException(); +#pragma warning restore CS0618, CS0672 + + public System.Collections.Generic.IEnumerable GetExtensions() where T : class + => throw new NotSupportedException(); + + public T GetService() where T : class => throw new NotSupportedException(); + } + + [Export(typeof(ITestExplorerStatsService))] + [PartCreationPolicy(CreationPolicy.Shared)] + private sealed class StubStatsService : ITestExplorerStatsService + { + internal TestExplorerStats Counts { get; set; } + + public bool IsTestExplorerStatsServiceRunning { get; set; } + + TestExplorerStats ITestExplorerStatsService.TestExplorerStats => Counts; + + public event EventHandler? TestExplorerStatsChanged; + + internal void RaiseStatsChanged() => TestExplorerStatsChanged?.Invoke(this, Counts); + } + + /// + /// Implements StateChanged explicitly, matching the real OperationBroker, so the interop's + /// interface-based event lookup is genuinely exercised. + /// + [Export(typeof(IOperationState))] + [PartCreationPolicy(CreationPolicy.Shared)] + private sealed class StubOperationState : IOperationState + { + private EventHandler? _stateChanged; + + internal int SubscriberCount { get; private set; } + + event EventHandler IOperationState.StateChanged + { + add + { + _stateChanged += value; + SubscriberCount++; + } + remove + { + _stateChanged -= value; + SubscriberCount--; + } + } + + internal void Raise(TestOperationStates state) => + _stateChanged?.Invoke(this, new OperationStateChangedEventArgs { State = state }); + } +} diff --git a/src/CodingWithCalvin.MCPServer.Tests/TestTargetSelectionTests.cs b/src/CodingWithCalvin.MCPServer.Tests/TestTargetSelectionTests.cs new file mode 100644 index 0000000..de6fefa --- /dev/null +++ b/src/CodingWithCalvin.MCPServer.Tests/TestTargetSelectionTests.cs @@ -0,0 +1,107 @@ +using System.Collections.Generic; +using CodingWithCalvin.MCPServer.Services; +using CodingWithCalvin.MCPServer.Shared.Models; +using Xunit; + +namespace CodingWithCalvin.MCPServer.Tests; + +/// +/// Covers how a caller-supplied test name is resolved to the symbol the caret gets placed on. +/// Test Explorer's run-in-context commands act on the caret, so picking the wrong symbol here +/// silently runs the wrong tests rather than failing. +/// +public class TestTargetSelectionTests +{ + [Fact] + public void SelectTestTarget_PrefersExactFullNameOverSimpleName() + { + var candidates = new List + { + Symbol("CalculatorTests", "Other.CalculatorTests", SymbolKind.Class), + Symbol("CalculatorTests", "MyApp.Tests.CalculatorTests", SymbolKind.Class) + }; + + var selected = VisualStudioService.SelectTestTarget(candidates, "MyApp.Tests.CalculatorTests"); + + Assert.Equal("MyApp.Tests.CalculatorTests", selected?.FullName); + } + + [Fact] + public void SelectTestTarget_MatchesSimpleName() + { + var candidates = new List + { + Symbol("Helper", "MyApp.Tests.Helper", SymbolKind.Class), + Symbol("Add_ReturnsSum", "MyApp.Tests.CalculatorTests.Add_ReturnsSum", SymbolKind.Function) + }; + + var selected = VisualStudioService.SelectTestTarget(candidates, "Add_ReturnsSum"); + + Assert.Equal("MyApp.Tests.CalculatorTests.Add_ReturnsSum", selected?.FullName); + } + + [Fact] + public void SelectTestTarget_FallsBackToTrailingSegmentMatch() + { + var candidates = new List + { + Symbol("Unrelated", "MyApp.Tests.Unrelated", SymbolKind.Class), + Symbol("Add_ReturnsSum", "MyApp.Tests.CalculatorTests.Add_ReturnsSum", SymbolKind.Function) + }; + + var selected = VisualStudioService.SelectTestTarget( + candidates, + "CalculatorTests.Add_ReturnsSum"); + + Assert.Equal("MyApp.Tests.CalculatorTests.Add_ReturnsSum", selected?.FullName); + } + + [Fact] + public void SelectTestTarget_PrefersExactSimpleNameOverTrailingSegment() + { + var candidates = new List + { + Symbol("Nested", "MyApp.Tests.Outer.Nested", SymbolKind.Class), + Symbol("Nested", "MyApp.Tests.Nested", SymbolKind.Class) + }; + + var selected = VisualStudioService.SelectTestTarget(candidates, "Nested"); + + // Both end with ".Nested"; the exact simple-name rule runs first and takes the earlier + // candidate, so the result is deterministic rather than dependent on suffix ordering. + Assert.Equal("MyApp.Tests.Outer.Nested", selected?.FullName); + } + + [Fact] + public void SelectTestTarget_ReturnsNull_WhenNoCandidates() + { + var selected = VisualStudioService.SelectTestTarget(new List(), "Anything"); + + Assert.Null(selected); + } + + [Fact] + public void SelectTestTarget_FallsBackToFirstCandidate_WhenNothingMatches() + { + var candidates = new List + { + Symbol("CalculatorTests", "MyApp.Tests.CalculatorTests", SymbolKind.Class) + }; + + // The caller's text already matched during the workspace symbol search, so a candidate + // that fails the stricter rules here is still a better answer than nothing. + var selected = VisualStudioService.SelectTestTarget(candidates, "Calculator"); + + Assert.Equal("MyApp.Tests.CalculatorTests", selected?.FullName); + } + + private static SymbolInfo Symbol(string name, string fullName, SymbolKind kind) => new() + { + Name = name, + FullName = fullName, + Kind = kind, + FilePath = @"C:\src\Tests.cs", + StartLine = 10, + StartColumn = 5 + }; +} diff --git a/src/CodingWithCalvin.MCPServer.Tests/TestWindowStandIns.cs b/src/CodingWithCalvin.MCPServer.Tests/TestWindowStandIns.cs new file mode 100644 index 0000000..6c04cc9 --- /dev/null +++ b/src/CodingWithCalvin.MCPServer.Tests/TestWindowStandIns.cs @@ -0,0 +1,55 @@ +using System; + +// Stand-ins for the Test Explorer extensibility types, declared in the real namespace so that +// their full names match the MEF contract names TestExplorerInterop looks up. +// +// Microsoft.VisualStudio.TestWindow.Interfaces.dll ships only inside the Visual Studio +// installation: it is absent from the Microsoft.VisualStudio.SDK metapackage, nuget.org carries +// nothing newer than 11.0.61030 (2012), and $(DevEnvDir) is undefined under `dotnet build`. The +// production code therefore resolves these services by contract name and reads them +// reflectively, and these declarations let the tests drive that path without Visual Studio. +// +// Shapes are transcribed from the shipped assembly and are identical in VS 2022 17.14 and +// VS 2026 18.0. Only the members the interop touches are reproduced. +namespace Microsoft.VisualStudio.TestWindow.Extensibility; + +public struct TestExplorerStats +{ + public int PassedTestCount { get; set; } + public int FailedTestCount { get; set; } + public int SkippedTestCount { get; set; } + public int NotRunTestCount { get; set; } +} + +public interface ITestExplorerStatsService +{ + TestExplorerStats TestExplorerStats { get; } + bool IsTestExplorerStatsServiceRunning { get; } + event EventHandler TestExplorerStatsChanged; +} + +public enum TestOperationStates +{ + None = 0x00000000, + Discovery = 0x00010000, + DiscoveryStarting = 0x00010008, + DiscoveryStarted = 0x00010001, + DiscoveryFinished = 0x00010004, + DiscoveryCanceled = 0x00010006, + TestExecution = 0x00020000, + TestExecutionStarting = 0x00020008, + TestExecutionStarted = 0x00020001, + TestExecutionCanceling = 0x00020003, + TestExecutionFinished = 0x00020004, + TestExecutionCancelAndFinished = 0x00020006 +} + +public class OperationStateChangedEventArgs : EventArgs +{ + public TestOperationStates State { get; set; } +} + +public interface IOperationState +{ + event EventHandler StateChanged; +} diff --git a/src/CodingWithCalvin.MCPServer/Services/IVisualStudioService.cs b/src/CodingWithCalvin.MCPServer/Services/IVisualStudioService.cs index 828be40..67fd4dc 100644 --- a/src/CodingWithCalvin.MCPServer/Services/IVisualStudioService.cs +++ b/src/CodingWithCalvin.MCPServer/Services/IVisualStudioService.cs @@ -68,6 +68,14 @@ public interface IVisualStudioService Task WriteOutputPaneAsync(string paneIdentifier, string message, bool activate = false); Task> GetOutputPanesAsync(); + // Test Explorer tools + Task RunAllTestsAsync(); + Task DebugAllTestsAsync(); + Task RunTestsInContextAsync(string target, bool debug); + Task CancelTestRunAsync(); + Task GetTestRunStatusAsync(); + Task GetTestStatsAsync(); + Task> GetWindowsAsync(); Task ActivateWindowAsync(string caption); Task ShowToolWindowAsync(string name); diff --git a/src/CodingWithCalvin.MCPServer/Services/RpcServer.cs b/src/CodingWithCalvin.MCPServer/Services/RpcServer.cs index 0d79ddd..104401d 100644 --- a/src/CodingWithCalvin.MCPServer/Services/RpcServer.cs +++ b/src/CodingWithCalvin.MCPServer/Services/RpcServer.cs @@ -279,6 +279,14 @@ public Task WriteOutputPaneAsync(string paneIdentifier, string message, bo => _vsService.WriteOutputPaneAsync(paneIdentifier, message, activate); public Task> GetOutputPanesAsync() => _vsService.GetOutputPanesAsync(); + public Task RunAllTestsAsync() => _vsService.RunAllTestsAsync(); + public Task DebugAllTestsAsync() => _vsService.DebugAllTestsAsync(); + public Task RunTestsInContextAsync(string target, bool debug) + => _vsService.RunTestsInContextAsync(target, debug); + public Task CancelTestRunAsync() => _vsService.CancelTestRunAsync(); + public Task GetTestRunStatusAsync() => _vsService.GetTestRunStatusAsync(); + public Task GetTestStatsAsync() => _vsService.GetTestStatsAsync(); + public Task> GetWindowsAsync() => _vsService.GetWindowsAsync(); public Task ActivateWindowAsync(string caption) => _vsService.ActivateWindowAsync(caption); public Task ShowToolWindowAsync(string name) => _vsService.ShowToolWindowAsync(name); diff --git a/src/CodingWithCalvin.MCPServer/Services/TestExplorerInterop.cs b/src/CodingWithCalvin.MCPServer/Services/TestExplorerInterop.cs new file mode 100644 index 0000000..fe4c0e7 --- /dev/null +++ b/src/CodingWithCalvin.MCPServer/Services/TestExplorerInterop.cs @@ -0,0 +1,321 @@ +using System; +using System.ComponentModel.Composition.Primitives; +using System.Linq; +using System.Reflection; +using CodingWithCalvin.MCPServer.Shared.Models; +using CodingWithCalvin.Otel4Vsix; +using Microsoft.VisualStudio.ComponentModelHost; + +namespace CodingWithCalvin.MCPServer.Services; + +/// +/// Reflection bridge to the Test Explorer extensibility services. +/// +/// +/// +/// The types used here live in Microsoft.VisualStudio.TestWindow.Interfaces.dll, which +/// ships only inside the Visual Studio installation. It cannot be referenced at compile time: +/// nuget.org carries nothing newer than 11.0.61030 (2012), it is absent from the +/// Microsoft.VisualStudio.SDK metapackage, and $(DevEnvDir) is undefined under +/// dotnet build โ€” which is how this repo builds both locally and in CI โ€” so a HintPath +/// would break the build outright. +/// +/// +/// The surface needed is small and stable: one bool, one struct of four ints, and one event. +/// It is byte-identical between VS 2022 17.14 and VS 2026 18.0, the range the VSIX manifest +/// targets. Both services are MEF parts (TestExplorerStatsService exports +/// ITestExplorerStatsService, OperationBroker exports IOperationState), so +/// they are resolved by contract name and read reflectively. +/// +/// +/// Every failure path degrades to "unavailable" rather than throwing. Test Explorer may never +/// have been opened, and on a future Visual Studio the shape could change; neither should take +/// an MCP tool call down with it. +/// +/// +internal sealed class TestExplorerInterop +{ + private const string StatsServiceContract = + "Microsoft.VisualStudio.TestWindow.Extensibility.ITestExplorerStatsService"; + private const string OperationStateContract = + "Microsoft.VisualStudio.TestWindow.Extensibility.IOperationState"; + + private const string DiscoveryPrefix = "Discovery"; + private const string ExecutionPrefix = "TestExecution"; + + private readonly object _gate = new object(); + private readonly Func _componentModelAccessor; + + private bool _statsResolved; + private object? _statsService; + private PropertyInfo? _isRunningProperty; + private PropertyInfo? _statsProperty; + private PropertyInfo? _passedProperty; + private PropertyInfo? _failedProperty; + private PropertyInfo? _skippedProperty; + private PropertyInfo? _notRunProperty; + + private bool _trackingAttempted; + private string? _lastOperation; + private string? _lastExecutionOperation; + + internal TestExplorerInterop(Func componentModelAccessor) + { + _componentModelAccessor = componentModelAccessor; + } + + /// + /// Reads the current Test Explorer counts. Returns false + /// when Test Explorer has not been initialized, so that an uninitialized window is not + /// reported as a solution with zero tests. + /// + internal TestStats GetStats() + { + try + { + lock (_gate) + { + if (!TryResolveStatsService()) + { + return new TestStats(); + } + + if (_isRunningProperty!.GetValue(_statsService) is not true) + { + return new TestStats(); + } + + var stats = _statsProperty!.GetValue(_statsService); + if (stats == null) + { + return new TestStats(); + } + + return new TestStats + { + Available = true, + Passed = ReadCount(_passedProperty, stats), + Failed = ReadCount(_failedProperty, stats), + Skipped = ReadCount(_skippedProperty, stats), + NotRun = ReadCount(_notRunProperty, stats) + }; + } + } + catch (Exception ex) + { + VsixTelemetry.TrackException(ex); + return new TestStats(); + } + } + + /// + /// Reports the tracked run state alongside the current counts. State is only observed from + /// the point first succeeds, so a run started outside this + /// extension before then reads as "NoRunObserved". + /// + internal TestRunStatus GetRunStatus() + { + EnsureTracking(); + + string? lastOperation; + string? lastExecution; + + lock (_gate) + { + lastOperation = _lastOperation; + lastExecution = _lastExecutionOperation; + } + + var state = DeriveState(lastOperation, lastExecution); + + return new TestRunStatus + { + State = state, + IsRunning = state is "Discovering" or "Running" or "Canceling", + LastOperation = lastOperation, + Stats = GetStats() + }; + } + + /// + /// Subscribes to Test Explorer operation state changes. Safe to call repeatedly; the + /// subscription is attempted once and the outcome cached either way. + /// + internal void EnsureTracking() + { + lock (_gate) + { + if (_trackingAttempted) + { + return; + } + + _trackingAttempted = true; + + try + { + var operationState = ResolveExport(OperationStateContract); + if (operationState == null) + { + return; + } + + // IOperationState.StateChanged is implemented explicitly on OperationBroker, so + // the event has to be reached through the interface rather than the class. + var contract = FindInterface(operationState, OperationStateContract); + var stateChanged = contract?.GetEvent("StateChanged"); + if (stateChanged?.EventHandlerType == null) + { + return; + } + + var callback = typeof(TestExplorerInterop).GetMethod( + nameof(OnOperationStateChanged), + BindingFlags.Instance | BindingFlags.NonPublic); + if (callback == null) + { + return; + } + + // The event is EventHandler; the callback takes + // the EventArgs base. Delegate creation allows that contravariance. + var handler = Delegate.CreateDelegate( + stateChanged.EventHandlerType, + this, + callback, + throwOnBindFailure: false); + if (handler == null) + { + return; + } + + stateChanged.AddEventHandler(operationState, handler); + } + catch (Exception ex) + { + VsixTelemetry.TrackException(ex); + } + } + } + + private void OnOperationStateChanged(object sender, EventArgs args) + { + try + { + var operation = args.GetType().GetProperty("State")?.GetValue(args)?.ToString(); + if (string.IsNullOrEmpty(operation)) + { + return; + } + + lock (_gate) + { + _lastOperation = operation; + + if (operation!.StartsWith(ExecutionPrefix, StringComparison.Ordinal)) + { + _lastExecutionOperation = operation; + } + } + } + catch (Exception ex) + { + VsixTelemetry.TrackException(ex); + } + } + + /// + /// Collapses the raw Test Explorer operation names into the small state set the tools + /// report. Discovery is only surfaced while it is in flight, so that a discovery pass + /// triggered after a run does not erase the outcome of that run. + /// + internal static string DeriveState(string? lastOperation, string? lastExecutionOperation) + { + if (lastOperation != null + && lastOperation.StartsWith(DiscoveryPrefix, StringComparison.Ordinal) + && IsInFlight(lastOperation)) + { + return "Discovering"; + } + + return lastExecutionOperation switch + { + null => "NoRunObserved", + "TestExecutionFinished" => "Completed", + "TestExecutionCancelAndFinished" => "Canceled", + "TestExecutionCanceling" => "Canceling", + _ => "Running" + }; + } + + private static bool IsInFlight(string operation) => + !operation.EndsWith("Finished", StringComparison.Ordinal) + && !operation.EndsWith("Canceled", StringComparison.Ordinal); + + private static int ReadCount(PropertyInfo? property, object stats) => + property?.GetValue(stats) is int value ? value : 0; + + private bool TryResolveStatsService() + { + if (_statsResolved) + { + return _statsService != null; + } + + _statsResolved = true; + + var service = ResolveExport(StatsServiceContract); + if (service == null) + { + return false; + } + + var contract = FindInterface(service, StatsServiceContract); + if (contract == null) + { + return false; + } + + _isRunningProperty = contract.GetProperty("IsTestExplorerStatsServiceRunning"); + _statsProperty = contract.GetProperty("TestExplorerStats"); + if (_isRunningProperty == null || _statsProperty == null) + { + return false; + } + + var statsType = _statsProperty.PropertyType; + _passedProperty = statsType.GetProperty("PassedTestCount"); + _failedProperty = statsType.GetProperty("FailedTestCount"); + _skippedProperty = statsType.GetProperty("SkippedTestCount"); + _notRunProperty = statsType.GetProperty("NotRunTestCount"); + + _statsService = service; + return true; + } + + private object? ResolveExport(string contractName) + { + var componentModel = _componentModelAccessor(); + if (componentModel == null) + { + return null; + } + + var definition = new ImportDefinition( + d => d.ContractName == contractName, + contractName, + ImportCardinality.ZeroOrMore, + isRecomposable: false, + isPrerequisite: false); + + return componentModel.DefaultExportProvider + .GetExports(definition) + .FirstOrDefault() + ?.Value; + } + + private static Type? FindInterface(object instance, string fullName) => + instance.GetType() + .GetInterfaces() + .FirstOrDefault(i => string.Equals(i.FullName, fullName, StringComparison.Ordinal)); +} diff --git a/src/CodingWithCalvin.MCPServer/Services/VisualStudioService.cs b/src/CodingWithCalvin.MCPServer/Services/VisualStudioService.cs index 15584b8..588689e 100644 --- a/src/CodingWithCalvin.MCPServer/Services/VisualStudioService.cs +++ b/src/CodingWithCalvin.MCPServer/Services/VisualStudioService.cs @@ -2587,4 +2587,224 @@ public static IReadOnlyCollection GetSupportedToolWindowNames() { return ToolWindowCommands.Keys; } + + /// + /// Test Explorer exposes no documented cancel command, so the first candidate that the + /// running Visual Studio actually recognises is used. + /// + private static readonly string[] CancelTestRunCommands = + { + "TestExplorer.CancelTestRun", + "TestExplorer.CancelTests" + }; + + private TestExplorerInterop? _testExplorer; + + private TestExplorerInterop TestExplorer => + _testExplorer ??= new TestExplorerInterop( + () => ServiceProvider.GetService(typeof(SComponentModel)) as IComponentModel); + + public async Task RunAllTestsAsync() + { + using var activity = VsixTelemetry.Tracer.StartActivity("RunAllTests"); + return await StartTestRunAsync("TestExplorer.RunAllTests", activity); + } + + public async Task DebugAllTestsAsync() + { + using var activity = VsixTelemetry.Tracer.StartActivity("DebugAllTests"); + return await StartTestRunAsync("TestExplorer.DebugAllTests", activity); + } + + private async Task StartTestRunAsync(string command, Activity? activity) + { + await ThreadHelper.JoinableTaskFactory.SwitchToMainThreadAsync(); + var dte = await GetDteAsync(); + + // Subscribe before issuing the command, otherwise the run's own state changes are + // missed and test_status reports NoRunObserved for a run that is already going. + TestExplorer.EnsureTracking(); + + try + { + if (!IsCommandAvailable(dte, command)) + { + return false; + } + + dte.ExecuteCommand(command); + return true; + } + catch (Exception ex) + { + activity?.SetStatus(ActivityStatusCode.Error, ex.Message); + activity?.RecordException(ex); + return false; + } + } + + public async Task RunTestsInContextAsync(string target, bool debug) + { + using var activity = VsixTelemetry.Tracer.StartActivity( + debug ? "DebugTestsInContext" : "RunTestsInContext"); + + if (string.IsNullOrWhiteSpace(target)) + { + return new TestTargetResult { Message = "A class or method name is required." }; + } + + await ThreadHelper.JoinableTaskFactory.SwitchToMainThreadAsync(); + var dte = await GetDteAsync(); + + try + { + var search = await SearchWorkspaceSymbolsAsync(target.Trim()); + var candidates = search.Symbols + .Where(s => s.Kind == SymbolKind.Class || s.Kind == SymbolKind.Function) + .ToList(); + + var symbol = SelectTestTarget(candidates, target.Trim()); + if (symbol == null) + { + return new TestTargetResult + { + Message = $"No class or method named '{target}' was found in the solution." + }; + } + + // The *InContext commands act on wherever the caret currently is, so the target has + // to be opened and the caret placed on it before the command is issued. + if (!await OpenDocumentAsync(symbol.FilePath)) + { + return new TestTargetResult + { + ResolvedTarget = symbol.FullName, + FilePath = symbol.FilePath, + Line = symbol.StartLine, + Message = $"Could not open '{symbol.FilePath}' to position the caret." + }; + } + + if (!await SetSelectionAsync( + symbol.FilePath, + symbol.StartLine, + symbol.StartColumn, + symbol.StartLine, + symbol.StartColumn)) + { + return new TestTargetResult + { + ResolvedTarget = symbol.FullName, + FilePath = symbol.FilePath, + Line = symbol.StartLine, + Message = $"Could not position the caret on '{symbol.FullName}'." + }; + } + + var command = debug + ? "TestExplorer.DebugAllTestsInContext" + : "TestExplorer.RunAllTestsInContext"; + + TestExplorer.EnsureTracking(); + + if (!IsCommandAvailable(dte, command)) + { + return new TestTargetResult + { + ResolvedTarget = symbol.FullName, + FilePath = symbol.FilePath, + Line = symbol.StartLine, + Message = $"'{command}' is unavailable. Test Explorer may still be discovering tests." + }; + } + + dte.ExecuteCommand(command); + + return new TestTargetResult + { + Started = true, + ResolvedTarget = symbol.FullName, + FilePath = symbol.FilePath, + Line = symbol.StartLine, + Message = candidates.Count > 1 + ? $"{candidates.Count} symbols matched '{target}'; used '{symbol.FullName}'." + : null + }; + } + catch (Exception ex) + { + activity?.SetStatus(ActivityStatusCode.Error, ex.Message); + activity?.RecordException(ex); + return new TestTargetResult { Message = ex.Message }; + } + } + + /// + /// Picks the best match for a caller-supplied class or method name, preferring an exact + /// fully qualified match over an exact simple name over a trailing-segment match. + /// + internal static SymbolInfo? SelectTestTarget(IReadOnlyList candidates, string target) + { + return candidates.FirstOrDefault(s => string.Equals(s.FullName, target, StringComparison.Ordinal)) + ?? candidates.FirstOrDefault(s => string.Equals(s.Name, target, StringComparison.Ordinal)) + ?? candidates.FirstOrDefault(s => s.FullName.EndsWith("." + target, StringComparison.Ordinal)) + ?? candidates.FirstOrDefault(); + } + + public async Task CancelTestRunAsync() + { + await ThreadHelper.JoinableTaskFactory.SwitchToMainThreadAsync(); + var dte = await GetDteAsync(); + + foreach (var command in CancelTestRunCommands) + { + try + { + if (!IsCommandAvailable(dte, command)) + { + continue; + } + + dte.ExecuteCommand(command); + return true; + } + catch (Exception ex) + { + VsixTelemetry.TrackException(ex); + } + } + + return false; + } + + public async Task GetTestRunStatusAsync() + { + await ThreadHelper.JoinableTaskFactory.SwitchToMainThreadAsync(); + return TestExplorer.GetRunStatus(); + } + + public async Task GetTestStatsAsync() + { + await ThreadHelper.JoinableTaskFactory.SwitchToMainThreadAsync(); + return TestExplorer.GetStats(); + } + + /// + /// ExecuteCommand throws when a command is disabled or unknown, which for Test Explorer is + /// the normal state before discovery finishes. Probing first keeps that an ordinary result + /// instead of an exception. + /// + private static bool IsCommandAvailable(DTE2 dte, string command) + { + ThreadHelper.ThrowIfNotOnUIThread(); + + try + { + return dte.Commands.Item(command)?.IsAvailable == true; + } + catch (Exception) + { + return false; + } + } }