diff --git a/CLAUDE.md b/CLAUDE.md index 6cb09ad..b6dbb20 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -124,6 +124,7 @@ dotnet build src/CodingWithCalvin.VSMCP/CodingWithCalvin.VSMCP.csproj - `Services/VisualStudioService.cs` - VS API wrapper for all VS operations - `Services/RpcServer.cs` - Named pipe JSON-RPC server - `Services/ServerProcessManager.cs` - Server process lifecycle management +- `Services/TestExplorerInterop.cs` - Reflection bridge to Test Explorer (see the file's remarks for why it cannot be a compile-time reference) - `Server/Tools/*.cs` - MCP tool definitions ## MCP Tools Available @@ -156,6 +157,15 @@ dotnet build src/CodingWithCalvin.VSMCP/CodingWithCalvin.VSMCP.csproj - `build_cancel` - Cancel build - `build_status` - Get build status +### Test Tools +- `test_run_all` - Run every test in the solution +- `test_debug_all` - Debug every test in the solution +- `test_run` - Run tests in a specific class or method +- `test_debug` - Debug tests in a specific class or method +- `test_cancel` - Cancel the test run in progress +- `test_status` - Get run state plus current counts +- `test_stats` - Get passed/failed/skipped/not-run counts + ## Technology Stack - .NET Framework 4.8 (VSIX) diff --git a/README.md b/README.md index 09bf98c..1acf91f 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,18 @@ | `output_read` | Read content from an Output window pane | | `output_write` | Write a message to an Output window pane | +### ๐Ÿงช Test Tools + +| Tool | Description | +|------|-------------| +| `test_cancel` | Cancel the test run in progress | +| `test_debug` | Debug the tests in a single class or method | +| `test_debug_all` | Debug every test in the solution | +| `test_run` | Run the tests in a single class or method | +| `test_run_all` | Run every test in the solution | +| `test_stats` | Get passed, failed, skipped, and not-run counts | +| `test_status` | Get the run state plus current counts | + ### ๐ŸชŸ Window Tools | Tool | Description | diff --git a/src/CodingWithCalvin.MCPServer.Server/Program.cs b/src/CodingWithCalvin.MCPServer.Server/Program.cs index fd48782..d410590 100644 --- a/src/CodingWithCalvin.MCPServer.Server/Program.cs +++ b/src/CodingWithCalvin.MCPServer.Server/Program.cs @@ -101,7 +101,8 @@ static async Task RunServerAsync(string pipeName, string host, int port, string .WithTools() .WithTools() .WithTools() - .WithTools(); + .WithTools() + .WithTools(); var app = builder.Build(); diff --git a/src/CodingWithCalvin.MCPServer.Server/RpcClient.cs b/src/CodingWithCalvin.MCPServer.Server/RpcClient.cs index 8cdf111..2cfcc61 100644 --- a/src/CodingWithCalvin.MCPServer.Server/RpcClient.cs +++ b/src/CodingWithCalvin.MCPServer.Server/RpcClient.cs @@ -65,7 +65,7 @@ public Task> GetAvailableToolsAsync() } var tools = new List(); - var toolTypes = new[] { typeof(Tools.SolutionTools), typeof(Tools.DocumentTools), typeof(Tools.BuildTools), typeof(Tools.NavigationTools), typeof(Tools.DebuggerTools), typeof(Tools.DiagnosticsTools), typeof(Tools.WindowTools) }; + var toolTypes = new[] { typeof(Tools.SolutionTools), typeof(Tools.DocumentTools), typeof(Tools.BuildTools), typeof(Tools.NavigationTools), typeof(Tools.DebuggerTools), typeof(Tools.DiagnosticsTools), typeof(Tools.WindowTools), typeof(Tools.TestTools) }; foreach (var toolType in toolTypes) { @@ -170,6 +170,14 @@ public Task WriteOutputPaneAsync(string paneIdentifier, string message, bo => Proxy.WriteOutputPaneAsync(paneIdentifier, message, activate); public Task> GetOutputPanesAsync() => Proxy.GetOutputPanesAsync(); + public Task RunAllTestsAsync() => Proxy.RunAllTestsAsync(); + public Task DebugAllTestsAsync() => Proxy.DebugAllTestsAsync(); + public Task RunTestsInContextAsync(string target, bool debug) + => Proxy.RunTestsInContextAsync(target, debug); + public Task CancelTestRunAsync() => Proxy.CancelTestRunAsync(); + public Task GetTestRunStatusAsync() => Proxy.GetTestRunStatusAsync(); + public Task GetTestStatsAsync() => Proxy.GetTestStatsAsync(); + public Task> GetWindowsAsync() => Proxy.GetWindowsAsync(); public Task ActivateWindowAsync(string caption) => Proxy.ActivateWindowAsync(caption); public Task ShowToolWindowAsync(string name) => Proxy.ShowToolWindowAsync(name); diff --git a/src/CodingWithCalvin.MCPServer.Server/Tools/TestTools.cs b/src/CodingWithCalvin.MCPServer.Server/Tools/TestTools.cs new file mode 100644 index 0000000..0372d09 --- /dev/null +++ b/src/CodingWithCalvin.MCPServer.Server/Tools/TestTools.cs @@ -0,0 +1,93 @@ +using System.ComponentModel; +using System.Text.Json; +using System.Threading.Tasks; +using ModelContextProtocol.Server; + +namespace CodingWithCalvin.MCPServer.Server.Tools; + +[McpServerToolType] +public class TestTools +{ + private const string StatsUnavailableMessage = + "Test Explorer has not been initialized in this Visual Studio session, so no counts are " + + "available. Open Test Explorer (window_show with 'TestExplorer') or start a run first. " + + "Note this is not the same as the solution having no tests."; + + private readonly RpcClient _rpcClient; + private readonly JsonSerializerOptions _jsonOptions; + + public TestTools(RpcClient rpcClient) + { + _rpcClient = rpcClient; + _jsonOptions = new JsonSerializerOptions { WriteIndented = true }; + } + + [McpServerTool(Name = "test_run_all", Destructive = false)] + [Description("Run every test in the solution via Test Explorer. The run starts asynchronously and this returns immediately; poll test_status to observe completion. Requires a solution with discovered tests.")] + public async Task RunAllTestsAsync() + { + var started = await _rpcClient.RunAllTestsAsync(); + return started + ? "Test run started. Poll test_status for progress." + : "Failed to start the test run. Test Explorer may still be discovering tests, or the solution has none."; + } + + [McpServerTool(Name = "test_debug_all", Destructive = false)] + [Description("Debug every test in the solution via Test Explorer, stopping on any breakpoints that are set. The run starts asynchronously and this returns immediately; poll test_status to observe completion.")] + public async Task DebugAllTestsAsync() + { + var started = await _rpcClient.DebugAllTestsAsync(); + return started + ? "Test debug run started. Poll test_status for progress." + : "Failed to start the test debug run. Test Explorer may still be discovering tests, or the solution has none."; + } + + [McpServerTool(Name = "test_run", Destructive = false)] + [Description("Run the tests in a single test class or test method. Accepts a simple name ('CalculatorTests', 'Add_ReturnsSum') or a fully qualified one ('MyApp.Tests.CalculatorTests.Add_ReturnsSum'); a fully qualified name is matched first and avoids ambiguity. The run starts asynchronously; poll test_status to observe completion.")] + public async Task RunTestAsync( + [Description("Test class or test method name to run. Fully qualified names are preferred when several types share a simple name.")] string target) + { + var result = await _rpcClient.RunTestsInContextAsync(target, debug: false); + return JsonSerializer.Serialize(result, _jsonOptions); + } + + [McpServerTool(Name = "test_debug", Destructive = false)] + [Description("Debug the tests in a single test class or test method, stopping on any breakpoints that are set. Accepts a simple or fully qualified name; a fully qualified name is matched first and avoids ambiguity. The run starts asynchronously; poll test_status to observe completion.")] + public async Task DebugTestAsync( + [Description("Test class or test method name to debug. Fully qualified names are preferred when several types share a simple name.")] string target) + { + var result = await _rpcClient.RunTestsInContextAsync(target, debug: true); + return JsonSerializer.Serialize(result, _jsonOptions); + } + + [McpServerTool(Name = "test_cancel", Destructive = false, Idempotent = true)] + [Description("Cancel the test run that is currently in progress.")] + public async Task CancelTestRunAsync() + { + var cancelled = await _rpcClient.CancelTestRunAsync(); + return cancelled + ? "Test run cancelled" + : "No test run is currently in progress"; + } + + [McpServerTool(Name = "test_status", ReadOnly = true)] + [Description("Get the Test Explorer run state plus current counts. State is one of 'NoRunObserved', 'Discovering', 'Running', 'Canceling', 'Completed', or 'Canceled'. Use this to poll for completion after test_run_all, test_run, or their debug equivalents. State is only tracked for runs started after this extension first contacted Test Explorer.")] + public async Task GetTestStatusAsync() + { + var status = await _rpcClient.GetTestRunStatusAsync(); + return JsonSerializer.Serialize(status, _jsonOptions); + } + + [McpServerTool(Name = "test_stats", ReadOnly = true)] + [Description("Get the passed, failed, skipped and not-run test counts from Test Explorer. Returns counts only - Visual Studio exposes no public API for the names of the individual failed or not-run tests.")] + public async Task GetTestStatsAsync() + { + var stats = await _rpcClient.GetTestStatsAsync(); + if (!stats.Available) + { + return StatsUnavailableMessage; + } + + return JsonSerializer.Serialize(stats, _jsonOptions); + } +} diff --git a/src/CodingWithCalvin.MCPServer.Shared/Models/TestModels.cs b/src/CodingWithCalvin.MCPServer.Shared/Models/TestModels.cs new file mode 100644 index 0000000..0e5f31e --- /dev/null +++ b/src/CodingWithCalvin.MCPServer.Shared/Models/TestModels.cs @@ -0,0 +1,64 @@ +namespace CodingWithCalvin.MCPServer.Shared.Models; + +/// +/// Aggregate test counts as reported by the Test Explorer window. +/// +public class TestStats +{ + /// + /// False when Test Explorer has not been initialized in this Visual Studio session. + /// The counts are meaningless when this is false, and must not be read as "no tests". + /// + public bool Available { get; set; } + + public int Passed { get; set; } + public int Failed { get; set; } + public int Skipped { get; set; } + public int NotRun { get; set; } +} + +/// +/// Test Explorer run state, plus the counts as of the most recent state change. +/// +public class TestRunStatus +{ + /// + /// One of "NoRunObserved", "Discovering", "Running", "Canceling", "Completed", or "Canceled". + /// + public string State { get; set; } = string.Empty; + + /// + /// True while discovery or execution is still in flight. + /// + public bool IsRunning { get; set; } + + /// + /// Raw Test Explorer operation name behind , retained for diagnostics. + /// Null until a state change has been observed. + /// + public string? LastOperation { get; set; } + + public TestStats Stats { get; set; } = new(); +} + +/// +/// Outcome of asking Test Explorer to start a run for a specific class or method. +/// +public class TestTargetResult +{ + public bool Started { get; set; } + + /// + /// Fully qualified name of the symbol the caret was placed on, when one was resolved. + /// + public string? ResolvedTarget { get; set; } + + public string? FilePath { get; set; } + public int Line { get; set; } + + /// + /// Populated when is false, or when the resolved target was + /// ambiguous and the first match was used. + /// + public string? Message { get; set; } +} diff --git a/src/CodingWithCalvin.MCPServer.Shared/RpcContracts.cs b/src/CodingWithCalvin.MCPServer.Shared/RpcContracts.cs index f21a96e..497b678 100644 --- a/src/CodingWithCalvin.MCPServer.Shared/RpcContracts.cs +++ b/src/CodingWithCalvin.MCPServer.Shared/RpcContracts.cs @@ -73,6 +73,14 @@ public interface IVisualStudioRpc Task WriteOutputPaneAsync(string paneIdentifier, string message, bool activate = false); Task> GetOutputPanesAsync(); + // Test Explorer tools + Task RunAllTestsAsync(); + Task DebugAllTestsAsync(); + Task RunTestsInContextAsync(string target, bool debug); + Task CancelTestRunAsync(); + Task GetTestRunStatusAsync(); + Task GetTestStatsAsync(); + // Window management tools Task> GetWindowsAsync(); Task ActivateWindowAsync(string caption); diff --git a/src/CodingWithCalvin.MCPServer.Tests/TestExplorerInteropTests.cs b/src/CodingWithCalvin.MCPServer.Tests/TestExplorerInteropTests.cs new file mode 100644 index 0000000..8eb6bf7 --- /dev/null +++ b/src/CodingWithCalvin.MCPServer.Tests/TestExplorerInteropTests.cs @@ -0,0 +1,266 @@ +using System; +using System.ComponentModel.Composition; +using System.ComponentModel.Composition.Hosting; +using System.ComponentModel.Composition.Primitives; +using CodingWithCalvin.MCPServer.Services; +using Microsoft.VisualStudio.ComponentModelHost; +using Microsoft.VisualStudio.TestWindow.Extensibility; +using Xunit; + +namespace CodingWithCalvin.MCPServer.Tests; + +/// +/// Covers the reflection bridge to Test Explorer. +/// +/// +/// Microsoft.VisualStudio.TestWindow.Interfaces.dll cannot be referenced at build time, so +/// resolves its services by MEF contract name and reads them +/// reflectively. These tests stand up a real MEF container holding stand-in parts declared in +/// the genuine namespace and shaped like the real ones, which exercises the contract-name +/// lookup, the interface discovery, the property reads and the explicitly implemented event +/// subscription rather than just the pure helpers. +/// +public class TestExplorerInteropTests +{ + [Fact] + public void GetStats_ReportsUnavailable_WhenComponentModelMissing() + { + var interop = new TestExplorerInterop(() => null); + + var stats = interop.GetStats(); + + Assert.False(stats.Available); + } + + [Fact] + public void GetStats_ReportsUnavailable_WhenTestExplorerNotRunning() + { + using var host = new StubHost(); + host.Stats.IsTestExplorerStatsServiceRunning = false; + host.Stats.Counts = new TestExplorerStats { PassedTestCount = 7 }; + + var stats = new TestExplorerInterop(host.Accessor).GetStats(); + + // Counts must not leak through as zeros that read like "solution has no tests". + Assert.False(stats.Available); + Assert.Equal(0, stats.Passed); + } + + [Fact] + public void GetStats_ReadsCounts_WhenTestExplorerRunning() + { + using var host = new StubHost(); + host.Stats.IsTestExplorerStatsServiceRunning = true; + host.Stats.Counts = new TestExplorerStats + { + PassedTestCount = 12, + FailedTestCount = 3, + SkippedTestCount = 1, + NotRunTestCount = 4 + }; + + var stats = new TestExplorerInterop(host.Accessor).GetStats(); + + Assert.True(stats.Available); + Assert.Equal(12, stats.Passed); + Assert.Equal(3, stats.Failed); + Assert.Equal(1, stats.Skipped); + Assert.Equal(4, stats.NotRun); + } + + [Fact] + public void GetRunStatus_ReportsNoRunObserved_BeforeAnyStateChange() + { + using var host = new StubHost(); + + var status = new TestExplorerInterop(host.Accessor).GetRunStatus(); + + Assert.Equal("NoRunObserved", status.State); + Assert.False(status.IsRunning); + Assert.Null(status.LastOperation); + } + + [Fact] + public void GetRunStatus_TracksExecutionThroughToCompletion() + { + using var host = new StubHost(); + var interop = new TestExplorerInterop(host.Accessor); + interop.EnsureTracking(); + + host.Operations.Raise(TestOperationStates.TestExecutionStarted); + var running = interop.GetRunStatus(); + + host.Operations.Raise(TestOperationStates.TestExecutionFinished); + var finished = interop.GetRunStatus(); + + Assert.Equal("Running", running.State); + Assert.True(running.IsRunning); + + Assert.Equal("Completed", finished.State); + Assert.False(finished.IsRunning); + Assert.Equal("TestExecutionFinished", finished.LastOperation); + } + + [Fact] + public void GetRunStatus_ReportsCanceled_AfterCancellation() + { + using var host = new StubHost(); + var interop = new TestExplorerInterop(host.Accessor); + interop.EnsureTracking(); + + host.Operations.Raise(TestOperationStates.TestExecutionStarted); + host.Operations.Raise(TestOperationStates.TestExecutionCancelAndFinished); + + Assert.Equal("Canceled", interop.GetRunStatus().State); + } + + [Fact] + public void GetRunStatus_KeepsRunOutcome_WhenDiscoveryRunsAfterwards() + { + using var host = new StubHost(); + var interop = new TestExplorerInterop(host.Accessor); + interop.EnsureTracking(); + + host.Operations.Raise(TestOperationStates.TestExecutionFinished); + host.Operations.Raise(TestOperationStates.DiscoveryFinished); + + // A discovery pass triggered by editing code must not erase the last run's outcome. + Assert.Equal("Completed", interop.GetRunStatus().State); + } + + [Fact] + public void EnsureTracking_SubscribesOnlyOnce() + { + using var host = new StubHost(); + var interop = new TestExplorerInterop(host.Accessor); + + interop.EnsureTracking(); + interop.EnsureTracking(); + interop.EnsureTracking(); + + Assert.Equal(1, host.Operations.SubscriberCount); + } + + [Fact] + public void EnsureTracking_DoesNotThrow_WhenComponentModelMissing() + { + var interop = new TestExplorerInterop(() => null); + + interop.EnsureTracking(); + + Assert.Equal("NoRunObserved", interop.GetRunStatus().State); + } + + [Theory] + [InlineData(null, null, "NoRunObserved")] + [InlineData("DiscoveryStarted", null, "Discovering")] + [InlineData("DiscoveryStarting", null, "Discovering")] + [InlineData("DiscoveryFinished", null, "NoRunObserved")] + [InlineData("TestExecutionStarted", "TestExecutionStarted", "Running")] + [InlineData("TestExecutionStarting", "TestExecutionStarting", "Running")] + [InlineData("TestExecutionCanceling", "TestExecutionCanceling", "Canceling")] + [InlineData("TestExecutionFinished", "TestExecutionFinished", "Completed")] + [InlineData("TestExecutionCancelAndFinished", "TestExecutionCancelAndFinished", "Canceled")] + [InlineData("DiscoveryStarted", "TestExecutionFinished", "Discovering")] + [InlineData("DiscoveryFinished", "TestExecutionFinished", "Completed")] + public void DeriveState_CollapsesOperationNames(string? last, string? lastExecution, string expected) + { + Assert.Equal(expected, TestExplorerInterop.DeriveState(last, lastExecution)); + } + + /// + /// A MEF container holding the stand-in Test Explorer parts, exposed through the + /// shape that consumes. + /// + private sealed class StubHost : IDisposable + { + private readonly CompositionContainer _container; + private readonly StubComponentModel _componentModel; + + internal StubHost() + { + _container = new CompositionContainer( + new TypeCatalog(typeof(StubStatsService), typeof(StubOperationState))); + _componentModel = new StubComponentModel(_container); + + Stats = (StubStatsService)_container.GetExportedValue(); + Operations = (StubOperationState)_container.GetExportedValue(); + } + + internal StubStatsService Stats { get; } + + internal StubOperationState Operations { get; } + + internal Func Accessor => () => _componentModel; + + public void Dispose() => _container.Dispose(); + } + + private sealed class StubComponentModel : IComponentModel + { + internal StubComponentModel(ExportProvider exportProvider) + { + DefaultExportProvider = exportProvider; + } + + public ExportProvider DefaultExportProvider { get; } + + public ComposablePartCatalog DefaultCatalog => throw new NotSupportedException(); + + public ICompositionService DefaultCompositionService => throw new NotSupportedException(); + +#pragma warning disable CS0618, CS0672 // Obsolete member is part of the interface being stubbed. + public ComposablePartCatalog GetCatalog(string catalogName) => throw new NotSupportedException(); +#pragma warning restore CS0618, CS0672 + + public System.Collections.Generic.IEnumerable GetExtensions() where T : class + => throw new NotSupportedException(); + + public T GetService() where T : class => throw new NotSupportedException(); + } + + [Export(typeof(ITestExplorerStatsService))] + [PartCreationPolicy(CreationPolicy.Shared)] + private sealed class StubStatsService : ITestExplorerStatsService + { + internal TestExplorerStats Counts { get; set; } + + public bool IsTestExplorerStatsServiceRunning { get; set; } + + TestExplorerStats ITestExplorerStatsService.TestExplorerStats => Counts; + + public event EventHandler? TestExplorerStatsChanged; + + internal void RaiseStatsChanged() => TestExplorerStatsChanged?.Invoke(this, Counts); + } + + /// + /// Implements StateChanged explicitly, matching the real OperationBroker, so the interop's + /// interface-based event lookup is genuinely exercised. + /// + [Export(typeof(IOperationState))] + [PartCreationPolicy(CreationPolicy.Shared)] + private sealed class StubOperationState : IOperationState + { + private EventHandler? _stateChanged; + + internal int SubscriberCount { get; private set; } + + event EventHandler IOperationState.StateChanged + { + add + { + _stateChanged += value; + SubscriberCount++; + } + remove + { + _stateChanged -= value; + SubscriberCount--; + } + } + + internal void Raise(TestOperationStates state) => + _stateChanged?.Invoke(this, new OperationStateChangedEventArgs { State = state }); + } +} diff --git a/src/CodingWithCalvin.MCPServer.Tests/TestTargetSelectionTests.cs b/src/CodingWithCalvin.MCPServer.Tests/TestTargetSelectionTests.cs new file mode 100644 index 0000000..de6fefa --- /dev/null +++ b/src/CodingWithCalvin.MCPServer.Tests/TestTargetSelectionTests.cs @@ -0,0 +1,107 @@ +using System.Collections.Generic; +using CodingWithCalvin.MCPServer.Services; +using CodingWithCalvin.MCPServer.Shared.Models; +using Xunit; + +namespace CodingWithCalvin.MCPServer.Tests; + +/// +/// Covers how a caller-supplied test name is resolved to the symbol the caret gets placed on. +/// Test Explorer's run-in-context commands act on the caret, so picking the wrong symbol here +/// silently runs the wrong tests rather than failing. +/// +public class TestTargetSelectionTests +{ + [Fact] + public void SelectTestTarget_PrefersExactFullNameOverSimpleName() + { + var candidates = new List + { + Symbol("CalculatorTests", "Other.CalculatorTests", SymbolKind.Class), + Symbol("CalculatorTests", "MyApp.Tests.CalculatorTests", SymbolKind.Class) + }; + + var selected = VisualStudioService.SelectTestTarget(candidates, "MyApp.Tests.CalculatorTests"); + + Assert.Equal("MyApp.Tests.CalculatorTests", selected?.FullName); + } + + [Fact] + public void SelectTestTarget_MatchesSimpleName() + { + var candidates = new List + { + Symbol("Helper", "MyApp.Tests.Helper", SymbolKind.Class), + Symbol("Add_ReturnsSum", "MyApp.Tests.CalculatorTests.Add_ReturnsSum", SymbolKind.Function) + }; + + var selected = VisualStudioService.SelectTestTarget(candidates, "Add_ReturnsSum"); + + Assert.Equal("MyApp.Tests.CalculatorTests.Add_ReturnsSum", selected?.FullName); + } + + [Fact] + public void SelectTestTarget_FallsBackToTrailingSegmentMatch() + { + var candidates = new List + { + Symbol("Unrelated", "MyApp.Tests.Unrelated", SymbolKind.Class), + Symbol("Add_ReturnsSum", "MyApp.Tests.CalculatorTests.Add_ReturnsSum", SymbolKind.Function) + }; + + var selected = VisualStudioService.SelectTestTarget( + candidates, + "CalculatorTests.Add_ReturnsSum"); + + Assert.Equal("MyApp.Tests.CalculatorTests.Add_ReturnsSum", selected?.FullName); + } + + [Fact] + public void SelectTestTarget_PrefersExactSimpleNameOverTrailingSegment() + { + var candidates = new List + { + Symbol("Nested", "MyApp.Tests.Outer.Nested", SymbolKind.Class), + Symbol("Nested", "MyApp.Tests.Nested", SymbolKind.Class) + }; + + var selected = VisualStudioService.SelectTestTarget(candidates, "Nested"); + + // Both end with ".Nested"; the exact simple-name rule runs first and takes the earlier + // candidate, so the result is deterministic rather than dependent on suffix ordering. + Assert.Equal("MyApp.Tests.Outer.Nested", selected?.FullName); + } + + [Fact] + public void SelectTestTarget_ReturnsNull_WhenNoCandidates() + { + var selected = VisualStudioService.SelectTestTarget(new List(), "Anything"); + + Assert.Null(selected); + } + + [Fact] + public void SelectTestTarget_FallsBackToFirstCandidate_WhenNothingMatches() + { + var candidates = new List + { + Symbol("CalculatorTests", "MyApp.Tests.CalculatorTests", SymbolKind.Class) + }; + + // The caller's text already matched during the workspace symbol search, so a candidate + // that fails the stricter rules here is still a better answer than nothing. + var selected = VisualStudioService.SelectTestTarget(candidates, "Calculator"); + + Assert.Equal("MyApp.Tests.CalculatorTests", selected?.FullName); + } + + private static SymbolInfo Symbol(string name, string fullName, SymbolKind kind) => new() + { + Name = name, + FullName = fullName, + Kind = kind, + FilePath = @"C:\src\Tests.cs", + StartLine = 10, + StartColumn = 5 + }; +} diff --git a/src/CodingWithCalvin.MCPServer.Tests/TestWindowStandIns.cs b/src/CodingWithCalvin.MCPServer.Tests/TestWindowStandIns.cs new file mode 100644 index 0000000..6c04cc9 --- /dev/null +++ b/src/CodingWithCalvin.MCPServer.Tests/TestWindowStandIns.cs @@ -0,0 +1,55 @@ +using System; + +// Stand-ins for the Test Explorer extensibility types, declared in the real namespace so that +// their full names match the MEF contract names TestExplorerInterop looks up. +// +// Microsoft.VisualStudio.TestWindow.Interfaces.dll ships only inside the Visual Studio +// installation: it is absent from the Microsoft.VisualStudio.SDK metapackage, nuget.org carries +// nothing newer than 11.0.61030 (2012), and $(DevEnvDir) is undefined under `dotnet build`. The +// production code therefore resolves these services by contract name and reads them +// reflectively, and these declarations let the tests drive that path without Visual Studio. +// +// Shapes are transcribed from the shipped assembly and are identical in VS 2022 17.14 and +// VS 2026 18.0. Only the members the interop touches are reproduced. +namespace Microsoft.VisualStudio.TestWindow.Extensibility; + +public struct TestExplorerStats +{ + public int PassedTestCount { get; set; } + public int FailedTestCount { get; set; } + public int SkippedTestCount { get; set; } + public int NotRunTestCount { get; set; } +} + +public interface ITestExplorerStatsService +{ + TestExplorerStats TestExplorerStats { get; } + bool IsTestExplorerStatsServiceRunning { get; } + event EventHandler TestExplorerStatsChanged; +} + +public enum TestOperationStates +{ + None = 0x00000000, + Discovery = 0x00010000, + DiscoveryStarting = 0x00010008, + DiscoveryStarted = 0x00010001, + DiscoveryFinished = 0x00010004, + DiscoveryCanceled = 0x00010006, + TestExecution = 0x00020000, + TestExecutionStarting = 0x00020008, + TestExecutionStarted = 0x00020001, + TestExecutionCanceling = 0x00020003, + TestExecutionFinished = 0x00020004, + TestExecutionCancelAndFinished = 0x00020006 +} + +public class OperationStateChangedEventArgs : EventArgs +{ + public TestOperationStates State { get; set; } +} + +public interface IOperationState +{ + event EventHandler StateChanged; +} diff --git a/src/CodingWithCalvin.MCPServer/Services/IVisualStudioService.cs b/src/CodingWithCalvin.MCPServer/Services/IVisualStudioService.cs index 828be40..67fd4dc 100644 --- a/src/CodingWithCalvin.MCPServer/Services/IVisualStudioService.cs +++ b/src/CodingWithCalvin.MCPServer/Services/IVisualStudioService.cs @@ -68,6 +68,14 @@ public interface IVisualStudioService Task WriteOutputPaneAsync(string paneIdentifier, string message, bool activate = false); Task> GetOutputPanesAsync(); + // Test Explorer tools + Task RunAllTestsAsync(); + Task DebugAllTestsAsync(); + Task RunTestsInContextAsync(string target, bool debug); + Task CancelTestRunAsync(); + Task GetTestRunStatusAsync(); + Task GetTestStatsAsync(); + Task> GetWindowsAsync(); Task ActivateWindowAsync(string caption); Task ShowToolWindowAsync(string name); diff --git a/src/CodingWithCalvin.MCPServer/Services/RpcServer.cs b/src/CodingWithCalvin.MCPServer/Services/RpcServer.cs index 0d79ddd..104401d 100644 --- a/src/CodingWithCalvin.MCPServer/Services/RpcServer.cs +++ b/src/CodingWithCalvin.MCPServer/Services/RpcServer.cs @@ -279,6 +279,14 @@ public Task WriteOutputPaneAsync(string paneIdentifier, string message, bo => _vsService.WriteOutputPaneAsync(paneIdentifier, message, activate); public Task> GetOutputPanesAsync() => _vsService.GetOutputPanesAsync(); + public Task RunAllTestsAsync() => _vsService.RunAllTestsAsync(); + public Task DebugAllTestsAsync() => _vsService.DebugAllTestsAsync(); + public Task RunTestsInContextAsync(string target, bool debug) + => _vsService.RunTestsInContextAsync(target, debug); + public Task CancelTestRunAsync() => _vsService.CancelTestRunAsync(); + public Task GetTestRunStatusAsync() => _vsService.GetTestRunStatusAsync(); + public Task GetTestStatsAsync() => _vsService.GetTestStatsAsync(); + public Task> GetWindowsAsync() => _vsService.GetWindowsAsync(); public Task ActivateWindowAsync(string caption) => _vsService.ActivateWindowAsync(caption); public Task ShowToolWindowAsync(string name) => _vsService.ShowToolWindowAsync(name); diff --git a/src/CodingWithCalvin.MCPServer/Services/TestExplorerInterop.cs b/src/CodingWithCalvin.MCPServer/Services/TestExplorerInterop.cs new file mode 100644 index 0000000..fe4c0e7 --- /dev/null +++ b/src/CodingWithCalvin.MCPServer/Services/TestExplorerInterop.cs @@ -0,0 +1,321 @@ +using System; +using System.ComponentModel.Composition.Primitives; +using System.Linq; +using System.Reflection; +using CodingWithCalvin.MCPServer.Shared.Models; +using CodingWithCalvin.Otel4Vsix; +using Microsoft.VisualStudio.ComponentModelHost; + +namespace CodingWithCalvin.MCPServer.Services; + +/// +/// Reflection bridge to the Test Explorer extensibility services. +/// +/// +/// +/// The types used here live in Microsoft.VisualStudio.TestWindow.Interfaces.dll, which +/// ships only inside the Visual Studio installation. It cannot be referenced at compile time: +/// nuget.org carries nothing newer than 11.0.61030 (2012), it is absent from the +/// Microsoft.VisualStudio.SDK metapackage, and $(DevEnvDir) is undefined under +/// dotnet build โ€” which is how this repo builds both locally and in CI โ€” so a HintPath +/// would break the build outright. +/// +/// +/// The surface needed is small and stable: one bool, one struct of four ints, and one event. +/// It is byte-identical between VS 2022 17.14 and VS 2026 18.0, the range the VSIX manifest +/// targets. Both services are MEF parts (TestExplorerStatsService exports +/// ITestExplorerStatsService, OperationBroker exports IOperationState), so +/// they are resolved by contract name and read reflectively. +/// +/// +/// Every failure path degrades to "unavailable" rather than throwing. Test Explorer may never +/// have been opened, and on a future Visual Studio the shape could change; neither should take +/// an MCP tool call down with it. +/// +/// +internal sealed class TestExplorerInterop +{ + private const string StatsServiceContract = + "Microsoft.VisualStudio.TestWindow.Extensibility.ITestExplorerStatsService"; + private const string OperationStateContract = + "Microsoft.VisualStudio.TestWindow.Extensibility.IOperationState"; + + private const string DiscoveryPrefix = "Discovery"; + private const string ExecutionPrefix = "TestExecution"; + + private readonly object _gate = new object(); + private readonly Func _componentModelAccessor; + + private bool _statsResolved; + private object? _statsService; + private PropertyInfo? _isRunningProperty; + private PropertyInfo? _statsProperty; + private PropertyInfo? _passedProperty; + private PropertyInfo? _failedProperty; + private PropertyInfo? _skippedProperty; + private PropertyInfo? _notRunProperty; + + private bool _trackingAttempted; + private string? _lastOperation; + private string? _lastExecutionOperation; + + internal TestExplorerInterop(Func componentModelAccessor) + { + _componentModelAccessor = componentModelAccessor; + } + + /// + /// Reads the current Test Explorer counts. Returns false + /// when Test Explorer has not been initialized, so that an uninitialized window is not + /// reported as a solution with zero tests. + /// + internal TestStats GetStats() + { + try + { + lock (_gate) + { + if (!TryResolveStatsService()) + { + return new TestStats(); + } + + if (_isRunningProperty!.GetValue(_statsService) is not true) + { + return new TestStats(); + } + + var stats = _statsProperty!.GetValue(_statsService); + if (stats == null) + { + return new TestStats(); + } + + return new TestStats + { + Available = true, + Passed = ReadCount(_passedProperty, stats), + Failed = ReadCount(_failedProperty, stats), + Skipped = ReadCount(_skippedProperty, stats), + NotRun = ReadCount(_notRunProperty, stats) + }; + } + } + catch (Exception ex) + { + VsixTelemetry.TrackException(ex); + return new TestStats(); + } + } + + /// + /// Reports the tracked run state alongside the current counts. State is only observed from + /// the point first succeeds, so a run started outside this + /// extension before then reads as "NoRunObserved". + /// + internal TestRunStatus GetRunStatus() + { + EnsureTracking(); + + string? lastOperation; + string? lastExecution; + + lock (_gate) + { + lastOperation = _lastOperation; + lastExecution = _lastExecutionOperation; + } + + var state = DeriveState(lastOperation, lastExecution); + + return new TestRunStatus + { + State = state, + IsRunning = state is "Discovering" or "Running" or "Canceling", + LastOperation = lastOperation, + Stats = GetStats() + }; + } + + /// + /// Subscribes to Test Explorer operation state changes. Safe to call repeatedly; the + /// subscription is attempted once and the outcome cached either way. + /// + internal void EnsureTracking() + { + lock (_gate) + { + if (_trackingAttempted) + { + return; + } + + _trackingAttempted = true; + + try + { + var operationState = ResolveExport(OperationStateContract); + if (operationState == null) + { + return; + } + + // IOperationState.StateChanged is implemented explicitly on OperationBroker, so + // the event has to be reached through the interface rather than the class. + var contract = FindInterface(operationState, OperationStateContract); + var stateChanged = contract?.GetEvent("StateChanged"); + if (stateChanged?.EventHandlerType == null) + { + return; + } + + var callback = typeof(TestExplorerInterop).GetMethod( + nameof(OnOperationStateChanged), + BindingFlags.Instance | BindingFlags.NonPublic); + if (callback == null) + { + return; + } + + // The event is EventHandler; the callback takes + // the EventArgs base. Delegate creation allows that contravariance. + var handler = Delegate.CreateDelegate( + stateChanged.EventHandlerType, + this, + callback, + throwOnBindFailure: false); + if (handler == null) + { + return; + } + + stateChanged.AddEventHandler(operationState, handler); + } + catch (Exception ex) + { + VsixTelemetry.TrackException(ex); + } + } + } + + private void OnOperationStateChanged(object sender, EventArgs args) + { + try + { + var operation = args.GetType().GetProperty("State")?.GetValue(args)?.ToString(); + if (string.IsNullOrEmpty(operation)) + { + return; + } + + lock (_gate) + { + _lastOperation = operation; + + if (operation!.StartsWith(ExecutionPrefix, StringComparison.Ordinal)) + { + _lastExecutionOperation = operation; + } + } + } + catch (Exception ex) + { + VsixTelemetry.TrackException(ex); + } + } + + /// + /// Collapses the raw Test Explorer operation names into the small state set the tools + /// report. Discovery is only surfaced while it is in flight, so that a discovery pass + /// triggered after a run does not erase the outcome of that run. + /// + internal static string DeriveState(string? lastOperation, string? lastExecutionOperation) + { + if (lastOperation != null + && lastOperation.StartsWith(DiscoveryPrefix, StringComparison.Ordinal) + && IsInFlight(lastOperation)) + { + return "Discovering"; + } + + return lastExecutionOperation switch + { + null => "NoRunObserved", + "TestExecutionFinished" => "Completed", + "TestExecutionCancelAndFinished" => "Canceled", + "TestExecutionCanceling" => "Canceling", + _ => "Running" + }; + } + + private static bool IsInFlight(string operation) => + !operation.EndsWith("Finished", StringComparison.Ordinal) + && !operation.EndsWith("Canceled", StringComparison.Ordinal); + + private static int ReadCount(PropertyInfo? property, object stats) => + property?.GetValue(stats) is int value ? value : 0; + + private bool TryResolveStatsService() + { + if (_statsResolved) + { + return _statsService != null; + } + + _statsResolved = true; + + var service = ResolveExport(StatsServiceContract); + if (service == null) + { + return false; + } + + var contract = FindInterface(service, StatsServiceContract); + if (contract == null) + { + return false; + } + + _isRunningProperty = contract.GetProperty("IsTestExplorerStatsServiceRunning"); + _statsProperty = contract.GetProperty("TestExplorerStats"); + if (_isRunningProperty == null || _statsProperty == null) + { + return false; + } + + var statsType = _statsProperty.PropertyType; + _passedProperty = statsType.GetProperty("PassedTestCount"); + _failedProperty = statsType.GetProperty("FailedTestCount"); + _skippedProperty = statsType.GetProperty("SkippedTestCount"); + _notRunProperty = statsType.GetProperty("NotRunTestCount"); + + _statsService = service; + return true; + } + + private object? ResolveExport(string contractName) + { + var componentModel = _componentModelAccessor(); + if (componentModel == null) + { + return null; + } + + var definition = new ImportDefinition( + d => d.ContractName == contractName, + contractName, + ImportCardinality.ZeroOrMore, + isRecomposable: false, + isPrerequisite: false); + + return componentModel.DefaultExportProvider + .GetExports(definition) + .FirstOrDefault() + ?.Value; + } + + private static Type? FindInterface(object instance, string fullName) => + instance.GetType() + .GetInterfaces() + .FirstOrDefault(i => string.Equals(i.FullName, fullName, StringComparison.Ordinal)); +} diff --git a/src/CodingWithCalvin.MCPServer/Services/VisualStudioService.cs b/src/CodingWithCalvin.MCPServer/Services/VisualStudioService.cs index 15584b8..588689e 100644 --- a/src/CodingWithCalvin.MCPServer/Services/VisualStudioService.cs +++ b/src/CodingWithCalvin.MCPServer/Services/VisualStudioService.cs @@ -2587,4 +2587,224 @@ public static IReadOnlyCollection GetSupportedToolWindowNames() { return ToolWindowCommands.Keys; } + + /// + /// Test Explorer exposes no documented cancel command, so the first candidate that the + /// running Visual Studio actually recognises is used. + /// + private static readonly string[] CancelTestRunCommands = + { + "TestExplorer.CancelTestRun", + "TestExplorer.CancelTests" + }; + + private TestExplorerInterop? _testExplorer; + + private TestExplorerInterop TestExplorer => + _testExplorer ??= new TestExplorerInterop( + () => ServiceProvider.GetService(typeof(SComponentModel)) as IComponentModel); + + public async Task RunAllTestsAsync() + { + using var activity = VsixTelemetry.Tracer.StartActivity("RunAllTests"); + return await StartTestRunAsync("TestExplorer.RunAllTests", activity); + } + + public async Task DebugAllTestsAsync() + { + using var activity = VsixTelemetry.Tracer.StartActivity("DebugAllTests"); + return await StartTestRunAsync("TestExplorer.DebugAllTests", activity); + } + + private async Task StartTestRunAsync(string command, Activity? activity) + { + await ThreadHelper.JoinableTaskFactory.SwitchToMainThreadAsync(); + var dte = await GetDteAsync(); + + // Subscribe before issuing the command, otherwise the run's own state changes are + // missed and test_status reports NoRunObserved for a run that is already going. + TestExplorer.EnsureTracking(); + + try + { + if (!IsCommandAvailable(dte, command)) + { + return false; + } + + dte.ExecuteCommand(command); + return true; + } + catch (Exception ex) + { + activity?.SetStatus(ActivityStatusCode.Error, ex.Message); + activity?.RecordException(ex); + return false; + } + } + + public async Task RunTestsInContextAsync(string target, bool debug) + { + using var activity = VsixTelemetry.Tracer.StartActivity( + debug ? "DebugTestsInContext" : "RunTestsInContext"); + + if (string.IsNullOrWhiteSpace(target)) + { + return new TestTargetResult { Message = "A class or method name is required." }; + } + + await ThreadHelper.JoinableTaskFactory.SwitchToMainThreadAsync(); + var dte = await GetDteAsync(); + + try + { + var search = await SearchWorkspaceSymbolsAsync(target.Trim()); + var candidates = search.Symbols + .Where(s => s.Kind == SymbolKind.Class || s.Kind == SymbolKind.Function) + .ToList(); + + var symbol = SelectTestTarget(candidates, target.Trim()); + if (symbol == null) + { + return new TestTargetResult + { + Message = $"No class or method named '{target}' was found in the solution." + }; + } + + // The *InContext commands act on wherever the caret currently is, so the target has + // to be opened and the caret placed on it before the command is issued. + if (!await OpenDocumentAsync(symbol.FilePath)) + { + return new TestTargetResult + { + ResolvedTarget = symbol.FullName, + FilePath = symbol.FilePath, + Line = symbol.StartLine, + Message = $"Could not open '{symbol.FilePath}' to position the caret." + }; + } + + if (!await SetSelectionAsync( + symbol.FilePath, + symbol.StartLine, + symbol.StartColumn, + symbol.StartLine, + symbol.StartColumn)) + { + return new TestTargetResult + { + ResolvedTarget = symbol.FullName, + FilePath = symbol.FilePath, + Line = symbol.StartLine, + Message = $"Could not position the caret on '{symbol.FullName}'." + }; + } + + var command = debug + ? "TestExplorer.DebugAllTestsInContext" + : "TestExplorer.RunAllTestsInContext"; + + TestExplorer.EnsureTracking(); + + if (!IsCommandAvailable(dte, command)) + { + return new TestTargetResult + { + ResolvedTarget = symbol.FullName, + FilePath = symbol.FilePath, + Line = symbol.StartLine, + Message = $"'{command}' is unavailable. Test Explorer may still be discovering tests." + }; + } + + dte.ExecuteCommand(command); + + return new TestTargetResult + { + Started = true, + ResolvedTarget = symbol.FullName, + FilePath = symbol.FilePath, + Line = symbol.StartLine, + Message = candidates.Count > 1 + ? $"{candidates.Count} symbols matched '{target}'; used '{symbol.FullName}'." + : null + }; + } + catch (Exception ex) + { + activity?.SetStatus(ActivityStatusCode.Error, ex.Message); + activity?.RecordException(ex); + return new TestTargetResult { Message = ex.Message }; + } + } + + /// + /// Picks the best match for a caller-supplied class or method name, preferring an exact + /// fully qualified match over an exact simple name over a trailing-segment match. + /// + internal static SymbolInfo? SelectTestTarget(IReadOnlyList candidates, string target) + { + return candidates.FirstOrDefault(s => string.Equals(s.FullName, target, StringComparison.Ordinal)) + ?? candidates.FirstOrDefault(s => string.Equals(s.Name, target, StringComparison.Ordinal)) + ?? candidates.FirstOrDefault(s => s.FullName.EndsWith("." + target, StringComparison.Ordinal)) + ?? candidates.FirstOrDefault(); + } + + public async Task CancelTestRunAsync() + { + await ThreadHelper.JoinableTaskFactory.SwitchToMainThreadAsync(); + var dte = await GetDteAsync(); + + foreach (var command in CancelTestRunCommands) + { + try + { + if (!IsCommandAvailable(dte, command)) + { + continue; + } + + dte.ExecuteCommand(command); + return true; + } + catch (Exception ex) + { + VsixTelemetry.TrackException(ex); + } + } + + return false; + } + + public async Task GetTestRunStatusAsync() + { + await ThreadHelper.JoinableTaskFactory.SwitchToMainThreadAsync(); + return TestExplorer.GetRunStatus(); + } + + public async Task GetTestStatsAsync() + { + await ThreadHelper.JoinableTaskFactory.SwitchToMainThreadAsync(); + return TestExplorer.GetStats(); + } + + /// + /// ExecuteCommand throws when a command is disabled or unknown, which for Test Explorer is + /// the normal state before discovery finishes. Probing first keeps that an ordinary result + /// instead of an exception. + /// + private static bool IsCommandAvailable(DTE2 dte, string command) + { + ThreadHelper.ThrowIfNotOnUIThread(); + + try + { + return dte.Commands.Item(command)?.IsAvailable == true; + } + catch (Exception) + { + return false; + } + } }