feat(agent-testing): add a Template case type and let turns attach files

src/Plugins/BotSharp.Plugin.AgentTesting/Controllers/AgentTestController.cs

@@ -413,6 +413,7 @@ public async Task<ActionResult<AgentTestRunDeleteResponse>> DeleteRuns(
                 continue;
             }
 
+            run = await FailIfAbandonedAsync(run);
             if (!IsTerminalStatus(run.Status))
             {
                 // Deleting a run that is still executing does not stop it: the queue keeps driving
@@ -637,7 +638,14 @@ public async Task<ActionResult<AgentTestRun>> TriggerRun(
 
     [HttpGet("runs")]
     public async Task<List<AgentTestRun>> ListRuns([FromQuery] string? suiteId)
-        => await _repo.ListRunsAsync(suiteId);
+    {
+        var runs = await _repo.ListRunsAsync(suiteId);
+        for (var i = 0; i < runs.Count; i++)
+        {
+            runs[i] = await FailIfAbandonedAsync(runs[i]);
+        }
+        return runs;
+    }
 
     [HttpGet("runs/{id}")]
     public async Task<ActionResult<AgentTestRunDetailDto>> GetRun(string id)
@@ -648,6 +656,7 @@ public async Task<ActionResult<AgentTestRunDetailDto>> GetRun(string id)
             return NotFound($"agent test run {id} not found");
         }
 
+        run = await FailIfAbandonedAsync(run);
         var results = await _repo.ListCaseResultsAsync(id);
         return new AgentTestRunDetailDto { Run = run, Results = results };
     }
@@ -669,8 +678,13 @@ public async Task<IActionResult> CancelRun(string id)
             return Conflict($"agent test run {id} has already finished ({run.Status}) and cannot be cancelled");
         }
 
-        run.CancelRequested = true;
-        await _repo.UpdateRunAsync(run);
+        // A single-field update, not a write-back of `run`: the executor keeps writing this document
+        // while the run goes, and replacing it with the copy read above would revert its progress.
+        if (!await _repo.RequestRunCancelAsync(id))
+        {
+            return Conflict($"agent test run {id} finished before it could be cancelled");
+        }
+
         return Ok();
     }
 
@@ -765,13 +779,16 @@ private static void ApplyCase(AgentTestCase testCase, AgentTestCaseUpsertRequest
     {
         testCase.SuiteId = request.SuiteId;
         testCase.Name = request.Name;
-        testCase.Enabled = request.Enabled;
+        // Null keeps the case's current value, as ApplySuite does: a partial PUT must not enable a
+        // recorded draft nobody has reviewed yet.
+        testCase.Enabled = request.Enabled ?? testCase.Enabled;
         // Normalised, not taken verbatim: CaseValidation.Validate has already rejected anything that is
         // neither blank nor a known type, so this only fixes casing ("routing" -> "Routing") and
         // maps blank onto the default. Storing the caller's casing would break every later
         // Ordinal comparison against CaseTypes.Routing.
         testCase.CaseType = CaseTypes.Normalize(request.CaseType) ?? CaseTypes.Agent;
         testCase.EntryAgentId = string.IsNullOrWhiteSpace(request.EntryAgentId) ? null : request.EntryAgentId.Trim();
+        testCase.TemplateName = string.IsNullOrWhiteSpace(request.TemplateName) ? null : request.TemplateName.Trim();
         testCase.Turns = request.Turns ?? [];
         testCase.Assertions = request.Assertions ?? [];
         testCase.InitialStates = request.InitialStates ?? [];
@@ -822,7 +839,63 @@ private static void ApplyCase(AgentTestCase testCase, AgentTestCaseUpsertRequest
         }
 
         var agent = await _agents.GetAgent(request.EntryAgentId.Trim());
-        return agent == null ? $"entry agent {request.EntryAgentId} not found" : null;
+        if (agent == null)
+        {
+            return $"entry agent {request.EntryAgentId} not found";
+        }
+
+        var isTemplateCase = string.Equals(
+            CaseTypes.Normalize(request.CaseType), CaseTypes.Template, StringComparison.Ordinal);
+        if (isTemplateCase
+            && !string.IsNullOrWhiteSpace(request.TemplateName)
+            // Ordinal, as AgentService.RenderTemplate looks templates up with ==: a name that only
+            // matches ignoring case would save fine and then render an empty instruction.
+            && agent.Templates?.Any(t => string.Equals(t.Name, request.TemplateName.Trim(), StringComparison.Ordinal)) != true)
+        {
+            return $"agent {agent.Name} has no template '{request.TemplateName}'";
+        }
+
+        return null;
+    }
+
+    /// <summary>
+    /// The run queue is in-process, so a run whose host died stays Running in the database. A live
+    /// executor stamps UpdateDate every AgentTestRunExecutor.HeartbeatInterval (30s), so this long
+    /// without one means nothing is executing it any more.
+    /// </summary>
+    private static readonly TimeSpan AbandonedAfter = TimeSpan.FromMinutes(3);
+
+    /// <summary>
+    /// Fails a Running run whose heartbeat stopped, and returns the run as it now stands. Checked
+    /// when a run is read rather than swept on host startup: hosts share one database, and a sweep
+    /// cannot tell another host's live run from a dead one.
+    /// </summary>
+    private async Task<AgentTestRun> FailIfAbandonedAsync(AgentTestRun run)
+    {
+        if (run.Status != AgentTestStatus.Running)
+        {
+            return run;
+        }
+
+        var lastProgress = run.UpdateDate ?? run.StartedAt ?? run.CreateDate;
+        if (DateTime.UtcNow - lastProgress <= AbandonedAfter)
+        {
+            return run;
+        }
+
+        var error = $"The run stopped reporting progress for over {AbandonedAfter.TotalMinutes:0} minutes, so "
+            + "the host running it has stopped. The queue is in-process and does not survive a restart -- "
+            + "trigger the run again.";
+        if (!await _repo.TryAbandonRunAsync(run.Id, run.UpdateDate, error))
+        {
+            // The executor wrote since this copy was read: it is alive, or already finished.
+            return await _repo.GetRunAsync(run.Id) ?? run;
+        }
+
+        run.Status = AgentTestStatus.Error;
+        run.Error = error;
+        run.CompletedAt = DateTime.UtcNow;
+        return run;
     }
 
     private static bool IsTerminalStatus(string status) =>

src/Plugins/BotSharp.Plugin.AgentTesting/Models/AgentTestAuthorDtos.cs

@@ -135,6 +135,7 @@ public static class AuthorFields
     public const string Enabled = "enabled";
     public const string CaseType = "caseType";
     public const string EntryAgentId = "entryAgentId";
+    public const string TemplateName = "templateName";
     public const string Turns = "turns";
     public const string Assertions = "assertions";
     public const string InitialStates = "initialStates";
@@ -150,7 +151,7 @@ public static class AuthorFields
 
     public static readonly string[] All =
     [
-        Name, Enabled, CaseType, EntryAgentId, Turns, Assertions, InitialStates, History, Mocks,
+        Name, Enabled, CaseType, EntryAgentId, TemplateName, Turns, Assertions, InitialStates, History, Mocks,
         Priority, Severity, Batch, CrossCutting, InvolvedAgents, BusinessDomain, ExpectedOutcome
     ];
 

src/Plugins/BotSharp.Plugin.AgentTesting/Models/AgentTestCase.cs

@@ -32,6 +32,11 @@ public class AgentTestCase : MongoBase
     /// </summary>
     public string? EntryAgentId { get; set; }
 
+    /// <summary>
+    /// The entry agent's template a Template case calls directly; ignored for every other type.
+    /// </summary>
+    public string? TemplateName { get; set; }
+
     /// <summary>A length of 1 is a single-turn case.</summary>
     public List<TestTurn> Turns { get; set; } = [];
 
@@ -139,9 +144,25 @@ public class TestTurn
 {
     public int Index { get; set; }
     public string UserMessage { get; set; } = default!;
+
+    /// <summary>Attached to the user message, e.g. photos for a vision template.</summary>
+    public List<TestFile> Files { get; set; } = [];
+
     public List<TestAssertion> Assertions { get; set; } = [];
 }
 
+[BsonIgnoreExtraElements(Inherited = true)]
+public class TestFile
+{
+    public string FileName { get; set; } = string.Empty;
+
+    /// <summary>
+    /// Must stay reachable for as long as the case exists: a signed URL that expires turns every
+    /// later run into a model looking at nothing.
+    /// </summary>
+    public string FileUrl { get; set; } = string.Empty;
+}
+
 [BsonIgnoreExtraElements(Inherited = true)]
 public class TestState
 {
@@ -321,13 +342,17 @@ public static int Effective(AgentTestCase testCase)
 /// Agent   -- one agent's own behaviour, normally entered directly so the router is not part of what
 ///            is being measured. Multi-agent journeys are Agent cases too; the agentChain assertion
 ///            is what describes their hand-offs.
+/// Template -- one of the entry agent's templates called directly, the way code such as a photo
+///            check calls it outside any conversation. Each turn is an independent call: no routing,
+///            no function calls, no dialog history.
 /// </summary>
 public static class CaseTypes
 {
     public const string Routing = "Routing";
     public const string Agent = "Agent";
+    public const string Template = "Template";
 
-    public static readonly string[] All = [Routing, Agent];
+    public static readonly string[] All = [Routing, Agent, Template];
 
     /// <summary>
     /// Maps any casing of a known type onto its canonical constant; null for anything unknown, so

src/Plugins/BotSharp.Plugin.AgentTesting/Models/AgentTestCaseResult.cs

@@ -71,6 +71,13 @@ public class AgentTestCaseResult : MongoBase
     /// </summary>
     public double Cost { get; set; }
 
+    /// <summary>
+    /// What this case's llmJudge assertions spent, kept out of <see cref="TotalTokens"/> and
+    /// <see cref="Cost"/> so those describe the model under test alone. Priced by the judge model.
+    /// </summary>
+    public long JudgeTokens { get; set; }
+    public double JudgeCost { get; set; }
+
     /// <summary>
     /// Infrastructure-level reason for failure (a timeout, a dead canary), kept distinct from an
     /// assertion failure.

src/Plugins/BotSharp.Plugin.AgentTesting/Models/AgentTestDtos.cs

@@ -43,7 +43,9 @@ public class AgentTestCaseUpsertRequest
 {
     public string SuiteId { get; set; } = string.Empty;
     public string Name { get; set; } = string.Empty;
-    public bool Enabled { get; set; } = true;
+
+    /// <summary>Null keeps the current value on update; a new case defaults to enabled.</summary>
+    public bool? Enabled { get; set; }
 
     /// <summary>
     /// See <see cref="CaseTypes"/>. Blank or omitted means Agent, so an existing client that never
@@ -67,6 +69,9 @@ public class AgentTestCaseUpsertRequest
     /// </summary>
     public string? EntryAgentId { get; set; }
 
+    /// <summary>See <see cref="AgentTestCase.TemplateName"/>. Required for a Template case.</summary>
+    public string? TemplateName { get; set; }
+
     public List<TestTurn> Turns { get; set; } = [];
     public List<TestAssertion> Assertions { get; set; } = [];
     public List<TestState> InitialStates { get; set; } = [];

src/Plugins/BotSharp.Plugin.AgentTesting/Models/AgentTestRun.cs

@@ -54,6 +54,12 @@ public class AgentTestRun : MongoBase
     /// </summary>
     public int MaxParallelCases { get; set; }
 
+    /// <summary>
+    /// How many case x model executions the run set out to do, set when it starts Running.
+    /// TotalCount is how many of them have finished. Zero on runs written before this field existed.
+    /// </summary>
+    public int PlannedCount { get; set; }
+
     public int TotalCount { get; set; }
     public int PassedCount { get; set; }
     public int FailedCount { get; set; }
@@ -90,6 +96,13 @@ public class AgentTestRun : MongoBase
     /// </summary>
     public List<ModelPricingSnapshot> ModelPricing { get; set; } = [];
 
+    /// <summary>
+    /// The judge model llmJudge assertions were scored with, and its unit costs. Taken from the suite
+    /// this run executed against, because the suite's setting can change afterwards and the run report
+    /// would then name a judge that never scored it. Null when the suite had no judge configured.
+    /// </summary>
+    public ModelPricingSnapshot? JudgePricing { get; set; }
+
     /// <summary>
     /// Why a run ended as <see cref="AgentTestStatus.Error"/> -- an infrastructure stop that
     /// happened before or instead of executing cases (suite gone, suite disabled, the CaseIds
@@ -106,6 +119,13 @@ public class AgentTestRun : MongoBase
     public DateTime? StartedAt { get; set; }
     public DateTime? CompletedAt { get; set; }
     public DateTime CreateDate { get; set; } = DateTime.UtcNow;
+
+    /// <summary>
+    /// Set on every run write and by the executor's heartbeat while cases run, so a Running run whose
+    /// UpdateDate falls far behind no longer has a live executor. Null on runs written before this
+    /// field existed.
+    /// </summary>
+    public DateTime? UpdateDate { get; set; }
 }
 
 /// <summary>
@@ -132,6 +152,10 @@ public class PerformanceSummary
 
     public long TotalTokens { get; set; }
     public double TotalCost { get; set; }
+
+    /// <summary>llmJudge spend for this model's cases, not included in TotalTokens/TotalCost.</summary>
+    public long JudgeTokens { get; set; }
+    public double JudgeCost { get; set; }
 }
 
 /// <summary>

src/Plugins/BotSharp.Plugin.AgentTesting/Repositories/AgentTestRepository.cs

@@ -24,16 +24,32 @@ public interface IAgentTestRepository
     Task<List<AgentTestRun>> ListRunsAsync(string? suiteId);
 
     /// <summary>
-    /// Bounded counterpart to <see cref="ListRunsAsync"/> for the startup reconciliation sweep
-    /// (AgentTestRunQueue.ReconcileStaleRunningRunsAsync), which only ever cares about one status
-    /// (AgentTestStatus.Running) across every suite -- ListRunsAsync(null) would otherwise pull
-    /// the entire, ever-growing AgentTestRuns collection into memory on every host startup just to
-    /// filter it down to a handful of rows client-side.
+    /// Whole-document replace that never reverts CancelRequested: if a cancel landed after
+    /// <paramref name="run"/> was read, the write keeps it and sets it on <paramref name="run"/>.
+    /// Stamps UpdateDate.
     /// </summary>
-    Task<List<AgentTestRun>> ListRunsByStatusAsync(string status);
-
     Task UpdateRunAsync(AgentTestRun run);
 
+    /// <summary>
+    /// Sets CancelRequested on a Pending or Running run and touches no other field. False when the
+    /// run does not exist or has already finished.
+    /// </summary>
+    Task<bool> RequestRunCancelAsync(string id);
+
+    /// <summary>
+    /// Stamps UpdateDate on a Running run and touches no other field. False when the run is gone or
+    /// no longer Running.
+    /// </summary>
+    Task<bool> TouchRunAsync(string id);
+
+    /// <summary>
+    /// Marks a Running run Error only while its UpdateDate still equals
+    /// <paramref name="observedUpdateDate"/>, setting just Status, Error and CompletedAt. False when
+    /// the executor wrote anything since it was observed, so a stale copy never overwrites newer
+    /// progress or a terminal status.
+    /// </summary>
+    Task<bool> TryAbandonRunAsync(string id, DateTime? observedUpdateDate, string error);
+
     /// <summary>
     /// Removes a run and every case result belonging to it, and reports how many results went with
     /// it.
@@ -151,13 +167,52 @@ public async Task<List<AgentTestRun>> ListRunsAsync(string? suiteId)
             .ToListAsync();
     }
 
-    public async Task<List<AgentTestRun>> ListRunsByStatusAsync(string status)
-        => await _mongoDbContext.AgentTestRuns
-            .Find(x => x.Status == status)
-            .ToListAsync();
-
     public async Task UpdateRunAsync(AgentTestRun run)
-        => await _mongoDbContext.AgentTestRuns.ReplaceOneAsync(x => x.Id == run.Id, run);
+    {
+        run.UpdateDate = DateTime.UtcNow;
+
+        // CancelRequested is the one field written outside the executor, and it only ever goes
+        // false -> true. Replacing only while the stored flag still matches ours means a cancel that
+        // arrived after this copy was read is never overwritten; on a miss, carry it over and write.
+        var replaced = await _mongoDbContext.AgentTestRuns.ReplaceOneAsync(
+            x => x.Id == run.Id && x.CancelRequested == run.CancelRequested, run);
+
+        if (replaced.MatchedCount == 0 && !run.CancelRequested)
+        {
+            run.CancelRequested = true;
+            await _mongoDbContext.AgentTestRuns.ReplaceOneAsync(x => x.Id == run.Id, run);
+        }
+    }
+
+    public async Task<bool> RequestRunCancelAsync(string id)
+    {
+        var updated = await _mongoDbContext.AgentTestRuns.UpdateOneAsync(
+            x => x.Id == id && (x.Status == AgentTestStatus.Pending || x.Status == AgentTestStatus.Running),
+            Builders<AgentTestRun>.Update.Set(x => x.CancelRequested, true));
+        return updated.MatchedCount > 0;
+    }
+
+    public async Task<bool> TouchRunAsync(string id)
+    {
+        var updated = await _mongoDbContext.AgentTestRuns.UpdateOneAsync(
+            x => x.Id == id && x.Status == AgentTestStatus.Running,
+            Builders<AgentTestRun>.Update.Set(x => x.UpdateDate, DateTime.UtcNow));
+        return updated.MatchedCount > 0;
+    }
+
+    public async Task<bool> TryAbandonRunAsync(string id, DateTime? observedUpdateDate, string error)
+    {
+        // observedUpdateDate was read back from this collection, so it carries the stored
+        // millisecond precision and compares equal. Null matches a run written before UpdateDate
+        // existed.
+        var updated = await _mongoDbContext.AgentTestRuns.UpdateOneAsync(
+            x => x.Id == id && x.Status == AgentTestStatus.Running && x.UpdateDate == observedUpdateDate,
+            Builders<AgentTestRun>.Update
+                .Set(x => x.Status, AgentTestStatus.Error)
+                .Set(x => x.Error, error)
+                .Set(x => x.CompletedAt, DateTime.UtcNow));
+        return updated.MatchedCount > 0;
+    }
 
     public async Task<long> DeleteRunAsync(string id)
     {