feat(agent-testing): add Template cases, turn file editor and llmJudge spend

src/lib/helpers/types/agentTestTypes.js

@@ -42,6 +42,7 @@
  * @typedef {Object} TestTurn
  * @property {number} index
  * @property {string} userMessage
+ * @property {{ fileName: string, fileUrl: string }[]} [files] - Attached to the user message.
  * @property {TestAssertion[]} assertions - Turn-level assertions, as opposed to the case-level
  *   AgentTestCase.assertions.
  */
@@ -104,7 +105,7 @@
  * @property {string} name
  * @property {boolean} enabled - Recorded drafts land as false; a human has to review and enable them
  *   before they join a normal run.
- * @property {string} caseType - Routing | Agent. Defaults to Agent, which is also what every case
+ * @property {string} caseType - Routing | Agent | Template. Defaults to Agent, which is also what every case
  *   stored before the field existed reads back as. Routing is validated more strictly (one turn,
  *   must assert a routing outcome, no llmJudge) and is the only type counted towards a run's routing
  *   accuracy. A journey across several agents is an Agent case whose `agentChain` assertion
@@ -113,6 +114,7 @@
  *   uses the suite's. This is the switch between testing routing and testing one agent alone:
  *   BotSharp sends a routing-type agent through the router (which can hand off) and any other agent
  *   straight into itself (the router never runs).
+ * @property {string?} templateName - The entry agent's template a Template case calls directly.
  * @property {TestTurn[]} turns - Length 1 is a single-turn case.
  * @property {TestAssertion[]} assertions - Case-level assertions, evaluated after every turn has run.
  * @property {TestState[]} initialStates - Injected before the conversation starts; maps to BotSharp's
@@ -156,10 +158,11 @@
  * @property {string} suiteId
  * @property {string} name
  * @property {boolean} [enabled] - Defaults to true.
- * @property {string} [caseType] - Routing | Agent; blank or omitted means Agent. Any other value is
- *   a 400 rather than a silent fallback.
+ * @property {string} [caseType] - Routing | Agent | Template; blank or omitted means Agent. Any other
+ *   value is a 400 rather than a silent fallback.
  * @property {string?} [entryAgentId] - Send null rather than "" to mean "use the suite's agent";
- *   validated to exist at save time.
+ *   validated to exist at save time. Required for a Template case.
+ * @property {string?} [templateName] - Required for a Template case; must exist on the entry agent.
  * @property {TestHistoryMessage[]} [history] - Authored prior turns; every message needs a
  *   user/assistant role and non-empty content.
  * @property {string} [priority] - P0 | P1 | P2; blank keeps P1. Any other value is a 400.
@@ -214,6 +217,9 @@
  *   and then this is the only place the reason exists.
  * @property {string?} triggeredBy - User id of whoever triggered it.
  * @property {string[]?} caseIds - Run only these cases; null/empty = every enabled case in the suite.
+ * @property {number?} plannedCount - Case x model executions the run set out to do, set when it
+ *   starts Running. `totalCount` is how many have finished, so "10 / 30" is the run's progress.
+ *   0 or absent on runs from before the field existed.
  * @property {number} totalCount
  * @property {number} passedCount
  * @property {number} failedCount
@@ -226,6 +232,9 @@
  *   cost figure is not comparable with another run's unless these match. Kept per model because a single run-wide figure would
  *   average a candidate model together with the baseline and hide the difference the run exists to
  *   measure.
+ * @property {ModelPricingSnapshot?} [judgePricing] - The judge model llmJudge scored with and its unit
+ *   costs, as of this run. Absent on runs recorded before the field existed, and when the suite had no
+ *   judge configured.
  * @property {boolean} cancelRequested
  * @property {string?} startedAt - ISO date string; null until it starts.
  * @property {string?} completedAt - ISO date string; null until it finishes.
@@ -246,6 +255,9 @@
  * @property {number} totalTokens - Over EVERY result, unlike latency: a case that errored still spent
  *   what it spent.
  * @property {number} totalCost
+ * @property {number} [judgeTokens] - llmJudge spend for these cases, not included in totalTokens.
+ *   Absent on runs recorded before it existed.
+ * @property {number} [judgeCost]
  */
 
 /**
@@ -324,8 +336,11 @@
  * @property {number} modelDurationMs - The agent calls alone, summed over the turns. This is what the
  *   run's latency percentiles are built from.
  * @property {number} totalTokens - Measured as a delta across this case's own execution. Total only:
- *   the input/output split is not reachable through ITokenStatistics.
+ *   the input/output split is not reachable through ITokenStatistics. Excludes llmJudge calls.
  * @property {number} cost
+ * @property {number} [judgeTokens] - What this case's llmJudge assertions spent, priced by the judge
+ *   model. Absent on results recorded before it existed.
+ * @property {number} [judgeCost]
  * @property {string?} error - Infrastructure-level reason for failure (timeout, a dead mock seam, a
  *   case with no turns), as distinct from an assertion failure. Show this text whenever `status` is
  *   `Error`.

src/lib/helpers/utils/agent-test.js

@@ -49,6 +49,9 @@ const EXPECTED_REQUIRED = [
 	'llmJudge'
 ];
 
+/** .NET regex constructs JS RegExp rejects; see the outputRegex check in validateAssertion. */
+const DOTNET_ONLY_REGEX_SYNTAX = /\(\?[imnsx-]+[:)]|\(\?>|\(\?#|\(\?\(|\\[AZzG]|-\[/;
+
 /** Types whose `target` the backend rejects as empty. */
 const TARGET_REQUIRED = ['toolCalled', 'toolNotCalled', 'stateEquals'];
 
@@ -72,7 +75,7 @@ export const OUTPUT_SOURCE_TYPES = ['outputContains', 'outputNotContains', 'outp
  * towards a run's routing accuracy. A multi-agent journey is an Agent case whose
  * `agentChain` assertion describes the hand-offs.
  */
-export const CASE_TYPES = ['Routing', 'Agent'];
+export const CASE_TYPES = ['Routing', 'Agent', 'Template'];
 
 /**
  * Roles an authored history message may take (HistoryRoles in AgentTestCase.cs).
@@ -231,11 +234,17 @@ export function validateAssertion(assertion) {
 		return t('Assertion "stateEquals" needs an expected value, otherwise it can never pass.');
 	}
 
+	// Checked with JS RegExp as an approximation of the backend's .NET Regex. A pattern JS cannot
+	// parse is let through when it uses .NET-only syntax -- inline options (?i), atomic (?>...),
+	// comments (?#...), conditionals (?(...)), anchors \A \Z \z \G, class subtraction -[...] -- since
+	// JS cannot judge it and the backend would accept it.
 	if (type === 'outputRegex' && assertion.expected?.trim()) {
 		try {
 			new RegExp(assertion.expected);
 		} catch {
-			return t('"{pattern}" is not a valid regular expression.', { pattern: assertion.expected });
+			if (!DOTNET_ONLY_REGEX_SYNTAX.test(assertion.expected)) {
+				return t('"{pattern}" is not a valid regular expression.', { pattern: assertion.expected });
+			}
 		}
 	}
 

src/lib/langs/en.json

@@ -495,6 +495,14 @@
     "Failed to load this test case.": "Failed to load this test case.",
     "Failed to load the suite this case belongs to.": "Failed to load the suite this case belongs to.",
     "Failed to load this run.": "Failed to load this run.",
+    "Rich content": "Rich content",
+    "Suite default ({provider} / {model})": "Suite default ({provider} / {model})",
+    "No suite default -- pick a model to use chat authoring": "No suite default -- pick a model to use chat authoring",
+    "The assistant could not answer.": "The assistant could not answer.",
+    "Queued {n} re-runs, one per model the failures ran on.": "Queued {n} re-runs, one per model the failures ran on.",
+    "Failed to load the test cases.": "Failed to load the test cases.",
+    "Failed to load the run history.": "Failed to load the run history.",
+    "A Template case cannot assert on richContent: a template call returns text only.": "A Template case cannot assert on richContent: a template call returns text only.",
     "Failed to create test suite.": "Failed to create test suite.",
     "Failed to delete test suite.": "Failed to delete test suite.",
     "Failed to save suite settings.": "Failed to save suite settings.",
@@ -550,6 +558,21 @@
     "Case type": "Case type",
     "Routing case": "Routing case",
     "Agent case": "Agent case",
+    "Template case": "Template case",
+    "Calls one of the entry agent's templates directly, the way code calls it outside a conversation. No routing, no function calls; each turn is an independent call.": "Calls one of the entry agent's templates directly, the way code calls it outside a conversation. No routing, no function calls; each turn is an independent call.",
+    "Template": "Template",
+    "Select a template": "Select a template",
+    "Search templates": "Search templates",
+    "Select an entry agent first": "Select an entry agent first",
+    "Add File": "Add File",
+    "File name": "File name",
+    "File URL": "File URL",
+    "Remove file": "Remove file",
+    "Turn {n}, file {m}: needs both a name and a URL.": "Turn {n}, file {m}: needs both a name and a URL.",
+    "A Template case needs an entry agent: the agent that owns the template.": "A Template case needs an entry agent: the agent that owns the template.",
+    "A Template case needs a template.": "A Template case needs a template.",
+    "A Template case cannot have history: a template call does not read the conversation.": "A Template case cannot have history: a template call does not read the conversation.",
+    "A Template case cannot use {types} assertions: a template call does not route or call functions, so their result is fixed.": "A Template case cannot use {types} assertions: a template call does not route or call functions, so their result is fixed.",
     "One turn, entered on the Copilot entry agent, asserting only which agent took the conversation. The only type counted towards a run's routing accuracy.": "One turn, entered on the Copilot entry agent, asserting only which agent took the conversation. The only type counted towards a run's routing accuracy.",
     "One agent's own behaviour. Enter directly on that agent to keep the router out of what is measured.": "One agent's own behaviour. Enter directly on that agent to keep the router out of what is measured.",
     "Entry agent": "Entry agent",
@@ -619,7 +642,11 @@
     "Percentiles are nearest-rank over the agent-call time of the cases that reached the model, so every figure is a duration some case actually took. Cases that failed before their first turn are excluded from the percentiles but still counted in tokens and cost.": "Percentiles are nearest-rank over the agent-call time of the cases that reached the model, so every figure is a duration some case actually took. Cases that failed before their first turn are excluded from the percentiles but still counted in tokens and cost.",
     "Wall clock for the whole case": "Wall clock for the whole case",
     "Agent call time only": "Agent call time only",
-    "Tokens / cost": "Tokens / cost",
+    "Agent tokens / cost": "Agent tokens / cost",
+    "Judge tokens / cost": "Judge tokens / cost",
+    "You edited the form while the assistant was working; the new draft replaced those edits. Press Undo to get them back.": "You edited the form while the assistant was working; the new draft replaced those edits. Press Undo to get them back.",
+    "Judge": "Judge",
+    "Tokens and cost cover the model under test only; llmJudge spend is listed separately.": "Tokens and cost cover the model under test only; llmJudge spend is listed separately.",
     "Plan a scope": "Plan a scope",
     "Which cases a change needs to run, and which it does not. Read-only -- it plans a run, it does not start one.": "Which cases a change needs to run, and which it does not. Read-only -- it plans a run, it does not start one.",
     "Changed agents": "Changed agents",

src/lib/langs/zh.json

@@ -594,7 +594,7 @@
   "Agent output": "Agent 输出",
   "Conversation id": "会话 id",
   "Judge provider": "判官 provider",
-  "Judge model": "判官 model",
+  "Judge model": "Judge 模型",
   "Case timeout": "用例超时",
   "Case timeout (seconds)": "用例超时（秒）",
   "Extra allowed functions": "额外放行的函数",
@@ -689,6 +689,14 @@
   "Failed to load this test case.": "加载这条测试用例失败。",
   "Failed to load the suite this case belongs to.": "加载这条用例所属的套件失败。",
   "Failed to load this run.": "加载这次运行失败。",
+  "Rich content": "富内容",
+  "Suite default ({provider} / {model})": "套件默认（{provider} / {model}）",
+  "No suite default -- pick a model to use chat authoring": "套件未设默认模型 -- 选择一个模型才能用对话写用例",
+  "The assistant could not answer.": "助手无法回答。",
+  "Queued {n} re-runs, one per model the failures ran on.": "已排队 {n} 次重跑，按失败用例所用的模型各一次。",
+  "Failed to load the test cases.": "加载测试用例失败。",
+  "Failed to load the run history.": "加载运行记录失败。",
+  "A Template case cannot assert on richContent: a template call returns text only.": "模板用例不能对 richContent 做断言：模板调用只返回文本。",
   "Failed to create test suite.": "创建测试套件失败。",
   "Failed to delete test suite.": "删除测试套件失败。",
   "Failed to save suite settings.": "保存套件设置失败。",
@@ -744,6 +752,21 @@
     "Case type": "用例类型",
     "Routing case": "路由用例",
     "Agent case": "Agent 用例",
+    "Template case": "模板用例",
+    "Calls one of the entry agent's templates directly, the way code calls it outside a conversation. No routing, no function calls; each turn is an independent call.": "直接调用入口 agent 的某个模板，和代码在对话之外调用它的方式一致。不走路由、不调函数；每一轮都是独立的一次调用。",
+    "Template": "模板",
+    "Select a template": "选择模板",
+    "Search templates": "搜索模板",
+    "Select an entry agent first": "请先选择入口 agent",
+    "Add File": "添加文件",
+    "File name": "文件名",
+    "File URL": "文件 URL",
+    "Remove file": "删除文件",
+    "Turn {n}, file {m}: needs both a name and a URL.": "第 {n} 轮，文件 {m}：文件名和 URL 都需要填写。",
+    "A Template case needs an entry agent: the agent that owns the template.": "模板用例需要指定入口 agent，即模板所属的 agent。",
+    "A Template case needs a template.": "模板用例需要选择一个模板。",
+    "A Template case cannot have history: a template call does not read the conversation.": "模板用例不能带历史消息：模板调用不读取对话。",
+    "A Template case cannot use {types} assertions: a template call does not route or call functions, so their result is fixed.": "模板用例不能使用 {types} 断言：模板调用不走路由、不调函数，这些断言的结果是固定的。",
     "One turn, entered on the Copilot entry agent, asserting only which agent took the conversation. The only type counted towards a run's routing accuracy.": "单轮，从 Copilot 入口 agent 进入，只断言由哪个 agent 接走了这段对话。只有这一类会计入跑批的路由准确率。",
     "One agent's own behaviour. Enter directly on that agent to keep the router out of what is measured.": "单个 agent 自身的行为。直接从该 agent 进入，把 Router 排除在被测范围之外。",
     "Entry agent": "入口 agent",
@@ -813,7 +836,11 @@
     "Percentiles are nearest-rank over the agent-call time of the cases that reached the model, so every figure is a duration some case actually took. Cases that failed before their first turn are excluded from the percentiles but still counted in tokens and cost.": "百分位取最近秩，基于真正调用到模型的那些用例的 agent 调用耗时，所以每个数字都是某条用例真实花掉的时间。第一轮之前就失败的用例不计入百分位，但仍计入 Token 与成本。",
     "Wall clock for the whole case": "整条用例的墙钟时间",
     "Agent call time only": "仅 agent 调用耗时",
-    "Tokens / cost": "Token / 成本",
+    "Agent tokens / cost": "Agent Token / 成本",
+    "Judge tokens / cost": "Judge Token / 成本",
+    "You edited the form while the assistant was working; the new draft replaced those edits. Press Undo to get them back.": "你在助手处理期间改过表单，新草稿已覆盖这些改动。点“撤销”可以找回。",
+    "Judge": "Judge",
+    "Tokens and cost cover the model under test only; llmJudge spend is listed separately.": "Token 与成本只统计被测模型；llmJudge 的消耗单独列出。",
     "Plan a scope": "规划评估范围",
     "Which cases a change needs to run, and which it does not. Read-only -- it plans a run, it does not start one.": "一次变更需要跑哪些用例、以及哪些不需要。只读 —— 它只做规划，不会启动跑批。",
     "Changed agents": "变更涉及的 agent",

src/lib/styles/pages/_agent-test.scss

@@ -609,6 +609,12 @@
     }
 }
 
+/* A summary row set apart from the data rows above it; pair with an ats-tone-* class. */
+.ats-table-row-tone > td {
+    background-color: color-mix(in srgb, var(--ats-tone) 10%, transparent);
+    border-top: 2px solid color-mix(in srgb, var(--ats-tone) 35%, transparent);
+}
+
 /* Row-level hover + pointer for the rows that navigate on click
    (was .clickable). */
 .ats-table-row-link {

src/routes/page/agent-test/+page.svelte

@@ -13,6 +13,11 @@
 	import { errorMessage, t } from '$lib/helpers/utils/agent-test.js';
 
 	const duration = 3000;
+	// One handle per banner, so a second message restarts its timer instead of being cleared early by the first one's.
+	/** @type {ReturnType<typeof setTimeout> | undefined} */
+	let successTimer;
+	/** @type {ReturnType<typeof setTimeout> | undefined} */
+	let errorTimer;
 	const nameMaxLength = 200;
 	const descriptionMaxLength = 2000;
 
@@ -73,9 +78,13 @@
 
 	function loadSuites() {
 		loadErrorText = null;
-		return getSuites(selectedAgentId).then(res => {
+		const agentId = selectedAgentId;
+		return getSuites(agentId).then(res => {
+			// Filters can change faster than the list loads; only the latest one may fill it.
+			if (agentId !== selectedAgentId) return;
 			suites = res || [];
 		}).catch(() => {
+			if (agentId !== selectedAgentId) return;
 			suites = [];
 			loadErrorText = t('Failed to load test suites. Please try again.');
 		});
@@ -97,6 +106,9 @@
 	function changeAgentFilter(e) {
 		const selecteds = e?.detail?.selecteds || [];
 		selectedAgentId = selecteds.length > 0 ? selecteds[0].value : null;
+		// Applied on pick: otherwise the picker shows one agent while the list still shows another,
+		// and New Suite already pre-fills from the picker.
+		refreshSuites();
 	}
 
 	function applyAgentFilter() {
@@ -146,15 +158,17 @@
 			isCreateModalOpen = false;
 			isComplete = true;
 			successText = t('Test suite created!');
-			setTimeout(() => {
+			clearTimeout(successTimer);
+			successTimer = setTimeout(() => {
 				isComplete = false;
 				successText = '';
 			}, duration);
 			refreshSuites();
 		}).catch(() => {
 			isError = true;
 			errorText = t('Failed to create test suite.');
-			setTimeout(() => {
+			clearTimeout(errorTimer);
+			errorTimer = setTimeout(() => {
 				isError = false;
 				errorText = '';
 			}, duration);
@@ -200,15 +214,17 @@
 		deleteSuite(suiteId).then(() => {
 			isComplete = true;
 			successText = t('Test suite deleted!');
-			setTimeout(() => {
+			clearTimeout(successTimer);
+			successTimer = setTimeout(() => {
 				isComplete = false;
 				successText = '';
 			}, duration);
 			return loadSuites();
 		}).catch(() => {
 			isError = true;
 			errorText = t('Failed to delete test suite.');
-			setTimeout(() => {
+			clearTimeout(errorTimer);
+			errorTimer = setTimeout(() => {
 				isError = false;
 				errorText = '';
 			}, duration);