From b2c5aee945f814e4665afe8e35207fd1ebfd8fec Mon Sep 17 00:00:00 2001 From: "nick.yi" Date: Sat, 31 Oct 2026 11:10:22 +0800 Subject: [PATCH 1/9] feat(agent-testing): add Template case type and keep turn files on save - Template cases call one of the entry agent's templates directly, so they need an entry agent and a template picked from it, and cannot have history - Turn files have no editor yet; carry them through load and save so editing a case in the UI does not clear files attached through the API Co-Authored-By: Claude Opus 5.5 --- src/lib/helpers/types/agentTestTypes.js | 11 ++- src/lib/langs/en.json | 9 ++ src/lib/langs/zh.json | 9 ++ .../page/agent-test/[suiteId]/+page.svelte | 2 + .../[suiteId]/case/[caseId]/+page.svelte | 82 ++++++++++++++++++- 5 files changed, 107 insertions(+), 6 deletions(-) diff --git a/src/lib/helpers/types/agentTestTypes.js b/src/lib/helpers/types/agentTestTypes.js index 063b0dae..c3ccf3a0 100644 --- a/src/lib/helpers/types/agentTestTypes.js +++ b/src/lib/helpers/types/agentTestTypes.js @@ -42,6 +42,7 @@ * @typedef {Object} TestTurn * @property {number} index * @property {string} userMessage + * @property {{ fileName: string, fileUrl: string }[]} [files] - Attached to the user message. * @property {TestAssertion[]} assertions - Turn-level assertions, as opposed to the case-level * AgentTestCase.assertions. */ @@ -104,7 +105,7 @@ * @property {string} name * @property {boolean} enabled - Recorded drafts land as false; a human has to review and enable them * before they join a normal run. - * @property {string} caseType - Routing | Agent. Defaults to Agent, which is also what every case + * @property {string} caseType - Routing | Agent | Template. Defaults to Agent, which is also what every case * stored before the field existed reads back as. Routing is validated more strictly (one turn, * must assert a routing outcome, no llmJudge) and is the only type counted towards a run's routing * accuracy. A journey across several agents is an Agent case whose `agentChain` assertion @@ -113,6 +114,7 @@ * uses the suite's. This is the switch between testing routing and testing one agent alone: * BotSharp sends a routing-type agent through the router (which can hand off) and any other agent * straight into itself (the router never runs). + * @property {string?} templateName - The entry agent's template a Template case calls directly. * @property {TestTurn[]} turns - Length 1 is a single-turn case. * @property {TestAssertion[]} assertions - Case-level assertions, evaluated after every turn has run. * @property {TestState[]} initialStates - Injected before the conversation starts; maps to BotSharp's @@ -156,10 +158,11 @@ * @property {string} suiteId * @property {string} name * @property {boolean} [enabled] - Defaults to true. - * @property {string} [caseType] - Routing | Agent; blank or omitted means Agent. Any other value is - * a 400 rather than a silent fallback. + * @property {string} [caseType] - Routing | Agent | Template; blank or omitted means Agent. Any other + * value is a 400 rather than a silent fallback. * @property {string?} [entryAgentId] - Send null rather than "" to mean "use the suite's agent"; - * validated to exist at save time. + * validated to exist at save time. Required for a Template case. + * @property {string?} [templateName] - Required for a Template case; must exist on the entry agent. * @property {TestHistoryMessage[]} [history] - Authored prior turns; every message needs a * user/assistant role and non-empty content. * @property {string} [priority] - P0 | P1 | P2; blank keeps P1. Any other value is a 400. diff --git a/src/lib/langs/en.json b/src/lib/langs/en.json index 9c694915..dbc84009 100644 --- a/src/lib/langs/en.json +++ b/src/lib/langs/en.json @@ -550,6 +550,15 @@ "Case type": "Case type", "Routing case": "Routing case", "Agent case": "Agent case", + "Template case": "Template case", + "Calls one of the entry agent's templates directly, the way code calls it outside a conversation. No routing, no function calls; each turn is an independent call.": "Calls one of the entry agent's templates directly, the way code calls it outside a conversation. No routing, no function calls; each turn is an independent call.", + "Template": "Template", + "Select a template": "Select a template", + "Select an entry agent first": "Select an entry agent first", + "read-only, maintained through the API": "read-only, maintained through the API", + "A Template case needs an entry agent: the agent that owns the template.": "A Template case needs an entry agent: the agent that owns the template.", + "A Template case needs a template.": "A Template case needs a template.", + "A Template case cannot have history: a template call does not read the conversation.": "A Template case cannot have history: a template call does not read the conversation.", "One turn, entered on the Copilot entry agent, asserting only which agent took the conversation. The only type counted towards a run's routing accuracy.": "One turn, entered on the Copilot entry agent, asserting only which agent took the conversation. The only type counted towards a run's routing accuracy.", "One agent's own behaviour. Enter directly on that agent to keep the router out of what is measured.": "One agent's own behaviour. Enter directly on that agent to keep the router out of what is measured.", "Entry agent": "Entry agent", diff --git a/src/lib/langs/zh.json b/src/lib/langs/zh.json index 9b55c231..dd29367a 100644 --- a/src/lib/langs/zh.json +++ b/src/lib/langs/zh.json @@ -744,6 +744,15 @@ "Case type": "用例类型", "Routing case": "路由用例", "Agent case": "Agent 用例", + "Template case": "模板用例", + "Calls one of the entry agent's templates directly, the way code calls it outside a conversation. No routing, no function calls; each turn is an independent call.": "直接调用入口 agent 的某个模板，和代码在对话之外调用它的方式一致。不走路由、不调函数；每一轮都是独立的一次调用。", + "Template": "模板", + "Select a template": "选择模板", + "Select an entry agent first": "请先选择入口 agent", + "read-only, maintained through the API": "只读，通过 API 维护", + "A Template case needs an entry agent: the agent that owns the template.": "模板用例需要指定入口 agent，即模板所属的 agent。", + "A Template case needs a template.": "模板用例需要选择一个模板。", + "A Template case cannot have history: a template call does not read the conversation.": "模板用例不能带历史消息：模板调用不读取对话。", "One turn, entered on the Copilot entry agent, asserting only which agent took the conversation. The only type counted towards a run's routing accuracy.": "单轮，从 Copilot 入口 agent 进入，只断言由哪个 agent 接走了这段对话。只有这一类会计入跑批的路由准确率。", "One agent's own behaviour. Enter directly on that agent to keep the router out of what is measured.": "单个 agent 自身的行为。直接从该 agent 进入，把 Router 排除在被测范围之外。", "Entry agent": "入口 agent", diff --git a/src/routes/page/agent-test/[suiteId]/+page.svelte b/src/routes/page/agent-test/[suiteId]/+page.svelte index b433a8d3..6ae60483 100644 --- a/src/routes/page/agent-test/[suiteId]/+page.svelte +++ b/src/routes/page/agent-test/[suiteId]/+page.svelte @@ -907,6 +907,8 @@ case to discover. --> {#if testCase.caseType === 'Routing'} {$_('Routing case')} + {:else if testCase.caseType === 'Template'} + {$_('Template case')} {:else} {$_('Agent case')} {/if} diff --git a/src/routes/page/agent-test/[suiteId]/case/[caseId]/+page.svelte b/src/routes/page/agent-test/[suiteId]/case/[caseId]/+page.svelte index ea6db9f3..87ebd637 100644 --- a/src/routes/page/agent-test/[suiteId]/case/[caseId]/+page.svelte +++ b/src/routes/page/agent-test/[suiteId]/case/[caseId]/+page.svelte @@ -8,7 +8,7 @@ import LoadingToComplete from '$lib/common/spinners/LoadingToComplete.svelte'; import Select from '$lib/common/dropdowns/Select.svelte'; import { getSuite, getCase, createCase, updateCase, getMockTargets, authorCase } from '$lib/services/agent-test-service.js'; - import { getAgentOptions } from '$lib/services/agent-service.js'; + import { getAgent, getAgentOptions } from '$lib/services/agent-service.js'; import { getLlmConfigs } from '$lib/services/llm-provider-service'; import { LlmModelCapability, LlmModelType } from '$lib/helpers/enums'; import { @@ -170,6 +170,7 @@ enabled: true, caseType: 'Agent', entryAgentId: '', + templateName: '', history: [], priority: 'P1', severity: 'S1', @@ -215,6 +216,39 @@ untrack(() => ensureMockTargets(ids)); }); + /** + * Template names per agent id, for the Template case picker. Cached like the tool lists: the entry + * agent picker gets flipped back and forth while an author settles on one. + * @type {Record} + */ + let templatesByAgent = $state({}); + + let templateOptions = $derived((templatesByAgent[form.entryAgentId] || []) + .map(name => ({ label: name, value: name }))); + + $effect(() => { + const id = form.caseType === 'Template' ? form.entryAgentId : ''; + untrack(() => ensureTemplates(id)); + }); + + /** @param {string} id */ + async function ensureTemplates(id) { + if (!id || id in templatesByAgent) return; + templatesByAgent[id] = []; + await getAgent(id) + .then((/** @type {any} */ agent) => { + templatesByAgent[id] = (agent?.templates || []).map((/** @type {any} */ x) => x.name).sort(); + }) + // Costs the picker its options and nothing more; the backend still rejects an unknown template. + .catch(() => {}); + } + + /** @param {any} e */ + function changeTemplate(e) { + const selecteds = e?.detail?.selecteds || []; + form.templateName = selecteds.length > 0 ? selecteds[0].value : ''; + } + /** Blocking problems, recomputed on every keystroke and shown before the user can save. */ let validationErrors = $derived(collectValidationErrors(form)); let canSave = $derived(validationErrors.length === 0 && !isSaving); @@ -353,6 +387,7 @@ // untriaged at P1/S1. caseType: res.caseType || 'Agent', entryAgentId: res.entryAgentId || '', + templateName: res.templateName || '', history: (res.history || []).map((/** @type {any} */ m) => ({ role: m.role || 'user', content: m.content || '' @@ -368,6 +403,8 @@ turns: (res.turns || []).map((/** @type {any} */ turn, /** @type {number} */ i) => ({ index: i, userMessage: turn.userMessage || '', + // No editor for these yet; carried through so a save does not clear them. + files: turn.files || [], assertions: (turn.assertions || []).map(normalizeAssertion) })), assertions: (res.assertions || []).map(normalizeAssertion), @@ -439,7 +476,7 @@ /** @param {number} index */ function newTurn(index) { - return { index: index, userMessage: '', assertions: [] }; + return { index: index, userMessage: '', files: [], assertions: [] }; } function newAssertion() { @@ -504,6 +541,18 @@ // drift from what the backend would reject on save. errors.push(...validateCaseType(value.caseType, value.turns, value.assertions)); + if (value.caseType === 'Template') { + if (!value.entryAgentId) { + errors.push(t('A Template case needs an entry agent: the agent that owns the template.')); + } + if (!value.templateName?.trim()) { + errors.push(t('A Template case needs a template.')); + } + if (value.history?.length) { + errors.push(t('A Template case cannot have history: a template call does not read the conversation.')); + } + } + // Rejected by the backend, and worth catching here: BotSharp dialog storage drops elements // with blank content, so an empty message would silently not be in the conversation and the // runner would then fail the whole case for a write that vanished. @@ -592,6 +641,7 @@ // Null, not '', so the backend "fall back to the suite agent" check reads it correctly -- // an empty string would be a request to enter on an agent whose id is empty. entryAgentId: form.entryAgentId?.trim() || null, + templateName: form.caseType === 'Template' ? form.templateName?.trim() || null : null, history: form.history.map(m => ({ role: m.role, content: m.content ?? '' })), priority: form.priority || 'P1', severity: form.severity || 'S1', @@ -608,6 +658,7 @@ turns: form.turns.map((turn, i) => ({ index: i, userMessage: turn.userMessage.trim(), + files: turn.files || [], assertions: (turn.assertions || []).map(cleanAssertion) })), assertions: form.assertions.map(cleanAssertion), @@ -1437,10 +1488,13 @@

{#if form.caseType === 'Routing'} {$_('One turn, entered on the Copilot entry agent, asserting only which agent took the conversation. The only type counted towards a run\'s routing accuracy.')} + {:else if form.caseType === 'Template'} + {$_('Calls one of the entry agent\'s templates directly, the way code calls it outside a conversation. No routing, no function calls; each turn is an independent call.')} {:else} {$_('One agent\'s own behaviour. Enter directly on that agent to keep the router out of what is measured. A journey across several agents is an Agent case too -- use an agentChain assertion for the hand-offs.')} {/if} @@ -1463,6 +1517,21 @@ {$_('A routing agent runs the router and can hand off; any other agent is entered directly, so the router never runs.')}

+ {#if form.caseType === 'Template'} +
+ + o.value === assertion.type || !TEMPLATE_UNUSABLE_TYPES.includes(o.value)) + : ASSERTION_TYPE_OPTIONS} onselect={e => changeAssertionType(assertion, e)} /> {#if assertion.type === 'llmJudge'} @@ -1627,16 +1657,55 @@ rows="2" bind:value={turn.userMessage} > - {#if turn.files?.length} -
- - {$_('Files')} ({turn.files.length}) · {$_('read-only, maintained through the API')} - {#each turn.files as file} -
{file.fileName}   {file.fileUrl}
- {/each} -
- {/if}
+
+ + {$_('Files')} ({turn.files.length}) + + +
+ {#each turn.files as file, j} +
+
+ + +
+
+ + +
+
+ +
+
+ {/each}
{$_('Assertions checked right after this turn')} ({turn.assertions.length}) From 68fdfaf72e158805ab8ae2faad384d86f9c26ce7 Mon Sep 17 00:00:00 2001 From: "nick.yi" Date: Sat, 31 Oct 2026 12:53:27 +0800 Subject: [PATCH 4/9] fix(agent-testing): fix run page reloads, re-run models and silent load failures - Run page reloads when the run id changes: Re-run Failures navigates on the same route, so the mount-only load kept showing the old run and never polled - Stop polling a run that never loaded, keep the last snapshot on a failed poll, and never overlap polls so an older response cannot overwrite a newer one - Re-run failures in one run per model set: a run applies its models to every case, so a single run forced one case's model onto the others - Key per-model stats and pricing by provider and model, not model alone - Suite page shows a retryable error when cases or runs fail to load instead of the empty states, which read as data loss - Clear the template when an AI draft changes the entry agent, and keep Template cases off richContent, which a template call never returns - Stop rejecting .NET-only regex syntax that JS RegExp cannot parse - Add missing en/zh strings Co-Authored-By: Claude Opus 5.5 --- src/lib/helpers/utils/agent-test.js | 11 +- src/lib/langs/en.json | 8 ++ src/lib/langs/zh.json | 8 ++ .../page/agent-test/[suiteId]/+page.svelte | 43 +++++-- .../[suiteId]/case/[caseId]/+page.svelte | 17 ++- .../page/agent-test/run/[runId]/+page.svelte | 105 +++++++++++++----- 6 files changed, 152 insertions(+), 40 deletions(-) diff --git a/src/lib/helpers/utils/agent-test.js b/src/lib/helpers/utils/agent-test.js index 55046c5a..f0420bc2 100644 --- a/src/lib/helpers/utils/agent-test.js +++ b/src/lib/helpers/utils/agent-test.js @@ -49,6 +49,9 @@ const EXPECTED_REQUIRED = [ 'llmJudge' ]; +/** .NET regex constructs JS RegExp rejects; see the outputRegex check in validateAssertion. */ +const DOTNET_ONLY_REGEX_SYNTAX = /\(\?[imnsx-]+[:)]|\(\?>|\(\?#|\(\?\(|\\[AZzG]|-\[/; + /** Types whose `target` the backend rejects as empty. */ const TARGET_REQUIRED = ['toolCalled', 'toolNotCalled', 'stateEquals']; @@ -231,11 +234,17 @@ export function validateAssertion(assertion) { return t('Assertion "stateEquals" needs an expected value, otherwise it can never pass.'); } + // Checked with JS RegExp as an approximation of the backend's .NET Regex. A pattern JS cannot + // parse is let through when it uses .NET-only syntax -- inline options (?i), atomic (?>...), + // comments (?#...), conditionals (?(...)), anchors \A \Z \z \G, class subtraction -[...] -- since + // JS cannot judge it and the backend would accept it. if (type === 'outputRegex' && assertion.expected?.trim()) { try { new RegExp(assertion.expected); } catch { - return t('"{pattern}" is not a valid regular expression.', { pattern: assertion.expected }); + if (!DOTNET_ONLY_REGEX_SYNTAX.test(assertion.expected)) { + return t('"{pattern}" is not a valid regular expression.', { pattern: assertion.expected }); + } } } diff --git a/src/lib/langs/en.json b/src/lib/langs/en.json index b802d3d7..c7845028 100644 --- a/src/lib/langs/en.json +++ b/src/lib/langs/en.json @@ -495,6 +495,14 @@ "Failed to load this test case.": "Failed to load this test case.", "Failed to load the suite this case belongs to.": "Failed to load the suite this case belongs to.", "Failed to load this run.": "Failed to load this run.", + "Rich content": "Rich content", + "Suite default ({provider} / {model})": "Suite default ({provider} / {model})", + "No suite default -- pick a model to use chat authoring": "No suite default -- pick a model to use chat authoring", + "The assistant could not answer.": "The assistant could not answer.", + "Queued {n} re-runs, one per model the failures ran on.": "Queued {n} re-runs, one per model the failures ran on.", + "Failed to load the test cases.": "Failed to load the test cases.", + "Failed to load the run history.": "Failed to load the run history.", + "A Template case cannot assert on richContent: a template call returns text only.": "A Template case cannot assert on richContent: a template call returns text only.", "Failed to create test suite.": "Failed to create test suite.", "Failed to delete test suite.": "Failed to delete test suite.", "Failed to save suite settings.": "Failed to save suite settings.", diff --git a/src/lib/langs/zh.json b/src/lib/langs/zh.json index 0a7653e1..d6ddc71b 100644 --- a/src/lib/langs/zh.json +++ b/src/lib/langs/zh.json @@ -689,6 +689,14 @@ "Failed to load this test case.": "加载这条测试用例失败。", "Failed to load the suite this case belongs to.": "加载这条用例所属的套件失败。", "Failed to load this run.": "加载这次运行失败。", + "Rich content": "富内容", + "Suite default ({provider} / {model})": "套件默认（{provider} / {model}）", + "No suite default -- pick a model to use chat authoring": "套件未设默认模型 -- 选择一个模型才能用对话写用例", + "The assistant could not answer.": "助手无法回答。", + "Queued {n} re-runs, one per model the failures ran on.": "已排队 {n} 次重跑，按失败用例所用的模型各一次。", + "Failed to load the test cases.": "加载测试用例失败。", + "Failed to load the run history.": "加载运行记录失败。", + "A Template case cannot assert on richContent: a template call returns text only.": "模板用例不能对 richContent 做断言：模板调用只返回文本。", "Failed to create test suite.": "创建测试套件失败。", "Failed to delete test suite.": "删除测试套件失败。", "Failed to save suite settings.": "保存套件设置失败。", diff --git a/src/routes/page/agent-test/[suiteId]/+page.svelte b/src/routes/page/agent-test/[suiteId]/+page.svelte index 6ae60483..e3edddcd 100644 --- a/src/routes/page/agent-test/[suiteId]/+page.svelte +++ b/src/routes/page/agent-test/[suiteId]/+page.svelte @@ -50,6 +50,15 @@ /** @type {string | null} */ let loadErrorText = $state(null); + /** + * Per-list load failures. Without these a failed fetch fell through to the empty states, which + * read as "this suite has no cases / was never run" -- as if the data were gone. + * @type {string | null} + */ + let casesLoadError = $state(null); + /** @type {string | null} */ + let runsLoadError = $state(null); + let suiteId = $derived(page.params.suiteId); /** @type {import('$agentTestTypes').AgentTestSuite | null} */ @@ -272,25 +281,28 @@ function loadCases() { return getCases(suiteId).then(res => { + casesLoadError = null; cases = res || []; // Drop ticks for cases that no longer exist, otherwise a stale id ends // up in caseIds and the run comes back Error with nothing matched. const ids = new Set(cases.map(x => x.id)); selectedCaseIds = selectedCaseIds.filter(id => ids.has(id)); - }).catch(() => { - cases = []; + }).catch(err => { + // The previous list, if any, stays: it is stale at worst, never wrong about what exists. + casesLoadError = errorMessage(err, t('Failed to load the test cases.')); }); } function loadRuns() { return getRuns(suiteId).then(res => { + runsLoadError = null; runs = res || []; // Drop selections for runs that are gone, or a later delete would post ids the server // can only answer "already deleted" for. const ids = new Set(runs.map(r => r.id)); selectedRunIds = selectedRunIds.filter(id => ids.has(id)); - }).catch(() => { - runs = []; + }).catch(err => { + runsLoadError = errorMessage(err, t('Failed to load the run history.')); }); } @@ -712,6 +724,15 @@ toggleModal={closeDeleteCaseModal} /> +{#snippet listLoadError(/** @type {string} */ text, /** @type {() => Promise} */ retry)} + +{/snippet} + {#if loadErrorText}
@@ -835,7 +856,10 @@
- {#if cases.length === 0} + {#if casesLoadError} + {@render listLoadError(casesLoadError, loadCases)} + {/if} + {#if cases.length === 0 && !casesLoadError}

{$_('No test cases in this suite yet.')}

@@ -850,7 +874,7 @@
- {:else} + {:else if cases.length > 0}
@@ -1054,11 +1078,14 @@
- {#if runs.length === 0} + {#if runsLoadError} + {@render listLoadError(runsLoadError, loadRuns)} + {/if} + {#if runs.length === 0 && !runsLoadError}

{$_('This suite has never been run.')}

- {:else} + {:else if runs.length > 0}
diff --git a/src/routes/page/agent-test/[suiteId]/case/[caseId]/+page.svelte b/src/routes/page/agent-test/[suiteId]/case/[caseId]/+page.svelte index acd4a54c..2e2cb9e7 100644 --- a/src/routes/page/agent-test/[suiteId]/case/[caseId]/+page.svelte +++ b/src/routes/page/agent-test/[suiteId]/case/[caseId]/+page.svelte @@ -574,6 +574,12 @@ if (used.length) { errors.push(t('A Template case cannot use {types} assertions: a template call does not route or call functions, so their result is fixed.', { types: [...new Set(used)].join(', ') })); } + // A template call returns text only (InstructAsync passes no rich content), so this source + // is always empty. Case-insensitive like the backend's source match. + if ([...(value.turns || []).flatMap(turn => turn.assertions || []), ...(value.assertions || [])] + .some(a => OUTPUT_SOURCE_TYPES.includes(a?.type) && a?.target?.trim().toLowerCase() === 'richcontent')) { + errors.push(t('A Template case cannot assert on richContent: a template call returns text only.')); + } } // Rejected by the backend, and worth catching here: BotSharp dialog storage drops elements @@ -944,7 +950,12 @@ // JSON round-trip rather than keeping the reactive object: history entries have to be inert // snapshots, or undo would restore something that has since been edited in place. draftHistory = [...draftHistory, JSON.parse(JSON.stringify(form))]; - form = toForm(draft); + const next = toForm(draft); + // Same rule as picking another entry agent by hand: the template belongs to the old one. + if (next.entryAgentId !== form.entryAgentId) { + next.templateName = ''; + } + form = next; } function undoDraft() { @@ -1017,7 +1028,9 @@