diff --git a/src/lib/helpers/types/agentTestTypes.js b/src/lib/helpers/types/agentTestTypes.js index 7def88b1..063b0dae 100644 --- a/src/lib/helpers/types/agentTestTypes.js +++ b/src/lib/helpers/types/agentTestTypes.js @@ -317,6 +317,8 @@ * LlmConfig was used. * @property {string?} model - As above. In a multi-model run the same `caseId` yields several * results, told apart by these two fields. + * @property {string[]?} usedModels - "provider/model" of every completion the agents under test + * actually ran on, filled even when `model` is null. Can hold several (router + hand-off target). * @property {number} durationMs - Wall clock for the whole case, including the canary and the * conversation reads. Comparable between models, but not a model-latency measurement. * @property {number} modelDurationMs - The agent calls alone, summed over the turns. This is what the diff --git a/src/lib/helpers/utils/agent-test.js b/src/lib/helpers/utils/agent-test.js index 67a078bd..a6ff1fc3 100644 --- a/src/lib/helpers/utils/agent-test.js +++ b/src/lib/helpers/utils/agent-test.js @@ -59,6 +59,13 @@ const TARGET_REQUIRED = ['toolCalled', 'toolNotCalled', 'stateEquals']; */ export const AGENT_CHAIN_MODES = ['contains', 'ordered', 'exact']; +/** + * Which text an output assertion or llmJudge reads (OutputSources in + * AssertionEvaluator.cs), carried in `target`. Blank means `reply`. + */ +export const OUTPUT_SOURCES = ['reply', 'richContent']; +export const OUTPUT_SOURCE_TYPES = ['outputContains', 'outputNotContains', 'outputRegex', 'llmJudge']; + /** * What a case is verifying (CaseTypes in AgentTestCase.cs). Routing is checked * more strictly -- see `validateCaseType` -- because it is the only type counted @@ -241,6 +248,13 @@ export function validateAssertion(assertion) { return t('Assertion "agentChain" mode must be one of {modes}.', { modes: AGENT_CHAIN_MODES.join(', ') }); } + // Same for the output source: an unrecognised value fails the assertion rather + // than falling back to the reply text. Blank is legal and means `reply`. + if (OUTPUT_SOURCE_TYPES.includes(type) && assertion.target?.trim() + && !OUTPUT_SOURCES.some(s => s.toLowerCase() === assertion.target.trim().toLowerCase())) { + return t('Assertion "{type}" target must be one of {sources}.', { type, sources: OUTPUT_SOURCES.join(', ') }); + } + if (assertion.argsMatchJson?.trim() && !isParsableJson(assertion.argsMatchJson)) { return t('The args match on an assertion is not valid JSON.'); } @@ -302,6 +316,20 @@ export function formatAgentChain(chain) { return chain?.length ? chain.join(' -> ') : '--'; } +/** + * A turn's rich content, pretty-printed. Stored as a JSON string; anything that does not parse is + * shown as-is rather than hidden. + * @param {string} richContent + * @returns {string} + */ +export function formatRichContent(richContent) { + try { + return JSON.stringify(JSON.parse(richContent), null, 2); + } catch { + return richContent; + } +} + /** * @param {string} text * @returns {boolean} diff --git a/src/lib/langs/en.json b/src/lib/langs/en.json index 75430d0d..9c694915 100644 --- a/src/lib/langs/en.json +++ b/src/lib/langs/en.json @@ -558,6 +558,7 @@ "A Routing case cannot use llmJudge: routing is judged only by which agent handled the conversation.": "A Routing case cannot use llmJudge: routing is judged only by which agent handled the conversation.", "Mode": "Mode", "Assertion \"agentChain\" mode must be one of {modes}.": "Assertion \"agentChain\" mode must be one of {modes}.", + "Assertion \"{type}\" target must be one of {sources}.": "Assertion \"{type}\" target must be one of {sources}.", "Copilot, Work Order Creator": "Copilot, Work Order Creator", "These agents appear somewhere in the chain, in any order.": "These agents appear somewhere in the chain, in any order.", "These agents appear in this order; other agents may come in between. This is the hand-off assertion.": "These agents appear in this order; other agents may come in between. This is the hand-off assertion.", diff --git a/src/lib/langs/zh.json b/src/lib/langs/zh.json index 574713a3..9b55c231 100644 --- a/src/lib/langs/zh.json +++ b/src/lib/langs/zh.json @@ -752,6 +752,7 @@ "A Routing case cannot use llmJudge: routing is judged only by which agent handled the conversation.": "路由用例不能用 llmJudge:路由只按由哪个 agent 处理了对话来判定。", "Mode": "比较方式", "Assertion \"agentChain\" mode must be one of {modes}.": "agentChain 断言的比较方式只能是 {modes} 之一。", + "Assertion \"{type}\" target must be one of {sources}.": "{type} 断言的 Target 只能是 {sources} 之一。", "Copilot, Work Order Creator": "Copilot, Work Order Creator", "These agents appear somewhere in the chain, in any order.": "这些 agent 出现在链路中的任意位置,不论顺序。", "These agents appear in this order; other agents may come in between. This is the hand-off assertion.": "这些 agent 按此顺序出现,中间允许有其他 agent。这条就是交接断言。", diff --git a/src/routes/page/agent-test/[suiteId]/case/[caseId]/+page.svelte b/src/routes/page/agent-test/[suiteId]/case/[caseId]/+page.svelte index c67e4205..ea6db9f3 100644 --- a/src/routes/page/agent-test/[suiteId]/case/[caseId]/+page.svelte +++ b/src/routes/page/agent-test/[suiteId]/case/[caseId]/+page.svelte @@ -14,6 +14,8 @@ import { ASSERTION_TYPES, AGENT_CHAIN_MODES, + OUTPUT_SOURCES, + OUTPUT_SOURCE_TYPES, HISTORY_ROLES, CASE_PRIORITIES, CASE_SEVERITIES, @@ -32,6 +34,8 @@ /** Assertion types as Select options. Constant -- the list never changes at runtime. */ const ASSERTION_TYPE_OPTIONS = ASSERTION_TYPES.map(name => ({ label: name, value: name })); const AGENT_CHAIN_MODE_OPTIONS = AGENT_CHAIN_MODES.map(name => ({ label: name, value: name })); + /** `reply` is shown as "content": it is the reply text, as opposed to the rich content. */ + const OUTPUT_SOURCE_OPTIONS = OUTPUT_SOURCES.map(name => ({ label: name === 'reply' ? 'content' : name, value: name })); let isLoading = $state(false); let isComplete = $state(false); @@ -665,6 +669,12 @@ assertion.target = ''; } + // Output types carry the output source in `target` the same way. + const isSource = OUTPUT_SOURCES.some(s => s.toLowerCase() === (assertion.target || '').trim().toLowerCase()); + if (OUTPUT_SOURCE_TYPES.includes(assertion.type) ? !isSource : isSource) { + assertion.target = ''; + } + // Both fields mean something different under the new type, so an explicit "let me type it" from // the old one does not carry over. A value the new picker cannot show still opens in the text // box on its own -- see isManualEntry. @@ -905,7 +915,7 @@