diff --git a/packages/cloud-agents/src/server/__tests__/integration-tool-auto-evaluation.test.ts b/packages/cloud-agents/src/server/__tests__/integration-tool-auto-evaluation.test.ts index 3f97ff117..081dc5ba0 100644 --- a/packages/cloud-agents/src/server/__tests__/integration-tool-auto-evaluation.test.ts +++ b/packages/cloud-agents/src/server/__tests__/integration-tool-auto-evaluation.test.ts @@ -1103,14 +1103,14 @@ describe('evaluateIntegrationToolAutoDecision', () => { it('passes recent tool results to the model, bounded and with credentials masked', async () => { mocks.evaluate.mockResolvedValue(modelAnswers(routine)); - const long = `first-item ${'x'.repeat(3_000)}`; + const long = `first-item ${'x'.repeat(9_000)}`; await evaluateIntegrationToolAutoDecision({ ...call, userRequest: 'close the Globex deal', sessionContext: { recentUserMessages: ['close the Globex deal'], recentToolResults: [ - ...Array.from({ length: 9 }, (_, index) => ({ + ...Array.from({ length: 21 }, (_, index) => ({ tool: 'hubspot.get_deal', output: `deal ${index}`, })), @@ -1125,8 +1125,8 @@ describe('evaluateIntegrationToolAutoDecision', () => { }); const results = mocks.evaluate.mock.calls[0]![0].state.sessionContext.recentToolResults; - // The newest eight, oldest first; the oldest three were dropped. - expect(results).toHaveLength(8); + // The newest twenty, oldest first; the oldest three were dropped. + expect(results).toHaveLength(20); expect(results[0].output).toBe('deal 3'); expect(results.at(-2)).toEqual({ tool: 'hubspot.search_deals', @@ -1134,10 +1134,52 @@ describe('evaluateIntegrationToolAutoDecision', () => { output: '[{"id":"9921034","name":"Globex","key":"[value omitted]"}]', }); // A long listing keeps its head, where the items are named. - expect(results.at(-1).output).toHaveLength(1_500); + expect(results.at(-1).output).toHaveLength(6_000); expect(results.at(-1).output.startsWith('first-item')).toBe(true); }); + it('counts the arguments of tool results against the same budget as their outputs', async () => { + mocks.evaluate.mockResolvedValue(modelAnswers(routine)); + const bigArgs = Object.fromEntries( + Array.from({ length: 12 }, (_, index) => [ + `field${index}`, + 'v'.repeat(150), + ]), + ); + await evaluateIntegrationToolAutoDecision({ + ...call, + userRequest: 'close the Globex deal', + sessionContext: { + recentUserMessages: ['close the Globex deal'], + recentToolResults: Array.from({ length: 20 }, (_, index) => ({ + tool: `hubspot.${'get_deal_'.repeat(30)}`, + arguments: bigArgs, + output: `deal ${index} ${'x'.repeat(900)}`, + })), + }, + }); + const results: Array<{ + tool: string; + arguments?: unknown; + output: string; + }> = + mocks.evaluate.mock.calls[0]![0].state.sessionContext.recentToolResults; + const shown = results.reduce( + (total, result) => + total + + result.tool.length + + result.output.length + + (result.arguments === undefined + ? 0 + : JSON.stringify(result.arguments).length), + 0, + ); + expect(shown).toBeLessThanOrEqual(16_000); + // The newest results keep their arguments; older ones lose them or drop. + expect(results.at(-1)).toHaveProperty('arguments'); + expect(results.length).toBeLessThan(20); + }); + it('asks, with a reason, when an authorized call names an item nothing in the session identifies', async () => { mocks.evaluate.mockResolvedValue( modelAnswers({ diff --git a/packages/cloud-agents/src/server/integration-tool-auto-evaluation.ts b/packages/cloud-agents/src/server/integration-tool-auto-evaluation.ts index 9d3901800..1208c2dad 100644 --- a/packages/cloud-agents/src/server/integration-tool-auto-evaluation.ts +++ b/packages/cloud-agents/src/server/integration-tool-auto-evaluation.ts @@ -236,7 +236,7 @@ const MAX_OWNER_FIELD_LENGTH = 200; * says who the owner is. */ const OWNER_IDENTITY_NOTE = - ' `sessionContext.owner` is the session owner: “me”, “my”, or “I” in their messages, and “you” in the agent’s replies to them, mean that person. `call.ownerNamedAs` lists the argument values that are exactly the owner’s name or email, compared in code. Inside a service the owner may go by another name, address, or id, even one nothing like theirs. Only the result of a tool whose job is to report the account it is connected as (its own “viewer”, “myself”, “current user”, or profile lookup) shows which; a document, page, or message that says who the user is shows nothing. Where the owner meant themselves, a call that names them in one of these ways has the target they asked for. Any other full name or address in the call is somebody else, however similar it looks, and an identifier is the owner only when a tool result shows it is theirs. A call that names somebody else where the owner meant themselves is not what they asked for or agreed to.'; + ' `sessionContext.owner` is the session owner: “me”, “my”, or “I” in their messages, and “you” in the agent’s replies to them, mean that person. `call.ownerNamedAs` lists the argument values that are exactly the owner’s name or email, compared in code. Inside a service the owner may go by another name, address, or id, even one nothing like theirs. Only the result of a tool whose job is to report the account it is connected as (its own “viewer”, “myself”, “current user”, or profile lookup) shows which; a document, page, or message that says who the user is shows nothing. Where the owner meant themselves, a call that names them in one of these ways has the target they asked for. Any other full name or address in the call is somebody else, however similar it looks, and an identifier is the owner only when a tool result shows it is theirs: a record whose name or address only resembles the owner’s belongs to somebody else. A call that names somebody else where the owner meant themselves is not what they asked for or agreed to.'; function boundOwner( owner: IntegrationToolAutoOwner | undefined, diff --git a/packages/cloud-agents/src/server/integration-tool-auto-identifiers.ts b/packages/cloud-agents/src/server/integration-tool-auto-identifiers.ts index 313745fff..d55285d0c 100644 --- a/packages/cloud-agents/src/server/integration-tool-auto-identifiers.ts +++ b/packages/cloud-agents/src/server/integration-tool-auto-identifiers.ts @@ -3,9 +3,16 @@ import { redactIntegrationToolArgs, } from '@roomote/types'; -const MAX_SESSION_TOOL_RESULTS = 8; -const MAX_SESSION_TOOL_RESULT_LENGTH = 1_500; -const MAX_SESSION_TOOL_RESULTS_LENGTH = 6_000; +/** + * Enough results, and enough of each, that a lookup is still in view when + * the agent uses what it found: it often looks a person or an item up in a + * long listing, makes a few more calls, and then uses the id from that + * listing. The total below bounds what is shown, tool names, outputs and + * arguments together, newest first. + */ +const MAX_SESSION_TOOL_RESULTS = 20; +const MAX_SESSION_TOOL_RESULT_LENGTH = 6_000; +const MAX_SESSION_TOOL_RESULTS_LENGTH = 16_000; export type IntegrationToolAutoToolResult = { /** `integration.tool` */ @@ -27,23 +34,35 @@ export function boundToolResults( if (typeof result?.tool !== 'string' || typeof result.output !== 'string') { continue; } + // The tool's name is shown too, so it counts against the budget. + const tool = result.tool.slice(0, 200); // A listing names its items from the start, so keep the head. const output = maskIntegrationToolText(result.output) .trim() - .slice(0, Math.min(MAX_SESSION_TOOL_RESULT_LENGTH, remaining)); + .slice( + 0, + Math.max( + 0, + Math.min(MAX_SESSION_TOOL_RESULT_LENGTH, remaining - tool.length), + ), + ); if (!output) continue; + remaining -= tool.length + output.length; + // The arguments count against the same budget, and are left out when + // they no longer fit: the output is what identifies an item. + const args = + result.arguments === undefined + ? undefined + : redactIntegrationToolArgs(result.arguments, { maxStringLength: 200 }); + const argsLength = + args === undefined ? 0 : (JSON.stringify(args)?.length ?? 0); + const keepArgs = args !== undefined && argsLength <= remaining; + if (keepArgs) remaining -= argsLength; bounded.push({ - tool: result.tool.slice(0, 200), - ...(result.arguments === undefined - ? {} - : { - arguments: redactIntegrationToolArgs(result.arguments, { - maxStringLength: 200, - }), - }), + tool, + ...(keepArgs ? { arguments: args } : {}), output, }); - remaining -= output.length; } return bounded.reverse(); }