|
| 1 | +/*--------------------------------------------------------------------------------------------- |
| 2 | + * A reasoning model gets the whole per-turn output ceiling, and its effort — run: node test/agentReasoningBudget.test.js |
| 3 | + * |
| 4 | + * Gemini 4 Argon reasons on every turn, and its reasoning comes out of the same max_tokens as the answer. |
| 5 | + * At the 8,192 default a hard turn could spend it all thinking and stop with nothing said. A model whose |
| 6 | + * caps say it reasons now asks for 32,768 (the ceiling, and the LevelCode gateway's own limit for a paid |
| 7 | + * request), and a model whose row names an effort sends it. Everything else keeps the user's setting. |
| 8 | + * |
| 9 | + * This drives the real agent loop with a stand-in for the provider and reads what each turn asked for. |
| 10 | + *--------------------------------------------------------------------------------------------*/ |
| 11 | +// @ts-check |
| 12 | +'use strict'; |
| 13 | + |
| 14 | +const assert = require('assert'); |
| 15 | +const fs = require('fs'); |
| 16 | +const os = require('os'); |
| 17 | +const path = require('path'); |
| 18 | +const Module = require('module'); |
| 19 | + |
| 20 | +const root = fs.mkdtempSync(path.join(os.tmpdir(), 'lc-reasoning-budget-')); |
| 21 | +const vscodeMock = { workspace: { workspaceFolders: [{ uri: { fsPath: root }, name: 'app' }] } }; |
| 22 | +const origLoad = Module._load; |
| 23 | +// @ts-ignore |
| 24 | +Module._load = function (request, parent, isMain) { |
| 25 | + if (request === 'vscode') { return vscodeMock; } |
| 26 | + return origLoad.call(this, request, parent, isMain); |
| 27 | +}; |
| 28 | +const providers = require('../providers/index'); |
| 29 | +/** @type {any[]} */ |
| 30 | +const asked = []; |
| 31 | +providers.streamAgentTurn = async (o) => { |
| 32 | + asked.push({ model: o.model, maxTokens: o.maxTokens, reasoningEffort: o.reasoningEffort }); |
| 33 | + return { stop_reason: 'end_turn', content: [{ type: 'text', text: 'Done: nothing to change.' }] }; |
| 34 | +}; |
| 35 | +const { runAgent } = require('../agent'); |
| 36 | + |
| 37 | +let n = 0; |
| 38 | +async function test(name, fn) { await fn(); n++; console.log(' ok - ' + name); } |
| 39 | + |
| 40 | +/** One single-turn run on `model` with the user's `maxTokens` setting; returns what that turn asked for. */ |
| 41 | +async function turnFor(model, maxTokens) { |
| 42 | + asked.length = 0; |
| 43 | + await runAgent({ |
| 44 | + providerId: 'openrouter', model, maxTokens, messages: [{ role: 'user', content: 'go' }], maxSteps: 2, |
| 45 | + post() {}, dbg() {}, approve: async () => true, commandRuns: new Map(), commandStops: new Map(), signal: new AbortController().signal, |
| 46 | + }); |
| 47 | + assert.strictEqual(asked.length, 1, 'one turn'); |
| 48 | + return asked[0]; |
| 49 | +} |
| 50 | + |
| 51 | +(async () => { |
| 52 | + try { |
| 53 | + await test('Gemini 4 Argon through OpenRouter or the gateway: 32,768 per turn and medium effort', async () => { |
| 54 | + assert.deepStrictEqual(await turnFor('google/gemini-4-argon', 8192), { model: 'google/gemini-4-argon', maxTokens: 32768, reasoningEffort: 'medium' }); |
| 55 | + }); |
| 56 | + await test('the bare Argon id also gets the ceiling, but asks for no effort', async () => { |
| 57 | + const t = await turnFor('gemini-4-argon', 8192); |
| 58 | + assert.strictEqual(t.maxTokens, 32768); |
| 59 | + assert.strictEqual(t.reasoningEffort, null); |
| 60 | + }); |
| 61 | + await test('other reasoning models get the ceiling too (their reasoning shares the budget the same way)', async () => { |
| 62 | + assert.strictEqual((await turnFor('moonshotai/kimi-k3', 8192)).maxTokens, 32768); |
| 63 | + assert.strictEqual((await turnFor('o3-mini', 4096)).maxTokens, 32768); |
| 64 | + }); |
| 65 | + await test('a model that does not reason keeps the user\'s setting, its clamp, and sends no effort', async () => { |
| 66 | + assert.deepStrictEqual(await turnFor('moonshotai/kimi-k2.7-code', 8192), { model: 'moonshotai/kimi-k2.7-code', maxTokens: 8192, reasoningEffort: null }); |
| 67 | + assert.strictEqual((await turnFor('moonshotai/kimi-k2.7-code', 100000)).maxTokens, 32768, 'still clamped to 32,768'); |
| 68 | + assert.strictEqual((await turnFor('moonshotai/kimi-k2.7-code', 10)).maxTokens, 1024, 'still at least 1,024'); |
| 69 | + assert.strictEqual((await turnFor('anthropic/claude-opus-5', 0)).maxTokens, 8192, 'the default when nothing is set'); |
| 70 | + }); |
| 71 | + console.log('\nagentReasoningBudget: ' + n + ' tests passed.'); |
| 72 | + } finally { |
| 73 | + Module._load = origLoad; |
| 74 | + fs.rmSync(root, { recursive: true, force: true }); |
| 75 | + } |
| 76 | +})().catch((e) => { console.error(e); process.exit(1); }); |
0 commit comments