Skip to content

Commit e5cd647

Browse files
TheGreatAxiosTheGreatAxios
committed
Add per-director prompt size budgets assembled per family
Assembled sizes drift silently as prompts grow; a numeric budget per director and model family catches bloat in CI while leaving copy edits free to land.
1 parent a61cd76 commit e5cd647

2 files changed

Lines changed: 267 additions & 0 deletions

File tree

‎src/agent/prompt-sizes.test.ts‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
import { describe, expect, test } from "bun:test";
2+
import { DIRECTOR_REGISTRY } from "./directors/registry.js";
3+
import { DIRECTOR_IDS, type DirectorId } from "./directors/types.js";
4+
import {
5+
directorPromptSizeTable,
6+
type PromptSizeFamily,
7+
} from "./prompt-sizes.js";
8+
9+
/**
10+
* Prompt size budget (CL-7664). Numeric asserts only — copy edits must not
11+
* fail this test. Baselines were captured from the canonical fixture in
12+
* src/agent/prompt-sizes.ts with a +2000 char / +3000 byte allowance; bytes
13+
* get the larger headroom because multibyte copy can shift them faster.
14+
*/
15+
const CHAR_BUDGET: Record<DirectorId, number> = {
16+
skywalker: 28000,
17+
builder: 49000,
18+
explorer: 14200,
19+
counsel: 52800,
20+
intern: 16800,
21+
critic: 54400,
22+
greybeard: 54300,
23+
neckbeard: 72300,
24+
bruckheimer: 23400,
25+
gaasbot: 31800,
26+
draper: 15200,
27+
emil: 16700,
28+
rand: 15100,
29+
shakespeare: 54900,
30+
testsmith: 16200,
31+
tester: 14000,
32+
};
33+
34+
const BYTE_BUDGET: Record<DirectorId, number> = {
35+
skywalker: 29100,
36+
builder: 50100,
37+
explorer: 15300,
38+
counsel: 53900,
39+
intern: 17900,
40+
critic: 55600,
41+
greybeard: 55500,
42+
neckbeard: 73500,
43+
bruckheimer: 24400,
44+
gaasbot: 32900,
45+
draper: 16300,
46+
emil: 17800,
47+
rand: 16200,
48+
shakespeare: 56000,
49+
testsmith: 17300,
50+
tester: 15100,
51+
};
52+
53+
function budgetMessage(
54+
directorId: DirectorId,
55+
family: PromptSizeFamily,
56+
chars: number,
57+
bytes: number,
58+
): string {
59+
return (
60+
`Director "${directorId}" [${family}]: ${chars} chars / ${bytes} bytes ` +
61+
`exceeds budget (${CHAR_BUDGET[directorId]} chars / ` +
62+
`${BYTE_BUDGET[directorId]} bytes). Trim the prompt (preferred) or ` +
63+
`consciously raise the budget here with justification. ` +
64+
`Repro: bun -e 'import { directorPromptSizeTable, ` +
65+
`formatPromptSizeTable } from "./src/agent/prompt-sizes.ts"; ` +
66+
`console.log(formatPromptSizeTable(directorPromptSizeTable()))'.`
67+
);
68+
}
69+
70+
describe("director prompt size budget", () => {
71+
const rows = directorPromptSizeTable();
72+
73+
test("covers every director in both families", () => {
74+
expect(rows.length).toBe(DIRECTOR_IDS.length * 2);
75+
for (const directorId of DIRECTOR_IDS) {
76+
for (const family of ["default", "grok"] as const) {
77+
expect(
78+
rows.some((r) => r.directorId === directorId && r.family === family),
79+
).toBe(true);
80+
}
81+
}
82+
});
83+
84+
test("every assembled prompt stays within budget", () => {
85+
for (const row of rows) {
86+
const overChars = row.chars > CHAR_BUDGET[row.directorId];
87+
const overBytes = row.bytes > BYTE_BUDGET[row.directorId];
88+
expect(
89+
overChars || overBytes,
90+
budgetMessage(row.directorId, row.family, row.chars, row.bytes),
91+
).toBe(false);
92+
}
93+
});
94+
95+
test("every assembled prompt is a real prompt, not an empty assembly", () => {
96+
for (const row of rows) {
97+
expect(row.chars).toBeGreaterThan(5000);
98+
expect(row.bytes).toBeGreaterThanOrEqual(row.chars);
99+
}
100+
});
101+
102+
test("grok family never shrinks a prompt; only leaves grow", () => {
103+
for (const directorId of DIRECTOR_IDS) {
104+
const base = rows.find(
105+
(r) => r.directorId === directorId && r.family === "default",
106+
);
107+
const grok = rows.find(
108+
(r) => r.directorId === directorId && r.family === "grok",
109+
);
110+
expect(grok?.chars ?? 0).toBeGreaterThanOrEqual(base?.chars ?? 0);
111+
if (DIRECTOR_REGISTRY[directorId].spawn.maySpawn) {
112+
expect(grok?.chars).toBe(base?.chars);
113+
} else {
114+
expect(grok?.chars ?? 0).toBeGreaterThan(base?.chars ?? 0);
115+
}
116+
}
117+
});
118+
119+
test("measurement is deterministic", () => {
120+
const again = directorPromptSizeTable();
121+
expect(again.map((r) => r.chars)).toEqual(rows.map((r) => r.chars));
122+
expect(again.map((r) => r.bytes)).toEqual(rows.map((r) => r.bytes));
123+
});
124+
});

‎src/agent/prompt-sizes.ts‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
import type { EnvironmentInfo } from "./environment.js";
2+
import { DIRECTOR_REGISTRY } from "./directors/registry.js";
3+
import { formatDirectorSystemPrompt } from "./directors/identity.js";
4+
import {
5+
DIRECTOR_IDS,
6+
type DirectorId,
7+
type DirectorPackage,
8+
} from "./directors/types.js";
9+
import { buildSubAgentSystemPrompt } from "./prompts.js";
10+
import { shouldApplyGrokAntiThrash } from "../subagent/provider-family.js";
11+
12+
/**
13+
* Canonical prompt-size fixture (CL-7664).
14+
*
15+
* Assembles each director prompt exactly as src/subagent/run.ts does:
16+
* extensions=[director systemPromptRole] + environment + tools +
17+
* appendix, with the Grok finish-bias note gated by
18+
* shouldApplyGrokAntiThrash (leaves on Grok-family providers only).
19+
*
20+
* The env and provider inputs are pinned here so sizes never drift with the
21+
* machine, date, or checkout — only real prompt changes move the numbers.
22+
*/
23+
export const CANONICAL_PROMPT_ENV: EnvironmentInfo = {
24+
cwd: "/repo",
25+
platform: "Darwin 25.0.0",
26+
arch: "arm64",
27+
runtime: "Bun 1.2.0",
28+
date: new Date("2026-01-15T12:00:00Z"),
29+
isGitRepo: true,
30+
gitBranch: "main",
31+
gitDirtyCount: 0,
32+
topLevel: "AGENTS.md CONTRIBUTING.md src/ tests/ docs/ plugins/",
33+
};
34+
35+
const GROK_PROVIDER = { providerName: "xai/default", model: "grok-4.6" };
36+
const DEFAULT_PROVIDER = {
37+
providerName: "anthropic",
38+
model: "claude-sonnet-4",
39+
};
40+
41+
/** Families in the size table: default assembly vs Grok (+finish-bias note). */
42+
export type PromptSizeFamily = "default" | "grok";
43+
44+
/**
45+
* Canonical tool names per director, mirroring the run.ts mount order:
46+
* package allowlist, then always-mounted manage_tasks, then leaf-only
47+
* submit_result + ask_director, then orchestrator fleet tools
48+
* (search_agents is Tier-1 skywalker only).
49+
*/
50+
export function canonicalToolNamesForDirector(
51+
pkg: DirectorPackage,
52+
): readonly string[] {
53+
const names = [...(pkg.tools?.allow ?? [])];
54+
names.push("manage_tasks");
55+
if (pkg.tier === "leaf") {
56+
names.push("submit_result", "ask_director");
57+
}
58+
if (pkg.spawn.maySpawn) {
59+
if (pkg.tier === "orchestrator") names.push("search_agents");
60+
names.push(
61+
"read_agent_trace",
62+
"spawn_agent",
63+
"wait_agents",
64+
"list_agents",
65+
"close_agent",
66+
"resume_agent",
67+
"interrupt_agent",
68+
"send_input",
69+
);
70+
}
71+
return names;
72+
}
73+
74+
/** Assemble one director prompt exactly as run.ts does. */
75+
export function assembleDirectorPrompt(
76+
directorId: DirectorId,
77+
family: PromptSizeFamily,
78+
): string {
79+
const pkg = DIRECTOR_REGISTRY[directorId];
80+
const orchestrator = pkg.spawn.maySpawn;
81+
const provider = family === "grok" ? GROK_PROVIDER : DEFAULT_PROVIDER;
82+
return buildSubAgentSystemPrompt(
83+
[formatDirectorSystemPrompt(pkg)],
84+
CANONICAL_PROMPT_ENV,
85+
undefined,
86+
{
87+
orchestrator,
88+
toolNames: canonicalToolNamesForDirector(pkg),
89+
grokAntiThrash: shouldApplyGrokAntiThrash({ ...provider, orchestrator }),
90+
},
91+
);
92+
}
93+
94+
export interface DirectorPromptSize {
95+
directorId: DirectorId;
96+
family: PromptSizeFamily;
97+
chars: number;
98+
bytes: number;
99+
}
100+
101+
export function measureDirectorPrompt(
102+
directorId: DirectorId,
103+
family: PromptSizeFamily,
104+
): DirectorPromptSize {
105+
const prompt = assembleDirectorPrompt(directorId, family);
106+
return {
107+
directorId,
108+
family,
109+
chars: prompt.length,
110+
bytes: Buffer.byteLength(prompt, "utf8"),
111+
};
112+
}
113+
114+
/** Full per-director x per-family size table. */
115+
export function directorPromptSizeTable(): DirectorPromptSize[] {
116+
const rows: DirectorPromptSize[] = [];
117+
for (const directorId of DIRECTOR_IDS) {
118+
for (const family of ["default", "grok"] as const) {
119+
rows.push(measureDirectorPrompt(directorId, family));
120+
}
121+
}
122+
return rows;
123+
}
124+
125+
/** Render the size table as markdown (for PR bodies and budget updates). */
126+
export function formatPromptSizeTable(rows: DirectorPromptSize[]): string {
127+
const lines = [
128+
"| director | default chars (bytes) | grok chars (bytes) |",
129+
"| --- | --- | --- |",
130+
];
131+
for (const directorId of DIRECTOR_IDS) {
132+
const base = rows.find(
133+
(r) => r.directorId === directorId && r.family === "default",
134+
);
135+
const grok = rows.find(
136+
(r) => r.directorId === directorId && r.family === "grok",
137+
);
138+
lines.push(
139+
`| ${directorId} | ${base?.chars} (${base?.bytes}) | ${grok?.chars} (${grok?.bytes}) |`,
140+
);
141+
}
142+
return lines.join("\n");
143+
}

0 commit comments

Comments
 (0)