Skip to content

Commit 9cb47ae

Browse files
committed
Overhaul Testsmith prompt
Teach Testsmith to design permanent cases from success_criteria with blinders on, refuse Tester/Builder drift despite mounted writes, and drop tool-schema restatement or fake caps. Closes CL-7033
1 parent 335afa2 commit 9cb47ae

2 files changed

Lines changed: 61 additions & 20 deletions

File tree

src/agent/directors/testsmith/package.test.ts

Lines changed: 37 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,39 @@ describe("testsmithPackage", () => {
1111
expect(testsmithPackage.systemPrompt.startsWith("Placeholder")).toBe(false);
1212
});
1313

14-
test("systemPrompt states PRIMARY INTENT for test design", () => {
15-
expect(testsmithPackage.systemPrompt).toContain("PRIMARY INTENT");
16-
expect(testsmithPackage.systemPrompt).toMatch(/test strategy|test cases|design/i);
17-
expect(testsmithPackage.systemPrompt).toMatch(/do not implement|not implement/i);
14+
test("systemPrompt identity is Testsmith / TestsmithDirector", () => {
15+
const p = testsmithPackage.systemPrompt;
16+
expect(p).toMatch(/TestsmithDirector \(Testsmith\)/);
17+
expect(p).toContain("PRIMARY INTENT");
18+
expect(p).toMatch(/permanent test cases/i);
19+
});
20+
21+
test("systemPrompt teaches permanent case design from success_criteria", () => {
22+
const p = testsmithPackage.systemPrompt;
23+
expect(p).toMatch(/Blinders on/i);
24+
expect(p).toContain("success_criteria");
25+
expect(p).toMatch(/name, setup, action, expected/i);
26+
expect(p).toMatch(/what not to test/i);
27+
expect(p).toMatch(/Hand off to Builder/i);
28+
expect(p).toContain("Blockers");
29+
});
30+
31+
test("systemPrompt is design lane only (not Tester / Builder / orchestrator)", () => {
32+
const p = testsmithPackage.systemPrompt;
33+
expect(p).toMatch(/Do not become Builder/i);
34+
expect(p).toMatch(/that is Tester/i);
35+
expect(p).toMatch(/do not use them/i);
36+
expect(p).toMatch(/fleet orchestration/i);
37+
expect(p).toMatch(/DONE GATE/i);
38+
});
39+
40+
test("systemPrompt has no tool-schema restatement or fake caps", () => {
41+
const p = testsmithPackage.systemPrompt;
42+
expect(p).not.toMatch(/parameters?:/i);
43+
expect(p).not.toMatch(/fan-out/i);
44+
expect(p).not.toMatch(/at most \d+/i);
45+
expect(p).not.toMatch(/turn budget/i);
46+
expect(p).not.toMatch(/scheduler/i);
1847
});
1948

2049
test("spawn.maySpawn is false (leaf)", () => {
@@ -33,14 +62,15 @@ describe("testsmithPackage", () => {
3362
expect(testsmithPackage.modelRole).toBe("test");
3463
});
3564

36-
test("primaryIntent is design-only and not primary verifier", () => {
37-
expect(testsmithPackage.primaryIntent).toMatch(/design/i);
65+
test("primaryIntent is permanent-design and not primary verifier", () => {
66+
expect(testsmithPackage.primaryIntent).toMatch(/permanent test cases/i);
3867
expect(testsmithPackage.primaryIntent).toMatch(/not.*verifier|do not run as primary verifier/i);
3968
});
4069

41-
test("outOfLane refuses product implement and runtime verify role", () => {
70+
test("outOfLane refuses product implement, verifier role, and landing tests", () => {
4271
const joined = testsmithPackage.outOfLane.join(" ");
4372
expect(joined).toMatch(/implement/i);
4473
expect(joined).toMatch(/verifier|tester/i);
74+
expect(joined).toMatch(/landing test/i);
4575
});
4676
});

src/agent/directors/testsmith/package.ts

Lines changed: 24 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,34 +2,45 @@ import type { DirectorPackage } from "../types.js";
22
import { REVIEW_TOOLS } from "../tool-sets.js";
33

44
/**
5-
* Testsmith: test design specialist — strategy and cases only; never implements product
6-
* and is not the runtime verifier (that is tester).
5+
* Testsmith leaf (CL-7033).
6+
* Design permanent test cases from the brief — report only; never implement product or act as Tester.
77
*/
88
export const testsmithPackage: DirectorPackage = {
99
id: "testsmith",
1010
primaryIntent:
11-
"Design test strategy and cases; do not implement product; do not run as primary verifier",
11+
"Design permanent test cases; do not implement product; do not run as primary verifier",
1212
outOfLane: [
1313
"implementing product code",
1414
"shipping features",
1515
"acting as primary runtime verifier (tester)",
1616
"fixing failing product code",
17+
"landing test files as the implementer",
1718
"orchestration",
1819
],
19-
description: "Test design specialist — strategy and cases in the report only",
20-
systemPrompt: `You are TestsmithDirector, a specialist in Corbits Code.
20+
description: "Test design specialist — permanent cases in the report only",
21+
systemPrompt: `You are TestsmithDirector (Testsmith), a specialist in Corbits Code.
2122
22-
PRIMARY INTENT: design test strategy and test cases for the brief. Produce clear, agent-ready coverage plans. Do not implement product code. Do not act as the primary runtime verifier (that is Tester).
23+
PRIMARY INTENT: design permanent test cases for the brief. Produce agent-ready coverage the suite should keep. Do not implement product code. Do not act as the primary runtime verifier (that is Tester). Do not become Builder.
2324
24-
Design in the report. Prefer:
25-
- risk-based coverage and acceptance criteria from the brief
26-
- unit / integration / e2e boundaries when relevant
27-
- concrete cases: setup, action, expected result, edge/failure modes
28-
- what not to test and why
25+
Blinders on:
26+
- Design from the brief's success_criteria / acceptance criteria and stated risks — not from "whatever the code does today."
27+
- Read/search only to ground paths, public APIs, and existing suite shape. Do not soften cases to match current buggy behavior.
28+
- Stay on this brief. Do not wander into peer work, unrelated packages, or fleet orchestration.
2929
30-
OUT OF LANE: fixing production code, becoming the implementer, running the full verify-and-fix loop, fleet orchestration.
30+
Design permanent cases in the report:
31+
1. Map every success_criteria item to concrete cases (or Blockers if you cannot).
32+
2. Prefer risk-based coverage; name unit / integration / e2e boundaries when relevant.
33+
3. Each case: name, setup, action, expected result, edge/failure modes.
34+
4. Say what not to test and why (impossible paths, over-engineering theater).
35+
5. Hand off to Builder to land tests; Tester runs them. You design.
3136
32-
Read and search the codebase to ground the design; do not mutate product code.`,
37+
Write tools are mounted with no path lock — do not use them. Leave product and test-file edits to Builder.
38+
39+
DONE GATE: Stop when every success_criteria item has permanent cases (or Blockers). Do not invent architecture or expand the brief after criteria are covered. If the brief is ambiguous, report Blockers — do not become Counsel or Greybeard.
40+
41+
Findings: permanent cases (name + setup/action/expected), coverage map to success_criteria, what not to test. Paths: files you read to ground the design.
42+
43+
OUT OF LANE: implementing product or tests, becoming Tester/Builder, running the full verify-and-fix loop, fleet orchestration.`,
3344
tools: { allow: REVIEW_TOOLS },
3445
spawn: { maySpawn: false },
3546
tier: "leaf",

0 commit comments

Comments
 (0)