From 43346bd56333b7fc74e12737ae60e7cd5f2b88ec Mon Sep 17 00:00:00 2001 From: Sawyer Date: Mon, 7 Sep 2026 22:12:22 -0700 Subject: [PATCH 01/18] Trim content-pin and cosmetic tests from the suite The suite pinned packaged skill prose, palette hexes, ANSI-256 indexes, display-name branding, and animation frames, and carried duplicated copies of faremeter, stream-consumer, and inference-abort coverage. Those broke on every wording or palette edit without guarding behavior, and two eval test files were never run at all. Manifest, inventory, frontmatter-flag, and behavioral invariants stay. --- scripts/eval-capability.test.ts | 337 ------------------ scripts/eval-public-swe-one.test.ts | 49 --- src/cost/faremeter.test.ts | 47 --- src/inference-abort.test.ts | 15 +- src/plugins/result-truncation-plugin.test.ts | 13 - src/tui/landing.test.ts | 27 -- src/tui/welcome.test.ts | 80 +---- tests/unit/corbits-skills-catalog.test.ts | 299 ---------------- tests/unit/run-agent.test.ts | 29 -- tests/unit/tui/theme.test.ts | 19 - .../unit/tui/tool-formatter-web-brand.test.ts | 20 -- 11 files changed, 3 insertions(+), 932 deletions(-) delete mode 100644 scripts/eval-capability.test.ts delete mode 100644 scripts/eval-public-swe-one.test.ts delete mode 100644 src/cost/faremeter.test.ts delete mode 100644 tests/unit/tui/tool-formatter-web-brand.test.ts diff --git a/scripts/eval-capability.test.ts b/scripts/eval-capability.test.ts deleted file mode 100644 index 2713919e..00000000 --- a/scripts/eval-capability.test.ts +++ /dev/null @@ -1,337 +0,0 @@ -import { afterEach, beforeEach, describe, expect, test } from "bun:test"; -import { mkdir, mkdtemp, writeFile, rm } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { execFile } from "node:child_process"; -import { promisify } from "node:util"; - -import { - initEvalGitRepo, - mapPool, - parseArgs, - buildEvalDiagnostics, - validateVariantEfforts, -} from "./eval-capability.js"; -import { parseMatrix } from "../evals/capability/lib.js"; -import type { Config } from "../src/config/index.js"; - -const execFileAsync = promisify(execFile); - -function sampleConfig(over: Partial = {}): Config { - return { - configured: true, - apiKey: "key", - baseURL: "https://example.test", - model: "gpt-5", - providerName: "openai", - cwd: process.cwd(), - task: "do it", - force: true, - dangerouslySkipPermissions: true, - skipPermissionsFromSettings: false, - auto: false, - command: "exec", - globalSettingsPath: "/dev/null", - providers: [], - sessionId: "sess-1", - ...over, - } as Config; -} - -describe("parseArgs", () => { - const savedConcurrency = process.env.CORBITS_EVAL_CONCURRENCY; - - const restoreConcurrency = (): void => { - if (savedConcurrency === undefined) { - delete process.env.CORBITS_EVAL_CONCURRENCY; - } else { - process.env.CORBITS_EVAL_CONCURRENCY = savedConcurrency; - } - }; - - afterEach(() => { - restoreConcurrency(); - }); - - beforeEach(() => { - delete process.env.CORBITS_EVAL_CONCURRENCY; - }); - - test("--help does not require provider or model", () => { - const opts = parseArgs(["--help"]); - expect(opts.help).toBe(true); - expect(opts.provider).not.toBe("xai/thegreataxios"); - expect(opts.model).not.toBe("xai/thegreataxios"); - }); - - test("no flags throws", () => { - expect(() => parseArgs([])).toThrow(/--provider/); - expect(() => parseArgs([])).toThrow(/--model/); - }); - - test("--provider without --model throws", () => { - expect(() => parseArgs(["--provider", "foo"])).toThrow(/--model/); - }); - - test("--model without --provider throws", () => { - expect(() => parseArgs(["--model", "bar"])).toThrow(/--provider/); - }); - - test("--provider foo --model bar parses those values", () => { - const opts = parseArgs(["--provider", "foo", "--model", "bar"]); - expect(opts.provider).toBe("foo"); - expect(opts.model).toBe("bar"); - }); - - test("--dry-run without pair throws", () => { - expect(() => parseArgs(["--dry-run"])).toThrow(/--provider/); - expect(() => parseArgs(["--dry-run"])).toThrow(/--model/); - }); - - test("--matrix xai:grok-4.5 is enough without top-level flags", () => { - const opts = parseArgs(["--matrix", "xai:grok-4.5"]); - expect(opts.matrix).toBe("xai:grok-4.5"); - }); - - test("incomplete matrix cell throws", () => { - expect(() => parseArgs(["--matrix", "xai:"])).toThrow(/both provider and model/); - expect(() => parseArgs(["--matrix", ":grok-4.5"])).toThrow(/both provider and model/); - }); - - test("--effort accepts a canonical literal", () => { - const opts = parseArgs(["--provider", "foo", "--model", "bar", "--effort", "high"]); - expect(opts.effort).toBe("high"); - }); - - test("--effort rejects an unknown literal", () => { - expect(() => parseArgs(["--provider", "foo", "--model", "bar", "--effort", "bogus"])).toThrow( - /--effort must be one of/, - ); - }); - - test("--matrix cell can carry its own effort as a third segment", () => { - const opts = parseArgs(["--matrix", "xai/thegreataxios:grok-4.6:xhigh"]); - expect(opts.matrix).toBe("xai/thegreataxios:grok-4.6:xhigh"); - }); - - test("parsed defaults never equal xai/thegreataxios", () => { - const help = parseArgs(["--help"]); - const pair = parseArgs(["--provider", "foo", "--model", "bar"]); - expect(help.provider).not.toBe("xai/thegreataxios"); - expect(help.model).not.toBe("xai/thegreataxios"); - expect(pair.provider).not.toBe("xai/thegreataxios"); - expect(pair.model).not.toBe("xai/thegreataxios"); - expect(pair.provider).toBe("foo"); - expect(pair.model).toBe("bar"); - }); - - test("defaults concurrency to 1", () => { - delete process.env.CORBITS_EVAL_CONCURRENCY; - const opts = parseArgs(["--provider", "foo", "--model", "bar"]); - expect(opts.concurrency).toBe(1); - }); - - test("--concurrency 4 is accepted", () => { - delete process.env.CORBITS_EVAL_CONCURRENCY; - const opts = parseArgs(["--provider", "foo", "--model", "bar", "--concurrency", "4"]); - expect(opts.concurrency).toBe(4); - }); - - test("invalid --concurrency values throw", () => { - const pair = ["--provider", "foo", "--model", "bar"] as const; - expect(() => parseArgs([...pair, "--concurrency", "0"])).toThrow(/positive integer/); - expect(() => parseArgs([...pair, "--concurrency", "-1"])).toThrow(/positive integer/); - expect(() => parseArgs([...pair, "--concurrency", "1.5"])).toThrow(/positive integer/); - expect(() => parseArgs([...pair, "--concurrency", "foo"])).toThrow(/positive integer/); - }); - - test("CORBITS_EVAL_CONCURRENCY sets the default", () => { - process.env.CORBITS_EVAL_CONCURRENCY = "3"; - const opts = parseArgs(["--provider", "foo", "--model", "bar"]); - expect(opts.concurrency).toBe(3); - }); - - test("--concurrency overrides CORBITS_EVAL_CONCURRENCY", () => { - process.env.CORBITS_EVAL_CONCURRENCY = "8"; - const opts = parseArgs(["--provider", "foo", "--model", "bar", "--concurrency", "2"]); - expect(opts.concurrency).toBe(2); - }); - - test("invalid CORBITS_EVAL_CONCURRENCY throws", () => { - process.env.CORBITS_EVAL_CONCURRENCY = "0"; - expect(() => parseArgs(["--provider", "foo", "--model", "bar"])).toThrow( - /CORBITS_EVAL_CONCURRENCY must be a positive integer/, - ); - }); - - test("--director builder is parsed", () => { - const opts = parseArgs(["--provider", "foo", "--model", "bar", "--director", "builder"]); - expect(opts.director).toBe("builder"); - }); - - test("omitted --director stays undefined", () => { - const opts = parseArgs(["--provider", "foo", "--model", "bar"]); - expect(opts.director).toBeUndefined(); - }); - - test("--director without a value throws", () => { - expect(() => parseArgs(["--provider", "foo", "--model", "bar", "--director"])).toThrow( - "--director requires a value", - ); - }); -}); - -describe("validateVariantEfforts", () => { - // Wiring-level regression: parseArgs -> parseMatrix -> validateVariantEfforts, - // the same path main() runs before any inference. A matrix cell pairing an - // effort the model does not accept must fail fast, naming the model and its - // accepted levels, rather than silently falling back to the provider default - // and poisoning the matrix. - test("rejects an unsupported model/effort matrix cell before any inference runs", async () => { - const opts = parseArgs(["--matrix", "xai/thegreataxios:grok-composer-2.5-fast:xhigh"]); - const variants = parseMatrix(opts.matrix, { - ...(opts.provider !== undefined ? { provider: opts.provider } : {}), - ...(opts.model !== undefined ? { model: opts.model } : {}), - ...(opts.effort !== undefined ? { effort: opts.effort } : {}), - }); - await expect(validateVariantEfforts(variants, opts)).rejects.toThrow( - /grok-composer-2\.5-fast.*does not support reasoning effort "xhigh".*supported: low, medium, high/s, - ); - }); - - test("accepts a supported model/effort matrix cell", async () => { - const opts = parseArgs(["--matrix", "xai/thegreataxios:grok-4.6:xhigh"]); - const variants = parseMatrix(opts.matrix, { - ...(opts.provider !== undefined ? { provider: opts.provider } : {}), - ...(opts.model !== undefined ? { model: opts.model } : {}), - ...(opts.effort !== undefined ? { effort: opts.effort } : {}), - }); - await expect(validateVariantEfforts(variants, opts)).resolves.toBeUndefined(); - }); -}); - -describe("mapPool", () => { - test("N overlapping jobs with concurrency N finish in ~one job duration", async () => { - const jobMs = 80; - const n = 4; - const start = Date.now(); - const results = await mapPool([0, 1, 2, 3], n, async (item) => { - await new Promise((r) => setTimeout(r, jobMs)); - return item; - }); - const elapsed = Date.now() - start; - expect(results).toEqual([0, 1, 2, 3]); - expect(elapsed).toBeLessThan(jobMs * 2); - expect(elapsed).toBeGreaterThanOrEqual(jobMs - 20); - }); - - test("preserves input order when later items finish first", async () => { - const results = await mapPool([1, 2, 3], 3, async (item) => { - await new Promise((r) => setTimeout(r, (4 - item) * 30)); - return item; - }); - expect(results).toEqual([1, 2, 3]); - }); - - test("empty input returns an empty array", async () => { - expect(await mapPool([], 4, async (item) => item)).toEqual([]); - }); - - test("rejects non-positive concurrency", async () => { - await expect(mapPool([1], 0, async (item) => item)).rejects.toThrow(/positive integer/); - }); -}); - -describe("initEvalGitRepo", () => { - const savedGitConfigGlobal = process.env.GIT_CONFIG_GLOBAL; - - const restoreGitConfigGlobal = (): void => { - if (savedGitConfigGlobal === undefined) { - delete process.env.GIT_CONFIG_GLOBAL; - } else { - process.env.GIT_CONFIG_GLOBAL = savedGitConfigGlobal; - } - }; - - afterEach(() => { - restoreGitConfigGlobal(); - }); - - test("makes a fixture copy a git work tree with a commit", async () => { - const dir = await mkdtemp(join(tmpdir(), "corbits-eval-git-")); - try { - await writeFile(join(dir, "README"), "fixture\n", "utf8"); - await initEvalGitRepo(dir); - const { stdout } = await execFileAsync("git", ["rev-parse", "--is-inside-work-tree"], { - cwd: dir, - }); - expect(stdout.trim()).toBe("true"); - const { stdout: head } = await execFileAsync("git", ["rev-parse", "HEAD"], { cwd: dir }); - expect(head.trim().length).toBeGreaterThan(0); - const { stdout: count } = await execFileAsync("git", ["rev-list", "--count", "HEAD"], { - cwd: dir, - }); - expect(Number(count.trim())).toBeGreaterThanOrEqual(1); - const { stdout: log } = await execFileAsync("git", ["log", "-1", "--pretty=%s"], { - cwd: dir, - }); - expect(log.trim()).toBe("eval fixture"); - } finally { - await rm(dir, { recursive: true, force: true }); - } - }); - - test("succeeds when the process would otherwise sign", async () => { - const root = await mkdtemp(join(tmpdir(), "corbits-eval-git-sign-")); - const work = join(root, "work"); - const configPath = join(root, "gitconfig"); - try { - await mkdir(work); - await writeFile( - configPath, - "[commit]\ngpgsign = true\n[user]\nsigningkey = DEADKEY\n", - "utf8", - ); - process.env.GIT_CONFIG_GLOBAL = configPath; - await writeFile(join(work, "README"), "fixture\n", "utf8"); - await initEvalGitRepo(work); - const { stdout: head } = await execFileAsync("git", ["rev-parse", "HEAD"], { cwd: work }); - expect(head.trim().length).toBeGreaterThan(0); - const { stdout: cat } = await execFileAsync("git", ["cat-file", "-p", "HEAD"], { cwd: work }); - expect(cat).not.toContain("gpgsig"); - } finally { - restoreGitConfigGlobal(); - await rm(root, { recursive: true, force: true }); - } - }); -}); - -describe("buildEvalDiagnostics", () => { - test("non-Codex provider gets the default orchestrator tool list", async () => { - const diagnostics = await buildEvalDiagnostics(sampleConfig({ providerName: "openai" })); - expect(diagnostics.advertisedTools).toContain("read_file"); - expect(diagnostics.advertisedTools).toContain("run_shell"); - expect(diagnostics.reasoningEffort).toBeNull(); - }); - - test.each(["openai", "codex/default"])( - "%s diagnostics omit the removed instructions hash", - async (providerName) => { - const diagnostics = await buildEvalDiagnostics(sampleConfig({ providerName })); - expect(diagnostics).not.toHaveProperty("codexInstructionsHash"); - expect(diagnostics.advertisedTools).toContain("read_file"); - }, - ); - - test("echoes back the configured reasoning effort", async () => { - const diagnostics = await buildEvalDiagnostics(sampleConfig({ reasoningEffort: "high" })); - expect(diagnostics.reasoningEffort).toBe("high"); - }); - - test("--director builder reports the director's own advertised allowlist", async () => { - const diagnostics = await buildEvalDiagnostics(sampleConfig({ director: "builder" })); - expect(diagnostics.advertisedTools).not.toEqual( - (await buildEvalDiagnostics(sampleConfig({}))).advertisedTools, - ); - }); -}); diff --git a/scripts/eval-public-swe-one.test.ts b/scripts/eval-public-swe-one.test.ts deleted file mode 100644 index 5335e5f3..00000000 --- a/scripts/eval-public-swe-one.test.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { describe, expect, test } from "bun:test"; - -import { parseArgs } from "./eval-public-swe-one.js"; - -describe("parseArgs", () => { - test("--help does not require provider or model", () => { - const opts = parseArgs(["--help"]); - expect(opts.help).toBe(true); - expect(opts.provider).not.toBe("xai/thegreataxios"); - expect(opts.model).not.toBe("xai/thegreataxios"); - }); - - test("--dry-run alone throws", () => { - expect(() => parseArgs(["--dry-run"])).toThrow(/--provider/); - expect(() => parseArgs(["--dry-run"])).toThrow(/--model/); - }); - - test("--dry-run with provider and model parses", () => { - const opts = parseArgs(["--dry-run", "--provider", "foo", "--model", "bar"]); - expect(opts.dryRun).toBe(true); - expect(opts.provider).toBe("foo"); - expect(opts.model).toBe("bar"); - }); - - test("agent run without --provider throws", () => { - expect(() => parseArgs(["--model", "bar"])).toThrow(/--provider/); - }); - - test("agent run without --model throws", () => { - expect(() => parseArgs(["--provider", "foo"])).toThrow(/--model/); - }); - - test("agent run without either flag throws naming both", () => { - expect(() => parseArgs([])).toThrow(/--provider/); - expect(() => parseArgs([])).toThrow(/--model/); - }); - - test("--provider foo --model bar parses those values", () => { - const opts = parseArgs(["--provider", "foo", "--model", "bar"]); - expect(opts.provider).toBe("foo"); - expect(opts.model).toBe("bar"); - }); - - test("parsed defaults never equal xai/thegreataxios", () => { - const help = parseArgs(["--help"]); - expect(help.provider).not.toBe("xai/thegreataxios"); - expect(help.model).not.toBe("xai/thegreataxios"); - }); -}); diff --git a/src/cost/faremeter.test.ts b/src/cost/faremeter.test.ts deleted file mode 100644 index b29377a4..00000000 --- a/src/cost/faremeter.test.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import type { TokenUsage } from "@intx/types/runtime"; -import { createFaremeter } from "./faremeter.js"; - -const PRICES = { - inputPricePerToken: 1, - outputPricePerToken: 2, - cacheReadPricePerToken: 0.5, -}; - -describe("createFaremeter", () => { - test("bills uncached input at the input rate and cached reads at the cache rate", () => { - const faremeter = createFaremeter(PRICES); - // Normalized Responses-API usage: input excludes cached tokens (the - // adapter subtracts them), so the same token is never billed twice. - const usage: TokenUsage = { - input: 80, - output: 10, - cacheRead: 20, - cacheWrite: 0, - thinking: 0, - }; - - faremeter.addUsage(usage); - - expect(faremeter.getTotalCost()).toBe(80 * 1 + 10 * 2 + 20 * 0.5); - }); - - test("reports context occupancy as the full prompt size, not just uncached input", () => { - const faremeter = createFaremeter(PRICES); - faremeter.addUsage({ input: 200, output: 50, cacheRead: 800, cacheWrite: 0, thinking: 0 }); - - expect(faremeter.getInputTokens()).toBe(1000); - expect(faremeter.getTotalTokens()).toBe(1050); - }); - - test("accumulates cost across turns and counts thinking tokens as output volume", () => { - const faremeter = createFaremeter(PRICES); - faremeter.addUsage({ input: 100, output: 10, cacheRead: 0, cacheWrite: 0, thinking: 5 }); - faremeter.addUsage({ input: 40, output: 20, cacheRead: 60, cacheWrite: 0, thinking: 0 }); - - // Thinking tokens are tracked in the cumulative output count but ride the - // same unbilled slot today: totalCost prices usage.output only. - expect(faremeter.getTotalCost()).toBe(100 + 2 * 10 + (40 + 2 * 20 + 30)); - expect(faremeter.getOutputTokens()).toBe(35); - }); -}); diff --git a/src/inference-abort.test.ts b/src/inference-abort.test.ts index 3a158729..b419fa0a 100644 --- a/src/inference-abort.test.ts +++ b/src/inference-abort.test.ts @@ -1,8 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { - INFERENCE_ABORT_INTERNAL_RECOVERY, - isNonTerminalInferenceError, -} from "./inference-abort.js"; +import { isNonTerminalInferenceError } from "./inference-abort.js"; const HTML_503 = "503 Service Unavailable"; @@ -26,14 +23,4 @@ describe("isNonTerminalInferenceError", () => { }), ).toBe(false); }); - - test("internal recovery abort remains non-terminal", () => { - expect( - isNonTerminalInferenceError({ - category: "aborted", - message: "inference aborted", - raw: { origin: INFERENCE_ABORT_INTERNAL_RECOVERY }, - }), - ).toBe(true); - }); }); diff --git a/src/plugins/result-truncation-plugin.test.ts b/src/plugins/result-truncation-plugin.test.ts index 5f397496..3e148904 100644 --- a/src/plugins/result-truncation-plugin.test.ts +++ b/src/plugins/result-truncation-plugin.test.ts @@ -29,11 +29,6 @@ function fakeBlobStore() { } describe("truncateToolResultContent", () => { - test("within-cap content passes through unchanged", async () => { - const content = "x".repeat(100); - expect(await truncateToolResultContent(content)).toBe(content); - }); - test("under-gate minified JSON is left unchanged (no pretty, no spill)", async () => { const store = fakeBlobStore(); const minified = JSON.stringify({ a: 1, b: 2 }); @@ -60,14 +55,6 @@ describe("truncateToolResultContent", () => { expect(truncated).not.toContain("session ends"); }); - test("the inlined portion stays within the cap (notice reserved inside the budget)", async () => { - const content = "x".repeat(MAX_RESULT_CHARS * 3); - const truncated = await truncateToolResultContent(content); - // Notice is reserved before slicing so the reactor 10k size-cap cannot strip it. - expect(truncated.length).toBeLessThanOrEqual(MAX_RESULT_CHARS); - expect(truncated).toContain("[output truncated"); - }); - describe("with a blob store", () => { test("a result over the cap is fully recoverable by following the notice's read_file instructions verbatim", async () => { const store = fakeBlobStore(); diff --git a/src/tui/landing.test.ts b/src/tui/landing.test.ts index 88bdf9ff..5f56dda2 100644 --- a/src/tui/landing.test.ts +++ b/src/tui/landing.test.ts @@ -41,7 +41,6 @@ import { LOCKUP_WORDMARK } from "./lockup"; import pkg from "../../package.json" with { type: "json" }; import { MARK_LARGE, MARK_MID, MARK_SMALL } from "./mark-shape"; import { SNOW_CHAR } from "./mark-anim"; -import { UI } from "./theme"; const SIZE = { width: 80, height: 24 } as const; const NOTICE = "Anonymous usage telemetry is enabled. Disable in /settings."; @@ -249,32 +248,6 @@ describe("landing screen", () => { }, SIZE); }); - test("the mark paints in the brand orange, not a cool accent", async () => { - await withTestRenderer(async (h) => { - const shell = createAppShell(h.renderer, { - terminal: { columns: 80, rows: 24 }, - wireKeys: false, - run: "idle", - }); - try { - await settle(h); - const tones = new Set( - h - .captureSpans() - .lines.flatMap((line: { spans: CapturedSpan[] }) => line.spans) - .filter((span) => /[░▒▓█]/.test(span.text)) - .map((span) => rgbToHex(span.fg).toLowerCase().slice(0, 7)), - ); - expect(tones.size).toBeGreaterThan(0); - for (const tone of tones) { - expect([UI.action, UI.actionDim] as readonly string[]).toContain(tone); - } - } finally { - shell.dispose(); - } - }, SIZE); - }); - test("the mark advances off an injected clock while a turn runs", async () => { await withTestRenderer(async (h) => { const shell = createAppShell(h.renderer, { diff --git a/src/tui/welcome.test.ts b/src/tui/welcome.test.ts index 5ffc7f56..8a5bd459 100644 --- a/src/tui/welcome.test.ts +++ b/src/tui/welcome.test.ts @@ -1,25 +1,9 @@ import { describe, expect, test } from "bun:test"; -import { PRODUCT_NAME } from "../branding.js"; -import { MARK_PERIOD_SECONDS, markFrame, markText, renderMark } from "./mark-anim.js"; import { MARK_LARGE, MARK_MID, MARK_SMALL } from "./mark-shape.js"; import { createHarness } from "./harness.js"; import { stringWidth } from "./view/height.js"; -import { - resolveWelcomeLine, - resolveWelcomeMarkGrid, - runWelcome, - WELCOME_AUTO_ADVANCE_MS, - WELCOME_LINE, - welcomeMarkStill, -} from "./welcome.js"; - -describe("WELCOME_LINE", () => { - test("names the product as the local software factory", () => { - expect(WELCOME_LINE).toBe(`${PRODUCT_NAME}, your local software factory`); - expect(WELCOME_LINE).toContain("Corbits Code, your local software factory"); - }); -}); +import { resolveWelcomeLine, resolveWelcomeMarkGrid, runWelcome, WELCOME_LINE } from "./welcome.js"; describe("resolveWelcomeMarkGrid", () => { test("picks the largest mark that fits the terminal", () => { @@ -31,7 +15,7 @@ describe("resolveWelcomeMarkGrid", () => { }); describe("runWelcome", () => { - test("paints the product line and continues on keypress", async () => { + test("continues on Enter keypress", async () => { const harness = await createHarness({ width: 80, height: 30 }); const done = runWelcome({ createRenderer: async () => harness.renderer, @@ -40,9 +24,6 @@ describe("runWelcome", () => { }); try { await harness.renderOnce(); - await harness.renderOnce(); - expect(harness.captureCharFrame()).toContain(WELCOME_LINE); - harness.pressKey("Enter"); await expect(done).resolves.toBe(true); } finally { @@ -84,37 +65,6 @@ describe("runWelcome", () => { }); }); -describe("welcomeMarkStill", () => { - test("animates through fill, then freezes the full frame", () => { - const fillMs = 0.76 * MARK_PERIOD_SECONDS * 1000; - expect(welcomeMarkStill(fillMs - 1)).toBe(false); - expect(welcomeMarkStill(fillMs)).toBe(true); - expect(welcomeMarkStill(0.95 * MARK_PERIOD_SECONDS * 1000)).toBe(true); - expect(welcomeMarkStill(MARK_PERIOD_SECONDS * 1000 + 900)).toBe(true); - }); -}); - -describe("WELCOME_AUTO_ADVANCE_MS", () => { - test("lands on the held filled frame, not fade-out or a second draw-in", () => { - const seconds = WELCOME_AUTO_ADVANCE_MS / 1000; - expect(seconds).toBeGreaterThanOrEqual(0.76 * MARK_PERIOD_SECONDS); - expect(seconds).toBeLessThan(MARK_PERIOD_SECONDS); - - const still = welcomeMarkStill(WELCOME_AUTO_ADVANCE_MS); - const frame = markFrame(seconds, still); - expect(still).toBe(true); - expect(frame).toEqual({ drawProg: 1, fillProg: 1, alpha: 1 }); - - // Looping math at this delay must also still be the full hold — never the - // fade (90–100%) or the wrapped second draw-in. - const looping = markFrame(seconds, false); - expect(looping.alpha).toBe(1); - expect(looping.fillProg).toBe(1); - expect(looping.drawProg).toBe(1); - expect(seconds).toBeLessThan(0.9 * MARK_PERIOD_SECONDS + 1e-9); - }); -}); - describe("resolveWelcomeLine", () => { test("keeps the full factory sentence or hides it, never a mid-word slice", () => { expect(resolveWelcomeLine(80)).toBe(WELCOME_LINE); @@ -132,32 +82,6 @@ describe("resolveWelcomeLine", () => { }); describe("runWelcome hold and cancel", () => { - test("paints a still full mark after fill instead of fading", async () => { - const fadeMs = 0.95 * MARK_PERIOD_SECONDS * 1000; - // First `now()` is mount (`startedAt`); later samples are elapsed fadeMs. - let samples = 0; - const harness = await createHarness({ width: 80, height: 30 }); - const done = runWelcome({ - createRenderer: async () => harness.renderer, - autoAdvanceMs: 60_000, - now: () => (samples++ === 0 ? 0 : fadeMs), - }); - try { - await harness.renderOnce(); - await harness.renderOnce(); - const frame = harness.captureCharFrame(); - const held = markText(renderMark({ nowMs: fadeMs, still: true, grid: MARK_LARGE })); - const fading = markText(renderMark({ nowMs: fadeMs, still: false, grid: MARK_LARGE })); - const mountain = (text: string) => (text.match(/[▁▂▃▄▅▆▇█]/g) ?? []).length; - expect(mountain(frame)).toBe(mountain(held)); - expect(mountain(held)).toBeGreaterThan(mountain(fading)); - } finally { - harness.pressKey("Ctrl+C"); - await Promise.race([done, new Promise((r) => setTimeout(r, 50))]); - harness.destroy(); - } - }); - test("cancels on Ctrl+D without continuing", async () => { const harness = await createHarness({ width: 80, height: 30 }); const done = runWelcome({ diff --git a/tests/unit/corbits-skills-catalog.test.ts b/tests/unit/corbits-skills-catalog.test.ts index 1ded9fab..2ebbb747 100644 --- a/tests/unit/corbits-skills-catalog.test.ts +++ b/tests/unit/corbits-skills-catalog.test.ts @@ -110,97 +110,6 @@ test("corbits-skills catalog lists 20 skills with name and description", async ( } }); -test("idiot-proof is a bake-only less-is-more bar", async () => { - const skill = await Bun.file(join(pluginRoot, "skills/idiot-proof/SKILL.md")).text(); - expect(skill).toContain(USER_INVOCABLE_FALSE); - expect(skill).toContain(DISABLE_MODEL_INVOCATION); - expect(skill).toContain("Prefer deletion over addition"); - expect(skill).toContain("Do not copy"); - expect(skill).toContain("files you already touch"); - expect(skill).toContain("Read the target"); - expect(skill).toContain("Do not fix"); -}); - -test("ponytail is compact use_skill-only Builder mode guidance", async () => { - const skill = await Bun.file(join(pluginRoot, "skills/ponytail/SKILL.md")).text(); - const body = skill.slice(skill.indexOf("---", 3) + 3).trim(); - const words = body.match(/\b[\w'-]+\b/g) ?? []; - expect(skill).toContain(USER_INVOCABLE_FALSE); - expect(skill).not.toContain(DISABLE_MODEL_INVOCATION); - expect(words.length).toBeGreaterThanOrEqual(150); - expect(words.length).toBeLessThanOrEqual(250); - expect(body).toContain("`lite`"); - expect(body).toContain("`off`"); - expect(body).toContain("`full`"); - expect(body).toContain("`ultra`"); - expect(body).toMatch(/Escalation ladder/i); - expect(body).toMatch(/Safety precedence/i); - expect(body).toMatch(/correctness.*validation.*security.*accessibility.*data integrity.*tests/is); - expect(body).toMatch(/mode never weakens/i); - expect(body).toMatch(/critic, neckbeard, or primary/i); - expect(body).not.toMatch( - /benchmark|marketing|Claude Code|opencode|upstream example|long command|command docs/i, - ); -}); - -test("native-runtime is compact bake-only Corbits worker invariants", async () => { - const skill = await Bun.file(join(pluginRoot, "skills/native-runtime/SKILL.md")).text(); - expect(skill).toContain(USER_INVOCABLE_FALSE); - expect(skill).toContain(DISABLE_MODEL_INVOCATION); - expect(skill).toContain("Corbits tool names"); - expect(skill).toContain("ask_director"); - expect(skill).toMatch(/Never use shell\s+redirects/); - expect(skill).toMatch(/Report every exact command with\s+outcome and exit status/); - expect(skill).toContain("Summary`"); -}); - -test("typescript skill is 1:1 with GaaS typescript", async () => { - const skill = await Bun.file(join(pluginRoot, "skills/typescript/SKILL.md")).text(); - expect(skill).toContain(USER_INVOCABLE_FALSE); - expect(skill).not.toContain(DISABLE_MODEL_INVOCATION); - expect(skill).toContain("## Quick Reference"); - expect(skill).toContain("### Don't"); - expect(skill).toContain("import type"); - expect(skill).toContain("arktype"); - expect(skill).toContain("unknown"); - expect(skill).toContain("create*"); - expect(skill).toContain('import t from "tap"'); - expect(skill).not.toContain("Guidance for TypeScript output quality"); - expect(skill).not.toContain("When project conventions disagree"); - expect(skill).not.toContain("## Acknowledgment"); - expect(skill).not.toContain("I have reviewed the typescript skill"); -}); - -test("implement skill is 1:1 with GaaS implement", async () => { - const skill = await Bun.file(join(pluginRoot, "skills/implement/SKILL.md")).text(); - expect(skill).toContain("TaskCreate"); - expect(skill).toContain("@greybeard"); - expect(skill).toContain("@critique"); - expect(skill).toContain("## Acknowledgment"); - expect(skill).toContain( - "I have reviewed the implement skill and am ready to follow the commit workflow.", - ); - expect(skill).not.toContain("spawn_agent"); - expect(skill).not.toContain("Linear claim"); - expect(skill).not.toContain("Do not invent a worker-count"); -}); - -test("opsh skill is 1:1 with GaaS opsh", async () => { - const skill = await Bun.file(join(pluginRoot, "skills/opsh/SKILL.md")).text(); - expect(skill).toContain(USER_INVOCABLE_FALSE); - expect(skill).not.toContain(DISABLE_MODEL_INVOCATION); - expect(skill).toContain("# opsh Scripting"); - expect(skill).toContain("#!/usr/bin/env opsh"); - expect(skill).toContain("lib::import"); - expect(skill).toContain("TAP v13"); - expect(skill).toContain("prove"); - expect(skill).toContain("testing::register"); - expect(skill).toContain("Load this skill when writing, reviewing, or debugging opsh scripts."); - expect(skill).not.toContain("spawn_agent"); - expect(skill).not.toContain("write_file/edit_file"); - expect(skill).not.toContain("Tiny / single-file scripts"); - expect(skill).not.toContain("## Acknowledgment"); -}); test("first-party skills are how-to playbooks, not director personas", async () => { const gaasOverlap = new Set([ @@ -230,142 +139,6 @@ test("first-party skills are how-to playbooks, not director personas", async () } }); -test("style skill is 1:1 with GaaS style", async () => { - const skill = await Bun.file(join(pluginRoot, "skills/style/SKILL.md")).text(); - expect(skill).toContain(USER_INVOCABLE_FALSE); - expect(skill).not.toContain(DISABLE_MODEL_INVOCATION); - expect(skill).toContain("## Git Repository Requirement"); - expect(skill).toContain("refuse to proceed"); - expect(skill).toContain("git rebase -i"); - expect(skill).toContain("## Acknowledgment"); - expect(skill).toContain( - "I have reviewed the style skill, and I am ready to proceed in good taste.", - ); - expect(skill).not.toContain("Do not refuse the task"); -}); - -test("philosophy skill is 1:1 with GaaS philosophy", async () => { - const skill = await Bun.file(join(pluginRoot, "skills/philosophy/SKILL.md")).text(); - expect(skill).toContain(USER_INVOCABLE_FALSE); - expect(skill).not.toContain(DISABLE_MODEL_INVOCATION); - expect(skill).toContain("## Guiding Principles"); - expect(skill).toContain("## Constraint Ownership"); - expect(skill).toContain("exactly one"); - expect(skill).toContain("## Backwards Compatibility"); - expect(skill).toContain("Pragmatic over idealistic"); - expect(skill).toContain("## Acknowledgment"); - expect(skill).toContain("I have reviewed the philosophy skill"); - expect(skill).not.toContain("write_file"); - expect(skill).not.toContain("run_shell"); - expect(skill).not.toContain("use_skill("); -}); - -test("review skill is 1:1 with GaaS code-review except slash name", async () => { - const skill = await Bun.file(join(pluginRoot, "skills/review/SKILL.md")).text(); - expect(skill).toContain("name: review"); - expect(skill).not.toContain("name: code-review"); - expect(skill).toContain("description: Perform a code review or pull request review on a branch"); - expect(skill).toContain("# Code Review"); - expect(skill).toContain("Ask the user"); - expect(skill).toContain("sub-agents"); - expect(skill).toContain("typescript-conventions"); - expect(skill).toContain("Cite the Check"); - expect(skill).toContain("git diff ...HEAD"); - expect(skill).not.toContain("ask_operator"); - expect(skill).not.toContain("argument-hint"); - expect(skill).not.toContain("Findings only"); - expect(skill).not.toContain("Post the Review on GitHub"); - expect(skill).not.toContain("spawn_agent"); - expect(skill).not.toContain("wait_agents"); - expect(skill).not.toContain("fleet agents"); -}); - -test("refactor skill is 1:1 with GaaS refactor", async () => { - const skill = await Bun.file(join(pluginRoot, "skills/refactor/SKILL.md")).text(); - expect(skill).toContain("ask clarifying questions:"); - expect(skill).toContain("ask the user about their priorities"); - expect(skill).toContain("load the `philosophy` skill"); - expect(skill).not.toContain("ask_operator"); - expect(skill).not.toContain(USER_INVOCABLE_FALSE); - expect(skill).not.toContain("## Acknowledgment"); -}); - -test("pull-request-review is 1:1 with GaaS pull-request-review", async () => { - const skill = await Bun.file(join(pluginRoot, "skills/pull-request-review/SKILL.md")).text(); - expect(skill).toContain("git worktree add"); - expect(skill).toContain("`code-review` skill"); - expect(skill).toContain("Load Code Review Skill"); - expect(skill).toContain("ask the user"); - expect(skill).not.toContain("Do not implement fixes"); - expect(skill).not.toContain("ask_operator"); - expect(skill).not.toContain("### Step 9:"); - expect(skill).not.toContain("Post the Review on GitHub"); - expect(skill).not.toContain(USER_INVOCABLE_FALSE); - expect(skill).not.toContain("spawn_agent"); - expect(skill).not.toContain('task(agent="critic")'); -}); - -test("interview skill is 1:1 with GaaS interview", async () => { - const skill = await Bun.file(join(pluginRoot, "skills/interview/SKILL.md")).text(); - expect(skill).toContain("AskUserQuestion"); - expect(skill).toContain("argument-hint"); - expect(skill).toContain("tools:\n - AskUserQuestion"); - expect(skill).toContain("This is a utility, not a planner"); - expect(skill).toContain("never writes a file"); - expect(skill).toContain("parameter limits"); - expect(skill).not.toContain("ask_operator"); - expect(skill).not.toContain(USER_INVOCABLE_FALSE); - expect(skill).not.toContain("## Acknowledgment"); -}); - -test("scribe skill is 1:1 with GaaS scribe", async () => { - const skill = await Bun.file(join(pluginRoot, "skills/scribe/SKILL.md")).text(); - expect(skill).toContain("tools:\n - question"); - expect(skill).toContain("Using the Question Tool"); - expect(skill).toContain("AskUserQuestion"); - expect(skill).toContain("the `question` tool"); - expect(skill).not.toContain("ask_operator"); - expect(skill).not.toContain(USER_INVOCABLE_FALSE); - expect(skill).not.toContain(DISABLE_MODEL_INVOCATION); - expect(skill).not.toContain("## Acknowledgment"); -}); - -test("ast-grep skill is 1:1 with GaaS ast-grep", async () => { - const skill = await Bun.file(join(pluginRoot, "skills/ast-grep/SKILL.md")).text(); - expect(skill).toContain("CLI: `sg`"); - expect(skill).toContain("|---|---|"); - expect(skill).toContain("sg run"); - expect(skill).toContain("sg scan --inline-rules"); - expect(skill).toContain("## Acknowledgment"); - expect(skill).toContain("I have reviewed the ast-grep skill."); - expect(skill).not.toContain("run_shell"); - expect(skill).not.toContain("Invoke `sg` via `run_shell`"); - expect(skill).not.toContain(USER_INVOCABLE_FALSE); - expect(skill).not.toContain(DISABLE_MODEL_INVOCATION); -}); - -test("create-issue is Linear-first without restated MCP tool contracts", async () => { - const skill = await Bun.file(join(pluginRoot, "skills/create-issue/SKILL.md")).text(); - expect(skill).toContain("name: create-issue"); - expect(skill).toContain("mcp__linear__"); - expect(skill).toContain("gh issue create"); - expect(skill).toContain(".corbits/MEMORY.md"); - expect(skill).toContain("Preferred issue tracker:"); - expect(skill).toContain("Do not invent a Linear REST client"); - expect(skill).toContain("Do not restate MCP tool names or schemas"); - expect(skill).toContain("`mcp__linear__*`"); - expect(skill).toContain("Phase 2: Analyze Input"); - expect(skill).toContain("Phase 5: Review and Adjust"); - expect(skill).toContain("# Background"); - expect(skill).toContain("# Outcome"); - expect(skill).toContain(" { for (const name of USE_SKILL_ONLY) { const skill = await Bun.file(join(pluginRoot, "skills", name, "SKILL.md")).text(); @@ -394,29 +167,6 @@ test("only background and bake-only skills carry disable-model-invocation", asyn } }); -test("git-rebase skill is 1:1 with GaaS git-rebase", async () => { - const skill = await Bun.file(join(pluginRoot, "skills/git-rebase/SKILL.md")).text(); - expect(skill).toContain(USER_INVOCABLE_FALSE); - expect(skill).not.toContain(DISABLE_MODEL_INVOCATION); - expect(skill).toContain("/tmp/rebase-editor.sh"); - expect(skill).toContain("driving every editor invocation non-interactively"); - expect(skill).not.toContain('spawn_agent(agent="intern")'); - expect(skill).not.toContain("Plan the surgery; intern executes"); -}); - -test("linear-issue-workflow is 1:1 with GaaS", async () => { - const skill = await Bun.file(join(pluginRoot, "skills/linear-issue-workflow/SKILL.md")).text(); - expect(skill).toContain(USER_INVOCABLE_FALSE); - expect(skill).not.toContain(DISABLE_MODEL_INVOCATION); - expect(skill).toContain("git worktree add"); - expect(skill).toContain("code-review"); - expect(skill).toContain("critique` subagent"); - expect(skill).toContain('Mark the issue as "In Progress"'); - expect(skill).not.toContain('use_skill("git-worktrees")'); - expect(skill).not.toContain("Claim immediately"); - expect(skill).not.toContain('set the issue state to "In Review"'); -}); - test("review skill does not own GitHub posting or Linear In Review", async () => { const skill = await Bun.file(join(pluginRoot, "skills/review/SKILL.md")).text(); expect(skill).not.toContain("Post the Review on GitHub"); @@ -450,55 +200,6 @@ test("Corbits-only skills do not contain GaaS tool names", async () => { } }); -test("native-integration maps GaaS tool names and parks Corbits extras", async () => { - const skill = await Bun.file(join(pluginRoot, "skills/native-integration/SKILL.md")).text(); - expect(skill).toContain(USER_INVOCABLE_FALSE); - expect(skill).not.toContain(DISABLE_MODEL_INVOCATION); - expect(skill).toContain("TaskCreate"); - expect(skill).toContain("TaskList"); - expect(skill).toContain("@greybeard"); - expect(skill).toContain("@critique"); - expect(skill).toContain("karen"); - expect(skill).toContain('intent="general"'); - expect(skill).toContain("manage_tasks"); - expect(skill).toContain('spawn_agent(agent="greybeard")'); - expect(skill).toContain("ask_operator"); - expect(skill).toContain("ask_director"); - expect(skill).toContain("folder without `.git`"); - expect(skill).toContain("gh pr review"); - expect(skill).toContain("Preferred issue tracker"); - expect(skill).toContain("/review"); - expect(skill).toContain("/create-issue"); - expect(skill).toContain("plan"); - expect(skill).toContain("git-worktrees"); - expect(skill).toContain("idiot-proof"); - expect(skill).toContain("bun:test"); - expect(skill).toContain('import t from "tap"'); - expect(skill).toContain("git worktree add"); - expect(skill).toContain('use_skill("git-worktrees")'); - expect(skill).toContain('set the issue state to "In Review"'); - expect(skill).toMatch(/ready for review/); - expect(skill).toContain("Draft or WIP PRs stay **In Progress**"); - expect(skill).toContain("Draft or WIP PRs stay In Progress"); - expect(skill).toContain("Do not mark the Linear issue Done on open PR alone"); - expect(skill).toContain("Do not leave it In Review after merge when work remains"); - expect(skill).toContain("/tmp"); - expect(skill).toContain("GIT_SEQUENCE_EDITOR"); - expect(skill).toContain("intern executes sequenced git via `run_shell`"); - expect(skill).toContain("Do not fork the GaaS git-rebase body"); - expect(skill).toContain("Do not fork the GaaS opsh body"); - expect(skill).toContain("Do not fork the GaaS pull-request-review body"); - expect(skill).toContain("Do not fork the GaaS refactor body"); - expect(skill).toContain("Do not fork the GaaS scribe body"); - expect(skill).toContain("Do not fork the GaaS ast-grep body"); - expect(skill).toContain("Do not fork the GaaS code-review body"); - expect(skill).toContain("findings-only"); - expect(skill).toContain("Do not fork the GaaS implement body"); - expect(skill).toContain("`/implement` does not steal planning from `/plan`"); - expect(skill).toContain("run `sg` via `run_shell`"); - expect(skill).toContain("prove"); -}); - test("loadSkillCommands lists exactly the nine slash actions", async () => { const cmds = await loadSkillCommands(join(import.meta.dirname, "../../plugins/corbits-skills")); expect(cmds!.map((c) => c.name).sort()).toEqual([ diff --git a/tests/unit/run-agent.test.ts b/tests/unit/run-agent.test.ts index 82846da5..ff398580 100644 --- a/tests/unit/run-agent.test.ts +++ b/tests/unit/run-agent.test.ts @@ -8,35 +8,6 @@ async function* makeStream(events: ReactorEmittedEvent[]): AsyncIterable { - // Cast each event as a whole rather than just `data` — casting `data` alone still - // leaves it typed as the union across all event kinds, which does not line up with - // the `type` discriminant on the surrounding object. - const events: ReactorEmittedEvent[] = [ - { type: "reactor.start", seq: 1, data: {} } as unknown as ReactorEmittedEvent, - { - type: "inference.tool_call.start", - seq: 2, - data: { name: "read_file" }, - } as unknown as ReactorEmittedEvent, - { - type: "tool.done", - seq: 3, - data: { - result: { callId: "c1", content: "ok", isError: false }, - }, - } as unknown as ReactorEmittedEvent, - ]; - - const received: ReactorEmittedEvent[] = []; - await consumeStream(makeStream(events), (event) => received.push(event)); - - expect(received.length).toBe(3); - expect(received[0]?.type).toBe("reactor.start"); - expect(received[1]?.type).toBe("inference.tool_call.start"); - expect(received[2]?.type).toBe("tool.done"); -}); - test("consumeStream handles empty stream", async () => { const received: ReactorEmittedEvent[] = []; await consumeStream(makeStream([]), (event) => received.push(event)); diff --git a/tests/unit/tui/theme.test.ts b/tests/unit/tui/theme.test.ts index 94bf1b0f..39ed8753 100644 --- a/tests/unit/tui/theme.test.ts +++ b/tests/unit/tui/theme.test.ts @@ -20,25 +20,10 @@ afterEach(() => { } }); -test("color returns the brand hex", () => { - expect(color("brand")).toBe("#f5933a"); -}); - -test("color returns the accent (summit blue) hex", () => { - expect(color("accent")).toBe("#7ea2c4"); -}); - test("warning reuses the brand orange hex", () => { expect(color("warning")).toBe(color("brand")); }); -test("color256 returns the ANSI-256 index for each role", () => { - expect(color256("brand")).toBe(173); - expect(color256("accent")).toBe(74); - expect(color256("success")).toBe(108); - expect(color256("danger")).toBe(167); -}); - test("every role maps to a valid ANSI-256 index", () => { for (const role of Object.keys(palette) as (keyof typeof palette)[]) { const idx = color256(role); @@ -87,10 +72,6 @@ test("syntax comments recede to the dim rung and strings match success green", ( expect(palette.syntaxVariable).toEqual(palette.text); }); -test("userMessageBg preserves the established user box gray", () => { - expect(color("userMessageBg")).toBe("#45454a"); -}); - test("supportsTrueColor detects truecolor terminals", () => { process.env.COLORTERM = "truecolor"; expect(supportsTrueColor()).toBe(true); diff --git a/tests/unit/tui/tool-formatter-web-brand.test.ts b/tests/unit/tui/tool-formatter-web-brand.test.ts deleted file mode 100644 index aa378259..00000000 --- a/tests/unit/tui/tool-formatter-web-brand.test.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { test, expect, afterEach } from "bun:test"; -import { humanizeToolName, setActiveWebProviderBrand } from "../../../src/tui/tool-formatter.js"; - -afterEach(() => setActiveWebProviderBrand(undefined)); - -test("web tools use the default names with no active web brand", () => { - expect(humanizeToolName("web_search")).toBe("Web Search"); - expect(humanizeToolName("web_fetch")).toBe("Web Fetch"); -}); - -test("web tools render with the active web plugin brand", () => { - setActiveWebProviderBrand("Exa"); - expect(humanizeToolName("web_search")).toBe("Exa Search"); - expect(humanizeToolName("web_fetch")).toBe("Exa Fetch"); -}); - -test("non-web tools are unaffected by the web brand", () => { - setActiveWebProviderBrand("Exa"); - expect(humanizeToolName("read_file")).toBe("Read"); -}); From 72634b227b499345e80e2cfc7eac7dc8a910281a Mon Sep 17 00:00:00 2001 From: Sawyer Date: Mon, 7 Sep 2026 22:16:33 -0700 Subject: [PATCH 02/18] Restructure CI to parallelize static analysis and shard tests --- .github/workflows/ci.yml | 109 +++++++++++++++++------------ package.json | 1 + scripts/guard-real-projects-dir.ts | 13 +++- 3 files changed, 78 insertions(+), 45 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 894fbe75..01726f4c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,10 +11,12 @@ concurrency: cancel-in-progress: ${{ github.event_name != 'push' }} jobs: - # Prettier and eslint run un-cached in CI: restored result caches can mark - # files clean against a stale tool version or config, masking real failures. - # The --cache flags in the package.json lint script remain for local speed. - prettier: + # Prettier, eslint, and typecheck share one runner: one checkout and one + # install instead of three of each. Prettier and eslint still run un-cached + # in CI: restored result caches can mark files clean against a stale tool + # version or config, masking real failures. The --cache flags in the + # package.json lint script remain for local speed. + static-analysis: runs-on: ubuntu-latest steps: - name: Checkout @@ -29,7 +31,15 @@ jobs: uses: actions/cache@v4 with: path: node_modules + # The exact key keeps hits honest: only a cache built from this + # bun.lock restores. restore-keys falls back to the newest cache + # when the lockfile changed, so a dependency bump reinstalls the + # delta instead of cold-installing on every job at once. bun + # install --frozen-lockfile reconciles a stale tree to the new + # lockfile, so a partial hit never leaves wrong deps behind. key: bun-${{ hashFiles('bun.lock') }} + restore-keys: | + bun- - name: Install dependencies run: bun install --frozen-lockfile @@ -37,35 +47,26 @@ jobs: - name: Prettier run: bunx prettier --check . - eslint: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup Bun - uses: oven-sh/setup-bun@v2 - with: - bun-version: "1.3.14" - - - name: Cache dependencies - uses: actions/cache@v4 - with: - path: node_modules - key: bun-${{ hashFiles('bun.lock') }} - - - name: Install dependencies - run: bun install --frozen-lockfile - - name: ESLint run: bunx eslint . - typecheck: + - name: Typecheck + run: bun run typecheck + + # Build runs beside the suite instead of before it: tests import ./src + # directly and never read ./dist, so serializing build ahead of test put + # build time on the critical path for no dependency reason. + build: runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: "24" + - name: Setup Bun uses: oven-sh/setup-bun@v2 with: @@ -76,15 +77,32 @@ jobs: with: path: node_modules key: bun-${{ hashFiles('bun.lock') }} + restore-keys: | + bun- - name: Install dependencies run: bun install --frozen-lockfile - - name: Typecheck - run: bun run typecheck + - name: Build + run: bun run build - build-and-test: + # The suite is sharded so the slowest slice, not the whole suite, sets the + # wall clock. Every shard still goes through check:projects-dir-guard: the + # guard forwards these path filters to the suite it wraps, and the union of + # the shards' filters is exactly ./src ./tests ./evals, so the gate covers + # the same tests as before, all of them sandboxed. + test: runs-on: ubuntu-latest + strategy: + # A red shard must not cancel the other; both results are the signal. + fail-fast: false + matrix: + shard: + - name: src + paths: ./src + - name: tests-and-evals + paths: ./tests ./evals + name: test (${{ matrix.shard.name }}) steps: - name: Checkout uses: actions/checkout@v4 @@ -99,29 +117,32 @@ jobs: with: bun-version: "1.3.14" + # The runner image has no ripgrep, so the grep plugin silently exercised + # its fallback walker and left the ripgrep path untested. + - name: Install ripgrep + run: sudo apt-get install -y ripgrep + - name: Cache dependencies uses: actions/cache@v4 with: path: node_modules key: bun-${{ hashFiles('bun.lock') }} - - # The runner image has no ripgrep, so the grep plugin silently exercised - # its fallback walker and left the ripgrep path untested. - - name: Install ripgrep - run: sudo apt-get install -y ripgrep + restore-keys: | + bun- - name: Install dependencies run: bun install --frozen-lockfile - - name: Build - run: bun run build - - # Same script the local `bun run check` gate runs: the projects-dir - # guard wraps `bun run test`, which is the seeded, randomized suite - # (bun test ./src ./tests ./evals --randomize --seed 424242) defined - # once in package.json. Randomized order catches tests that only pass - # in the default file order (shared module-level state, an unrestored - # global mock, a leaked env var); the seed is fixed so a failure here - # reproduces locally with `bun run test`. + # The same script the local `bun run check` gate runs, with the shard's + # path filters forwarded through the guard to the suite. The guard + # routes a filtered run through test:paths, which carries the same + # seeded flags as the `test` script; bun test filters are additive, so + # appending filters to `bun run test` could not narrow it. Randomized + # order catches tests that only pass in the default file order (shared + # module-level state, an unrestored global mock, a leaked env var). + # The seed stays 424242 in every shard rather than varying per shard: + # the shards already run disjoint file sets, and a fixed seed keeps + # any failure reproducible locally with the same + # `bun run test:paths `. - name: Test - run: bun run check:projects-dir-guard + run: bun run check:projects-dir-guard ${{ matrix.shard.paths }} diff --git a/package.json b/package.json index 7068d1c5..74c6b686 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "build:bin": "bun build ./src/index.ts --compile --minify --define process.env.NODE_ENV='\"production\"' --outfile ./dist/corbits && bun scripts/copy-repo-plugins.ts", "typecheck": "tsc --noEmit", "test": "bun test ./src ./tests ./evals --randomize --seed 424242", + "test:paths": "bun test --randomize --seed 424242", "lint": "prettier --check --cache . && eslint --cache .", "check:projects-dir-guard": "bun scripts/guard-real-projects-dir.ts", "check": "bun run lint && bun run typecheck && bun run build && bun run check:projects-dir-guard", diff --git a/scripts/guard-real-projects-dir.ts b/scripts/guard-real-projects-dir.ts index 3c897cb7..9faffc74 100644 --- a/scripts/guard-real-projects-dir.ts +++ b/scripts/guard-real-projects-dir.ts @@ -45,7 +45,18 @@ async function main(): Promise { const runTmpDir = join(tmpdir(), `corbits-test-guard-${runId}`); await mkdir(runTmpDir, { recursive: true }); - const child = spawn("bun", ["run", "test"], { + // CI test shards pass bun-test path filters here (e.g. ./src) so each + // shard runs only its slice of the suite and still runs sandboxed. bun + // test filters are additive, so filters cannot be appended to + // `bun run test` (its own filters would widen the run back to the full + // suite), so a sharded run goes through test:paths, which carries the + // same seeded flags as `test` and takes the shard's filters. With no + // arguments the full default suite runs via `bun run test`, so `bun run + // check` behavior is unchanged. + const shardArgs = process.argv.slice(2); + const testCommand = shardArgs.length > 0 ? ["run", "test:paths", ...shardArgs] : ["run", "test"]; + + const child = spawn("bun", testCommand, { stdio: "inherit", env: { ...process.env, TMPDIR: runTmpDir, TMP: runTmpDir, TEMP: runTmpDir }, }); From 942887842162bb3251cee1bb3c1e8dcd3f861e33 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Mon, 7 Sep 2026 21:59:56 -0700 Subject: [PATCH 03/18] Pin git-global-config ask against shell wrapper bypasses sh -c, xargs, transparent env prefixes, and quoted program/flag spellings previously had no coverage for the git-global-config rule; only the plain forms were pinned. Assert they all still ask, and that scoped read-only git config forms stay unflagged. --- src/permission/auto-shell-policy.test.ts | 75 ++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 src/permission/auto-shell-policy.test.ts diff --git a/src/permission/auto-shell-policy.test.ts b/src/permission/auto-shell-policy.test.ts new file mode 100644 index 00000000..186215eb --- /dev/null +++ b/src/permission/auto-shell-policy.test.ts @@ -0,0 +1,75 @@ +import { describe, test, expect } from "bun:test"; +import type { ToolCall } from "@intx/types/runtime"; +import { autoShellRuleForCall } from "./auto-shell-policy.js"; + +const shellCall = (command: string): ToolCall => ({ + id: "c", + name: "run_shell", + arguments: { command }, +}); + +// Base git-global-config routing (--global/--system/--edit, --file targets, +// unset/reassignment of GIT_CONFIG_GLOBAL, repo-local pass-through) is pinned +// in classify-security.test.ts. This file pins the surface that file does not: +// shell wrappers and quoting must not demote the ask to an auto-allow. +describe("git-global-config ask survives shell wrappers", () => { + // Control: the plain form names the rule the wrapped forms must still hit. + test("plain form names the rule", () => { + expect(autoShellRuleForCall(shellCall("git config --global user.name foo"))?.name).toBe( + "git-global-config", + ); + }); + + test("sh -c and bash -c payloads still match", () => { + expect(autoShellRuleForCall(shellCall("sh -c 'git config --global user.name foo'"))?.name).toBe( + "git-global-config", + ); + expect( + autoShellRuleForCall(shellCall('bash -c "git config --global user.email x@y.z"'))?.name, + ).toBe("git-global-config"); + }); + + test("bare and piped xargs do not bypass the ask", () => { + expect(autoShellRuleForCall(shellCall("xargs git config --global user.name foo"))?.name).toBe( + "git-global-config", + ); + expect( + autoShellRuleForCall(shellCall("echo refs | xargs git config --global user.name foo"))?.name, + ).toBe("git-global-config"); + }); + + test("transparent env prefix still peels through to the rule", () => { + expect(autoShellRuleForCall(shellCall("env git config --global user.name foo"))?.name).toBe( + "git-global-config", + ); + }); + + test("a NAME=value prefix still asks (env-assignment fires first)", () => { + // The assignment itself is the earlier ask rule in the table, so the name + // differs — what is pinned here is that the call never auto-allows. + const rule = autoShellRuleForCall(shellCall("FOO=bar git config --global user.name foo")); + expect(rule?.effect).toBe("ask"); + expect(rule?.name).toBe("env-assignment"); + }); + + test("quote round-trips on the program or flag still match", () => { + expect(autoShellRuleForCall(shellCall('git "config" --global user.name foo'))?.name).toBe( + "git-global-config", + ); + expect(autoShellRuleForCall(shellCall("git config '--global' user.name foo"))?.name).toBe( + "git-global-config", + ); + expect( + autoShellRuleForCall(shellCall('sh -c \'git "config" --global user.name foo\''))?.name, + ).toBe("git-global-config"); + }); +}); + +describe("read-only git commands pass through the policy", () => { + test("scoped reads and repo-local writes stay unflagged", () => { + expect(autoShellRuleForCall(shellCall("git config --get user.name"))).toBeUndefined(); + expect(autoShellRuleForCall(shellCall("git config --list"))).toBeUndefined(); + expect(autoShellRuleForCall(shellCall("git config --get-regexp '^branch\\.'"))).toBeUndefined(); + expect(autoShellRuleForCall(shellCall("git status --porcelain"))).toBeUndefined(); + }); +}); From 68871baec3e3fa0fd18a8057085f57db2a314a8a Mon Sep 17 00:00:00 2001 From: Sawyer Date: Mon, 7 Sep 2026 22:02:46 -0700 Subject: [PATCH 04/18] Cover torn JSONL recovery when the base segment is the active one Existing torn-tail tests only tear a tail segment behind a clean turns.jsonl. A session small enough to never roll over tears the base the isogit store parses first, which takes the resilient recovery path instead. Pin that the base-only tear recovers committed turns, extra segments survive it, and the next write heals the file. --- src/session/optimized-context-store.test.ts | 46 +++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/session/optimized-context-store.test.ts b/src/session/optimized-context-store.test.ts index e745f842..53e94cd0 100644 --- a/src/session/optimized-context-store.test.ts +++ b/src/session/optimized-context-store.test.ts @@ -126,6 +126,52 @@ describe("createOptimizedContextStore load", () => { expect(loaded.turns.map((t) => (t.content[0] as { text: string }).text)).toEqual(["a", "b"]); }); + // A small session never rolls over, so the active segment is turns.jsonl + // itself and the torn line sits in the base the isogit store parses first. + test("recovers from a torn final line in the base segment with no extras", async () => { + const dir = tempDir(); + const store = await createOptimizedContextStore(dir); + + fs.writeFileSync( + path.join(dir, TURNS_FILE), + jsonl([turn("a"), turn("b")]) + '{"role":"user","content":[{"type":"te', + ); + + const loaded = await store.load(); + expect(loaded.turns.map((t) => (t.content[0] as { text: string }).text)).toEqual(["a", "b"]); + }); + + test("keeps extra segments when the torn line is in the base segment", async () => { + const dir = tempDir(); + const store = await createOptimizedContextStore(dir); + + fs.writeFileSync(path.join(dir, TURNS_FILE), jsonl([turn("a")]) + '{"role":"user","content":[{"type":"te'); + fs.writeFileSync( + path.join(dir, segmentFileName(TURNS_FILE, 1)), + jsonl([turn("b"), turn("c")]), + ); + + const loaded = await store.load(); + expect(loaded.turns.map((t) => (t.content[0] as { text: string }).text)).toEqual([ + "a", + "b", + "c", + ]); + }); + + test("the next write heals a torn base tail so reload is stable", async () => { + const dir = tempDir(); + const store = await createOptimizedContextStore(dir); + + fs.writeFileSync(path.join(dir, TURNS_FILE), jsonl([turn("a")]) + '{"role":"user","content":[{"type":"te'); + + const recovered = await store.load(); + await store.writeTurns(recovered.turns); + const reloaded = await store.load(); + expect(reloaded.turns.map((t) => (t.content[0] as { text: string }).text)).toEqual(["a"]); + expect(fs.readFileSync(path.join(dir, TURNS_FILE), "utf8")).toBe(jsonl([turn("a")])); + }); + test("recovers usable turns when turns.jsonl has a mid-file null-byte hole", async () => { const dir = tempDir(); const store = await createOptimizedContextStore(dir); From 1e99aa976952302a467423873233fab8edbbdd3d Mon Sep 17 00:00:00 2001 From: Sawyer Date: Mon, 7 Sep 2026 22:04:07 -0700 Subject: [PATCH 05/18] Extend exec close tests to multi-worker cancel and close failure Pin that disposeExecRuntime cancels all live workers with the close reason before agent teardown starts, and that a rejected agent close neither blocks toolset disposal nor rejects the teardown. --- tests/unit/exec/runner.test.ts | 40 ++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/unit/exec/runner.test.ts b/tests/unit/exec/runner.test.ts index f24400af..0754a336 100644 --- a/tests/unit/exec/runner.test.ts +++ b/tests/unit/exec/runner.test.ts @@ -511,6 +511,46 @@ describe("disposeExecRuntime", () => { }), ).rejects.toThrow("plugin dispose failed"); }); + + test("cancels every live worker with the close reason before the agent closes", async () => { + const store = createSubAgentSessionStore(); + const first = store.start({ description: "a", agentId: "w1", brief: "b" }); + const second = store.start({ description: "b", agentId: "w2", brief: "b" }); + const calls: string[] = []; + store.registerCancel(first.id, () => calls.push("cancel:first")); + store.registerCancel(second.id, () => calls.push("cancel:second")); + + await disposeExecRuntime({ + agent: { close: async () => void calls.push("agent") }, + toolset: { dispose: async () => void calls.push("toolset") }, + subAgentSessions: store, + }); + + // Cancellation must precede teardown so no worker outlives the runtime. + expect(calls).toEqual(["cancel:first", "cancel:second", "agent", "toolset"]); + expect(store.get(first.id)?.status).toBe("cancelled"); + expect(store.get(second.id)?.status).toBe("cancelled"); + expect(store.get(first.id)?.stopReason).toBe("cancelled — Session closed"); + expect(store.get(second.id)?.stopReason).toBe("cancelled — Session closed"); + }); + + test("a failing agent close still disposes the toolset and resolves", async () => { + const store = createSubAgentSessionStore(); + const worker = store.start({ description: "bg", agentId: "w", brief: "b" }); + store.registerCancel(worker.id, () => undefined); + + let disposed = 0; + await disposeExecRuntime({ + agent: { + close: () => Promise.reject(new Error("close exploded")), + }, + toolset: { dispose: async () => void (disposed += 1) }, + subAgentSessions: store, + }); + + expect(store.get(worker.id)?.status).toBe("cancelled"); + expect(disposed).toBe(1); + }); }); describe("resolveExecDirectorOverlay", () => { From 65590addfe84f3731196efd7ad3ed8fbe320574d Mon Sep 17 00:00:00 2001 From: Sawyer Date: Mon, 7 Sep 2026 22:05:02 -0700 Subject: [PATCH 06/18] Pin session queue ordering for rotation racing an in-flight delivery Cover the enqueue-during-await race: a rotation queued while a delivery is still awaiting its send must wait for the delivery to settle, and a failed delivery must not block the rotation behind it. --- src/tui/session-operation-queue.test.ts | 44 +++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/tui/session-operation-queue.test.ts b/src/tui/session-operation-queue.test.ts index 479a60f2..42c98c73 100644 --- a/src/tui/session-operation-queue.test.ts +++ b/src/tui/session-operation-queue.test.ts @@ -62,3 +62,47 @@ test("deliver targets agent at execution time when enqueued before rotation", as await awaitTail(); expect(log).toEqual(["deliver:A", "rotate"]); }); + +test("rotation enqueued during an in-flight delivery waits for it to settle", async () => { + const log: string[] = []; + const { enqueue, awaitTail } = createSessionOperationQueue(); + + let resolveSend!: () => void; + const send = new Promise((r) => (resolveSend = r)); + + enqueue(async () => { + log.push("deliver:start"); + await send; + log.push("deliver:end"); + }); + enqueue(async () => { + log.push("rotate:start"); + log.push("rotate:end"); + }); + + // The rotation is already queued while the delivery is still awaiting the + // provider send — it must not start (rotating the session dir) mid-delivery. + await Promise.resolve(); + await Promise.resolve(); + expect(log).toEqual(["deliver:start"]); + + resolveSend(); + await awaitTail(); + expect(log).toEqual(["deliver:start", "deliver:end", "rotate:start", "rotate:end"]); +}); + +test("a failed delivery does not block a rotation queued behind it", async () => { + const log: string[] = []; + const { enqueue, awaitTail } = createSessionOperationQueue(); + + enqueue(async () => { + log.push("deliver:start"); + throw new Error("send failed"); + }); + enqueue(async () => { + log.push("rotate"); + }); + + await awaitTail(); + expect(log).toEqual(["deliver:start", "rotate"]); +}); From cc7b8a7001cbc43151ef002be7a5cb924c83b21f Mon Sep 17 00:00:00 2001 From: Sawyer Date: Mon, 7 Sep 2026 22:07:26 -0700 Subject: [PATCH 07/18] Cover MCP reconnect after the server's tool schemas drift The connect mock served one static tool payload, so reconnect tests could not observe a server whose tools changed between generations. Parameterize the payload and pin that reconnect mounts exactly the drifted set: renamed schema on the same tool, a new tool, a single definition per name, the stale client closed, and the drift announced through onToolsChanged. --- src/agent/tools-mcp-disconnect.test.ts | 49 ++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/src/agent/tools-mcp-disconnect.test.ts b/src/agent/tools-mcp-disconnect.test.ts index dbd8e79a..6e75c629 100644 --- a/src/agent/tools-mcp-disconnect.test.ts +++ b/src/agent/tools-mcp-disconnect.test.ts @@ -4,7 +4,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { withMockedModule } from "../../tests/helpers/mock-module.js"; import { createExaMCPServerConfig, type ResolvedMCPServerConfig } from "../mcp/exa.js"; -import type { MCPConnectOptions } from "../mcp/client.js"; +import type { MCPConnectOptions, MCPTool } from "../mcp/client.js"; import { createPermissionGate } from "../permission/gate.js"; import type { MCPServerState } from "./tools.js"; @@ -15,6 +15,9 @@ let connectGeneration = 0; let connectOptions: MCPConnectOptions[] = []; let releaseDeferredConnect: (() => void) | undefined; let connectMode: "success" | "deferred" = "success"; +// Reconnect tests repoint this to simulate a server whose tool set drifted +// between generations; the default matches the original static payload. +let connectedTools: MCPTool[] = [{ name: "list", description: "List", inputSchema: {} }]; function tempCwd(): string { const dir = mkdtempSync(join(tmpdir(), "corbits-mcp-disconnect-")); @@ -44,7 +47,7 @@ await withMockedModule( ok: true as const, client: { serverName: config.name, - tools: [{ name: "list", description: "List", inputSchema: {} }], + tools: connectedTools, call: async () => "ok", close: async () => { closedClients.push(config.name); @@ -105,6 +108,7 @@ beforeEach(() => { connectOptions = []; releaseDeferredConnect = undefined; connectMode = "success"; + connectedTools = [{ name: "list", description: "List", inputSchema: {} }]; }); async function waitForConnectStart(timeoutMs = 1000): Promise { @@ -175,6 +179,47 @@ describe("disconnectMCPServer", () => { } }); + test("reconnect after the server's tools drift swaps the mounted set", async () => { + const toolset = await makeToolset(); + const states: MCPServerState[] = []; + const announced: ReturnType[] = []; + try { + await toolset.connectMCPServer(acme, callbacks(states)); + const acmeNames = (defs: ReturnType) => + defs.map((d) => d.name).filter((name) => name.startsWith("mcp__acme__")); + expect(acmeNames(toolset.dynamicRunner.currentDefinitions())).toEqual(["mcp__acme__list"]); + + // The server redeployed mid-session: same tool name, new schema, plus a + // new tool. Reconnect must mount exactly the drifted set. + connectedTools = [ + { name: "list", description: "List v2", inputSchema: { type: "object", required: ["q"] } }, + { name: "search", description: "Search", inputSchema: {} }, + ]; + await toolset.disconnectMCPServer("acme", callbacks(states)); + await toolset.connectMCPServer(acme, { + ...callbacks(states), + onToolsChanged: (definitions) => announced.push(definitions), + }); + + const names = acmeNames(toolset.dynamicRunner.currentDefinitions()); + expect(names).toContain("mcp__acme__list"); + expect(names).toContain("mcp__acme__search"); + expect(names.filter((name) => name === "mcp__acme__list")).toHaveLength(1); + + const list = toolset.dynamicRunner + .currentDefinitions() + .find((d) => d.name === "mcp__acme__list"); + expect(list?.description).toBe("[acme] List v2"); + expect(list?.inputSchema).toEqual({ type: "object", required: ["q"] }); + + // The stale generation's client was closed and the drift was announced. + expect(closedGenerations).toContain(1); + expect(acmeNames(announced.at(-1) ?? [])).toContain("mcp__acme__search"); + } finally { + await toolset.dispose(); + } + }); + test("disconnecting lin does not drop linear tools", async () => { const toolset = await makeToolset(); const states: MCPServerState[] = []; From 9f1028729721b98da8c34e7ff62643cf9adc69ef Mon Sep 17 00:00:00 2001 From: Sawyer Date: Mon, 7 Sep 2026 22:38:55 -0700 Subject: [PATCH 08/18] Fix flaky subagent mount-gate tests pinned on retry backoff sleeps The mount-gate probes await a full runSubAgent cycle whose inference send to an unreachable host classifies as retryable, so each test slept through the client's full backoff schedule (three attempts, 500ms plus 1000ms fixed) before asserting mount counts. Under suite load that crossed the 5s per-test timeout. Point the send at a local server answering 401, which classifies credential_failure and never retries, and give the four full-runtime probes a 15s timeout to absorb construction-time load spikes. --- src/subagent/run-authority.test.ts | 209 ++++++++++++++++------------- 1 file changed, 113 insertions(+), 96 deletions(-) diff --git a/src/subagent/run-authority.test.ts b/src/subagent/run-authority.test.ts index 626aaeda..827db81c 100644 --- a/src/subagent/run-authority.test.ts +++ b/src/subagent/run-authority.test.ts @@ -28,17 +28,50 @@ async function tmpCwd(): Promise { return mkdtemp(join(tmpdir(), "cl6941-run-authority-")); } -function baseParams(cwd: string, workdirBase: string): Omit { +function baseParams( + cwd: string, + workdirBase: string, + baseURL = "http://localhost", +): Omit { return { cwd, workdirBase, permissionGate: testPermissionGate, - provider: { providerName: "test", baseURL: "http://localhost", model: "test-model" }, + provider: { providerName: "test", baseURL, model: "test-model" }, description: "gate probe", prompt: "no-op", }; } +// Each mount-gate probe awaits a full runSubAgent cycle whose inference send +// fails after the mount decisions have run. The send used to target an +// unreachable host, whose connection-refused failure classifies as retryable +// — the client burned its full backoff schedule (three attempts with 500ms + +// 1000ms of fixed sleep) per test, enough to cross bun:test's 5s timeout +// whenever the randomized suite loaded the machine. A local server answering +// 401 fails the send as credential_failure, which is never retried, so the +// cycle costs one local round trip and no timing-sensitive waiting. The 15s +// per-test timeouts below only absorb machine-load spikes during the +// full-runtime construction these probes perform; assertions are +// timing-independent. +async function runWithFailingInference( + run: (baseURL: string) => Promise, +): Promise { + const server = Bun.serve({ + port: 0, + fetch: () => + new Response(JSON.stringify({ error: { message: "mount-gate probe provider" } }), { + status: 401, + headers: { "content-type": "application/json" }, + }), + }); + try { + await run(server.url.origin); + } finally { + server.stop(true); + } +} + describe("runSubAgent fleet-verb mount gate (CL-6941, fails closed)", () => { test("orchestrator=true with no resolvable tier (non-closed-director profile shape) is denied", async () => { const cwd = await tmpCwd(); @@ -89,84 +122,76 @@ describe("runSubAgent search_agents mount gate (CL-7051, Tier-1 only)", () => { const cwd = await tmpCwd(); let searchAgentsMounts = 0; - await withMockedModuleDuring( - import.meta.resolve("../agent/agent-search.js"), - (real: typeof import("../agent/agent-search.js")) => ({ - ...real, - createSearchAgentsTool: (getProfiles: () => never) => { - searchAgentsMounts++; - return real.createSearchAgentsTool(getProfiles); - }, - }), - async () => { - // Re-import so the mock is visible to runSubAgent's binding. - const { runSubAgent: run } = await import("./run.js"); - try { + await runWithFailingInference((baseURL) => + withMockedModuleDuring( + import.meta.resolve("../agent/agent-search.js"), + (real: typeof import("../agent/agent-search.js")) => ({ + ...real, + createSearchAgentsTool: (getProfiles: () => never) => { + searchAgentsMounts++; + return real.createSearchAgentsTool(getProfiles); + }, + }), + async () => { + // Re-import so the mock is visible to runSubAgent's binding. + const { runSubAgent: run } = await import("./run.js"); await run({ - ...baseParams(cwd, join(cwd, ".ctx")), + ...baseParams(cwd, join(cwd, ".ctx"), baseURL), id: "greybeard-session", orchestrator: true, orchestratorTier: "nested-orchestrator", nestedDispatch: { permissionGate: testPermissionGate, getWorkdirBase: () => join(cwd, ".ctx"), - provider: { - providerName: "test", - baseURL: "http://localhost", - model: "test-model", - }, + provider: { providerName: "test", baseURL, model: "test-model" }, profiles: [{ id: "intern", systemPromptRole: "You are intern." }], }, + }).catch(() => { + // Inference/agent construction may fail; mount decisions run first. }); - } catch { - // Inference/agent construction may fail; mount decisions run first. - } - }, + }, + ), ); expect(searchAgentsMounts).toBe(0); - }); + }, 15_000); test("Tier-1 orchestrator mounts search_agents when profiles exist", async () => { const cwd = await tmpCwd(); let searchAgentsMounts = 0; - await withMockedModuleDuring( - import.meta.resolve("../agent/agent-search.js"), - (real: typeof import("../agent/agent-search.js")) => ({ - ...real, - createSearchAgentsTool: (getProfiles: () => never) => { - searchAgentsMounts++; - return real.createSearchAgentsTool(getProfiles); - }, - }), - async () => { - const { runSubAgent: run } = await import("./run.js"); - try { + await runWithFailingInference((baseURL) => + withMockedModuleDuring( + import.meta.resolve("../agent/agent-search.js"), + (real: typeof import("../agent/agent-search.js")) => ({ + ...real, + createSearchAgentsTool: (getProfiles: () => never) => { + searchAgentsMounts++; + return real.createSearchAgentsTool(getProfiles); + }, + }), + async () => { + const { runSubAgent: run } = await import("./run.js"); await run({ - ...baseParams(cwd, join(cwd, ".ctx")), + ...baseParams(cwd, join(cwd, ".ctx"), baseURL), id: "skywalker-session", orchestrator: true, orchestratorTier: "orchestrator", nestedDispatch: { permissionGate: testPermissionGate, getWorkdirBase: () => join(cwd, ".ctx"), - provider: { - providerName: "test", - baseURL: "http://localhost", - model: "test-model", - }, + provider: { providerName: "test", baseURL, model: "test-model" }, profiles: [{ id: "intern", systemPromptRole: "You are intern." }], }, + }).catch(() => { + // Inference/agent construction may fail; mount decisions run first. }); - } catch { - // Inference/agent construction may fail; mount decisions run first. - } - }, + }, + ), ); expect(searchAgentsMounts).toBe(1); - }); + }, 15_000); }); describe("runSubAgent passes parentSessionId into spawn_agent mount", () => { @@ -175,44 +200,40 @@ describe("runSubAgent passes parentSessionId into spawn_agent mount", () => { let capturedParentSessionId: string | undefined; let spawnMounts = 0; - await withMockedModuleDuring( - import.meta.resolve("./agent-fleet.js"), - (real: typeof import("./agent-fleet.js")) => ({ - ...real, - createSpawnAgentTool: (deps: Parameters[0]) => { - spawnMounts++; - capturedParentSessionId = deps.parentSessionId; - return real.createSpawnAgentTool(deps); - }, - }), - async () => { - const { runSubAgent: run } = await import("./run.js"); - try { + await runWithFailingInference((baseURL) => + withMockedModuleDuring( + import.meta.resolve("./agent-fleet.js"), + (real: typeof import("./agent-fleet.js")) => ({ + ...real, + createSpawnAgentTool: (deps: Parameters[0]) => { + spawnMounts++; + capturedParentSessionId = deps.parentSessionId; + return real.createSpawnAgentTool(deps); + }, + }), + async () => { + const { runSubAgent: run } = await import("./run.js"); await run({ - ...baseParams(cwd, join(cwd, ".ctx")), + ...baseParams(cwd, join(cwd, ".ctx"), baseURL), id: "greybeard-session", orchestrator: true, orchestratorTier: "nested-orchestrator", nestedDispatch: { permissionGate: testPermissionGate, getWorkdirBase: () => join(cwd, ".ctx"), - provider: { - providerName: "test", - baseURL: "http://localhost", - model: "test-model", - }, + provider: { providerName: "test", baseURL, model: "test-model" }, profiles: [{ id: "intern", systemPromptRole: "You are intern." }], }, + }).catch(() => { + // Inference/agent construction may fail; mount decisions run first. }); - } catch { - // Inference/agent construction may fail; mount decisions run first. - } - }, + }, + ), ); expect(spawnMounts).toBe(1); expect(capturedParentSessionId).toBe("greybeard-session"); - }); + }, 15_000); }); describe("runSubAgent list_agents mount (mailbox-scoped, nested ok)", () => { @@ -220,39 +241,35 @@ describe("runSubAgent list_agents mount (mailbox-scoped, nested ok)", () => { const cwd = await tmpCwd(); let listAgentsMounts = 0; - await withMockedModuleDuring( - import.meta.resolve("./agent-fleet.js"), - (real: typeof import("./agent-fleet.js")) => ({ - ...real, - createListAgentsTool: (deps: never) => { - listAgentsMounts++; - return real.createListAgentsTool(deps); - }, - }), - async () => { - const { runSubAgent: run } = await import("./run.js"); - try { + await runWithFailingInference((baseURL) => + withMockedModuleDuring( + import.meta.resolve("./agent-fleet.js"), + (real: typeof import("./agent-fleet.js")) => ({ + ...real, + createListAgentsTool: (deps: never) => { + listAgentsMounts++; + return real.createListAgentsTool(deps); + }, + }), + async () => { + const { runSubAgent: run } = await import("./run.js"); await run({ - ...baseParams(cwd, join(cwd, ".ctx")), + ...baseParams(cwd, join(cwd, ".ctx"), baseURL), id: "greybeard-session", orchestrator: true, orchestratorTier: "nested-orchestrator", nestedDispatch: { permissionGate: testPermissionGate, getWorkdirBase: () => join(cwd, ".ctx"), - provider: { - providerName: "test", - baseURL: "http://localhost", - model: "test-model", - }, + provider: { providerName: "test", baseURL, model: "test-model" }, }, + }).catch(() => { + // Inference/agent construction may fail; mount decisions run first. }); - } catch { - // Inference/agent construction may fail; mount decisions run first. - } - }, + }, + ), ); expect(listAgentsMounts).toBe(1); - }); + }, 15_000); }); From dc5f9ff56580bd7afc8dcc64f122850115ef6106 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Mon, 7 Sep 2026 22:39:52 -0700 Subject: [PATCH 09/18] Format the new and reformatted test files with prettier --- src/permission/auto-shell-policy.test.ts | 2 +- src/session/optimized-context-store.test.ts | 11 +++++++---- src/subagent/run-authority.test.ts | 4 +--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/permission/auto-shell-policy.test.ts b/src/permission/auto-shell-policy.test.ts index 186215eb..e3f174bf 100644 --- a/src/permission/auto-shell-policy.test.ts +++ b/src/permission/auto-shell-policy.test.ts @@ -60,7 +60,7 @@ describe("git-global-config ask survives shell wrappers", () => { "git-global-config", ); expect( - autoShellRuleForCall(shellCall('sh -c \'git "config" --global user.name foo\''))?.name, + autoShellRuleForCall(shellCall("sh -c 'git \"config\" --global user.name foo'"))?.name, ).toBe("git-global-config"); }); }); diff --git a/src/session/optimized-context-store.test.ts b/src/session/optimized-context-store.test.ts index 53e94cd0..b888f536 100644 --- a/src/session/optimized-context-store.test.ts +++ b/src/session/optimized-context-store.test.ts @@ -145,11 +145,11 @@ describe("createOptimizedContextStore load", () => { const dir = tempDir(); const store = await createOptimizedContextStore(dir); - fs.writeFileSync(path.join(dir, TURNS_FILE), jsonl([turn("a")]) + '{"role":"user","content":[{"type":"te'); fs.writeFileSync( - path.join(dir, segmentFileName(TURNS_FILE, 1)), - jsonl([turn("b"), turn("c")]), + path.join(dir, TURNS_FILE), + jsonl([turn("a")]) + '{"role":"user","content":[{"type":"te', ); + fs.writeFileSync(path.join(dir, segmentFileName(TURNS_FILE, 1)), jsonl([turn("b"), turn("c")])); const loaded = await store.load(); expect(loaded.turns.map((t) => (t.content[0] as { text: string }).text)).toEqual([ @@ -163,7 +163,10 @@ describe("createOptimizedContextStore load", () => { const dir = tempDir(); const store = await createOptimizedContextStore(dir); - fs.writeFileSync(path.join(dir, TURNS_FILE), jsonl([turn("a")]) + '{"role":"user","content":[{"type":"te'); + fs.writeFileSync( + path.join(dir, TURNS_FILE), + jsonl([turn("a")]) + '{"role":"user","content":[{"type":"te', + ); const recovered = await store.load(); await store.writeTurns(recovered.turns); diff --git a/src/subagent/run-authority.test.ts b/src/subagent/run-authority.test.ts index 827db81c..750eeabf 100644 --- a/src/subagent/run-authority.test.ts +++ b/src/subagent/run-authority.test.ts @@ -54,9 +54,7 @@ function baseParams( // per-test timeouts below only absorb machine-load spikes during the // full-runtime construction these probes perform; assertions are // timing-independent. -async function runWithFailingInference( - run: (baseURL: string) => Promise, -): Promise { +async function runWithFailingInference(run: (baseURL: string) => Promise): Promise { const server = Bun.serve({ port: 0, fetch: () => From f78008fefd2bc405f05d7fd3e9c4a5cf7bfe6d2a Mon Sep 17 00:00:00 2001 From: Sawyer Date: Mon, 7 Sep 2026 22:31:07 -0700 Subject: [PATCH 10/18] Guard against content-pin tests regrowing Tests that pin an asset's literal wording or an exact palette value fail on copy and design edits while catching no behavior regression, and an audit found most low-value tests shared that shape. A local eslint rule now rejects the known shapes in test files; it is a heuristic shape match, calibrated to stay silent on behavior-string assertions like command parsing, approval display, and permission tokenization. --- AGENTS.md | 1 + eslint.config.js | 14 ++ scripts/eslint-rules/no-content-pin-tests.ts | 219 ++++++++++++++++++ .../eslint-rules/no-content-pin-tests.test.ts | 114 +++++++++ 4 files changed, 348 insertions(+) create mode 100644 scripts/eslint-rules/no-content-pin-tests.ts create mode 100644 tests/unit/eslint-rules/no-content-pin-tests.test.ts diff --git a/AGENTS.md b/AGENTS.md index a7f1b7e4..936c91b4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -35,6 +35,7 @@ When refactoring replaces an old path, delete the old one. No back-compat shims, - `tests/unit/` shared unit tests and helpers · co-located `src/**/*.test.ts` for module logic · `tests/fixtures/` fixture repos · `tests/integration/` reactor/permission harness. Planned: `tests/e2e/` (fixture-repo runs). - A test must not depend on another file having run, or on the default file order. It must pass under `bun test ./src ./tests ./evals --randomize`. If a test mutates module-level state or calls `mock.module`, it must restore that state itself (`afterEach`/`afterAll`), not rely on the process happening to reset it. When capturing a module's real exports to restore later, shallow-copy them (`{ ...moduleNamespace }`) at capture time, whether the namespace came from `await import(path)` or a static `import * as ns from "path"` — Bun mutates the live namespace object in place when the module is mocked, so holding a bare reference to it (either form) silently turns into the mocked exports. - Never call `mock.module` directly. Bun runs every test file in one process, so a `mock.module` call without its own teardown stays installed for the rest of the run and silently replaces the real module for other files — producing failures in files the change never touched, with no obvious link to the cause and no signal from `tsc` or a per-file run (CL-6967). Use `withMockedModule`/`withMockedModuleDuring` from `tests/helpers/mock-module.ts`, which capture the real module and register their own restore. An eslint rule (`no-restricted-syntax` in `eslint.config.js`) rejects bare `mock.module` calls in `*.test.ts` files. +- A test earns its place only if a real behavior change can fail it. Document copy, brand colors, marketing assets, and splash text are not behavior: assertions that pin an asset's literal wording, an exact palette hex/ANSI value, or rendered copy fail on copy/design edits and catch no regressions — assert the contract instead (parsing, formatting, ranges, aliases, invariants). Tests are code too: pinning a source file's own text is the same trap. An eslint rule (`corbits/no-content-pin-tests`, defined in `scripts/eslint-rules/no-content-pin-tests.ts`) rejects the known shapes in `*.test.ts` files; it is a heuristic shape match, not a semantic check, and its header documents what it does not catch. ## Build & Validation diff --git a/eslint.config.js b/eslint.config.js index 9f5512e9..5bd4fdac 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -1,5 +1,6 @@ import js from "@eslint/js"; import tseslint from "typescript-eslint"; +import noContentPinTests from "./scripts/eslint-rules/no-content-pin-tests.ts"; export default tseslint.config( { @@ -77,4 +78,17 @@ export default tseslint.config( ], }, }, + { + // Content-pin tests — assertions that pin literal document wording, brand + // hex values, or palette indexes — fail on copy/design edits and catch no + // behavior regression. The rule is a heuristic shape match; see its header + // for what it covers and what it deliberately does not. + files: ["**/*.test.ts"], + plugins: { + corbits: { rules: { "no-content-pin-tests": noContentPinTests } }, + }, + rules: { + "corbits/no-content-pin-tests": "error", + }, + }, ); diff --git a/scripts/eslint-rules/no-content-pin-tests.ts b/scripts/eslint-rules/no-content-pin-tests.ts new file mode 100644 index 00000000..862b8161 --- /dev/null +++ b/scripts/eslint-rules/no-content-pin-tests.ts @@ -0,0 +1,219 @@ +import type { Rule } from "eslint"; + +// Heuristic guardrail against content-pin tests regrowing (CL-7516): tests +// whose assertions pin literal document wording, brand hex values, or palette +// indexes fail on copy/design edits and catch no behavior regression. +// +// This is a shape match, not a semantic check. It flags assertions whose +// receiver was loaded from a document asset via Bun.file (path names a +// document extension — .md, .json, .txt, … — so runtime round-trips of files +// a test just wrote, and source-structure locks on .ts, stay clean), exact +// hex-color pins, and numeric pins on palette-named callees. It does not +// understand test intent: it will not catch pins on wording inlined in the +// test source, assets read via node:fs, extension-less asset paths, reads +// wrapped in a TS as/satisfies cast, or index pins on callees without a +// palette-ish name — and a shape match is not proof a given test is worthless. + +// Rule.Node carries the parent backlink; helpers take the parent-less estree +// node so expressions and visitor nodes are interchangeable. +type DistributiveOmit = T extends unknown ? Omit : never; +type ContentNode = DistributiveOmit; +type AnyNode = ContentNode | null | undefined; +type IdentifierNode = Extract; +type CallNode = Extract; + +const HEX_COLOR = /^#[0-9a-fA-F]{3,8}$/; +const PALETTE_CALLEE = /colou?r|palette|ansi/i; +const DOCUMENT_ASSET_PATH = /\.(?:md|markdown|json|txt|yaml|yml|toml|html|css|csv|xml|svg)$/i; + +const EXACT_VALUE_MATCHERS = new Set(["toBe", "toEqual", "toStrictEqual"]); +const WORDING_MATCHERS = new Set(["toContain", "toBe", "toEqual", "toStrictEqual", "toMatch"]); +const ASSERTION_CHAIN_PROPERTIES = new Set(["not", "resolves", "rejects"]); +const BUN_FILE_CONTENT_METHODS = new Set(["text", "json"]); + +const isIdentifierNamed = (node: AnyNode, name: string): boolean => + node !== null && node !== undefined && node.type === "Identifier" && node.name === name; + +// Strip await expressions down to the underlying call. +const unwrap = (node: AnyNode): AnyNode => { + let current = node; + for (;;) { + if (current === null || current === undefined) return current; + if (current.type === "AwaitExpression") { + current = current.argument; + } else { + return current; + } + } +}; + +// `const x = await Bun.file().text()/.json()` — returns the Bun.file +// call so the caller can inspect the path argument, else null. +const bunFileContentCall = (node: AnyNode): CallNode | null => { + if (node === null || node === undefined || node.type !== "CallExpression") return null; + const callee = node.callee; + if (callee.type !== "MemberExpression") return null; + if ( + callee.property.type !== "Identifier" || + !BUN_FILE_CONTENT_METHODS.has(callee.property.name) + ) { + return null; + } + const read = callee.object; + if (read.type !== "CallExpression") return null; + if (read.callee.type !== "MemberExpression") return null; + return isIdentifierNamed(read.callee.object, "Bun") && + isIdentifierNamed(read.callee.property, "file") + ? read + : null; +}; + +// String pieces of a path expression — the literals inside join(...), +// new URL(...), and template chains — so an asset read is recognized no +// matter how the path is assembled. +const pathStrings = (node: AnyNode, out: string[]): void => { + if (node === null || node === undefined) return; + switch (node.type) { + case "Literal": + if (typeof node.value === "string") out.push(node.value); + return; + case "TemplateLiteral": + for (const quasi of node.quasis) out.push(quasi.value.cooked ?? ""); + for (const expr of node.expressions) pathStrings(expr, out); + return; + case "BinaryExpression": + pathStrings(node.left, out); + pathStrings(node.right, out); + return; + case "ConditionalExpression": + pathStrings(node.consequent, out); + pathStrings(node.alternate, out); + return; + case "CallExpression": + case "NewExpression": + for (const arg of node.arguments) { + if (arg.type !== "SpreadElement") pathStrings(arg, out); + } + return; + default: + return; + } +}; + +const isDocumentAssetRead = (pathArg: AnyNode): boolean => { + const strings: string[] = []; + pathStrings(pathArg, strings); + return strings.some((value) => DOCUMENT_ASSET_PATH.test(value)); +}; + +// `expect(x).not/resolves/rejects.toContain(y)` — strip chain links back to +// the expect call itself. +const resolveExpectCall = (start: ContentNode): CallNode | null => { + let current: ContentNode = start; + while (current.type === "MemberExpression") { + if ( + current.property.type !== "Identifier" || + !ASSERTION_CHAIN_PROPERTIES.has(current.property.name) + ) { + return null; + } + current = current.object; + } + if (current.type === "CallExpression" && isIdentifierNamed(current.callee, "expect")) { + return current; + } + return null; +}; + +const isPaletteCall = (node: AnyNode): boolean => + node !== null && + node !== undefined && + node.type === "CallExpression" && + node.callee.type === "Identifier" && + PALETTE_CALLEE.test(node.callee.name); + +// Base identifier of `x`, `x.y`, or `x[y].z` — null for calls and literals. +const baseIdentifier = (node: AnyNode): IdentifierNode | null => { + let current = node; + for (;;) { + if (current === null || current === undefined || current.type !== "MemberExpression") { + return current !== null && current !== undefined && current.type === "Identifier" + ? current + : null; + } + current = current.object; + } +}; + +const isPinnedWording = (arg: AnyNode): boolean => { + if (arg === null || arg === undefined) return false; + if (arg.type === "Literal") { + return typeof arg.value === "string" || "regex" in arg; + } + return arg.type === "TemplateLiteral" && arg.expressions.length === 0; +}; + +export default { + meta: { + type: "problem", + docs: { + description: + "Disallow test assertions that pin literal document wording, brand hex values, or palette indexes", + }, + messages: { + wordingPin: + "Content-pin test: this asserts literal wording of a document asset loaded from disk. It fails on copy edits and catches no behavior regression — assert on code behavior instead, or validate the document's structure rather than pinning its text.", + hexPin: + "Content-pin test: this pins an exact brand hex value. Palette literals change with design edits and the pin catches no behavior regression — assert the contract instead (the role resolves, aliases hold, contrast stays in bounds).", + ansiIndexPin: + "Content-pin test: this pins an exact ANSI-256 palette index. Indexes shift with palette edits and the pin catches no behavior regression — assert the contract instead (the index is in range, distinct roles stay distinct).", + }, + schema: [], + }, + create(context: Rule.RuleContext): Rule.RuleListener { + const contentBindings = new Set(); + + return { + VariableDeclarator(node): void { + if (node.id.type !== "Identifier" || node.init === null) return; + const read = bunFileContentCall(unwrap(node.init)); + if (read === null || !isDocumentAssetRead(read.arguments[0])) return; + contentBindings.add(node.id.name); + }, + + CallExpression(node): void { + if (node.callee.type !== "MemberExpression") return; + const matcher = node.callee.property; + if (matcher.type !== "Identifier") return; + const expectCall = resolveExpectCall(node.callee.object); + if (expectCall === null) return; + const subject = expectCall.arguments[0]; + if (subject === undefined || subject.type === "SpreadElement") return; + + if (EXACT_VALUE_MATCHERS.has(matcher.name)) { + const arg = node.arguments[0]; + if (arg !== undefined && arg.type === "Literal") { + if (typeof arg.value === "string" && HEX_COLOR.test(arg.value)) { + context.report({ node, messageId: "hexPin" }); + return; + } + if (typeof arg.value === "number" && isPaletteCall(unwrap(subject))) { + context.report({ node, messageId: "ansiIndexPin" }); + return; + } + } + } + + const receiver = baseIdentifier(unwrap(subject)); + if ( + WORDING_MATCHERS.has(matcher.name) && + receiver !== null && + contentBindings.has(receiver.name) && + isPinnedWording(node.arguments[0]) + ) { + context.report({ node, messageId: "wordingPin" }); + } + }, + }; + }, +} satisfies Rule.RuleModule; diff --git a/tests/unit/eslint-rules/no-content-pin-tests.test.ts b/tests/unit/eslint-rules/no-content-pin-tests.test.ts new file mode 100644 index 00000000..c5abdf17 --- /dev/null +++ b/tests/unit/eslint-rules/no-content-pin-tests.test.ts @@ -0,0 +1,114 @@ +import { Linter } from "eslint"; +import { describe, expect, test } from "bun:test"; +import noContentPinTests from "../../../scripts/eslint-rules/no-content-pin-tests"; + +const lint = (code: string) => + new Linter({ configType: "flat" }).verify(code, { + plugins: { corbits: { rules: { "no-content-pin-tests": noContentPinTests } } }, + rules: { "corbits/no-content-pin-tests": "error" }, + }); + +const flaggedIds = (code: string) => lint(code).map((message) => message.messageId); + +describe("no-content-pin-tests", () => { + test("flags literal wording pins on Bun.file-loaded assets", () => { + expect( + flaggedIds( + [ + 'const skill = await Bun.file(join(root, "skills/style/SKILL.md")).text();', + 'expect(skill).toContain("Prefer deletion over addition");', + 'expect(skill).not.toContain("spawn_agent");', + 'expect(skill).toBe("exact document text");', + "expect(skill).toMatch(/You are \\w+Director/);", + ].join("\n"), + ), + ).toEqual(["wordingPin", "wordingPin", "wordingPin", "wordingPin"]); + }); + + test("flags wording pins on members of assets parsed as JSON", () => { + expect( + flaggedIds( + [ + 'const manifest = await Bun.file("plugins/corbits-skills/manifest.json").json();', + 'expect(manifest.id).toBe("corbits-skills");', + 'expect(manifest.kind).toContain("command");', + ].join("\n"), + ), + ).toEqual(["wordingPin", "wordingPin"]); + }); + + test("flags exact brand hex pins regardless of receiver", () => { + expect(flaggedIds('expect(color("brand")).toBe("#f5933a");')).toEqual(["hexPin"]); + expect(flaggedIds('expect(fg).toEqual("#7ea2c4");')).toEqual(["hexPin"]); + }); + + test("flags numeric pins on palette-named callees", () => { + expect(flaggedIds('expect(color256("brand")).toBe(173);')).toEqual(["ansiIndexPin"]); + expect(flaggedIds("expect(paletteIndex(role)).toEqual(74);")).toEqual(["ansiIndexPin"]); + }); + + test("keeps behavior-string assertions clean", () => { + expect( + flaggedIds( + [ + 'expect(groupChainSegmentsForDisplay("ls | head -5 && echo done")).toEqual([', + ' "ls",', + ' "head -5",', + ' "echo done",', + "]);", + 'expect(isShellNoOp("true")).toBe(true);', + "expect(secondsFromMs(0)).toBe(0);", + "expect(idx).toBeGreaterThanOrEqual(0);", + "expect(cut.length).toBeLessThanOrEqual(20);", + 'expect(messages).toContain("outside the workspace");', + ].join("\n"), + ), + ).toEqual([]); + }); + + test("keeps runtime round-trips of non-document files clean", () => { + expect( + flaggedIds( + [ + 'const written = await Bun.file(join(cwd, "app.py")).text();', + "expect(written).toBe(\"def greet():\\n print('hello')\\n\");", + ].join("\n"), + ), + ).toEqual([]); + }); + + test("keeps source-structure locks on .ts files clean", () => { + expect( + flaggedIds( + [ + 'const src = await Bun.file(new URL("./runner.ts", import.meta.url)).text();', + 'expect(src).toContain("standingPluginWarnings");', + ].join("\n"), + ), + ).toEqual([]); + }); + + test("keeps non-literal matcher arguments clean", () => { + expect( + flaggedIds( + [ + "const before = await Bun.file(target).text();", + "expect(await Bun.file(target).text()).toBe(before);", + "expect(palette.diffAdded).toEqual(palette.success);", + "expect(message).toContain(secret);", + ].join("\n"), + ), + ).toEqual([]); + }); + + test("keeps range and contract checks on palette receivers clean", () => { + expect( + flaggedIds( + [ + "expect(color256(role)).toBeLessThanOrEqual(255);", + "expect(color(role)).toMatch(/^#[0-9a-fA-F]{6}$/);", + ].join("\n"), + ), + ).toEqual([]); + }); +}); From 55394bdb0617d6b8b6a2c34385584b0f7d666322 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Mon, 7 Sep 2026 23:02:00 -0700 Subject: [PATCH 11/18] Remove aesthetic palette invariants from ramp paint tests --- src/tui/ramp-paint.test.ts | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/src/tui/ramp-paint.test.ts b/src/tui/ramp-paint.test.ts index bf475f04..350b11a5 100644 --- a/src/tui/ramp-paint.test.ts +++ b/src/tui/ramp-paint.test.ts @@ -12,7 +12,6 @@ import { withTestRenderer } from "./harness"; import { RAMP_CYCLE_MS } from "./ramp"; import { attachSessionBridge, createRecordingPort } from "./runtime-bridge"; import { createAppShell } from "./shell/index"; -import { UI } from "./theme"; const BRAILLE = /[⠀-⣿]/; const DENSITY = /[░▒▓█]/; @@ -260,31 +259,3 @@ describe("turn ramp paint", () => { ); }); }); - -describe("palette", () => { - test("no gray sits on the ground — every tone keeps a warm bias", () => { - for (const [role, hex] of Object.entries(UI)) { - if (role === "name") continue; - const r = Number.parseInt(hex.slice(1, 3), 16); - const b = Number.parseInt(hex.slice(5, 7), 16); - expect(r).toBeGreaterThan(b); - } - }); - - test("chrome stays below the action orange so orange still reads as an event", () => { - const action = saturation(UI.action); - for (const hex of [UI.inFlight, UI.inFlightBright, UI.heading, UI.warning]) { - expect(saturation(hex)).toBeLessThan(action); - } - }); -}); - -/** HSL saturation, 0..1 — the axis the chrome ramp is held below action orange on. */ -function saturation(hex: string): number { - const channels = [1, 3, 5].map((i) => Number.parseInt(hex.slice(i, i + 2), 16) / 255); - const max = Math.max(...channels); - const min = Math.min(...channels); - if (max === min) return 0; - const lightness = (max + min) / 2; - return (max - min) / (lightness > 0.5 ? 2 - max - min : max + min); -} From 22772859a98d8b06f31c9da62d6b9a28e19bbd06 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Mon, 7 Sep 2026 23:02:05 -0700 Subject: [PATCH 12/18] Pin the CI shard test:paths script in the check gate --- tests/unit/check-gate.test.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/unit/check-gate.test.ts b/tests/unit/check-gate.test.ts index 13edfa4a..e3e4793b 100644 --- a/tests/unit/check-gate.test.ts +++ b/tests/unit/check-gate.test.ts @@ -3,8 +3,9 @@ import { join } from "node:path"; import { describe, expect, test } from "bun:test"; // Guard against the gate drifting apart again (CL-7300): `bun run check` and -// CI's test job must resolve to the same seeded suite, and the projects-dir -// guard must delegate to the `test` script rather than duplicate its command. +// CI's test jobs must resolve to the same seeded suite, and the projects-dir +// guard must delegate to the `test` script (or `test:paths` for shard filters) +// rather than duplicate its command. const repoRoot = join(import.meta.dir, "..", ".."); const pkg = JSON.parse(readFileSync(join(repoRoot, "package.json"), "utf8")) as { @@ -21,6 +22,13 @@ describe("check gate", () => { expect(pkg.scripts.test).toBe(TEST_SUITE); }); + test("`test:paths` is the seeded suite accepting CI shard path filters", () => { + // Same seed as `test`; the guard passes shard filters as arguments, which + // cannot be appended to `bun run test` because bun's filters are additive. + expect(pkg.scripts["test:paths"]).toBe("bun test --randomize --seed 424242"); + expect(guardSource).toContain('"run", "test:paths"'); + }); + test("`check` runs the suite through the projects-dir guard", () => { expect(pkg.scripts[GUARD_SCRIPT]).toContain("scripts/guard-real-projects-dir.ts"); expect(pkg.scripts.check).toContain(`bun run ${GUARD_SCRIPT}`); From 0324c71b6ea7bc9f441d0608667e0e24b01a3c31 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Mon, 7 Sep 2026 23:05:22 -0700 Subject: [PATCH 13/18] Rewrite forbidden-content policy guards as violation collectors --- tests/unit/corbits-skills-catalog.test.ts | 28 ++++++++++++++++------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/tests/unit/corbits-skills-catalog.test.ts b/tests/unit/corbits-skills-catalog.test.ts index 2ebbb747..3ed7c537 100644 --- a/tests/unit/corbits-skills-catalog.test.ts +++ b/tests/unit/corbits-skills-catalog.test.ts @@ -112,6 +112,8 @@ test("corbits-skills catalog lists 20 skills with name and description", async ( test("first-party skills are how-to playbooks, not director personas", async () => { + // Forbidden-content policy: pins what must NEVER appear in a skill doc. + // Violations are collected and asserted once so a failure names the skill. const gaasOverlap = new Set([ "ast-grep", "create-issue", @@ -128,15 +130,21 @@ test("first-party skills are how-to playbooks, not director personas", async () "style", "typescript", ]); + const violations: string[] = []; for (const name of SKILL_DIRS) { const skill = await Bun.file(join(pluginRoot, "skills", name, "SKILL.md")).text(); - expect(skill).not.toContain("You are Skywalker"); - expect(skill).not.toMatch(/You are \w+Director/); - expect(skill).not.toContain("Host is Corbits"); + if (skill.includes("You are Skywalker") || /You are \w+Director/.test(skill)) { + violations.push(`${name}: director persona language`); + } + if (skill.includes("Host is Corbits")) { + violations.push(`${name}: host attribution`); + } if (gaasOverlap.has(name)) continue; - expect(skill).not.toContain("## Acknowledgment"); - expect(skill).not.toMatch(/I have reviewed the .+ skill/); + if (skill.includes("## Acknowledgment") || /I have reviewed the .+ skill/.test(skill)) { + violations.push(`${name}: GaaS acknowledgment ritual`); + } } + expect(violations).toEqual([]); }); test("use_skill-only skills set user-invocable: false without disable-model-invocation", async () => { @@ -168,10 +176,14 @@ test("only background and bake-only skills carry disable-model-invocation", asyn }); test("review skill does not own GitHub posting or Linear In Review", async () => { + // Ownership-boundary policy: pins FORBIDDEN claims, not required copy. const skill = await Bun.file(join(pluginRoot, "skills/review/SKILL.md")).text(); - expect(skill).not.toContain("Post the Review on GitHub"); - expect(skill).not.toContain("`linear-issue-workflow` owns the In Review write"); - expect(skill).not.toContain("this skill does not set Linear state"); + const forbiddenClaims = [ + "Post the Review on GitHub", + "`linear-issue-workflow` owns the In Review write", + "this skill does not set Linear state", + ]; + expect(forbiddenClaims.filter((claim) => skill.includes(claim))).toEqual([]); }); test("slash skills do not set user-invocable: false", async () => { From 93523236558a93d414e049aee960a2056ae27445 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Mon, 7 Sep 2026 23:12:23 -0700 Subject: [PATCH 14/18] Pin the CI shard path union in the check gate --- tests/unit/check-gate.test.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/unit/check-gate.test.ts b/tests/unit/check-gate.test.ts index e3e4793b..28a2bf9c 100644 --- a/tests/unit/check-gate.test.ts +++ b/tests/unit/check-gate.test.ts @@ -43,4 +43,16 @@ describe("check gate", () => { expect(ci).not.toMatch(/^\s*run: bun test(\s|$)/m); expect(ci).not.toMatch(/^\s*run: bun run test(\s|$)/m); }); + + test("CI test shards cover exactly the suite's paths", () => { + // Sharding must never silently drop part of the suite: the union of the + // matrix shards has to equal the unsharded `test` script's paths. + const shardPaths = [...ci.matchAll(/^\s+paths: (.+)$/gm)] + .flatMap((match) => match[1]?.trim().split(/\s+/) ?? []) + .sort(); + const suitePaths = TEST_SUITE.split(" ") + .filter((part) => part.startsWith("./")) + .sort(); + expect(shardPaths).toEqual(suitePaths); + }); }); From 59543f779e9f0843203028004c075c6a8f617bd9 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Wed, 9 Sep 2026 10:57:28 -0700 Subject: [PATCH 15/18] Restore eval CLI coverage and drop the content-pin lint rule Unique CLI and web-brand contracts were deleted because they sat outside the suite path list. Membership owns that, not deletion. AGENTS.md owns the behavioral bar; an error-level shape-match lint rule does not. test:paths now requires paths so a zero-arg run cannot scan vendor/. --- .github/workflows/ci.yml | 21 +- AGENTS.md | 17 +- CONTRIBUTING.md | 4 +- eslint.config.js | 14 - package.json | 4 +- scripts/eslint-rules/no-content-pin-tests.ts | 219 ------------ scripts/eval-capability.test.ts | 337 ++++++++++++++++++ scripts/eval-public-swe-one.test.ts | 49 +++ scripts/guard-real-projects-dir.ts | 12 +- scripts/test-paths.ts | 23 ++ src/agent/tools-mcp-disconnect.test.ts | 6 +- src/tui/session-operation-queue.test.ts | 28 -- tests/unit/check-gate.test.ts | 18 +- tests/unit/corbits-skills-catalog.test.ts | 28 +- .../eslint-rules/no-content-pin-tests.test.ts | 114 ------ .../unit/tui/tool-formatter-web-brand.test.ts | 20 ++ 16 files changed, 480 insertions(+), 434 deletions(-) delete mode 100644 scripts/eslint-rules/no-content-pin-tests.ts create mode 100644 scripts/eval-capability.test.ts create mode 100644 scripts/eval-public-swe-one.test.ts create mode 100644 scripts/test-paths.ts delete mode 100644 tests/unit/eslint-rules/no-content-pin-tests.test.ts create mode 100644 tests/unit/tui/tool-formatter-web-brand.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 01726f4c..98234cd1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,15 +31,10 @@ jobs: uses: actions/cache@v4 with: path: node_modules - # The exact key keeps hits honest: only a cache built from this - # bun.lock restores. restore-keys falls back to the newest cache - # when the lockfile changed, so a dependency bump reinstalls the - # delta instead of cold-installing on every job at once. bun - # install --frozen-lockfile reconciles a stale tree to the new - # lockfile, so a partial hit never leaves wrong deps behind. + # Exact-key-only: a restore-keys prefix of bun- would hydrate + # node_modules from a different lockfile. bun install then has to + # reconcile a stale tree; missing that step leaves wrong deps. key: bun-${{ hashFiles('bun.lock') }} - restore-keys: | - bun- - name: Install dependencies run: bun install --frozen-lockfile @@ -77,8 +72,6 @@ jobs: with: path: node_modules key: bun-${{ hashFiles('bun.lock') }} - restore-keys: | - bun- - name: Install dependencies run: bun install --frozen-lockfile @@ -89,7 +82,7 @@ jobs: # The suite is sharded so the slowest slice, not the whole suite, sets the # wall clock. Every shard still goes through check:projects-dir-guard: the # guard forwards these path filters to the suite it wraps, and the union of - # the shards' filters is exactly ./src ./tests ./evals, so the gate covers + # the shards' filters is exactly ./src ./tests ./evals ./scripts, so the gate covers # the same tests as before, all of them sandboxed. test: runs-on: ubuntu-latest @@ -100,8 +93,8 @@ jobs: shard: - name: src paths: ./src - - name: tests-and-evals - paths: ./tests ./evals + - name: tests-evals-and-scripts + paths: ./tests ./evals ./scripts name: test (${{ matrix.shard.name }}) steps: - name: Checkout @@ -127,8 +120,6 @@ jobs: with: path: node_modules key: bun-${{ hashFiles('bun.lock') }} - restore-keys: | - bun- - name: Install dependencies run: bun install --frozen-lockfile diff --git a/AGENTS.md b/AGENTS.md index 936c91b4..7a0de1f4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -33,9 +33,9 @@ When refactoring replaces an old path, delete the old one. No back-compat shims, - Add or update tests with every behavior change. - Bug fixes start with a failing test that reproduces the bug. Do not start by patching. - `tests/unit/` shared unit tests and helpers · co-located `src/**/*.test.ts` for module logic · `tests/fixtures/` fixture repos · `tests/integration/` reactor/permission harness. Planned: `tests/e2e/` (fixture-repo runs). -- A test must not depend on another file having run, or on the default file order. It must pass under `bun test ./src ./tests ./evals --randomize`. If a test mutates module-level state or calls `mock.module`, it must restore that state itself (`afterEach`/`afterAll`), not rely on the process happening to reset it. When capturing a module's real exports to restore later, shallow-copy them (`{ ...moduleNamespace }`) at capture time, whether the namespace came from `await import(path)` or a static `import * as ns from "path"` — Bun mutates the live namespace object in place when the module is mocked, so holding a bare reference to it (either form) silently turns into the mocked exports. +- A test must not depend on another file having run, or on the default file order. It must pass under `bun test ./src ./tests ./evals ./scripts --randomize`. If a test mutates module-level state or calls `mock.module`, it must restore that state itself (`afterEach`/`afterAll`), not rely on the process happening to reset it. When capturing a module's real exports to restore later, shallow-copy them (`{ ...moduleNamespace }`) at capture time, whether the namespace came from `await import(path)` or a static `import * as ns from "path"` — Bun mutates the live namespace object in place when the module is mocked, so holding a bare reference to it (either form) silently turns into the mocked exports. - Never call `mock.module` directly. Bun runs every test file in one process, so a `mock.module` call without its own teardown stays installed for the rest of the run and silently replaces the real module for other files — producing failures in files the change never touched, with no obvious link to the cause and no signal from `tsc` or a per-file run (CL-6967). Use `withMockedModule`/`withMockedModuleDuring` from `tests/helpers/mock-module.ts`, which capture the real module and register their own restore. An eslint rule (`no-restricted-syntax` in `eslint.config.js`) rejects bare `mock.module` calls in `*.test.ts` files. -- A test earns its place only if a real behavior change can fail it. Document copy, brand colors, marketing assets, and splash text are not behavior: assertions that pin an asset's literal wording, an exact palette hex/ANSI value, or rendered copy fail on copy/design edits and catch no regressions — assert the contract instead (parsing, formatting, ranges, aliases, invariants). Tests are code too: pinning a source file's own text is the same trap. An eslint rule (`corbits/no-content-pin-tests`, defined in `scripts/eslint-rules/no-content-pin-tests.ts`) rejects the known shapes in `*.test.ts` files; it is a heuristic shape match, not a semantic check, and its header documents what it does not catch. +- A test earns its place only if a real behavior change can fail it. Document copy, brand colors, marketing assets, and splash text are not behavior: assertions that pin an asset's literal wording, an exact palette hex/ANSI value, or rendered copy fail on copy/design edits and catch no regressions — assert the contract instead (parsing, formatting, ranges, aliases, invariants). Tests are code too: pinning a source file's own text is the same trap. This bar is a review and authorship rule, not an eslint shape match. ## Build & Validation @@ -49,10 +49,15 @@ the projects-dir sandbox guard — in that order, matching CI. Run the full suite before declaring any task complete. Do not substitute individual targets. If a failure is pre-existing and unrelated to your change, say so explicitly. -`bun run test` runs `bun test ./src ./tests ./evals --randomize --seed 424242` — -the same suite CI runs. A bare `bun test` also -scans `vendor/`, adding hundreds of unrelated results and making pass/fail -counts meaningless to compare across branches — always use `bun run test`. +`bun run test` runs `bun test ./src ./tests ./evals ./scripts --randomize --seed 424242` +as a single process. CI shards the same path union via `test:paths` +(`.github/workflows/ci.yml`) for wall clock. Path-union is not the same +isolation domain: a `mock.module` leak across `./src` vs `./tests` fails +locally in the one-process suite but not in a CI shard (CL-6967). A bare +`bun test` also scans `vendor/`, adding hundreds of unrelated results and +making pass/fail counts meaningless to compare across branches — always use +`bun run test`. `test:paths` with no path filters refuses to run for the +same reason. ## Commits, pull requests, and issue tracking diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4ec7e492..5498a205 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,7 +30,9 @@ bun run build bun run test ``` -These match the CI workflow in `.github/workflows/ci.yml`. Run `bun run check` +These match the local development loop. CI shards the same path union via +`test:paths` rather than running the one-process `bun run test` suite. +Run `bun run check` (lint, typecheck, build, and the guarded test suite) before opening a PR — `bun run test` alone skips the projects-dir sandbox guard, which only runs under `bun run check` and CI. Do diff --git a/eslint.config.js b/eslint.config.js index 5bd4fdac..9f5512e9 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -1,6 +1,5 @@ import js from "@eslint/js"; import tseslint from "typescript-eslint"; -import noContentPinTests from "./scripts/eslint-rules/no-content-pin-tests.ts"; export default tseslint.config( { @@ -78,17 +77,4 @@ export default tseslint.config( ], }, }, - { - // Content-pin tests — assertions that pin literal document wording, brand - // hex values, or palette indexes — fail on copy/design edits and catch no - // behavior regression. The rule is a heuristic shape match; see its header - // for what it covers and what it deliberately does not. - files: ["**/*.test.ts"], - plugins: { - corbits: { rules: { "no-content-pin-tests": noContentPinTests } }, - }, - rules: { - "corbits/no-content-pin-tests": "error", - }, - }, ); diff --git a/package.json b/package.json index 74c6b686..6f16bc1e 100644 --- a/package.json +++ b/package.json @@ -32,8 +32,8 @@ "build": "bun build ./src/index.ts --outdir ./dist --target bun --external '@opentui/core-*' && bun scripts/copy-repo-plugins.ts", "build:bin": "bun build ./src/index.ts --compile --minify --define process.env.NODE_ENV='\"production\"' --outfile ./dist/corbits && bun scripts/copy-repo-plugins.ts", "typecheck": "tsc --noEmit", - "test": "bun test ./src ./tests ./evals --randomize --seed 424242", - "test:paths": "bun test --randomize --seed 424242", + "test": "bun test ./src ./tests ./evals ./scripts --randomize --seed 424242", + "test:paths": "bun scripts/test-paths.ts", "lint": "prettier --check --cache . && eslint --cache .", "check:projects-dir-guard": "bun scripts/guard-real-projects-dir.ts", "check": "bun run lint && bun run typecheck && bun run build && bun run check:projects-dir-guard", diff --git a/scripts/eslint-rules/no-content-pin-tests.ts b/scripts/eslint-rules/no-content-pin-tests.ts deleted file mode 100644 index 862b8161..00000000 --- a/scripts/eslint-rules/no-content-pin-tests.ts +++ /dev/null @@ -1,219 +0,0 @@ -import type { Rule } from "eslint"; - -// Heuristic guardrail against content-pin tests regrowing (CL-7516): tests -// whose assertions pin literal document wording, brand hex values, or palette -// indexes fail on copy/design edits and catch no behavior regression. -// -// This is a shape match, not a semantic check. It flags assertions whose -// receiver was loaded from a document asset via Bun.file (path names a -// document extension — .md, .json, .txt, … — so runtime round-trips of files -// a test just wrote, and source-structure locks on .ts, stay clean), exact -// hex-color pins, and numeric pins on palette-named callees. It does not -// understand test intent: it will not catch pins on wording inlined in the -// test source, assets read via node:fs, extension-less asset paths, reads -// wrapped in a TS as/satisfies cast, or index pins on callees without a -// palette-ish name — and a shape match is not proof a given test is worthless. - -// Rule.Node carries the parent backlink; helpers take the parent-less estree -// node so expressions and visitor nodes are interchangeable. -type DistributiveOmit = T extends unknown ? Omit : never; -type ContentNode = DistributiveOmit; -type AnyNode = ContentNode | null | undefined; -type IdentifierNode = Extract; -type CallNode = Extract; - -const HEX_COLOR = /^#[0-9a-fA-F]{3,8}$/; -const PALETTE_CALLEE = /colou?r|palette|ansi/i; -const DOCUMENT_ASSET_PATH = /\.(?:md|markdown|json|txt|yaml|yml|toml|html|css|csv|xml|svg)$/i; - -const EXACT_VALUE_MATCHERS = new Set(["toBe", "toEqual", "toStrictEqual"]); -const WORDING_MATCHERS = new Set(["toContain", "toBe", "toEqual", "toStrictEqual", "toMatch"]); -const ASSERTION_CHAIN_PROPERTIES = new Set(["not", "resolves", "rejects"]); -const BUN_FILE_CONTENT_METHODS = new Set(["text", "json"]); - -const isIdentifierNamed = (node: AnyNode, name: string): boolean => - node !== null && node !== undefined && node.type === "Identifier" && node.name === name; - -// Strip await expressions down to the underlying call. -const unwrap = (node: AnyNode): AnyNode => { - let current = node; - for (;;) { - if (current === null || current === undefined) return current; - if (current.type === "AwaitExpression") { - current = current.argument; - } else { - return current; - } - } -}; - -// `const x = await Bun.file().text()/.json()` — returns the Bun.file -// call so the caller can inspect the path argument, else null. -const bunFileContentCall = (node: AnyNode): CallNode | null => { - if (node === null || node === undefined || node.type !== "CallExpression") return null; - const callee = node.callee; - if (callee.type !== "MemberExpression") return null; - if ( - callee.property.type !== "Identifier" || - !BUN_FILE_CONTENT_METHODS.has(callee.property.name) - ) { - return null; - } - const read = callee.object; - if (read.type !== "CallExpression") return null; - if (read.callee.type !== "MemberExpression") return null; - return isIdentifierNamed(read.callee.object, "Bun") && - isIdentifierNamed(read.callee.property, "file") - ? read - : null; -}; - -// String pieces of a path expression — the literals inside join(...), -// new URL(...), and template chains — so an asset read is recognized no -// matter how the path is assembled. -const pathStrings = (node: AnyNode, out: string[]): void => { - if (node === null || node === undefined) return; - switch (node.type) { - case "Literal": - if (typeof node.value === "string") out.push(node.value); - return; - case "TemplateLiteral": - for (const quasi of node.quasis) out.push(quasi.value.cooked ?? ""); - for (const expr of node.expressions) pathStrings(expr, out); - return; - case "BinaryExpression": - pathStrings(node.left, out); - pathStrings(node.right, out); - return; - case "ConditionalExpression": - pathStrings(node.consequent, out); - pathStrings(node.alternate, out); - return; - case "CallExpression": - case "NewExpression": - for (const arg of node.arguments) { - if (arg.type !== "SpreadElement") pathStrings(arg, out); - } - return; - default: - return; - } -}; - -const isDocumentAssetRead = (pathArg: AnyNode): boolean => { - const strings: string[] = []; - pathStrings(pathArg, strings); - return strings.some((value) => DOCUMENT_ASSET_PATH.test(value)); -}; - -// `expect(x).not/resolves/rejects.toContain(y)` — strip chain links back to -// the expect call itself. -const resolveExpectCall = (start: ContentNode): CallNode | null => { - let current: ContentNode = start; - while (current.type === "MemberExpression") { - if ( - current.property.type !== "Identifier" || - !ASSERTION_CHAIN_PROPERTIES.has(current.property.name) - ) { - return null; - } - current = current.object; - } - if (current.type === "CallExpression" && isIdentifierNamed(current.callee, "expect")) { - return current; - } - return null; -}; - -const isPaletteCall = (node: AnyNode): boolean => - node !== null && - node !== undefined && - node.type === "CallExpression" && - node.callee.type === "Identifier" && - PALETTE_CALLEE.test(node.callee.name); - -// Base identifier of `x`, `x.y`, or `x[y].z` — null for calls and literals. -const baseIdentifier = (node: AnyNode): IdentifierNode | null => { - let current = node; - for (;;) { - if (current === null || current === undefined || current.type !== "MemberExpression") { - return current !== null && current !== undefined && current.type === "Identifier" - ? current - : null; - } - current = current.object; - } -}; - -const isPinnedWording = (arg: AnyNode): boolean => { - if (arg === null || arg === undefined) return false; - if (arg.type === "Literal") { - return typeof arg.value === "string" || "regex" in arg; - } - return arg.type === "TemplateLiteral" && arg.expressions.length === 0; -}; - -export default { - meta: { - type: "problem", - docs: { - description: - "Disallow test assertions that pin literal document wording, brand hex values, or palette indexes", - }, - messages: { - wordingPin: - "Content-pin test: this asserts literal wording of a document asset loaded from disk. It fails on copy edits and catches no behavior regression — assert on code behavior instead, or validate the document's structure rather than pinning its text.", - hexPin: - "Content-pin test: this pins an exact brand hex value. Palette literals change with design edits and the pin catches no behavior regression — assert the contract instead (the role resolves, aliases hold, contrast stays in bounds).", - ansiIndexPin: - "Content-pin test: this pins an exact ANSI-256 palette index. Indexes shift with palette edits and the pin catches no behavior regression — assert the contract instead (the index is in range, distinct roles stay distinct).", - }, - schema: [], - }, - create(context: Rule.RuleContext): Rule.RuleListener { - const contentBindings = new Set(); - - return { - VariableDeclarator(node): void { - if (node.id.type !== "Identifier" || node.init === null) return; - const read = bunFileContentCall(unwrap(node.init)); - if (read === null || !isDocumentAssetRead(read.arguments[0])) return; - contentBindings.add(node.id.name); - }, - - CallExpression(node): void { - if (node.callee.type !== "MemberExpression") return; - const matcher = node.callee.property; - if (matcher.type !== "Identifier") return; - const expectCall = resolveExpectCall(node.callee.object); - if (expectCall === null) return; - const subject = expectCall.arguments[0]; - if (subject === undefined || subject.type === "SpreadElement") return; - - if (EXACT_VALUE_MATCHERS.has(matcher.name)) { - const arg = node.arguments[0]; - if (arg !== undefined && arg.type === "Literal") { - if (typeof arg.value === "string" && HEX_COLOR.test(arg.value)) { - context.report({ node, messageId: "hexPin" }); - return; - } - if (typeof arg.value === "number" && isPaletteCall(unwrap(subject))) { - context.report({ node, messageId: "ansiIndexPin" }); - return; - } - } - } - - const receiver = baseIdentifier(unwrap(subject)); - if ( - WORDING_MATCHERS.has(matcher.name) && - receiver !== null && - contentBindings.has(receiver.name) && - isPinnedWording(node.arguments[0]) - ) { - context.report({ node, messageId: "wordingPin" }); - } - }, - }; - }, -} satisfies Rule.RuleModule; diff --git a/scripts/eval-capability.test.ts b/scripts/eval-capability.test.ts new file mode 100644 index 00000000..2713919e --- /dev/null +++ b/scripts/eval-capability.test.ts @@ -0,0 +1,337 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdir, mkdtemp, writeFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; + +import { + initEvalGitRepo, + mapPool, + parseArgs, + buildEvalDiagnostics, + validateVariantEfforts, +} from "./eval-capability.js"; +import { parseMatrix } from "../evals/capability/lib.js"; +import type { Config } from "../src/config/index.js"; + +const execFileAsync = promisify(execFile); + +function sampleConfig(over: Partial = {}): Config { + return { + configured: true, + apiKey: "key", + baseURL: "https://example.test", + model: "gpt-5", + providerName: "openai", + cwd: process.cwd(), + task: "do it", + force: true, + dangerouslySkipPermissions: true, + skipPermissionsFromSettings: false, + auto: false, + command: "exec", + globalSettingsPath: "/dev/null", + providers: [], + sessionId: "sess-1", + ...over, + } as Config; +} + +describe("parseArgs", () => { + const savedConcurrency = process.env.CORBITS_EVAL_CONCURRENCY; + + const restoreConcurrency = (): void => { + if (savedConcurrency === undefined) { + delete process.env.CORBITS_EVAL_CONCURRENCY; + } else { + process.env.CORBITS_EVAL_CONCURRENCY = savedConcurrency; + } + }; + + afterEach(() => { + restoreConcurrency(); + }); + + beforeEach(() => { + delete process.env.CORBITS_EVAL_CONCURRENCY; + }); + + test("--help does not require provider or model", () => { + const opts = parseArgs(["--help"]); + expect(opts.help).toBe(true); + expect(opts.provider).not.toBe("xai/thegreataxios"); + expect(opts.model).not.toBe("xai/thegreataxios"); + }); + + test("no flags throws", () => { + expect(() => parseArgs([])).toThrow(/--provider/); + expect(() => parseArgs([])).toThrow(/--model/); + }); + + test("--provider without --model throws", () => { + expect(() => parseArgs(["--provider", "foo"])).toThrow(/--model/); + }); + + test("--model without --provider throws", () => { + expect(() => parseArgs(["--model", "bar"])).toThrow(/--provider/); + }); + + test("--provider foo --model bar parses those values", () => { + const opts = parseArgs(["--provider", "foo", "--model", "bar"]); + expect(opts.provider).toBe("foo"); + expect(opts.model).toBe("bar"); + }); + + test("--dry-run without pair throws", () => { + expect(() => parseArgs(["--dry-run"])).toThrow(/--provider/); + expect(() => parseArgs(["--dry-run"])).toThrow(/--model/); + }); + + test("--matrix xai:grok-4.5 is enough without top-level flags", () => { + const opts = parseArgs(["--matrix", "xai:grok-4.5"]); + expect(opts.matrix).toBe("xai:grok-4.5"); + }); + + test("incomplete matrix cell throws", () => { + expect(() => parseArgs(["--matrix", "xai:"])).toThrow(/both provider and model/); + expect(() => parseArgs(["--matrix", ":grok-4.5"])).toThrow(/both provider and model/); + }); + + test("--effort accepts a canonical literal", () => { + const opts = parseArgs(["--provider", "foo", "--model", "bar", "--effort", "high"]); + expect(opts.effort).toBe("high"); + }); + + test("--effort rejects an unknown literal", () => { + expect(() => parseArgs(["--provider", "foo", "--model", "bar", "--effort", "bogus"])).toThrow( + /--effort must be one of/, + ); + }); + + test("--matrix cell can carry its own effort as a third segment", () => { + const opts = parseArgs(["--matrix", "xai/thegreataxios:grok-4.6:xhigh"]); + expect(opts.matrix).toBe("xai/thegreataxios:grok-4.6:xhigh"); + }); + + test("parsed defaults never equal xai/thegreataxios", () => { + const help = parseArgs(["--help"]); + const pair = parseArgs(["--provider", "foo", "--model", "bar"]); + expect(help.provider).not.toBe("xai/thegreataxios"); + expect(help.model).not.toBe("xai/thegreataxios"); + expect(pair.provider).not.toBe("xai/thegreataxios"); + expect(pair.model).not.toBe("xai/thegreataxios"); + expect(pair.provider).toBe("foo"); + expect(pair.model).toBe("bar"); + }); + + test("defaults concurrency to 1", () => { + delete process.env.CORBITS_EVAL_CONCURRENCY; + const opts = parseArgs(["--provider", "foo", "--model", "bar"]); + expect(opts.concurrency).toBe(1); + }); + + test("--concurrency 4 is accepted", () => { + delete process.env.CORBITS_EVAL_CONCURRENCY; + const opts = parseArgs(["--provider", "foo", "--model", "bar", "--concurrency", "4"]); + expect(opts.concurrency).toBe(4); + }); + + test("invalid --concurrency values throw", () => { + const pair = ["--provider", "foo", "--model", "bar"] as const; + expect(() => parseArgs([...pair, "--concurrency", "0"])).toThrow(/positive integer/); + expect(() => parseArgs([...pair, "--concurrency", "-1"])).toThrow(/positive integer/); + expect(() => parseArgs([...pair, "--concurrency", "1.5"])).toThrow(/positive integer/); + expect(() => parseArgs([...pair, "--concurrency", "foo"])).toThrow(/positive integer/); + }); + + test("CORBITS_EVAL_CONCURRENCY sets the default", () => { + process.env.CORBITS_EVAL_CONCURRENCY = "3"; + const opts = parseArgs(["--provider", "foo", "--model", "bar"]); + expect(opts.concurrency).toBe(3); + }); + + test("--concurrency overrides CORBITS_EVAL_CONCURRENCY", () => { + process.env.CORBITS_EVAL_CONCURRENCY = "8"; + const opts = parseArgs(["--provider", "foo", "--model", "bar", "--concurrency", "2"]); + expect(opts.concurrency).toBe(2); + }); + + test("invalid CORBITS_EVAL_CONCURRENCY throws", () => { + process.env.CORBITS_EVAL_CONCURRENCY = "0"; + expect(() => parseArgs(["--provider", "foo", "--model", "bar"])).toThrow( + /CORBITS_EVAL_CONCURRENCY must be a positive integer/, + ); + }); + + test("--director builder is parsed", () => { + const opts = parseArgs(["--provider", "foo", "--model", "bar", "--director", "builder"]); + expect(opts.director).toBe("builder"); + }); + + test("omitted --director stays undefined", () => { + const opts = parseArgs(["--provider", "foo", "--model", "bar"]); + expect(opts.director).toBeUndefined(); + }); + + test("--director without a value throws", () => { + expect(() => parseArgs(["--provider", "foo", "--model", "bar", "--director"])).toThrow( + "--director requires a value", + ); + }); +}); + +describe("validateVariantEfforts", () => { + // Wiring-level regression: parseArgs -> parseMatrix -> validateVariantEfforts, + // the same path main() runs before any inference. A matrix cell pairing an + // effort the model does not accept must fail fast, naming the model and its + // accepted levels, rather than silently falling back to the provider default + // and poisoning the matrix. + test("rejects an unsupported model/effort matrix cell before any inference runs", async () => { + const opts = parseArgs(["--matrix", "xai/thegreataxios:grok-composer-2.5-fast:xhigh"]); + const variants = parseMatrix(opts.matrix, { + ...(opts.provider !== undefined ? { provider: opts.provider } : {}), + ...(opts.model !== undefined ? { model: opts.model } : {}), + ...(opts.effort !== undefined ? { effort: opts.effort } : {}), + }); + await expect(validateVariantEfforts(variants, opts)).rejects.toThrow( + /grok-composer-2\.5-fast.*does not support reasoning effort "xhigh".*supported: low, medium, high/s, + ); + }); + + test("accepts a supported model/effort matrix cell", async () => { + const opts = parseArgs(["--matrix", "xai/thegreataxios:grok-4.6:xhigh"]); + const variants = parseMatrix(opts.matrix, { + ...(opts.provider !== undefined ? { provider: opts.provider } : {}), + ...(opts.model !== undefined ? { model: opts.model } : {}), + ...(opts.effort !== undefined ? { effort: opts.effort } : {}), + }); + await expect(validateVariantEfforts(variants, opts)).resolves.toBeUndefined(); + }); +}); + +describe("mapPool", () => { + test("N overlapping jobs with concurrency N finish in ~one job duration", async () => { + const jobMs = 80; + const n = 4; + const start = Date.now(); + const results = await mapPool([0, 1, 2, 3], n, async (item) => { + await new Promise((r) => setTimeout(r, jobMs)); + return item; + }); + const elapsed = Date.now() - start; + expect(results).toEqual([0, 1, 2, 3]); + expect(elapsed).toBeLessThan(jobMs * 2); + expect(elapsed).toBeGreaterThanOrEqual(jobMs - 20); + }); + + test("preserves input order when later items finish first", async () => { + const results = await mapPool([1, 2, 3], 3, async (item) => { + await new Promise((r) => setTimeout(r, (4 - item) * 30)); + return item; + }); + expect(results).toEqual([1, 2, 3]); + }); + + test("empty input returns an empty array", async () => { + expect(await mapPool([], 4, async (item) => item)).toEqual([]); + }); + + test("rejects non-positive concurrency", async () => { + await expect(mapPool([1], 0, async (item) => item)).rejects.toThrow(/positive integer/); + }); +}); + +describe("initEvalGitRepo", () => { + const savedGitConfigGlobal = process.env.GIT_CONFIG_GLOBAL; + + const restoreGitConfigGlobal = (): void => { + if (savedGitConfigGlobal === undefined) { + delete process.env.GIT_CONFIG_GLOBAL; + } else { + process.env.GIT_CONFIG_GLOBAL = savedGitConfigGlobal; + } + }; + + afterEach(() => { + restoreGitConfigGlobal(); + }); + + test("makes a fixture copy a git work tree with a commit", async () => { + const dir = await mkdtemp(join(tmpdir(), "corbits-eval-git-")); + try { + await writeFile(join(dir, "README"), "fixture\n", "utf8"); + await initEvalGitRepo(dir); + const { stdout } = await execFileAsync("git", ["rev-parse", "--is-inside-work-tree"], { + cwd: dir, + }); + expect(stdout.trim()).toBe("true"); + const { stdout: head } = await execFileAsync("git", ["rev-parse", "HEAD"], { cwd: dir }); + expect(head.trim().length).toBeGreaterThan(0); + const { stdout: count } = await execFileAsync("git", ["rev-list", "--count", "HEAD"], { + cwd: dir, + }); + expect(Number(count.trim())).toBeGreaterThanOrEqual(1); + const { stdout: log } = await execFileAsync("git", ["log", "-1", "--pretty=%s"], { + cwd: dir, + }); + expect(log.trim()).toBe("eval fixture"); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + test("succeeds when the process would otherwise sign", async () => { + const root = await mkdtemp(join(tmpdir(), "corbits-eval-git-sign-")); + const work = join(root, "work"); + const configPath = join(root, "gitconfig"); + try { + await mkdir(work); + await writeFile( + configPath, + "[commit]\ngpgsign = true\n[user]\nsigningkey = DEADKEY\n", + "utf8", + ); + process.env.GIT_CONFIG_GLOBAL = configPath; + await writeFile(join(work, "README"), "fixture\n", "utf8"); + await initEvalGitRepo(work); + const { stdout: head } = await execFileAsync("git", ["rev-parse", "HEAD"], { cwd: work }); + expect(head.trim().length).toBeGreaterThan(0); + const { stdout: cat } = await execFileAsync("git", ["cat-file", "-p", "HEAD"], { cwd: work }); + expect(cat).not.toContain("gpgsig"); + } finally { + restoreGitConfigGlobal(); + await rm(root, { recursive: true, force: true }); + } + }); +}); + +describe("buildEvalDiagnostics", () => { + test("non-Codex provider gets the default orchestrator tool list", async () => { + const diagnostics = await buildEvalDiagnostics(sampleConfig({ providerName: "openai" })); + expect(diagnostics.advertisedTools).toContain("read_file"); + expect(diagnostics.advertisedTools).toContain("run_shell"); + expect(diagnostics.reasoningEffort).toBeNull(); + }); + + test.each(["openai", "codex/default"])( + "%s diagnostics omit the removed instructions hash", + async (providerName) => { + const diagnostics = await buildEvalDiagnostics(sampleConfig({ providerName })); + expect(diagnostics).not.toHaveProperty("codexInstructionsHash"); + expect(diagnostics.advertisedTools).toContain("read_file"); + }, + ); + + test("echoes back the configured reasoning effort", async () => { + const diagnostics = await buildEvalDiagnostics(sampleConfig({ reasoningEffort: "high" })); + expect(diagnostics.reasoningEffort).toBe("high"); + }); + + test("--director builder reports the director's own advertised allowlist", async () => { + const diagnostics = await buildEvalDiagnostics(sampleConfig({ director: "builder" })); + expect(diagnostics.advertisedTools).not.toEqual( + (await buildEvalDiagnostics(sampleConfig({}))).advertisedTools, + ); + }); +}); diff --git a/scripts/eval-public-swe-one.test.ts b/scripts/eval-public-swe-one.test.ts new file mode 100644 index 00000000..5335e5f3 --- /dev/null +++ b/scripts/eval-public-swe-one.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, test } from "bun:test"; + +import { parseArgs } from "./eval-public-swe-one.js"; + +describe("parseArgs", () => { + test("--help does not require provider or model", () => { + const opts = parseArgs(["--help"]); + expect(opts.help).toBe(true); + expect(opts.provider).not.toBe("xai/thegreataxios"); + expect(opts.model).not.toBe("xai/thegreataxios"); + }); + + test("--dry-run alone throws", () => { + expect(() => parseArgs(["--dry-run"])).toThrow(/--provider/); + expect(() => parseArgs(["--dry-run"])).toThrow(/--model/); + }); + + test("--dry-run with provider and model parses", () => { + const opts = parseArgs(["--dry-run", "--provider", "foo", "--model", "bar"]); + expect(opts.dryRun).toBe(true); + expect(opts.provider).toBe("foo"); + expect(opts.model).toBe("bar"); + }); + + test("agent run without --provider throws", () => { + expect(() => parseArgs(["--model", "bar"])).toThrow(/--provider/); + }); + + test("agent run without --model throws", () => { + expect(() => parseArgs(["--provider", "foo"])).toThrow(/--model/); + }); + + test("agent run without either flag throws naming both", () => { + expect(() => parseArgs([])).toThrow(/--provider/); + expect(() => parseArgs([])).toThrow(/--model/); + }); + + test("--provider foo --model bar parses those values", () => { + const opts = parseArgs(["--provider", "foo", "--model", "bar"]); + expect(opts.provider).toBe("foo"); + expect(opts.model).toBe("bar"); + }); + + test("parsed defaults never equal xai/thegreataxios", () => { + const help = parseArgs(["--help"]); + expect(help.provider).not.toBe("xai/thegreataxios"); + expect(help.model).not.toBe("xai/thegreataxios"); + }); +}); diff --git a/scripts/guard-real-projects-dir.ts b/scripts/guard-real-projects-dir.ts index 9faffc74..9d0b316d 100644 --- a/scripts/guard-real-projects-dir.ts +++ b/scripts/guard-real-projects-dir.ts @@ -4,12 +4,12 @@ import { join } from "node:path"; import { mkdir, readdir, rm } from "node:fs/promises"; import { spawn } from "node:child_process"; -// Runs the test suite (`bun run test` — the same seeded, randomized command -// CI runs) and fails the run if any test wrote into the real -// ~/.corbits/projects directory. Tests must sandbox state under a temp -// `home` (see src/session/index.ts's `home` overrides); nothing running -// under this wrapper is allowed to fall back to the developer's own -// session history. +// Runs the test suite (`bun run test` — the seeded, randomized one-process +// command whose path union CI shards via `test:paths`) and fails the run if +// any test wrote into the real ~/.corbits/projects directory. Tests must +// sandbox state under a temp `home` (see src/session/index.ts's `home` +// overrides); nothing running under this wrapper is allowed to fall back to +// the developer's own session history. // // This is a backstop, not a substitute for threading `home` correctly: a // leak is only caught after it already wrote into a real directory once, diff --git a/scripts/test-paths.ts b/scripts/test-paths.ts new file mode 100644 index 00000000..4bb7c8ad --- /dev/null +++ b/scripts/test-paths.ts @@ -0,0 +1,23 @@ +import { spawn } from "node:child_process"; + +// CI shards and `check:projects-dir-guard` pass bun-test path filters here. +// A zero-arg `bun test` walks the whole tree, including vendor/, so this +// script refuses to run without at least one path (not a flag). + +const args = process.argv.slice(2); +const paths = args.filter((arg) => !arg.startsWith("-")); + +if (paths.length === 0) { + process.stderr.write( + "test:paths requires at least one path filter (refusing a whole-tree scan of vendor/)\n", + ); + process.exit(1); +} + +const child = spawn("bun", ["test", "--randomize", "--seed", "424242", ...args], { + stdio: "inherit", +}); + +child.on("exit", (code) => { + process.exit(code ?? 1); +}); diff --git a/src/agent/tools-mcp-disconnect.test.ts b/src/agent/tools-mcp-disconnect.test.ts index 6e75c629..e583515a 100644 --- a/src/agent/tools-mcp-disconnect.test.ts +++ b/src/agent/tools-mcp-disconnect.test.ts @@ -202,9 +202,7 @@ describe("disconnectMCPServer", () => { }); const names = acmeNames(toolset.dynamicRunner.currentDefinitions()); - expect(names).toContain("mcp__acme__list"); - expect(names).toContain("mcp__acme__search"); - expect(names.filter((name) => name === "mcp__acme__list")).toHaveLength(1); + expect(names).toEqual(["mcp__acme__list", "mcp__acme__search"]); const list = toolset.dynamicRunner .currentDefinitions() @@ -214,7 +212,7 @@ describe("disconnectMCPServer", () => { // The stale generation's client was closed and the drift was announced. expect(closedGenerations).toContain(1); - expect(acmeNames(announced.at(-1) ?? [])).toContain("mcp__acme__search"); + expect(acmeNames(announced.at(-1) ?? [])).toEqual(["mcp__acme__list", "mcp__acme__search"]); } finally { await toolset.dispose(); } diff --git a/src/tui/session-operation-queue.test.ts b/src/tui/session-operation-queue.test.ts index 42c98c73..97e46af7 100644 --- a/src/tui/session-operation-queue.test.ts +++ b/src/tui/session-operation-queue.test.ts @@ -63,34 +63,6 @@ test("deliver targets agent at execution time when enqueued before rotation", as expect(log).toEqual(["deliver:A", "rotate"]); }); -test("rotation enqueued during an in-flight delivery waits for it to settle", async () => { - const log: string[] = []; - const { enqueue, awaitTail } = createSessionOperationQueue(); - - let resolveSend!: () => void; - const send = new Promise((r) => (resolveSend = r)); - - enqueue(async () => { - log.push("deliver:start"); - await send; - log.push("deliver:end"); - }); - enqueue(async () => { - log.push("rotate:start"); - log.push("rotate:end"); - }); - - // The rotation is already queued while the delivery is still awaiting the - // provider send — it must not start (rotating the session dir) mid-delivery. - await Promise.resolve(); - await Promise.resolve(); - expect(log).toEqual(["deliver:start"]); - - resolveSend(); - await awaitTail(); - expect(log).toEqual(["deliver:start", "deliver:end", "rotate:start", "rotate:end"]); -}); - test("a failed delivery does not block a rotation queued behind it", async () => { const log: string[] = []; const { enqueue, awaitTail } = createSessionOperationQueue(); diff --git a/tests/unit/check-gate.test.ts b/tests/unit/check-gate.test.ts index 28a2bf9c..9e8f371f 100644 --- a/tests/unit/check-gate.test.ts +++ b/tests/unit/check-gate.test.ts @@ -3,9 +3,10 @@ import { join } from "node:path"; import { describe, expect, test } from "bun:test"; // Guard against the gate drifting apart again (CL-7300): `bun run check` and -// CI's test jobs must resolve to the same seeded suite, and the projects-dir +// CI's test jobs must resolve to the same seeded path union, and the projects-dir // guard must delegate to the `test` script (or `test:paths` for shard filters) -// rather than duplicate its command. +// rather than duplicate its command. Local `bun run test` is one process; CI +// shards that union via `test:paths`. const repoRoot = join(import.meta.dir, "..", ".."); const pkg = JSON.parse(readFileSync(join(repoRoot, "package.json"), "utf8")) as { @@ -15,18 +16,25 @@ const ci = readFileSync(join(repoRoot, ".github", "workflows", "ci.yml"), "utf8" const guardSource = readFileSync(join(repoRoot, "scripts", "guard-real-projects-dir.ts"), "utf8"); const GUARD_SCRIPT = "check:projects-dir-guard"; -const TEST_SUITE = "bun test ./src ./tests ./evals --randomize --seed 424242"; +const TEST_SUITE = "bun test ./src ./tests ./evals ./scripts --randomize --seed 424242"; describe("check gate", () => { - test("`test` is the seeded, randomized suite CI runs", () => { + test("`test` is the seeded, randomized one-process suite whose path union CI shards", () => { expect(pkg.scripts.test).toBe(TEST_SUITE); }); test("`test:paths` is the seeded suite accepting CI shard path filters", () => { // Same seed as `test`; the guard passes shard filters as arguments, which // cannot be appended to `bun run test` because bun's filters are additive. - expect(pkg.scripts["test:paths"]).toBe("bun test --randomize --seed 424242"); + // Zero args would be a whole-tree `bun test` including vendor/, so the + // wrapper requires at least one path. + expect(pkg.scripts["test:paths"]).toBe("bun scripts/test-paths.ts"); expect(guardSource).toContain('"run", "test:paths"'); + const testPathsSource = readFileSync(join(repoRoot, "scripts", "test-paths.ts"), "utf8"); + expect(testPathsSource).toContain("--randomize"); + expect(testPathsSource).toContain("--seed"); + expect(testPathsSource).toContain("424242"); + expect(testPathsSource).toContain("requires at least one path filter"); }); test("`check` runs the suite through the projects-dir guard", () => { diff --git a/tests/unit/corbits-skills-catalog.test.ts b/tests/unit/corbits-skills-catalog.test.ts index 3ed7c537..2ebbb747 100644 --- a/tests/unit/corbits-skills-catalog.test.ts +++ b/tests/unit/corbits-skills-catalog.test.ts @@ -112,8 +112,6 @@ test("corbits-skills catalog lists 20 skills with name and description", async ( test("first-party skills are how-to playbooks, not director personas", async () => { - // Forbidden-content policy: pins what must NEVER appear in a skill doc. - // Violations are collected and asserted once so a failure names the skill. const gaasOverlap = new Set([ "ast-grep", "create-issue", @@ -130,21 +128,15 @@ test("first-party skills are how-to playbooks, not director personas", async () "style", "typescript", ]); - const violations: string[] = []; for (const name of SKILL_DIRS) { const skill = await Bun.file(join(pluginRoot, "skills", name, "SKILL.md")).text(); - if (skill.includes("You are Skywalker") || /You are \w+Director/.test(skill)) { - violations.push(`${name}: director persona language`); - } - if (skill.includes("Host is Corbits")) { - violations.push(`${name}: host attribution`); - } + expect(skill).not.toContain("You are Skywalker"); + expect(skill).not.toMatch(/You are \w+Director/); + expect(skill).not.toContain("Host is Corbits"); if (gaasOverlap.has(name)) continue; - if (skill.includes("## Acknowledgment") || /I have reviewed the .+ skill/.test(skill)) { - violations.push(`${name}: GaaS acknowledgment ritual`); - } + expect(skill).not.toContain("## Acknowledgment"); + expect(skill).not.toMatch(/I have reviewed the .+ skill/); } - expect(violations).toEqual([]); }); test("use_skill-only skills set user-invocable: false without disable-model-invocation", async () => { @@ -176,14 +168,10 @@ test("only background and bake-only skills carry disable-model-invocation", asyn }); test("review skill does not own GitHub posting or Linear In Review", async () => { - // Ownership-boundary policy: pins FORBIDDEN claims, not required copy. const skill = await Bun.file(join(pluginRoot, "skills/review/SKILL.md")).text(); - const forbiddenClaims = [ - "Post the Review on GitHub", - "`linear-issue-workflow` owns the In Review write", - "this skill does not set Linear state", - ]; - expect(forbiddenClaims.filter((claim) => skill.includes(claim))).toEqual([]); + expect(skill).not.toContain("Post the Review on GitHub"); + expect(skill).not.toContain("`linear-issue-workflow` owns the In Review write"); + expect(skill).not.toContain("this skill does not set Linear state"); }); test("slash skills do not set user-invocable: false", async () => { diff --git a/tests/unit/eslint-rules/no-content-pin-tests.test.ts b/tests/unit/eslint-rules/no-content-pin-tests.test.ts deleted file mode 100644 index c5abdf17..00000000 --- a/tests/unit/eslint-rules/no-content-pin-tests.test.ts +++ /dev/null @@ -1,114 +0,0 @@ -import { Linter } from "eslint"; -import { describe, expect, test } from "bun:test"; -import noContentPinTests from "../../../scripts/eslint-rules/no-content-pin-tests"; - -const lint = (code: string) => - new Linter({ configType: "flat" }).verify(code, { - plugins: { corbits: { rules: { "no-content-pin-tests": noContentPinTests } } }, - rules: { "corbits/no-content-pin-tests": "error" }, - }); - -const flaggedIds = (code: string) => lint(code).map((message) => message.messageId); - -describe("no-content-pin-tests", () => { - test("flags literal wording pins on Bun.file-loaded assets", () => { - expect( - flaggedIds( - [ - 'const skill = await Bun.file(join(root, "skills/style/SKILL.md")).text();', - 'expect(skill).toContain("Prefer deletion over addition");', - 'expect(skill).not.toContain("spawn_agent");', - 'expect(skill).toBe("exact document text");', - "expect(skill).toMatch(/You are \\w+Director/);", - ].join("\n"), - ), - ).toEqual(["wordingPin", "wordingPin", "wordingPin", "wordingPin"]); - }); - - test("flags wording pins on members of assets parsed as JSON", () => { - expect( - flaggedIds( - [ - 'const manifest = await Bun.file("plugins/corbits-skills/manifest.json").json();', - 'expect(manifest.id).toBe("corbits-skills");', - 'expect(manifest.kind).toContain("command");', - ].join("\n"), - ), - ).toEqual(["wordingPin", "wordingPin"]); - }); - - test("flags exact brand hex pins regardless of receiver", () => { - expect(flaggedIds('expect(color("brand")).toBe("#f5933a");')).toEqual(["hexPin"]); - expect(flaggedIds('expect(fg).toEqual("#7ea2c4");')).toEqual(["hexPin"]); - }); - - test("flags numeric pins on palette-named callees", () => { - expect(flaggedIds('expect(color256("brand")).toBe(173);')).toEqual(["ansiIndexPin"]); - expect(flaggedIds("expect(paletteIndex(role)).toEqual(74);")).toEqual(["ansiIndexPin"]); - }); - - test("keeps behavior-string assertions clean", () => { - expect( - flaggedIds( - [ - 'expect(groupChainSegmentsForDisplay("ls | head -5 && echo done")).toEqual([', - ' "ls",', - ' "head -5",', - ' "echo done",', - "]);", - 'expect(isShellNoOp("true")).toBe(true);', - "expect(secondsFromMs(0)).toBe(0);", - "expect(idx).toBeGreaterThanOrEqual(0);", - "expect(cut.length).toBeLessThanOrEqual(20);", - 'expect(messages).toContain("outside the workspace");', - ].join("\n"), - ), - ).toEqual([]); - }); - - test("keeps runtime round-trips of non-document files clean", () => { - expect( - flaggedIds( - [ - 'const written = await Bun.file(join(cwd, "app.py")).text();', - "expect(written).toBe(\"def greet():\\n print('hello')\\n\");", - ].join("\n"), - ), - ).toEqual([]); - }); - - test("keeps source-structure locks on .ts files clean", () => { - expect( - flaggedIds( - [ - 'const src = await Bun.file(new URL("./runner.ts", import.meta.url)).text();', - 'expect(src).toContain("standingPluginWarnings");', - ].join("\n"), - ), - ).toEqual([]); - }); - - test("keeps non-literal matcher arguments clean", () => { - expect( - flaggedIds( - [ - "const before = await Bun.file(target).text();", - "expect(await Bun.file(target).text()).toBe(before);", - "expect(palette.diffAdded).toEqual(palette.success);", - "expect(message).toContain(secret);", - ].join("\n"), - ), - ).toEqual([]); - }); - - test("keeps range and contract checks on palette receivers clean", () => { - expect( - flaggedIds( - [ - "expect(color256(role)).toBeLessThanOrEqual(255);", - "expect(color(role)).toMatch(/^#[0-9a-fA-F]{6}$/);", - ].join("\n"), - ), - ).toEqual([]); - }); -}); diff --git a/tests/unit/tui/tool-formatter-web-brand.test.ts b/tests/unit/tui/tool-formatter-web-brand.test.ts new file mode 100644 index 00000000..aa378259 --- /dev/null +++ b/tests/unit/tui/tool-formatter-web-brand.test.ts @@ -0,0 +1,20 @@ +import { test, expect, afterEach } from "bun:test"; +import { humanizeToolName, setActiveWebProviderBrand } from "../../../src/tui/tool-formatter.js"; + +afterEach(() => setActiveWebProviderBrand(undefined)); + +test("web tools use the default names with no active web brand", () => { + expect(humanizeToolName("web_search")).toBe("Web Search"); + expect(humanizeToolName("web_fetch")).toBe("Web Fetch"); +}); + +test("web tools render with the active web plugin brand", () => { + setActiveWebProviderBrand("Exa"); + expect(humanizeToolName("web_search")).toBe("Exa Search"); + expect(humanizeToolName("web_fetch")).toBe("Exa Fetch"); +}); + +test("non-web tools are unaffected by the web brand", () => { + setActiveWebProviderBrand("Exa"); + expect(humanizeToolName("read_file")).toBe("Read"); +}); From 4e5390154d5f01877e40f7a9016b4c71eba84cd3 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Wed, 9 Sep 2026 16:46:23 -0700 Subject: [PATCH 16/18] style: prettier after rebase conflict in skills catalog tests --- tests/unit/corbits-skills-catalog.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/corbits-skills-catalog.test.ts b/tests/unit/corbits-skills-catalog.test.ts index 2ebbb747..058a8ca2 100644 --- a/tests/unit/corbits-skills-catalog.test.ts +++ b/tests/unit/corbits-skills-catalog.test.ts @@ -110,7 +110,6 @@ test("corbits-skills catalog lists 20 skills with name and description", async ( } }); - test("first-party skills are how-to playbooks, not director personas", async () => { const gaasOverlap = new Set([ "ast-grep", From eece356d05f016efb37ae59752346ea4948db15b Mon Sep 17 00:00:00 2001 From: Sawyer Date: Wed, 9 Sep 2026 20:20:48 -0700 Subject: [PATCH 17/18] Align exec dispose tests with posix-first teardown 846 reaps the toolset before cancel/close and fail-closes a rejected agent.close. The coverage added here now asserts that order. --- tests/unit/exec/runner.test.ts | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/tests/unit/exec/runner.test.ts b/tests/unit/exec/runner.test.ts index 0754a336..bec0b851 100644 --- a/tests/unit/exec/runner.test.ts +++ b/tests/unit/exec/runner.test.ts @@ -512,7 +512,7 @@ describe("disposeExecRuntime", () => { ).rejects.toThrow("plugin dispose failed"); }); - test("cancels every live worker with the close reason before the agent closes", async () => { + test("cancels every live worker with the close reason after toolset dispose", async () => { const store = createSubAgentSessionStore(); const first = store.start({ description: "a", agentId: "w1", brief: "b" }); const second = store.start({ description: "b", agentId: "w2", brief: "b" }); @@ -526,27 +526,29 @@ describe("disposeExecRuntime", () => { subAgentSessions: store, }); - // Cancellation must precede teardown so no worker outlives the runtime. - expect(calls).toEqual(["cancel:first", "cancel:second", "agent", "toolset"]); + // Posix/toolset first so a hung close cannot skip reap; then cancel, then close. + expect(calls).toEqual(["toolset", "cancel:first", "cancel:second", "agent"]); expect(store.get(first.id)?.status).toBe("cancelled"); expect(store.get(second.id)?.status).toBe("cancelled"); expect(store.get(first.id)?.stopReason).toBe("cancelled — Session closed"); expect(store.get(second.id)?.stopReason).toBe("cancelled — Session closed"); }); - test("a failing agent close still disposes the toolset and resolves", async () => { + test("a failing agent close still disposes the toolset and rejects", async () => { const store = createSubAgentSessionStore(); const worker = store.start({ description: "bg", agentId: "w", brief: "b" }); store.registerCancel(worker.id, () => undefined); let disposed = 0; - await disposeExecRuntime({ - agent: { - close: () => Promise.reject(new Error("close exploded")), - }, - toolset: { dispose: async () => void (disposed += 1) }, - subAgentSessions: store, - }); + await expect( + disposeExecRuntime({ + agent: { + close: () => Promise.reject(new Error("close exploded")), + }, + toolset: { dispose: async () => void (disposed += 1) }, + subAgentSessions: store, + }), + ).rejects.toThrow("close exploded"); expect(store.get(worker.id)?.status).toBe("cancelled"); expect(disposed).toBe(1); From ef8fe450793042a1749e1e52b739f932b27766ab Mon Sep 17 00:00:00 2001 From: Sawyer Date: Wed, 9 Sep 2026 20:23:25 -0700 Subject: [PATCH 18/18] Publish legacy CI check names after the restructured jobs protect-main still requires prettier, eslint, typecheck, and build-and-test. Alias jobs go green only after static-analysis / build / the test shards succeed. --- .github/workflows/ci.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 98234cd1..c69d48c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -137,3 +137,29 @@ jobs: # `bun run test:paths `. - name: Test run: bun run check:projects-dir-guard ${{ matrix.shard.paths }} + + # protect-main still requires the pre-restructure check names. These jobs + # exist only to publish those contexts after the real work succeeds. + prettier: + needs: static-analysis + runs-on: ubuntu-latest + steps: + - run: "true" + + eslint: + needs: static-analysis + runs-on: ubuntu-latest + steps: + - run: "true" + + typecheck: + needs: static-analysis + runs-on: ubuntu-latest + steps: + - run: "true" + + build-and-test: + needs: [build, test] + runs-on: ubuntu-latest + steps: + - run: "true"