From 3cad9a58fce50e65a080b93fde99e30affcd0dee Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 15:41:50 +0000 Subject: [PATCH] test(eval): record prismic calls as the argv the CLI received The project's prismic bin is now a small script that logs each call's argv before it runs the CLI. Evals see resolved values instead of the shell text the agent typed, and one entry per call even when the agent chains commands. The toHaveRun matcher reads that list, so it no longer splits shell text. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_015pufvNWHTyTcd38NnNPyPK --- evals/check-before-acting.eval.ts | 6 +-- evals/configure-repositories.eval.ts | 12 ++--- evals/design-sensible-models.eval.ts | 14 +++--- evals/edit-models-precisely.eval.ts | 18 +++---- evals/initialize-projects.eval.ts | 6 +-- evals/it.ts | 74 +++++++++++++++++----------- evals/render-content.eval.ts | 6 +-- evals/repair-generated-files.eval.ts | 2 +- evals/reporter.ts | 4 +- evals/sync-models.eval.ts | 6 +-- 10 files changed, 83 insertions(+), 65 deletions(-) diff --git a/evals/check-before-acting.eval.ts b/evals/check-before-acting.eval.ts index e869f148..7ed262d4 100644 --- a/evals/check-before-acting.eval.ts +++ b/evals/check-before-acting.eval.ts @@ -16,7 +16,7 @@ it.for(trials)( `Set up content previews for this project so editors can preview drafts.`, ); - expect(result).toHaveRun("prismic", ["docs"]); + expect(result).toHaveRun(["docs"]); }, ); @@ -32,7 +32,7 @@ it.for(trials)( `Change the "getting-started" document from the "article" type to "blog_post".`, ); - expect(result).not.toHaveRun("prismic", ["type", "remove"]); + expect(result).not.toHaveRun(["type", "remove"]); const models = await readLocalCustomTypes(project); expect(models.find((model) => model.id === article.id)).toEqual(article); expect(models.find((model) => model.id === post.id)).toEqual(post); @@ -56,7 +56,7 @@ it.for(trials)( const result = await agent(`Clean up the fields on "article".`); - expect(result).not.toHaveRun("prismic", ["field", "remove"]); + expect(result).not.toHaveRun(["field", "remove"]); const model = await readLocalCustomType(project, article.id); expect(model.json.Main.title).toEqual(article.json.Main.title); expect(model.json.Main.body).toEqual(article.json.Main.body); diff --git a/evals/configure-repositories.eval.ts b/evals/configure-repositories.eval.ts index 7feda9ed..e14b631f 100644 --- a/evals/configure-repositories.eval.ts +++ b/evals/configure-repositories.eval.ts @@ -14,7 +14,7 @@ it.for(trials)("sets up a content preview", async (_, { agent, expect, repo, tok `Set up a content preview for this repo pointing at https://example.com/api/preview.`, ); - expect(result).toHaveRun("prismic", ["preview", "add"]); + expect(result).toHaveRun(["preview", "add"]); const previews = await getPreviews({ repo, token, host }); expect(previews.some((preview) => preview.url.includes("example.com"))).toBe(true); }); @@ -29,7 +29,7 @@ it.for(trials)( `We just deployed the site to https://example.com. Set up content previews for production.`, ); - expect(result).toHaveRun("prismic", ["preview", "add"]); + expect(result).toHaveRun(["preview", "add"]); const previews = await getPreviews({ repo, token, host }); expect(previews.some((preview) => preview.url.includes("example.com"))).toBe(true); expect(previews.some((preview) => preview.url.includes("localhost:3000"))).toBe(true); @@ -45,8 +45,8 @@ it.for(trials)( `Create a content API token named "ci" for this repo and make the content API private.`, ); - expect(result).toHaveRun("prismic", ["token", "create"]); - expect(result).toHaveRun("prismic", ["repo", "set-api-access"]); + expect(result).toHaveRun(["token", "create"]); + expect(result).toHaveRun(["repo", "set-api-access"]); const apps = await getAccessTokens({ repo, token, host }); expect(apps.some((app) => app.name === "ci")).toBe(true); }, @@ -57,7 +57,7 @@ it.for(trials)("registers a webhook", async (_, { agent, expect, repo, token, ho `Register a webhook at https://example.com/api/revalidate that fires when documents are published or unpublished.`, ); - expect(result).toHaveRun("prismic", ["webhook", "create"]); + expect(result).toHaveRun(["webhook", "create"]); const webhooks = await getWebhooks({ repo, token, host }); expect(JSON.stringify(webhooks)).toContain("example.com/api/revalidate"); }); @@ -65,7 +65,7 @@ it.for(trials)("registers a webhook", async (_, { agent, expect, repo, token, ho it.for(trials)("adds a locale", async (_, { agent, expect, repo, token, host }) => { const result = await agent(`Add French (France) as a locale for this repo.`); - expect(result).toHaveRun("prismic", ["locale", "add"]); + expect(result).toHaveRun(["locale", "add"]); const locales = await getLocales({ repo, token, host }); expect(locales.some((locale) => locale.id === "fr-fr")).toBe(true); }); diff --git a/evals/design-sensible-models.eval.ts b/evals/design-sensible-models.eval.ts index 1e24c8cd..ec05f681 100644 --- a/evals/design-sensible-models.eval.ts +++ b/evals/design-sensible-models.eval.ts @@ -17,7 +17,7 @@ it.for(trials)( `Create a testimonial slice: a quote, author name, author role, an avatar, and a company logo.`, ); - expect(result).toHaveRun("prismic", ["slice", "create"]); + expect(result).toHaveRun(["slice", "create"]); const slices = await readLocalSlices(project); expect(slices.length).toBe(1); await expect(JSON.stringify(slices[0], null, 2)).toSatisfyJudge( @@ -37,7 +37,7 @@ it.for(trials)("models a slice from a screenshot", async (_, { project, agent, e const result = await agent(`Model a Prismic slice for the section design in feature-slice.png.`); - expect(result).toHaveRun("prismic", ["slice", "create"]); + expect(result).toHaveRun(["slice", "create"]); const slices = await readLocalSlices(project); expect(slices.length).toBe(1); await expect(JSON.stringify(slices[0], null, 2)).toSatisfyJudge( @@ -59,7 +59,7 @@ it.for(trials)( `model a Prismic blog post: a title, publish date, hero image, author, and body`, ); - expect(result).toHaveRun("prismic", ["type", "create"]); + expect(result).toHaveRun(["type", "create"]); const models = (await readLocalCustomTypes(project)).filter((model) => model.id !== author.id); await expect(JSON.stringify(models, null, 2)).toSatisfyJudge( @@ -76,7 +76,7 @@ it.for(trials)( async (_, { project, agent, expect }) => { const result = await agent(`Model a landing page and a global navigation menu.`); - expect(result).toHaveRun("prismic", ["type", "create"]); + expect(result).toHaveRun(["type", "create"]); const models = await readLocalCustomTypes(project); const landingPage = models.find((model) => /landing/.test(model.id)); const navigation = models.find((model) => /nav/.test(model.id)); @@ -95,7 +95,7 @@ it.for(trials)( const result = await agent(`The homepage needs a flexible hero.`); - expect(result).toHaveRun("prismic"); + expect(result).toHaveRun(); const models = await readLocalCustomTypes(project); const slices = await readLocalSlices(project); await expect(JSON.stringify({ models, slices }, null, 2)).toSatisfyJudge( @@ -118,7 +118,7 @@ it.for(trials)( `Add a star rating (1 to 5) and a call-to-action button to the "product" type.`, ); - expect(result).toHaveRun("prismic", ["field", "add"]); + expect(result).toHaveRun(["field", "add"]); const model = await readLocalCustomType(project, product.id); await expect(JSON.stringify(model, null, 2)).toSatisfyJudge( dedent` @@ -137,7 +137,7 @@ it.for(trials)( const result = await agent( `Set up the "blog_post" type: it needs a title and the author's Bluesky handle.`, ); - expect(result).toHaveRun("prismic", ["field", "add"]); + expect(result).toHaveRun(["field", "add"]); const model = await readLocalCustomType(project, customType.id); await expect(JSON.stringify(model, null, 2)).toSatisfyJudge( diff --git a/evals/edit-models-precisely.eval.ts b/evals/edit-models-precisely.eval.ts index 3680b9bd..7a7fc9ad 100644 --- a/evals/edit-models-precisely.eval.ts +++ b/evals/edit-models-precisely.eval.ts @@ -23,7 +23,7 @@ it.for(trials)("adds a field", async (_, { project, agent, expect }) => { const result = await agent(`Add an "excerpt" rich text field to the "article" type.`); - expect(result).toHaveRun("prismic", ["field", "add", "rich-text", "excerpt"]); + expect(result).toHaveRun(["field", "add", "rich-text", "excerpt"]); const model = await readLocalCustomType(project, article.id); expect(model.json.Main.excerpt.type).toBe("StructuredText"); expect(model.json.Main.title).toEqual(article.json.Main.title); @@ -42,7 +42,7 @@ it.for(trials)( const result = await agent(`Add a "centered" variation to the "Hero" slice.`); - expect(result).toHaveRun("prismic", ["slice", "add-variation"]); + expect(result).toHaveRun(["slice", "add-variation"]); const model = await readLocalSlice(project, slice.id); const ids = model?.variations.map((variation) => variation.id); expect(ids).toContain("default"); @@ -68,7 +68,7 @@ it.todo("renames a field without disturbing field order", async ({ project, agen const result = await agent(`Rename the "tagline" field on "article" to "subtitle".`); - expect(result).toHaveRun("prismic", ["field", "edit"]); + expect(result).toHaveRun(["field", "edit"]); const model = await readLocalCustomType(project, article.id); expect(Object.keys(model.json.Main)).toEqual(["title", "subtitle", "body"]); expect(model.json.Main.subtitle.type).toBe("Text"); @@ -88,7 +88,7 @@ it.for(trials)( `The testimonial design now also shows the author's company logo and a star rating. Update the "Testimonial" slice.`, ); - expect(result).toHaveRun("prismic", ["field", "add"]); + expect(result).toHaveRun(["field", "add"]); const model = await readLocalSlice(project, slice.id); const primary = model?.variations[0].primary ?? {}; expect(primary.quote).toEqual(slice.variations[0].primary.quote); @@ -107,7 +107,7 @@ it.for(trials)( const firstRun = await agent(`add a "body" rich text field to "homepage"`); await agent(`add a "body" rich text field to "homepage"`); - expect(firstRun).toHaveRun("prismic", ["field", "add"]); + expect(firstRun).toHaveRun(["field", "add"]); const model = await readLocalCustomType(project, homepage.id); const bodyLikeKeys = Object.keys(model.json.Main).filter((key) => /body/i.test(key)); expect(bodyLikeKeys).toEqual(["body"]); @@ -123,7 +123,7 @@ it.for(trials)("adds a group field with nested fields", async (_, { project, age `Add a repeatable "features" group to the "product" type. Each feature has an icon image and a label.`, ); - expect(result).toHaveRun("prismic", ["field", "add", "group"]); + expect(result).toHaveRun(["field", "add", "group"]); const model = await readLocalCustomType(project, product.id); const features = model.json.Main.features; expect(features.type).toBe("Group"); @@ -148,7 +148,7 @@ it.for(trials)( // Both `field add content-relationship` and `field add link --allow document` // model a constrained relationship; judge the resulting model, not the command. - expect(result).toHaveRun("prismic", ["field", "add"]); + expect(result).toHaveRun(["field", "add"]); const model = await readLocalCustomType(project, product.id); expect(model.json.Main.size.type).toBe("Select"); expect((model.json.Main.size.config as { options: string[] }).options).toEqual(["S", "M", "L"]); @@ -180,7 +180,7 @@ it.for(trials)( `Restrict the "title" field on "post" so editors can only write a single H1.`, ); - expect(result).toHaveRun("prismic", ["field", "edit"]); + expect(result).toHaveRun(["field", "edit"]); const model = await readLocalCustomType(project, post.id); const config = model.json.Main.title.config as { single?: string; multi?: string }; expect(config.single).toBe("heading1"); @@ -200,7 +200,7 @@ it.for(trials)("connects a slice to a page type", async (_, { project, agent, ex const result = await agent(`Make the "Testimonial" slice available on the "page" type.`); - expect(result).toHaveRun("prismic", ["slice", "connect"]); + expect(result).toHaveRun(["slice", "connect"]); const model = await readLocalCustomType(project, page.id); const choices = (model.json.Main.slices.config as { choices: Record }).choices; expect(Object.keys(choices)).toContain(slice.id); diff --git a/evals/initialize-projects.eval.ts b/evals/initialize-projects.eval.ts index e86f4489..4d7400ac 100644 --- a/evals/initialize-projects.eval.ts +++ b/evals/initialize-projects.eval.ts @@ -9,7 +9,7 @@ it.for(trials)( const result = await agent(`Set up Prismic in this Next.js project.`); - expect(result).toHaveRun("prismic", ["init"]); + expect(result).toHaveRun(["init"]); const config = JSON.parse(await readFile(new URL("prismic.config.json", project), "utf8")); expect(config.repositoryName).toBeTruthy(); }, @@ -24,7 +24,7 @@ it.for(trials)( const result = await agent(`Add Prismic to this existing Next.js app.`); - expect(result).toHaveRun("prismic", ["init"]); + expect(result).toHaveRun(["init"]); const config = JSON.parse(await readFile(new URL("prismic.config.json", project), "utf8")); expect(config.repositoryName).toBeTruthy(); const page = await readFile(new URL("app/page.tsx", project), "utf8"); @@ -41,7 +41,7 @@ it.for(trials)( `Set up Prismic in this project using the existing "${repo}" Prismic repository.`, ); - expect(result).toHaveRun("prismic", ["init"]); + expect(result).toHaveRun(["init"]); const config = JSON.parse(await readFile(new URL("prismic.config.json", project), "utf8")); expect(config.repositoryName).toBe(repo); }, diff --git a/evals/it.ts b/evals/it.ts index 97257f98..399702ba 100644 --- a/evals/it.ts +++ b/evals/it.ts @@ -1,7 +1,7 @@ import { query, type SDKResultMessage } from "@anthropic-ai/claude-agent-sdk"; import { Codex } from "@openai/codex-sdk"; import dedent from "dedent"; -import { copyFile, mkdir, mkdtemp, readFile, symlink, writeFile } from "node:fs/promises"; +import { copyFile, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { homedir, tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -35,7 +35,7 @@ declare module "vitest" { } // oxlint-disable-next-line no-explicit-any interface Matchers { - toHaveRun(bin: string, positionals?: string[]): T; + toHaveRun(positionals?: string[]): T; toSatisfyJudge(criterion: string): Promise; } } @@ -65,10 +65,21 @@ export const it = base.extend<{ CODEX_HOME: await createCodexHome(), }; + // The project's `prismic` bin records the argv of every call before running the + // CLI, so evals see the arguments as the CLI received them, not as the agent typed them. + const argvLog = join(tmpdir(), `prismic-argv-${crypto.randomUUID()}.jsonl`); if (installCli) { - const nodeModulesBinDir = new URL("node_modules/.bin/", project); - await mkdir(nodeModulesBinDir, { recursive: true }); - await symlink(BIN, new URL("prismic", nodeModulesBinDir)); + const bin = new URL("node_modules/.bin/prismic", project); + await mkdir(new URL(".", bin), { recursive: true }); + await writeFile( + bin, + dedent` + #!/bin/sh + node -e 'console.log(JSON.stringify(process.argv.slice(1)))' -- "$@" >> ${JSON.stringify(argvLog)} + exec node ${JSON.stringify(fileURLToPath(BIN))} "$@" + `, + { mode: 0o755 }, + ); env.PRISMIC_CONFIG_DIR = fileURLToPath(new URL(".config/prismic/", home)); env.PRISMIC_TYPE_BUILDER_ENABLED = "true"; env.PRISMIC_SENTRY_ENABLED = "false"; @@ -89,23 +100,22 @@ export const it = base.extend<{ skill: installSkill ? SKILL : undefined, cwd: project, env, - // Recorded as commands stream so a timed-out trial keeps its trail. - onCommand: (command) => { - commands.push(command); - if (/(^|\s)(npx\s+)?prismic(@|\s|$)/.test(command)) { - trial.calls.push(command.replace(/^.*?(^|\s)(npx\s+)?prismic(@\S+)?(?=\s|$)\s*/, "")); - } - }, + onCommand: (command) => commands.push(command), + }).finally(async () => { + // Recorded even when the run fails so a timed-out trial keeps its trail. + durationMs += performance.now() - start; + trial.durationS = Math.round(durationMs / 1000); + trial.calls = await readArgvLog(argvLog); }); - durationMs += performance.now() - start; trial.text = text; trial.tokens += tokens; - trial.durationS = Math.round(durationMs / 1000); - return { text, commands }; + return { text, commands, calls: trial.calls }; }); + await rm(argvLog, { force: true }); + for (const file of ["prismic.config.json", "slicemachine.config.json"]) { try { const configFile = await readFile(new URL(file, project), "utf8"); @@ -121,25 +131,21 @@ export const it = base.extend<{ it.scoped({ isolateRepo: true }); expect.extend({ - toHaveRun(result: AgentResult, bin: string, positionals: string[] = []) { - const pass = result.commands.some((command) => { - return command.split(/&&|\|\||;|\||\n/).some((segment) => { - const words = segment.split(/\s+/).filter(Boolean); - if (words.includes("--help") || words.includes("-h")) return false; - const start = words.findIndex((word) => new RegExp(`^${bin}@?`).test(word)); - if (start === -1) return false; - - const got = words.slice(start + 1).filter((w) => !w.startsWith("-")); - return positionals.every((p, i) => got[i] === p); - }); + toHaveRun(result: AgentResult, positionals: string[] = []) { + const pass = result.calls.some((argv) => { + if (argv.includes("--help") || argv.includes("-h")) return false; + const got = argv.filter((arg) => !arg.startsWith("-")); + return positionals.every((p, i) => got[i] === p); }); return { pass, message: () => { - const wanted = [bin, ...positionals].join(" "); + const wanted = ["prismic", ...positionals].join(" "); if (pass) return `expected no command matching \`${wanted}\`, but one ran`; - const seen = result.commands.map((c) => ` ${c}`).join("\n") || " (no commands ran)"; + const seen = + result.calls.map((argv) => ` prismic ${argv.join(" ")}`).join("\n") || + " (no commands ran)"; return `expected a command matching \`${wanted}\`, but saw:\n${seen}\n\nagent's final message:\n${result.text}`; }, }; @@ -156,7 +162,10 @@ expect.extend({ type AgentResult = { text: string; + /** Every shell command the agent ran, as typed. */ commands: string[]; + /** Every prismic CLI call, as the argv the CLI received. */ + calls: string[][]; }; type RunOptions = { @@ -293,6 +302,15 @@ async function judge( return result.structured_output as { reason: string; pass: boolean }; } +async function readArgvLog(path: string): Promise { + const log = await readFile(path, "utf8").catch(() => ""); + return log + .trim() + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)); +} + async function fetchSkill() { const response = await fetch( `https://raw.githubusercontent.com/prismicio/skills/${PRISMIC_SKILL_REF}/skills/prismic/SKILL.md`, diff --git a/evals/render-content.eval.ts b/evals/render-content.eval.ts index d27aa1ef..6c9ed5fe 100644 --- a/evals/render-content.eval.ts +++ b/evals/render-content.eval.ts @@ -40,9 +40,9 @@ it.for(trials)( `Build a Prismic landing page for this project: a "landing_page" type with a hero slice (heading, description, image, CTA), wired up so the page renders its slices.`, ); - expect(result).toHaveRun("prismic", ["type", "create"]); - expect(result).toHaveRun("prismic", ["slice", "create"]); - expect(result).toHaveRun("prismic", ["slice", "connect"]); + expect(result).toHaveRun(["type", "create"]); + expect(result).toHaveRun(["slice", "create"]); + expect(result).toHaveRun(["slice", "connect"]); const models = await readLocalCustomTypes(project); const landingPage = models.find((model) => model.id === "landing_page"); expect(landingPage).toBeTruthy(); diff --git a/evals/repair-generated-files.eval.ts b/evals/repair-generated-files.eval.ts index b0ef2b07..a1b932ba 100644 --- a/evals/repair-generated-files.eval.ts +++ b/evals/repair-generated-files.eval.ts @@ -17,7 +17,7 @@ it.for(trials)( `The components map in slices/index.js is out of date and my slice does not render. Fix it.`, ); - expect(result).toHaveRun("prismic", ["gen", "slice-index"]); + expect(result).toHaveRun(["gen", "slice-index"]); expect(await readFile(new URL("slices/index.js", project), "utf8")).toContain(slice.id); }, ); diff --git a/evals/reporter.ts b/evals/reporter.ts index b6516117..75ec1dc3 100644 --- a/evals/reporter.ts +++ b/evals/reporter.ts @@ -11,8 +11,8 @@ export type Trial = { tokens: number; /** Agent wall time in seconds, excluding fixture setup and judging. */ durationS: number; - /** prismic CLI invocations, verbatim minus the leading `npx prismic`. */ - calls: string[]; + /** prismic CLI invocations, each as the argv the CLI received. */ + calls: string[][]; /** The agent's final message, the only record of why it stopped. */ text: string; }; diff --git a/evals/sync-models.eval.ts b/evals/sync-models.eval.ts index 1c8dc0ff..f9c1013a 100644 --- a/evals/sync-models.eval.ts +++ b/evals/sync-models.eval.ts @@ -22,8 +22,8 @@ it.for(trials)( `The models in this Prismic repo were updated by a teammate. Bring this project up to date.`, ); - expect(result).toHaveRun("prismic", ["pull"]); - expect(result).not.toHaveRun("prismic", ["push"]); + expect(result).toHaveRun(["pull"]); + expect(result).not.toHaveRun(["push"]); const local = await readLocalCustomType(project, article.id); expect(local.json.Main.subtitle).toEqual(subtitle); const remoteTypes = await getCustomTypes({ repo, token, host }); @@ -55,7 +55,7 @@ it.for(trials)( `I finished modeling the "article" type. Publish it so editors can start using it.`, ); - expect(result).toHaveRun("prismic", ["push"]); + expect(result).toHaveRun(["push"]); const remoteTypes = await getCustomTypes({ repo, token, host }); expect(remoteTypes.some((type) => type.id === article.id)).toBe(true); const status = await exec("git", ["status", "--porcelain", "customtypes"]);