diff --git a/.changeset/demo-runtime-rule.md b/.changeset/demo-runtime-rule.md index d61945c0..adf6fccc 100644 --- a/.changeset/demo-runtime-rule.md +++ b/.changeset/demo-runtime-rule.md @@ -28,7 +28,7 @@ never issued one, so it is skipped with the reason `check` already gives. Running it uses the documented `--dangerously-run-scripts`. The `sg` and `vale` samples are inert data and run under `check` like any other rule. -The same three rules are published as `@taskless/cli/demo-reference.json`, as a +The same three rules are published as `@taskless/cli/reference.json`, as a conformance corpus rather than a pile of examples. Each entry carries the generation prompt it answers, the rule itself, and the held-out cases — kept apart, because the useful comparison is the cross: run a generated rule against diff --git a/packages/cli/assets/demo-reference.json b/packages/cli/assets/reference.json similarity index 100% rename from packages/cli/assets/demo-reference.json rename to packages/cli/assets/reference.json diff --git a/packages/cli/package.json b/packages/cli/package.json index e3bf677b..65a3b64e 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -20,8 +20,8 @@ "prebuild": "tsx scripts/fetch-rule-hash-vectors.ts", "test": "vitest run", "typecheck": "tsc --noEmit", - "demo:reference": "tsx scripts/write-demo-reference.ts", - "demo:manifests": "tsx scripts/print-demo-manifests.ts" + "demo:manifests": "tsx scripts/print-demo-manifests.ts", + "reference": "tsx scripts/write-reference.ts" }, "type": "module", "module": "./dist/index.js", @@ -41,11 +41,11 @@ "types": "./dist/node/runtimes/index.d.ts", "import": "./dist/node/runtimes.js" }, - "./demo-reference.json": "./assets/demo-reference.json" + "./reference.json": "./assets/reference.json" }, "files": [ "dist", - "assets/demo-reference.json" + "assets/reference.json" ], "engines": { "node": ">=22.22.0" diff --git a/packages/cli/scripts/write-demo-reference.ts b/packages/cli/scripts/write-reference.ts similarity index 78% rename from packages/cli/scripts/write-demo-reference.ts rename to packages/cli/scripts/write-reference.ts index 7748b29f..58de1a7d 100644 --- a/packages/cli/scripts/write-demo-reference.ts +++ b/packages/cli/scripts/write-reference.ts @@ -1,5 +1,5 @@ /** - * Regenerate `assets/demo-reference.json` from the shipped rules. + * Regenerate `assets/reference.json` from the shipped rules. * * The corpus is every demonstration rule with the prompt it answers, the rule * itself, and the held-out cases — separately, so another team can run their @@ -7,15 +7,15 @@ * * Reads the asset files directly rather than importing `rule.ts`, whose `?raw` * specifiers only resolve under a vite transform. Both read `DEMO_MANIFESTS`, - * and `test/demo-reference.test.ts` asserts they agree. + * and `test/reference.test.ts` asserts they agree. * - * Run: pnpm --filter @taskless/cli demo:reference + * Run: pnpm --filter @taskless/cli reference */ import { readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { buildDemoReference } from "../src/rules/demo/reference"; +import { buildReference } from "../src/rules/reference"; import { DEMO_MANIFESTS } from "../src/rules/demo/manifest"; const packageRoot = join(import.meta.dirname, ".."); @@ -45,10 +45,10 @@ const rules = await Promise.all( })) ); -const target = join(packageRoot, "assets", "demo-reference.json"); +const target = join(packageRoot, "assets", "reference.json"); await writeFile( target, - `${JSON.stringify(buildDemoReference(rules), undefined, 2)}\n`, + `${JSON.stringify(buildReference(rules), undefined, 2)}\n`, "utf8" ); console.log(`Wrote ${target}`); diff --git a/packages/cli/src/rules/demo/constraints.ts b/packages/cli/src/rules/constraints.ts similarity index 97% rename from packages/cli/src/rules/demo/constraints.ts rename to packages/cli/src/rules/constraints.ts index e87141ba..77802006 100644 --- a/packages/cli/src/rules/demo/constraints.ts +++ b/packages/cli/src/rules/constraints.ts @@ -1,4 +1,4 @@ -import type { EngineName } from "../layout"; +import type { EngineName } from "./layout"; /** * What `verify` enforces beyond the engine's own schema. @@ -19,7 +19,7 @@ import type { EngineName } from "../layout"; * "`verify` never reads `language`", which stopped being true when * `validateLanguage` landed. Nothing failed, because prose has no test. * - * `test/rule-constraints.test.ts` builds a rule that violates each entry and + * `test/constraints.test.ts` builds a rule that violates each entry and * asserts `verify` rejects it, keyed on `id`. An entry describing a check that * no longer fires fails the suite; a check with no entry is invisible to that * test and is the gap this list is trying to close, so add one when you add a diff --git a/packages/cli/src/rules/demo/manifest.ts b/packages/cli/src/rules/demo/manifest.ts index b22c0f4f..19913863 100644 --- a/packages/cli/src/rules/demo/manifest.ts +++ b/packages/cli/src/rules/demo/manifest.ts @@ -8,7 +8,7 @@ * * The paths therefore live here, stated once. `rule.ts` embeds exactly these * lists, the generator reads exactly these lists off disk, and - * `test/demo-reference.test.ts` asserts the two agree — so the rules the CLI + * `test/reference.test.ts` asserts the two agree — so the rules the CLI * writes and the payload handed to other teams cannot come to describe * different things. * diff --git a/packages/cli/src/rules/demo/reference.ts b/packages/cli/src/rules/reference.ts similarity index 87% rename from packages/cli/src/rules/demo/reference.ts rename to packages/cli/src/rules/reference.ts index b4790c89..35989246 100644 --- a/packages/cli/src/rules/demo/reference.ts +++ b/packages/cli/src/rules/reference.ts @@ -1,5 +1,5 @@ -import type { DeliveredFile } from "../deliver"; -import type { EngineName } from "../layout"; +import type { DeliveredFile } from "./deliver"; +import type { EngineName } from "./layout"; import { RULE_CONSTRAINTS, type RuleConstraint } from "./constraints"; /** @@ -21,10 +21,10 @@ import { RULE_CONSTRAINTS, type RuleConstraint } from "./constraints"; * and both tiers are inert, so a signature on them would imply a gate that does * not exist. */ -export const DEMO_REFERENCE_SIGNATURE = `1;h=sha-256;d=${"0".repeat(64)}`; +export const REFERENCE_SIGNATURE = `1;h=sha-256;d=${"0".repeat(64)}`; /** Bumped when a consumer would have to change code to keep reading this. */ -export const DEMO_REFERENCE_VERSION = 1; +export const REFERENCE_VERSION = 1; /** * The check this corpus exists to make possible, stated in the artifact itself. @@ -44,7 +44,7 @@ export const DEMO_REFERENCE_VERSION = 1; * disagreement about our validator. Any consumer would have hit it on their * first run. */ -export const DEMO_REFERENCE_PROTOCOL = [ +export const REFERENCE_PROTOCOL = [ "Generate a rule from `prompt`, using your own pipeline.", "Run `taskless verify` over what you generated. It enforces constraints beyond the engine's own schema, listed in `constraints` below, so a rule the engine executes correctly can still be refused. A rule that fails here is not deliverable however well it behaves, and every later step would be measuring the wrong thing. Check `enforcedBy` before concluding anything: some constraints are only decided once the fixtures run.", "Run your generated rule against your own cases. It should pass; if it does not, the disagreement is inside your pipeline and nothing below will be informative.", @@ -53,7 +53,7 @@ export const DEMO_REFERENCE_PROTOCOL = [ ]; /** One rule as this corpus carries it. */ -export interface DemoReferenceRule { +export interface ReferenceRule { engine: EngineName; id: string; /** The generation request, as a caller would phrase it. */ @@ -66,7 +66,7 @@ export interface DemoReferenceRule { signature?: string; } -export interface DemoReference { +export interface Reference { version: number; protocol: string[]; /** @@ -78,11 +78,11 @@ export interface DemoReference { * enforces it, which decides the order an eval has to run in. */ constraints: RuleConstraint[]; - rules: DemoReferenceRule[]; + rules: ReferenceRule[]; } -/** The input shape `buildDemoReference` reads, satisfied by both callers. */ -export interface DemoReferenceInput { +/** The input shape `buildReference` reads, satisfied by both callers. */ +export interface ReferenceInput { engine: EngineName; ruleId: string; prompt: string; @@ -106,12 +106,10 @@ const plain = (files: readonly DeliveredFile[]) => * Built from the rules rather than restating them, so the corpus and what the * CLI writes cannot describe different things. */ -export function buildDemoReference( - rules: readonly DemoReferenceInput[] -): DemoReference { +export function buildReference(rules: readonly ReferenceInput[]): Reference { return { - version: DEMO_REFERENCE_VERSION, - protocol: DEMO_REFERENCE_PROTOCOL, + version: REFERENCE_VERSION, + protocol: REFERENCE_PROTOCOL, constraints: [...RULE_CONSTRAINTS], rules: rules.map((rule) => ({ engine: rule.engine, @@ -119,9 +117,7 @@ export function buildDemoReference( prompt: rule.prompt, rule: plain(rule.ruleFiles), tests: plain(rule.testFiles), - ...(rule.engine === "runtime" - ? { signature: DEMO_REFERENCE_SIGNATURE } - : {}), + ...(rule.engine === "runtime" ? { signature: REFERENCE_SIGNATURE } : {}), })), }; } diff --git a/packages/cli/test/rule-constraints.test.ts b/packages/cli/test/constraints.test.ts similarity index 99% rename from packages/cli/test/rule-constraints.test.ts rename to packages/cli/test/constraints.test.ts index 463419c9..b7b0f6d6 100644 --- a/packages/cli/test/rule-constraints.test.ts +++ b/packages/cli/test/constraints.test.ts @@ -5,7 +5,7 @@ import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { stringify } from "yaml"; -import { RULE_CONSTRAINTS } from "../src/rules/demo/constraints"; +import { RULE_CONSTRAINTS } from "../src/rules/constraints"; import { testOneRule, verifyOneRule } from "../src/rules/inspect"; /** diff --git a/packages/cli/test/demo-reference.test.ts b/packages/cli/test/reference.test.ts similarity index 91% rename from packages/cli/test/demo-reference.test.ts rename to packages/cli/test/reference.test.ts index 856a1306..51806dc7 100644 --- a/packages/cli/test/demo-reference.test.ts +++ b/packages/cli/test/reference.test.ts @@ -4,11 +4,8 @@ import { join } from "node:path"; import { describe, expect, it } from "vitest"; import { parseSignature } from "../src/rules/rule-hash"; -import { - buildDemoReference, - type DemoReference, -} from "../src/rules/demo/reference"; -import { RULE_CONSTRAINTS } from "../src/rules/demo/constraints"; +import { buildReference, type Reference } from "../src/rules/reference"; +import { RULE_CONSTRAINTS } from "../src/rules/constraints"; import { DEMO_MANIFESTS, writtenPaths } from "../src/rules/demo/manifest"; import { DEMO_RULES } from "../src/rules/demo/rule"; @@ -25,22 +22,22 @@ const referencePath = join( import.meta.dirname, "..", "assets", - "demo-reference.json" + "reference.json" ); -async function readReference(): Promise { - return JSON.parse(await readFile(referencePath, "utf8")) as DemoReference; +async function readReference(): Promise { + return JSON.parse(await readFile(referencePath, "utf8")) as Reference; } -function ruleFor(reference: DemoReference, engine: string) { +function ruleFor(reference: Reference, engine: string) { const rule = reference.rules.find((entry) => entry.engine === engine); if (rule === undefined) throw new Error(`reference has no ${engine} rule`); return rule; } describe("the demo reference payload", () => { - it("is current — regenerate with `pnpm --filter @taskless/cli demo:reference`", async () => { - expect(await readReference()).toEqual(buildDemoReference(DEMO_RULES)); + it("is current — regenerate with `pnpm --filter @taskless/cli reference`", async () => { + expect(await readReference()).toEqual(buildReference(DEMO_RULES)); }); it("embeds exactly the files each manifest lists", () => { @@ -177,9 +174,9 @@ describe("the reference payload is reachable by another team", () => { // Both halves are needed and neither implies the other: `exports` without // `files` names a path npm does not ship, and `files` without `exports` is // unreachable under this package's strict export map. - expect(manifest.files).toContain("assets/demo-reference.json"); - expect(manifest.exports["./demo-reference.json"]).toBe( - "./assets/demo-reference.json" + expect(manifest.files).toContain("assets/reference.json"); + expect(manifest.exports["./reference.json"]).toBe( + "./assets/reference.json" ); }); }); diff --git a/packages/cli/vite.config.ts b/packages/cli/vite.config.ts index 59f3d4db..6f1dc4ea 100644 --- a/packages/cli/vite.config.ts +++ b/packages/cli/vite.config.ts @@ -177,7 +177,7 @@ const HOST_BOUND_ENTRIES = [NODE_RUNTIMES_ENTRY]; * it. An `exports` entry npm does not publish resolves for nobody. */ const ASSET_EXPORTS: Record = { - "./demo-reference.json": "./assets/demo-reference.json", + "./reference.json": "./assets/reference.json", }; /**