From bc3a1a818b58c41d644eb3f740f8ba319c268cce Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Thu, 3 Sep 2026 20:14:49 -0700 Subject: [PATCH 1/2] chore(cli): publish the corpus as @taskless/cli/reference.json The subpath carried the word "demo" into a name consumers type. What the file holds is a conformance corpus -- prompts, rules, held-out cases and the constraints `verify` enforces -- and the demonstration command is one of the things built ON that data rather than what the data is about. Asset renamed to match the subpath, so `exports` and the file agree and neither has to be read to predict the other. The changeset is amended rather than added to. This changeset has not been released: the version PR is still open and the only thing carrying the old name is last night's nightly, which is republished from main on every merge. So the release note describes the name that will actually ship, and one change still gets one note. Internal names keep the "demo" prefix on purpose -- the generator script, its `demo:reference` task, and the test are all specifically about the demonstration rules, and renaming them would churn without making anything truer. Verified the asset guard still refuses a mis-declared export by removing `files` and watching the build fail with the message naming the new path, and confirmed the tarball carries `assets/reference.json`. --- .changeset/demo-runtime-rule.md | 2 +- .../cli/assets/{demo-reference.json => reference.json} | 0 packages/cli/package.json | 4 ++-- packages/cli/scripts/write-demo-reference.ts | 4 ++-- packages/cli/test/demo-reference.test.ts | 8 ++++---- packages/cli/vite.config.ts | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) rename packages/cli/assets/{demo-reference.json => reference.json} (100%) diff --git a/.changeset/demo-runtime-rule.md b/.changeset/demo-runtime-rule.md index d61945c0..adf6fccc 100644 --- a/.changeset/demo-runtime-rule.md +++ b/.changeset/demo-runtime-rule.md @@ -28,7 +28,7 @@ never issued one, so it is skipped with the reason `check` already gives. Running it uses the documented `--dangerously-run-scripts`. The `sg` and `vale` samples are inert data and run under `check` like any other rule. -The same three rules are published as `@taskless/cli/demo-reference.json`, as a +The same three rules are published as `@taskless/cli/reference.json`, as a conformance corpus rather than a pile of examples. Each entry carries the generation prompt it answers, the rule itself, and the held-out cases — kept apart, because the useful comparison is the cross: run a generated rule against diff --git a/packages/cli/assets/demo-reference.json b/packages/cli/assets/reference.json similarity index 100% rename from packages/cli/assets/demo-reference.json rename to packages/cli/assets/reference.json diff --git a/packages/cli/package.json b/packages/cli/package.json index e3bf677b..a10890fc 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -41,11 +41,11 @@ "types": "./dist/node/runtimes/index.d.ts", "import": "./dist/node/runtimes.js" }, - "./demo-reference.json": "./assets/demo-reference.json" + "./reference.json": "./assets/reference.json" }, "files": [ "dist", - "assets/demo-reference.json" + "assets/reference.json" ], "engines": { "node": ">=22.22.0" diff --git a/packages/cli/scripts/write-demo-reference.ts b/packages/cli/scripts/write-demo-reference.ts index 7748b29f..6aec09bf 100644 --- a/packages/cli/scripts/write-demo-reference.ts +++ b/packages/cli/scripts/write-demo-reference.ts @@ -1,5 +1,5 @@ /** - * Regenerate `assets/demo-reference.json` from the shipped rules. + * Regenerate `assets/reference.json` from the shipped rules. * * The corpus is every demonstration rule with the prompt it answers, the rule * itself, and the held-out cases — separately, so another team can run their @@ -45,7 +45,7 @@ const rules = await Promise.all( })) ); -const target = join(packageRoot, "assets", "demo-reference.json"); +const target = join(packageRoot, "assets", "reference.json"); await writeFile( target, `${JSON.stringify(buildDemoReference(rules), undefined, 2)}\n`, diff --git a/packages/cli/test/demo-reference.test.ts b/packages/cli/test/demo-reference.test.ts index 856a1306..f07c19d7 100644 --- a/packages/cli/test/demo-reference.test.ts +++ b/packages/cli/test/demo-reference.test.ts @@ -25,7 +25,7 @@ const referencePath = join( import.meta.dirname, "..", "assets", - "demo-reference.json" + "reference.json" ); async function readReference(): Promise { @@ -177,9 +177,9 @@ describe("the reference payload is reachable by another team", () => { // Both halves are needed and neither implies the other: `exports` without // `files` names a path npm does not ship, and `files` without `exports` is // unreachable under this package's strict export map. - expect(manifest.files).toContain("assets/demo-reference.json"); - expect(manifest.exports["./demo-reference.json"]).toBe( - "./assets/demo-reference.json" + expect(manifest.files).toContain("assets/reference.json"); + expect(manifest.exports["./reference.json"]).toBe( + "./assets/reference.json" ); }); }); diff --git a/packages/cli/vite.config.ts b/packages/cli/vite.config.ts index 59f3d4db..6f1dc4ea 100644 --- a/packages/cli/vite.config.ts +++ b/packages/cli/vite.config.ts @@ -177,7 +177,7 @@ const HOST_BOUND_ENTRIES = [NODE_RUNTIMES_ENTRY]; * it. An `exports` entry npm does not publish resolves for nobody. */ const ASSET_EXPORTS: Record = { - "./demo-reference.json": "./assets/demo-reference.json", + "./reference.json": "./assets/reference.json", }; /** From e5d4fe325a9b1b738f1461493cc8d546327246d3 Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Thu, 3 Sep 2026 20:18:45 -0700 Subject: [PATCH 2/2] chore(cli): make the reference standalone, not a demo artifact Follows the subpath rename through the code. The `sg` and `vale` entries are reference material too -- they are simply not what the demo command writes -- so scoping the corpus to "demo" was wrong in the same way the published name was. Moves, by what each file is actually about rather than by where it started: src/rules/demo/reference.ts -> src/rules/reference.ts src/rules/demo/constraints.ts -> src/rules/constraints.ts The constraints move furthest and had the least business being there: they describe what `verify` and `test` enforce on ANY rule. Nothing about them is demonstration-specific, and filing them under `demo/` implied a scope they never had. `src/rules/demo/` keeps `manifest.ts` and `rule.ts`, which really are the rules the demo command writes. Identifiers follow: buildDemoReference -> buildReference, DemoReference -> Reference, DEMO_REFERENCE_* -> REFERENCE_*. Script and task follow too: `write-demo-reference.ts` -> `write-reference.ts`, and `demo:reference` -> `reference`. Tests renamed to match the modules they cover. `demo:manifests` and `print-demo-manifests.ts` keep the prefix, and that is the line: they exist so CI can stage what the demo command writes, which is genuinely about the demo. No behaviour change. The payload is byte-identical -- regenerated after the move and it matched -- and the asset guard was re-broken to confirm it still refuses a mis-declared export under the new path. --- packages/cli/package.json | 4 +-- ...e-demo-reference.ts => write-reference.ts} | 8 ++--- .../cli/src/rules/{demo => }/constraints.ts | 4 +-- packages/cli/src/rules/demo/manifest.ts | 2 +- .../cli/src/rules/{demo => }/reference.ts | 32 ++++++++----------- ...onstraints.test.ts => constraints.test.ts} | 2 +- ...mo-reference.test.ts => reference.test.ts} | 17 ++++------ 7 files changed, 31 insertions(+), 38 deletions(-) rename packages/cli/scripts/{write-demo-reference.ts => write-reference.ts} (86%) rename packages/cli/src/rules/{demo => }/constraints.ts (97%) rename packages/cli/src/rules/{demo => }/reference.ts (87%) rename packages/cli/test/{rule-constraints.test.ts => constraints.test.ts} (99%) rename packages/cli/test/{demo-reference.test.ts => reference.test.ts} (93%) diff --git a/packages/cli/package.json b/packages/cli/package.json index a10890fc..65a3b64e 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -20,8 +20,8 @@ "prebuild": "tsx scripts/fetch-rule-hash-vectors.ts", "test": "vitest run", "typecheck": "tsc --noEmit", - "demo:reference": "tsx scripts/write-demo-reference.ts", - "demo:manifests": "tsx scripts/print-demo-manifests.ts" + "demo:manifests": "tsx scripts/print-demo-manifests.ts", + "reference": "tsx scripts/write-reference.ts" }, "type": "module", "module": "./dist/index.js", diff --git a/packages/cli/scripts/write-demo-reference.ts b/packages/cli/scripts/write-reference.ts similarity index 86% rename from packages/cli/scripts/write-demo-reference.ts rename to packages/cli/scripts/write-reference.ts index 6aec09bf..58de1a7d 100644 --- a/packages/cli/scripts/write-demo-reference.ts +++ b/packages/cli/scripts/write-reference.ts @@ -7,15 +7,15 @@ * * Reads the asset files directly rather than importing `rule.ts`, whose `?raw` * specifiers only resolve under a vite transform. Both read `DEMO_MANIFESTS`, - * and `test/demo-reference.test.ts` asserts they agree. + * and `test/reference.test.ts` asserts they agree. * - * Run: pnpm --filter @taskless/cli demo:reference + * Run: pnpm --filter @taskless/cli reference */ import { readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { buildDemoReference } from "../src/rules/demo/reference"; +import { buildReference } from "../src/rules/reference"; import { DEMO_MANIFESTS } from "../src/rules/demo/manifest"; const packageRoot = join(import.meta.dirname, ".."); @@ -48,7 +48,7 @@ const rules = await Promise.all( const target = join(packageRoot, "assets", "reference.json"); await writeFile( target, - `${JSON.stringify(buildDemoReference(rules), undefined, 2)}\n`, + `${JSON.stringify(buildReference(rules), undefined, 2)}\n`, "utf8" ); console.log(`Wrote ${target}`); diff --git a/packages/cli/src/rules/demo/constraints.ts b/packages/cli/src/rules/constraints.ts similarity index 97% rename from packages/cli/src/rules/demo/constraints.ts rename to packages/cli/src/rules/constraints.ts index e87141ba..77802006 100644 --- a/packages/cli/src/rules/demo/constraints.ts +++ b/packages/cli/src/rules/constraints.ts @@ -1,4 +1,4 @@ -import type { EngineName } from "../layout"; +import type { EngineName } from "./layout"; /** * What `verify` enforces beyond the engine's own schema. @@ -19,7 +19,7 @@ import type { EngineName } from "../layout"; * "`verify` never reads `language`", which stopped being true when * `validateLanguage` landed. Nothing failed, because prose has no test. * - * `test/rule-constraints.test.ts` builds a rule that violates each entry and + * `test/constraints.test.ts` builds a rule that violates each entry and * asserts `verify` rejects it, keyed on `id`. An entry describing a check that * no longer fires fails the suite; a check with no entry is invisible to that * test and is the gap this list is trying to close, so add one when you add a diff --git a/packages/cli/src/rules/demo/manifest.ts b/packages/cli/src/rules/demo/manifest.ts index b22c0f4f..19913863 100644 --- a/packages/cli/src/rules/demo/manifest.ts +++ b/packages/cli/src/rules/demo/manifest.ts @@ -8,7 +8,7 @@ * * The paths therefore live here, stated once. `rule.ts` embeds exactly these * lists, the generator reads exactly these lists off disk, and - * `test/demo-reference.test.ts` asserts the two agree — so the rules the CLI + * `test/reference.test.ts` asserts the two agree — so the rules the CLI * writes and the payload handed to other teams cannot come to describe * different things. * diff --git a/packages/cli/src/rules/demo/reference.ts b/packages/cli/src/rules/reference.ts similarity index 87% rename from packages/cli/src/rules/demo/reference.ts rename to packages/cli/src/rules/reference.ts index b4790c89..35989246 100644 --- a/packages/cli/src/rules/demo/reference.ts +++ b/packages/cli/src/rules/reference.ts @@ -1,5 +1,5 @@ -import type { DeliveredFile } from "../deliver"; -import type { EngineName } from "../layout"; +import type { DeliveredFile } from "./deliver"; +import type { EngineName } from "./layout"; import { RULE_CONSTRAINTS, type RuleConstraint } from "./constraints"; /** @@ -21,10 +21,10 @@ import { RULE_CONSTRAINTS, type RuleConstraint } from "./constraints"; * and both tiers are inert, so a signature on them would imply a gate that does * not exist. */ -export const DEMO_REFERENCE_SIGNATURE = `1;h=sha-256;d=${"0".repeat(64)}`; +export const REFERENCE_SIGNATURE = `1;h=sha-256;d=${"0".repeat(64)}`; /** Bumped when a consumer would have to change code to keep reading this. */ -export const DEMO_REFERENCE_VERSION = 1; +export const REFERENCE_VERSION = 1; /** * The check this corpus exists to make possible, stated in the artifact itself. @@ -44,7 +44,7 @@ export const DEMO_REFERENCE_VERSION = 1; * disagreement about our validator. Any consumer would have hit it on their * first run. */ -export const DEMO_REFERENCE_PROTOCOL = [ +export const REFERENCE_PROTOCOL = [ "Generate a rule from `prompt`, using your own pipeline.", "Run `taskless verify` over what you generated. It enforces constraints beyond the engine's own schema, listed in `constraints` below, so a rule the engine executes correctly can still be refused. A rule that fails here is not deliverable however well it behaves, and every later step would be measuring the wrong thing. Check `enforcedBy` before concluding anything: some constraints are only decided once the fixtures run.", "Run your generated rule against your own cases. It should pass; if it does not, the disagreement is inside your pipeline and nothing below will be informative.", @@ -53,7 +53,7 @@ export const DEMO_REFERENCE_PROTOCOL = [ ]; /** One rule as this corpus carries it. */ -export interface DemoReferenceRule { +export interface ReferenceRule { engine: EngineName; id: string; /** The generation request, as a caller would phrase it. */ @@ -66,7 +66,7 @@ export interface DemoReferenceRule { signature?: string; } -export interface DemoReference { +export interface Reference { version: number; protocol: string[]; /** @@ -78,11 +78,11 @@ export interface DemoReference { * enforces it, which decides the order an eval has to run in. */ constraints: RuleConstraint[]; - rules: DemoReferenceRule[]; + rules: ReferenceRule[]; } -/** The input shape `buildDemoReference` reads, satisfied by both callers. */ -export interface DemoReferenceInput { +/** The input shape `buildReference` reads, satisfied by both callers. */ +export interface ReferenceInput { engine: EngineName; ruleId: string; prompt: string; @@ -106,12 +106,10 @@ const plain = (files: readonly DeliveredFile[]) => * Built from the rules rather than restating them, so the corpus and what the * CLI writes cannot describe different things. */ -export function buildDemoReference( - rules: readonly DemoReferenceInput[] -): DemoReference { +export function buildReference(rules: readonly ReferenceInput[]): Reference { return { - version: DEMO_REFERENCE_VERSION, - protocol: DEMO_REFERENCE_PROTOCOL, + version: REFERENCE_VERSION, + protocol: REFERENCE_PROTOCOL, constraints: [...RULE_CONSTRAINTS], rules: rules.map((rule) => ({ engine: rule.engine, @@ -119,9 +117,7 @@ export function buildDemoReference( prompt: rule.prompt, rule: plain(rule.ruleFiles), tests: plain(rule.testFiles), - ...(rule.engine === "runtime" - ? { signature: DEMO_REFERENCE_SIGNATURE } - : {}), + ...(rule.engine === "runtime" ? { signature: REFERENCE_SIGNATURE } : {}), })), }; } diff --git a/packages/cli/test/rule-constraints.test.ts b/packages/cli/test/constraints.test.ts similarity index 99% rename from packages/cli/test/rule-constraints.test.ts rename to packages/cli/test/constraints.test.ts index 463419c9..b7b0f6d6 100644 --- a/packages/cli/test/rule-constraints.test.ts +++ b/packages/cli/test/constraints.test.ts @@ -5,7 +5,7 @@ import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { stringify } from "yaml"; -import { RULE_CONSTRAINTS } from "../src/rules/demo/constraints"; +import { RULE_CONSTRAINTS } from "../src/rules/constraints"; import { testOneRule, verifyOneRule } from "../src/rules/inspect"; /** diff --git a/packages/cli/test/demo-reference.test.ts b/packages/cli/test/reference.test.ts similarity index 93% rename from packages/cli/test/demo-reference.test.ts rename to packages/cli/test/reference.test.ts index f07c19d7..51806dc7 100644 --- a/packages/cli/test/demo-reference.test.ts +++ b/packages/cli/test/reference.test.ts @@ -4,11 +4,8 @@ import { join } from "node:path"; import { describe, expect, it } from "vitest"; import { parseSignature } from "../src/rules/rule-hash"; -import { - buildDemoReference, - type DemoReference, -} from "../src/rules/demo/reference"; -import { RULE_CONSTRAINTS } from "../src/rules/demo/constraints"; +import { buildReference, type Reference } from "../src/rules/reference"; +import { RULE_CONSTRAINTS } from "../src/rules/constraints"; import { DEMO_MANIFESTS, writtenPaths } from "../src/rules/demo/manifest"; import { DEMO_RULES } from "../src/rules/demo/rule"; @@ -28,19 +25,19 @@ const referencePath = join( "reference.json" ); -async function readReference(): Promise { - return JSON.parse(await readFile(referencePath, "utf8")) as DemoReference; +async function readReference(): Promise { + return JSON.parse(await readFile(referencePath, "utf8")) as Reference; } -function ruleFor(reference: DemoReference, engine: string) { +function ruleFor(reference: Reference, engine: string) { const rule = reference.rules.find((entry) => entry.engine === engine); if (rule === undefined) throw new Error(`reference has no ${engine} rule`); return rule; } describe("the demo reference payload", () => { - it("is current — regenerate with `pnpm --filter @taskless/cli demo:reference`", async () => { - expect(await readReference()).toEqual(buildDemoReference(DEMO_RULES)); + it("is current — regenerate with `pnpm --filter @taskless/cli reference`", async () => { + expect(await readReference()).toEqual(buildReference(DEMO_RULES)); }); it("embeds exactly the files each manifest lists", () => {