test(run-engine): bind each waitpoint mint site to its anchor, and make the census site-granular

An adversarial review found the census guard was file-granular where the
requirement is site-granular, and that no test bound a create site to its
anchor. Both were real: a fifth mint added inside an already-catalogued file
passed, and swapping any site's anchor for undefined passed every test on the
branch while silently reverting that site to a cuid.

The catalog now records the exact mint expression per site, and the proof test
counts each one per file. It walks the whole engine tree rather than the
coordinator directory alone, so a mint moved back into systems/ -- where they
all lived before the coordinator seam -- is visible. Test-support trees are
excluded explicitly, since a helper writing through raw Prisma never reaches
the routing store. Both holes were confirmed closed by reintroducing them and
watching the guard fail.

The site tests now drive the real create sites through a capturing run store
rather than calling the mint helper with a hand-written literal, including the
standalone-token arms and the precedence of an owning run over the
environment shard.

Also: deletes a test that duplicated another file while claiming to guard the
failed-run path it never imported; adds the missing gen-2 batch-anchor case
for batch items; corrects the standaloneShardKey contract text, which stated a
rule its only caller does not follow; and corrects the BATCH comment, which
claimed stamping from the run "would throw" when on the normal path both
stamps agree and it would not.

Author: d-cs

apps/webapp/app/v3/runOpsMigration/mintAnchoredRunFriendlyId.server.test.ts

@@ -28,4 +28,16 @@ describe("mintAnchoredRunFriendlyId", () => {
     expect(parsed.format).toBe("b32hex");
     expect(parsed.format === "b32hex" && parsed.region).toBe(REGION_CODES["us-east-1"]);
   });
+
+  it("a gen-2 batch anchor mints an item on the batch's shard", () => {
+    const body = mintAnchoredRunFriendlyId(`batch_${"a".repeat(24)}a2`).slice("run_".length);
+    expect(body).toHaveLength(26);
+    expect(body[24]).toBe("a");
+    expect(body[25]).toBe("2");
+  });
+
+  it("a gen-2 batch anchor ignores a caller region: the shard owns index 24", () => {
+    const body = mintAnchoredRunFriendlyId(`batch_${"a".repeat(24)}a2`, "us-east-1").slice(4);
+    expect(body[24]).toBe("a");
+  });
 });

apps/webapp/test/engine/gen2ChildMintInheritance.test.ts

@@ -1824,10 +1824,10 @@ export class RunEngine {
     standaloneResidency?: "NEW" | "LEGACY";
     /**
      * The environment's mint shard, for a STANDALONE token with no owning run. It selects the
-     * shard the token's id is stamped for. When it names a gen-2 shard the caller must NOT also
-     * set `standaloneResidency`: a residency hint outranks the id shape in the router and can
-     * only name a gen-1 store, so the row would land there while its completion routes to the
-     * shard. Only a Postgres implementation reads this.
+     * shard the token's id is stamped for. When it names a gen-2 shard the implementation must
+     * IGNORE `standaloneResidency`: a residency hint outranks the id shape in the router and
+     * can only name a gen-1 store, so honouring it would land the row there while its
+     * completion routes to the shard. Only a Postgres implementation reads this.
      */
     standaloneShardKey?: ShardKey;
   }): Promise<{ waitpoint: Waitpoint; isCached: boolean }> {
@@ -1868,8 +1868,10 @@ export class RunEngine {
           data: {
             // Stamped from the BATCH, not the blocked run: this create passes only
             // completedByBatchId, so the routing store resolves the owner from the batch and
-            // validates the stamp against the batch's shard. The two match, because the batch
-            // inherited this run's shard when it was minted.
+            // validates the stamp against the BATCH's shard. On the normal path the two are
+            // the same char anyway, because the batch inherited this run's shard. They differ
+            // only if a batch ever blocks a run from another shard -- and then this is the
+            // stamp that matches the owner the router actually checks.
             ...mintWaitpointIdFor(batchId),
             type: "BATCH",
             idempotencyKey: batchId,

internal-packages/run-engine/src/engine/index.ts

@@ -200,10 +200,10 @@ export class WaitpointSystem {
     standaloneResidency?: "NEW" | "LEGACY";
     /**
      * The environment's mint shard, for a STANDALONE token with no owning run. It selects the
-     * shard the token's id is stamped for. When it names a gen-2 shard the caller must NOT also
-     * set `standaloneResidency`: a residency hint outranks the id shape in the router and can
-     * only name a gen-1 store, so the row would land there while its completion routes to the
-     * shard. Only a Postgres implementation reads this.
+     * shard the token's id is stamped for. When it names a gen-2 shard the implementation must
+     * IGNORE `standaloneResidency`: a residency hint outranks the id shape in the router and
+     * can only name a gen-1 store, so honouring it would land the row there while its
+     * completion routes to the shard. Only a Postgres implementation reads this.
      */
     standaloneShardKey?: ShardKey;
   }): Promise<{ waitpoint: Waitpoint; isCached: boolean }> {

internal-packages/run-engine/src/engine/systems/waitpointSystem.ts

@@ -138,10 +138,10 @@ export type CreateManualWaitpointParams = {
   standaloneResidency?: "NEW" | "LEGACY";
   /**
    * The environment's mint shard, for a STANDALONE token with no owning run. It selects the
-   * shard the token's id is stamped for. When it names a gen-2 shard the caller must NOT also
-   * set `standaloneResidency`: a residency hint outranks the id shape in the router and can
-   * only name a gen-1 store, so the row would land there while its completion routes to the
-   * shard. Only a Postgres implementation reads this.
+   * shard the token's id is stamped for. When it names a gen-2 shard the implementation must
+   * IGNORE `standaloneResidency`: a residency hint outranks the id shape in the router and
+   * can only name a gen-1 store, so honouring it would land the row there while its
+   * completion routes to the shard. Only a Postgres implementation reads this.
    */
   standaloneShardKey?: ShardKey;
 };

internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts

@@ -1,12 +1,8 @@
-import { existsSync, readdirSync, readFileSync } from "node:fs";
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
 import path from "node:path";
 import { describe, expect, it } from "vitest";
-import { mintWaitpointIdFor } from "@trigger.dev/core/v3/isomorphic";
 import { WAITPOINT_MINT_SITES } from "./waitpointMintCatalog";
 
-const GEN2_ANCHOR = `${"a".repeat(24)}a2`;
-const GEN1_ANCHOR = `${"a".repeat(24)}01`;
-
 function repoRoot(): string {
   let dir = process.cwd();
   while (!existsSync(path.join(dir, "pnpm-workspace.yaml"))) {
@@ -25,63 +21,92 @@ function count(source: string, pattern: RegExp): number {
   return (source.match(pattern) ?? []).length;
 }
 
-// Every source file that may create a Postgres waitpoint row. The coordinator directory is
-// WALKED rather than listed, so a mint added in a new coordinator file cannot hide here.
-function scannedFiles(): string[] {
-  const coordinatorDir = "internal-packages/run-engine/src/engine/waitpointCoordinator";
-  const walked = readdirSync(path.join(repoRoot(), coordinatorDir))
-    .filter((name) => name.endsWith(".ts") && !name.includes(".test."))
-    .map((name) => `${coordinatorDir}/${name}`);
-
-  return [
-    ...walked,
-    "internal-packages/run-engine/src/engine/index.ts",
-    "internal-packages/run-store/src/PostgresRunStore.ts",
-  ];
+// Every production `.ts` under a root, walked rather than listed: a mint added in a new
+// file, or moved back into `systems/` where these all lived until the coordinator seam was
+// extracted, has to be visible here or the census is decorative.
+//
+// Test-support trees are excluded deliberately. A helper that writes a row through raw
+// Prisma never reaches the routing store, so it cannot misroute; requiring it to be
+// catalogued would fill the census with sites that carry no risk.
+const TEST_SUPPORT_DIRS = new Set(["tests", "__tests__", "fixtures"]);
+
+function walk(relativeRoot: string): string[] {
+  const absolute = path.join(repoRoot(), relativeRoot);
+  return readdirSync(absolute).flatMap((name) => {
+    const child = `${relativeRoot}/${name}`;
+    if (statSync(path.join(absolute, name)).isDirectory()) {
+      return TEST_SUPPORT_DIRS.has(name) ? [] : walk(child);
+    }
+    return name.endsWith(".ts") && !name.includes(".test.") ? [child] : [];
+  });
 }
 
-describe("waitpoint mint census — behaviour per catalogued site", () => {
-  for (const site of WAITPOINT_MINT_SITES) {
-    it(`${site.id} (${site.type}) stamps a gen-2 anchor's shard char`, () => {
-      const r = mintWaitpointIdFor(GEN2_ANCHOR);
-      expect(r.id[24]).toBe("a");
-      expect(r.id[25]).toBe("2");
-    });
-
-    it(`${site.id} (${site.type}) keeps a cuid for a gen-1 anchor`, () => {
-      expect(mintWaitpointIdFor(GEN1_ANCHOR).id.length).toBe(25);
-    });
+// The mint helpers are the only sanctioned way to produce a Postgres waitpoint id.
+const MINT_CALL = /mintWaitpointIdFor(?:Shard)?\(/g;
+const UNSTAMPED_MINT = /WaitpointId\.generate\(/g;
+const WAITPOINT_WRITE = /waitpoint\.create\(|upsertWaitpoint\(|createWaitpoint\(/g;
+
+// The catalog holds the mint expressions as string data, so scanning it would count them.
+const CATALOG_ITSELF =
+  "internal-packages/run-engine/src/engine/waitpointCoordinator/waitpointMintCatalog.ts";
+
+const ENGINE_SOURCES = walk("internal-packages/run-engine/src/engine").filter(
+  (f) => f !== CATALOG_ITSELF
+);
+const SCANNED = [...ENGINE_SOURCES, "internal-packages/run-store/src/PostgresRunStore.ts"];
+
+// expression -> how many times the catalog says it appears in this file
+function expectedMints(file: string): Map<string, number> {
+  const expected = new Map<string, number>();
+  for (const site of WAITPOINT_MINT_SITES.filter((s) => s.site === file)) {
+    for (const expr of site.mints) {
+      expected.set(expr, (expected.get(expr) ?? 0) + 1);
+    }
   }
-});
+  return expected;
+}
 
-describe("waitpoint mint census — source drift guard", () => {
-  it("no scanned source mints a waitpoint id with the un-stamped helper", () => {
-    // The regex matches tokens inside comments too — deliberate. Any textual addition
-    // forces the census to be reconciled, so a new site cannot land without an entry.
-    for (const file of scannedFiles()) {
-      expect({ file, hits: count(read(file), /WaitpointId\.generate\(/g) }).toEqual({
-        file,
-        hits: 0,
+function escapeRegExp(value: string): string {
+  return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+describe("waitpoint mint census — the catalog matches the source", () => {
+  it("scans the engine tree and the run-store writer, and finds files to scan", () => {
+    expect(ENGINE_SOURCES.length).toBeGreaterThan(10);
+    expect(SCANNED).toContain("internal-packages/run-engine/src/engine/systems/waitpointSystem.ts");
+  });
+
+  // Per EXPRESSION, not per file: this fails for a fifth mint added inside an
+  // already-catalogued file, AND for a swapped anchor — mintWaitpointIdFor(undefined) in
+  // place of the run id — which a bare call-count would wave through.
+  it.each(SCANNED)("%s has exactly the mint expressions the catalog claims", (file) => {
+    const source = read(file);
+    const expected = expectedMints(file);
+
+    for (const [expr, n] of expected) {
+      expect({ expr, found: count(source, new RegExp(escapeRegExp(expr), "g")) }).toEqual({
+        expr,
+        found: n,
       });
     }
+
+    // No mint in the file beyond the ones the catalog accounts for.
+    const accounted = [...expected.values()].reduce((a, b) => a + b, 0);
+    expect(count(source, MINT_CALL)).toBe(accounted);
   });
 
-  it("every file that writes a waitpoint row is catalogued", () => {
-    const catalogued = new Set(WAITPOINT_MINT_SITES.map((s) => s.site));
-
-    for (const file of scannedFiles()) {
-      const source = read(file);
-      // A create with NO id is the worst case: Prisma's @default(cuid()) then mints a cuid
-      // on a gen-2 shard after the write, which no stamp check can see.
-      const writes =
-        count(source, /waitpoint\.create\(/g) +
-        count(source, /upsertWaitpoint\(/g) +
-        count(source, /createWaitpoint\(/g);
-
-      if (writes > 0) {
-        expect({ file, catalogued: catalogued.has(file) }).toEqual({ file, catalogued: true });
-      }
-    }
+  it.each(SCANNED)("%s mints no waitpoint id with the un-stamped helper", (file) => {
+    // The regex matches tokens inside comments too — deliberate. Any textual addition
+    // forces the census to be reconciled, so a new site cannot land unnoticed.
+    expect(count(read(file), UNSTAMPED_MINT)).toBe(0);
+  });
+
+  it.each(SCANNED)("%s writes a waitpoint row only if it is catalogued", (file) => {
+    // A create with NO id is the worst case: Prisma's @default(cuid()) then mints a cuid on
+    // a gen-2 shard after the write, which no stamp check can see.
+    const writes = count(read(file), WAITPOINT_WRITE);
+    const catalogued = WAITPOINT_MINT_SITES.some((s) => s.site === file);
+    expect(writes === 0 || catalogued).toBe(true);
   });
 
   it("every catalogued site names a file that exists", () => {
@@ -93,6 +118,16 @@ describe("waitpoint mint census — source drift guard", () => {
     }
   });
 
+  it("every catalogued site names its enclosing symbol in that file", () => {
+    for (const site of WAITPOINT_MINT_SITES) {
+      const symbol = site.symbol.split(" ")[0]!.replace("#", "");
+      expect({ site: site.id, present: read(site.site).includes(symbol) }).toEqual({
+        site: site.id,
+        present: true,
+      });
+    }
+  });
+
   it("no catalogued symbol is a line number", () => {
     for (const site of WAITPOINT_MINT_SITES) {
       expect(site.symbol).not.toMatch(/:\d+/);

internal-packages/run-engine/src/engine/waitpointCoordinator/waitpointMint.proof.test.ts

@@ -16,6 +16,13 @@ export type WaitpointMintSite = {
   site: string;
   /** Enclosing method or symbol name — NEVER a line number. */
   symbol: string;
+  /**
+   * The exact mint expressions this site contains, verbatim. The proof test counts each one
+   * per file, so both a new mint inside an already-catalogued file and a swapped anchor
+   * (`mintWaitpointIdFor(undefined)` in place of the run id) fail until reconciled here.
+   * Empty for a site that writes a row from an id minted elsewhere.
+   */
+  mints: readonly string[];
 };
 
 const COORDINATOR =
@@ -26,35 +33,52 @@ const RUN_STORE = "internal-packages/run-store/src/PostgresRunStore.ts";
 export const WAITPOINT_MINT_SITES: readonly WaitpointMintSite[] = [
   {
     id: "coordinator.datetime",
+    mints: ["mintWaitpointIdFor(runId)"],
     type: "DATETIME",
     site: COORDINATOR,
     symbol: "createDateTimeWaitpoint",
   },
-  { id: "coordinator.manual", type: "MANUAL", site: COORDINATOR, symbol: "createManualWaitpoint" },
+  {
+    id: "coordinator.manual",
+    mints: ["mintWaitpointIdForShard(standaloneShard)", "mintWaitpointIdFor(runId)"],
+    type: "MANUAL",
+    site: COORDINATOR,
+    symbol: "createManualWaitpoint",
+  },
   {
     id: "coordinator.associated.mint",
+    mints: ["mintWaitpointIdFor(anchorRunId)"],
     type: "RUN",
     site: COORDINATOR,
     symbol: "mintAssociatedWaitpointData",
   },
   {
     id: "coordinator.associated.create",
+    mints: [],
     type: "RUN",
     site: COORDINATOR,
     symbol: "createAssociatedWaitpoint",
   },
-  { id: "engine.batch", type: "BATCH", site: ENGINE, symbol: "blockRunWithCreatedBatch" },
+  {
+    id: "engine.batch",
+    mints: ["mintWaitpointIdFor(batchId)"],
+    type: "BATCH",
+    site: ENGINE,
+    symbol: "blockRunWithCreatedBatch",
+  },
   // The physical writers of the RUN row. They take an already-minted id rather than
   // minting one, but they are the writes that bypass the routing store's stamp check, so a
   // new writer here must be seen.
   {
     id: "runStore.createRun.nested",
+    mints: [],
     type: "RUN",
     site: RUN_STORE,
     symbol: "createRun (nested associatedWaitpoint create)",
   },
   {
     id: "runStore.createRun.dedicated",
+    mints: [],
     type: "RUN",
     site: RUN_STORE,
     symbol: "#createAssociatedWaitpoint",