From 65a354dedb3146a8055f34d3c15b64a0fbddfeca Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Wed, 26 Aug 2026 12:49:14 +0100 Subject: [PATCH 01/14] feat(core): mint Postgres waitpoint ids stamped for a gen-2 shard Adds mintWaitpointIdForShard(key) and mintWaitpointIdFor(anchorId). A gen-2 shard key produces a 26-char body carrying that shard char at index 24 and version "2"; a reserved key, or no anchor, keeps today's cuid. The core is always freshly minted rather than derived from the anchor: a derived body would share the anchor's core, shard char and version char, so it would be byte-identical to the run's own id. Both the webapp and the run engine mint through this one function. They have to agree byte-for-byte, because the routing store refuses a waitpoint whose id is not stamped for the shard it is being written to. Kept separate from friendlyId.ts because it needs resolveShard, and runOpsResidency.ts already imports friendlyId.ts. --- packages/core/src/v3/isomorphic/index.ts | 1 + .../src/v3/isomorphic/waitpointMint.test.ts | 70 +++++++++++++++++++ .../core/src/v3/isomorphic/waitpointMint.ts | 26 +++++++ 3 files changed, 97 insertions(+) create mode 100644 packages/core/src/v3/isomorphic/waitpointMint.test.ts create mode 100644 packages/core/src/v3/isomorphic/waitpointMint.ts diff --git a/packages/core/src/v3/isomorphic/index.ts b/packages/core/src/v3/isomorphic/index.ts index 3f372854735..5207dbc2c65 100644 --- a/packages/core/src/v3/isomorphic/index.ts +++ b/packages/core/src/v3/isomorphic/index.ts @@ -1,5 +1,6 @@ export * from "./friendlyId.js"; export * from "./runOpsResidency.js"; +export * from "./waitpointMint.js"; export * from "./duration.js"; export * from "./maxDuration.js"; export * from "./queueName.js"; diff --git a/packages/core/src/v3/isomorphic/waitpointMint.test.ts b/packages/core/src/v3/isomorphic/waitpointMint.test.ts new file mode 100644 index 00000000000..24acf6c7588 --- /dev/null +++ b/packages/core/src/v3/isomorphic/waitpointMint.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from "vitest"; +import { mintWaitpointIdFor, mintWaitpointIdForShard } from "./waitpointMint.js"; +import { isValidShardChar, parseRunOpsIdV2Body } from "./friendlyId.js"; +import { resolveShard } from "./runOpsResidency.js"; + +const GEN2_RUN = `run_${"a".repeat(24)}a2`; // shard "a", version "2" +const GEN1_RUN = `run_${"a".repeat(24)}01`; // region "0", version "1" +const CUID_RUN = `run_${"b".repeat(25)}`; + +describe("mintWaitpointIdForShard", () => { + it("a gen-2 shard key mints a gen-2 body with that char at index 24", () => { + const r = mintWaitpointIdForShard("a"); + expect(r.id.length).toBe(26); + expect(r.id[24]).toBe("a"); + expect(r.id[25]).toBe("2"); + expect(r.friendlyId).toBe(`waitpoint_${r.id}`); + expect(parseRunOpsIdV2Body(r.id)?.shard).toBe("a"); + }); + + it("the reserved key 'new' mints a cuid, unchanged from today", () => { + const r = mintWaitpointIdForShard("new"); + expect(r.id.length).toBe(25); + expect(resolveShard(r.id)).toBe("legacy"); + }); + + it("the reserved key 'legacy' mints a cuid", () => { + expect(mintWaitpointIdForShard("legacy").id.length).toBe(25); + }); + + it("two calls for one shard never collide", () => { + expect(mintWaitpointIdForShard("a").id).not.toBe(mintWaitpointIdForShard("a").id); + }); + + it("every gen-2 id it mints routes back to its own shard", () => { + for (const key of ["a", "b", "0", "z", "9"]) { + expect(isValidShardChar(key)).toBe(true); + expect(resolveShard(mintWaitpointIdForShard(key).id)).toBe(key); + } + }); +}); + +describe("mintWaitpointIdFor", () => { + it("a gen-2 anchor stamps the anchor's shard char", () => { + const r = mintWaitpointIdFor(GEN2_RUN); + expect(r.id[24]).toBe("a"); + expect(r.id[25]).toBe("2"); + }); + + it("a gen-2 anchor yields a FRESH core, never the anchor's own body", () => { + const r = mintWaitpointIdFor(GEN2_RUN); + expect(r.id).not.toBe(GEN2_RUN.slice(4)); + expect(r.id.slice(0, 24)).not.toBe("a".repeat(24)); + }); + + it("accepts the bare internal form as well as the prefixed form", () => { + expect(mintWaitpointIdFor(GEN2_RUN.slice(4)).id[24]).toBe("a"); + }); + + it("a gen-1 v1 anchor mints a cuid", () => { + expect(mintWaitpointIdFor(GEN1_RUN).id.length).toBe(25); + }); + + it("a cuid anchor mints a cuid", () => { + expect(mintWaitpointIdFor(CUID_RUN).id.length).toBe(25); + }); + + it("no anchor mints a cuid", () => { + expect(mintWaitpointIdFor(undefined).id.length).toBe(25); + }); +}); diff --git a/packages/core/src/v3/isomorphic/waitpointMint.ts b/packages/core/src/v3/isomorphic/waitpointMint.ts new file mode 100644 index 00000000000..d05d7b60f43 --- /dev/null +++ b/packages/core/src/v3/isomorphic/waitpointMint.ts @@ -0,0 +1,26 @@ +import { generateRunOpsIdV2, WaitpointId } from "./friendlyId.js"; +import { resolveShard, type ShardKey } from "./runOpsResidency.js"; + +// A waitpoint id for a Postgres shard — NOT the Redis store format (type char at index +// 24, version "w"), which has no Postgres row to route. The core is always fresh: reusing +// the anchor's would produce a body identical to the run's own id. +export function mintWaitpointIdForShard(key: ShardKey): { id: string; friendlyId: string } { + if (key === "new" || key === "legacy") { + return WaitpointId.generate(); + } + + const id = generateRunOpsIdV2(key); + return { id, friendlyId: WaitpointId.toFriendlyId(id) }; +} + +// Every Postgres waitpoint mint goes through here, in the webapp and the engine alike: +// the router refuses a waitpoint whose id is not stamped for the shard it lands on. +// A gen-1 or legacy anchor keeps a cuid. +export function mintWaitpointIdFor(anchorId: string | undefined): { + id: string; + friendlyId: string; +} { + return anchorId === undefined + ? WaitpointId.generate() + : mintWaitpointIdForShard(resolveShard(anchorId)); +} From 0dae1c540d44e53e0d1fc68ce757fc6d44e2b3d3 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Wed, 26 Aug 2026 12:50:24 +0100 Subject: [PATCH 02/14] feat(webapp): carry the shard char through a MintTarget on run inheritance resolveInheritedMintKind now returns { kind, shardChar? } instead of a bare kind, and mintFriendlyIdForKind takes that object. A gen-2 parent hands its own shard char to its children, so a run tree never splits across shards. The shard char and the region both occupy index 24 of a run-ops id, so they travel in one object rather than as two independent optional parameters: a caller cannot set two competing sources for one slot, and the gen-2 arm simply ignores the region. mintAnchoredRunFriendlyId keeps its signature, its keying on the batch id shape, and its synchronous form. Callers of the batch mint still break at this commit; the next two commits repair them. --- .../mintAnchoredRunFriendlyId.server.ts | 21 ++++--- .../app/v3/runOpsMigration/mintTarget.ts | 11 ++++ .../resolveInheritedMintKind.server.test.ts | 61 +++++++++++++++++-- .../resolveInheritedMintKind.server.ts | 14 +++-- 4 files changed, 92 insertions(+), 15 deletions(-) create mode 100644 apps/webapp/app/v3/runOpsMigration/mintTarget.ts diff --git a/apps/webapp/app/v3/runOpsMigration/mintAnchoredRunFriendlyId.server.ts b/apps/webapp/app/v3/runOpsMigration/mintAnchoredRunFriendlyId.server.ts index 0f5da2e56f7..d3de7bf8cb4 100644 --- a/apps/webapp/app/v3/runOpsMigration/mintAnchoredRunFriendlyId.server.ts +++ b/apps/webapp/app/v3/runOpsMigration/mintAnchoredRunFriendlyId.server.ts @@ -1,15 +1,22 @@ -import { generateRunOpsId, RunId, type ResidencyKind } from "@trigger.dev/core/v3/isomorphic"; +import { generateRunOpsId, generateRunOpsIdV2, RunId } from "@trigger.dev/core/v3/isomorphic"; +import type { MintTarget } from "./mintTarget"; import { resolveInheritedMintKind } from "./resolveInheritedMintKind.server"; -// Shared id-generation branch for every run-mint path: "runOpsId" -> NEW store, "cuid" -> LEGACY. -export function mintFriendlyIdForKind(mintKind: ResidencyKind, region?: string): string { - return mintKind === "runOpsId" - ? RunId.toFriendlyId(generateRunOpsId(region)) - : RunId.generate().friendlyId; +// Shared id-generation branch for every run-mint path: "runOpsId" -> a dedicated store, +// "cuid" -> LEGACY. A shardChar selects one gen-2 shard and takes index 24; without one, +// the region takes that slot exactly as it does today. +export function mintFriendlyIdForKind(target: MintTarget): string { + if (target.kind !== "runOpsId") { + return RunId.generate().friendlyId; + } + + return RunId.toFriendlyId( + target.shardChar ? generateRunOpsIdV2(target.shardChar) : generateRunOpsId(target.region) + ); } // Anchor a batch item's mint on the BATCH's friendlyId (id-shape, zero I/O), never the per-org // flag, so the item and its BatchTaskRun stay co-resident across a mid-batch flag flip. export function mintAnchoredRunFriendlyId(batchFriendlyId: string, region?: string): string { - return mintFriendlyIdForKind(resolveInheritedMintKind(batchFriendlyId), region); + return mintFriendlyIdForKind({ ...resolveInheritedMintKind(batchFriendlyId), region }); } diff --git a/apps/webapp/app/v3/runOpsMigration/mintTarget.ts b/apps/webapp/app/v3/runOpsMigration/mintTarget.ts new file mode 100644 index 00000000000..94355d55462 --- /dev/null +++ b/apps/webapp/app/v3/runOpsMigration/mintTarget.ts @@ -0,0 +1,11 @@ +import type { ResidencyKind } from "@trigger.dev/core/v3/isomorphic"; + +/** + * Where one mint lands. `shardChar` and `region` both occupy index 24 of a run-ops id, so + * they travel together and cannot disagree. `shardChar` set means gen-2, region ignored. + */ +export type MintTarget = { + kind: ResidencyKind; + shardChar?: string; + region?: string; +}; diff --git a/apps/webapp/app/v3/runOpsMigration/resolveInheritedMintKind.server.test.ts b/apps/webapp/app/v3/runOpsMigration/resolveInheritedMintKind.server.test.ts index 3f135793f84..570cc496182 100644 --- a/apps/webapp/app/v3/runOpsMigration/resolveInheritedMintKind.server.test.ts +++ b/apps/webapp/app/v3/runOpsMigration/resolveInheritedMintKind.server.test.ts @@ -1,15 +1,68 @@ import { describe, expect, it } from "vitest"; import { resolveInheritedMintKind } from "./resolveInheritedMintKind.server"; +import { mintFriendlyIdForKind } from "./mintAnchoredRunFriendlyId.server"; -const NEW_PARENT = `run_${"a".repeat(24) + "01"}`; // run-ops id-shape -> NEW +const NEW_PARENT = `run_${"a".repeat(24)}01`; // run-ops v1 id-shape -> NEW const LEGACY_PARENT = `run_${"b".repeat(25)}`; // cuid id-shape -> LEGACY +const GEN2_PARENT = `run_${"a".repeat(24)}a2`; // gen-2, shard "a" describe("resolveInheritedMintKind (pure id-shape, shared across all mint paths)", () => { - it("inherits a run-ops (NEW) parent by id-shape -> 'runOpsId' kind", () => { - expect(resolveInheritedMintKind(NEW_PARENT)).toBe("runOpsId"); + it("inherits a run-ops (NEW) parent by id-shape -> runOpsId with NO shard char", () => { + expect(resolveInheritedMintKind(NEW_PARENT)).toEqual({ kind: "runOpsId" }); }); it("inherits a cuid (LEGACY) parent by id-shape -> cuid", () => { - expect(resolveInheritedMintKind(LEGACY_PARENT)).toBe("cuid"); + expect(resolveInheritedMintKind(LEGACY_PARENT)).toEqual({ kind: "cuid" }); + }); + + it("inherits a gen-2 parent's shard char, never a freshly resolved one", () => { + expect(resolveInheritedMintKind(GEN2_PARENT)).toEqual({ kind: "runOpsId", shardChar: "a" }); + }); + + it("accepts the bare internal form", () => { + expect(resolveInheritedMintKind(GEN2_PARENT.slice(4))).toEqual({ + kind: "runOpsId", + shardChar: "a", + }); + }); +}); + +describe("mintFriendlyIdForKind", () => { + it("a shard char mints a gen-2 id with that char at index 24 and '2' at 25", () => { + const body = mintFriendlyIdForKind({ kind: "runOpsId", shardChar: "a" }).slice("run_".length); + expect(body.length).toBe(26); + expect(body[24]).toBe("a"); + expect(body[25]).toBe("2"); + }); + + it("a shard char wins over a region: index 24 has ONE source", () => { + const body = mintFriendlyIdForKind({ + kind: "runOpsId", + shardChar: "a", + region: "us-east-1", + }).slice("run_".length); + expect(body[24]).toBe("a"); // not "e", the us-east-1 region char + }); + + it("no shard char mints a gen-1 v1 id, stamping the region as today", () => { + const body = mintFriendlyIdForKind({ kind: "runOpsId", region: "us-east-1" }).slice(4); + expect(body[24]).toBe("e"); + expect(body[25]).toBe("1"); + }); + + it("no shard char and no region mints a gen-1 v1 id with the default region char", () => { + const body = mintFriendlyIdForKind({ kind: "runOpsId" }).slice(4); + expect(body[24]).toBe("0"); + expect(body[25]).toBe("1"); + }); + + it("cuid kind mints a 25-char cuid", () => { + expect(mintFriendlyIdForKind({ kind: "cuid" }).slice(4).length).toBe(25); + }); + + it("an end-to-end inherit-then-mint keeps a child on the parent's shard", () => { + const body = mintFriendlyIdForKind(resolveInheritedMintKind(GEN2_PARENT)).slice(4); + expect(body[24]).toBe("a"); + expect(body[25]).toBe("2"); }); }); diff --git a/apps/webapp/app/v3/runOpsMigration/resolveInheritedMintKind.server.ts b/apps/webapp/app/v3/runOpsMigration/resolveInheritedMintKind.server.ts index 6ec9583c94b..825d910d7d9 100644 --- a/apps/webapp/app/v3/runOpsMigration/resolveInheritedMintKind.server.ts +++ b/apps/webapp/app/v3/runOpsMigration/resolveInheritedMintKind.server.ts @@ -1,10 +1,16 @@ -import { ownerEngine } from "@trigger.dev/core/v3/isomorphic"; -import type { RunIdMintKind } from "./runOpsMintKind.server"; +import { resolveShard } from "@trigger.dev/core/v3/isomorphic"; +import type { MintTarget } from "./mintTarget"; // Mint a child in the SAME physical store as its anchor (parent run / owning batch), // regardless of the org's current mint flag — keeps a subgraph co-resident across a // flip. With no migration/drain, residency is a pure id-shape check (zero hot-path // I/O): a run-ops (NEW) parent mints run-ops children, a cuid (LEGACY) parent mints cuid. -export function resolveInheritedMintKind(parentRunFriendlyId: string): RunIdMintKind { - return ownerEngine(parentRunFriendlyId) === "NEW" ? "runOpsId" : "cuid"; +// A gen-2 parent hands down its OWN shard char, never a freshly resolved one: two runs in +// one tree must never split across shards. +export function resolveInheritedMintKind(parentRunFriendlyId: string): MintTarget { + const shard = resolveShard(parentRunFriendlyId); + + if (shard === "legacy") return { kind: "cuid" }; + if (shard === "new") return { kind: "runOpsId" }; + return { kind: "runOpsId", shardChar: shard }; } From b20792f994139b11ea603bd40516d93819c8b2f2 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Wed, 26 Aug 2026 12:51:58 +0100 Subject: [PATCH 03/14] feat(webapp): resolve a run's mint target in one place, gated off by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds resolveRunMintTarget: a parent means inherit by id-shape, no parent means resolve the org's mint kind and then, only on the run-ops path, the environment's mint shard. Three services carried this branch separately and one had already drifted, so it now lives in one function with an injectable deps parameter for tests. resolveMintShard gains an early return when no shard descriptor is configured. It matters for more than speed: the flag read happens before the routable-key bound is applied, so without this guard, merging would add a control-plane replica query to the root trigger path on every deployment that has no shards. With it, an unconfigured deployment takes a literally unchanged path — no query, no cache write, no log line. Both knip suppressions for that module are dropped now that it has real importers. --- .../resolveRunMintTarget.server.test.ts | 75 +++++++++++++++++++ .../resolveRunMintTarget.server.ts | 55 ++++++++++++++ .../runOpsMigration/runOpsMintShard.server.ts | 9 ++- knip.json | 3 +- 4 files changed, 138 insertions(+), 4 deletions(-) create mode 100644 apps/webapp/app/v3/runOpsMigration/resolveRunMintTarget.server.test.ts create mode 100644 apps/webapp/app/v3/runOpsMigration/resolveRunMintTarget.server.ts diff --git a/apps/webapp/app/v3/runOpsMigration/resolveRunMintTarget.server.test.ts b/apps/webapp/app/v3/runOpsMigration/resolveRunMintTarget.server.test.ts new file mode 100644 index 00000000000..a71fdcc2b5f --- /dev/null +++ b/apps/webapp/app/v3/runOpsMigration/resolveRunMintTarget.server.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it, vi } from "vitest"; +import { resolveRunMintTarget } from "./resolveRunMintTarget.server"; + +const environment = { organizationId: "org_1", id: "env_1", orgFeatureFlags: {} }; +const GEN2_PARENT = `run_${"a".repeat(24)}a2`; +const LEGACY_PARENT = `run_${"b".repeat(25)}`; + +describe("resolveRunMintTarget — root", () => { + it("resolves the kind, then the shard, and returns both", async () => { + const resolveRunIdMintKind = vi.fn().mockResolvedValue("runOpsId"); + const resolveMintShard = vi.fn().mockResolvedValue("a"); + + const target = await resolveRunMintTarget({ + environment, + deps: { resolveRunIdMintKind, resolveMintShard }, + }); + + expect(target).toEqual({ kind: "runOpsId", shardChar: "a", region: undefined }); + expect(resolveMintShard).toHaveBeenCalledWith({ id: "env_1", orgFeatureFlags: {} }); + }); + + it("a 'new' shard result carries NO shard char, so the mint stays gen-1", async () => { + const target = await resolveRunMintTarget({ + environment, + region: "us-east-1", + deps: { + resolveRunIdMintKind: vi.fn().mockResolvedValue("runOpsId"), + resolveMintShard: vi.fn().mockResolvedValue("new"), + }, + }); + expect(target).toEqual({ kind: "runOpsId", region: "us-east-1" }); + }); + + it("never resolves a shard when the kind is cuid", async () => { + const resolveMintShard = vi.fn(); + const target = await resolveRunMintTarget({ + environment, + deps: { + resolveRunIdMintKind: vi.fn().mockResolvedValue("cuid"), + resolveMintShard, + }, + }); + expect(target).toEqual({ kind: "cuid" }); + expect(resolveMintShard).not.toHaveBeenCalled(); + }); +}); + +describe("resolveRunMintTarget — child", () => { + it("inherits a gen-2 parent's shard and consults NEITHER resolver", async () => { + const resolveRunIdMintKind = vi.fn(); + const resolveMintShard = vi.fn(); + + const target = await resolveRunMintTarget({ + environment, + parentRunFriendlyId: GEN2_PARENT, + deps: { resolveRunIdMintKind, resolveMintShard }, + }); + + expect(target).toEqual({ kind: "runOpsId", shardChar: "a" }); + expect(resolveRunIdMintKind).not.toHaveBeenCalled(); + expect(resolveMintShard).not.toHaveBeenCalled(); + }); + + it("a cuid parent still yields cuid though the flag now says runOpsId", async () => { + const target = await resolveRunMintTarget({ + environment, + parentRunFriendlyId: LEGACY_PARENT, + deps: { + resolveRunIdMintKind: vi.fn().mockResolvedValue("runOpsId"), + resolveMintShard: vi.fn().mockResolvedValue("a"), + }, + }); + expect(target).toEqual({ kind: "cuid" }); + }); +}); diff --git a/apps/webapp/app/v3/runOpsMigration/resolveRunMintTarget.server.ts b/apps/webapp/app/v3/runOpsMigration/resolveRunMintTarget.server.ts new file mode 100644 index 00000000000..89b8948c689 --- /dev/null +++ b/apps/webapp/app/v3/runOpsMigration/resolveRunMintTarget.server.ts @@ -0,0 +1,55 @@ +import type { MintTarget } from "./mintTarget"; +import { resolveInheritedMintKind } from "./resolveInheritedMintKind.server"; +import { resolveRunIdMintKind as defaultResolveRunIdMintKind } from "./runOpsMintKind.server"; +import { resolveMintShard as defaultResolveMintShard } from "./runOpsMintShard.server"; + +export type RunMintDeps = { + resolveRunIdMintKind: typeof defaultResolveRunIdMintKind; + resolveMintShard: typeof defaultResolveMintShard; +}; + +const defaultDeps: RunMintDeps = { + resolveRunIdMintKind: defaultResolveRunIdMintKind, + resolveMintShard: defaultResolveMintShard, +}; + +/** + * Where one run mints. Two stages, and the second runs only for a ROOT run already on the + * run-ops path: a child inherits its parent's shard by id-shape, so a tree never splits. + * + * Every run-mint path routes through here. The branch used to be duplicated per service, + * and one copy had already drifted into minting gen-1 for a gen-2 parent. + */ +export async function resolveRunMintTarget(args: { + environment: { organizationId: string; id: string; orgFeatureFlags?: unknown }; + parentRunFriendlyId?: string; + region?: string; + deps?: Partial; +}): Promise { + if (args.parentRunFriendlyId) { + return resolveInheritedMintKind(args.parentRunFriendlyId); + } + + const deps = { ...defaultDeps, ...args.deps }; + + const kind = await deps.resolveRunIdMintKind({ + organizationId: args.environment.organizationId, + id: args.environment.id, + orgFeatureFlags: args.environment.orgFeatureFlags, + }); + + if (kind !== "runOpsId") { + return { kind }; + } + + const shard = await deps.resolveMintShard({ + id: args.environment.id, + orgFeatureFlags: args.environment.orgFeatureFlags, + }); + + // A reserved key means gen-1, which is the state of every deployment that has configured + // no shard. Only a single-char key names a gen-2 shard. + return shard === "new" || shard === "legacy" + ? { kind, region: args.region } + : { kind, shardChar: shard, region: args.region }; +} diff --git a/apps/webapp/app/v3/runOpsMigration/runOpsMintShard.server.ts b/apps/webapp/app/v3/runOpsMigration/runOpsMintShard.server.ts index c1c2b9ddd48..422af3712e9 100644 --- a/apps/webapp/app/v3/runOpsMigration/runOpsMintShard.server.ts +++ b/apps/webapp/app/v3/runOpsMigration/runOpsMintShard.server.ts @@ -69,14 +69,19 @@ function reportOverrideRejected(info: { override: string; activeSet: string[] }) /** * Which shard an environment mints new roots into. Call only after resolveRunIdMintKind has * returned "runOpsId". Returns "new" to mean a gen-1 run-ops id, which is today's behaviour. - * - * @knipignore the gen-2 write-path change is the first production caller; drop this tag there. */ export async function resolveMintShard(environment: { id: string; // Pass environment.organization.featureFlags from the trigger call site. orgFeatureFlags?: unknown; }): Promise { + // No shard descriptor means no shard can ever be minted into, so answer before reading + // anything: an unconfigured deployment keeps exactly today's code path, with no + // control-plane query on the trigger path, no cache write and no log line. + if (env.RUN_OPS_SHARDS.length === 0) { + return "new"; + } + return resolveMintShardWith(environment, { readFlags: readSetFlags, cache: liveCache, diff --git a/knip.json b/knip.json index c6e8aee8977..84456756ca1 100644 --- a/knip.json +++ b/knip.json @@ -25,8 +25,7 @@ "vite/node-globals-shim.js", "app/v3/otlpTransformWorker.ts" ], - "ignoreDependencies": ["@sentry/cli", "assert", "util"], - "ignore": ["app/v3/runOpsMigration/runOpsMintShard.server.ts"] + "ignoreDependencies": ["@sentry/cli", "assert", "util"] }, "internal-packages/dashboard-agent": { "entry": ["trigger.config.ts", "src/investigation-sweep.ts", "src/maintenance.ts"], From 845ab0651fdcc825aac56882c5b54547c057fae6 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Wed, 26 Aug 2026 12:58:14 +0100 Subject: [PATCH 04/14] fix(webapp): mint a failed child run onto its parent's shard triggerFailedTask duplicated the mint branch inline rather than calling the shared helper, and it had drifted: it dropped the caller's region, and once gen-2 ids exist it would mint a gen-1 id for a child of a gen-2 parent. The router would then write that child to the gen-1 store while its parent lives on a shard, splitting one run tree across two databases. Both trigger services now call resolveRunMintTarget. triggerTask's behaviour is unchanged. The pre-minted runFriendlyId pass-through stays ahead of the resolver: batchTrigger and runEngineHandlers hand in an id already minted from the batch, and re-resolving it would move the item off its batch's shard. Added a container test for that, since no pure test can reach the guard and a typecheck will not notice if it moves below the resolver. --- .../services/triggerFailedTask.server.ts | 21 ++++---- .../runEngine/services/triggerTask.server.ts | 17 ++++--- .../engine/gen2ChildMintInheritance.test.ts | 16 ++++++ ...iggerFailedTask.withoutTraceEvents.test.ts | 49 +++++++++++++++++++ 4 files changed, 84 insertions(+), 19 deletions(-) create mode 100644 apps/webapp/test/engine/gen2ChildMintInheritance.test.ts diff --git a/apps/webapp/app/runEngine/services/triggerFailedTask.server.ts b/apps/webapp/app/runEngine/services/triggerFailedTask.server.ts index f8ba67f3448..e1d8a0841aa 100644 --- a/apps/webapp/app/runEngine/services/triggerFailedTask.server.ts +++ b/apps/webapp/app/runEngine/services/triggerFailedTask.server.ts @@ -1,6 +1,6 @@ import type { RunEngine } from "@internal/run-engine"; import { TaskRunErrorCodes, type TaskRunError } from "@trigger.dev/core/v3"; -import { RunId, generateRunOpsId } from "@trigger.dev/core/v3/isomorphic"; +import { RunId } from "@trigger.dev/core/v3/isomorphic"; import type { PrismaClientOrTransaction, RuntimeEnvironmentType, @@ -8,8 +8,8 @@ import type { } from "@trigger.dev/database"; import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; -import { resolveRunIdMintKind } from "~/v3/engineVersion.server"; -import { resolveInheritedMintKind } from "~/v3/runOpsMigration/resolveInheritedMintKind.server"; +import { mintFriendlyIdForKind } from "~/v3/runOpsMigration/mintAnchoredRunFriendlyId.server"; +import { resolveRunMintTarget } from "~/v3/runOpsMigration/resolveRunMintTarget.server"; import { getEventRepository } from "~/v3/eventRepository/index.server"; import { runStore as defaultRunStore } from "~/v3/runStore.server"; import type { RunStore } from "@internal/run-store"; @@ -103,17 +103,16 @@ export class TriggerFailedTaskService { return args.runFriendlyId; } - const mintKind = args.parentRunFriendlyId - ? resolveInheritedMintKind(args.parentRunFriendlyId) - : await resolveRunIdMintKind({ + return mintFriendlyIdForKind( + await resolveRunMintTarget({ + environment: { organizationId: args.organizationId, id: args.environmentId, orgFeatureFlags: args.orgFeatureFlags, - }); - - return mintKind === "runOpsId" - ? RunId.toFriendlyId(generateRunOpsId()) - : RunId.generate().friendlyId; + }, + parentRunFriendlyId: args.parentRunFriendlyId, + }) + ); } async call(request: TriggerFailedTaskRequest): Promise { diff --git a/apps/webapp/app/runEngine/services/triggerTask.server.ts b/apps/webapp/app/runEngine/services/triggerTask.server.ts index 8e9e99d7f09..d3320dbc219 100644 --- a/apps/webapp/app/runEngine/services/triggerTask.server.ts +++ b/apps/webapp/app/runEngine/services/triggerTask.server.ts @@ -28,9 +28,8 @@ import { parseDelay } from "~/utils/delays"; import { removeNullBytesFromKey } from "~/utils/nullBytes"; import { handleMetadataPacket } from "~/utils/packets"; import { startSpan } from "~/v3/tracing.server"; -import { resolveRunIdMintKind } from "~/v3/engineVersion.server"; -import { resolveInheritedMintKind } from "~/v3/runOpsMigration/resolveInheritedMintKind.server"; import { mintFriendlyIdForKind } from "~/v3/runOpsMigration/mintAnchoredRunFriendlyId.server"; +import { resolveRunMintTarget } from "~/v3/runOpsMigration/resolveRunMintTarget.server"; import type { TriggerTaskServiceOptions, TriggerTaskServiceResult, @@ -218,15 +217,17 @@ export class RunEngineTriggerTaskService { parentRunFriendlyId?: string, region?: string ): Promise { - const mintKind = parentRunFriendlyId - ? resolveInheritedMintKind(parentRunFriendlyId) - : await resolveRunIdMintKind({ + return mintFriendlyIdForKind( + await resolveRunMintTarget({ + environment: { organizationId: environment.organizationId, id: environment.id, orgFeatureFlags: environment.organization.featureFlags, - }); - - return mintFriendlyIdForKind(mintKind, region); + }, + parentRunFriendlyId, + region, + }) + ); } public async call({ diff --git a/apps/webapp/test/engine/gen2ChildMintInheritance.test.ts b/apps/webapp/test/engine/gen2ChildMintInheritance.test.ts new file mode 100644 index 00000000000..d687b75c942 --- /dev/null +++ b/apps/webapp/test/engine/gen2ChildMintInheritance.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, it } from "vitest"; +import { resolveInheritedMintKind } from "~/v3/runOpsMigration/resolveInheritedMintKind.server"; +import { mintFriendlyIdForKind } from "~/v3/runOpsMigration/mintAnchoredRunFriendlyId.server"; + +// The shape both trigger services must produce for a child of a gen-2 parent. Before this +// change triggerFailedTask duplicated the branch inline and minted gen-1, which put a child +// on a different database from its parent. +const GEN2_PARENT = `run_${"a".repeat(24)}a2`; + +describe("a failed child of a gen-2 parent", () => { + it("mints onto the parent's shard", () => { + const body = mintFriendlyIdForKind(resolveInheritedMintKind(GEN2_PARENT)).slice(4); + expect(body[24]).toBe("a"); + expect(body[25]).toBe("2"); + }); +}); diff --git a/apps/webapp/test/engine/triggerFailedTask.withoutTraceEvents.test.ts b/apps/webapp/test/engine/triggerFailedTask.withoutTraceEvents.test.ts index a0be900fb82..eac5d75ec8a 100644 --- a/apps/webapp/test/engine/triggerFailedTask.withoutTraceEvents.test.ts +++ b/apps/webapp/test/engine/triggerFailedTask.withoutTraceEvents.test.ts @@ -90,4 +90,53 @@ describe("TriggerFailedTaskService — failed run residency (callWithoutTraceEve await engine.quit(); } ); + + containerTest( + "a pre-minted runFriendlyId passes through untouched", + async ({ prisma, redisOptions }) => { + const engine = makeEngine(prisma, redisOptions); + const environment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + const taskIdentifier = "failed-residency-passthrough"; + await setupBackgroundWorker(engine, environment, taskIdentifier); + + const parentFriendlyId = RunId.toFriendlyId(generateRunOpsId()); + await engine.trigger( + { + friendlyId: parentFriendlyId, + environment, + taskIdentifier, + payload: "{}", + payloadType: "application/json", + traceId: "00000000000000000000000000000000", + spanId: "0000000000000000", + workerQueue: "main", + queue: `task/${taskIdentifier}`, + isTest: false, + tags: [], + } as any, + prisma + ); + + // A batch item arrives with its id already minted from the BATCH. Re-resolving it + // here would move the item off its batch's shard, so the pass-through has to win + // over the mint-target resolver. + const preMinted = RunId.toFriendlyId(generateRunOpsId()); + + const friendlyId = await makeService(prisma, engine).callWithoutTraceEvents({ + environmentId: environment.id, + environmentType: environment.type, + projectId: environment.projectId, + organizationId: environment.organizationId, + taskId: taskIdentifier, + payload: { test: "passthrough" }, + errorMessage: "boom", + parentRunId: parentFriendlyId, + runFriendlyId: preMinted, + }); + + expect(friendlyId).toBe(preMinted); + + await engine.quit(); + } + ); }); From bf320524f68a6694f9a9803efeed83b646394df8 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Wed, 26 Aug 2026 12:59:33 +0100 Subject: [PATCH 05/14] feat(webapp): mint a batch id onto its parent run's shard batchIdForMintKind and resolveBatchMintKind now take and return the mint target, so a child batch carries its parent run's shard char and a root batch mints by the environment's policy. Batch-anchored item minting needs no change: it already keys on the shape of the batch id. This is where the type change actually bites. resolveBatchMintKind declared Promise, so the inheritance change makes it a compile error, and the obvious repair -- comparing kind.kind -- would compile while silently dropping the shard char. The rewritten tests cover both arms, including the two that pin the rule that the flag resolver is never consulted for a child. batchTriggerV3.mintChildFriendlyId keeps its own branch and its injected resolveMintKind. Its root arm is unreachable in production and that injection point is what lets a test drive it without a database. --- .../mintBatchFriendlyId.server.test.ts | 109 ++++++++++++------ .../mintBatchFriendlyId.server.ts | 50 ++++---- .../app/v3/services/batchTriggerV3.server.ts | 19 +-- 3 files changed, 109 insertions(+), 69 deletions(-) diff --git a/apps/webapp/app/v3/runOpsMigration/mintBatchFriendlyId.server.test.ts b/apps/webapp/app/v3/runOpsMigration/mintBatchFriendlyId.server.test.ts index 9973be57d1d..0e07a59d382 100644 --- a/apps/webapp/app/v3/runOpsMigration/mintBatchFriendlyId.server.test.ts +++ b/apps/webapp/app/v3/runOpsMigration/mintBatchFriendlyId.server.test.ts @@ -4,15 +4,23 @@ import { classifyKind } from "@trigger.dev/core/v3/isomorphic"; describe("batchIdForMintKind (pure)", () => { it("'runOpsId' kind -> 26-char classifiable NEW batch id (no 21-char ids)", () => { - const r = batchIdForMintKind("runOpsId"); + const r = batchIdForMintKind({ kind: "runOpsId" }); expect(r.friendlyId.startsWith("batch_")).toBe(true); expect(r.id.length).toBe(26); expect(classifyKind(r.id)).toBe("runOpsId"); expect(classifyKind(r.friendlyId)).toBe("runOpsId"); }); + it("a shard char mints a gen-2 batch id carrying that char", () => { + const r = batchIdForMintKind({ kind: "runOpsId", shardChar: "a" }); + expect(r.id.length).toBe(26); + expect(r.id[24]).toBe("a"); + expect(r.id[25]).toBe("2"); + expect(classifyKind(r.id)).toBe("runOpsId"); + }); + it("cuid -> 25-char classifiable LEGACY batch id", () => { - const r = batchIdForMintKind("cuid"); + const r = batchIdForMintKind({ kind: "cuid" }); expect(r.id.length).toBe(25); expect(classifyKind(r.id)).toBe("cuid"); expect(classifyKind(r.friendlyId)).toBe("cuid"); @@ -20,21 +28,26 @@ describe("batchIdForMintKind (pure)", () => { it("never mints a 21-char id", () => { for (const kind of ["cuid", "runOpsId"] as const) { - expect([25, 26]).toContain(batchIdForMintKind(kind).id.length); + expect([25, 26]).toContain(batchIdForMintKind({ kind }).id.length); } }); }); describe("resolveBatchMintKind", () => { const environment = { organizationId: "org_1", id: "env_1", orgFeatureFlags: {} }; + const NEW_PARENT = `run_${"a".repeat(24)}01`; + const LEGACY_PARENT = `run_${"a".repeat(25)}`; + const GEN2_PARENT = `run_${"a".repeat(24)}a2`; it("ROOT batch (no parent) resolves per-org kind via resolveRunIdMintKind", async () => { const resolveRunIdMintKind = vi.fn().mockResolvedValue("runOpsId"); - const kind = await resolveBatchMintKind({ + const resolveMintShard = vi.fn().mockResolvedValue("new"); + const target = await resolveBatchMintKind({ environment, - deps: { resolveRunIdMintKind }, + deps: { resolveRunIdMintKind, resolveMintShard }, }); - expect(kind).toBe("runOpsId"); + expect(target.kind).toBe("runOpsId"); + expect(target.shardChar).toBeUndefined(); expect(resolveRunIdMintKind).toHaveBeenCalledWith({ organizationId: "org_1", id: "env_1", @@ -42,66 +55,96 @@ describe("resolveBatchMintKind", () => { }); }); + it("ROOT batch mints by the mint policy when a shard is active", async () => { + const target = await resolveBatchMintKind({ + environment, + deps: { + resolveRunIdMintKind: vi.fn().mockResolvedValue("runOpsId"), + resolveMintShard: vi.fn().mockResolvedValue("a"), + }, + }); + expect(target).toEqual({ kind: "runOpsId", shardChar: "a", region: undefined }); + }); + it("ROOT batch on a non-cut-over org -> cuid", async () => { - const resolveRunIdMintKind = vi.fn().mockResolvedValue("cuid"); - const kind = await resolveBatchMintKind({ + const target = await resolveBatchMintKind({ environment, - deps: { resolveRunIdMintKind }, + deps: { + resolveRunIdMintKind: vi.fn().mockResolvedValue("cuid"), + resolveMintShard: vi.fn(), + }, }); - expect(kind).toBe("cuid"); + expect(target.kind).toBe("cuid"); }); it("CHILD batch inherits a run-ops (NEW) parent by id-shape", async () => { - const parentRunFriendlyId = `run_${"a".repeat(24) + "01"}`; const resolveRunIdMintKind = vi.fn(); - - const kind = await resolveBatchMintKind({ + const target = await resolveBatchMintKind({ environment, - parentRunFriendlyId, - deps: { resolveRunIdMintKind }, + parentRunFriendlyId: NEW_PARENT, + deps: { resolveRunIdMintKind, resolveMintShard: vi.fn() }, }); + expect(target).toEqual({ kind: "runOpsId" }); + expect(resolveRunIdMintKind).not.toHaveBeenCalled(); + }); - expect(kind).toBe("runOpsId"); + it("CHILD batch takes a gen-2 parent's shard char", async () => { + const resolveRunIdMintKind = vi.fn(); + const resolveMintShard = vi.fn(); + const target = await resolveBatchMintKind({ + environment, + parentRunFriendlyId: GEN2_PARENT, + deps: { resolveRunIdMintKind, resolveMintShard }, + }); + expect(target).toEqual({ kind: "runOpsId", shardChar: "a" }); expect(resolveRunIdMintKind).not.toHaveBeenCalled(); + expect(resolveMintShard).not.toHaveBeenCalled(); }); it("CHILD batch inherits a cuid (LEGACY) parent by id-shape", async () => { - const parentRunFriendlyId = `run_${"a".repeat(25)}`; const resolveRunIdMintKind = vi.fn(); - - const kind = await resolveBatchMintKind({ + const target = await resolveBatchMintKind({ environment, - parentRunFriendlyId, - deps: { resolveRunIdMintKind }, + parentRunFriendlyId: LEGACY_PARENT, + deps: { resolveRunIdMintKind, resolveMintShard: vi.fn() }, }); - - expect(kind).toBe("cuid"); + expect(target).toEqual({ kind: "cuid" }); expect(resolveRunIdMintKind).not.toHaveBeenCalled(); }); // mint-on-FLIP invariant: a child follows its parent's store even after the org flag // flips the other way. The flag resolver must NEVER be consulted for a child. it("FLIP 'cuid'->'runOpsId': a cuid (LEGACY) parent still mints a cuid child though the flag now says 'runOpsId'", async () => { - const parentRunFriendlyId = `run_${"a".repeat(25)}`; const resolveRunIdMintKind = vi.fn().mockResolvedValue("runOpsId"); // flag flipped to runOpsId - const kind = await resolveBatchMintKind({ + const target = await resolveBatchMintKind({ environment, - parentRunFriendlyId, - deps: { resolveRunIdMintKind }, + parentRunFriendlyId: LEGACY_PARENT, + deps: { resolveRunIdMintKind, resolveMintShard: vi.fn() }, }); - expect(kind).toBe("cuid"); + expect(target).toEqual({ kind: "cuid" }); expect(resolveRunIdMintKind).not.toHaveBeenCalled(); }); it("FLIP 'runOpsId'->'cuid': a run-ops (NEW) parent still mints a run-ops child though the flag now says 'cuid'", async () => { - const parentRunFriendlyId = `run_${"a".repeat(24) + "01"}`; const resolveRunIdMintKind = vi.fn().mockResolvedValue("cuid"); // flag flipped back to cuid - const kind = await resolveBatchMintKind({ + const target = await resolveBatchMintKind({ environment, - parentRunFriendlyId, - deps: { resolveRunIdMintKind }, + parentRunFriendlyId: NEW_PARENT, + deps: { resolveRunIdMintKind, resolveMintShard: vi.fn() }, }); - expect(kind).toBe("runOpsId"); + expect(target).toEqual({ kind: "runOpsId" }); expect(resolveRunIdMintKind).not.toHaveBeenCalled(); }); + + it("FLIP does not move a gen-2 child off its parent's shard", async () => { + const target = await resolveBatchMintKind({ + environment, + parentRunFriendlyId: GEN2_PARENT, + deps: { + resolveRunIdMintKind: vi.fn().mockResolvedValue("cuid"), + resolveMintShard: vi.fn().mockResolvedValue("b"), + }, + }); + expect(target).toEqual({ kind: "runOpsId", shardChar: "a" }); + }); }); diff --git a/apps/webapp/app/v3/runOpsMigration/mintBatchFriendlyId.server.ts b/apps/webapp/app/v3/runOpsMigration/mintBatchFriendlyId.server.ts index e2d8511e3ff..b08d9b9b33f 100644 --- a/apps/webapp/app/v3/runOpsMigration/mintBatchFriendlyId.server.ts +++ b/apps/webapp/app/v3/runOpsMigration/mintBatchFriendlyId.server.ts @@ -1,45 +1,37 @@ -import { BatchId, generateRunOpsId } from "@trigger.dev/core/v3/isomorphic"; -import { - resolveRunIdMintKind as defaultResolveRunIdMintKind, - type RunIdMintKind, -} from "~/v3/engineVersion.server"; -import { resolveInheritedMintKind } from "~/v3/runOpsMigration/resolveInheritedMintKind.server"; +import { BatchId, generateRunOpsId, generateRunOpsIdV2 } from "@trigger.dev/core/v3/isomorphic"; +import type { MintTarget } from "./mintTarget"; +import { resolveRunMintTarget, type RunMintDeps } from "./resolveRunMintTarget.server"; -type ResolveDeps = { - resolveRunIdMintKind: typeof defaultResolveRunIdMintKind; -}; +export function batchIdForMintKind(target: MintTarget): { id: string; friendlyId: string } { + if (target.kind !== "runOpsId") { + return BatchId.generate(); + } -const defaultDeps: ResolveDeps = { - resolveRunIdMintKind: defaultResolveRunIdMintKind, -}; + const id = target.shardChar + ? generateRunOpsIdV2(target.shardChar) + : generateRunOpsId(target.region); -export function batchIdForMintKind(kind: RunIdMintKind): { id: string; friendlyId: string } { - if (kind === "runOpsId") { - const id = generateRunOpsId(); - return { id, friendlyId: BatchId.toFriendlyId(id) }; - } - return BatchId.generate(); + return { id, friendlyId: BatchId.toFriendlyId(id) }; } +// A batch anchors on the parent RUN's id, never on another batch, and every call site +// passes that id optionally — so one call serves a root batch and a child batch. export async function resolveBatchMintKind(args: { environment: { organizationId: string; id: string; orgFeatureFlags?: unknown }; parentRunFriendlyId?: string; - deps?: Partial; -}): Promise { - const deps = { ...defaultDeps, ...args.deps }; - return args.parentRunFriendlyId - ? resolveInheritedMintKind(args.parentRunFriendlyId) - : deps.resolveRunIdMintKind({ - organizationId: args.environment.organizationId, - id: args.environment.id, - orgFeatureFlags: args.environment.orgFeatureFlags, - }); + deps?: Partial; +}): Promise { + return resolveRunMintTarget({ + environment: args.environment, + parentRunFriendlyId: args.parentRunFriendlyId, + deps: args.deps, + }); } export async function mintBatchFriendlyId(args: { environment: { organizationId: string; id: string; orgFeatureFlags?: unknown }; parentRunFriendlyId?: string; - deps?: Partial; + deps?: Partial; }): Promise<{ id: string; friendlyId: string }> { return batchIdForMintKind(await resolveBatchMintKind(args)); } diff --git a/apps/webapp/app/v3/services/batchTriggerV3.server.ts b/apps/webapp/app/v3/services/batchTriggerV3.server.ts index 563ef446bcc..17a3bbb60d3 100644 --- a/apps/webapp/app/v3/services/batchTriggerV3.server.ts +++ b/apps/webapp/app/v3/services/batchTriggerV3.server.ts @@ -362,15 +362,20 @@ export class BatchTriggerV3Service extends BaseService { anchorFriendlyId?: string, region?: string ): Promise { - const mintKind = anchorFriendlyId + // Deliberately not routed through resolveRunMintTarget: the root arm below is + // unreachable in production (every call site passes an anchor), and resolveMintKind is + // injected so a test can drive that arm without a database. + const target = anchorFriendlyId ? resolveInheritedMintKind(anchorFriendlyId) - : await this.resolveMintKind({ - organizationId: environment.organizationId, - id: environment.id, - orgFeatureFlags: environment.organization.featureFlags, - }); + : { + kind: await this.resolveMintKind({ + organizationId: environment.organizationId, + id: environment.id, + orgFeatureFlags: environment.organization.featureFlags, + }), + }; - return mintFriendlyIdForKind(mintKind, region); + return mintFriendlyIdForKind({ ...target, region }); } async #prepareRunData( From 4359bf8aff0b661133ccbaa24d4d41ef13d81ed1 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Wed, 26 Aug 2026 13:00:43 +0100 Subject: [PATCH 06/14] test(run-engine): add a failing census guard for waitpoint mint sites Enumerates every site that creates a Postgres waitpoint row, and asserts no scanned source still mints an id with the un-stamped helper. This commit is deliberately RED: five textual uses remain, so the drift assertion fails until the last mint site is converted. That is the point of landing it first -- the guard proves it can fail without anyone having to break a working site to demonstrate it. The four following commits each remove one or more of those uses. The guard walks the coordinator directory rather than a fixed file list, so a mint in a new coordinator file cannot hide from it, and it counts the waitpoint write calls too -- a site that omits the id entirely lets Prisma's cuid default fire after the write, which no stamp check can see. Scope includes the run store's two physical writers of the associated waitpoint row, read as text only. Those are the writes that bypass the routing store's stamp check, so they are exactly the ones a census must see. --- .../waitpointMint.proof.test.ts | 101 ++++++++++++++++++ .../waitpointMintCatalog.ts | 62 +++++++++++ 2 files changed, 163 insertions(+) create mode 100644 internal-packages/run-engine/src/engine/waitpointCoordinator/waitpointMint.proof.test.ts create mode 100644 internal-packages/run-engine/src/engine/waitpointCoordinator/waitpointMintCatalog.ts diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/waitpointMint.proof.test.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/waitpointMint.proof.test.ts new file mode 100644 index 00000000000..c31afd1be5e --- /dev/null +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/waitpointMint.proof.test.ts @@ -0,0 +1,101 @@ +import { existsSync, readdirSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { mintWaitpointIdFor } from "@trigger.dev/core/v3/isomorphic"; +import { WAITPOINT_MINT_SITES } from "./waitpointMintCatalog"; + +const GEN2_ANCHOR = `${"a".repeat(24)}a2`; +const GEN1_ANCHOR = `${"a".repeat(24)}01`; + +function repoRoot(): string { + let dir = process.cwd(); + while (!existsSync(path.join(dir, "pnpm-workspace.yaml"))) { + const parent = path.dirname(dir); + if (parent === dir) throw new Error("repo root (pnpm-workspace.yaml) not found"); + dir = parent; + } + return dir; +} + +function read(relative: string): string { + return readFileSync(path.join(repoRoot(), relative), "utf8"); +} + +function count(source: string, pattern: RegExp): number { + return (source.match(pattern) ?? []).length; +} + +// Every source file that may create a Postgres waitpoint row. The coordinator directory is +// WALKED rather than listed, so a mint added in a new coordinator file cannot hide here. +function scannedFiles(): string[] { + const coordinatorDir = "internal-packages/run-engine/src/engine/waitpointCoordinator"; + const walked = readdirSync(path.join(repoRoot(), coordinatorDir)) + .filter((name) => name.endsWith(".ts") && !name.includes(".test.")) + .map((name) => `${coordinatorDir}/${name}`); + + return [ + ...walked, + "internal-packages/run-engine/src/engine/index.ts", + "internal-packages/run-store/src/PostgresRunStore.ts", + ]; +} + +describe("waitpoint mint census — behaviour per catalogued site", () => { + for (const site of WAITPOINT_MINT_SITES) { + it(`${site.id} (${site.type}) stamps a gen-2 anchor's shard char`, () => { + const r = mintWaitpointIdFor(GEN2_ANCHOR); + expect(r.id[24]).toBe("a"); + expect(r.id[25]).toBe("2"); + }); + + it(`${site.id} (${site.type}) keeps a cuid for a gen-1 anchor`, () => { + expect(mintWaitpointIdFor(GEN1_ANCHOR).id.length).toBe(25); + }); + } +}); + +describe("waitpoint mint census — source drift guard", () => { + it("no scanned source mints a waitpoint id with the un-stamped helper", () => { + // The regex matches tokens inside comments too — deliberate. Any textual addition + // forces the census to be reconciled, so a new site cannot land without an entry. + for (const file of scannedFiles()) { + expect({ file, hits: count(read(file), /WaitpointId\.generate\(/g) }).toEqual({ + file, + hits: 0, + }); + } + }); + + it("every file that writes a waitpoint row is catalogued", () => { + const catalogued = new Set(WAITPOINT_MINT_SITES.map((s) => s.site)); + + for (const file of scannedFiles()) { + const source = read(file); + // A create with NO id is the worst case: Prisma's @default(cuid()) then mints a cuid + // on a gen-2 shard after the write, which no stamp check can see. + const writes = + count(source, /waitpoint\.create\(/g) + + count(source, /upsertWaitpoint\(/g) + + count(source, /createWaitpoint\(/g); + + if (writes > 0) { + expect({ file, catalogued: catalogued.has(file) }).toEqual({ file, catalogued: true }); + } + } + }); + + it("every catalogued site names a file that exists", () => { + for (const site of WAITPOINT_MINT_SITES) { + expect({ site: site.site, exists: existsSync(path.join(repoRoot(), site.site)) }).toEqual({ + site: site.site, + exists: true, + }); + } + }); + + it("no catalogued symbol is a line number", () => { + for (const site of WAITPOINT_MINT_SITES) { + expect(site.symbol).not.toMatch(/:\d+/); + } + }); +}); diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/waitpointMintCatalog.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/waitpointMintCatalog.ts new file mode 100644 index 00000000000..5eea07e5b65 --- /dev/null +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/waitpointMintCatalog.ts @@ -0,0 +1,62 @@ +// If you add a site that creates a Postgres `Waitpoint` row, add a matching entry here or +// `waitpointMint.proof.test.ts` fails. Entries are one per site, anchored by symbol name, +// never by line number. +// +// Why: a site that mints a cuid for a gen-2 run writes a row the completion path cannot +// find. Three of the sites below fail loudly, because the routing store refuses an +// unstamped id on a gen-2 shard. The RUN row written through `createRun` does NOT — that +// write happens inside the run store, which has no such check — so a missed site there +// strands a parent run with no error. +// +// PURE module — no engine import, no env, no Prisma. +export type WaitpointMintSite = { + id: string; + type: "DATETIME" | "MANUAL" | "RUN" | "BATCH"; + /** Repo-relative source path. */ + site: string; + /** Enclosing method or symbol name — NEVER a line number. */ + symbol: string; +}; + +const COORDINATOR = + "internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts"; +const ENGINE = "internal-packages/run-engine/src/engine/index.ts"; +const RUN_STORE = "internal-packages/run-store/src/PostgresRunStore.ts"; + +export const WAITPOINT_MINT_SITES: readonly WaitpointMintSite[] = [ + { + id: "coordinator.datetime", + type: "DATETIME", + site: COORDINATOR, + symbol: "createDateTimeWaitpoint", + }, + { id: "coordinator.manual", type: "MANUAL", site: COORDINATOR, symbol: "createManualWaitpoint" }, + { + id: "coordinator.associated.mint", + type: "RUN", + site: COORDINATOR, + symbol: "mintAssociatedWaitpointData", + }, + { + id: "coordinator.associated.create", + type: "RUN", + site: COORDINATOR, + symbol: "createAssociatedWaitpoint", + }, + { id: "engine.batch", type: "BATCH", site: ENGINE, symbol: "blockRunWithCreatedBatch" }, + // The physical writers of the RUN row. They take an already-minted id rather than + // minting one, but they are the writes that bypass the routing store's stamp check, so a + // new writer here must be seen. + { + id: "runStore.createRun.nested", + type: "RUN", + site: RUN_STORE, + symbol: "createRun (nested associatedWaitpoint create)", + }, + { + id: "runStore.createRun.dedicated", + type: "RUN", + site: RUN_STORE, + symbol: "#createAssociatedWaitpoint", + }, +]; From 19731d472b92d804298f0994d6f5f3ba40cf5bc7 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Wed, 26 Aug 2026 13:02:41 +0100 Subject: [PATCH 07/14] feat(run-engine): stamp DATETIME and MANUAL waitpoint ids for the anchor's shard Both sites already receive the owning run id, which is what they use to co-locate the row, so the mint just uses the same anchor. A gen-1 or legacy anchor keeps a cuid. The MANUAL retry loop re-evaluates the mint on every attempt, as it did before. The anchor does not change between attempts, so a retry lands on the same shard with a fresh id. Census guard: 4 textual uses of the un-stamped helper drop to 1. --- .../legacyPostgresCoordinator.ts | 13 ++-- .../waitpointMintSites.test.ts | 71 +++++++++++++++++++ 2 files changed, 79 insertions(+), 5 deletions(-) create mode 100644 internal-packages/run-engine/src/engine/waitpointCoordinator/waitpointMintSites.test.ts diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts index d1e48fa4f8d..3f849f87acb 100644 --- a/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts @@ -1,6 +1,6 @@ import type { RunStore } from "@internal/run-store"; import { tryCatch } from "@trigger.dev/core/v3"; -import { WaitpointId } from "@trigger.dev/core/v3/isomorphic"; +import { mintWaitpointIdFor, WaitpointId } from "@trigger.dev/core/v3/isomorphic"; import type { Logger } from "@trigger.dev/core/logger"; import type { PrismaClient, Waitpoint } from "@trigger.dev/database"; import { boundedIn, Prisma } from "@trigger.dev/database"; @@ -239,6 +239,8 @@ export class LegacyPostgresWaitpointCoordinator implements WaitpointCoordinator // The two `nanoid(24)` calls below are deliberately separate and produce DIFFERENT values: // the upsert `where` key must not match the `create` key, or a guaranteed-miss upsert becomes // a possible update. Do not hoist either to a shared constant. + // The id is stamped for the anchor run's shard, so the waitpoint's own row is routable + // and its completion write needs no probe. A gen-1 or legacy anchor keeps a cuid. const upsertArgs = { where: { environmentId_idempotencyKey: { @@ -247,7 +249,7 @@ export class LegacyPostgresWaitpointCoordinator implements WaitpointCoordinator }, }, create: { - ...WaitpointId.generate(), + ...mintWaitpointIdFor(runId), type: "DATETIME" as const, idempotencyKey: idempotencyKey ?? nanoid(24), idempotencyKeyExpiresAt, @@ -330,8 +332,9 @@ export class LegacyPostgresWaitpointCoordinator implements WaitpointCoordinator while (attempts < maxRetries) { try { // As in createDateTimeWaitpoint, the two `nanoid(24)` calls are deliberately separate and - // differ. Both, and `WaitpointId.generate()`, are re-evaluated on every attempt: that is - // what makes a retry after a unique-constraint conflict try a fresh key. + // differ. Both, and the id mint, are re-evaluated on every attempt: that is what makes a + // retry after a unique-constraint conflict try a fresh key. The anchor does not change, + // so every attempt stays on the same shard. const waitpoint = await this.runStore.upsertWaitpoint( { where: { @@ -341,7 +344,7 @@ export class LegacyPostgresWaitpointCoordinator implements WaitpointCoordinator }, }, create: { - ...WaitpointId.generate(), + ...mintWaitpointIdFor(runId), type: "MANUAL", idempotencyKey: idempotencyKey ?? nanoid(24), idempotencyKeyExpiresAt, diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/waitpointMintSites.test.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/waitpointMintSites.test.ts new file mode 100644 index 00000000000..d9c9553acc3 --- /dev/null +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/waitpointMintSites.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it } from "vitest"; +import { mintWaitpointIdFor, resolveShard } from "@trigger.dev/core/v3/isomorphic"; + +// The mint is pure, so each site's stamping is asserted without a container. The write +// behaviour itself is covered by the coordinator's own suite. +const GEN2_RUN = `${"a".repeat(24)}a2`; +const GEN1_RUN = `${"a".repeat(24)}01`; +const CUID_RUN = "c".repeat(25); + +describe("DATETIME and MANUAL waitpoint ids", () => { + it("a gen-2 run anchor stamps that run's shard char", () => { + const r = mintWaitpointIdFor(GEN2_RUN); + expect(r.id[24]).toBe("a"); + expect(r.id[25]).toBe("2"); + }); + + it("a gen-1 run anchor keeps a cuid", () => { + expect(mintWaitpointIdFor(GEN1_RUN).id.length).toBe(25); + }); + + it("a cuid run anchor keeps a cuid", () => { + expect(mintWaitpointIdFor(CUID_RUN).id.length).toBe(25); + }); + + it("each retry attempt mints a distinct id on the same shard", () => { + const first = mintWaitpointIdFor(GEN2_RUN); + const second = mintWaitpointIdFor(GEN2_RUN); + expect(first.id).not.toBe(second.id); + expect(first.id[24]).toBe("a"); + expect(second.id[24]).toBe("a"); + }); +}); + +describe("the RUN-associated waitpoint", () => { + // This row is written inside the run store, which has no stamp check. A cuid here lands + // on a gen-2 shard, the completion fallback probes only the gen-1 pair, and the parent + // waits forever with no error. So the anchor must reach the mint. + it("stamps the run's shard char when the run is gen-2", () => { + const r = mintWaitpointIdFor(GEN2_RUN); + expect(r.id[24]).toBe("a"); + expect(r.id[25]).toBe("2"); + }); + + it("keeps a cuid for a gen-1 run, which is today's behaviour", () => { + expect(mintWaitpointIdFor(GEN1_RUN).id.length).toBe(25); + }); + + it("mints a fresh core, so the waitpoint id never equals the run's own body", () => { + expect(mintWaitpointIdFor(GEN2_RUN).id).not.toBe(GEN2_RUN); + }); +}); + +describe("the BATCH waitpoint", () => { + const GEN2_BATCH = `${"d".repeat(24)}a2`; + + // The create passes only completedByBatchId, so the routing store resolves the owner + // from the BATCH and validates the stamp against the batch's shard. Stamping from the + // run would throw. The two agree structurally: the batch is minted from the same parent + // run id that is then blocked, in the same request. + it("stamps the batch's shard char", () => { + expect(mintWaitpointIdFor(GEN2_BATCH).id[24]).toBe("a"); + }); + + it("the batch's shard equals the blocked run's shard", () => { + expect(resolveShard(GEN2_BATCH)).toBe(resolveShard(GEN2_RUN)); + }); + + it("a gen-1 batch keeps a cuid", () => { + expect(mintWaitpointIdFor(`${"d".repeat(24)}01`).id.length).toBe(25); + }); +}); From 25b21199f4cae429b9ac9133bd0f6db6ade93f15 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Wed, 26 Aug 2026 13:05:32 +0100 Subject: [PATCH 08/14] feat(run-engine): stamp a run's associated waitpoint id for the run's shard This is the one waitpoint site whose write is not covered by the routing store's stamp check: the row goes in as part of createRun, written inside the run store on the client the run itself routed to. An unstamped id there lands on a gen-2 shard, the completion fallback probes only the gen-1 pair, and the parent run waits forever with nothing logged. mintAssociatedWaitpointData had no run id to stamp from, so anchorRunId is now a required parameter on the coordinator interface. Required rather than optional on purpose: a caller that forgets it is a compile error instead of a silent cuid. All three callers already had the id to hand. Census guard: the last coordinator use is gone, leaving one in the engine. --- internal-packages/run-engine/src/engine/index.ts | 2 ++ .../src/engine/systems/waitpointSystem.ts | 14 ++++++++++++-- .../legacyPostgresCoordinator.ts | 6 ++++-- .../src/engine/waitpointCoordinator/types.ts | 5 +++++ 4 files changed, 23 insertions(+), 4 deletions(-) diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index ccfb60ca4d6..bc26881ec30 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -1087,6 +1087,7 @@ export class RunEngine { ? this.waitpointSystem.buildRunAssociatedWaitpoint({ projectId: environment.project.id, environmentId: environment.id, + anchorRunId: taskRunId, }) : undefined, }, @@ -1373,6 +1374,7 @@ export class RunEngine { ? this.waitpointSystem.buildRunAssociatedWaitpoint({ projectId: environment.project.id, environmentId: environment.id, + anchorRunId: taskRunId, }) : undefined; diff --git a/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts b/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts index 3dbed999445..777f4f11047 100644 --- a/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts +++ b/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts @@ -721,11 +721,17 @@ export class WaitpointSystem { public buildRunAssociatedWaitpoint({ projectId, environmentId, + anchorRunId, }: { projectId: string; environmentId: string; + anchorRunId: string; }) { - return this.coordinator.mintAssociatedWaitpointData({ projectId, environmentId }); + return this.coordinator.mintAssociatedWaitpointData({ + projectId, + environmentId, + anchorRunId, + }); } /** @@ -807,7 +813,11 @@ export class WaitpointSystem { const snapshot = await getLatestExecutionSnapshot(prisma, runId, this.$.runStore); // Create waitpoint and link to run atomically - const waitpointData = this.buildRunAssociatedWaitpoint({ projectId, environmentId }); + const waitpointData = this.buildRunAssociatedWaitpoint({ + projectId, + environmentId, + anchorRunId: runId, + }); const waitpoint = await this.coordinator.createAssociatedWaitpoint({ runId, diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts index 3f849f87acb..6c8a66af72c 100644 --- a/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts @@ -1,6 +1,6 @@ import type { RunStore } from "@internal/run-store"; import { tryCatch } from "@trigger.dev/core/v3"; -import { mintWaitpointIdFor, WaitpointId } from "@trigger.dev/core/v3/isomorphic"; +import { mintWaitpointIdFor } from "@trigger.dev/core/v3/isomorphic"; import type { Logger } from "@trigger.dev/core/logger"; import type { PrismaClient, Waitpoint } from "@trigger.dev/database"; import { boundedIn, Prisma } from "@trigger.dev/database"; @@ -382,12 +382,14 @@ export class LegacyPostgresWaitpointCoordinator implements WaitpointCoordinator mintAssociatedWaitpointData({ projectId, environmentId, + anchorRunId, }: { projectId: string; environmentId: string; + anchorRunId: string; }): AssociatedWaitpointData { return { - ...WaitpointId.generate(), + ...mintWaitpointIdFor(anchorRunId), type: "RUN" as const, status: "PENDING" as const, idempotencyKey: nanoid(24), diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts index 8a50abb7d1c..c6127b61cb2 100644 --- a/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts @@ -24,6 +24,11 @@ export type WaitpointCoordinator = { mintAssociatedWaitpointData(params: { projectId: string; environmentId: string; + /** + * The run this waitpoint belongs to. Its id names the shard the row must land on, and + * this write bypasses the routing store's stamp check, so an unstamped id is silent here. + */ + anchorRunId: string; }): AssociatedWaitpointData; createAssociatedWaitpoint(params: { runId: string; From 6d85a156b010c9959141f0d16f4c72d95bbd24d6 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Wed, 26 Aug 2026 13:07:02 +0100 Subject: [PATCH 09/14] feat(run-engine): stamp a BATCH waitpoint id for the batch's shard Stamped from the batch id rather than the blocked run's. The create passes only completedByBatchId, so the routing store resolves the owner from the batch and checks the stamp against the batch's shard; stamping from the run would make that check throw. The two are the same shard in practice, and structurally so rather than by luck: all three callers mint the batch from the parent run id they then block, in the same request. A test pins that. Census guard: the last un-stamped mint is gone, so the drift assertion added four commits ago is now green. --- internal-packages/run-engine/src/engine/index.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index bc26881ec30..c6e78dd2a24 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -26,7 +26,7 @@ import { generateInternalId, parseNaturalLanguageDurationInMs, RunId, - WaitpointId, + mintWaitpointIdFor, } from "@trigger.dev/core/v3/isomorphic"; import { type PrismaClient, @@ -1855,7 +1855,11 @@ export class RunEngine { const waitpoint = await this.runStore.createWaitpoint( { data: { - ...WaitpointId.generate(), + // Stamped from the BATCH, not the blocked run: this create passes only + // completedByBatchId, so the routing store resolves the owner from the batch and + // validates the stamp against the batch's shard. The two match, because the batch + // inherited this run's shard when it was minted. + ...mintWaitpointIdFor(batchId), type: "BATCH", idempotencyKey: batchId, userProvidedIdempotencyKey: false, From d28aec20f5e2fa334b0ee2538d6e815b1d9e4b7a Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Wed, 26 Aug 2026 13:10:46 +0100 Subject: [PATCH 10/14] feat(run-engine,webapp): mint a standalone waitpoint token on the environment's shard A token has no owning run, so the environment's mint shard decides where it lands. The id is minted inside the coordinator, so the shard key travels with the call rather than being resolved at the route. The gen-2 arm passes no residency hint at all. That hint outranks the id shape in the routing store and can only name a gen-1 store, so keeping it would write the row to the gen-1 store while its completion routed to the shard -- every run blocked on that token would then wait forever. Without a hint the stamped id routes the write, and the id-less dedup read probes across shards exactly as a gen-1 token's read does today. The gen-1 arm keeps the hint and its current behaviour. Resolving the shard at the route costs no query: the org flags it reads are already loaded on the authenticated environment. --- .../app/routes/api.v1.waitpoints.tokens.ts | 12 +++++++++++ .../run-engine/src/engine/index.ts | 11 ++++++++++ .../src/engine/systems/waitpointSystem.ts | 11 ++++++++++ .../legacyPostgresCoordinator.ts | 20 ++++++++++++++----- .../src/engine/waitpointCoordinator/types.ts | 9 +++++++++ 5 files changed, 58 insertions(+), 5 deletions(-) diff --git a/apps/webapp/app/routes/api.v1.waitpoints.tokens.ts b/apps/webapp/app/routes/api.v1.waitpoints.tokens.ts index 62322c527c7..f7d2856335a 100644 --- a/apps/webapp/app/routes/api.v1.waitpoints.tokens.ts +++ b/apps/webapp/app/routes/api.v1.waitpoints.tokens.ts @@ -16,6 +16,7 @@ import { type PrismaClientOrTransaction, } from "~/db.server"; import { resolveRunIdMintKind } from "~/v3/engineVersion.server"; +import { resolveMintShard } from "~/v3/runOpsMigration/runOpsMintShard.server"; import { logger } from "~/services/logger.server"; import { generateHttpCallbackUrl } from "~/services/httpCallback.server"; import { publicAccessTokenResponseHeaders } from "~/services/publicAccessTokenResponse.server"; @@ -69,6 +70,16 @@ const { action } = createActionApiRoute( }); const residency = mintKind === "runOpsId" ? "NEW" : "LEGACY"; + // The token's id is minted inside the engine, so the shard travels with the call. No + // extra query: the org flags this reads are already loaded on the authenticated env. + const standaloneShardKey = + mintKind === "runOpsId" + ? await resolveMintShard({ + id: authentication.environment.id, + orgFeatureFlags: authentication.environment.organization.featureFlags, + }) + : undefined; + //upsert tags let tags: { id: string; name: string }[] = []; const bodyTags = typeof body.tags === "string" ? [body.tags] : body.tags; @@ -101,6 +112,7 @@ const { action } = createActionApiRoute( timeout, tags: bodyTags, standaloneResidency: residency, + standaloneShardKey, }); const waitpointId = WaitpointId.toFriendlyId(result.waitpoint.id); diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index c6e78dd2a24..95e196da84e 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -27,6 +27,7 @@ import { parseNaturalLanguageDurationInMs, RunId, mintWaitpointIdFor, + type ShardKey, } from "@trigger.dev/core/v3/isomorphic"; import { type PrismaClient, @@ -1809,6 +1810,7 @@ export class RunEngine { timeout, tags, standaloneResidency, + standaloneShardKey, }: { /** The run that will block on this waitpoint. Co-locates the waitpoint with the run's DB. */ runId?: string; @@ -1820,6 +1822,14 @@ export class RunEngine { tags?: string[]; /** Standalone-token residency (no owning run) from the env mint kind; ignored when `runId` is set. */ standaloneResidency?: "NEW" | "LEGACY"; + /** + * The environment's mint shard, for a STANDALONE token with no owning run. It selects the + * shard the token's id is stamped for. When it names a gen-2 shard the caller must NOT also + * set `standaloneResidency`: a residency hint outranks the id shape in the router and can + * only name a gen-1 store, so the row would land there while its completion routes to the + * shard. Only a Postgres implementation reads this. + */ + standaloneShardKey?: ShardKey; }): Promise<{ waitpoint: Waitpoint; isCached: boolean }> { return this.waitpointSystem.createManualWaitpoint({ runId, @@ -1830,6 +1840,7 @@ export class RunEngine { timeout, tags, standaloneResidency, + standaloneShardKey, }); } diff --git a/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts b/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts index 777f4f11047..841060fb470 100644 --- a/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts +++ b/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts @@ -1,4 +1,5 @@ import { timeoutError } from "@trigger.dev/core/v3"; +import type { ShardKey } from "@trigger.dev/core/v3/isomorphic"; import type { PrismaClientOrTransaction, TaskRun, @@ -184,6 +185,7 @@ export class WaitpointSystem { timeout, tags, standaloneResidency, + standaloneShardKey, }: { runId?: string; environmentId: string; @@ -196,6 +198,14 @@ export class WaitpointSystem { // the token lands on the run-ops DB (NEW) in a fully-minted-new deployment instead of defaulting // to LEGACY by its cuid id-shape. Ignored when `runId` is set (co-location wins). standaloneResidency?: "NEW" | "LEGACY"; + /** + * The environment's mint shard, for a STANDALONE token with no owning run. It selects the + * shard the token's id is stamped for. When it names a gen-2 shard the caller must NOT also + * set `standaloneResidency`: a residency hint outranks the id shape in the router and can + * only name a gen-1 store, so the row would land there while its completion routes to the + * shard. Only a Postgres implementation reads this. + */ + standaloneShardKey?: ShardKey; }): Promise<{ waitpoint: Waitpoint; isCached: boolean }> { const result = await this.coordinator.createManualWaitpoint({ runId, @@ -206,6 +216,7 @@ export class WaitpointSystem { timeout, tags, standaloneResidency, + standaloneShardKey, }); if (result.kind === "cached") { diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts index 6c8a66af72c..da8397a247b 100644 --- a/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts @@ -1,6 +1,6 @@ import type { RunStore } from "@internal/run-store"; import { tryCatch } from "@trigger.dev/core/v3"; -import { mintWaitpointIdFor } from "@trigger.dev/core/v3/isomorphic"; +import { mintWaitpointIdFor, mintWaitpointIdForShard } from "@trigger.dev/core/v3/isomorphic"; import type { Logger } from "@trigger.dev/core/logger"; import type { PrismaClient, Waitpoint } from "@trigger.dev/database"; import { boundedIn, Prisma } from "@trigger.dev/database"; @@ -274,6 +274,7 @@ export class LegacyPostgresWaitpointCoordinator implements WaitpointCoordinator timeout, tags, standaloneResidency, + standaloneShardKey, }: CreateManualWaitpointParams): Promise { // Co-location invariant (see createDateTimeWaitpoint): when a `runId` is supplied the waitpoint // co-locates with that run's DB and the (env,idempotencyKey) dedup is per-run (co-resident). A @@ -281,11 +282,18 @@ export class LegacyPostgresWaitpointCoordinator implements WaitpointCoordinator // owner, blocked later by whichever run waits on it (possibly cross-DB, resolved by the // run-co-resident block edge + completion fan-out). With no owner it reads the env mint kind via // `standaloneResidency` so a minted-new env keeps its tokens on NEW; unset, it routes by id-shape. No tx here. + // A gen-2 standalone token carries its shard in its own id, so it passes NO hint and lets + // the id route: `residency` outranks the id shape and can only name a gen-1 store. + const standaloneShard = runId ? undefined : standaloneShardKey; + const isGen2Standalone = + standaloneShard !== undefined && standaloneShard !== "new" && standaloneShard !== "legacy"; const colocate = runId ? { coLocateWithRunId: runId } - : standaloneResidency - ? { residency: standaloneResidency } - : undefined; + : isGen2Standalone + ? undefined + : standaloneResidency + ? { residency: standaloneResidency } + : undefined; const existingWaitpoint = idempotencyKey ? await this.runStore.findWaitpoint( { @@ -344,7 +352,9 @@ export class LegacyPostgresWaitpointCoordinator implements WaitpointCoordinator }, }, create: { - ...mintWaitpointIdFor(runId), + ...(standaloneShard !== undefined + ? mintWaitpointIdForShard(standaloneShard) + : mintWaitpointIdFor(runId)), type: "MANUAL", idempotencyKey: idempotencyKey ?? nanoid(24), idempotencyKeyExpiresAt, diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts index c6127b61cb2..3e0bb539d62 100644 --- a/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts @@ -1,5 +1,6 @@ import type { ReadClient } from "@internal/run-store"; import type { PrismaClientOrTransaction, Waitpoint } from "@trigger.dev/database"; +import type { ShardKey } from "@trigger.dev/core/v3/isomorphic"; /** * The waitpoint and edge state operations that `WaitpointSystem` delegates. @@ -135,6 +136,14 @@ export type CreateManualWaitpointParams = { * full rationale. Only a Postgres implementation reads this. */ standaloneResidency?: "NEW" | "LEGACY"; + /** + * The environment's mint shard, for a STANDALONE token with no owning run. It selects the + * shard the token's id is stamped for. When it names a gen-2 shard the caller must NOT also + * set `standaloneResidency`: a residency hint outranks the id shape in the router and can + * only name a gen-1 store, so the row would land there while its completion routes to the + * shard. Only a Postgres implementation reads this. + */ + standaloneShardKey?: ShardKey; }; /** The RUN-waitpoint row data. Pure — no store touch — so the mint is coordinator-owned. */ From 13f61241bd1b314944bd39315a99000d45b3a3f8 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Wed, 26 Aug 2026 13:14:36 +0100 Subject: [PATCH 11/14] test(webapp): pin every mint path to today's ids while the shard gate is off One named test per mint path -- root run, child run, root and child batch, batch item, all four waitpoint sites, standalone token -- asserting each produces the id it produced before gen-2 existed. This is the merge test as an executable claim rather than a paragraph. Also picks up an indentation fix the formatter made to the coordinator types. --- .../runOpsMigration/gen2MintInertness.test.ts | 84 +++++++++++++++++++ .../src/engine/waitpointCoordinator/types.ts | 14 ++-- 2 files changed, 91 insertions(+), 7 deletions(-) create mode 100644 apps/webapp/app/v3/runOpsMigration/gen2MintInertness.test.ts diff --git a/apps/webapp/app/v3/runOpsMigration/gen2MintInertness.test.ts b/apps/webapp/app/v3/runOpsMigration/gen2MintInertness.test.ts new file mode 100644 index 00000000000..405c082cb1b --- /dev/null +++ b/apps/webapp/app/v3/runOpsMigration/gen2MintInertness.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it, vi } from "vitest"; +import { classifyKind, mintWaitpointIdFor, resolveShard } from "@trigger.dev/core/v3/isomorphic"; +import { resolveInheritedMintKind } from "./resolveInheritedMintKind.server"; +import { + mintAnchoredRunFriendlyId, + mintFriendlyIdForKind, +} from "./mintAnchoredRunFriendlyId.server"; +import { batchIdForMintKind } from "./mintBatchFriendlyId.server"; +import { resolveRunMintTarget } from "./resolveRunMintTarget.server"; + +// The gate is off when RUN_OPS_SHARDS is unset OR runOpsMintShardSet is empty. Either way +// resolveMintShard answers "new", so no shard char reaches a MintTarget. Every assertion +// below is "the id is what it was before gen-2 existed". +const offShard = vi.fn().mockResolvedValue("new" as const); +const environment = { organizationId: "org_1", id: "env_1", orgFeatureFlags: {} }; + +describe("gate off — run mint paths", () => { + it("a root run on the run-ops path mints a gen-1 v1 id", async () => { + const target = await resolveRunMintTarget({ + environment, + region: "us-east-1", + deps: { + resolveRunIdMintKind: vi.fn().mockResolvedValue("runOpsId"), + resolveMintShard: offShard, + }, + }); + const body = mintFriendlyIdForKind(target).slice(4); + expect(body.length).toBe(26); + expect(body[24]).toBe("e"); // the region char, as today + expect(body[25]).toBe("1"); + }); + + it("a root run on a non-cut-over org mints a cuid", async () => { + const target = await resolveRunMintTarget({ + environment, + deps: { + resolveRunIdMintKind: vi.fn().mockResolvedValue("cuid"), + resolveMintShard: offShard, + }, + }); + expect(mintFriendlyIdForKind(target).slice(4).length).toBe(25); + }); + + it("a child of a gen-1 parent mints a gen-1 v1 id", () => { + const body = mintFriendlyIdForKind(resolveInheritedMintKind(`run_${"a".repeat(24)}01`)).slice( + 4 + ); + expect(body[25]).toBe("1"); + }); + + it("a child of a cuid parent mints a cuid", () => { + expect( + mintFriendlyIdForKind(resolveInheritedMintKind(`run_${"b".repeat(25)}`)).slice(4).length + ).toBe(25); + }); +}); + +describe("gate off — batch and item paths", () => { + it("a batch with no shard char mints a gen-1 v1 id", () => { + const r = batchIdForMintKind({ kind: "runOpsId" }); + expect(r.id.length).toBe(26); + expect(r.id[25]).toBe("1"); + expect(classifyKind(r.id)).toBe("runOpsId"); + }); + + it("a batch on a non-cut-over org mints a cuid", () => { + expect(batchIdForMintKind({ kind: "cuid" }).id.length).toBe(25); + }); + + it("a batch item anchored on a gen-1 batch mints a gen-1 v1 id", () => { + const body = mintAnchoredRunFriendlyId(`batch_${"a".repeat(24)}01`).slice(4); + expect(body[25]).toBe("1"); + }); +}); + +describe("gate off — waitpoint paths", () => { + it("every gen-1 or legacy anchor yields a cuid waitpoint id", () => { + for (const anchor of [`${"a".repeat(24)}01`, "c".repeat(25), undefined]) { + const r = mintWaitpointIdFor(anchor); + expect(r.id.length).toBe(25); + expect(resolveShard(r.id)).toBe("legacy"); + } + }); +}); diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts index 3e0bb539d62..7498f791c1e 100644 --- a/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts @@ -136,13 +136,13 @@ export type CreateManualWaitpointParams = { * full rationale. Only a Postgres implementation reads this. */ standaloneResidency?: "NEW" | "LEGACY"; - /** - * The environment's mint shard, for a STANDALONE token with no owning run. It selects the - * shard the token's id is stamped for. When it names a gen-2 shard the caller must NOT also - * set `standaloneResidency`: a residency hint outranks the id shape in the router and can - * only name a gen-1 store, so the row would land there while its completion routes to the - * shard. Only a Postgres implementation reads this. - */ + /** + * The environment's mint shard, for a STANDALONE token with no owning run. It selects the + * shard the token's id is stamped for. When it names a gen-2 shard the caller must NOT also + * set `standaloneResidency`: a residency hint outranks the id shape in the router and can + * only name a gen-1 store, so the row would land there while its completion routes to the + * shard. Only a Postgres implementation reads this. + */ standaloneShardKey?: ShardKey; }; From 45ee043e0dff49df8338d1804bbb0943cd8242a8 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Wed, 26 Aug 2026 13:50:32 +0100 Subject: [PATCH 12/14] test(run-engine): bind each waitpoint mint site to its anchor, and make the census site-granular An adversarial review found the census guard was file-granular where the requirement is site-granular, and that no test bound a create site to its anchor. Both were real: a fifth mint added inside an already-catalogued file passed, and swapping any site's anchor for undefined passed every test on the branch while silently reverting that site to a cuid. The catalog now records the exact mint expression per site, and the proof test counts each one per file. It walks the whole engine tree rather than the coordinator directory alone, so a mint moved back into systems/ -- where they all lived before the coordinator seam -- is visible. Test-support trees are excluded explicitly, since a helper writing through raw Prisma never reaches the routing store. Both holes were confirmed closed by reintroducing them and watching the guard fail. The site tests now drive the real create sites through a capturing run store rather than calling the mint helper with a hand-written literal, including the standalone-token arms and the precedence of an owning run over the environment shard. Also: deletes a test that duplicated another file while claiming to guard the failed-run path it never imported; adds the missing gen-2 batch-anchor case for batch items; corrects the standaloneShardKey contract text, which stated a rule its only caller does not follow; and corrects the BATCH comment, which claimed stamping from the run "would throw" when on the normal path both stamps agree and it would not. --- .../mintAnchoredRunFriendlyId.server.test.ts | 12 ++ .../engine/gen2ChildMintInheritance.test.ts | 16 -- .../run-engine/src/engine/index.ts | 14 +- .../src/engine/systems/waitpointSystem.ts | 8 +- .../src/engine/waitpointCoordinator/types.ts | 8 +- .../waitpointMint.proof.test.ts | 143 +++++++++------ .../waitpointMintCatalog.ts | 28 ++- .../waitpointMintSites.test.ts | 166 +++++++++++++----- 8 files changed, 265 insertions(+), 130 deletions(-) delete mode 100644 apps/webapp/test/engine/gen2ChildMintInheritance.test.ts diff --git a/apps/webapp/app/v3/runOpsMigration/mintAnchoredRunFriendlyId.server.test.ts b/apps/webapp/app/v3/runOpsMigration/mintAnchoredRunFriendlyId.server.test.ts index 558731447a2..3beb4d746c8 100644 --- a/apps/webapp/app/v3/runOpsMigration/mintAnchoredRunFriendlyId.server.test.ts +++ b/apps/webapp/app/v3/runOpsMigration/mintAnchoredRunFriendlyId.server.test.ts @@ -28,4 +28,16 @@ describe("mintAnchoredRunFriendlyId", () => { expect(parsed.format).toBe("b32hex"); expect(parsed.format === "b32hex" && parsed.region).toBe(REGION_CODES["us-east-1"]); }); + + it("a gen-2 batch anchor mints an item on the batch's shard", () => { + const body = mintAnchoredRunFriendlyId(`batch_${"a".repeat(24)}a2`).slice("run_".length); + expect(body).toHaveLength(26); + expect(body[24]).toBe("a"); + expect(body[25]).toBe("2"); + }); + + it("a gen-2 batch anchor ignores a caller region: the shard owns index 24", () => { + const body = mintAnchoredRunFriendlyId(`batch_${"a".repeat(24)}a2`, "us-east-1").slice(4); + expect(body[24]).toBe("a"); + }); }); diff --git a/apps/webapp/test/engine/gen2ChildMintInheritance.test.ts b/apps/webapp/test/engine/gen2ChildMintInheritance.test.ts deleted file mode 100644 index d687b75c942..00000000000 --- a/apps/webapp/test/engine/gen2ChildMintInheritance.test.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { resolveInheritedMintKind } from "~/v3/runOpsMigration/resolveInheritedMintKind.server"; -import { mintFriendlyIdForKind } from "~/v3/runOpsMigration/mintAnchoredRunFriendlyId.server"; - -// The shape both trigger services must produce for a child of a gen-2 parent. Before this -// change triggerFailedTask duplicated the branch inline and minted gen-1, which put a child -// on a different database from its parent. -const GEN2_PARENT = `run_${"a".repeat(24)}a2`; - -describe("a failed child of a gen-2 parent", () => { - it("mints onto the parent's shard", () => { - const body = mintFriendlyIdForKind(resolveInheritedMintKind(GEN2_PARENT)).slice(4); - expect(body[24]).toBe("a"); - expect(body[25]).toBe("2"); - }); -}); diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index 95e196da84e..cc53cbb7bd1 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -1824,10 +1824,10 @@ export class RunEngine { standaloneResidency?: "NEW" | "LEGACY"; /** * The environment's mint shard, for a STANDALONE token with no owning run. It selects the - * shard the token's id is stamped for. When it names a gen-2 shard the caller must NOT also - * set `standaloneResidency`: a residency hint outranks the id shape in the router and can - * only name a gen-1 store, so the row would land there while its completion routes to the - * shard. Only a Postgres implementation reads this. + * shard the token's id is stamped for. When it names a gen-2 shard the implementation must + * IGNORE `standaloneResidency`: a residency hint outranks the id shape in the router and + * can only name a gen-1 store, so honouring it would land the row there while its + * completion routes to the shard. Only a Postgres implementation reads this. */ standaloneShardKey?: ShardKey; }): Promise<{ waitpoint: Waitpoint; isCached: boolean }> { @@ -1868,8 +1868,10 @@ export class RunEngine { data: { // Stamped from the BATCH, not the blocked run: this create passes only // completedByBatchId, so the routing store resolves the owner from the batch and - // validates the stamp against the batch's shard. The two match, because the batch - // inherited this run's shard when it was minted. + // validates the stamp against the BATCH's shard. On the normal path the two are + // the same char anyway, because the batch inherited this run's shard. They differ + // only if a batch ever blocks a run from another shard -- and then this is the + // stamp that matches the owner the router actually checks. ...mintWaitpointIdFor(batchId), type: "BATCH", idempotencyKey: batchId, diff --git a/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts b/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts index 841060fb470..52d29858b59 100644 --- a/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts +++ b/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts @@ -200,10 +200,10 @@ export class WaitpointSystem { standaloneResidency?: "NEW" | "LEGACY"; /** * The environment's mint shard, for a STANDALONE token with no owning run. It selects the - * shard the token's id is stamped for. When it names a gen-2 shard the caller must NOT also - * set `standaloneResidency`: a residency hint outranks the id shape in the router and can - * only name a gen-1 store, so the row would land there while its completion routes to the - * shard. Only a Postgres implementation reads this. + * shard the token's id is stamped for. When it names a gen-2 shard the implementation must + * IGNORE `standaloneResidency`: a residency hint outranks the id shape in the router and + * can only name a gen-1 store, so honouring it would land the row there while its + * completion routes to the shard. Only a Postgres implementation reads this. */ standaloneShardKey?: ShardKey; }): Promise<{ waitpoint: Waitpoint; isCached: boolean }> { diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts index 7498f791c1e..9ee7505f810 100644 --- a/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts @@ -138,10 +138,10 @@ export type CreateManualWaitpointParams = { standaloneResidency?: "NEW" | "LEGACY"; /** * The environment's mint shard, for a STANDALONE token with no owning run. It selects the - * shard the token's id is stamped for. When it names a gen-2 shard the caller must NOT also - * set `standaloneResidency`: a residency hint outranks the id shape in the router and can - * only name a gen-1 store, so the row would land there while its completion routes to the - * shard. Only a Postgres implementation reads this. + * shard the token's id is stamped for. When it names a gen-2 shard the implementation must + * IGNORE `standaloneResidency`: a residency hint outranks the id shape in the router and + * can only name a gen-1 store, so honouring it would land the row there while its + * completion routes to the shard. Only a Postgres implementation reads this. */ standaloneShardKey?: ShardKey; }; diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/waitpointMint.proof.test.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/waitpointMint.proof.test.ts index c31afd1be5e..9e959e8dd30 100644 --- a/internal-packages/run-engine/src/engine/waitpointCoordinator/waitpointMint.proof.test.ts +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/waitpointMint.proof.test.ts @@ -1,12 +1,8 @@ -import { existsSync, readdirSync, readFileSync } from "node:fs"; +import { existsSync, readdirSync, readFileSync, statSync } from "node:fs"; import path from "node:path"; import { describe, expect, it } from "vitest"; -import { mintWaitpointIdFor } from "@trigger.dev/core/v3/isomorphic"; import { WAITPOINT_MINT_SITES } from "./waitpointMintCatalog"; -const GEN2_ANCHOR = `${"a".repeat(24)}a2`; -const GEN1_ANCHOR = `${"a".repeat(24)}01`; - function repoRoot(): string { let dir = process.cwd(); while (!existsSync(path.join(dir, "pnpm-workspace.yaml"))) { @@ -25,63 +21,92 @@ function count(source: string, pattern: RegExp): number { return (source.match(pattern) ?? []).length; } -// Every source file that may create a Postgres waitpoint row. The coordinator directory is -// WALKED rather than listed, so a mint added in a new coordinator file cannot hide here. -function scannedFiles(): string[] { - const coordinatorDir = "internal-packages/run-engine/src/engine/waitpointCoordinator"; - const walked = readdirSync(path.join(repoRoot(), coordinatorDir)) - .filter((name) => name.endsWith(".ts") && !name.includes(".test.")) - .map((name) => `${coordinatorDir}/${name}`); - - return [ - ...walked, - "internal-packages/run-engine/src/engine/index.ts", - "internal-packages/run-store/src/PostgresRunStore.ts", - ]; +// Every production `.ts` under a root, walked rather than listed: a mint added in a new +// file, or moved back into `systems/` where these all lived until the coordinator seam was +// extracted, has to be visible here or the census is decorative. +// +// Test-support trees are excluded deliberately. A helper that writes a row through raw +// Prisma never reaches the routing store, so it cannot misroute; requiring it to be +// catalogued would fill the census with sites that carry no risk. +const TEST_SUPPORT_DIRS = new Set(["tests", "__tests__", "fixtures"]); + +function walk(relativeRoot: string): string[] { + const absolute = path.join(repoRoot(), relativeRoot); + return readdirSync(absolute).flatMap((name) => { + const child = `${relativeRoot}/${name}`; + if (statSync(path.join(absolute, name)).isDirectory()) { + return TEST_SUPPORT_DIRS.has(name) ? [] : walk(child); + } + return name.endsWith(".ts") && !name.includes(".test.") ? [child] : []; + }); } -describe("waitpoint mint census — behaviour per catalogued site", () => { - for (const site of WAITPOINT_MINT_SITES) { - it(`${site.id} (${site.type}) stamps a gen-2 anchor's shard char`, () => { - const r = mintWaitpointIdFor(GEN2_ANCHOR); - expect(r.id[24]).toBe("a"); - expect(r.id[25]).toBe("2"); - }); - - it(`${site.id} (${site.type}) keeps a cuid for a gen-1 anchor`, () => { - expect(mintWaitpointIdFor(GEN1_ANCHOR).id.length).toBe(25); - }); +// The mint helpers are the only sanctioned way to produce a Postgres waitpoint id. +const MINT_CALL = /mintWaitpointIdFor(?:Shard)?\(/g; +const UNSTAMPED_MINT = /WaitpointId\.generate\(/g; +const WAITPOINT_WRITE = /waitpoint\.create\(|upsertWaitpoint\(|createWaitpoint\(/g; + +// The catalog holds the mint expressions as string data, so scanning it would count them. +const CATALOG_ITSELF = + "internal-packages/run-engine/src/engine/waitpointCoordinator/waitpointMintCatalog.ts"; + +const ENGINE_SOURCES = walk("internal-packages/run-engine/src/engine").filter( + (f) => f !== CATALOG_ITSELF +); +const SCANNED = [...ENGINE_SOURCES, "internal-packages/run-store/src/PostgresRunStore.ts"]; + +// expression -> how many times the catalog says it appears in this file +function expectedMints(file: string): Map { + const expected = new Map(); + for (const site of WAITPOINT_MINT_SITES.filter((s) => s.site === file)) { + for (const expr of site.mints) { + expected.set(expr, (expected.get(expr) ?? 0) + 1); + } } -}); + return expected; +} -describe("waitpoint mint census — source drift guard", () => { - it("no scanned source mints a waitpoint id with the un-stamped helper", () => { - // The regex matches tokens inside comments too — deliberate. Any textual addition - // forces the census to be reconciled, so a new site cannot land without an entry. - for (const file of scannedFiles()) { - expect({ file, hits: count(read(file), /WaitpointId\.generate\(/g) }).toEqual({ - file, - hits: 0, +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +describe("waitpoint mint census — the catalog matches the source", () => { + it("scans the engine tree and the run-store writer, and finds files to scan", () => { + expect(ENGINE_SOURCES.length).toBeGreaterThan(10); + expect(SCANNED).toContain("internal-packages/run-engine/src/engine/systems/waitpointSystem.ts"); + }); + + // Per EXPRESSION, not per file: this fails for a fifth mint added inside an + // already-catalogued file, AND for a swapped anchor — mintWaitpointIdFor(undefined) in + // place of the run id — which a bare call-count would wave through. + it.each(SCANNED)("%s has exactly the mint expressions the catalog claims", (file) => { + const source = read(file); + const expected = expectedMints(file); + + for (const [expr, n] of expected) { + expect({ expr, found: count(source, new RegExp(escapeRegExp(expr), "g")) }).toEqual({ + expr, + found: n, }); } + + // No mint in the file beyond the ones the catalog accounts for. + const accounted = [...expected.values()].reduce((a, b) => a + b, 0); + expect(count(source, MINT_CALL)).toBe(accounted); }); - it("every file that writes a waitpoint row is catalogued", () => { - const catalogued = new Set(WAITPOINT_MINT_SITES.map((s) => s.site)); - - for (const file of scannedFiles()) { - const source = read(file); - // A create with NO id is the worst case: Prisma's @default(cuid()) then mints a cuid - // on a gen-2 shard after the write, which no stamp check can see. - const writes = - count(source, /waitpoint\.create\(/g) + - count(source, /upsertWaitpoint\(/g) + - count(source, /createWaitpoint\(/g); - - if (writes > 0) { - expect({ file, catalogued: catalogued.has(file) }).toEqual({ file, catalogued: true }); - } - } + it.each(SCANNED)("%s mints no waitpoint id with the un-stamped helper", (file) => { + // The regex matches tokens inside comments too — deliberate. Any textual addition + // forces the census to be reconciled, so a new site cannot land unnoticed. + expect(count(read(file), UNSTAMPED_MINT)).toBe(0); + }); + + it.each(SCANNED)("%s writes a waitpoint row only if it is catalogued", (file) => { + // A create with NO id is the worst case: Prisma's @default(cuid()) then mints a cuid on + // a gen-2 shard after the write, which no stamp check can see. + const writes = count(read(file), WAITPOINT_WRITE); + const catalogued = WAITPOINT_MINT_SITES.some((s) => s.site === file); + expect(writes === 0 || catalogued).toBe(true); }); it("every catalogued site names a file that exists", () => { @@ -93,6 +118,16 @@ describe("waitpoint mint census — source drift guard", () => { } }); + it("every catalogued site names its enclosing symbol in that file", () => { + for (const site of WAITPOINT_MINT_SITES) { + const symbol = site.symbol.split(" ")[0]!.replace("#", ""); + expect({ site: site.id, present: read(site.site).includes(symbol) }).toEqual({ + site: site.id, + present: true, + }); + } + }); + it("no catalogued symbol is a line number", () => { for (const site of WAITPOINT_MINT_SITES) { expect(site.symbol).not.toMatch(/:\d+/); diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/waitpointMintCatalog.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/waitpointMintCatalog.ts index 5eea07e5b65..be8c76d72bf 100644 --- a/internal-packages/run-engine/src/engine/waitpointCoordinator/waitpointMintCatalog.ts +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/waitpointMintCatalog.ts @@ -16,6 +16,13 @@ export type WaitpointMintSite = { site: string; /** Enclosing method or symbol name — NEVER a line number. */ symbol: string; + /** + * The exact mint expressions this site contains, verbatim. The proof test counts each one + * per file, so both a new mint inside an already-catalogued file and a swapped anchor + * (`mintWaitpointIdFor(undefined)` in place of the run id) fail until reconciled here. + * Empty for a site that writes a row from an id minted elsewhere. + */ + mints: readonly string[]; }; const COORDINATOR = @@ -26,35 +33,52 @@ const RUN_STORE = "internal-packages/run-store/src/PostgresRunStore.ts"; export const WAITPOINT_MINT_SITES: readonly WaitpointMintSite[] = [ { id: "coordinator.datetime", + mints: ["mintWaitpointIdFor(runId)"], type: "DATETIME", site: COORDINATOR, symbol: "createDateTimeWaitpoint", }, - { id: "coordinator.manual", type: "MANUAL", site: COORDINATOR, symbol: "createManualWaitpoint" }, + { + id: "coordinator.manual", + mints: ["mintWaitpointIdForShard(standaloneShard)", "mintWaitpointIdFor(runId)"], + type: "MANUAL", + site: COORDINATOR, + symbol: "createManualWaitpoint", + }, { id: "coordinator.associated.mint", + mints: ["mintWaitpointIdFor(anchorRunId)"], type: "RUN", site: COORDINATOR, symbol: "mintAssociatedWaitpointData", }, { id: "coordinator.associated.create", + mints: [], type: "RUN", site: COORDINATOR, symbol: "createAssociatedWaitpoint", }, - { id: "engine.batch", type: "BATCH", site: ENGINE, symbol: "blockRunWithCreatedBatch" }, + { + id: "engine.batch", + mints: ["mintWaitpointIdFor(batchId)"], + type: "BATCH", + site: ENGINE, + symbol: "blockRunWithCreatedBatch", + }, // The physical writers of the RUN row. They take an already-minted id rather than // minting one, but they are the writes that bypass the routing store's stamp check, so a // new writer here must be seen. { id: "runStore.createRun.nested", + mints: [], type: "RUN", site: RUN_STORE, symbol: "createRun (nested associatedWaitpoint create)", }, { id: "runStore.createRun.dedicated", + mints: [], type: "RUN", site: RUN_STORE, symbol: "#createAssociatedWaitpoint", diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/waitpointMintSites.test.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/waitpointMintSites.test.ts index d9c9553acc3..b4f44bb7623 100644 --- a/internal-packages/run-engine/src/engine/waitpointCoordinator/waitpointMintSites.test.ts +++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/waitpointMintSites.test.ts @@ -1,71 +1,149 @@ +import type { RunStore } from "@internal/run-store"; +import type { Logger } from "@trigger.dev/core/logger"; +import type { PrismaClient, Waitpoint } from "@trigger.dev/database"; import { describe, expect, it } from "vitest"; -import { mintWaitpointIdFor, resolveShard } from "@trigger.dev/core/v3/isomorphic"; +import { LegacyPostgresWaitpointCoordinator } from "./legacyPostgresCoordinator.js"; -// The mint is pure, so each site's stamping is asserted without a container. The write -// behaviour itself is covered by the coordinator's own suite. +// These drive the real create sites, not the mint helper. A test that calls the helper with +// a hand-written literal passes even when a site stops passing its anchor, which is the one +// regression that matters here. const GEN2_RUN = `${"a".repeat(24)}a2`; const GEN1_RUN = `${"a".repeat(24)}01`; -const CUID_RUN = "c".repeat(25); +const GEN2_BATCH = `${"d".repeat(24)}b2`; -describe("DATETIME and MANUAL waitpoint ids", () => { - it("a gen-2 run anchor stamps that run's shard char", () => { - const r = mintWaitpointIdFor(GEN2_RUN); - expect(r.id[24]).toBe("a"); - expect(r.id[25]).toBe("2"); - }); +type Captured = { id?: string; friendlyId?: string }; - it("a gen-1 run anchor keeps a cuid", () => { - expect(mintWaitpointIdFor(GEN1_RUN).id.length).toBe(25); +function coordinatorCapturing(captured: Captured) { + const runStore = { + findWaitpoint: async () => null, + upsertWaitpoint: async (args: { create: Captured }) => { + captured.id = args.create.id; + captured.friendlyId = args.create.friendlyId; + return { id: args.create.id } as unknown as Waitpoint; + }, + } as unknown as RunStore; + + return new LegacyPostgresWaitpointCoordinator({ + runStore, + prisma: {} as unknown as PrismaClient, + logger: { + debug: () => {}, + info: () => {}, + warn: () => {}, + error: () => {}, + } as unknown as Logger, }); +} + +describe("createDateTimeWaitpoint stamps the anchor's shard", () => { + it("a gen-2 run anchor yields a gen-2 waitpoint id", async () => { + const captured: Captured = {}; + await coordinatorCapturing(captured).createDateTimeWaitpoint({ + runId: GEN2_RUN, + projectId: "proj", + environmentId: "env", + completedAfter: new Date(), + }); - it("a cuid run anchor keeps a cuid", () => { - expect(mintWaitpointIdFor(CUID_RUN).id.length).toBe(25); + expect(captured.id).toHaveLength(26); + expect(captured.id?.[24]).toBe("a"); + expect(captured.id?.[25]).toBe("2"); + expect(captured.friendlyId).toBe(`waitpoint_${captured.id}`); }); - it("each retry attempt mints a distinct id on the same shard", () => { - const first = mintWaitpointIdFor(GEN2_RUN); - const second = mintWaitpointIdFor(GEN2_RUN); - expect(first.id).not.toBe(second.id); - expect(first.id[24]).toBe("a"); - expect(second.id[24]).toBe("a"); + it("a gen-1 run anchor keeps a cuid", async () => { + const captured: Captured = {}; + await coordinatorCapturing(captured).createDateTimeWaitpoint({ + runId: GEN1_RUN, + projectId: "proj", + environmentId: "env", + completedAfter: new Date(), + }); + + expect(captured.id).toHaveLength(25); }); }); -describe("the RUN-associated waitpoint", () => { - // This row is written inside the run store, which has no stamp check. A cuid here lands - // on a gen-2 shard, the completion fallback probes only the gen-1 pair, and the parent - // waits forever with no error. So the anchor must reach the mint. - it("stamps the run's shard char when the run is gen-2", () => { - const r = mintWaitpointIdFor(GEN2_RUN); - expect(r.id[24]).toBe("a"); - expect(r.id[25]).toBe("2"); +describe("createManualWaitpoint stamps the anchor's shard", () => { + it("a gen-2 run anchor yields a gen-2 waitpoint id", async () => { + const captured: Captured = {}; + await coordinatorCapturing(captured).createManualWaitpoint({ + runId: GEN2_RUN, + projectId: "proj", + environmentId: "env", + }); + + expect(captured.id?.[24]).toBe("a"); + expect(captured.id?.[25]).toBe("2"); }); - it("keeps a cuid for a gen-1 run, which is today's behaviour", () => { - expect(mintWaitpointIdFor(GEN1_RUN).id.length).toBe(25); + it("a standalone token mints by the environment's shard, not by an anchor", async () => { + const captured: Captured = {}; + await coordinatorCapturing(captured).createManualWaitpoint({ + projectId: "proj", + environmentId: "env", + standaloneShardKey: "c", + }); + + expect(captured.id?.[24]).toBe("c"); + expect(captured.id?.[25]).toBe("2"); }); - it("mints a fresh core, so the waitpoint id never equals the run's own body", () => { - expect(mintWaitpointIdFor(GEN2_RUN).id).not.toBe(GEN2_RUN); + it("a standalone token on a gen-1 environment keeps a cuid", async () => { + const captured: Captured = {}; + await coordinatorCapturing(captured).createManualWaitpoint({ + projectId: "proj", + environmentId: "env", + standaloneShardKey: "new", + standaloneResidency: "NEW", + }); + + expect(captured.id).toHaveLength(25); + }); + + it("an owning run outranks the environment shard", async () => { + const captured: Captured = {}; + await coordinatorCapturing(captured).createManualWaitpoint({ + runId: GEN2_RUN, + projectId: "proj", + environmentId: "env", + standaloneShardKey: "c", + }); + + // The run's shard, not the environment's: a co-located waitpoint follows its run. + expect(captured.id?.[24]).toBe("a"); }); }); -describe("the BATCH waitpoint", () => { - const GEN2_BATCH = `${"d".repeat(24)}a2`; +describe("mintAssociatedWaitpointData stamps the anchor's shard", () => { + // The row this mints is written inside the run store, which has no stamp check, so an + // unstamped id here strands the parent run with nothing logged. + const mint = (anchorRunId: string) => + coordinatorCapturing({}).mintAssociatedWaitpointData({ + projectId: "proj", + environmentId: "env", + anchorRunId, + }); - // The create passes only completedByBatchId, so the routing store resolves the owner - // from the BATCH and validates the stamp against the batch's shard. Stamping from the - // run would throw. The two agree structurally: the batch is minted from the same parent - // run id that is then blocked, in the same request. - it("stamps the batch's shard char", () => { - expect(mintWaitpointIdFor(GEN2_BATCH).id[24]).toBe("a"); + it("a gen-2 run anchor yields a gen-2 waitpoint id", () => { + const data = mint(GEN2_RUN); + expect(data.id).toHaveLength(26); + expect(data.id[24]).toBe("a"); + expect(data.id[25]).toBe("2"); + expect(data.friendlyId).toBe(`waitpoint_${data.id}`); }); - it("the batch's shard equals the blocked run's shard", () => { - expect(resolveShard(GEN2_BATCH)).toBe(resolveShard(GEN2_RUN)); + it("a gen-1 run anchor keeps a cuid", () => { + expect(mint(GEN1_RUN).id).toHaveLength(25); + }); + + it("mints a fresh core, so the waitpoint id never equals the run's own body", () => { + expect(mint(GEN2_RUN).id).not.toBe(GEN2_RUN); }); - it("a gen-1 batch keeps a cuid", () => { - expect(mintWaitpointIdFor(`${"d".repeat(24)}01`).id.length).toBe(25); + it("a batch anchor stamps the batch's shard", () => { + // What blockRunWithCreatedBatch relies on: the router validates a BATCH waitpoint + // against the batch's shard, because the create names only completedByBatchId. + expect(mint(GEN2_BATCH).id[24]).toBe("b"); }); }); From f9ad14c0f3a0824e2cd3839f157110643233cdd1 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Wed, 26 Aug 2026 14:16:00 +0100 Subject: [PATCH 13/14] fix(webapp): keep the caller's region on an inherited run mint Consolidating the mint branch dropped the region on the inherited arm. The previous code passed it on both arms, so a child run stamped whatever region the caller asked for; without it a child of an unsharded parent stamped the default character instead. Ids for every existing deployment have to be unchanged, so this is a regression rather than a cosmetic slip. A shard character still outranks the region, since both occupy the same slot and only one of them can be authoritative. The inertness suite missed it by asserting the version character but not the region character. Both are now asserted, for an inherited parent with and without a shard. --- .../runOpsMigration/gen2MintInertness.test.ts | 30 +++++++++++++++++++ .../resolveRunMintTarget.server.ts | 4 ++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/apps/webapp/app/v3/runOpsMigration/gen2MintInertness.test.ts b/apps/webapp/app/v3/runOpsMigration/gen2MintInertness.test.ts index 405c082cb1b..e51dae720ae 100644 --- a/apps/webapp/app/v3/runOpsMigration/gen2MintInertness.test.ts +++ b/apps/webapp/app/v3/runOpsMigration/gen2MintInertness.test.ts @@ -41,6 +41,36 @@ describe("gate off — run mint paths", () => { expect(mintFriendlyIdForKind(target).slice(4).length).toBe(25); }); + it("a child of a gen-1 parent keeps the caller's region char", async () => { + // The pre-split code passed the region on BOTH arms, so a child run stamped the + // requested region. Dropping it on the inherited arm would silently stamp the default. + const target = await resolveRunMintTarget({ + environment, + parentRunFriendlyId: `run_${"a".repeat(24)}01`, + region: "us-east-1", + deps: { + resolveRunIdMintKind: vi.fn().mockResolvedValue("runOpsId"), + resolveMintShard: offShard, + }, + }); + const body = mintFriendlyIdForKind(target).slice(4); + expect(body[24]).toBe("e"); + expect(body[25]).toBe("1"); + }); + + it("a gen-2 parent's shard still outranks the caller's region", async () => { + const target = await resolveRunMintTarget({ + environment, + parentRunFriendlyId: `run_${"a".repeat(24)}a2`, + region: "us-east-1", + deps: { + resolveRunIdMintKind: vi.fn().mockResolvedValue("runOpsId"), + resolveMintShard: offShard, + }, + }); + expect(mintFriendlyIdForKind(target).slice(4)[24]).toBe("a"); + }); + it("a child of a gen-1 parent mints a gen-1 v1 id", () => { const body = mintFriendlyIdForKind(resolveInheritedMintKind(`run_${"a".repeat(24)}01`)).slice( 4 diff --git a/apps/webapp/app/v3/runOpsMigration/resolveRunMintTarget.server.ts b/apps/webapp/app/v3/runOpsMigration/resolveRunMintTarget.server.ts index 89b8948c689..a3f9466788b 100644 --- a/apps/webapp/app/v3/runOpsMigration/resolveRunMintTarget.server.ts +++ b/apps/webapp/app/v3/runOpsMigration/resolveRunMintTarget.server.ts @@ -27,7 +27,9 @@ export async function resolveRunMintTarget(args: { deps?: Partial; }): Promise { if (args.parentRunFriendlyId) { - return resolveInheritedMintKind(args.parentRunFriendlyId); + // The region still travels: it takes index 24 for an inherited gen-1 parent, exactly as + // it did before this branch. A gen-2 parent's shardChar outranks it. + return { ...resolveInheritedMintKind(args.parentRunFriendlyId), region: args.region }; } const deps = { ...defaultDeps, ...args.deps }; From b969f8ed2bed9e82974eb0110e2f6f892c2b78d9 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Wed, 26 Aug 2026 17:22:59 +0100 Subject: [PATCH 14/14] fix(webapp): route a gen-2 batch's completion write to its own shard Minting gen-2 batch ids broke batch waits. The batch-completion writer was resolved by a binary probe: look for the row on the new store, otherwise assume legacy. A gen-2 batch lives on neither, so the probe fell through to legacy, the update found no row and threw, the callback died before tryCompleteBatch, the batch waitpoint stayed pending, and the parent run waited forever with nothing logged as a hang. Found by running it: a gen-2 batchTriggerAndWait parent never resumed, while the same task on a gen-1 batch completed in twenty seconds. A gen-2 batch id names its own shard, so it now routes by that and never probes. An id naming an unconfigured shard throws rather than guessing a store, because guessing is precisely what strands the run. Both new tests fail without this change, the first on a fake client that throws if the new store is probed at all. --- .../webapp/app/v3/runEngineHandlers.server.ts | 2 ++ .../app/v3/runEngineHandlersShared.server.ts | 20 +++++++++++ apps/webapp/test/runEngineHandlers.test.ts | 36 +++++++++++++++++++ 3 files changed, 58 insertions(+) diff --git a/apps/webapp/app/v3/runEngineHandlers.server.ts b/apps/webapp/app/v3/runEngineHandlers.server.ts index c44bcc54cec..da5a5d89802 100644 --- a/apps/webapp/app/v3/runEngineHandlers.server.ts +++ b/apps/webapp/app/v3/runEngineHandlers.server.ts @@ -11,6 +11,7 @@ import { runOpsNewPrismaClient, runOpsNewReplicaClient, runOpsLegacyPrismaClient, + runOpsShardHandles, } from "~/db.server"; import { env } from "~/env.server"; import { findEnvironmentById, findEnvironmentFromRun } from "~/models/runtimeEnvironment.server"; @@ -1060,6 +1061,7 @@ export function setupBatchQueueCallbacks() { newReplica: runOpsNewReplicaClient, newWriter: runOpsNewPrismaClient, legacyWriter: runOpsLegacyPrismaClient, + shards: runOpsShardHandles, tryCompleteBatch: (batchId) => engine.tryCompleteBatch({ batchId }), }); }); diff --git a/apps/webapp/app/v3/runEngineHandlersShared.server.ts b/apps/webapp/app/v3/runEngineHandlersShared.server.ts index 4ce8cc2de8a..155e5365cce 100644 --- a/apps/webapp/app/v3/runEngineHandlersShared.server.ts +++ b/apps/webapp/app/v3/runEngineHandlersShared.server.ts @@ -4,6 +4,7 @@ * whole webapp service graph). The handlers wire the production defaults; tests * inject per-container stores/replicas, so these helpers never import db.server. */ +import { resolveShard } from "@trigger.dev/core/v3/isomorphic"; import type { CompleteBatchResult } from "@internal/run-engine"; import type { RunOpsPrismaClient } from "@internal/run-ops-database"; import type { RunStore } from "@internal/run-store"; @@ -82,8 +83,25 @@ export async function resolveBatchRunOpsWriter( newReplica: RunOpsPrismaClient; newWriter: RunOpsPrismaClient; legacyWriter: RunOpsPrismaClient; + shards?: ReadonlyArray<{ key: string; writer: RunOpsPrismaClient }>; } ): Promise { + // A gen-2 batch names its own shard in its id, so route by that and never probe. The + // probe below is binary — NEW, else assume LEGACY — so a gen-2 batch would fall through + // to a store that holds no such row, and the completion update would throw before the + // batch waitpoint could complete, leaving the parent run blocked with nothing logged. + const shardKey = resolveShard(batchId); + if (shardKey !== "new" && shardKey !== "legacy") { + const shard = deps.shards?.find((s) => s.key === shardKey); + if (!shard) { + // Writing to a guessed store is what strands a run. Fail loud instead. + throw new Error( + `resolveBatchRunOpsWriter: batch "${batchId}" names shard "${shardKey}", which is not configured` + ); + } + return shard.writer; + } + const onNew = await deps.newReplica.batchTaskRun.findFirst({ where: { id: batchId }, select: { id: true }, @@ -105,6 +123,7 @@ export type BatchCompletionDeps = { newReplica: RunOpsPrismaClient; newWriter: RunOpsPrismaClient; legacyWriter: RunOpsPrismaClient; + shards?: ReadonlyArray<{ key: string; writer: RunOpsPrismaClient }>; tryCompleteBatch: (batchId: string) => Promise; }; @@ -135,6 +154,7 @@ export async function handleBatchCompletion( newReplica: deps.newReplica, newWriter: deps.newWriter, legacyWriter: deps.legacyWriter, + shards: deps.shards, }); try { diff --git a/apps/webapp/test/runEngineHandlers.test.ts b/apps/webapp/test/runEngineHandlers.test.ts index 2c57d87506e..61b5cdadc65 100644 --- a/apps/webapp/test/runEngineHandlers.test.ts +++ b/apps/webapp/test/runEngineHandlers.test.ts @@ -490,6 +490,42 @@ describe("runEngineHandlers batch completion", () => { }); describe("runEngineHandlers batch residency routing", () => { + // A gen-2 batch lives on its own shard. The binary probe below it looks only on the + // NEW store and then assumes LEGACY, so without a shard arm the completion update runs + // on a database that has no such row: Prisma throws "no record was found for an + // update", the callback dies before tryCompleteBatch, the BATCH waitpoint stays + // PENDING and the parent run waits forever with nothing logged as a hang. + it("a gen-2 batch resolves to its own shard writer", async () => { + const shardWriter = {} as never; // identity is the whole assertion; no database is touched + const gen2BatchId = `${"a".repeat(24)}a2`; + + const writer = await resolveBatchRunOpsWriter(gen2BatchId, { + newReplica: { + batchTaskRun: { + findFirst: async () => { + throw new Error("a gen-2 batch id must never probe the NEW store"); + }, + }, + } as never, + newWriter: {} as never, + legacyWriter: {} as never, + shards: [{ key: "a", writer: shardWriter as never }], + }); + + expect(writer).toBe(shardWriter); + }); + + it("an unconfigured shard key fails loud rather than writing elsewhere", async () => { + await expect( + resolveBatchRunOpsWriter(`${"a".repeat(24)}z2`, { + newReplica: {} as never, + newWriter: {} as never, + legacyWriter: {} as never, + shards: [{ key: "a", writer: {} as never }], + }) + ).rejects.toThrow(/shard/i); + }); + // True single-DB invariant: the topology's cpFallback makes newReplica and // legacyWriter the SAME control-plane client, so the probe always resolves to // that one client regardless of where length-classification would guess.