From 5da48f9e90052707277275ebea0a3b15705dbfe3 Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Fri, 21 Aug 2026 12:44:09 +0100
Subject: [PATCH 01/49] feat(run-store): keyspace and entry helpers for the
 Redis snapshot store

---
 internal-packages/run-store/package.json      |  1 +
 internal-packages/run-store/src/index.ts      |  1 +
 .../run-store/src/redisSnapshotStore.test.ts  | 48 ++++++++++
 .../run-store/src/redisSnapshotStore.ts       | 92 +++++++++++++++++++
 pnpm-lock.yaml                                |  3 +
 5 files changed, 145 insertions(+)
 create mode 100644 internal-packages/run-store/src/redisSnapshotStore.test.ts
 create mode 100644 internal-packages/run-store/src/redisSnapshotStore.ts

diff --git a/internal-packages/run-store/package.json b/internal-packages/run-store/package.json
index 110c3b49058..7263a6de05c 100644
--- a/internal-packages/run-store/package.json
+++ b/internal-packages/run-store/package.json
@@ -14,6 +14,7 @@
     }
   },
   "dependencies": {
+    "@internal/redis": "workspace:*",
     "@trigger.dev/core": "workspace:*",
     "@trigger.dev/database": "workspace:*"
   },
diff --git a/internal-packages/run-store/src/index.ts b/internal-packages/run-store/src/index.ts
index 160f9cdada2..3717dc01527 100644
--- a/internal-packages/run-store/src/index.ts
+++ b/internal-packages/run-store/src/index.ts
@@ -2,3 +2,4 @@ export * from "./types.js";
 export * from "./PostgresRunStore.js";
 export * from "./runOpsStore.js";
 export * from "./readReplicaClient.js";
+export * from "./redisSnapshotStore.js";
diff --git a/internal-packages/run-store/src/redisSnapshotStore.test.ts b/internal-packages/run-store/src/redisSnapshotStore.test.ts
new file mode 100644
index 00000000000..7f93e0d5412
--- /dev/null
+++ b/internal-packages/run-store/src/redisSnapshotStore.test.ts
@@ -0,0 +1,48 @@
+// Unit suite for the raw Redis execution-snapshot store. Redis-only: the store holds no Prisma
+// reference, so no Postgres container is needed.
+import { expect, describe } from "vitest";
+import { snapshotKeys, deriveOrder, isValidFor } from "./redisSnapshotStore.js";
+
+describe("snapshotKeys", () => {
+  it("puts every core key under one hash tag", () => {
+    const k = snapshotKeys("run_abc123");
+    expect(k.e).toBe("snap:{run_abc123}:e");
+    expect(k.idx).toBe("snap:{run_abc123}:idx");
+    expect(k.cur).toBe("snap:{run_abc123}:cur");
+    expect(k.seq).toBe("snap:{run_abc123}:seq");
+  });
+});
+
+describe("deriveOrder", () => {
+  it("drops entries with no index, sorts by index, and maps to id", () => {
+    expect(
+      deriveOrder([
+        { id: "w_c", index: 2 },
+        { id: "w_a", index: 0 },
+        { id: "w_no" },
+        { id: "w_b", index: 1 },
+      ])
+    ).toEqual(["w_a", "w_b", "w_c"]);
+  });
+
+  it("preserves a repeated id at each of its positions", () => {
+    expect(
+      deriveOrder([
+        { id: "w_x", index: 0 },
+        { id: "w_x", index: 1 },
+      ])
+    ).toEqual(["w_x", "w_x"]);
+  });
+
+  it("returns an empty list when nothing carries an index", () => {
+    expect(deriveOrder([{ id: "w_a" }, { id: "w_b" }])).toEqual([]);
+  });
+});
+
+describe("isValidFor", () => {
+  it("is false when the entry carries an error and true otherwise", () => {
+    expect(isValidFor({ error: "boom" })).toBe(false);
+    expect(isValidFor({})).toBe(true);
+    expect(isValidFor({ error: undefined })).toBe(true);
+  });
+});
diff --git a/internal-packages/run-store/src/redisSnapshotStore.ts b/internal-packages/run-store/src/redisSnapshotStore.ts
new file mode 100644
index 00000000000..a5a2e609213
--- /dev/null
+++ b/internal-packages/run-store/src/redisSnapshotStore.ts
@@ -0,0 +1,92 @@
+import { createRedisClient, type Redis, type RedisOptions } from "@internal/redis";
+import { Logger } from "@trigger.dev/core/logger";
+
+export type SnapshotKeys = { e: string; idx: string; cur: string; seq: string };
+
+// All four core keys plus every snap:{runId}:wp:<n> key share the {runId} hash tag, so a run's whole
+// state sits in one cluster slot and every mutation is one atomic script.
+export function snapshotKeys(runId: string): SnapshotKeys {
+  const base = `snap:{${runId}}`;
+  return { e: `${base}:e`, idx: `${base}:idx`, cur: `${base}:cur`, seq: `${base}:seq` };
+}
+
+export type CompletedWaitpointRef = { id: string; index?: number };
+
+// Reproduces PostgresRunStore.#createExecutionSnapshot's completedWaitpointOrder derivation exactly:
+// drop anything without an index, sort ascending by index, map to id. Repeats are preserved, because
+// the same run can sit in one batch more than once under a single idempotency key.
+export function deriveOrder(completedWaitpoints: CompletedWaitpointRef[]): string[] {
+  return completedWaitpoints
+    .filter((w) => w.index !== undefined)
+    .sort((a, b) => a.index! - b.index!)
+    .map((w) => w.id);
+}
+
+// isValid is derived, never stored, so the entry JSON stays byte-identical to the caller's document.
+export function isValidFor(entry: { error?: unknown }): boolean {
+  return !entry.error;
+}
+
+export type SnapshotEntryInput = {
+  id: string;
+  engine: "V2";
+  executionStatus: string;
+  description: string;
+  runId: string;
+  runStatus: string;
+  createdAt: string;
+  attemptNumber?: number | null;
+  previousSnapshotId?: string;
+  batchId?: string;
+  environmentId: string;
+  environmentType: string;
+  projectId: string;
+  organizationId: string;
+  checkpointId?: string;
+  workerId?: string;
+  runnerId?: string;
+  metadata?: unknown;
+  error?: string;
+};
+
+export type WaitpointIds = { present: boolean; distinctIds: string[]; order: string[] };
+
+export type SnapshotRead = {
+  id: string;
+  seq: number;
+  isValid: boolean;
+  entry: Record<string, unknown>;
+  raw: string;
+  cycle?: { cycleSeq: number; count: number };
+  completedWaitpointIds?: WaitpointIds;
+};
+
+export type AppendResult =
+  | {
+      outcome: "written";
+      seq: number;
+      cycleSeq?: number;
+      ttl: "none" | "completion" | "reapplied";
+      cycleMismatch: boolean;
+    }
+  | { outcome: "skippedNoKeyspace" }
+  | { outcome: "forked"; actualCur: string };
+
+export type SnapshotStoreMetrics = {
+  recordAppend(outcome: string, ttl: string): void;
+  recordEntryBytes(bytes: number): void;
+  recordCycleKeyBytes(bytes: number): void;
+  recordCycleCount(count: number): void;
+  recordSkippedNoKeyspace(): void;
+  recordCycleMismatch(): void;
+  recordLatency(op: string, ms: number): void;
+};
+
+export type RedisSnapshotStoreOptions = {
+  redisOptions: RedisOptions;
+  completedTtlMs: number;
+  sinceLimit?: number;
+  highWater?: { entryBytes?: number; cycleKeyBytes?: number; cycleCount?: number };
+  metrics?: SnapshotStoreMetrics;
+  logger?: Logger;
+};
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 7d74cbbe905..297cb7897ff 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -1290,6 +1290,9 @@ importers:
 
   internal-packages/run-store:
     dependencies:
+      '@internal/redis':
+        specifier: workspace:*
+        version: link:../redis
       '@trigger.dev/core':
         specifier: workspace:*
         version: link:../../packages/core

From 030acd1368b78a4ea52b27f1c8eb7472689ef0ee Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Fri, 21 Aug 2026 12:50:40 +0100
Subject: [PATCH 02/49] fix(run-store): import only what the helpers use

---
 internal-packages/run-store/src/redisSnapshotStore.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/internal-packages/run-store/src/redisSnapshotStore.ts b/internal-packages/run-store/src/redisSnapshotStore.ts
index a5a2e609213..b6bd4424ca1 100644
--- a/internal-packages/run-store/src/redisSnapshotStore.ts
+++ b/internal-packages/run-store/src/redisSnapshotStore.ts
@@ -1,5 +1,5 @@
-import { createRedisClient, type Redis, type RedisOptions } from "@internal/redis";
-import { Logger } from "@trigger.dev/core/logger";
+import type { RedisOptions } from "@internal/redis";
+import type { Logger } from "@trigger.dev/core/logger";
 
 export type SnapshotKeys = { e: string; idx: string; cur: string; seq: string };
 

From 92028d54416ee9b9f8a2c3168a6175060ab18ad8 Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Fri, 21 Aug 2026 12:55:25 +0100
Subject: [PATCH 03/49] feat(run-store): append and id-keyed reads for the
 Redis snapshot store

---
 .../run-store/src/redisSnapshotStore.test.ts  | 100 ++++-
 .../run-store/src/redisSnapshotStore.ts       | 377 +++++++++++++++++-
 2 files changed, 474 insertions(+), 3 deletions(-)

diff --git a/internal-packages/run-store/src/redisSnapshotStore.test.ts b/internal-packages/run-store/src/redisSnapshotStore.test.ts
index 7f93e0d5412..d1a7d8fe71f 100644
--- a/internal-packages/run-store/src/redisSnapshotStore.test.ts
+++ b/internal-packages/run-store/src/redisSnapshotStore.test.ts
@@ -1,7 +1,14 @@
 // Unit suite for the raw Redis execution-snapshot store. Redis-only: the store holds no Prisma
 // reference, so no Postgres container is needed.
 import { expect, describe } from "vitest";
-import { snapshotKeys, deriveOrder, isValidFor } from "./redisSnapshotStore.js";
+import { redisTest } from "@internal/testcontainers";
+import {
+  snapshotKeys,
+  deriveOrder,
+  isValidFor,
+  RedisSnapshotStore,
+  type SnapshotEntryInput,
+} from "./redisSnapshotStore.js";
 
 describe("snapshotKeys", () => {
   it("puts every core key under one hash tag", () => {
@@ -46,3 +53,94 @@ describe("isValidFor", () => {
     expect(isValidFor({ error: undefined })).toBe(true);
   });
 });
+
+function entry(over: Partial<SnapshotEntryInput> = {}): SnapshotEntryInput {
+  return {
+    id: "snap_1",
+    engine: "V2",
+    executionStatus: "RUN_CREATED",
+    description: "created",
+    runId: "run_1",
+    runStatus: "PENDING",
+    createdAt: "2026-08-21T00:00:00.000Z",
+    environmentId: "env_1",
+    environmentType: "PRODUCTION",
+    projectId: "proj_1",
+    organizationId: "org_1",
+    ...over,
+  };
+}
+
+describe("append", () => {
+  redisTest("assigns a monotonic seq and reads the entry back by id", async ({ redisOptions }) => {
+    const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 72 * 3600 * 1000 });
+    try {
+      const a = await store.append({
+        entry: entry({ id: "snap_1" }),
+        kind: "birth",
+        isTerminal: false,
+      });
+      const b = await store.append({
+        entry: entry({ id: "snap_2" }),
+        kind: "transition",
+        isTerminal: false,
+      });
+      expect(a).toMatchObject({ outcome: "written", seq: 1 });
+      expect(b).toMatchObject({ outcome: "written", seq: 2 });
+
+      const read = await store.getById("run_1", "snap_2");
+      expect(read?.seq).toBe(2);
+      expect(read?.isValid).toBe(true);
+      expect(read?.entry.description).toBe("created");
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest("preserves the entry JSON byte for byte", async ({ redisOptions }) => {
+    const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000 });
+    try {
+      const e = entry({ id: "snap_1", metadata: { empty: [], nested: { a: 1 } } });
+      await store.append({ entry: e, kind: "birth", isTerminal: false });
+      const read = await store.getById("run_1", "snap_1");
+      expect(read?.raw).toBe(JSON.stringify(e));
+      expect(read?.entry).toEqual(e);
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest("advances cur only for a valid entry", async ({ redisOptions }) => {
+    const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000 });
+    try {
+      await store.append({ entry: entry({ id: "snap_1" }), kind: "birth", isTerminal: false });
+      await store.append({
+        entry: entry({ id: "snap_bad", error: "nope" }),
+        kind: "transition",
+        isTerminal: false,
+      });
+      const latest = await store.getLatest("run_1");
+      expect(latest?.id).toBe("snap_1");
+
+      const invalid = await store.getById("run_1", "snap_bad");
+      expect(invalid?.isValid).toBe(false);
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest("skips a transition against an absent keyspace", async ({ redisOptions }) => {
+    const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000 });
+    try {
+      const r = await store.append({
+        entry: entry({ id: "snap_1", runId: "run_never" }),
+        kind: "transition",
+        isTerminal: false,
+      });
+      expect(r).toEqual({ outcome: "skippedNoKeyspace" });
+      expect(await store.getLatest("run_never")).toBeNull();
+    } finally {
+      await store.quit();
+    }
+  });
+});
diff --git a/internal-packages/run-store/src/redisSnapshotStore.ts b/internal-packages/run-store/src/redisSnapshotStore.ts
index b6bd4424ca1..72aae1c2578 100644
--- a/internal-packages/run-store/src/redisSnapshotStore.ts
+++ b/internal-packages/run-store/src/redisSnapshotStore.ts
@@ -1,5 +1,11 @@
-import type { RedisOptions } from "@internal/redis";
-import type { Logger } from "@trigger.dev/core/logger";
+import {
+  createRedisClient,
+  type Callback,
+  type Redis,
+  type RedisOptions,
+  type Result,
+} from "@internal/redis";
+import { Logger } from "@trigger.dev/core/logger";
 
 export type SnapshotKeys = { e: string; idx: string; cur: string; seq: string };
 
@@ -90,3 +96,370 @@ export type RedisSnapshotStoreOptions = {
   metrics?: SnapshotStoreMetrics;
   logger?: Logger;
 };
+
+const SKIPPED = "skipped";
+const FORKED = "forked";
+const WRITTEN = "written";
+
+export class RedisSnapshotStore {
+  private readonly redis: Redis;
+  private readonly logger: Logger;
+  private readonly completedTtlMs: number;
+  private readonly sinceLimit: number;
+  private readonly metrics?: SnapshotStoreMetrics;
+  private readonly highWater: NonNullable<RedisSnapshotStoreOptions["highWater"]>;
+  #quit?: Promise<void>;
+
+  constructor(options: RedisSnapshotStoreOptions) {
+    this.logger = options.logger ?? new Logger("RedisSnapshotStore", "debug");
+    this.completedTtlMs = options.completedTtlMs;
+    this.sinceLimit = options.sinceLimit ?? 50;
+    this.metrics = options.metrics;
+    this.highWater = options.highWater ?? {};
+    this.redis = createRedisClient(options.redisOptions, {
+      onError: (error) => this.logger.error("RedisSnapshotStore redis client error", { error }),
+    });
+    this.#registerCommands();
+  }
+
+  async quit(): Promise<void> {
+    // Idempotent and error-swallowing: every test calls this in a `finally`, and a double quit()
+    // (or one after a failed connect) must never mask the real assertion failure.
+    if (!this.#quit) {
+      this.#quit = this.redis.quit().then(
+        () => undefined,
+        () => undefined
+      );
+    }
+    await this.#quit;
+  }
+
+  async #timed<T>(op: string, fn: () => Promise<T>): Promise<T> {
+    const started = Date.now();
+    try {
+      return await fn();
+    } finally {
+      this.metrics?.recordLatency(op, Date.now() - started);
+    }
+  }
+
+  async append(args: {
+    entry: SnapshotEntryInput;
+    kind: "birth" | "transition";
+    isTerminal: boolean;
+    expectedCur?: string;
+    cycle?:
+      | { kind: "new"; completedWaitpoints: CompletedWaitpointRef[]; records?: string }
+      | { kind: "carryForward"; cycleSeq: number };
+  }): Promise<AppendResult> {
+    return this.#timed("append", async () => {
+      const k = snapshotKeys(args.entry.runId);
+      const raw = JSON.stringify(args.entry);
+      const valid = isValidFor(args.entry);
+
+      let cycleMode = "none";
+      let cycleSeqIn = "0";
+      let orderJson = "";
+      let records = "";
+      let orderCount = "0";
+      if (args.cycle?.kind === "new") {
+        const order = deriveOrder(args.cycle.completedWaitpoints);
+        cycleMode = "new";
+        orderJson = JSON.stringify(order);
+        records = args.cycle.records ?? "";
+        orderCount = String(order.length);
+      } else if (args.cycle?.kind === "carryForward") {
+        cycleMode = "carry";
+        cycleSeqIn = String(args.cycle.cycleSeq);
+      }
+
+      const reply = (await this.redis.appendSnapshotEntry(
+        k.e,
+        k.idx,
+        k.cur,
+        k.seq,
+        args.kind,
+        args.entry.id,
+        raw,
+        valid ? "1" : "0",
+        args.isTerminal ? "1" : "0",
+        String(this.completedTtlMs),
+        cycleMode,
+        cycleSeqIn,
+        orderJson,
+        records,
+        orderCount,
+        args.expectedCur ?? ""
+      )) as string[];
+
+      return this.#interpretAppend(reply, raw, orderJson);
+    });
+  }
+
+  #interpretAppend(reply: string[], raw: string, orderJson: string): AppendResult {
+    if (reply[0] === SKIPPED) {
+      this.metrics?.recordSkippedNoKeyspace();
+      this.metrics?.recordAppend("skippedNoKeyspace", "none");
+      return { outcome: "skippedNoKeyspace" };
+    }
+    if (reply[0] === FORKED) {
+      this.metrics?.recordAppend("forked", "none");
+      return { outcome: "forked", actualCur: reply[1] ?? "" };
+    }
+    const seq = Number(reply[1]);
+    const cycleSeq = Number(reply[2]);
+    const ttl = reply[3] as "none" | "completion" | "reapplied";
+    const cycleMismatch = reply[4] === "1";
+    if (cycleMismatch) {
+      this.metrics?.recordCycleMismatch();
+    }
+    this.#observeSizes(raw, orderJson, cycleSeq);
+    this.metrics?.recordAppend("written", ttl);
+    return {
+      outcome: "written",
+      seq,
+      ...(cycleSeq > 0 ? { cycleSeq } : {}),
+      ttl,
+      cycleMismatch,
+    };
+  }
+
+  #observeSizes(raw: string, orderJson: string, cycleSeq: number): void {
+    const entryBytes = Buffer.byteLength(raw, "utf8");
+    this.metrics?.recordEntryBytes(entryBytes);
+    if (this.highWater.entryBytes !== undefined && entryBytes > this.highWater.entryBytes) {
+      this.logger.warn("RedisSnapshotStore entry above high-water mark", { entryBytes });
+    }
+    if (orderJson !== "") {
+      const cycleBytes = Buffer.byteLength(orderJson, "utf8");
+      this.metrics?.recordCycleKeyBytes(cycleBytes);
+      if (this.highWater.cycleKeyBytes !== undefined && cycleBytes > this.highWater.cycleKeyBytes) {
+        this.logger.warn("RedisSnapshotStore cycle key above high-water mark", { cycleBytes });
+      }
+    }
+    if (cycleSeq > 0) {
+      this.metrics?.recordCycleCount(cycleSeq);
+      if (this.highWater.cycleCount !== undefined && cycleSeq > this.highWater.cycleCount) {
+        this.logger.warn("RedisSnapshotStore cycle count above high-water mark", { cycleSeq });
+      }
+    }
+  }
+
+  async getById(
+    runId: string,
+    snapshotId: string,
+    opts?: { environmentId?: string }
+  ): Promise<SnapshotRead | null> {
+    return this.#timed("getById", async () => {
+      const k = snapshotKeys(runId);
+      const reply = await this.redis.readSnapshotById(k.e, k.idx, k.cur, k.seq, snapshotId);
+      return this.#decode(reply, opts?.environmentId);
+    });
+  }
+
+  async getLatest(runId: string, opts?: { environmentId?: string }): Promise<SnapshotRead | null> {
+    return this.#timed("getLatest", async () => {
+      const k = snapshotKeys(runId);
+      const reply = await this.redis.readLatestSnapshot(k.e, k.idx, k.cur, k.seq);
+      return this.#decode(reply, opts?.environmentId);
+    });
+  }
+
+  // [id, raw, seq, pointer, order] -> SnapshotRead. The environment compare is app-side, per the
+  // plan: the store returns null for a foreign environment and the 404 throw stays in the engine.
+  #decode(reply: string[] | null, environmentId?: string): SnapshotRead | null {
+    if (!reply || reply.length === 0) return null;
+    const [id, raw, seqStr, pointer, orderJson] = reply;
+    const entry = JSON.parse(raw) as Record<string, unknown>;
+    if (environmentId !== undefined && entry.environmentId !== environmentId) return null;
+    const read: SnapshotRead = {
+      id,
+      seq: Number(seqStr),
+      isValid: isValidFor(entry as { error?: unknown }),
+      entry,
+      raw,
+    };
+    if (pointer) {
+      const [cs, count] = pointer.split(":");
+      read.cycle = { cycleSeq: Number(cs), count: Number(count) };
+      read.completedWaitpointIds = decodeWaitpointIds(true, orderJson);
+    }
+    return read;
+  }
+
+  #registerCommands() {
+    // Every script declares exactly these four keys and derives snap:{runId}:wp:<n> from KEYS[1] by
+    // string surgery. ioredis prefixes only the KEYS array, so a key minted inside Lua would be
+    // UNPREFIXED while the client wrote a prefixed one.
+    const PRELUDE = `
+      local eKey, idxKey, curKey, seqKey = KEYS[1], KEYS[2], KEYS[3], KEYS[4]
+      local base = string.sub(eKey, 1, #eKey - 2)
+      local function wpKey(n) return base .. ':wp:' .. n end
+      local function orderFor(pointer)
+        if not pointer then return '' end
+        local cs = string.match(pointer, '^(%d+):')
+        if not cs then return '' end
+        return redis.call('HGET', wpKey(cs), 'order') or ''
+      end
+    `;
+
+    this.redis.defineCommand("appendSnapshotEntry", {
+      numberOfKeys: 4,
+      lua: `
+        ${PRELUDE}
+        local kind        = ARGV[1]
+        local id          = ARGV[2]
+        local raw         = ARGV[3]
+        local isValid     = ARGV[4] == '1'
+        local isTerminal  = ARGV[5] == '1'
+        local ttlMs       = tonumber(ARGV[6])
+        local cycleMode   = ARGV[7]
+        local cycleSeqIn  = tonumber(ARGV[8])
+        local orderJson   = ARGV[9]
+        local records     = ARGV[10]
+        local orderCount  = ARGV[11]
+        local expectedCur = ARGV[12]
+
+        -- Liveness is ONE anchor. All keys get the same PEXPIRE but expire independently, so seq can
+        -- vanish while e and cur linger; anchoring on e treats a partly expired keyspace as gone,
+        -- once and consistently. A birth creates the keyspace; a transition that finds none writes
+        -- nothing. That state has two causes the caller must not merge: a completed run whose TTL
+        -- fired, and a run that predates this org's dual-write.
+        if kind == 'transition' and redis.call('EXISTS', eKey) == 0 then
+          return { '${SKIPPED}' }
+        end
+
+        -- Optional compare-and-set on cur, checked BEFORE any mutation. Absent by default, which
+        -- matches Postgres: it has no such guard either.
+        if expectedCur ~= '' then
+          local actual = redis.call('GET', curKey)
+          if (actual or '') ~= expectedCur then
+            return { '${FORKED}', actual or '' }
+          end
+        end
+
+        local seq = redis.call('HINCRBY', seqKey, 'e', 1)
+
+        local cycleSeq = 0
+        local mismatch = 0
+        if cycleMode == 'new' then
+          -- The STORE mints cycleSeq, so the sequence is dense by construction and the terminal
+          -- PEXPIRE loop from 1..c is correct.
+          cycleSeq = redis.call('HINCRBY', seqKey, 'c', 1)
+          redis.call('HSET', wpKey(cycleSeq), 'order', orderJson)
+          if records ~= '' then
+            redis.call('HSET', wpKey(cycleSeq), 'records', records)
+          end
+        elseif cycleMode == 'carry' then
+          cycleSeq = cycleSeqIn
+          if redis.call('EXISTS', wpKey(cycleSeq)) == 0 then
+            mismatch = 1
+          end
+        end
+
+        redis.call('HSET', eKey, id, raw, id .. '#s', seq)
+        if cycleSeq > 0 then
+          redis.call('HSET', eKey, id .. '#c', cycleSeq .. ':' .. orderCount)
+        end
+
+        -- idx indexes VALID entries only, which makes the since-cap exact. An invalid entry is still
+        -- reachable by id, and its seq is still readable from its own '#s' field.
+        if isValid then
+          redis.call('ZADD', idxKey, seq, id)
+          redis.call('SET', curKey, id)
+        end
+
+        local wasTerminal = redis.call('HGET', seqKey, 't') == '1'
+        local ttl = 'none'
+        if isTerminal then
+          redis.call('HSET', seqKey, 't', '1')
+        end
+        if isTerminal or wasTerminal then
+          redis.call('PEXPIRE', eKey, ttlMs)
+          redis.call('PEXPIRE', idxKey, ttlMs)
+          redis.call('PEXPIRE', curKey, ttlMs)
+          redis.call('PEXPIRE', seqKey, ttlMs)
+          local high = tonumber(redis.call('HGET', seqKey, 'c') or '0')
+          for i = 1, high do
+            redis.call('PEXPIRE', wpKey(i), ttlMs)
+          end
+          if isTerminal and not wasTerminal then
+            ttl = 'completion'
+          else
+            ttl = 'reapplied'
+          end
+        end
+
+        return { '${WRITTEN}', tostring(seq), tostring(cycleSeq), ttl, tostring(mismatch) }
+      `,
+    });
+
+    this.redis.defineCommand("readSnapshotById", {
+      numberOfKeys: 4,
+      lua: `
+        ${PRELUDE}
+        local id = ARGV[1]
+        local vals = redis.call('HMGET', eKey, id, id .. '#s', id .. '#c')
+        if not vals[1] then return nil end
+        -- Coerce every element: a Lua false TRUNCATES the returned array at that position.
+        return { id, vals[1], vals[2] or '', vals[3] or '', orderFor(vals[3]) }
+      `,
+    });
+
+    this.redis.defineCommand("readLatestSnapshot", {
+      numberOfKeys: 4,
+      lua: `
+        ${PRELUDE}
+        local cur = redis.call('GET', curKey)
+        if not cur then return nil end
+        local vals = redis.call('HMGET', eKey, cur, cur .. '#s', cur .. '#c')
+        if not vals[1] then return nil end
+        return { cur, vals[1], vals[2] or '', vals[3] or '', orderFor(vals[3]) }
+      `,
+    });
+  }
+}
+
+export function decodeWaitpointIds(present: boolean, orderJson: string): WaitpointIds {
+  const order: string[] = orderJson === "" ? [] : (JSON.parse(orderJson) as string[]);
+  return { present, distinctIds: [...new Set(order)], order };
+}
+
+declare module "@internal/redis" {
+  interface RedisCommander<Context> {
+    appendSnapshotEntry(
+      eKey: string,
+      idxKey: string,
+      curKey: string,
+      seqKey: string,
+      kind: string,
+      id: string,
+      raw: string,
+      isValid: string,
+      isTerminal: string,
+      ttlMs: string,
+      cycleMode: string,
+      cycleSeqIn: string,
+      orderJson: string,
+      records: string,
+      orderCount: string,
+      expectedCur: string,
+      callback?: Callback<string[]>
+    ): Result<string[], Context>;
+    readSnapshotById(
+      eKey: string,
+      idxKey: string,
+      curKey: string,
+      seqKey: string,
+      id: string,
+      callback?: Callback<string[] | null>
+    ): Result<string[] | null, Context>;
+    readLatestSnapshot(
+      eKey: string,
+      idxKey: string,
+      curKey: string,
+      seqKey: string,
+      callback?: Callback<string[] | null>
+    ): Result<string[] | null, Context>;
+  }
+}

From 66479aa9f0bbb2a7c2056c74bc0ae18562e110eb Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Fri, 21 Aug 2026 13:08:39 +0100
Subject: [PATCH 04/49] fix(run-store): anchor liveness on the counter and make
 append idempotent

---
 .../run-store/src/redisSnapshotStore.test.ts  | 99 +++++++++++++++++++
 .../run-store/src/redisSnapshotStore.ts       | 49 ++++++---
 2 files changed, 134 insertions(+), 14 deletions(-)

diff --git a/internal-packages/run-store/src/redisSnapshotStore.test.ts b/internal-packages/run-store/src/redisSnapshotStore.test.ts
index d1a7d8fe71f..d3be34b199c 100644
--- a/internal-packages/run-store/src/redisSnapshotStore.test.ts
+++ b/internal-packages/run-store/src/redisSnapshotStore.test.ts
@@ -2,6 +2,7 @@
 // reference, so no Postgres container is needed.
 import { expect, describe } from "vitest";
 import { redisTest } from "@internal/testcontainers";
+import { createRedisClient } from "@internal/redis";
 import {
   snapshotKeys,
   deriveOrder,
@@ -139,8 +140,106 @@ describe("append", () => {
       });
       expect(r).toEqual({ outcome: "skippedNoKeyspace" });
       expect(await store.getLatest("run_never")).toBeNull();
+
+      const k = snapshotKeys("run_never");
+      const raw = createRedisClient(redisOptions);
+      try {
+        expect(await raw.exists(k.e, k.idx, k.cur, k.seq)).toBe(0);
+      } finally {
+        await raw.quit();
+      }
     } finally {
       await store.quit();
     }
   });
+
+  redisTest("skips a transition when only the seq key has expired", async ({ redisOptions }) => {
+    const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000 });
+    try {
+      await store.append({ entry: entry({ id: "snap_1" }), kind: "birth", isTerminal: false });
+
+      const k = snapshotKeys("run_1");
+      const raw = createRedisClient(redisOptions);
+      try {
+        await raw.del(k.seq);
+      } finally {
+        await raw.quit();
+      }
+
+      const r = await store.append({
+        entry: entry({ id: "snap_2" }),
+        kind: "transition",
+        isTerminal: false,
+      });
+      expect(r).toEqual({ outcome: "skippedNoKeyspace" });
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest(
+    "carries the original count forward on a carryForward append",
+    async ({ redisOptions }) => {
+      const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000 });
+      try {
+        await store.append({
+          entry: entry({ id: "snap_1" }),
+          kind: "birth",
+          isTerminal: false,
+          cycle: {
+            kind: "new",
+            completedWaitpoints: [
+              { id: "w_a", index: 0 },
+              { id: "w_b", index: 1 },
+            ],
+          },
+        });
+        await store.append({
+          entry: entry({ id: "snap_2" }),
+          kind: "transition",
+          isTerminal: false,
+          cycle: { kind: "carryForward", cycleSeq: 1 },
+        });
+
+        const read = await store.getById("run_1", "snap_2");
+        expect(read?.cycle).toEqual({ cycleSeq: 1, count: 2 });
+      } finally {
+        await store.quit();
+      }
+    }
+  );
+
+  redisTest(
+    "reports a duplicate id without overwriting the original entry",
+    async ({ redisOptions }) => {
+      const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000 });
+      try {
+        const first = await store.append({
+          entry: entry({ id: "snap_1", description: "created" }),
+          kind: "birth",
+          isTerminal: false,
+        });
+        expect(first).toMatchObject({ outcome: "written", seq: 1 });
+
+        const dup = await store.append({
+          entry: entry({ id: "snap_1", description: "different" }),
+          kind: "transition",
+          isTerminal: false,
+        });
+        expect(dup).toEqual({ outcome: "duplicate", seq: 1 });
+
+        const read = await store.getById("run_1", "snap_1");
+        expect(read?.entry.description).toBe("created");
+
+        const next = await store.append({
+          entry: entry({ id: "snap_2" }),
+          kind: "transition",
+          isTerminal: false,
+        });
+        expect(next).toMatchObject({ outcome: "written", seq: 2 });
+      } finally {
+        await store.quit();
+      }
+    }
+  );
 });
diff --git a/internal-packages/run-store/src/redisSnapshotStore.ts b/internal-packages/run-store/src/redisSnapshotStore.ts
index 72aae1c2578..5aa7e8b9716 100644
--- a/internal-packages/run-store/src/redisSnapshotStore.ts
+++ b/internal-packages/run-store/src/redisSnapshotStore.ts
@@ -76,7 +76,8 @@ export type AppendResult =
       cycleMismatch: boolean;
     }
   | { outcome: "skippedNoKeyspace" }
-  | { outcome: "forked"; actualCur: string };
+  | { outcome: "forked"; actualCur: string }
+  | { outcome: "duplicate"; seq: number };
 
 export type SnapshotStoreMetrics = {
   recordAppend(outcome: string, ttl: string): void;
@@ -100,6 +101,7 @@ export type RedisSnapshotStoreOptions = {
 const SKIPPED = "skipped";
 const FORKED = "forked";
 const WRITTEN = "written";
+const DUPLICATE = "duplicate";
 
 export class RedisSnapshotStore {
   private readonly redis: Redis;
@@ -189,7 +191,8 @@ export class RedisSnapshotStore {
         orderJson,
         records,
         orderCount,
-        args.expectedCur ?? ""
+        args.expectedCur ?? "",
+        args.expectedCur !== undefined ? "1" : "0"
       )) as string[];
 
       return this.#interpretAppend(reply, raw, orderJson);
@@ -206,6 +209,10 @@ export class RedisSnapshotStore {
       this.metrics?.recordAppend("forked", "none");
       return { outcome: "forked", actualCur: reply[1] ?? "" };
     }
+    if (reply[0] === DUPLICATE) {
+      this.metrics?.recordAppend("duplicate", "none");
+      return { outcome: "duplicate", seq: Number(reply[1]) };
+    }
     const seq = Number(reply[1]);
     const cycleSeq = Number(reply[2]);
     const ttl = reply[3] as "none" | "completion" | "reapplied";
@@ -319,25 +326,33 @@ export class RedisSnapshotStore {
         local records     = ARGV[10]
         local orderCount  = ARGV[11]
         local expectedCur = ARGV[12]
+        local casEnabled  = ARGV[13] == '1'
 
-        -- Liveness is ONE anchor. All keys get the same PEXPIRE but expire independently, so seq can
-        -- vanish while e and cur linger; anchoring on e treats a partly expired keyspace as gone,
-        -- once and consistently. A birth creates the keyspace; a transition that finds none writes
-        -- nothing. That state has two causes the caller must not merge: a completed run whose TTL
-        -- fired, and a run that predates this org's dual-write.
-        if kind == 'transition' and redis.call('EXISTS', eKey) == 0 then
+        -- Liveness is TWO anchors: e and seq. All keys get the same PEXPIRE but expire independently
+        -- (or seq can vanish under maxmemory eviction while e survives), so checking e alone lets a
+        -- late transition recreate seq with no TTL and restart it at 1 beside a surviving idx. A
+        -- birth always creates both in this same script, so this never rejects a live keyspace.
+        if kind == 'transition' and (redis.call('EXISTS', eKey) == 0 or redis.call('EXISTS', seqKey) == 0) then
           return { '${SKIPPED}' }
         end
 
-        -- Optional compare-and-set on cur, checked BEFORE any mutation. Absent by default, which
-        -- matches Postgres: it has no such guard either.
-        if expectedCur ~= '' then
+        -- Optional compare-and-set on cur, checked BEFORE any mutation. Gated on an explicit flag
+        -- (not on expectedCur ~= ''), so a caller asserting cur is unset (expectedCur = '') still
+        -- gets a real check instead of silently skipping it.
+        if casEnabled then
           local actual = redis.call('GET', curKey)
           if (actual or '') ~= expectedCur then
             return { '${FORKED}', actual or '' }
           end
         end
 
+        -- Append-only: a retried append (eg. ioredis reconnect-and-retry on a READONLY/UNBLOCKED
+        -- reply error) must not overwrite an existing entry's bytes or rescore it in idx.
+        local prior = redis.call('HGET', eKey, id .. '#s')
+        if prior then
+          return { '${DUPLICATE}', prior }
+        end
+
         local seq = redis.call('HINCRBY', seqKey, 'e', 1)
 
         local cycleSeq = 0
@@ -346,14 +361,17 @@ export class RedisSnapshotStore {
           -- The STORE mints cycleSeq, so the sequence is dense by construction and the terminal
           -- PEXPIRE loop from 1..c is correct.
           cycleSeq = redis.call('HINCRBY', seqKey, 'c', 1)
-          redis.call('HSET', wpKey(cycleSeq), 'order', orderJson)
+          redis.call('HSET', wpKey(cycleSeq), 'order', orderJson, 'count', orderCount)
           if records ~= '' then
             redis.call('HSET', wpKey(cycleSeq), 'records', records)
           end
         elseif cycleMode == 'carry' then
           cycleSeq = cycleSeqIn
-          if redis.call('EXISTS', wpKey(cycleSeq)) == 0 then
+          local c = redis.call('HGET', wpKey(cycleSeq), 'count')
+          if not c then
             mismatch = 1
+          else
+            orderCount = c
           end
         end
 
@@ -363,7 +381,9 @@ export class RedisSnapshotStore {
         end
 
         -- idx indexes VALID entries only, which makes the since-cap exact. An invalid entry is still
-        -- reachable by id, and its seq is still readable from its own '#s' field.
+        -- reachable by id, and its seq is still readable from its own '#s' field. ZADD before SET cur
+        -- because Redis never rolls back a partially applied script: if a later call in this script
+        -- errored, having idx already written is the recoverable half of the pair.
         if isValid then
           redis.call('ZADD', idxKey, seq, id)
           redis.call('SET', curKey, id)
@@ -444,6 +464,7 @@ declare module "@internal/redis" {
       records: string,
       orderCount: string,
       expectedCur: string,
+      casEnabled: string,
       callback?: Callback<string[]>
     ): Result<string[], Context>;
     readSnapshotById(

From 9f6772438dbac5a2d9776dcc1ab3a1b1dbb2908d Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Fri, 21 Aug 2026 13:15:46 +0100
Subject: [PATCH 05/49] feat(run-store): wait-cycle waitpoint id reads

---
 .../run-store/src/redisSnapshotStore.test.ts  | 101 ++++++++++++++++++
 .../run-store/src/redisSnapshotStore.ts       |  33 ++++++
 2 files changed, 134 insertions(+)

diff --git a/internal-packages/run-store/src/redisSnapshotStore.test.ts b/internal-packages/run-store/src/redisSnapshotStore.test.ts
index d3be34b199c..e279e4f807e 100644
--- a/internal-packages/run-store/src/redisSnapshotStore.test.ts
+++ b/internal-packages/run-store/src/redisSnapshotStore.test.ts
@@ -243,3 +243,104 @@ describe("append", () => {
     }
   );
 });
+
+describe("cycle keys", () => {
+  redisTest(
+    "mints an increasing cycleSeq across successive new cycles",
+    async ({ redisOptions }) => {
+      const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000 });
+      try {
+        await store.append({ entry: entry({ id: "snap_1" }), kind: "birth", isTerminal: false });
+        const a = await store.append({
+          entry: entry({ id: "snap_2" }),
+          kind: "transition",
+          isTerminal: false,
+          cycle: { kind: "new", completedWaitpoints: [{ id: "w_a", index: 0 }] },
+        });
+        const b = await store.append({
+          entry: entry({ id: "snap_3" }),
+          kind: "transition",
+          isTerminal: false,
+          cycle: { kind: "new", completedWaitpoints: [{ id: "w_b", index: 0 }] },
+        });
+        expect(a).toMatchObject({ cycleSeq: 1 });
+        expect(b).toMatchObject({ cycleSeq: 2 });
+      } finally {
+        await store.quit();
+      }
+    }
+  );
+
+  redisTest(
+    "a carry-forward reuses the cycle and does not rewrite it",
+    async ({ redisOptions }) => {
+      const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000 });
+      try {
+        await store.append({ entry: entry({ id: "snap_1" }), kind: "birth", isTerminal: false });
+        await store.append({
+          entry: entry({ id: "snap_2" }),
+          kind: "transition",
+          isTerminal: false,
+          cycle: {
+            kind: "new",
+            completedWaitpoints: [
+              { id: "w_a", index: 0 },
+              { id: "w_a", index: 1 },
+            ],
+          },
+        });
+        const carried = await store.append({
+          entry: entry({ id: "snap_3" }),
+          kind: "transition",
+          isTerminal: false,
+          cycle: { kind: "carryForward", cycleSeq: 1 },
+        });
+        expect(carried).toMatchObject({ cycleSeq: 1, cycleMismatch: false });
+
+        // Both entries resolve to the SAME cycle contents, written once.
+        const first = await store.getSnapshotWaitpointIds("run_1", "snap_2");
+        const second = await store.getSnapshotWaitpointIds("run_1", "snap_3");
+        expect(first.order).toEqual(["w_a", "w_a"]);
+        expect(first.distinctIds).toEqual(["w_a"]);
+        expect(second).toEqual(first);
+      } finally {
+        await store.quit();
+      }
+    }
+  );
+
+  redisTest("a carry-forward naming a missing cycle still appends", async ({ redisOptions }) => {
+    const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000 });
+    try {
+      await store.append({ entry: entry({ id: "snap_1" }), kind: "birth", isTerminal: false });
+      const r = await store.append({
+        entry: entry({ id: "snap_2" }),
+        kind: "transition",
+        isTerminal: false,
+        cycle: { kind: "carryForward", cycleSeq: 99 },
+      });
+      expect(r).toMatchObject({ outcome: "written", cycleMismatch: true });
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest("reports presence and emptiness separately", async ({ redisOptions }) => {
+    const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000 });
+    try {
+      await store.append({ entry: entry({ id: "snap_1" }), kind: "birth", isTerminal: false });
+      expect(await store.getSnapshotWaitpointIds("run_1", "nope")).toEqual({
+        present: false,
+        distinctIds: [],
+        order: [],
+      });
+      expect(await store.getSnapshotWaitpointIds("run_1", "snap_1")).toEqual({
+        present: true,
+        distinctIds: [],
+        order: [],
+      });
+    } finally {
+      await store.quit();
+    }
+  });
+});
diff --git a/internal-packages/run-store/src/redisSnapshotStore.ts b/internal-packages/run-store/src/redisSnapshotStore.ts
index 5aa7e8b9716..35d828b29f0 100644
--- a/internal-packages/run-store/src/redisSnapshotStore.ts
+++ b/internal-packages/run-store/src/redisSnapshotStore.ts
@@ -272,6 +272,18 @@ export class RedisSnapshotStore {
     });
   }
 
+  // Returns all three shapes the Postgres surface needs from one read: `distinctIds` matches the
+  // deduped join that findSnapshotCompletedWaitpointIds returns, `present` serves the WithPresence
+  // variant (which distinguishes "no waitpoints" from "snapshot not visible"), and `order` keeps the
+  // repeats that the engine expands into one CompletedWaitpoint per position.
+  async getSnapshotWaitpointIds(runId: string, snapshotId: string): Promise<WaitpointIds> {
+    return this.#timed("getSnapshotWaitpointIds", async () => {
+      const k = snapshotKeys(runId);
+      const reply = await this.redis.readSnapshotWaitpointIds(k.e, k.idx, k.cur, k.seq, snapshotId);
+      return decodeWaitpointIds(reply[0] === "1", reply[1] ?? "");
+    });
+  }
+
   // [id, raw, seq, pointer, order] -> SnapshotRead. The environment compare is app-side, per the
   // plan: the store returns null for a foreign environment and the 404 throw stays in the engine.
   #decode(reply: string[] | null, environmentId?: string): SnapshotRead | null {
@@ -437,6 +449,19 @@ export class RedisSnapshotStore {
         return { cur, vals[1], vals[2] or '', vals[3] or '', orderFor(vals[3]) }
       `,
     });
+
+    this.redis.defineCommand("readSnapshotWaitpointIds", {
+      numberOfKeys: 4,
+      lua: `
+        ${PRELUDE}
+        local id = ARGV[1]
+        if redis.call('HEXISTS', eKey, id) == 0 then
+          return { '0', '' }
+        end
+        local pointer = redis.call('HGET', eKey, id .. '#c')
+        return { '1', orderFor(pointer) }
+      `,
+    });
   }
 }
 
@@ -482,5 +507,13 @@ declare module "@internal/redis" {
       seqKey: string,
       callback?: Callback<string[] | null>
     ): Result<string[] | null, Context>;
+    readSnapshotWaitpointIds(
+      eKey: string,
+      idxKey: string,
+      curKey: string,
+      seqKey: string,
+      id: string,
+      callback?: Callback<string[]>
+    ): Result<string[], Context>;
   }
 }

From 47dab9623e7d1e54a66ad6df16523a08ef5d694d Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Fri, 21 Aug 2026 13:23:10 +0100
Subject: [PATCH 06/49] test(run-store): the snapshot store TTL rule and
 keyspace liveness

---
 .../run-store/src/redisSnapshotStore.test.ts  | 141 ++++++++++++++++++
 1 file changed, 141 insertions(+)

diff --git a/internal-packages/run-store/src/redisSnapshotStore.test.ts b/internal-packages/run-store/src/redisSnapshotStore.test.ts
index e279e4f807e..9c8903091fe 100644
--- a/internal-packages/run-store/src/redisSnapshotStore.test.ts
+++ b/internal-packages/run-store/src/redisSnapshotStore.test.ts
@@ -177,6 +177,32 @@ describe("append", () => {
     }
   });
 
+  // Pairs with "skips a transition when only the seq key has expired" above: liveness is checked
+  // against BOTH anchors, so either one missing alone must skip.
+  redisTest("skips a transition when only the e key has expired", async ({ redisOptions }) => {
+    const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000 });
+    try {
+      await store.append({ entry: entry({ id: "snap_1" }), kind: "birth", isTerminal: false });
+
+      const k = snapshotKeys("run_1");
+      const raw = createRedisClient(redisOptions);
+      try {
+        await raw.del(k.e);
+      } finally {
+        await raw.quit();
+      }
+
+      const r = await store.append({
+        entry: entry({ id: "snap_2" }),
+        kind: "transition",
+        isTerminal: false,
+      });
+      expect(r).toEqual({ outcome: "skippedNoKeyspace" });
+    } finally {
+      await store.quit();
+    }
+  });
+
   redisTest(
     "carries the original count forward on a carryForward append",
     async ({ redisOptions }) => {
@@ -344,3 +370,118 @@ describe("cycle keys", () => {
     }
   });
 });
+
+describe("TTL rule", () => {
+  redisTest("a non-terminal append leaves every key unexpiring", async ({ redisOptions }) => {
+    const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 60_000 });
+    const raw = createRedisClient(redisOptions);
+    try {
+      await store.append({
+        entry: entry({ id: "s1" }),
+        kind: "birth",
+        isTerminal: false,
+        cycle: { kind: "new", completedWaitpoints: [{ id: "w_a", index: 0 }] },
+      });
+      for (const key of [
+        "snap:{run_1}:e",
+        "snap:{run_1}:idx",
+        "snap:{run_1}:cur",
+        "snap:{run_1}:seq",
+        "snap:{run_1}:wp:1",
+      ]) {
+        expect(await raw.pttl(key)).toBe(-1);
+      }
+    } finally {
+      raw.disconnect();
+      await store.quit();
+    }
+  });
+
+  redisTest(
+    "a terminal append expires every key, cycle keys included",
+    async ({ redisOptions }) => {
+      const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 60_000 });
+      const raw = createRedisClient(redisOptions);
+      try {
+        await store.append({
+          entry: entry({ id: "s1" }),
+          kind: "birth",
+          isTerminal: false,
+          cycle: { kind: "new", completedWaitpoints: [{ id: "w_a", index: 0 }] },
+        });
+        const r = await store.append({
+          entry: entry({ id: "s2", executionStatus: "FINISHED" }),
+          kind: "transition",
+          isTerminal: true,
+        });
+        expect(r).toMatchObject({ ttl: "completion" });
+        for (const key of [
+          "snap:{run_1}:e",
+          "snap:{run_1}:idx",
+          "snap:{run_1}:cur",
+          "snap:{run_1}:seq",
+          "snap:{run_1}:wp:1",
+        ]) {
+          const ttl = await raw.pttl(key);
+          expect(ttl).toBeGreaterThan(0);
+          expect(ttl).toBeLessThanOrEqual(60_000);
+        }
+      } finally {
+        raw.disconnect();
+        await store.quit();
+      }
+    }
+  );
+
+  redisTest("a post-completion append re-applies the completion TTL", async ({ redisOptions }) => {
+    const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 60_000 });
+    const raw = createRedisClient(redisOptions);
+    try {
+      await store.append({ entry: entry({ id: "s1" }), kind: "birth", isTerminal: false });
+      await store.append({
+        entry: entry({ id: "s2", executionStatus: "FINISHED" }),
+        kind: "transition",
+        isTerminal: true,
+      });
+      // A stale client appends a non-terminal, invalid row after FINISHED.
+      const late = await store.append({
+        entry: entry({ id: "s3", error: "stale" }),
+        kind: "transition",
+        isTerminal: false,
+      });
+      expect(late).toMatchObject({ outcome: "written", ttl: "reapplied" });
+      // Never a live TTL, and never cleared: the key stays bounded.
+      const ttl = await raw.pttl("snap:{run_1}:e");
+      expect(ttl).toBeGreaterThan(0);
+      expect(ttl).toBeLessThanOrEqual(60_000);
+    } finally {
+      raw.disconnect();
+      await store.quit();
+    }
+  });
+
+  redisTest("a transition after the keyspace expired writes nothing", async ({ redisOptions }) => {
+    const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 60_000 });
+    const raw = createRedisClient(redisOptions);
+    try {
+      await store.append({ entry: entry({ id: "s1" }), kind: "birth", isTerminal: false });
+      await store.append({
+        entry: entry({ id: "s2", executionStatus: "FINISHED" }),
+        kind: "transition",
+        isTerminal: true,
+      });
+      // Simulate the completion TTL firing.
+      await raw.del("snap:{run_1}:e", "snap:{run_1}:idx", "snap:{run_1}:cur", "snap:{run_1}:seq");
+      const after = await store.append({
+        entry: entry({ id: "s4" }),
+        kind: "transition",
+        isTerminal: false,
+      });
+      expect(after).toEqual({ outcome: "skippedNoKeyspace" });
+      expect(await raw.exists("snap:{run_1}:e")).toBe(0);
+    } finally {
+      raw.disconnect();
+      await store.quit();
+    }
+  });
+});

From a4e9ededdaec2a8fe65cd8bdffb8209885c1111e Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Fri, 21 Aug 2026 13:33:40 +0100
Subject: [PATCH 07/49] test(run-store): prove the completion TTL is
 re-applied, not merely uncleared

---
 .../run-store/src/redisSnapshotStore.test.ts  | 52 +++++++++++++++----
 1 file changed, 43 insertions(+), 9 deletions(-)

diff --git a/internal-packages/run-store/src/redisSnapshotStore.test.ts b/internal-packages/run-store/src/redisSnapshotStore.test.ts
index 9c8903091fe..86e458796ed 100644
--- a/internal-packages/run-store/src/redisSnapshotStore.test.ts
+++ b/internal-packages/run-store/src/redisSnapshotStore.test.ts
@@ -392,7 +392,7 @@ describe("TTL rule", () => {
         expect(await raw.pttl(key)).toBe(-1);
       }
     } finally {
-      raw.disconnect();
+      await raw.quit();
       await store.quit();
     }
   });
@@ -409,6 +409,13 @@ describe("TTL rule", () => {
           isTerminal: false,
           cycle: { kind: "new", completedWaitpoints: [{ id: "w_a", index: 0 }] },
         });
+        // Second cycle, so the terminal PEXPIRE loop runs past its first iteration.
+        await store.append({
+          entry: entry({ id: "s1b" }),
+          kind: "transition",
+          isTerminal: false,
+          cycle: { kind: "new", completedWaitpoints: [{ id: "w_b", index: 0 }] },
+        });
         const r = await store.append({
           entry: entry({ id: "s2", executionStatus: "FINISHED" }),
           kind: "transition",
@@ -421,13 +428,14 @@ describe("TTL rule", () => {
           "snap:{run_1}:cur",
           "snap:{run_1}:seq",
           "snap:{run_1}:wp:1",
+          "snap:{run_1}:wp:2",
         ]) {
           const ttl = await raw.pttl(key);
           expect(ttl).toBeGreaterThan(0);
           expect(ttl).toBeLessThanOrEqual(60_000);
         }
       } finally {
-        raw.disconnect();
+        await raw.quit();
         await store.quit();
       }
     }
@@ -437,12 +445,37 @@ describe("TTL rule", () => {
     const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 60_000 });
     const raw = createRedisClient(redisOptions);
     try {
-      await store.append({ entry: entry({ id: "s1" }), kind: "birth", isTerminal: false });
+      await store.append({
+        entry: entry({ id: "s1" }),
+        kind: "birth",
+        isTerminal: false,
+        cycle: { kind: "new", completedWaitpoints: [{ id: "w_a", index: 0 }] },
+      });
+      await store.append({
+        entry: entry({ id: "s1b" }),
+        kind: "transition",
+        isTerminal: false,
+        cycle: { kind: "new", completedWaitpoints: [{ id: "w_b", index: 0 }] },
+      });
       await store.append({
         entry: entry({ id: "s2", executionStatus: "FINISHED" }),
         kind: "transition",
         isTerminal: true,
       });
+
+      const keys = [
+        "snap:{run_1}:e",
+        "snap:{run_1}:idx",
+        "snap:{run_1}:cur",
+        "snap:{run_1}:seq",
+        "snap:{run_1}:wp:1",
+        "snap:{run_1}:wp:2",
+      ];
+      // Shrink first: a re-apply is then the only way the TTL can go back up.
+      for (const key of keys) {
+        await raw.pexpire(key, 5_000);
+      }
+
       // A stale client appends a non-terminal, invalid row after FINISHED.
       const late = await store.append({
         entry: entry({ id: "s3", error: "stale" }),
@@ -450,12 +483,13 @@ describe("TTL rule", () => {
         isTerminal: false,
       });
       expect(late).toMatchObject({ outcome: "written", ttl: "reapplied" });
-      // Never a live TTL, and never cleared: the key stays bounded.
-      const ttl = await raw.pttl("snap:{run_1}:e");
-      expect(ttl).toBeGreaterThan(0);
-      expect(ttl).toBeLessThanOrEqual(60_000);
+      for (const key of keys) {
+        const ttl = await raw.pttl(key);
+        expect(ttl).toBeGreaterThan(55_000);
+        expect(ttl).toBeLessThanOrEqual(60_000);
+      }
     } finally {
-      raw.disconnect();
+      await raw.quit();
       await store.quit();
     }
   });
@@ -480,7 +514,7 @@ describe("TTL rule", () => {
       expect(after).toEqual({ outcome: "skippedNoKeyspace" });
       expect(await raw.exists("snap:{run_1}:e")).toBe(0);
     } finally {
-      raw.disconnect();
+      await raw.quit();
       await store.quit();
     }
   });

From 005c76c0f588d0b7160a62886a020d0305b79c73 Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Fri, 21 Aug 2026 13:41:21 +0100
Subject: [PATCH 08/49] feat(run-store): getSince with a newest-first window
 and head-only waitpoints

---
 .../run-store/src/redisSnapshotStore.test.ts  | 111 ++++++++++++++++++
 .../run-store/src/redisSnapshotStore.ts       |  97 +++++++++++++++
 2 files changed, 208 insertions(+)

diff --git a/internal-packages/run-store/src/redisSnapshotStore.test.ts b/internal-packages/run-store/src/redisSnapshotStore.test.ts
index 86e458796ed..a34b2f4e5ad 100644
--- a/internal-packages/run-store/src/redisSnapshotStore.test.ts
+++ b/internal-packages/run-store/src/redisSnapshotStore.test.ts
@@ -519,3 +519,114 @@ describe("TTL rule", () => {
     }
   });
 });
+
+describe("getSince", () => {
+  redisTest("misses on an unknown since id", async ({ redisOptions }) => {
+    const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000 });
+    try {
+      await store.append({ entry: entry({ id: "s1" }), kind: "birth", isTerminal: false });
+      expect(await store.getSince("run_1", "unknown")).toEqual({ kind: "miss" });
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest("resolves an INVALID since id through its own seq field", async ({ redisOptions }) => {
+    const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000 });
+    try {
+      await store.append({ entry: entry({ id: "s1" }), kind: "birth", isTerminal: false });
+      await store.append({
+        entry: entry({ id: "s_bad", error: "x" }),
+        kind: "transition",
+        isTerminal: false,
+      });
+      await store.append({ entry: entry({ id: "s3" }), kind: "transition", isTerminal: false });
+
+      // s_bad is not in the valid-only index, so ZSCORE misses and the '#s' field answers instead.
+      const r = await store.getSince("run_1", "s_bad");
+      expect(r.kind).toBe("hit");
+      if (r.kind !== "hit") throw new Error("unreachable");
+      expect(r.entries.map((e) => e.id)).toEqual(["s3"]);
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest("returns the NEWEST N ascending, not the oldest", async ({ redisOptions }) => {
+    const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000, sinceLimit: 5 });
+    try {
+      await store.append({ entry: entry({ id: "s0" }), kind: "birth", isTerminal: false });
+      for (let i = 1; i <= 12; i++) {
+        await store.append({
+          entry: entry({ id: `s${i}` }),
+          kind: "transition",
+          isTerminal: false,
+        });
+      }
+      const r = await store.getSince("run_1", "s0");
+      if (r.kind !== "hit") throw new Error("expected a hit");
+      // The engine reads createdAt desc / take N / reverse, so the window is the newest N ascending.
+      expect(r.entries.map((e) => e.id)).toEqual(["s8", "s9", "s10", "s11", "s12"]);
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest("excludes invalid entries from the window", async ({ redisOptions }) => {
+    const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000 });
+    try {
+      await store.append({ entry: entry({ id: "s0" }), kind: "birth", isTerminal: false });
+      await store.append({
+        entry: entry({ id: "s_bad", error: "x" }),
+        kind: "transition",
+        isTerminal: false,
+      });
+      await store.append({ entry: entry({ id: "s2" }), kind: "transition", isTerminal: false });
+      const r = await store.getSince("run_1", "s0");
+      if (r.kind !== "hit") throw new Error("expected a hit");
+      expect(r.entries.map((e) => e.id)).toEqual(["s2"]);
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest("resolves waitpoint ids for the HEAD only", async ({ redisOptions }) => {
+    const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000 });
+    try {
+      await store.append({ entry: entry({ id: "s0" }), kind: "birth", isTerminal: false });
+      await store.append({
+        entry: entry({ id: "s1" }),
+        kind: "transition",
+        isTerminal: false,
+        cycle: { kind: "new", completedWaitpoints: [{ id: "w_old", index: 0 }] },
+      });
+      await store.append({
+        entry: entry({ id: "s2" }),
+        kind: "transition",
+        isTerminal: false,
+        cycle: { kind: "new", completedWaitpoints: [{ id: "w_new", index: 0 }] },
+      });
+      const r = await store.getSince("run_1", "s0");
+      if (r.kind !== "hit") throw new Error("expected a hit");
+      // The head is the NEWEST entry, and only it carries resolved ids.
+      expect(r.headWaitpointIds.order).toEqual(["w_new"]);
+      expect(r.entries.at(-1)?.id).toBe("s2");
+      expect(r.entries[0]?.completedWaitpointIds).toBeUndefined();
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest("misses for a foreign environment", async ({ redisOptions }) => {
+    const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000 });
+    try {
+      await store.append({ entry: entry({ id: "s0" }), kind: "birth", isTerminal: false });
+      await store.append({ entry: entry({ id: "s1" }), kind: "transition", isTerminal: false });
+      expect(await store.getSince("run_1", "s0", { environmentId: "env_other" })).toEqual({
+        kind: "miss",
+      });
+    } finally {
+      await store.quit();
+    }
+  });
+});
diff --git a/internal-packages/run-store/src/redisSnapshotStore.ts b/internal-packages/run-store/src/redisSnapshotStore.ts
index 35d828b29f0..df05717a479 100644
--- a/internal-packages/run-store/src/redisSnapshotStore.ts
+++ b/internal-packages/run-store/src/redisSnapshotStore.ts
@@ -57,6 +57,10 @@ export type SnapshotEntryInput = {
 
 export type WaitpointIds = { present: boolean; distinctIds: string[]; order: string[] };
 
+export type GetSinceResult =
+  | { kind: "miss" }
+  | { kind: "hit"; entries: SnapshotRead[]; headWaitpointIds: WaitpointIds };
+
 export type SnapshotRead = {
   id: string;
   seq: number;
@@ -284,6 +288,55 @@ export class RedisSnapshotStore {
     });
   }
 
+  // A miss is not an error. It is the coexistence path: a pre-cutover snapshot id, expired history,
+  // or an org not yet enabled. The caller falls back to Postgres.
+  async getSince(
+    runId: string,
+    sinceId: string,
+    opts?: { environmentId?: string; limit?: number }
+  ): Promise<GetSinceResult> {
+    return this.#timed("getSince", async () => {
+      const k = snapshotKeys(runId);
+      const limit = opts?.limit ?? this.sinceLimit;
+      const reply = await this.redis.readSnapshotsSince(
+        k.e,
+        k.idx,
+        k.cur,
+        k.seq,
+        sinceId,
+        String(limit)
+      );
+      if (reply === null) return { kind: "miss" };
+
+      const headOrder = reply[0] ?? "";
+      const rows: SnapshotRead[] = [];
+      for (let i = 1; i + 3 < reply.length + 1; i += 4) {
+        const decoded = this.#decode(
+          [reply[i], reply[i + 1], reply[i + 2], reply[i + 3], ""],
+          opts?.environmentId
+        );
+        if (decoded) rows.push(decoded);
+      }
+
+      // The since id itself is env-scoped in the engine, so a foreign environment must miss rather
+      // than return an empty hit: an empty hit would read as "nothing new", not "not found".
+      if (opts?.environmentId !== undefined && rows.length === 0 && reply.length > 1) {
+        return { kind: "miss" };
+      }
+
+      rows.reverse();
+      const head = rows[rows.length - 1];
+      const headWaitpointIds = decodeWaitpointIds(head !== undefined, headOrder);
+      if (head) {
+        head.completedWaitpointIds = headWaitpointIds;
+      }
+      for (const row of rows.slice(0, -1)) {
+        delete row.completedWaitpointIds;
+      }
+      return { kind: "hit", entries: rows, headWaitpointIds };
+    });
+  }
+
   // [id, raw, seq, pointer, order] -> SnapshotRead. The environment compare is app-side, per the
   // plan: the store returns null for a foreign environment and the 404 throw stays in the engine.
   #decode(reply: string[] | null, environmentId?: string): SnapshotRead | null {
@@ -462,6 +515,41 @@ export class RedisSnapshotStore {
         return { '1', orderFor(pointer) }
       `,
     });
+
+    this.redis.defineCommand("readSnapshotsSince", {
+      numberOfKeys: 4,
+      lua: `
+        ${PRELUDE}
+        local sinceId = ARGV[1]
+        local limit = tonumber(ARGV[2])
+
+        -- The index holds valid entries only, so an invalid since id misses ZSCORE. Its seq is still
+        -- on its own '#s' field, which keeps the id resolvable without indexing invalid rows.
+        local score = redis.call('ZSCORE', idxKey, sinceId)
+        if not score then
+          score = redis.call('HGET', eKey, sinceId .. '#s')
+          if not score then return nil end
+        end
+
+        -- NEWEST-first with a limit, then reversed app-side. The engine reads createdAt desc /
+        -- take N / reverse, so the oldest-first form would return the wrong window entirely.
+        local ids = redis.call('ZREVRANGEBYSCORE', idxKey, '+inf', '(' .. score, 'LIMIT', 0, limit)
+        if #ids == 0 then return { '' } end
+
+        -- The head is the newest entry, and it is the ONLY one whose cycle key is read. That makes
+        -- head-only hydration structural: the tail's cycle keys are never touched.
+        local out = { orderFor(redis.call('HGET', eKey, ids[1] .. '#c')) }
+        for i = 1, #ids do
+          local id = ids[i]
+          local vals = redis.call('HMGET', eKey, id, id .. '#s', id .. '#c')
+          out[#out + 1] = id
+          out[#out + 1] = vals[1] or ''
+          out[#out + 1] = vals[2] or ''
+          out[#out + 1] = vals[3] or ''
+        end
+        return out
+      `,
+    });
   }
 }
 
@@ -515,5 +603,14 @@ declare module "@internal/redis" {
       id: string,
       callback?: Callback<string[]>
     ): Result<string[], Context>;
+    readSnapshotsSince(
+      eKey: string,
+      idxKey: string,
+      curKey: string,
+      seqKey: string,
+      sinceId: string,
+      limit: string,
+      callback?: Callback<string[] | null>
+    ): Result<string[] | null, Context>;
   }
 }

From 42d9a64b7e2423266ee8c0e1444c3c441a9e29b9 Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Fri, 21 Aug 2026 13:54:24 +0100
Subject: [PATCH 09/49] fix(run-store): scope getSince by the since entry, not
 by the window

---
 .../run-store/src/redisSnapshotStore.test.ts  | 52 +++++++++++++++++++
 .../run-store/src/redisSnapshotStore.ts       | 38 +++++++++-----
 2 files changed, 76 insertions(+), 14 deletions(-)

diff --git a/internal-packages/run-store/src/redisSnapshotStore.test.ts b/internal-packages/run-store/src/redisSnapshotStore.test.ts
index a34b2f4e5ad..6a7c9a928e2 100644
--- a/internal-packages/run-store/src/redisSnapshotStore.test.ts
+++ b/internal-packages/run-store/src/redisSnapshotStore.test.ts
@@ -629,4 +629,56 @@ describe("getSince", () => {
       await store.quit();
     }
   });
+
+  redisTest("misses for a foreign environment even at the newest id", async ({ redisOptions }) => {
+    const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000 });
+    try {
+      await store.append({ entry: entry({ id: "s0" }), kind: "birth", isTerminal: false });
+      await store.append({ entry: entry({ id: "s1" }), kind: "transition", isTerminal: false });
+      // The window here is empty (s1 is the newest), so this is the case the old reply.length > 1
+      // guard could never catch: an empty window must not silently coerce a foreign miss into a hit.
+      expect(await store.getSince("run_1", "s1", { environmentId: "env_other" })).toEqual({
+        kind: "miss",
+      });
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest(
+    "hits with zero entries when nothing follows the since id",
+    async ({ redisOptions }) => {
+      const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000 });
+      try {
+        await store.append({ entry: entry({ id: "s0" }), kind: "birth", isTerminal: false });
+        // Resolves, nothing after it: "nothing new", NOT "not found".
+        expect(await store.getSince("run_1", "s0")).toEqual({
+          kind: "hit",
+          entries: [],
+          headWaitpointIds: { present: false, distinctIds: [], order: [] },
+        });
+      } finally {
+        await store.quit();
+      }
+    }
+  );
+
+  redisTest(
+    "hits with zero entries when scoped to the since entry's own environment",
+    async ({ redisOptions }) => {
+      const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000 });
+      try {
+        await store.append({ entry: entry({ id: "s0" }), kind: "birth", isTerminal: false });
+        // Matching environment, nothing after it: pins that an empty window resolves via sinceRaw,
+        // not by falling through to the "sinceRaw missing" miss path.
+        expect(await store.getSince("run_1", "s0", { environmentId: "env_1" })).toEqual({
+          kind: "hit",
+          entries: [],
+          headWaitpointIds: { present: false, distinctIds: [], order: [] },
+        });
+      } finally {
+        await store.quit();
+      }
+    }
+  );
 });
diff --git a/internal-packages/run-store/src/redisSnapshotStore.ts b/internal-packages/run-store/src/redisSnapshotStore.ts
index df05717a479..e76963ef379 100644
--- a/internal-packages/run-store/src/redisSnapshotStore.ts
+++ b/internal-packages/run-store/src/redisSnapshotStore.ts
@@ -308,9 +308,18 @@ export class RedisSnapshotStore {
       );
       if (reply === null) return { kind: "miss" };
 
-      const headOrder = reply[0] ?? "";
+      const sinceRaw = reply[0] ?? "";
+      if (opts?.environmentId !== undefined) {
+        // Scoped by the since entry itself, same as Postgres's step-1 lookup: a foreign since id
+        // is NOT FOUND regardless of what follows it, never an empty "nothing new" hit.
+        if (sinceRaw === "") return { kind: "miss" };
+        const since = JSON.parse(sinceRaw) as { environmentId?: string };
+        if (since.environmentId !== opts.environmentId) return { kind: "miss" };
+      }
+
+      const headOrder = reply[1] ?? "";
       const rows: SnapshotRead[] = [];
-      for (let i = 1; i + 3 < reply.length + 1; i += 4) {
+      for (let i = 2; i + 3 < reply.length; i += 4) {
         const decoded = this.#decode(
           [reply[i], reply[i + 1], reply[i + 2], reply[i + 3], ""],
           opts?.environmentId
@@ -318,12 +327,6 @@ export class RedisSnapshotStore {
         if (decoded) rows.push(decoded);
       }
 
-      // The since id itself is env-scoped in the engine, so a foreign environment must miss rather
-      // than return an empty hit: an empty hit would read as "nothing new", not "not found".
-      if (opts?.environmentId !== undefined && rows.length === 0 && reply.length > 1) {
-        return { kind: "miss" };
-      }
-
       rows.reverse();
       const head = rows[rows.length - 1];
       const headWaitpointIds = decodeWaitpointIds(head !== undefined, headOrder);
@@ -531,21 +534,28 @@ export class RedisSnapshotStore {
           if not score then return nil end
         end
 
+        -- Env scoping is decided from the since entry itself, not from the window it produces.
+        local sinceRaw = redis.call('HGET', eKey, sinceId) or ''
+
         -- NEWEST-first with a limit, then reversed app-side. The engine reads createdAt desc /
         -- take N / reverse, so the oldest-first form would return the wrong window entirely.
         local ids = redis.call('ZREVRANGEBYSCORE', idxKey, '+inf', '(' .. score, 'LIMIT', 0, limit)
-        if #ids == 0 then return { '' } end
+        if #ids == 0 then return { sinceRaw, '' } end
 
         -- The head is the newest entry, and it is the ONLY one whose cycle key is read. That makes
         -- head-only hydration structural: the tail's cycle keys are never touched.
-        local out = { orderFor(redis.call('HGET', eKey, ids[1] .. '#c')) }
+        local out = { sinceRaw, orderFor(redis.call('HGET', eKey, ids[1] .. '#c')) }
         for i = 1, #ids do
           local id = ids[i]
           local vals = redis.call('HMGET', eKey, id, id .. '#s', id .. '#c')
-          out[#out + 1] = id
-          out[#out + 1] = vals[1] or ''
-          out[#out + 1] = vals[2] or ''
-          out[#out + 1] = vals[3] or ''
+          -- A nil body (e survived only partially, eg. idx outlived e) must drop the row, not emit
+          -- an unparseable '' that would throw out of JSON.parse in #decode.
+          if vals[1] then
+            out[#out + 1] = id
+            out[#out + 1] = vals[1]
+            out[#out + 1] = vals[2] or ''
+            out[#out + 1] = vals[3] or ''
+          end
         end
         return out
       `,

From ef4fb1382b644a9b25e12ba13702cf4514352748 Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Fri, 21 Aug 2026 13:57:53 +0100
Subject: [PATCH 10/49] test(run-store): cover getSince's evicted-body skip
 guard

---
 .../run-store/src/redisSnapshotStore.test.ts  | 25 +++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/internal-packages/run-store/src/redisSnapshotStore.test.ts b/internal-packages/run-store/src/redisSnapshotStore.test.ts
index 6a7c9a928e2..00615482da7 100644
--- a/internal-packages/run-store/src/redisSnapshotStore.test.ts
+++ b/internal-packages/run-store/src/redisSnapshotStore.test.ts
@@ -663,6 +663,31 @@ describe("getSince", () => {
     }
   );
 
+  redisTest(
+    "skips an entry whose body was evicted rather than throwing",
+    async ({ redisOptions }) => {
+      const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000 });
+      const raw = createRedisClient(redisOptions);
+      try {
+        await store.append({ entry: entry({ id: "s0" }), kind: "birth", isTerminal: false });
+        await store.append({ entry: entry({ id: "s1" }), kind: "transition", isTerminal: false });
+        await store.append({ entry: entry({ id: "s2" }), kind: "transition", isTerminal: false });
+
+        // The mirror of the case the append script documents: idx survives while the entry body in
+        // `e` is gone. The seq field is left in place so the id still resolves.
+        await raw.hdel("snap:{run_1}:e", "s1");
+
+        const r = await store.getSince("run_1", "s0");
+        expect(r.kind).toBe("hit");
+        if (r.kind !== "hit") throw new Error("unreachable");
+        expect(r.entries.map((e) => e.id)).toEqual(["s2"]);
+      } finally {
+        await raw.quit();
+        await store.quit();
+      }
+    }
+  );
+
   redisTest(
     "hits with zero entries when scoped to the since entry's own environment",
     async ({ redisOptions }) => {

From a5ad85a36bb18cec3f3a29af3ba162db439b081e Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Fri, 21 Aug 2026 14:10:22 +0100
Subject: [PATCH 11/49] fix(run-store): pair the head waitpoint order with the
 surviving head row

---
 .../run-store/src/redisSnapshotStore.test.ts  | 36 +++++++++++++++++++
 .../run-store/src/redisSnapshotStore.ts       | 14 +++++---
 2 files changed, 45 insertions(+), 5 deletions(-)

diff --git a/internal-packages/run-store/src/redisSnapshotStore.test.ts b/internal-packages/run-store/src/redisSnapshotStore.test.ts
index 00615482da7..72f67a463e3 100644
--- a/internal-packages/run-store/src/redisSnapshotStore.test.ts
+++ b/internal-packages/run-store/src/redisSnapshotStore.test.ts
@@ -688,6 +688,42 @@ describe("getSince", () => {
     }
   );
 
+  redisTest(
+    "does not donate the evicted head's waitpoints to the surviving head",
+    async ({ redisOptions }) => {
+      const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000 });
+      const raw = createRedisClient(redisOptions);
+      try {
+        await store.append({ entry: entry({ id: "s0" }), kind: "birth", isTerminal: false });
+        await store.append({
+          entry: entry({ id: "s1" }),
+          kind: "transition",
+          isTerminal: false,
+          cycle: { kind: "new", completedWaitpoints: [{ id: "w_old", index: 0 }] },
+        });
+        await store.append({
+          entry: entry({ id: "s2" }),
+          kind: "transition",
+          isTerminal: false,
+          cycle: { kind: "new", completedWaitpoints: [{ id: "w_new", index: 0 }] },
+        });
+
+        // s2 is the newest and its body is gone. s1 must come back with ITS OWN waitpoints,
+        // never s2's -- a dropped row must not donate its cycle data to the next one.
+        await raw.hdel("snap:{run_1}:e", "s2");
+
+        const r = await store.getSince("run_1", "s0");
+        expect(r.kind).toBe("hit");
+        if (r.kind !== "hit") throw new Error("unreachable");
+        expect(r.entries.map((e) => e.id)).toEqual(["s1"]);
+        expect(r.headWaitpointIds.order).toEqual(["w_old"]);
+      } finally {
+        await raw.quit();
+        await store.quit();
+      }
+    }
+  );
+
   redisTest(
     "hits with zero entries when scoped to the since entry's own environment",
     async ({ redisOptions }) => {
diff --git a/internal-packages/run-store/src/redisSnapshotStore.ts b/internal-packages/run-store/src/redisSnapshotStore.ts
index e76963ef379..b0190c3da30 100644
--- a/internal-packages/run-store/src/redisSnapshotStore.ts
+++ b/internal-packages/run-store/src/redisSnapshotStore.ts
@@ -542,21 +542,25 @@ export class RedisSnapshotStore {
         local ids = redis.call('ZREVRANGEBYSCORE', idxKey, '+inf', '(' .. score, 'LIMIT', 0, limit)
         if #ids == 0 then return { sinceRaw, '' } end
 
-        -- The head is the newest entry, and it is the ONLY one whose cycle key is read. That makes
-        -- head-only hydration structural: the tail's cycle keys are never touched.
-        local out = { sinceRaw, orderFor(redis.call('HGET', eKey, ids[1] .. '#c')) }
+        -- The head is the newest SURVIVING entry, and it is the only one whose cycle key is read.
+        -- Deriving the order after the loop keeps it paired with the row it is attached to: a row
+        -- dropped for a missing body must not donate its cycle data to the next one.
+        local out = { sinceRaw, '' }
+        local headId = nil
         for i = 1, #ids do
           local id = ids[i]
           local vals = redis.call('HMGET', eKey, id, id .. '#s', id .. '#c')
-          -- A nil body (e survived only partially, eg. idx outlived e) must drop the row, not emit
-          -- an unparseable '' that would throw out of JSON.parse in #decode.
           if vals[1] then
+            if not headId then headId = id end
             out[#out + 1] = id
             out[#out + 1] = vals[1]
             out[#out + 1] = vals[2] or ''
             out[#out + 1] = vals[3] or ''
           end
         end
+        if headId then
+          out[2] = orderFor(redis.call('HGET', eKey, headId .. '#c'))
+        end
         return out
       `,
     });

From fa944e6850c1308ffa3dcc63f38bc122b40e74b1 Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Fri, 21 Aug 2026 14:21:58 +0100
Subject: [PATCH 12/49] test(run-store): environment scoping on the snapshot
 store reads

---
 .../run-store/src/redisSnapshotStore.test.ts  | 26 +++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/internal-packages/run-store/src/redisSnapshotStore.test.ts b/internal-packages/run-store/src/redisSnapshotStore.test.ts
index 72f67a463e3..9065be64a9f 100644
--- a/internal-packages/run-store/src/redisSnapshotStore.test.ts
+++ b/internal-packages/run-store/src/redisSnapshotStore.test.ts
@@ -743,3 +743,29 @@ describe("getSince", () => {
     }
   );
 });
+
+describe("environment scoping", () => {
+  redisTest("getLatest and getById return null for a foreign env", async ({ redisOptions }) => {
+    const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000 });
+    try {
+      await store.append({ entry: entry({ id: "s1" }), kind: "birth", isTerminal: false });
+
+      expect(await store.getLatest("run_1", { environmentId: "env_1" })).not.toBeNull();
+      expect(await store.getLatest("run_1", { environmentId: "env_other" })).toBeNull();
+      expect(await store.getById("run_1", "s1", { environmentId: "env_1" })).not.toBeNull();
+      expect(await store.getById("run_1", "s1", { environmentId: "env_other" })).toBeNull();
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest("getLatest returns null for a run with no keys", async ({ redisOptions }) => {
+    const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000 });
+    try {
+      expect(await store.getLatest("run_absent")).toBeNull();
+      expect(await store.getById("run_absent", "nope")).toBeNull();
+    } finally {
+      await store.quit();
+    }
+  });
+});

From db08cf5f20433973040e797775187cd61e6a9479 Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Fri, 21 Aug 2026 14:22:37 +0100
Subject: [PATCH 13/49] test(run-store): the optional compare-and-set on the
 current snapshot pointer

---
 .../run-store/src/redisSnapshotStore.test.ts  | 84 +++++++++++++++++++
 1 file changed, 84 insertions(+)

diff --git a/internal-packages/run-store/src/redisSnapshotStore.test.ts b/internal-packages/run-store/src/redisSnapshotStore.test.ts
index 9065be64a9f..088fc5cb541 100644
--- a/internal-packages/run-store/src/redisSnapshotStore.test.ts
+++ b/internal-packages/run-store/src/redisSnapshotStore.test.ts
@@ -769,3 +769,87 @@ describe("environment scoping", () => {
     }
   });
 });
+
+describe("expectedCur compare-and-set", () => {
+  redisTest("absent by default: cur advances unconditionally", async ({ redisOptions }) => {
+    const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000 });
+    try {
+      await store.append({ entry: entry({ id: "s1" }), kind: "birth", isTerminal: false });
+      const r = await store.append({
+        entry: entry({ id: "s2", previousSnapshotId: "stale" }),
+        kind: "transition",
+        isTerminal: false,
+      });
+      expect(r).toMatchObject({ outcome: "written" });
+      expect((await store.getLatest("run_1"))?.id).toBe("s2");
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest("supplied and matching: the append proceeds", async ({ redisOptions }) => {
+    const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000 });
+    try {
+      await store.append({ entry: entry({ id: "s1" }), kind: "birth", isTerminal: false });
+      const r = await store.append({
+        entry: entry({ id: "s2" }),
+        kind: "transition",
+        isTerminal: false,
+        expectedCur: "s1",
+      });
+      expect(r).toMatchObject({ outcome: "written", seq: 2 });
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest("supplied and stale: writes NOTHING and reports the fork", async ({ redisOptions }) => {
+    const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000 });
+    try {
+      await store.append({ entry: entry({ id: "s1" }), kind: "birth", isTerminal: false });
+      await store.append({ entry: entry({ id: "s2" }), kind: "transition", isTerminal: false });
+
+      // A second concurrent transition that read cur = s1 before s2 landed.
+      const r = await store.append({
+        entry: entry({ id: "s3" }),
+        kind: "transition",
+        isTerminal: false,
+        expectedCur: "s1",
+      });
+      expect(r).toEqual({ outcome: "forked", actualCur: "s2" });
+
+      // Nothing was written: no entry, and the seq counter did not move.
+      expect(await store.getById("run_1", "s3")).toBeNull();
+      const next = await store.append({
+        entry: entry({ id: "s4" }),
+        kind: "transition",
+        isTerminal: false,
+      });
+      expect(next).toMatchObject({ seq: 3 });
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest(
+    "supplied as empty string: still enforces a check against an unset cur",
+    async ({ redisOptions }) => {
+      const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000 });
+      try {
+        // The birth sets cur to "s1", so a caller claiming cur is UNSET (expectedCur: "") must
+        // fork rather than have "" silently treated as "no compare-and-set requested".
+        await store.append({ entry: entry({ id: "s1" }), kind: "birth", isTerminal: false });
+        const r = await store.append({
+          entry: entry({ id: "s2" }),
+          kind: "transition",
+          isTerminal: false,
+          expectedCur: "",
+        });
+        expect(r).toEqual({ outcome: "forked", actualCur: "s1" });
+        expect(await store.getById("run_1", "s2")).toBeNull();
+      } finally {
+        await store.quit();
+      }
+    }
+  );
+});

From d87187db9569a052efed920b1c163fd7060bd278 Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Fri, 21 Aug 2026 14:23:54 +0100
Subject: [PATCH 14/49] test(run-store): hash-tag slot and keyPrefix guards for
 the Lua key derivation

---
 .../run-store/src/redisSnapshotStore.test.ts  | 85 +++++++++++++++++++
 1 file changed, 85 insertions(+)

diff --git a/internal-packages/run-store/src/redisSnapshotStore.test.ts b/internal-packages/run-store/src/redisSnapshotStore.test.ts
index 088fc5cb541..a2b608b09a1 100644
--- a/internal-packages/run-store/src/redisSnapshotStore.test.ts
+++ b/internal-packages/run-store/src/redisSnapshotStore.test.ts
@@ -853,3 +853,88 @@ describe("expectedCur compare-and-set", () => {
     }
   );
 });
+
+// CRC16/XMODEM over a key's hash tag, per Redis's cluster hashing rule. CLUSTER KEYSLOT is
+// unavailable on this standalone container ("cluster support disabled"), so the slot is computed
+// here instead. Verified against the `cluster-key-slot` package's output for our key shapes.
+function crc16(str: string): number {
+  let crc = 0;
+  for (let i = 0; i < str.length; i++) {
+    crc ^= str.charCodeAt(i) << 8;
+    for (let j = 0; j < 8; j++) {
+      crc = crc & 0x8000 ? (crc << 1) ^ 0x1021 : crc << 1;
+      crc &= 0xffff;
+    }
+  }
+  return crc;
+}
+
+function hashSlot(key: string): number {
+  const start = key.indexOf("{");
+  const end = start === -1 ? -1 : key.indexOf("}", start + 1);
+  const tag = start !== -1 && end !== -1 && end > start + 1 ? key.slice(start + 1, end) : key;
+  return crc16(tag) % 16384;
+}
+
+describe("hash tag and keyPrefix", () => {
+  it("every key for one run lands in one cluster slot", () => {
+    // Keys come from snapshotKeys() plus the wp:<n> suffix the Lua prelude derives the same way,
+    // with a keyPrefix prepended by hand as ioredis would. A dropped hash tag would split the slots.
+    const k = snapshotKeys("run_1");
+    const base = k.e.slice(0, -2);
+    const keys = [k.e, k.idx, k.cur, k.seq, `${base}:wp:1`, `${base}:wp:2`].map(
+      (key) => `engine:${key}`
+    );
+    const slots = new Set(keys.map(hashSlot));
+    expect(slots.size).toBe(1);
+  });
+
+  redisTest("the terminal append expires the PREFIXED cycle keys", async ({ redisOptions }) => {
+    // This is the guard for the trap: ioredis prefixes only the KEYS array, so a cycle key minted
+    // inside Lua would be UNPREFIXED while the client wrote a prefixed one. Deriving it from KEYS[1]
+    // inherits both the prefix and the hash tag. If someone later mints it in Lua, this fails.
+    const prefixed = { ...redisOptions, keyPrefix: "engine:" };
+    const store = new RedisSnapshotStore({ redisOptions: prefixed, completedTtlMs: 60_000 });
+    const raw = createRedisClient(redisOptions);
+    try {
+      await store.append({
+        entry: entry({ id: "s1" }),
+        kind: "birth",
+        isTerminal: false,
+        cycle: { kind: "new", completedWaitpoints: [{ id: "w_a", index: 0 }] },
+      });
+      expect(await raw.exists("engine:snap:{run_1}:wp:1")).toBe(1);
+      expect(await raw.exists("snap:{run_1}:wp:1")).toBe(0);
+
+      await store.append({
+        entry: entry({ id: "s2", executionStatus: "FINISHED" }),
+        kind: "transition",
+        isTerminal: true,
+      });
+      const ttl = await raw.pttl("engine:snap:{run_1}:wp:1");
+      expect(ttl).toBeGreaterThan(0);
+    } finally {
+      raw.disconnect();
+      await store.quit();
+    }
+  });
+
+  redisTest("reads work through a keyPrefix", async ({ redisOptions }) => {
+    const store = new RedisSnapshotStore({
+      redisOptions: { ...redisOptions, keyPrefix: "engine:" },
+      completedTtlMs: 60_000,
+    });
+    try {
+      await store.append({
+        entry: entry({ id: "s1" }),
+        kind: "birth",
+        isTerminal: false,
+        cycle: { kind: "new", completedWaitpoints: [{ id: "w_a", index: 0 }] },
+      });
+      expect((await store.getLatest("run_1"))?.id).toBe("s1");
+      expect((await store.getSnapshotWaitpointIds("run_1", "s1")).order).toEqual(["w_a"]);
+    } finally {
+      await store.quit();
+    }
+  });
+});

From 39cd9ad68ac8663c94489dfdd8b56fcdcf518098 Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Fri, 21 Aug 2026 14:24:31 +0100
Subject: [PATCH 15/49] test(run-store): snapshot store size metrics and
 high-water logging

---
 .../run-store/src/redisSnapshotStore.test.ts  | 50 +++++++++++++++++++
 1 file changed, 50 insertions(+)

diff --git a/internal-packages/run-store/src/redisSnapshotStore.test.ts b/internal-packages/run-store/src/redisSnapshotStore.test.ts
index a2b608b09a1..477ad48d803 100644
--- a/internal-packages/run-store/src/redisSnapshotStore.test.ts
+++ b/internal-packages/run-store/src/redisSnapshotStore.test.ts
@@ -938,3 +938,53 @@ describe("hash tag and keyPrefix", () => {
     }
   });
 });
+
+describe("observability", () => {
+  redisTest("records sizes and outcomes without ever rejecting", async ({ redisOptions }) => {
+    const calls: string[] = [];
+    const metrics = {
+      recordAppend: (o: string, t: string) => calls.push(`append:${o}:${t}`),
+      recordEntryBytes: (b: number) => calls.push(`entryBytes:${b > 0}`),
+      recordCycleKeyBytes: (b: number) => calls.push(`cycleBytes:${b > 0}`),
+      recordCycleCount: (c: number) => calls.push(`cycleCount:${c}`),
+      recordSkippedNoKeyspace: () => calls.push("skipped"),
+      recordCycleMismatch: () => calls.push("mismatch"),
+      recordLatency: (op: string) => calls.push(`latency:${op}`),
+    };
+    const store = new RedisSnapshotStore({
+      redisOptions,
+      completedTtlMs: 60_000,
+      metrics,
+      highWater: { entryBytes: 1 },
+    });
+    try {
+      // A huge inline value is observed, never rejected or truncated: Postgres had no cap either.
+      const big = "x".repeat(20_000);
+      const r = await store.append({
+        entry: entry({ id: "s1", description: big }),
+        kind: "birth",
+        isTerminal: false,
+        cycle: { kind: "new", completedWaitpoints: [{ id: "w_a", index: 0 }] },
+      });
+      expect(r).toMatchObject({ outcome: "written" });
+      expect((await store.getById("run_1", "s1"))?.entry.description).toBe(big);
+
+      await store.append({
+        entry: entry({ id: "s2", runId: "run_absent" }),
+        kind: "transition",
+        isTerminal: false,
+      });
+
+      expect(calls).toContain("append:written:none");
+      expect(calls).toContain("entryBytes:true");
+      expect(calls).toContain("cycleBytes:true");
+      expect(calls).toContain("cycleCount:1");
+      expect(calls).toContain("skipped");
+      // recordLatency is wired through #timed for every public method, not just a no-op stub.
+      expect(calls).toContain("latency:append");
+      expect(calls).toContain("latency:getById");
+    } finally {
+      await store.quit();
+    }
+  });
+});

From b61cb4d64fbe95e2c29d4f5fa529c91cd171ffb8 Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Fri, 21 Aug 2026 14:52:42 +0100
Subject: [PATCH 16/49] fix(run-store): name the run in high-water warnings

Thread runId into #observeSizes so all three high-water logger.warn
payloads name the run, per spec. Adds a capturing-logger test proving
the warning fires with the run id above the mark, and stays silent
under a high threshold.
---
 .../run-store/src/redisSnapshotStore.test.ts  | 50 ++++++++++++++++++-
 .../run-store/src/redisSnapshotStore.ts       | 14 +++---
 2 files changed, 56 insertions(+), 8 deletions(-)

diff --git a/internal-packages/run-store/src/redisSnapshotStore.test.ts b/internal-packages/run-store/src/redisSnapshotStore.test.ts
index 477ad48d803..b0b9f2c7dbc 100644
--- a/internal-packages/run-store/src/redisSnapshotStore.test.ts
+++ b/internal-packages/run-store/src/redisSnapshotStore.test.ts
@@ -1,8 +1,9 @@
 // Unit suite for the raw Redis execution-snapshot store. Redis-only: the store holds no Prisma
 // reference, so no Postgres container is needed.
-import { expect, describe } from "vitest";
+import { expect, describe, vi } from "vitest";
 import { redisTest } from "@internal/testcontainers";
 import { createRedisClient } from "@internal/redis";
+import { Logger } from "@trigger.dev/core/logger";
 import {
   snapshotKeys,
   deriveOrder,
@@ -987,4 +988,51 @@ describe("observability", () => {
       await store.quit();
     }
   });
+  redisTest(
+    "names the run in a high-water warning, and stays silent under a high threshold",
+    async ({ redisOptions }) => {
+      const loudLogger = new Logger("test", "debug");
+      const loudWarn = vi.spyOn(loudLogger, "warn");
+      const loud = new RedisSnapshotStore({
+        redisOptions,
+        completedTtlMs: 1000,
+        logger: loudLogger,
+        highWater: { entryBytes: 1, cycleKeyBytes: 1, cycleCount: 0 },
+      });
+
+      const quietLogger = new Logger("test", "debug");
+      const quietWarn = vi.spyOn(quietLogger, "warn");
+      const quiet = new RedisSnapshotStore({
+        redisOptions,
+        completedTtlMs: 1000,
+        logger: quietLogger,
+        highWater: { entryBytes: 1_000_000, cycleKeyBytes: 1_000_000, cycleCount: 1_000_000 },
+      });
+
+      try {
+        await loud.append({
+          entry: entry({ id: "s1", runId: "run_loud" }),
+          kind: "birth",
+          isTerminal: false,
+          cycle: { kind: "new", completedWaitpoints: [{ id: "w_a", index: 0 }] },
+        });
+        expect(loudWarn).toHaveBeenCalledTimes(3);
+        for (const [, payload] of loudWarn.mock.calls) {
+          expect(payload).toMatchObject({ runId: "run_loud" });
+        }
+
+        // Same shape of append, high thresholds: proves the mark is respected, not just logged.
+        await quiet.append({
+          entry: entry({ id: "s1", runId: "run_quiet" }),
+          kind: "birth",
+          isTerminal: false,
+          cycle: { kind: "new", completedWaitpoints: [{ id: "w_a", index: 0 }] },
+        });
+        expect(quietWarn).not.toHaveBeenCalled();
+      } finally {
+        await loud.quit();
+        await quiet.quit();
+      }
+    }
+  );
 });
diff --git a/internal-packages/run-store/src/redisSnapshotStore.ts b/internal-packages/run-store/src/redisSnapshotStore.ts
index b0190c3da30..a6bd3e328a3 100644
--- a/internal-packages/run-store/src/redisSnapshotStore.ts
+++ b/internal-packages/run-store/src/redisSnapshotStore.ts
@@ -199,11 +199,11 @@ export class RedisSnapshotStore {
         args.expectedCur !== undefined ? "1" : "0"
       )) as string[];
 
-      return this.#interpretAppend(reply, raw, orderJson);
+      return this.#interpretAppend(reply, raw, orderJson, args.entry.runId);
     });
   }
 
-  #interpretAppend(reply: string[], raw: string, orderJson: string): AppendResult {
+  #interpretAppend(reply: string[], raw: string, orderJson: string, runId: string): AppendResult {
     if (reply[0] === SKIPPED) {
       this.metrics?.recordSkippedNoKeyspace();
       this.metrics?.recordAppend("skippedNoKeyspace", "none");
@@ -224,7 +224,7 @@ export class RedisSnapshotStore {
     if (cycleMismatch) {
       this.metrics?.recordCycleMismatch();
     }
-    this.#observeSizes(raw, orderJson, cycleSeq);
+    this.#observeSizes(raw, orderJson, cycleSeq, runId);
     this.metrics?.recordAppend("written", ttl);
     return {
       outcome: "written",
@@ -235,23 +235,23 @@ export class RedisSnapshotStore {
     };
   }
 
-  #observeSizes(raw: string, orderJson: string, cycleSeq: number): void {
+  #observeSizes(raw: string, orderJson: string, cycleSeq: number, runId: string): void {
     const entryBytes = Buffer.byteLength(raw, "utf8");
     this.metrics?.recordEntryBytes(entryBytes);
     if (this.highWater.entryBytes !== undefined && entryBytes > this.highWater.entryBytes) {
-      this.logger.warn("RedisSnapshotStore entry above high-water mark", { entryBytes });
+      this.logger.warn("RedisSnapshotStore entry above high-water mark", { runId, entryBytes });
     }
     if (orderJson !== "") {
       const cycleBytes = Buffer.byteLength(orderJson, "utf8");
       this.metrics?.recordCycleKeyBytes(cycleBytes);
       if (this.highWater.cycleKeyBytes !== undefined && cycleBytes > this.highWater.cycleKeyBytes) {
-        this.logger.warn("RedisSnapshotStore cycle key above high-water mark", { cycleBytes });
+        this.logger.warn("RedisSnapshotStore cycle key above high-water mark", { runId, cycleBytes });
       }
     }
     if (cycleSeq > 0) {
       this.metrics?.recordCycleCount(cycleSeq);
       if (this.highWater.cycleCount !== undefined && cycleSeq > this.highWater.cycleCount) {
-        this.logger.warn("RedisSnapshotStore cycle count above high-water mark", { cycleSeq });
+        this.logger.warn("RedisSnapshotStore cycle count above high-water mark", { runId, cycleSeq });
       }
     }
   }

From 1bf9dbaedfe65b7a9233fb11723ccd7aee9057ec Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Fri, 21 Aug 2026 14:53:00 +0100
Subject: [PATCH 17/49] test(run-store): pin the metric, CAS and slot
 assertions to their values

Record actual byte values instead of booleans and partition per append
so a mis-wired metric can't hide behind a flat toContain. Cover the
succeeding direction of expectedCur: "" against a genuinely unset cur,
assert recordCycleMismatch fires, pin the CRC16 helper against a known
vector plus a negative control, bound the prefixed cycle-key TTL, prove
cur is untouched by a stale CAS, and add a matching-environment getSince
with a non-empty window.
---
 .../run-store/src/redisSnapshotStore.test.ts  | 117 +++++++++++++++---
 1 file changed, 97 insertions(+), 20 deletions(-)

diff --git a/internal-packages/run-store/src/redisSnapshotStore.test.ts b/internal-packages/run-store/src/redisSnapshotStore.test.ts
index b0b9f2c7dbc..51dcf6b6b00 100644
--- a/internal-packages/run-store/src/redisSnapshotStore.test.ts
+++ b/internal-packages/run-store/src/redisSnapshotStore.test.ts
@@ -337,7 +337,17 @@ describe("cycle keys", () => {
   );
 
   redisTest("a carry-forward naming a missing cycle still appends", async ({ redisOptions }) => {
-    const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000 });
+    const calls: string[] = [];
+    const metrics = {
+      recordAppend: () => {},
+      recordEntryBytes: () => {},
+      recordCycleKeyBytes: () => {},
+      recordCycleCount: () => {},
+      recordSkippedNoKeyspace: () => {},
+      recordCycleMismatch: () => calls.push("mismatch"),
+      recordLatency: () => {},
+    };
+    const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000, metrics });
     try {
       await store.append({ entry: entry({ id: "snap_1" }), kind: "birth", isTerminal: false });
       const r = await store.append({
@@ -347,6 +357,8 @@ describe("cycle keys", () => {
         cycle: { kind: "carryForward", cycleSeq: 99 },
       });
       expect(r).toMatchObject({ outcome: "written", cycleMismatch: true });
+      // recordCycleMismatch is required by the spec and was previously stubbed but never checked.
+      expect(calls).toEqual(["mismatch"]);
     } finally {
       await store.quit();
     }
@@ -769,6 +781,25 @@ describe("environment scoping", () => {
       await store.quit();
     }
   });
+
+  redisTest(
+    "getSince returns entries when scoped to a matching, non-empty environment",
+    async ({ redisOptions }) => {
+      const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000 });
+      try {
+        await store.append({ entry: entry({ id: "s0" }), kind: "birth", isTerminal: false });
+        await store.append({ entry: entry({ id: "s1" }), kind: "transition", isTerminal: false });
+        await store.append({ entry: entry({ id: "s2" }), kind: "transition", isTerminal: false });
+        // Every existing matching-env getSince test used an EMPTY window, so the per-row compare
+        // in #decode never ran in the passing direction. This is the first to exercise it with rows.
+        const r = await store.getSince("run_1", "s0", { environmentId: "env_1" });
+        if (r.kind !== "hit") throw new Error("expected a hit");
+        expect(r.entries.map((e) => e.id)).toEqual(["s1", "s2"]);
+      } finally {
+        await store.quit();
+      }
+    }
+  );
 });
 
 describe("expectedCur compare-and-set", () => {
@@ -819,8 +850,10 @@ describe("expectedCur compare-and-set", () => {
       });
       expect(r).toEqual({ outcome: "forked", actualCur: "s2" });
 
-      // Nothing was written: no entry, and the seq counter did not move.
+      // Nothing was written: no entry, cur is still s2 (not overwritten by s3, and not cleared),
+      // and the seq counter did not move.
       expect(await store.getById("run_1", "s3")).toBeNull();
+      expect((await store.getLatest("run_1"))?.id).toBe("s2");
       const next = await store.append({
         entry: entry({ id: "s4" }),
         kind: "transition",
@@ -853,6 +886,26 @@ describe("expectedCur compare-and-set", () => {
       }
     }
   );
+
+  redisTest(
+    "supplied as empty string against a genuinely unset cur: the append proceeds",
+    async ({ redisOptions }) => {
+      const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000 });
+      try {
+        // The load-bearing succeeding direction: expectedCur: "" asserts "cur is unset", and on a
+        // fresh keyspace that assertion is TRUE, so the append must proceed, not fork.
+        const r = await store.append({
+          entry: entry({ id: "s1" }),
+          kind: "birth",
+          isTerminal: false,
+          expectedCur: "",
+        });
+        expect(r).toMatchObject({ outcome: "written", seq: 1 });
+      } finally {
+        await store.quit();
+      }
+    }
+  );
 });
 
 // CRC16/XMODEM over a key's hash tag, per Redis's cluster hashing rule. CLUSTER KEYSLOT is
@@ -881,6 +934,13 @@ describe("hash tag and keyPrefix", () => {
   it("every key for one run lands in one cluster slot", () => {
     // Keys come from snapshotKeys() plus the wp:<n> suffix the Lua prelude derives the same way,
     // with a keyPrefix prepended by hand as ioredis would. A dropped hash tag would split the slots.
+    // Pin the helper itself before trusting it: the published XMODEM check value, and two known
+    // slots (one matching cluster-key-slot, one a different run's tag as a negative control --
+    // otherwise a constant-valued crc16 would satisfy slots.size === 1 for the wrong reason).
+    expect(crc16("123456789")).toBe(0x31c3);
+    expect(hashSlot("engine:snap:{run_1}:e")).toBe(8108);
+    expect(hashSlot("engine:snap:{run_2}:e")).toBe(12239);
+
     const k = snapshotKeys("run_1");
     const base = k.e.slice(0, -2);
     const keys = [k.e, k.idx, k.cur, k.seq, `${base}:wp:1`, `${base}:wp:2`].map(
@@ -913,7 +973,8 @@ describe("hash tag and keyPrefix", () => {
         isTerminal: true,
       });
       const ttl = await raw.pttl("engine:snap:{run_1}:wp:1");
-      expect(ttl).toBeGreaterThan(0);
+      expect(ttl).toBeGreaterThan(50_000);
+      expect(ttl).toBeLessThanOrEqual(60_000);
     } finally {
       raw.disconnect();
       await store.quit();
@@ -942,15 +1003,15 @@ describe("hash tag and keyPrefix", () => {
 
 describe("observability", () => {
   redisTest("records sizes and outcomes without ever rejecting", async ({ redisOptions }) => {
-    const calls: string[] = [];
+    const calls: unknown[][] = [];
     const metrics = {
-      recordAppend: (o: string, t: string) => calls.push(`append:${o}:${t}`),
-      recordEntryBytes: (b: number) => calls.push(`entryBytes:${b > 0}`),
-      recordCycleKeyBytes: (b: number) => calls.push(`cycleBytes:${b > 0}`),
-      recordCycleCount: (c: number) => calls.push(`cycleCount:${c}`),
-      recordSkippedNoKeyspace: () => calls.push("skipped"),
-      recordCycleMismatch: () => calls.push("mismatch"),
-      recordLatency: (op: string) => calls.push(`latency:${op}`),
+      recordAppend: (o: string, t: string) => calls.push(["append", o, t]),
+      recordEntryBytes: (b: number) => calls.push(["entryBytes", b]),
+      recordCycleKeyBytes: (b: number) => calls.push(["cycleBytes", b]),
+      recordCycleCount: (c: number) => calls.push(["cycleCount", c]),
+      recordSkippedNoKeyspace: () => calls.push(["skipped"]),
+      recordCycleMismatch: () => calls.push(["mismatch"]),
+      recordLatency: (op: string) => calls.push(["latency", op]),
     };
     const store = new RedisSnapshotStore({
       redisOptions,
@@ -961,8 +1022,12 @@ describe("observability", () => {
     try {
       // A huge inline value is observed, never rejected or truncated: Postgres had no cap either.
       const big = "x".repeat(20_000);
+      const bigEntry = entry({ id: "s1", description: big });
+      const rawBytes = Buffer.byteLength(JSON.stringify(bigEntry), "utf8");
+      const orderBytes = Buffer.byteLength(JSON.stringify(["w_a"]), "utf8");
+
       const r = await store.append({
-        entry: entry({ id: "s1", description: big }),
+        entry: bigEntry,
         kind: "birth",
         isTerminal: false,
         cycle: { kind: "new", completedWaitpoints: [{ id: "w_a", index: 0 }] },
@@ -970,24 +1035,36 @@ describe("observability", () => {
       expect(r).toMatchObject({ outcome: "written" });
       expect((await store.getById("run_1", "s1"))?.entry.description).toBe(big);
 
+      // Exact values, not just `b > 0`: a swapped recordEntryBytes/recordCycleKeyBytes wiring
+      // would still pass a `b > 0` check but fails this, since the two sizes are wildly different.
+      expect(calls).toEqual([
+        ["entryBytes", rawBytes],
+        ["cycleBytes", orderBytes],
+        ["cycleCount", 1],
+        ["append", "written", "none"],
+        ["latency", "append"],
+        ["latency", "getById"],
+      ]);
+      calls.length = 0;
+
       await store.append({
         entry: entry({ id: "s2", runId: "run_absent" }),
         kind: "transition",
         isTerminal: false,
       });
 
-      expect(calls).toContain("append:written:none");
-      expect(calls).toContain("entryBytes:true");
-      expect(calls).toContain("cycleBytes:true");
-      expect(calls).toContain("cycleCount:1");
-      expect(calls).toContain("skipped");
-      // recordLatency is wired through #timed for every public method, not just a no-op stub.
-      expect(calls).toContain("latency:append");
-      expect(calls).toContain("latency:getById");
+      // Partitioned from the first append's calls: proves recordSkippedNoKeyspace fires ONLY on
+      // this skip, not (also, harmlessly) on the earlier successful append.
+      expect(calls).toEqual([
+        ["skipped"],
+        ["append", "skippedNoKeyspace", "none"],
+        ["latency", "append"],
+      ]);
     } finally {
       await store.quit();
     }
   });
+
   redisTest(
     "names the run in a high-water warning, and stays silent under a high threshold",
     async ({ redisOptions }) => {

From aa16e6e43a0dc06afd7864172f1e9a8307b5f3a0 Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Fri, 21 Aug 2026 15:13:14 +0100
Subject: [PATCH 18/49] style(run-store): wrap the high-water warnings after
 oxfmt

---
 internal-packages/run-store/src/redisSnapshotStore.ts | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/internal-packages/run-store/src/redisSnapshotStore.ts b/internal-packages/run-store/src/redisSnapshotStore.ts
index a6bd3e328a3..e25aec2d432 100644
--- a/internal-packages/run-store/src/redisSnapshotStore.ts
+++ b/internal-packages/run-store/src/redisSnapshotStore.ts
@@ -245,13 +245,19 @@ export class RedisSnapshotStore {
       const cycleBytes = Buffer.byteLength(orderJson, "utf8");
       this.metrics?.recordCycleKeyBytes(cycleBytes);
       if (this.highWater.cycleKeyBytes !== undefined && cycleBytes > this.highWater.cycleKeyBytes) {
-        this.logger.warn("RedisSnapshotStore cycle key above high-water mark", { runId, cycleBytes });
+        this.logger.warn("RedisSnapshotStore cycle key above high-water mark", {
+          runId,
+          cycleBytes,
+        });
       }
     }
     if (cycleSeq > 0) {
       this.metrics?.recordCycleCount(cycleSeq);
       if (this.highWater.cycleCount !== undefined && cycleSeq > this.highWater.cycleCount) {
-        this.logger.warn("RedisSnapshotStore cycle count above high-water mark", { runId, cycleSeq });
+        this.logger.warn("RedisSnapshotStore cycle count above high-water mark", {
+          runId,
+          cycleSeq,
+        });
       }
     }
   }

From b4c411ad84e73a6d56735ea7c04ea3d70e99f4e6 Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Fri, 21 Aug 2026 15:30:35 +0100
Subject: [PATCH 19/49] fix(run-store): check for a duplicate id before the
 compare-and-set

A retried append whose write already succeeded advanced cur to its own
id, so the CAS above the duplicate guard saw its own id as a stale
expectedCur and reported forked instead of duplicate. Snapshot ids are
unique per append, so checking duplicate first is always correct.
---
 .../run-store/src/redisSnapshotStore.test.ts  | 28 +++++++++++++++++++
 .../run-store/src/redisSnapshotStore.ts       | 14 +++++-----
 2 files changed, 35 insertions(+), 7 deletions(-)

diff --git a/internal-packages/run-store/src/redisSnapshotStore.test.ts b/internal-packages/run-store/src/redisSnapshotStore.test.ts
index 51dcf6b6b00..c827c859aaa 100644
--- a/internal-packages/run-store/src/redisSnapshotStore.test.ts
+++ b/internal-packages/run-store/src/redisSnapshotStore.test.ts
@@ -887,6 +887,34 @@ describe("expectedCur compare-and-set", () => {
     }
   );
 
+  redisTest(
+    "a duplicate id wins over a stale CAS: retrying your own successful write is not a fork",
+    async ({ redisOptions }) => {
+      const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000 });
+      try {
+        await store.append({ entry: entry({ id: "s1" }), kind: "birth", isTerminal: false });
+        await store.append({
+          entry: entry({ id: "s2" }),
+          kind: "transition",
+          isTerminal: false,
+          expectedCur: "s1",
+        });
+
+        // Retry of the same append: cur has since moved to s2, so a naive CAS-first check would
+        // see actual=s2 != expected=s1 and report a fork -- but s2 is THIS write, not a rival's.
+        const retry = await store.append({
+          entry: entry({ id: "s2" }),
+          kind: "transition",
+          isTerminal: false,
+          expectedCur: "s1",
+        });
+        expect(retry).toEqual({ outcome: "duplicate", seq: 2 });
+      } finally {
+        await store.quit();
+      }
+    }
+  );
+
   redisTest(
     "supplied as empty string against a genuinely unset cur: the append proceeds",
     async ({ redisOptions }) => {
diff --git a/internal-packages/run-store/src/redisSnapshotStore.ts b/internal-packages/run-store/src/redisSnapshotStore.ts
index e25aec2d432..dad60be592f 100644
--- a/internal-packages/run-store/src/redisSnapshotStore.ts
+++ b/internal-packages/run-store/src/redisSnapshotStore.ts
@@ -410,6 +410,13 @@ export class RedisSnapshotStore {
           return { '${SKIPPED}' }
         end
 
+        -- Append-only: a retried append must not overwrite an existing entry. Checked BEFORE the
+        -- CAS below -- a present id can only be this same retry, never a competitor's write.
+        local prior = redis.call('HGET', eKey, id .. '#s')
+        if prior then
+          return { '${DUPLICATE}', prior }
+        end
+
         -- Optional compare-and-set on cur, checked BEFORE any mutation. Gated on an explicit flag
         -- (not on expectedCur ~= ''), so a caller asserting cur is unset (expectedCur = '') still
         -- gets a real check instead of silently skipping it.
@@ -420,13 +427,6 @@ export class RedisSnapshotStore {
           end
         end
 
-        -- Append-only: a retried append (eg. ioredis reconnect-and-retry on a READONLY/UNBLOCKED
-        -- reply error) must not overwrite an existing entry's bytes or rescore it in idx.
-        local prior = redis.call('HGET', eKey, id .. '#s')
-        if prior then
-          return { '${DUPLICATE}', prior }
-        end
-
         local seq = redis.call('HINCRBY', seqKey, 'e', 1)
 
         local cycleSeq = 0

From 9bec7376143bd51bd37092e78b23f397529a0423 Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Fri, 21 Aug 2026 15:31:44 +0100
Subject: [PATCH 20/49] test(run-store): prove getSince drops a
 foreign-environment row

Adds the reachable-in-tests, unreachable-in-prod case where the Lua-
chosen head is dropped by the TS env filter. It surfaced a real bug:
headOrder stayed attached to whatever row ended up last after
filtering, donating the dropped head's waitpoints to it. Track whether
the actual head row survives and only then attach its order.
---
 .../run-store/src/redisSnapshotStore.test.ts  | 31 +++++++++++++++++++
 .../run-store/src/redisSnapshotStore.ts       | 12 +++++--
 2 files changed, 40 insertions(+), 3 deletions(-)

diff --git a/internal-packages/run-store/src/redisSnapshotStore.test.ts b/internal-packages/run-store/src/redisSnapshotStore.test.ts
index c827c859aaa..55d06fb46c0 100644
--- a/internal-packages/run-store/src/redisSnapshotStore.test.ts
+++ b/internal-packages/run-store/src/redisSnapshotStore.test.ts
@@ -737,6 +737,37 @@ describe("getSince", () => {
     }
   );
 
+  redisTest(
+    "does not donate a foreign-environment head's waitpoints to the query's window",
+    async ({ redisOptions }) => {
+      const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000 });
+      try {
+        await store.append({
+          entry: entry({ id: "s1", environmentId: "env_a" }),
+          kind: "birth",
+          isTerminal: false,
+          cycle: { kind: "new", completedWaitpoints: [{ id: "w_a", index: 0 }] },
+        });
+        // Same run, a different environment -- unreachable in production, but exercises the branch
+        // where the Lua-chosen head is dropped by the TS-side environment filter.
+        await store.append({
+          entry: entry({ id: "s2", environmentId: "env_b" }),
+          kind: "transition",
+          isTerminal: false,
+          cycle: { kind: "new", completedWaitpoints: [{ id: "w_b", index: 0 }] },
+        });
+
+        const r = await store.getSince("run_1", "s1", { environmentId: "env_a" });
+        expect(r.kind).toBe("hit");
+        if (r.kind !== "hit") throw new Error("unreachable");
+        expect(r.entries).toEqual([]);
+        expect(r.headWaitpointIds.order).toEqual([]);
+      } finally {
+        await store.quit();
+      }
+    }
+  );
+
   redisTest(
     "hits with zero entries when scoped to the since entry's own environment",
     async ({ redisOptions }) => {
diff --git a/internal-packages/run-store/src/redisSnapshotStore.ts b/internal-packages/run-store/src/redisSnapshotStore.ts
index dad60be592f..7659a1cb002 100644
--- a/internal-packages/run-store/src/redisSnapshotStore.ts
+++ b/internal-packages/run-store/src/redisSnapshotStore.ts
@@ -325,17 +325,23 @@ export class RedisSnapshotStore {
 
       const headOrder = reply[1] ?? "";
       const rows: SnapshotRead[] = [];
+      // Tracks whether the Lua-chosen head row (always the first, i === 2) itself survives the
+      // env filter below -- headOrder must never be attributed to a different, surviving row.
+      let headSurvived = false;
       for (let i = 2; i + 3 < reply.length; i += 4) {
         const decoded = this.#decode(
           [reply[i], reply[i + 1], reply[i + 2], reply[i + 3], ""],
           opts?.environmentId
         );
-        if (decoded) rows.push(decoded);
+        if (decoded) {
+          rows.push(decoded);
+          if (i === 2) headSurvived = true;
+        }
       }
 
       rows.reverse();
-      const head = rows[rows.length - 1];
-      const headWaitpointIds = decodeWaitpointIds(head !== undefined, headOrder);
+      const head = headSurvived ? rows[rows.length - 1] : undefined;
+      const headWaitpointIds = decodeWaitpointIds(head !== undefined, head ? headOrder : "");
       if (head) {
         head.completedWaitpointIds = headWaitpointIds;
       }

From 8b62d086b299e9ae1830ab71795b1dba26a5764e Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Fri, 21 Aug 2026 15:33:53 +0100
Subject: [PATCH 21/49] feat(run-store): warn when a cycle pointer's count
 disagrees with its order

Implements the spec's read-side check that was previously unwritten: a
sentinel problem (an empty order string meant both "read as empty" and
"not read for this row" in getSince's tail rows) blocked it. #decode
now takes an explicit orderKnown flag, runs the count-vs-length check
only when the order was actually read, and never sets
completedWaitpointIds on a row whose order wasn't read -- which also
removes the need to delete it again afterward.
---
 .../run-store/src/redisSnapshotStore.test.ts  | 52 +++++++++++++++++++
 .../run-store/src/redisSnapshotStore.ts       | 41 ++++++++++++---
 2 files changed, 85 insertions(+), 8 deletions(-)

diff --git a/internal-packages/run-store/src/redisSnapshotStore.test.ts b/internal-packages/run-store/src/redisSnapshotStore.test.ts
index 55d06fb46c0..369d79e5338 100644
--- a/internal-packages/run-store/src/redisSnapshotStore.test.ts
+++ b/internal-packages/run-store/src/redisSnapshotStore.test.ts
@@ -384,6 +384,58 @@ describe("cycle keys", () => {
   });
 });
 
+describe("read-side cycle mismatch", () => {
+  redisTest(
+    "warns and records a metric when a cycle's count disagrees with its order",
+    async ({ redisOptions }) => {
+      const calls: string[] = [];
+      const metrics = {
+        recordAppend: () => {},
+        recordEntryBytes: () => {},
+        recordCycleKeyBytes: () => {},
+        recordCycleCount: () => {},
+        recordSkippedNoKeyspace: () => {},
+        recordCycleMismatch: () => calls.push("mismatch"),
+        recordLatency: () => {},
+      };
+      const logger = new Logger("test", "debug");
+      const warnSpy = vi.spyOn(logger, "warn");
+      const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 1000, metrics, logger });
+      const raw = createRedisClient(redisOptions);
+      try {
+        await store.append({
+          entry: entry({ id: "s1" }),
+          kind: "birth",
+          isTerminal: false,
+          cycle: { kind: "new", completedWaitpoints: [{ id: "w_a", index: 0 }] },
+        });
+        await store.append({
+          entry: entry({ id: "s2" }),
+          kind: "transition",
+          isTerminal: false,
+          cycle: { kind: "carryForward", cycleSeq: 1 },
+        });
+
+        // The pointer's count field (written at append time) survives; only the cycle key's order
+        // field is wiped, so a read must catch the disagreement instead of reporting count 1.
+        await raw.hdel("snap:{run_1}:wp:1", "order");
+
+        const read = await store.getById("run_1", "s2");
+        expect(read?.cycle).toEqual({ cycleSeq: 1, count: 1 });
+        expect(read?.completedWaitpointIds?.order).toEqual([]);
+        expect(calls).toEqual(["mismatch"]);
+        expect(warnSpy).toHaveBeenCalledWith(
+          expect.stringContaining("cycle"),
+          expect.objectContaining({ runId: "run_1" })
+        );
+      } finally {
+        await raw.quit();
+        await store.quit();
+      }
+    }
+  );
+});
+
 describe("TTL rule", () => {
   redisTest("a non-terminal append leaves every key unexpiring", async ({ redisOptions }) => {
     const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 60_000 });
diff --git a/internal-packages/run-store/src/redisSnapshotStore.ts b/internal-packages/run-store/src/redisSnapshotStore.ts
index 7659a1cb002..7b60843e2b4 100644
--- a/internal-packages/run-store/src/redisSnapshotStore.ts
+++ b/internal-packages/run-store/src/redisSnapshotStore.ts
@@ -270,7 +270,7 @@ export class RedisSnapshotStore {
     return this.#timed("getById", async () => {
       const k = snapshotKeys(runId);
       const reply = await this.redis.readSnapshotById(k.e, k.idx, k.cur, k.seq, snapshotId);
-      return this.#decode(reply, opts?.environmentId);
+      return this.#decode(reply, opts?.environmentId, runId, true);
     });
   }
 
@@ -278,7 +278,7 @@ export class RedisSnapshotStore {
     return this.#timed("getLatest", async () => {
       const k = snapshotKeys(runId);
       const reply = await this.redis.readLatestSnapshot(k.e, k.idx, k.cur, k.seq);
-      return this.#decode(reply, opts?.environmentId);
+      return this.#decode(reply, opts?.environmentId, runId, true);
     });
   }
 
@@ -329,9 +329,12 @@ export class RedisSnapshotStore {
       // env filter below -- headOrder must never be attributed to a different, surviving row.
       let headSurvived = false;
       for (let i = 2; i + 3 < reply.length; i += 4) {
+        // orderKnown is false here: headOrder covers only the head row, resolved separately below.
         const decoded = this.#decode(
           [reply[i], reply[i + 1], reply[i + 2], reply[i + 3], ""],
-          opts?.environmentId
+          opts?.environmentId,
+          runId,
+          false
         );
         if (decoded) {
           rows.push(decoded);
@@ -344,17 +347,35 @@ export class RedisSnapshotStore {
       const headWaitpointIds = decodeWaitpointIds(head !== undefined, head ? headOrder : "");
       if (head) {
         head.completedWaitpointIds = headWaitpointIds;
-      }
-      for (const row of rows.slice(0, -1)) {
-        delete row.completedWaitpointIds;
+        if (head.cycle) {
+          this.#checkCycleMismatch(runId, head.cycle.count, headWaitpointIds.order.length);
+        }
       }
       return { kind: "hit", entries: rows, headWaitpointIds };
     });
   }
 
+  #checkCycleMismatch(runId: string, count: number, orderLength: number): void {
+    if (orderLength === count) return;
+    this.metrics?.recordCycleMismatch();
+    this.logger.warn("RedisSnapshotStore cycle count disagrees with its order", {
+      runId,
+      count,
+      orderLength,
+    });
+  }
+
   // [id, raw, seq, pointer, order] -> SnapshotRead. The environment compare is app-side, per the
   // plan: the store returns null for a foreign environment and the 404 throw stays in the engine.
-  #decode(reply: string[] | null, environmentId?: string): SnapshotRead | null {
+  // orderKnown distinguishes "order field is genuinely empty" from "order was not read for this
+  // row" (getSince's tail rows use the same empty string for the latter) -- the mismatch check and
+  // completedWaitpointIds must both be skipped when the order was never read.
+  #decode(
+    reply: string[] | null,
+    environmentId: string | undefined,
+    runId: string,
+    orderKnown: boolean
+  ): SnapshotRead | null {
     if (!reply || reply.length === 0) return null;
     const [id, raw, seqStr, pointer, orderJson] = reply;
     const entry = JSON.parse(raw) as Record<string, unknown>;
@@ -369,7 +390,11 @@ export class RedisSnapshotStore {
     if (pointer) {
       const [cs, count] = pointer.split(":");
       read.cycle = { cycleSeq: Number(cs), count: Number(count) };
-      read.completedWaitpointIds = decodeWaitpointIds(true, orderJson);
+      if (orderKnown) {
+        const ids = decodeWaitpointIds(true, orderJson);
+        read.completedWaitpointIds = ids;
+        this.#checkCycleMismatch(runId, Number(count), ids.order.length);
+      }
     }
     return read;
   }

From 4c1f6e513fe3ff5d1f5a52cb18d1944e1a580cf6 Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Fri, 21 Aug 2026 18:37:43 +0100
Subject: [PATCH 22/49] feat(core): run-ops format waitpoint ids with a type
 char and version w

---
 .../core/src/v3/isomorphic/friendlyId.test.ts | 139 ++++++++++++++++++
 packages/core/src/v3/isomorphic/friendlyId.ts |  88 +++++++++++
 2 files changed, 227 insertions(+)

diff --git a/packages/core/src/v3/isomorphic/friendlyId.test.ts b/packages/core/src/v3/isomorphic/friendlyId.test.ts
index 2e3ba4d83a5..a0be5a69f79 100644
--- a/packages/core/src/v3/isomorphic/friendlyId.test.ts
+++ b/packages/core/src/v3/isomorphic/friendlyId.test.ts
@@ -11,13 +11,20 @@ import {
   RUN_OPS_ID_VERSION,
   RUN_OPS_ID_VERSION_2,
   RUN_OPS_ID_VERSION_INDEX,
+  WAITPOINT_ID_TYPE_INDEX,
+  WAITPOINT_ID_VERSION,
   base32hexDecode,
   base32hexEncode,
+  deriveWaitpointIdFromAnchor,
+  generateFriendlyId,
   generateRunOpsId,
   generateRunOpsIdV2,
+  generateWaitpointId,
   parseRunId,
   parseRunOpsIdBody,
   parseRunOpsIdV2Body,
+  parseWaitpointId,
+  type WaitpointIdType,
 } from "./friendlyId.js";
 
 /** Every legal gen-2 shard char: the full DNS-safe lowercase range. */
@@ -410,3 +417,135 @@ describe("parseRunId — v2 arm", () => {
     expect(parseRunId(`waitpoint_${generateRunOpsIdV2("a")}`).format).toBe("legacy");
   });
 });
+
+describe("waitpoint ids: run-ops format with version char w", () => {
+  it("mints a 26-char body per type, with the type char at index 24 and version w at 25", () => {
+    const cases: Array<[WaitpointIdType, string]> = [
+      ["RUN", "r"],
+      ["BATCH", "b"],
+      ["DATETIME", "d"],
+      ["MANUAL", "m"],
+    ];
+
+    for (const [type, typeChar] of cases) {
+      const body = generateWaitpointId(type);
+      expect(body.length).toBe(RUN_OPS_ID_LENGTH);
+      expect(body[WAITPOINT_ID_TYPE_INDEX]).toBe(typeChar);
+      expect(body[RUN_OPS_ID_VERSION_INDEX]).toBe(WAITPOINT_ID_VERSION);
+    }
+  });
+
+  it("round-trips every type char through parseWaitpointId", () => {
+    for (const type of ["RUN", "BATCH", "DATETIME", "MANUAL"] as WaitpointIdType[]) {
+      const parsed = parseWaitpointId(generateWaitpointId(type));
+      expect(parsed.format).toBe("b32hexW");
+      if (parsed.format !== "b32hexW") throw new Error("unreachable");
+      expect(parsed.type).toBe(type);
+    }
+  });
+
+  it("classifies both the prefixed and the bare form identically", () => {
+    const body = generateWaitpointId("MANUAL");
+    expect(parseWaitpointId(body)).toEqual(parseWaitpointId(`waitpoint_${body}`));
+  });
+
+  it("recovers the mint timestamp from the core", () => {
+    vi.useFakeTimers();
+    try {
+      vi.setSystemTime(new Date("2026-08-21T12:00:00.000Z"));
+      const parsed = parseWaitpointId(generateWaitpointId("DATETIME"));
+      if (parsed.format !== "b32hexW") throw new Error("unreachable");
+      expect(parsed.timestamp.toISOString()).toBe("2026-08-21T12:00:00.000Z");
+    } finally {
+      vi.useRealTimers();
+    }
+  });
+
+  it("classifies every legacy shape as legacy", () => {
+    const legacy = [
+      WaitpointId.generate().id,
+      WaitpointId.generate().friendlyId,
+      generateFriendlyId("waitpoint"),
+      "",
+      "waitpoint_",
+      "a".repeat(27),
+      "a".repeat(26),
+    ];
+
+    for (const id of legacy) {
+      expect(parseWaitpointId(id).format).toBe("legacy");
+    }
+  });
+
+  it("rejects a 26-char body whose version is w but whose type char is not r/b/d/m", () => {
+    const body = generateWaitpointId("RUN");
+    const bad = `${body.slice(0, WAITPOINT_ID_TYPE_INDEX)}x${WAITPOINT_ID_VERSION}`;
+    expect(parseWaitpointId(bad).format).toBe("legacy");
+  });
+
+  it("rejects a body whose core is outside the base32hex alphabet", () => {
+    const body = generateWaitpointId("RUN");
+    // "w" is outside [0-9a-v], so the core no longer decodes.
+    expect(parseWaitpointId(`w${body.slice(1)}`).format).toBe("legacy");
+  });
+
+  it("never parses a run id as a waitpoint id, or the reverse", () => {
+    expect(parseWaitpointId(generateRunOpsId()).format).toBe("legacy");
+    expect(parseWaitpointId(generateRunOpsIdV2("7")).format).toBe("legacy");
+    expect(parseRunId(`run_${generateWaitpointId("RUN")}`).format).toBe("legacy");
+  });
+});
+
+describe("deriveWaitpointIdFromAnchor", () => {
+  it("is deterministic: the same anchor and type always give the same id", () => {
+    const anchor = `run_${generateRunOpsId("us-east-1")}`;
+    const first = deriveWaitpointIdFromAnchor(anchor, "RUN");
+    expect(first).toBeDefined();
+    expect(first).toBe(deriveWaitpointIdFromAnchor(anchor, "RUN"));
+  });
+
+  it("shares the anchor's 24-char core and replaces the region and version chars", () => {
+    const anchorBody = generateRunOpsId("us-east-1");
+    const derived = deriveWaitpointIdFromAnchor(`run_${anchorBody}`, "RUN");
+    expect(derived).toBeDefined();
+    expect(derived!.slice(0, WAITPOINT_ID_TYPE_INDEX)).toBe(
+      anchorBody.slice(0, WAITPOINT_ID_TYPE_INDEX)
+    );
+    expect(derived![WAITPOINT_ID_TYPE_INDEX]).toBe("r");
+    expect(derived![RUN_OPS_ID_VERSION_INDEX]).toBe(WAITPOINT_ID_VERSION);
+  });
+
+  it("accepts a bare anchor body as well as a prefixed one", () => {
+    const anchorBody = generateRunOpsId();
+    expect(deriveWaitpointIdFromAnchor(anchorBody, "RUN")).toBe(
+      deriveWaitpointIdFromAnchor(`run_${anchorBody}`, "RUN")
+    );
+  });
+
+  it("accepts a gen-2 anchor", () => {
+    const derived = deriveWaitpointIdFromAnchor(`run_${generateRunOpsIdV2("7")}`, "RUN");
+    expect(derived).toBeDefined();
+    expect(parseWaitpointId(derived!).format).toBe("b32hexW");
+  });
+
+  it("derives a BATCH id from a run-ops format batch anchor", () => {
+    const derived = deriveWaitpointIdFromAnchor(`batch_${generateRunOpsId()}`, "BATCH");
+    expect(derived).toBeDefined();
+    const parsed = parseWaitpointId(derived!);
+    if (parsed.format !== "b32hexW") throw new Error("unreachable");
+    expect(parsed.type).toBe("BATCH");
+  });
+
+  it("returns undefined for a legacy anchor, so the caller falls back to a legacy mint", () => {
+    expect(deriveWaitpointIdFromAnchor(RunId.generate().friendlyId, "RUN")).toBeUndefined();
+    expect(deriveWaitpointIdFromAnchor("run_", "RUN")).toBeUndefined();
+    expect(deriveWaitpointIdFromAnchor("", "RUN")).toBeUndefined();
+  });
+
+  it("gives a different id per type from one anchor", () => {
+    const anchor = `run_${generateRunOpsId()}`;
+    expect(deriveWaitpointIdFromAnchor(anchor, "RUN")).not.toBe(
+      deriveWaitpointIdFromAnchor(anchor, "BATCH")
+    );
+  });
+});
diff --git a/packages/core/src/v3/isomorphic/friendlyId.ts b/packages/core/src/v3/isomorphic/friendlyId.ts
index c468de65319..443ca9fc472 100644
--- a/packages/core/src/v3/isomorphic/friendlyId.ts
+++ b/packages/core/src/v3/isomorphic/friendlyId.ts
@@ -238,6 +238,94 @@ export function parseRunId(id: string): ParsedRunId {
   return LEGACY_RUN_ID;
 }
 
+// Waitpoint ids reuse the run-ops body layout — 24-char base32hex core, then a
+// positional char, then a version char — so the body parses positionally instead of
+// splitting on "_". Index 24 carries the TYPE (the slot a run uses for its region or
+// shard char), which leaves room to move to a shard char under a later version.
+export const WAITPOINT_ID_VERSION = "w";
+export const WAITPOINT_ID_TYPE_INDEX = RUN_OPS_ID_REGION_INDEX;
+
+export type WaitpointIdType = "RUN" | "BATCH" | "DATETIME" | "MANUAL";
+
+// "w" sits OUTSIDE the base32hex alphabet [0-9a-v], so the version char can never be
+// mistaken for a core char, and it can never collide with a numeric run generation.
+const WAITPOINT_TYPE_CHARS: Readonly<Record<WaitpointIdType, string>> = {
+  RUN: "r",
+  BATCH: "b",
+  DATETIME: "d",
+  MANUAL: "m",
+};
+
+const WAITPOINT_TYPES_BY_CHAR: Readonly<Record<string, WaitpointIdType>> = {
+  r: "RUN",
+  b: "BATCH",
+  d: "DATETIME",
+  m: "MANUAL",
+};
+
+export type ParsedWaitpointId =
+  | { format: "b32hexW"; type: WaitpointIdType; timestamp: Date }
+  | { format: "legacy" };
+
+const LEGACY_WAITPOINT_ID: ParsedWaitpointId = { format: "legacy" };
+
+/**
+ * Mint a standalone waitpoint id body (26 chars, no prefix) for DATETIME and MANUAL: a
+ * fresh core, the type char, then the waitpoint version char.
+ */
+export function generateWaitpointId(type: WaitpointIdType): string {
+  return `${mintRunOpsIdCore()}${WAITPOINT_TYPE_CHARS[type]}${WAITPOINT_ID_VERSION}`;
+}
+
+/**
+ * Derive the 1:1 waitpoint id body for a RUN or BATCH anchor by reusing the anchor's
+ * 24-char core. Pure, so create-if-absent is idempotent without a lock. Returns
+ * undefined when the anchor is not a run-ops id, which is the caller's signal to mint a
+ * legacy waitpoint instead.
+ *
+ * Only the core survives: the anchor's region or shard char and its version char are
+ * both replaced. So the anchor id is NOT recoverable from the waitpoint id — the reverse
+ * direction uses the completedBy* back-pointer.
+ */
+export function deriveWaitpointIdFromAnchor(
+  anchorId: string,
+  type: WaitpointIdType
+): string | undefined {
+  const body = stripIdPrefix(anchorId);
+  if (!parseRunOpsIdBody(body) && !parseRunOpsIdV2Body(body)) {
+    return undefined;
+  }
+
+  return `${body.slice(0, RUN_OPS_ID_CORE_LENGTH)}${WAITPOINT_TYPE_CHARS[type]}${WAITPOINT_ID_VERSION}`;
+}
+
+/**
+ * Classify a waitpoint id. Accepts the prefixed form (`waitpoint_<body>`) and the bare
+ * internal form, because both circulate: IdUtil.generate() returns each, and the store's
+ * own call sites carry the internal one. Total: never throws.
+ */
+export function parseWaitpointId(id: string): ParsedWaitpointId {
+  const body = stripIdPrefix(id);
+  if (body.length !== RUN_OPS_ID_LENGTH) return LEGACY_WAITPOINT_ID;
+  if (body[RUN_OPS_ID_VERSION_INDEX] !== WAITPOINT_ID_VERSION) return LEGACY_WAITPOINT_ID;
+
+  const type = WAITPOINT_TYPES_BY_CHAR[body[WAITPOINT_ID_TYPE_INDEX] ?? ""];
+  if (!type) return LEGACY_WAITPOINT_ID;
+
+  const timestamp = parseRunOpsIdCoreTimestamp(body);
+  if (timestamp === undefined) return LEGACY_WAITPOINT_ID;
+
+  return { format: "b32hexW", type, timestamp };
+}
+
+// Strip a single leading `<prefix>_` if present, so the friendly and internal forms
+// classify identically. Only the FIRST underscore separates the prefix, mirroring
+// fromFriendlyId's two-part contract.
+function stripIdPrefix(id: string): string {
+  const underscore = id.indexOf("_");
+  return underscore === -1 ? id : id.slice(underscore + 1);
+}
+
 export function generateInternalId(): string {
   return cuid();
 }

From d801da8c1a995d76387b0010c69bcff1aadcd560 Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Fri, 21 Aug 2026 18:46:55 +0100
Subject: [PATCH 23/49] fix(core): reject a foreign prefix in parseWaitpointId

parseWaitpointId no longer strips an arbitrary <prefix>_ before
classifying a body, so a run_ or batch_ id can never be misread as a
waitpoint id. deriveWaitpointIdFromAnchor keeps its own prefix-agnostic
stripping, since its input is always a known run/batch anchor.
---
 .../core/src/v3/isomorphic/friendlyId.test.ts | 25 +++++++++++++++-
 packages/core/src/v3/isomorphic/friendlyId.ts | 29 +++++++++++++------
 2 files changed, 44 insertions(+), 10 deletions(-)

diff --git a/packages/core/src/v3/isomorphic/friendlyId.test.ts b/packages/core/src/v3/isomorphic/friendlyId.test.ts
index a0be5a69f79..b5ea7a51971 100644
--- a/packages/core/src/v3/isomorphic/friendlyId.test.ts
+++ b/packages/core/src/v3/isomorphic/friendlyId.test.ts
@@ -446,7 +446,10 @@ describe("waitpoint ids: run-ops format with version char w", () => {
 
   it("classifies both the prefixed and the bare form identically", () => {
     const body = generateWaitpointId("MANUAL");
-    expect(parseWaitpointId(body)).toEqual(parseWaitpointId(`waitpoint_${body}`));
+    const bare = parseWaitpointId(body);
+    const prefixed = parseWaitpointId(`waitpoint_${body}`);
+    expect(bare).toEqual(prefixed);
+    expect(bare).toEqual({ format: "b32hexW", type: "MANUAL", timestamp: expect.any(Date) });
   });
 
   it("recovers the mint timestamp from the core", () => {
@@ -494,6 +497,26 @@ describe("waitpoint ids: run-ops format with version char w", () => {
     expect(parseWaitpointId(generateRunOpsIdV2("7")).format).toBe("legacy");
     expect(parseRunId(`run_${generateWaitpointId("RUN")}`).format).toBe("legacy");
   });
+
+  it("rejects a well-formed waitpoint body wearing a foreign prefix", () => {
+    const body = `${"0".repeat(24)}rw`; // valid core + RUN type char + version w
+    expect(parseWaitpointId(`run_${body}`).format).toBe("legacy");
+    expect(parseWaitpointId(`batch_${body}`).format).toBe("legacy");
+    expect(parseWaitpointId(`waitpoint_${body}`)).toEqual({
+      format: "b32hexW",
+      type: "RUN",
+      timestamp: expect.any(Date),
+    });
+    expect(parseWaitpointId(body).format).toBe("b32hexW");
+  });
+
+  it("handles a bare body that happens to contain an underscore sanely (never throws, never misclassifies)", () => {
+    const body = generateWaitpointId("BATCH");
+    const withUnderscore = `_${body.slice(1)}`;
+    expect(() => parseWaitpointId(withUnderscore)).not.toThrow();
+    // "_" is outside the base32hex alphabet, so this can never be a real waitpoint id.
+    expect(parseWaitpointId(withUnderscore).format).toBe("legacy");
+  });
 });
 
 describe("deriveWaitpointIdFromAnchor", () => {
diff --git a/packages/core/src/v3/isomorphic/friendlyId.ts b/packages/core/src/v3/isomorphic/friendlyId.ts
index 443ca9fc472..2f436b93a3e 100644
--- a/packages/core/src/v3/isomorphic/friendlyId.ts
+++ b/packages/core/src/v3/isomorphic/friendlyId.ts
@@ -291,7 +291,7 @@ export function deriveWaitpointIdFromAnchor(
   anchorId: string,
   type: WaitpointIdType
 ): string | undefined {
-  const body = stripIdPrefix(anchorId);
+  const body = stripAnchorPrefix(anchorId);
   if (!parseRunOpsIdBody(body) && !parseRunOpsIdV2Body(body)) {
     return undefined;
   }
@@ -300,12 +300,14 @@ export function deriveWaitpointIdFromAnchor(
 }
 
 /**
- * Classify a waitpoint id. Accepts the prefixed form (`waitpoint_<body>`) and the bare
- * internal form, because both circulate: IdUtil.generate() returns each, and the store's
- * own call sites carry the internal one. Total: never throws.
+ * Classify a waitpoint id. Accepts the prefixed (`waitpoint_<body>`) and bare forms, but
+ * NOT another entity's prefix (`run_`, `batch_`, ...) — this is the discriminator a
+ * later ticket uses to route a possibly customer-supplied id, so a foreign prefix must
+ * classify legacy rather than have its body reinterpreted as a waitpoint id. Total:
+ * never throws.
  */
 export function parseWaitpointId(id: string): ParsedWaitpointId {
-  const body = stripIdPrefix(id);
+  const body = stripWaitpointIdPrefix(id);
   if (body.length !== RUN_OPS_ID_LENGTH) return LEGACY_WAITPOINT_ID;
   if (body[RUN_OPS_ID_VERSION_INDEX] !== WAITPOINT_ID_VERSION) return LEGACY_WAITPOINT_ID;
 
@@ -318,14 +320,23 @@ export function parseWaitpointId(id: string): ParsedWaitpointId {
   return { format: "b32hexW", type, timestamp };
 }
 
-// Strip a single leading `<prefix>_` if present, so the friendly and internal forms
-// classify identically. Only the FIRST underscore separates the prefix, mirroring
-// fromFriendlyId's two-part contract.
-function stripIdPrefix(id: string): string {
+// Strip any `<prefix>_` if present. Prefix-agnostic is correct ONLY here: the caller
+// already knows anchorId names a run or batch anchor, so there is no foreign prefix to
+// guard against. Do not reuse for parseWaitpointId — see stripWaitpointIdPrefix.
+function stripAnchorPrefix(id: string): string {
   const underscore = id.indexOf("_");
   return underscore === -1 ? id : id.slice(underscore + 1);
 }
 
+const WAITPOINT_ID_PREFIX = "waitpoint_";
+
+// Strip the `waitpoint_` prefix if present; any other prefix, or a bare body, is left
+// as-is. Unlike stripAnchorPrefix, this must never strip a foreign prefix down to a body
+// that then happens to pass the run-ops shape check.
+function stripWaitpointIdPrefix(id: string): string {
+  return id.startsWith(WAITPOINT_ID_PREFIX) ? id.slice(WAITPOINT_ID_PREFIX.length) : id;
+}
+
 export function generateInternalId(): string {
   return cuid();
 }

From 10a80fae38c223e24984086a07271c43157ad346 Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Fri, 21 Aug 2026 18:53:43 +0100
Subject: [PATCH 24/49] feat(run-engine): waitpoint coordination keyspace and
 single-slot assertion

---
 .../engine/waitpointCoordinator/keys.test.ts  | 109 ++++++++++++++++++
 .../src/engine/waitpointCoordinator/keys.ts   |  84 ++++++++++++++
 2 files changed, 193 insertions(+)
 create mode 100644 internal-packages/run-engine/src/engine/waitpointCoordinator/keys.test.ts
 create mode 100644 internal-packages/run-engine/src/engine/waitpointCoordinator/keys.ts

diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.test.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.test.ts
new file mode 100644
index 00000000000..7bd9505c9ef
--- /dev/null
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.test.ts
@@ -0,0 +1,109 @@
+import { describe, expect, it } from "vitest";
+import {
+  WaitpointKeyTagError,
+  assertSingleSlot,
+  edgeField,
+  idempotencyKey,
+  runBlockKeys,
+  waitpointIdFromEdgeField,
+  waitpointKeys,
+  watcherField,
+} from "./keys.js";
+
+describe("waitpointKeys", () => {
+  it("puts the record and its watchers under one hash tag", () => {
+    const k = waitpointKeys("abc123w");
+    expect(k.record).toBe("wp:{abc123w}");
+    expect(k.watchers).toBe("wp:{abc123w}:w");
+  });
+});
+
+describe("runBlockKeys", () => {
+  it("puts all three run keys under one hash tag", () => {
+    const k = runBlockKeys("run_abc");
+    expect(k.pend).toBe("wp:run:{run_abc}:pend");
+    expect(k.done).toBe("wp:run:{run_abc}:done");
+    expect(k.edge).toBe("wp:run:{run_abc}:edge");
+  });
+});
+
+describe("idempotencyKey", () => {
+  it("tags by environment, so one environment's reservations share a slot", () => {
+    expect(idempotencyKey("env_1", "my-key")).toBe("wp:idem:{env_1}:my-key");
+  });
+});
+
+describe("edgeField", () => {
+  it("keys by waitpoint id and batch index, matching the Postgres unique key", () => {
+    expect(edgeField("w_a", 3)).toBe("w_a#3");
+  });
+
+  it("collapses a null or absent batch index onto one field", () => {
+    expect(edgeField("w_a")).toBe("w_a#");
+    expect(edgeField("w_a", null)).toBe("w_a#");
+  });
+
+  it("distinguishes index 0 from an absent index", () => {
+    expect(edgeField("w_a", 0)).not.toBe(edgeField("w_a"));
+  });
+
+  it("round-trips back to the waitpoint id", () => {
+    for (const index of [undefined, null, 0, 7]) {
+      expect(waitpointIdFromEdgeField(edgeField("w_a", index))).toBe("w_a");
+    }
+  });
+
+  it("returns undefined for a field with no separator", () => {
+    expect(waitpointIdFromEdgeField("nope")).toBeUndefined();
+  });
+});
+
+describe("watcherField", () => {
+  it("keys by run id and batch index, so one run can watch at several indexes", () => {
+    expect(watcherField("run_a", 2)).toBe("run_a#2");
+    expect(watcherField("run_a")).toBe("run_a#");
+    expect(watcherField("run_a", 0)).not.toBe(watcherField("run_a"));
+  });
+});
+
+describe("assertSingleSlot", () => {
+  it("accepts keys that share one tag", () => {
+    const k = runBlockKeys("run_abc");
+    expect(() => assertSingleSlot("runReadBlockState", [k.pend, k.done, k.edge])).not.toThrow();
+  });
+
+  it("accepts a single tagged key", () => {
+    expect(() => assertSingleSlot("wpIdemReserve", [idempotencyKey("env_1", "k")])).not.toThrow();
+  });
+
+  it("accepts an empty key list", () => {
+    expect(() => assertSingleSlot("noKeys", [])).not.toThrow();
+  });
+
+  it("rejects keys from two different tags", () => {
+    const wp = waitpointKeys("w_a");
+    const run = runBlockKeys("run_abc");
+    expect(() => assertSingleSlot("bad", [wp.record, run.pend])).toThrow(WaitpointKeyTagError);
+  });
+
+  it("rejects an untagged key", () => {
+    expect(() => assertSingleSlot("bad", ["wp:no-tag"])).toThrow(WaitpointKeyTagError);
+  });
+
+  it("rejects an empty tag", () => {
+    expect(() => assertSingleSlot("bad", ["wp:{}"])).toThrow(WaitpointKeyTagError);
+  });
+
+  it("names the operation and the offending key in the error", () => {
+    const wp = waitpointKeys("w_a");
+    const run = runBlockKeys("run_abc");
+    try {
+      assertSingleSlot("myOperation", [wp.record, run.pend]);
+      throw new Error("should have thrown");
+    } catch (error) {
+      expect(error).toBeInstanceOf(WaitpointKeyTagError);
+      expect((error as Error).message).toContain("myOperation");
+      expect((error as Error).message).toContain(run.pend);
+    }
+  });
+});
diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.ts
new file mode 100644
index 00000000000..2b3a8f17eac
--- /dev/null
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.ts
@@ -0,0 +1,84 @@
+/**
+ * Waitpoint coordination keyspace. Two hash tags, deliberately:
+ *
+ *  - `wp:{waitpointId}` — the record, its status and completion envelope, plus the
+ *    watcher hash. A waitpoint has N watcher runs, so it cannot live under any single
+ *    run's tag.
+ *  - `wp:run:{runId}:*` — one run's pending set, delivered set and edge set. The pending
+ *    set's cardinality is the blocked-versus-unblocked signal, so it has to be readable
+ *    atomically, which means one slot.
+ *
+ * Every script therefore touches exactly one tag, and assertSingleSlot enforces it on
+ * every invocation. A cluster would reject a cross-slot script; a single-node test server
+ * would not, so this assertion is the only thing standing between a cross-slot bug and
+ * production.
+ */
+
+export type WaitpointKeys = { record: string; watchers: string };
+export type RunBlockKeys = { pend: string; done: string; edge: string };
+
+export function waitpointKeys(waitpointId: string): WaitpointKeys {
+  const base = `wp:{${waitpointId}}`;
+  return { record: base, watchers: `${base}:w` };
+}
+
+export function runBlockKeys(runId: string): RunBlockKeys {
+  const base = `wp:run:{${runId}}`;
+  return { pend: `${base}:pend`, done: `${base}:done`, edge: `${base}:edge` };
+}
+
+export function idempotencyKey(environmentId: string, key: string): string {
+  return `wp:idem:{${environmentId}}:${key}`;
+}
+
+// "#" separates the id from the index. A waitpoint id and a run id never contain "#", so
+// the split is unambiguous. An absent index collapses onto the empty suffix, which is how
+// the partial unique index on a null batchIndex behaves; index 0 keeps its own field,
+// because "0" and "" are different strings.
+const SEPARATOR = "#";
+
+export function edgeField(waitpointId: string, batchIndex?: number | null): string {
+  return `${waitpointId}${SEPARATOR}${batchIndex ?? ""}`;
+}
+
+export function watcherField(runId: string, batchIndex?: number | null): string {
+  return `${runId}${SEPARATOR}${batchIndex ?? ""}`;
+}
+
+export function waitpointIdFromEdgeField(field: string): string | undefined {
+  const separator = field.lastIndexOf(SEPARATOR);
+  return separator === -1 ? undefined : field.slice(0, separator);
+}
+
+export class WaitpointKeyTagError extends Error {
+  constructor(operation: string, keys: string[], offending: string) {
+    super(
+      `Waitpoint operation ${operation} would span more than one cluster slot: ` +
+        `key ${JSON.stringify(offending)} does not share the tag of ${JSON.stringify(keys)}`
+    );
+    this.name = "WaitpointKeyTagError";
+  }
+}
+
+const HASH_TAG = /\{([^}]+)\}/;
+
+/**
+ * Throw unless every key carries the same non-empty hash tag. Called on every script
+ * invocation, because the keys embed ids and are only known at call time.
+ */
+export function assertSingleSlot(operation: string, keys: string[]): void {
+  let tag: string | undefined;
+
+  for (const key of keys) {
+    const match = HASH_TAG.exec(key);
+    const found = match?.[1];
+    if (!found) {
+      throw new WaitpointKeyTagError(operation, keys, key);
+    }
+    if (tag === undefined) {
+      tag = found;
+    } else if (found !== tag) {
+      throw new WaitpointKeyTagError(operation, keys, key);
+    }
+  }
+}

From a5fecde311962f226ec30d749c7a995c9bc5bd3e Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Fri, 21 Aug 2026 19:04:48 +0100
Subject: [PATCH 25/49] feat(run-engine): waitpoint shard scripts for create,
 register and complete

---
 .../engine/waitpointCoordinator/scripts.ts    | 336 +++++++++++++++++
 .../storeCoordinator.test.ts                  | 340 ++++++++++++++++++
 .../waitpointCoordinator/storeCoordinator.ts  | 239 ++++++++++++
 3 files changed, 915 insertions(+)
 create mode 100644 internal-packages/run-engine/src/engine/waitpointCoordinator/scripts.ts
 create mode 100644 internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts
 create mode 100644 internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts

diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/scripts.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/scripts.ts
new file mode 100644
index 00000000000..1e60f6516de
--- /dev/null
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/scripts.ts
@@ -0,0 +1,336 @@
+import type { Callback, Redis, Result } from "@internal/redis";
+
+/**
+ * Lua for the waitpoint coordination protocol. Three rules hold throughout:
+ *
+ *  1. Every key a script touches is declared in KEYS. No script builds a key name inside
+ *     Lua. ioredis prefixes only the KEYS array, so a key minted in Lua would be
+ *     unprefixed while the client wrote a prefixed one — and a script with a single
+ *     declared key gives the caller's single-slot assertion nothing to compare.
+ *  2. Lua never parses JSON. Each script branches only on a short status string and moves
+ *     opaque blobs, so every encoding decision stays in TypeScript.
+ *  3. Every returned slot is coerced with `or ''`. A Lua false or nil TRUNCATES the reply
+ *     array at that position, silently shortening it.
+ *
+ * STORED_COMPLETED is the value written into the record's `status` field and is
+ * UPPERCASE. The outcome tokens below are lowercase and are a separate vocabulary: they
+ * name what a script DID, not what a record IS. Sharing one constant between the two
+ * makes an already-completed record invisible to every script.
+ */
+
+const STORED_COMPLETED = "COMPLETED";
+
+const MISSING = "missing";
+const CREATED = "created";
+const EXISTS = "exists";
+const REGISTERED = "registered";
+const DID_COMPLETE = "completed";
+const ALREADY = "already";
+const RESERVED = "reserved";
+const CLEARED = "cleared";
+const DRAINED = "drained";
+
+export function registerWaitpointCommands(redis: Redis): void {
+  // KEYS: record. ARGV: recordJson, status ('PENDING'|'COMPLETED'), completionJson ('').
+  redis.defineCommand("wpCreateIfAbsent", {
+    numberOfKeys: 1,
+    lua: `
+      local record = KEYS[1]
+
+      -- EXISTS-then-HSET inside one script, rather than a field-by-field HSETNX: the
+      -- record and its status must appear together or not at all.
+      if redis.call('EXISTS', record) == 1 then
+        local vals = redis.call('HMGET', record, 'r', 'status', 'c')
+        return { '${EXISTS}', vals[1] or '', vals[2] or '', vals[3] or '' }
+      end
+
+      redis.call('HSET', record, 'r', ARGV[1], 'status', ARGV[2])
+      if ARGV[3] ~= '' then
+        redis.call('HSET', record, 'c', ARGV[3])
+      end
+
+      return { '${CREATED}' }
+    `,
+  });
+
+  // KEYS: record, watchers. ARGV: watcherField, watcherJson.
+  redis.defineCommand("wpRegisterOrReport", {
+    numberOfKeys: 2,
+    lua: `
+      local record, watchers = KEYS[1], KEYS[2]
+
+      -- A missing waitpoint is never a silent no-op: the caller throws. Defaulting to
+      -- "not blocked" here would resume a run whose waitpoint never completed.
+      if redis.call('EXISTS', record) == 0 then
+        return { '${MISSING}' }
+      end
+
+      if redis.call('HGET', record, 'status') == '${STORED_COMPLETED}' then
+        return { '${DID_COMPLETE}', redis.call('HGET', record, 'c') or '' }
+      end
+
+      -- The watcher lands before any flip can read the watcher hash, because this script
+      -- and wpComplete are both atomic on this same shard. So a register either appears
+      -- in the flip's watcher list, or it observes COMPLETED above.
+      redis.call('HSET', watchers, ARGV[1], ARGV[2])
+      return { '${REGISTERED}' }
+    `,
+  });
+
+  // KEYS: record, watchers. ARGV: completionJson.
+  redis.defineCommand("wpComplete", {
+    numberOfKeys: 2,
+    lua: `
+      local record, watchers = KEYS[1], KEYS[2]
+
+      if redis.call('EXISTS', record) == 0 then
+        return { '${MISSING}' }
+      end
+
+      local outcome = '${DID_COMPLETE}'
+      if redis.call('HGET', record, 'status') == '${STORED_COMPLETED}' then
+        -- Double completion is not an error, and the FIRST completion wins. This is the
+        -- guard a conditional UPDATE ... WHERE status = 'PENDING' used to provide.
+        outcome = '${ALREADY}'
+      else
+        redis.call('HSET', record, 'status', '${STORED_COMPLETED}', 'c', ARGV[1])
+      end
+
+      -- Returning the watchers here is what removes the reverse fan-out query. The
+      -- envelope comes back too, because delivery runs on each watcher's own shard and
+      -- cannot read this key.
+      local out = { outcome, redis.call('HGET', record, 'c') or '' }
+      local entries = redis.call('HVALS', watchers)
+      for i = 1, #entries do
+        out[#out + 1] = entries[i]
+      end
+
+      return out
+    `,
+  });
+
+  // KEYS: idempotency key. ARGV: waitpointId, expiresAtMs ('' for no expiry).
+  redis.defineCommand("wpIdemReserve", {
+    numberOfKeys: 1,
+    lua: `
+      local key = KEYS[1]
+
+      -- SET NX returns a status reply on success and false on conflict.
+      if redis.call('SET', key, ARGV[1], 'NX') then
+        -- Expiry only when the caller has one. A reservation with no expiry is the common
+        -- case and must never grow one here.
+        if ARGV[2] ~= '' then
+          redis.call('PEXPIREAT', key, tonumber(ARGV[2]))
+        end
+        return { '${RESERVED}', ARGV[1] }
+      end
+
+      return { '${EXISTS}', redis.call('GET', key) or '' }
+    `,
+  });
+
+  // KEYS: pend, done, edge.
+  // ARGV: n, then n groups of 4 — waitpointId, edgeField, edgeJson, reportedJson ('').
+  redis.defineCommand("runAbsorbBlockers", {
+    numberOfKeys: 3,
+    lua: `
+      local pend, done, edge = KEYS[1], KEYS[2], KEYS[3]
+      local n = tonumber(ARGV[1])
+
+      -- countedPending and seenDelivered make both outputs DISTINCT BY ID. The count this
+      -- replaces was a COUNT(*) over waitpoint rows, so two edges for one waitpoint
+      -- contributed one. Counting per edge would inflate it.
+      local countedPending = {}
+      local seenDelivered = {}
+      local pendingOfRequested = 0
+      local out = { '0', '0' }
+
+      for i = 0, n - 1 do
+        local id       = ARGV[2 + i * 4]
+        local field    = ARGV[3 + i * 4]
+        local edgeJson = ARGV[4 + i * 4]
+        local reported = ARGV[5 + i * 4]
+
+        -- HSETNX is the ON CONFLICT DO NOTHING of the edge write: a retry must not
+        -- overwrite the first attempt's metadata.
+        redis.call('HSETNX', edge, field, edgeJson)
+
+        if reported ~= '' then
+          -- Already COMPLETED when the watcher registered. It never becomes pending.
+          redis.call('HSET', done, id, reported)
+          redis.call('SREM', pend, id)
+          if not seenDelivered[id] then
+            seenDelivered[id] = true
+            out[#out + 1] = id
+            out[#out + 1] = reported
+          end
+        else
+          -- Check the delivered set FIRST. A completion that landed between register and
+          -- absorb has already delivered here, and that delivery wins.
+          local delivered = redis.call('HGET', done, id)
+          if delivered then
+            if not seenDelivered[id] then
+              seenDelivered[id] = true
+              out[#out + 1] = id
+              out[#out + 1] = delivered
+            end
+          else
+            redis.call('SADD', pend, id)
+            if not countedPending[id] then
+              countedPending[id] = true
+              pendingOfRequested = pendingOfRequested + 1
+            end
+          end
+        end
+      end
+
+      out[1] = tostring(pendingOfRequested)
+      out[2] = tostring(redis.call('SCARD', pend))
+      return out
+    `,
+  });
+
+  // KEYS: pend, done. ARGV: waitpointId, completionJson.
+  redis.defineCommand("runDeliverCompletion", {
+    numberOfKeys: 2,
+    lua: `
+      local pend, done = KEYS[1], KEYS[2]
+
+      redis.call('HSET', done, ARGV[1], ARGV[2])
+      redis.call('SREM', pend, ARGV[1])
+
+      -- The caller treats this as a wakeup trigger, not as the resume decision: the
+      -- resume is decided under the run lock, and this count covers store-resident
+      -- blockers only.
+      return { tostring(redis.call('SCARD', pend)) }
+    `,
+  });
+
+  // KEYS: pend, done, edge.
+  redis.defineCommand("runReadBlockState", {
+    numberOfKeys: 3,
+    lua: `
+      local pend, done, edge = KEYS[1], KEYS[2], KEYS[3]
+
+      local pendIds = redis.call('SMEMBERS', pend)
+      -- HKEYS, never HGETALL: the delivered set's values are completion envelopes with
+      -- inline outputs, and materializing those inside a single-threaded script would
+      -- block the shard.
+      local doneIds = redis.call('HKEYS', done)
+      local edges   = redis.call('HGETALL', edge)
+
+      local out = { tostring(#pendIds), tostring(#doneIds), tostring(#edges) }
+      for i = 1, #pendIds do out[#out + 1] = pendIds[i] end
+      for i = 1, #doneIds do out[#out + 1] = doneIds[i] end
+      for i = 1, #edges   do out[#out + 1] = edges[i]   end
+      return out
+    `,
+  });
+
+  // KEYS: pend, done, edge. ARGV: n, then n edge fields. n = 0 clears everything.
+  redis.defineCommand("runClear", {
+    numberOfKeys: 3,
+    lua: `
+      local pend, done, edge = KEYS[1], KEYS[2], KEYS[3]
+      local n = tonumber(ARGV[1])
+
+      if n == 0 then
+        redis.call('DEL', pend, done, edge)
+        return { '${CLEARED}' }
+      end
+
+      for i = 1, n do
+        redis.call('HDEL', edge, ARGV[1 + i])
+      end
+
+      -- Reconcile rather than delete by name. The edge set is the authority: after the
+      -- drain, pend and done may only hold ids that some surviving edge still references.
+      --
+      -- Two reasons this is a superset of "remove the drained ids". First, one waitpoint
+      -- can hold several edges at different batch indexes, so a drained field must not
+      -- evict a delivery another edge still needs. Second, runDeliverCompletion writes
+      -- done[id] unconditionally, so a crash between register and absorb can leave a
+      -- delivered entry with no edge at all, which no name-derived drain could reach.
+      local remaining = {}
+      local fields = redis.call('HKEYS', edge)
+      for i = 1, #fields do
+        local sep = string.find(fields[i], '#[^#]*$')
+        if sep then
+          remaining[string.sub(fields[i], 1, sep - 1)] = true
+        end
+      end
+
+      local doneIds = redis.call('HKEYS', done)
+      for i = 1, #doneIds do
+        if not remaining[doneIds[i]] then
+          redis.call('HDEL', done, doneIds[i])
+        end
+      end
+
+      local pendIds = redis.call('SMEMBERS', pend)
+      for i = 1, #pendIds do
+        if not remaining[pendIds[i]] then
+          redis.call('SREM', pend, pendIds[i])
+        end
+      end
+
+      return { '${DRAINED}' }
+    `,
+  });
+}
+
+declare module "@internal/redis" {
+  interface RedisCommander<Context> {
+    wpCreateIfAbsent(
+      recordKey: string,
+      recordJson: string,
+      status: string,
+      completionJson: string,
+      callback?: Callback<string[]>
+    ): Result<string[], Context>;
+    wpRegisterOrReport(
+      recordKey: string,
+      watchersKey: string,
+      watcherField: string,
+      watcherJson: string,
+      callback?: Callback<string[]>
+    ): Result<string[], Context>;
+    wpComplete(
+      recordKey: string,
+      watchersKey: string,
+      completionJson: string,
+      callback?: Callback<string[]>
+    ): Result<string[], Context>;
+    wpIdemReserve(
+      key: string,
+      waitpointId: string,
+      expiresAtMs: string,
+      callback?: Callback<string[]>
+    ): Result<string[], Context>;
+    runAbsorbBlockers(
+      pendKey: string,
+      doneKey: string,
+      edgeKey: string,
+      ...args: Array<string | Callback<string[]>>
+    ): Result<string[], Context>;
+    runDeliverCompletion(
+      pendKey: string,
+      doneKey: string,
+      waitpointId: string,
+      completionJson: string,
+      callback?: Callback<string[]>
+    ): Result<string[], Context>;
+    runReadBlockState(
+      pendKey: string,
+      doneKey: string,
+      edgeKey: string,
+      callback?: Callback<string[]>
+    ): Result<string[], Context>;
+    runClear(
+      pendKey: string,
+      doneKey: string,
+      edgeKey: string,
+      ...args: Array<string | Callback<string[]>>
+    ): Result<string[], Context>;
+  }
+}
diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts
new file mode 100644
index 00000000000..9ec5e73c723
--- /dev/null
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts
@@ -0,0 +1,340 @@
+// Redis-only suite: the coordinator holds no Prisma reference, so no Postgres container
+// is needed. redisTest FLUSHALLs before every test, so ids may be reused across describes.
+import { createRedisClient, type RedisOptions } from "@internal/redis";
+import { redisTest } from "@internal/testcontainers";
+import { describe, expect } from "vitest";
+import { WaitpointKeyTagError } from "./keys.js";
+import {
+  WaitpointNotFoundError,
+  WaitpointStoreCoordinator,
+  type WaitpointCompletion,
+  type WaitpointRecordInput,
+} from "./storeCoordinator.js";
+
+const ENV_ID = "env_1";
+const PROJECT_ID = "proj_1";
+const NOW = "2026-08-21T12:00:00.000Z";
+
+function coordinator(redisOptions: RedisOptions) {
+  return new WaitpointStoreCoordinator({ redisOptions });
+}
+
+function record(id: string, overrides: Partial<WaitpointRecordInput> = {}): WaitpointRecordInput {
+  return {
+    id,
+    friendlyId: `waitpoint_${id}`,
+    type: "MANUAL",
+    environmentId: ENV_ID,
+    projectId: PROJECT_ID,
+    createdAt: NOW,
+    updatedAt: NOW,
+    userProvidedIdempotencyKey: false,
+    tags: [],
+    ...overrides,
+  };
+}
+
+function completion(overrides: Partial<WaitpointCompletion> = {}): WaitpointCompletion {
+  return {
+    completedAt: NOW,
+    outputType: "application/json",
+    outputIsError: false,
+    output: { inline: '{"ok":true}' },
+    ...overrides,
+  };
+}
+
+describe("createIfAbsent", () => {
+  redisTest("creates a PENDING record and reports created", async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    try {
+      const result = await store.createIfAbsent({ record: record("w_a"), status: "PENDING" });
+      expect(result.outcome).toBe("created");
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest("returns the existing record on a second call", async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    try {
+      await store.createIfAbsent({ record: record("w_a"), status: "PENDING" });
+      const second = await store.createIfAbsent({
+        record: record("w_a", { friendlyId: "waitpoint_DIFFERENT" }),
+        status: "PENDING",
+      });
+
+      expect(second.outcome).toBe("exists");
+      if (second.outcome !== "exists") throw new Error("unreachable");
+      // The first write wins: a retry must not overwrite the stored record.
+      expect(second.record.friendlyId).toBe("waitpoint_w_a");
+      expect(second.status).toBe("PENDING");
+      expect(second.completion).toBeUndefined();
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest("preserves every record field through a round trip", async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    try {
+      const full = record("w_a", {
+        type: "RUN",
+        idempotencyKey: "key-1",
+        userProvidedIdempotencyKey: true,
+        idempotencyKeyExpiresAt: NOW,
+        completedAfter: NOW,
+        completedByTaskRunId: "run_child",
+        completedByBatchId: "batch_1",
+        tags: ["one", "two"],
+      });
+
+      await store.createIfAbsent({ record: full, status: "PENDING" });
+      const read = await store.createIfAbsent({ record: record("w_a"), status: "PENDING" });
+
+      expect(read.outcome).toBe("exists");
+      if (read.outcome !== "exists") throw new Error("unreachable");
+      // Every field the frozen return shapes need must survive the blob round trip.
+      expect(read.record).toEqual(full);
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest(
+    "can create an already-COMPLETED record with no completion envelope",
+    async ({ redisOptions }) => {
+      const store = coordinator(redisOptions);
+      try {
+        // This is the shape that catches a status-casing mismatch: the record is stored
+        // COMPLETED, and a register must see it as completed rather than pending.
+        await store.createIfAbsent({ record: record("w_a"), status: "COMPLETED" });
+
+        const reported = await store.registerOrReport({
+          waitpointId: "w_a",
+          runId: "run_1",
+          createdAt: NOW,
+        });
+
+        expect(reported.outcome).toBe("completed");
+        if (reported.outcome !== "completed") throw new Error("unreachable");
+        expect(reported.completion).toBeUndefined();
+      } finally {
+        await store.quit();
+      }
+    }
+  );
+
+  redisTest(
+    "can create an already-COMPLETED record with a completion",
+    async ({ redisOptions }) => {
+      const store = coordinator(redisOptions);
+      try {
+        await store.createIfAbsent({
+          record: record("w_a", { type: "RUN" }),
+          status: "COMPLETED",
+          completion: completion(),
+        });
+
+        const reported = await store.registerOrReport({
+          waitpointId: "w_a",
+          runId: "run_1",
+          createdAt: NOW,
+        });
+
+        expect(reported.outcome).toBe("completed");
+        if (reported.outcome !== "completed") throw new Error("unreachable");
+        expect(reported.completion?.output).toEqual({ inline: '{"ok":true}' });
+      } finally {
+        await store.quit();
+      }
+    }
+  );
+});
+
+describe("registerOrReport", () => {
+  redisTest("registers a watcher against a PENDING waitpoint", async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    try {
+      await store.createIfAbsent({ record: record("w_a"), status: "PENDING" });
+      const result = await store.registerOrReport({
+        waitpointId: "w_a",
+        runId: "run_1",
+        createdAt: NOW,
+      });
+      expect(result.outcome).toBe("registered");
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest("reports the completion inline for a COMPLETED waitpoint", async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    try {
+      await store.createIfAbsent({ record: record("w_a"), status: "PENDING" });
+      await store.complete({ waitpointId: "w_a", completion: completion() });
+
+      const result = await store.registerOrReport({
+        waitpointId: "w_a",
+        runId: "run_1",
+        createdAt: NOW,
+      });
+
+      expect(result.outcome).toBe("completed");
+      if (result.outcome !== "completed") throw new Error("unreachable");
+      expect(result.completion?.output).toEqual({ inline: '{"ok":true}' });
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest("throws for a waitpoint that does not exist", async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    try {
+      await expect(
+        store.registerOrReport({ waitpointId: "w_missing", runId: "run_1", createdAt: NOW })
+      ).rejects.toThrow(WaitpointNotFoundError);
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest("keeps one watcher entry per batch index", async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    try {
+      await store.createIfAbsent({ record: record("w_a"), status: "PENDING" });
+      await store.registerOrReport({
+        waitpointId: "w_a",
+        runId: "run_1",
+        batchIndex: 0,
+        createdAt: NOW,
+      });
+      await store.registerOrReport({
+        waitpointId: "w_a",
+        runId: "run_1",
+        batchIndex: 2,
+        createdAt: NOW,
+      });
+
+      const completed = await store.complete({ waitpointId: "w_a", completion: completion() });
+      expect(completed.watchers).toHaveLength(2);
+      expect(completed.watchers.map((w) => w.batchIndex).sort()).toEqual([0, 2]);
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest("carries spanIdToComplete through to the watcher entry", async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    try {
+      await store.createIfAbsent({ record: record("w_a"), status: "PENDING" });
+      await store.registerOrReport({
+        waitpointId: "w_a",
+        runId: "run_1",
+        spanIdToComplete: "span_abc",
+        createdAt: NOW,
+      });
+
+      const completed = await store.complete({ waitpointId: "w_a", completion: completion() });
+      expect(completed.watchers[0]!.spanIdToComplete).toBe("span_abc");
+      expect(completed.watchers[0]!.runId).toBe("run_1");
+      expect(completed.watchers[0]!.createdAt).toBe(NOW);
+    } finally {
+      await store.quit();
+    }
+  });
+});
+
+describe("complete", () => {
+  redisTest("flips PENDING to COMPLETED and returns the watchers", async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    try {
+      await store.createIfAbsent({ record: record("w_a"), status: "PENDING" });
+      await store.registerOrReport({ waitpointId: "w_a", runId: "run_1", createdAt: NOW });
+      await store.registerOrReport({ waitpointId: "w_a", runId: "run_2", createdAt: NOW });
+
+      const result = await store.complete({ waitpointId: "w_a", completion: completion() });
+
+      expect(result.outcome).toBe("completed");
+      expect(result.watchers.map((w) => w.runId).sort()).toEqual(["run_1", "run_2"]);
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest("is idempotent and returns the watchers again", async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    try {
+      await store.createIfAbsent({ record: record("w_a"), status: "PENDING" });
+      await store.registerOrReport({ waitpointId: "w_a", runId: "run_1", createdAt: NOW });
+
+      const first = await store.complete({ waitpointId: "w_a", completion: completion() });
+      const second = await store.complete({
+        waitpointId: "w_a",
+        completion: completion({ output: { inline: '{"second":true}' } }),
+      });
+
+      expect(first.outcome).toBe("completed");
+      expect(second.outcome).toBe("already");
+      // The FIRST completion wins, matching the guard on status = PENDING.
+      expect(second.completion?.output).toEqual({ inline: '{"ok":true}' });
+      expect(second.watchers.map((w) => w.runId)).toEqual(["run_1"]);
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest("throws for a waitpoint that does not exist", async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    try {
+      await expect(
+        store.complete({ waitpointId: "w_missing", completion: completion() })
+      ).rejects.toThrow(WaitpointNotFoundError);
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest("returns an empty watcher list when nobody is blocked", async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    try {
+      await store.createIfAbsent({ record: record("w_a"), status: "PENDING" });
+      const result = await store.complete({ waitpointId: "w_a", completion: completion() });
+      expect(result.watchers).toEqual([]);
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest("sets no TTL on the record or the watcher key", async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    const probe = createRedisClient(redisOptions);
+    try {
+      await store.createIfAbsent({ record: record("w_a"), status: "PENDING" });
+      await store.registerOrReport({ waitpointId: "w_a", runId: "run_1", createdAt: NOW });
+      await store.complete({ waitpointId: "w_a", completion: completion() });
+
+      // -1 means the key exists with no expiry. Anything >= 0 breaks the retention rule.
+      expect(await probe.pttl("wp:{w_a}")).toBe(-1);
+      expect(await probe.pttl("wp:{w_a}:w")).toBe(-1);
+    } finally {
+      probe.disconnect();
+      await store.quit();
+    }
+  });
+});
+
+describe("the single-slot guard", () => {
+  redisTest("rejects an invocation whose keys span two tags", async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    try {
+      // Reaches the same wrapper every operation goes through, so this proves the guard
+      // is live at the call path and not only in the pure unit test.
+      expect(() =>
+        store.assertKeysForTest("wpComplete", ["wp:{w_a}", "wp:run:{run_1}:pend"])
+      ).toThrow(WaitpointKeyTagError);
+    } finally {
+      await store.quit();
+    }
+  });
+});
diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts
new file mode 100644
index 00000000000..f98436e6dd9
--- /dev/null
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts
@@ -0,0 +1,239 @@
+import { createRedisClient, type Redis, type RedisOptions } from "@internal/redis";
+import { Logger } from "@trigger.dev/core/logger";
+import { assertSingleSlot, waitpointKeys, watcherField } from "./keys.js";
+import { registerWaitpointCommands } from "./scripts.js";
+
+/** The values written into a record's `status` field. Uppercase, and never a token. */
+export type WaitpointStatus = "PENDING" | "COMPLETED";
+
+/** Every script this coordinator may invoke. The wrapper below is the only entry point. */
+type ScriptName =
+  | "wpCreateIfAbsent"
+  | "wpRegisterOrReport"
+  | "wpComplete"
+  | "wpIdemReserve"
+  | "runAbsorbBlockers"
+  | "runDeliverCompletion"
+  | "runReadBlockState"
+  | "runClear";
+
+/**
+ * The immutable half of a waitpoint, written once at creation. Carries every field the
+ * legacy-shaped return types need, including the two that gate the executor-visible
+ * idempotency key and the token surface.
+ */
+export type WaitpointRecordInput = {
+  id: string;
+  friendlyId: string;
+  type: "RUN" | "BATCH" | "DATETIME" | "MANUAL";
+  environmentId: string;
+  projectId: string;
+  createdAt: string;
+  updatedAt: string;
+  userProvidedIdempotencyKey: boolean;
+  tags: string[];
+  idempotencyKey?: string;
+  idempotencyKeyExpiresAt?: string;
+  completedAfter?: string;
+  completedByTaskRunId?: string;
+  completedByBatchId?: string;
+};
+
+/**
+ * A stored output: a small inline value, an already-offloaded reference, or null when the
+ * value is re-derivable from a business fact and is therefore never copied forward.
+ */
+export type WaitpointCompletionOutput = { inline: string } | { ref: string } | null;
+
+/**
+ * The completion half of a waitpoint, written at the flip.
+ *
+ * This is the coordinator's OWN type, deliberately not a projection of any frozen record
+ * type. The store treats a completion as an opaque blob: it writes it, returns it, and
+ * never inspects a field. Whoever owns the read-time resolver maps between this and the
+ * frozen record shape, so the two can evolve without a type dependency in either
+ * direction.
+ */
+export type WaitpointCompletion = {
+  /** ISO 8601. */
+  completedAt: string;
+  outputType: string;
+  outputIsError: boolean;
+  output: WaitpointCompletionOutput;
+};
+
+export type WatcherEntry = {
+  runId: string;
+  batchIndex?: number;
+  spanIdToComplete?: string;
+  createdAt: string;
+};
+
+export type CreateIfAbsentResult =
+  | { outcome: "created" }
+  | {
+      outcome: "exists";
+      record: WaitpointRecordInput;
+      status: WaitpointStatus;
+      completion?: WaitpointCompletion;
+    };
+
+export type RegisterOrReportResult =
+  | { outcome: "registered" }
+  | { outcome: "completed"; completion?: WaitpointCompletion };
+
+export type CompleteResult = {
+  outcome: "completed" | "already";
+  completion?: WaitpointCompletion;
+  watchers: WatcherEntry[];
+};
+
+export class WaitpointNotFoundError extends Error {
+  constructor(waitpointId: string) {
+    super(`Waitpoint ${waitpointId} is not present in the store`);
+    this.name = "WaitpointNotFoundError";
+  }
+}
+
+export type WaitpointStoreCoordinatorOptions = {
+  redisOptions: RedisOptions;
+  logger?: Logger;
+};
+
+// Lua returns '' for an absent value, never nil, because every reply slot is coerced to
+// keep the array from truncating. So a nullish check would not fire and JSON.parse('')
+// throws. One helper, used at every decode site.
+function parseJson<T>(raw: string | undefined): T | undefined {
+  return raw ? (JSON.parse(raw) as T) : undefined;
+}
+
+export class WaitpointStoreCoordinator {
+  private readonly redis: Redis;
+  private readonly logger: Logger;
+  #quit?: Promise<void>;
+
+  constructor(options: WaitpointStoreCoordinatorOptions) {
+    this.logger = options.logger ?? new Logger("WaitpointStoreCoordinator", "debug");
+    this.redis = createRedisClient(options.redisOptions, {
+      onError: (error) =>
+        this.logger.error("WaitpointStoreCoordinator redis client error", { error }),
+    });
+    registerWaitpointCommands(this.redis);
+  }
+
+  // Idempotent and error-swallowing: every test calls this in a finally, and a double quit
+  // must never mask the real assertion failure.
+  async quit(): Promise<void> {
+    if (!this.#quit) {
+      this.#quit = this.redis.quit().then(
+        () => undefined,
+        () => undefined
+      );
+    }
+    await this.#quit;
+  }
+
+  /**
+   * The ONLY way this class invokes a script. Routing every call through one place is what
+   * makes the single-slot guard un-forgettable: a method added later cannot reach a script
+   * without passing its keys through this assertion.
+   *
+   * Every script's signature is (...keys, ...argv) => string[], so one cast covers them
+   * all. The typed RedisCommander augmentation in scripts.ts documents each shape.
+   */
+  #call(script: ScriptName, keys: string[], ...argv: string[]): Promise<string[]> {
+    assertSingleSlot(script, keys);
+    const command = this.redis[script] as (...args: string[]) => Promise<string[]>;
+    return command.call(this.redis, ...keys, ...argv);
+  }
+
+  /** Exposed for the guard's own test. Asserts and returns; never invokes a script. */
+  assertKeysForTest(operation: string, keys: string[]): void {
+    assertSingleSlot(operation, keys);
+  }
+
+  async createIfAbsent(args: {
+    record: WaitpointRecordInput;
+    status: WaitpointStatus;
+    completion?: WaitpointCompletion;
+  }): Promise<CreateIfAbsentResult> {
+    const keys = waitpointKeys(args.record.id);
+
+    const reply = await this.#call(
+      "wpCreateIfAbsent",
+      [keys.record],
+      JSON.stringify(args.record),
+      args.status,
+      args.completion ? JSON.stringify(args.completion) : ""
+    );
+
+    if (reply[0] === "created") {
+      return { outcome: "created" };
+    }
+
+    return {
+      outcome: "exists",
+      record: JSON.parse(reply[1] ?? "{}") as WaitpointRecordInput,
+      status: (reply[2] ?? "PENDING") as WaitpointStatus,
+      completion: parseJson<WaitpointCompletion>(reply[3]),
+    };
+  }
+
+  async registerOrReport(args: {
+    waitpointId: string;
+    runId: string;
+    batchIndex?: number | null;
+    spanIdToComplete?: string;
+    createdAt: string;
+  }): Promise<RegisterOrReportResult> {
+    const keys = waitpointKeys(args.waitpointId);
+
+    // batchIndex is nullable at the boundary (matching the column) and undefined inside,
+    // because JSON.stringify drops an undefined field but keeps a null one.
+    const watcher: WatcherEntry = {
+      runId: args.runId,
+      batchIndex: args.batchIndex ?? undefined,
+      spanIdToComplete: args.spanIdToComplete,
+      createdAt: args.createdAt,
+    };
+
+    const reply = await this.#call(
+      "wpRegisterOrReport",
+      [keys.record, keys.watchers],
+      watcherField(args.runId, args.batchIndex),
+      JSON.stringify(watcher)
+    );
+
+    if (reply[0] === "missing") {
+      throw new WaitpointNotFoundError(args.waitpointId);
+    }
+    if (reply[0] === "completed") {
+      return { outcome: "completed", completion: parseJson<WaitpointCompletion>(reply[1]) };
+    }
+
+    return { outcome: "registered" };
+  }
+
+  async complete(args: {
+    waitpointId: string;
+    completion: WaitpointCompletion;
+  }): Promise<CompleteResult> {
+    const keys = waitpointKeys(args.waitpointId);
+
+    const reply = await this.#call(
+      "wpComplete",
+      [keys.record, keys.watchers],
+      JSON.stringify(args.completion)
+    );
+
+    if (reply[0] === "missing") {
+      throw new WaitpointNotFoundError(args.waitpointId);
+    }
+
+    return {
+      outcome: reply[0] as "completed" | "already",
+      completion: parseJson<WaitpointCompletion>(reply[1]),
+      watchers: reply.slice(2).map((entry) => JSON.parse(entry) as WatcherEntry),
+    };
+  }
+}

From 72062a838f890c4776ce85f0fde00136ef5f216d Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Fri, 21 Aug 2026 19:34:19 +0100
Subject: [PATCH 26/49] fix(run-engine): order-independent absorb count, arity
 guards, and guard-test coverage
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- runAbsorbBlockers now computes pendingOfRequested once after every write in the
  batch lands, as distinct requested ids with no entry in done, instead of
  incrementing during the loop — the incremental count was order-dependent and could
  report a waitpoint as both pending and delivered.
- runAbsorbBlockers, runClear and wpIdemReserve reject a bad arity/expiry before their
  first write, so a caller mistake cannot half-apply a script.
- wpRegisterOrReport now uses HSETNX for the watcher write, matching the edge's
  ON CONFLICT DO NOTHING semantics: the first registration wins.
- assertKeysForTest now delegates through the private #call funnel instead of calling
  assertSingleSlot directly, so its own test fails if the guard inside #call is ever
  removed.
- createIfAbsent decodes the record and status fields explicitly instead of relying on
  ?? against a Lua '' sentinel, and throws a diagnosable error naming the waitpoint id
  if the record blob is unexpectedly missing.
- Adds direct-Lua coverage for the two straddle orderings and the three arity guards,
  a decode-correctness test for an absent completion field, and fixes a lexicographic
  sort in an existing assertion.
---
 .../engine/waitpointCoordinator/scripts.ts    |  45 ++++-
 .../storeCoordinator.test.ts                  | 174 +++++++++++++++++-
 .../waitpointCoordinator/storeCoordinator.ts  |  22 ++-
 3 files changed, 224 insertions(+), 17 deletions(-)

diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/scripts.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/scripts.ts
index 1e60f6516de..96492494541 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/scripts.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/scripts.ts
@@ -72,7 +72,9 @@ export function registerWaitpointCommands(redis: Redis): void {
       -- The watcher lands before any flip can read the watcher hash, because this script
       -- and wpComplete are both atomic on this same shard. So a register either appears
       -- in the flip's watcher list, or it observes COMPLETED above.
-      redis.call('HSET', watchers, ARGV[1], ARGV[2])
+      --
+      -- HSETNX: the first registration wins, mirroring the edge's ON CONFLICT DO NOTHING.
+      redis.call('HSETNX', watchers, ARGV[1], ARGV[2])
       return { '${REGISTERED}' }
     `,
   });
@@ -115,6 +117,12 @@ export function registerWaitpointCommands(redis: Redis): void {
     lua: `
       local key = KEYS[1]
 
+      -- Guard before the SET: a non-numeric expiry must not land a reservation that can
+      -- never expire because PEXPIREAT then errors out after the write already happened.
+      if ARGV[2] ~= '' and tonumber(ARGV[2]) == nil then
+        return redis.error_reply('wpIdemReserve: ARGV[2] must be numeric or empty')
+      end
+
       -- SET NX returns a status reply on success and false on conflict.
       if redis.call('SET', key, ARGV[1], 'NX') then
         -- Expiry only when the caller has one. A reservation with no expiry is the common
@@ -137,12 +145,16 @@ export function registerWaitpointCommands(redis: Redis): void {
       local pend, done, edge = KEYS[1], KEYS[2], KEYS[3]
       local n = tonumber(ARGV[1])
 
-      -- countedPending and seenDelivered make both outputs DISTINCT BY ID. The count this
-      -- replaces was a COUNT(*) over waitpoint rows, so two edges for one waitpoint
-      -- contributed one. Counting per edge would inflate it.
-      local countedPending = {}
+      -- Guard before any write: a wrong n must not half-apply the script. HDEL/HSETNX below
+      -- are irreversible mid-script, and Redis does not roll back a script that errors.
+      if #ARGV ~= 1 + n * 4 then
+        return redis.error_reply('runAbsorbBlockers: arity mismatch')
+      end
+
+      -- seenDelivered makes the delivered-pair output DISTINCT BY ID: two edges for one
+      -- waitpoint must contribute one pair, not two.
+      local requestedIds = {}
       local seenDelivered = {}
-      local pendingOfRequested = 0
       local out = { '0', '0' }
 
       for i = 0, n - 1 do
@@ -154,6 +166,7 @@ export function registerWaitpointCommands(redis: Redis): void {
         -- HSETNX is the ON CONFLICT DO NOTHING of the edge write: a retry must not
         -- overwrite the first attempt's metadata.
         redis.call('HSETNX', edge, field, edgeJson)
+        requestedIds[id] = true
 
         if reported ~= '' then
           -- Already COMPLETED when the watcher registered. It never becomes pending.
@@ -176,14 +189,21 @@ export function registerWaitpointCommands(redis: Redis): void {
             end
           else
             redis.call('SADD', pend, id)
-            if not countedPending[id] then
-              countedPending[id] = true
-              pendingOfRequested = pendingOfRequested + 1
-            end
           end
         end
       end
 
+      -- Computed AFTER every write in this batch, as the count of distinct requested ids
+      -- with no entry in done. Counting incrementally during the loop is order-dependent:
+      -- a later group's completion for an id already counted as pending would leave the
+      -- count stale, reporting a waitpoint as both pending and delivered.
+      local pendingOfRequested = 0
+      for id in pairs(requestedIds) do
+        if redis.call('HEXISTS', done, id) == 0 then
+          pendingOfRequested = pendingOfRequested + 1
+        end
+      end
+
       out[1] = tostring(pendingOfRequested)
       out[2] = tostring(redis.call('SCARD', pend))
       return out
@@ -234,6 +254,11 @@ export function registerWaitpointCommands(redis: Redis): void {
       local pend, done, edge = KEYS[1], KEYS[2], KEYS[3]
       local n = tonumber(ARGV[1])
 
+      -- Guard before any write, same reasoning as runAbsorbBlockers.
+      if #ARGV ~= 1 + n then
+        return redis.error_reply('runClear: arity mismatch')
+      end
+
       if n == 0 then
         redis.call('DEL', pend, done, edge)
         return { '${CLEARED}' }
diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts
index 9ec5e73c723..0ad4f6e0f24 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts
@@ -3,12 +3,20 @@
 import { createRedisClient, type RedisOptions } from "@internal/redis";
 import { redisTest } from "@internal/testcontainers";
 import { describe, expect } from "vitest";
-import { WaitpointKeyTagError } from "./keys.js";
+import {
+  edgeField,
+  idempotencyKey,
+  runBlockKeys,
+  watcherField,
+  WaitpointKeyTagError,
+} from "./keys.js";
+import { registerWaitpointCommands } from "./scripts.js";
 import {
   WaitpointNotFoundError,
   WaitpointStoreCoordinator,
   type WaitpointCompletion,
   type WaitpointRecordInput,
+  type WatcherEntry,
 } from "./storeCoordinator.js";
 
 const ENV_ID = "env_1";
@@ -218,7 +226,7 @@ describe("registerOrReport", () => {
 
       const completed = await store.complete({ waitpointId: "w_a", completion: completion() });
       expect(completed.watchers).toHaveLength(2);
-      expect(completed.watchers.map((w) => w.batchIndex).sort()).toEqual([0, 2]);
+      expect(completed.watchers.map((w) => w.batchIndex).sort((a, b) => a! - b!)).toEqual([0, 2]);
     } finally {
       await store.quit();
     }
@@ -306,6 +314,33 @@ describe("complete", () => {
     }
   });
 
+  redisTest(
+    "keeps the watcher list intact when the completion field is absent on an already-completed record",
+    async ({ redisOptions }) => {
+      const store = coordinator(redisOptions);
+      const probe = createRedisClient(redisOptions);
+      try {
+        // registerOrReport never lets a watcher land once status is COMPLETED, so this
+        // shape is forced by hand: it pins that an absent 'c' field decodes to an
+        // undefined completion without disturbing the watchers that follow it in the
+        // reply array.
+        await store.createIfAbsent({ record: record("w_a"), status: "COMPLETED" });
+        const watcher: WatcherEntry = { runId: "run_1", createdAt: NOW };
+        await probe.hset("wp:{w_a}:w", watcherField("run_1"), JSON.stringify(watcher));
+
+        const result = await store.complete({ waitpointId: "w_a", completion: completion() });
+
+        expect(result.outcome).toBe("already");
+        expect(result.completion).toBeUndefined();
+        expect(result.watchers).toHaveLength(1);
+        expect(result.watchers[0]!.runId).toBe("run_1");
+      } finally {
+        probe.disconnect();
+        await store.quit();
+      }
+    }
+  );
+
   redisTest("sets no TTL on the record or the watcher key", async ({ redisOptions }) => {
     const store = coordinator(redisOptions);
     const probe = createRedisClient(redisOptions);
@@ -324,6 +359,141 @@ describe("complete", () => {
   });
 });
 
+// No coordinator method calls runAbsorbBlockers/runClear/wpIdemReserve yet — a later task
+// wires those in. Registered directly on a raw client so the Lua itself is exercised now.
+describe("runAbsorbBlockers (direct Lua)", () => {
+  const envelope = JSON.stringify(completion());
+
+  redisTest(
+    "does not double-count a waitpoint reported pending then delivered in the same batch",
+    async ({ redisOptions }) => {
+      const client = createRedisClient(redisOptions);
+      registerWaitpointCommands(client);
+      try {
+        const keys = runBlockKeys("run_1");
+        const fieldA = edgeField("w_solo", 0);
+        const fieldB = edgeField("w_solo", 1);
+
+        // Group 0 arrives unreported (still pending); group 1 for the SAME waitpoint
+        // arrives already reported. This is the straddle that broke pendingOfRequested.
+        const reply = await client.runAbsorbBlockers(
+          keys.pend,
+          keys.done,
+          keys.edge,
+          "2",
+          "w_solo",
+          fieldA,
+          "{}",
+          "",
+          "w_solo",
+          fieldB,
+          "{}",
+          envelope
+        );
+
+        expect(reply).toEqual(["0", "0", "w_solo", envelope]);
+        expect(await client.scard(keys.pend)).toBe(0);
+      } finally {
+        client.disconnect();
+      }
+    }
+  );
+
+  redisTest(
+    "produces the identical result when the same two groups arrive in reverse order",
+    async ({ redisOptions }) => {
+      const client = createRedisClient(redisOptions);
+      registerWaitpointCommands(client);
+      try {
+        const keys = runBlockKeys("run_1");
+        const fieldA = edgeField("w_solo", 0);
+        const fieldB = edgeField("w_solo", 1);
+
+        const reply = await client.runAbsorbBlockers(
+          keys.pend,
+          keys.done,
+          keys.edge,
+          "2",
+          "w_solo",
+          fieldB,
+          "{}",
+          envelope,
+          "w_solo",
+          fieldA,
+          "{}",
+          ""
+        );
+
+        expect(reply).toEqual(["0", "0", "w_solo", envelope]);
+        expect(await client.scard(keys.pend)).toBe(0);
+      } finally {
+        client.disconnect();
+      }
+    }
+  );
+
+  redisTest("rejects an arity mismatch before writing anything", async ({ redisOptions }) => {
+    const client = createRedisClient(redisOptions);
+    registerWaitpointCommands(client);
+    try {
+      const keys = runBlockKeys("run_1");
+      const field = edgeField("w_solo", 0);
+
+      // n says 2 groups but only one group (4 ARGV entries) is supplied.
+      await expect(
+        client.runAbsorbBlockers(keys.pend, keys.done, keys.edge, "2", "w_solo", field, "{}", "")
+      ).rejects.toThrow();
+
+      expect(await client.exists(keys.pend)).toBe(0);
+      expect(await client.exists(keys.done)).toBe(0);
+      expect(await client.exists(keys.edge)).toBe(0);
+    } finally {
+      client.disconnect();
+    }
+  });
+
+  redisTest(
+    "runClear rejects an arity mismatch before writing anything",
+    async ({ redisOptions }) => {
+      const client = createRedisClient(redisOptions);
+      registerWaitpointCommands(client);
+      try {
+        const keys = runBlockKeys("run_1");
+        const field = edgeField("w_solo", 0);
+        await client.hset(keys.edge, field, "{}");
+        await client.sadd(keys.pend, "w_solo");
+
+        // n says 2 fields but only one field is supplied.
+        await expect(
+          client.runClear(keys.pend, keys.done, keys.edge, "2", field)
+        ).rejects.toThrow();
+
+        expect(await client.hexists(keys.edge, field)).toBe(1);
+        expect(await client.sismember(keys.pend, "w_solo")).toBe(1);
+      } finally {
+        client.disconnect();
+      }
+    }
+  );
+
+  redisTest(
+    "wpIdemReserve rejects a non-numeric expiry and does not create the reservation",
+    async ({ redisOptions }) => {
+      const client = createRedisClient(redisOptions);
+      registerWaitpointCommands(client);
+      try {
+        const key = idempotencyKey(ENV_ID, "key-1");
+
+        await expect(client.wpIdemReserve(key, "w_a", "not-a-number")).rejects.toThrow();
+
+        expect(await client.exists(key)).toBe(0);
+      } finally {
+        client.disconnect();
+      }
+    }
+  );
+});
+
 describe("the single-slot guard", () => {
   redisTest("rejects an invocation whose keys span two tags", async ({ redisOptions }) => {
     const store = coordinator(redisOptions);
diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts
index f98436e6dd9..b8d4f81786f 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts
@@ -147,9 +147,13 @@ export class WaitpointStoreCoordinator {
     return command.call(this.redis, ...keys, ...argv);
   }
 
-  /** Exposed for the guard's own test. Asserts and returns; never invokes a script. */
-  assertKeysForTest(operation: string, keys: string[]): void {
-    assertSingleSlot(operation, keys);
+  /**
+   * Exposed for the guard's own test. Delegates through #call rather than calling
+   * assertSingleSlot directly, so a mutation to the guard inside #call fails this test too
+   * — not only the tests that happen to exercise a real script.
+   */
+  assertKeysForTest(operation: string, keys: string[]) {
+    return this.#call(operation as ScriptName, keys);
   }
 
   async createIfAbsent(args: {
@@ -171,10 +175,18 @@ export class WaitpointStoreCoordinator {
       return { outcome: "created" };
     }
 
+    // reply[1] is '' only if the record hash exists with no 'r' field, which should never
+    // happen — but ?? never fires on '', so a bare JSON.parse('') would throw an
+    // undiagnosable SyntaxError instead of naming the waitpoint.
+    const record = parseJson<WaitpointRecordInput>(reply[1]);
+    if (!record) {
+      throw new Error(`Waitpoint ${args.record.id} exists in the store with no record blob`);
+    }
+
     return {
       outcome: "exists",
-      record: JSON.parse(reply[1] ?? "{}") as WaitpointRecordInput,
-      status: (reply[2] ?? "PENDING") as WaitpointStatus,
+      record,
+      status: reply[2] === "COMPLETED" ? "COMPLETED" : "PENDING",
       completion: parseJson<WaitpointCompletion>(reply[3]),
     };
   }

From 79d14ca14a9449c2870fd17a9b7fceabc4c4aa09 Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Fri, 21 Aug 2026 19:49:23 +0100
Subject: [PATCH 27/49] fix(run-engine): correct the Lua truncation claim and
 widen absorb coverage
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- scripts.ts's header rule 3 stated that a Lua false or nil truncates the reply
  array. Measured against a live Redis: a missing HGET returns Lua false, not nil,
  and false does not truncate anything after it — only a genuine Lua nil does. The
  or '' coercion exists to give absent values one decoded shape, not to prevent
  truncation. Corrected the comment so it states what was actually measured.
- Adds four more direct-Lua cases for runAbsorbBlockers's pendingOfRequested/pend
  count: two distinct unreported ids, one reported plus one unreported, the same
  unreported id passed twice, and an id already in done passed unreported. The
  round 1 fix only had straddle-ordering coverage; these round out the behaviour
  that was hand-verified but unguarded.
- assertKeysForTest still delegates through #call so a mutation removing the guard
  from #call fails its own test, but no longer returns or awaits that call's
  promise: a valid-key invocation's eventual settlement is swallowed instead of
  risking an unhandled rejection.
- Renames the direct-Lua describe block to name all three scripts it covers.
---
 .../engine/waitpointCoordinator/scripts.ts    |  10 +-
 .../storeCoordinator.test.ts                  | 122 +++++++++++++++++-
 .../waitpointCoordinator/storeCoordinator.ts  |  10 +-
 3 files changed, 137 insertions(+), 5 deletions(-)

diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/scripts.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/scripts.ts
index 96492494541..6818fd029f8 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/scripts.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/scripts.ts
@@ -9,8 +9,14 @@ import type { Callback, Redis, Result } from "@internal/redis";
  *     declared key gives the caller's single-slot assertion nothing to compare.
  *  2. Lua never parses JSON. Each script branches only on a short status string and moves
  *     opaque blobs, so every encoding decision stays in TypeScript.
- *  3. Every returned slot is coerced with `or ''`. A Lua false or nil TRUNCATES the reply
- *     array at that position, silently shortening it.
+ *  3. A missing HGET returns Lua `false`, not `nil` — measured directly against a live
+ *     Redis: `EVAL "return {'a', false, 'c'}"` and a table holding a missing-field HGET
+ *     result both come back as 3 elements; only `EVAL "return {'a', nil, 'c'}"` comes back
+ *     as 1. A `false` element converts to a reply-array null and does NOT shorten anything
+ *     after it — only a genuine Lua nil truncates. Every returned slot is still coerced
+ *     with `or ''` regardless, not to prevent truncation, but so an absent value arrives
+ *     as `''` rather than `null`, giving the TypeScript one shape to decode instead of
+ *     two.
  *
  * STORED_COMPLETED is the value written into the record's `status` field and is
  * UPPERCASE. The outcome tokens below are lowercase and are a separate vocabulary: they
diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts
index 0ad4f6e0f24..2a62b491a82 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts
@@ -361,7 +361,7 @@ describe("complete", () => {
 
 // No coordinator method calls runAbsorbBlockers/runClear/wpIdemReserve yet — a later task
 // wires those in. Registered directly on a raw client so the Lua itself is exercised now.
-describe("runAbsorbBlockers (direct Lua)", () => {
+describe("runAbsorbBlockers, runClear and wpIdemReserve (direct Lua)", () => {
   const envelope = JSON.stringify(completion());
 
   redisTest(
@@ -432,6 +432,126 @@ describe("runAbsorbBlockers (direct Lua)", () => {
     }
   );
 
+  redisTest("counts two distinct unreported ids as fully pending", async ({ redisOptions }) => {
+    const client = createRedisClient(redisOptions);
+    registerWaitpointCommands(client);
+    try {
+      const keys = runBlockKeys("run_1");
+
+      const reply = await client.runAbsorbBlockers(
+        keys.pend,
+        keys.done,
+        keys.edge,
+        "2",
+        "w_a",
+        edgeField("w_a", 0),
+        "{}",
+        "",
+        "w_b",
+        edgeField("w_b", 0),
+        "{}",
+        ""
+      );
+
+      expect(reply).toEqual(["2", "2"]);
+      expect(await client.scard(keys.pend)).toBe(2);
+    } finally {
+      client.disconnect();
+    }
+  });
+
+  redisTest(
+    "counts one reported and one unreported id as one pending, one delivered",
+    async ({ redisOptions }) => {
+      const client = createRedisClient(redisOptions);
+      registerWaitpointCommands(client);
+      try {
+        const keys = runBlockKeys("run_1");
+
+        const reply = await client.runAbsorbBlockers(
+          keys.pend,
+          keys.done,
+          keys.edge,
+          "2",
+          "w_a",
+          edgeField("w_a", 0),
+          "{}",
+          "",
+          "w_b",
+          edgeField("w_b", 0),
+          "{}",
+          envelope
+        );
+
+        expect(reply).toEqual(["1", "1", "w_b", envelope]);
+        expect(await client.scard(keys.pend)).toBe(1);
+      } finally {
+        client.disconnect();
+      }
+    }
+  );
+
+  redisTest(
+    "counts the same unreported id passed twice as one pending, not two",
+    async ({ redisOptions }) => {
+      const client = createRedisClient(redisOptions);
+      registerWaitpointCommands(client);
+      try {
+        const keys = runBlockKeys("run_1");
+
+        const reply = await client.runAbsorbBlockers(
+          keys.pend,
+          keys.done,
+          keys.edge,
+          "2",
+          "w_a",
+          edgeField("w_a", 0),
+          "{}",
+          "",
+          "w_a",
+          edgeField("w_a", 1),
+          "{}",
+          ""
+        );
+
+        expect(reply).toEqual(["1", "1"]);
+        expect(await client.scard(keys.pend)).toBe(1);
+      } finally {
+        client.disconnect();
+      }
+    }
+  );
+
+  redisTest(
+    "counts an id already in done, passed unreported, as delivered rather than pending",
+    async ({ redisOptions }) => {
+      const client = createRedisClient(redisOptions);
+      registerWaitpointCommands(client);
+      try {
+        const keys = runBlockKeys("run_1");
+        // A completion that landed between register and absorb — the delivered set
+        // already has this id before the absorb call ever sees it.
+        await client.hset(keys.done, "w_a", envelope);
+
+        const reply = await client.runAbsorbBlockers(
+          keys.pend,
+          keys.done,
+          keys.edge,
+          "1",
+          "w_a",
+          edgeField("w_a", 0),
+          "{}",
+          ""
+        );
+
+        expect(reply).toEqual(["0", "0", "w_a", envelope]);
+        expect(await client.scard(keys.pend)).toBe(0);
+      } finally {
+        client.disconnect();
+      }
+    }
+  );
+
   redisTest("rejects an arity mismatch before writing anything", async ({ redisOptions }) => {
     const client = createRedisClient(redisOptions);
     registerWaitpointCommands(client);
diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts
index b8d4f81786f..506849fd8a7 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts
@@ -151,9 +151,15 @@ export class WaitpointStoreCoordinator {
    * Exposed for the guard's own test. Delegates through #call rather than calling
    * assertSingleSlot directly, so a mutation to the guard inside #call fails this test too
    * — not only the tests that happen to exercise a real script.
+   *
+   * With cross-tag (invalid) keys, assertSingleSlot throws synchronously inside #call,
+   * before any promise exists, and that throw propagates straight out of this method. With
+   * same-tag (valid) keys, #call would go on to dispatch a real script call; this method
+   * never returns or awaits that promise, and swallows whatever it eventually settles to,
+   * so a valid-key call here can never surface as an unhandled rejection in the caller.
    */
-  assertKeysForTest(operation: string, keys: string[]) {
-    return this.#call(operation as ScriptName, keys);
+  assertKeysForTest(operation: string, keys: string[]): void {
+    this.#call(operation as ScriptName, keys).catch(() => undefined);
   }
 
   async createIfAbsent(args: {

From 6d624d89dd1ba9c12d67c4e34079c095a7d0a552 Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Fri, 21 Aug 2026 20:03:43 +0100
Subject: [PATCH 28/49] feat(run-engine): idempotency-keyed waitpoint creation,
 record before reservation

---
 .../engine/waitpointCoordinator/scripts.ts    |  15 +++
 .../storeCoordinator.test.ts                  | 105 ++++++++++++++++++
 .../waitpointCoordinator/storeCoordinator.ts  |  52 ++++++++-
 3 files changed, 171 insertions(+), 1 deletion(-)

diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/scripts.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/scripts.ts
index 6818fd029f8..5608b722699 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/scripts.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/scripts.ts
@@ -35,6 +35,7 @@ const ALREADY = "already";
 const RESERVED = "reserved";
 const CLEARED = "cleared";
 const DRAINED = "drained";
+const DISCARDED = "discarded";
 
 export function registerWaitpointCommands(redis: Redis): void {
   // KEYS: record. ARGV: recordJson, status ('PENDING'|'COMPLETED'), completionJson ('').
@@ -143,6 +144,15 @@ export function registerWaitpointCommands(redis: Redis): void {
     `,
   });
 
+  // KEYS: record, watchers. No ARGV. Discards a losing reservation's orphan record.
+  redis.defineCommand("wpDiscard", {
+    numberOfKeys: 2,
+    lua: `
+      redis.call('DEL', KEYS[1], KEYS[2])
+      return { '${DISCARDED}' }
+    `,
+  });
+
   // KEYS: pend, done, edge.
   // ARGV: n, then n groups of 4 — waitpointId, edgeField, edgeJson, reportedJson ('').
   redis.defineCommand("runAbsorbBlockers", {
@@ -338,6 +348,11 @@ declare module "@internal/redis" {
       expiresAtMs: string,
       callback?: Callback<string[]>
     ): Result<string[], Context>;
+    wpDiscard(
+      recordKey: string,
+      watchersKey: string,
+      callback?: Callback<string[]>
+    ): Result<string[], Context>;
     runAbsorbBlockers(
       pendKey: string,
       doneKey: string,
diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts
index 2a62b491a82..b6aabb690a4 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts
@@ -614,6 +614,111 @@ describe("runAbsorbBlockers, runClear and wpIdemReserve (direct Lua)", () => {
   );
 });
 
+describe("createWithIdempotencyKey", () => {
+  redisTest("creates the waitpoint and wins the reservation", async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    try {
+      const result = await store.createWithIdempotencyKey({
+        record: record("w_a", { idempotencyKey: "key-1", userProvidedIdempotencyKey: true }),
+        environmentId: ENV_ID,
+        idempotencyKey: "key-1",
+      });
+
+      expect(result).toEqual({ waitpointId: "w_a", created: true });
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest("returns the winner's id and deletes the loser", async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    const probe = createRedisClient(redisOptions);
+    try {
+      await store.createWithIdempotencyKey({
+        record: record("w_first", { idempotencyKey: "key-1", userProvidedIdempotencyKey: true }),
+        environmentId: ENV_ID,
+        idempotencyKey: "key-1",
+      });
+
+      const second = await store.createWithIdempotencyKey({
+        record: record("w_second", { idempotencyKey: "key-1", userProvidedIdempotencyKey: true }),
+        environmentId: ENV_ID,
+        idempotencyKey: "key-1",
+      });
+
+      expect(second).toEqual({ waitpointId: "w_first", created: false });
+      // The loser cleans up after itself: nothing ever referenced its id.
+      expect(await probe.exists("wp:{w_second}")).toBe(0);
+      expect(await probe.exists("wp:{w_first}")).toBe(1);
+    } finally {
+      probe.disconnect();
+      await store.quit();
+    }
+  });
+
+  redisTest("sets no expiry when the record carries none", async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    const probe = createRedisClient(redisOptions);
+    try {
+      await store.createWithIdempotencyKey({
+        record: record("w_a", { idempotencyKey: "key-1", userProvidedIdempotencyKey: true }),
+        environmentId: ENV_ID,
+        idempotencyKey: "key-1",
+      });
+
+      // The common case. An expiry appearing here would be a retention rule violation.
+      expect(await probe.pttl(`wp:idem:{${ENV_ID}}:key-1`)).toBe(-1);
+    } finally {
+      probe.disconnect();
+      await store.quit();
+    }
+  });
+
+  redisTest("sets the expiry the record carries", async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    const probe = createRedisClient(redisOptions);
+    try {
+      await store.createWithIdempotencyKey({
+        record: record("w_a", {
+          idempotencyKey: "key-1",
+          userProvidedIdempotencyKey: true,
+          idempotencyKeyExpiresAt: new Date(Date.now() + 60_000).toISOString(),
+        }),
+        environmentId: ENV_ID,
+        idempotencyKey: "key-1",
+      });
+
+      const ttl = await probe.pttl(`wp:idem:{${ENV_ID}}:key-1`);
+      expect(ttl).toBeGreaterThan(0);
+      expect(ttl).toBeLessThanOrEqual(60_000);
+    } finally {
+      probe.disconnect();
+      await store.quit();
+    }
+  });
+
+  redisTest("scopes reservations by environment", async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    try {
+      await store.createWithIdempotencyKey({
+        record: record("w_a", { idempotencyKey: "key-1" }),
+        environmentId: "env_1",
+        idempotencyKey: "key-1",
+      });
+
+      const other = await store.createWithIdempotencyKey({
+        record: record("w_b", { idempotencyKey: "key-1", environmentId: "env_2" }),
+        environmentId: "env_2",
+        idempotencyKey: "key-1",
+      });
+
+      expect(other).toEqual({ waitpointId: "w_b", created: true });
+    } finally {
+      await store.quit();
+    }
+  });
+});
+
 describe("the single-slot guard", () => {
   redisTest("rejects an invocation whose keys span two tags", async ({ redisOptions }) => {
     const store = coordinator(redisOptions);
diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts
index 506849fd8a7..d2c3f0b50c9 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts
@@ -1,6 +1,6 @@
 import { createRedisClient, type Redis, type RedisOptions } from "@internal/redis";
 import { Logger } from "@trigger.dev/core/logger";
-import { assertSingleSlot, waitpointKeys, watcherField } from "./keys.js";
+import { assertSingleSlot, idempotencyKey, waitpointKeys, watcherField } from "./keys.js";
 import { registerWaitpointCommands } from "./scripts.js";
 
 /** The values written into a record's `status` field. Uppercase, and never a token. */
@@ -12,6 +12,7 @@ type ScriptName =
   | "wpRegisterOrReport"
   | "wpComplete"
   | "wpIdemReserve"
+  | "wpDiscard"
   | "runAbsorbBlockers"
   | "runDeliverCompletion"
   | "runReadBlockState"
@@ -254,4 +255,53 @@ export class WaitpointStoreCoordinator {
       watchers: reply.slice(2).map((entry) => JSON.parse(entry) as WatcherEntry),
     };
   }
+
+  /**
+   * Create a waitpoint under an idempotency key.
+   *
+   * The reservation and the record sit under different hash tags, so no script spans
+   * them. That makes the ORDER load-bearing: create first, then reserve.
+   *
+   * Reserve-first would mean a crash between the two steps leaves a reservation naming a
+   * waitpoint that does not exist. Every later request with that key loses the
+   * reservation, blocks on the winner's id, and throws when it registers — correctly, but
+   * forever, because an idempotency key commonly carries no expiry to clear it.
+   *
+   * Create-first inverts the failure: a crash leaves an orphan record that nothing ever
+   * referenced, because its id is random and unpublished. No caller hangs, and the orphan
+   * is reaped by the store's own garbage collection rather than by an expiry, which
+   * pending keys never carry.
+   */
+  async createWithIdempotencyKey(args: {
+    record: WaitpointRecordInput;
+    environmentId: string;
+    idempotencyKey: string;
+  }): Promise<{ waitpointId: string; created: boolean }> {
+    await this.createIfAbsent({ record: args.record, status: "PENDING" });
+
+    const expiresAtMs = args.record.idempotencyKeyExpiresAt
+      ? String(new Date(args.record.idempotencyKeyExpiresAt).getTime())
+      : "";
+
+    const reply = await this.#call(
+      "wpIdemReserve",
+      [idempotencyKey(args.environmentId, args.idempotencyKey)],
+      args.record.id,
+      expiresAtMs
+    );
+
+    if (reply[0] === "reserved") {
+      return { waitpointId: args.record.id, created: true };
+    }
+
+    const winner = reply[1] ?? args.record.id;
+    if (winner !== args.record.id) {
+      // Safe to discard: this id is random and was never handed to any caller, so no
+      // watcher can reference it. Both keys share the record's tag.
+      const keys = waitpointKeys(args.record.id);
+      await this.#call("wpDiscard", [keys.record, keys.watchers]);
+    }
+
+    return { waitpointId: winner, created: false };
+  }
 }

From 6aea717ade638e3b47176f35be5708b8d92adf8d Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Fri, 21 Aug 2026 20:14:07 +0100
Subject: [PATCH 29/49] fix(run-engine): tolerance-band expiry assertion,
 honest orphan comment, drop dead ?? on Lua reply

---
 .../waitpointCoordinator/storeCoordinator.test.ts     | 11 +++++++++--
 .../engine/waitpointCoordinator/storeCoordinator.ts   | 11 +++++++----
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts
index b6aabb690a4..23dca517a59 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts
@@ -689,8 +689,15 @@ describe("createWithIdempotencyKey", () => {
       });
 
       const ttl = await probe.pttl(`wp:idem:{${ENV_ID}}:key-1`);
-      expect(ttl).toBeGreaterThan(0);
-      expect(ttl).toBeLessThanOrEqual(60_000);
+      // Wide band, deliberately: the deadline is computed from the test process's clock
+      // and applied as an absolute PEXPIREAT, while PTTL is computed against the Redis
+      // server's own clock. A few ms of disagreement between those two clocks is normal
+      // and shows up as overshoot on this read, not as a bug in the reservation. The
+      // band still catches every failure worth catching — wrong units, no expiry
+      // applied, a negative TTL — without re-asserting that two independent clocks
+      // agree to the millisecond.
+      expect(ttl).toBeGreaterThan(55_000);
+      expect(ttl).toBeLessThanOrEqual(65_000);
     } finally {
       probe.disconnect();
       await store.quit();
diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts
index d2c3f0b50c9..0ef39b522df 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts
@@ -268,9 +268,12 @@ export class WaitpointStoreCoordinator {
    * forever, because an idempotency key commonly carries no expiry to clear it.
    *
    * Create-first inverts the failure: a crash leaves an orphan record that nothing ever
-   * referenced, because its id is random and unpublished. No caller hangs, and the orphan
-   * is reaped by the store's own garbage collection rather than by an expiry, which
-   * pending keys never carry.
+   * referenced, because its id is random and unpublished. No caller hangs, but nothing
+   * currently reclaims that record either: the backstop collector the wider plan
+   * describes is keyed off a run's status, and this orphan has no owning run, so that
+   * collector never sees it. The record is harmless — inert, unreferenced, never
+   * returned to anyone — but it is a real leak until a later ticket adds a reaper for
+   * standalone idempotency-keyed orphans specifically.
    */
   async createWithIdempotencyKey(args: {
     record: WaitpointRecordInput;
@@ -294,7 +297,7 @@ export class WaitpointStoreCoordinator {
       return { waitpointId: args.record.id, created: true };
     }
 
-    const winner = reply[1] ?? args.record.id;
+    const winner = reply[1];
     if (winner !== args.record.id) {
       // Safe to discard: this id is random and was never handed to any caller, so no
       // watcher can reference it. Both keys share the record's tag.

From 77f0e4dd2b7b4693e614eb09719f4e861be6fe24 Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Fri, 21 Aug 2026 20:23:10 +0100
Subject: [PATCH 30/49] feat(run-engine): run shard operations for absorb,
 deliver, read and clear

---
 .../storeCoordinator.test.ts                  | 378 ++++++++++++++++++
 .../waitpointCoordinator/storeCoordinator.ts  | 149 ++++++-
 2 files changed, 526 insertions(+), 1 deletion(-)

diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts
index 23dca517a59..0f951ed215f 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts
@@ -14,6 +14,7 @@ import { registerWaitpointCommands } from "./scripts.js";
 import {
   WaitpointNotFoundError,
   WaitpointStoreCoordinator,
+  type BlockEdge,
   type WaitpointCompletion,
   type WaitpointRecordInput,
   type WatcherEntry,
@@ -740,3 +741,380 @@ describe("the single-slot guard", () => {
     }
   });
 });
+
+const RUN_ID = "run_1";
+
+function edge(waitpointId: string, overrides: Partial<BlockEdge> = {}): BlockEdge {
+  return { waitpointId, createdAt: NOW, type: "MANUAL", ...overrides };
+}
+
+describe("absorbBlockers", () => {
+  redisTest("counts pending blockers and reports the store total", async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    try {
+      const result = await store.absorbBlockers({
+        runId: RUN_ID,
+        edges: [edge("w_a"), edge("w_b")],
+      });
+
+      expect(result.pendingOfRequested).toBe(2);
+      expect(result.storePendingTotal).toBe(2);
+      expect(result.alreadyDelivered).toEqual([]);
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest(
+    "counts a repeated waitpoint id once, matching a count over distinct rows",
+    async ({ redisOptions }) => {
+      const store = coordinator(redisOptions);
+      try {
+        const result = await store.absorbBlockers({
+          runId: RUN_ID,
+          edges: [edge("w_a", { batchIndex: 0 }), edge("w_a", { batchIndex: 2 })],
+        });
+
+        // The count this replaces was a COUNT(*) over waitpoint rows, so two edges for
+        // one waitpoint contributed one. Both numbers must say 1, not 2.
+        expect(result.pendingOfRequested).toBe(1);
+        expect(result.storePendingTotal).toBe(1);
+
+        // The edges themselves stay distinct — that multiplicity is what produces the
+        // repeats in the cycle's ordered id list.
+        const state = await store.readBlockState(RUN_ID);
+        expect(state.edges).toHaveLength(2);
+        expect(state.edges.map((e) => e.batchIndex).sort()).toEqual([0, 2]);
+      } finally {
+        await store.quit();
+      }
+    }
+  );
+
+  redisTest(
+    "does not add a reported-complete blocker to the pending set",
+    async ({ redisOptions }) => {
+      const store = coordinator(redisOptions);
+      try {
+        const result = await store.absorbBlockers({
+          runId: RUN_ID,
+          edges: [edge("w_a", { reported: completion() }), edge("w_b")],
+        });
+
+        expect(result.pendingOfRequested).toBe(1);
+        expect(result.storePendingTotal).toBe(1);
+        expect(result.alreadyDelivered.map((d) => d.waitpointId)).toEqual(["w_a"]);
+      } finally {
+        await store.quit();
+      }
+    }
+  );
+
+  redisTest("reports a repeated already-delivered id once", async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    try {
+      const result = await store.absorbBlockers({
+        runId: RUN_ID,
+        edges: [
+          edge("w_a", { batchIndex: 0, reported: completion() }),
+          edge("w_a", { batchIndex: 1, reported: completion() }),
+        ],
+      });
+
+      expect(result.alreadyDelivered).toHaveLength(1);
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest("lets a delivery that raced ahead of the absorb win", async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    try {
+      // The completion landed between register and absorb, so it is already delivered.
+      await store.deliverCompletion({
+        runId: RUN_ID,
+        waitpointId: "w_a",
+        completion: completion(),
+      });
+
+      const result = await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_a")] });
+
+      expect(result.pendingOfRequested).toBe(0);
+      expect(result.storePendingTotal).toBe(0);
+      expect(result.alreadyDelivered.map((d) => d.waitpointId)).toEqual(["w_a"]);
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest("is idempotent when run twice", async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    try {
+      const first = await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_a")] });
+      const second = await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_a")] });
+
+      expect(first.storePendingTotal).toBe(1);
+      expect(second.storePendingTotal).toBe(1);
+      expect((await store.readBlockState(RUN_ID)).edges).toHaveLength(1);
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest("keeps the first edge's metadata on a retry", async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    try {
+      await store.absorbBlockers({
+        runId: RUN_ID,
+        edges: [edge("w_a", { spanIdToComplete: "span_first" })],
+      });
+      await store.absorbBlockers({
+        runId: RUN_ID,
+        edges: [edge("w_a", { spanIdToComplete: "span_second" })],
+      });
+
+      expect((await store.readBlockState(RUN_ID)).edges[0]!.spanIdToComplete).toBe("span_first");
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest("reports the run's real total for an empty edge list", async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    try {
+      await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_a")] });
+
+      const result = await store.absorbBlockers({ runId: RUN_ID, edges: [] });
+
+      // pendingOfRequested is 0 because nothing was requested. storePendingTotal is the
+      // run's whole store-resident set, which is NOT empty.
+      expect(result.pendingOfRequested).toBe(0);
+      expect(result.storePendingTotal).toBe(1);
+      expect(result.alreadyDelivered).toEqual([]);
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest("sets no TTL on any run key", async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    const probe = createRedisClient(redisOptions);
+    try {
+      await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_a")] });
+      await store.deliverCompletion({
+        runId: RUN_ID,
+        waitpointId: "w_a",
+        completion: completion(),
+      });
+
+      // -1 is "exists, no expiry"; -2 is "no key". Neither is a TTL. `pend` is emptied by
+      // the delivery, and Redis deletes an empty set, so -2 is expected there.
+      for (const key of [
+        `wp:run:{${RUN_ID}}:pend`,
+        `wp:run:{${RUN_ID}}:done`,
+        `wp:run:{${RUN_ID}}:edge`,
+      ]) {
+        expect(await probe.pttl(key)).toBeLessThan(0);
+      }
+    } finally {
+      probe.disconnect();
+      await store.quit();
+    }
+  });
+});
+
+describe("deliverCompletion", () => {
+  redisTest("removes the blocker and returns the new store total", async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    try {
+      await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_a"), edge("w_b")] });
+
+      expect(
+        (
+          await store.deliverCompletion({
+            runId: RUN_ID,
+            waitpointId: "w_a",
+            completion: completion(),
+          })
+        ).storePendingTotal
+      ).toBe(1);
+
+      expect(
+        (
+          await store.deliverCompletion({
+            runId: RUN_ID,
+            waitpointId: "w_b",
+            completion: completion(),
+          })
+        ).storePendingTotal
+      ).toBe(0);
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest("is idempotent", async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    try {
+      await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_a")] });
+      await store.deliverCompletion({
+        runId: RUN_ID,
+        waitpointId: "w_a",
+        completion: completion(),
+      });
+      const again = await store.deliverCompletion({
+        runId: RUN_ID,
+        waitpointId: "w_a",
+        completion: completion(),
+      });
+
+      expect(again.storePendingTotal).toBe(0);
+    } finally {
+      await store.quit();
+    }
+  });
+});
+
+describe("readBlockState", () => {
+  redisTest(
+    "returns the pending ids, the delivered ids and the edges",
+    async ({ redisOptions }) => {
+      const store = coordinator(redisOptions);
+      try {
+        await store.absorbBlockers({
+          runId: RUN_ID,
+          edges: [
+            edge("w_a", { batchIndex: 0, completedAfter: NOW, type: "DATETIME" }),
+            edge("w_b"),
+          ],
+        });
+        await store.deliverCompletion({
+          runId: RUN_ID,
+          waitpointId: "w_a",
+          completion: completion(),
+        });
+
+        const state = await store.readBlockState(RUN_ID);
+
+        expect(state.pendingIds).toEqual(["w_b"]);
+        expect(state.deliveredIds).toEqual(["w_a"]);
+        expect(state.edges).toHaveLength(2);
+
+        const datetime = state.edges.find((e) => e.waitpointId === "w_a");
+        // type and completedAfter must ride the edge: a frozen return type needs them, and
+        // they live on the waitpoint's own shard, which this read cannot touch.
+        expect(datetime?.type).toBe("DATETIME");
+        expect(datetime?.completedAfter).toBe(NOW);
+        expect(datetime?.edgeId).toBe("w_a#0");
+      } finally {
+        await store.quit();
+      }
+    }
+  );
+
+  redisTest("returns empty collections for a run with no blockers", async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    try {
+      expect(await store.readBlockState("run_unknown")).toEqual({
+        pendingIds: [],
+        deliveredIds: [],
+        edges: [],
+      });
+    } finally {
+      await store.quit();
+    }
+  });
+});
+
+describe("clearBlockState", () => {
+  redisTest("drains the named edges and reconciles", async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    try {
+      await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_a"), edge("w_b")] });
+      await store.deliverCompletion({
+        runId: RUN_ID,
+        waitpointId: "w_a",
+        completion: completion(),
+      });
+
+      expect((await store.clearBlockState({ runId: RUN_ID, edgeIds: ["w_a#"] })).outcome).toBe(
+        "drained"
+      );
+
+      const state = await store.readBlockState(RUN_ID);
+      expect(state.edges.map((e) => e.waitpointId)).toEqual(["w_b"]);
+      expect(state.deliveredIds).toEqual([]);
+      expect(state.pendingIds).toEqual(["w_b"]);
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest(
+    "keeps a waitpoint's delivery while another edge for it survives",
+    async ({ redisOptions }) => {
+      const store = coordinator(redisOptions);
+      try {
+        await store.absorbBlockers({
+          runId: RUN_ID,
+          edges: [edge("w_a", { batchIndex: 0 }), edge("w_a", { batchIndex: 1 })],
+        });
+        await store.deliverCompletion({
+          runId: RUN_ID,
+          waitpointId: "w_a",
+          completion: completion(),
+        });
+
+        await store.clearBlockState({ runId: RUN_ID, edgeIds: ["w_a#0"] });
+
+        // One edge remains, so the delivery must remain too — dropping it would make the
+        // surviving edge look undelivered.
+        const state = await store.readBlockState(RUN_ID);
+        expect(state.edges.map((e) => e.edgeId)).toEqual(["w_a#1"]);
+        expect(state.deliveredIds).toEqual(["w_a"]);
+      } finally {
+        await store.quit();
+      }
+    }
+  );
+
+  redisTest("reaps a delivered entry that no edge references", async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    try {
+      // The register-before-absorb window: a delivery can land for a waitpoint whose edge
+      // was never written. A name-derived drain could never reach it.
+      await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_kept")] });
+      await store.deliverCompletion({
+        runId: RUN_ID,
+        waitpointId: "w_orphan",
+        completion: completion(),
+      });
+
+      expect((await store.readBlockState(RUN_ID)).deliveredIds).toEqual(["w_orphan"]);
+
+      await store.clearBlockState({ runId: RUN_ID, edgeIds: ["w_nothing#"] });
+
+      const state = await store.readBlockState(RUN_ID);
+      expect(state.deliveredIds).toEqual([]);
+      expect(state.edges.map((e) => e.waitpointId)).toEqual(["w_kept"]);
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest("clears everything when no edge ids are given", async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    try {
+      await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_a"), edge("w_b")] });
+
+      expect((await store.clearBlockState({ runId: RUN_ID })).outcome).toBe("cleared");
+      expect(await store.readBlockState(RUN_ID)).toEqual({
+        pendingIds: [],
+        deliveredIds: [],
+        edges: [],
+      });
+    } finally {
+      await store.quit();
+    }
+  });
+});
diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts
index 0ef39b522df..32a00e355e8 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts
@@ -1,6 +1,13 @@
 import { createRedisClient, type Redis, type RedisOptions } from "@internal/redis";
 import { Logger } from "@trigger.dev/core/logger";
-import { assertSingleSlot, idempotencyKey, waitpointKeys, watcherField } from "./keys.js";
+import {
+  assertSingleSlot,
+  edgeField,
+  idempotencyKey,
+  runBlockKeys,
+  waitpointKeys,
+  watcherField,
+} from "./keys.js";
 import { registerWaitpointCommands } from "./scripts.js";
 
 /** The values written into a record's `status` field. Uppercase, and never a token. */
@@ -89,6 +96,44 @@ export type CompleteResult = {
   watchers: WatcherEntry[];
 };
 
+/** One run-to-waitpoint edge. The metadata a frozen return type needs travels here. */
+export type BlockEdge = {
+  waitpointId: string;
+  batchIndex?: number | null;
+  batchId?: string;
+  spanIdToComplete?: string;
+  createdAt: string;
+  type: WaitpointRecordInput["type"];
+  completedAfter?: string;
+  /** Set when the register step already reported this waitpoint COMPLETED. */
+  reported?: WaitpointCompletion;
+};
+
+export type AbsorbResult = {
+  /**
+   * How many DISTINCT requested ids were still pending. Equivalent to the count the
+   * previous path took over this call's ids, which was a COUNT over waitpoint rows — so
+   * two edges for one waitpoint contribute one. This is the number a caller should use to
+   * keep today's block-time gate unchanged.
+   */
+  pendingOfRequested: number;
+  /**
+   * The run's whole pending set, counting STORE-RESIDENT blockers only. A run can also be
+   * blocked by a legacy waitpoint, which this number cannot see, so it is never on its own
+   * a decision to resume.
+   */
+  storePendingTotal: number;
+  alreadyDelivered: Array<{ waitpointId: string; completion?: WaitpointCompletion }>;
+};
+
+export type BlockStateEdge = BlockEdge & { edgeId: string };
+
+export type BlockState = {
+  pendingIds: string[];
+  deliveredIds: string[];
+  edges: BlockStateEdge[];
+};
+
 export class WaitpointNotFoundError extends Error {
   constructor(waitpointId: string) {
     super(`Waitpoint ${waitpointId} is not present in the store`);
@@ -307,4 +352,106 @@ export class WaitpointStoreCoordinator {
 
     return { waitpointId: winner, created: false };
   }
+
+  async absorbBlockers(args: { runId: string; edges: BlockEdge[] }): Promise<AbsorbResult> {
+    const keys = runBlockKeys(args.runId);
+
+    // No fast path for an empty list: storePendingTotal is defined as the run's WHOLE
+    // store-resident pending set, so it has to be read even when nothing is requested.
+    const argv: string[] = [String(args.edges.length)];
+    for (const item of args.edges) {
+      const { reported, ...stored } = item;
+      argv.push(
+        item.waitpointId,
+        edgeField(item.waitpointId, item.batchIndex),
+        JSON.stringify(stored),
+        reported ? JSON.stringify(reported) : ""
+      );
+    }
+
+    const reply = await this.#call("runAbsorbBlockers", [keys.pend, keys.done, keys.edge], ...argv);
+
+    const alreadyDelivered: AbsorbResult["alreadyDelivered"] = [];
+    for (let i = 2; i < reply.length; i += 2) {
+      alreadyDelivered.push({
+        waitpointId: reply[i]!,
+        completion: parseJson<WaitpointCompletion>(reply[i + 1]),
+      });
+    }
+
+    return {
+      pendingOfRequested: Number(reply[0]),
+      storePendingTotal: Number(reply[1]),
+      alreadyDelivered,
+    };
+  }
+
+  async deliverCompletion(args: {
+    runId: string;
+    waitpointId: string;
+    completion: WaitpointCompletion;
+  }): Promise<{ storePendingTotal: number }> {
+    const keys = runBlockKeys(args.runId);
+
+    const reply = await this.#call(
+      "runDeliverCompletion",
+      [keys.pend, keys.done],
+      args.waitpointId,
+      JSON.stringify(args.completion)
+    );
+
+    return { storePendingTotal: Number(reply[0]) };
+  }
+
+  async readBlockState(runId: string): Promise<BlockState> {
+    const keys = runBlockKeys(runId);
+    const reply = await this.#call("runReadBlockState", [keys.pend, keys.done, keys.edge]);
+
+    // Slots 0 and 1 are true element counts, but slot 2 is the FLAT length of the edge
+    // HGETALL — two entries per edge, field then value. The cursor arithmetic below relies
+    // on that asymmetry, so do not "normalise" it without changing the Lua too.
+    const pendCount = Number(reply[0]);
+    const doneCount = Number(reply[1]);
+    const edgeCount = Number(reply[2]);
+
+    let cursor = 3;
+    const pendingIds = reply.slice(cursor, cursor + pendCount);
+    cursor += pendCount;
+    const deliveredIds = reply.slice(cursor, cursor + doneCount);
+    cursor += doneCount;
+
+    const edges: BlockStateEdge[] = [];
+    for (let i = 0; i < edgeCount; i += 2) {
+      const edgeId = reply[cursor + i]!;
+      const stored = JSON.parse(reply[cursor + i + 1] ?? "{}") as BlockEdge;
+      edges.push({ ...stored, edgeId });
+    }
+
+    return { pendingIds, deliveredIds, edges };
+  }
+
+  /**
+   * Drain one cycle's edges, or clear the run entirely when no edge ids are given.
+   *
+   * The selective form RECONCILES: after the named edges go, any pending or delivered
+   * entry that no surviving edge references goes too. That is wider than deleting the
+   * named ids, and it has to be — a delivery is written unconditionally, so the window
+   * between register and absorb can leave a delivered entry with no edge at all.
+   */
+  async clearBlockState(args: {
+    runId: string;
+    edgeIds?: string[];
+  }): Promise<{ outcome: "cleared" | "drained" }> {
+    const keys = runBlockKeys(args.runId);
+    const edgeIds = args.edgeIds ?? [];
+
+    const reply = await this.#call(
+      "runClear",
+      [keys.pend, keys.done, keys.edge],
+      String(edgeIds.length),
+      ...edgeIds
+    );
+
+    return { outcome: reply[0] as "cleared" | "drained" };
+  }
 }

From 7e1142a187b6c6aae7ced0c78829b0caab686b09 Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Fri, 21 Aug 2026 20:38:59 +0100
Subject: [PATCH 31/49] fix(run-engine): fail loudly on a missing edge reply
 slot, guard empty-array clear
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Removes the one ?? on a Lua reply decode in readBlockState — a missing edge
slot now throws instead of silently decoding an empty BlockEdge, matching
the convention everywhere else in this file.

clearBlockState now distinguishes an explicitly empty edgeIds array (a
no-op) from an omitted one (the terminal clear-everything), since both
previously collapsed onto the Lua's n === 0 clear-everything branch.

Also: BlockStateEdge no longer advertises a reported field it can never
carry, adds a test where pendingOfRequested and storePendingTotal diverge
for a reason other than an empty request list, and trims/renames a few
comments per review.
---
 .../storeCoordinator.test.ts                  | 56 +++++++++++++++++--
 .../waitpointCoordinator/storeCoordinator.ts  | 38 ++++++++++---
 2 files changed, 80 insertions(+), 14 deletions(-)

diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts
index 0f951ed215f..039ad78a0e9 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts
@@ -360,9 +360,10 @@ describe("complete", () => {
   });
 });
 
-// No coordinator method calls runAbsorbBlockers/runClear/wpIdemReserve yet — a later task
-// wires those in. Registered directly on a raw client so the Lua itself is exercised now.
-describe("runAbsorbBlockers, runClear and wpIdemReserve (direct Lua)", () => {
+// A coordinator method now drives each of these scripts, but this block stays: it is the
+// only place asserting the RAW reply shape, so a Lua/TypeScript framing change made on
+// both sides at once would still fail here even though every class-level test passed.
+describe("reply framing (direct Lua — pins the wire shape the coordinator decodes)", () => {
   const envelope = JSON.stringify(completion());
 
   redisTest(
@@ -780,8 +781,6 @@ describe("absorbBlockers", () => {
         expect(result.pendingOfRequested).toBe(1);
         expect(result.storePendingTotal).toBe(1);
 
-        // The edges themselves stay distinct — that multiplicity is what produces the
-        // repeats in the cycle's ordered id list.
         const state = await store.readBlockState(RUN_ID);
         expect(state.edges).toHaveLength(2);
         expect(state.edges.map((e) => e.batchIndex).sort()).toEqual([0, 2]);
@@ -830,7 +829,6 @@ describe("absorbBlockers", () => {
   redisTest("lets a delivery that raced ahead of the absorb win", async ({ redisOptions }) => {
     const store = coordinator(redisOptions);
     try {
-      // The completion landed between register and absorb, so it is already delivered.
       await store.deliverCompletion({
         runId: RUN_ID,
         waitpointId: "w_a",
@@ -896,6 +894,31 @@ describe("absorbBlockers", () => {
     }
   });
 
+  redisTest(
+    "reports a smaller pendingOfRequested than storePendingTotal when an unrelated blocker is already pending",
+    async ({ redisOptions }) => {
+      const store = coordinator(redisOptions);
+      try {
+        // w_x is a live blocker from an earlier absorb, unrelated to this call's request.
+        await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_x")] });
+
+        const result = await store.absorbBlockers({
+          runId: RUN_ID,
+          edges: [edge("w_a", { reported: completion() })],
+        });
+
+        // Nothing THIS call requested is pending (w_a arrived already delivered), but the
+        // run's whole store-resident set still holds w_x — a divergence for a different
+        // reason than an empty request list, so a reply[0]/reply[1] swap or a
+        // re-derived-in-TypeScript pendingOfRequested would both be caught here too.
+        expect(result.pendingOfRequested).toBe(0);
+        expect(result.storePendingTotal).toBe(1);
+      } finally {
+        await store.quit();
+      }
+    }
+  );
+
   redisTest("sets no TTL on any run key", async ({ redisOptions }) => {
     const store = coordinator(redisOptions);
     const probe = createRedisClient(redisOptions);
@@ -1117,4 +1140,25 @@ describe("clearBlockState", () => {
       await store.quit();
     }
   });
+
+  redisTest(
+    "is a no-op for an explicitly empty edge id list, unlike an omitted one",
+    async ({ redisOptions }) => {
+      const store = coordinator(redisOptions);
+      try {
+        await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_a"), edge("w_b")] });
+
+        // Omitting edgeIds reaches the Lua's n === 0 branch and clears everything (proven
+        // above). A caller-computed EMPTY array must not collapse onto that: it means
+        // "nothing to drain", not "clear the run".
+        expect((await store.clearBlockState({ runId: RUN_ID, edgeIds: [] })).outcome).toBe("noop");
+
+        const state = await store.readBlockState(RUN_ID);
+        expect(state.edges.map((e) => e.waitpointId).sort()).toEqual(["w_a", "w_b"]);
+        expect(state.pendingIds.sort()).toEqual(["w_a", "w_b"]);
+      } finally {
+        await store.quit();
+      }
+    }
+  );
 });
diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts
index 32a00e355e8..8c4eca86ba0 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts
@@ -96,7 +96,10 @@ export type CompleteResult = {
   watchers: WatcherEntry[];
 };
 
-/** One run-to-waitpoint edge. The metadata a frozen return type needs travels here. */
+/**
+ * One run-to-waitpoint edge. The metadata a frozen return type — an existing API response
+ * shape this store must keep reproducing — needs travels here.
+ */
 export type BlockEdge = {
   waitpointId: string;
   batchIndex?: number | null;
@@ -126,7 +129,10 @@ export type AbsorbResult = {
   alreadyDelivered: Array<{ waitpointId: string; completion?: WaitpointCompletion }>;
 };
 
-export type BlockStateEdge = BlockEdge & { edgeId: string };
+// absorbBlockers strips `reported` before writing the edge blob, so a value read back
+// here can never carry it — Omit says so instead of inheriting a field that is always
+// undefined.
+export type BlockStateEdge = Omit<BlockEdge, "reported"> & { edgeId: string };
 
 export type BlockState = {
   pendingIds: string[];
@@ -423,7 +429,17 @@ export class WaitpointStoreCoordinator {
     const edges: BlockStateEdge[] = [];
     for (let i = 0; i < edgeCount; i += 2) {
       const edgeId = reply[cursor + i]!;
-      const stored = JSON.parse(reply[cursor + i + 1] ?? "{}") as BlockEdge;
+      // An edge value is always a non-empty JSON.stringify, so a missing slot here means
+      // the cursor walked off the end of the reply. That must fail loudly, not decode a
+      // BlockEdge with no waitpointId — the exact off-by-one this task's arithmetic guards
+      // against.
+      const edgeJson = reply[cursor + i + 1];
+      if (!edgeJson) {
+        throw new Error(
+          `readBlockState(${runId}): missing edge payload at reply index ${cursor + i + 1}`
+        );
+      }
+      const stored = JSON.parse(edgeJson) as BlockEdge;
       edges.push({ ...stored, edgeId });
     }
 
@@ -433,15 +449,21 @@ export class WaitpointStoreCoordinator {
   /**
    * Drain one cycle's edges, or clear the run entirely when no edge ids are given.
    *
-   * The selective form RECONCILES: after the named edges go, any pending or delivered
-   * entry that no surviving edge references goes too. That is wider than deleting the
-   * named ids, and it has to be — a delivery is written unconditionally, so the window
-   * between register and absorb can leave a delivered entry with no edge at all.
+   * The selective form RECONCILES: any pending or delivered entry that no surviving edge
+   * references goes too, not only the named ones. See runClear in scripts.ts for why.
    */
   async clearBlockState(args: {
     runId: string;
     edgeIds?: string[];
-  }): Promise<{ outcome: "cleared" | "drained" }> {
+  }): Promise<{ outcome: "cleared" | "drained" | "noop" }> {
+    // `omitted` and `explicitly empty` must not collapse onto each other: the Lua's
+    // n === 0 means "clear the whole run", so an omitted edgeIds stays the terminal clear,
+    // but a caller that computed zero edges to drain gets a genuine no-op that never
+    // reaches Redis.
+    if (args.edgeIds && args.edgeIds.length === 0) {
+      return { outcome: "noop" };
+    }
+
     const keys = runBlockKeys(args.runId);
     const edgeIds = args.edgeIds ?? [];
 

From 5ef87efb26049aac74f99ca8c1f3cc3807aad593 Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Fri, 21 Aug 2026 20:58:43 +0100
Subject: [PATCH 32/49] feat(run-engine): register-then-absorb across the
 waitpoint and run shards

Adds registerBlocks, which registers on each waitpoint's own shard before
absorbing once on the run's shard, plus the protocol acceptance tests.

Also fixes a hang: a waitpoint COMPLETED with no completion envelope (the
FINISHED-healing shape) used to be threaded into the pending set instead of
being reported, blocking the run forever on something already done.
runAbsorbBlockers now carries an explicit reportedFlag per group so the
reported/pending branch is keyed on outcome, never on envelope presence.
---
 .../engine/waitpointCoordinator/scripts.ts    |  22 +-
 .../storeCoordinator.test.ts                  | 253 +++++++++++++++++-
 .../waitpointCoordinator/storeCoordinator.ts  |  51 +++-
 3 files changed, 314 insertions(+), 12 deletions(-)

diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/scripts.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/scripts.ts
index 5608b722699..820b4145f86 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/scripts.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/scripts.ts
@@ -154,7 +154,11 @@ export function registerWaitpointCommands(redis: Redis): void {
   });
 
   // KEYS: pend, done, edge.
-  // ARGV: n, then n groups of 4 — waitpointId, edgeField, edgeJson, reportedJson ('').
+  // ARGV: n, then n groups of 5 — waitpointId, edgeField, edgeJson, reportedFlag
+  // ('1'|'0'), reportedJson (''). reportedFlag, not the emptiness of reportedJson, is what
+  // decides the branch: a waitpoint can be reported COMPLETED with no completion envelope
+  // (see the FINISHED-healing path), and that case must still take the reported branch —
+  // flag '1', reportedJson '' — or the run would block forever on something already done.
   redis.defineCommand("runAbsorbBlockers", {
     numberOfKeys: 3,
     lua: `
@@ -163,7 +167,7 @@ export function registerWaitpointCommands(redis: Redis): void {
 
       -- Guard before any write: a wrong n must not half-apply the script. HDEL/HSETNX below
       -- are irreversible mid-script, and Redis does not roll back a script that errors.
-      if #ARGV ~= 1 + n * 4 then
+      if #ARGV ~= 1 + n * 5 then
         return redis.error_reply('runAbsorbBlockers: arity mismatch')
       end
 
@@ -174,18 +178,20 @@ export function registerWaitpointCommands(redis: Redis): void {
       local out = { '0', '0' }
 
       for i = 0, n - 1 do
-        local id       = ARGV[2 + i * 4]
-        local field    = ARGV[3 + i * 4]
-        local edgeJson = ARGV[4 + i * 4]
-        local reported = ARGV[5 + i * 4]
+        local id           = ARGV[2 + i * 5]
+        local field        = ARGV[3 + i * 5]
+        local edgeJson     = ARGV[4 + i * 5]
+        local reportedFlag = ARGV[5 + i * 5]
+        local reported     = ARGV[6 + i * 5]
 
         -- HSETNX is the ON CONFLICT DO NOTHING of the edge write: a retry must not
         -- overwrite the first attempt's metadata.
         redis.call('HSETNX', edge, field, edgeJson)
         requestedIds[id] = true
 
-        if reported ~= '' then
-          -- Already COMPLETED when the watcher registered. It never becomes pending.
+        if reportedFlag == '1' then
+          -- Already COMPLETED when the watcher registered. It never becomes pending, even
+          -- when reported ('' here) carries no envelope.
           redis.call('HSET', done, id, reported)
           redis.call('SREM', pend, id)
           if not seenDelivered[id] then
diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts
index 039ad78a0e9..5a1258b0611 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts
@@ -386,10 +386,12 @@ describe("reply framing (direct Lua — pins the wire shape the coordinator deco
           "w_solo",
           fieldA,
           "{}",
+          "0",
           "",
           "w_solo",
           fieldB,
           "{}",
+          "1",
           envelope
         );
 
@@ -419,10 +421,12 @@ describe("reply framing (direct Lua — pins the wire shape the coordinator deco
           "w_solo",
           fieldB,
           "{}",
+          "1",
           envelope,
           "w_solo",
           fieldA,
           "{}",
+          "0",
           ""
         );
 
@@ -448,10 +452,12 @@ describe("reply framing (direct Lua — pins the wire shape the coordinator deco
         "w_a",
         edgeField("w_a", 0),
         "{}",
+        "0",
         "",
         "w_b",
         edgeField("w_b", 0),
         "{}",
+        "0",
         ""
       );
 
@@ -478,10 +484,12 @@ describe("reply framing (direct Lua — pins the wire shape the coordinator deco
           "w_a",
           edgeField("w_a", 0),
           "{}",
+          "0",
           "",
           "w_b",
           edgeField("w_b", 0),
           "{}",
+          "1",
           envelope
         );
 
@@ -509,10 +517,12 @@ describe("reply framing (direct Lua — pins the wire shape the coordinator deco
           "w_a",
           edgeField("w_a", 0),
           "{}",
+          "0",
           "",
           "w_a",
           edgeField("w_a", 1),
           "{}",
+          "0",
           ""
         );
 
@@ -543,6 +553,7 @@ describe("reply framing (direct Lua — pins the wire shape the coordinator deco
           "w_a",
           edgeField("w_a", 0),
           "{}",
+          "0",
           ""
         );
 
@@ -561,9 +572,20 @@ describe("reply framing (direct Lua — pins the wire shape the coordinator deco
       const keys = runBlockKeys("run_1");
       const field = edgeField("w_solo", 0);
 
-      // n says 2 groups but only one group (4 ARGV entries) is supplied.
+      // n says 2 groups (1 + 2 * 5 = 11 ARGV entries expected) but only one group (5
+      // ARGV entries) is supplied.
       await expect(
-        client.runAbsorbBlockers(keys.pend, keys.done, keys.edge, "2", "w_solo", field, "{}", "")
+        client.runAbsorbBlockers(
+          keys.pend,
+          keys.done,
+          keys.edge,
+          "2",
+          "w_solo",
+          field,
+          "{}",
+          "0",
+          ""
+        )
       ).rejects.toThrow();
 
       expect(await client.exists(keys.pend)).toBe(0);
@@ -1162,3 +1184,230 @@ describe("clearBlockState", () => {
     }
   );
 });
+
+describe("registerBlocks: a COMPLETED waitpoint with no envelope never blocks (regression)", () => {
+  redisTest(
+    "created COMPLETED with no envelope: registerBlocks does not block the run",
+    async ({ redisOptions }) => {
+      const store = coordinator(redisOptions);
+      try {
+        // No `completion` at all — the FINISHED-healing shape from Task 4's "can create an
+        // already-COMPLETED record with no completion envelope" test.
+        await store.createIfAbsent({ record: record("w_a"), status: "COMPLETED" });
+
+        const result = await store.registerBlocks({ runId: RUN_ID, edges: [edge("w_a")] });
+
+        expect(result.pendingOfRequested).toBe(0);
+        expect(result.storePendingTotal).toBe(0);
+        expect(result.alreadyDelivered.map((d) => d.waitpointId)).toEqual(["w_a"]);
+      } finally {
+        await store.quit();
+      }
+    }
+  );
+
+  redisTest(
+    "created COMPLETED with an envelope: behaves identically with respect to blocking",
+    async ({ redisOptions }) => {
+      const store = coordinator(redisOptions);
+      try {
+        await store.createIfAbsent({
+          record: record("w_a"),
+          status: "COMPLETED",
+          completion: completion(),
+        });
+
+        const result = await store.registerBlocks({ runId: RUN_ID, edges: [edge("w_a")] });
+
+        expect(result.pendingOfRequested).toBe(0);
+        expect(result.storePendingTotal).toBe(0);
+        expect(result.alreadyDelivered.map((d) => d.waitpointId)).toEqual(["w_a"]);
+      } finally {
+        await store.quit();
+      }
+    }
+  );
+});
+
+describe("registerBlocks: the two orderings", () => {
+  redisTest("block first, then complete: the run blocks, then wakes", async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    try {
+      await store.createIfAbsent({ record: record("w_a"), status: "PENDING" });
+
+      const blocked = await store.registerBlocks({ runId: RUN_ID, edges: [edge("w_a")] });
+      expect(blocked.pendingOfRequested).toBe(1);
+      expect(blocked.storePendingTotal).toBe(1);
+
+      const completed = await store.complete({ waitpointId: "w_a", completion: completion() });
+      expect(completed.watchers.map((w) => w.runId)).toEqual([RUN_ID]);
+
+      const delivered = await store.deliverCompletion({
+        runId: RUN_ID,
+        waitpointId: "w_a",
+        completion: completed.completion!,
+      });
+      expect(delivered.storePendingTotal).toBe(0);
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest("complete first, then block: the run never goes pending", async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    try {
+      await store.createIfAbsent({ record: record("w_a"), status: "PENDING" });
+      await store.complete({ waitpointId: "w_a", completion: completion() });
+
+      const result = await store.registerBlocks({ runId: RUN_ID, edges: [edge("w_a")] });
+
+      expect(result.pendingOfRequested).toBe(0);
+      expect(result.storePendingTotal).toBe(0);
+      expect(result.alreadyDelivered.map((d) => d.waitpointId)).toEqual(["w_a"]);
+
+      const state = await store.readBlockState(RUN_ID);
+      expect(state.pendingIds).toEqual([]);
+      expect(state.deliveredIds).toEqual(["w_a"]);
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest("throws when a blocking waitpoint does not exist", async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    try {
+      await expect(
+        store.registerBlocks({ runId: RUN_ID, edges: [edge("w_missing")] })
+      ).rejects.toThrow(WaitpointNotFoundError);
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest("is idempotent when run twice", async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    try {
+      await store.createIfAbsent({ record: record("w_a"), status: "PENDING" });
+
+      const first = await store.registerBlocks({ runId: RUN_ID, edges: [edge("w_a")] });
+      const second = await store.registerBlocks({ runId: RUN_ID, edges: [edge("w_a")] });
+
+      expect(first.storePendingTotal).toBe(1);
+      expect(second.storePendingTotal).toBe(1);
+      expect((await store.readBlockState(RUN_ID)).edges).toHaveLength(1);
+    } finally {
+      await store.quit();
+    }
+  });
+
+  redisTest(
+    "mixed set: one pending and one already complete blocks the run once",
+    async ({ redisOptions }) => {
+      const store = coordinator(redisOptions);
+      try {
+        await store.createIfAbsent({ record: record("w_pending"), status: "PENDING" });
+        await store.createIfAbsent({ record: record("w_done"), status: "PENDING" });
+        await store.complete({ waitpointId: "w_done", completion: completion() });
+
+        const result = await store.registerBlocks({
+          runId: RUN_ID,
+          edges: [edge("w_pending"), edge("w_done")],
+        });
+
+        expect(result.pendingOfRequested).toBe(1);
+        expect(result.storePendingTotal).toBe(1);
+        expect(result.alreadyDelivered.map((d) => d.waitpointId)).toEqual(["w_done"]);
+      } finally {
+        await store.quit();
+      }
+    }
+  );
+});
+
+describe("multi-index merge, end to end into the executor shape", () => {
+  redisTest(
+    "a run blocked on one waitpoint at two indexes resolves to two entries",
+    async ({ redisOptions }) => {
+      const store = coordinator(redisOptions);
+      try {
+        await store.createIfAbsent({
+          record: record("w_child", { type: "RUN", completedByTaskRunId: "run_child" }),
+          status: "PENDING",
+        });
+
+        await store.registerBlocks({
+          runId: RUN_ID,
+          edges: [
+            edge("w_child", { batchIndex: 0, batchId: "batch_1", type: "RUN" }),
+            edge("w_child", { batchIndex: 2, batchId: "batch_1", type: "RUN" }),
+          ],
+        });
+
+        const completed = await store.complete({
+          waitpointId: "w_child",
+          completion: completion({ output: null }),
+        });
+        await store.deliverCompletion({
+          runId: RUN_ID,
+          waitpointId: "w_child",
+          completion: completed.completion!,
+        });
+
+        const state = await store.readBlockState(RUN_ID);
+        expect(state.pendingIds).toEqual([]);
+        expect(state.deliveredIds).toEqual(["w_child"]);
+
+        // Derive the cycle's ordered id list the way the read path does: keep only edges
+        // that carry a batch index, sort ascending, map to id. Derived inline on purpose —
+        // another lane owns the order rule and its resolver, and this test's job is to
+        // prove the COORDINATOR preserved the edge multiplicity across two shards, not to
+        // own that rule.
+        const order = state.edges
+          .filter((e) => e.batchIndex !== undefined && e.batchIndex !== null)
+          .sort((a, b) => a.batchIndex! - b.batchIndex!)
+          .map((e) => e.waitpointId);
+
+        // One waitpoint, two edges, so the id repeats — that repeat is what expands into
+        // two entries for the executor, and losing it would silently drop a batch item.
+        expect(order).toEqual(["w_child", "w_child"]);
+        expect(state.edges.map((e) => e.edgeId).sort()).toEqual(["w_child#0", "w_child#2"]);
+        expect(state.edges.every((e) => e.batchId === "batch_1")).toBe(true);
+      } finally {
+        await store.quit();
+      }
+    }
+  );
+});
+
+describe("the resume cycle drains and can start again", () => {
+  redisTest("a second wait on the same waitpoint blocks nothing", async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    try {
+      await store.createIfAbsent({ record: record("w_a"), status: "PENDING" });
+      await store.registerBlocks({ runId: RUN_ID, edges: [edge("w_a")] });
+
+      const completed = await store.complete({ waitpointId: "w_a", completion: completion() });
+      await store.deliverCompletion({
+        runId: RUN_ID,
+        waitpointId: "w_a",
+        completion: completed.completion!,
+      });
+
+      const first = await store.readBlockState(RUN_ID);
+      await store.clearBlockState({ runId: RUN_ID, edgeIds: first.edges.map((e) => e.edgeId) });
+
+      // Cycle two. The waitpoint is COMPLETED for good, so the register reports it and the
+      // run is never blocked.
+      const second = await store.registerBlocks({
+        runId: RUN_ID,
+        edges: [edge("w_a", { batchIndex: 5 })],
+      });
+
+      expect(second.storePendingTotal).toBe(0);
+      expect(second.alreadyDelivered.map((d) => d.waitpointId)).toEqual(["w_a"]);
+      expect((await store.readBlockState(RUN_ID)).edges.map((e) => e.edgeId)).toEqual(["w_a#5"]);
+    } finally {
+      await store.quit();
+    }
+  });
+});
diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts
index 8c4eca86ba0..58724f9ffd3 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts
@@ -77,6 +77,13 @@ export type WatcherEntry = {
   createdAt: string;
 };
 
+/**
+ * Marks an edge as reported COMPLETED with no completion envelope — the shape a
+ * FINISHED-healing create can produce. Distinct from `undefined` (never reported), so the
+ * pending decision can key on outcome alone rather than on envelope presence.
+ */
+export const EMPTY_REPORTED_MARKER = Symbol("waitpoint-reported-no-envelope");
+
 export type CreateIfAbsentResult =
   | { outcome: "created" }
   | {
@@ -109,7 +116,7 @@ export type BlockEdge = {
   type: WaitpointRecordInput["type"];
   completedAfter?: string;
   /** Set when the register step already reported this waitpoint COMPLETED. */
-  reported?: WaitpointCompletion;
+  reported?: WaitpointCompletion | typeof EMPTY_REPORTED_MARKER;
 };
 
 export type AbsorbResult = {
@@ -367,11 +374,17 @@ export class WaitpointStoreCoordinator {
     const argv: string[] = [String(args.edges.length)];
     for (const item of args.edges) {
       const { reported, ...stored } = item;
+      const reportedFlag = reported !== undefined ? "1" : "0";
+      const reportedJson =
+        reported !== undefined && reported !== EMPTY_REPORTED_MARKER
+          ? JSON.stringify(reported)
+          : "";
       argv.push(
         item.waitpointId,
         edgeField(item.waitpointId, item.batchIndex),
         JSON.stringify(stored),
-        reported ? JSON.stringify(reported) : ""
+        reportedFlag,
+        reportedJson
       );
     }
 
@@ -392,6 +405,40 @@ export class WaitpointStoreCoordinator {
     };
   }
 
+  /**
+   * Block a run on a set of waitpoints.
+   *
+   * Register on every waitpoint's own shard FIRST, then absorb on the run's shard. The
+   * order is the protocol: a completion that lands in between finds the watcher already
+   * registered, so it delivers onto the run's shard, and the absorb sees that delivery and
+   * never marks the waitpoint pending. Reversing the two would open the window where a
+   * completion is missed by both steps.
+   *
+   * The register keys the decision to skip the pending set on OUTCOME, never on whether a
+   * completion envelope came back — a waitpoint can be reported COMPLETED with none.
+   */
+  async registerBlocks(args: { runId: string; edges: BlockEdge[] }): Promise<AbsorbResult> {
+    const registered: BlockEdge[] = [];
+
+    for (const item of args.edges) {
+      const result = await this.registerOrReport({
+        waitpointId: item.waitpointId,
+        runId: args.runId,
+        batchIndex: item.batchIndex,
+        spanIdToComplete: item.spanIdToComplete,
+        createdAt: item.createdAt,
+      });
+
+      registered.push(
+        result.outcome === "completed"
+          ? { ...item, reported: result.completion ?? EMPTY_REPORTED_MARKER }
+          : item
+      );
+    }
+
+    return this.absorbBlockers({ runId: args.runId, edges: registered });
+  }
+
   async deliverCompletion(args: {
     runId: string;
     waitpointId: string;

From bedce8b763ecbcdb11d17cf94c0444fbf24b2cf7 Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Fri, 21 Aug 2026 21:15:02 +0100
Subject: [PATCH 33/49] fix(run-engine): replace the reported-envelope symbol
 with a JSON-safe box

Review fix round 1 on registerBlocks: swap EMPTY_REPORTED_MARKER for
reported?: { completion?: WaitpointCompletion }, which round-trips through
JSON.stringify/parse instead of silently dropping. Tightens the no-envelope
regression test to check for a fabricated completion, pins the new
reportedFlag='1'/empty-envelope wire shape at the direct-Lua level, documents
and tests the safe partial-failure residue when registerBlocks throws
mid-loop, and asserts the watcher fan-out in the multi-index merge test.
---
 .../storeCoordinator.test.ts                  | 84 ++++++++++++++++++-
 .../waitpointCoordinator/storeCoordinator.ts  | 29 +++----
 2 files changed, 93 insertions(+), 20 deletions(-)

diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts
index 5a1258b0611..f8b02381ed6 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts
@@ -501,6 +501,36 @@ describe("reply framing (direct Lua — pins the wire shape the coordinator deco
     }
   );
 
+  redisTest(
+    "reported flag '1' with an empty envelope still delivers, not pends",
+    async ({ redisOptions }) => {
+      const client = createRedisClient(redisOptions);
+      registerWaitpointCommands(client);
+      try {
+        const keys = runBlockKeys("run_1");
+
+        // The bug this task fixed: COMPLETED-with-no-envelope must take the reported
+        // branch on the flag alone, not on the envelope being non-empty.
+        const reply = await client.runAbsorbBlockers(
+          keys.pend,
+          keys.done,
+          keys.edge,
+          "1",
+          "w_a",
+          edgeField("w_a", 0),
+          "{}",
+          "1",
+          ""
+        );
+
+        expect(reply).toEqual(["0", "0", "w_a", ""]);
+        expect(await client.scard(keys.pend)).toBe(0);
+      } finally {
+        client.disconnect();
+      }
+    }
+  );
+
   redisTest(
     "counts the same unreported id passed twice as one pending, not two",
     async ({ redisOptions }) => {
@@ -819,7 +849,7 @@ describe("absorbBlockers", () => {
       try {
         const result = await store.absorbBlockers({
           runId: RUN_ID,
-          edges: [edge("w_a", { reported: completion() }), edge("w_b")],
+          edges: [edge("w_a", { reported: { completion: completion() } }), edge("w_b")],
         });
 
         expect(result.pendingOfRequested).toBe(1);
@@ -837,8 +867,8 @@ describe("absorbBlockers", () => {
       const result = await store.absorbBlockers({
         runId: RUN_ID,
         edges: [
-          edge("w_a", { batchIndex: 0, reported: completion() }),
-          edge("w_a", { batchIndex: 1, reported: completion() }),
+          edge("w_a", { batchIndex: 0, reported: { completion: completion() } }),
+          edge("w_a", { batchIndex: 1, reported: { completion: completion() } }),
         ],
       });
 
@@ -926,7 +956,7 @@ describe("absorbBlockers", () => {
 
         const result = await store.absorbBlockers({
           runId: RUN_ID,
-          edges: [edge("w_a", { reported: completion() })],
+          edges: [edge("w_a", { reported: { completion: completion() } })],
         });
 
         // Nothing THIS call requested is pending (w_a arrived already delivered), but the
@@ -1200,6 +1230,10 @@ describe("registerBlocks: a COMPLETED waitpoint with no envelope never blocks (r
         expect(result.pendingOfRequested).toBe(0);
         expect(result.storePendingTotal).toBe(0);
         expect(result.alreadyDelivered.map((d) => d.waitpointId)).toEqual(["w_a"]);
+        // The whole point: no fabricated envelope, and the delivery is real on the run
+        // shard, not just absent from pending.
+        expect(result.alreadyDelivered[0]!.completion).toBeUndefined();
+        expect((await store.readBlockState(RUN_ID)).deliveredIds).toEqual(["w_a"]);
       } finally {
         await store.quit();
       }
@@ -1284,6 +1318,43 @@ describe("registerBlocks: the two orderings", () => {
     }
   });
 
+  redisTest(
+    "a throw mid-loop leaves the earlier watcher registered, and that residue is safe",
+    async ({ redisOptions }) => {
+      const store = coordinator(redisOptions);
+      try {
+        await store.createIfAbsent({ record: record("w_ok"), status: "PENDING" });
+
+        await expect(
+          store.registerBlocks({ runId: RUN_ID, edges: [edge("w_ok"), edge("w_missing")] })
+        ).rejects.toThrow(WaitpointNotFoundError);
+
+        // registerBlocks throws before absorbBlockers ever runs, so the run's own shard
+        // is untouched.
+        const state = await store.readBlockState(RUN_ID);
+        expect(state.pendingIds).toEqual([]);
+        expect(state.edges).toEqual([]);
+
+        // But w_ok's watcher WAS registered on w_ok's own shard before the throw.
+        const completed = await store.complete({ waitpointId: "w_ok", completion: completion() });
+        expect(completed.watchers.map((w) => w.runId)).toEqual([RUN_ID]);
+
+        // Delivering it writes a `done` entry for a run that was never blocked on it —
+        // inert residue, not a false resume: no edge ever named it, and clearBlockState's
+        // reconcile would drop it the moment this run's block state is next drained.
+        const delivered = await store.deliverCompletion({
+          runId: RUN_ID,
+          waitpointId: "w_ok",
+          completion: completed.completion!,
+        });
+        expect(delivered.storePendingTotal).toBe(0);
+        expect((await store.readBlockState(RUN_ID)).deliveredIds).toEqual(["w_ok"]);
+      } finally {
+        await store.quit();
+      }
+    }
+  );
+
   redisTest("is idempotent when run twice", async ({ redisOptions }) => {
     const store = coordinator(redisOptions);
     try {
@@ -1347,6 +1418,11 @@ describe("multi-index merge, end to end into the executor shape", () => {
           waitpointId: "w_child",
           completion: completion({ output: null }),
         });
+        // The cross-shard fact this test claims to prove: two registers for the same
+        // waitpoint at different indexes fanned out into two distinct watcher entries.
+        expect(
+          completed.watchers.map((w) => w.batchIndex).sort((a, b) => (a ?? 0) - (b ?? 0))
+        ).toEqual([0, 2]);
         await store.deliverCompletion({
           runId: RUN_ID,
           waitpointId: "w_child",
diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts
index 58724f9ffd3..da79d3c5162 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts
@@ -77,13 +77,6 @@ export type WatcherEntry = {
   createdAt: string;
 };
 
-/**
- * Marks an edge as reported COMPLETED with no completion envelope — the shape a
- * FINISHED-healing create can produce. Distinct from `undefined` (never reported), so the
- * pending decision can key on outcome alone rather than on envelope presence.
- */
-export const EMPTY_REPORTED_MARKER = Symbol("waitpoint-reported-no-envelope");
-
 export type CreateIfAbsentResult =
   | { outcome: "created" }
   | {
@@ -115,8 +108,10 @@ export type BlockEdge = {
   createdAt: string;
   type: WaitpointRecordInput["type"];
   completedAfter?: string;
-  /** Set when the register step already reported this waitpoint COMPLETED. */
-  reported?: WaitpointCompletion | typeof EMPTY_REPORTED_MARKER;
+  // Set when the register step already reported this waitpoint COMPLETED. The box, not
+  // `completion`, carries the "reported" fact: box present + no completion means
+  // COMPLETED-with-no-envelope, box absent means never reported.
+  reported?: { completion?: WaitpointCompletion };
 };
 
 export type AbsorbResult = {
@@ -375,10 +370,7 @@ export class WaitpointStoreCoordinator {
     for (const item of args.edges) {
       const { reported, ...stored } = item;
       const reportedFlag = reported !== undefined ? "1" : "0";
-      const reportedJson =
-        reported !== undefined && reported !== EMPTY_REPORTED_MARKER
-          ? JSON.stringify(reported)
-          : "";
+      const reportedJson = reported?.completion ? JSON.stringify(reported.completion) : "";
       argv.push(
         item.waitpointId,
         edgeField(item.waitpointId, item.batchIndex),
@@ -411,11 +403,16 @@ export class WaitpointStoreCoordinator {
    * Register on every waitpoint's own shard FIRST, then absorb on the run's shard. The
    * order is the protocol: a completion that lands in between finds the watcher already
    * registered, so it delivers onto the run's shard, and the absorb sees that delivery and
-   * never marks the waitpoint pending. Reversing the two would open the window where a
-   * completion is missed by both steps.
+   * never marks the waitpoint pending.
    *
    * The register keys the decision to skip the pending set on OUTCOME, never on whether a
    * completion envelope came back — a waitpoint can be reported COMPLETED with none.
+   *
+   * A throw partway through (a missing waitpoint) intentionally leaves any
+   * already-registered watchers in place rather than unwinding them. That's safe: a later
+   * `complete` on one of those waitpoints still delivers correctly, and if it lands before
+   * this run ever retries `registerBlocks`, the stray `done` entry it writes is inert until
+   * a future absorb or `clearBlockState`'s reconcile reads it — never a false resume.
    */
   async registerBlocks(args: { runId: string; edges: BlockEdge[] }): Promise<AbsorbResult> {
     const registered: BlockEdge[] = [];
@@ -431,7 +428,7 @@ export class WaitpointStoreCoordinator {
 
       registered.push(
         result.outcome === "completed"
-          ? { ...item, reported: result.completion ?? EMPTY_REPORTED_MARKER }
+          ? { ...item, reported: { completion: result.completion } }
           : item
       );
     }

From ccd17ff0ba4b02d510537a8f74d1a480a25d8f75 Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Fri, 21 Aug 2026 21:27:21 +0100
Subject: [PATCH 34/49] chore(run-engine): export the waitpoint store
 coordinator surface

---
 internal-packages/run-engine/src/index.ts | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/internal-packages/run-engine/src/index.ts b/internal-packages/run-engine/src/index.ts
index 2c54e4c20c0..2c98edf6866 100644
--- a/internal-packages/run-engine/src/index.ts
+++ b/internal-packages/run-engine/src/index.ts
@@ -38,3 +38,26 @@ export type {
   ProcessBatchItemCallback,
   BatchCompletionCallback,
 } from "./batch-queue/types.js";
+
+// Waitpoint store coordinator. Exported but not yet wired: a later ticket routes
+// WaitpointSystem onto it behind a per-organisation flag.
+export {
+  WaitpointStoreCoordinator,
+  WaitpointNotFoundError,
+} from "./engine/waitpointCoordinator/storeCoordinator.js";
+export type {
+  AbsorbResult,
+  BlockEdge,
+  BlockState,
+  BlockStateEdge,
+  CompleteResult,
+  CreateIfAbsentResult,
+  RegisterOrReportResult,
+  WaitpointCompletion,
+  WaitpointCompletionOutput,
+  WaitpointRecordInput,
+  WaitpointStatus,
+  WaitpointStoreCoordinatorOptions,
+  WatcherEntry,
+} from "./engine/waitpointCoordinator/storeCoordinator.js";
+export { WaitpointKeyTagError } from "./engine/waitpointCoordinator/keys.js";

From ef7f3fc57de07f1c8ebc90ac33c15504cbce5d6a Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Fri, 21 Aug 2026 21:41:21 +0100
Subject: [PATCH 35/49] test(run-engine): benchmark waitpoint pending count,
 hydration and fan-out

---
 .../bench/waitpointCoordinator.bench.test.ts  | 259 ++++++++++++++++++
 1 file changed, 259 insertions(+)
 create mode 100644 internal-packages/run-engine/src/engine/bench/waitpointCoordinator.bench.test.ts

diff --git a/internal-packages/run-engine/src/engine/bench/waitpointCoordinator.bench.test.ts b/internal-packages/run-engine/src/engine/bench/waitpointCoordinator.bench.test.ts
new file mode 100644
index 00000000000..dcce73074a1
--- /dev/null
+++ b/internal-packages/run-engine/src/engine/bench/waitpointCoordinator.bench.test.ts
@@ -0,0 +1,259 @@
+/**
+ * Waitpoint coordination benchmark. Reports numbers; asserts nothing — on a shared runner
+ * the timings swing far more than any threshold worth gating on.
+ *
+ * Four groups, and only the first two are pairs:
+ *
+ *  1. Pending count — the store's SCARD gate against the previous path's
+ *     `COUNT(*) ... WHERE status='PENDING'`, over the same population. Like for like.
+ *  2. Read amplification — the store's `readBlockState` against a full-payload `SELECT`
+ *     of the same waitpoints. Like for like.
+ *  3. Store-only write paths — block+complete+deliver and K-watcher fan-out. Absolute
+ *     numbers with NO Postgres counterpart: no single statement on the previous path
+ *     corresponds to a Redis round trip that both blocks a run and delivers to watchers.
+ *  4. Register cost versus edge count — `registerBlocks` registers each edge with its own
+ *     round trip before the single absorb. This measures whether that serial loop is a
+ *     real cost at a wide fan-in, or a non-issue, at several fan-in widths.
+ *
+ * Every Postgres measurement here runs against rows this file inserts. A baseline over an
+ * empty table measures nothing.
+ *
+ * Knobs: BENCH_WP_ITERATIONS, BENCH_WP_FANIN, BENCH_WP_WATCHERS, BENCH_WP_REGISTER_WIDTHS,
+ * BENCH_WP_REGISTER_SAMPLES.
+ */
+import { containerTest } from "@internal/testcontainers";
+import type { PrismaClient } from "@trigger.dev/database";
+import {
+  WaitpointStoreCoordinator,
+  type BlockEdge,
+  type WaitpointRecordInput,
+} from "../waitpointCoordinator/storeCoordinator.js";
+import { setupAuthenticatedEnvironment } from "../tests/setup.js";
+
+vi.setConfig({ testTimeout: 900_000 });
+
+const ITERATIONS = Number(process.env.BENCH_WP_ITERATIONS ?? 100);
+const FANIN = Number(process.env.BENCH_WP_FANIN ?? 1001);
+const WATCHERS = Number(process.env.BENCH_WP_WATCHERS ?? 100);
+const REGISTER_WIDTHS = (process.env.BENCH_WP_REGISTER_WIDTHS ?? "1,10,100,1001")
+  .split(",")
+  .map((raw) => Number(raw.trim()))
+  .filter((width) => Number.isFinite(width) && width > 0);
+const REGISTER_SAMPLES = Number(process.env.BENCH_WP_REGISTER_SAMPLES ?? 20);
+const NOW = new Date().toISOString();
+
+type Sample = { label: string; count: number; p50: number; p99: number; totalMs: number };
+
+function percentile(sorted: number[], p: number): number {
+  if (sorted.length === 0) return 0;
+  return sorted[Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length))]!;
+}
+
+async function measure(label: string, count: number, run: (i: number) => Promise<void>) {
+  const durations: number[] = [];
+  const started = Date.now();
+  for (let i = 0; i < count; i++) {
+    const t0 = performance.now();
+    await run(i);
+    durations.push(performance.now() - t0);
+  }
+  durations.sort((a, b) => a - b);
+  const sample: Sample = {
+    label,
+    count,
+    p50: percentile(durations, 50),
+    p99: percentile(durations, 99),
+    totalMs: Date.now() - started,
+  };
+  console.log(
+    `[bench] ${sample.label} n=${sample.count} p50=${sample.p50.toFixed(2)}ms ` +
+      `p99=${sample.p99.toFixed(2)}ms total=${sample.totalMs}ms`
+  );
+  return sample;
+}
+
+function record(id: string, environmentId: string, projectId: string): WaitpointRecordInput {
+  return {
+    id,
+    friendlyId: `waitpoint_${id}`,
+    type: "MANUAL",
+    environmentId,
+    projectId,
+    createdAt: NOW,
+    updatedAt: NOW,
+    userProvidedIdempotencyKey: false,
+    tags: [],
+  };
+}
+
+const completion = {
+  completedAt: NOW,
+  outputType: "application/json",
+  outputIsError: false,
+  output: { inline: '{"ok":true}' },
+};
+
+function edge(waitpointId: string, batchIndex?: number): BlockEdge {
+  return { waitpointId, batchIndex, createdAt: NOW, type: "MANUAL" };
+}
+
+async function insertWaitpoints(
+  prisma: PrismaClient,
+  ids: string[],
+  environmentId: string,
+  projectId: string
+) {
+  await prisma.waitpoint.createMany({
+    data: ids.map((id) => ({
+      id,
+      friendlyId: `waitpoint_${id}`,
+      type: "MANUAL" as const,
+      idempotencyKey: id,
+      userProvidedIdempotencyKey: false,
+      projectId,
+      environmentId,
+    })),
+  });
+}
+
+containerTest(
+  "waitpoint coordination: pending count, read amplification, store write paths, register cost",
+  async ({ prisma, redisOptions }) => {
+    const env = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
+    const store = new WaitpointStoreCoordinator({ redisOptions });
+    const samples: Sample[] = [];
+    const registerCost: Array<{ width: number; p50Ms: number; p99Ms: number; perEdgeMsP50: number }> =
+      [];
+
+    try {
+      const ids = Array.from({ length: FANIN }, (_, i) => `bench_w_${i}`);
+
+      // Both stores get the SAME population. A Postgres baseline over an empty table
+      // measures an index probe against nothing.
+      await insertWaitpoints(prisma, ids, env.id, env.project.id);
+      for (const id of ids) {
+        await store.createIfAbsent({ record: record(id, env.id, env.project.id), status: "PENDING" });
+      }
+      await store.registerBlocks({
+        runId: "bench_run_fanin",
+        edges: ids.map((id, index) => edge(id, index)),
+      });
+
+      // --- group 1: the pending-count gate, like for like ---
+      samples.push(
+        await measure("store.pendingCount", ITERATIONS, async () => {
+          await store.absorbBlockers({ runId: "bench_run_fanin", edges: [] });
+        })
+      );
+      samples.push(
+        await measure("postgres.pendingCount", ITERATIONS, async () => {
+          await prisma.$queryRaw`SELECT COUNT(*) FROM "Waitpoint" WHERE id = ANY(${ids}::text[]) AND status = 'PENDING'`;
+        })
+      );
+
+      // --- group 2: read amplification, like for like ---
+      samples.push(
+        await measure("store.readBlockState", ITERATIONS, async () => {
+          await store.readBlockState("bench_run_fanin");
+        })
+      );
+      samples.push(
+        await measure("postgres.hydrateFullPayload", ITERATIONS, async () => {
+          // Every column of every waitpoint — the amplification the store removes.
+          await prisma.waitpoint.findMany({ where: { id: { in: ids } } });
+        })
+      );
+
+      // --- group 3: store-only write paths, no Postgres counterpart ---
+      samples.push(
+        await measure("store.block+complete+deliver", ITERATIONS, async (i) => {
+          const id = `bench_cycle_${i}`;
+          await store.createIfAbsent({
+            record: record(id, env.id, env.project.id),
+            status: "PENDING",
+          });
+          await store.registerBlocks({ runId: `bench_run_${i}`, edges: [edge(id)] });
+          const done = await store.complete({ waitpointId: id, completion });
+          for (const watcher of done.watchers) {
+            await store.deliverCompletion({
+              runId: watcher.runId,
+              waitpointId: id,
+              completion: done.completion!,
+            });
+          }
+        })
+      );
+
+      const fanOutId = "bench_fanout_w";
+      await store.createIfAbsent({
+        record: record(fanOutId, env.id, env.project.id),
+        status: "PENDING",
+      });
+      for (let i = 0; i < WATCHERS; i++) {
+        await store.registerBlocks({ runId: `bench_watcher_${i}`, edges: [edge(fanOutId)] });
+      }
+      samples.push(
+        await measure(`store.complete+deliver(watchers=${WATCHERS})`, 1, async () => {
+          const done = await store.complete({ waitpointId: fanOutId, completion });
+          // Serial on purpose: this is the worst case, and it is the number that says
+          // whether delivery needs to pipeline.
+          for (const watcher of done.watchers) {
+            await store.deliverCompletion({
+              runId: watcher.runId,
+              waitpointId: fanOutId,
+              completion: done.completion!,
+            });
+          }
+        })
+      );
+
+      // --- group 4: register cost versus edge count ---
+      // registerBlocks registers each edge with its own round trip, serially, before the
+      // single absorb. A review flagged that a wide fan-in therefore serializes one round
+      // trip per edge. This measures the real cost at several widths rather than predicting
+      // it, so the decision about bounded concurrency is made against a number.
+      const registerPoolWidth = Math.max(0, ...REGISTER_WIDTHS);
+      const registerIds = Array.from({ length: registerPoolWidth }, (_, i) => `bench_reg_w_${i}`);
+      await insertWaitpoints(prisma, registerIds, env.id, env.project.id);
+      for (const id of registerIds) {
+        await store.createIfAbsent({
+          record: record(id, env.id, env.project.id),
+          status: "PENDING",
+        });
+      }
+
+      for (const width of REGISTER_WIDTHS) {
+        const edges = registerIds.slice(0, width).map((id, index) => edge(id, index));
+        let call = 0;
+        const sample = await measure(
+          `store.registerBlocks(edges=${width})`,
+          REGISTER_SAMPLES,
+          async () => {
+            await store.registerBlocks({ runId: `bench_register_${width}_${call++}`, edges });
+          }
+        );
+        samples.push(sample);
+        registerCost.push({
+          width,
+          p50Ms: sample.p50,
+          p99Ms: sample.p99,
+          perEdgeMsP50: sample.p50 / width,
+        });
+        console.log(
+          `[bench] store.registerBlocks(edges=${width}) implied per-edge cost ` +
+            `p50=${(sample.p50 / width).toFixed(3)}ms p99=${(sample.p99 / width).toFixed(3)}ms`
+        );
+      }
+
+      console.log(
+        `[bench] groups 1 and 2 are like-for-like pairs. Group 3 and the register-cost ` +
+          `group (4) have no Postgres counterpart: no single statement on the previous ` +
+          `path corresponds to a Redis round trip that blocks, completes and delivers, ` +
+          `or to a serial per-edge register loop.`
+      );
+      console.log(`[bench] summary\n${JSON.stringify({ samples, registerCost }, null, 2)}`);
+    } finally {
+      await store.quit();
+    }
+  }
+);

From 8d25d662e149a15e4dc880495f6acbb4d788c78f Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Fri, 21 Aug 2026 22:29:13 +0100
Subject: [PATCH 36/49] style(run-engine): wrap bench test lines over
 printWidth

---
 .../engine/bench/waitpointCoordinator.bench.test.ts | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/internal-packages/run-engine/src/engine/bench/waitpointCoordinator.bench.test.ts b/internal-packages/run-engine/src/engine/bench/waitpointCoordinator.bench.test.ts
index dcce73074a1..01393ba4f96 100644
--- a/internal-packages/run-engine/src/engine/bench/waitpointCoordinator.bench.test.ts
+++ b/internal-packages/run-engine/src/engine/bench/waitpointCoordinator.bench.test.ts
@@ -122,8 +122,12 @@ containerTest(
     const env = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
     const store = new WaitpointStoreCoordinator({ redisOptions });
     const samples: Sample[] = [];
-    const registerCost: Array<{ width: number; p50Ms: number; p99Ms: number; perEdgeMsP50: number }> =
-      [];
+    const registerCost: Array<{
+      width: number;
+      p50Ms: number;
+      p99Ms: number;
+      perEdgeMsP50: number;
+    }> = [];
 
     try {
       const ids = Array.from({ length: FANIN }, (_, i) => `bench_w_${i}`);
@@ -132,7 +136,10 @@ containerTest(
       // measures an index probe against nothing.
       await insertWaitpoints(prisma, ids, env.id, env.project.id);
       for (const id of ids) {
-        await store.createIfAbsent({ record: record(id, env.id, env.project.id), status: "PENDING" });
+        await store.createIfAbsent({
+          record: record(id, env.id, env.project.id),
+          status: "PENDING",
+        });
       }
       await store.registerBlocks({
         runId: "bench_run_fanin",

From 4530122a22ff23500ad30b8428c4295045e827c0 Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Fri, 21 Aug 2026 22:33:53 +0100
Subject: [PATCH 37/49] test(run-engine): guard the watcher first-write-wins
 rule and pin the edge-field split

---
 .../engine/waitpointCoordinator/keys.test.ts  |  6 +++++
 .../src/engine/waitpointCoordinator/keys.ts   | 12 ++++++---
 .../storeCoordinator.test.ts                  | 27 +++++++++++++++++++
 3 files changed, 41 insertions(+), 4 deletions(-)

diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.test.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.test.ts
index 7bd9505c9ef..641fd4c80c4 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.test.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.test.ts
@@ -46,7 +46,9 @@ describe("edgeField", () => {
   it("distinguishes index 0 from an absent index", () => {
     expect(edgeField("w_a", 0)).not.toBe(edgeField("w_a"));
   });
+});
 
+describe("waitpointIdFromEdgeField", () => {
   it("round-trips back to the waitpoint id", () => {
     for (const index of [undefined, null, 0, 7]) {
       expect(waitpointIdFromEdgeField(edgeField("w_a", index))).toBe("w_a");
@@ -56,6 +58,10 @@ describe("edgeField", () => {
   it("returns undefined for a field with no separator", () => {
     expect(waitpointIdFromEdgeField("nope")).toBeUndefined();
   });
+
+  it("splits on the last separator, tolerating a '#' inside the waitpoint id", () => {
+    expect(waitpointIdFromEdgeField("a#b#3")).toBe("a#b");
+  });
 });
 
 describe("watcherField", () => {
diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.ts
index 2b3a8f17eac..38447d107a1 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.ts
@@ -31,10 +31,11 @@ export function idempotencyKey(environmentId: string, key: string): string {
   return `wp:idem:{${environmentId}}:${key}`;
 }
 
-// "#" separates the id from the index. A waitpoint id and a run id never contain "#", so
-// the split is unambiguous. An absent index collapses onto the empty suffix, which is how
-// the partial unique index on a null batchIndex behaves; index 0 keeps its own field,
-// because "0" and "" are different strings.
+// "#" separates the id from the index. An absent index collapses onto the empty suffix,
+// which is how the partial unique index on a null batchIndex behaves; index 0 keeps its
+// own field, because "0" and "" are different strings. The split back to an id below is
+// taken from the LAST "#", not the first, so this stays unambiguous even if a waitpoint id
+// or a run id ever contains "#" itself.
 const SEPARATOR = "#";
 
 export function edgeField(waitpointId: string, batchIndex?: number | null): string {
@@ -45,6 +46,9 @@ export function watcherField(runId: string, batchIndex?: number | null): string
   return `${runId}${SEPARATOR}${batchIndex ?? ""}`;
 }
 
+// The last-"#" rule here is re-implemented as a Lua pattern in runClear (scripts.ts). This
+// function has no caller besides its own test, so that test is what pins the rule as a
+// specification the Lua mirrors, not just documentation of this helper.
 export function waitpointIdFromEdgeField(field: string): string | undefined {
   const separator = field.lastIndexOf(SEPARATOR);
   return separator === -1 ? undefined : field.slice(0, separator);
diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts
index f8b02381ed6..9674339a606 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts
@@ -252,6 +252,33 @@ describe("registerOrReport", () => {
       await store.quit();
     }
   });
+
+  redisTest("keeps the first registration's watcher on a re-register", async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    try {
+      await store.createIfAbsent({ record: record("w_a"), status: "PENDING" });
+      await store.registerOrReport({
+        waitpointId: "w_a",
+        runId: "run_1",
+        spanIdToComplete: "span_first",
+        createdAt: NOW,
+      });
+      // Same run, same (absent) batch index, so the watcher field collides. HSETNX must
+      // not let this second registration overwrite the first one's span.
+      await store.registerOrReport({
+        waitpointId: "w_a",
+        runId: "run_1",
+        spanIdToComplete: "span_second",
+        createdAt: NOW,
+      });
+
+      const completed = await store.complete({ waitpointId: "w_a", completion: completion() });
+      expect(completed.watchers).toHaveLength(1);
+      expect(completed.watchers[0]!.spanIdToComplete).toBe("span_first");
+    } finally {
+      await store.quit();
+    }
+  });
 });
 
 describe("complete", () => {

From 3c0f344dfdc04171736ef43aa0075676cf65f847 Mon Sep 17 00:00:00 2001
From: Dan Sutton <dansutton@trigger.dev>
Date: Fri, 21 Aug 2026 13:17:12 +0100
Subject: [PATCH 38/49] refactor(run-engine): add WaitpointCoordinator seam
 with clearRunBlockState

---
 .../src/engine/systems/waitpointSystem.ts     | 27 +++++-----
 .../legacyPostgresCoordinator.ts              | 51 +++++++++++++++++++
 .../src/engine/waitpointCoordinator/types.ts  | 28 ++++++++++
 3 files changed, 92 insertions(+), 14 deletions(-)
 create mode 100644 internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts
 create mode 100644 internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts

diff --git a/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts b/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts
index 5d5a80772a6..d43e811c6fd 100644
--- a/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts
+++ b/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts
@@ -7,13 +7,15 @@ import type {
   TaskRunExecutionStatus,
   Waitpoint,
 } from "@trigger.dev/database";
-import { Prisma, boundedIn } from "@trigger.dev/database";
+import { Prisma } from "@trigger.dev/database";
 import type { RunStore } from "@internal/run-store";
 import { assertNever } from "assert-never";
 import { nanoid } from "nanoid";
 import { UnclassifiableWaitpointId } from "../errors.js";
 import { sendNotificationToWorker } from "../eventBus.js";
 import { isFinalRunStatus } from "../statuses.js";
+import { LegacyPostgresWaitpointCoordinator } from "../waitpointCoordinator/legacyPostgresCoordinator.js";
+import type { WaitpointCoordinator } from "../waitpointCoordinator/types.js";
 import type { EnqueueSystem } from "./enqueueSystem.js";
 import type { ExecutionSnapshotSystem } from "./executionSnapshotSystem.js";
 import { getLatestExecutionSnapshot } from "./executionSnapshotSystem.js";
@@ -45,11 +47,17 @@ export class WaitpointSystem {
   private readonly $: SystemResources;
   private readonly executionSnapshotSystem: ExecutionSnapshotSystem;
   private readonly enqueueSystem: EnqueueSystem;
+  private readonly coordinator: WaitpointCoordinator;
 
   constructor(private readonly options: WaitpointSystemOptions) {
     this.$ = options.resources;
     this.executionSnapshotSystem = options.executionSnapshotSystem;
     this.enqueueSystem = options.enqueueSystem;
+    this.coordinator = new LegacyPostgresWaitpointCoordinator({
+      runStore: this.$.runStore,
+      prisma: this.$.prisma,
+      logger: this.$.logger,
+    });
   }
 
   public async clearBlockingWaitpoints({
@@ -59,14 +67,7 @@ export class WaitpointSystem {
     runId: string;
     tx?: PrismaClientOrTransaction;
   }) {
-    // A run's edges co-locate with the run (the edge write routes by runId), so the router routes this
-    // taskRunId-keyed delete to the run's store rather than fanning out. The caller's `tx` is not
-    // forwarded — the delete runs on the owning store's own client (the router never threads a
-    // control-plane tx into a routed write).
-    const deleted = await this.$.runStore.deleteManyTaskRunWaitpoints(
-      { where: { taskRunId: runId } },
-      tx
-    );
+    const deleted = await this.coordinator.clearRunBlockState({ runId, tx });
 
     return deleted.count;
   }
@@ -926,11 +927,9 @@ export class WaitpointSystem {
 
       if (blockingWaitpoints.length > 0) {
         //5. Remove the blocking waitpoints
-        await this.$.runStore.deleteManyTaskRunWaitpoints({
-          where: {
-            taskRunId: runId,
-            id: { in: boundedIn(blockingWaitpoints.map((b) => b.id)) },
-          },
+        await this.coordinator.clearRunBlockState({
+          runId,
+          edgeIds: blockingWaitpoints.map((b) => b.id),
         });
 
         this.$.logger.debug(`continueRunIfUnblocked: removed blocking waitpoints`, {
diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts
new file mode 100644
index 00000000000..b2e6aadb26b
--- /dev/null
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts
@@ -0,0 +1,51 @@
+import type { RunStore } from "@internal/run-store";
+import type { Logger } from "@trigger.dev/core/logger";
+import type { PrismaClient } from "@trigger.dev/database";
+import { boundedIn } from "@trigger.dev/database";
+import type { ClearRunBlockStateParams, WaitpointCoordinator } from "./types.js";
+
+export type LegacyPostgresWaitpointCoordinatorOptions = {
+  runStore: RunStore;
+  prisma: PrismaClient;
+  logger: Logger;
+};
+
+/**
+ * Waitpoint coordination against Postgres, through the run-ops store.
+ *
+ * Dependencies are deliberately narrow: no run lock, no worker, no event bus.
+ * That makes "this owns waitpoint state only" structural rather than a convention.
+ */
+export class LegacyPostgresWaitpointCoordinator implements WaitpointCoordinator {
+  private readonly runStore: RunStore;
+  private readonly prisma: PrismaClient;
+  private readonly logger: Logger;
+
+  constructor(options: LegacyPostgresWaitpointCoordinatorOptions) {
+    this.runStore = options.runStore;
+    this.prisma = options.prisma;
+    this.logger = options.logger;
+  }
+
+  async clearRunBlockState({
+    runId,
+    edgeIds,
+    tx,
+  }: ClearRunBlockStateParams): Promise<{ count: number }> {
+    if (edgeIds) {
+      // Bounded delete of named edges, on the unblock path. No tx: that path is not inside a
+      // caller transaction, and boundedIn caps the id-list arity for Prisma.
+      return this.runStore.deleteManyTaskRunWaitpoints({
+        where: {
+          taskRunId: runId,
+          id: { in: boundedIn(edgeIds) },
+        },
+      });
+    }
+
+    // A run's edges co-locate with the run (the edge write routes by runId), so the router routes
+    // this taskRunId-keyed delete to the run's store rather than fanning out. The caller's `tx` is
+    // passed through: a routing store strips it, and a single store joins it.
+    return this.runStore.deleteManyTaskRunWaitpoints({ where: { taskRunId: runId } }, tx);
+  }
+}
diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts
new file mode 100644
index 00000000000..8db4b986bbe
--- /dev/null
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts
@@ -0,0 +1,28 @@
+import type { PrismaClientOrTransaction } from "@trigger.dev/database";
+
+/**
+ * The waitpoint and edge state operations that `WaitpointSystem` delegates.
+ *
+ * Orchestration stays in `WaitpointSystem`: the run lock, snapshot transitions,
+ * worker-job enqueues, event emissions and racepoints. This owns waitpoint and
+ * edge state only, so a non-Postgres implementation can replace it without any
+ * caller learning that it changed.
+ *
+ * The residency hints and `tx` are opaque pass-throughs. Opaque does not mean
+ * type-free — a Prisma type appears here — it means a non-Postgres implementation
+ * never reads the value.
+ */
+export type WaitpointCoordinator = {
+  clearRunBlockState(params: ClearRunBlockStateParams): Promise<{ count: number }>;
+};
+
+export type ClearRunBlockStateParams = {
+  runId: string;
+  /** Edge ids to delete. Omit to clear every edge for the run. */
+  edgeIds?: string[];
+  /**
+   * Forwarded verbatim on the full-clear leg only, and never on the bounded leg
+   * or an edge write. A routing store strips it; a single store joins it.
+   */
+  tx?: PrismaClientOrTransaction;
+};

From cef4d700ae0dac21f24f37ed5e3515427e141560 Mon Sep 17 00:00:00 2001
From: Dan Sutton <dansutton@trigger.dev>
Date: Fri, 21 Aug 2026 13:26:28 +0100
Subject: [PATCH 39/49] refactor(run-engine): move the run block-state read
 behind the coordinator

---
 .../src/engine/systems/waitpointSystem.ts     | 15 +--------------
 .../legacyPostgresCoordinator.ts              | 19 ++++++++++++++++++-
 .../src/engine/waitpointCoordinator/types.ts  | 16 +++++++++++++++-
 3 files changed, 34 insertions(+), 16 deletions(-)

diff --git a/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts b/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts
index d43e811c6fd..752dbe90da4 100644
--- a/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts
+++ b/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts
@@ -683,20 +683,7 @@ export class WaitpointSystem {
 
     return await this.$.runLock.lock("continueRunIfUnblocked", [runId], async () => {
       // 1. Get the any blocking waitpoints
-      const blockingWaitpoints = await this.$.runStore.findManyTaskRunWaitpoints(
-        {
-          where: { taskRunId: runId },
-          select: {
-            id: true,
-            batchId: true,
-            batchIndex: true,
-            waitpoint: {
-              select: { id: true, status: true, type: true, completedAfter: true },
-            },
-          },
-        },
-        this.$.prisma
-      );
+      const blockingWaitpoints = await this.coordinator.readRunBlockState(runId);
 
       // 2. There are blockers still, so do nothing
       if (blockingWaitpoints.some((w) => w.waitpoint.status !== "COMPLETED")) {
diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts
index b2e6aadb26b..b7ca49f1541 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts
@@ -2,7 +2,7 @@ import type { RunStore } from "@internal/run-store";
 import type { Logger } from "@trigger.dev/core/logger";
 import type { PrismaClient } from "@trigger.dev/database";
 import { boundedIn } from "@trigger.dev/database";
-import type { ClearRunBlockStateParams, WaitpointCoordinator } from "./types.js";
+import type { ClearRunBlockStateParams, RunBlockEdge, WaitpointCoordinator } from "./types.js";
 
 export type LegacyPostgresWaitpointCoordinatorOptions = {
   runStore: RunStore;
@@ -48,4 +48,21 @@ export class LegacyPostgresWaitpointCoordinator implements WaitpointCoordinator
     // passed through: a routing store strips it, and a single store joins it.
     return this.runStore.deleteManyTaskRunWaitpoints({ where: { taskRunId: runId } }, tx);
   }
+
+  async readRunBlockState(runId: string): Promise<RunBlockEdge[]> {
+    return this.runStore.findManyTaskRunWaitpoints(
+      {
+        where: { taskRunId: runId },
+        select: {
+          id: true,
+          batchId: true,
+          batchIndex: true,
+          waitpoint: {
+            select: { id: true, status: true, type: true, completedAfter: true },
+          },
+        },
+      },
+      this.prisma
+    );
+  }
 }
diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts
index 8db4b986bbe..f4c63063893 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts
@@ -1,4 +1,4 @@
-import type { PrismaClientOrTransaction } from "@trigger.dev/database";
+import type { PrismaClientOrTransaction, Waitpoint } from "@trigger.dev/database";
 
 /**
  * The waitpoint and edge state operations that `WaitpointSystem` delegates.
@@ -14,6 +14,7 @@ import type { PrismaClientOrTransaction } from "@trigger.dev/database";
  */
 export type WaitpointCoordinator = {
   clearRunBlockState(params: ClearRunBlockStateParams): Promise<{ count: number }>;
+  readRunBlockState(runId: string): Promise<RunBlockEdge[]>;
 };
 
 export type ClearRunBlockStateParams = {
@@ -26,3 +27,16 @@ export type ClearRunBlockStateParams = {
    */
   tx?: PrismaClientOrTransaction;
 };
+
+/**
+ * One block edge, with the fields the unblock decision reads.
+ *
+ * `batchId` is read by no logic. It rides inside two `logger.debug` payloads
+ * (`waitpointSystem.ts:702-705` and `:936-939`), so removing it changes log output.
+ */
+export type RunBlockEdge = {
+  id: string;
+  batchId: string | null;
+  batchIndex: number | null;
+  waitpoint: Pick<Waitpoint, "id" | "status" | "type" | "completedAfter">;
+};

From 3da076dcd84191a3e1e2e9fa9e1c0034de52280b Mon Sep 17 00:00:00 2001
From: Dan Sutton <dansutton@trigger.dev>
Date: Fri, 21 Aug 2026 13:37:49 +0100
Subject: [PATCH 40/49] refactor(run-engine): move block-edge registration
 behind the coordinator

---
 .../src/engine/systems/waitpointSystem.ts     | 24 +++-----
 .../legacyPostgresCoordinator.ts              | 59 ++++++++++++++++++-
 .../src/engine/waitpointCoordinator/types.ts  | 25 ++++++++
 3 files changed, 92 insertions(+), 16 deletions(-)

diff --git a/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts b/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts
index 752dbe90da4..2a9b1d55ac9 100644
--- a/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts
+++ b/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts
@@ -490,25 +490,19 @@ export class WaitpointSystem {
         this.$.runStore
       );
 
-      // Insert the blocking + historical connections via the run-ops store, routed by the owning
-      // run id so the edge co-resides with the run. Never pinned to the caller's control-plane tx:
-      // that joined `Waitpoint` on the wrong DB and wrote 0 edges. The pending check stays a
-      // SEPARATE store call so it gets its own READ COMMITTED snapshot (see the doc comment above).
-      await this.$.runStore.blockRunWithWaitpointEdges({
+      // Insert the blocking + historical connections and re-check the pending count. The
+      // coordinator keeps these as two separate store statements, in this order, for the READ
+      // COMMITTED reason documented on the method and in the doc comment above.
+      const { pendingCount } = await this.coordinator.registerBlocks({
         runId,
         waitpointIds: $waitpoints,
         projectId,
         spanIdToComplete,
         batchId: batch?.id,
         batchIndex: batch?.index,
+        client: prisma,
       });
 
-      // Check if the run is actually blocked using a separate query (see above). Pass the writer so the
-      // pending re-read is read-your-writes on the owning PRIMARY (a lagging replica can strand the run).
-      // Route by the blocked run id: its blocking waitpoints co-locate with the run, so the router
-      // counts on the run's store and only falls back to the other DB for a cross-tree token.
-      const pendingCount = await this.$.runStore.countPendingWaitpoints($waitpoints, prisma, runId);
-
       const isRunBlocked = pendingCount > 0;
 
       let newStatus: TaskRunExecutionStatus = "SUSPENDED";
@@ -606,10 +600,10 @@ export class WaitpointSystem {
   }): Promise<void> {
     const $waitpoints = typeof waitpoints === "string" ? [waitpoints] : waitpoints;
 
-    // Same routed edge write as blockRunWithWaitpoint, routed by the owning run id. No lock
-    // needed: ON CONFLICT DO NOTHING makes concurrent inserts safe, and the parent snapshot is
-    // already EXECUTING_WITH_WAITPOINTS from blockRunWithCreatedBatch.
-    await this.$.runStore.blockRunWithWaitpointEdges({
+    // Same routed edge write as blockRunWithWaitpoint. No lock needed: ON CONFLICT DO NOTHING
+    // makes concurrent inserts safe, and the parent snapshot is already
+    // EXECUTING_WITH_WAITPOINTS from blockRunWithCreatedBatch. No pending count here.
+    await this.coordinator.registerBlocksLockless({
       runId,
       waitpointIds: $waitpoints,
       projectId,
diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts
index b7ca49f1541..d310c4e2b9f 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts
@@ -2,7 +2,13 @@ import type { RunStore } from "@internal/run-store";
 import type { Logger } from "@trigger.dev/core/logger";
 import type { PrismaClient } from "@trigger.dev/database";
 import { boundedIn } from "@trigger.dev/database";
-import type { ClearRunBlockStateParams, RunBlockEdge, WaitpointCoordinator } from "./types.js";
+import type {
+  ClearRunBlockStateParams,
+  RegisterBlocksLocklessParams,
+  RegisterBlocksParams,
+  RunBlockEdge,
+  WaitpointCoordinator,
+} from "./types.js";
 
 export type LegacyPostgresWaitpointCoordinatorOptions = {
   runStore: RunStore;
@@ -65,4 +71,55 @@ export class LegacyPostgresWaitpointCoordinator implements WaitpointCoordinator
       this.prisma
     );
   }
+
+  async registerBlocks({
+    client,
+    ...edge
+  }: RegisterBlocksParams): Promise<{ pendingCount: number }> {
+    await this.#writeBlockEdges(edge);
+
+    // Check if the run is actually blocked using a separate query. The separate statement is the
+    // point: under PostgreSQL READ COMMITTED each statement gets its own snapshot, so a
+    // concurrent completion that commits between the edge write and this check is still seen.
+    // It queries ALL requested ids, not just inserted ones: a row that already existed (ON
+    // CONFLICT skipped the insert) but is still PENDING must still block. Pass the caller's
+    // client so the re-read is read-your-writes on the owning PRIMARY, and pass the run id so
+    // the router counts on the run's store instead of fanning out to both DBs.
+    const pendingCount = await this.runStore.countPendingWaitpoints(
+      edge.waitpointIds,
+      client,
+      edge.runId
+    );
+
+    return { pendingCount };
+  }
+
+  async registerBlocksLockless(params: RegisterBlocksLocklessParams): Promise<void> {
+    await this.#writeBlockEdges(params);
+  }
+
+  /**
+   * The edge write, shared by both register paths so they cannot drift.
+   *
+   * Routed by the owning run id so the edge co-resides with the run. Never pinned to a caller
+   * transaction: that joined `Waitpoint` on the wrong DB, wrote 0 edges, and silently never
+   * suspended the parent. The write is idempotent (ON CONFLICT DO NOTHING).
+   */
+  #writeBlockEdges({
+    runId,
+    waitpointIds,
+    projectId,
+    spanIdToComplete,
+    batchId,
+    batchIndex,
+  }: RegisterBlocksLocklessParams): Promise<void> {
+    return this.runStore.blockRunWithWaitpointEdges({
+      runId,
+      waitpointIds,
+      projectId,
+      spanIdToComplete,
+      batchId,
+      batchIndex,
+    });
+  }
 }
diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts
index f4c63063893..faaa77f40ed 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts
@@ -1,3 +1,4 @@
+import type { ReadClient } from "@internal/run-store";
 import type { PrismaClientOrTransaction, Waitpoint } from "@trigger.dev/database";
 
 /**
@@ -15,6 +16,8 @@ import type { PrismaClientOrTransaction, Waitpoint } from "@trigger.dev/database
 export type WaitpointCoordinator = {
   clearRunBlockState(params: ClearRunBlockStateParams): Promise<{ count: number }>;
   readRunBlockState(runId: string): Promise<RunBlockEdge[]>;
+  registerBlocks(params: RegisterBlocksParams): Promise<{ pendingCount: number }>;
+  registerBlocksLockless(params: RegisterBlocksLocklessParams): Promise<void>;
 };
 
 export type ClearRunBlockStateParams = {
@@ -40,3 +43,25 @@ export type RunBlockEdge = {
   batchIndex: number | null;
   waitpoint: Pick<Waitpoint, "id" | "status" | "type" | "completedAfter">;
 };
+
+export type RegisterBlocksParams = {
+  runId: string;
+  waitpointIds: string[];
+  projectId: string;
+  spanIdToComplete?: string;
+  batchId?: string;
+  batchIndex?: number;
+  /**
+   * Read client for the pending count only. The caller resolves `tx ?? prisma` once
+   * and passes the result, so the writer is used when the caller is inside a
+   * transaction and the pending re-read is read-your-writes on the owning primary.
+   * Never forwarded to the edge write.
+   */
+  client: ReadClient;
+};
+
+/**
+ * The lockless variant writes the edge and does not count. Two methods rather than
+ * one method with a flag, so "the batch path issues no extra query" is structural.
+ */
+export type RegisterBlocksLocklessParams = Omit<RegisterBlocksParams, "client">;

From ee58cd886134868f7e9cf36f7c5aba81b810955e Mon Sep 17 00:00:00 2001
From: Dan Sutton <dansutton@trigger.dev>
Date: Fri, 21 Aug 2026 13:47:49 +0100
Subject: [PATCH 41/49] refactor(run-engine): move waitpoint completion behind
 the coordinator

---
 .../src/engine/systems/waitpointSystem.ts     | 81 ++-----------------
 .../legacyPostgresCoordinator.ts              | 78 ++++++++++++++++++
 .../src/engine/waitpointCoordinator/types.ts  | 22 +++++
 3 files changed, 106 insertions(+), 75 deletions(-)

diff --git a/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts b/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts
index 2a9b1d55ac9..fc2372b53a1 100644
--- a/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts
+++ b/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts
@@ -1,4 +1,4 @@
-import { timeoutError, tryCatch } from "@trigger.dev/core/v3";
+import { timeoutError } from "@trigger.dev/core/v3";
 import { WaitpointId } from "@trigger.dev/core/v3/isomorphic";
 import type {
   PrismaClientOrTransaction,
@@ -8,10 +8,8 @@ import type {
   Waitpoint,
 } from "@trigger.dev/database";
 import { Prisma } from "@trigger.dev/database";
-import type { RunStore } from "@internal/run-store";
 import { assertNever } from "assert-never";
 import { nanoid } from "nanoid";
-import { UnclassifiableWaitpointId } from "../errors.js";
 import { sendNotificationToWorker } from "../eventBus.js";
 import { isFinalRunStatus } from "../statuses.js";
 import { LegacyPostgresWaitpointCoordinator } from "../waitpointCoordinator/legacyPostgresCoordinator.js";
@@ -85,86 +83,19 @@ export class WaitpointSystem {
       isError: boolean;
     };
   }): Promise<Waitpoint> {
-    // Residency store-selection guard. completeWaitpoint arrives with only
-    // (waitpointId, output) — no run id — so the owning run-ops store is selected
-    // by the waitpoint's own residency. In single-DB this is the one store
-    // (no classification). An unclassifiable id throws loud — never default-routes.
-    let store: RunStore;
-    try {
-      store = await this.$.runStore.forWaitpointCompletion(id, { routeKind: "MANUAL" });
-    } catch (error) {
-      this.$.logger.error("completeWaitpoint: unclassifiable waitpointId", {
-        waitpointId: id,
-        error,
-      });
-      throw new UnclassifiableWaitpointId(id, { cause: error });
-    }
-
-    // 1. Complete the Waitpoint (if not completed)
-    const [updateError, updateResult] = await tryCatch(
-      store.updateManyWaitpoints({
-        where: { id, status: "PENDING" },
-        data: {
-          status: "COMPLETED",
-          completedAt: new Date(),
-          output: output?.value,
-          outputType: output?.type,
-          outputIsError: output?.isError,
-        },
-      })
-    );
-
-    if (updateError) {
-      this.$.logger.error("completeWaitpoint: error updating waitpoint:", { updateError });
-      throw updateError;
-    }
-
-    if (updateResult.count === 0) {
-      this.$.logger.info(
-        "completeWaitpoint: attempted to complete a waitpoint that is not PENDING",
-        { waitpointId: id }
-      );
-    }
-
-    // Re-read the just-written row from the RESOLVED store's PRIMARY: the replica (findWaitpoint's
-    // default) can miss it under lag → false "not found" → the parent hangs; this.$.prisma would
-    // instead hit the wrong DB. findWaitpointOnPrimary reads the owning store's primary.
-    const waitpoint = await store.findWaitpointOnPrimary({
-      where: { id },
+    const { waitpoint, blockedRuns } = await this.coordinator.complete({
+      waitpointId: id,
+      output,
     });
 
-    if (!waitpoint) {
-      this.$.logger.error("completeWaitpoint: waitpoint not found", { waitpointId: id });
-      throw new Error("Waitpoint not found");
-    }
-
-    if (waitpoint.status !== "COMPLETED") {
-      this.$.logger.error(`completeWaitpoint: waitpoint is not completed`, {
-        waitpointId: id,
-      });
-      throw new Error("Waitpoint not completed");
-    }
-
-    // 2. Find the TaskRuns blocked by this waitpoint. The edge (TaskRunWaitpoint) co-locates
-    // with its RUN, not this token, so it can live on the OTHER run-ops DB: read via the router
-    // (which fans the waitpointId lookup across both DBs) rather than the token's own `store`,
-    // or a cross-DB blocked run is never found and hangs forever.
-    const affectedTaskRuns = await this.$.runStore.findManyTaskRunWaitpoints(
-      {
-        where: { waitpointId: id },
-        select: { taskRunId: true, spanIdToComplete: true, createdAt: true },
-      },
-      this.$.prisma
-    );
-
-    if (affectedTaskRuns.length === 0) {
+    if (blockedRuns.length === 0) {
       this.$.logger.debug(`completeWaitpoint: no TaskRunWaitpoints found for waitpoint`, {
         waitpointId: id,
       });
     }
 
     // 3. Schedule trying to continue the runs
-    for (const run of affectedTaskRuns) {
+    for (const run of blockedRuns) {
       const jobId = `continueRunIfUnblocked:${run.taskRunId}`;
       //50ms in the future
       const availableAt = new Date(Date.now() + 50);
diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts
index d310c4e2b9f..de1cebb51c7 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts
@@ -1,9 +1,13 @@
 import type { RunStore } from "@internal/run-store";
+import { tryCatch } from "@trigger.dev/core/v3";
 import type { Logger } from "@trigger.dev/core/logger";
 import type { PrismaClient } from "@trigger.dev/database";
 import { boundedIn } from "@trigger.dev/database";
+import { UnclassifiableWaitpointId } from "../errors.js";
 import type {
   ClearRunBlockStateParams,
+  CompleteParams,
+  CompleteResult,
   RegisterBlocksLocklessParams,
   RegisterBlocksParams,
   RunBlockEdge,
@@ -98,6 +102,80 @@ export class LegacyPostgresWaitpointCoordinator implements WaitpointCoordinator
     await this.#writeBlockEdges(params);
   }
 
+  async complete({ waitpointId, output }: CompleteParams): Promise<CompleteResult> {
+    // Residency store-selection guard. complete arrives with only (waitpointId, output) — no run
+    // id — so the owning run-ops store is selected by the waitpoint's own residency. In single-DB
+    // this is the one store (no classification). An unclassifiable id throws loud — never
+    // default-routes. The try wraps ONLY the resolve: widening it would swallow the
+    // "Waitpoint not found" path that a single store relies on.
+    let store: RunStore;
+    try {
+      store = await this.runStore.forWaitpointCompletion(waitpointId, { routeKind: "MANUAL" });
+    } catch (error) {
+      this.logger.error("completeWaitpoint: unclassifiable waitpointId", {
+        waitpointId,
+        error,
+      });
+      throw new UnclassifiableWaitpointId(waitpointId, { cause: error });
+    }
+
+    // 1. Complete the Waitpoint (if not completed)
+    const [updateError, updateResult] = await tryCatch(
+      store.updateManyWaitpoints({
+        where: { id: waitpointId, status: "PENDING" },
+        data: {
+          status: "COMPLETED",
+          completedAt: new Date(),
+          output: output?.value,
+          outputType: output?.type,
+          outputIsError: output?.isError,
+        },
+      })
+    );
+
+    if (updateError) {
+      this.logger.error("completeWaitpoint: error updating waitpoint:", { updateError });
+      throw updateError;
+    }
+
+    if (updateResult.count === 0) {
+      this.logger.info("completeWaitpoint: attempted to complete a waitpoint that is not PENDING", {
+        waitpointId,
+      });
+    }
+
+    // Re-read the just-written row from the RESOLVED store's PRIMARY: the replica (findWaitpoint's
+    // default) can miss it under lag → false "not found" → the parent hangs. Going back through
+    // the router would re-resolve the store and change the routing, so use the handle.
+    const waitpoint = await store.findWaitpointOnPrimary({
+      where: { id: waitpointId },
+    });
+
+    if (!waitpoint) {
+      this.logger.error("completeWaitpoint: waitpoint not found", { waitpointId });
+      throw new Error("Waitpoint not found");
+    }
+
+    if (waitpoint.status !== "COMPLETED") {
+      this.logger.error(`completeWaitpoint: waitpoint is not completed`, { waitpointId });
+      throw new Error("Waitpoint not completed");
+    }
+
+    // 2. Find the TaskRuns blocked by this waitpoint. The edge (TaskRunWaitpoint) co-locates
+    // with its RUN, not this token, so it can live on the OTHER run-ops DB: read via the router
+    // (which fans the waitpointId lookup across both DBs) rather than the token's own `store`,
+    // or a cross-DB blocked run is never found and hangs forever.
+    const blockedRuns = await this.runStore.findManyTaskRunWaitpoints(
+      {
+        where: { waitpointId },
+        select: { taskRunId: true, spanIdToComplete: true, createdAt: true },
+      },
+      this.prisma
+    );
+
+    return { waitpoint, blockedRuns };
+  }
+
   /**
    * The edge write, shared by both register paths so they cannot drift.
    *
diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts
index faaa77f40ed..e7b9e53a1a3 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts
@@ -18,6 +18,7 @@ export type WaitpointCoordinator = {
   readRunBlockState(runId: string): Promise<RunBlockEdge[]>;
   registerBlocks(params: RegisterBlocksParams): Promise<{ pendingCount: number }>;
   registerBlocksLockless(params: RegisterBlocksLocklessParams): Promise<void>;
+  complete(params: CompleteParams): Promise<CompleteResult>;
 };
 
 export type ClearRunBlockStateParams = {
@@ -65,3 +66,24 @@ export type RegisterBlocksParams = {
  * one method with a flag, so "the batch path issues no extra query" is structural.
  */
 export type RegisterBlocksLocklessParams = Omit<RegisterBlocksParams, "client">;
+
+export type CompleteParams = {
+  waitpointId: string;
+  output?: {
+    value: string;
+    type?: string;
+    isError: boolean;
+  };
+};
+
+/** One run blocked by the completed waitpoint, with the fields the caller's fan-out loop reads. */
+export type BlockedRun = {
+  taskRunId: string;
+  spanIdToComplete: string | null;
+  createdAt: Date;
+};
+
+export type CompleteResult = {
+  waitpoint: Waitpoint;
+  blockedRuns: BlockedRun[];
+};

From d55610f3328586cd5f9e3d1ab9c543f44cf4c66b Mon Sep 17 00:00:00 2001
From: Dan Sutton <dansutton@trigger.dev>
Date: Fri, 21 Aug 2026 13:59:27 +0100
Subject: [PATCH 42/49] refactor(run-engine): move waitpoint creation and
 minting behind the coordinator

---
 .../src/engine/systems/waitpointSystem.ts     | 232 +++--------------
 .../legacyPostgresCoordinator.ts              | 237 +++++++++++++++++-
 .../src/engine/waitpointCoordinator/types.ts  |  57 +++++
 3 files changed, 333 insertions(+), 193 deletions(-)

diff --git a/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts b/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts
index fc2372b53a1..3dbed999445 100644
--- a/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts
+++ b/internal-packages/run-engine/src/engine/systems/waitpointSystem.ts
@@ -1,5 +1,4 @@
 import { timeoutError } from "@trigger.dev/core/v3";
-import { WaitpointId } from "@trigger.dev/core/v3/isomorphic";
 import type {
   PrismaClientOrTransaction,
   TaskRun,
@@ -7,9 +6,7 @@ import type {
   TaskRunExecutionStatus,
   Waitpoint,
 } from "@trigger.dev/database";
-import { Prisma } from "@trigger.dev/database";
 import { assertNever } from "assert-never";
-import { nanoid } from "nanoid";
 import { sendNotificationToWorker } from "../eventBus.js";
 import { isFinalRunStatus } from "../statuses.js";
 import { LegacyPostgresWaitpointCoordinator } from "../waitpointCoordinator/legacyPostgresCoordinator.js";
@@ -152,81 +149,27 @@ export class WaitpointSystem {
     idempotencyKey?: string;
     idempotencyKeyExpiresAt?: Date;
   }) {
-    // Co-location invariant: a DATETIME wait waitpoint lives on the same run-ops DB as the run that
-    // blocks on it (so the block edge's local `Waitpoint` join resolves and completion/resume stay
-    // local). The minted waitpoint id is always a cuid, so without `coLocateWithRunId` the upsert
-    // would always route to LEGACY and a run-ops run on NEW would hang. The (env,idempotencyKey) dedup
-    // is within the owning run/tree (co-resident on one DB), so the dedup probe + rotation target the
-    // SAME store. With no run id (a standalone token has no owning run yet) the lookup falls back to
-    // a cross-DB NEW-then-LEGACY scan and the upsert routes by id-shape. Always routed through the
-    // run store (never a caller tx) so it can never bypass residency onto the wrong DB.
-    const colocate = runId ? { coLocateWithRunId: runId } : undefined;
-    const existingWaitpoint = idempotencyKey
-      ? await this.$.runStore.findWaitpoint(
-          {
-            where: {
-              environmentId,
-              idempotencyKey,
-            },
-          },
-          undefined,
-          colocate
-        )
-      : undefined;
-
-    if (existingWaitpoint) {
-      if (
-        existingWaitpoint.idempotencyKeyExpiresAt &&
-        new Date() > existingWaitpoint.idempotencyKeyExpiresAt
-      ) {
-        //the idempotency key has expired
-        //remove the waitpoint idempotencyKey
-        const rotateArgs = {
-          where: {
-            id: existingWaitpoint.id,
-          },
-          data: {
-            idempotencyKey: nanoid(24),
-            inactiveIdempotencyKey: existingWaitpoint.idempotencyKey,
-          },
-        };
-        await this.$.runStore.updateWaitpoint(rotateArgs, undefined, colocate);
+    const result = await this.coordinator.createDateTimeWaitpoint({
+      runId,
+      projectId,
+      environmentId,
+      completedAfter,
+      idempotencyKey,
+      idempotencyKeyExpiresAt,
+    });
 
-        //let it fall through to create a new waitpoint
-      } else {
-        return { waitpoint: existingWaitpoint, isCached: true };
-      }
+    if (result.kind === "cached") {
+      return { waitpoint: result.waitpoint, isCached: true };
     }
 
-    const upsertArgs = {
-      where: {
-        environmentId_idempotencyKey: {
-          environmentId,
-          idempotencyKey: idempotencyKey ?? nanoid(24),
-        },
-      },
-      create: {
-        ...WaitpointId.generate(),
-        type: "DATETIME" as const,
-        idempotencyKey: idempotencyKey ?? nanoid(24),
-        idempotencyKeyExpiresAt,
-        userProvidedIdempotencyKey: !!idempotencyKey,
-        environmentId,
-        projectId,
-        completedAfter,
-      },
-      update: {},
-    };
-    const waitpoint = await this.$.runStore.upsertWaitpoint(upsertArgs, undefined, colocate);
-
     await this.$.worker.enqueue({
-      id: `finishWaitpoint.${waitpoint.id}`,
+      id: `finishWaitpoint.${result.waitpoint.id}`,
       job: "finishWaitpoint",
-      payload: { waitpointId: waitpoint.id },
+      payload: { waitpointId: result.waitpoint.id },
       availableAt: completedAfter,
     });
 
-    return { waitpoint, isCached: false };
+    return { waitpoint: result.waitpoint, isCached: false };
   }
 
   /** This creates a MANUAL waitpoint, that can be explicitly completed (or failed).
@@ -254,117 +197,35 @@ export class WaitpointSystem {
     // to LEGACY by its cuid id-shape. Ignored when `runId` is set (co-location wins).
     standaloneResidency?: "NEW" | "LEGACY";
   }): Promise<{ waitpoint: Waitpoint; isCached: boolean }> {
-    // Co-location invariant (see createDateTimeWaitpoint): when a `runId` is supplied the waitpoint
-    // co-locates with that run's DB and the (env,idempotencyKey) dedup is per-run (co-resident). A
-    // standalone token (api.v1.waitpoints.tokens.ts) passes no run id — it is created without an
-    // owner, blocked later by whichever run waits on it (possibly cross-DB, resolved by the
-    // run-co-resident block edge + completion fan-out). With no owner it reads the env mint kind via
-    // `standaloneResidency` so a minted-new env keeps its tokens on NEW; unset, it routes by id-shape. No tx here.
-    const colocate = runId
-      ? { coLocateWithRunId: runId }
-      : standaloneResidency
-        ? { residency: standaloneResidency }
-        : undefined;
-    const existingWaitpoint = idempotencyKey
-      ? await this.$.runStore.findWaitpoint(
-          {
-            where: {
-              environmentId,
-              idempotencyKey,
-            },
-          },
-          undefined,
-          colocate
-        )
-      : undefined;
-
-    if (existingWaitpoint) {
-      if (
-        existingWaitpoint.idempotencyKeyExpiresAt &&
-        new Date() > existingWaitpoint.idempotencyKeyExpiresAt
-      ) {
-        //the idempotency key has expired
-        //remove the waitpoint idempotencyKey
-        await this.$.runStore.updateWaitpoint(
-          {
-            where: {
-              id: existingWaitpoint.id,
-            },
-            data: {
-              idempotencyKey: nanoid(24),
-              inactiveIdempotencyKey: existingWaitpoint.idempotencyKey,
-            },
-          },
-          undefined,
-          colocate
-        );
+    const result = await this.coordinator.createManualWaitpoint({
+      runId,
+      environmentId,
+      projectId,
+      idempotencyKey,
+      idempotencyKeyExpiresAt,
+      timeout,
+      tags,
+      standaloneResidency,
+    });
 
-        //let it fall through to create a new waitpoint
-      } else {
-        return { waitpoint: existingWaitpoint, isCached: true };
-      }
+    if (result.kind === "cached") {
+      return { waitpoint: result.waitpoint, isCached: true };
     }
 
-    const maxRetries = 5;
-    let attempts = 0;
-
-    while (attempts < maxRetries) {
-      try {
-        const waitpoint = await this.$.runStore.upsertWaitpoint(
-          {
-            where: {
-              environmentId_idempotencyKey: {
-                environmentId,
-                idempotencyKey: idempotencyKey ?? nanoid(24),
-              },
-            },
-            create: {
-              ...WaitpointId.generate(),
-              type: "MANUAL",
-              idempotencyKey: idempotencyKey ?? nanoid(24),
-              idempotencyKeyExpiresAt,
-              userProvidedIdempotencyKey: !!idempotencyKey,
-              environmentId,
-              projectId,
-              completedAfter: timeout,
-              tags,
-            },
-            update: {},
-          },
-          undefined,
-          colocate
-        );
-
-        //schedule the timeout
-        if (timeout) {
-          await this.$.worker.enqueue({
-            id: `finishWaitpoint.${waitpoint.id}`,
-            job: "finishWaitpoint",
-            payload: {
-              waitpointId: waitpoint.id,
-              error: JSON.stringify(timeoutError(timeout)),
-            },
-            availableAt: timeout,
-          });
-        }
-
-        return { waitpoint, isCached: false };
-      } catch (error) {
-        if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === "P2002") {
-          // Handle unique constraint violation (conflict)
-          attempts++;
-          if (attempts >= maxRetries) {
-            throw new Error(
-              `Failed to create waitpoint after ${maxRetries} attempts due to conflicts.`
-            );
-          }
-        } else {
-          throw error; // Re-throw other errors
-        }
-      }
+    //schedule the timeout
+    if (timeout) {
+      await this.$.worker.enqueue({
+        id: `finishWaitpoint.${result.waitpoint.id}`,
+        job: "finishWaitpoint",
+        payload: {
+          waitpointId: result.waitpoint.id,
+          error: JSON.stringify(timeoutError(timeout)),
+        },
+        availableAt: timeout,
+      });
     }
 
-    throw new Error(`Failed to create waitpoint after ${maxRetries} attempts due to conflicts.`);
+    return { waitpoint: result.waitpoint, isCached: false };
   }
 
   /**
@@ -864,15 +725,7 @@ export class WaitpointSystem {
     projectId: string;
     environmentId: string;
   }) {
-    return {
-      ...WaitpointId.generate(),
-      type: "RUN" as const,
-      status: "PENDING" as const,
-      idempotencyKey: nanoid(24),
-      userProvidedIdempotencyKey: false,
-      projectId,
-      environmentId,
-    };
+    return this.coordinator.mintAssociatedWaitpointData({ projectId, environmentId });
   }
 
   /**
@@ -956,12 +809,9 @@ export class WaitpointSystem {
       // Create waitpoint and link to run atomically
       const waitpointData = this.buildRunAssociatedWaitpoint({ projectId, environmentId });
 
-      // RUN-type within-tree waitpoint that belongs to runId; routes by owning run id.
-      const waitpoint = await this.$.runStore.createWaitpoint({
-        data: {
-          ...waitpointData,
-          completedByTaskRunId: runId,
-        },
+      const waitpoint = await this.coordinator.createAssociatedWaitpoint({
+        runId,
+        data: waitpointData,
       });
 
       // If run has already finished (per snapshot), complete the waitpoint immediately so the parent can resume
diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts
index de1cebb51c7..97dc8055445 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts
@@ -1,13 +1,19 @@
 import type { RunStore } from "@internal/run-store";
 import { tryCatch } from "@trigger.dev/core/v3";
+import { WaitpointId } from "@trigger.dev/core/v3/isomorphic";
 import type { Logger } from "@trigger.dev/core/logger";
-import type { PrismaClient } from "@trigger.dev/database";
-import { boundedIn } from "@trigger.dev/database";
+import type { PrismaClient, Waitpoint } from "@trigger.dev/database";
+import { boundedIn, Prisma } from "@trigger.dev/database";
+import { nanoid } from "nanoid";
 import { UnclassifiableWaitpointId } from "../errors.js";
 import type {
+  AssociatedWaitpointData,
   ClearRunBlockStateParams,
   CompleteParams,
   CompleteResult,
+  CreateDateTimeWaitpointParams,
+  CreateManualWaitpointParams,
+  CreateWaitpointResult,
   RegisterBlocksLocklessParams,
   RegisterBlocksParams,
   RunBlockEdge,
@@ -176,6 +182,233 @@ export class LegacyPostgresWaitpointCoordinator implements WaitpointCoordinator
     return { waitpoint, blockedRuns };
   }
 
+  async createDateTimeWaitpoint({
+    runId,
+    projectId,
+    environmentId,
+    completedAfter,
+    idempotencyKey,
+    idempotencyKeyExpiresAt,
+  }: CreateDateTimeWaitpointParams): Promise<CreateWaitpointResult> {
+    // Co-location invariant: a DATETIME wait waitpoint lives on the same run-ops DB as the run
+    // that blocks on it. The minted waitpoint id is always a cuid, so without `coLocateWithRunId`
+    // the upsert would always route to LEGACY and a run-ops run on NEW would hang. The
+    // (env,idempotencyKey) dedup is within the owning run/tree, so the dedup probe + rotation
+    // target the SAME store. With no run id the lookup falls back to a cross-DB NEW-then-LEGACY
+    // scan and the upsert routes by id-shape. Always routed through the run store (never a caller
+    // tx) so it can never bypass residency onto the wrong DB.
+    const colocate = runId ? { coLocateWithRunId: runId } : undefined;
+    const existingWaitpoint = idempotencyKey
+      ? await this.runStore.findWaitpoint(
+          {
+            where: {
+              environmentId,
+              idempotencyKey,
+            },
+          },
+          undefined,
+          colocate
+        )
+      : undefined;
+
+    if (existingWaitpoint) {
+      if (
+        existingWaitpoint.idempotencyKeyExpiresAt &&
+        new Date() > existingWaitpoint.idempotencyKeyExpiresAt
+      ) {
+        //the idempotency key has expired
+        //remove the waitpoint idempotencyKey
+        const rotateArgs = {
+          where: {
+            id: existingWaitpoint.id,
+          },
+          data: {
+            idempotencyKey: nanoid(24),
+            inactiveIdempotencyKey: existingWaitpoint.idempotencyKey,
+          },
+        };
+        await this.runStore.updateWaitpoint(rotateArgs, undefined, colocate);
+
+        //let it fall through to create a new waitpoint
+      } else {
+        return { kind: "cached", waitpoint: existingWaitpoint };
+      }
+    }
+
+    // The two `nanoid(24)` calls below are deliberately separate and produce DIFFERENT values:
+    // the upsert `where` key must not match the `create` key, or a guaranteed-miss upsert becomes
+    // a possible update. Do not hoist either to a shared constant.
+    const upsertArgs = {
+      where: {
+        environmentId_idempotencyKey: {
+          environmentId,
+          idempotencyKey: idempotencyKey ?? nanoid(24),
+        },
+      },
+      create: {
+        ...WaitpointId.generate(),
+        type: "DATETIME" as const,
+        idempotencyKey: idempotencyKey ?? nanoid(24),
+        idempotencyKeyExpiresAt,
+        userProvidedIdempotencyKey: !!idempotencyKey,
+        environmentId,
+        projectId,
+        completedAfter,
+      },
+      update: {},
+    };
+    const waitpoint = await this.runStore.upsertWaitpoint(upsertArgs, undefined, colocate);
+
+    return { kind: "created", waitpoint };
+  }
+
+  async createManualWaitpoint({
+    runId,
+    environmentId,
+    projectId,
+    idempotencyKey,
+    idempotencyKeyExpiresAt,
+    timeout,
+    tags,
+    standaloneResidency,
+  }: CreateManualWaitpointParams): Promise<CreateWaitpointResult> {
+    // Co-location invariant (see createDateTimeWaitpoint): when a `runId` is supplied the
+    // waitpoint co-locates with that run's DB and the (env,idempotencyKey) dedup is per-run. A
+    // standalone token passes no run id — it is created without an owner, blocked later by
+    // whichever run waits on it (possibly cross-DB, resolved by the run-co-resident block edge +
+    // completion fan-out). With no owner it reads the env mint kind via `standaloneResidency` so
+    // a minted-new env keeps its tokens on NEW; unset, it routes by id-shape. No tx here.
+    const colocate = runId
+      ? { coLocateWithRunId: runId }
+      : standaloneResidency
+        ? { residency: standaloneResidency }
+        : undefined;
+    const existingWaitpoint = idempotencyKey
+      ? await this.runStore.findWaitpoint(
+          {
+            where: {
+              environmentId,
+              idempotencyKey,
+            },
+          },
+          undefined,
+          colocate
+        )
+      : undefined;
+
+    if (existingWaitpoint) {
+      if (
+        existingWaitpoint.idempotencyKeyExpiresAt &&
+        new Date() > existingWaitpoint.idempotencyKeyExpiresAt
+      ) {
+        //the idempotency key has expired
+        //remove the waitpoint idempotencyKey
+        await this.runStore.updateWaitpoint(
+          {
+            where: {
+              id: existingWaitpoint.id,
+            },
+            data: {
+              idempotencyKey: nanoid(24),
+              inactiveIdempotencyKey: existingWaitpoint.idempotencyKey,
+            },
+          },
+          undefined,
+          colocate
+        );
+
+        //let it fall through to create a new waitpoint
+      } else {
+        return { kind: "cached", waitpoint: existingWaitpoint };
+      }
+    }
+
+    const maxRetries = 5;
+    let attempts = 0;
+
+    while (attempts < maxRetries) {
+      try {
+        // As in createDateTimeWaitpoint, the two `nanoid(24)` calls are deliberately separate and
+        // differ. Both, and `WaitpointId.generate()`, are re-evaluated on every attempt: that is
+        // what makes a retry after a unique-constraint conflict try a fresh key.
+        const waitpoint = await this.runStore.upsertWaitpoint(
+          {
+            where: {
+              environmentId_idempotencyKey: {
+                environmentId,
+                idempotencyKey: idempotencyKey ?? nanoid(24),
+              },
+            },
+            create: {
+              ...WaitpointId.generate(),
+              type: "MANUAL",
+              idempotencyKey: idempotencyKey ?? nanoid(24),
+              idempotencyKeyExpiresAt,
+              userProvidedIdempotencyKey: !!idempotencyKey,
+              environmentId,
+              projectId,
+              completedAfter: timeout,
+              tags,
+            },
+            update: {},
+          },
+          undefined,
+          colocate
+        );
+
+        return { kind: "created", waitpoint };
+      } catch (error) {
+        if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === "P2002") {
+          // Handle unique constraint violation (conflict)
+          attempts++;
+          if (attempts >= maxRetries) {
+            throw new Error(
+              `Failed to create waitpoint after ${maxRetries} attempts due to conflicts.`
+            );
+          }
+        } else {
+          throw error; // Re-throw other errors
+        }
+      }
+    }
+
+    throw new Error(`Failed to create waitpoint after ${maxRetries} attempts due to conflicts.`);
+  }
+
+  mintAssociatedWaitpointData({
+    projectId,
+    environmentId,
+  }: {
+    projectId: string;
+    environmentId: string;
+  }): AssociatedWaitpointData {
+    return {
+      ...WaitpointId.generate(),
+      type: "RUN" as const,
+      status: "PENDING" as const,
+      idempotencyKey: nanoid(24),
+      userProvidedIdempotencyKey: false,
+      projectId,
+      environmentId,
+    };
+  }
+
+  async createAssociatedWaitpoint({
+    runId,
+    data,
+  }: {
+    runId: string;
+    data: AssociatedWaitpointData;
+  }): Promise<Waitpoint> {
+    // RUN-type within-tree waitpoint that belongs to runId; routes by owning run id.
+    return this.runStore.createWaitpoint({
+      data: {
+        ...data,
+        completedByTaskRunId: runId,
+      },
+    });
+  }
+
   /**
    * The edge write, shared by both register paths so they cannot drift.
    *
diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts
index e7b9e53a1a3..36b0ea4b315 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts
@@ -19,6 +19,16 @@ export type WaitpointCoordinator = {
   registerBlocks(params: RegisterBlocksParams): Promise<{ pendingCount: number }>;
   registerBlocksLockless(params: RegisterBlocksLocklessParams): Promise<void>;
   complete(params: CompleteParams): Promise<CompleteResult>;
+  createDateTimeWaitpoint(params: CreateDateTimeWaitpointParams): Promise<CreateWaitpointResult>;
+  createManualWaitpoint(params: CreateManualWaitpointParams): Promise<CreateWaitpointResult>;
+  mintAssociatedWaitpointData(params: {
+    projectId: string;
+    environmentId: string;
+  }): AssociatedWaitpointData;
+  createAssociatedWaitpoint(params: {
+    runId: string;
+    data: AssociatedWaitpointData;
+  }): Promise<Waitpoint>;
 };
 
 export type ClearRunBlockStateParams = {
@@ -87,3 +97,50 @@ export type CompleteResult = {
   waitpoint: Waitpoint;
   blockedRuns: BlockedRun[];
 };
+
+/**
+ * Discriminated on purpose. The caller enqueues the `finishWaitpoint` job only in the
+ * `created` branch, because today's create methods return before their enqueue on the
+ * cached path. A boolean would let a later edit enqueue on both branches.
+ */
+export type CreateWaitpointResult =
+  | { kind: "cached"; waitpoint: Waitpoint }
+  | { kind: "created"; waitpoint: Waitpoint };
+
+export type CreateDateTimeWaitpointParams = {
+  /** When set, the waitpoint co-locates with this run's DB and the dedup probe targets it. */
+  runId?: string;
+  projectId: string;
+  environmentId: string;
+  completedAfter: Date;
+  idempotencyKey?: string;
+  idempotencyKeyExpiresAt?: Date;
+};
+
+export type CreateManualWaitpointParams = {
+  runId?: string;
+  environmentId: string;
+  projectId: string;
+  idempotencyKey?: string;
+  idempotencyKeyExpiresAt?: Date;
+  timeout?: Date;
+  tags?: string[];
+  /**
+   * For a STANDALONE token (no owning `runId`): the residency the env's mint kind resolves
+   * to. Ignored when `runId` is set, because co-location wins. Only a Postgres
+   * implementation reads this.
+   */
+  standaloneResidency?: "NEW" | "LEGACY";
+};
+
+/** The RUN-waitpoint row data. Pure — no store touch — so the mint is coordinator-owned. */
+export type AssociatedWaitpointData = {
+  id: string;
+  friendlyId: string;
+  type: "RUN";
+  status: "PENDING";
+  idempotencyKey: string;
+  userProvidedIdempotencyKey: false;
+  projectId: string;
+  environmentId: string;
+};

From 073d655260b48f4e123aaeb3576faa175412fda2 Mon Sep 17 00:00:00 2001
From: Dan Sutton <dansutton@trigger.dev>
Date: Fri, 21 Aug 2026 14:09:25 +0100
Subject: [PATCH 43/49] refactor(run-engine): restore dropped residency comment
 clauses

---
 .../legacyPostgresCoordinator.ts              | 27 ++++++++++---------
 .../src/engine/waitpointCoordinator/types.ts  |  5 ++--
 2 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts
index 97dc8055445..d1e48fa4f8d 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/legacyPostgresCoordinator.ts
@@ -190,13 +190,14 @@ export class LegacyPostgresWaitpointCoordinator implements WaitpointCoordinator
     idempotencyKey,
     idempotencyKeyExpiresAt,
   }: CreateDateTimeWaitpointParams): Promise<CreateWaitpointResult> {
-    // Co-location invariant: a DATETIME wait waitpoint lives on the same run-ops DB as the run
-    // that blocks on it. The minted waitpoint id is always a cuid, so without `coLocateWithRunId`
-    // the upsert would always route to LEGACY and a run-ops run on NEW would hang. The
-    // (env,idempotencyKey) dedup is within the owning run/tree, so the dedup probe + rotation
-    // target the SAME store. With no run id the lookup falls back to a cross-DB NEW-then-LEGACY
-    // scan and the upsert routes by id-shape. Always routed through the run store (never a caller
-    // tx) so it can never bypass residency onto the wrong DB.
+    // Co-location invariant: a DATETIME wait waitpoint lives on the same run-ops DB as the run that
+    // blocks on it (so the block edge's local `Waitpoint` join resolves and completion/resume stay
+    // local). The minted waitpoint id is always a cuid, so without `coLocateWithRunId` the upsert
+    // would always route to LEGACY and a run-ops run on NEW would hang. The (env,idempotencyKey) dedup
+    // is within the owning run/tree (co-resident on one DB), so the dedup probe + rotation target the
+    // SAME store. With no run id (a standalone token has no owning run yet) the lookup falls back to
+    // a cross-DB NEW-then-LEGACY scan and the upsert routes by id-shape. Always routed through the
+    // run store (never a caller tx) so it can never bypass residency onto the wrong DB.
     const colocate = runId ? { coLocateWithRunId: runId } : undefined;
     const existingWaitpoint = idempotencyKey
       ? await this.runStore.findWaitpoint(
@@ -272,12 +273,12 @@ export class LegacyPostgresWaitpointCoordinator implements WaitpointCoordinator
     tags,
     standaloneResidency,
   }: CreateManualWaitpointParams): Promise<CreateWaitpointResult> {
-    // Co-location invariant (see createDateTimeWaitpoint): when a `runId` is supplied the
-    // waitpoint co-locates with that run's DB and the (env,idempotencyKey) dedup is per-run. A
-    // standalone token passes no run id — it is created without an owner, blocked later by
-    // whichever run waits on it (possibly cross-DB, resolved by the run-co-resident block edge +
-    // completion fan-out). With no owner it reads the env mint kind via `standaloneResidency` so
-    // a minted-new env keeps its tokens on NEW; unset, it routes by id-shape. No tx here.
+    // Co-location invariant (see createDateTimeWaitpoint): when a `runId` is supplied the waitpoint
+    // co-locates with that run's DB and the (env,idempotencyKey) dedup is per-run (co-resident). A
+    // standalone token (api.v1.waitpoints.tokens.ts) passes no run id — it is created without an
+    // owner, blocked later by whichever run waits on it (possibly cross-DB, resolved by the
+    // run-co-resident block edge + completion fan-out). With no owner it reads the env mint kind via
+    // `standaloneResidency` so a minted-new env keeps its tokens on NEW; unset, it routes by id-shape. No tx here.
     const colocate = runId
       ? { coLocateWithRunId: runId }
       : standaloneResidency
diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts
index 36b0ea4b315..9b89a065c1c 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts
@@ -126,9 +126,8 @@ export type CreateManualWaitpointParams = {
   timeout?: Date;
   tags?: string[];
   /**
-   * For a STANDALONE token (no owning `runId`): the residency the env's mint kind resolves
-   * to. Ignored when `runId` is set, because co-location wins. Only a Postgres
-   * implementation reads this.
+   * See the `standaloneResidency` param doc on `WaitpointSystem.createManualWaitpoint` for the
+   * full rationale. Only a Postgres implementation reads this.
    */
   standaloneResidency?: "NEW" | "LEGACY";
 };

From a913a5e58cf02e1b90dcfcd67001a0d5b814842e Mon Sep 17 00:00:00 2001
From: Dan Sutton <dansutton@trigger.dev>
Date: Fri, 21 Aug 2026 14:34:04 +0100
Subject: [PATCH 44/49] refactor(run-engine): cite the batchId log sites by
 symbol, not line number

The RunBlockEdge comment pointed at stale waitpointSystem.ts line
numbers that no longer match the file after this branch shrank it.
Name the continueRunIfUnblocked method instead so the citation can't
drift again.
---
 .../run-engine/src/engine/waitpointCoordinator/types.ts       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts
index 9b89a065c1c..bc2f02e264f 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts
@@ -45,8 +45,8 @@ export type ClearRunBlockStateParams = {
 /**
  * One block edge, with the fields the unblock decision reads.
  *
- * `batchId` is read by no logic. It rides inside two `logger.debug` payloads
- * (`waitpointSystem.ts:702-705` and `:936-939`), so removing it changes log output.
+ * `batchId` is read by no logic. It rides inside the two `logger.debug` payloads in
+ * `continueRunIfUnblocked`, so removing it changes log output.
  */
 export type RunBlockEdge = {
   id: string;

From 7b04f860141909239b3888380c32f48d9d4ab0c6 Mon Sep 17 00:00:00 2001
From: Dan Sutton <dansutton@trigger.dev>
Date: Fri, 21 Aug 2026 16:10:58 +0100
Subject: [PATCH 45/49] refactor(run-engine): stop exporting the internal
 BlockedRun type
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

`BlockedRun` is only named inside types.ts, by CompleteResult. The repo's
knip gate rejects unused exports, so drop the export keyword rather than
add a knip.json exception — nothing outside this file needs the name yet.
---
 .../run-engine/src/engine/waitpointCoordinator/types.ts         | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts
index bc2f02e264f..8a50abb7d1c 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/types.ts
@@ -87,7 +87,7 @@ export type CompleteParams = {
 };
 
 /** One run blocked by the completed waitpoint, with the fields the caller's fan-out loop reads. */
-export type BlockedRun = {
+type BlockedRun = {
   taskRunId: string;
   spanIdToComplete: string | null;
   createdAt: Date;

From 80a6dab96c9c6af19ea6a6b2c7bde57c8b99dcb3 Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Mon, 24 Aug 2026 12:41:37 +0100
Subject: [PATCH 46/49] fix(run-engine): match Redis hash-tag semantics without
 a backtracking regex

The tag scan reproduced the first NON-empty brace pair, where Redis stops at
the first pair and treats an empty one as no tag at all. The two disagreed
about the slot for a key like wp:{}{a}. The pattern also backtracked
quadratically on a key made of many opening braces, which CodeQL flagged.

Replaces it with a two-indexOf scan that mirrors keyHashSlot exactly.
---
 .../engine/waitpointCoordinator/keys.test.ts  | 21 +++++++++++++++++++
 .../src/engine/waitpointCoordinator/keys.ts   | 16 +++++++++++---
 2 files changed, 34 insertions(+), 3 deletions(-)

diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.test.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.test.ts
index 641fd4c80c4..01bd2c064aa 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.test.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.test.ts
@@ -100,6 +100,27 @@ describe("assertSingleSlot", () => {
     expect(() => assertSingleSlot("bad", ["wp:{}"])).toThrow(WaitpointKeyTagError);
   });
 
+  it("rejects an empty first pair, matching Redis rather than skipping to a later one", () => {
+    // Redis stops at the first `{`/`}` pair. An empty one means no tag at all, so it hashes
+    // the whole key. A regex would have found `a` here and wrongly claimed a shared slot.
+    expect(() => assertSingleSlot("bad", ["wp:{}{a}", "wp:{}{a}"])).toThrow(
+      WaitpointKeyTagError
+    );
+  });
+
+  it("takes the first pair when several are present", () => {
+    expect(() => assertSingleSlot("ok", ["wp:{a}{b}", "wp:{a}:w"])).not.toThrow();
+    expect(() => assertSingleSlot("bad", ["wp:{a}{b}", "wp:{b}:w"])).toThrow(
+      WaitpointKeyTagError
+    );
+  });
+
+  it("does not degrade on a key made of many opening braces", () => {
+    const started = performance.now();
+    expect(() => assertSingleSlot("bad", ["{".repeat(50_000)])).toThrow(WaitpointKeyTagError);
+    expect(performance.now() - started).toBeLessThan(1_000);
+  });
+
   it("names the operation and the offending key in the error", () => {
     const wp = waitpointKeys("w_a");
     const run = runBlockKeys("run_abc");
diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.ts
index 38447d107a1..28eac087b4b 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.ts
@@ -64,7 +64,18 @@ export class WaitpointKeyTagError extends Error {
   }
 }
 
-const HASH_TAG = /\{([^}]+)\}/;
+// Redis's own keyHashSlot rule: the FIRST `{`, then the FIRST `}` after it. A missing brace
+// or an empty pair means no tag, and Redis hashes the whole key. A regex would instead find
+// the first NON-empty pair, disagreeing with Redis on `wp:{}{a}`.
+function hashTag(key: string): string | undefined {
+  const open = key.indexOf("{");
+  if (open === -1) return undefined;
+
+  const close = key.indexOf("}", open + 1);
+  if (close === -1 || close === open + 1) return undefined;
+
+  return key.slice(open + 1, close);
+}
 
 /**
  * Throw unless every key carries the same non-empty hash tag. Called on every script
@@ -74,8 +85,7 @@ export function assertSingleSlot(operation: string, keys: string[]): void {
   let tag: string | undefined;
 
   for (const key of keys) {
-    const match = HASH_TAG.exec(key);
-    const found = match?.[1];
+    const found = hashTag(key);
     if (!found) {
       throw new WaitpointKeyTagError(operation, keys, key);
     }

From b0c93c4bc6828d9a2c4be83e2678fd431a8d85dc Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Mon, 24 Aug 2026 12:42:01 +0100
Subject: [PATCH 47/49] style(run-engine): oxfmt the new hash-tag assertions

---
 .../src/engine/waitpointCoordinator/keys.test.ts          | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.test.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.test.ts
index 01bd2c064aa..463e1ecd388 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.test.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/keys.test.ts
@@ -103,16 +103,12 @@ describe("assertSingleSlot", () => {
   it("rejects an empty first pair, matching Redis rather than skipping to a later one", () => {
     // Redis stops at the first `{`/`}` pair. An empty one means no tag at all, so it hashes
     // the whole key. A regex would have found `a` here and wrongly claimed a shared slot.
-    expect(() => assertSingleSlot("bad", ["wp:{}{a}", "wp:{}{a}"])).toThrow(
-      WaitpointKeyTagError
-    );
+    expect(() => assertSingleSlot("bad", ["wp:{}{a}", "wp:{}{a}"])).toThrow(WaitpointKeyTagError);
   });
 
   it("takes the first pair when several are present", () => {
     expect(() => assertSingleSlot("ok", ["wp:{a}{b}", "wp:{a}:w"])).not.toThrow();
-    expect(() => assertSingleSlot("bad", ["wp:{a}{b}", "wp:{b}:w"])).toThrow(
-      WaitpointKeyTagError
-    );
+    expect(() => assertSingleSlot("bad", ["wp:{a}{b}", "wp:{b}:w"])).toThrow(WaitpointKeyTagError);
   });
 
   it("does not degrade on a key made of many opening braces", () => {

From e1bc664d9134703683ca13b079a76549debf7213 Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Mon, 24 Aug 2026 13:25:36 +0100
Subject: [PATCH 48/49] fix(run-engine): reject a derived id in the
 idempotency-keyed create

The loser-discard deletes this call's own record, which is only safe for a
freshly minted id that was never handed out. A RUN or BATCH id is derived from
its anchor, so any caller can recompute it and register a watcher on it, and
discarding one could delete a record already in use.

Also documents that `created` means this call won the reservation, not that
the id is new: a retry by the original creator loses to its own reservation and
reports false.
---
 .../storeCoordinator.test.ts                  | 54 ++++++++++++++-----
 .../waitpointCoordinator/storeCoordinator.ts  | 15 ++++++
 2 files changed, 57 insertions(+), 12 deletions(-)

diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts
index 9674339a606..133129bf636 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts
@@ -2,6 +2,11 @@
 // is needed. redisTest FLUSHALLs before every test, so ids may be reused across describes.
 import { createRedisClient, type RedisOptions } from "@internal/redis";
 import { redisTest } from "@internal/testcontainers";
+import {
+  deriveWaitpointIdFromAnchor,
+  generateRunOpsId,
+  generateWaitpointId,
+} from "@trigger.dev/core/v3/isomorphic";
 import { describe, expect } from "vitest";
 import {
   edgeField,
@@ -696,16 +701,20 @@ describe("reply framing (direct Lua — pins the wire shape the coordinator deco
 });
 
 describe("createWithIdempotencyKey", () => {
+  // Real minted ids. The method rejects anything but a standalone DATETIME/MANUAL id, because
+  // its loser-discard is only safe for an id that was never handed out.
+  const idA = generateWaitpointId("MANUAL");
+  const idB = generateWaitpointId("MANUAL");
   redisTest("creates the waitpoint and wins the reservation", async ({ redisOptions }) => {
     const store = coordinator(redisOptions);
     try {
       const result = await store.createWithIdempotencyKey({
-        record: record("w_a", { idempotencyKey: "key-1", userProvidedIdempotencyKey: true }),
+        record: record(idA, { idempotencyKey: "key-1", userProvidedIdempotencyKey: true }),
         environmentId: ENV_ID,
         idempotencyKey: "key-1",
       });
 
-      expect(result).toEqual({ waitpointId: "w_a", created: true });
+      expect(result).toEqual({ waitpointId: idA, created: true });
     } finally {
       await store.quit();
     }
@@ -716,21 +725,21 @@ describe("createWithIdempotencyKey", () => {
     const probe = createRedisClient(redisOptions);
     try {
       await store.createWithIdempotencyKey({
-        record: record("w_first", { idempotencyKey: "key-1", userProvidedIdempotencyKey: true }),
+        record: record(idA, { idempotencyKey: "key-1", userProvidedIdempotencyKey: true }),
         environmentId: ENV_ID,
         idempotencyKey: "key-1",
       });
 
       const second = await store.createWithIdempotencyKey({
-        record: record("w_second", { idempotencyKey: "key-1", userProvidedIdempotencyKey: true }),
+        record: record(idB, { idempotencyKey: "key-1", userProvidedIdempotencyKey: true }),
         environmentId: ENV_ID,
         idempotencyKey: "key-1",
       });
 
-      expect(second).toEqual({ waitpointId: "w_first", created: false });
+      expect(second).toEqual({ waitpointId: idA, created: false });
       // The loser cleans up after itself: nothing ever referenced its id.
-      expect(await probe.exists("wp:{w_second}")).toBe(0);
-      expect(await probe.exists("wp:{w_first}")).toBe(1);
+      expect(await probe.exists(`wp:{${idB}}`)).toBe(0);
+      expect(await probe.exists(`wp:{${idA}}`)).toBe(1);
     } finally {
       probe.disconnect();
       await store.quit();
@@ -742,7 +751,7 @@ describe("createWithIdempotencyKey", () => {
     const probe = createRedisClient(redisOptions);
     try {
       await store.createWithIdempotencyKey({
-        record: record("w_a", { idempotencyKey: "key-1", userProvidedIdempotencyKey: true }),
+        record: record(idA, { idempotencyKey: "key-1", userProvidedIdempotencyKey: true }),
         environmentId: ENV_ID,
         idempotencyKey: "key-1",
       });
@@ -760,7 +769,7 @@ describe("createWithIdempotencyKey", () => {
     const probe = createRedisClient(redisOptions);
     try {
       await store.createWithIdempotencyKey({
-        record: record("w_a", {
+        record: record(idA, {
           idempotencyKey: "key-1",
           userProvidedIdempotencyKey: true,
           idempotencyKeyExpiresAt: new Date(Date.now() + 60_000).toISOString(),
@@ -789,24 +798,45 @@ describe("createWithIdempotencyKey", () => {
     const store = coordinator(redisOptions);
     try {
       await store.createWithIdempotencyKey({
-        record: record("w_a", { idempotencyKey: "key-1" }),
+        record: record(idA, { idempotencyKey: "key-1" }),
         environmentId: "env_1",
         idempotencyKey: "key-1",
       });
 
       const other = await store.createWithIdempotencyKey({
-        record: record("w_b", { idempotencyKey: "key-1", environmentId: "env_2" }),
+        record: record(idB, { idempotencyKey: "key-1", environmentId: "env_2" }),
         environmentId: "env_2",
         idempotencyKey: "key-1",
       });
 
-      expect(other).toEqual({ waitpointId: "w_b", created: true });
+      expect(other).toEqual({ waitpointId: idB, created: true });
     } finally {
       await store.quit();
     }
   });
 });
 
+redisTest(
+  "rejects a derived RUN id, whose loser-discard would be unsafe",
+  async ({ redisOptions }) => {
+    const store = coordinator(redisOptions);
+    try {
+      // A derived id is recomputable from its anchor, so another caller can register a
+      // watcher on it. Discarding one could delete a record already in use.
+      const derived = deriveWaitpointIdFromAnchor(`run_${generateRunOpsId()}`, "RUN")!;
+      await expect(
+        store.createWithIdempotencyKey({
+          record: record(derived, { type: "RUN", idempotencyKey: "key-1" }),
+          environmentId: ENV_ID,
+          idempotencyKey: "key-1",
+        })
+      ).rejects.toThrow(/freshly minted DATETIME or MANUAL/);
+    } finally {
+      await store.quit();
+    }
+  }
+);
+
 describe("the single-slot guard", () => {
   redisTest("rejects an invocation whose keys span two tags", async ({ redisOptions }) => {
     const store = coordinator(redisOptions);
diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts
index da79d3c5162..723552c57ab 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.ts
@@ -1,5 +1,6 @@
 import { createRedisClient, type Redis, type RedisOptions } from "@internal/redis";
 import { Logger } from "@trigger.dev/core/logger";
+import { parseWaitpointId } from "@trigger.dev/core/v3/isomorphic";
 import {
   assertSingleSlot,
   edgeField,
@@ -332,7 +333,21 @@ export class WaitpointStoreCoordinator {
     record: WaitpointRecordInput;
     environmentId: string;
     idempotencyKey: string;
+    // `created` means THIS CALL won the reservation, not that the id is new. A retry by the
+    // original creator reports false, because the reservation it is losing to is its own. A
+    // caller must not gate one-time side effects on it without handling that.
   }): Promise<{ waitpointId: string; created: boolean }> {
+    // Standalone ids only. The discard below deletes this call's own record, and that is
+    // only safe because a freshly minted id was never handed out, so nothing can reference
+    // it. A RUN or BATCH id is DERIVED from its anchor, so any caller can recompute it and
+    // register a watcher on it — discarding one could delete a record already in use.
+    const parsed = parseWaitpointId(args.record.id);
+    if (parsed.format !== "b32hexW" || (parsed.type !== "DATETIME" && parsed.type !== "MANUAL")) {
+      throw new Error(
+        `createWithIdempotencyKey requires a freshly minted DATETIME or MANUAL id, got ${args.record.id}`
+      );
+    }
+
     await this.createIfAbsent({ record: args.record, status: "PENDING" });
 
     const expiresAtMs = args.record.idempotencyKeyExpiresAt

From 55dd5dfa46fc61fea0db1037188ed0cf7e345cf5 Mon Sep 17 00:00:00 2001
From: Daniel Sutton <dcsu@pm.me>
Date: Mon, 24 Aug 2026 15:43:07 +0100
Subject: [PATCH 49/49] test(run-engine): close four coverage holes in the
 waitpoint store coordinator

Add tests for the winner's own retry in createWithIdempotencyKey, a
COMPLETED record round-tripping through createIfAbsent, absorbBlockers
reading back a stored delivery envelope rather than its flag, and a new
genuine-concurrency suite (real Promise.all races, no mocks) covering
complete, registerOrReport, createWithIdempotencyKey, and registerBlocks
under contention. Each was proven against its mutant and restored clean.
---
 .../storeCoordinator.test.ts                  | 291 ++++++++++++++++++
 1 file changed, 291 insertions(+)

diff --git a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts
index 133129bf636..f0e9c0c297d 100644
--- a/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts
+++ b/internal-packages/run-engine/src/engine/waitpointCoordinator/storeCoordinator.test.ts
@@ -164,6 +164,48 @@ describe("createIfAbsent", () => {
       }
     }
   );
+
+  redisTest(
+    "reads a COMPLETED record back through createIfAbsent, with an envelope",
+    async ({ redisOptions }) => {
+      const store = coordinator(redisOptions);
+      try {
+        await store.createIfAbsent({
+          record: record("w_a"),
+          status: "COMPLETED",
+          completion: completion(),
+        });
+
+        const second = await store.createIfAbsent({ record: record("w_a"), status: "PENDING" });
+
+        expect(second.outcome).toBe("exists");
+        if (second.outcome !== "exists") throw new Error("unreachable");
+        expect(second.status).toBe("COMPLETED");
+        expect(second.completion?.output).toEqual({ inline: '{"ok":true}' });
+      } finally {
+        await store.quit();
+      }
+    }
+  );
+
+  redisTest(
+    "reads a COMPLETED record back through createIfAbsent, with no envelope",
+    async ({ redisOptions }) => {
+      const store = coordinator(redisOptions);
+      try {
+        await store.createIfAbsent({ record: record("w_a"), status: "COMPLETED" });
+
+        const second = await store.createIfAbsent({ record: record("w_a"), status: "PENDING" });
+
+        expect(second.outcome).toBe("exists");
+        if (second.outcome !== "exists") throw new Error("unreachable");
+        expect(second.status).toBe("COMPLETED");
+        expect(second.completion).toBeUndefined();
+      } finally {
+        await store.quit();
+      }
+    }
+  );
 });
 
 describe("registerOrReport", () => {
@@ -746,6 +788,51 @@ describe("createWithIdempotencyKey", () => {
     }
   });
 
+  redisTest(
+    "the original creator's own retry does not discard its own record",
+    async ({ redisOptions }) => {
+      const store = coordinator(redisOptions);
+      const probe = createRedisClient(redisOptions);
+      try {
+        const withKey = record(idA, {
+          idempotencyKey: "key-1",
+          userProvidedIdempotencyKey: true,
+        });
+
+        const first = await store.createWithIdempotencyKey({
+          record: withKey,
+          environmentId: ENV_ID,
+          idempotencyKey: "key-1",
+        });
+        expect(first).toEqual({ waitpointId: idA, created: true });
+
+        // The SAME caller, retrying with the SAME record id and the SAME key — not a
+        // different id racing for the same reservation.
+        const retry = await store.createWithIdempotencyKey({
+          record: withKey,
+          environmentId: ENV_ID,
+          idempotencyKey: "key-1",
+        });
+
+        expect(retry).toEqual({ waitpointId: idA, created: false });
+        // The record must survive: a wrongly-discarded record would delete this too.
+        expect(await probe.exists(`wp:{${idA}}`)).toBe(1);
+
+        // The real proof: something usable is still there for every later caller that
+        // blocks on this id.
+        const registered = await store.registerOrReport({
+          waitpointId: idA,
+          runId: "run_1",
+          createdAt: NOW,
+        });
+        expect(registered.outcome).toBe("registered");
+      } finally {
+        probe.disconnect();
+        await store.quit();
+      }
+    }
+  );
+
   redisTest("sets no expiry when the record carries none", async ({ redisOptions }) => {
     const store = coordinator(redisOptions);
     const probe = createRedisClient(redisOptions);
@@ -918,6 +1005,51 @@ describe("absorbBlockers", () => {
     }
   );
 
+  redisTest(
+    "a later absorb reads back the stored envelope, not a bare flag",
+    async ({ redisOptions }) => {
+      const store = coordinator(redisOptions);
+      try {
+        const envelope = completion({ output: { inline: '{"first":true}' } });
+
+        // Reported once, with an envelope — this write is what's under test.
+        await store.absorbBlockers({
+          runId: RUN_ID,
+          edges: [edge("w_a", { reported: { completion: envelope } })],
+        });
+
+        // Same waitpoint id, arriving unreported this time: takes the "read `done` back"
+        // path, exposing whatever the first call actually stored under that id.
+        const second = await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_a")] });
+
+        expect(second.alreadyDelivered).toHaveLength(1);
+        expect(second.alreadyDelivered[0]!.completion).toEqual(envelope);
+      } finally {
+        await store.quit();
+      }
+    }
+  );
+
+  redisTest(
+    "a later absorb for a no-envelope delivery reads back no completion",
+    async ({ redisOptions }) => {
+      const store = coordinator(redisOptions);
+      try {
+        await store.absorbBlockers({
+          runId: RUN_ID,
+          edges: [edge("w_a", { reported: {} })],
+        });
+
+        const second = await store.absorbBlockers({ runId: RUN_ID, edges: [edge("w_a")] });
+
+        expect(second.alreadyDelivered).toHaveLength(1);
+        expect(second.alreadyDelivered[0]!.completion).toBeUndefined();
+      } finally {
+        await store.quit();
+      }
+    }
+  );
+
   redisTest("reports a repeated already-delivered id once", async ({ redisOptions }) => {
     const store = coordinator(redisOptions);
     try {
@@ -1544,3 +1676,162 @@ describe("the resume cycle drains and can start again", () => {
     }
   });
 });
+
+// Every test above is a sequence of awaits. Redis guarantees atomicity WITHIN a script, so
+// those tests can only ever prove single-script invariants. These races drive real
+// concurrent calls (Promise.all over N copies) against the multi-script TypeScript
+// sequences, and assert an invariant that holds regardless of who wins — never a timing.
+describe("genuine concurrency", () => {
+  const CONCURRENCY = 8;
+
+  redisTest(
+    "exactly one of N concurrent completers wins, and every caller sees its completion",
+    async ({ redisOptions }) => {
+      const store = coordinator(redisOptions);
+      try {
+        await store.createIfAbsent({ record: record("w_a"), status: "PENDING" });
+        await store.registerOrReport({ waitpointId: "w_a", runId: "run_1", createdAt: NOW });
+        await store.registerOrReport({ waitpointId: "w_a", runId: "run_2", createdAt: NOW });
+
+        const results = await Promise.all(
+          Array.from({ length: CONCURRENCY }, (_, i) =>
+            store.complete({
+              waitpointId: "w_a",
+              completion: completion({ output: { inline: `{"racer":${i}}` } }),
+            })
+          )
+        );
+
+        const winners = results.filter((r) => r.outcome === "completed");
+        const losers = results.filter((r) => r.outcome === "already");
+        expect(winners).toHaveLength(1);
+        expect(losers).toHaveLength(CONCURRENCY - 1);
+
+        // Every caller, winner and losers alike, reads back the SAME stored completion.
+        const stored = winners[0]!.completion;
+        for (const r of results) {
+          expect(r.completion).toEqual(stored);
+        }
+
+        // And every caller returns the full watcher list — a race must never truncate it.
+        for (const r of results) {
+          expect(r.watchers.map((w) => w.runId).sort()).toEqual(["run_1", "run_2"]);
+        }
+      } finally {
+        await store.quit();
+      }
+    }
+  );
+
+  redisTest(
+    "a pre-existing registration survives N concurrent attempts to re-register its field",
+    async ({ redisOptions }) => {
+      const store = coordinator(redisOptions);
+      try {
+        await store.createIfAbsent({ record: record("w_a"), status: "PENDING" });
+        await store.registerOrReport({
+          waitpointId: "w_a",
+          runId: "run_1",
+          spanIdToComplete: "span_first",
+          createdAt: NOW,
+        });
+
+        // Same run, same (absent) batch index as the registration above, so every one of
+        // these collides on the exact same watcher field.
+        await Promise.all(
+          Array.from({ length: CONCURRENCY }, (_, i) =>
+            store.registerOrReport({
+              waitpointId: "w_a",
+              runId: "run_1",
+              spanIdToComplete: `span_racer_${i}`,
+              createdAt: NOW,
+            })
+          )
+        );
+
+        const completed = await store.complete({ waitpointId: "w_a", completion: completion() });
+        const forRun1 = completed.watchers.filter((w) => w.runId === "run_1");
+        expect(forRun1).toHaveLength(1);
+        expect(forRun1[0]!.spanIdToComplete).toBe("span_first");
+      } finally {
+        await store.quit();
+      }
+    }
+  );
+
+  redisTest(
+    "exactly one of N concurrent idempotency-keyed creators wins, and every loser cleans up",
+    async ({ redisOptions }) => {
+      const store = coordinator(redisOptions);
+      const probe = createRedisClient(redisOptions);
+      try {
+        const ids = Array.from({ length: CONCURRENCY }, () => generateWaitpointId("MANUAL"));
+
+        const results = await Promise.all(
+          ids.map((id) =>
+            store.createWithIdempotencyKey({
+              record: record(id, { idempotencyKey: "key-1", userProvidedIdempotencyKey: true }),
+              environmentId: ENV_ID,
+              idempotencyKey: "key-1",
+            })
+          )
+        );
+
+        const winners = results.filter((r) => r.created);
+        expect(winners).toHaveLength(1);
+
+        const winnerId = winners[0]!.waitpointId;
+        for (const r of results) {
+          expect(r.waitpointId).toBe(winnerId);
+        }
+        expect(await probe.exists(`wp:{${winnerId}}`)).toBe(1);
+
+        for (const id of ids) {
+          if (id === winnerId) continue;
+          expect(await probe.exists(`wp:{${id}}`)).toBe(0);
+          expect(await probe.exists(`wp:{${id}}:w`)).toBe(0);
+        }
+      } finally {
+        probe.disconnect();
+        await store.quit();
+      }
+    }
+  );
+
+  redisTest(
+    "registerBlocks racing complete never leaves a waitpoint double-booked or the pending count negative",
+    async ({ redisOptions }) => {
+      const store = coordinator(redisOptions);
+      try {
+        for (let i = 0; i < 30; i++) {
+          const waitpointId = `w_race_${i}`;
+          const runId = `run_race_${i}`;
+          await store.createIfAbsent({ record: record(waitpointId), status: "PENDING" });
+
+          // Two edges for the SAME waitpoint: registerBlocks registers them one at a
+          // time, so a concurrent complete() has a real window to land between the two
+          // registrations — the exact straddle that makes absorbBlockers' per-group
+          // reported/unreported split matter, rather than racing a single all-or-nothing
+          // group.
+          const [blocked] = await Promise.all([
+            store.registerBlocks({
+              runId,
+              edges: [edge(waitpointId, { batchIndex: 0 }), edge(waitpointId, { batchIndex: 1 })],
+            }),
+            store.complete({ waitpointId, completion: completion() }),
+          ]);
+
+          const state = await store.readBlockState(runId);
+          const delivered = state.deliveredIds.includes(waitpointId);
+          const pending = state.pendingIds.includes(waitpointId);
+
+          expect(delivered && pending).toBe(false);
+          expect(blocked.storePendingTotal).toBeGreaterThanOrEqual(0);
+          expect(blocked.storePendingTotal).toBeLessThanOrEqual(1);
+        }
+      } finally {
+        await store.quit();
+      }
+    }
+  );
+});