From 8ce500e5bdb4a2a6de527740b4e9872236a4fb8f Mon Sep 17 00:00:00 2001 From: JJ Eaton <58495321+jayleaton@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:22:46 +0700 Subject: [PATCH 1/5] fix(server): history trim stops on activity and publishes what it removed Co-Authored-By: Claude Opus 5.5 --- apps/server/src/orchestration-v2/EventSink.ts | 23 ++ .../src/orchestration-v2/ProjectionStore.ts | 32 ++ .../ThreadHistoryTrim.test.ts | 351 ++++++++++++++++-- .../src/orchestration-v2/ThreadHistoryTrim.ts | 327 ++++++++-------- .../state/orchestrationV2Projection.test.ts | 12 +- .../src/state/orchestrationV2Projection.ts | 14 + packages/contracts/src/orchestrationV2.ts | 16 + 7 files changed, 601 insertions(+), 174 deletions(-) diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index fff4fd3e3db76..0b655ff30697b 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -107,6 +107,10 @@ export interface EventSinkV2Shape { readonly commandId?: CommandId; readonly events: ReadonlyArray; }) => Effect.Effect, EventSinkV2Error>; + /** Builds the events inside the commit transaction, so they reflect what is stored; none commits nothing. */ + readonly writeBuilt: ( + build: Effect.Effect, E>, + ) => Effect.Effect, EventSinkV2Error>; readonly writeWithEffects: (input: { /** * Rebuilds each run.updated from the stored row with only its delegatedCompletion replaced. @@ -520,6 +524,21 @@ const layerBase: Layer.Layer< return committed; }); + const writeBuiltEffect = Effect.fn("orchestrationV2.EventSink.writeBuilt")(function* ( + build: Effect.Effect, E>, + ) { + const { committed } = yield* commitThenPublish( + Effect.gen(function* () { + const events = yield* build; + if (events.length === 0) return { committed: [], outcomes: [] }; + const committed = yield* eventStore.append({ events: yield* normalizeEvents(events) }); + return { committed, outcomes: yield* applyStoredEvents(committed) }; + }), + ({ committed, outcomes }) => publishStoredEvents(committed, outcomes), + ); + return committed; + }); + const writeIfRunCurrentEffect = Effect.fn("orchestrationV2.EventSink.writeIfRunCurrent")( function* (input: Parameters[0]) { yield* Effect.annotateCurrentSpan({ @@ -1137,6 +1156,10 @@ const layerBase: Layer.Layer< }), ), ), + writeBuilt: (build) => + writeBuiltEffect(build).pipe( + Effect.mapError((cause) => new EventSinkWriteError({ eventCount: 0, cause })), + ), writeWithEffects: (input) => writeEffect(input).pipe( Effect.mapError( diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 404e4c1b0f948..b7dfb8835f108 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -1102,6 +1102,18 @@ export function applyToProjection( ...projection, thread: event.payload, }; + case "thread.history-trimmed": { + const items = new Set(event.payload.turnItemIds); + const nodes = new Set(event.payload.nodeIds); + return { + ...projection, + turnItems: projection.turnItems.filter((item) => !items.has(item.id)), + nodes: projection.nodes.filter((node) => !nodes.has(node.id)), + visibleTurnItems: projection.visibleTurnItems + .filter((row) => row.sourceThreadId !== event.threadId || !items.has(row.sourceItemId)) + .map((row, position) => (row.position === position ? row : { ...row, position })), + }; + } case "thread.visit-recorded": return { ...projection, @@ -3337,6 +3349,26 @@ export const layer: Layer.Layer = } case "checkpoint.rollback-requested": break; + case "thread.history-trimmed": { + const { turnItemIds, nodeIds } = event.payload; + if (turnItemIds.length > 0) { + yield* sql` + DELETE FROM orchestration_v2_projection_turn_items + WHERE thread_id = ${event.threadId} AND turn_item_id IN ${sql.in(turnItemIds)} + `; + yield* sql` + DELETE FROM orchestration_v2_turn_item_positions + WHERE thread_id = ${event.threadId} AND turn_item_id IN ${sql.in(turnItemIds)} + `; + } + if (nodeIds.length > 0) { + yield* sql` + DELETE FROM orchestration_v2_projection_nodes + WHERE thread_id = ${event.threadId} AND node_id IN ${sql.in(nodeIds)} + `; + } + break; + } case "context-handoff.updated": { const payloadJson = yield* encodeContextHandoffPayload(event.payload); const payload = parseEncodedPayload(payloadJson); diff --git a/apps/server/src/orchestration-v2/ThreadHistoryTrim.test.ts b/apps/server/src/orchestration-v2/ThreadHistoryTrim.test.ts index 667a6e8e2ef0f..7842753b87913 100644 --- a/apps/server/src/orchestration-v2/ThreadHistoryTrim.test.ts +++ b/apps/server/src/orchestration-v2/ThreadHistoryTrim.test.ts @@ -6,6 +6,7 @@ import { type OrchestrationV2AppThread, type OrchestrationV2DomainEvent, type OrchestrationV2Run, + type OrchestrationV2ThreadProjection, type OrchestrationV2TurnItem, ProjectId, ProviderInstanceId, @@ -25,16 +26,20 @@ import * as SqlClient from "effect/sql/SqlClient"; import * as OrchestrationEventStore from "../persistence/OrchestrationEventStore.ts"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; import * as ServerSettings from "../serverSettings.ts"; +import { visibleConversationItems } from "./ContextHandoffService.ts"; import * as EventSink from "./EventSink.ts"; import * as EventStore from "./EventStore.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; +import * as ProjectStore from "./ProjectStore.ts"; import * as ThreadHistoryTrim from "./ThreadHistoryTrim.ts"; +import { buildBoundedThreadStreamSnapshot } from "./ThreadStream.ts"; const database = SqlitePersistence.layerMemory; const stores = Layer.mergeAll( database, EventStore.layer.pipe(Layer.provideMerge(database)), ProjectionStore.layer.pipe(Layer.provideMerge(database)), + ProjectStore.layer.pipe(Layer.provideMerge(database)), ); const dependencies = (afterDays: number) => Layer.mergeAll( @@ -57,6 +62,7 @@ type SeedOptions = { readonly thread?: Partial; readonly runStatus?: OrchestrationV2Run["status"]; readonly userMessageAt?: DateTime.Utc; + readonly extraReasoning?: number; }; const seedThread = Effect.fn(function* (name: string, options: SeedOptions = {}) { @@ -201,6 +207,14 @@ const seedThread = Effect.fn(function* (name: string, options: SeedOptions = {}) attachments: [], }), item({ ...itemBase("reasoning"), type: "reasoning", text: "thinking", streaming: false }), + ...Array.from({ length: options.extraReasoning ?? 0 }, (_, index) => + item({ + ...itemBase(`reasoning-${index}`), + type: "reasoning", + text: "more thinking", + streaming: false, + }), + ), item({ ...itemBase("command", commandNodeId), type: "command_execution", @@ -226,6 +240,27 @@ const seedThread = Effect.fn(function* (name: string, options: SeedOptions = {}) input: null, output: "The draft the user was asked to read.", }), + item({ + ...itemBase("html"), + type: "dynamic_tool", + toolName: "mcp__t3-code__html_render", + input: null, + output: { htmlRender: { title: "Chart" } }, + }), + item({ + ...itemBase("app"), + type: "dynamic_tool", + toolName: "mcp__charts__show", + input: null, + output: { t3McpApp: { resourceUri: "ui://charts/show" } }, + }), + item({ + ...itemBase("task"), + type: "dynamic_tool", + toolName: "mcp__t3-code__delegate_task", + input: null, + output: "The delegated task's result.", + }), item({ ...itemBase("assistant"), type: "assistant_message", @@ -238,6 +273,49 @@ const seedThread = Effect.fn(function* (name: string, options: SeedOptions = {}) return { threadId, runId }; }); +const openPullRequest = { + host: "github.com", + repository: "acme/app", + number: 7, + url: "https://github.com/acme/app/pull/7", + source: "agent", + linkedAt: DateTime.formatIso(old), + snapshot: { + state: "open", + title: "Fix harness", + headBranch: "fix", + baseBranch: "main", + isDraft: false, + updatedAt: null, + syncedAt: DateTime.formatIso(old), + }, + stack: null, +} satisfies NonNullable[number]; + +const createProject = Effect.flatMap(ProjectStore.ProjectStoreV2, (projects) => + projects.apply({ + sequence: 0, + eventId: EventId.make("created:project:trim"), + aggregateKind: "project", + aggregateId: ProjectId.make("project:trim"), + occurredAt: DateTime.formatIso(old), + commandId: null, + causationEventId: null, + correlationId: null, + metadata: {}, + type: "project.created", + payload: { + projectId: ProjectId.make("project:trim"), + title: "trim", + workspaceRoot: "/work/trim", + defaultModelSelection: null, + scripts: [], + createdAt: DateTime.formatIso(old), + updatedAt: DateTime.formatIso(old), + }, + }), +); + const itemTitles = Effect.fn(function* (threadId: ThreadId) { const sql = yield* SqlClient.SqlClient; const rows = yield* sql<{ readonly title: string }>` @@ -249,9 +327,17 @@ const itemTitles = Effect.fn(function* (threadId: ThreadId) { return rows.map((row) => row.title); }); -const everyItem = ["user", "reasoning", "command", "tool", "edit", "sent", "assistant"]; -// `sent` stays: the timeline always shows that output inline. -const trimmedItems = ["user", "tool", "edit", "sent", "assistant", "Older detail was trimmed"]; +const inlineItems = ["sent", "html", "app", "task"]; +const everyItem = ["user", "reasoning", "command", "tool", "edit", ...inlineItems, "assistant"]; +// Inline items stay: the timeline always shows that output. +const trimmedItems = [ + "user", + "tool", + "edit", + ...inlineItems, + "assistant", + "Older detail was trimmed", +]; const trim = Effect.gen(function* () { const service = yield* ThreadHistoryTrim.ThreadHistoryTrim; @@ -272,13 +358,30 @@ it.layer(layerTest(30))("ThreadHistoryTrim", (it) => { const preview = yield* service.preview; assert.deepEqual( { dryRun: preview.dryRun, threadCount: preview.threadCount, itemCount: preview.itemCount }, - { dryRun: true, threadCount: 1, itemCount: 3 }, + { dryRun: true, threadCount: 1, itemCount: everyItem.length - trimmedItems.length + 1 }, ); - assert.isAbove(preview.estimatedBytes, 8_000); + assert.isAbove(preview.estimatedBytes, 4_000); assert.deepEqual(yield* itemTitles(threadId), everyItem); + const inlineIds = inlineItems.map((key) => `item:idle:${key}`); const report = yield* trim; assert.equal(report.dryRun, false); + // What the web and mobile timelines, exports and transfers read keeps every inline output. + const snapshot = yield* store.getThreadSnapshot(threadId); + const wire = buildBoundedThreadStreamSnapshot(snapshot); + const { turnItems, ...records } = yield* store.getThreadRecords(threadId, [ + "turnItems", + "runs", + "attempts", + ]); + const readers = { + timeline: snapshot.projection.visibleTurnItems.map((row) => row.sourceItemId), + wire: wire.projection.visibleTurnItems.map((row) => row.sourceItemId), + transfer: visibleConversationItems(records, turnItems).map((item) => item.id), + }; + for (const [reader, shown] of Object.entries(readers)) { + assert.includeMembers(shown, inlineIds, reader); + } // `tool` stays: the kept file change hangs off it. assert.deepEqual(yield* itemTitles(threadId), trimmedItems); const nodes = yield* sql<{ readonly node_id: string }>` @@ -334,26 +437,7 @@ it.layer(layerTest(30))("ThreadHistoryTrim", (it) => { }), yield* seedThread("open-pr", { thread: { - pullRequests: [ - { - host: "github.com", - repository: "acme/app", - number: 7, - url: "https://github.com/acme/app/pull/7", - source: "agent", - linkedAt: DateTime.formatIso(old), - snapshot: { - state: "open", - title: "Fix harness", - headBranch: "fix", - baseBranch: "main", - isDraft: false, - updatedAt: null, - syncedAt: DateTime.formatIso(old), - }, - stack: null, - }, - ], + pullRequests: [openPullRequest], }, }), yield* seedThread("side-chat", { @@ -398,6 +482,223 @@ it.layer(layerTest(30))("ThreadHistoryTrim", (it) => { ); }); +type Hook = Effect.Effect< + void, + unknown, + | EventSink.EventSinkV2 + | EventStore.EventStoreV2 + | ProjectionStore.ProjectionStoreV2 + | ServerSettings.ServerSettingsService +>; +const isNotice = (events: ReadonlyArray) => + events.some( + (event) => event.type === "turn-item.updated" && event.payload.type === "system_notice", + ); +/** Runs `hook` at the pass's `atWrite`th write: after it chose what to remove, before removing it. */ +const duringTrim = (hook: Hook, atWrite = 1) => { + const hooked = Layer.effect( + EventSink.EventSinkV2, + Effect.gen(function* () { + const sink = yield* EventSink.EventSinkV2; + const context = yield* Effect.context>(); + let writes = 0; + const next = Effect.suspend(() => + ++writes === atWrite ? hook.pipe(Effect.provide(context), Effect.orDie) : Effect.void, + ); + return EventSink.EventSinkV2.of({ + ...sink, + write: (input) => + sink.write(input).pipe(Effect.tap(() => (isNotice(input.events) ? next : Effect.void))), + writeBuilt: (build) => next.pipe(Effect.andThen(sink.writeBuilt(build))), + }); + }), + ); + return ThreadHistoryTrim.layer.pipe(Layer.provide(hooked), Layer.provideMerge(dependencies(30))); +}; + +const updateThread = ( + name: string, + type: OrchestrationV2DomainEvent["type"], + patch: Partial, +): Hook => + Effect.gen(function* () { + const threadId = ThreadId.make(`thread:trim:${name}`); + const thread = yield* (yield* ProjectionStore.ProjectionStoreV2).getThread(threadId); + yield* (yield* EventSink.EventSinkV2).write({ + events: [ + { + id: EventId.make(`event:${name}:activity`), + type, + threadId, + occurredAt: now, + payload: { ...thread, ...patch }, + } as OrchestrationV2DomainEvent, + ], + }); + }); +const writeActivity = (name: string, event: (threadId: ThreadId, runId: RunId) => object): Hook => + Effect.gen(function* () { + const threadId = ThreadId.make(`thread:trim:${name}`); + const runId = RunId.make(`run:${threadId}:2`); + yield* (yield* EventSink.EventSinkV2).write({ + events: [ + { + id: EventId.make(`event:${name}:activity`), + threadId, + runId, + occurredAt: now, + ...event(threadId, runId), + } as OrchestrationV2DomainEvent, + ], + }); + }); + +const activities: ReadonlyArray = [ + ["opened", updateThread("opened", "thread.visited", { lastVisitedAt: now })], + ["pinned", updateThread("pinned", "thread.pinned", { pinnedAt: now })], + [ + "snoozed", + updateThread("snoozed", "thread.snoozed", { + snoozedAt: now, + snoozedUntil: DateTime.add(now, { days: 1 }), + }), + ], + [ + "pr-linked", + updateThread("pr-linked", "thread.pull-request-synced", { pullRequests: [openPullRequest] }), + ], + [ + "messaged", + writeActivity("messaged", (threadId, runId) => ({ + type: "message.updated", + payload: { + id: MessageId.make("message:messaged:again"), + threadId, + runId, + nodeId: null, + role: "user", + text: "one more thing", + attachments: [], + streaming: false, + createdBy: "user", + creationSource: "web", + createdAt: now, + updatedAt: now, + }, + })), + ], + [ + "run", + writeActivity("run", (threadId, runId) => ({ + type: "run.created", + payload: { + id: runId, + threadId, + ordinal: 2, + providerInstanceId, + modelSelection, + providerThreadId: null, + userMessageId: MessageId.make("message:run:again"), + rootNodeId: NodeId.make("node:run:again"), + activeAttemptId: null, + status: "queued", + requestedAt: now, + startedAt: null, + completedAt: null, + checkpointId: null, + contextHandoffId: null, + }, + })), + ], + [ + "trimming-off", + Effect.flatMap(ServerSettings.ServerSettingsService, (settings) => + settings.updateSettings({ threadHistoryTrimAfterDays: 0 }), + ), + ], +]; + +it.effect.each(activities)( + "stops trimming a thread that is %s during the pass", + ([name, activity]) => + Effect.gen(function* () { + yield* TestClock.setTime(DateTime.toEpochMillis(now)); + const { threadId } = yield* seedThread(name); + yield* trim; + assert.deepEqual(yield* itemTitles(threadId), everyItem); + }).pipe(Effect.provide(duringTrim(activity))), +); + +it.effect("stops between pages once the thread is opened", () => + Effect.gen(function* () { + yield* TestClock.setTime(DateTime.toEpochMillis(now)); + const { threadId } = yield* seedThread("opened", { extraReasoning: 250 }); + yield* trim; + // The first page of 200 went; the rest stays. + assert.lengthOf(yield* itemTitles(threadId), everyItem.length + 250 - 200 + 1); + }).pipe( + Effect.provide(duringTrim(updateThread("opened", "thread.visited", { lastVisitedAt: now }), 2)), + ), +); + +it.effect( + "advances the thread's revision so reconnects, windows and search drop trimmed rows", + () => { + const name = "reconnect"; + const threadId = ThreadId.make(`thread:trim:${name}`); + const removed = [`item:${name}:reasoning`, `item:${name}:command`]; + let warmed: + | { readonly snapshotSequence: number; readonly projection: OrchestrationV2ThreadProjection } + | undefined; + // A client that read the thread mid-pass, and left the timeline and search caches warm. + const read: Hook = Effect.gen(function* () { + const store = yield* ProjectionStore.ProjectionStoreV2; + warmed = yield* store.getThreadSnapshot(threadId); + yield* store.getThreadSnapshotWindow(threadId, { rowLimit: 50 }); + yield* store.searchThread({ threadId, query: "flaky" }); + }); + return Effect.gen(function* () { + yield* TestClock.setTime(DateTime.toEpochMillis(now)); + const store = yield* ProjectionStore.ProjectionStoreV2; + const events = yield* EventStore.EventStoreV2; + yield* createProject; + yield* seedThread(name); + yield* trim; + const before = warmed!; + assert.includeMembers( + before.projection.turnItems.map((item) => item.id), + removed, + ); + + const after = yield* store.getThreadSnapshot(threadId); + assert.isAbove(after.snapshotSequence, before.snapshotSequence); + const ids = (projection: OrchestrationV2ThreadProjection) => ({ + turnItems: projection.turnItems.map((item) => item.id).toSorted(), + nodes: projection.nodes.map((node) => node.id).toSorted(), + visible: projection.visibleTurnItems.map((row) => row.sourceItemId), + }); + const replay = yield* Stream.runCollect( + events.read({ threadId, afterSequence: before.snapshotSequence }), + ); + const resumed = replay.reduce( + (projection, stored) => ProjectionStore.applyToProjection(projection, stored.event), + before.projection, + ); + assert.deepEqual(ids(resumed), ids(after.projection)); + assert.notIncludeMembers(ids(after.projection).turnItems, removed); + + const window = yield* store.getThreadSnapshotWindow(threadId, { rowLimit: 50 }); + assert.equal(window.snapshotSequence, after.snapshotSequence); + assert.notIncludeMembers( + window.projection.turnItems.map((item) => item.id), + removed, + ); + const found = yield* store.searchThread({ threadId, query: "flaky" }); + assert.equal(found.snapshotSequence, after.snapshotSequence); + }).pipe(Effect.provide(duringTrim(read))); + }, +); + it.layer(layerTest(0))("ThreadHistoryTrim turned off", (it) => { it.effect("removes nothing", () => Effect.gen(function* () { diff --git a/apps/server/src/orchestration-v2/ThreadHistoryTrim.ts b/apps/server/src/orchestration-v2/ThreadHistoryTrim.ts index 0d84c25d15e41..d7815d22d31a7 100644 --- a/apps/server/src/orchestration-v2/ThreadHistoryTrim.ts +++ b/apps/server/src/orchestration-v2/ThreadHistoryTrim.ts @@ -1,8 +1,11 @@ import { + type NodeId, + type OrchestrationV2DomainEvent, type RunId, type ThreadHistoryTrimReport, type ThreadHistoryTrimStatus, type ThreadId, + type TurnItemId, } from "@t3tools/contracts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import { makeDrainableWorker } from "@t3tools/shared/DrainableWorker"; @@ -71,7 +74,9 @@ type Candidate = { readonly updatedAt: string; }; type Mode = "background" | "manual"; -type Removed = { readonly itemCount: number; readonly bytes: number }; + +const sumBytes = (rows: ReadonlyArray<{ readonly bytes: number }>) => + rows.reduce((total, row) => total + row.bytes, 0); // Space a row gives back: its payload plus each copy of its ids in the row, its indexes and its position row. const ITEM_BYTES = `(octet_length(payload_json) + 5 * length(turn_item_id) + 8 * length(thread_id) @@ -163,7 +168,7 @@ const make = Effect.gen(function* () { OR COALESCE(recent.completed_at, '') >= ${input.idleBefore} ) ) - AND EXISTS ( + AND (${input.threadId !== undefined} OR EXISTS ( SELECT 1 FROM orchestration_v2_projection_turn_items AS item WHERE item.thread_id = thread.thread_id AND item.type IN ${sql.in(TRIMMED_ITEM_TYPES)} @@ -174,7 +179,7 @@ const make = Effect.gen(function* () { SELECT 1 FROM orchestration_v2_projection_turn_items AS child WHERE child.thread_id = item.thread_id AND child.parent_item_id = item.turn_item_id ) - ) + )) ORDER BY thread.thread_id ASC `; @@ -188,114 +193,12 @@ const make = Effect.gen(function* () { }; }); - const estimate = Effect.fn("ThreadHistoryTrim.estimate")(function* (dryRun: boolean) { - const { afterDays, now, idleBefore } = yield* window; - const candidates = afterDays <= 0 ? [] : yield* readCandidates({ idleBefore, now }); - let itemCount = 0; - let estimatedBytes = 0; - for (const candidate of candidates) { - const rows = yield* sql<{ readonly items: number; readonly bytes: number }>` - SELECT - (SELECT COUNT(*) FROM orchestration_v2_projection_turn_items - WHERE thread_id = ${candidate.threadId} - AND type IN ${sql.in(TRIMMED_ITEM_TYPES)} - AND inline_work_output = 0 - AND status NOT IN ${sql.in(LIVE_ITEM_STATUSES)}) AS items, - COALESCE((SELECT SUM(${sql.literal(ITEM_BYTES)}) FROM orchestration_v2_projection_turn_items - WHERE thread_id = ${candidate.threadId} - AND type IN ${sql.in(TRIMMED_ITEM_TYPES)} - AND inline_work_output = 0 - AND status NOT IN ${sql.in(LIVE_ITEM_STATUSES)}), 0) - + COALESCE((SELECT SUM(${sql.literal(nodeBytes(""))}) FROM orchestration_v2_projection_nodes - WHERE thread_id = ${candidate.threadId} - AND kind IN ${sql.in(TRIMMED_NODE_KINDS)} - AND status NOT IN ${sql.in(LIVE_NODE_STATUSES)}), 0) AS bytes - `; - itemCount += rows[0]?.items ?? 0; - estimatedBytes += rows[0]?.bytes ?? 0; - yield* Effect.yieldNow; - } - return { - dryRun, - afterDays, - threadCount: candidates.length, - itemCount, - estimatedBytes, - } satisfies ThreadHistoryTrimReport; - }); - - const writeNotice = Effect.fn("ThreadHistoryTrim.writeNotice")(function* ( - candidate: Candidate, - afterDays: number, - ) { - const now = yield* DateTime.now; - yield* eventSink.write({ - events: [ - { - id: yield* ids.allocate.event({ threadId: candidate.threadId }), - type: "turn-item.updated", - threadId: candidate.threadId, - runId: candidate.runId, - // Every event stamps the thread with its time; reusing the thread's own keeps a trimmed - // thread where it was in lists sorted by activity. - occurredAt: DateTime.makeUnsafe(candidate.updatedAt), - payload: { - id: ids.derive.runSignalTurnItem({ runId: candidate.runId, signal: NOTICE_SIGNAL }), - threadId: candidate.threadId, - runId: candidate.runId, - nodeId: null, - providerThreadId: null, - providerTurnId: null, - nativeItemRef: null, - parentItemId: null, - // EventSink allocates the real position, after the run's existing items. - ordinal: 0, - status: "completed", - title: "Older detail was trimmed", - startedAt: now, - completedAt: now, - updatedAt: now, - type: "system_notice", - message: trimNoticeMessage(afterDays), - }, - }, - ], - }); - }); - - const deleteInPages = Effect.fnUntraced(function* ( - rowIds: ReadonlyArray, - deletePage: (page: ReadonlyArray) => Effect.Effect, - rest: Effect.Effect, - ) { - for (let offset = 0; offset < rowIds.length; offset += DELETE_PAGE_SIZE) { - yield* deletePage(rowIds.slice(offset, offset + DELETE_PAGE_SIZE)); - // Never hold the writer across a rest: foreground commits run between pages. - yield* rest; - } - }); - - const trimThread = Effect.fn("ThreadHistoryTrim.trimThread")(function* ( - candidate: Candidate, - afterDays: number, - rest: Effect.Effect, - ) { + const select = Effect.fn("ThreadHistoryTrim.select")(function* (candidate: Candidate) { const { threadId } = candidate; - const noticeId = ids.derive.runSignalTurnItem({ - runId: candidate.runId, - signal: NOTICE_SIGNAL, - }); - const readBoundary = sql<{ readonly ordinal: number }>` - SELECT ordinal FROM orchestration_v2_projection_turn_items WHERE turn_item_id = ${noticeId} - `; - // The notice goes first: a pass that stops halfway leaves a marked thread the next pass finishes. - if ((yield* readBoundary)[0] === undefined) yield* writeNotice(candidate, afterDays); - const boundary = (yield* readBoundary)[0]?.ordinal; - if (boundary === undefined) return { itemCount: 0, bytes: 0 } satisfies Removed; - const items = yield* sql<{ readonly id: string; readonly parent: string | null; + readonly node: string | null; readonly type: string; readonly status: string; readonly inline: number; @@ -304,13 +207,14 @@ const make = Effect.gen(function* () { SELECT turn_item_id AS id, parent_item_id AS parent, + node_id AS node, type, status, inline_work_output AS inline, ${sql.literal(ITEM_BYTES)} AS bytes FROM orchestration_v2_projection_turn_items - WHERE thread_id = ${threadId} AND ordinal < ${boundary} - `.pipe(NodeSqliteClient.readOnlyBackground); + WHERE thread_id = ${threadId} + `; const trimmedItemTypes: ReadonlyArray = TRIMMED_ITEM_TYPES; const liveItemStatuses: ReadonlyArray = LIVE_ITEM_STATUSES; const removedItems = unreferenced( @@ -321,25 +225,7 @@ const make = Effect.gen(function* () { !liveItemStatuses.includes(item.status), [], ); - yield* deleteInPages( - removedItems.map((item) => item.id), - (page) => - sql.withTransaction( - Effect.gen(function* () { - yield* sql` - DELETE FROM orchestration_v2_projection_turn_items - WHERE turn_item_id IN ${sql.in(page)} - AND status NOT IN ${sql.in(LIVE_ITEM_STATUSES)} - AND inline_work_output = 0 - `; - yield* sql` - DELETE FROM orchestration_v2_turn_item_positions - WHERE thread_id = ${threadId} AND turn_item_id IN ${sql.in(page)} - `; - }), - ), - rest, - ); + const removedItemIds = new Set(removedItems.map((item) => item.id)); const nodes = yield* sql<{ readonly id: string; @@ -359,11 +245,9 @@ const make = Effect.gen(function* () { FROM orchestration_v2_projection_nodes AS node LEFT JOIN orchestration_v2_projection_runs AS run ON run.run_id = node.run_id WHERE node.thread_id = ${threadId} - `.pipe(NodeSqliteClient.readOnlyBackground); + `; const referencedNodes = yield* sql<{ readonly id: string }>` - SELECT node_id AS id FROM orchestration_v2_projection_turn_items - WHERE thread_id = ${threadId} AND node_id IS NOT NULL - UNION SELECT node_id FROM orchestration_v2_projection_messages + SELECT node_id AS id FROM orchestration_v2_projection_messages WHERE thread_id = ${threadId} AND node_id IS NOT NULL UNION SELECT node_id FROM orchestration_v2_projection_plans WHERE thread_id = ${threadId} AND node_id IS NOT NULL @@ -381,7 +265,7 @@ const make = Effect.gen(function* () { WHERE thread_id = ${threadId} AND root_node_id IS NOT NULL UNION SELECT owner_node_id FROM orchestration_v2_projection_provider_threads WHERE thread_id = ${threadId} AND owner_node_id IS NOT NULL - `.pipe(NodeSqliteClient.readOnlyBackground); + `; const trimmedNodeKinds: ReadonlyArray = TRIMMED_NODE_KINDS; const liveNodeStatuses: ReadonlyArray = LIVE_NODE_STATUSES; const removedNodes = unreferenced( @@ -390,23 +274,168 @@ const make = Effect.gen(function* () { node.settled === 1 && trimmedNodeKinds.includes(node.kind) && !liveNodeStatuses.includes(node.status), - referencedNodes.map((row) => row.id), - ); - yield* deleteInPages( - removedNodes.map((node) => node.id), - (page) => sql` - DELETE FROM orchestration_v2_projection_nodes - WHERE node_id IN ${sql.in(page)} AND status NOT IN ${sql.in(LIVE_NODE_STATUSES)} - `, - rest, + [ + ...items.flatMap((item) => + item.node === null || removedItemIds.has(item.id) ? [] : [item.node], + ), + ...referencedNodes.map((row) => row.id), + ], ); + return { items: removedItems, nodes: removedNodes }; + }); + + const estimate = Effect.fn("ThreadHistoryTrim.estimate")(function* (dryRun: boolean) { + const { afterDays, now, idleBefore } = yield* window; + const candidates = afterDays <= 0 ? [] : yield* readCandidates({ idleBefore, now }); + let itemCount = 0; + let estimatedBytes = 0; + for (const candidate of candidates) { + const selection = yield* select(candidate); + itemCount += selection.items.length; + estimatedBytes += sumBytes(selection.items) + sumBytes(selection.nodes); + yield* Effect.yieldNow; + } + return { + dryRun, + afterDays, + threadCount: candidates.length, + itemCount, + estimatedBytes, + } satisfies ThreadHistoryTrimReport; + }); - const bytes = (rows: ReadonlyArray<{ readonly bytes: number }>) => - rows.reduce((total, row) => total + row.bytes, 0); + const noticeEvent = Effect.fn("ThreadHistoryTrim.noticeEvent")(function* ( + candidate: Candidate, + afterDays: number, + ) { + const now = yield* DateTime.now; return { - itemCount: removedItems.length, - bytes: bytes(removedItems) + bytes(removedNodes), - } satisfies Removed; + id: yield* ids.allocate.event({ threadId: candidate.threadId }), + type: "turn-item.updated", + threadId: candidate.threadId, + runId: candidate.runId, + // Every event stamps the thread with its time; reusing the thread's own keeps a trimmed + // thread where it was in lists sorted by activity. + occurredAt: DateTime.makeUnsafe(candidate.updatedAt), + payload: { + id: ids.derive.runSignalTurnItem({ runId: candidate.runId, signal: NOTICE_SIGNAL }), + threadId: candidate.threadId, + runId: candidate.runId, + nodeId: null, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + // EventSink allocates the real position, after the run's existing items. + ordinal: 0, + status: "completed", + title: "Older detail was trimmed", + startedAt: now, + completedAt: now, + updatedAt: now, + type: "system_notice", + message: trimNoticeMessage(afterDays), + }, + } satisfies OrchestrationV2DomainEvent; + }); + + /** Removes one page in a transaction that first confirms the thread is still idle and trimming on. */ + const trimPage = Effect.fn("ThreadHistoryTrim.trimPage")(function* ( + threadId: ThreadId, + page: { readonly itemIds: ReadonlyArray; readonly nodeIds: ReadonlyArray }, + ) { + // Assigned inside the transaction, which narrowing cannot see. + let outcome = "trimmed" as "trimmed" | "thread-in-use" | "trimming-off"; + let removed: { readonly itemIds: ReadonlySet; readonly nodeIds: ReadonlySet } = + { + itemIds: new Set(), + nodeIds: new Set(), + }; + yield* eventSink.writeBuilt( + Effect.gen(function* () { + const { afterDays, now, idleBefore } = yield* window; + if (afterDays <= 0) { + outcome = "trimming-off"; + return []; + } + const current = (yield* readCandidates({ idleBefore, now, threadId }))[0]; + if (current === undefined) { + outcome = "thread-in-use"; + return []; + } + const itemRows = + page.itemIds.length === 0 + ? [] + : yield* sql<{ readonly id: TurnItemId }>` + SELECT turn_item_id AS id FROM orchestration_v2_projection_turn_items + WHERE thread_id = ${threadId} AND turn_item_id IN ${sql.in(page.itemIds)} + AND status NOT IN ${sql.in(LIVE_ITEM_STATUSES)} AND inline_work_output = 0 + `; + const nodeRows = + page.nodeIds.length === 0 + ? [] + : yield* sql<{ readonly id: NodeId }>` + SELECT node_id AS id FROM orchestration_v2_projection_nodes + WHERE thread_id = ${threadId} AND node_id IN ${sql.in(page.nodeIds)} + AND status NOT IN ${sql.in(LIVE_NODE_STATUSES)} + `; + if (itemRows.length === 0 && nodeRows.length === 0) return []; + removed = { + itemIds: new Set(itemRows.map((row) => row.id)), + nodeIds: new Set(nodeRows.map((row) => row.id)), + }; + const noticeId = ids.derive.runSignalTurnItem({ + runId: current.runId, + signal: NOTICE_SIGNAL, + }); + const noticed = yield* sql` + SELECT 1 FROM orchestration_v2_projection_turn_items WHERE turn_item_id = ${noticeId} + `; + const events: Array = + noticed.length === 0 ? [yield* noticeEvent(current, afterDays)] : []; + events.push({ + id: yield* ids.allocate.event({ threadId }), + type: "thread.history-trimmed", + threadId, + occurredAt: DateTime.makeUnsafe(current.updatedAt), + payload: { + turnItemIds: itemRows.map((row) => row.id), + nodeIds: nodeRows.map((row) => row.id), + }, + }); + return events; + }), + ); + return { outcome, removed }; + }); + + const trimThread = Effect.fn("ThreadHistoryTrim.trimThread")(function* ( + candidate: Candidate, + rest: Effect.Effect, + ) { + const selection = yield* select(candidate).pipe(NodeSqliteClient.readOnlyBackground); + const rows = [ + ...selection.items.map((row) => ({ ...row, kind: "item" as const })), + ...selection.nodes.map((row) => ({ ...row, kind: "node" as const })), + ]; + let itemCount = 0; + let bytes = 0; + for (let offset = 0; offset < rows.length; offset += DELETE_PAGE_SIZE) { + const page = rows.slice(offset, offset + DELETE_PAGE_SIZE); + const { outcome, removed } = yield* trimPage(candidate.threadId, { + itemIds: page.flatMap((row) => (row.kind === "item" ? [row.id] : [])), + nodeIds: page.flatMap((row) => (row.kind === "node" ? [row.id] : [])), + }); + if (outcome !== "trimmed") return { outcome, itemCount, bytes }; + for (const row of page) { + if (!(row.kind === "item" ? removed.itemIds : removed.nodeIds).has(row.id)) continue; + if (row.kind === "item") itemCount += 1; + bytes += row.bytes; + } + // Never hold the writer across a rest: foreground commits run between pages. + yield* rest; + } + return { outcome: "trimmed" as const, itemCount, bytes }; }); const readLastPass = sql<{ readonly updated_at: string }>` @@ -470,7 +499,7 @@ const make = Effect.gen(function* () { const current = yield* readCandidates({ idleBefore, now, threadId: listed.threadId }); const candidate = current[0]; if (candidate === undefined) continue; - const removed = yield* trimThread(candidate, afterDays, rest).pipe( + const removed = yield* trimThread(candidate, rest).pipe( Effect.catchCauseIf( (cause) => !Cause.hasInterruptsOnly(cause), (cause) => @@ -481,6 +510,8 @@ const make = Effect.gen(function* () { ), ); if (removed === null) continue; + if (removed.outcome === "trimming-off") return; + if (removed.itemCount === 0 && removed.bytes === 0) continue; threadCount += 1; itemCount += removed.itemCount; estimatedBytes += removed.bytes; diff --git a/packages/client-runtime/src/state/orchestrationV2Projection.test.ts b/packages/client-runtime/src/state/orchestrationV2Projection.test.ts index 16301157a6a4e..8bac0693cc80e 100644 --- a/packages/client-runtime/src/state/orchestrationV2Projection.test.ts +++ b/packages/client-runtime/src/state/orchestrationV2Projection.test.ts @@ -583,7 +583,7 @@ describe("applyOrchestrationV2ProjectionEvent", () => { expect(visibleIds(apply(bounded, [failed], partial))).toEqual([reply.id]); }); - it("removes only hidden local items while preserving inherited rows", () => { + it("removes only hidden or trimmed local items while preserving inherited rows", () => { const inherited = commandItem("item-inherited"); const local = commandItem("item-local"); const inheritedRow = { @@ -618,6 +618,16 @@ describe("applyOrchestrationV2ProjectionEvent", () => { const next = applyOrchestrationV2ProjectionEvent(projection, event); expect(next?.visibleTurnItems).toEqual([inheritedRow]); expect(next?.visibleTurnItems[0]).toBe(inheritedRow); + + const trimmed = applyOrchestrationV2ProjectionEvent(projection, { + id: "event-history-trimmed", + type: "thread.history-trimmed", + threadId, + occurredAt: now, + payload: { turnItemIds: [local.id, inherited.id], nodeIds: [] }, + } as OrchestrationV2DomainEvent); + expect(trimmed?.turnItems).toEqual([]); + expect(trimmed?.visibleTurnItems).toEqual([inheritedRow]); }); }); diff --git a/packages/client-runtime/src/state/orchestrationV2Projection.ts b/packages/client-runtime/src/state/orchestrationV2Projection.ts index 37da408a1c35b..17870ec54850d 100644 --- a/packages/client-runtime/src/state/orchestrationV2Projection.ts +++ b/packages/client-runtime/src/state/orchestrationV2Projection.ts @@ -330,6 +330,20 @@ export function applyOrchestrationV2ProjectionEvent( ...projection, thread: applyOrchestrationV2ThreadVisit(projection.thread, event.payload), }; + case "thread.history-trimmed": { + const items = new Set(event.payload.turnItemIds); + const nodes = new Set(event.payload.nodeIds); + return { + ...projection, + turnItems: projection.turnItems.filter((item) => !items.has(item.id)), + nodes: projection.nodes.filter((node) => !nodes.has(node.id)), + visibleTurnItems: renumberVisibleItems( + projection.visibleTurnItems.filter( + (row) => row.sourceThreadId !== projection.thread.id || !items.has(row.sourceItemId), + ), + ), + }; + } case "run.created": case "run.updated": { const next = { ...base, runs: upsertEntity(base.runs, event.payload) }; diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index 765db9d8a5fb1..42ac289b96e02 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -1584,6 +1584,12 @@ export const OrchestrationV2CheckpointRollbackRequest = Schema.Struct({ export type OrchestrationV2CheckpointRollbackRequest = typeof OrchestrationV2CheckpointRollbackRequest.Type; +/** Rows a history trim removed from the thread's projection. */ +const OrchestrationV2ThreadHistoryTrimmed = Schema.Struct({ + turnItemIds: Schema.Array(TurnItemId), + nodeIds: Schema.Array(NodeId), +}); + export class OrchestrationV2CheckpointUnavailableError extends Schema.TaggedError()( "OrchestrationV2CheckpointUnavailableError", { @@ -2182,6 +2188,11 @@ export const OrchestrationV2DomainEvent = Schema.Union([ type: Schema.Literal("checkpoint.rollback-requested"), payload: OrchestrationV2CheckpointRollbackRequest, }), + Schema.Struct({ + ...OrchestrationV2EventBase.fields, + type: Schema.Literal("thread.history-trimmed"), + payload: OrchestrationV2ThreadHistoryTrimmed, + }), Schema.Struct({ ...OrchestrationV2EventBase.fields, type: Schema.Literal("context-handoff.updated"), @@ -3130,6 +3141,11 @@ export const OrchestrationV2DomainEventJson = Schema.Union([ type: Schema.Literal("checkpoint.rollback-requested"), payload: OrchestrationV2CheckpointRollbackRequestJson, }), + Schema.Struct({ + ...OrchestrationV2JsonEventBaseFields, + type: Schema.Literal("thread.history-trimmed"), + payload: OrchestrationV2ThreadHistoryTrimmed, + }), Schema.Struct({ ...OrchestrationV2JsonEventBaseFields, type: Schema.Literal("context-handoff.updated"), From 6940c4a8c8641c6b590724ef36166ecf4e70954a Mon Sep 17 00:00:00 2001 From: JJ Eaton <58495321+jayleaton@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:30:30 +0700 Subject: [PATCH 2/5] test(server): type the trim test hooks Co-Authored-By: Claude Opus 5.5 --- apps/server/src/orchestration-v2/ThreadHistoryTrim.test.ts | 3 ++- .../client-runtime/src/state/orchestrationV2Projection.test.ts | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/server/src/orchestration-v2/ThreadHistoryTrim.test.ts b/apps/server/src/orchestration-v2/ThreadHistoryTrim.test.ts index 7842753b87913..f96743c993c7f 100644 --- a/apps/server/src/orchestration-v2/ThreadHistoryTrim.test.ts +++ b/apps/server/src/orchestration-v2/ThreadHistoryTrim.test.ts @@ -1,6 +1,7 @@ import { assert, it } from "@effect/vitest"; import { EventId, + ServerSettingsError, MessageId, NodeId, type OrchestrationV2AppThread, @@ -484,7 +485,7 @@ it.layer(layerTest(30))("ThreadHistoryTrim", (it) => { type Hook = Effect.Effect< void, - unknown, + EventSink.EventSinkV2Error | ProjectionStore.ProjectionStoreV2Error | ServerSettingsError, | EventSink.EventSinkV2 | EventStore.EventStoreV2 | ProjectionStore.ProjectionStoreV2 diff --git a/packages/client-runtime/src/state/orchestrationV2Projection.test.ts b/packages/client-runtime/src/state/orchestrationV2Projection.test.ts index 8bac0693cc80e..f9acf3becf81e 100644 --- a/packages/client-runtime/src/state/orchestrationV2Projection.test.ts +++ b/packages/client-runtime/src/state/orchestrationV2Projection.test.ts @@ -625,7 +625,7 @@ describe("applyOrchestrationV2ProjectionEvent", () => { threadId, occurredAt: now, payload: { turnItemIds: [local.id, inherited.id], nodeIds: [] }, - } as OrchestrationV2DomainEvent); + } as unknown as OrchestrationV2DomainEvent); expect(trimmed?.turnItems).toEqual([]); expect(trimmed?.visibleTurnItems).toEqual([inheritedRow]); }); From 5363779f8215a4513c66acbf888bca8349d0f6bb Mon Sep 17 00:00:00 2001 From: JJ Eaton <58495321+jayleaton@users.noreply.github.com> Date: Wed, 18 Nov 2026 00:14:05 +0700 Subject: [PATCH 3/5] fix(server): history trim keeps ancestors and revises forks Co-Authored-By: Claude Opus 5.5 --- .../src/orchestration-v2/ProjectionStore.ts | 43 ++- .../ThreadHistoryTrim.test.ts | 321 ++++++++++++------ .../src/orchestration-v2/ThreadHistoryTrim.ts | 213 +++++++++--- .../state/orchestrationV2Projection.test.ts | 35 +- .../src/state/orchestrationV2Projection.ts | 38 ++- packages/contracts/src/orchestrationV2.ts | 18 +- 6 files changed, 507 insertions(+), 161 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index b7dfb8835f108..65eb9703174be 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -1046,6 +1046,27 @@ export function emptyProjection( }; } +/** A fork shows its source's trim notice after the last row it inherits from the notice's run. */ +function withInheritedNotice( + rows: OrchestrationV2ThreadProjection["visibleTurnItems"], + notice: OrchestrationV2TurnItem | undefined, + sourceThreadId: ThreadId, +): OrchestrationV2ThreadProjection["visibleTurnItems"] { + if (notice === undefined || rows.some((row) => row.sourceItemId === notice.id)) return rows; + const after = rows.findLastIndex( + (row) => row.sourceThreadId === sourceThreadId && row.item.runId === notice.runId, + ); + if (after === -1) return rows; + const row = { + position: 0, + visibility: "inherited" as const, + sourceThreadId, + sourceItemId: notice.id, + item: notice, + }; + return [...rows.slice(0, after + 1), row, ...rows.slice(after + 1)]; +} + export function applyToProjection( projection: OrchestrationV2ThreadProjection, event: OrchestrationV2DomainEvent, @@ -1103,15 +1124,22 @@ export function applyToProjection( thread: event.payload, }; case "thread.history-trimmed": { + const { sourceThreadId } = event.payload; const items = new Set(event.payload.turnItemIds); - const nodes = new Set(event.payload.nodeIds); + const nodes = new Set(sourceThreadId === event.threadId ? event.payload.nodeIds : []); return { ...projection, - turnItems: projection.turnItems.filter((item) => !items.has(item.id)), + turnItems: projection.turnItems.filter( + (item) => item.threadId !== sourceThreadId || !items.has(item.id), + ), nodes: projection.nodes.filter((node) => !nodes.has(node.id)), - visibleTurnItems: projection.visibleTurnItems - .filter((row) => row.sourceThreadId !== event.threadId || !items.has(row.sourceItemId)) - .map((row, position) => (row.position === position ? row : { ...row, position })), + visibleTurnItems: withInheritedNotice( + projection.visibleTurnItems.filter( + (row) => row.sourceThreadId !== sourceThreadId || !items.has(row.sourceItemId), + ), + sourceThreadId === event.threadId ? undefined : event.payload.notice, + sourceThreadId, + ).map((row, position) => (row.position === position ? row : { ...row, position })), }; } case "thread.visit-recorded": @@ -3351,6 +3379,8 @@ export const layer: Layer.Layer = break; case "thread.history-trimmed": { const { turnItemIds, nodeIds } = event.payload; + // A fork's copy only revises the fork; the source's own copy removed the rows. + if (event.payload.sourceThreadId !== event.threadId) break; if (turnItemIds.length > 0) { yield* sql` DELETE FROM orchestration_v2_projection_turn_items @@ -3469,7 +3499,8 @@ export const layer: Layer.Layer = event.type !== "thread.interaction-mode-updated" && event.type !== "thread.model-selection-updated" && event.type !== "thread.provider-switched" && - event.type !== "thread.guest-share-updated" + event.type !== "thread.guest-share-updated" && + event.type !== "thread.history-trimmed" ) { const updatedAt = yield* encodeThreadUpdatedAt(event.occurredAt); yield* sql` diff --git a/apps/server/src/orchestration-v2/ThreadHistoryTrim.test.ts b/apps/server/src/orchestration-v2/ThreadHistoryTrim.test.ts index f96743c993c7f..6950867b75abf 100644 --- a/apps/server/src/orchestration-v2/ThreadHistoryTrim.test.ts +++ b/apps/server/src/orchestration-v2/ThreadHistoryTrim.test.ts @@ -16,6 +16,7 @@ import { TurnItemId, } from "@t3tools/contracts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; +import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; @@ -64,6 +65,8 @@ type SeedOptions = { readonly runStatus?: OrchestrationV2Run["status"]; readonly userMessageAt?: DateTime.Utc; readonly extraReasoning?: number; + /** A reasoning row under `command`, written after every other row. */ + readonly commandChild?: boolean; }; const seedThread = Effect.fn(function* (name: string, options: SeedOptions = {}) { @@ -208,14 +211,6 @@ const seedThread = Effect.fn(function* (name: string, options: SeedOptions = {}) attachments: [], }), item({ ...itemBase("reasoning"), type: "reasoning", text: "thinking", streaming: false }), - ...Array.from({ length: options.extraReasoning ?? 0 }, (_, index) => - item({ - ...itemBase(`reasoning-${index}`), - type: "reasoning", - text: "more thinking", - streaming: false, - }), - ), item({ ...itemBase("command", commandNodeId), type: "command_execution", @@ -269,6 +264,24 @@ const seedThread = Effect.fn(function* (name: string, options: SeedOptions = {}) text: "Fixed the harness.", streaming: false, }), + ...Array.from({ length: options.extraReasoning ?? 0 }, (_, index) => + item({ + ...itemBase(`reasoning-${index}`), + type: "reasoning", + text: "more thinking", + streaming: false, + }), + ), + ...(options.commandChild === true + ? [ + item({ + ...itemBase("command-child", null, "command"), + type: "reasoning", + text: "why the command ran", + streaming: false, + }), + ] + : []), ], }); return { threadId, runId }; @@ -491,31 +504,65 @@ type Hook = Effect.Effect< | ProjectionStore.ProjectionStoreV2 | ServerSettings.ServerSettingsService >; -const isNotice = (events: ReadonlyArray) => - events.some( - (event) => event.type === "turn-item.updated" && event.payload.type === "system_notice", - ); -/** Runs `hook` at the pass's `atWrite`th write: after it chose what to remove, before removing it. */ -const duringTrim = (hook: Hook, atWrite = 1) => { - const hooked = Layer.effect( - EventSink.EventSinkV2, - Effect.gen(function* () { - const sink = yield* EventSink.EventSinkV2; - const context = yield* Effect.context>(); - let writes = 0; - const next = Effect.suspend(() => - ++writes === atWrite ? hook.pipe(Effect.provide(context), Effect.orDie) : Effect.void, - ); - return EventSink.EventSinkV2.of({ - ...sink, - write: (input) => - sink.write(input).pipe(Effect.tap(() => (isNotice(input.events) ? next : Effect.void))), - writeBuilt: (build) => next.pipe(Effect.andThen(sink.writeBuilt(build))), - }); - }), - ); - return ThreadHistoryTrim.layer.pipe(Layer.provide(hooked), Layer.provideMerge(dependencies(30))); -}; +class TrimHook extends Context.Service< + TrimHook, + { + /** Runs `hook` before the pass's `atWrite`th page: after it chose rows, before it removes them. */ + readonly arm: (hook: Hook, atWrite?: number) => Effect.Effect; + readonly disarm: Effect.Effect; + readonly next: Hook; + } +>()("t3/orchestration-v2/ThreadHistoryTrim.test/TrimHook") {} +const trimHook = Layer.effect( + TrimHook, + Effect.sync(() => { + let armed: { readonly hook: Hook; readonly atWrite: number; writes: number } | null = null; + return TrimHook.of({ + arm: (hook, atWrite = 1) => + Effect.sync(() => { + armed = { hook, atWrite, writes: 0 }; + }), + disarm: Effect.sync(() => { + armed = null; + }), + next: Effect.suspend(() => + armed !== null && ++armed.writes === armed.atWrite ? armed.hook : Effect.void, + ), + }); + }), +); +const hookedSink = Layer.effect( + EventSink.EventSinkV2, + Effect.gen(function* () { + const sink = yield* EventSink.EventSinkV2; + const hooks = yield* TrimHook; + const context = yield* Effect.context>(); + return EventSink.EventSinkV2.of({ + ...sink, + writeBuilt: (build) => + hooks.next.pipe( + Effect.provide(context), + Effect.orDie, + Effect.andThen(sink.writeBuilt(build)), + ), + }); + }), +); +// One migrated database for every hooked case; `isolate` clears what an earlier case left. +const hookedFixture = Layer.mergeAll( + ThreadHistoryTrim.layer.pipe(Layer.provide(hookedSink)), + Layer.effectDiscard(createProject), +).pipe(Layer.provideMerge(Layer.mergeAll(trimHook, dependencies(30)))); +const isolate = Effect.gen(function* () { + yield* TestClock.setTime(DateTime.toEpochMillis(now)); + yield* (yield* TrimHook).disarm; + yield* (yield* ServerSettings.ServerSettingsService).updateSettings({ + threadHistoryTrimAfterDays: 30, + }); + yield* trim; +}); +const arm = (hook: Hook, atWrite?: number) => + Effect.flatMap(TrimHook, (hooks) => hooks.arm(hook, atWrite)); const updateThread = ( name: string, @@ -619,86 +666,170 @@ const activities: ReadonlyArray = [ ], ]; -it.effect.each(activities)( - "stops trimming a thread that is %s during the pass", - ([name, activity]) => - Effect.gen(function* () { - yield* TestClock.setTime(DateTime.toEpochMillis(now)); - const { threadId } = yield* seedThread(name); - yield* trim; - assert.deepEqual(yield* itemTitles(threadId), everyItem); - }).pipe(Effect.provide(duringTrim(activity))), -); - -it.effect("stops between pages once the thread is opened", () => - Effect.gen(function* () { - yield* TestClock.setTime(DateTime.toEpochMillis(now)); - const { threadId } = yield* seedThread("opened", { extraReasoning: 250 }); - yield* trim; - // The first page of 200 went; the rest stays. - assert.lengthOf(yield* itemTitles(threadId), everyItem.length + 250 - 200 + 1); - }).pipe( - Effect.provide(duringTrim(updateThread("opened", "thread.visited", { lastVisitedAt: now }), 2)), - ), -); +const ids = (projection: OrchestrationV2ThreadProjection) => ({ + turnItems: projection.turnItems.map((item) => item.id).toSorted(), + nodes: projection.nodes.map((node) => node.id).toSorted(), + visible: projection.visibleTurnItems.map((row) => row.sourceItemId), +}); +type Snapshot = { + readonly snapshotSequence: number; + readonly projection: OrchestrationV2ThreadProjection; +}; -it.effect( - "advances the thread's revision so reconnects, windows and search drop trimmed rows", - () => { - const name = "reconnect"; - const threadId = ThreadId.make(`thread:trim:${name}`); - const removed = [`item:${name}:reasoning`, `item:${name}:command`]; - let warmed: - | { readonly snapshotSequence: number; readonly projection: OrchestrationV2ThreadProjection } - | undefined; - // A client that read the thread mid-pass, and left the timeline and search caches warm. - const read: Hook = Effect.gen(function* () { - const store = yield* ProjectionStore.ProjectionStoreV2; - warmed = yield* store.getThreadSnapshot(threadId); - yield* store.getThreadSnapshotWindow(threadId, { rowLimit: 50 }); - yield* store.searchThread({ threadId, query: "flaky" }); - }); - return Effect.gen(function* () { - yield* TestClock.setTime(DateTime.toEpochMillis(now)); +/** A client that read `threadId` mid-pass, left the caches warm, and then replays its stream. */ +const midPassReader = (threadId: ThreadId) => { + let warmed: Snapshot | undefined; + const read: Hook = Effect.gen(function* () { + const store = yield* ProjectionStore.ProjectionStoreV2; + warmed = yield* store.getThreadSnapshot(threadId); + yield* store.getThreadSnapshotWindow(threadId, { rowLimit: 50 }); + yield* store.searchThread({ threadId, query: "flaky" }); + }); + const verify = (removed: Array) => + Effect.gen(function* () { const store = yield* ProjectionStore.ProjectionStoreV2; - const events = yield* EventStore.EventStoreV2; - yield* createProject; - yield* seedThread(name); - yield* trim; const before = warmed!; - assert.includeMembers( - before.projection.turnItems.map((item) => item.id), - removed, - ); + assert.includeMembers(ids(before.projection).visible, removed); const after = yield* store.getThreadSnapshot(threadId); assert.isAbove(after.snapshotSequence, before.snapshotSequence); - const ids = (projection: OrchestrationV2ThreadProjection) => ({ - turnItems: projection.turnItems.map((item) => item.id).toSorted(), - nodes: projection.nodes.map((node) => node.id).toSorted(), - visible: projection.visibleTurnItems.map((row) => row.sourceItemId), - }); + assert.notIncludeMembers(ids(after.projection).visible, removed); + // The events a live subscriber hears, and a reconnect after `before` replays. const replay = yield* Stream.runCollect( - events.read({ threadId, afterSequence: before.snapshotSequence }), + (yield* EventStore.EventStoreV2).read({ threadId, afterSequence: before.snapshotSequence }), ); const resumed = replay.reduce( (projection, stored) => ProjectionStore.applyToProjection(projection, stored.event), before.projection, ); assert.deepEqual(ids(resumed), ids(after.projection)); - assert.notIncludeMembers(ids(after.projection).turnItems, removed); const window = yield* store.getThreadSnapshotWindow(threadId, { rowLimit: 50 }); assert.equal(window.snapshotSequence, after.snapshotSequence); - assert.notIncludeMembers( - window.projection.turnItems.map((item) => item.id), - removed, - ); + assert.notIncludeMembers(ids(window.projection).visible, removed); const found = yield* store.searchThread({ threadId, query: "flaky" }); assert.equal(found.snapshotSequence, after.snapshotSequence); - }).pipe(Effect.provide(duringTrim(read))); - }, -); + }); + return { read, verify }; +}; + +const orphanedItems = Effect.fn(function* (threadId: ThreadId) { + const sql = yield* SqlClient.SqlClient; + const rows = yield* sql<{ readonly id: string }>` + SELECT child.turn_item_id AS id FROM orchestration_v2_projection_turn_items AS child + WHERE child.thread_id = ${threadId} AND child.parent_item_id IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM orchestration_v2_projection_turn_items AS parent + WHERE parent.turn_item_id = child.parent_item_id + ) + `; + return rows.map((row) => row.id); +}); + +it.layer(hookedFixture)("ThreadHistoryTrim during a pass", (it) => { + it.effect.each(activities)("stops trimming a thread that is %s", ([name, activity]) => + Effect.gen(function* () { + yield* isolate; + const { threadId } = yield* seedThread(name); + yield* arm(activity); + yield* trim; + assert.deepEqual(yield* itemTitles(threadId), everyItem); + }), + ); + + it.effect("stops between pages without orphaning a kept row", () => + Effect.gen(function* () { + yield* isolate; + const name = "opened-between-pages"; + // `command` sorts into the first page and its child into the second, unless children go first. + const { threadId } = yield* seedThread(name, { extraReasoning: 250, commandChild: true }); + yield* arm(updateThread(name, "thread.visited", { lastVisitedAt: now }), 2); + yield* trim; + assert.lengthOf(yield* itemTitles(threadId), everyItem.length + 251 - 200 + 1); + assert.deepEqual(yield* orphanedItems(threadId), []); + }), + ); + + it.effect("keeps a row that gains a child after the pass chose it", () => + Effect.gen(function* () { + yield* isolate; + const name = "gains-child"; + const { threadId, runId } = yield* seedThread(name); + const child: Hook = Effect.flatMap(EventSink.EventSinkV2, (sink) => + sink.write({ + events: [ + { + id: EventId.make(`event:${name}:child`), + type: "turn-item.updated", + threadId, + runId, + occurredAt: old, + payload: { + id: TurnItemId.make(`item:${name}:late-edit`), + threadId, + runId, + nodeId: null, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: TurnItemId.make(`item:${name}:command`), + ordinal: 0, + status: "completed", + title: "late-edit", + startedAt: old, + completedAt: old, + updatedAt: old, + type: "file_change", + fileName: "late.ts", + }, + }, + ], + }), + ); + yield* arm(child); + yield* trim; + assert.include(yield* itemTitles(threadId), "command"); + assert.deepEqual(yield* orphanedItems(threadId), []); + }), + ); + + it.effect( + "advances the thread's revision so reconnects, windows and search drop trimmed rows", + () => + Effect.gen(function* () { + yield* isolate; + const name = "reconnect"; + const { threadId } = yield* seedThread(name); + const reader = midPassReader(threadId); + yield* arm(reader.read); + yield* trim; + yield* reader.verify([`item:${name}:reasoning`, `item:${name}:command`]); + }), + ); + + it.effect("advances an active fork's revision when the rows it inherits are trimmed", () => + Effect.gen(function* () { + yield* isolate; + const source = yield* seedThread("fork-source"); + const fork = yield* seedThread("fork", { + userMessageAt: recent, + thread: { + lineage: { + parentThreadId: source.threadId, + relationshipToParent: "fork", + rootThreadId: source.threadId, + }, + forkedFrom: { type: "run", threadId: source.threadId, runId: source.runId }, + }, + }); + const reader = midPassReader(fork.threadId); + yield* arm(reader.read); + yield* trim; + yield* reader.verify(["item:fork-source:reasoning", "item:fork-source:command"]); + assert.deepEqual(yield* itemTitles(fork.threadId), everyItem); + }), + ); +}); it.layer(layerTest(0))("ThreadHistoryTrim turned off", (it) => { it.effect("removes nothing", () => diff --git a/apps/server/src/orchestration-v2/ThreadHistoryTrim.ts b/apps/server/src/orchestration-v2/ThreadHistoryTrim.ts index d7815d22d31a7..91a89f61b868a 100644 --- a/apps/server/src/orchestration-v2/ThreadHistoryTrim.ts +++ b/apps/server/src/orchestration-v2/ThreadHistoryTrim.ts @@ -1,6 +1,7 @@ import { type NodeId, type OrchestrationV2DomainEvent, + type OrchestrationV2TurnItem, type RunId, type ThreadHistoryTrimReport, type ThreadHistoryTrimStatus, @@ -26,6 +27,7 @@ import * as ServerSettingsModule from "../serverSettings.ts"; import { forkParked } from "../serverActivation.ts"; import * as EventSink from "./EventSink.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; +import * as TurnItemPositionStore from "./TurnItemPositionStore.ts"; export class ThreadHistoryTrimError extends Schema.TaggedError()( "ThreadHistoryTrimError", @@ -93,7 +95,7 @@ function trimNoticeMessage(afterDays: number): string { return `Tool output and reasoning above this point were removed to save space after ${afterDays} ${afterDays === 1 ? "day" : "days"} without use. Messages, plans and file changes are kept.`; } -/** Ids no kept row points at, following `parent` links so a kept row never loses an ancestor. */ +/** Ids no kept row points at, children first, so a pass that stops early never orphans a row. */ function unreferenced( rows: ReadonlyArray, removable: (row: Row) => boolean, @@ -109,7 +111,37 @@ function unreferenced !kept.has(row.id)); + const depth = (row: Row) => { + let count = 0; + for (let current = row.parent; current !== null; current = byId.get(current)?.parent ?? null) + count += 1; + return count; + }; + return rows + .filter((row) => !kept.has(row.id)) + .map((row) => ({ row, depth: depth(row) })) + .toSorted((left, right) => right.depth - left.depth) + .map(({ row }) => row); +} + +/** Drops each id that a row outside `removable` still hangs off, until none does. */ +function withoutParentsOfKept( + removable: ReadonlyArray, + children: ReadonlyArray<{ readonly id: string; readonly parent: string }>, + blocked: Iterable, +): ReadonlySet { + const removed = new Set(removable); + for (const id of blocked) removed.delete(id); + for (let changed = true; changed;) { + changed = false; + for (const child of children) { + if (removed.has(child.parent) && !removed.has(child.id)) { + removed.delete(child.parent); + changed = true; + } + } + } + return removed; } const make = Effect.gen(function* () { @@ -117,6 +149,7 @@ const make = Effect.gen(function* () { const eventSink = yield* EventSink.EventSinkV2; const ids = yield* IdAllocator.IdAllocatorV2; const settingsService = yield* ServerSettingsModule.ServerSettingsService; + const positions = yield* TurnItemPositionStore.TurnItemPositionStoreV2; // A subagent has no user of its own, so it ages with the thread that started it. const readCandidates = (input: { @@ -193,6 +226,42 @@ const make = Effect.gen(function* () { }; }); + // Rows other than turn items and nodes that keep a node. + const nodeReferences = (threadId: ThreadId) => sql` + SELECT node_id AS id FROM orchestration_v2_projection_messages + WHERE thread_id = ${threadId} AND node_id IS NOT NULL + UNION SELECT node_id FROM orchestration_v2_projection_plans + WHERE thread_id = ${threadId} AND node_id IS NOT NULL + UNION SELECT parent_node_id FROM orchestration_v2_projection_subagents + WHERE thread_id = ${threadId} + UNION SELECT node_id FROM orchestration_v2_projection_runtime_requests + WHERE thread_id = ${threadId} AND node_id IS NOT NULL + UNION SELECT node_id FROM orchestration_v2_projection_checkpoints + WHERE thread_id = ${threadId} + UNION SELECT node_id FROM orchestration_v2_projection_checkpoint_scopes + WHERE thread_id = ${threadId} AND node_id IS NOT NULL + UNION SELECT node_id FROM orchestration_v2_projection_provider_turns + WHERE thread_id = ${threadId} + UNION SELECT root_node_id FROM orchestration_v2_projection_run_attempts + WHERE thread_id = ${threadId} AND root_node_id IS NOT NULL + UNION SELECT owner_node_id FROM orchestration_v2_projection_provider_threads + WHERE thread_id = ${threadId} AND owner_node_id IS NOT NULL + `; + + // Forks that show this thread's rows, through any depth of forking; side chats inherit none. + const readForks = (threadId: ThreadId) => sql<{ readonly id: ThreadId }>` + WITH RECURSIVE forks(thread_id) AS ( + SELECT ${threadId} + UNION + SELECT fork.thread_id FROM orchestration_v2_projection_threads AS fork + JOIN forks ON json_extract(fork.payload_json, '$.forkedFrom.threadId') = forks.thread_id + WHERE json_extract(fork.payload_json, '$.forkedFrom.type') = 'run' + AND COALESCE(json_extract(fork.payload_json, '$.lineage.sideChat'), 0) = 0 + AND fork.deleted_at IS NULL + ) + SELECT thread_id AS id FROM forks WHERE thread_id <> ${threadId} + `; + const select = Effect.fn("ThreadHistoryTrim.select")(function* (candidate: Candidate) { const { threadId } = candidate; const items = yield* sql<{ @@ -246,26 +315,7 @@ const make = Effect.gen(function* () { LEFT JOIN orchestration_v2_projection_runs AS run ON run.run_id = node.run_id WHERE node.thread_id = ${threadId} `; - const referencedNodes = yield* sql<{ readonly id: string }>` - SELECT node_id AS id FROM orchestration_v2_projection_messages - WHERE thread_id = ${threadId} AND node_id IS NOT NULL - UNION SELECT node_id FROM orchestration_v2_projection_plans - WHERE thread_id = ${threadId} AND node_id IS NOT NULL - UNION SELECT parent_node_id FROM orchestration_v2_projection_subagents - WHERE thread_id = ${threadId} - UNION SELECT node_id FROM orchestration_v2_projection_runtime_requests - WHERE thread_id = ${threadId} AND node_id IS NOT NULL - UNION SELECT node_id FROM orchestration_v2_projection_checkpoints - WHERE thread_id = ${threadId} - UNION SELECT node_id FROM orchestration_v2_projection_checkpoint_scopes - WHERE thread_id = ${threadId} AND node_id IS NOT NULL - UNION SELECT node_id FROM orchestration_v2_projection_provider_turns - WHERE thread_id = ${threadId} - UNION SELECT root_node_id FROM orchestration_v2_projection_run_attempts - WHERE thread_id = ${threadId} AND root_node_id IS NOT NULL - UNION SELECT owner_node_id FROM orchestration_v2_projection_provider_threads - WHERE thread_id = ${threadId} AND owner_node_id IS NOT NULL - `; + const referencedNodes = yield* sql<{ readonly id: string }>`${nodeReferences(threadId)}`; const trimmedNodeKinds: ReadonlyArray = TRIMMED_NODE_KINDS; const liveNodeStatuses: ReadonlyArray = LIVE_NODE_STATUSES; const removedNodes = unreferenced( @@ -363,27 +413,75 @@ const make = Effect.gen(function* () { outcome = "thread-in-use"; return []; } - const itemRows = + const itemIds = page.itemIds.length === 0 ? [] : yield* sql<{ readonly id: TurnItemId }>` - SELECT turn_item_id AS id FROM orchestration_v2_projection_turn_items - WHERE thread_id = ${threadId} AND turn_item_id IN ${sql.in(page.itemIds)} - AND status NOT IN ${sql.in(LIVE_ITEM_STATUSES)} AND inline_work_output = 0 - `; - const nodeRows = + SELECT turn_item_id AS id FROM orchestration_v2_projection_turn_items + WHERE thread_id = ${threadId} AND turn_item_id IN ${sql.in(page.itemIds)} + AND status NOT IN ${sql.in(LIVE_ITEM_STATUSES)} AND inline_work_output = 0 + `; + const itemChildren = + page.itemIds.length === 0 + ? [] + : yield* sql<{ + readonly id: string; + readonly parent: string; + }>` + SELECT turn_item_id AS id, parent_item_id AS parent + FROM orchestration_v2_projection_turn_items + WHERE thread_id = ${threadId} AND parent_item_id IN ${sql.in(page.itemIds)} + `; + const items = withoutParentsOfKept( + itemIds.map((row) => row.id), + itemChildren, + [], + ); + + const nodeIds = page.nodeIds.length === 0 ? [] : yield* sql<{ readonly id: NodeId }>` - SELECT node_id AS id FROM orchestration_v2_projection_nodes - WHERE thread_id = ${threadId} AND node_id IN ${sql.in(page.nodeIds)} - AND status NOT IN ${sql.in(LIVE_NODE_STATUSES)} - `; - if (itemRows.length === 0 && nodeRows.length === 0) return []; - removed = { - itemIds: new Set(itemRows.map((row) => row.id)), - nodeIds: new Set(nodeRows.map((row) => row.id)), - }; + SELECT node_id AS id FROM orchestration_v2_projection_nodes + WHERE thread_id = ${threadId} AND node_id IN ${sql.in(page.nodeIds)} + AND status NOT IN ${sql.in(LIVE_NODE_STATUSES)} + `; + const nodeChildren = + page.nodeIds.length === 0 + ? [] + : yield* sql<{ + readonly id: string; + readonly parent: string; + }>` + SELECT node_id AS id, parent_node_id AS parent FROM orchestration_v2_projection_nodes + WHERE thread_id = ${threadId} AND parent_node_id IN ${sql.in(page.nodeIds)} + `; + const nodeHolders = + page.nodeIds.length === 0 + ? [] + : yield* sql<{ + readonly id: string; + readonly node: string; + }>` + SELECT turn_item_id AS id, node_id AS node FROM orchestration_v2_projection_turn_items + WHERE thread_id = ${threadId} AND node_id IN ${sql.in(page.nodeIds)} + `; + const otherReferences = + page.nodeIds.length === 0 + ? [] + : yield* sql<{ readonly id: string }>` + SELECT id FROM (${nodeReferences(threadId)}) WHERE id IN ${sql.in(page.nodeIds)} + `; + const nodes = withoutParentsOfKept( + nodeIds.map((row) => row.id), + nodeChildren, + [ + ...nodeHolders.flatMap((row) => (items.has(row.id) ? [] : [row.node])), + ...otherReferences.map((row) => row.id), + ], + ); + if (items.size === 0 && nodes.size === 0) return []; + removed = { itemIds: items, nodeIds: nodes }; const noticeId = ids.derive.runSignalTurnItem({ runId: current.runId, signal: NOTICE_SIGNAL, @@ -391,18 +489,31 @@ const make = Effect.gen(function* () { const noticed = yield* sql` SELECT 1 FROM orchestration_v2_projection_turn_items WHERE turn_item_id = ${noticeId} `; - const events: Array = - noticed.length === 0 ? [yield* noticeEvent(current, afterDays)] : []; - events.push({ - id: yield* ids.allocate.event({ threadId }), - type: "thread.history-trimmed", - threadId, - occurredAt: DateTime.makeUnsafe(current.updatedAt), - payload: { - turnItemIds: itemRows.map((row) => row.id), - nodeIds: nodeRows.map((row) => row.id), - }, - }); + const events: Array = []; + let notice: OrchestrationV2TurnItem | undefined; + if (noticed.length === 0) { + const event = yield* noticeEvent(current, afterDays); + // Placed here rather than by EventSink, so forks get the position the source shows. + notice = yield* positions.normalize(event.payload, current.runOrdinal); + events.push({ ...event, payload: notice }); + } + const payload = { + sourceThreadId: threadId, + turnItemIds: itemIds.flatMap((row) => (items.has(row.id) ? [row.id] : [])), + nodeIds: nodeIds.flatMap((row) => (nodes.has(row.id) ? [row.id] : [])), + ...(notice === undefined ? {} : { notice }), + }; + const occurredAt = yield* DateTime.now; + // Each fork gets its own copy, so its revision moves with the rows it inherits. + for (const target of [threadId, ...(yield* readForks(threadId)).map((row) => row.id)]) { + events.push({ + id: yield* ids.allocate.event({ threadId: target }), + type: "thread.history-trimmed", + threadId: target, + occurredAt, + payload, + }); + } return events; }), ); @@ -560,4 +671,6 @@ const make = Effect.gen(function* () { }); }); -export const layer = Layer.effect(ThreadHistoryTrim, make); +export const layer = Layer.effect(ThreadHistoryTrim, make).pipe( + Layer.provide(TurnItemPositionStore.layer), +); diff --git a/packages/client-runtime/src/state/orchestrationV2Projection.test.ts b/packages/client-runtime/src/state/orchestrationV2Projection.test.ts index f9acf3becf81e..45e4a0fdb4b31 100644 --- a/packages/client-runtime/src/state/orchestrationV2Projection.test.ts +++ b/packages/client-runtime/src/state/orchestrationV2Projection.test.ts @@ -624,10 +624,43 @@ describe("applyOrchestrationV2ProjectionEvent", () => { type: "thread.history-trimmed", threadId, occurredAt: now, - payload: { turnItemIds: [local.id, inherited.id], nodeIds: [] }, + payload: { sourceThreadId: threadId, turnItemIds: [local.id, inherited.id], nodeIds: [] }, } as unknown as OrchestrationV2DomainEvent); expect(trimmed?.turnItems).toEqual([]); expect(trimmed?.visibleTurnItems).toEqual([inheritedRow]); + // A fork hears its source's trim on its own stream and drops only the inherited rows. + const sourceTrimmed = applyOrchestrationV2ProjectionEvent(projection, { + id: "event-source-trimmed", + type: "thread.history-trimmed", + threadId, + occurredAt: now, + payload: { + sourceThreadId: inheritedRow.sourceThreadId, + turnItemIds: [inherited.id, local.id], + nodeIds: [], + }, + } as unknown as OrchestrationV2DomainEvent); + expect(sourceTrimmed?.turnItems).toEqual([local]); + expect(sourceTrimmed?.visibleTurnItems).toEqual([{ ...localRow, position: 0 }]); + // The source's notice lands after the last row the fork inherits from its run. + const notice = commandItem("item-notice"); + const noticed = applyOrchestrationV2ProjectionEvent(projection, { + id: "event-source-noticed", + type: "thread.history-trimmed", + threadId, + occurredAt: now, + payload: { + sourceThreadId: inheritedRow.sourceThreadId, + turnItemIds: [], + nodeIds: [], + notice, + }, + } as unknown as OrchestrationV2DomainEvent); + expect(noticed?.visibleTurnItems.map((row) => [row.sourceItemId, row.position])).toEqual([ + [inherited.id, 0], + [notice.id, 1], + [local.id, 2], + ]); }); }); diff --git a/packages/client-runtime/src/state/orchestrationV2Projection.ts b/packages/client-runtime/src/state/orchestrationV2Projection.ts index 17870ec54850d..16901c93d29f4 100644 --- a/packages/client-runtime/src/state/orchestrationV2Projection.ts +++ b/packages/client-runtime/src/state/orchestrationV2Projection.ts @@ -84,6 +84,27 @@ function removeVisibleItem( return renumberVisibleItems([...rows.slice(0, index), ...rows.slice(index + 1)]); } +/** A fork shows its source's trim notice after the last row it inherits from the notice's run. */ +function withInheritedNotice( + rows: OrchestrationV2ThreadProjection["visibleTurnItems"], + notice: OrchestrationV2TurnItem | undefined, + sourceThreadId: OrchestrationV2TurnItem["threadId"], +): OrchestrationV2ThreadProjection["visibleTurnItems"] { + if (notice === undefined || rows.some((row) => row.sourceItemId === notice.id)) return rows; + const after = rows.findLastIndex( + (row) => row.sourceThreadId === sourceThreadId && row.item.runId === notice.runId, + ); + if (after === -1) return rows; + const row = { + position: 0, + visibility: "inherited" as const, + sourceThreadId, + sourceItemId: notice.id, + item: notice, + }; + return [...rows.slice(0, after + 1), row, ...rows.slice(after + 1)]; +} + function renumberVisibleItems( rows: OrchestrationV2ThreadProjection["visibleTurnItems"], ): OrchestrationV2ThreadProjection["visibleTurnItems"] { @@ -331,15 +352,24 @@ export function applyOrchestrationV2ProjectionEvent( thread: applyOrchestrationV2ThreadVisit(projection.thread, event.payload), }; case "thread.history-trimmed": { + const { sourceThreadId } = event.payload; const items = new Set(event.payload.turnItemIds); - const nodes = new Set(event.payload.nodeIds); + const nodes = new Set( + sourceThreadId === projection.thread.id ? event.payload.nodeIds : [], + ); return { ...projection, - turnItems: projection.turnItems.filter((item) => !items.has(item.id)), + turnItems: projection.turnItems.filter( + (item) => item.threadId !== sourceThreadId || !items.has(item.id), + ), nodes: projection.nodes.filter((node) => !nodes.has(node.id)), visibleTurnItems: renumberVisibleItems( - projection.visibleTurnItems.filter( - (row) => row.sourceThreadId !== projection.thread.id || !items.has(row.sourceItemId), + withInheritedNotice( + projection.visibleTurnItems.filter( + (row) => row.sourceThreadId !== sourceThreadId || !items.has(row.sourceItemId), + ), + sourceThreadId === projection.thread.id ? undefined : event.payload.notice, + sourceThreadId, ), ), }; diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index 42ac289b96e02..bf7300e0a0de8 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -1584,11 +1584,12 @@ export const OrchestrationV2CheckpointRollbackRequest = Schema.Struct({ export type OrchestrationV2CheckpointRollbackRequest = typeof OrchestrationV2CheckpointRollbackRequest.Type; -/** Rows a history trim removed from the thread's projection. */ -const OrchestrationV2ThreadHistoryTrimmed = Schema.Struct({ +/** Rows a history trim removed from `sourceThreadId`; its forks hear it on their own streams. */ +const OrchestrationV2ThreadHistoryTrimmedFields = { + sourceThreadId: ThreadId, turnItemIds: Schema.Array(TurnItemId), nodeIds: Schema.Array(NodeId), -}); +}; export class OrchestrationV2CheckpointUnavailableError extends Schema.TaggedError()( "OrchestrationV2CheckpointUnavailableError", @@ -2191,7 +2192,11 @@ export const OrchestrationV2DomainEvent = Schema.Union([ Schema.Struct({ ...OrchestrationV2EventBase.fields, type: Schema.Literal("thread.history-trimmed"), - payload: OrchestrationV2ThreadHistoryTrimmed, + payload: Schema.Struct({ + ...OrchestrationV2ThreadHistoryTrimmedFields, + /** The notice the source gained, for forks that show the run it is in. */ + notice: Schema.optional(OrchestrationV2TurnItem), + }), }), Schema.Struct({ ...OrchestrationV2EventBase.fields, @@ -3144,7 +3149,10 @@ export const OrchestrationV2DomainEventJson = Schema.Union([ Schema.Struct({ ...OrchestrationV2JsonEventBaseFields, type: Schema.Literal("thread.history-trimmed"), - payload: OrchestrationV2ThreadHistoryTrimmed, + payload: Schema.Struct({ + ...OrchestrationV2ThreadHistoryTrimmedFields, + notice: Schema.optional(OrchestrationV2TurnItemJson), + }), }), Schema.Struct({ ...OrchestrationV2JsonEventBaseFields, From 2b5b815f1816201110cde0aba963a54c1c11b56a Mon Sep 17 00:00:00 2001 From: JJ Eaton <58495321+jayleaton@users.noreply.github.com> Date: Wed, 18 Nov 2026 06:09:05 +0700 Subject: [PATCH 4/5] fix(server): trim reaches forks past deleted ones with compact invalidations Co-Authored-By: Claude Opus 5.5 --- .../src/orchestration-v2/ProjectionStore.ts | 39 ++--------- .../ThreadHistoryTrim.test.ts | 64 +++++++++++------ .../src/orchestration-v2/ThreadHistoryTrim.ts | 56 +++++++-------- .../src/orchestration-v2/ThreadStream.ts | 10 +++ apps/server/src/ws.test.ts | 69 +++++++++++++++++++ apps/server/src/ws.ts | 28 +++++++- .../state/orchestrationV2Projection.test.ts | 32 ++------- .../src/state/orchestrationV2Projection.ts | 40 ++--------- packages/contracts/src/orchestrationV2.ts | 20 +++--- 9 files changed, 197 insertions(+), 161 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 65eb9703174be..ae3964efe18cf 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -1046,27 +1046,6 @@ export function emptyProjection( }; } -/** A fork shows its source's trim notice after the last row it inherits from the notice's run. */ -function withInheritedNotice( - rows: OrchestrationV2ThreadProjection["visibleTurnItems"], - notice: OrchestrationV2TurnItem | undefined, - sourceThreadId: ThreadId, -): OrchestrationV2ThreadProjection["visibleTurnItems"] { - if (notice === undefined || rows.some((row) => row.sourceItemId === notice.id)) return rows; - const after = rows.findLastIndex( - (row) => row.sourceThreadId === sourceThreadId && row.item.runId === notice.runId, - ); - if (after === -1) return rows; - const row = { - position: 0, - visibility: "inherited" as const, - sourceThreadId, - sourceItemId: notice.id, - item: notice, - }; - return [...rows.slice(0, after + 1), row, ...rows.slice(after + 1)]; -} - export function applyToProjection( projection: OrchestrationV2ThreadProjection, event: OrchestrationV2DomainEvent, @@ -1124,22 +1103,16 @@ export function applyToProjection( thread: event.payload, }; case "thread.history-trimmed": { - const { sourceThreadId } = event.payload; + if (event.payload.sourceThreadId !== event.threadId) return projection; const items = new Set(event.payload.turnItemIds); - const nodes = new Set(sourceThreadId === event.threadId ? event.payload.nodeIds : []); + const nodes = new Set(event.payload.nodeIds); return { ...projection, - turnItems: projection.turnItems.filter( - (item) => item.threadId !== sourceThreadId || !items.has(item.id), - ), + turnItems: projection.turnItems.filter((item) => !items.has(item.id)), nodes: projection.nodes.filter((node) => !nodes.has(node.id)), - visibleTurnItems: withInheritedNotice( - projection.visibleTurnItems.filter( - (row) => row.sourceThreadId !== sourceThreadId || !items.has(row.sourceItemId), - ), - sourceThreadId === event.threadId ? undefined : event.payload.notice, - sourceThreadId, - ).map((row, position) => (row.position === position ? row : { ...row, position })), + visibleTurnItems: projection.visibleTurnItems + .filter((row) => row.sourceThreadId !== event.threadId || !items.has(row.sourceItemId)) + .map((row, position) => (row.position === position ? row : { ...row, position })), }; } case "thread.visit-recorded": diff --git a/apps/server/src/orchestration-v2/ThreadHistoryTrim.test.ts b/apps/server/src/orchestration-v2/ThreadHistoryTrim.test.ts index 6950867b75abf..d50fed591d354 100644 --- a/apps/server/src/orchestration-v2/ThreadHistoryTrim.test.ts +++ b/apps/server/src/orchestration-v2/ThreadHistoryTrim.test.ts @@ -34,7 +34,7 @@ import * as EventStore from "./EventStore.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; import * as ProjectStore from "./ProjectStore.ts"; import * as ThreadHistoryTrim from "./ThreadHistoryTrim.ts"; -import { buildBoundedThreadStreamSnapshot } from "./ThreadStream.ts"; +import { buildBoundedThreadStreamSnapshot, invalidatesInheritedRows } from "./ThreadStream.ts"; const database = SqlitePersistence.layerMemory; const stores = Layer.mergeAll( @@ -685,7 +685,8 @@ const midPassReader = (threadId: ThreadId) => { yield* store.getThreadSnapshotWindow(threadId, { rowLimit: 50 }); yield* store.searchThread({ threadId, query: "flaky" }); }); - const verify = (removed: Array) => + /** A fork's replay holds compact copies, which subscribers answer with a fresh snapshot. */ + const verify = (removed: Array, fork = false) => Effect.gen(function* () { const store = yield* ProjectionStore.ProjectionStoreV2; const before = warmed!; @@ -698,11 +699,19 @@ const midPassReader = (threadId: ThreadId) => { const replay = yield* Stream.runCollect( (yield* EventStore.EventStoreV2).read({ threadId, afterSequence: before.snapshotSequence }), ); - const resumed = replay.reduce( - (projection, stored) => ProjectionStore.applyToProjection(projection, stored.event), - before.projection, - ); - assert.deepEqual(ids(resumed), ids(after.projection)); + if (fork) { + const copies = replay.filter((stored) => invalidatesInheritedRows(stored.event, threadId)); + assert.isNotEmpty(copies); + for (const copy of copies) { + assert.isAtMost(JSON.stringify(copy.event.payload).length, 128); + } + } else { + const resumed = replay.reduce( + (projection, stored) => ProjectionStore.applyToProjection(projection, stored.event), + before.projection, + ); + assert.deepEqual(ids(resumed), ids(after.projection)); + } const window = yield* store.getThreadSnapshotWindow(threadId, { rowLimit: 50 }); assert.equal(window.snapshotSequence, after.snapshotSequence); @@ -807,25 +816,38 @@ it.layer(hookedFixture)("ThreadHistoryTrim during a pass", (it) => { }), ); - it.effect("advances an active fork's revision when the rows it inherits are trimmed", () => + it.effect("advances the revision of every live fork, through deleted forks, compactly", () => Effect.gen(function* () { yield* isolate; const source = yield* seedThread("fork-source"); - const fork = yield* seedThread("fork", { - userMessageAt: recent, - thread: { - lineage: { - parentThreadId: source.threadId, - relationshipToParent: "fork", - rootThreadId: source.threadId, + const forkOf = (name: string, parent: { threadId: ThreadId; runId: RunId }) => + seedThread(name, { + userMessageAt: recent, + thread: { + lineage: { + parentThreadId: parent.threadId, + relationshipToParent: "fork", + rootThreadId: source.threadId, + }, + forkedFrom: { type: "run", threadId: parent.threadId, runId: parent.runId }, }, - forkedFrom: { type: "run", threadId: source.threadId, runId: source.runId }, - }, - }); - const reader = midPassReader(fork.threadId); - yield* arm(reader.read); + }); + const fork = yield* forkOf("fork", source); + const deleted = yield* forkOf("deleted-fork", source); + const grandchild = yield* forkOf("fork-of-deleted", deleted); + yield* updateThread("deleted-fork", "thread.deleted", { deletedAt: now }); + const deletedEvents = Stream.runCollect( + (yield* EventStore.EventStoreV2).read({ threadId: deleted.threadId }), + ); + const deletedBefore = (yield* deletedEvents).length; + + const readers = [midPassReader(grandchild.threadId), midPassReader(fork.threadId)]; + yield* arm(Effect.forEach(readers, (reader) => reader.read, { discard: true })); yield* trim; - yield* reader.verify(["item:fork-source:reasoning", "item:fork-source:command"]); + for (const reader of readers) { + yield* reader.verify(["item:fork-source:reasoning", "item:fork-source:command"], true); + } + assert.lengthOf(yield* deletedEvents, deletedBefore); assert.deepEqual(yield* itemTitles(fork.threadId), everyItem); }), ); diff --git a/apps/server/src/orchestration-v2/ThreadHistoryTrim.ts b/apps/server/src/orchestration-v2/ThreadHistoryTrim.ts index 91a89f61b868a..a1539c915939c 100644 --- a/apps/server/src/orchestration-v2/ThreadHistoryTrim.ts +++ b/apps/server/src/orchestration-v2/ThreadHistoryTrim.ts @@ -1,7 +1,6 @@ import { type NodeId, type OrchestrationV2DomainEvent, - type OrchestrationV2TurnItem, type RunId, type ThreadHistoryTrimReport, type ThreadHistoryTrimStatus, @@ -27,7 +26,6 @@ import * as ServerSettingsModule from "../serverSettings.ts"; import { forkParked } from "../serverActivation.ts"; import * as EventSink from "./EventSink.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; -import * as TurnItemPositionStore from "./TurnItemPositionStore.ts"; export class ThreadHistoryTrimError extends Schema.TaggedError()( "ThreadHistoryTrimError", @@ -149,7 +147,6 @@ const make = Effect.gen(function* () { const eventSink = yield* EventSink.EventSinkV2; const ids = yield* IdAllocator.IdAllocatorV2; const settingsService = yield* ServerSettingsModule.ServerSettingsService; - const positions = yield* TurnItemPositionStore.TurnItemPositionStoreV2; // A subagent has no user of its own, so it ages with the thread that started it. const readCandidates = (input: { @@ -248,18 +245,18 @@ const make = Effect.gen(function* () { WHERE thread_id = ${threadId} AND owner_node_id IS NOT NULL `; - // Forks that show this thread's rows, through any depth of forking; side chats inherit none. + // Live forks that show this thread's rows, at any depth; a deleted fork still passes them on. const readForks = (threadId: ThreadId) => sql<{ readonly id: ThreadId }>` - WITH RECURSIVE forks(thread_id) AS ( - SELECT ${threadId} + WITH RECURSIVE forks(thread_id, deleted) AS ( + SELECT ${threadId}, 0 UNION - SELECT fork.thread_id FROM orchestration_v2_projection_threads AS fork + SELECT fork.thread_id, fork.deleted_at IS NOT NULL + FROM orchestration_v2_projection_threads AS fork JOIN forks ON json_extract(fork.payload_json, '$.forkedFrom.threadId') = forks.thread_id WHERE json_extract(fork.payload_json, '$.forkedFrom.type') = 'run' AND COALESCE(json_extract(fork.payload_json, '$.lineage.sideChat'), 0) = 0 - AND fork.deleted_at IS NULL ) - SELECT thread_id AS id FROM forks WHERE thread_id <> ${threadId} + SELECT thread_id AS id FROM forks WHERE thread_id <> ${threadId} AND deleted = 0 `; const select = Effect.fn("ThreadHistoryTrim.select")(function* (candidate: Candidate) { @@ -489,29 +486,28 @@ const make = Effect.gen(function* () { const noticed = yield* sql` SELECT 1 FROM orchestration_v2_projection_turn_items WHERE turn_item_id = ${noticeId} `; - const events: Array = []; - let notice: OrchestrationV2TurnItem | undefined; - if (noticed.length === 0) { - const event = yield* noticeEvent(current, afterDays); - // Placed here rather than by EventSink, so forks get the position the source shows. - notice = yield* positions.normalize(event.payload, current.runOrdinal); - events.push({ ...event, payload: notice }); - } - const payload = { - sourceThreadId: threadId, - turnItemIds: itemIds.flatMap((row) => (items.has(row.id) ? [row.id] : [])), - nodeIds: nodeIds.flatMap((row) => (nodes.has(row.id) ? [row.id] : [])), - ...(notice === undefined ? {} : { notice }), - }; + const events: Array = + noticed.length === 0 ? [yield* noticeEvent(current, afterDays)] : []; const occurredAt = yield* DateTime.now; - // Each fork gets its own copy, so its revision moves with the rows it inherits. - for (const target of [threadId, ...(yield* readForks(threadId)).map((row) => row.id)]) { + events.push({ + id: yield* ids.allocate.event({ threadId }), + type: "thread.history-trimmed", + threadId, + occurredAt, + payload: { + sourceThreadId: threadId, + turnItemIds: itemIds.flatMap((row) => (items.has(row.id) ? [row.id] : [])), + nodeIds: nodeIds.flatMap((row) => (nodes.has(row.id) ? [row.id] : [])), + }, + }); + // Each fork's revision moves with the rows it inherits; its copy stays a few bytes. + for (const { id: fork } of yield* readForks(threadId)) { events.push({ - id: yield* ids.allocate.event({ threadId: target }), + id: yield* ids.allocate.event({ threadId: fork }), type: "thread.history-trimmed", - threadId: target, + threadId: fork, occurredAt, - payload, + payload: { sourceThreadId: threadId, turnItemIds: [], nodeIds: [] }, }); } return events; @@ -671,6 +667,4 @@ const make = Effect.gen(function* () { }); }); -export const layer = Layer.effect(ThreadHistoryTrim, make).pipe( - Layer.provide(TurnItemPositionStore.layer), -); +export const layer = Layer.effect(ThreadHistoryTrim, make); diff --git a/apps/server/src/orchestration-v2/ThreadStream.ts b/apps/server/src/orchestration-v2/ThreadStream.ts index c16a1b9e095aa..a04e9db317974 100644 --- a/apps/server/src/orchestration-v2/ThreadStream.ts +++ b/apps/server/src/orchestration-v2/ThreadStream.ts @@ -1,5 +1,7 @@ import type { + OrchestrationV2DomainEvent, OrchestrationV2ThreadProjection, + ThreadId, OrchestrationV2ThreadStreamItem, } from "@t3tools/contracts"; @@ -116,3 +118,11 @@ export function decideThreadResume(input: { throughSequence: input.highWater, }; } + +/** A fork's copy of its source's history trim, which names no rows: read a fresh snapshot. */ +export function invalidatesInheritedRows( + event: OrchestrationV2DomainEvent, + threadId: ThreadId, +): boolean { + return event.type === "thread.history-trimmed" && event.payload.sourceThreadId !== threadId; +} diff --git a/apps/server/src/ws.test.ts b/apps/server/src/ws.test.ts index 1c37de19c34a9..fc7b5d7a7a98f 100644 --- a/apps/server/src/ws.test.ts +++ b/apps/server/src/ws.test.ts @@ -35,6 +35,7 @@ import { type OrchestrationV2AppThread, type OrchestrationV2DomainEvent, type OrchestrationV2StoredEvent, + type OrchestrationV2ThreadProjection, type OrchestrationV2ThreadShell, type OrchestrationProjectShell, type ServerConfig, @@ -484,6 +485,74 @@ it.effect("coalesces streaming replay before synchronization and the live tail", }).pipe(Effect.scoped, Effect.provide(ReplayTestLayer)), ); +it.effect("a fork reads a fresh snapshot when its source's history is trimmed", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const store = yield* OrchestrationEventStore.OrchestrationEventStore; + const threadId = ThreadId.make("thread:trimmed-fork"); + const now = "2026-11-17T00:00:00.000Z"; + const insert = (version: number, type: string, payload: object) => + sql<{ sequence: number }>` + INSERT INTO orchestration_events ( + event_id, aggregate_kind, stream_id, stream_version, event_type, + occurred_at, actor_kind, payload_json, metadata_json, application_event_version + ) VALUES ( + ${`trimmed-fork:${version}`}, 'thread', ${threadId}, ${version}, ${type}, + ${now}, 'server', ${JSON.stringify(payload)}, '{}', 2 + ) RETURNING sequence + `.pipe(Effect.map(([row]) => row!.sequence)); + const forkCopy = { sourceThreadId: "thread:trimmed-source", turnItemIds: [], nodeIds: [] }; + const seen = yield* insert(0, "provider-session.detached", { + providerSessionId: "session:trimmed-fork", + detachedAt: now, + }); + const replayed = yield* insert(1, "thread.history-trimmed", forkCopy); + let snapshotSequence = replayed; + const snapshots: Array = []; + const items = yield* subscribeOrchestrationV2Thread({ + threadId, + afterSequence: seen, + requestCompletionMarker: true, + }).pipe( + Effect.flatMap(Stream.runCollect), + Effect.provideService(ThreadManagement.ThreadManagementService, { + ensureLegacyTranscript: () => Effect.void, + getThreadSnapshot: () => + Effect.sync(() => { + snapshots.push(snapshotSequence); + return { + schemaVersion: 1, + snapshotSequence, + projection: { + checkpoints: [], + contextHandoffs: [], + turnItems: [], + visibleTurnItems: [], + } as unknown as OrchestrationV2ThreadProjection, + }; + }), + streamStoredEventsFrom: () => + Stream.unwrap( + insert(2, "thread.history-trimmed", forkCopy).pipe( + Effect.map((live) => { + snapshotSequence = live; + return store.readAgentEvents({ threadId, afterSequence: replayed }); + }), + ), + ), + } as unknown as ThreadManagement.ThreadManagementService["Service"]), + ); + // Replay and the live tail each answer the fork's copy with what the fork shows now. + assert.deepEqual( + items.map((item) => + item.kind === "snapshot" ? [item.kind, item.snapshotSequence] : [item.kind], + ), + [["snapshot", replayed], ["synchronized"], ["snapshot", replayed + 1]], + ); + assert.lengthOf(snapshots, 2); + }).pipe(Effect.scoped, Effect.provide(ReplayTestLayer)), +); + it.effect("shell groups read distinct threads once and keep each event's own sequence", () => Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 186c6b1040733..2996271294c70 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -153,6 +153,7 @@ import { import { buildBoundedThreadStreamSnapshot, decideThreadResume, + invalidatesInheritedRows, isThreadReplayRawPayloadSafe, threadReplayEncodedBytes, THREAD_RESUME_MAX_REPLAY_EVENTS, @@ -854,6 +855,18 @@ export const subscribeOrchestrationV2Thread = Effect.fn("ws.orchestrationV2.subs .pipe( Stream.map(threadStreamEventItem), coalesceThreadLiveStream, + // Rows this fork inherits were trimmed at their source: send what it shows now. + Stream.mapEffect( + ( + item, + ): Effect.Effect< + OrchestrationV2ThreadStreamItem, + OrchestrationV2GetThreadProjectionError + > => + item.kind === "event" && invalidatesInheritedRows(item.event, input.threadId) + ? loadSnapshotItem() + : Effect.succeed(item), + ), Stream.mapError( (cause) => new OrchestrationV2GetThreadProjectionError({ @@ -897,7 +910,7 @@ export const subscribeOrchestrationV2Thread = Effect.fn("ws.orchestrationV2.subs ? Stream.make({ kind: "synchronized" as const }) : Stream.empty; - const snapshotThenLive = Effect.fn("ws.orchestrationV2.threadSnapshotThenLive")(function* () { + const loadSnapshotItem = Effect.fn("ws.orchestrationV2.loadThreadSnapshotItem")(function* () { const useBoundedSnapshot = shouldUseBoundedThreadSnapshot(input); const snapshot = yield* ( useBoundedSnapshot @@ -933,9 +946,13 @@ export const subscribeOrchestrationV2Thread = Effect.fn("ws.orchestrationV2.subs projection: dedupeBodies ? dedupeSnapshotBodies(projection) : projection, }; })(); + return snapshotItem; + }); + const snapshotThenLive = Effect.fn("ws.orchestrationV2.threadSnapshotThenLive")(function* () { + const snapshotItem = yield* loadSnapshotItem(); return rpcInitialItems([snapshotItem]).pipe( Stream.concat(completionMarker), - Stream.concat(eventStreamFrom(snapshotSequence)), + Stream.concat(eventStreamFrom(snapshotItem.snapshotSequence)), withLiveDedupe, ); }); @@ -1026,6 +1043,13 @@ export const subscribeOrchestrationV2Thread = Effect.fn("ws.orchestrationV2.subs return yield* snapshotThenLive(); } const replay = coalesceLiveUpdatedEvents(replayed); + if ( + replay.some( + (item) => item.kind === "event" && invalidatesInheritedRows(item.event, input.threadId), + ) + ) { + return yield* snapshotThenLive(); + } const plan = decideThreadResume({ afterSequence: input.afterSequence, highWater, diff --git a/packages/client-runtime/src/state/orchestrationV2Projection.test.ts b/packages/client-runtime/src/state/orchestrationV2Projection.test.ts index 45e4a0fdb4b31..76e48253e35b5 100644 --- a/packages/client-runtime/src/state/orchestrationV2Projection.test.ts +++ b/packages/client-runtime/src/state/orchestrationV2Projection.test.ts @@ -628,39 +628,15 @@ describe("applyOrchestrationV2ProjectionEvent", () => { } as unknown as OrchestrationV2DomainEvent); expect(trimmed?.turnItems).toEqual([]); expect(trimmed?.visibleTurnItems).toEqual([inheritedRow]); - // A fork hears its source's trim on its own stream and drops only the inherited rows. - const sourceTrimmed = applyOrchestrationV2ProjectionEvent(projection, { + // A fork's copy names no rows; the server sends a fresh snapshot in its place. + const forkCopy = applyOrchestrationV2ProjectionEvent(projection, { id: "event-source-trimmed", type: "thread.history-trimmed", threadId, occurredAt: now, - payload: { - sourceThreadId: inheritedRow.sourceThreadId, - turnItemIds: [inherited.id, local.id], - nodeIds: [], - }, + payload: { sourceThreadId: inheritedRow.sourceThreadId, turnItemIds: [], nodeIds: [] }, } as unknown as OrchestrationV2DomainEvent); - expect(sourceTrimmed?.turnItems).toEqual([local]); - expect(sourceTrimmed?.visibleTurnItems).toEqual([{ ...localRow, position: 0 }]); - // The source's notice lands after the last row the fork inherits from its run. - const notice = commandItem("item-notice"); - const noticed = applyOrchestrationV2ProjectionEvent(projection, { - id: "event-source-noticed", - type: "thread.history-trimmed", - threadId, - occurredAt: now, - payload: { - sourceThreadId: inheritedRow.sourceThreadId, - turnItemIds: [], - nodeIds: [], - notice, - }, - } as unknown as OrchestrationV2DomainEvent); - expect(noticed?.visibleTurnItems.map((row) => [row.sourceItemId, row.position])).toEqual([ - [inherited.id, 0], - [notice.id, 1], - [local.id, 2], - ]); + expect(forkCopy).toBe(projection); }); }); diff --git a/packages/client-runtime/src/state/orchestrationV2Projection.ts b/packages/client-runtime/src/state/orchestrationV2Projection.ts index 16901c93d29f4..7e0bcaebd5d39 100644 --- a/packages/client-runtime/src/state/orchestrationV2Projection.ts +++ b/packages/client-runtime/src/state/orchestrationV2Projection.ts @@ -84,27 +84,6 @@ function removeVisibleItem( return renumberVisibleItems([...rows.slice(0, index), ...rows.slice(index + 1)]); } -/** A fork shows its source's trim notice after the last row it inherits from the notice's run. */ -function withInheritedNotice( - rows: OrchestrationV2ThreadProjection["visibleTurnItems"], - notice: OrchestrationV2TurnItem | undefined, - sourceThreadId: OrchestrationV2TurnItem["threadId"], -): OrchestrationV2ThreadProjection["visibleTurnItems"] { - if (notice === undefined || rows.some((row) => row.sourceItemId === notice.id)) return rows; - const after = rows.findLastIndex( - (row) => row.sourceThreadId === sourceThreadId && row.item.runId === notice.runId, - ); - if (after === -1) return rows; - const row = { - position: 0, - visibility: "inherited" as const, - sourceThreadId, - sourceItemId: notice.id, - item: notice, - }; - return [...rows.slice(0, after + 1), row, ...rows.slice(after + 1)]; -} - function renumberVisibleItems( rows: OrchestrationV2ThreadProjection["visibleTurnItems"], ): OrchestrationV2ThreadProjection["visibleTurnItems"] { @@ -352,24 +331,17 @@ export function applyOrchestrationV2ProjectionEvent( thread: applyOrchestrationV2ThreadVisit(projection.thread, event.payload), }; case "thread.history-trimmed": { - const { sourceThreadId } = event.payload; + // A fork's copy names no rows: the server answers it with a fresh snapshot. + if (event.payload.sourceThreadId !== projection.thread.id) return projection; const items = new Set(event.payload.turnItemIds); - const nodes = new Set( - sourceThreadId === projection.thread.id ? event.payload.nodeIds : [], - ); + const nodes = new Set(event.payload.nodeIds); return { ...projection, - turnItems: projection.turnItems.filter( - (item) => item.threadId !== sourceThreadId || !items.has(item.id), - ), + turnItems: projection.turnItems.filter((item) => !items.has(item.id)), nodes: projection.nodes.filter((node) => !nodes.has(node.id)), visibleTurnItems: renumberVisibleItems( - withInheritedNotice( - projection.visibleTurnItems.filter( - (row) => row.sourceThreadId !== sourceThreadId || !items.has(row.sourceItemId), - ), - sourceThreadId === projection.thread.id ? undefined : event.payload.notice, - sourceThreadId, + projection.visibleTurnItems.filter( + (row) => row.sourceThreadId !== projection.thread.id || !items.has(row.sourceItemId), ), ), }; diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index bf7300e0a0de8..946d1ec9e9588 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -1584,12 +1584,15 @@ export const OrchestrationV2CheckpointRollbackRequest = Schema.Struct({ export type OrchestrationV2CheckpointRollbackRequest = typeof OrchestrationV2CheckpointRollbackRequest.Type; -/** Rows a history trim removed from `sourceThreadId`; its forks hear it on their own streams. */ -const OrchestrationV2ThreadHistoryTrimmedFields = { +/** + * Rows a history trim removed from `sourceThreadId`. A fork that shows them gets a copy on its own + * stream that names no rows; subscribers read a fresh snapshot in its place. + */ +const OrchestrationV2ThreadHistoryTrimmed = Schema.Struct({ sourceThreadId: ThreadId, turnItemIds: Schema.Array(TurnItemId), nodeIds: Schema.Array(NodeId), -}; +}); export class OrchestrationV2CheckpointUnavailableError extends Schema.TaggedError()( "OrchestrationV2CheckpointUnavailableError", @@ -2192,11 +2195,7 @@ export const OrchestrationV2DomainEvent = Schema.Union([ Schema.Struct({ ...OrchestrationV2EventBase.fields, type: Schema.Literal("thread.history-trimmed"), - payload: Schema.Struct({ - ...OrchestrationV2ThreadHistoryTrimmedFields, - /** The notice the source gained, for forks that show the run it is in. */ - notice: Schema.optional(OrchestrationV2TurnItem), - }), + payload: OrchestrationV2ThreadHistoryTrimmed, }), Schema.Struct({ ...OrchestrationV2EventBase.fields, @@ -3149,10 +3148,7 @@ export const OrchestrationV2DomainEventJson = Schema.Union([ Schema.Struct({ ...OrchestrationV2JsonEventBaseFields, type: Schema.Literal("thread.history-trimmed"), - payload: Schema.Struct({ - ...OrchestrationV2ThreadHistoryTrimmedFields, - notice: Schema.optional(OrchestrationV2TurnItemJson), - }), + payload: OrchestrationV2ThreadHistoryTrimmed, }), Schema.Struct({ ...OrchestrationV2JsonEventBaseFields, From b4868b61ff3fd2a6c1b679f28654ec7a41925cb7 Mon Sep 17 00:00:00 2001 From: JJ Eaton <58495321+jayleaton@users.noreply.github.com> Date: Wed, 18 Nov 2026 06:21:00 +0700 Subject: [PATCH 5/5] fix(server): keep thread live batches whole while checking for fork invalidations Co-Authored-By: Claude Opus 5.5 --- apps/server/src/ws.ts | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 2996271294c70..2e3acdc68bef6 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -16,6 +16,7 @@ import * as PushRegistrations from "./push/PushRegistrations.ts"; import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; import * as Base64 from "effect/encoding/Base64"; +import * as Arr from "effect/Array"; import * as Effect from "effect/Effect"; import * as NodeHttpServerRequest from "@effect/platform-node/NodeHttpServerRequest"; import * as Fiber from "effect/Fiber"; @@ -856,16 +857,29 @@ export const subscribeOrchestrationV2Thread = Effect.fn("ws.orchestrationV2.subs Stream.map(threadStreamEventItem), coalesceThreadLiveStream, // Rows this fork inherits were trimmed at their source: send what it shows now. - Stream.mapEffect( + // Per array, so an ordinary batch passes through as one message. + Stream.mapArrayEffect( ( - item, + items, ): Effect.Effect< - OrchestrationV2ThreadStreamItem, + Arr.NonEmptyReadonlyArray, OrchestrationV2GetThreadProjectionError - > => - item.kind === "event" && invalidatesInheritedRows(item.event, input.threadId) - ? loadSnapshotItem() - : Effect.succeed(item), + > => { + const invalidates = (item: OrchestrationV2ThreadStreamItem) => + item.kind === "event" && invalidatesInheritedRows(item.event, input.threadId); + if (!items.some(invalidates)) return Effect.succeed(items); + return Effect.forEach( + items, + ( + item, + ): Effect.Effect< + OrchestrationV2ThreadStreamItem, + OrchestrationV2GetThreadProjectionError + > => (invalidates(item) ? loadSnapshotItem() : Effect.succeed(item)), + ).pipe( + Effect.map((mapped) => (Arr.isReadonlyArrayNonEmpty(mapped) ? mapped : items)), + ); + }, ), Stream.mapError( (cause) =>