diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index fff4fd3e3db76..0b655ff30697b 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -107,6 +107,10 @@ export interface EventSinkV2Shape { readonly commandId?: CommandId; readonly events: ReadonlyArray; }) => Effect.Effect, EventSinkV2Error>; + /** Builds the events inside the commit transaction, so they reflect what is stored; none commits nothing. */ + readonly writeBuilt: ( + build: Effect.Effect, E>, + ) => Effect.Effect, EventSinkV2Error>; readonly writeWithEffects: (input: { /** * Rebuilds each run.updated from the stored row with only its delegatedCompletion replaced. @@ -520,6 +524,21 @@ const layerBase: Layer.Layer< return committed; }); + const writeBuiltEffect = Effect.fn("orchestrationV2.EventSink.writeBuilt")(function* ( + build: Effect.Effect, E>, + ) { + const { committed } = yield* commitThenPublish( + Effect.gen(function* () { + const events = yield* build; + if (events.length === 0) return { committed: [], outcomes: [] }; + const committed = yield* eventStore.append({ events: yield* normalizeEvents(events) }); + return { committed, outcomes: yield* applyStoredEvents(committed) }; + }), + ({ committed, outcomes }) => publishStoredEvents(committed, outcomes), + ); + return committed; + }); + const writeIfRunCurrentEffect = Effect.fn("orchestrationV2.EventSink.writeIfRunCurrent")( function* (input: Parameters[0]) { yield* Effect.annotateCurrentSpan({ @@ -1137,6 +1156,10 @@ const layerBase: Layer.Layer< }), ), ), + writeBuilt: (build) => + writeBuiltEffect(build).pipe( + Effect.mapError((cause) => new EventSinkWriteError({ eventCount: 0, cause })), + ), writeWithEffects: (input) => writeEffect(input).pipe( Effect.mapError( diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 404e4c1b0f948..ae3964efe18cf 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -1102,6 +1102,19 @@ export function applyToProjection( ...projection, thread: event.payload, }; + case "thread.history-trimmed": { + if (event.payload.sourceThreadId !== event.threadId) return projection; + const items = new Set(event.payload.turnItemIds); + const nodes = new Set(event.payload.nodeIds); + return { + ...projection, + turnItems: projection.turnItems.filter((item) => !items.has(item.id)), + nodes: projection.nodes.filter((node) => !nodes.has(node.id)), + visibleTurnItems: projection.visibleTurnItems + .filter((row) => row.sourceThreadId !== event.threadId || !items.has(row.sourceItemId)) + .map((row, position) => (row.position === position ? row : { ...row, position })), + }; + } case "thread.visit-recorded": return { ...projection, @@ -3337,6 +3350,28 @@ export const layer: Layer.Layer = } case "checkpoint.rollback-requested": break; + case "thread.history-trimmed": { + const { turnItemIds, nodeIds } = event.payload; + // A fork's copy only revises the fork; the source's own copy removed the rows. + if (event.payload.sourceThreadId !== event.threadId) break; + if (turnItemIds.length > 0) { + yield* sql` + DELETE FROM orchestration_v2_projection_turn_items + WHERE thread_id = ${event.threadId} AND turn_item_id IN ${sql.in(turnItemIds)} + `; + yield* sql` + DELETE FROM orchestration_v2_turn_item_positions + WHERE thread_id = ${event.threadId} AND turn_item_id IN ${sql.in(turnItemIds)} + `; + } + if (nodeIds.length > 0) { + yield* sql` + DELETE FROM orchestration_v2_projection_nodes + WHERE thread_id = ${event.threadId} AND node_id IN ${sql.in(nodeIds)} + `; + } + break; + } case "context-handoff.updated": { const payloadJson = yield* encodeContextHandoffPayload(event.payload); const payload = parseEncodedPayload(payloadJson); @@ -3437,7 +3472,8 @@ export const layer: Layer.Layer = event.type !== "thread.interaction-mode-updated" && event.type !== "thread.model-selection-updated" && event.type !== "thread.provider-switched" && - event.type !== "thread.guest-share-updated" + event.type !== "thread.guest-share-updated" && + event.type !== "thread.history-trimmed" ) { const updatedAt = yield* encodeThreadUpdatedAt(event.occurredAt); yield* sql` diff --git a/apps/server/src/orchestration-v2/ThreadHistoryTrim.test.ts b/apps/server/src/orchestration-v2/ThreadHistoryTrim.test.ts index 667a6e8e2ef0f..d50fed591d354 100644 --- a/apps/server/src/orchestration-v2/ThreadHistoryTrim.test.ts +++ b/apps/server/src/orchestration-v2/ThreadHistoryTrim.test.ts @@ -1,11 +1,13 @@ import { assert, it } from "@effect/vitest"; import { EventId, + ServerSettingsError, MessageId, NodeId, type OrchestrationV2AppThread, type OrchestrationV2DomainEvent, type OrchestrationV2Run, + type OrchestrationV2ThreadProjection, type OrchestrationV2TurnItem, ProjectId, ProviderInstanceId, @@ -14,6 +16,7 @@ import { TurnItemId, } from "@t3tools/contracts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; +import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; @@ -25,16 +28,20 @@ import * as SqlClient from "effect/sql/SqlClient"; import * as OrchestrationEventStore from "../persistence/OrchestrationEventStore.ts"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; import * as ServerSettings from "../serverSettings.ts"; +import { visibleConversationItems } from "./ContextHandoffService.ts"; import * as EventSink from "./EventSink.ts"; import * as EventStore from "./EventStore.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; +import * as ProjectStore from "./ProjectStore.ts"; import * as ThreadHistoryTrim from "./ThreadHistoryTrim.ts"; +import { buildBoundedThreadStreamSnapshot, invalidatesInheritedRows } from "./ThreadStream.ts"; const database = SqlitePersistence.layerMemory; const stores = Layer.mergeAll( database, EventStore.layer.pipe(Layer.provideMerge(database)), ProjectionStore.layer.pipe(Layer.provideMerge(database)), + ProjectStore.layer.pipe(Layer.provideMerge(database)), ); const dependencies = (afterDays: number) => Layer.mergeAll( @@ -57,6 +64,9 @@ type SeedOptions = { readonly thread?: Partial; readonly runStatus?: OrchestrationV2Run["status"]; readonly userMessageAt?: DateTime.Utc; + readonly extraReasoning?: number; + /** A reasoning row under `command`, written after every other row. */ + readonly commandChild?: boolean; }; const seedThread = Effect.fn(function* (name: string, options: SeedOptions = {}) { @@ -226,6 +236,27 @@ const seedThread = Effect.fn(function* (name: string, options: SeedOptions = {}) input: null, output: "The draft the user was asked to read.", }), + item({ + ...itemBase("html"), + type: "dynamic_tool", + toolName: "mcp__t3-code__html_render", + input: null, + output: { htmlRender: { title: "Chart" } }, + }), + item({ + ...itemBase("app"), + type: "dynamic_tool", + toolName: "mcp__charts__show", + input: null, + output: { t3McpApp: { resourceUri: "ui://charts/show" } }, + }), + item({ + ...itemBase("task"), + type: "dynamic_tool", + toolName: "mcp__t3-code__delegate_task", + input: null, + output: "The delegated task's result.", + }), item({ ...itemBase("assistant"), type: "assistant_message", @@ -233,11 +264,72 @@ const seedThread = Effect.fn(function* (name: string, options: SeedOptions = {}) text: "Fixed the harness.", streaming: false, }), + ...Array.from({ length: options.extraReasoning ?? 0 }, (_, index) => + item({ + ...itemBase(`reasoning-${index}`), + type: "reasoning", + text: "more thinking", + streaming: false, + }), + ), + ...(options.commandChild === true + ? [ + item({ + ...itemBase("command-child", null, "command"), + type: "reasoning", + text: "why the command ran", + streaming: false, + }), + ] + : []), ], }); return { threadId, runId }; }); +const openPullRequest = { + host: "github.com", + repository: "acme/app", + number: 7, + url: "https://github.com/acme/app/pull/7", + source: "agent", + linkedAt: DateTime.formatIso(old), + snapshot: { + state: "open", + title: "Fix harness", + headBranch: "fix", + baseBranch: "main", + isDraft: false, + updatedAt: null, + syncedAt: DateTime.formatIso(old), + }, + stack: null, +} satisfies NonNullable[number]; + +const createProject = Effect.flatMap(ProjectStore.ProjectStoreV2, (projects) => + projects.apply({ + sequence: 0, + eventId: EventId.make("created:project:trim"), + aggregateKind: "project", + aggregateId: ProjectId.make("project:trim"), + occurredAt: DateTime.formatIso(old), + commandId: null, + causationEventId: null, + correlationId: null, + metadata: {}, + type: "project.created", + payload: { + projectId: ProjectId.make("project:trim"), + title: "trim", + workspaceRoot: "/work/trim", + defaultModelSelection: null, + scripts: [], + createdAt: DateTime.formatIso(old), + updatedAt: DateTime.formatIso(old), + }, + }), +); + const itemTitles = Effect.fn(function* (threadId: ThreadId) { const sql = yield* SqlClient.SqlClient; const rows = yield* sql<{ readonly title: string }>` @@ -249,9 +341,17 @@ const itemTitles = Effect.fn(function* (threadId: ThreadId) { return rows.map((row) => row.title); }); -const everyItem = ["user", "reasoning", "command", "tool", "edit", "sent", "assistant"]; -// `sent` stays: the timeline always shows that output inline. -const trimmedItems = ["user", "tool", "edit", "sent", "assistant", "Older detail was trimmed"]; +const inlineItems = ["sent", "html", "app", "task"]; +const everyItem = ["user", "reasoning", "command", "tool", "edit", ...inlineItems, "assistant"]; +// Inline items stay: the timeline always shows that output. +const trimmedItems = [ + "user", + "tool", + "edit", + ...inlineItems, + "assistant", + "Older detail was trimmed", +]; const trim = Effect.gen(function* () { const service = yield* ThreadHistoryTrim.ThreadHistoryTrim; @@ -272,13 +372,30 @@ it.layer(layerTest(30))("ThreadHistoryTrim", (it) => { const preview = yield* service.preview; assert.deepEqual( { dryRun: preview.dryRun, threadCount: preview.threadCount, itemCount: preview.itemCount }, - { dryRun: true, threadCount: 1, itemCount: 3 }, + { dryRun: true, threadCount: 1, itemCount: everyItem.length - trimmedItems.length + 1 }, ); - assert.isAbove(preview.estimatedBytes, 8_000); + assert.isAbove(preview.estimatedBytes, 4_000); assert.deepEqual(yield* itemTitles(threadId), everyItem); + const inlineIds = inlineItems.map((key) => `item:idle:${key}`); const report = yield* trim; assert.equal(report.dryRun, false); + // What the web and mobile timelines, exports and transfers read keeps every inline output. + const snapshot = yield* store.getThreadSnapshot(threadId); + const wire = buildBoundedThreadStreamSnapshot(snapshot); + const { turnItems, ...records } = yield* store.getThreadRecords(threadId, [ + "turnItems", + "runs", + "attempts", + ]); + const readers = { + timeline: snapshot.projection.visibleTurnItems.map((row) => row.sourceItemId), + wire: wire.projection.visibleTurnItems.map((row) => row.sourceItemId), + transfer: visibleConversationItems(records, turnItems).map((item) => item.id), + }; + for (const [reader, shown] of Object.entries(readers)) { + assert.includeMembers(shown, inlineIds, reader); + } // `tool` stays: the kept file change hangs off it. assert.deepEqual(yield* itemTitles(threadId), trimmedItems); const nodes = yield* sql<{ readonly node_id: string }>` @@ -334,26 +451,7 @@ it.layer(layerTest(30))("ThreadHistoryTrim", (it) => { }), yield* seedThread("open-pr", { thread: { - pullRequests: [ - { - host: "github.com", - repository: "acme/app", - number: 7, - url: "https://github.com/acme/app/pull/7", - source: "agent", - linkedAt: DateTime.formatIso(old), - snapshot: { - state: "open", - title: "Fix harness", - headBranch: "fix", - baseBranch: "main", - isDraft: false, - updatedAt: null, - syncedAt: DateTime.formatIso(old), - }, - stack: null, - }, - ], + pullRequests: [openPullRequest], }, }), yield* seedThread("side-chat", { @@ -398,6 +496,363 @@ it.layer(layerTest(30))("ThreadHistoryTrim", (it) => { ); }); +type Hook = Effect.Effect< + void, + EventSink.EventSinkV2Error | ProjectionStore.ProjectionStoreV2Error | ServerSettingsError, + | EventSink.EventSinkV2 + | EventStore.EventStoreV2 + | ProjectionStore.ProjectionStoreV2 + | ServerSettings.ServerSettingsService +>; +class TrimHook extends Context.Service< + TrimHook, + { + /** Runs `hook` before the pass's `atWrite`th page: after it chose rows, before it removes them. */ + readonly arm: (hook: Hook, atWrite?: number) => Effect.Effect; + readonly disarm: Effect.Effect; + readonly next: Hook; + } +>()("t3/orchestration-v2/ThreadHistoryTrim.test/TrimHook") {} +const trimHook = Layer.effect( + TrimHook, + Effect.sync(() => { + let armed: { readonly hook: Hook; readonly atWrite: number; writes: number } | null = null; + return TrimHook.of({ + arm: (hook, atWrite = 1) => + Effect.sync(() => { + armed = { hook, atWrite, writes: 0 }; + }), + disarm: Effect.sync(() => { + armed = null; + }), + next: Effect.suspend(() => + armed !== null && ++armed.writes === armed.atWrite ? armed.hook : Effect.void, + ), + }); + }), +); +const hookedSink = Layer.effect( + EventSink.EventSinkV2, + Effect.gen(function* () { + const sink = yield* EventSink.EventSinkV2; + const hooks = yield* TrimHook; + const context = yield* Effect.context>(); + return EventSink.EventSinkV2.of({ + ...sink, + writeBuilt: (build) => + hooks.next.pipe( + Effect.provide(context), + Effect.orDie, + Effect.andThen(sink.writeBuilt(build)), + ), + }); + }), +); +// One migrated database for every hooked case; `isolate` clears what an earlier case left. +const hookedFixture = Layer.mergeAll( + ThreadHistoryTrim.layer.pipe(Layer.provide(hookedSink)), + Layer.effectDiscard(createProject), +).pipe(Layer.provideMerge(Layer.mergeAll(trimHook, dependencies(30)))); +const isolate = Effect.gen(function* () { + yield* TestClock.setTime(DateTime.toEpochMillis(now)); + yield* (yield* TrimHook).disarm; + yield* (yield* ServerSettings.ServerSettingsService).updateSettings({ + threadHistoryTrimAfterDays: 30, + }); + yield* trim; +}); +const arm = (hook: Hook, atWrite?: number) => + Effect.flatMap(TrimHook, (hooks) => hooks.arm(hook, atWrite)); + +const updateThread = ( + name: string, + type: OrchestrationV2DomainEvent["type"], + patch: Partial, +): Hook => + Effect.gen(function* () { + const threadId = ThreadId.make(`thread:trim:${name}`); + const thread = yield* (yield* ProjectionStore.ProjectionStoreV2).getThread(threadId); + yield* (yield* EventSink.EventSinkV2).write({ + events: [ + { + id: EventId.make(`event:${name}:activity`), + type, + threadId, + occurredAt: now, + payload: { ...thread, ...patch }, + } as OrchestrationV2DomainEvent, + ], + }); + }); +const writeActivity = (name: string, event: (threadId: ThreadId, runId: RunId) => object): Hook => + Effect.gen(function* () { + const threadId = ThreadId.make(`thread:trim:${name}`); + const runId = RunId.make(`run:${threadId}:2`); + yield* (yield* EventSink.EventSinkV2).write({ + events: [ + { + id: EventId.make(`event:${name}:activity`), + threadId, + runId, + occurredAt: now, + ...event(threadId, runId), + } as OrchestrationV2DomainEvent, + ], + }); + }); + +const activities: ReadonlyArray = [ + ["opened", updateThread("opened", "thread.visited", { lastVisitedAt: now })], + ["pinned", updateThread("pinned", "thread.pinned", { pinnedAt: now })], + [ + "snoozed", + updateThread("snoozed", "thread.snoozed", { + snoozedAt: now, + snoozedUntil: DateTime.add(now, { days: 1 }), + }), + ], + [ + "pr-linked", + updateThread("pr-linked", "thread.pull-request-synced", { pullRequests: [openPullRequest] }), + ], + [ + "messaged", + writeActivity("messaged", (threadId, runId) => ({ + type: "message.updated", + payload: { + id: MessageId.make("message:messaged:again"), + threadId, + runId, + nodeId: null, + role: "user", + text: "one more thing", + attachments: [], + streaming: false, + createdBy: "user", + creationSource: "web", + createdAt: now, + updatedAt: now, + }, + })), + ], + [ + "run", + writeActivity("run", (threadId, runId) => ({ + type: "run.created", + payload: { + id: runId, + threadId, + ordinal: 2, + providerInstanceId, + modelSelection, + providerThreadId: null, + userMessageId: MessageId.make("message:run:again"), + rootNodeId: NodeId.make("node:run:again"), + activeAttemptId: null, + status: "queued", + requestedAt: now, + startedAt: null, + completedAt: null, + checkpointId: null, + contextHandoffId: null, + }, + })), + ], + [ + "trimming-off", + Effect.flatMap(ServerSettings.ServerSettingsService, (settings) => + settings.updateSettings({ threadHistoryTrimAfterDays: 0 }), + ), + ], +]; + +const ids = (projection: OrchestrationV2ThreadProjection) => ({ + turnItems: projection.turnItems.map((item) => item.id).toSorted(), + nodes: projection.nodes.map((node) => node.id).toSorted(), + visible: projection.visibleTurnItems.map((row) => row.sourceItemId), +}); +type Snapshot = { + readonly snapshotSequence: number; + readonly projection: OrchestrationV2ThreadProjection; +}; + +/** A client that read `threadId` mid-pass, left the caches warm, and then replays its stream. */ +const midPassReader = (threadId: ThreadId) => { + let warmed: Snapshot | undefined; + const read: Hook = Effect.gen(function* () { + const store = yield* ProjectionStore.ProjectionStoreV2; + warmed = yield* store.getThreadSnapshot(threadId); + yield* store.getThreadSnapshotWindow(threadId, { rowLimit: 50 }); + yield* store.searchThread({ threadId, query: "flaky" }); + }); + /** A fork's replay holds compact copies, which subscribers answer with a fresh snapshot. */ + const verify = (removed: Array, fork = false) => + Effect.gen(function* () { + const store = yield* ProjectionStore.ProjectionStoreV2; + const before = warmed!; + assert.includeMembers(ids(before.projection).visible, removed); + + const after = yield* store.getThreadSnapshot(threadId); + assert.isAbove(after.snapshotSequence, before.snapshotSequence); + assert.notIncludeMembers(ids(after.projection).visible, removed); + // The events a live subscriber hears, and a reconnect after `before` replays. + const replay = yield* Stream.runCollect( + (yield* EventStore.EventStoreV2).read({ threadId, afterSequence: before.snapshotSequence }), + ); + if (fork) { + const copies = replay.filter((stored) => invalidatesInheritedRows(stored.event, threadId)); + assert.isNotEmpty(copies); + for (const copy of copies) { + assert.isAtMost(JSON.stringify(copy.event.payload).length, 128); + } + } else { + const resumed = replay.reduce( + (projection, stored) => ProjectionStore.applyToProjection(projection, stored.event), + before.projection, + ); + assert.deepEqual(ids(resumed), ids(after.projection)); + } + + const window = yield* store.getThreadSnapshotWindow(threadId, { rowLimit: 50 }); + assert.equal(window.snapshotSequence, after.snapshotSequence); + assert.notIncludeMembers(ids(window.projection).visible, removed); + const found = yield* store.searchThread({ threadId, query: "flaky" }); + assert.equal(found.snapshotSequence, after.snapshotSequence); + }); + return { read, verify }; +}; + +const orphanedItems = Effect.fn(function* (threadId: ThreadId) { + const sql = yield* SqlClient.SqlClient; + const rows = yield* sql<{ readonly id: string }>` + SELECT child.turn_item_id AS id FROM orchestration_v2_projection_turn_items AS child + WHERE child.thread_id = ${threadId} AND child.parent_item_id IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM orchestration_v2_projection_turn_items AS parent + WHERE parent.turn_item_id = child.parent_item_id + ) + `; + return rows.map((row) => row.id); +}); + +it.layer(hookedFixture)("ThreadHistoryTrim during a pass", (it) => { + it.effect.each(activities)("stops trimming a thread that is %s", ([name, activity]) => + Effect.gen(function* () { + yield* isolate; + const { threadId } = yield* seedThread(name); + yield* arm(activity); + yield* trim; + assert.deepEqual(yield* itemTitles(threadId), everyItem); + }), + ); + + it.effect("stops between pages without orphaning a kept row", () => + Effect.gen(function* () { + yield* isolate; + const name = "opened-between-pages"; + // `command` sorts into the first page and its child into the second, unless children go first. + const { threadId } = yield* seedThread(name, { extraReasoning: 250, commandChild: true }); + yield* arm(updateThread(name, "thread.visited", { lastVisitedAt: now }), 2); + yield* trim; + assert.lengthOf(yield* itemTitles(threadId), everyItem.length + 251 - 200 + 1); + assert.deepEqual(yield* orphanedItems(threadId), []); + }), + ); + + it.effect("keeps a row that gains a child after the pass chose it", () => + Effect.gen(function* () { + yield* isolate; + const name = "gains-child"; + const { threadId, runId } = yield* seedThread(name); + const child: Hook = Effect.flatMap(EventSink.EventSinkV2, (sink) => + sink.write({ + events: [ + { + id: EventId.make(`event:${name}:child`), + type: "turn-item.updated", + threadId, + runId, + occurredAt: old, + payload: { + id: TurnItemId.make(`item:${name}:late-edit`), + threadId, + runId, + nodeId: null, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: TurnItemId.make(`item:${name}:command`), + ordinal: 0, + status: "completed", + title: "late-edit", + startedAt: old, + completedAt: old, + updatedAt: old, + type: "file_change", + fileName: "late.ts", + }, + }, + ], + }), + ); + yield* arm(child); + yield* trim; + assert.include(yield* itemTitles(threadId), "command"); + assert.deepEqual(yield* orphanedItems(threadId), []); + }), + ); + + it.effect( + "advances the thread's revision so reconnects, windows and search drop trimmed rows", + () => + Effect.gen(function* () { + yield* isolate; + const name = "reconnect"; + const { threadId } = yield* seedThread(name); + const reader = midPassReader(threadId); + yield* arm(reader.read); + yield* trim; + yield* reader.verify([`item:${name}:reasoning`, `item:${name}:command`]); + }), + ); + + it.effect("advances the revision of every live fork, through deleted forks, compactly", () => + Effect.gen(function* () { + yield* isolate; + const source = yield* seedThread("fork-source"); + const forkOf = (name: string, parent: { threadId: ThreadId; runId: RunId }) => + seedThread(name, { + userMessageAt: recent, + thread: { + lineage: { + parentThreadId: parent.threadId, + relationshipToParent: "fork", + rootThreadId: source.threadId, + }, + forkedFrom: { type: "run", threadId: parent.threadId, runId: parent.runId }, + }, + }); + const fork = yield* forkOf("fork", source); + const deleted = yield* forkOf("deleted-fork", source); + const grandchild = yield* forkOf("fork-of-deleted", deleted); + yield* updateThread("deleted-fork", "thread.deleted", { deletedAt: now }); + const deletedEvents = Stream.runCollect( + (yield* EventStore.EventStoreV2).read({ threadId: deleted.threadId }), + ); + const deletedBefore = (yield* deletedEvents).length; + + const readers = [midPassReader(grandchild.threadId), midPassReader(fork.threadId)]; + yield* arm(Effect.forEach(readers, (reader) => reader.read, { discard: true })); + yield* trim; + for (const reader of readers) { + yield* reader.verify(["item:fork-source:reasoning", "item:fork-source:command"], true); + } + assert.lengthOf(yield* deletedEvents, deletedBefore); + assert.deepEqual(yield* itemTitles(fork.threadId), everyItem); + }), + ); +}); + it.layer(layerTest(0))("ThreadHistoryTrim turned off", (it) => { it.effect("removes nothing", () => Effect.gen(function* () { diff --git a/apps/server/src/orchestration-v2/ThreadHistoryTrim.ts b/apps/server/src/orchestration-v2/ThreadHistoryTrim.ts index 0d84c25d15e41..a1539c915939c 100644 --- a/apps/server/src/orchestration-v2/ThreadHistoryTrim.ts +++ b/apps/server/src/orchestration-v2/ThreadHistoryTrim.ts @@ -1,8 +1,11 @@ import { + type NodeId, + type OrchestrationV2DomainEvent, type RunId, type ThreadHistoryTrimReport, type ThreadHistoryTrimStatus, type ThreadId, + type TurnItemId, } from "@t3tools/contracts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import { makeDrainableWorker } from "@t3tools/shared/DrainableWorker"; @@ -71,7 +74,9 @@ type Candidate = { readonly updatedAt: string; }; type Mode = "background" | "manual"; -type Removed = { readonly itemCount: number; readonly bytes: number }; + +const sumBytes = (rows: ReadonlyArray<{ readonly bytes: number }>) => + rows.reduce((total, row) => total + row.bytes, 0); // Space a row gives back: its payload plus each copy of its ids in the row, its indexes and its position row. const ITEM_BYTES = `(octet_length(payload_json) + 5 * length(turn_item_id) + 8 * length(thread_id) @@ -88,7 +93,7 @@ function trimNoticeMessage(afterDays: number): string { return `Tool output and reasoning above this point were removed to save space after ${afterDays} ${afterDays === 1 ? "day" : "days"} without use. Messages, plans and file changes are kept.`; } -/** Ids no kept row points at, following `parent` links so a kept row never loses an ancestor. */ +/** Ids no kept row points at, children first, so a pass that stops early never orphans a row. */ function unreferenced( rows: ReadonlyArray, removable: (row: Row) => boolean, @@ -104,7 +109,37 @@ function unreferenced !kept.has(row.id)); + const depth = (row: Row) => { + let count = 0; + for (let current = row.parent; current !== null; current = byId.get(current)?.parent ?? null) + count += 1; + return count; + }; + return rows + .filter((row) => !kept.has(row.id)) + .map((row) => ({ row, depth: depth(row) })) + .toSorted((left, right) => right.depth - left.depth) + .map(({ row }) => row); +} + +/** Drops each id that a row outside `removable` still hangs off, until none does. */ +function withoutParentsOfKept( + removable: ReadonlyArray, + children: ReadonlyArray<{ readonly id: string; readonly parent: string }>, + blocked: Iterable, +): ReadonlySet { + const removed = new Set(removable); + for (const id of blocked) removed.delete(id); + for (let changed = true; changed;) { + changed = false; + for (const child of children) { + if (removed.has(child.parent) && !removed.has(child.id)) { + removed.delete(child.parent); + changed = true; + } + } + } + return removed; } const make = Effect.gen(function* () { @@ -163,7 +198,7 @@ const make = Effect.gen(function* () { OR COALESCE(recent.completed_at, '') >= ${input.idleBefore} ) ) - AND EXISTS ( + AND (${input.threadId !== undefined} OR EXISTS ( SELECT 1 FROM orchestration_v2_projection_turn_items AS item WHERE item.thread_id = thread.thread_id AND item.type IN ${sql.in(TRIMMED_ITEM_TYPES)} @@ -174,7 +209,7 @@ const make = Effect.gen(function* () { SELECT 1 FROM orchestration_v2_projection_turn_items AS child WHERE child.thread_id = item.thread_id AND child.parent_item_id = item.turn_item_id ) - ) + )) ORDER BY thread.thread_id ASC `; @@ -188,114 +223,48 @@ const make = Effect.gen(function* () { }; }); - const estimate = Effect.fn("ThreadHistoryTrim.estimate")(function* (dryRun: boolean) { - const { afterDays, now, idleBefore } = yield* window; - const candidates = afterDays <= 0 ? [] : yield* readCandidates({ idleBefore, now }); - let itemCount = 0; - let estimatedBytes = 0; - for (const candidate of candidates) { - const rows = yield* sql<{ readonly items: number; readonly bytes: number }>` - SELECT - (SELECT COUNT(*) FROM orchestration_v2_projection_turn_items - WHERE thread_id = ${candidate.threadId} - AND type IN ${sql.in(TRIMMED_ITEM_TYPES)} - AND inline_work_output = 0 - AND status NOT IN ${sql.in(LIVE_ITEM_STATUSES)}) AS items, - COALESCE((SELECT SUM(${sql.literal(ITEM_BYTES)}) FROM orchestration_v2_projection_turn_items - WHERE thread_id = ${candidate.threadId} - AND type IN ${sql.in(TRIMMED_ITEM_TYPES)} - AND inline_work_output = 0 - AND status NOT IN ${sql.in(LIVE_ITEM_STATUSES)}), 0) - + COALESCE((SELECT SUM(${sql.literal(nodeBytes(""))}) FROM orchestration_v2_projection_nodes - WHERE thread_id = ${candidate.threadId} - AND kind IN ${sql.in(TRIMMED_NODE_KINDS)} - AND status NOT IN ${sql.in(LIVE_NODE_STATUSES)}), 0) AS bytes - `; - itemCount += rows[0]?.items ?? 0; - estimatedBytes += rows[0]?.bytes ?? 0; - yield* Effect.yieldNow; - } - return { - dryRun, - afterDays, - threadCount: candidates.length, - itemCount, - estimatedBytes, - } satisfies ThreadHistoryTrimReport; - }); - - const writeNotice = Effect.fn("ThreadHistoryTrim.writeNotice")(function* ( - candidate: Candidate, - afterDays: number, - ) { - const now = yield* DateTime.now; - yield* eventSink.write({ - events: [ - { - id: yield* ids.allocate.event({ threadId: candidate.threadId }), - type: "turn-item.updated", - threadId: candidate.threadId, - runId: candidate.runId, - // Every event stamps the thread with its time; reusing the thread's own keeps a trimmed - // thread where it was in lists sorted by activity. - occurredAt: DateTime.makeUnsafe(candidate.updatedAt), - payload: { - id: ids.derive.runSignalTurnItem({ runId: candidate.runId, signal: NOTICE_SIGNAL }), - threadId: candidate.threadId, - runId: candidate.runId, - nodeId: null, - providerThreadId: null, - providerTurnId: null, - nativeItemRef: null, - parentItemId: null, - // EventSink allocates the real position, after the run's existing items. - ordinal: 0, - status: "completed", - title: "Older detail was trimmed", - startedAt: now, - completedAt: now, - updatedAt: now, - type: "system_notice", - message: trimNoticeMessage(afterDays), - }, - }, - ], - }); - }); + // Rows other than turn items and nodes that keep a node. + const nodeReferences = (threadId: ThreadId) => sql` + SELECT node_id AS id FROM orchestration_v2_projection_messages + WHERE thread_id = ${threadId} AND node_id IS NOT NULL + UNION SELECT node_id FROM orchestration_v2_projection_plans + WHERE thread_id = ${threadId} AND node_id IS NOT NULL + UNION SELECT parent_node_id FROM orchestration_v2_projection_subagents + WHERE thread_id = ${threadId} + UNION SELECT node_id FROM orchestration_v2_projection_runtime_requests + WHERE thread_id = ${threadId} AND node_id IS NOT NULL + UNION SELECT node_id FROM orchestration_v2_projection_checkpoints + WHERE thread_id = ${threadId} + UNION SELECT node_id FROM orchestration_v2_projection_checkpoint_scopes + WHERE thread_id = ${threadId} AND node_id IS NOT NULL + UNION SELECT node_id FROM orchestration_v2_projection_provider_turns + WHERE thread_id = ${threadId} + UNION SELECT root_node_id FROM orchestration_v2_projection_run_attempts + WHERE thread_id = ${threadId} AND root_node_id IS NOT NULL + UNION SELECT owner_node_id FROM orchestration_v2_projection_provider_threads + WHERE thread_id = ${threadId} AND owner_node_id IS NOT NULL + `; - const deleteInPages = Effect.fnUntraced(function* ( - rowIds: ReadonlyArray, - deletePage: (page: ReadonlyArray) => Effect.Effect, - rest: Effect.Effect, - ) { - for (let offset = 0; offset < rowIds.length; offset += DELETE_PAGE_SIZE) { - yield* deletePage(rowIds.slice(offset, offset + DELETE_PAGE_SIZE)); - // Never hold the writer across a rest: foreground commits run between pages. - yield* rest; - } - }); + // Live forks that show this thread's rows, at any depth; a deleted fork still passes them on. + const readForks = (threadId: ThreadId) => sql<{ readonly id: ThreadId }>` + WITH RECURSIVE forks(thread_id, deleted) AS ( + SELECT ${threadId}, 0 + UNION + SELECT fork.thread_id, fork.deleted_at IS NOT NULL + FROM orchestration_v2_projection_threads AS fork + JOIN forks ON json_extract(fork.payload_json, '$.forkedFrom.threadId') = forks.thread_id + WHERE json_extract(fork.payload_json, '$.forkedFrom.type') = 'run' + AND COALESCE(json_extract(fork.payload_json, '$.lineage.sideChat'), 0) = 0 + ) + SELECT thread_id AS id FROM forks WHERE thread_id <> ${threadId} AND deleted = 0 + `; - const trimThread = Effect.fn("ThreadHistoryTrim.trimThread")(function* ( - candidate: Candidate, - afterDays: number, - rest: Effect.Effect, - ) { + const select = Effect.fn("ThreadHistoryTrim.select")(function* (candidate: Candidate) { const { threadId } = candidate; - const noticeId = ids.derive.runSignalTurnItem({ - runId: candidate.runId, - signal: NOTICE_SIGNAL, - }); - const readBoundary = sql<{ readonly ordinal: number }>` - SELECT ordinal FROM orchestration_v2_projection_turn_items WHERE turn_item_id = ${noticeId} - `; - // The notice goes first: a pass that stops halfway leaves a marked thread the next pass finishes. - if ((yield* readBoundary)[0] === undefined) yield* writeNotice(candidate, afterDays); - const boundary = (yield* readBoundary)[0]?.ordinal; - if (boundary === undefined) return { itemCount: 0, bytes: 0 } satisfies Removed; - const items = yield* sql<{ readonly id: string; readonly parent: string | null; + readonly node: string | null; readonly type: string; readonly status: string; readonly inline: number; @@ -304,13 +273,14 @@ const make = Effect.gen(function* () { SELECT turn_item_id AS id, parent_item_id AS parent, + node_id AS node, type, status, inline_work_output AS inline, ${sql.literal(ITEM_BYTES)} AS bytes FROM orchestration_v2_projection_turn_items - WHERE thread_id = ${threadId} AND ordinal < ${boundary} - `.pipe(NodeSqliteClient.readOnlyBackground); + WHERE thread_id = ${threadId} + `; const trimmedItemTypes: ReadonlyArray = TRIMMED_ITEM_TYPES; const liveItemStatuses: ReadonlyArray = LIVE_ITEM_STATUSES; const removedItems = unreferenced( @@ -321,25 +291,7 @@ const make = Effect.gen(function* () { !liveItemStatuses.includes(item.status), [], ); - yield* deleteInPages( - removedItems.map((item) => item.id), - (page) => - sql.withTransaction( - Effect.gen(function* () { - yield* sql` - DELETE FROM orchestration_v2_projection_turn_items - WHERE turn_item_id IN ${sql.in(page)} - AND status NOT IN ${sql.in(LIVE_ITEM_STATUSES)} - AND inline_work_output = 0 - `; - yield* sql` - DELETE FROM orchestration_v2_turn_item_positions - WHERE thread_id = ${threadId} AND turn_item_id IN ${sql.in(page)} - `; - }), - ), - rest, - ); + const removedItemIds = new Set(removedItems.map((item) => item.id)); const nodes = yield* sql<{ readonly id: string; @@ -359,29 +311,8 @@ const make = Effect.gen(function* () { FROM orchestration_v2_projection_nodes AS node LEFT JOIN orchestration_v2_projection_runs AS run ON run.run_id = node.run_id WHERE node.thread_id = ${threadId} - `.pipe(NodeSqliteClient.readOnlyBackground); - const referencedNodes = yield* sql<{ readonly id: string }>` - SELECT node_id AS id FROM orchestration_v2_projection_turn_items - WHERE thread_id = ${threadId} AND node_id IS NOT NULL - UNION SELECT node_id FROM orchestration_v2_projection_messages - WHERE thread_id = ${threadId} AND node_id IS NOT NULL - UNION SELECT node_id FROM orchestration_v2_projection_plans - WHERE thread_id = ${threadId} AND node_id IS NOT NULL - UNION SELECT parent_node_id FROM orchestration_v2_projection_subagents - WHERE thread_id = ${threadId} - UNION SELECT node_id FROM orchestration_v2_projection_runtime_requests - WHERE thread_id = ${threadId} AND node_id IS NOT NULL - UNION SELECT node_id FROM orchestration_v2_projection_checkpoints - WHERE thread_id = ${threadId} - UNION SELECT node_id FROM orchestration_v2_projection_checkpoint_scopes - WHERE thread_id = ${threadId} AND node_id IS NOT NULL - UNION SELECT node_id FROM orchestration_v2_projection_provider_turns - WHERE thread_id = ${threadId} - UNION SELECT root_node_id FROM orchestration_v2_projection_run_attempts - WHERE thread_id = ${threadId} AND root_node_id IS NOT NULL - UNION SELECT owner_node_id FROM orchestration_v2_projection_provider_threads - WHERE thread_id = ${threadId} AND owner_node_id IS NOT NULL - `.pipe(NodeSqliteClient.readOnlyBackground); + `; + const referencedNodes = yield* sql<{ readonly id: string }>`${nodeReferences(threadId)}`; const trimmedNodeKinds: ReadonlyArray = TRIMMED_NODE_KINDS; const liveNodeStatuses: ReadonlyArray = LIVE_NODE_STATUSES; const removedNodes = unreferenced( @@ -390,23 +321,228 @@ const make = Effect.gen(function* () { node.settled === 1 && trimmedNodeKinds.includes(node.kind) && !liveNodeStatuses.includes(node.status), - referencedNodes.map((row) => row.id), - ); - yield* deleteInPages( - removedNodes.map((node) => node.id), - (page) => sql` - DELETE FROM orchestration_v2_projection_nodes - WHERE node_id IN ${sql.in(page)} AND status NOT IN ${sql.in(LIVE_NODE_STATUSES)} - `, - rest, + [ + ...items.flatMap((item) => + item.node === null || removedItemIds.has(item.id) ? [] : [item.node], + ), + ...referencedNodes.map((row) => row.id), + ], ); + return { items: removedItems, nodes: removedNodes }; + }); + + const estimate = Effect.fn("ThreadHistoryTrim.estimate")(function* (dryRun: boolean) { + const { afterDays, now, idleBefore } = yield* window; + const candidates = afterDays <= 0 ? [] : yield* readCandidates({ idleBefore, now }); + let itemCount = 0; + let estimatedBytes = 0; + for (const candidate of candidates) { + const selection = yield* select(candidate); + itemCount += selection.items.length; + estimatedBytes += sumBytes(selection.items) + sumBytes(selection.nodes); + yield* Effect.yieldNow; + } + return { + dryRun, + afterDays, + threadCount: candidates.length, + itemCount, + estimatedBytes, + } satisfies ThreadHistoryTrimReport; + }); - const bytes = (rows: ReadonlyArray<{ readonly bytes: number }>) => - rows.reduce((total, row) => total + row.bytes, 0); + const noticeEvent = Effect.fn("ThreadHistoryTrim.noticeEvent")(function* ( + candidate: Candidate, + afterDays: number, + ) { + const now = yield* DateTime.now; return { - itemCount: removedItems.length, - bytes: bytes(removedItems) + bytes(removedNodes), - } satisfies Removed; + id: yield* ids.allocate.event({ threadId: candidate.threadId }), + type: "turn-item.updated", + threadId: candidate.threadId, + runId: candidate.runId, + // Every event stamps the thread with its time; reusing the thread's own keeps a trimmed + // thread where it was in lists sorted by activity. + occurredAt: DateTime.makeUnsafe(candidate.updatedAt), + payload: { + id: ids.derive.runSignalTurnItem({ runId: candidate.runId, signal: NOTICE_SIGNAL }), + threadId: candidate.threadId, + runId: candidate.runId, + nodeId: null, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + // EventSink allocates the real position, after the run's existing items. + ordinal: 0, + status: "completed", + title: "Older detail was trimmed", + startedAt: now, + completedAt: now, + updatedAt: now, + type: "system_notice", + message: trimNoticeMessage(afterDays), + }, + } satisfies OrchestrationV2DomainEvent; + }); + + /** Removes one page in a transaction that first confirms the thread is still idle and trimming on. */ + const trimPage = Effect.fn("ThreadHistoryTrim.trimPage")(function* ( + threadId: ThreadId, + page: { readonly itemIds: ReadonlyArray; readonly nodeIds: ReadonlyArray }, + ) { + // Assigned inside the transaction, which narrowing cannot see. + let outcome = "trimmed" as "trimmed" | "thread-in-use" | "trimming-off"; + let removed: { readonly itemIds: ReadonlySet; readonly nodeIds: ReadonlySet } = + { + itemIds: new Set(), + nodeIds: new Set(), + }; + yield* eventSink.writeBuilt( + Effect.gen(function* () { + const { afterDays, now, idleBefore } = yield* window; + if (afterDays <= 0) { + outcome = "trimming-off"; + return []; + } + const current = (yield* readCandidates({ idleBefore, now, threadId }))[0]; + if (current === undefined) { + outcome = "thread-in-use"; + return []; + } + const itemIds = + page.itemIds.length === 0 + ? [] + : yield* sql<{ readonly id: TurnItemId }>` + SELECT turn_item_id AS id FROM orchestration_v2_projection_turn_items + WHERE thread_id = ${threadId} AND turn_item_id IN ${sql.in(page.itemIds)} + AND status NOT IN ${sql.in(LIVE_ITEM_STATUSES)} AND inline_work_output = 0 + `; + const itemChildren = + page.itemIds.length === 0 + ? [] + : yield* sql<{ + readonly id: string; + readonly parent: string; + }>` + SELECT turn_item_id AS id, parent_item_id AS parent + FROM orchestration_v2_projection_turn_items + WHERE thread_id = ${threadId} AND parent_item_id IN ${sql.in(page.itemIds)} + `; + const items = withoutParentsOfKept( + itemIds.map((row) => row.id), + itemChildren, + [], + ); + + const nodeIds = + page.nodeIds.length === 0 + ? [] + : yield* sql<{ readonly id: NodeId }>` + SELECT node_id AS id FROM orchestration_v2_projection_nodes + WHERE thread_id = ${threadId} AND node_id IN ${sql.in(page.nodeIds)} + AND status NOT IN ${sql.in(LIVE_NODE_STATUSES)} + `; + const nodeChildren = + page.nodeIds.length === 0 + ? [] + : yield* sql<{ + readonly id: string; + readonly parent: string; + }>` + SELECT node_id AS id, parent_node_id AS parent FROM orchestration_v2_projection_nodes + WHERE thread_id = ${threadId} AND parent_node_id IN ${sql.in(page.nodeIds)} + `; + const nodeHolders = + page.nodeIds.length === 0 + ? [] + : yield* sql<{ + readonly id: string; + readonly node: string; + }>` + SELECT turn_item_id AS id, node_id AS node FROM orchestration_v2_projection_turn_items + WHERE thread_id = ${threadId} AND node_id IN ${sql.in(page.nodeIds)} + `; + const otherReferences = + page.nodeIds.length === 0 + ? [] + : yield* sql<{ readonly id: string }>` + SELECT id FROM (${nodeReferences(threadId)}) WHERE id IN ${sql.in(page.nodeIds)} + `; + const nodes = withoutParentsOfKept( + nodeIds.map((row) => row.id), + nodeChildren, + [ + ...nodeHolders.flatMap((row) => (items.has(row.id) ? [] : [row.node])), + ...otherReferences.map((row) => row.id), + ], + ); + if (items.size === 0 && nodes.size === 0) return []; + removed = { itemIds: items, nodeIds: nodes }; + const noticeId = ids.derive.runSignalTurnItem({ + runId: current.runId, + signal: NOTICE_SIGNAL, + }); + const noticed = yield* sql` + SELECT 1 FROM orchestration_v2_projection_turn_items WHERE turn_item_id = ${noticeId} + `; + const events: Array = + noticed.length === 0 ? [yield* noticeEvent(current, afterDays)] : []; + const occurredAt = yield* DateTime.now; + events.push({ + id: yield* ids.allocate.event({ threadId }), + type: "thread.history-trimmed", + threadId, + occurredAt, + payload: { + sourceThreadId: threadId, + turnItemIds: itemIds.flatMap((row) => (items.has(row.id) ? [row.id] : [])), + nodeIds: nodeIds.flatMap((row) => (nodes.has(row.id) ? [row.id] : [])), + }, + }); + // Each fork's revision moves with the rows it inherits; its copy stays a few bytes. + for (const { id: fork } of yield* readForks(threadId)) { + events.push({ + id: yield* ids.allocate.event({ threadId: fork }), + type: "thread.history-trimmed", + threadId: fork, + occurredAt, + payload: { sourceThreadId: threadId, turnItemIds: [], nodeIds: [] }, + }); + } + return events; + }), + ); + return { outcome, removed }; + }); + + const trimThread = Effect.fn("ThreadHistoryTrim.trimThread")(function* ( + candidate: Candidate, + rest: Effect.Effect, + ) { + const selection = yield* select(candidate).pipe(NodeSqliteClient.readOnlyBackground); + const rows = [ + ...selection.items.map((row) => ({ ...row, kind: "item" as const })), + ...selection.nodes.map((row) => ({ ...row, kind: "node" as const })), + ]; + let itemCount = 0; + let bytes = 0; + for (let offset = 0; offset < rows.length; offset += DELETE_PAGE_SIZE) { + const page = rows.slice(offset, offset + DELETE_PAGE_SIZE); + const { outcome, removed } = yield* trimPage(candidate.threadId, { + itemIds: page.flatMap((row) => (row.kind === "item" ? [row.id] : [])), + nodeIds: page.flatMap((row) => (row.kind === "node" ? [row.id] : [])), + }); + if (outcome !== "trimmed") return { outcome, itemCount, bytes }; + for (const row of page) { + if (!(row.kind === "item" ? removed.itemIds : removed.nodeIds).has(row.id)) continue; + if (row.kind === "item") itemCount += 1; + bytes += row.bytes; + } + // Never hold the writer across a rest: foreground commits run between pages. + yield* rest; + } + return { outcome: "trimmed" as const, itemCount, bytes }; }); const readLastPass = sql<{ readonly updated_at: string }>` @@ -470,7 +606,7 @@ const make = Effect.gen(function* () { const current = yield* readCandidates({ idleBefore, now, threadId: listed.threadId }); const candidate = current[0]; if (candidate === undefined) continue; - const removed = yield* trimThread(candidate, afterDays, rest).pipe( + const removed = yield* trimThread(candidate, rest).pipe( Effect.catchCauseIf( (cause) => !Cause.hasInterruptsOnly(cause), (cause) => @@ -481,6 +617,8 @@ const make = Effect.gen(function* () { ), ); if (removed === null) continue; + if (removed.outcome === "trimming-off") return; + if (removed.itemCount === 0 && removed.bytes === 0) continue; threadCount += 1; itemCount += removed.itemCount; estimatedBytes += removed.bytes; diff --git a/apps/server/src/orchestration-v2/ThreadStream.ts b/apps/server/src/orchestration-v2/ThreadStream.ts index c16a1b9e095aa..a04e9db317974 100644 --- a/apps/server/src/orchestration-v2/ThreadStream.ts +++ b/apps/server/src/orchestration-v2/ThreadStream.ts @@ -1,5 +1,7 @@ import type { + OrchestrationV2DomainEvent, OrchestrationV2ThreadProjection, + ThreadId, OrchestrationV2ThreadStreamItem, } from "@t3tools/contracts"; @@ -116,3 +118,11 @@ export function decideThreadResume(input: { throughSequence: input.highWater, }; } + +/** A fork's copy of its source's history trim, which names no rows: read a fresh snapshot. */ +export function invalidatesInheritedRows( + event: OrchestrationV2DomainEvent, + threadId: ThreadId, +): boolean { + return event.type === "thread.history-trimmed" && event.payload.sourceThreadId !== threadId; +} diff --git a/apps/server/src/ws.test.ts b/apps/server/src/ws.test.ts index 1c37de19c34a9..fc7b5d7a7a98f 100644 --- a/apps/server/src/ws.test.ts +++ b/apps/server/src/ws.test.ts @@ -35,6 +35,7 @@ import { type OrchestrationV2AppThread, type OrchestrationV2DomainEvent, type OrchestrationV2StoredEvent, + type OrchestrationV2ThreadProjection, type OrchestrationV2ThreadShell, type OrchestrationProjectShell, type ServerConfig, @@ -484,6 +485,74 @@ it.effect("coalesces streaming replay before synchronization and the live tail", }).pipe(Effect.scoped, Effect.provide(ReplayTestLayer)), ); +it.effect("a fork reads a fresh snapshot when its source's history is trimmed", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const store = yield* OrchestrationEventStore.OrchestrationEventStore; + const threadId = ThreadId.make("thread:trimmed-fork"); + const now = "2026-11-17T00:00:00.000Z"; + const insert = (version: number, type: string, payload: object) => + sql<{ sequence: number }>` + INSERT INTO orchestration_events ( + event_id, aggregate_kind, stream_id, stream_version, event_type, + occurred_at, actor_kind, payload_json, metadata_json, application_event_version + ) VALUES ( + ${`trimmed-fork:${version}`}, 'thread', ${threadId}, ${version}, ${type}, + ${now}, 'server', ${JSON.stringify(payload)}, '{}', 2 + ) RETURNING sequence + `.pipe(Effect.map(([row]) => row!.sequence)); + const forkCopy = { sourceThreadId: "thread:trimmed-source", turnItemIds: [], nodeIds: [] }; + const seen = yield* insert(0, "provider-session.detached", { + providerSessionId: "session:trimmed-fork", + detachedAt: now, + }); + const replayed = yield* insert(1, "thread.history-trimmed", forkCopy); + let snapshotSequence = replayed; + const snapshots: Array = []; + const items = yield* subscribeOrchestrationV2Thread({ + threadId, + afterSequence: seen, + requestCompletionMarker: true, + }).pipe( + Effect.flatMap(Stream.runCollect), + Effect.provideService(ThreadManagement.ThreadManagementService, { + ensureLegacyTranscript: () => Effect.void, + getThreadSnapshot: () => + Effect.sync(() => { + snapshots.push(snapshotSequence); + return { + schemaVersion: 1, + snapshotSequence, + projection: { + checkpoints: [], + contextHandoffs: [], + turnItems: [], + visibleTurnItems: [], + } as unknown as OrchestrationV2ThreadProjection, + }; + }), + streamStoredEventsFrom: () => + Stream.unwrap( + insert(2, "thread.history-trimmed", forkCopy).pipe( + Effect.map((live) => { + snapshotSequence = live; + return store.readAgentEvents({ threadId, afterSequence: replayed }); + }), + ), + ), + } as unknown as ThreadManagement.ThreadManagementService["Service"]), + ); + // Replay and the live tail each answer the fork's copy with what the fork shows now. + assert.deepEqual( + items.map((item) => + item.kind === "snapshot" ? [item.kind, item.snapshotSequence] : [item.kind], + ), + [["snapshot", replayed], ["synchronized"], ["snapshot", replayed + 1]], + ); + assert.lengthOf(snapshots, 2); + }).pipe(Effect.scoped, Effect.provide(ReplayTestLayer)), +); + it.effect("shell groups read distinct threads once and keep each event's own sequence", () => Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 186c6b1040733..2e3acdc68bef6 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -16,6 +16,7 @@ import * as PushRegistrations from "./push/PushRegistrations.ts"; import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; import * as Base64 from "effect/encoding/Base64"; +import * as Arr from "effect/Array"; import * as Effect from "effect/Effect"; import * as NodeHttpServerRequest from "@effect/platform-node/NodeHttpServerRequest"; import * as Fiber from "effect/Fiber"; @@ -153,6 +154,7 @@ import { import { buildBoundedThreadStreamSnapshot, decideThreadResume, + invalidatesInheritedRows, isThreadReplayRawPayloadSafe, threadReplayEncodedBytes, THREAD_RESUME_MAX_REPLAY_EVENTS, @@ -854,6 +856,31 @@ export const subscribeOrchestrationV2Thread = Effect.fn("ws.orchestrationV2.subs .pipe( Stream.map(threadStreamEventItem), coalesceThreadLiveStream, + // Rows this fork inherits were trimmed at their source: send what it shows now. + // Per array, so an ordinary batch passes through as one message. + Stream.mapArrayEffect( + ( + items, + ): Effect.Effect< + Arr.NonEmptyReadonlyArray, + OrchestrationV2GetThreadProjectionError + > => { + const invalidates = (item: OrchestrationV2ThreadStreamItem) => + item.kind === "event" && invalidatesInheritedRows(item.event, input.threadId); + if (!items.some(invalidates)) return Effect.succeed(items); + return Effect.forEach( + items, + ( + item, + ): Effect.Effect< + OrchestrationV2ThreadStreamItem, + OrchestrationV2GetThreadProjectionError + > => (invalidates(item) ? loadSnapshotItem() : Effect.succeed(item)), + ).pipe( + Effect.map((mapped) => (Arr.isReadonlyArrayNonEmpty(mapped) ? mapped : items)), + ); + }, + ), Stream.mapError( (cause) => new OrchestrationV2GetThreadProjectionError({ @@ -897,7 +924,7 @@ export const subscribeOrchestrationV2Thread = Effect.fn("ws.orchestrationV2.subs ? Stream.make({ kind: "synchronized" as const }) : Stream.empty; - const snapshotThenLive = Effect.fn("ws.orchestrationV2.threadSnapshotThenLive")(function* () { + const loadSnapshotItem = Effect.fn("ws.orchestrationV2.loadThreadSnapshotItem")(function* () { const useBoundedSnapshot = shouldUseBoundedThreadSnapshot(input); const snapshot = yield* ( useBoundedSnapshot @@ -933,9 +960,13 @@ export const subscribeOrchestrationV2Thread = Effect.fn("ws.orchestrationV2.subs projection: dedupeBodies ? dedupeSnapshotBodies(projection) : projection, }; })(); + return snapshotItem; + }); + const snapshotThenLive = Effect.fn("ws.orchestrationV2.threadSnapshotThenLive")(function* () { + const snapshotItem = yield* loadSnapshotItem(); return rpcInitialItems([snapshotItem]).pipe( Stream.concat(completionMarker), - Stream.concat(eventStreamFrom(snapshotSequence)), + Stream.concat(eventStreamFrom(snapshotItem.snapshotSequence)), withLiveDedupe, ); }); @@ -1026,6 +1057,13 @@ export const subscribeOrchestrationV2Thread = Effect.fn("ws.orchestrationV2.subs return yield* snapshotThenLive(); } const replay = coalesceLiveUpdatedEvents(replayed); + if ( + replay.some( + (item) => item.kind === "event" && invalidatesInheritedRows(item.event, input.threadId), + ) + ) { + return yield* snapshotThenLive(); + } const plan = decideThreadResume({ afterSequence: input.afterSequence, highWater, diff --git a/packages/client-runtime/src/state/orchestrationV2Projection.test.ts b/packages/client-runtime/src/state/orchestrationV2Projection.test.ts index 16301157a6a4e..76e48253e35b5 100644 --- a/packages/client-runtime/src/state/orchestrationV2Projection.test.ts +++ b/packages/client-runtime/src/state/orchestrationV2Projection.test.ts @@ -583,7 +583,7 @@ describe("applyOrchestrationV2ProjectionEvent", () => { expect(visibleIds(apply(bounded, [failed], partial))).toEqual([reply.id]); }); - it("removes only hidden local items while preserving inherited rows", () => { + it("removes only hidden or trimmed local items while preserving inherited rows", () => { const inherited = commandItem("item-inherited"); const local = commandItem("item-local"); const inheritedRow = { @@ -618,6 +618,25 @@ describe("applyOrchestrationV2ProjectionEvent", () => { const next = applyOrchestrationV2ProjectionEvent(projection, event); expect(next?.visibleTurnItems).toEqual([inheritedRow]); expect(next?.visibleTurnItems[0]).toBe(inheritedRow); + + const trimmed = applyOrchestrationV2ProjectionEvent(projection, { + id: "event-history-trimmed", + type: "thread.history-trimmed", + threadId, + occurredAt: now, + payload: { sourceThreadId: threadId, turnItemIds: [local.id, inherited.id], nodeIds: [] }, + } as unknown as OrchestrationV2DomainEvent); + expect(trimmed?.turnItems).toEqual([]); + expect(trimmed?.visibleTurnItems).toEqual([inheritedRow]); + // A fork's copy names no rows; the server sends a fresh snapshot in its place. + const forkCopy = applyOrchestrationV2ProjectionEvent(projection, { + id: "event-source-trimmed", + type: "thread.history-trimmed", + threadId, + occurredAt: now, + payload: { sourceThreadId: inheritedRow.sourceThreadId, turnItemIds: [], nodeIds: [] }, + } as unknown as OrchestrationV2DomainEvent); + expect(forkCopy).toBe(projection); }); }); diff --git a/packages/client-runtime/src/state/orchestrationV2Projection.ts b/packages/client-runtime/src/state/orchestrationV2Projection.ts index 37da408a1c35b..7e0bcaebd5d39 100644 --- a/packages/client-runtime/src/state/orchestrationV2Projection.ts +++ b/packages/client-runtime/src/state/orchestrationV2Projection.ts @@ -330,6 +330,22 @@ export function applyOrchestrationV2ProjectionEvent( ...projection, thread: applyOrchestrationV2ThreadVisit(projection.thread, event.payload), }; + case "thread.history-trimmed": { + // A fork's copy names no rows: the server answers it with a fresh snapshot. + if (event.payload.sourceThreadId !== projection.thread.id) return projection; + const items = new Set(event.payload.turnItemIds); + const nodes = new Set(event.payload.nodeIds); + return { + ...projection, + turnItems: projection.turnItems.filter((item) => !items.has(item.id)), + nodes: projection.nodes.filter((node) => !nodes.has(node.id)), + visibleTurnItems: renumberVisibleItems( + projection.visibleTurnItems.filter( + (row) => row.sourceThreadId !== projection.thread.id || !items.has(row.sourceItemId), + ), + ), + }; + } case "run.created": case "run.updated": { const next = { ...base, runs: upsertEntity(base.runs, event.payload) }; diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index 765db9d8a5fb1..946d1ec9e9588 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -1584,6 +1584,16 @@ export const OrchestrationV2CheckpointRollbackRequest = Schema.Struct({ export type OrchestrationV2CheckpointRollbackRequest = typeof OrchestrationV2CheckpointRollbackRequest.Type; +/** + * Rows a history trim removed from `sourceThreadId`. A fork that shows them gets a copy on its own + * stream that names no rows; subscribers read a fresh snapshot in its place. + */ +const OrchestrationV2ThreadHistoryTrimmed = Schema.Struct({ + sourceThreadId: ThreadId, + turnItemIds: Schema.Array(TurnItemId), + nodeIds: Schema.Array(NodeId), +}); + export class OrchestrationV2CheckpointUnavailableError extends Schema.TaggedError()( "OrchestrationV2CheckpointUnavailableError", { @@ -2182,6 +2192,11 @@ export const OrchestrationV2DomainEvent = Schema.Union([ type: Schema.Literal("checkpoint.rollback-requested"), payload: OrchestrationV2CheckpointRollbackRequest, }), + Schema.Struct({ + ...OrchestrationV2EventBase.fields, + type: Schema.Literal("thread.history-trimmed"), + payload: OrchestrationV2ThreadHistoryTrimmed, + }), Schema.Struct({ ...OrchestrationV2EventBase.fields, type: Schema.Literal("context-handoff.updated"), @@ -3130,6 +3145,11 @@ export const OrchestrationV2DomainEventJson = Schema.Union([ type: Schema.Literal("checkpoint.rollback-requested"), payload: OrchestrationV2CheckpointRollbackRequestJson, }), + Schema.Struct({ + ...OrchestrationV2JsonEventBaseFields, + type: Schema.Literal("thread.history-trimmed"), + payload: OrchestrationV2ThreadHistoryTrimmed, + }), Schema.Struct({ ...OrchestrationV2JsonEventBaseFields, type: Schema.Literal("context-handoff.updated"),