fix(file-search): seek the backfill cursor instead of rescanning each page

The hourly backfill walks every live workspace file by `(workspace_id, id)`,
but no index supplied that order under its predicate, so each page sorted the
whole remaining set and the dispatcher's 10s statement timeout aborted the
transaction before any page committed.

Adds the matching partial index and compares the cursor row-wise. The previous
`workspace_id > :ws OR (workspace_id = :ws AND id > :id)` spelling is only ever
an index filter, never an index condition, so even with the index each page
restarted at the low end and rescanned every page before it.

On a prod-shaped fixture (5.5M files, 94k live) a full 95-page walk goes from
786ms to 42ms; the index alone accounts for 786ms -> 267ms and the row-wise
cursor for the rest.

Author: waleedlatif1

apps/sim/lib/workspace-files/search/dispatcher.integration.ts

@@ -25,6 +25,7 @@ vi.mock('@trigger.dev/sdk', () => ({ tasks: { batchTrigger: mocks.batchTrigger }
 vi.mock('@/lib/core/config/env-flags', () => ({ isTriggerDevEnabled: true }))
 vi.mock('@/lib/core/async-jobs/region', () => ({ resolveTriggerRegion: async () => 'us-east-1' }))
 
+import { FILE_SEARCH_BACKFILL_PAGE_SIZE } from '@/lib/workspace-files/search/constants'
 import {
   dispatchWorkspaceFileSearchIndexJobs,
   prepareWorkspaceFileSearchDispatch,
@@ -64,14 +65,19 @@ describe('workspace file search dispatch PostgreSQL deadlines', () => {
       deleted_at timestamp, content_updated_at timestamp NOT NULL
     )`
     await connection`CREATE TABLE workspace_file_search_revision (
-      file_id text NOT NULL, workspace_id text NOT NULL, source_content_updated_at timestamp NOT NULL,
-      status text NOT NULL, dispatched_at timestamp, updated_at timestamp NOT NULL,
-      PRIMARY KEY (file_id, source_content_updated_at)
+      file_id text PRIMARY KEY, workspace_id text NOT NULL,
+      source_content_updated_at timestamp NOT NULL, status text NOT NULL DEFAULT 'pending',
+      build_id text, failure_reason text, line_count integer NOT NULL DEFAULT 0,
+      indexed_bytes integer NOT NULL DEFAULT 0, chunk_count integer NOT NULL DEFAULT 0,
+      dispatched_at timestamp, updated_at timestamp NOT NULL DEFAULT now()
     )`
     await connection`CREATE TABLE workspace_file_search_dispatch_queue (
       workspace_id text PRIMARY KEY, enqueued_at timestamp NOT NULL,
       updated_at timestamp NOT NULL, last_dispatched_at timestamp
     )`
+    await connection`CREATE INDEX workspace_files_workspace_active_keyset_idx
+      ON workspace_files (workspace_id, id)
+      WHERE deleted_at IS NULL AND context = 'workspace' AND workspace_id IS NOT NULL`
     await connection`CREATE INDEX ON workspace_file_search_revision
       (workspace_id, updated_at, file_id, source_content_updated_at)
       WHERE status = 'pending' AND dispatched_at IS NULL`
@@ -89,7 +95,8 @@ describe('workspace file search dispatch PostgreSQL deadlines', () => {
     await connection`DROP TRIGGER IF EXISTS slow_backfill ON workspace_file_search_backfill`
     await connection`TRUNCATE workspace_files, workspace_file_search_revision, workspace_file_search_dispatch_queue`
     await connection`UPDATE workspace_file_search_backfill
-      SET updated_at = '2026-09-16 00:00:00', completed_at = NULL`
+      SET updated_at = '2026-09-16 00:00:00', completed_at = NULL,
+        after_workspace_id = NULL, after_file_id = NULL`
   })
 
   afterAll(async () => {
@@ -176,6 +183,37 @@ describe('workspace file search dispatch PostgreSQL deadlines', () => {
     expect(row.active).toBe(100)
   })
 
+  it('walks every live workspace file exactly once across backfill pages', async () => {
+    const files = 2 * FILE_SEARCH_BACKFILL_PAGE_SIZE + FILE_SEARCH_BACKFILL_PAGE_SIZE / 2
+    await connection`INSERT INTO workspace_files (id, workspace_id, context, content_updated_at)
+      SELECT md5(n::text), 'workspace-' || lpad((n % 7)::text, 2, '0'), 'workspace', '2026-09-16'
+      FROM generate_series(1, ${files}) n`
+    await connection`INSERT INTO workspace_files
+      (id, workspace_id, context, deleted_at, content_updated_at)
+      VALUES ('skipped-deleted', 'workspace-00', 'workspace', now(), '2026-09-16')`
+    await connection`INSERT INTO workspace_files (id, workspace_id, context, content_updated_at)
+      VALUES ('skipped-context', 'workspace-00', 'execution', '2026-09-16')`
+
+    let pages = 0
+    for (;;) {
+      await prepareWorkspaceFileSearchDispatch()
+      pages += 1
+      const [cursor] = await connection`SELECT completed_at FROM workspace_file_search_backfill`
+      if (cursor.completed_at) break
+      expect(pages).toBeLessThanOrEqual(files)
+    }
+
+    expect(pages).toBe(Math.ceil(files / FILE_SEARCH_BACKFILL_PAGE_SIZE))
+    const [seeded] = await connection`SELECT count(*)::int AS total,
+      count(DISTINCT file_id)::int AS distinct_files FROM workspace_file_search_revision`
+    expect(seeded).toEqual({ total: files, distinct_files: files })
+    const [skipped] = await connection`SELECT count(*)::int AS missed FROM workspace_files file
+      WHERE file.context = 'workspace' AND file.deleted_at IS NULL
+        AND NOT EXISTS (SELECT 1 FROM workspace_file_search_revision revision
+          WHERE revision.file_id = file.id)`
+    expect(skipped.missed).toBe(0)
+  }, 30_000)
+
   it('fails on a locked backfill row and releases the dispatcher lock', async () => {
     let release = () => {}
     let locked = () => {}

apps/sim/lib/workspace-files/search/dispatcher.ts

@@ -13,7 +13,6 @@ import {
   asc,
   eq,
   exists,
-  gt,
   inArray,
   isNotNull,
   isNull,
@@ -154,6 +153,15 @@ async function enqueueWorkspaces(
     })
 }
 
+/**
+ * Seeds one page of the backfill that walks every live workspace file into the revision table.
+ *
+ * The cursor is compared row-wise so the walk seeks straight to it using
+ * `workspace_files_workspace_active_keyset_idx`. The equivalent
+ * `workspace_id > :ws OR (workspace_id = :ws AND id > :id)` spelling is not something the planner
+ * can turn into an index condition: it stays a filter, so every page rescans the pages before it
+ * and the walk degrades to O(files^2) until it exceeds the dispatch statement timeout.
+ */
 async function seedBackfillPage(tx: DbTransaction, now: Date): Promise<number> {
   await tx
     .insert(workspaceFileSearchBackfill)
@@ -188,13 +196,7 @@ async function seedBackfillPage(tx: DbTransaction, now: Date): Promise<number> {
         isNull(workspaceFiles.deletedAt),
         isNotNull(workspaceFiles.workspaceId),
         afterWorkspaceId && afterFileId
-          ? or(
-              gt(workspaceFiles.workspaceId, afterWorkspaceId),
-              and(
-                eq(workspaceFiles.workspaceId, afterWorkspaceId),
-                gt(workspaceFiles.id, afterFileId)
-              )
-            )
+          ? sql`(${workspaceFiles.workspaceId}, ${workspaceFiles.id}) > (${afterWorkspaceId}, ${afterFileId})`
           : undefined
       )
     )

packages/db/migrations/0361_workspace_file_search_backfill_keyset.sql

@@ -0,0 +1,6 @@
+COMMIT;--> statement-breakpoint
+SET lock_timeout = 0;--> statement-breakpoint
+-- migration-safe: replay replaces only this new index to recover an interrupted concurrent build; existing indexes remain available.
+DROP INDEX CONCURRENTLY IF EXISTS "workspace_files_workspace_active_keyset_idx";--> statement-breakpoint
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "workspace_files_workspace_active_keyset_idx" ON "workspace_files" USING btree ("workspace_id","id") WHERE "workspace_files"."deleted_at" IS NULL AND "workspace_files"."context" = 'workspace' AND "workspace_files"."workspace_id" IS NOT NULL;--> statement-breakpoint
+SET lock_timeout = '5s';