improvement(file-search): publish complete indexes in bounded text chunks

Author: icecrasher321

apps/docs/content/docs/integrations/file.mdx

@@ -77,7 +77,7 @@ Extract the text content of workspace files selected directly, identified by can
 
 ### File Search
 
-Search the indexed text of active workspace files for lines matching a query, and return each matching line once with its file ID and line number. By default the query is a regular expression; in exact mode it is matched verbatim and metacharacters are literal. Coverage is what the index currently holds. A term that is not found is only authoritative when "complete" is true AND "indexStatus" reports no skipped or partial files; otherwise it is unknown rather than absent, so re-check before creating something on the assumption it is missing. Narrow the search with folderPaths to confine it to one or more folder trees, which also narrows "indexStatus" to those trees.
+Search the indexed text of active workspace files for lines matching a query, and return each matching line once with its file ID and line number. By default the query is a regular expression; in exact mode it is matched verbatim and metacharacters are literal. Coverage is what the index currently holds. A term that is not found is only authoritative when "complete" is true AND "indexStatus" reports no skipped files; otherwise it is unknown rather than absent, so re-check before creating something on the assumption it is missing. Narrow the search with folderPaths to confine it to one or more folder trees, which also narrows "indexStatus" to those trees.
 
 #### Input
 
@@ -99,13 +99,13 @@ Search the indexed text of active workspace files for lines matching a query, an
 |   ↳ `text` | string | Matching line or bounded match-centered preview. |
 | `count` | number | Number of returned matching lines. |
 | `truncated` | boolean | Whether more matching lines exist beyond the configured hard cap. |
-| `complete` | boolean | Whether indexing has no pending or failed current revisions; skipped and partial coverage is reported separately. |
+| `complete` | boolean | Whether indexing has no pending or failed current revisions; excluded files are reported separately. |
 | `indexStatus` | object | Current workspace search-index coverage by file status. |
-|   ↳ `readyFiles` | number | Files whose current revision is searchable. |
+|   ↳ `readyFiles` | number | Files whose entire current extracted text is searchable. |
 |   ↳ `pendingFiles` | number | Files still waiting to be indexed. |
 |   ↳ `failedFiles` | number | Files whose current indexing attempt failed. |
-|   ↳ `skippedFiles` | number | Files intentionally excluded because they are unsupported or oversized. |
-|   ↳ `partialFiles` | number | Searchable files whose extracted text was truncated by the parser or cap. |
+|   ↳ `skippedFiles` | number | Files excluded in full because they are oversized, unsupported, or cannot be completely extracted. |
+|   ↳ `partialFiles` | number | Always zero; retained for compatibility. Files are never partially indexed. |
 
 ### File Fetch
 
@@ -380,4 +380,14 @@ Move an existing workspace file into a folder. Moves the file itself; use Move F
 | `fileId` | string | The file that was moved. |
 | `folderPath` | string | The folder the file now lives in. |
 
+{/* MANUAL-CONTENT-START:search_limits */}
+## Search coverage and limits
 
+Search indexes the complete extracted text of each eligible file. The source file and its extracted UTF-8 text must each be at most **25 MiB (26,214,400 bytes)**. Oversized files, unsupported binary formats, and documents that cannot be completely extracted within parser safety limits are excluded as whole files and counted in `skippedFiles`. Search never indexes only the first rows, lines, or characters. CSV search preserves decoded source text; spreadsheet search includes populated cells beyond the preview limits. Image-only documents require searchable text; search does not perform OCR.
+
+Existing parser safeguards also apply to complete extraction. PDFs allow at most 10,000 pages, 20 MiB of extracted text, 250,000 characters on one page, and 60 seconds of extraction. Office archives allow at most 150 MiB expanded in total, 64 MiB for one archive entry, and 10,000 entries; malformed archives and excessive compression ratios are rejected. Hitting any of these limits excludes the whole file from search.
+
+Updates are indexed asynchronously. `pendingFiles` and `failedFiles` indicate revisions that are not yet searchable; a new revision becomes searchable only when its full index is ready. An empty result proves absence only within the searched scope when `complete` is true and `skippedFiles` is zero.
+
+Regex is evaluated against complete logical lines, including long lines, and cannot span line breaks. Returned lines may use a shortened preview. The result limit (up to 200 lines) and a 10-second query deadline limit an individual request, not the amount of text indexed. An expensive query fails explicitly instead of returning an apparently complete subset; narrow its literal text or folder scope and retry.
+{/* MANUAL-CONTENT-END */}

apps/sim/lib/copilot/tools/server/files/doc-compile.ts

@@ -3,6 +3,7 @@ import { createLogger } from '@sim/logger'
 import { sha256Hex } from '@sim/security/hash'
 import { DocCompileUserError } from '@/lib/copilot/tools/server/files/doc-compile-error'
 import {
+  type CompiledDocReadOptions,
   loadCompiledDoc,
   loadPublishedCompiledDoc,
   publishCompiledDocArtifact,
@@ -736,26 +737,32 @@ export async function loadCompiledDocByExt(
   workspaceId: string,
   source: string,
   ext: string,
-  options: {
+  options: CompiledDocReadOptions & {
     allowLegacyReferencedArtifact?: boolean
     allowPublishedReferencedArtifact?: boolean
     filePrincipal?: Principal
   } = {}
 ): Promise<{ buffer: Buffer; contentType: string } | null> {
   const fmt = await getE2BDocFormat(`x.${ext}`)
   if (!fmt) return null
+  const readOptions: CompiledDocReadOptions = { maxBytes: options.maxBytes, signal: options.signal }
   const referencedFileIds = collectReferencedFileIds(source)
   if (!options.filePrincipal) {
     if (referencedFileIds.size === 0) {
-      const buffer = await loadCompiledDoc(workspaceId, source, fmt.ext)
+      const buffer = await loadCompiledDoc(workspaceId, source, fmt.ext, undefined, readOptions)
       return buffer ? { buffer, contentType: fmt.contentType } : null
     }
     if (options.allowPublishedReferencedArtifact) {
-      const publishedBuffer = await loadPublishedCompiledDoc(workspaceId, source, fmt.ext)
+      const publishedBuffer = await loadPublishedCompiledDoc(
+        workspaceId,
+        source,
+        fmt.ext,
+        readOptions
+      )
       if (publishedBuffer) return { buffer: publishedBuffer, contentType: fmt.contentType }
     }
     if (!options.allowLegacyReferencedArtifact) return null
-    const legacyBuffer = await loadCompiledDoc(workspaceId, source, fmt.ext)
+    const legacyBuffer = await loadCompiledDoc(workspaceId, source, fmt.ext, undefined, readOptions)
     return legacyBuffer ? { buffer: legacyBuffer, contentType: fmt.contentType } : null
   }
   const referencedImages = await resolveReferencedImages(
@@ -768,11 +775,12 @@ export async function loadCompiledDocByExt(
     workspaceId,
     source,
     fmt.ext,
-    referencedImages.artifactIdentity
+    referencedImages.artifactIdentity,
+    readOptions
   )
   if (buffer) return { buffer, contentType: fmt.contentType }
   if (referencedImages.artifactIdentity && options.allowLegacyReferencedArtifact) {
-    const legacyBuffer = await loadCompiledDoc(workspaceId, source, fmt.ext)
+    const legacyBuffer = await loadCompiledDoc(workspaceId, source, fmt.ext, undefined, readOptions)
     if (legacyBuffer) return { buffer: legacyBuffer, contentType: fmt.contentType }
   }
   return null
@@ -799,7 +807,8 @@ export type ServableDoc =
 export async function resolveServableDoc(
   workspaceId: string,
   storedBytes: Buffer,
-  fileName: string
+  fileName: string,
+  options: CompiledDocReadOptions = {}
 ): Promise<ServableDoc> {
   const fmt = await getE2BDocFormat(fileName)
   if (!fmt) return { kind: 'passthrough' }
@@ -810,7 +819,7 @@ export async function resolveServableDoc(
       workspaceId,
       storedBytes.toString('utf-8'),
       fmt.ext,
-      { allowLegacyReferencedArtifact: true, allowPublishedReferencedArtifact: true }
+      { ...options, allowLegacyReferencedArtifact: true, allowPublishedReferencedArtifact: true }
     )
     return artifact ? { kind: 'artifact', ...artifact } : { kind: 'unavailable' }
   } catch (error) {

apps/sim/lib/copilot/tools/server/files/doc-compiled-store.test.ts

@@ -16,6 +16,7 @@ vi.mock('@/lib/uploads/core/storage-service', () => ({
 }))
 
 import {
+  loadCompiledDoc,
   loadPublishedCompiledDoc,
   storeCompiledDoc,
 } from '@/lib/copilot/tools/server/files/doc-compiled-store'
@@ -105,6 +106,35 @@ describe('compiled document publication', () => {
     )
   })
 
+  it('applies the caller budget and cancellation to both pointer and artifact downloads', async () => {
+    const signal = new AbortController().signal
+    const maxBytes = 25 * 1024 * 1024
+    mockHeadObject.mockResolvedValue({ size: 1 })
+    mockDownloadFile
+      .mockResolvedValueOnce(
+        Buffer.from(JSON.stringify({ version: 1, referencedInputIdentity: 'x'.repeat(8192) }))
+      )
+      .mockResolvedValueOnce(Buffer.from('%PDF-artifact'))
+
+    await loadPublishedCompiledDoc('workspace-1', 'source', 'pdf', { maxBytes, signal })
+
+    expect(mockDownloadFile).toHaveBeenCalledTimes(2)
+    for (const [options] of mockDownloadFile.mock.calls) {
+      expect(options).toMatchObject({ maxBytes, signal })
+    }
+  })
+
+  it('does not turn an interrupted artifact download into a cache miss', async () => {
+    const controller = new AbortController()
+    mockDownloadFile.mockImplementationOnce(async () => {
+      controller.abort()
+      throw new Error('download interrupted')
+    })
+    await expect(
+      loadCompiledDoc('workspace-1', 'source', 'pdf', undefined, { signal: controller.signal })
+    ).rejects.toMatchObject({ name: 'AbortError' })
+  })
+
   it('still reports a missing artifact as not yet built', async () => {
     mockHeadObject.mockResolvedValue({ size: 1 })
     mockDownloadFile.mockResolvedValueOnce(

apps/sim/lib/copilot/tools/server/files/doc-compiled-store.ts

@@ -37,15 +37,32 @@ function publishedArtifactPointerKey(workspaceId: string, source: string, ext: s
   return `copilot-doc-compiled/${workspaceId}/${sourceHash}.${ext}.published.json`
 }
 
+export interface CompiledDocReadOptions {
+  maxBytes?: number
+  signal?: AbortSignal
+}
+
 interface PublishedArtifactPointer {
   version: 1
   referencedInputIdentity: string
 }
 
-async function loadPublishedArtifactPointer(key: string): Promise<PublishedArtifactPointer | null> {
+async function loadPublishedArtifactPointer(
+  key: string,
+  options: CompiledDocReadOptions = {}
+): Promise<PublishedArtifactPointer | null> {
+  options.signal?.throwIfAborted()
   const stored = await headObject(key, 'copilot')
   if (!stored) return null
-  const encoded = await downloadFile({ key, context: 'copilot' })
+  const encoded = await downloadFile({
+    key,
+    context: 'copilot',
+    maxBytes: Math.min(
+      options.maxBytes ?? MAX_BUFFERED_TRANSFER_BYTES,
+      MAX_BUFFERED_TRANSFER_BYTES
+    ),
+    signal: options.signal,
+  })
 
   let decoded: unknown
   try {
@@ -75,11 +92,8 @@ async function loadPublishedArtifactPointer(key: string): Promise<PublishedArtif
  * about the size of this. Bounding it here rather than on the finished response is
  * what keeps an oversized artifact from being materialized before it is refused.
  *
- * The bound is the WIDEST ceiling any consumer of this funnel allows, because it is a
- * memory backstop and not a policy: a consumer that permits less enforces its own
- * limit on what it got back (the workspace download path holds artifacts to
- * `MAX_RENDERED_DOCUMENT_BYTES`, half of this). Using the tighter figure here instead
- * would reject artifacts the serving routes are willing to return.
+ * The default is the widest ceiling consumers allow. Callers can tighten it before
+ * downloading, so indexing does not materialize an artifact it will immediately reject.
  *
  * A size breach is rethrown rather than folded into `null`: null means "not built
  * yet", which callers answer with "still being prepared, try again", and an artifact
@@ -89,12 +103,22 @@ export async function loadCompiledDoc(
   workspaceId: string,
   source: string,
   ext: string,
-  referencedInputIdentity?: string
+  referencedInputIdentity?: string,
+  options: CompiledDocReadOptions = {}
 ): Promise<Buffer | null> {
   const key = compiledArtifactKey(workspaceId, source, ext, referencedInputIdentity)
   try {
-    return await downloadFile({ key, context: 'copilot', maxBytes: MAX_BUFFERED_TRANSFER_BYTES })
+    return await downloadFile({
+      key,
+      context: 'copilot',
+      maxBytes: Math.min(
+        options.maxBytes ?? MAX_BUFFERED_TRANSFER_BYTES,
+        MAX_BUFFERED_TRANSFER_BYTES
+      ),
+      signal: options.signal,
+    })
   } catch (error) {
+    options.signal?.throwIfAborted()
     if (isPayloadSizeLimitError(error)) throw error
     return null
   }
@@ -140,12 +164,19 @@ export async function publishCompiledDocArtifact(
 export async function loadPublishedCompiledDoc(
   workspaceId: string,
   source: string,
-  ext: string
+  ext: string,
+  options: CompiledDocReadOptions = {}
 ): Promise<Buffer | null> {
   const key = publishedArtifactPointerKey(workspaceId, source, ext)
-  const pointer = await loadPublishedArtifactPointer(key)
+  const pointer = await loadPublishedArtifactPointer(key, options)
   if (!pointer) return null
-  const artifact = await loadCompiledDoc(workspaceId, source, ext, pointer.referencedInputIdentity)
+  const artifact = await loadCompiledDoc(
+    workspaceId,
+    source,
+    ext,
+    pointer.referencedInputIdentity,
+    options
+  )
   if (!artifact) throw new Error(`Published compiled document artifact is missing: ${key}`)
   return artifact
 }

apps/sim/lib/copilot/tools/server/files/doc-servable.test.ts

@@ -96,7 +96,9 @@ describe('resolveServableDocBytes', () => {
     expect(mockLoadCompiledDoc).toHaveBeenCalledWith(
       WORKSPACE_ID,
       PDF_SOURCE.toString('utf-8'),
-      'pdf'
+      'pdf',
+      undefined,
+      { maxBytes: undefined, signal: undefined }
     )
     expect(mockLoadCompiledDoc).toHaveBeenCalledTimes(1)
   })
@@ -123,7 +125,9 @@ describe('resolveServableDocBytes', () => {
     expect(mockLoadCompiledDoc).toHaveBeenCalledWith(
       WORKSPACE_ID,
       PDF_SOURCE.toString('utf-8'),
-      'pdf'
+      'pdf',
+      undefined,
+      { maxBytes: undefined, signal: undefined }
     )
   })
 
@@ -202,7 +206,13 @@ describe('resolveServableDocBytes', () => {
       buffer: legacyArtifact,
       contentType: 'application/pdf',
     })
-    expect(mockLoadCompiledDoc).toHaveBeenCalledWith(WORKSPACE_ID, source.toString('utf-8'), 'pdf')
+    expect(mockLoadCompiledDoc).toHaveBeenCalledWith(
+      WORKSPACE_ID,
+      source.toString('utf-8'),
+      'pdf',
+      undefined,
+      { maxBytes: undefined, signal: undefined }
+    )
     expect(mockReadWorkspaceFileMetadata).not.toHaveBeenCalled()
     expect(mockExecuteInSandbox).not.toHaveBeenCalled()
     expect(mockStoreCompiledDoc).not.toHaveBeenCalled()
@@ -221,7 +231,8 @@ describe('resolveServableDocBytes', () => {
     expect(mockLoadPublishedCompiledDoc).toHaveBeenCalledWith(
       WORKSPACE_ID,
       source.toString('utf-8'),
-      'pdf'
+      'pdf',
+      { maxBytes: undefined, signal: undefined }
     )
     expect(mockLoadCompiledDoc).not.toHaveBeenCalled()
     expect(mockReadWorkspaceFileMetadata).not.toHaveBeenCalled()
@@ -242,7 +253,8 @@ describe('resolveServableDocBytes', () => {
     expect(mockLoadPublishedCompiledDoc).toHaveBeenCalledWith(
       WORKSPACE_ID,
       source.toString('utf-8'),
-      'pdf'
+      'pdf',
+      { maxBytes: undefined, signal: undefined }
     )
     expect(mockLoadCompiledDoc).not.toHaveBeenCalled()
   })
@@ -307,7 +319,13 @@ describe('resolveServableDocBytes', () => {
       contentType: 'application/pdf',
     })
     expect(mockReadWorkspaceFileMetadata).not.toHaveBeenCalled()
-    expect(mockLoadCompiledDoc).toHaveBeenCalledWith(WORKSPACE_ID, source.toString('utf-8'), 'pdf')
+    expect(mockLoadCompiledDoc).toHaveBeenCalledWith(
+      WORKSPACE_ID,
+      source.toString('utf-8'),
+      'pdf',
+      undefined,
+      { maxBytes: undefined, signal: undefined }
+    )
     expect(mockExecuteInSandbox).not.toHaveBeenCalled()
   })