fix(file-search): skip base64-dominated files and redact query errors from task failures (#8004)

* fix(file-search): skip base64-dominated files and redact query errors from task failures

* fix(file-search): count wrapped base64 blocks and verify trigram estimate against pg_trgm

Author: waleedlatif1

.github/workflows/test-build.yml

@@ -149,6 +149,12 @@ jobs:
           KNOWLEDGE_ACL_TEST_DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/sim_auth_scim
         run: bunx vitest run --mode integration lib/workspace-files/search/dispatcher.integration.ts
 
+      - name: Verify file search trigram estimate against pg_trgm
+        working-directory: apps/sim
+        env:
+          KNOWLEDGE_ACL_TEST_DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/sim_auth_scim
+        run: bunx vitest run --mode integration lib/workspace-files/search/index-plan.integration.ts
+
       - name: Verify SCIM and administration over real HTTP
         working-directory: apps/sim
         env:

apps/sim/lib/core/errors/database-query-error.test.ts

@@ -0,0 +1,37 @@
+/**
+ * @vitest-environment node
+ */
+import { DrizzleQueryError } from 'drizzle-orm/errors'
+import { describe, expect, it } from 'vitest'
+import { redactDatabaseQueryError } from '@/lib/core/errors/database-query-error'
+
+const BOUND_VALUE = 'confidential user text'
+
+function queryError(code: string, message: string): DrizzleQueryError {
+  return new DrizzleQueryError(
+    'insert into "t" values ($1)',
+    [BOUND_VALUE],
+    Object.assign(new Error(message), { code })
+  )
+}
+
+describe('redactDatabaseQueryError', () => {
+  it('reports a query failure by code and known cancellation reason only', () => {
+    const original = queryError('57014', 'canceling statement due to statement timeout')
+    const redacted = redactDatabaseQueryError(new Error('wrapper', { cause: original }), 'Insert')
+    expect(redacted).toBeInstanceOf(Error)
+    expect((redacted as Error).message).toBe('Insert failed (57014, statement_timeout)')
+    expect((redacted as Error).stack).not.toContain(BOUND_VALUE)
+  })
+  it('omits an unrecognized reason', () => {
+    const redacted = redactDatabaseQueryError(
+      queryError('23505', `duplicate key value ${BOUND_VALUE}`),
+      'Insert'
+    )
+    expect((redacted as Error).message).toBe('Insert failed (23505)')
+  })
+  it('returns errors without a query unchanged', () => {
+    const error = new Error('storage unavailable')
+    expect(redactDatabaseQueryError(error, 'Insert')).toBe(error)
+  })
+})

apps/sim/lib/core/errors/database-query-error.ts

@@ -1,4 +1,4 @@
-import { findCause } from '@sim/utils/errors'
+import { findCause, getPostgresCancellationReason, getPostgresErrorCode } from '@sim/utils/errors'
 import { DrizzleQueryError } from 'drizzle-orm/errors'
 
 /**
@@ -8,3 +8,18 @@ import { DrizzleQueryError } from 'drizzle-orm/errors'
 export function findDatabaseQueryError(error: unknown): DrizzleQueryError | undefined {
   return findCause(error, (cause): cause is DrizzleQueryError => cause instanceof DrizzleQueryError)
 }
+
+/**
+ * Replaces a Drizzle query failure with an error naming only its database code, for surfaces such as
+ * task runners that record a thrown error's message and stack verbatim. The original stays in
+ * `cause` for in-process diagnostics. Errors without a query are returned unchanged.
+ */
+export function redactDatabaseQueryError(error: unknown, operation: string): unknown {
+  const queryError = findDatabaseQueryError(error)
+  if (!queryError) return error
+  const code = getPostgresErrorCode(queryError) ?? 'no error code'
+  const reason = getPostgresCancellationReason(queryError)
+  return new Error(`${operation} failed (${reason ? `${code}, ${reason}` : code})`, {
+    cause: error,
+  })
+}

apps/sim/lib/workspace-files/search/README.md

@@ -1,6 +1,6 @@
 # Workspace file search
 
-PostgreSQL stores complete extracted text in bounded chunks. Object storage remains the source of truth. A source and its extracted UTF-8 text must each fit within 25 MiB. Unsupported, degraded, oversized, or partially extracted files are excluded in full. There is no row-count or line-count coverage limit.
+PostgreSQL stores complete extracted text in bounded chunks. Object storage remains the source of truth. A source and its extracted UTF-8 text must each fit within 25 MiB. Unsupported, degraded, oversized, or partially extracted files are excluded in full. So is text that is mostly an encoded payload: when unbroken base64-alphabet runs of at least 256 characters that mix upper case, lower case, and digits make up half the text and more than 32 KiB, as with binaries stored as base64 text or SVGs embedding data URIs, the file is skipped as `encoded_content`. A run that fills whole lines at least 60 characters wide continues across line breaks, so base64 wrapped at the usual 64 or 76 columns counts as one payload. Smaller encoded values, such as a signature in a config file or lockfile integrity hashes, stay searchable, as do single-case runs such as hex digests and DNA sequences. There is no row-count or line-count coverage limit.
 
 ## Storage and publication
 
@@ -12,7 +12,7 @@ PostgreSQL stores complete extracted text in bounded chunks. Object storage rema
 
 Workers download and extract outside database transactions, then insert batches of at most 250 rows / 128 KiB. Each batch checks the build token and lease. Publication locks the canonical file, build, and revision in that order, verifies the stored chunk count, and changes the visible pointer only after every batch succeeds. Old dispatch failure callbacks cannot overwrite newer dispatches or successful builds.
 
-The chunk GIN index uses `fastupdate = off`. Each bounded insert updates the main index directly instead of appending to a shared pending list. With deferred updates enabled, even a small insert can cross the pending-list threshold and synchronously merge accumulated work from other files. Direct updates trade some bulk-write throughput for avoiding that foreground cleanup cliff. They do not eliminate normal index I/O, vacuum, or storage contention; the row and worker limits still apply. The 128 KiB batch budget bounds direct index work per transaction without changing the 25 MiB file coverage limit. Dense text with many distinct trigrams and an index working set larger than the available cache can still exceed the statement deadline. Capacity validation must include that cache pressure, not only a small corpus or a row count.
+The chunk GIN index uses `fastupdate = off`. Each bounded insert updates the main index directly instead of appending to a shared pending list. With deferred updates enabled, even a small insert can cross the pending-list threshold and synchronously merge accumulated work from other files. Direct updates trade some bulk-write throughput for avoiding that foreground cleanup cliff. They do not eliminate normal index I/O, vacuum, or storage contention; the row and worker limits still apply. The 128 KiB batch budget bounds direct index work per transaction without changing the 25 MiB file coverage limit. Dense text with many distinct trigrams and an index working set larger than the available cache can still exceed the statement deadline. Capacity validation must include that cache pressure, not only a small corpus or a row count. A batch that takes at least two seconds, including one a statement timeout cancels, is logged with its rows, bytes, and estimated trigram key count (pg_trgm's extraction, reproduced exactly for the database's `en_US.UTF-8` ctype), so a key-based batch budget can be sized from production timings. A failed query is reported to the task runner by error code only; Drizzle's message carries the bound file text.
 
 Indexing transactions have separate limits from search: ten seconds per statement, five seconds waiting for a lock, and thirty seconds total on PostgreSQL 17. The outer limit leaves time for ordinary statement cancellation and rollback instead of terminating the connection at the same ten-second deadline. PostgreSQL 16 uses the compatible idle-transaction guard. A canceled batch remains unpublished; the existing task retry starts a fresh fenced build, and cleanup retires the previous attempt. This does not automatically retry revisions already marked failed.
 

apps/sim/lib/workspace-files/search/constants.ts

@@ -19,6 +19,21 @@ export const FILE_SEARCH_MAX_RESULTS = 200
 
 export const FILE_SEARCH_MAX_SOURCE_BYTES = 25 * 1024 * 1024
 export const FILE_SEARCH_MAX_EXTRACTED_BYTES = 25 * 1024 * 1024
+/**
+ * Unbroken base64-alphabet runs at least this long are encoded payloads (data URIs, binaries
+ * stored as base64 text), not searchable text. Each chunk of one yields thousands of distinct
+ * rare trigrams, which makes its direct GIN insert far slower than ordinary text.
+ */
+export const FILE_SEARCH_ENCODED_RUN_MIN_CHARS = 256
+/**
+ * A run continues across a line break when it fills a line at least this wide, so base64 wrapped
+ * at the usual 64 or 76 columns (MIME, PEM, Python's `encodebytes`) counts as one payload.
+ */
+export const FILE_SEARCH_ENCODED_WRAP_MIN_CHARS = 60
+/** Text is excluded when encoded runs are at least this share of it... */
+export const FILE_SEARCH_ENCODED_EXCLUSION_RATIO = 0.5
+/** ...and span more than a few chunks, so a small config carrying one signature stays searchable. */
+export const FILE_SEARCH_ENCODED_EXCLUSION_MIN_BYTES = 32 * 1024
 export const FILE_SEARCH_MAX_PREVIEW_BYTES = 2 * 1024
 export const FILE_SEARCH_CHUNK_BYTES = 8 * 1024
 export const FILE_SEARCH_CANDIDATE_PAGE_SIZE = 16
@@ -48,6 +63,8 @@ export const FILE_SEARCH_RECONCILE_INTERVAL_MS = 60 * 60 * 1000
 export const FILE_SEARCH_INSERT_BATCH_ROWS = 250
 /** Direct GIN writes perform index work in each insert, so transactions use smaller byte batches. */
 export const FILE_SEARCH_INSERT_BATCH_BYTES = 128 * 1024
+/** Batches at least this slow are logged with their estimated trigram key count. */
+export const FILE_SEARCH_SLOW_INSERT_BATCH_MS = 2000
 
 /** Index writes allow statement cancellation before the outer transaction terminates its session. */
 export const FILE_SEARCH_INDEX_TRANSACTION_LIMITS = {

apps/sim/lib/workspace-files/search/index-plan.integration.ts

@@ -0,0 +1,49 @@
+/**
+ * @vitest-environment node
+ */
+import postgres from 'postgres'
+import { afterAll, beforeAll, describe, expect, it } from 'vitest'
+import { estimateTrigramKeys } from '@/lib/workspace-files/search/index-plan'
+
+describe('trigram key estimate against pg_trgm', () => {
+  const databaseUrl = process.env.KNOWLEDGE_ACL_TEST_DATABASE_URL
+  if (!databaseUrl) throw new Error('Trigram estimate tests require a disposable local database')
+  const target = new URL(databaseUrl)
+  if (
+    !['postgres:', 'postgresql:'].includes(target.protocol) ||
+    !['localhost', '127.0.0.1'].includes(target.hostname) ||
+    (!target.pathname.startsWith('/sim_acl_test') && target.pathname !== '/sim_auth_scim')
+  ) {
+    throw new Error('File search tests require a disposable local integration database')
+  }
+  const connection = postgres(databaseUrl, { max: 1, onnotice: () => {} })
+
+  beforeAll(async () => {
+    await connection`CREATE EXTENSION IF NOT EXISTS pg_trgm`
+  })
+
+  afterAll(async () => {
+    await connection.end()
+  })
+
+  it('runs where the estimate is specified', async () => {
+    const [{ ctype }] = await connection<{ ctype: string }[]>`
+      SELECT datctype AS ctype FROM pg_database WHERE datname = current_database()`
+    expect(ctype).toMatch(/^en_US\.utf-?8$/i)
+  })
+
+  it.each([
+    ['prose', 'The quick brown fox jumps over the lazy dog. The dog sleeps.'],
+    ['identifiers', 'snake_case CamelCase kebab-case 123abc x86_64 v2.1.0'],
+    ['a lockfile hash', '"sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd=="'],
+    ['accented text', 'Café naïve ÉCOLE über straße'],
+    ['CJK text', '日本語のテキスト検索 テスト'],
+    ['Greek text', 'Ελληνικά κείμενα ΑΒΓ'],
+    ['emoji', 'emoji 🙂🙂 party 🎉 done'],
+    ['punctuation only', '--- ___ ... !!!'],
+  ])('matches show_trgm for %s', async (_, text) => {
+    const [{ keys }] = await connection<{ keys: number }[]>`
+      SELECT coalesce(array_length(show_trgm(${text}), 1), 0)::int AS keys`
+    expect(estimateTrigramKeys(text)).toBe(keys)
+  })
+})

apps/sim/lib/workspace-files/search/index-plan.test.ts

@@ -1,5 +1,6 @@
 import { describe, expect, it } from 'vitest'
 import {
+  estimateTrigramKeys,
   iterateFileSearchChunks,
   planFileSearchIndex,
 } from '@/lib/workspace-files/search/index-plan'
@@ -56,6 +57,52 @@ describe('file search chunk packing', () => {
       planFileSearchIndex({ text: 'x'.repeat(25 * 1024 * 1024 + 1), partial: false }, signal)
     ).toThrow('extracted_text_too_large')
   })
+  it.each([
+    ['a binary stored as base64 text', 'iVBORw0KGgo'.repeat(10_000)],
+    [
+      'an SVG embedding a data URI',
+      `<svg><title>sheet</title><image href="data:image/jpeg;base64,${'/9j/4AAQ'.repeat(8000)}"/></svg>`,
+    ],
+    ['space-separated runs at the minimum run length', `${'aB3'.repeat(86)} `.repeat(500)],
+    ['base64 wrapped at 76 columns', `${'aB3d'.repeat(19)}\n`.repeat(2000)],
+    [
+      'a PEM-style block wrapped at 64 columns with CRLF',
+      `-----BEGIN DATA-----\r\n${`${'Qk9z'.repeat(16)}\r\n`.repeat(2000)}-----END DATA-----\r\n`,
+    ],
+  ])('excludes %s before producing any chunks', (_, text) => {
+    expect(() => planFileSearchIndex({ text, partial: false }, signal)).toThrow('encoded_content')
+  })
+  it.each([
+    [
+      'a small config carrying one signature',
+      `{"$schema":"https://example.com/schema.json","signature":"${'Qk9'.repeat(200)}"}`,
+    ],
+    [
+      'a lockfile whose integrity hashes are short runs',
+      `"pkg": ["pkg@1.0.0", "", {}, "sha512-${'Ab1+'.repeat(22)}=="],\n\n`.repeat(2000),
+    ],
+    ['space-separated runs just short of the run length', `${'aB3'.repeat(85)} `.repeat(500)],
+    ['one short token per line', `${'aB3d'.repeat(10)}\n`.repeat(5000)],
+    ['lines that each end in a long hash', `checksum ${'aB3d'.repeat(19)}\n`.repeat(2000)],
+    ['an unwrapped DNA sequence', `>chr1\n${'ACGT'.repeat(20_000)}\n`],
+    ['a hex digest dump', `${'deadbeef0123'.repeat(5000)}\n`],
+    [
+      'prose that embeds an encoded payload smaller than itself',
+      `${'Quarterly results and notes. '.repeat(5000)}${'aB3d'.repeat(10_000)}`,
+    ],
+  ])('keeps %s searchable', (_, text) => {
+    expect(() => planFileSearchIndex({ text, partial: false }, signal)).not.toThrow()
+  })
+  it.each([
+    ['cat', 4],
+    ['Cat CAT cat', 4],
+    ['foo|bar', 8],
+    ['a', 2],
+    ['', 0],
+    ['--- ___ ...', 0],
+  ])('estimates pg_trgm keys for %j', (text, expected) => {
+    expect(estimateTrigramKeys(text)).toBe(expected)
+  })
   it('stops iteration when aborted', () => {
     const controller = new AbortController()
     const chunks = iterateFileSearchChunks(

apps/sim/lib/workspace-files/search/index-plan.ts

@@ -2,11 +2,79 @@ import { Buffer } from 'node:buffer'
 import {
   FILE_SEARCH_CANDIDATE_LITERAL_CHARS,
   FILE_SEARCH_CHUNK_BYTES,
+  FILE_SEARCH_ENCODED_EXCLUSION_MIN_BYTES,
+  FILE_SEARCH_ENCODED_EXCLUSION_RATIO,
+  FILE_SEARCH_ENCODED_RUN_MIN_CHARS,
+  FILE_SEARCH_ENCODED_WRAP_MIN_CHARS,
   FILE_SEARCH_MAX_EXTRACTED_BYTES,
 } from '@/lib/workspace-files/search/constants'
 import type { ExtractedIndexText } from '@/lib/workspace-files/search/extract'
 
-export type FileSearchExclusionReason = 'extracted_text_too_large' | 'incomplete_extraction'
+export type FileSearchExclusionReason =
+  | 'extracted_text_too_large'
+  | 'incomplete_extraction'
+  | 'encoded_content'
+
+const TRIGRAM_WORD = /[\p{L}\p{N}]+/gu
+
+/**
+ * Bytes inside long base64 runs, in one linear pass. A run counts only when it mixes upper case,
+ * lower case, and digits, as real base64 does; single-case runs such as hex digests or DNA
+ * sequences are searchable text with few distinct trigrams. A run that fills whole lines of at
+ * least the wrap width continues across line breaks, so wrapped base64 counts as one payload while
+ * a hash inside a line still ends with it. Runs are ASCII, so chars are bytes.
+ */
+function countEncodedBytes(text: string): number {
+  let encoded = 0
+  let run = 0
+  let runFillsLines = false
+  let lineChars = 0
+  let upper = false
+  let lower = false
+  let digit = false
+  const endRun = () => {
+    if (run >= FILE_SEARCH_ENCODED_RUN_MIN_CHARS && upper && lower && digit) encoded += run
+    run = 0
+    upper = lower = digit = false
+  }
+  for (let i = 0; i < text.length; i++) {
+    const code = text.charCodeAt(i)
+    const isUpper = code >= 65 && code <= 90
+    const isLower = code >= 97 && code <= 122
+    const isDigit = code >= 48 && code <= 57
+    if (isUpper || isLower || isDigit || code === 43 || code === 47) {
+      if (run === 0) runFillsLines = lineChars === 0
+      run++
+      lineChars++
+      upper ||= isUpper
+      lower ||= isLower
+      digit ||= isDigit
+    } else if (code === 10) {
+      if (!(run > 0 && runFillsLines && lineChars >= FILE_SEARCH_ENCODED_WRAP_MIN_CHARS)) endRun()
+      lineChars = 0
+    } else if (code !== 13 || text.charCodeAt(i + 1) !== 10) {
+      endRun()
+      lineChars++
+    }
+  }
+  endRun()
+  return encoded
+}
+
+/**
+ * Distinct keys `gin_trgm_ops` extracts from `content`, mirroring pg_trgm: lowercased alphanumeric
+ * words, each padded with two leading spaces and one trailing space. This matches PostgreSQL under
+ * the `en_US.UTF-8` ctype; its hashing of multibyte trigrams can only merge keys, so the count is
+ * never an underestimate.
+ */
+export function estimateTrigramKeys(content: string): number {
+  const keys = new Set<string>()
+  for (const [word] of content.toLowerCase().matchAll(TRIGRAM_WORD)) {
+    const padded = [' ', ' ', ...word, ' ']
+    for (let i = 2; i < padded.length; i++) keys.add(padded[i - 2] + padded[i - 1] + padded[i])
+  }
+  return keys.size
+}
 
 export class FileSearchExclusionError extends Error {
   constructor(readonly reason: FileSearchExclusionReason) {
@@ -30,16 +98,27 @@ export interface FileSearchIndexPlan {
   indexedBytes: number
 }
 
-/** Admission happens before any chunks are written; incomplete extraction never becomes searchable. */
+/**
+ * Admission happens before any chunks are written; incomplete extraction never becomes searchable,
+ * and neither does text that is mostly an encoded payload.
+ */
 export function planFileSearchIndex(
   extracted: ExtractedIndexText,
   signal: AbortSignal
 ): FileSearchIndexPlan {
   signal.throwIfAborted()
   if (extracted.partial) throw new FileSearchExclusionError('incomplete_extraction')
-  if (Buffer.byteLength(extracted.text, 'utf8') > FILE_SEARCH_MAX_EXTRACTED_BYTES) {
+  const textBytes = Buffer.byteLength(extracted.text, 'utf8')
+  if (textBytes > FILE_SEARCH_MAX_EXTRACTED_BYTES) {
     throw new FileSearchExclusionError('extracted_text_too_large')
   }
+  const encodedBytes = countEncodedBytes(extracted.text)
+  if (
+    encodedBytes > FILE_SEARCH_ENCODED_EXCLUSION_MIN_BYTES &&
+    encodedBytes >= textBytes * FILE_SEARCH_ENCODED_EXCLUSION_RATIO
+  ) {
+    throw new FileSearchExclusionError('encoded_content')
+  }
   const bytes = Buffer.from(extracted.text.replace(/\r(?=\n|$)/g, ''), 'utf8')
   let lineCount = 1
   for (const byte of bytes) if (byte === 10) lineCount++