fix(file-search): allow larger concurrent search bursts

Author: icecrasher321

apps/sim/lib/workspace-files/search/README.md

@@ -20,7 +20,7 @@ Search joins the current file revision and resolved workspace/folder scope. A re
 
 For regular chunks, PostgreSQL checks the pattern with newline-aware semantics, then verifies individual logical lines. Long-line fragments use only necessary three-character literals as a conservative prefilter, including all required alternation branches. Two-code-point overlap preserves those literals at every boundary. PostgreSQL reconstructs the complete candidate line and evaluates the original regex, so anchors, word boundaries, repetitions, and arbitrarily long match spans retain line semantics. Fixed overlap alone is never treated as proof of a match. The supported regex grammar and minimum literal requirement are unchanged.
 
-Regular blocks are verified in batches of at most 16 (128 KiB of indexed text); long lines are reconstructed one at a time. Only bounded match-centered previews leave PostgreSQL: at most 201 rows to detect truncation, and at most 2 KiB per rendered result. A single search has a ten-second application deadline with per-statement guards. PostgreSQL 17 additionally enforces a total transaction timeout; PostgreSQL 16 uses the compatible idle-transaction guard. Transaction advisory locks admit at most two simultaneous searches per workspace and ten globally per database. Busy and timed-out searches fail explicitly; they never report an incomplete scan as an authoritative empty result. The reader uses the normal application database connection, so admission is coordinated on the same database as the index.
+Regular blocks are verified in batches of at most 16 (128 KiB of indexed text); long lines are reconstructed one at a time. Only bounded match-centered previews leave PostgreSQL: at most 201 rows to detect truncation, and at most 2 KiB per rendered result. A single search has a ten-second application deadline with per-statement guards. PostgreSQL 17 additionally enforces a total transaction timeout; PostgreSQL 16 uses the compatible idle-transaction guard. Transaction advisory locks admit at most 20 simultaneous searches per workspace and 5,000 globally per database. The application connection pool queues requests when its connections are occupied. These admission ceilings are operational safeguards, not throughput guarantees. Busy and timed-out searches fail explicitly; they never report an incomplete scan as an authoritative empty result. The reader uses the normal application database connection, so admission is coordinated on the same database as the index.
 
 Arbitrary regex cannot have a fixed latency guarantee. Common terms, broad alternatives, and punctuation-only literals may require scanning significant scoped text. Larger capacity decisions need representative query plans and workload measurements; neither a per-file byte cap nor a PostgreSQL row-count claim establishes a total corpus capacity.
 

apps/sim/lib/workspace-files/search/chunks.integration.ts

@@ -26,6 +26,8 @@ vi.mock('@/lib/file-parsers', () => ({ parseBuffer: vi.fn(), isSupportedFileType
 import {
   FILE_SEARCH_CLEANUP_BATCH_ROWS,
   FILE_SEARCH_CLEANUP_MAX_BATCHES,
+  FILE_SEARCH_QUERY_GLOBAL_CONCURRENCY,
+  FILE_SEARCH_QUERY_WORKSPACE_CONCURRENCY,
 } from '@/lib/workspace-files/search/constants'
 import { prepareWorkspaceFileSearchDispatch } from '@/lib/workspace-files/search/dispatcher'
 import {
@@ -429,11 +431,27 @@ describe('chunked workspace file search on PostgreSQL', () => {
     )
   })
 
-  it('releases query admission slots after a busy response', async () => {
+  it('admits a search up to the workspace and global ceilings', async () => {
     const held = await connection.reserve()
     try {
       await held`BEGIN`
-      await held`SELECT pg_advisory_xact_lock(hashtextextended('workspace-file-search-read:workspace:workspace-1:' || n::text, 0)) FROM generate_series(1, 2) n`
+      await held`SELECT pg_advisory_xact_lock(hashtextextended('workspace-file-search-read:workspace:workspace-1:' || n::text, 0)) FROM generate_series(1, ${FILE_SEARCH_QUERY_WORKSPACE_CONCURRENCY - 1}) n`
+      await held`SELECT pg_advisory_xact_lock(hashtextextended('workspace-file-search-read:global:' || n::text, 0)) FROM generate_series(1, ${FILE_SEARCH_QUERY_GLOBAL_CONCURRENCY - 1}) n`
+      expect((await search('needle')).results).toEqual([])
+    } finally {
+      await held`ROLLBACK`
+      held.release()
+    }
+  })
+
+  it.each([
+    ['workspace:workspace-1', FILE_SEARCH_QUERY_WORKSPACE_CONCURRENCY],
+    ['global', FILE_SEARCH_QUERY_GLOBAL_CONCURRENCY],
+  ])('releases query admission slots after %s saturation', async (scope, capacity) => {
+    const held = await connection.reserve()
+    try {
+      await held`BEGIN`
+      await held`SELECT pg_advisory_xact_lock(hashtextextended('workspace-file-search-read:' || ${scope} || ':' || n::text, 0)) FROM generate_series(1, ${capacity}) n`
       await expect(search('needle')).rejects.toThrow('busy')
     } finally {
       await held`ROLLBACK`

apps/sim/lib/workspace-files/search/constants.ts

@@ -23,8 +23,8 @@ export const FILE_SEARCH_MAX_PREVIEW_BYTES = 2 * 1024
 export const FILE_SEARCH_CHUNK_BYTES = 8 * 1024
 export const FILE_SEARCH_CANDIDATE_PAGE_SIZE = 16
 export const FILE_SEARCH_CANDIDATE_PROBE_SIZE = 256
-export const FILE_SEARCH_QUERY_GLOBAL_CONCURRENCY = 10
-export const FILE_SEARCH_QUERY_WORKSPACE_CONCURRENCY = 2
+export const FILE_SEARCH_QUERY_GLOBAL_CONCURRENCY = 5000
+export const FILE_SEARCH_QUERY_WORKSPACE_CONCURRENCY = 20
 export const FILE_SEARCH_CANDIDATE_LITERAL_CHARS = 3
 export const FILE_SEARCH_BUILD_LEASE_MS = 20 * 60 * 1000
 export const FILE_SEARCH_CLEANUP_BATCH_ROWS = 1000