improvement(knowledge): rank each readable source in its own vector index

pgvector post-filters, so a walk over every source spends its scan budget on the
sources a caller cannot read and returns few of their true neighbours: measured
recall for a member reading half an organization index ranged from 0.00 to 1.00,
averaging 0.80 across nine queries, with two queries returning nothing of the
exact page.

Readability follows sources, so each one is ranked on its own terms. A member of
a source reads essentially all of it and is ranked by walking that source's index,
built after a sync grows it past the threshold and dropped with the connector.
Every other source is sliced — mirrored permissions give a caller their own mail,
their own files — and those slices are ranked exactly in one statement, which is
cheaper than a walk and exact by construction. The lists merge by distance.

Recall over the same nine queries rises to 0.95 with no query below 0.75, at 398ms
of database time against 95ms for the walk it replaces. Retrieval never waits on an
index existing: a source without one is ranked exactly.

Author: waleedlatif1

apps/sim/lib/knowledge/connectors/sync-engine.ts

@@ -6,7 +6,7 @@ import {
   knowledgeConnectorSyncLog,
 } from '@sim/db/schema'
 import { createLogger } from '@sim/logger'
-import { toError } from '@sim/utils/errors'
+import { getErrorMessage, toError } from '@sim/utils/errors'
 import { generateId } from '@sim/utils/id'
 import { randomInt } from '@sim/utils/random'
 import { and, asc, eq, exists, gt, inArray, isNotNull, isNull, or, sql } from 'drizzle-orm'
@@ -85,6 +85,7 @@ import {
 } from '@/lib/knowledge/connectors/sync-primitives'
 import { hardDeleteDocuments } from '@/lib/knowledge/documents/service'
 import { getRetryAfterMs, isRateLimitError } from '@/lib/knowledge/documents/utils'
+import { ensureSourceVectorIndex } from '@/lib/knowledge/search/source-vector-indexes'
 import { CONNECTOR_REGISTRY } from '@/connectors/registry.server'
 import type {
   ConnectorAuthConfig,
@@ -391,7 +392,7 @@ export async function completeSuccessfulSync(
   const completionNotice =
     [directoryNotice, listingNotice, contentNotice].filter(Boolean).join('\n') || null
   try {
-    return await db.transaction(async (tx) => {
+    const completed = await db.transaction(async (tx) => {
       const [lockedKnowledgeBase] = await tx
         .select({ id: knowledgeBase.id })
         .from(knowledgeBase)
@@ -493,6 +494,18 @@ export async function completeSuccessfulSync(
 
       return true
     })
+    /**
+     * A source that has grown past the threshold gets its own vector index, so a member who reads
+     * it whole is ranked through a walk of their own documents. Retrieval ranks exactly without
+     * it, so a failure here is logged and left for the next sync.
+     */
+    await ensureSourceVectorIndex(connectorId).catch((error: unknown) => {
+      logger.warn('Could not ensure the source vector index', {
+        connectorId,
+        error: getErrorMessage(error),
+      })
+    })
+    return completed
   } catch (error) {
     if (error instanceof SyncCompletionOwnershipLost) return false
     throw error

apps/sim/lib/knowledge/orchestration/connectors.ts

@@ -9,7 +9,7 @@ import {
   knowledgeConnectorMember,
 } from '@sim/db/schema'
 import { createLogger } from '@sim/logger'
-import { getPostgresErrorCode } from '@sim/utils/errors'
+import { getErrorMessage, getPostgresErrorCode } from '@sim/utils/errors'
 import { generateId } from '@sim/utils/id'
 import { and, eq, isNull, sql } from 'drizzle-orm'
 import { encryptApiKey } from '@/lib/api-key/crypto'
@@ -60,6 +60,7 @@ import {
   type KnowledgeOperationContext,
   type KnowledgeOrchestrationResult,
 } from '@/lib/knowledge/orchestration/shared'
+import { dropSourceVectorIndex } from '@/lib/knowledge/search/source-vector-indexes'
 import { createTagDefinition } from '@/lib/knowledge/tags/service'
 import { captureServerEvent } from '@/lib/posthog/server'
 import { searchSourceIdentity } from '@/lib/sim-search/source-identity'
@@ -1411,6 +1412,14 @@ export async function performDeleteKnowledgeConnector(
     })
   }
 
+  /** The source is gone, so its vector index is too; ranking falls back to the exact path. */
+  await dropSourceVectorIndex(connectorId).catch((error: unknown) => {
+    logger.warn('Could not drop the source vector index', {
+      connectorId,
+      error: getErrorMessage(error),
+    })
+  })
+
   return {
     success: true,
     documentsDeleted: deleteDocuments ? docCount : 0,

apps/sim/lib/knowledge/search/diagnostics.ts

@@ -29,6 +29,9 @@ export type SearchStage =
   | 'defaults'
   | 'retrieval'
   | 'connector_eligibility'
+  | 'vector.source_plan'
+  | 'vector.source_exact'
+  | 'vector.source_walk'
   | 'permitted_documents'
   | 'result_provenance'
   | 'reranking'
@@ -82,7 +85,7 @@ export interface SearchDiagnosticMetadata {
   searchMode?: 'hybrid' | 'vector'
   boostRecency?: boolean
   embeddingDimensions?: number
-  vectorRanking?: 'exact' | 'exact-candidates' | 'candidate-rerank'
+  vectorRanking?: 'exact' | 'exact-candidates' | 'candidate-rerank' | 'per-source'
   vectorCandidateStorage?: 'stored-halfvec'
   /**
    * Whether the bounded traversal filled its candidate limit. `underfilled` means visibility
@@ -102,6 +105,8 @@ export interface SearchDiagnosticMetadata {
   permittedDocuments?: 'bounded' | 'unbounded'
   /** Documents in a bounded permitted set. */
   permittedDocumentCount?: number
+  vectorSourcesSliced?: number
+  vectorSourcesWalked?: number
   /**
    * Which index ranked an unbounded keyword leg: `tin` ranks by BM25 and checks access on the top
    * of that ranking; `gin` ranks every match. Absent when the leg ranked inside a bounded set.

apps/sim/lib/knowledge/search/queries.test.ts

@@ -1432,19 +1432,27 @@ describe('permitted-document planner', () => {
 
   let probeRows: Array<{ id: string | null; connectorId: string | null; saturated: boolean }>
   let exactRows: Array<{ id: string }>
-  let traversedRows: Array<{ id: string }>
+  let traversedRows: Array<{ id: string; distance?: number }>
   let rerankRows: Array<ReturnType<typeof hit>>
+  let memberedSources: Array<{ connectorId: string }>
+  let indexedSourceRows: Array<{ name: string; connectorId: string }>
+  let sourceExactRows: Array<{ id: string; distance: number }>
 
   beforeEach(() => {
     resetDbChainMock()
     probeRows = []
     exactRows = []
     traversedRows = []
     rerankRows = []
+    sourceExactRows = []
+    memberedSources = []
+    indexedSourceRows = []
     dbChainMockFns.execute.mockImplementation(async (query) => {
       const statement = render(query).sql
+      if (statement.includes('pg_index')) return indexedSourceRows
       if (statement.includes('AS visible')) return traversedRows
       if (statement.includes('WITH scored_search_candidates')) return rerankRows
+      if (statement.includes('WITH readable_documents')) return sourceExactRows
       if (isExactRanking(statement)) return exactRows
       if (isProbeStatement(statement)) return probeRows
       return []
@@ -1485,6 +1493,55 @@ describe('permitted-document planner', () => {
     expect(sqls.some(isExactRanking)).toBe(false)
   })
 
+  it('walks a source the caller is a member of and ranks every other source exactly', async () => {
+    const eligibility = {
+      workspace: [],
+      admin: ['sliced-src'],
+      members: ['member-src'],
+      liveProofRequired: [],
+    }
+    /** Membership decides the walk; the sliced source contributes enumerated documents. */
+    memberedSources = [{ connectorId: 'member-src' }]
+    indexedSourceRows = [{ name: 'idx', connectorId: 'member-src' }]
+    queueTableRows(schemaMock.knowledgeConnectorMember, memberedSources)
+    sourceExactRows = [{ id: 'sliced-hit', distance: 0.05 }]
+    traversedRows = [{ id: 'walked-hit', distance: 0.2 }]
+    rerankRows = [hit('sliced-hit', 'sliced-src'), hit('walked-hit', 'member-src')]
+    queueTableRows(schemaMock.embedding, rerankRows)
+    await handleVectorOnlySearch({
+      ...params,
+      permitted: { kind: 'unbounded' },
+      connectorEligibility: eligibility,
+    })
+    const walks = statements().filter((query) => query.sql.includes('AS visible'))
+    expect(walks).toHaveLength(1)
+    expect(
+      walks[0].params.some((param) => JSON.stringify(param).includes('"right":"member-src"'))
+    ).toBe(true)
+    /** The sliced sources resolve their documents inside one statement, not through the app. */
+    const exact = statements().filter((query) => query.sql.includes('WITH readable_documents'))
+    expect(exact).toHaveLength(1)
+    expect(JSON.stringify(exact[0])).toContain('sliced-src')
+  })
+
+  it('ranks every source exactly when the caller is a member of none', async () => {
+    memberedSources = []
+    sourceExactRows = [{ id: 'sliced-hit', distance: 0.05 }]
+    rerankRows = [hit('sliced-hit', 'sliced-src')]
+    queueTableRows(schemaMock.embedding, rerankRows)
+    await handleVectorOnlySearch({
+      ...params,
+      permitted: { kind: 'unbounded' },
+      connectorEligibility: {
+        workspace: [],
+        admin: ['sliced-src'],
+        members: [],
+        liveProofRequired: [],
+      },
+    })
+    expect(statements().filter((query) => query.sql.includes('AS visible'))).toHaveLength(0)
+  })
+
   it('confines keyword matching to the bounded permitted set', async () => {
     await executeKeywordSearch({
       ...params,