improvement(knowledge): rank organization keyword search with Tin

.github/workflows/test-build.yml

@@ -241,6 +241,7 @@ jobs:
           bunx vitest run
           script-migrations/0016_backfill_search_vectors.postgres.test.ts
           script-migrations/0018_repair_workspace_file_content_revision.postgres.test.ts
+          script-migrations/0019_tin_keyword_projection.postgres.test.ts
 
       - name: Verify Search progress, pagination, and outbox scheduling in PostgreSQL
         working-directory: apps/sim

apps/sim/lib/core/config/env.ts

@@ -629,6 +629,7 @@ export const env = createEnv({
     TABLE_ROW_TTL:                        z.boolean().optional(),
     CREDENTIAL_GROUPS:                    z.boolean().optional(),                 // Enable enterprise Credential Groups globally
     KNOWLEDGE_MEMBER_ACCESS:              z.boolean().optional(),                 // Enable per-member knowledge connectors and hybrid-by-default retrieval globally
+    KNOWLEDGE_TIN_KEYWORD:                z.boolean().optional(),                 // Rank large-scope keyword retrieval through the Tin text index where it exists
 
     // Organizations - for self-hosted deployments
     ORGANIZATIONS_ENABLED:                 z.boolean().optional(),                 // Enable organizations on self-hosted (bypasses plan requirements)

apps/sim/lib/core/config/feature-flags.test.ts

@@ -15,6 +15,7 @@ const { mockFetch, mockIsPlatformAdmin, envRef } = vi.hoisted(() => ({
     TABLE_ROW_TTL: undefined as boolean | undefined,
     CREDENTIAL_GROUPS: undefined as boolean | undefined,
     KNOWLEDGE_MEMBER_ACCESS: undefined as boolean | undefined,
+    KNOWLEDGE_TIN_KEYWORD: undefined as boolean | undefined,
     SLACK_SEARCH_SHARED_APP: undefined as boolean | undefined,
   },
 }))
@@ -126,6 +127,7 @@ describe('isFeatureEnabled', () => {
     setEnvFlags({ isAppConfigEnabled: false })
     envRef.CREDENTIAL_GROUPS = undefined
     envRef.KNOWLEDGE_MEMBER_ACCESS = undefined
+    envRef.KNOWLEDGE_TIN_KEYWORD = undefined
     envRef.SLACK_SEARCH_SHARED_APP = undefined
   })
 
@@ -162,6 +164,19 @@ describe('isFeatureEnabled', () => {
     })
   })
 
+  describe('knowledge-tin-keyword flag', () => {
+    it('is a global switch', async () => {
+      expect(await isFeatureEnabled('knowledge-tin-keyword')).toBe(false)
+      envRef.KNOWLEDGE_TIN_KEYWORD = true
+      expect(await isFeatureEnabled('knowledge-tin-keyword')).toBe(true)
+    })
+
+    it('follows an AppConfig global rule', async () => {
+      withAppConfig({ 'knowledge-tin-keyword': { enabled: true } })
+      expect(await isFeatureEnabled('knowledge-tin-keyword')).toBe(true)
+    })
+  })
+
   describe('knowledge-member-access flag', () => {
     it('uses a global fallback switch off AppConfig', async () => {
       expect(await isFeatureEnabled('knowledge-member-access')).toBe(false)

apps/sim/lib/core/config/feature-flags.ts

@@ -93,6 +93,13 @@ const FEATURE_FLAGS = {
       'KNOWLEDGE_MEMBER_ACCESS.',
     fallback: 'KNOWLEDGE_MEMBER_ACCESS',
   },
+  'knowledge-tin-keyword': {
+    description:
+      'Rank keyword retrieval for members whose permitted set is too large to enumerate through ' +
+      'the Tin text index instead of GIN. Has no effect where the Tin keyword index is absent or ' +
+      'invalid. Off-AppConfig falls back to KNOWLEDGE_TIN_KEYWORD.',
+    fallback: 'KNOWLEDGE_TIN_KEYWORD',
+  },
 } satisfies Record<string, FeatureFlagDefinition>
 
 /**

apps/sim/lib/knowledge/search/diagnostics.ts

@@ -51,6 +51,9 @@ export type SearchStage =
   | 'vector.exact_candidates'
   | 'vector.exact'
   | 'vector.candidate_search'
+  | 'keyword.tin'
+  | 'keyword.tin_readiness'
+  | 'keyword.tin_query'
   | 'source_overview'
   | 'source_overview.availability'
   | 'source_overview.providers'
@@ -98,6 +101,13 @@ export interface SearchDiagnosticMetadata {
   permittedDocuments?: 'bounded' | 'unbounded'
   /** Documents in a bounded permitted set. */
   permittedDocumentCount?: number
+  /**
+   * Which index ranked an unbounded keyword leg: `tin` ranks by BM25 and checks access on the top
+   * of that ranking; `gin` ranks every match. Absent when the leg ranked inside a bounded set.
+   */
+  keywordRanking?: 'tin' | 'gin'
+  /** Candidates Tin ranked before access was checked on the last keyword page. */
+  keywordTinWindow?: number
   vectorCandidateCount?: number
   vectorCandidateDimensions?: number
   resultCount?: number

apps/sim/lib/knowledge/search/queries.test.ts

@@ -10,6 +10,15 @@ import {
   schemaMock,
 } from '@sim/testing'
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { mockResolveTinKeywordQuery } = vi.hoisted(() => ({
+  mockResolveTinKeywordQuery: vi.fn<() => Promise<string | null>>(async () => null),
+}))
+
+vi.mock('@/lib/knowledge/search/tin-keyword', () => ({
+  resolveTinKeywordQuery: mockResolveTinKeywordQuery,
+}))
+
 import {
   type KnowledgeAccessProvider,
   type UserAccessScope,
@@ -1489,6 +1498,106 @@ describe('permitted-document planner', () => {
     expect(JSON.stringify(keyword)).toContain('doc-a')
   })
 
+  describe('Tin keyword ranking for an unbounded caller', () => {
+    const unbounded: PermittedDocuments = { kind: 'unbounded' }
+    const keyword = (overrides: Partial<Parameters<typeof executeKeywordSearch>[0]> = {}) =>
+      executeKeywordSearch({
+        ...params,
+        topK: 1,
+        query: 'release',
+        queryVector: params.queryVector!,
+        permitted: unbounded,
+        ...overrides,
+      })
+    const tinStatements = () =>
+      statements().filter((query) => query.sql.includes('ranked_tin_chunks'))
+    const ginStatements = () =>
+      statements().filter((query) => query.sql.includes('WITH matched_keyword_chunks'))
+    let tinPages: Array<{ ranked: number; candidates: ReturnType<typeof hit>[] }>
+
+    beforeEach(() => {
+      mockResolveTinKeywordQuery.mockReset()
+      mockResolveTinKeywordQuery.mockResolvedValue('"releas"')
+      tinPages = []
+      dbChainMockFns.execute.mockImplementation(async (query) =>
+        render(query).sql.includes('ranked_tin_chunks')
+          ? [tinPages.shift() ?? { ranked: 0, candidates: [] }]
+          : []
+      )
+    })
+
+    it('ranks with Tin and checks access only on the top of that ranking', async () => {
+      tinPages = [{ ranked: 1500, candidates: [hit('a', null)] }]
+      queueTableRows(schemaMock.embedding, [{ ...hit('a', null), content: 'release notes' }])
+      const results = await keyword()
+      expect(results.map((row) => row.id)).toEqual(['a'])
+      expect(mockResolveTinKeywordQuery).toHaveBeenCalledWith(
+        ['org-index'],
+        'release',
+        'english',
+        params.budget
+      )
+      expect(ginStatements()).toHaveLength(0)
+      expect(JSON.stringify(tinStatements()[0])).toContain('2000')
+      /** `==>` binds tighter than `||`, so the concatenated query must be parenthesized. */
+      expect(tinStatements()[0].sql).toContain('==> (?)')
+    })
+
+    it('widens the ranked window while too few ranked chunks are readable', async () => {
+      tinPages = [
+        { ranked: 2000, candidates: [] },
+        { ranked: 4000, candidates: [hit('b', null)] },
+      ]
+      queueTableRows(schemaMock.embedding, [{ ...hit('b', null), content: 'release notes' }])
+      expect((await keyword()).map((row) => row.id)).toEqual(['b'])
+      const windows = tinStatements().map((query) => JSON.stringify(query))
+      expect(windows[0]).toContain('2000')
+      expect(windows[1]).toContain('10000')
+      expect(ginStatements()).toHaveLength(0)
+    })
+
+    it('stops widening once Tin ranked every match', async () => {
+      tinPages = [{ ranked: 12, candidates: [] }]
+      expect(await keyword()).toEqual([])
+      expect(tinStatements()).toHaveLength(1)
+      expect(ginStatements()).toHaveLength(0)
+    })
+
+    it('leaves the page to the GIN ranking when the widest window cannot fill it', async () => {
+      tinPages = [
+        { ranked: 2000, candidates: [] },
+        { ranked: 10_000, candidates: [] },
+        { ranked: 50_000, candidates: [] },
+      ]
+      await keyword()
+      expect(tinStatements()).toHaveLength(3)
+      expect(ginStatements()).toHaveLength(1)
+    })
+
+    it.each([
+      ['a bounded permitted set', { permitted: bounded({ id: 'doc-a', connectorId: null }) }],
+      [
+        'structured tag filters',
+        {
+          structuredFilters: [
+            { tagSlot: 'tag1', fieldType: 'text', operator: 'eq', value: 'x' },
+          ] as StructuredFilter[],
+        },
+      ],
+    ])('keeps GIN ranking for %s', async (_case, overrides) => {
+      await keyword(overrides)
+      expect(mockResolveTinKeywordQuery).not.toHaveBeenCalled()
+      expect(tinStatements()).toHaveLength(0)
+    })
+
+    it('keeps GIN ranking when Tin is not ready or cannot express the query', async () => {
+      mockResolveTinKeywordQuery.mockResolvedValue(null)
+      await keyword()
+      expect(tinStatements()).toHaveLength(0)
+      expect(ginStatements()).toHaveLength(1)
+    })
+  })
+
   it('skips keyword SQL entirely when nothing is permitted', async () => {
     expect(
       await executeKeywordSearch({

apps/sim/lib/knowledge/search/queries.ts

@@ -3,6 +3,7 @@ import {
   document,
   embedding,
   embeddingKeywordSearch,
+  embeddingKeywordTin,
   embeddingSearch,
   knowledgeConnector,
 } from '@sim/db/schema'
@@ -35,6 +36,7 @@ import {
 import { workspaceSearchFilterConditions } from '@/lib/knowledge/search/filter-conditions'
 import type { WorkspaceSearchFilters } from '@/lib/knowledge/search/filters'
 import { applyRecencyBoost, RRF_K } from '@/lib/knowledge/search/recency'
+import { resolveTinKeywordQuery } from '@/lib/knowledge/search/tin-keyword'
 import {
   coerceTagFilterValue,
   escapeLikePattern,
@@ -491,6 +493,13 @@ export function getStructuredTagFilters(filters: StructuredFilter[], embeddingTa
  */
 const FTS_CONFIG = 'english'
 
+/**
+ * Chunks Tin ranks before access is checked, widening while too few are readable to fill a page.
+ * A caller past the permitted-set limit reads a large share of the index, so the first window
+ * almost always fills; the widest bounds the work before the GIN ranking takes over.
+ */
+const TIN_KEYWORD_WINDOWS = [2000, 10_000, 50_000] as const
+
 /**
  * Row visibility predicates shared by every search leg: a chunk is only
  * retrievable when both it and its document are enabled, the document finished
@@ -1348,6 +1357,96 @@ export async function executeKeywordSearch(params: KeywordSearchParams): Promise
       ...tagFilterConditions,
     ]
     const candidateRank = sql<number>`ts_rank_cd(${embeddingKeywordSearch.contentTsv}, ${tsQuery})`
+    /**
+     * A caller reaching past the permitted-set limit reads much of the index, so ranking every
+     * match before checking access is the leg's whole cost for a common term. Where the Tin
+     * projection is complete, BM25 ranks inside the bases first and access is checked only on the
+     * top of that ranking.
+     */
+    const tinQuery =
+      params.permitted?.kind === 'unbounded' && tagFilterConditions.length === 0
+        ? await resolveTinKeywordQuery(knowledgeBaseIds, query, FTS_CONFIG, params.budget)
+        : null
+    annotateSearchDiagnostics({
+      ...(params.permitted?.kind === 'unbounded'
+        ? { keywordRanking: tinQuery ? 'tin' : 'gin' }
+        : {}),
+    })
+    const documentConditions = (excludedSources: readonly string[]) =>
+      and(
+        ...candidateDocumentConditions(
+          knowledgeBaseIds,
+          access,
+          params.filters,
+          knowledgeMetadataCandidateAccessCondition(access)
+        ),
+        excludeSearchSources(excludedSources)
+      )
+    /**
+     * One page from the top of Tin's ranking. The window of ranked chunks widens while too few of
+     * them are readable to fill the page; if the widest window still cannot, the page is left to
+     * the GIN ranking, which covers every match.
+     */
+    const selectTinPage = async (
+      scopedQuery: SQL,
+      limit: number,
+      offset: number,
+      excludedSources: readonly string[]
+    ): Promise<SearchReadCandidatePage | null> => {
+      for (const window of TIN_KEYWORD_WINDOWS) {
+        if (window < offset + limit) continue
+        const [page] = await runSearchQuery(params.budget, 'keyword.tin', (executor) =>
+          executor.execute<{ ranked: number; candidates: SearchReadCandidate[] }>(sql`
+            WITH ranked_tin_chunks AS MATERIALIZED (
+              SELECT ${embeddingKeywordTin.id} AS id, ${embeddingKeywordTin.documentId} AS document_id,
+                ${embeddingKeywordTin.enabled} AS enabled,
+                tin.full_score(${embeddingKeywordTin}.ctid) AS keyword_rank
+              FROM ${embeddingKeywordTin}
+              WHERE ${embeddingKeywordTin.content} ==> (${scopedQuery})
+              ORDER BY keyword_rank DESC
+              LIMIT ${window}
+            ), visible_keyword_documents AS MATERIALIZED (
+              SELECT ${document.id} AS id FROM ${document}
+              WHERE ${and(
+                sql`${document.id} = ANY (ARRAY(SELECT document_id FROM ranked_tin_chunks))`,
+                documentConditions(excludedSources)
+              )}
+            ), page AS (
+              SELECT ranked_tin_chunks.id, ${document.id} AS "documentId",
+                ${document.connectorId} AS "connectorId",
+                ${SEARCH_READ_CANDIDATE_FIELDS.liveAuthorizationSource} AS "liveAuthorizationSource",
+                ranked_tin_chunks.keyword_rank
+              FROM ranked_tin_chunks INNER JOIN ${document}
+                ON ${document.id} = ranked_tin_chunks.document_id
+              WHERE ranked_tin_chunks.enabled
+                AND ranked_tin_chunks.document_id IN (SELECT id FROM visible_keyword_documents)
+              ORDER BY ranked_tin_chunks.keyword_rank DESC, ranked_tin_chunks.id
+              LIMIT ${limit} OFFSET ${offset}
+            )
+            SELECT (SELECT count(*)::int FROM ranked_tin_chunks) AS ranked,
+              coalesce((
+                SELECT json_agg(json_build_object(
+                  'id', page.id, 'documentId', page."documentId", 'connectorId', page."connectorId",
+                  'liveAuthorizationSource', page."liveAuthorizationSource"
+                ) ORDER BY page.keyword_rank DESC, page.id)
+                FROM page
+              ), '[]'::json) AS candidates
+          `)
+        )
+        annotateSearchDiagnostics({ keywordTinWindow: window })
+        if (page.candidates.length === limit || page.ranked < window) {
+          return { candidates: page.candidates, nextOffset: offset + page.candidates.length }
+        }
+      }
+      return null
+    }
+    /** Parenthesized where used: `==>` binds tighter than `||`. */
+    const tinScope = tinQuery
+      ? sql`'(' || ${sql.join(
+          knowledgeBaseIds.map((id) => sql`knowledge_tin_base_token(${id}) || '^0'`),
+          sql` || ' OR ' || `
+        )} || ') AND (' || ${tinQuery} || ')'`
+      : undefined
     /** Keep readable identities and rank scalars separate so sorts never carry full text-search vectors. */
     return selectAuthorizedSearchResults({
       leg: 'keyword',
@@ -1368,6 +1467,11 @@ export async function executeKeywordSearch(params: KeywordSearchParams): Promise
             ? permittedDocumentIds(params.permitted.documents, excludedSources)
             : undefined
         if (permittedIds?.length === 0) return { candidates: [], nextOffset: offset }
+        if (tinScope && !permittedIds) {
+          const tinPage = await selectTinPage(tinScope, limit, offset, excludedSources)
+          if (tinPage) return tinPage
+          annotateSearchDiagnostics({ keywordRanking: 'gin' })
+        }
         const baseScope = and(
           inArray(embeddingKeywordSearch.knowledgeBaseId, knowledgeBaseIds),
           eq(embeddingKeywordSearch.enabled, true)
@@ -1411,13 +1515,7 @@ export async function executeKeywordSearch(params: KeywordSearchParams): Promise
               SELECT ${document.id} AS id FROM ${document}
               WHERE ${and(
                 sql`${document.id} = ANY (ARRAY(SELECT document_id FROM matched_keyword_chunks))`,
-                ...candidateDocumentConditions(
-                  knowledgeBaseIds,
-                  access,
-                  params.filters,
-                  knowledgeMetadataCandidateAccessCondition(access)
-                ),
-                excludeSearchSources(excludedSources)
+                documentConditions(excludedSources)
               )}
             ), ranked_keyword_candidates AS MATERIALIZED (
               SELECT matched_keyword_chunks.id, matched_keyword_chunks.document_id,

apps/sim/lib/knowledge/search/tin-keyword-readiness.test.ts

@@ -0,0 +1,31 @@
+/**
+ * @vitest-environment node
+ */
+import { dbChainMockFns, queueTableRows, resetDbChainMock, schemaMock } from '@sim/testing'
+import { expect, it, vi } from 'vitest'
+
+vi.mock('@/lib/core/config/feature-flags', () => ({
+  isFeatureEnabled: vi.fn(async () => true),
+}))
+
+import { resolveTinKeywordQuery } from '@/lib/knowledge/search/tin-keyword'
+
+/** Its own file, so the process-wide readiness cache starts empty. */
+it('stays on the GIN projection while the Tin index is incomplete, and remembers that', async () => {
+  resetDbChainMock()
+  let indexValid = false
+  dbChainMockFns.execute.mockImplementation(async (query) => {
+    const text = JSON.stringify(query)
+    if (text.includes('indisvalid')) return [{ valid: indexValid }]
+    if (text.includes('websearch_to_tsquery')) return [{ rendered: "'releas'" }]
+    return []
+  })
+  queueTableRows(schemaMock.knowledgeBase, [{ id: 'kb-index', isSearchIndex: true }])
+  expect(await resolveTinKeywordQuery(['kb-index'], 'release', 'english', undefined)).toBeNull()
+  indexValid = true
+  expect(await resolveTinKeywordQuery(['kb-index'], 'release', 'english', undefined)).toBeNull()
+  const readinessReads = dbChainMockFns.execute.mock.calls.filter(([query]) =>
+    JSON.stringify(query).includes('indisvalid')
+  )
+  expect(readinessReads).toHaveLength(1)
+})