fix(knowledge): keep the GIN ranking for the rest of a search once a page was handed to it (#8110)

Author: waleedlatif1

apps/sim/lib/knowledge/search/queries.test.ts

@@ -1867,6 +1867,35 @@ describe('permitted-document planner', () => {
         expect(ginStatements()).toHaveLength(0)
       })
 
+      it('stays with the GIN ranking for the rest of a search once a page was handed to it', async () => {
+        /** Tin cannot fill the first page; GIN supplies it, and hydration keeps only half, so a second page follows. */
+        tinPages = [
+          { ranked: 2000, candidates: [] },
+          { ranked: 20_000, candidates: [] },
+          { ranked: 2000, candidates: [hit('never', 'src-a')] },
+        ]
+        const ginRows = Array.from({ length: 40 }, (_, index) => hit(`g-${index}`, 'src-a'))
+        const ginPages = [ginRows, []]
+        const execute = dbChainMockFns.execute.getMockImplementation()!
+        dbChainMockFns.execute.mockImplementation(async (query) =>
+          render(query).sql.includes('WITH matched_keyword_chunks')
+            ? (ginPages.shift() ?? [])
+            : execute(query)
+        )
+        queueTableRows(
+          schemaMock.embedding,
+          ginRows.slice(0, 20).map((row) => ({ ...row, content: 'release notes' }))
+        )
+        const results = await keyword({
+          topK: 40,
+          permitted: { kind: 'bounded', documents: large },
+          accessPlan,
+        })
+        expect(results.map((row) => row.id)).not.toContain('never')
+        expect(tinStatements()).toHaveLength(2)
+        expect(ginStatements().length).toBeGreaterThanOrEqual(2)
+      })
+
       it('keeps the bounded read for a set under the size', async () => {
         mockResolveTinKeywordQuery.mockResolvedValue(null)
         await keyword({ permitted: { kind: 'bounded', documents: large.slice(0, -1) }, accessPlan })

apps/sim/lib/knowledge/search/queries.ts

@@ -2239,6 +2239,11 @@ export async function executeKeywordSearch(params: KeywordSearchParams): Promise
           sql` || ' OR ' || `
         )} || ') AND (' || ${tinQuery} || ')'`
       : undefined
+    /**
+     * Tin and GIN order candidates differently, so a search that once handed a page to GIN stays
+     * with GIN: an offset advanced through one ranking cannot resume the other.
+     */
+    let handedToGin = false
     /** Keep readable identities and rank scalars separate so sorts never carry full text-search vectors. */
     return selectAuthorizedSearchResults({
       leg: 'keyword',
@@ -2259,9 +2264,10 @@ export async function executeKeywordSearch(params: KeywordSearchParams): Promise
             ? params.permitted.documents.map((entry) => entry.id)
             : undefined
         if (permittedIds?.length === 0) return { candidates: [], nextOffset: offset }
-        if (tinScope) {
+        if (tinScope && !handedToGin) {
           const tinPage = await selectTinPage(tinScope, limit, offset, excludedSources)
           if (tinPage) return tinPage
+          handedToGin = true
           annotateSearchDiagnostics({ keywordRanking: 'gin' })
         }
         const baseScope = and(