fix(knowledge): hand only a large bounded set's first page to the GIN ranking

Tin and GIN order candidates differently, so an offset advanced through one ranking cannot resume the other. A large bounded set's first page that Tin's widest window cannot fill goes to GIN; a later page stays with Tin and is left short as a narrow reader's is.

Author: waleedlatif1

apps/sim/lib/knowledge/search/queries.test.ts

@@ -1845,6 +1845,28 @@ describe('permitted-document planner', () => {
         expect(JSON.stringify(ginStatements()[0])).not.toContain('doc-4999')
       })
 
+      it('leaves a later page short rather than resuming a different ranking at its offset', async () => {
+        /** The first page fills from Tin; hydration keeps half, so a second page is asked for. */
+        const first = Array.from({ length: 40 }, (_, index) => hit(`t-${index}`, 'src-a'))
+        tinPages = [
+          { ranked: 2000, candidates: first },
+          { ranked: 2000, candidates: [] },
+          { ranked: 20_000, candidates: [] },
+        ]
+        queueTableRows(
+          schemaMock.embedding,
+          first.slice(0, 20).map((row) => ({ ...row, content: 'release notes' }))
+        )
+        const results = await keyword({
+          topK: 40,
+          permitted: { kind: 'bounded', documents: large },
+          accessPlan,
+        })
+        expect(results).toHaveLength(20)
+        expect(tinStatements()).toHaveLength(3)
+        expect(ginStatements()).toHaveLength(0)
+      })
+
       it('keeps the bounded read for a set under the size', async () => {
         mockResolveTinKeywordQuery.mockResolvedValue(null)
         await keyword({ permitted: { kind: 'bounded', documents: large.slice(0, -1) }, accessPlan })

apps/sim/lib/knowledge/search/queries.ts

@@ -2155,8 +2155,10 @@ export async function executeKeywordSearch(params: KeywordSearchParams): Promise
        * A resolved scope decides readability on the ranked row. The windows widen while the page
        * is short, a narrow reader's to a wide one sooner and no further, and what the widest
        * cannot fill is left short rather than handed to a ranking over every match. A large
-       * bounded set is the exception: its bounded read was exhaustive, so the widest window that
-       * still falls short hands the page to the GIN ranking, which covers every match.
+       * bounded set is the exception on its first page: its bounded read was exhaustive, so the
+       * widest window that still falls short hands that page to the GIN ranking, which covers
+       * every match. A later page stays with Tin: the two rankers order differently, so an offset
+       * advanced through one cannot resume the other.
        */
       const narrow =
         accessPlan !== undefined &&
@@ -2221,7 +2223,9 @@ export async function executeKeywordSearch(params: KeywordSearchParams): Promise
         if (
           page.candidates.length >= limit ||
           page.ranked < window ||
-          (accessPlan !== undefined && !largePermittedSet && window === windows[windows.length - 1])
+          (accessPlan !== undefined &&
+            !(largePermittedSet && offset === 0) &&
+            window === windows[windows.length - 1])
         ) {
           return { candidates: page.candidates, nextOffset: offset + page.candidates.length }
         }