From 2c1cdb9071578c59749ae47491070a724035583d Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 21 Sep 2026 10:53:09 -0700 Subject: [PATCH] fix(knowledge): keep the GIN ranking for the rest of a search once a page was handed to it Tin and GIN order candidates differently, so a keyword search whose first page Tin could not fill, and which GIN supplied, must not resume Tin at GIN's offset on a later page. The leg now stays with GIN for the rest of that search. --- apps/sim/lib/knowledge/search/queries.test.ts | 29 +++++++++++++++++++ apps/sim/lib/knowledge/search/queries.ts | 8 ++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/apps/sim/lib/knowledge/search/queries.test.ts b/apps/sim/lib/knowledge/search/queries.test.ts index 3535574312e..6e76b50d677 100644 --- a/apps/sim/lib/knowledge/search/queries.test.ts +++ b/apps/sim/lib/knowledge/search/queries.test.ts @@ -1867,6 +1867,35 @@ describe('permitted-document planner', () => { expect(ginStatements()).toHaveLength(0) }) + it('stays with the GIN ranking for the rest of a search once a page was handed to it', async () => { + /** Tin cannot fill the first page; GIN supplies it, and hydration keeps only half, so a second page follows. */ + tinPages = [ + { ranked: 2000, candidates: [] }, + { ranked: 20_000, candidates: [] }, + { ranked: 2000, candidates: [hit('never', 'src-a')] }, + ] + const ginRows = Array.from({ length: 40 }, (_, index) => hit(`g-${index}`, 'src-a')) + const ginPages = [ginRows, []] + const execute = dbChainMockFns.execute.getMockImplementation()! + dbChainMockFns.execute.mockImplementation(async (query) => + render(query).sql.includes('WITH matched_keyword_chunks') + ? (ginPages.shift() ?? []) + : execute(query) + ) + queueTableRows( + schemaMock.embedding, + ginRows.slice(0, 20).map((row) => ({ ...row, content: 'release notes' })) + ) + const results = await keyword({ + topK: 40, + permitted: { kind: 'bounded', documents: large }, + accessPlan, + }) + expect(results.map((row) => row.id)).not.toContain('never') + expect(tinStatements()).toHaveLength(2) + expect(ginStatements().length).toBeGreaterThanOrEqual(2) + }) + it('keeps the bounded read for a set under the size', async () => { mockResolveTinKeywordQuery.mockResolvedValue(null) await keyword({ permitted: { kind: 'bounded', documents: large.slice(0, -1) }, accessPlan }) diff --git a/apps/sim/lib/knowledge/search/queries.ts b/apps/sim/lib/knowledge/search/queries.ts index a92a5eb1ec7..af01100b26e 100644 --- a/apps/sim/lib/knowledge/search/queries.ts +++ b/apps/sim/lib/knowledge/search/queries.ts @@ -2239,6 +2239,11 @@ export async function executeKeywordSearch(params: KeywordSearchParams): Promise sql` || ' OR ' || ` )} || ') AND (' || ${tinQuery} || ')'` : undefined + /** + * Tin and GIN order candidates differently, so a search that once handed a page to GIN stays + * with GIN: an offset advanced through one ranking cannot resume the other. + */ + let handedToGin = false /** Keep readable identities and rank scalars separate so sorts never carry full text-search vectors. */ return selectAuthorizedSearchResults({ leg: 'keyword', @@ -2259,9 +2264,10 @@ export async function executeKeywordSearch(params: KeywordSearchParams): Promise ? params.permitted.documents.map((entry) => entry.id) : undefined if (permittedIds?.length === 0) return { candidates: [], nextOffset: offset } - if (tinScope) { + if (tinScope && !handedToGin) { const tinPage = await selectTinPage(tinScope, limit, offset, excludedSources) if (tinPage) return tinPage + handedToGin = true annotateSearchDiagnostics({ keywordRanking: 'gin' }) } const baseScope = and(