From 7ccefe2014425eece072ffe88e2c3a845dd21c69 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 18 Sep 2026 00:32:33 -0700 Subject: [PATCH] improvement(knowledge): stop re-probing known indexing providers per access batch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The search source overview loops over access batches and, for each one, ran the indexing probe in full. That probe's result is only ever read back as set membership when projecting `isSyncing`, so a provider type already found cannot change the answer — every later batch re-paid an EXISTS scan over `document` carrying the full ACL predicate for nothing. Mirror the searchable probe's existing guard: skip the indexing probe once every configured provider type is accounted for, and exclude already-found types from the query on later batches. Both queries build on the same configured condition and the probe only adds narrowing predicates, so the resulting set is unchanged. --- .../search-source-overview.test.ts | 36 +++++++++++++++++++ .../application/search-source-overview.ts | 16 ++++++++- 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/apps/sim/lib/knowledge/application/search-source-overview.test.ts b/apps/sim/lib/knowledge/application/search-source-overview.test.ts index 7b4fbfff729..08b28960654 100644 --- a/apps/sim/lib/knowledge/application/search-source-overview.test.ts +++ b/apps/sim/lib/knowledge/application/search-source-overview.test.ts @@ -24,6 +24,7 @@ vi.mock('@/lib/knowledge/read-access', () => ({ })) import { readSearchSourceOverview } from '@/lib/knowledge/application/search-source-overview' +import { MAX_SEARCH_SOURCE_PROVIDER_TYPES } from '@/lib/knowledge/constants' const principal = { kind: 'session', userId: 'reader', sessionId: 'session' } as const const input = { organizationId: 'org-1', workspaceId: null } @@ -36,6 +37,14 @@ const searchableProbeCount = () => dbChainMockFns.limit.mock.calls.filter(([rows]) => rows === 1).length - AUTHORIZATION_SINGLE_ROW_READS +/** The configured-provider list is read once, before the batches, under the same bound. */ +const CONFIGURED_PROVIDER_READS = 1 + +/** Every other provider-bounded read in this use case is the indexing probe. */ +const indexingProbeCount = () => + dbChainMockFns.limit.mock.calls.filter(([rows]) => rows === MAX_SEARCH_SOURCE_PROVIDER_TYPES) + .length - CONFIGURED_PROVIDER_READS + function yieldBatches(count: number) { mocks.batches.mockImplementation(async function* () { for (let index = 0; index < count; index += 1) yield sql`batch-${sql.raw(String(index))}` @@ -73,4 +82,31 @@ describe('readSearchSourceOverview', () => { expect(result.hasSearchableDocuments).toBe(false) expect(searchableProbeCount()).toBe(3) }) + + it('stops probing for indexing once every configured provider type is known', async () => { + yieldBatches(3) + queueTableRows(member, [{ role: 'owner' }]) + queueTableRows(knowledgeConnector, [{ connectorType: 'gmail' }]) + queueTableRows(knowledgeConnector, [{ connectorType: 'gmail' }]) + + const result = await readSearchSourceOverview.execute({ principal, input }) + + expect(result.providers).toEqual([{ connectorType: 'gmail', isSyncing: true }]) + expect(indexingProbeCount()).toBe(1) + }) + + it('keeps probing every batch while a configured provider type is still unaccounted for', async () => { + yieldBatches(3) + queueTableRows(member, [{ role: 'owner' }]) + queueTableRows(knowledgeConnector, [{ connectorType: 'gmail' }, { connectorType: 'notion' }]) + queueTableRows(knowledgeConnector, [{ connectorType: 'gmail' }]) + + const result = await readSearchSourceOverview.execute({ principal, input }) + + expect(result.providers).toEqual([ + { connectorType: 'gmail', isSyncing: true }, + { connectorType: 'notion', isSyncing: false }, + ]) + expect(indexingProbeCount()).toBe(3) + }) }) diff --git a/apps/sim/lib/knowledge/application/search-source-overview.ts b/apps/sim/lib/knowledge/application/search-source-overview.ts index 0cfb6795313..c0892fcb856 100644 --- a/apps/sim/lib/knowledge/application/search-source-overview.ts +++ b/apps/sim/lib/knowledge/application/search-source-overview.ts @@ -117,16 +117,30 @@ export const readSearchSourceOverview = instrumentSourceOverviewUseCase( /** One searchable document is the whole answer, so later batches skip the probe entirely. */ const probesSearchable: boolean = probesSources && !hasSearchableDocuments if (probesSearchable) searchableProbes += 1 + /** + * A provider type is only read back as membership of `indexingTypes`, so once every + * configured type is in the set no later batch can change the answer. + */ + const probesIndexing: boolean = + probesSources && providers.some(({ connectorType }) => !indexingTypes.has(connectorType)) /** Annotated so the searchable probe's guard does not infer through its own result. */ const [indexing, searchable]: [{ connectorType: string }[], { id: string }[]] = await Promise.all([ - probesSources + probesIndexing ? measureSearchStage('source_overview.indexing', () => configuredProvidersQuery() .where( and( configured, syncingEnabled, + /** + * The probe narrows the configured set the provider list came from, so a + * type already found stays found; excluding it only drops repeated work. + * An empty set adds no predicate rather than a no-op one. + */ + indexingTypes.size > 0 + ? notInArray(knowledgeConnector.connectorType, [...indexingTypes]) + : undefined, or( inArray(knowledgeConnector.status, ['pending', 'syncing']), and(