From aa9eec67f6bec000e12428daad89c678f7f7b7aa Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 14 Sep 2026 16:17:23 -0700 Subject: [PATCH] improvement(search): speed up organization integration and source queries --- apps/sim/app/api/files/authorization.test.ts | 9 +- .../sim-search/sources/route.test.ts | 2 +- .../integrations/integrations.test.tsx | 2 +- .../[connectorType]/provider-detail.test.tsx | 3 +- .../kb/search-source-progress.test.tsx | 18 +- .../lib/api/contracts/knowledge/connectors.ts | 6 +- .../knowledge/personal-integrations.ts | 1 - .../load-search-integrations.test.ts | 1 - .../github-member.integration.ts | 14 +- ...rganization-search-overview.integration.ts | 2 +- .../lib/knowledge/access/confluence-site.ts | 13 +- .../knowledge/access/github-installation.ts | 16 +- apps/sim/lib/knowledge/access/live-sources.ts | 41 + apps/sim/lib/knowledge/access/scope.test.ts | 61 +- apps/sim/lib/knowledge/access/scope.ts | 35 +- apps/sim/lib/knowledge/access/types.ts | 11 +- .../knowledge/application/connectors.test.ts | 8 +- .../lib/knowledge/application/connectors.ts | 66 +- .../knowledge/application/documents.test.ts | 3 +- .../personal-search-integrations.test.ts | 15 +- .../personal-search-integrations.ts | 14 +- .../application/search-source-overview.ts | 63 +- .../application/search-sources.test.ts | 95 +- .../knowledge/application/search-sources.ts | 79 +- .../lib/knowledge/documents/service.test.ts | 1 + apps/sim/lib/knowledge/read-access.test.ts | 45 +- apps/sim/lib/knowledge/read-access.ts | 43 +- apps/sim/lib/knowledge/service.test.ts | 25 +- apps/sim/lib/sim-search/source-status.ts | 3 +- ...5_document_connector_processing_status.sql | 6 + .../db/migrations/meta/0345_snapshot.json | 26613 ++++++++++++++++ packages/db/migrations/meta/_journal.json | 7 + packages/db/schema.ts | 10 + 33 files changed, 27108 insertions(+), 223 deletions(-) create mode 100644 apps/sim/lib/knowledge/access/live-sources.ts create mode 100644 packages/db/migrations/0345_document_connector_processing_status.sql create mode 100644 packages/db/migrations/meta/0345_snapshot.json diff --git a/apps/sim/app/api/files/authorization.test.ts b/apps/sim/app/api/files/authorization.test.ts index b449eab962f..a8738342a8d 100644 --- a/apps/sim/app/api/files/authorization.test.ts +++ b/apps/sim/app/api/files/authorization.test.ts @@ -436,13 +436,15 @@ describe('KB file live source authorization', () => { async (allowed) => { const scope = { kind: 'user' as const, userId: USER_ID, tokens: ['reader-token'] } const getForConnectors = vi.fn().mockResolvedValue(scope) + const liveSources = { type: 'live-sources' } const access: KnowledgeAccessProvider = { get: async () => scope, getForConnectors, getForDocuments: async () => scope, + liveSourceConnectorCondition: async () => liveSources as never, } queueTableRows(schemaMock.document, []) - queueTableRows(schemaMock.document, [{ connectorId: 'confluence-source' }]) + queueTableRows(schemaMock.knowledgeConnector, [{ connectorId: 'confluence-source' }]) queueTableRows(schemaMock.document, allowed ? [{ id: 'doc-1' }] : []) await expect( verifyFileAccess(CLOUD_KEY, USER_ID, undefined, 'knowledge-base', false, { @@ -450,7 +452,12 @@ describe('KB file live source authorization', () => { }) ).resolves.toBe(allowed) expect(getForConnectors).toHaveBeenCalledExactlyOnceWith(['confluence-source'], undefined) + const discovery = dbChainMockFns.where.mock.calls.filter(([condition]) => + hasMockCondition(condition, (node) => node === liveSources) + ) + expect(discovery).toHaveLength(1) for (const [condition] of dbChainMockFns.where.mock.calls) { + if (discovery.some(([live]) => live === condition)) continue expect( hasMockCondition( condition, diff --git a/apps/sim/app/api/knowledge/sim-search/sources/route.test.ts b/apps/sim/app/api/knowledge/sim-search/sources/route.test.ts index 44ee57d8279..8456c037206 100644 --- a/apps/sim/app/api/knowledge/sim-search/sources/route.test.ts +++ b/apps/sim/app/api/knowledge/sim-search/sources/route.test.ts @@ -59,7 +59,7 @@ const source = { isSyncing: false, lastSyncAt: null, hasSyncError: false, - viewerDocumentCount: 0, + hasViewerDocuments: false, viewerFailedDocumentCount: 0, viewerEmailVerified: true, viewerAccounts: [], diff --git a/apps/sim/app/o/[organizationId]/integrations/integrations.test.tsx b/apps/sim/app/o/[organizationId]/integrations/integrations.test.tsx index fa22c0f7467..22a26645a65 100644 --- a/apps/sim/app/o/[organizationId]/integrations/integrations.test.tsx +++ b/apps/sim/app/o/[organizationId]/integrations/integrations.test.tsx @@ -149,7 +149,7 @@ const memberSource: SearchSourceSummary = { isSyncing: false, lastSyncAt: null, hasSyncError: false, - viewerDocumentCount: 0, + hasViewerDocuments: false, viewerFailedDocumentCount: 0, viewerEmailVerified: true, viewerAccounts: [], diff --git a/apps/sim/app/o/[organizationId]/settings/integrations/providers/[connectorType]/provider-detail.test.tsx b/apps/sim/app/o/[organizationId]/settings/integrations/providers/[connectorType]/provider-detail.test.tsx index 7d4d860870c..45566763b4c 100644 --- a/apps/sim/app/o/[organizationId]/settings/integrations/providers/[connectorType]/provider-detail.test.tsx +++ b/apps/sim/app/o/[organizationId]/settings/integrations/providers/[connectorType]/provider-detail.test.tsx @@ -136,7 +136,7 @@ const source = { lastSyncAt: '2026-09-08T12:00:00.000Z', connectionRequired: true, viewerMembership: 'needs_reauth', - viewerDocumentCount: 0, + hasViewerDocuments: false, } const credentialGroup = { id: 'accounts-one', @@ -429,7 +429,6 @@ describe('organization provider management', () => { ) expect(container.textContent).not.toContain('Connect account') expect(container.textContent).not.toContain('Reconnect') - expect(container.textContent).not.toContain('0 searchable documents') }) it('loads sources for a nonpersonal provider even when an accounts view URL is supplied', async () => { diff --git a/apps/sim/hooks/queries/kb/search-source-progress.test.tsx b/apps/sim/hooks/queries/kb/search-source-progress.test.tsx index 153b7e3907e..62d13f3779b 100644 --- a/apps/sim/hooks/queries/kb/search-source-progress.test.tsx +++ b/apps/sim/hooks/queries/kb/search-source-progress.test.tsx @@ -23,7 +23,7 @@ function Probe() { const sync = useTriggerSync() return (
- {result.data?.[0]?.viewerDocumentCount ?? 0} + {String(result.data?.[0]?.hasViewerDocuments ?? false)}