diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index eb467bfea33..dacc08540b9 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -187,6 +187,7 @@ jobs: bunx vitest run --mode integration lib/knowledge/__integration__/search-source-progress.integration.ts lib/knowledge/__integration__/search-source-pagination.integration.ts + lib/knowledge/__integration__/search-reference-batching.integration.ts lib/core/outbox/service.integration.ts lib/knowledge/__integration__/connector-upload.integration.ts diff --git a/apps/sim/app/api/v2/knowledge/search/route.provenance.test.ts b/apps/sim/app/api/v2/knowledge/search/route.provenance.test.ts index 6289e012046..3971a4c4ca5 100644 --- a/apps/sim/app/api/v2/knowledge/search/route.provenance.test.ts +++ b/apps/sim/app/api/v2/knowledge/search/route.provenance.test.ts @@ -48,7 +48,8 @@ vi.mock('@/lib/knowledge/application/contexts', () => ({ })) vi.mock('@/lib/knowledge/service', () => ({ - getActiveKnowledgeBaseReference: mocks.getKnowledgeBase, + getActiveKnowledgeBaseReferences: (ids: string[]) => + Promise.all(ids.map((id) => mocks.getKnowledgeBase(id))), })) vi.mock('@/lib/knowledge/embeddings', () => ({ @@ -63,7 +64,8 @@ vi.mock('@/lib/knowledge/search/queries', () => ({ })) vi.mock('@/lib/knowledge/tags/service', () => ({ - getDocumentTagDefinitions: mocks.getTagDefinitions, + getDocumentTagDefinitionsByKnowledgeBaseIds: async (ids: string[]) => + new Map(await Promise.all(ids.map(async (id) => [id, await mocks.getTagDefinitions(id)]))), })) vi.mock('@/lib/knowledge/tags/utils', () => ({ diff --git a/apps/sim/lib/knowledge/__integration__/search-reference-batching.integration.ts b/apps/sim/lib/knowledge/__integration__/search-reference-batching.integration.ts new file mode 100644 index 00000000000..a08d83b195a --- /dev/null +++ b/apps/sim/lib/knowledge/__integration__/search-reference-batching.integration.ts @@ -0,0 +1,123 @@ +import { db } from '@sim/db' +import { + knowledgeBase, + knowledgeBaseTagDefinitions, + organization, + user, + workspace, +} from '@sim/db/schema' +import { generateId } from '@sim/utils/id' +import { and, eq, inArray } from 'drizzle-orm' +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' +import { + createKnowledgeAclFixtureIds, + seedKnowledgeAclFixture, +} from '@/lib/knowledge/__integration__/seed-source-access-fixture' +import { + getActiveKnowledgeBaseReference, + getActiveKnowledgeBaseReferences, +} from '@/lib/knowledge/service' +import { + getDocumentTagDefinitions, + getDocumentTagDefinitionsByKnowledgeBaseIds, +} from '@/lib/knowledge/tags/service' + +describe('batched search reference reads', () => { + const fixture = createKnowledgeAclFixtureIds() + const baseIds = [fixture.knowledgeBaseId, ...Array.from({ length: 19 }, () => generateId())] + const deletedBaseId = generateId() + + beforeAll(async () => { + await seedKnowledgeAclFixture(fixture) + await db.insert(knowledgeBase).values( + [...baseIds.slice(1), deletedBaseId].map((id, index) => ({ + id, + userId: fixture.aliceId, + workspaceId: fixture.workspaceId, + name: `Batched search fixture ${index}`, + deletedAt: id === deletedBaseId ? new Date() : null, + })) + ) + await db.insert(knowledgeBaseTagDefinitions).values( + baseIds.slice(0, -1).flatMap((knowledgeBaseId) => + (['tag3', 'tag2'] as const).map((tagSlot) => ({ + id: generateId(), + knowledgeBaseId, + tagSlot, + displayName: tagSlot, + fieldType: 'text' as const, + })) + ) + ) + }) + + afterAll(async () => { + vi.restoreAllMocks() + await db.delete(workspace).where(eq(workspace.id, fixture.workspaceId)) + await db.delete(organization).where(eq(organization.id, fixture.organizationId)) + await db.delete(user).where(inArray(user.id, [fixture.aliceId, fixture.bobId])) + await db.$client.end() + }) + + it('returns identical active references with one query instead of twenty', async () => { + const select = vi.spyOn(db, 'select') + try { + const expected = await Promise.all(baseIds.map(getActiveKnowledgeBaseReference)) + expect(select).toHaveBeenCalledTimes(20) + select.mockClear() + expect(await getActiveKnowledgeBaseReferences(baseIds)).toEqual(expected) + expect(select).toHaveBeenCalledOnce() + } finally { + select.mockRestore() + } + }) + + it('preserves input order, duplicates, missing identities, and soft deletion', async () => { + const ids = [baseIds[8], generateId(), baseIds[0], deletedBaseId, baseIds[8]] + expect(await getActiveKnowledgeBaseReferences(ids)).toEqual( + await Promise.all(ids.map(getActiveKnowledgeBaseReference)) + ) + }) + + it('returns identical ordered tag definitions with one query instead of twenty', async () => { + const select = vi.spyOn(db, 'select') + try { + const expected = new Map( + await Promise.all( + baseIds.map(async (id) => [id, await getDocumentTagDefinitions(id)] as const) + ) + ) + expect(select).toHaveBeenCalledTimes(20) + select.mockClear() + expect(await getDocumentTagDefinitionsByKnowledgeBaseIds(baseIds)).toEqual(expected) + expect(select).toHaveBeenCalledOnce() + expect(expected.get(baseIds.at(-1)!)).toEqual([]) + } finally { + select.mockRestore() + } + }) + + it('does not cache updated references or tag definitions across reads', async () => { + await getActiveKnowledgeBaseReferences(baseIds) + await getDocumentTagDefinitionsByKnowledgeBaseIds(baseIds) + await db + .update(knowledgeBase) + .set({ name: 'Updated reference' }) + .where(eq(knowledgeBase.id, baseIds[0])) + await db + .update(knowledgeBaseTagDefinitions) + .set({ displayName: 'Updated definition' }) + .where( + and( + eq(knowledgeBaseTagDefinitions.knowledgeBaseId, baseIds[0]), + eq(knowledgeBaseTagDefinitions.tagSlot, 'tag1') + ) + ) + const references = await getActiveKnowledgeBaseReferences(baseIds) + const tags = await getDocumentTagDefinitionsByKnowledgeBaseIds(baseIds) + expect(references[0]?.name).toBe('Updated reference') + expect( + tags.get(baseIds[0])?.find((definition) => definition.tagSlot === 'tag1')?.displayName + ).toBe('Updated definition') + }) +}) diff --git a/apps/sim/lib/knowledge/application/documents.test.ts b/apps/sim/lib/knowledge/application/documents.test.ts index 751ea4c7426..4bded55be3c 100644 --- a/apps/sim/lib/knowledge/application/documents.test.ts +++ b/apps/sim/lib/knowledge/application/documents.test.ts @@ -79,6 +79,10 @@ vi.mock('@/lib/knowledge/documents/service', () => ({ vi.mock('@/lib/knowledge/tags/service', () => ({ getDocumentTagDefinitions: mocks.getDocumentTagDefinitions, + getDocumentTagDefinitionsByKnowledgeBaseIds: async (ids: string[]) => + new Map( + await Promise.all(ids.map(async (id) => [id, await mocks.getDocumentTagDefinitions(id)])) + ), })) vi.mock('@/lib/knowledge/orchestration/documents', () => ({ diff --git a/apps/sim/lib/knowledge/application/search.test.ts b/apps/sim/lib/knowledge/application/search.test.ts index 185e4d64760..86bc9e8a76f 100644 --- a/apps/sim/lib/knowledge/application/search.test.ts +++ b/apps/sim/lib/knowledge/application/search.test.ts @@ -13,6 +13,7 @@ const mocks = vi.hoisted(() => ({ requireOrganizationSearch: vi.fn(), resolvePermission: vi.fn(), getKnowledgeBase: vi.fn(), + getKnowledgeBases: vi.fn(), resolveBilling: vi.fn(), checkUsage: vi.fn(), checkActorUsage: vi.fn(), @@ -20,6 +21,7 @@ const mocks = vi.hoisted(() => ({ executeSearch: vi.fn(), getDocumentMetadata: vi.fn(), getTagDefinitions: vi.fn(), + getTagDefinitionsBatch: vi.fn(), recordEmbeddingUsage: vi.fn(), importProvenance: vi.fn(), rerank: vi.fn(), @@ -72,7 +74,7 @@ vi.mock('@/lib/permission-groups/resolve.server', () => ({ })) vi.mock('@/lib/knowledge/service', () => ({ - getActiveKnowledgeBaseReference: mocks.getKnowledgeBase, + getActiveKnowledgeBaseReferences: mocks.getKnowledgeBases, })) vi.mock('@/lib/knowledge/embeddings', () => ({ @@ -87,7 +89,7 @@ vi.mock('@/lib/knowledge/search/queries', () => ({ })) vi.mock('@/lib/knowledge/tags/service', () => ({ - getDocumentTagDefinitions: mocks.getTagDefinitions, + getDocumentTagDefinitionsByKnowledgeBaseIds: mocks.getTagDefinitionsBatch, })) vi.mock('@/lib/knowledge/tags/utils', () => ({ @@ -130,6 +132,13 @@ describe('knowledge search application use case', () => { mocks.resolveWorkspace.mockResolvedValue(workspace) mocks.resolvePermission.mockResolvedValue('read') mocks.getKnowledgeBase.mockResolvedValue(knowledgeBase) + mocks.getKnowledgeBases.mockImplementation((ids: string[]) => + Promise.all(ids.map((id) => mocks.getKnowledgeBase(id))) + ) + mocks.getTagDefinitionsBatch.mockImplementation( + async (ids: string[]) => + new Map(await Promise.all(ids.map(async (id) => [id, await mocks.getTagDefinitions(id)]))) + ) mocks.resolveBilling.mockResolvedValue({ actorUserId: 'user-1', workspaceId: 'workspace-1', @@ -554,6 +563,118 @@ describe('knowledge search application use case', () => { expect(mocks.resolveWorkspace).not.toHaveBeenCalled() expect(mocks.getKnowledgeBase).not.toHaveBeenCalled() + expect(mocks.getKnowledgeBases).not.toHaveBeenCalled() + expect(mocks.getTagDefinitionsBatch).not.toHaveBeenCalled() + }) + + it('loads references and tags once for twenty bases while preserving requested order', async () => { + const ids = Array.from({ length: 20 }, (_, index) => `knowledge-${20 - index}`) + mocks.getKnowledgeBases.mockResolvedValue(ids.map((id) => ({ ...knowledgeBase, id }))) + mocks.getTagDefinitionsBatch.mockResolvedValue(new Map(ids.map((id) => [id, []]))) + + const result = await searchKnowledge.execute({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: { knowledgeBaseIds: ids, query: 'answer', topK: 5 }, + }) + + expect(mocks.getKnowledgeBases).toHaveBeenCalledExactlyOnceWith(ids) + expect(mocks.getTagDefinitionsBatch).toHaveBeenCalledExactlyOnceWith(ids) + expect(result.knowledgeBaseIds).toEqual(ids) + expect(result.knowledgeBaseId).toBe(ids[0]) + expect(result.knowledgeBases.map((base) => base.id)).toEqual(ids) + expect(mocks.executeSearch).toHaveBeenCalledWith( + expect.objectContaining({ knowledgeBaseIds: ids }) + ) + }) + + it('preserves duplicate requested bases in retrieval and the response', async () => { + const ids = ['knowledge-2', 'knowledge-1', 'knowledge-2'] + mocks.getKnowledgeBases.mockResolvedValue(ids.map((id) => ({ ...knowledgeBase, id }))) + + const result = await searchKnowledge.execute({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: { knowledgeBaseIds: ids, query: 'answer', topK: 5 }, + }) + + expect(result.knowledgeBaseIds).toEqual(ids) + expect(mocks.executeSearch).toHaveBeenCalledWith( + expect.objectContaining({ knowledgeBaseIds: ids }) + ) + }) + + it('preserves missing-id order and duplicates in the concealed error before authorization or billing', async () => { + mocks.getKnowledgeBases.mockResolvedValue([null, knowledgeBase, null, null]) + + await expect( + searchKnowledge.execute({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: { + knowledgeBaseIds: ['missing-2', 'knowledge-1', 'missing-1', 'missing-2'], + query: 'answer', + topK: 5, + }, + }) + ).rejects.toMatchObject({ + code: 'not_found', + message: 'Knowledge bases not found or access denied: missing-2, missing-1, missing-2', + }) + expect(mocks.resolvePermission).not.toHaveBeenCalled() + expect(mocks.resolveBilling).not.toHaveBeenCalled() + expect(mocks.executeSearch).not.toHaveBeenCalled() + }) + + it('rejects a batch spanning different canonical workspaces before billing', async () => { + mocks.getKnowledgeBases.mockResolvedValue([ + knowledgeBase, + { ...knowledgeBase, id: 'knowledge-2', workspaceId: 'workspace-2' }, + ]) + + await expect( + searchKnowledge.execute({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: { knowledgeBaseIds: ['knowledge-1', 'knowledge-2'], query: 'answer', topK: 5 }, + }) + ).rejects.toMatchObject({ + code: 'validation', + message: 'Selected knowledge bases must belong to the same workspace', + }) + expect(mocks.resolveBilling).not.toHaveBeenCalled() + expect(mocks.executeSearch).not.toHaveBeenCalled() + }) + + it('reuses the tag filter batch when naming result metadata', async () => { + const ids = ['knowledge-1', 'knowledge-2'] + mocks.getKnowledgeBases.mockResolvedValue(ids.map((id) => ({ ...knowledgeBase, id }))) + mocks.getTagDefinitionsBatch.mockResolvedValue( + new Map( + ids.map((id) => [ + id, + [{ knowledgeBaseId: id, tagSlot: 'tag1', displayName: 'team', fieldType: 'text' }], + ]) + ) + ) + mocks.executeSearch.mockResolvedValue([ + { + id: 'chunk-1', + documentId: 'document-1', + knowledgeBaseId: ids[0], + content: 'answer', + tag1: 'docs', + }, + ]) + + const result = await searchKnowledge.execute({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: { + knowledgeBaseIds: ids, + topK: 5, + tagFilters: [{ tagName: 'team', operator: 'eq', value: 'docs' }], + }, + }) + + expect(mocks.getTagDefinitionsBatch).toHaveBeenCalledExactlyOnceWith(ids) + expect(result.results[0].metadata).toEqual({ team: 'docs' }) + expect(mocks.generateEmbedding).not.toHaveBeenCalled() }) it('rejects multi-knowledge-base tag filters without embedding spend', async () => { diff --git a/apps/sim/lib/knowledge/application/search.ts b/apps/sim/lib/knowledge/application/search.ts index a47c6f7141c..c5d83ca908a 100644 --- a/apps/sim/lib/knowledge/application/search.ts +++ b/apps/sim/lib/knowledge/application/search.ts @@ -49,13 +49,13 @@ import { import { importKnowledgeSearchResultSecretProvenance } from '@/lib/knowledge/secret-provenance' import { type ActiveKnowledgeBaseReference, - getActiveKnowledgeBaseReference, + getActiveKnowledgeBaseReferences, } from '@/lib/knowledge/service' import { type KnowledgeTagNameFilter, resolveKnowledgeTagFilters, } from '@/lib/knowledge/tags/filter-resolution' -import { getDocumentTagDefinitions } from '@/lib/knowledge/tags/service' +import { getDocumentTagDefinitionsByKnowledgeBaseIds } from '@/lib/knowledge/tags/service' import type { DocumentTagDefinition } from '@/lib/knowledge/tags/types' import type { StructuredFilter } from '@/lib/knowledge/types' import { estimateTokenCount } from '@/lib/tokenization/estimators' @@ -189,9 +189,7 @@ async function resolveKnowledgeSearchContext( `topK must be an integer between 1 and ${KNOWLEDGE_SEARCH_COST_POLICY.maxTopK}` ) } - const knowledgeBases = await Promise.all( - input.knowledgeBaseIds.map(getActiveKnowledgeBaseReference) - ) + const knowledgeBases = await getActiveKnowledgeBaseReferences(input.knowledgeBaseIds) const missingIds = input.knowledgeBaseIds.filter((_, index) => { const knowledgeBase = knowledgeBases[index] return !knowledgeBase || (!knowledgeBase.workspaceId && !knowledgeBase.organizationId) @@ -558,18 +556,16 @@ export const searchKnowledge = defineAuthorizedKnowledgeUseCase({ } } - const tagDefinitionEntries = await Promise.all( - knowledgeBaseIds.map(async (knowledgeBaseId) => { - const definitions = - definitionsByKnowledgeBase.get(knowledgeBaseId) ?? - (await getDocumentTagDefinitions(knowledgeBaseId)) - return [ - knowledgeBaseId, - new Map(definitions.map((definition) => [definition.tagSlot, definition.displayName])), - ] as const - }) + if (filters.length === 0) { + definitionsByKnowledgeBase = + await getDocumentTagDefinitionsByKnowledgeBaseIds(knowledgeBaseIds) + } + const tagMaps = new Map( + [...definitionsByKnowledgeBase].map(([knowledgeBaseId, definitions]) => [ + knowledgeBaseId, + new Map(definitions.map((definition) => [definition.tagSlot, definition.displayName])), + ]) ) - const tagMaps = new Map(tagDefinitionEntries) /** * Always read: the provenance snapshot vouches for the name, URL, and tags * a model may see, but the source card's modified time and connector type diff --git a/apps/sim/lib/knowledge/search/queries-github-discovery.test.ts b/apps/sim/lib/knowledge/search/queries-github-discovery.test.ts new file mode 100644 index 00000000000..76e6ef77d8c --- /dev/null +++ b/apps/sim/lib/knowledge/search/queries-github-discovery.test.ts @@ -0,0 +1,162 @@ +/** @vitest-environment node */ +import { + dbChainMockFns, + hasMockCondition, + queueTableRows, + resetDbChainMock, + schemaMock, +} from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { createKnowledgeAccessProvider } from '@/lib/knowledge/access/scope' +import { handleTagOnlySearch } from '@/lib/knowledge/search/queries' + +vi.mock('@/lib/knowledge/access/availability', () => ({ + resolveKnowledgeAccessAvailability: async () => ({ memberScoped: true, sourceMirrored: false }), +})) +vi.mock('@/lib/workspaces/permissions/utils', () => ({ checkWorkspaceAccess: vi.fn() })) +vi.mock('@/lib/credentials/managed-oauth', () => ({ resolveManagedOAuthToken: vi.fn() })) +vi.mock('@/lib/core/security/encryption', () => ({ decryptSecret: vi.fn() })) +vi.mock('@/lib/oauth/github-installation', () => ({ + parseGitHubInstallationBinding: vi.fn(), + assertGitHubInstallationActive: vi.fn(), + assertGitHubInstallationRepositoryActive: vi.fn(), +})) + +async function createReaderProvider(hasGitHubReader = true) { + queueTableRows(schemaMock.member, [{ id: 'membership-1' }]) + queueTableRows(schemaMock.user, [ + { + credentialId: 'github-reader', + providerId: hasGitHubReader ? 'github-repositories' : 'gmail', + providerSubjectId: '42', + providerTenantId: null, + }, + ]) + const provider = createKnowledgeAccessProvider( + { kind: 'session', userId: 'reader', sessionId: 'session-1' }, + { organizationId: 'org-1', knowledgeBaseIds: ['index-1'] } + ) + const access = await provider.get() + return { provider, access } +} + +describe('GitHub discovery through the request access provider', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + }) + + it.each([true, false])( + 'skips discovery for explicit Gmail with GitHub reader present: %s', + async (hasGitHubReader) => { + const { provider, access } = await createReaderProvider(hasGitHubReader) + await provider.getForConnectors(['gmail-source']) + expect( + dbChainMockFns.from.mock.calls.filter(([table]) => table === schemaMock.knowledgeConnector) + ).toHaveLength(hasGitHubReader ? 1 : 0) + dbChainMockFns.from.mockClear() + + queueTableRows(schemaMock.embedding, [ + { + id: 'gmail', + documentId: 'gmail-doc', + connectorId: 'gmail-source', + installationSource: false, + }, + ]) + const hydrated = [{ id: 'gmail', content: 'permitted content' }] + queueTableRows(schemaMock.embedding, hydrated) + + expect( + await handleTagOnlySearch({ + knowledgeBaseIds: ['index-1'], + topK: 1, + access, + accessProvider: provider, + filters: { source: 'gmail' }, + structuredFilters: [ + { tagSlot: 'tag1', fieldType: 'text', operator: 'eq', value: 'release' }, + ], + }) + ).toEqual(hydrated) + expect(dbChainMockFns.from.mock.calls.map(([table]) => table)).toEqual([ + schemaMock.embedding, + schemaMock.embedding, + ]) + } + ) + + it.each([undefined, '', 'github'])( + 'keeps current discovery for classic GitHub with source: %s', + async (source) => { + const { provider, access } = await createReaderProvider() + queueTableRows(schemaMock.embedding, [ + { + id: 'github', + documentId: 'github-doc', + connectorId: 'classic-github', + installationSource: false, + }, + { + id: 'github-second', + documentId: 'github-doc', + connectorId: 'classic-github', + installationSource: false, + }, + ]) + queueTableRows(schemaMock.embedding, [{ id: 'github', content: 'permitted content' }]) + + await handleTagOnlySearch({ + knowledgeBaseIds: ['index-1'], + topK: 1, + access, + accessProvider: provider, + filters: { source }, + structuredFilters: [ + { tagSlot: 'tag1', fieldType: 'text', operator: 'eq', value: 'release' }, + ], + }) + expect( + dbChainMockFns.from.mock.calls.filter(([table]) => table === schemaMock.knowledgeConnector) + ).toHaveLength(1) + expect( + dbChainMockFns.where.mock.calls.some(([condition]) => + hasMockCondition( + condition, + (node) => + node.type === 'inArray' && + node.column === schemaMock.knowledgeConnector.id && + JSON.stringify(node.values) === JSON.stringify(['classic-github']) + ) + ) + ).toBe(true) + } + ) + + it('keeps upload hydration without source discovery', async () => { + const { provider, access } = await createReaderProvider() + dbChainMockFns.from.mockClear() + queueTableRows(schemaMock.embedding, [ + { id: 'upload', documentId: 'upload-doc', connectorId: null, installationSource: false }, + ]) + const hydrated = [{ id: 'upload', content: 'permitted content' }] + queueTableRows(schemaMock.embedding, hydrated) + + expect( + await handleTagOnlySearch({ + knowledgeBaseIds: ['index-1'], + topK: 1, + access, + accessProvider: provider, + filters: { source: 'upload' }, + structuredFilters: [ + { tagSlot: 'tag1', fieldType: 'text', operator: 'eq', value: 'release' }, + ], + }) + ).toEqual(hydrated) + expect(dbChainMockFns.from.mock.calls.map(([table]) => table)).toEqual([ + schemaMock.embedding, + schemaMock.embedding, + ]) + }) +}) diff --git a/apps/sim/lib/knowledge/search/queries.test.ts b/apps/sim/lib/knowledge/search/queries.test.ts index e8edea4491d..49dd48de8c9 100644 --- a/apps/sim/lib/knowledge/search/queries.test.ts +++ b/apps/sim/lib/knowledge/search/queries.test.ts @@ -576,6 +576,77 @@ describe('live repository authorization follows ranked candidates', () => { } ) + it.each(['vector', 'tag-vector', 'tags', 'keyword'] as const)( + '%s skips discovery for an explicit non-GitHub source and retains full hydration', + async (mode) => { + getForConnectors.mockResolvedValue(identity) + queueTableRows(schemaMock.embedding, [ + { ...candidate('gmail', 'gmail-source'), installationSource: false }, + ]) + const hydrated = [{ id: 'gmail', content: 'current permitted content' }] + queueTableRows(schemaMock.embedding, hydrated) + const searchParams = { ...params, filters: { source: 'gmail' } } + const rows = + mode === 'vector' + ? await handleVectorOnlySearch(searchParams) + : mode === 'tag-vector' + ? await handleTagAndVectorSearch(searchParams) + : mode === 'tags' + ? await handleTagOnlySearch(searchParams) + : await executeKeywordSearch({ + ...searchParams, + query: 'release', + queryVector: searchParams.queryVector!, + }) + + expect(rows).toEqual(hydrated) + expect(getForConnectors).toHaveBeenCalledExactlyOnceWith([], undefined) + for (const [condition] of dbChainMockFns.where.mock.calls) { + expect(JSON.stringify(condition)).toContain('gmail') + } + const hydration = JSON.stringify(dbChainMockFns.where.mock.calls[1][0]) + expect(hydration).toContain('acl') + expect(hydration).toContain('knowledgeConnectorMember') + expect(dbChainMockFns.select).toHaveBeenCalledTimes(2) + } + ) + + it.each([undefined, '', 'github'])( + 'retains discovery for classic GitHub when source is %s', + async (source) => { + queueTableRows(schemaMock.embedding, [ + { ...candidate('selected', 'allowed-source'), installationSource: false }, + { ...candidate('second', 'allowed-source'), installationSource: false }, + ]) + queueTableRows(schemaMock.embedding, [{ id: 'selected', content: 'verified result' }]) + + expect(await handleTagOnlySearch({ ...params, filters: { source } })).toEqual([ + { id: 'selected', content: 'verified result' }, + ]) + expect(getForConnectors).toHaveBeenCalledExactlyOnceWith(['allowed-source'], undefined) + expect(JSON.stringify(dbChainMockFns.where.mock.calls[1][0])).toContain('github_read_grant') + } + ) + + it('retains every connector in unfiltered mixed pages', async () => { + queueTableRows(schemaMock.embedding, [ + { ...candidate('gmail', 'gmail-source'), installationSource: false }, + { ...candidate('classic', 'classic-source'), installationSource: false }, + candidate('selected', 'allowed-source'), + candidate('selected-second-chunk', 'allowed-source'), + { ...candidate('upload', 'unused'), connectorId: null, installationSource: false }, + ]) + queueTableRows(schemaMock.embedding, [{ id: 'selected', content: 'verified result' }]) + + expect(await handleTagOnlySearch(params)).toEqual([ + { id: 'selected', content: 'verified result' }, + ]) + expect(getForConnectors).toHaveBeenCalledExactlyOnceWith( + ['gmail-source', 'classic-source', 'allowed-source'], + undefined + ) + }) + it('refills after a denied repository instead of letting its matches consume the result limit', async () => { getForConnectors.mockResolvedValueOnce(identity) queueTableRows(schemaMock.embedding, [candidate('denied', 'revoked-source')]) @@ -612,16 +683,23 @@ describe('live repository authorization follows ranked candidates', () => { expect(getForConnectors).toHaveBeenCalledOnce() }) - it('propagates caller cancellation before content hydration', async () => { - const cancellation = new AbortController() - getForConnectors.mockImplementation(async () => { - cancellation.abort(new Error('Search cancelled')) - return allowed - }) - queueTableRows(schemaMock.embedding, [candidate('selected', 'allowed-source')]) - await expect(handleTagOnlySearch({ ...params, signal: cancellation.signal })).rejects.toThrow( - 'Search cancelled' - ) - expect(dbChainMockFns.select).toHaveBeenCalledOnce() - }) + it.each([undefined, 'gmail'])( + 'propagates caller cancellation before hydration with source %s', + async (source) => { + const cancellation = new AbortController() + getForConnectors.mockImplementation(async () => { + cancellation.abort(new Error('Search cancelled')) + return allowed + }) + queueTableRows(schemaMock.embedding, [candidate('selected', 'allowed-source')]) + await expect( + handleTagOnlySearch({ ...params, filters: { source }, signal: cancellation.signal }) + ).rejects.toThrow('Search cancelled') + expect(getForConnectors).toHaveBeenCalledExactlyOnceWith( + source ? [] : ['allowed-source'], + cancellation.signal + ) + expect(dbChainMockFns.select).toHaveBeenCalledOnce() + } + ) }) diff --git a/apps/sim/lib/knowledge/search/queries.ts b/apps/sim/lib/knowledge/search/queries.ts index dbe5d64735b..cecc89281b1 100644 --- a/apps/sim/lib/knowledge/search/queries.ts +++ b/apps/sim/lib/knowledge/search/queries.ts @@ -454,6 +454,7 @@ const LIVE_SEARCH_BUDGET_MS = 8000 */ async function selectAuthorizedSearchResults(input: { accessProvider: KnowledgeAccessProvider + filters?: WorkspaceSearchFilters signal?: AbortSignal topK: number selectPage: ( @@ -478,11 +479,17 @@ async function selectAuthorizedSearchResults(input: { const candidates = await input.selectPage(pageSize, offset, [...excludedSources]) if (!candidates.length) break scanned += candidates.length - const connectorIds = [ - ...new Set( - candidates.flatMap((candidate) => (candidate.connectorId ? [candidate.connectorId] : [])) - ), - ] + /** Candidate and hydration queries enforce this source filter; connector types are immutable. */ + const connectorIds = + input.filters?.source && input.filters.source !== 'github' + ? [] + : [ + ...new Set( + candidates.flatMap((candidate) => + candidate.connectorId ? [candidate.connectorId] : [] + ) + ), + ] const access = await input.accessProvider.getForConnectors(connectorIds, input.signal) input.signal?.throwIfAborted() const grantedSources = new Set( @@ -578,6 +585,7 @@ export async function handleTagOnlySearch(params: SearchParams): Promise @@ -705,6 +713,7 @@ function selectLiveVectorResults( const conditions = [inArray(embedding.knowledgeBaseId, params.knowledgeBaseIds), ...filters] return selectAuthorizedSearchResults({ accessProvider, + filters: params.filters, signal: params.signal, topK: params.topK, selectPage: (limit, offset, excludedSources) => @@ -818,6 +827,7 @@ export async function executeKeywordSearch(params: KeywordSearchParams): Promise ] return selectAuthorizedSearchResults({ accessProvider: params.accessProvider, + filters: params.filters, signal: params.signal, topK, selectPage: (limit, offset, excludedSources) => diff --git a/apps/sim/lib/knowledge/service.test.ts b/apps/sim/lib/knowledge/service.test.ts index 8d05b403c38..b27489f6e9e 100644 --- a/apps/sim/lib/knowledge/service.test.ts +++ b/apps/sim/lib/knowledge/service.test.ts @@ -6,6 +6,7 @@ import { hasMockCondition, permissionsMock, permissionsMockFns, + queueTableRows, resetDbChainMock, schemaMock, } from '@sim/testing' @@ -31,6 +32,7 @@ vi.mock('@/lib/billing/storage', () => ({ import { findActiveKnowledgeBasesByExactName, getActiveKnowledgeBaseReference, + getActiveKnowledgeBaseReferences, getKnowledgeBaseById, getWorkspaceKnowledgeBases, KnowledgeBasePermissionError, @@ -81,6 +83,71 @@ describe('knowledge base references', () => { await expect(getActiveKnowledgeBaseReference('missing')).resolves.toBeNull() }) + it('loads twenty references in one query without changing their projection or input order', async () => { + const ids = Array.from({ length: 20 }, (_, index) => `kb-${index}`) + const references = ids.map((id) => ({ id, chunkingConfig: { maxSize: 512 } })) + queueTableRows(schemaMock.knowledgeBase, [...references].reverse()) + + await expect(getActiveKnowledgeBaseReferences(ids)).resolves.toEqual(references) + + expect(dbChainMockFns.select).toHaveBeenCalledOnce() + expect(dbChainMockFns.from).toHaveBeenCalledOnce() + const projection = dbChainMockFns.select.mock.calls[0][0] + const [condition] = dbChainMockFns.where.mock.calls[0] + expect( + hasMockCondition( + condition, + (node) => node.type === 'isNull' && node.column === schemaMock.knowledgeBase.deletedAt + ) + ).toBe(true) + expect( + hasMockCondition( + condition, + (node) => + node.type === 'inArray' && + node.column === schemaMock.knowledgeBase.id && + JSON.stringify(node.values) === JSON.stringify(ids) + ) + ).toBe(true) + expect(dbChainMockFns.leftJoin).not.toHaveBeenCalled() + expect(dbChainMockFns.groupBy).not.toHaveBeenCalled() + + await getActiveKnowledgeBaseReference(ids[0]) + expect(dbChainMockFns.select.mock.calls[1][0]).toEqual(projection) + }) + + it('preserves duplicate and absent reference positions without querying duplicate ids', async () => { + const reference = { id: 'kb-1', chunkingConfig: {} } + queueTableRows(schemaMock.knowledgeBase, [reference]) + + await expect( + getActiveKnowledgeBaseReferences(['missing', 'kb-1', 'missing', 'kb-1']) + ).resolves.toEqual([null, reference, null, reference]) + expect(dbChainMockFns.select).toHaveBeenCalledOnce() + expect( + hasMockCondition( + dbChainMockFns.where.mock.calls[0][0], + (node) => + node.type === 'inArray' && + JSON.stringify(node.values) === JSON.stringify(['missing', 'kb-1']) + ) + ).toBe(true) + }) + + it('does not query an empty reference batch and retains the singleton query shape', async () => { + await expect(getActiveKnowledgeBaseReferences([])).resolves.toEqual([]) + expect(dbChainMockFns.select).not.toHaveBeenCalled() + await expect(getActiveKnowledgeBaseReferences(['missing'])).resolves.toEqual([null]) + expect(dbChainMockFns.select).toHaveBeenCalledOnce() + expect(dbChainMockFns.limit).toHaveBeenCalledWith(1) + }) + + it('propagates reference batch database failures', async () => { + const failure = new Error('reference database unavailable') + dbChainMockFns.where.mockRejectedValueOnce(failure) + await expect(getActiveKnowledgeBaseReferences(['kb-1', 'kb-2'])).rejects.toBe(failure) + }) + it('preserves aggregate counts for knowledge-base detail consumers', async () => { dbChainMockFns.limit.mockResolvedValueOnce([ { diff --git a/apps/sim/lib/knowledge/service.ts b/apps/sim/lib/knowledge/service.ts index 146f3c466ab..1bd2285032b 100644 --- a/apps/sim/lib/knowledge/service.ts +++ b/apps/sim/lib/knowledge/service.ts @@ -875,6 +875,23 @@ export type ActiveKnowledgeBaseReference = Omit< 'tokenCount' | 'docCount' | 'connectorTypes' | 'hasPermissionScopedConnector' > +const ACTIVE_KNOWLEDGE_BASE_REFERENCE_FIELDS = { + id: knowledgeBase.id, + userId: knowledgeBase.userId, + name: knowledgeBase.name, + isSearchIndex: knowledgeBase.isSearchIndex, + description: knowledgeBase.description, + embeddingModel: knowledgeBase.embeddingModel, + embeddingDimension: knowledgeBase.embeddingDimension, + chunkingConfig: knowledgeBase.chunkingConfig, + createdAt: knowledgeBase.createdAt, + updatedAt: knowledgeBase.updatedAt, + deletedAt: knowledgeBase.deletedAt, + workspaceId: knowledgeBase.workspaceId, + organizationId: knowledgeBase.organizationId, + folderId: knowledgeBase.folderId, +} + /** * Canonical identity and configuration for application authorization and retrieval. * Reading a reference never scans the base's documents to compute display counts. @@ -883,22 +900,7 @@ export async function getActiveKnowledgeBaseReference( knowledgeBaseId: string ): Promise { const [row] = await db - .select({ - id: knowledgeBase.id, - userId: knowledgeBase.userId, - name: knowledgeBase.name, - isSearchIndex: knowledgeBase.isSearchIndex, - description: knowledgeBase.description, - embeddingModel: knowledgeBase.embeddingModel, - embeddingDimension: knowledgeBase.embeddingDimension, - chunkingConfig: knowledgeBase.chunkingConfig, - createdAt: knowledgeBase.createdAt, - updatedAt: knowledgeBase.updatedAt, - deletedAt: knowledgeBase.deletedAt, - workspaceId: knowledgeBase.workspaceId, - organizationId: knowledgeBase.organizationId, - folderId: knowledgeBase.folderId, - }) + .select(ACTIVE_KNOWLEDGE_BASE_REFERENCE_FIELDS) .from(knowledgeBase) .where(and(eq(knowledgeBase.id, knowledgeBaseId), isNull(knowledgeBase.deletedAt))) .limit(1) @@ -906,6 +908,29 @@ export async function getActiveKnowledgeBaseReference( return row ? { ...row, chunkingConfig: row.chunkingConfig as ChunkingConfig } : null } +/** Loads active references in one statement while preserving requested order and missing entries. */ +export async function getActiveKnowledgeBaseReferences( + knowledgeBaseIds: readonly string[] +): Promise> { + if (knowledgeBaseIds.length === 0) return [] + if (knowledgeBaseIds.length === 1) + return [await getActiveKnowledgeBaseReference(knowledgeBaseIds[0])] + + const rows = await db + .select(ACTIVE_KNOWLEDGE_BASE_REFERENCE_FIELDS) + .from(knowledgeBase) + .where( + and( + inArray(knowledgeBase.id, [...new Set(knowledgeBaseIds)]), + isNull(knowledgeBase.deletedAt) + ) + ) + const byId = new Map( + rows.map((row) => [row.id, { ...row, chunkingConfig: row.chunkingConfig as ChunkingConfig }]) + ) + return knowledgeBaseIds.map((id) => byId.get(id) ?? null) +} + /** * Get a single knowledge base by ID */ diff --git a/apps/sim/lib/knowledge/tags/filter-resolution.test.ts b/apps/sim/lib/knowledge/tags/filter-resolution.test.ts index c1a4322cf57..7a88bfc8817 100644 --- a/apps/sim/lib/knowledge/tags/filter-resolution.test.ts +++ b/apps/sim/lib/knowledge/tags/filter-resolution.test.ts @@ -3,12 +3,13 @@ */ import { beforeEach, describe, expect, it, vi } from 'vitest' -const { mockGetDocumentTagDefinitions } = vi.hoisted(() => ({ +const { mockGetDocumentTagDefinitions, mockGetDocumentTagDefinitionsBatch } = vi.hoisted(() => ({ mockGetDocumentTagDefinitions: vi.fn(), + mockGetDocumentTagDefinitionsBatch: vi.fn(), })) vi.mock('@/lib/knowledge/tags/service', () => ({ - getDocumentTagDefinitions: mockGetDocumentTagDefinitions, + getDocumentTagDefinitionsByKnowledgeBaseIds: mockGetDocumentTagDefinitionsBatch, })) import { @@ -38,6 +39,12 @@ function definition( describe('resolveKnowledgeTagFilters', () => { beforeEach(() => { vi.clearAllMocks() + mockGetDocumentTagDefinitionsBatch.mockImplementation( + async (ids: string[]) => + new Map( + await Promise.all(ids.map(async (id) => [id, await mockGetDocumentTagDefinitions(id)])) + ) + ) }) it('resolves a display name to the slot it is stored in', async () => { diff --git a/apps/sim/lib/knowledge/tags/filter-resolution.ts b/apps/sim/lib/knowledge/tags/filter-resolution.ts index 57b4bde8854..c374105b284 100644 --- a/apps/sim/lib/knowledge/tags/filter-resolution.ts +++ b/apps/sim/lib/knowledge/tags/filter-resolution.ts @@ -2,7 +2,7 @@ import { KNOWLEDGE_TAG_FILTER_OPERATORS_BY_FIELD_TYPE } from '@/lib/api/contract import { OrchestrationError } from '@/lib/core/orchestration/types' import { SUPPORTED_FIELD_TYPES } from '@/lib/knowledge/constants' import type { TagFilterCondition } from '@/lib/knowledge/documents/tag-filter' -import { getDocumentTagDefinitions } from '@/lib/knowledge/tags/service' +import { getDocumentTagDefinitionsByKnowledgeBaseIds } from '@/lib/knowledge/tags/service' import type { DocumentTagDefinition } from '@/lib/knowledge/tags/types' import { buildUndefinedTagsError, validateTagValue } from '@/lib/knowledge/tags/utils' import type { StructuredFilter } from '@/lib/knowledge/types' @@ -72,15 +72,11 @@ export async function resolveKnowledgeTagFilters( filters: KnowledgeTagNameFilter[], knowledgeBaseIds: string[] ): Promise { - const definitionEntries = await Promise.all( - knowledgeBaseIds.map( - async (knowledgeBaseId) => - [knowledgeBaseId, await getDocumentTagDefinitions(knowledgeBaseId)] as const - ) - ) - const definitionsByKnowledgeBase = new Map(definitionEntries) + const definitionsByKnowledgeBase = + await getDocumentTagDefinitionsByKnowledgeBaseIds(knowledgeBaseIds) const sharedDefinitions = new Map() - for (const [, definitions] of definitionEntries) { + for (const knowledgeBaseId of knowledgeBaseIds) { + const definitions = definitionsByKnowledgeBase.get(knowledgeBaseId)! const currentByName = new Map( definitions.map((definition) => [ definition.displayName, diff --git a/apps/sim/lib/knowledge/tags/service.test.ts b/apps/sim/lib/knowledge/tags/service.test.ts index 2dc314fd369..92e8c31f515 100644 --- a/apps/sim/lib/knowledge/tags/service.test.ts +++ b/apps/sim/lib/knowledge/tags/service.test.ts @@ -3,7 +3,7 @@ */ import { knowledgeBaseTagDefinitions } from '@sim/db/schema' -import { dbChainMockFns, queueTableRows, resetDbChainMock } from '@sim/testing' +import { dbChainMockFns, hasMockCondition, queueTableRows, resetDbChainMock } from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' vi.mock('@sim/utils/id', () => ({ @@ -14,6 +14,8 @@ vi.mock('@sim/utils/id', () => ({ import { createOrUpdateTagDefinitionsBulk, createTagDefinition, + getDocumentTagDefinitions, + getDocumentTagDefinitionsByKnowledgeBaseIds, updateTagDefinition, } from '@/lib/knowledge/tags/service' @@ -32,6 +34,83 @@ function existingDefinition(overrides: Record) { } } +describe('getDocumentTagDefinitionsByKnowledgeBaseIds', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + }) + + it('loads twenty bases in one query and retains each base in requested order', async () => { + const ids = Array.from({ length: 20 }, (_, index) => `kb-${index}`) + queueTableRows(knowledgeBaseTagDefinitions, [ + existingDefinition({ knowledgeBaseId: 'kb-2', tagSlot: 'number1', fieldType: 'number' }), + existingDefinition({ knowledgeBaseId: 'kb-1', tagSlot: 'tag1' }), + existingDefinition({ knowledgeBaseId: 'kb-2', tagSlot: 'tag1' }), + ]) + + const result = await getDocumentTagDefinitionsByKnowledgeBaseIds(ids) + + expect([...result.keys()]).toEqual(ids) + expect(result.get('kb-0')).toEqual([]) + expect(result.get('kb-1')?.map((definition) => definition.tagSlot)).toEqual(['tag1']) + expect(result.get('kb-2')?.map((definition) => definition.tagSlot)).toEqual(['number1', 'tag1']) + expect(dbChainMockFns.select).toHaveBeenCalledOnce() + expect(dbChainMockFns.orderBy).toHaveBeenCalledWith(knowledgeBaseTagDefinitions.tagSlot) + expect( + hasMockCondition( + dbChainMockFns.where.mock.calls[0][0], + (node) => + node.type === 'inArray' && + node.column === knowledgeBaseTagDefinitions.knowledgeBaseId && + JSON.stringify(node.values) === JSON.stringify(ids) + ) + ).toBe(true) + const projection = dbChainMockFns.select.mock.calls[0][0] + await getDocumentTagDefinitions('kb-0') + expect(dbChainMockFns.select.mock.calls[1][0]).toEqual(projection) + }) + + it('keeps empty bases and deduplicates requested ids', async () => { + const result = await getDocumentTagDefinitionsByKnowledgeBaseIds(['kb-2', 'kb-1', 'kb-2']) + expect([...result]).toEqual([ + ['kb-2', []], + ['kb-1', []], + ]) + expect(dbChainMockFns.select).toHaveBeenCalledOnce() + expect( + hasMockCondition( + dbChainMockFns.where.mock.calls[0][0], + (node) => + node.type === 'inArray' && + JSON.stringify(node.values) === JSON.stringify(['kb-2', 'kb-1']) + ) + ).toBe(true) + }) + + it('skips an empty batch and preserves the singleton equality predicate', async () => { + expect(await getDocumentTagDefinitionsByKnowledgeBaseIds([])).toEqual(new Map()) + expect(dbChainMockFns.select).not.toHaveBeenCalled() + expect(await getDocumentTagDefinitionsByKnowledgeBaseIds(['kb-1'])).toEqual( + new Map([['kb-1', []]]) + ) + expect(dbChainMockFns.select).toHaveBeenCalledOnce() + expect( + hasMockCondition( + dbChainMockFns.where.mock.calls[0][0], + (node) => node.type === 'eq' && node.right === 'kb-1' + ) + ).toBe(true) + }) + + it('propagates batch database failures', async () => { + const failure = new Error('tag database unavailable') + dbChainMockFns.orderBy.mockRejectedValueOnce(failure) + await expect(getDocumentTagDefinitionsByKnowledgeBaseIds(['kb-1', 'kb-2'])).rejects.toBe( + failure + ) + }) +}) + describe('createOrUpdateTagDefinitionsBulk', () => { beforeEach(() => { vi.clearAllMocks() diff --git a/apps/sim/lib/knowledge/tags/service.ts b/apps/sim/lib/knowledge/tags/service.ts index 8f837de9ccb..7a599fea365 100644 --- a/apps/sim/lib/knowledge/tags/service.ts +++ b/apps/sim/lib/knowledge/tags/service.ts @@ -9,7 +9,7 @@ import { import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' -import { and, eq, isNotNull, isNull, or, sql } from 'drizzle-orm' +import { and, eq, inArray, isNotNull, isNull, or, sql } from 'drizzle-orm' import { OrchestrationError } from '@/lib/core/orchestration/types' import type { DbOrTx, DbTransaction } from '@/lib/db/types' import { @@ -201,23 +201,23 @@ export async function getNextAvailableSlot( return null // All slots for this field type are used } -/** - * Get all tag definitions for a knowledge base - */ +const DOCUMENT_TAG_DEFINITION_FIELDS = { + id: knowledgeBaseTagDefinitions.id, + knowledgeBaseId: knowledgeBaseTagDefinitions.knowledgeBaseId, + tagSlot: knowledgeBaseTagDefinitions.tagSlot, + displayName: knowledgeBaseTagDefinitions.displayName, + fieldType: knowledgeBaseTagDefinitions.fieldType, + createdAt: knowledgeBaseTagDefinitions.createdAt, + updatedAt: knowledgeBaseTagDefinitions.updatedAt, +} + +/** Get all tag definitions for a knowledge base. */ export async function getDocumentTagDefinitions( knowledgeBaseId: string, txDb?: DbOrTx ): Promise { const definitions = await (txDb ?? db) - .select({ - id: knowledgeBaseTagDefinitions.id, - knowledgeBaseId: knowledgeBaseTagDefinitions.knowledgeBaseId, - tagSlot: knowledgeBaseTagDefinitions.tagSlot, - displayName: knowledgeBaseTagDefinitions.displayName, - fieldType: knowledgeBaseTagDefinitions.fieldType, - createdAt: knowledgeBaseTagDefinitions.createdAt, - updatedAt: knowledgeBaseTagDefinitions.updatedAt, - }) + .select(DOCUMENT_TAG_DEFINITION_FIELDS) .from(knowledgeBaseTagDefinitions) .where(eq(knowledgeBaseTagDefinitions.knowledgeBaseId, knowledgeBaseId)) .orderBy(knowledgeBaseTagDefinitions.tagSlot) @@ -228,6 +228,35 @@ export async function getDocumentTagDefinitions( })) } +/** Loads each requested base's slot-ordered definitions in one statement, including empty bases. */ +export async function getDocumentTagDefinitionsByKnowledgeBaseIds( + knowledgeBaseIds: readonly string[] +): Promise> { + const definitionsByKnowledgeBase = new Map( + knowledgeBaseIds.map((id) => [id, []]) + ) + if (definitionsByKnowledgeBase.size === 0) return definitionsByKnowledgeBase + if (definitionsByKnowledgeBase.size === 1) { + const id = knowledgeBaseIds[0] + definitionsByKnowledgeBase.set(id, await getDocumentTagDefinitions(id)) + return definitionsByKnowledgeBase + } + const definitions = await db + .select(DOCUMENT_TAG_DEFINITION_FIELDS) + .from(knowledgeBaseTagDefinitions) + .where( + inArray(knowledgeBaseTagDefinitions.knowledgeBaseId, [...definitionsByKnowledgeBase.keys()]) + ) + .orderBy(knowledgeBaseTagDefinitions.tagSlot) + for (const definition of definitions) { + definitionsByKnowledgeBase.get(definition.knowledgeBaseId)!.push({ + ...definition, + tagSlot: definition.tagSlot as string, + }) + } + return definitionsByKnowledgeBase +} + /** * Get all tag definitions for a knowledge base (alias for compatibility) */