diff --git a/apps/sim/app/api/knowledge/search/route.test.ts b/apps/sim/app/api/knowledge/search/route.test.ts index 445fe171c63..34af2782b41 100644 --- a/apps/sim/app/api/knowledge/search/route.test.ts +++ b/apps/sim/app/api/knowledge/search/route.test.ts @@ -47,6 +47,9 @@ describe('workspace search route', () => { expect(call.input.signal).toBe(request.signal) expect(call.input.allowPartialResults).toBe(true) expect(call.input.vectorBudgetMs).toBe(3000) + /** A person's search asks for reranking; the use case reranks when a credential exists. */ + expect(call.input.rerankerEnabled).toBe(true) + expect(call.input.rerankerModel).toBe('rerank-v4.0-fast') controller.abort() expect(call.input.signal.aborted).toBe(true) await expect(response.json()).resolves.toEqual({ diff --git a/apps/sim/app/api/knowledge/search/route.ts b/apps/sim/app/api/knowledge/search/route.ts index b1db5e04d39..0e2ed73f3ef 100644 --- a/apps/sim/app/api/knowledge/search/route.ts +++ b/apps/sim/app/api/knowledge/search/route.ts @@ -7,6 +7,7 @@ import { import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' import { knowledgeOperations } from '@/lib/knowledge/application/operations' import { searchScopedKnowledge } from '@/lib/knowledge/application/workspace-search' +import { DEFAULT_RERANKER_MODEL } from '@/lib/knowledge/reranker-models' import { sourceAuthor } from '@/lib/knowledge/search/author' const DIRECT_SEARCH_VECTOR_BUDGET_MS = 3000 @@ -28,6 +29,13 @@ export const POST = defineInternalJsonRoute({ topK: body.topK, allowPartialResults: true, vectorBudgetMs: DIRECT_SEARCH_VECTOR_BUDGET_MS, + /** + * A person's search is reranked by a cross-encoder whenever the workspace or the platform + * holds a key for one; the use case checks that before spending a call, and reranking stays + * best-effort, so a provider outage leaves the fused order in place. + */ + rerankerEnabled: true, + rerankerModel: DEFAULT_RERANKER_MODEL, surface: 'dashboard' as const, signal: request.signal, }), diff --git a/apps/sim/app/o/[organizationId]/search/search.test.tsx b/apps/sim/app/o/[organizationId]/search/search.test.tsx index ddf950eb08b..e61511d7e27 100644 --- a/apps/sim/app/o/[organizationId]/search/search.test.tsx +++ b/apps/sim/app/o/[organizationId]/search/search.test.tsx @@ -135,7 +135,7 @@ async function editDraft(value: string) { function expectVisibleQuery(query: string) { expect(searchInput().value).toBe(query) expect(container.querySelector('a[data-source-link]')?.textContent).toBe(`${query} launch plan`) - expect(mocks.search).toHaveBeenLastCalledWith(scope, query, {}) + expect(mocks.search).toHaveBeenLastCalledWith(scope, query, {}, 20) expect(document.activeElement).toBe(searchInput()) } diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx index c4a857f3c7d..93e3ba1f46d 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx @@ -48,10 +48,13 @@ afterEach(() => { act(() => root.unmount()) vi.unstubAllGlobals() }) -async function render(scope: ResourceScope = { kind: 'workspace', workspaceId: 'workspace' }) { +async function render( + scope: ResourceScope = { kind: 'workspace', workspaceId: 'workspace' }, + searchParams = '' +) { await act(async () => root.render( - + ) @@ -148,3 +151,133 @@ describe('source setup navigation', () => { expect(container.querySelector('a')?.getAttribute('href')).toBe(href) }) }) + +describe('result paging and the custom window', () => { + const result = (n: number) => ({ + documentId: `doc-${n}`, + knowledgeBaseId: 'kb', + knowledgeBaseName: 'Index', + documentName: `Document ${n}`, + sourceUrl: null, + connectorType: 'slack', + sourceModifiedAt: null, + author: null, + content: 'launch notes', + chunkIndex: 0, + similarity: 0.5, + }) + + it('offers more only after a full first page, and asks for the wider search on request', async () => { + mocks.overview.mockReturnValue({ data: { providers: [], hasSearchableDocuments: true } }) + const page = (length: number) => ({ + data: { + query: 'launch', + results: Array.from({ length }, (_, n) => result(n)), + retrieval: { status: 'complete', timedOutLegs: [] }, + }, + isPending: false, + isFetching: false, + isPlaceholderData: false, + isError: false, + refetch: mocks.retry, + }) + mocks.search.mockReturnValue(page(20)) + await render() + expect(mocks.search.mock.calls.at(-1)![3]).toBe(20) + const more = () => + [...container.querySelectorAll('button')].find((b) => b.textContent === 'Show more') + expect(more()).toBeDefined() + await act(async () => more()!.click()) + /** The wider search is its own request; the first paint was never widened. */ + expect(mocks.search.mock.calls.at(-1)![3]).toBe(50) + expect(more()).toBeUndefined() + mocks.search.mockReturnValue(page(7)) + await render() + expect(more()).toBeUndefined() + }) + + it('starts a refined search over at the first page after the reader asked for more', async () => { + mocks.overview.mockReturnValue({ + data: { + providers: [{ connectorType: 'slack', isSyncing: false }], + hasSearchableDocuments: true, + }, + }) + mocks.search.mockReturnValue({ + data: { + query: 'launch', + results: Array.from({ length: 20 }, (_, n) => result(n)), + retrieval: { status: 'complete', timedOutLegs: [] }, + }, + isPending: false, + isFetching: false, + isPlaceholderData: false, + isError: false, + refetch: mocks.retry, + }) + await render() + const button = (label: string) => + [...container.querySelectorAll('button')].find((b) => b.textContent === label)! + await act(async () => button('Show more').click()) + expect(mocks.search.mock.calls.at(-1)![3]).toBe(50) + await act(async () => button('Slack').click()) + expect(mocks.search.mock.calls.at(-1)![2]).toEqual({ source: 'slack' }) + expect(mocks.search.mock.calls.at(-1)![3]).toBe(20) + }) + + it('drops the custom days when another window is chosen', async () => { + mocks.overview.mockReturnValue({ data: { providers: [], hasSearchableDocuments: true } }) + mocks.search.mockReturnValue({ + data: { query: 'launch', results: [], retrieval: { status: 'complete', timedOutLegs: [] } }, + isPending: false, + isFetching: false, + isPlaceholderData: false, + isError: false, + refetch: mocks.retry, + }) + await render(undefined, '?updated=custom&from=2026-09-01&to=2026-09-10') + expect(mocks.search.mock.calls.at(-1)![2]).toHaveProperty('modifiedBefore') + const anyTime = [...container.querySelectorAll('button')].find( + (b) => b.textContent === 'Any time' + )! + await act(async () => anyTime.click()) + expect(mocks.search.mock.calls.at(-1)![2]).toEqual({}) + }) + + it('searches nothing while a custom window has no days yet', async () => { + mocks.overview.mockReturnValue({ data: { providers: [], hasSearchableDocuments: true } }) + mocks.search.mockReturnValue({ + data: undefined, + isPending: true, + isFetching: false, + isPlaceholderData: false, + isError: false, + refetch: mocks.retry, + }) + await render(undefined, '?updated=custom') + expect(mocks.search.mock.calls.at(-1)![1]).toBe('') + expect(container.textContent).toContain('Choose the days to search.') + /** The filters, and the picker among them, are shown so the days can be chosen. */ + expect(container.textContent).toContain('Updated between') + /** One day alone is not a window either; a deep link with only `from` waits for `to`. */ + await render(undefined, '?updated=custom&from=2026-09-01') + expect(mocks.search.mock.calls.at(-1)![1]).toBe('') + }) + + it('searches a custom window as an inclusive range of days', async () => { + mocks.overview.mockReturnValue({ data: { providers: [], hasSearchableDocuments: true } }) + mocks.search.mockReturnValue({ + data: { query: 'launch', results: [], retrieval: { status: 'complete', timedOutLegs: [] } }, + isPending: false, + isFetching: false, + isPlaceholderData: false, + isError: false, + refetch: mocks.retry, + }) + await render(undefined, '?updated=custom&from=2026-09-01&to=2026-09-10') + const filters = mocks.search.mock.calls.at(-1)![2] + /** The days are the reader's own: local midnight to the last millisecond of the local day. */ + expect(filters.modifiedAfter).toBe(new Date(2026, 8, 1).toISOString()) + expect(filters.modifiedBefore).toBe(new Date(2026, 8, 11, 0, 0, 0, -1).toISOString()) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx index 8ac809b11ec..64ac4323031 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx @@ -1,12 +1,13 @@ 'use client' import { useState } from 'react' -import { Chip, ChipLink, cn } from '@sim/emcn' +import { Chip, ChipDatePicker, ChipLink, cn } from '@sim/emcn' import { useQueryStates } from 'nuqs' import { ActivityStatus } from '@/components/ui/activity-status' -import type { - WorkspaceKnowledgeSearchResult, - WorkspaceSearchFilters, +import { + WORKSPACE_KNOWLEDGE_SEARCH_LIMITS, + type WorkspaceKnowledgeSearchResult, + type WorkspaceSearchFilters, } from '@/lib/api/contracts/knowledge' import { useSession } from '@/lib/auth/auth-client' import { type ResourceScope, resourceScopeKey } from '@/lib/core/resource-scope' @@ -27,6 +28,17 @@ import { useSearchIndex, useSearchSourceOverview } from '@/hooks/queries/kb/conn import { useWorkspaceKnowledgeSearch } from '@/hooks/queries/kb/knowledge' const DAY_MS = 24 * 60 * 60 * 1000 + +/** + * The picker names calendar days; the URL keeps them as dates. A day's bounds are its local + * midnight and the last millisecond before the next, so "September 1" means the reader's own day. + */ +function startOfLocalDay(day: Date): Date { + return new Date(day.getUTCFullYear(), day.getUTCMonth(), day.getUTCDate()) +} +function endOfLocalDay(day: Date): Date { + return new Date(day.getUTCFullYear(), day.getUTCMonth(), day.getUTCDate() + 1, 0, 0, 0, -1) +} /** Every result without a connector is an upload; the filter names them so. */ const UPLOAD_SOURCE = 'upload' @@ -127,6 +139,11 @@ interface SearchResultsProps { function SearchResults({ scope, query, onSummarize }: SearchResultsProps) { const [hasShownFilters, setHasShownFilters] = useState(false) const [searchedAt] = useState(Date.now) + /** + * More results are a second, wider search: the first paint stays as quick as it is, and a + * refinement of the filters starts over at the first page. + */ + const [expandedFor, setExpandedFor] = useState(null) const { data: index, isPending: basesPending, @@ -136,12 +153,24 @@ function SearchResults({ scope, query, onSummarize }: SearchResultsProps) { } = useSearchIndex(scope) const [filters, setFilters] = useQueryStates(searchFilterParsers, resourceUrlKeys) const window = UPDATED_WINDOWS.find((entry) => entry.id === filters.updated) + /** A custom window is inclusive of both days; `to` runs to the end of its day. */ + const custom = filters.updated === 'custom' const searchFilters: WorkspaceSearchFilters = { ...(filters.source ? { source: filters.source } : {}), ...(window?.days ? { modifiedAfter: new Date(searchedAt - window.days * DAY_MS).toISOString() } : {}), + ...(custom && filters.from && filters.to + ? { + modifiedAfter: startOfLocalDay(filters.from).toISOString(), + modifiedBefore: endOfLocalDay(filters.to).toISOString(), + } + : {}), } + const filtersKey = JSON.stringify(searchFilters) + const expanded = expandedFor === filtersKey + /** A custom window is two-ended: until both days are chosen, nothing is searched. */ + const awaitingRange = custom && !(filters.from && filters.to) const { data: search, isPending, @@ -149,7 +178,17 @@ function SearchResults({ scope, query, onSummarize }: SearchResultsProps) { isPlaceholderData, isError: searchFailed, refetch: refetchSearch, - } = useWorkspaceKnowledgeSearch(scope, query, searchFilters) + } = useWorkspaceKnowledgeSearch( + scope, + awaitingRange ? '' : query, + searchFilters, + expanded + ? WORKSPACE_KNOWLEDGE_SEARCH_LIMITS.expanded + : WORKSPACE_KNOWLEDGE_SEARCH_LIMITS.initial + ) + /** A full first page may collapse to few cards, yet more documents may still match. */ + const mayHaveMore = + !expanded && (search?.results.length ?? 0) >= WORKSPACE_KNOWLEDGE_SEARCH_LIMITS.initial const { data: overview } = useSearchSourceOverview(scope) const indexing = (overview?.providers ?? []) .filter((provider) => provider.isSyncing) @@ -175,8 +214,12 @@ function SearchResults({ scope, query, onSummarize }: SearchResultsProps) { : null const showResults = !noSources && !failed && !basesPending && documents.length > 0 + /** A custom window waiting for its days must show the filters, or the picker is unreachable. */ const showFilters = - hasShownFilters || showResults || (!noSources && !pending && !failed && !!search && !partial) + hasShownFilters || + showResults || + awaitingRange || + (!noSources && !pending && !failed && !!search && !partial) if (showFilters && !hasShownFilters) setHasShownFilters(true) return noSources ? ( @@ -196,7 +239,11 @@ function SearchResults({ scope, query, onSummarize }: SearchResultsProps) {
- {fetching || (pending && !failed) ? ( + {awaitingRange ? ( +

+ Choose the days to search. +

+ ) : fetching || (pending && !failed) ? ( ) : (

@@ -257,11 +304,29 @@ function SearchResults({ scope, query, onSummarize }: SearchResultsProps) { shape='round' active={filters.updated === window.id} aria-pressed={filters.updated === window.id} - onClick={() => setFilters({ updated: window.id })} + onClick={() => + setFilters( + window.id === 'custom' + ? { updated: window.id } + : { updated: window.id, from: null, to: null } + ) + } > {window.label} ))} + {custom && ( + + void setFilters({ from: new Date(start), to: new Date(end) }) + } + onClear={() => void setFilters({ from: null, to: null })} + /> + )}

)} {showResults && ( @@ -291,6 +356,17 @@ function SearchResults({ scope, query, onSummarize }: SearchResultsProps) { /> ) })} + {mayHaveMore && ( +
+ setExpandedFor(filtersKey)} + > + Show more + +
+ )}
)}
diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/search-transitions.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/search-transitions.test.tsx index 6b435b90369..8da1cf1620f 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/search-transitions.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/search-transitions.test.tsx @@ -212,6 +212,7 @@ describe('search refinement with the real query cache and URL state', () => { expect(requests.at(-1)?.body).toEqual({ organizationId: 'organization', query: 'launch', + topK: 20, filters: expectedFilters, }) expect(container.querySelector('h1')).toBeNull() diff --git a/apps/sim/app/workspace/[workspaceId]/home/search-params.ts b/apps/sim/app/workspace/[workspaceId]/home/search-params.ts index 19401fa6e00..3c439d7a7a2 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/search-params.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/search-params.ts @@ -1,4 +1,4 @@ -import { parseAsString, parseAsStringLiteral } from 'nuqs/server' +import { parseAsIsoDate, parseAsString, parseAsStringLiteral } from 'nuqs/server' /** * Co-located, typed URL query-param definition for the home/Chat surface. @@ -26,19 +26,23 @@ export const resourceUrlKeys = { clearOnDefault: true, } as const -/** The recency windows a search can be narrowed to. */ +/** The recency windows a search can be narrowed to; `custom` reads its bounds from `from` and `to`. */ export const UPDATED_WINDOWS = [ { id: 'any', label: 'Any time', days: null }, { id: '7d', label: 'Past week', days: 7 }, { id: '30d', label: 'Past month', days: 30 }, + { id: 'custom', label: 'Custom range', days: null }, ] as const const UPDATED_WINDOW_IDS = UPDATED_WINDOWS.map((window) => window.id) /** * Shared result filters for organization search. `source` is a connector type - * or `upload`, absent for every source. + * or `upload`, absent for every source; `from` and `to` are the days of a custom + * window, inclusive, and mean nothing unless `updated` is `custom`. */ export const searchFilterParsers = { source: parseAsString, updated: parseAsStringLiteral(UPDATED_WINDOW_IDS).withDefault('any'), + from: parseAsIsoDate, + to: parseAsIsoDate, } as const diff --git a/apps/sim/hooks/queries/kb/knowledge.test.ts b/apps/sim/hooks/queries/kb/knowledge.test.ts index 07e2c6ca6e5..80379feb5c4 100644 --- a/apps/sim/hooks/queries/kb/knowledge.test.ts +++ b/apps/sim/hooks/queries/kb/knowledge.test.ts @@ -214,9 +214,11 @@ describe('knowledge query placeholder scope', () => { const query = captureQuery(() => useWorkspaceKnowledgeSearch('workspace-1', 'new query', { source: 'slack' }) ) - expect(query.queryKey).toEqual( - knowledgeKeys.search('workspace-1', 'new query', { source: 'slack' }, 'reader') - ) + /** The limit is the key's last part, so the wider search never evicts the first paint. */ + expect(query.queryKey).toEqual([ + ...knowledgeKeys.search('workspace-1', 'new query', { source: 'slack' }, 'reader'), + 20, + ]) expect(knowledgeKeys.search('workspace-1', 'query', { source: 'slack' })).not.toEqual( knowledgeKeys.search('workspace-1', 'query', { source: 'gitlab' }) ) diff --git a/apps/sim/hooks/queries/kb/knowledge.ts b/apps/sim/hooks/queries/kb/knowledge.ts index 5af98a7da79..2408154a166 100644 --- a/apps/sim/hooks/queries/kb/knowledge.ts +++ b/apps/sim/hooks/queries/kb/knowledge.ts @@ -51,8 +51,10 @@ import { updateKnowledgeChunkContract, updateKnowledgeDocumentContract, updateKnowledgeDocumentTagsContract, + WORKSPACE_KNOWLEDGE_SEARCH_LIMITS, type WorkspaceKnowledgeSearchBody, type WorkspaceKnowledgeSearchData, + type WorkspaceKnowledgeSearchLimit, } from '@/lib/api/contracts/knowledge' import type { WorkspaceSearchFilters } from '@/lib/api/contracts/knowledge/search' import { useSession } from '@/lib/auth/auth-client' @@ -1207,7 +1209,8 @@ async function searchWorkspaceKnowledge( export function useWorkspaceKnowledgeSearch( owner: string | ResourceScope | undefined, query: string, - filters?: WorkspaceSearchFilters + filters?: WorkspaceSearchFilters, + limit: WorkspaceKnowledgeSearchLimit = WORKSPACE_KNOWLEDGE_SEARCH_LIMITS.initial ) { const { data: session } = useSession() const queryClient = useQueryClient() @@ -1222,13 +1225,15 @@ export function useWorkspaceKnowledgeSearch( const scopeKey = scope?.kind === 'workspace' ? scope.workspaceId : scope ? resourceScopeKey(scope) : undefined return useQuery({ - queryKey: knowledgeKeys.search(scopeKey, trimmed, filters, userId), + /** The limit is the key's last part, so asking for more never evicts the first paint. */ + queryKey: [...knowledgeKeys.search(scopeKey, trimmed, filters, userId), limit], queryFn: ({ signal }) => searchWorkspaceKnowledge( { ...(scope ? resourceScopeFields(scope) : {}), query: trimmed, filters, + topK: limit, }, signal ), diff --git a/apps/sim/lib/api/contracts/knowledge/search.test.ts b/apps/sim/lib/api/contracts/knowledge/search.test.ts index acae92a414d..ce47b60c16d 100644 --- a/apps/sim/lib/api/contracts/knowledge/search.test.ts +++ b/apps/sim/lib/api/contracts/knowledge/search.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from 'vitest' -import { internalKnowledgeSearchBodySchema } from '@/lib/api/contracts/knowledge/search' +import { + internalKnowledgeSearchBodySchema, + workspaceKnowledgeSearchBodySchema, + workspaceSearchFiltersSchema, +} from '@/lib/api/contracts/knowledge/search' import { RESOLVED_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata' describe('internal Knowledge search contract', () => { @@ -20,3 +24,22 @@ describe('internal Knowledge search contract', () => { ).toMatchObject({ [RESOLVED_SECRET_PROVENANCE_FIELD]: provenance }) }) }) + +describe('workspaceKnowledgeSearchBodySchema', () => { + it('refuses a custom window whose end precedes its start, on the request rather than the filters', () => { + const parsed = workspaceKnowledgeSearchBodySchema.safeParse({ + workspaceId: 'workspace-1', + query: 'launch', + filters: { + modifiedAfter: '2026-09-10T00:00:00.000Z', + modifiedBefore: '2026-09-01T00:00:00.000Z', + }, + }) + expect(parsed.success).toBe(false) + if (!parsed.success) { + expect(parsed.error.issues[0]?.path).toEqual(['filters', 'modifiedBefore']) + } + /** The filters schema stays a plain object, so the Assistant's search input can still extend it. */ + expect(typeof workspaceSearchFiltersSchema.extend).toBe('function') + }) +}) diff --git a/apps/sim/lib/api/contracts/knowledge/search.ts b/apps/sim/lib/api/contracts/knowledge/search.ts index 912ca4690aa..70807dbe008 100644 --- a/apps/sim/lib/api/contracts/knowledge/search.ts +++ b/apps/sim/lib/api/contracts/knowledge/search.ts @@ -172,18 +172,36 @@ export const workspaceKnowledgeSearchResultSchema = z.object({ }) export type WorkspaceKnowledgeSearchResult = z.output +/** A plain object, so the Assistant's search input may still extend it; the window's order is checked on the request. */ export const workspaceSearchFiltersSchema = z.object({ source: z.string().trim().min(1, 'Source cannot be empty').max(100).optional(), modifiedAfter: z.string().datetime({ offset: true }).optional(), + modifiedBefore: z.string().datetime({ offset: true }).optional(), documentIds: z.array(z.string().min(1).max(200)).min(1).max(20).optional(), }) export type WorkspaceSearchFilters = z.output -export const workspaceKnowledgeSearchBodySchema = resourceOwnerSchema.safeExtend({ - filters: workspaceSearchFiltersSchema.optional(), - query: z.string().trim().min(1, 'A search query is required').max(2000, 'Query is too long'), - topK: z.number().int().min(1).max(50).optional().default(20), -}) +/** Chunks a search asks for at first paint, and once the reader asks for more; both within `topK`'s bound. */ +export const WORKSPACE_KNOWLEDGE_SEARCH_LIMITS = { initial: 20, expanded: 50 } as const +export type WorkspaceKnowledgeSearchLimit = + (typeof WORKSPACE_KNOWLEDGE_SEARCH_LIMITS)[keyof typeof WORKSPACE_KNOWLEDGE_SEARCH_LIMITS] + +export const workspaceKnowledgeSearchBodySchema = resourceOwnerSchema + .safeExtend({ + filters: workspaceSearchFiltersSchema.optional(), + query: z.string().trim().min(1, 'A search query is required').max(2000, 'Query is too long'), + topK: z.number().int().min(1).max(50).optional().default(20), + }) + .superRefine((body, ctx) => { + const { modifiedAfter, modifiedBefore } = body.filters ?? {} + if (modifiedAfter && modifiedBefore && Date.parse(modifiedBefore) < Date.parse(modifiedAfter)) { + ctx.addIssue({ + code: 'custom', + path: ['filters', 'modifiedBefore'], + message: 'modifiedBefore must not precede modifiedAfter', + }) + } + }) export type WorkspaceKnowledgeSearchBody = z.input export const workspaceKnowledgeSearchDataSchema = z.object({ diff --git a/apps/sim/lib/core/config/api-keys.ts b/apps/sim/lib/core/config/api-keys.ts index 3135374ae0a..01c04bc096a 100644 --- a/apps/sim/lib/core/config/api-keys.ts +++ b/apps/sim/lib/core/config/api-keys.ts @@ -1,6 +1,14 @@ import { env } from '@/lib/core/config/env' import { LLM_KEY_POOLS } from '@/lib/core/config/env-capabilities' +/** Whether the platform holds at least one key for a provider, without selecting one. */ +export function hasRotatingApiKey(provider: string): boolean { + if (!(provider in LLM_KEY_POOLS)) return false + const definition = LLM_KEY_POOLS[provider as keyof typeof LLM_KEY_POOLS] + if (definition.keys.some((key) => Boolean(env[key]))) return true + return 'fallbackKey' in definition && Boolean(env[definition.fallbackKey]) +} + /** * Rotates through available API keys for a provider * @param provider - The provider to get a key for (e.g., 'openai') diff --git a/apps/sim/lib/knowledge/application/search.test.ts b/apps/sim/lib/knowledge/application/search.test.ts index 3b76d732c7f..2defebd93e1 100644 --- a/apps/sim/lib/knowledge/application/search.test.ts +++ b/apps/sim/lib/knowledge/application/search.test.ts @@ -8,6 +8,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import { OrchestrationError } from '@/lib/core/orchestration/types' const mocks = vi.hoisted(() => ({ + hasRerankerCredential: vi.fn(async () => true), resolveWorkspace: vi.fn(), resolveOrganization: vi.fn(), requireOrganizationSearch: vi.fn(), @@ -39,6 +40,7 @@ vi.mock('@/lib/knowledge/search/activity', () => ({ })) vi.mock('@/lib/knowledge/reranker', () => ({ + hasRerankerCredential: mocks.hasRerankerCredential, rerank: mocks.rerank, })) @@ -866,6 +868,18 @@ describe('knowledge search application use case', () => { * `rerank`, the use case swallowed it, and the caller got a 200 whose results * were byte-identical to an unreranked search with nothing to distinguish them. */ + it('never calls the reranker when neither the workspace nor the platform holds a key', async () => { + mocks.hasRerankerCredential.mockResolvedValueOnce(false) + + const result = await rerankedSearch(true) + + /** A caller's own key is judged by the same policy the resolver applies, not taken on faith. */ + expect(mocks.hasRerankerCredential).toHaveBeenLastCalledWith(expect.anything(), undefined) + expect(mocks.rerank).not.toHaveBeenCalled() + expect(result.rerankerStatus).toBe('unavailable') + expect(result.results[0]).not.toHaveProperty('rerankerScore') + }) + it('reports unavailable rather than silently falling back to vector ordering', async () => { mocks.rerank.mockRejectedValueOnce(new Error('No Cohere API key configured.')) diff --git a/apps/sim/lib/knowledge/application/search.ts b/apps/sim/lib/knowledge/application/search.ts index a8c150f53da..1d355381606 100644 --- a/apps/sim/lib/knowledge/application/search.ts +++ b/apps/sim/lib/knowledge/application/search.ts @@ -34,7 +34,7 @@ import { ALL_TAG_SLOTS } from '@/lib/knowledge/constants' import { getEmbeddingModelInfo, toKbEmbeddingDimensions } from '@/lib/knowledge/embedding-models' import { generateSearchEmbedding, type KbEmbeddingTarget } from '@/lib/knowledge/embeddings' import { runWithKnowledgeModelInputProvenance } from '@/lib/knowledge/model-input-provenance' -import { rerank } from '@/lib/knowledge/reranker' +import { hasRerankerCredential, rerank } from '@/lib/knowledge/reranker' import type { RerankerStatus } from '@/lib/knowledge/reranker-models' import { recordOrganizationSearchActivity } from '@/lib/knowledge/search/activity' import { SearchDeadlineError } from '@/lib/knowledge/search/budget' @@ -391,7 +391,10 @@ const searchKnowledgeUseCase = defineAuthorizedKnowledgeUseCase({ boostRecency: searchDefaults.boostRecency, embeddingDimensions: embeddingTarget?.dimensions, }) - const useReranker = Boolean(input.rerankerEnabled && hasQuery) + /** A surface may ask to rerank; without a key for the workspace or the platform there is nothing to ask. */ + const useReranker = + Boolean(input.rerankerEnabled && hasQuery) && + (await hasRerankerCredential(context.workspaceId, input.rerankerApiKey)) const candidateTopK = useReranker ? input.rerankerInputCount !== undefined ? Math.min( diff --git a/apps/sim/lib/knowledge/reranker.ts b/apps/sim/lib/knowledge/reranker.ts index 085f8da7d47..2a28d1a4ce5 100644 --- a/apps/sim/lib/knowledge/reranker.ts +++ b/apps/sim/lib/knowledge/reranker.ts @@ -2,7 +2,7 @@ import { createLogger } from '@sim/logger' import { sha256Hex } from '@sim/security/hash' import { backoffWithJitter, parseRetryAfter } from '@sim/utils/retry' import { getBYOKKey } from '@/lib/api-key/byok' -import { getRotatingApiKey } from '@/lib/core/config/api-keys' +import { getRotatingApiKey, hasRotatingApiKey } from '@/lib/core/config/api-keys' import { env } from '@/lib/core/config/env' import { isHosted } from '@/lib/core/config/env-flags' import { @@ -30,6 +30,20 @@ const logger = createLogger('Reranker') const RERANK_OPERATION_TIMEOUT_MS = 30_000 +/** + * Whether a search for this workspace could be reranked at all: a workspace key, or one of the + * platform's. A surface that reranks "when configured" asks this before spending a call on it. + */ +export async function hasRerankerCredential( + workspaceId?: string, + userApiKey?: string +): Promise { + /** The same policy as the key resolver: a caller's own key counts only off hosted Sim. */ + if (!isHosted && userApiKey) return true + if (env.COHERE_API_KEY || hasRotatingApiKey('cohere')) return true + return Boolean(workspaceId && (await getBYOKKey(workspaceId, 'cohere'))) +} + /** * Cohere bills per "search unit" = one query with up to 100 documents. * We cap at 100 so each rerank call costs exactly 1 unit and matches diff --git a/apps/sim/lib/knowledge/search/filter-conditions.ts b/apps/sim/lib/knowledge/search/filter-conditions.ts index 8b39f98ef97..b23a78bf099 100644 --- a/apps/sim/lib/knowledge/search/filter-conditions.ts +++ b/apps/sim/lib/knowledge/search/filter-conditions.ts @@ -1,5 +1,5 @@ import { document, knowledgeConnector } from '@sim/db/schema' -import { eq, gte, inArray, isNull, type SQL, sql } from 'drizzle-orm' +import { eq, gte, inArray, isNull, lte, type SQL, sql } from 'drizzle-orm' import type { WorkspaceSearchFilters } from '@/lib/knowledge/search/filters' /** Filters the document in every retrieval leg, alongside its current ACL. */ @@ -9,6 +9,9 @@ export function workspaceSearchFilterConditions(filters?: WorkspaceSearchFilters if (filters?.modifiedAfter) { conditions.push(gte(document.sourceModifiedAt, new Date(filters.modifiedAfter))) } + if (filters?.modifiedBefore) { + conditions.push(lte(document.sourceModifiedAt, new Date(filters.modifiedBefore))) + } if (filters?.source === 'upload') conditions.push(isNull(document.connectorId)) else if (filters?.source) { conditions.push( diff --git a/apps/sim/lib/knowledge/search/filters.test.ts b/apps/sim/lib/knowledge/search/filters.test.ts index f630ab046cb..e083e0f3419 100644 --- a/apps/sim/lib/knowledge/search/filters.test.ts +++ b/apps/sim/lib/knowledge/search/filters.test.ts @@ -19,6 +19,24 @@ describe('Assistant search scope', () => { ) ).toEqual({ documentIds: ['a'], modifiedAfter: '2026-09-01T23:00:00Z' }) }) + it('keeps the narrower end of a date window and refuses an empty one', () => { + expect( + intersectWorkspaceSearchFilters( + { modifiedAfter: '2026-09-01T00:00:00.000Z', modifiedBefore: '2026-09-30T00:00:00.000Z' }, + { modifiedAfter: '2026-09-10T00:00:00.000Z', modifiedBefore: '2026-09-20T00:00:00.000Z' } + ) + ).toEqual({ + modifiedAfter: '2026-09-10T00:00:00.000Z', + modifiedBefore: '2026-09-20T00:00:00.000Z', + }) + expect(() => + intersectWorkspaceSearchFilters( + { modifiedAfter: '2026-09-25T00:00:00.000Z' }, + { modifiedBefore: '2026-09-20T00:00:00.000Z' } + ) + ).toThrow('outside this search') + }) + it('rejects a different source or disjoint document selection', () => { expect(() => intersectWorkspaceSearchFilters({ source: 'gitlab' }, { source: 'slack' }) diff --git a/apps/sim/lib/knowledge/search/filters.ts b/apps/sim/lib/knowledge/search/filters.ts index a62f3eefa92..4d318ee53a1 100644 --- a/apps/sim/lib/knowledge/search/filters.ts +++ b/apps/sim/lib/knowledge/search/filters.ts @@ -4,6 +4,7 @@ import { OrchestrationError } from '@/lib/core/orchestration/types' export interface WorkspaceSearchFilters { source?: string modifiedAfter?: string + modifiedBefore?: string documentIds?: string[] } @@ -27,9 +28,18 @@ export function intersectWorkspaceSearchFilters( .filter((value): value is string => Boolean(value)) .sort((a, b) => Date.parse(a) - Date.parse(b)) .at(-1) + /** The narrower end of each bound wins, so the intersection can only shrink the window. */ + const modifiedBefore = [requested.modifiedBefore, scope.modifiedBefore] + .filter((value): value is string => Boolean(value)) + .sort((a, b) => Date.parse(a) - Date.parse(b)) + .at(0) + if (modifiedAfter && modifiedBefore && Date.parse(modifiedBefore) < Date.parse(modifiedAfter)) { + throw new OrchestrationError('validation', 'The requested dates are outside this search') + } return { ...(scope.source || requested.source ? { source: scope.source ?? requested.source } : {}), ...(modifiedAfter ? { modifiedAfter } : {}), + ...(modifiedBefore ? { modifiedBefore } : {}), ...(documentIds ? { documentIds: [...new Set(documentIds)] } : {}), } } diff --git a/apps/sim/lib/knowledge/search/queries.test.ts b/apps/sim/lib/knowledge/search/queries.test.ts index f91a2b4f8d4..fbb5c368c68 100644 --- a/apps/sim/lib/knowledge/search/queries.test.ts +++ b/apps/sim/lib/knowledge/search/queries.test.ts @@ -35,6 +35,7 @@ import { import type { SearchStage } from '@/lib/knowledge/search/diagnostics' import { executeKeywordSearch, + forgetProjectionFilled, forgetSearchReach, getStructuredTagFilters, handleTagAndVectorSearch, @@ -1262,6 +1263,7 @@ describe('permitted-document planner', () => { indexedSourceRows = [] forgetIndexedVectorSources() forgetSearchReach() + forgetProjectionFilled() dbChainMockFns.execute.mockImplementation(async (query) => { const statement = render(query).sql if (statement.includes('pg_index')) return indexedSourceRows @@ -2120,6 +2122,7 @@ describe('filters on a resolved scope', () => { resetDbChainMock() forgetIndexedVectorSources() forgetSearchReach() + forgetProjectionFilled() probeRows = [] traversedRows = [] rerankRows = [] @@ -2177,6 +2180,9 @@ describe('filters on a resolved scope', () => { expect(probes).toHaveLength(1) /** Filter first, over the date index: never the reach count that reports a broad reader saturated. */ expect(probes[0].sql).not.toContain('WITH reach') + /** An index-driven probe earns its own budget: a window at the document limit fits inside it. */ + const deadlines = statements().filter((query) => query.sql.includes('statement_timeout')) + expect(deadlines.at(-1)?.params[0]).toBe('1500') expect(JSON.stringify(probes[0])).toContain('"type":"gte"') }) @@ -2263,6 +2269,7 @@ describe('filters on a resolved scope', () => { expect(probes()).toHaveLength(1) estimated = 1_000_000 forgetSearchReach() + forgetProjectionFilled() dbChainMockFns.execute.mockClear() await search() expect(probes()).toHaveLength(0) @@ -2312,6 +2319,29 @@ describe('filters on a resolved scope', () => { ).toHaveLength(1) }) + it('keeps the default scan while the projection still holds unfilled rows', async () => { + traversedRows = [{ id: 'a' }] + rerankRows = [hit('a', 'src-a')] + queueTableRows(schemaMock.embedding, rerankRows) + dbChainMockFns.execute.mockImplementation(async (query) => { + const statement = render(query).sql + if (statement.includes('AS unfilled')) return [{ unfilled: true }] + if (isWalk(statement)) return traversedRows + if (statement.includes('WITH scored_search_candidates')) return rerankRows + return [] + }) + await handleVectorOnlySearch({ + ...params, + permitted: { kind: 'unbounded', broad: true }, + accessPlan: plan(), + }) + /** An unfilled row is decided through its document, so the walk keeps the cap sized for that. */ + const caps = statements() + .filter((query) => query.sql.includes('hnsw.max_scan_tuples')) + .map((query) => query.params.find((param) => param === '20000' || param === '100000')) + expect(caps.at(-1)).toBe('20000') + }) + it('tests the date through the document inside an on-row walk when the filtered set is unbounded', async () => { traversedRows = [{ id: 'a' }] rerankRows = [hit('a', 'src-a')] @@ -2328,6 +2358,12 @@ describe('filters on a resolved scope', () => { /** The mock renders a nested condition into the params; the date test is the only `gte`. */ const datesDocument = (statement: unknown) => JSON.stringify(statement).includes('"type":"gte"') expect(datesDocument(walks[0])).toBe(true) + /** A walk that asks the document per tuple keeps the default scan cap, not the on-row one. */ + const scanCaps = () => + statements() + .filter((query) => query.sql.includes('hnsw.max_scan_tuples')) + .map((query) => query.params.find((param) => param === '20000' || param === '100000')) + expect(scanCaps().at(-1)).toBe('20000') resetDbChainMock() queueTableRows(schemaMock.embedding, rerankRows) dbChainMockFns.execute.mockImplementation(async (query) => { @@ -2342,6 +2378,7 @@ describe('filters on a resolved scope', () => { accessPlan: plan(), }) expect(datesDocument(statements().filter((query) => isWalk(query.sql))[0])).toBe(false) + expect(scanCaps().at(-1)).toBe('100000') }) it('leaves the keyword leg short when its deadline passes before the ranking is resolved', async () => { diff --git a/apps/sim/lib/knowledge/search/queries.ts b/apps/sim/lib/knowledge/search/queries.ts index 001ba08487f..b5a101c94ec 100644 --- a/apps/sim/lib/knowledge/search/queries.ts +++ b/apps/sim/lib/knowledge/search/queries.ts @@ -10,7 +10,7 @@ import { import { createLogger } from '@sim/logger' import { sha256Hex } from '@sim/security/hash' import { getErrorMessage, getPostgresErrorCode } from '@sim/utils/errors' -import { and, eq, gte, inArray, isNull, type SQL, sql } from 'drizzle-orm' +import { and, eq, gte, inArray, isNull, lte, type SQL, sql } from 'drizzle-orm' import { LRUCache } from 'lru-cache' import { mapWithConcurrency } from '@/lib/core/utils/concurrency' import { resolveSearchAccessPlan } from '@/lib/knowledge/access/connector-eligibility' @@ -89,6 +89,44 @@ const CANDIDATE_HNSW_MAX_SCAN_TUPLES = '20000' * left with what the neighbourhood happened to hold. */ const ON_ROW_WALK_SCAN_TUPLES = 100_000 + +/** + * How far a walk may go when readability is on the row: the on-row cap, unless the walk still + * has to ask the document about tuples — a tag or date filter, or rows the backfill has not + * filled yet — in which case such a tuple costs what it did before the columns were mirrored, + * and the default cap keeps a walk through a mostly-excluded neighbourhood at a short answer + * rather than a missed deadline. + */ +function onRowWalkScanTuples( + documentCondition: SQL | undefined, + projectionFilled: boolean +): number { + return documentCondition === undefined && projectionFilled + ? ON_ROW_WALK_SCAN_TUPLES + : Number(CANDIDATE_HNSW_MAX_SCAN_TUPLES) +} + +/** How long a fully filled projection is taken on trust before its unfilled rows are looked for again. */ +const PROJECTION_FILLED_TTL_MS = 60_000 + +/** + * Whether the ranking projection still holds rows the backfill has not filled. Read off the + * unfilled-rows index in microseconds and remembered briefly: the answer only ever changes once. + */ +const projectionFilled = new LRUCache({ + max: 1, + ttl: PROJECTION_FILLED_TTL_MS, + fetchMethod: async () => { + const [row] = await db.execute<{ unfilled: boolean }>(sql` + SELECT EXISTS (SELECT 1 FROM ${embeddingSearch} WHERE ${embeddingSearch.acl} IS NULL) AS unfilled`) + return !row?.unfilled + }, +}) + +/** Forgets whether the projection was filled, after its rows changed. */ +export function forgetProjectionFilled(): void { + projectionFilled.clear() +} /** * Beam width per iteration. A beam is the granularity of cancellation: pgvector calls * `CHECK_FOR_INTERRUPTS` only while building an index, never inside `hnswgettuple`, so neither @@ -114,6 +152,14 @@ const VECTOR_PROBE_BUDGET_MS = 600 * comparable cache pressure: the access predicate, evaluated once per document. */ const VECTOR_PROBE_MICROSECONDS_PER_DOCUMENT = 6 +/** + * What a filter-first probe may spend: it reads the filtered documents off their own index and + * tests each one's access, bounded by the same document limit, and measures around 2 µs per + * document to enumerate plus the access test — a window at the limit fits with room. Its result + * is ranked exactly, at a cost that is predictable where a walk through a mostly-excluded + * neighbourhood is not. + */ +const FILTERED_PROBE_BUDGET_MS = 1500 /** * Documents the probe enumerates before it concludes the permitted set is too large to rank * exactly. Derived so that reaching it is what spends the probe's budget, rather than a separate @@ -1001,7 +1047,9 @@ async function probeVisibleDocuments( stage: 'vector.probe' | 'permitted_documents', shape: 'reach-first' | 'direct' = 'reach-first' ): Promise { - const probeBudget = budget?.capped(VECTOR_PROBE_BUDGET_MS) + const probeBudget = budget?.capped( + shape === 'direct' ? FILTERED_PROBE_BUDGET_MS : VECTOR_PROBE_BUDGET_MS + ) try { const probed = await runSearchQuery(probeBudget, stage, (executor) => executor.execute( @@ -1152,6 +1200,19 @@ const indexDocumentCounts = new LRUCache({ }, }) +/** The date window a filter asks for, on the document row; nothing when none is asked. */ +function dateFilterCondition(filters: WorkspaceSearchFilters | undefined): SQL | undefined { + if (!filters?.modifiedAfter && !filters?.modifiedBefore) return undefined + return and( + filters.modifiedAfter + ? gte(document.sourceModifiedAt, new Date(filters.modifiedAfter)) + : undefined, + filters.modifiedBefore + ? lte(document.sourceModifiedAt, new Date(filters.modifiedBefore)) + : undefined + ) +} + /** * The planner's estimate of the documents a filter leaves in the bases — a date filter from the * statistics on its index, a source filter from its connectors' — so whether the filtered set is @@ -1169,9 +1230,7 @@ async function estimateFilteredDocuments( WHERE ${and( inArray(document.knowledgeBaseId, knowledgeBaseIds), isNull(document.deletedAt), - filters.modifiedAfter - ? gte(document.sourceModifiedAt, new Date(filters.modifiedAfter)) - : undefined, + dateFilterCondition(filters), filters.source ? planSourceCondition(plan) : undefined )}`) ) @@ -1290,7 +1349,7 @@ export async function resolvePermittedDocuments(params: { * change; the filtered set still has to be enumerated, so under one the probe always runs. */ const remembered = - key && !(params.accessPlan && (params.filters?.modifiedAfter || params.filters?.source)) + key && !(params.accessPlan && (dateFilterCondition(params.filters) || params.filters?.source)) ? saturatedReach.get(key) : undefined if (remembered) { @@ -1309,7 +1368,7 @@ export async function resolvePermittedDocuments(params: { params.access, params.budget, 'permitted_documents', - params.accessPlan && (params.filters?.modifiedAfter || params.filters?.source) + params.accessPlan && (dateFilterCondition(params.filters) || params.filters?.source) ? 'direct' : 'reach-first' ) @@ -1409,6 +1468,8 @@ async function selectSourceVectorCandidates(input: { plan: SearchAccessPlan tagCondition: SQL | undefined documentCondition: SQL | undefined + /** Whether every projection row carries its mirrored columns, so a walk needs no document. */ + projectionFilled: boolean candidateDistance: SQL candidateLimit: number budget?: SearchBudget @@ -1457,7 +1518,7 @@ async function selectSourceVectorCandidates(input: { ORDER BY ${input.candidateDistance} LIMIT ${input.candidateLimit}`), input.budget, 'vector.source_walk', - ON_ROW_WALK_SCAN_TUPLES + onRowWalkScanTuples(input.documentCondition, input.projectionFilled) ) const walks: Array<() => RankedChunks> = sources.walked.map((connectorId) => walk(eq(embeddingSearch.connectorId, connectorId)) @@ -1576,12 +1637,9 @@ async function selectVectorResults(params: SearchParams): Promise const plan = params.access.kind === 'user' ? params.accessPlan : undefined + const filled = plan ? ((await projectionFilled.fetch('embedding_search')) ?? false) : false /** * A source the caller is a member of that has its own index is walked on its own, which * beats ranking it exactly once it is large enough to have earned that index. @@ -1676,7 +1735,7 @@ async function selectVectorResults(params: SearchParams): Promise estimate <= VECTOR_PROBE_DOCUMENT_LIMIT) .catch((error) => { if (!estimateBudget.isTimeout(error)) throw error diff --git a/packages/emcn/src/components/chip-date-picker/chip-date-picker.tsx b/packages/emcn/src/components/chip-date-picker/chip-date-picker.tsx index d6bec05ca7d..ca65061ab31 100644 --- a/packages/emcn/src/components/chip-date-picker/chip-date-picker.tsx +++ b/packages/emcn/src/components/chip-date-picker/chip-date-picker.tsx @@ -62,6 +62,8 @@ interface ChipDatePickerRangeProps extends ChipDatePickerBaseProps { showTime?: boolean /** Called on Apply with the ordered range bounds. */ onRangeChange: (start: string, end: string) => void + /** Called on Clear, so a committed range can be dropped by whoever owns it. */ + onClear?: () => void } export type ChipDatePickerProps = ChipDatePickerSingleProps | ChipDatePickerRangeProps @@ -161,6 +163,7 @@ const ChipDatePicker = forwardRef( setOpen(false) }} onCancel={() => setOpen(false)} + onClear={props.onClear} /> ) : (