From 8ed7bd4f7e3dd10b75ec6ced26789ba279dbd9c9 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sun, 20 Sep 2026 10:35:59 -0700 Subject: [PATCH 01/10] improvement(knowledge): page search results, take a custom date window, and rerank a person's search - the Search page asks for 50 chunks, collapses them to one card per document, shows ten at a time and reveals more on request - `updated` gains a custom range: `from`/`to` days in the URL, sent as an inclusive `modifiedAfter`/`modifiedBefore` window; `modifiedBefore` joins the filter contract, the filter intersection, the document conditions, the on-row date test, the filtered-set estimate and the bounded probe - the dashboard search opts into the platform's cross-encoder reranker whenever a Cohere key is configured; reranking stays best-effort --- .../app/api/knowledge/search/route.test.ts | 7 +++ apps/sim/app/api/knowledge/search/route.ts | 8 +++ .../knowledge-search-results.test.tsx | 57 ++++++++++++++++++- .../knowledge-search-results.tsx | 31 +++++++++- .../search-transitions.test.tsx | 1 + .../[workspaceId]/home/search-params.ts | 10 +++- apps/sim/hooks/queries/kb/knowledge.ts | 3 + .../sim/lib/api/contracts/knowledge/search.ts | 25 ++++++-- apps/sim/lib/core/config/api-keys.ts | 8 +++ .../lib/knowledge/search/filter-conditions.ts | 5 +- apps/sim/lib/knowledge/search/filters.test.ts | 18 ++++++ apps/sim/lib/knowledge/search/filters.ts | 10 ++++ apps/sim/lib/knowledge/search/queries.ts | 45 ++++++++------- 13 files changed, 193 insertions(+), 35 deletions(-) diff --git a/apps/sim/app/api/knowledge/search/route.test.ts b/apps/sim/app/api/knowledge/search/route.test.ts index 445fe171c63..b58a8ec40ad 100644 --- a/apps/sim/app/api/knowledge/search/route.test.ts +++ b/apps/sim/app/api/knowledge/search/route.test.ts @@ -6,6 +6,9 @@ import { NextRequest } from 'next/server' import { beforeEach, describe, expect, it, vi } from 'vitest' const mocks = vi.hoisted(() => ({ search: vi.fn() })) +const { mockHasRotatingApiKey } = vi.hoisted(() => ({ mockHasRotatingApiKey: vi.fn(() => true) })) +vi.mock('@/lib/core/config/api-keys', () => ({ hasRotatingApiKey: mockHasRotatingApiKey })) + vi.mock('@/lib/knowledge/application/workspace-search', () => ({ searchScopedKnowledge: { operation: { id: 'knowledge.search' }, execute: mocks.search }, })) @@ -47,6 +50,10 @@ describe('workspace search route', () => { expect(call.input.signal).toBe(request.signal) expect(call.input.allowPartialResults).toBe(true) expect(call.input.vectorBudgetMs).toBe(3000) + /** A person's search is reranked by the platform's cross-encoder whenever one is configured. */ + expect(mockHasRotatingApiKey).toHaveBeenCalledWith('cohere') + expect(call.input.rerankerEnabled).toBe(true) + expect(call.input.rerankerModel).toBe('rerank-v4.0-fast') controller.abort() expect(call.input.signal.aborted).toBe(true) await expect(response.json()).resolves.toEqual({ diff --git a/apps/sim/app/api/knowledge/search/route.ts b/apps/sim/app/api/knowledge/search/route.ts index b1db5e04d39..0c9aa4237b9 100644 --- a/apps/sim/app/api/knowledge/search/route.ts +++ b/apps/sim/app/api/knowledge/search/route.ts @@ -4,9 +4,11 @@ import { internalRateLimits, internalSessionAuth, } from '@/lib/api/server/routes' +import { hasRotatingApiKey } from '@/lib/core/config/api-keys' import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' import { knowledgeOperations } from '@/lib/knowledge/application/operations' import { searchScopedKnowledge } from '@/lib/knowledge/application/workspace-search' +import { DEFAULT_RERANKER_MODEL } from '@/lib/knowledge/reranker-models' import { sourceAuthor } from '@/lib/knowledge/search/author' const DIRECT_SEARCH_VECTOR_BUDGET_MS = 3000 @@ -28,6 +30,12 @@ export const POST = defineInternalJsonRoute({ topK: body.topK, allowPartialResults: true, vectorBudgetMs: DIRECT_SEARCH_VECTOR_BUDGET_MS, + /** + * A person's search is reranked by the platform's cross-encoder when one is configured; + * reranking is best-effort, so a provider outage leaves the fused order in place. + */ + rerankerEnabled: hasRotatingApiKey('cohere'), + rerankerModel: DEFAULT_RERANKER_MODEL, surface: 'dashboard' as const, signal: request.signal, }), diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx index c4a857f3c7d..83127835707 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx @@ -48,10 +48,13 @@ afterEach(() => { act(() => root.unmount()) vi.unstubAllGlobals() }) -async function render(scope: ResourceScope = { kind: 'workspace', workspaceId: 'workspace' }) { +async function render( + scope: ResourceScope = { kind: 'workspace', workspaceId: 'workspace' }, + searchParams = '' +) { await act(async () => root.render( - + ) @@ -148,3 +151,53 @@ describe('source setup navigation', () => { expect(container.querySelector('a')?.getAttribute('href')).toBe(href) }) }) + +describe('result paging and the custom window', () => { + const result = (n: number) => ({ + documentId: `doc-${n}`, + knowledgeBaseId: 'kb', + knowledgeBaseName: 'Index', + documentName: `Document ${n}`, + sourceUrl: null, + connectorType: 'slack', + sourceModifiedAt: null, + author: null, + content: 'launch notes', + chunkIndex: 0, + similarity: 0.5, + }) + + it('shows ten documents at a time and reveals more on request', async () => { + mocks.overview.mockReturnValue({ data: { providers: [], hasSearchableDocuments: true } }) + mocks.search.mockReturnValue({ + data: { + query: 'launch', + results: Array.from({ length: 25 }, (_, n) => result(n)), + retrieval: { status: 'complete', timedOutLegs: [] }, + }, + isPending: false, + isFetching: false, + isPlaceholderData: false, + isError: false, + refetch: mocks.retry, + }) + await render() + expect(container.textContent).toContain('Document 9') + expect(container.textContent).not.toContain('Document 10') + const more = [...container.querySelectorAll('button')].find( + (b) => b.textContent === 'Show more' + )! + expect(more).toBeDefined() + await act(async () => more.click()) + expect(container.textContent).toContain('Document 19') + expect(container.textContent).not.toContain('Document 20') + }) + + it('searches a custom window as an inclusive range of days', async () => { + mocks.overview.mockReturnValue({ data: { providers: [], hasSearchableDocuments: true } }) + await render(undefined, '?updated=custom&from=2026-09-01&to=2026-09-10') + const filters = mocks.search.mock.calls.at(-1)![2] + expect(filters.modifiedAfter).toBe('2026-09-01T00:00:00.000Z') + expect(filters.modifiedBefore).toBe('2026-09-10T23:59:59.999Z') + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx index 8ac809b11ec..6b240c434c3 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx @@ -1,7 +1,7 @@ 'use client' import { useState } from 'react' -import { Chip, ChipLink, cn } from '@sim/emcn' +import { Chip, ChipDatePicker, ChipLink, cn } from '@sim/emcn' import { useQueryStates } from 'nuqs' import { ActivityStatus } from '@/components/ui/activity-status' import type { @@ -27,6 +27,8 @@ import { useSearchIndex, useSearchSourceOverview } from '@/hooks/queries/kb/conn import { useWorkspaceKnowledgeSearch } from '@/hooks/queries/kb/knowledge' const DAY_MS = 24 * 60 * 60 * 1000 +/** Cards shown before the reader asks for more; the search itself returns several pages' worth. */ +const RESULTS_PAGE_SIZE = 10 /** Every result without a connector is an upload; the filter names them so. */ const UPLOAD_SOURCE = 'upload' @@ -127,6 +129,7 @@ interface SearchResultsProps { function SearchResults({ scope, query, onSummarize }: SearchResultsProps) { const [hasShownFilters, setHasShownFilters] = useState(false) const [searchedAt] = useState(Date.now) + const [shown, setShown] = useState(RESULTS_PAGE_SIZE) const { data: index, isPending: basesPending, @@ -136,11 +139,17 @@ function SearchResults({ scope, query, onSummarize }: SearchResultsProps) { } = useSearchIndex(scope) const [filters, setFilters] = useQueryStates(searchFilterParsers, resourceUrlKeys) const window = UPDATED_WINDOWS.find((entry) => entry.id === filters.updated) + /** A custom window is inclusive of both days; `to` runs to the end of its day. */ + const custom = filters.updated === 'custom' const searchFilters: WorkspaceSearchFilters = { ...(filters.source ? { source: filters.source } : {}), ...(window?.days ? { modifiedAfter: new Date(searchedAt - window.days * DAY_MS).toISOString() } : {}), + ...(custom && filters.from ? { modifiedAfter: filters.from.toISOString() } : {}), + ...(custom && filters.to + ? { modifiedBefore: new Date(filters.to.getTime() + DAY_MS - 1).toISOString() } + : {}), } const { data: search, @@ -262,6 +271,17 @@ function SearchResults({ scope, query, onSummarize }: SearchResultsProps) { {window.label} ))} + {custom && ( + + void setFilters({ from: new Date(start), to: new Date(end) }) + } + /> + )} )} {showResults && ( @@ -272,7 +292,7 @@ function SearchResults({ scope, query, onSummarize }: SearchResultsProps) { className={cn('flex flex-col', isPlaceholderData && 'opacity-60')} onKeyDown={handleResultsKeyDown} > - {documents.map((result) => { + {documents.slice(0, shown).map((result) => { const source = toSource(result, query, scope) return ( ) })} + {documents.length > shown && ( +
+ setShown((count) => count + RESULTS_PAGE_SIZE)}> + Show more + +
+ )} )} diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/search-transitions.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/search-transitions.test.tsx index 6b435b90369..a9d604f59c5 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/search-transitions.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/search-transitions.test.tsx @@ -212,6 +212,7 @@ describe('search refinement with the real query cache and URL state', () => { expect(requests.at(-1)?.body).toEqual({ organizationId: 'organization', query: 'launch', + topK: 50, filters: expectedFilters, }) expect(container.querySelector('h1')).toBeNull() diff --git a/apps/sim/app/workspace/[workspaceId]/home/search-params.ts b/apps/sim/app/workspace/[workspaceId]/home/search-params.ts index 19401fa6e00..3c439d7a7a2 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/search-params.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/search-params.ts @@ -1,4 +1,4 @@ -import { parseAsString, parseAsStringLiteral } from 'nuqs/server' +import { parseAsIsoDate, parseAsString, parseAsStringLiteral } from 'nuqs/server' /** * Co-located, typed URL query-param definition for the home/Chat surface. @@ -26,19 +26,23 @@ export const resourceUrlKeys = { clearOnDefault: true, } as const -/** The recency windows a search can be narrowed to. */ +/** The recency windows a search can be narrowed to; `custom` reads its bounds from `from` and `to`. */ export const UPDATED_WINDOWS = [ { id: 'any', label: 'Any time', days: null }, { id: '7d', label: 'Past week', days: 7 }, { id: '30d', label: 'Past month', days: 30 }, + { id: 'custom', label: 'Custom range', days: null }, ] as const const UPDATED_WINDOW_IDS = UPDATED_WINDOWS.map((window) => window.id) /** * Shared result filters for organization search. `source` is a connector type - * or `upload`, absent for every source. + * or `upload`, absent for every source; `from` and `to` are the days of a custom + * window, inclusive, and mean nothing unless `updated` is `custom`. */ export const searchFilterParsers = { source: parseAsString, updated: parseAsStringLiteral(UPDATED_WINDOW_IDS).withDefault('any'), + from: parseAsIsoDate, + to: parseAsIsoDate, } as const diff --git a/apps/sim/hooks/queries/kb/knowledge.ts b/apps/sim/hooks/queries/kb/knowledge.ts index 5af98a7da79..b59433c2166 100644 --- a/apps/sim/hooks/queries/kb/knowledge.ts +++ b/apps/sim/hooks/queries/kb/knowledge.ts @@ -89,6 +89,8 @@ export const KNOWLEDGE_DOCUMENT_LIST_STALE_TIME = 60 * 1000 export const KNOWLEDGE_CHUNK_LIST_STALE_TIME = 60 * 1000 export const KNOWLEDGE_CHUNK_SEARCH_STALE_TIME = 60 * 1000 export const WORKSPACE_KNOWLEDGE_SEARCH_STALE_TIME = 60 * 1000 +/** Chunks one search asks for: several pages of documents once collapsed to one card each. */ +export const WORKSPACE_KNOWLEDGE_SEARCH_RESULT_LIMIT = 50 export const KNOWLEDGE_TAG_DEFINITION_LIST_STALE_TIME = 60 * 1000 export const KNOWLEDGE_TAG_USAGE_STALE_TIME = 60 * 1000 export const KNOWLEDGE_DOCUMENT_TAG_DEFINITION_LIST_STALE_TIME = 60 * 1000 @@ -1229,6 +1231,7 @@ export function useWorkspaceKnowledgeSearch( ...(scope ? resourceScopeFields(scope) : {}), query: trimmed, filters, + topK: WORKSPACE_KNOWLEDGE_SEARCH_RESULT_LIMIT, }, signal ), diff --git a/apps/sim/lib/api/contracts/knowledge/search.ts b/apps/sim/lib/api/contracts/knowledge/search.ts index 912ca4690aa..127014355db 100644 --- a/apps/sim/lib/api/contracts/knowledge/search.ts +++ b/apps/sim/lib/api/contracts/knowledge/search.ts @@ -172,11 +172,26 @@ export const workspaceKnowledgeSearchResultSchema = z.object({ }) export type WorkspaceKnowledgeSearchResult = z.output -export const workspaceSearchFiltersSchema = z.object({ - source: z.string().trim().min(1, 'Source cannot be empty').max(100).optional(), - modifiedAfter: z.string().datetime({ offset: true }).optional(), - documentIds: z.array(z.string().min(1).max(200)).min(1).max(20).optional(), -}) +export const workspaceSearchFiltersSchema = z + .object({ + source: z.string().trim().min(1, 'Source cannot be empty').max(100).optional(), + modifiedAfter: z.string().datetime({ offset: true }).optional(), + modifiedBefore: z.string().datetime({ offset: true }).optional(), + documentIds: z.array(z.string().min(1).max(200)).min(1).max(20).optional(), + }) + .superRefine((filters, ctx) => { + if ( + filters.modifiedAfter && + filters.modifiedBefore && + Date.parse(filters.modifiedBefore) < Date.parse(filters.modifiedAfter) + ) { + ctx.addIssue({ + code: 'custom', + path: ['modifiedBefore'], + message: 'modifiedBefore must not precede modifiedAfter', + }) + } + }) export type WorkspaceSearchFilters = z.output export const workspaceKnowledgeSearchBodySchema = resourceOwnerSchema.safeExtend({ diff --git a/apps/sim/lib/core/config/api-keys.ts b/apps/sim/lib/core/config/api-keys.ts index 3135374ae0a..01c04bc096a 100644 --- a/apps/sim/lib/core/config/api-keys.ts +++ b/apps/sim/lib/core/config/api-keys.ts @@ -1,6 +1,14 @@ import { env } from '@/lib/core/config/env' import { LLM_KEY_POOLS } from '@/lib/core/config/env-capabilities' +/** Whether the platform holds at least one key for a provider, without selecting one. */ +export function hasRotatingApiKey(provider: string): boolean { + if (!(provider in LLM_KEY_POOLS)) return false + const definition = LLM_KEY_POOLS[provider as keyof typeof LLM_KEY_POOLS] + if (definition.keys.some((key) => Boolean(env[key]))) return true + return 'fallbackKey' in definition && Boolean(env[definition.fallbackKey]) +} + /** * Rotates through available API keys for a provider * @param provider - The provider to get a key for (e.g., 'openai') diff --git a/apps/sim/lib/knowledge/search/filter-conditions.ts b/apps/sim/lib/knowledge/search/filter-conditions.ts index 8b39f98ef97..b23a78bf099 100644 --- a/apps/sim/lib/knowledge/search/filter-conditions.ts +++ b/apps/sim/lib/knowledge/search/filter-conditions.ts @@ -1,5 +1,5 @@ import { document, knowledgeConnector } from '@sim/db/schema' -import { eq, gte, inArray, isNull, type SQL, sql } from 'drizzle-orm' +import { eq, gte, inArray, isNull, lte, type SQL, sql } from 'drizzle-orm' import type { WorkspaceSearchFilters } from '@/lib/knowledge/search/filters' /** Filters the document in every retrieval leg, alongside its current ACL. */ @@ -9,6 +9,9 @@ export function workspaceSearchFilterConditions(filters?: WorkspaceSearchFilters if (filters?.modifiedAfter) { conditions.push(gte(document.sourceModifiedAt, new Date(filters.modifiedAfter))) } + if (filters?.modifiedBefore) { + conditions.push(lte(document.sourceModifiedAt, new Date(filters.modifiedBefore))) + } if (filters?.source === 'upload') conditions.push(isNull(document.connectorId)) else if (filters?.source) { conditions.push( diff --git a/apps/sim/lib/knowledge/search/filters.test.ts b/apps/sim/lib/knowledge/search/filters.test.ts index f630ab046cb..e083e0f3419 100644 --- a/apps/sim/lib/knowledge/search/filters.test.ts +++ b/apps/sim/lib/knowledge/search/filters.test.ts @@ -19,6 +19,24 @@ describe('Assistant search scope', () => { ) ).toEqual({ documentIds: ['a'], modifiedAfter: '2026-09-01T23:00:00Z' }) }) + it('keeps the narrower end of a date window and refuses an empty one', () => { + expect( + intersectWorkspaceSearchFilters( + { modifiedAfter: '2026-09-01T00:00:00.000Z', modifiedBefore: '2026-09-30T00:00:00.000Z' }, + { modifiedAfter: '2026-09-10T00:00:00.000Z', modifiedBefore: '2026-09-20T00:00:00.000Z' } + ) + ).toEqual({ + modifiedAfter: '2026-09-10T00:00:00.000Z', + modifiedBefore: '2026-09-20T00:00:00.000Z', + }) + expect(() => + intersectWorkspaceSearchFilters( + { modifiedAfter: '2026-09-25T00:00:00.000Z' }, + { modifiedBefore: '2026-09-20T00:00:00.000Z' } + ) + ).toThrow('outside this search') + }) + it('rejects a different source or disjoint document selection', () => { expect(() => intersectWorkspaceSearchFilters({ source: 'gitlab' }, { source: 'slack' }) diff --git a/apps/sim/lib/knowledge/search/filters.ts b/apps/sim/lib/knowledge/search/filters.ts index a62f3eefa92..4d318ee53a1 100644 --- a/apps/sim/lib/knowledge/search/filters.ts +++ b/apps/sim/lib/knowledge/search/filters.ts @@ -4,6 +4,7 @@ import { OrchestrationError } from '@/lib/core/orchestration/types' export interface WorkspaceSearchFilters { source?: string modifiedAfter?: string + modifiedBefore?: string documentIds?: string[] } @@ -27,9 +28,18 @@ export function intersectWorkspaceSearchFilters( .filter((value): value is string => Boolean(value)) .sort((a, b) => Date.parse(a) - Date.parse(b)) .at(-1) + /** The narrower end of each bound wins, so the intersection can only shrink the window. */ + const modifiedBefore = [requested.modifiedBefore, scope.modifiedBefore] + .filter((value): value is string => Boolean(value)) + .sort((a, b) => Date.parse(a) - Date.parse(b)) + .at(0) + if (modifiedAfter && modifiedBefore && Date.parse(modifiedBefore) < Date.parse(modifiedAfter)) { + throw new OrchestrationError('validation', 'The requested dates are outside this search') + } return { ...(scope.source || requested.source ? { source: scope.source ?? requested.source } : {}), ...(modifiedAfter ? { modifiedAfter } : {}), + ...(modifiedBefore ? { modifiedBefore } : {}), ...(documentIds ? { documentIds: [...new Set(documentIds)] } : {}), } } diff --git a/apps/sim/lib/knowledge/search/queries.ts b/apps/sim/lib/knowledge/search/queries.ts index 001ba08487f..16e1dd3b5f4 100644 --- a/apps/sim/lib/knowledge/search/queries.ts +++ b/apps/sim/lib/knowledge/search/queries.ts @@ -10,7 +10,7 @@ import { import { createLogger } from '@sim/logger' import { sha256Hex } from '@sim/security/hash' import { getErrorMessage, getPostgresErrorCode } from '@sim/utils/errors' -import { and, eq, gte, inArray, isNull, type SQL, sql } from 'drizzle-orm' +import { and, eq, gte, inArray, isNull, lte, type SQL, sql } from 'drizzle-orm' import { LRUCache } from 'lru-cache' import { mapWithConcurrency } from '@/lib/core/utils/concurrency' import { resolveSearchAccessPlan } from '@/lib/knowledge/access/connector-eligibility' @@ -1152,6 +1152,19 @@ const indexDocumentCounts = new LRUCache({ }, }) +/** The date window a filter asks for, on the document row; nothing when none is asked. */ +function dateFilterCondition(filters: WorkspaceSearchFilters | undefined): SQL | undefined { + if (!filters?.modifiedAfter && !filters?.modifiedBefore) return undefined + return and( + filters.modifiedAfter + ? gte(document.sourceModifiedAt, new Date(filters.modifiedAfter)) + : undefined, + filters.modifiedBefore + ? lte(document.sourceModifiedAt, new Date(filters.modifiedBefore)) + : undefined + ) +} + /** * The planner's estimate of the documents a filter leaves in the bases — a date filter from the * statistics on its index, a source filter from its connectors' — so whether the filtered set is @@ -1169,9 +1182,7 @@ async function estimateFilteredDocuments( WHERE ${and( inArray(document.knowledgeBaseId, knowledgeBaseIds), isNull(document.deletedAt), - filters.modifiedAfter - ? gte(document.sourceModifiedAt, new Date(filters.modifiedAfter)) - : undefined, + dateFilterCondition(filters), filters.source ? planSourceCondition(plan) : undefined )}`) ) @@ -1290,7 +1301,7 @@ export async function resolvePermittedDocuments(params: { * change; the filtered set still has to be enumerated, so under one the probe always runs. */ const remembered = - key && !(params.accessPlan && (params.filters?.modifiedAfter || params.filters?.source)) + key && !(params.accessPlan && (dateFilterCondition(params.filters) || params.filters?.source)) ? saturatedReach.get(key) : undefined if (remembered) { @@ -1309,7 +1320,7 @@ export async function resolvePermittedDocuments(params: { params.access, params.budget, 'permitted_documents', - params.accessPlan && (params.filters?.modifiedAfter || params.filters?.source) + params.accessPlan && (dateFilterCondition(params.filters) || params.filters?.source) ? 'direct' : 'reach-first' ) @@ -1576,12 +1587,7 @@ async function selectVectorResults(params: SearchParams): Promise estimate <= VECTOR_PROBE_DOCUMENT_LIMIT) .catch((error) => { if (!estimateBudget.isTimeout(error)) throw error From 414fdc256a725578ee13d3a31e35cb23e18015f4 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sun, 20 Sep 2026 10:41:24 -0700 Subject: [PATCH 02/10] improvement(knowledge): widen a search to 50 chunks only when the reader asks for more --- .../o/[organizationId]/search/search.test.tsx | 2 +- .../knowledge-search-results.test.tsx | 35 ++++++++++++------- .../knowledge-search-results.tsx | 30 ++++++++++------ .../search-transitions.test.tsx | 2 +- apps/sim/hooks/queries/kb/knowledge.test.ts | 8 +++-- apps/sim/hooks/queries/kb/knowledge.ts | 12 ++++--- .../sim/lib/api/contracts/knowledge/search.ts | 5 +++ 7 files changed, 62 insertions(+), 32 deletions(-) diff --git a/apps/sim/app/o/[organizationId]/search/search.test.tsx b/apps/sim/app/o/[organizationId]/search/search.test.tsx index ddf950eb08b..e61511d7e27 100644 --- a/apps/sim/app/o/[organizationId]/search/search.test.tsx +++ b/apps/sim/app/o/[organizationId]/search/search.test.tsx @@ -135,7 +135,7 @@ async function editDraft(value: string) { function expectVisibleQuery(query: string) { expect(searchInput().value).toBe(query) expect(container.querySelector('a[data-source-link]')?.textContent).toBe(`${query} launch plan`) - expect(mocks.search).toHaveBeenLastCalledWith(scope, query, {}) + expect(mocks.search).toHaveBeenLastCalledWith(scope, query, {}, 20) expect(document.activeElement).toBe(searchInput()) } diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx index 83127835707..0a68f35acdf 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx @@ -167,12 +167,12 @@ describe('result paging and the custom window', () => { similarity: 0.5, }) - it('shows ten documents at a time and reveals more on request', async () => { + it('offers more only after a full first page, and asks for the wider search on request', async () => { mocks.overview.mockReturnValue({ data: { providers: [], hasSearchableDocuments: true } }) - mocks.search.mockReturnValue({ + const page = (length: number) => ({ data: { query: 'launch', - results: Array.from({ length: 25 }, (_, n) => result(n)), + results: Array.from({ length }, (_, n) => result(n)), retrieval: { status: 'complete', timedOutLegs: [] }, }, isPending: false, @@ -181,20 +181,31 @@ describe('result paging and the custom window', () => { isError: false, refetch: mocks.retry, }) + mocks.search.mockReturnValue(page(20)) + await render() + expect(mocks.search.mock.calls.at(-1)![3]).toBe(20) + const more = () => + [...container.querySelectorAll('button')].find((b) => b.textContent === 'Show more') + expect(more()).toBeDefined() + await act(async () => more()!.click()) + /** The wider search is its own request; the first paint was never widened. */ + expect(mocks.search.mock.calls.at(-1)![3]).toBe(50) + expect(more()).toBeUndefined() + mocks.search.mockReturnValue(page(7)) await render() - expect(container.textContent).toContain('Document 9') - expect(container.textContent).not.toContain('Document 10') - const more = [...container.querySelectorAll('button')].find( - (b) => b.textContent === 'Show more' - )! - expect(more).toBeDefined() - await act(async () => more.click()) - expect(container.textContent).toContain('Document 19') - expect(container.textContent).not.toContain('Document 20') + expect(more()).toBeUndefined() }) it('searches a custom window as an inclusive range of days', async () => { mocks.overview.mockReturnValue({ data: { providers: [], hasSearchableDocuments: true } }) + mocks.search.mockReturnValue({ + data: { query: 'launch', results: [], retrieval: { status: 'complete', timedOutLegs: [] } }, + isPending: false, + isFetching: false, + isPlaceholderData: false, + isError: false, + refetch: mocks.retry, + }) await render(undefined, '?updated=custom&from=2026-09-01&to=2026-09-10') const filters = mocks.search.mock.calls.at(-1)![2] expect(filters.modifiedAfter).toBe('2026-09-01T00:00:00.000Z') diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx index 6b240c434c3..4f57c51b205 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx @@ -4,9 +4,10 @@ import { useState } from 'react' import { Chip, ChipDatePicker, ChipLink, cn } from '@sim/emcn' import { useQueryStates } from 'nuqs' import { ActivityStatus } from '@/components/ui/activity-status' -import type { - WorkspaceKnowledgeSearchResult, - WorkspaceSearchFilters, +import { + WORKSPACE_KNOWLEDGE_SEARCH_LIMITS, + type WorkspaceKnowledgeSearchResult, + type WorkspaceSearchFilters, } from '@/lib/api/contracts/knowledge' import { useSession } from '@/lib/auth/auth-client' import { type ResourceScope, resourceScopeKey } from '@/lib/core/resource-scope' @@ -27,8 +28,6 @@ import { useSearchIndex, useSearchSourceOverview } from '@/hooks/queries/kb/conn import { useWorkspaceKnowledgeSearch } from '@/hooks/queries/kb/knowledge' const DAY_MS = 24 * 60 * 60 * 1000 -/** Cards shown before the reader asks for more; the search itself returns several pages' worth. */ -const RESULTS_PAGE_SIZE = 10 /** Every result without a connector is an upload; the filter names them so. */ const UPLOAD_SOURCE = 'upload' @@ -129,7 +128,8 @@ interface SearchResultsProps { function SearchResults({ scope, query, onSummarize }: SearchResultsProps) { const [hasShownFilters, setHasShownFilters] = useState(false) const [searchedAt] = useState(Date.now) - const [shown, setShown] = useState(RESULTS_PAGE_SIZE) + /** More results are a second, wider search: the first paint stays as quick as it is. */ + const [expanded, setExpanded] = useState(false) const { data: index, isPending: basesPending, @@ -158,7 +158,17 @@ function SearchResults({ scope, query, onSummarize }: SearchResultsProps) { isPlaceholderData, isError: searchFailed, refetch: refetchSearch, - } = useWorkspaceKnowledgeSearch(scope, query, searchFilters) + } = useWorkspaceKnowledgeSearch( + scope, + query, + searchFilters, + expanded + ? WORKSPACE_KNOWLEDGE_SEARCH_LIMITS.expanded + : WORKSPACE_KNOWLEDGE_SEARCH_LIMITS.initial + ) + /** A full first page may collapse to few cards, yet more documents may still match. */ + const mayHaveMore = + !expanded && (search?.results.length ?? 0) >= WORKSPACE_KNOWLEDGE_SEARCH_LIMITS.initial const { data: overview } = useSearchSourceOverview(scope) const indexing = (overview?.providers ?? []) .filter((provider) => provider.isSyncing) @@ -292,7 +302,7 @@ function SearchResults({ scope, query, onSummarize }: SearchResultsProps) { className={cn('flex flex-col', isPlaceholderData && 'opacity-60')} onKeyDown={handleResultsKeyDown} > - {documents.slice(0, shown).map((result) => { + {documents.map((result) => { const source = toSource(result, query, scope) return ( ) })} - {documents.length > shown && ( + {mayHaveMore && (
- setShown((count) => count + RESULTS_PAGE_SIZE)}> + setExpanded(true)}> Show more
diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/search-transitions.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/search-transitions.test.tsx index a9d604f59c5..8da1cf1620f 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/search-transitions.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/search-transitions.test.tsx @@ -212,7 +212,7 @@ describe('search refinement with the real query cache and URL state', () => { expect(requests.at(-1)?.body).toEqual({ organizationId: 'organization', query: 'launch', - topK: 50, + topK: 20, filters: expectedFilters, }) expect(container.querySelector('h1')).toBeNull() diff --git a/apps/sim/hooks/queries/kb/knowledge.test.ts b/apps/sim/hooks/queries/kb/knowledge.test.ts index 07e2c6ca6e5..80379feb5c4 100644 --- a/apps/sim/hooks/queries/kb/knowledge.test.ts +++ b/apps/sim/hooks/queries/kb/knowledge.test.ts @@ -214,9 +214,11 @@ describe('knowledge query placeholder scope', () => { const query = captureQuery(() => useWorkspaceKnowledgeSearch('workspace-1', 'new query', { source: 'slack' }) ) - expect(query.queryKey).toEqual( - knowledgeKeys.search('workspace-1', 'new query', { source: 'slack' }, 'reader') - ) + /** The limit is the key's last part, so the wider search never evicts the first paint. */ + expect(query.queryKey).toEqual([ + ...knowledgeKeys.search('workspace-1', 'new query', { source: 'slack' }, 'reader'), + 20, + ]) expect(knowledgeKeys.search('workspace-1', 'query', { source: 'slack' })).not.toEqual( knowledgeKeys.search('workspace-1', 'query', { source: 'gitlab' }) ) diff --git a/apps/sim/hooks/queries/kb/knowledge.ts b/apps/sim/hooks/queries/kb/knowledge.ts index b59433c2166..2408154a166 100644 --- a/apps/sim/hooks/queries/kb/knowledge.ts +++ b/apps/sim/hooks/queries/kb/knowledge.ts @@ -51,8 +51,10 @@ import { updateKnowledgeChunkContract, updateKnowledgeDocumentContract, updateKnowledgeDocumentTagsContract, + WORKSPACE_KNOWLEDGE_SEARCH_LIMITS, type WorkspaceKnowledgeSearchBody, type WorkspaceKnowledgeSearchData, + type WorkspaceKnowledgeSearchLimit, } from '@/lib/api/contracts/knowledge' import type { WorkspaceSearchFilters } from '@/lib/api/contracts/knowledge/search' import { useSession } from '@/lib/auth/auth-client' @@ -89,8 +91,6 @@ export const KNOWLEDGE_DOCUMENT_LIST_STALE_TIME = 60 * 1000 export const KNOWLEDGE_CHUNK_LIST_STALE_TIME = 60 * 1000 export const KNOWLEDGE_CHUNK_SEARCH_STALE_TIME = 60 * 1000 export const WORKSPACE_KNOWLEDGE_SEARCH_STALE_TIME = 60 * 1000 -/** Chunks one search asks for: several pages of documents once collapsed to one card each. */ -export const WORKSPACE_KNOWLEDGE_SEARCH_RESULT_LIMIT = 50 export const KNOWLEDGE_TAG_DEFINITION_LIST_STALE_TIME = 60 * 1000 export const KNOWLEDGE_TAG_USAGE_STALE_TIME = 60 * 1000 export const KNOWLEDGE_DOCUMENT_TAG_DEFINITION_LIST_STALE_TIME = 60 * 1000 @@ -1209,7 +1209,8 @@ async function searchWorkspaceKnowledge( export function useWorkspaceKnowledgeSearch( owner: string | ResourceScope | undefined, query: string, - filters?: WorkspaceSearchFilters + filters?: WorkspaceSearchFilters, + limit: WorkspaceKnowledgeSearchLimit = WORKSPACE_KNOWLEDGE_SEARCH_LIMITS.initial ) { const { data: session } = useSession() const queryClient = useQueryClient() @@ -1224,14 +1225,15 @@ export function useWorkspaceKnowledgeSearch( const scopeKey = scope?.kind === 'workspace' ? scope.workspaceId : scope ? resourceScopeKey(scope) : undefined return useQuery({ - queryKey: knowledgeKeys.search(scopeKey, trimmed, filters, userId), + /** The limit is the key's last part, so asking for more never evicts the first paint. */ + queryKey: [...knowledgeKeys.search(scopeKey, trimmed, filters, userId), limit], queryFn: ({ signal }) => searchWorkspaceKnowledge( { ...(scope ? resourceScopeFields(scope) : {}), query: trimmed, filters, - topK: WORKSPACE_KNOWLEDGE_SEARCH_RESULT_LIMIT, + topK: limit, }, signal ), diff --git a/apps/sim/lib/api/contracts/knowledge/search.ts b/apps/sim/lib/api/contracts/knowledge/search.ts index 127014355db..c7e333d49f6 100644 --- a/apps/sim/lib/api/contracts/knowledge/search.ts +++ b/apps/sim/lib/api/contracts/knowledge/search.ts @@ -194,6 +194,11 @@ export const workspaceSearchFiltersSchema = z }) export type WorkspaceSearchFilters = z.output +/** Chunks a search asks for at first paint, and once the reader asks for more; both within `topK`'s bound. */ +export const WORKSPACE_KNOWLEDGE_SEARCH_LIMITS = { initial: 20, expanded: 50 } as const +export type WorkspaceKnowledgeSearchLimit = + (typeof WORKSPACE_KNOWLEDGE_SEARCH_LIMITS)[keyof typeof WORKSPACE_KNOWLEDGE_SEARCH_LIMITS] + export const workspaceKnowledgeSearchBodySchema = resourceOwnerSchema.safeExtend({ filters: workspaceSearchFiltersSchema.optional(), query: z.string().trim().min(1, 'A search query is required').max(2000, 'Query is too long'), From d30b7905cc2af30801452905c818644508bbbe95 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sun, 20 Sep 2026 10:51:02 -0700 Subject: [PATCH 03/10] improvement(knowledge): give a filter-first probe the budget its index-driven read can use --- apps/sim/lib/knowledge/search/queries.test.ts | 3 +++ apps/sim/lib/knowledge/search/queries.ts | 12 +++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/apps/sim/lib/knowledge/search/queries.test.ts b/apps/sim/lib/knowledge/search/queries.test.ts index f91a2b4f8d4..f0b88b4d3b8 100644 --- a/apps/sim/lib/knowledge/search/queries.test.ts +++ b/apps/sim/lib/knowledge/search/queries.test.ts @@ -2177,6 +2177,9 @@ describe('filters on a resolved scope', () => { expect(probes).toHaveLength(1) /** Filter first, over the date index: never the reach count that reports a broad reader saturated. */ expect(probes[0].sql).not.toContain('WITH reach') + /** An index-driven probe earns its own budget: a window at the document limit fits inside it. */ + const deadlines = statements().filter((query) => query.sql.includes('statement_timeout')) + expect(deadlines.at(-1)?.params[0]).toBe('1500') expect(JSON.stringify(probes[0])).toContain('"type":"gte"') }) diff --git a/apps/sim/lib/knowledge/search/queries.ts b/apps/sim/lib/knowledge/search/queries.ts index 16e1dd3b5f4..16fa3681f9f 100644 --- a/apps/sim/lib/knowledge/search/queries.ts +++ b/apps/sim/lib/knowledge/search/queries.ts @@ -114,6 +114,14 @@ const VECTOR_PROBE_BUDGET_MS = 600 * comparable cache pressure: the access predicate, evaluated once per document. */ const VECTOR_PROBE_MICROSECONDS_PER_DOCUMENT = 6 +/** + * What a filter-first probe may spend: it reads the filtered documents off their own index and + * tests each one's access, bounded by the same document limit, and measures around 2 µs per + * document to enumerate plus the access test — a window at the limit fits with room. Its result + * is ranked exactly, at a cost that is predictable where a walk through a mostly-excluded + * neighbourhood is not. + */ +const FILTERED_PROBE_BUDGET_MS = 1500 /** * Documents the probe enumerates before it concludes the permitted set is too large to rank * exactly. Derived so that reaching it is what spends the probe's budget, rather than a separate @@ -1001,7 +1009,9 @@ async function probeVisibleDocuments( stage: 'vector.probe' | 'permitted_documents', shape: 'reach-first' | 'direct' = 'reach-first' ): Promise { - const probeBudget = budget?.capped(VECTOR_PROBE_BUDGET_MS) + const probeBudget = budget?.capped( + shape === 'direct' ? FILTERED_PROBE_BUDGET_MS : VECTOR_PROBE_BUDGET_MS + ) try { const probed = await runSearchQuery(probeBudget, stage, (executor) => executor.execute( From 86ed6e19a9d912668deaa2983ef70cf8ea251a8f Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sun, 20 Sep 2026 10:54:32 -0700 Subject: [PATCH 04/10] improvement(knowledge): keep the default scan for a walk that asks the document per tuple --- apps/sim/lib/knowledge/search/queries.test.ts | 7 +++++++ apps/sim/lib/knowledge/search/queries.ts | 20 ++++++++++++++++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/apps/sim/lib/knowledge/search/queries.test.ts b/apps/sim/lib/knowledge/search/queries.test.ts index f0b88b4d3b8..9cafd195382 100644 --- a/apps/sim/lib/knowledge/search/queries.test.ts +++ b/apps/sim/lib/knowledge/search/queries.test.ts @@ -2331,6 +2331,12 @@ describe('filters on a resolved scope', () => { /** The mock renders a nested condition into the params; the date test is the only `gte`. */ const datesDocument = (statement: unknown) => JSON.stringify(statement).includes('"type":"gte"') expect(datesDocument(walks[0])).toBe(true) + /** A walk that asks the document per tuple keeps the default scan cap, not the on-row one. */ + const scanCaps = () => + statements() + .filter((query) => query.sql.includes('hnsw.max_scan_tuples')) + .map((query) => query.params.find((param) => param === '20000' || param === '100000')) + expect(scanCaps().at(-1)).toBe('20000') resetDbChainMock() queueTableRows(schemaMock.embedding, rerankRows) dbChainMockFns.execute.mockImplementation(async (query) => { @@ -2345,6 +2351,7 @@ describe('filters on a resolved scope', () => { accessPlan: plan(), }) expect(datesDocument(statements().filter((query) => isWalk(query.sql))[0])).toBe(false) + expect(scanCaps().at(-1)).toBe('100000') }) it('leaves the keyword leg short when its deadline passes before the ranking is resolved', async () => { diff --git a/apps/sim/lib/knowledge/search/queries.ts b/apps/sim/lib/knowledge/search/queries.ts index 16fa3681f9f..be37c18ddee 100644 --- a/apps/sim/lib/knowledge/search/queries.ts +++ b/apps/sim/lib/knowledge/search/queries.ts @@ -89,6 +89,18 @@ const CANDIDATE_HNSW_MAX_SCAN_TUPLES = '20000' * left with what the neighbourhood happened to hold. */ const ON_ROW_WALK_SCAN_TUPLES = 100_000 + +/** + * How far a walk may go when readability is on the row: the on-row cap, unless the walk still + * has to ask the document about each tuple — a tag or date filter — in which case a tuple costs + * what it did before the columns were mirrored, and the default cap keeps a walk through a + * mostly-excluded neighbourhood at a short answer rather than a missed deadline. + */ +function onRowWalkScanTuples(documentCondition: SQL | undefined): number { + return documentCondition === undefined + ? ON_ROW_WALK_SCAN_TUPLES + : Number(CANDIDATE_HNSW_MAX_SCAN_TUPLES) +} /** * Beam width per iteration. A beam is the granularity of cancellation: pgvector calls * `CHECK_FOR_INTERRUPTS` only while building an index, never inside `hnswgettuple`, so neither @@ -1478,7 +1490,7 @@ async function selectSourceVectorCandidates(input: { ORDER BY ${input.candidateDistance} LIMIT ${input.candidateLimit}`), input.budget, 'vector.source_walk', - ON_ROW_WALK_SCAN_TUPLES + onRowWalkScanTuples(input.documentCondition) ) const walks: Array<() => RankedChunks> = sources.walked.map((connectorId) => walk(eq(embeddingSearch.connectorId, connectorId)) @@ -1597,7 +1609,9 @@ async function selectVectorResults(params: SearchParams): Promise Date: Sun, 20 Sep 2026 11:10:59 -0700 Subject: [PATCH 05/10] fix(knowledge): check the window on the request, rerank with any key, and start a refined search at its first page - the filters schema stays a plain object so the Assistant's search input can still extend it; the window's order is checked on the request body - the dashboard asks for reranking outright, and the use case reranks only when the workspace or the platform holds a key - a refinement of the filters starts over at the first page after the reader asked for more - a custom window's days are the reader's local days --- .../app/api/knowledge/search/route.test.ts | 6 +-- apps/sim/app/api/knowledge/search/route.ts | 8 ++-- .../knowledge-search-results.test.tsx | 34 ++++++++++++- .../knowledge-search-results.tsx | 32 ++++++++++--- .../api/contracts/knowledge/search.test.ts | 25 +++++++++- .../sim/lib/api/contracts/knowledge/search.ts | 48 +++++++++---------- .../lib/knowledge/application/search.test.ts | 12 +++++ apps/sim/lib/knowledge/application/search.ts | 7 ++- apps/sim/lib/knowledge/reranker.ts | 11 ++++- 9 files changed, 137 insertions(+), 46 deletions(-) diff --git a/apps/sim/app/api/knowledge/search/route.test.ts b/apps/sim/app/api/knowledge/search/route.test.ts index b58a8ec40ad..34af2782b41 100644 --- a/apps/sim/app/api/knowledge/search/route.test.ts +++ b/apps/sim/app/api/knowledge/search/route.test.ts @@ -6,9 +6,6 @@ import { NextRequest } from 'next/server' import { beforeEach, describe, expect, it, vi } from 'vitest' const mocks = vi.hoisted(() => ({ search: vi.fn() })) -const { mockHasRotatingApiKey } = vi.hoisted(() => ({ mockHasRotatingApiKey: vi.fn(() => true) })) -vi.mock('@/lib/core/config/api-keys', () => ({ hasRotatingApiKey: mockHasRotatingApiKey })) - vi.mock('@/lib/knowledge/application/workspace-search', () => ({ searchScopedKnowledge: { operation: { id: 'knowledge.search' }, execute: mocks.search }, })) @@ -50,8 +47,7 @@ describe('workspace search route', () => { expect(call.input.signal).toBe(request.signal) expect(call.input.allowPartialResults).toBe(true) expect(call.input.vectorBudgetMs).toBe(3000) - /** A person's search is reranked by the platform's cross-encoder whenever one is configured. */ - expect(mockHasRotatingApiKey).toHaveBeenCalledWith('cohere') + /** A person's search asks for reranking; the use case reranks when a credential exists. */ expect(call.input.rerankerEnabled).toBe(true) expect(call.input.rerankerModel).toBe('rerank-v4.0-fast') controller.abort() diff --git a/apps/sim/app/api/knowledge/search/route.ts b/apps/sim/app/api/knowledge/search/route.ts index 0c9aa4237b9..0e2ed73f3ef 100644 --- a/apps/sim/app/api/knowledge/search/route.ts +++ b/apps/sim/app/api/knowledge/search/route.ts @@ -4,7 +4,6 @@ import { internalRateLimits, internalSessionAuth, } from '@/lib/api/server/routes' -import { hasRotatingApiKey } from '@/lib/core/config/api-keys' import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' import { knowledgeOperations } from '@/lib/knowledge/application/operations' import { searchScopedKnowledge } from '@/lib/knowledge/application/workspace-search' @@ -31,10 +30,11 @@ export const POST = defineInternalJsonRoute({ allowPartialResults: true, vectorBudgetMs: DIRECT_SEARCH_VECTOR_BUDGET_MS, /** - * A person's search is reranked by the platform's cross-encoder when one is configured; - * reranking is best-effort, so a provider outage leaves the fused order in place. + * A person's search is reranked by a cross-encoder whenever the workspace or the platform + * holds a key for one; the use case checks that before spending a call, and reranking stays + * best-effort, so a provider outage leaves the fused order in place. */ - rerankerEnabled: hasRotatingApiKey('cohere'), + rerankerEnabled: true, rerankerModel: DEFAULT_RERANKER_MODEL, surface: 'dashboard' as const, signal: request.signal, diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx index 0a68f35acdf..643cb5353cd 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx @@ -196,6 +196,35 @@ describe('result paging and the custom window', () => { expect(more()).toBeUndefined() }) + it('starts a refined search over at the first page after the reader asked for more', async () => { + mocks.overview.mockReturnValue({ + data: { + providers: [{ connectorType: 'slack', isSyncing: false }], + hasSearchableDocuments: true, + }, + }) + mocks.search.mockReturnValue({ + data: { + query: 'launch', + results: Array.from({ length: 20 }, (_, n) => result(n)), + retrieval: { status: 'complete', timedOutLegs: [] }, + }, + isPending: false, + isFetching: false, + isPlaceholderData: false, + isError: false, + refetch: mocks.retry, + }) + await render() + const button = (label: string) => + [...container.querySelectorAll('button')].find((b) => b.textContent === label)! + await act(async () => button('Show more').click()) + expect(mocks.search.mock.calls.at(-1)![3]).toBe(50) + await act(async () => button('Slack').click()) + expect(mocks.search.mock.calls.at(-1)![2]).toEqual({ source: 'slack' }) + expect(mocks.search.mock.calls.at(-1)![3]).toBe(20) + }) + it('searches a custom window as an inclusive range of days', async () => { mocks.overview.mockReturnValue({ data: { providers: [], hasSearchableDocuments: true } }) mocks.search.mockReturnValue({ @@ -208,7 +237,8 @@ describe('result paging and the custom window', () => { }) await render(undefined, '?updated=custom&from=2026-09-01&to=2026-09-10') const filters = mocks.search.mock.calls.at(-1)![2] - expect(filters.modifiedAfter).toBe('2026-09-01T00:00:00.000Z') - expect(filters.modifiedBefore).toBe('2026-09-10T23:59:59.999Z') + /** The days are the reader's own: local midnight to the last millisecond of the local day. */ + expect(filters.modifiedAfter).toBe(new Date(2026, 8, 1).toISOString()) + expect(filters.modifiedBefore).toBe(new Date(2026, 8, 11, 0, 0, 0, -1).toISOString()) }) }) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx index 4f57c51b205..91b25901e0a 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx @@ -28,6 +28,17 @@ import { useSearchIndex, useSearchSourceOverview } from '@/hooks/queries/kb/conn import { useWorkspaceKnowledgeSearch } from '@/hooks/queries/kb/knowledge' const DAY_MS = 24 * 60 * 60 * 1000 + +/** + * The picker names calendar days; the URL keeps them as dates. A day's bounds are its local + * midnight and the last millisecond before the next, so "September 1" means the reader's own day. + */ +function startOfLocalDay(day: Date): Date { + return new Date(day.getUTCFullYear(), day.getUTCMonth(), day.getUTCDate()) +} +function endOfLocalDay(day: Date): Date { + return new Date(day.getUTCFullYear(), day.getUTCMonth(), day.getUTCDate() + 1, 0, 0, 0, -1) +} /** Every result without a connector is an upload; the filter names them so. */ const UPLOAD_SOURCE = 'upload' @@ -128,8 +139,11 @@ interface SearchResultsProps { function SearchResults({ scope, query, onSummarize }: SearchResultsProps) { const [hasShownFilters, setHasShownFilters] = useState(false) const [searchedAt] = useState(Date.now) - /** More results are a second, wider search: the first paint stays as quick as it is. */ - const [expanded, setExpanded] = useState(false) + /** + * More results are a second, wider search: the first paint stays as quick as it is, and a + * refinement of the filters starts over at the first page. + */ + const [expandedFor, setExpandedFor] = useState(null) const { data: index, isPending: basesPending, @@ -146,11 +160,13 @@ function SearchResults({ scope, query, onSummarize }: SearchResultsProps) { ...(window?.days ? { modifiedAfter: new Date(searchedAt - window.days * DAY_MS).toISOString() } : {}), - ...(custom && filters.from ? { modifiedAfter: filters.from.toISOString() } : {}), - ...(custom && filters.to - ? { modifiedBefore: new Date(filters.to.getTime() + DAY_MS - 1).toISOString() } + ...(custom && filters.from + ? { modifiedAfter: startOfLocalDay(filters.from).toISOString() } : {}), + ...(custom && filters.to ? { modifiedBefore: endOfLocalDay(filters.to).toISOString() } : {}), } + const filtersKey = JSON.stringify(searchFilters) + const expanded = expandedFor === filtersKey const { data: search, isPending, @@ -323,7 +339,11 @@ function SearchResults({ scope, query, onSummarize }: SearchResultsProps) { })} {mayHaveMore && (
- setExpanded(true)}> + setExpandedFor(filtersKey)} + > Show more
diff --git a/apps/sim/lib/api/contracts/knowledge/search.test.ts b/apps/sim/lib/api/contracts/knowledge/search.test.ts index acae92a414d..ce47b60c16d 100644 --- a/apps/sim/lib/api/contracts/knowledge/search.test.ts +++ b/apps/sim/lib/api/contracts/knowledge/search.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from 'vitest' -import { internalKnowledgeSearchBodySchema } from '@/lib/api/contracts/knowledge/search' +import { + internalKnowledgeSearchBodySchema, + workspaceKnowledgeSearchBodySchema, + workspaceSearchFiltersSchema, +} from '@/lib/api/contracts/knowledge/search' import { RESOLVED_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata' describe('internal Knowledge search contract', () => { @@ -20,3 +24,22 @@ describe('internal Knowledge search contract', () => { ).toMatchObject({ [RESOLVED_SECRET_PROVENANCE_FIELD]: provenance }) }) }) + +describe('workspaceKnowledgeSearchBodySchema', () => { + it('refuses a custom window whose end precedes its start, on the request rather than the filters', () => { + const parsed = workspaceKnowledgeSearchBodySchema.safeParse({ + workspaceId: 'workspace-1', + query: 'launch', + filters: { + modifiedAfter: '2026-09-10T00:00:00.000Z', + modifiedBefore: '2026-09-01T00:00:00.000Z', + }, + }) + expect(parsed.success).toBe(false) + if (!parsed.success) { + expect(parsed.error.issues[0]?.path).toEqual(['filters', 'modifiedBefore']) + } + /** The filters schema stays a plain object, so the Assistant's search input can still extend it. */ + expect(typeof workspaceSearchFiltersSchema.extend).toBe('function') + }) +}) diff --git a/apps/sim/lib/api/contracts/knowledge/search.ts b/apps/sim/lib/api/contracts/knowledge/search.ts index c7e333d49f6..70807dbe008 100644 --- a/apps/sim/lib/api/contracts/knowledge/search.ts +++ b/apps/sim/lib/api/contracts/knowledge/search.ts @@ -172,26 +172,13 @@ export const workspaceKnowledgeSearchResultSchema = z.object({ }) export type WorkspaceKnowledgeSearchResult = z.output -export const workspaceSearchFiltersSchema = z - .object({ - source: z.string().trim().min(1, 'Source cannot be empty').max(100).optional(), - modifiedAfter: z.string().datetime({ offset: true }).optional(), - modifiedBefore: z.string().datetime({ offset: true }).optional(), - documentIds: z.array(z.string().min(1).max(200)).min(1).max(20).optional(), - }) - .superRefine((filters, ctx) => { - if ( - filters.modifiedAfter && - filters.modifiedBefore && - Date.parse(filters.modifiedBefore) < Date.parse(filters.modifiedAfter) - ) { - ctx.addIssue({ - code: 'custom', - path: ['modifiedBefore'], - message: 'modifiedBefore must not precede modifiedAfter', - }) - } - }) +/** A plain object, so the Assistant's search input may still extend it; the window's order is checked on the request. */ +export const workspaceSearchFiltersSchema = z.object({ + source: z.string().trim().min(1, 'Source cannot be empty').max(100).optional(), + modifiedAfter: z.string().datetime({ offset: true }).optional(), + modifiedBefore: z.string().datetime({ offset: true }).optional(), + documentIds: z.array(z.string().min(1).max(200)).min(1).max(20).optional(), +}) export type WorkspaceSearchFilters = z.output /** Chunks a search asks for at first paint, and once the reader asks for more; both within `topK`'s bound. */ @@ -199,11 +186,22 @@ export const WORKSPACE_KNOWLEDGE_SEARCH_LIMITS = { initial: 20, expanded: 50 } a export type WorkspaceKnowledgeSearchLimit = (typeof WORKSPACE_KNOWLEDGE_SEARCH_LIMITS)[keyof typeof WORKSPACE_KNOWLEDGE_SEARCH_LIMITS] -export const workspaceKnowledgeSearchBodySchema = resourceOwnerSchema.safeExtend({ - filters: workspaceSearchFiltersSchema.optional(), - query: z.string().trim().min(1, 'A search query is required').max(2000, 'Query is too long'), - topK: z.number().int().min(1).max(50).optional().default(20), -}) +export const workspaceKnowledgeSearchBodySchema = resourceOwnerSchema + .safeExtend({ + filters: workspaceSearchFiltersSchema.optional(), + query: z.string().trim().min(1, 'A search query is required').max(2000, 'Query is too long'), + topK: z.number().int().min(1).max(50).optional().default(20), + }) + .superRefine((body, ctx) => { + const { modifiedAfter, modifiedBefore } = body.filters ?? {} + if (modifiedAfter && modifiedBefore && Date.parse(modifiedBefore) < Date.parse(modifiedAfter)) { + ctx.addIssue({ + code: 'custom', + path: ['filters', 'modifiedBefore'], + message: 'modifiedBefore must not precede modifiedAfter', + }) + } + }) export type WorkspaceKnowledgeSearchBody = z.input export const workspaceKnowledgeSearchDataSchema = z.object({ diff --git a/apps/sim/lib/knowledge/application/search.test.ts b/apps/sim/lib/knowledge/application/search.test.ts index 3b76d732c7f..58a048ef57b 100644 --- a/apps/sim/lib/knowledge/application/search.test.ts +++ b/apps/sim/lib/knowledge/application/search.test.ts @@ -8,6 +8,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import { OrchestrationError } from '@/lib/core/orchestration/types' const mocks = vi.hoisted(() => ({ + hasRerankerCredential: vi.fn(async () => true), resolveWorkspace: vi.fn(), resolveOrganization: vi.fn(), requireOrganizationSearch: vi.fn(), @@ -39,6 +40,7 @@ vi.mock('@/lib/knowledge/search/activity', () => ({ })) vi.mock('@/lib/knowledge/reranker', () => ({ + hasRerankerCredential: mocks.hasRerankerCredential, rerank: mocks.rerank, })) @@ -866,6 +868,16 @@ describe('knowledge search application use case', () => { * `rerank`, the use case swallowed it, and the caller got a 200 whose results * were byte-identical to an unreranked search with nothing to distinguish them. */ + it('never calls the reranker when neither the workspace nor the platform holds a key', async () => { + mocks.hasRerankerCredential.mockResolvedValueOnce(false) + + const result = await rerankedSearch(true) + + expect(mocks.rerank).not.toHaveBeenCalled() + expect(result.rerankerStatus).toBe('unavailable') + expect(result.results[0]).not.toHaveProperty('rerankerScore') + }) + it('reports unavailable rather than silently falling back to vector ordering', async () => { mocks.rerank.mockRejectedValueOnce(new Error('No Cohere API key configured.')) diff --git a/apps/sim/lib/knowledge/application/search.ts b/apps/sim/lib/knowledge/application/search.ts index a8c150f53da..47e80c408f2 100644 --- a/apps/sim/lib/knowledge/application/search.ts +++ b/apps/sim/lib/knowledge/application/search.ts @@ -34,7 +34,7 @@ import { ALL_TAG_SLOTS } from '@/lib/knowledge/constants' import { getEmbeddingModelInfo, toKbEmbeddingDimensions } from '@/lib/knowledge/embedding-models' import { generateSearchEmbedding, type KbEmbeddingTarget } from '@/lib/knowledge/embeddings' import { runWithKnowledgeModelInputProvenance } from '@/lib/knowledge/model-input-provenance' -import { rerank } from '@/lib/knowledge/reranker' +import { hasRerankerCredential, rerank } from '@/lib/knowledge/reranker' import type { RerankerStatus } from '@/lib/knowledge/reranker-models' import { recordOrganizationSearchActivity } from '@/lib/knowledge/search/activity' import { SearchDeadlineError } from '@/lib/knowledge/search/budget' @@ -391,7 +391,10 @@ const searchKnowledgeUseCase = defineAuthorizedKnowledgeUseCase({ boostRecency: searchDefaults.boostRecency, embeddingDimensions: embeddingTarget?.dimensions, }) - const useReranker = Boolean(input.rerankerEnabled && hasQuery) + /** A surface may ask to rerank; without a key for the workspace or the platform there is nothing to ask. */ + const useReranker = + Boolean(input.rerankerEnabled && hasQuery) && + (Boolean(input.rerankerApiKey) || (await hasRerankerCredential(context.workspaceId))) const candidateTopK = useReranker ? input.rerankerInputCount !== undefined ? Math.min( diff --git a/apps/sim/lib/knowledge/reranker.ts b/apps/sim/lib/knowledge/reranker.ts index 085f8da7d47..95eed731605 100644 --- a/apps/sim/lib/knowledge/reranker.ts +++ b/apps/sim/lib/knowledge/reranker.ts @@ -2,7 +2,7 @@ import { createLogger } from '@sim/logger' import { sha256Hex } from '@sim/security/hash' import { backoffWithJitter, parseRetryAfter } from '@sim/utils/retry' import { getBYOKKey } from '@/lib/api-key/byok' -import { getRotatingApiKey } from '@/lib/core/config/api-keys' +import { getRotatingApiKey, hasRotatingApiKey } from '@/lib/core/config/api-keys' import { env } from '@/lib/core/config/env' import { isHosted } from '@/lib/core/config/env-flags' import { @@ -30,6 +30,15 @@ const logger = createLogger('Reranker') const RERANK_OPERATION_TIMEOUT_MS = 30_000 +/** + * Whether a search for this workspace could be reranked at all: a workspace key, or one of the + * platform's. A surface that reranks "when configured" asks this before spending a call on it. + */ +export async function hasRerankerCredential(workspaceId?: string): Promise { + if (workspaceId && (await getBYOKKey(workspaceId, 'cohere'))) return true + return Boolean(env.COHERE_API_KEY) || hasRotatingApiKey('cohere') +} + /** * Cohere bills per "search unit" = one query with up to 100 documents. * We cap at 100 so each rerank call costs exactly 1 unit and matches From 0dc12eeba14829d380378101cd6f1176993cfc5f Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sun, 20 Sep 2026 11:26:13 -0700 Subject: [PATCH 06/10] fix(knowledge): judge a caller's reranker key by the resolver's policy, and let a custom window be cleared - the credential check tries the platform key before a workspace lookup, and counts a caller's own key only off hosted Sim, as the resolver does - a custom window with no days yet searches nothing and says so; another window drops the days; the picker's Clear reaches the URL through a forwarded `onClear` --- .../knowledge-search-results.test.tsx | 41 +++++++++++++++++++ .../knowledge-search-results.tsx | 23 ++++++++--- .../lib/knowledge/application/search.test.ts | 2 + apps/sim/lib/knowledge/application/search.ts | 2 +- apps/sim/lib/knowledge/reranker.ts | 11 +++-- .../chip-date-picker/chip-date-picker.tsx | 3 ++ 6 files changed, 73 insertions(+), 9 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx index 643cb5353cd..9f4e679104f 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx @@ -225,6 +225,47 @@ describe('result paging and the custom window', () => { expect(mocks.search.mock.calls.at(-1)![3]).toBe(20) }) + it('drops the custom days when another window is chosen', async () => { + mocks.overview.mockReturnValue({ data: { providers: [], hasSearchableDocuments: true } }) + mocks.search.mockReturnValue({ + data: { query: 'launch', results: [], retrieval: { status: 'complete', timedOutLegs: [] } }, + isPending: false, + isFetching: false, + isPlaceholderData: false, + isError: false, + refetch: mocks.retry, + }) + await render(undefined, '?updated=custom&from=2026-09-01&to=2026-09-10') + expect(mocks.search.mock.calls.at(-1)![2]).toHaveProperty('modifiedBefore') + const anyTime = [...container.querySelectorAll('button')].find( + (b) => b.textContent === 'Any time' + )! + await act(async () => anyTime.click()) + expect(mocks.search.mock.calls.at(-1)![2]).toEqual({}) + await act(async () => + [...container.querySelectorAll('button')] + .find((b) => b.textContent === 'Custom range')! + .click() + ) + /** Back on the custom window, the old days are gone: nothing is searched until new ones are chosen. */ + expect(mocks.search.mock.calls.at(-1)![1]).toBe('') + }) + + it('searches nothing while a custom window has no days yet', async () => { + mocks.overview.mockReturnValue({ data: { providers: [], hasSearchableDocuments: true } }) + mocks.search.mockReturnValue({ + data: undefined, + isPending: true, + isFetching: false, + isPlaceholderData: false, + isError: false, + refetch: mocks.retry, + }) + await render(undefined, '?updated=custom') + expect(mocks.search.mock.calls.at(-1)![1]).toBe('') + expect(container.textContent).toContain('Choose the days to search.') + }) + it('searches a custom window as an inclusive range of days', async () => { mocks.overview.mockReturnValue({ data: { providers: [], hasSearchableDocuments: true } }) mocks.search.mockReturnValue({ diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx index 91b25901e0a..31c44f1cda2 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx @@ -167,6 +167,8 @@ function SearchResults({ scope, query, onSummarize }: SearchResultsProps) { } const filtersKey = JSON.stringify(searchFilters) const expanded = expandedFor === filtersKey + /** A custom window with no days chosen yet is not "any time": nothing is searched until it has them. */ + const awaitingRange = custom && !filters.from && !filters.to const { data: search, isPending, @@ -176,7 +178,7 @@ function SearchResults({ scope, query, onSummarize }: SearchResultsProps) { refetch: refetchSearch, } = useWorkspaceKnowledgeSearch( scope, - query, + awaitingRange ? '' : query, searchFilters, expanded ? WORKSPACE_KNOWLEDGE_SEARCH_LIMITS.expanded @@ -231,7 +233,11 @@ function SearchResults({ scope, query, onSummarize }: SearchResultsProps) {
- {fetching || (pending && !failed) ? ( + {awaitingRange ? ( +

+ Choose the days to search. +

+ ) : fetching || (pending && !failed) ? ( ) : (

@@ -292,7 +298,13 @@ function SearchResults({ scope, query, onSummarize }: SearchResultsProps) { shape='round' active={filters.updated === window.id} aria-pressed={filters.updated === window.id} - onClick={() => setFilters({ updated: window.id })} + onClick={() => + setFilters( + window.id === 'custom' + ? { updated: window.id } + : { updated: window.id, from: null, to: null } + ) + } > {window.label} @@ -301,11 +313,12 @@ function SearchResults({ scope, query, onSummarize }: SearchResultsProps) { void setFilters({ from: new Date(start), to: new Date(end) }) } + onClear={() => void setFilters({ from: null, to: null })} /> )}

diff --git a/apps/sim/lib/knowledge/application/search.test.ts b/apps/sim/lib/knowledge/application/search.test.ts index 58a048ef57b..2defebd93e1 100644 --- a/apps/sim/lib/knowledge/application/search.test.ts +++ b/apps/sim/lib/knowledge/application/search.test.ts @@ -873,6 +873,8 @@ describe('knowledge search application use case', () => { const result = await rerankedSearch(true) + /** A caller's own key is judged by the same policy the resolver applies, not taken on faith. */ + expect(mocks.hasRerankerCredential).toHaveBeenLastCalledWith(expect.anything(), undefined) expect(mocks.rerank).not.toHaveBeenCalled() expect(result.rerankerStatus).toBe('unavailable') expect(result.results[0]).not.toHaveProperty('rerankerScore') diff --git a/apps/sim/lib/knowledge/application/search.ts b/apps/sim/lib/knowledge/application/search.ts index 47e80c408f2..1d355381606 100644 --- a/apps/sim/lib/knowledge/application/search.ts +++ b/apps/sim/lib/knowledge/application/search.ts @@ -394,7 +394,7 @@ const searchKnowledgeUseCase = defineAuthorizedKnowledgeUseCase({ /** A surface may ask to rerank; without a key for the workspace or the platform there is nothing to ask. */ const useReranker = Boolean(input.rerankerEnabled && hasQuery) && - (Boolean(input.rerankerApiKey) || (await hasRerankerCredential(context.workspaceId))) + (await hasRerankerCredential(context.workspaceId, input.rerankerApiKey)) const candidateTopK = useReranker ? input.rerankerInputCount !== undefined ? Math.min( diff --git a/apps/sim/lib/knowledge/reranker.ts b/apps/sim/lib/knowledge/reranker.ts index 95eed731605..2a28d1a4ce5 100644 --- a/apps/sim/lib/knowledge/reranker.ts +++ b/apps/sim/lib/knowledge/reranker.ts @@ -34,9 +34,14 @@ const RERANK_OPERATION_TIMEOUT_MS = 30_000 * Whether a search for this workspace could be reranked at all: a workspace key, or one of the * platform's. A surface that reranks "when configured" asks this before spending a call on it. */ -export async function hasRerankerCredential(workspaceId?: string): Promise { - if (workspaceId && (await getBYOKKey(workspaceId, 'cohere'))) return true - return Boolean(env.COHERE_API_KEY) || hasRotatingApiKey('cohere') +export async function hasRerankerCredential( + workspaceId?: string, + userApiKey?: string +): Promise { + /** The same policy as the key resolver: a caller's own key counts only off hosted Sim. */ + if (!isHosted && userApiKey) return true + if (env.COHERE_API_KEY || hasRotatingApiKey('cohere')) return true + return Boolean(workspaceId && (await getBYOKKey(workspaceId, 'cohere'))) } /** diff --git a/packages/emcn/src/components/chip-date-picker/chip-date-picker.tsx b/packages/emcn/src/components/chip-date-picker/chip-date-picker.tsx index d6bec05ca7d..ca65061ab31 100644 --- a/packages/emcn/src/components/chip-date-picker/chip-date-picker.tsx +++ b/packages/emcn/src/components/chip-date-picker/chip-date-picker.tsx @@ -62,6 +62,8 @@ interface ChipDatePickerRangeProps extends ChipDatePickerBaseProps { showTime?: boolean /** Called on Apply with the ordered range bounds. */ onRangeChange: (start: string, end: string) => void + /** Called on Clear, so a committed range can be dropped by whoever owns it. */ + onClear?: () => void } export type ChipDatePickerProps = ChipDatePickerSingleProps | ChipDatePickerRangeProps @@ -161,6 +163,7 @@ const ChipDatePicker = forwardRef( setOpen(false) }} onCancel={() => setOpen(false)} + onClear={props.onClear} /> ) : ( Date: Sun, 20 Sep 2026 11:26:55 -0700 Subject: [PATCH 07/10] test(knowledge): assert only the dropped days, not the URL adapter's next flush --- .../knowledge-search-results.test.tsx | 7 ------- 1 file changed, 7 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx index 9f4e679104f..16f72711dac 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx @@ -242,13 +242,6 @@ describe('result paging and the custom window', () => { )! await act(async () => anyTime.click()) expect(mocks.search.mock.calls.at(-1)![2]).toEqual({}) - await act(async () => - [...container.querySelectorAll('button')] - .find((b) => b.textContent === 'Custom range')! - .click() - ) - /** Back on the custom window, the old days are gone: nothing is searched until new ones are chosen. */ - expect(mocks.search.mock.calls.at(-1)![1]).toBe('') }) it('searches nothing while a custom window has no days yet', async () => { From 3d5cb80b2d7c663c9f4d1da03d4bf9f33484aefb Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sun, 20 Sep 2026 11:34:08 -0700 Subject: [PATCH 08/10] chore(knowledge): narrow the filters once before estimating them --- apps/sim/lib/knowledge/search/queries.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/sim/lib/knowledge/search/queries.ts b/apps/sim/lib/knowledge/search/queries.ts index be37c18ddee..e19fc39a1c5 100644 --- a/apps/sim/lib/knowledge/search/queries.ts +++ b/apps/sim/lib/knowledge/search/queries.ts @@ -2456,9 +2456,10 @@ export async function retrieveKnowledgeSearch( */ /** Planning only, so a short cap of its own: running past it answers as the wide window it may be. */ const estimateBudget = budgets.vector.capped(VECTOR_PROBE_BUDGET_MS) + const filters = params.filters const enumerateFiltered = - accessPlan && (dateFilterCondition(params.filters) || params.filters?.source) - ? await estimateFilteredDocuments(knowledgeBaseIds, params.filters, accessPlan, estimateBudget) + accessPlan && filters && (dateFilterCondition(filters) || filters.source) + ? await estimateFilteredDocuments(knowledgeBaseIds, filters, accessPlan, estimateBudget) .then((estimate) => estimate <= VECTOR_PROBE_DOCUMENT_LIMIT) .catch((error) => { if (!estimateBudget.isTimeout(error)) throw error From c0761e90d833f7b7e30bfbcf2be625539bf8db6f Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sun, 20 Sep 2026 11:43:29 -0700 Subject: [PATCH 09/10] fix(knowledge): wait for both days of a custom window, show the chosen range, and keep the default scan while rows are unfilled - a custom window searches only once both days are chosen; the picker shows the chosen range instead of a fixed label - while the projection still holds rows the backfill has not filled, an on-row walk keeps the default scan cap, since an unfilled row is decided through its document; the answer is read off the unfilled-rows index and remembered for a minute --- .../knowledge-search-results.test.tsx | 3 ++ .../knowledge-search-results.tsx | 14 +++--- apps/sim/lib/knowledge/search/queries.test.ts | 27 ++++++++++++ apps/sim/lib/knowledge/search/queries.ts | 44 ++++++++++++++++--- 4 files changed, 75 insertions(+), 13 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx index 16f72711dac..e8dceb6b9cc 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx @@ -257,6 +257,9 @@ describe('result paging and the custom window', () => { await render(undefined, '?updated=custom') expect(mocks.search.mock.calls.at(-1)![1]).toBe('') expect(container.textContent).toContain('Choose the days to search.') + /** One day alone is not a window either; a deep link with only `from` waits for `to`. */ + await render(undefined, '?updated=custom&from=2026-09-01') + expect(mocks.search.mock.calls.at(-1)![1]).toBe('') }) it('searches a custom window as an inclusive range of days', async () => { diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx index 31c44f1cda2..b7320a2d3ed 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx @@ -160,15 +160,17 @@ function SearchResults({ scope, query, onSummarize }: SearchResultsProps) { ...(window?.days ? { modifiedAfter: new Date(searchedAt - window.days * DAY_MS).toISOString() } : {}), - ...(custom && filters.from - ? { modifiedAfter: startOfLocalDay(filters.from).toISOString() } + ...(custom && filters.from && filters.to + ? { + modifiedAfter: startOfLocalDay(filters.from).toISOString(), + modifiedBefore: endOfLocalDay(filters.to).toISOString(), + } : {}), - ...(custom && filters.to ? { modifiedBefore: endOfLocalDay(filters.to).toISOString() } : {}), } const filtersKey = JSON.stringify(searchFilters) const expanded = expandedFor === filtersKey - /** A custom window with no days chosen yet is not "any time": nothing is searched until it has them. */ - const awaitingRange = custom && !filters.from && !filters.to + /** A custom window is two-ended: until both days are chosen, nothing is searched. */ + const awaitingRange = custom && !(filters.from && filters.to) const { data: search, isPending, @@ -312,7 +314,7 @@ function SearchResults({ scope, query, onSummarize }: SearchResultsProps) { {custom && ( diff --git a/apps/sim/lib/knowledge/search/queries.test.ts b/apps/sim/lib/knowledge/search/queries.test.ts index 9cafd195382..fbb5c368c68 100644 --- a/apps/sim/lib/knowledge/search/queries.test.ts +++ b/apps/sim/lib/knowledge/search/queries.test.ts @@ -35,6 +35,7 @@ import { import type { SearchStage } from '@/lib/knowledge/search/diagnostics' import { executeKeywordSearch, + forgetProjectionFilled, forgetSearchReach, getStructuredTagFilters, handleTagAndVectorSearch, @@ -1262,6 +1263,7 @@ describe('permitted-document planner', () => { indexedSourceRows = [] forgetIndexedVectorSources() forgetSearchReach() + forgetProjectionFilled() dbChainMockFns.execute.mockImplementation(async (query) => { const statement = render(query).sql if (statement.includes('pg_index')) return indexedSourceRows @@ -2120,6 +2122,7 @@ describe('filters on a resolved scope', () => { resetDbChainMock() forgetIndexedVectorSources() forgetSearchReach() + forgetProjectionFilled() probeRows = [] traversedRows = [] rerankRows = [] @@ -2266,6 +2269,7 @@ describe('filters on a resolved scope', () => { expect(probes()).toHaveLength(1) estimated = 1_000_000 forgetSearchReach() + forgetProjectionFilled() dbChainMockFns.execute.mockClear() await search() expect(probes()).toHaveLength(0) @@ -2315,6 +2319,29 @@ describe('filters on a resolved scope', () => { ).toHaveLength(1) }) + it('keeps the default scan while the projection still holds unfilled rows', async () => { + traversedRows = [{ id: 'a' }] + rerankRows = [hit('a', 'src-a')] + queueTableRows(schemaMock.embedding, rerankRows) + dbChainMockFns.execute.mockImplementation(async (query) => { + const statement = render(query).sql + if (statement.includes('AS unfilled')) return [{ unfilled: true }] + if (isWalk(statement)) return traversedRows + if (statement.includes('WITH scored_search_candidates')) return rerankRows + return [] + }) + await handleVectorOnlySearch({ + ...params, + permitted: { kind: 'unbounded', broad: true }, + accessPlan: plan(), + }) + /** An unfilled row is decided through its document, so the walk keeps the cap sized for that. */ + const caps = statements() + .filter((query) => query.sql.includes('hnsw.max_scan_tuples')) + .map((query) => query.params.find((param) => param === '20000' || param === '100000')) + expect(caps.at(-1)).toBe('20000') + }) + it('tests the date through the document inside an on-row walk when the filtered set is unbounded', async () => { traversedRows = [{ id: 'a' }] rerankRows = [hit('a', 'src-a')] diff --git a/apps/sim/lib/knowledge/search/queries.ts b/apps/sim/lib/knowledge/search/queries.ts index e19fc39a1c5..b5a101c94ec 100644 --- a/apps/sim/lib/knowledge/search/queries.ts +++ b/apps/sim/lib/knowledge/search/queries.ts @@ -92,15 +92,41 @@ const ON_ROW_WALK_SCAN_TUPLES = 100_000 /** * How far a walk may go when readability is on the row: the on-row cap, unless the walk still - * has to ask the document about each tuple — a tag or date filter — in which case a tuple costs - * what it did before the columns were mirrored, and the default cap keeps a walk through a - * mostly-excluded neighbourhood at a short answer rather than a missed deadline. + * has to ask the document about tuples — a tag or date filter, or rows the backfill has not + * filled yet — in which case such a tuple costs what it did before the columns were mirrored, + * and the default cap keeps a walk through a mostly-excluded neighbourhood at a short answer + * rather than a missed deadline. */ -function onRowWalkScanTuples(documentCondition: SQL | undefined): number { - return documentCondition === undefined +function onRowWalkScanTuples( + documentCondition: SQL | undefined, + projectionFilled: boolean +): number { + return documentCondition === undefined && projectionFilled ? ON_ROW_WALK_SCAN_TUPLES : Number(CANDIDATE_HNSW_MAX_SCAN_TUPLES) } + +/** How long a fully filled projection is taken on trust before its unfilled rows are looked for again. */ +const PROJECTION_FILLED_TTL_MS = 60_000 + +/** + * Whether the ranking projection still holds rows the backfill has not filled. Read off the + * unfilled-rows index in microseconds and remembered briefly: the answer only ever changes once. + */ +const projectionFilled = new LRUCache({ + max: 1, + ttl: PROJECTION_FILLED_TTL_MS, + fetchMethod: async () => { + const [row] = await db.execute<{ unfilled: boolean }>(sql` + SELECT EXISTS (SELECT 1 FROM ${embeddingSearch} WHERE ${embeddingSearch.acl} IS NULL) AS unfilled`) + return !row?.unfilled + }, +}) + +/** Forgets whether the projection was filled, after its rows changed. */ +export function forgetProjectionFilled(): void { + projectionFilled.clear() +} /** * Beam width per iteration. A beam is the granularity of cancellation: pgvector calls * `CHECK_FOR_INTERRUPTS` only while building an index, never inside `hnswgettuple`, so neither @@ -1442,6 +1468,8 @@ async function selectSourceVectorCandidates(input: { plan: SearchAccessPlan tagCondition: SQL | undefined documentCondition: SQL | undefined + /** Whether every projection row carries its mirrored columns, so a walk needs no document. */ + projectionFilled: boolean candidateDistance: SQL candidateLimit: number budget?: SearchBudget @@ -1490,7 +1518,7 @@ async function selectSourceVectorCandidates(input: { ORDER BY ${input.candidateDistance} LIMIT ${input.candidateLimit}`), input.budget, 'vector.source_walk', - onRowWalkScanTuples(input.documentCondition) + onRowWalkScanTuples(input.documentCondition, input.projectionFilled) ) const walks: Array<() => RankedChunks> = sources.walked.map((connectorId) => walk(eq(embeddingSearch.connectorId, connectorId)) @@ -1697,6 +1725,7 @@ async function selectVectorResults(params: SearchParams): Promise const plan = params.access.kind === 'user' ? params.accessPlan : undefined + const filled = plan ? ((await projectionFilled.fetch('embedding_search')) ?? false) : false /** * A source the caller is a member of that has its own index is walked on its own, which * beats ranking it exactly once it is large enough to have earned that index. @@ -1728,6 +1757,7 @@ async function selectVectorResults(params: SearchParams): Promise Date: Sun, 20 Sep 2026 11:52:52 -0700 Subject: [PATCH 10/10] fix(knowledge): show the filters while a custom window waits for its days --- .../knowledge-search-results.test.tsx | 2 ++ .../knowledge-search-results/knowledge-search-results.tsx | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx index e8dceb6b9cc..93e3ba1f46d 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx @@ -257,6 +257,8 @@ describe('result paging and the custom window', () => { await render(undefined, '?updated=custom') expect(mocks.search.mock.calls.at(-1)![1]).toBe('') expect(container.textContent).toContain('Choose the days to search.') + /** The filters, and the picker among them, are shown so the days can be chosen. */ + expect(container.textContent).toContain('Updated between') /** One day alone is not a window either; a deep link with only `from` waits for `to`. */ await render(undefined, '?updated=custom&from=2026-09-01') expect(mocks.search.mock.calls.at(-1)![1]).toBe('') diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx index b7320a2d3ed..64ac4323031 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx @@ -214,8 +214,12 @@ function SearchResults({ scope, query, onSummarize }: SearchResultsProps) { : null const showResults = !noSources && !failed && !basesPending && documents.length > 0 + /** A custom window waiting for its days must show the filters, or the picker is unreachable. */ const showFilters = - hasShownFilters || showResults || (!noSources && !pending && !failed && !!search && !partial) + hasShownFilters || + showResults || + awaitingRange || + (!noSources && !pending && !failed && !!search && !partial) if (showFilters && !hasShownFilters) setHasShownFilters(true) return noSources ? (