fix(knowledge): wait for both days of a custom window, show the chosen range, and keep the default scan while rows are unfilled

- a custom window searches only once both days are chosen; the picker shows the chosen range instead of a fixed label
- while the projection still holds rows the backfill has not filled, an on-row walk keeps the default scan cap, since an unfilled row is decided through its document; the answer is read off the unfilled-rows index and remembered for a minute

Author: waleedlatif1

apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx

@@ -257,6 +257,9 @@ describe('result paging and the custom window', () => {
     await render(undefined, '?updated=custom')
     expect(mocks.search.mock.calls.at(-1)![1]).toBe('')
     expect(container.textContent).toContain('Choose the days to search.')
+    /** One day alone is not a window either; a deep link with only `from` waits for `to`. */
+    await render(undefined, '?updated=custom&from=2026-09-01')
+    expect(mocks.search.mock.calls.at(-1)![1]).toBe('')
   })
 
   it('searches a custom window as an inclusive range of days', async () => {

apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx

@@ -160,15 +160,17 @@ function SearchResults({ scope, query, onSummarize }: SearchResultsProps) {
     ...(window?.days
       ? { modifiedAfter: new Date(searchedAt - window.days * DAY_MS).toISOString() }
       : {}),
-    ...(custom && filters.from
-      ? { modifiedAfter: startOfLocalDay(filters.from).toISOString() }
+    ...(custom && filters.from && filters.to
+      ? {
+          modifiedAfter: startOfLocalDay(filters.from).toISOString(),
+          modifiedBefore: endOfLocalDay(filters.to).toISOString(),
+        }
       : {}),
-    ...(custom && filters.to ? { modifiedBefore: endOfLocalDay(filters.to).toISOString() } : {}),
   }
   const filtersKey = JSON.stringify(searchFilters)
   const expanded = expandedFor === filtersKey
-  /** A custom window with no days chosen yet is not "any time": nothing is searched until it has them. */
-  const awaitingRange = custom && !filters.from && !filters.to
+  /** A custom window is two-ended: until both days are chosen, nothing is searched. */
+  const awaitingRange = custom && !(filters.from && filters.to)
   const {
     data: search,
     isPending,
@@ -312,7 +314,7 @@ function SearchResults({ scope, query, onSummarize }: SearchResultsProps) {
           {custom && (
             <ChipDatePicker
               mode='range'
-              label='Updated between'
+              placeholder='Updated between'
               startDate={filters.from?.toISOString().slice(0, 10)}
               endDate={filters.to?.toISOString().slice(0, 10)}
               onRangeChange={(start, end) =>

apps/sim/lib/knowledge/search/queries.test.ts

@@ -35,6 +35,7 @@ import {
 import type { SearchStage } from '@/lib/knowledge/search/diagnostics'
 import {
   executeKeywordSearch,
+  forgetProjectionFilled,
   forgetSearchReach,
   getStructuredTagFilters,
   handleTagAndVectorSearch,
@@ -1262,6 +1263,7 @@ describe('permitted-document planner', () => {
     indexedSourceRows = []
     forgetIndexedVectorSources()
     forgetSearchReach()
+    forgetProjectionFilled()
     dbChainMockFns.execute.mockImplementation(async (query) => {
       const statement = render(query).sql
       if (statement.includes('pg_index')) return indexedSourceRows
@@ -2120,6 +2122,7 @@ describe('filters on a resolved scope', () => {
     resetDbChainMock()
     forgetIndexedVectorSources()
     forgetSearchReach()
+    forgetProjectionFilled()
     probeRows = []
     traversedRows = []
     rerankRows = []
@@ -2266,6 +2269,7 @@ describe('filters on a resolved scope', () => {
     expect(probes()).toHaveLength(1)
     estimated = 1_000_000
     forgetSearchReach()
+    forgetProjectionFilled()
     dbChainMockFns.execute.mockClear()
     await search()
     expect(probes()).toHaveLength(0)
@@ -2315,6 +2319,29 @@ describe('filters on a resolved scope', () => {
     ).toHaveLength(1)
   })
 
+  it('keeps the default scan while the projection still holds unfilled rows', async () => {
+    traversedRows = [{ id: 'a' }]
+    rerankRows = [hit('a', 'src-a')]
+    queueTableRows(schemaMock.embedding, rerankRows)
+    dbChainMockFns.execute.mockImplementation(async (query) => {
+      const statement = render(query).sql
+      if (statement.includes('AS unfilled')) return [{ unfilled: true }]
+      if (isWalk(statement)) return traversedRows
+      if (statement.includes('WITH scored_search_candidates')) return rerankRows
+      return []
+    })
+    await handleVectorOnlySearch({
+      ...params,
+      permitted: { kind: 'unbounded', broad: true },
+      accessPlan: plan(),
+    })
+    /** An unfilled row is decided through its document, so the walk keeps the cap sized for that. */
+    const caps = statements()
+      .filter((query) => query.sql.includes('hnsw.max_scan_tuples'))
+      .map((query) => query.params.find((param) => param === '20000' || param === '100000'))
+    expect(caps.at(-1)).toBe('20000')
+  })
+
   it('tests the date through the document inside an on-row walk when the filtered set is unbounded', async () => {
     traversedRows = [{ id: 'a' }]
     rerankRows = [hit('a', 'src-a')]

apps/sim/lib/knowledge/search/queries.ts

@@ -92,15 +92,41 @@ const ON_ROW_WALK_SCAN_TUPLES = 100_000
 
 /**
  * How far a walk may go when readability is on the row: the on-row cap, unless the walk still
- * has to ask the document about each tuple — a tag or date filter — in which case a tuple costs
- * what it did before the columns were mirrored, and the default cap keeps a walk through a
- * mostly-excluded neighbourhood at a short answer rather than a missed deadline.
+ * has to ask the document about tuples — a tag or date filter, or rows the backfill has not
+ * filled yet — in which case such a tuple costs what it did before the columns were mirrored,
+ * and the default cap keeps a walk through a mostly-excluded neighbourhood at a short answer
+ * rather than a missed deadline.
  */
-function onRowWalkScanTuples(documentCondition: SQL | undefined): number {
-  return documentCondition === undefined
+function onRowWalkScanTuples(
+  documentCondition: SQL | undefined,
+  projectionFilled: boolean
+): number {
+  return documentCondition === undefined && projectionFilled
     ? ON_ROW_WALK_SCAN_TUPLES
     : Number(CANDIDATE_HNSW_MAX_SCAN_TUPLES)
 }
+
+/** How long a fully filled projection is taken on trust before its unfilled rows are looked for again. */
+const PROJECTION_FILLED_TTL_MS = 60_000
+
+/**
+ * Whether the ranking projection still holds rows the backfill has not filled. Read off the
+ * unfilled-rows index in microseconds and remembered briefly: the answer only ever changes once.
+ */
+const projectionFilled = new LRUCache<string, boolean>({
+  max: 1,
+  ttl: PROJECTION_FILLED_TTL_MS,
+  fetchMethod: async () => {
+    const [row] = await db.execute<{ unfilled: boolean }>(sql`
+      SELECT EXISTS (SELECT 1 FROM ${embeddingSearch} WHERE ${embeddingSearch.acl} IS NULL) AS unfilled`)
+    return !row?.unfilled
+  },
+})
+
+/** Forgets whether the projection was filled, after its rows changed. */
+export function forgetProjectionFilled(): void {
+  projectionFilled.clear()
+}
 /**
  * Beam width per iteration. A beam is the granularity of cancellation: pgvector calls
  * `CHECK_FOR_INTERRUPTS` only while building an index, never inside `hnswgettuple`, so neither
@@ -1442,6 +1468,8 @@ async function selectSourceVectorCandidates(input: {
   plan: SearchAccessPlan
   tagCondition: SQL | undefined
   documentCondition: SQL | undefined
+  /** Whether every projection row carries its mirrored columns, so a walk needs no document. */
+  projectionFilled: boolean
   candidateDistance: SQL<number>
   candidateLimit: number
   budget?: SearchBudget
@@ -1490,7 +1518,7 @@ async function selectSourceVectorCandidates(input: {
             ORDER BY ${input.candidateDistance} LIMIT ${input.candidateLimit}`),
         input.budget,
         'vector.source_walk',
-        onRowWalkScanTuples(input.documentCondition)
+        onRowWalkScanTuples(input.documentCondition, input.projectionFilled)
       )
   const walks: Array<() => RankedChunks> = sources.walked.map((connectorId) =>
     walk(eq(embeddingSearch.connectorId, connectorId))
@@ -1697,6 +1725,7 @@ async function selectVectorResults(params: SearchParams): Promise<SearchResult[]
         }
         let selected: Array<{ id: string }>
         const plan = params.access.kind === 'user' ? params.accessPlan : undefined
+        const filled = plan ? ((await projectionFilled.fetch('embedding_search')) ?? false) : false
         /**
          * A source the caller is a member of that has its own index is walked on its own, which
          * beats ranking it exactly once it is large enough to have earned that index.
@@ -1728,6 +1757,7 @@ async function selectVectorResults(params: SearchParams): Promise<SearchResult[]
             access: params.access,
             knowledgeBaseIds: params.knowledgeBaseIds,
             plan,
+            projectionFilled: filled,
             tagCondition: candidateTagCondition,
             documentCondition,
             candidateDistance,
@@ -1771,7 +1801,7 @@ async function selectVectorResults(params: SearchParams): Promise<SearchResult[]
               ),
             params.budget,
             'vector.candidate_search',
-            plan ? onRowWalkScanTuples(documentCondition) : undefined
+            plan ? onRowWalkScanTuples(documentCondition, filled) : undefined
           )
           /**
            * A full traversal is already the nearest permitted chunks, so nothing else is worth