feat(kb): Adding support for more tags to the KB (#2433)

* creating boolean, number and date tags with different equality matchings

* feat: add UI for tag field types with filter operators

- Update base-tags-modal with field type selector dropdown
- Update document-tags-modal with different input types per fieldType
- Update knowledge-tag-filters with operator dropdown and type-specific inputs
- Update search routes to support all tag slot types
- Update hook to use AllTagSlot type

* feat: add field type support to document-tag-entry component

- Add dropdown with all field types (Text, Number, Date, Boolean)
- Render different value inputs based on field type
- Update slot counting to include all field types (28 total)

* fix: resolve MAX_TAG_SLOTS error and z-index dropdown issue

- Replace MAX_TAG_SLOTS with totalSlots in document-tag-entry
- Add z-index to SelectContent in base-tags-modal for proper layering

* fix: handle non-text columns in getTagUsage query

- Only apply empty string check for text columns (tag1-tag7)
- Numeric/date/boolean columns only check IS NOT NULL
- Cast values to text for consistent output

* refactor: use EMCN components for KB UI

- Replace @/components/ui imports with @/components/emcn
- Use Combobox instead of Select for dropdowns
- Use EMCN Switch, Button, Input, Label components
- Remove unsupported 'size' prop from EMCN Button

* fix: layout for delete button next to date picker

- Change delete button from absolute to inline positioning
- Add proper column width (w-10) for delete button
- Add empty header cell for delete column
- Apply fix to both document-tag-entry and knowledge-tag-filters

* fix: clear value when switching tag field type

- Reset value to empty when changing type (e.g., boolean to text)
- Reset value when tag name changes and type differs
- Prevents 'true'/'false' from sticking in text inputs

* feat: add full support for number/date/boolean tag filtering in KB search

- Copy all tag types (number, date, boolean) from document to embedding records
- Update processDocumentTags to handle all field types with proper type conversion
- Add number/date/boolean columns to document queries in checkDocumentWriteAccess
- Update chunk creation to inherit all tag types from parent document
- Add getSearchResultFields helper for consistent query result selection
- Support structured filters with operators (eq, gt, lt, between, etc.)
- Fix search queries to include all 28 tag fields in results

* fixing tags import issue

* fix rm file

* reduced number to 3 and date to 2

* fixing lint

* fixed the prop size issue

* increased number from 3 to 5 and boolean from 7 to 2

* fixed number the sql stuff

* progress

* fix document tag and kb tag modals

* update datepicker emcn component

* fix ui

* progress on KB block tags UI

* fix issues with date filters

* fix execution parsing of types for KB tags

* remove migration before merge

* regen migrations

* fix tests and types

* address greptile comments

* fix more greptile comments

* fix filtering logic for multiple of same row

* fix tests

---------

Co-authored-by: priyanshu.solanki <priyanshu.solanki@saviynt.com>
Co-authored-by: Vikhyath Mondreti <vikhyath@simstudio.ai>

Author: 3 people

apps/sim/app/api/knowledge/[id]/documents/[documentId]/chunks/route.ts

@@ -156,13 +156,27 @@ export async function POST(
       const validatedData = CreateChunkSchema.parse(searchParams)
 
       const docTags = {
+        // Text tags (7 slots)
         tag1: doc.tag1 ?? null,
         tag2: doc.tag2 ?? null,
         tag3: doc.tag3 ?? null,
         tag4: doc.tag4 ?? null,
         tag5: doc.tag5 ?? null,
         tag6: doc.tag6 ?? null,
         tag7: doc.tag7 ?? null,
+        // Number tags (5 slots)
+        number1: doc.number1 ?? null,
+        number2: doc.number2 ?? null,
+        number3: doc.number3 ?? null,
+        number4: doc.number4 ?? null,
+        number5: doc.number5 ?? null,
+        // Date tags (2 slots)
+        date1: doc.date1 ?? null,
+        date2: doc.date2 ?? null,
+        // Boolean tags (3 slots)
+        boolean1: doc.boolean1 ?? null,
+        boolean2: doc.boolean2 ?? null,
+        boolean3: doc.boolean3 ?? null,
       }
 
       const newChunk = await createChunk(

apps/sim/app/api/knowledge/[id]/documents/[documentId]/route.test.ts

@@ -72,6 +72,16 @@ describe('Document By ID API Route', () => {
     tag5: null,
     tag6: null,
     tag7: null,
+    number1: null,
+    number2: null,
+    number3: null,
+    number4: null,
+    number5: null,
+    date1: null,
+    date2: null,
+    boolean1: null,
+    boolean2: null,
+    boolean3: null,
     deletedAt: null,
   }
 

apps/sim/app/api/knowledge/[id]/documents/[documentId]/route.ts

@@ -23,14 +23,27 @@ const UpdateDocumentSchema = z.object({
   processingError: z.string().optional(),
   markFailedDueToTimeout: z.boolean().optional(),
   retryProcessing: z.boolean().optional(),
-  // Tag fields
+  // Text tag fields
   tag1: z.string().optional(),
   tag2: z.string().optional(),
   tag3: z.string().optional(),
   tag4: z.string().optional(),
   tag5: z.string().optional(),
   tag6: z.string().optional(),
   tag7: z.string().optional(),
+  // Number tag fields
+  number1: z.string().optional(),
+  number2: z.string().optional(),
+  number3: z.string().optional(),
+  number4: z.string().optional(),
+  number5: z.string().optional(),
+  // Date tag fields
+  date1: z.string().optional(),
+  date2: z.string().optional(),
+  // Boolean tag fields
+  boolean1: z.string().optional(),
+  boolean2: z.string().optional(),
+  boolean3: z.string().optional(),
 })
 
 export async function GET(

apps/sim/app/api/knowledge/[id]/documents/route.test.ts

@@ -80,6 +80,16 @@ describe('Knowledge Base Documents API Route', () => {
     tag5: null,
     tag6: null,
     tag7: null,
+    number1: null,
+    number2: null,
+    number3: null,
+    number4: null,
+    number5: null,
+    date1: null,
+    date2: null,
+    boolean1: null,
+    boolean2: null,
+    boolean3: null,
     deletedAt: null,
   }
 

apps/sim/app/api/knowledge/search/route.test.ts

@@ -64,6 +64,11 @@ vi.mock('@/app/api/knowledge/utils', () => ({
   checkKnowledgeBaseAccess: mockCheckKnowledgeBaseAccess,
 }))
 
+const mockGetDocumentTagDefinitions = vi.fn()
+vi.mock('@/lib/knowledge/tags/service', () => ({
+  getDocumentTagDefinitions: mockGetDocumentTagDefinitions,
+}))
+
 const mockHandleTagOnlySearch = vi.fn()
 const mockHandleVectorOnlySearch = vi.fn()
 const mockHandleTagAndVectorSearch = vi.fn()
@@ -156,6 +161,7 @@ describe('Knowledge Search API Route', () => {
       doc1: 'Document 1',
       doc2: 'Document 2',
     })
+    mockGetDocumentTagDefinitions.mockClear()
 
     vi.stubGlobal('crypto', {
       randomUUID: vi.fn().mockReturnValue('mock-uuid-1234-5678'),
@@ -659,8 +665,8 @@ describe('Knowledge Search API Route', () => {
 
   describe('Optional Query Search', () => {
     const mockTagDefinitions = [
-      { tagSlot: 'tag1', displayName: 'category' },
-      { tagSlot: 'tag2', displayName: 'priority' },
+      { tagSlot: 'tag1', displayName: 'category', fieldType: 'text' },
+      { tagSlot: 'tag2', displayName: 'priority', fieldType: 'text' },
     ]
 
     const mockTaggedResults = [
@@ -689,9 +695,7 @@ describe('Knowledge Search API Route', () => {
     it('should perform tag-only search without query', async () => {
       const tagOnlyData = {
         knowledgeBaseIds: 'kb-123',
-        filters: {
-          category: 'api',
-        },
+        tagFilters: [{ tagName: 'category', value: 'api', fieldType: 'text', operator: 'eq' }],
         topK: 10,
       }
 
@@ -706,10 +710,11 @@ describe('Knowledge Search API Route', () => {
         },
       })
 
-      // Mock tag definitions queries for filter mapping and display mapping
-      mockDbChain.limit
-        .mockResolvedValueOnce(mockTagDefinitions) // Tag definitions for filter mapping
-        .mockResolvedValueOnce(mockTagDefinitions) // Tag definitions for display mapping
+      // Mock tag definitions for validation
+      mockGetDocumentTagDefinitions.mockResolvedValue(mockTagDefinitions)
+
+      // Mock tag definitions queries for display mapping
+      mockDbChain.limit.mockResolvedValueOnce(mockTagDefinitions)
 
       // Mock the tag-only search handler
       mockHandleTagOnlySearch.mockResolvedValue(mockTaggedResults)
@@ -729,17 +734,17 @@ describe('Knowledge Search API Route', () => {
       expect(mockHandleTagOnlySearch).toHaveBeenCalledWith({
         knowledgeBaseIds: ['kb-123'],
         topK: 10,
-        filters: { category: 'api' }, // Note: When no tag definitions are found, it uses the original filter key
+        structuredFilters: [
+          { tagSlot: 'tag1', fieldType: 'text', operator: 'eq', value: 'api', valueTo: undefined },
+        ],
       })
     })
 
     it('should perform query + tag combination search', async () => {
       const combinedData = {
         knowledgeBaseIds: 'kb-123',
         query: 'test search',
-        filters: {
-          category: 'api',
-        },
+        tagFilters: [{ tagName: 'category', value: 'api', fieldType: 'text', operator: 'eq' }],
         topK: 10,
       }
 
@@ -754,10 +759,11 @@ describe('Knowledge Search API Route', () => {
         },
       })
 
-      // Mock tag definitions queries for filter mapping and display mapping
-      mockDbChain.limit
-        .mockResolvedValueOnce(mockTagDefinitions) // Tag definitions for filter mapping
-        .mockResolvedValueOnce(mockTagDefinitions) // Tag definitions for display mapping
+      // Mock tag definitions for validation
+      mockGetDocumentTagDefinitions.mockResolvedValue(mockTagDefinitions)
+
+      // Mock tag definitions queries for display mapping
+      mockDbChain.limit.mockResolvedValueOnce(mockTagDefinitions)
 
       // Mock the tag + vector search handler
       mockHandleTagAndVectorSearch.mockResolvedValue(mockSearchResults)
@@ -784,7 +790,9 @@ describe('Knowledge Search API Route', () => {
       expect(mockHandleTagAndVectorSearch).toHaveBeenCalledWith({
         knowledgeBaseIds: ['kb-123'],
         topK: 10,
-        filters: { category: 'api' }, // Note: When no tag definitions are found, it uses the original filter key
+        structuredFilters: [
+          { tagSlot: 'tag1', fieldType: 'text', operator: 'eq', value: 'api', valueTo: undefined },
+        ],
         queryVector: JSON.stringify(mockEmbedding),
         distanceThreshold: 1, // Single KB uses threshold of 1.0
       })
@@ -928,10 +936,10 @@ describe('Knowledge Search API Route', () => {
     it('should handle tag-only search with multiple knowledge bases', async () => {
       const multiKbTagData = {
         knowledgeBaseIds: ['kb-123', 'kb-456'],
-        filters: {
-          category: 'docs',
-          priority: 'high',
-        },
+        tagFilters: [
+          { tagName: 'category', value: 'docs', fieldType: 'text', operator: 'eq' },
+          { tagName: 'priority', value: 'high', fieldType: 'text', operator: 'eq' },
+        ],
         topK: 10,
       }
 
@@ -951,37 +959,14 @@ describe('Knowledge Search API Route', () => {
           knowledgeBase: { id: 'kb-456', userId: 'user-123', name: 'Test KB 2' },
         })
 
-      // Reset all mocks before setting up specific behavior
-      Object.values(mockDbChain).forEach((fn) => {
-        if (typeof fn === 'function') {
-          fn.mockClear().mockReturnThis()
-        }
-      })
+      // Mock tag definitions for validation
+      mockGetDocumentTagDefinitions.mockResolvedValue(mockTagDefinitions)
 
-      // Create fresh mocks for multiple database calls needed for multi-KB tag search
-      const mockTagDefsQuery1 = {
-        ...mockDbChain,
-        limit: vi.fn().mockResolvedValue(mockTagDefinitions),
-      }
-      const mockTagSearchQuery = {
-        ...mockDbChain,
-        limit: vi.fn().mockResolvedValue(mockTaggedResults),
-      }
-      const mockTagDefsQuery2 = {
-        ...mockDbChain,
-        limit: vi.fn().mockResolvedValue(mockTagDefinitions),
-      }
-      const mockTagDefsQuery3 = {
-        ...mockDbChain,
-        limit: vi.fn().mockResolvedValue(mockTagDefinitions),
-      }
+      // Mock the tag-only search handler
+      mockHandleTagOnlySearch.mockResolvedValue(mockTaggedResults)
 
-      // Chain the mocks for: tag defs, search, display mapping KB1, display mapping KB2
-      mockDbChain.select
-        .mockReturnValueOnce(mockTagDefsQuery1)
-        .mockReturnValueOnce(mockTagSearchQuery)
-        .mockReturnValueOnce(mockTagDefsQuery2)
-        .mockReturnValueOnce(mockTagDefsQuery3)
+      // Mock tag definitions queries for display mapping
+      mockDbChain.limit.mockResolvedValueOnce(mockTagDefinitions)
 
       const req = createMockRequest('POST', multiKbTagData)
       const { POST } = await import('@/app/api/knowledge/search/route')
@@ -1076,6 +1061,11 @@ describe('Knowledge Search API Route', () => {
         },
       })
 
+      // Mock tag definitions for validation
+      mockGetDocumentTagDefinitions.mockResolvedValue([
+        { tagSlot: 'tag1', displayName: 'tag1', fieldType: 'text' },
+      ])
+
       mockHandleTagOnlySearch.mockResolvedValue([
         {
           id: 'chunk-2',
@@ -1108,13 +1098,15 @@ describe('Knowledge Search API Route', () => {
       const mockTagDefs = {
         select: vi.fn().mockReturnThis(),
         from: vi.fn().mockReturnThis(),
-        where: vi.fn().mockResolvedValue([]),
+        where: vi
+          .fn()
+          .mockResolvedValue([{ tagSlot: 'tag1', displayName: 'tag1', fieldType: 'text' }]),
       }
       mockDbChain.select.mockReturnValueOnce(mockTagDefs)
 
       const req = createMockRequest('POST', {
         knowledgeBaseIds: ['kb-123'],
-        filters: { tag1: 'api' },
+        tagFilters: [{ tagName: 'tag1', value: 'api', fieldType: 'text', operator: 'eq' }],
         topK: 10,
       })
 
@@ -1143,6 +1135,11 @@ describe('Knowledge Search API Route', () => {
         },
       })
 
+      // Mock tag definitions for validation
+      mockGetDocumentTagDefinitions.mockResolvedValue([
+        { tagSlot: 'tag1', displayName: 'tag1', fieldType: 'text' },
+      ])
+
       mockHandleTagAndVectorSearch.mockResolvedValue([
         {
           id: 'chunk-3',
@@ -1176,14 +1173,16 @@ describe('Knowledge Search API Route', () => {
       const mockTagDefs = {
         select: vi.fn().mockReturnThis(),
         from: vi.fn().mockReturnThis(),
-        where: vi.fn().mockResolvedValue([]),
+        where: vi
+          .fn()
+          .mockResolvedValue([{ tagSlot: 'tag1', displayName: 'tag1', fieldType: 'text' }]),
       }
       mockDbChain.select.mockReturnValueOnce(mockTagDefs)
 
       const req = createMockRequest('POST', {
         knowledgeBaseIds: ['kb-123'],
         query: 'relevant content',
-        filters: { tag1: 'guide' },
+        tagFilters: [{ tagName: 'tag1', value: 'guide', fieldType: 'text', operator: 'eq' }],
         topK: 10,
       })