From 24f061116f606c3262ee575d499a61a5e881c12f Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 21 Sep 2026 19:39:03 -0700 Subject: [PATCH 1/5] fix(knowledge): bound source titles and tag values so an oversized index row cannot fail a connector sync --- .../connectors/sync-persistence.test.ts | 40 ++++++++++++++++++- .../knowledge/connectors/sync-persistence.ts | 23 +++++++++-- 2 files changed, 58 insertions(+), 5 deletions(-) diff --git a/apps/sim/lib/knowledge/connectors/sync-persistence.test.ts b/apps/sim/lib/knowledge/connectors/sync-persistence.test.ts index 34c9e6a2423..f4593174554 100644 --- a/apps/sim/lib/knowledge/connectors/sync-persistence.test.ts +++ b/apps/sim/lib/knowledge/connectors/sync-persistence.test.ts @@ -27,7 +27,11 @@ vi.mock('@/lib/knowledge/documents/storage-cleanup', () => ({ }), isKnowledgeBaseOwnedStorageKey: (key: string) => key.startsWith('kb/'), })) -vi.mock('@/connectors/registry.server', () => ({ CONNECTOR_REGISTRY: {} })) +vi.mock('@/connectors/registry.server', () => ({ + CONNECTOR_REGISTRY: { + fixture: { mapTags: (metadata: Record) => ({ label: metadata.label }) }, + }, +})) import { MAX_ACL_TOKENS } from '@/lib/knowledge/access/tokens' import { getConnectorFailureDiagnostic } from '@/lib/knowledge/connectors/connector-error' @@ -35,6 +39,7 @@ import { addDocument, persistDocumentAcls, persistSourceDocumentFailures, + resolveTagMapping, } from '@/lib/knowledge/connectors/sync-persistence' const CONNECTOR = 'connector-1' @@ -350,6 +355,18 @@ describe('persistSourceDocumentFailures', () => { expect(JSON.stringify(dbChainMockFns.set.mock.calls)).not.toContain('private body') expect(dbChainMockFns.delete).not.toHaveBeenCalled() }) + it('bounds a source title that would exceed the filename index row limit', async () => { + leaseHeld() + const title = 'x'.repeat(5000) + await persistSourceDocumentFailures({ + ...input, + documents: [{ ...input.documents[0], title }], + priorByExternalId: new Map(), + }) + const [rows] = dbChainMockFns.values.mock.calls[0] as [Array<{ filename: string }>] + expect(rows[0].filename.length).toBeLessThan(600) + expect(rows[0].filename.startsWith('xxxx')).toBe(true) + }) it('refuses to commit a failure under a reclaimed lease', async () => { queueTableRows(schemaMock.knowledgeBase, [{ id: 'kb' }]) await expect( @@ -416,3 +433,24 @@ describe('organization source cache persistence', () => { expect(mockUploadFile).not.toHaveBeenCalled() }) }) + +describe('resolveTagMapping', () => { + it('bounds a mapped tag value that would exceed its index row limit', () => { + const tags = resolveTagMapping( + 'fixture', + { label: 'y'.repeat(5000) }, + { tagSlotMapping: { label: 'tag1' } } + ) + expect(tags?.tag1?.length).toBeLessThan(600) + expect(tags?.tag1?.startsWith('yyyy')).toBe(true) + }) + + it('keeps a short mapped tag value intact', () => { + const tags = resolveTagMapping( + 'fixture', + { label: 'Purchasing' }, + { tagSlotMapping: { label: 'tag1' } } + ) + expect(tags?.tag1).toBe('Purchasing') + }) +}) diff --git a/apps/sim/lib/knowledge/connectors/sync-persistence.ts b/apps/sim/lib/knowledge/connectors/sync-persistence.ts index 33b53184ac2..526861107bd 100644 --- a/apps/sim/lib/knowledge/connectors/sync-persistence.ts +++ b/apps/sim/lib/knowledge/connectors/sync-persistence.ts @@ -3,6 +3,7 @@ import { document, embedding, knowledgeBase, knowledgeConnector } from '@sim/db/ import { createLogger } from '@sim/logger' import { chunkArray } from '@sim/utils/helpers' import { generateId } from '@sim/utils/id' +import { truncate } from '@sim/utils/string' import { and, eq, exists, inArray, isNull, lt, or, sql } from 'drizzle-orm' import { getInternalApiBaseUrl } from '@/lib/core/utils/urls' import type { DbOrTx } from '@/lib/db/types' @@ -173,6 +174,19 @@ export async function persistDocumentAcls( const MAX_SAFE_TITLE_LENGTH = 200 +/** + * A document's filename and text tags sit under btree indexes, and Postgres refuses an index + * row past about 2.7 KB. A source title or tag value beyond that fails the row and, with it, + * every sync that lists the document again. 512 characters keeps a four-byte-per-character + * value inside the ceiling. + */ +const MAX_INDEXED_TEXT_LENGTH = 512 + +/** Bounds a source-supplied value that lands in an indexed text column. */ +function boundIndexedText(value: string): string { + return truncate(value, MAX_INDEXED_TEXT_LENGTH) +} + function sanitizeStorageTitle(title: string): string { return title.replace(/[^a-zA-Z0-9.-]/g, '_').slice(0, MAX_SAFE_TITLE_LENGTH) } @@ -250,7 +264,8 @@ export function resolveTagMapping( const result: Partial = {} for (const [semanticKey, slot] of Object.entries(mapping)) { const value = semanticTags[semanticKey] - ;(result as Record)[slot] = value != null ? value : null + ;(result as Record)[slot] = + typeof value === 'string' ? boundIndexedText(value) : value != null ? value : null } return result } @@ -278,7 +293,7 @@ function buildSkippedDocumentRow( return { id: generateId(), knowledgeBaseId, - filename: extDoc.title, + filename: boundIndexedText(extDoc.title), fileUrl: '', storageKey: null, /** No artifact was stored; a provider's reported source size is not local storage usage. */ @@ -630,7 +645,7 @@ export async function addDocument( await tx.insert(document).values({ id: documentId, knowledgeBaseId, - filename: extDoc.title, + filename: boundIndexedText(extDoc.title), fileUrl, storageKey: fileInfo.key, fileSize: artifact.bytes.length, @@ -745,7 +760,7 @@ export async function updateDocument( await tx .update(document) .set({ - filename: extDoc.title, + filename: boundIndexedText(extDoc.title), fileUrl, storageKey: fileInfo.key, fileSize: artifact.bytes.length, From 720797482d2d5098a4c115b420fdd37a14ff895d Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 21 Sep 2026 19:44:48 -0700 Subject: [PATCH 2/5] test(knowledge): pin the exact indexed-text boundary --- apps/sim/lib/knowledge/connectors/sync-persistence.test.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/apps/sim/lib/knowledge/connectors/sync-persistence.test.ts b/apps/sim/lib/knowledge/connectors/sync-persistence.test.ts index f4593174554..23a98dd5482 100644 --- a/apps/sim/lib/knowledge/connectors/sync-persistence.test.ts +++ b/apps/sim/lib/knowledge/connectors/sync-persistence.test.ts @@ -364,8 +364,7 @@ describe('persistSourceDocumentFailures', () => { priorByExternalId: new Map(), }) const [rows] = dbChainMockFns.values.mock.calls[0] as [Array<{ filename: string }>] - expect(rows[0].filename.length).toBeLessThan(600) - expect(rows[0].filename.startsWith('xxxx')).toBe(true) + expect(rows[0].filename).toBe(`${'x'.repeat(512)}...`) }) it('refuses to commit a failure under a reclaimed lease', async () => { queueTableRows(schemaMock.knowledgeBase, [{ id: 'kb' }]) @@ -441,8 +440,7 @@ describe('resolveTagMapping', () => { { label: 'y'.repeat(5000) }, { tagSlotMapping: { label: 'tag1' } } ) - expect(tags?.tag1?.length).toBeLessThan(600) - expect(tags?.tag1?.startsWith('yyyy')).toBe(true) + expect(tags?.tag1).toBe(`${'y'.repeat(512)}...`) }) it('keeps a short mapped tag value intact', () => { From 20f9277171f3058338bb6551aea6f1e9ba4ec3fc Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 21 Sep 2026 20:18:16 -0700 Subject: [PATCH 3/5] fix(knowledge): share the indexed-text bound, cut by code point, and reject oversized filenames and tags at the document APIs --- .../api/contracts/knowledge/documents.test.ts | 35 ++++++++++++ .../lib/api/contracts/knowledge/documents.ts | 54 ++++++++++++------- .../connectors/sync-persistence.test.ts | 17 +++--- .../knowledge/connectors/sync-persistence.ts | 19 ++++--- apps/sim/lib/knowledge/constants.ts | 8 +++ .../sim/lib/knowledge/upload-metadata.test.ts | 11 ++++ apps/sim/lib/knowledge/upload-metadata.ts | 6 ++- 7 files changed, 114 insertions(+), 36 deletions(-) diff --git a/apps/sim/lib/api/contracts/knowledge/documents.test.ts b/apps/sim/lib/api/contracts/knowledge/documents.test.ts index 9a12967d460..e6a3a52fc5b 100644 --- a/apps/sim/lib/api/contracts/knowledge/documents.test.ts +++ b/apps/sim/lib/api/contracts/knowledge/documents.test.ts @@ -5,11 +5,14 @@ import { describe, expect, it } from 'vitest' import { z } from 'zod' import { bulkCreateDocumentsBodySchema, + createDocumentBodySchema, documentDataSchema, listKnowledgeDocumentsQuerySchema, parseDocumentTagFiltersParam, + updateDocumentBodySchema, upsertDocumentBodySchema, } from '@/lib/api/contracts/knowledge/documents' +import { MAX_DOCUMENT_INDEXED_TEXT_LENGTH } from '@/lib/knowledge/constants' import { getDocumentIndexingStatus } from '@/lib/knowledge/documents/types' describe('document processing response compatibility', () => { @@ -255,3 +258,35 @@ describe('internal document processingOptions', () => { }) }) }) + +describe('document filename and tag bounds', () => { + const base = { fileUrl: 'https://example.com/a.txt', fileSize: 1, mimeType: 'text/plain' } + const atLimit = 'a'.repeat(MAX_DOCUMENT_INDEXED_TEXT_LENGTH) + const overLimit = `${atLimit}a` + + it('accepts a filename and tag exactly at the indexed-text limit', () => { + expect( + createDocumentBodySchema.safeParse({ ...base, filename: atLimit, tag1: atLimit }).success + ).toBe(true) + }) + + it('rejects a filename over the limit on create, upsert, and update with a descriptive message', () => { + for (const schema of [createDocumentBodySchema, upsertDocumentBodySchema, updateDocumentBodySchema]) { + const result = schema.safeParse({ ...base, filename: overLimit }) + expect(result.success).toBe(false) + expect(result.error?.issues[0]?.message).toBe( + `Filename cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters` + ) + } + }) + + it('rejects a tag value over the limit on create and update', () => { + for (const schema of [createDocumentBodySchema, updateDocumentBodySchema]) { + const result = schema.safeParse({ ...base, filename: 'a.txt', tag3: overLimit }) + expect(result.success).toBe(false) + expect(result.error?.issues[0]?.message).toBe( + `Tag values cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters` + ) + } + }) +}) diff --git a/apps/sim/lib/api/contracts/knowledge/documents.ts b/apps/sim/lib/api/contracts/knowledge/documents.ts index 4b586ff6765..3d8de902eeb 100644 --- a/apps/sim/lib/api/contracts/knowledge/documents.ts +++ b/apps/sim/lib/api/contracts/knowledge/documents.ts @@ -15,7 +15,11 @@ import { import { privateSecretProvenanceBundleSchema } from '@/lib/api/contracts/primitives' import { defineRouteContract } from '@/lib/api/contracts/types' import { PRIVATE_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata' -import { getFieldTypeForSlot, MAX_KNOWLEDGE_DOCUMENTS_PER_CREATE } from '@/lib/knowledge/constants' +import { + getFieldTypeForSlot, + MAX_DOCUMENT_INDEXED_TEXT_LENGTH, + MAX_KNOWLEDGE_DOCUMENTS_PER_CREATE, +} from '@/lib/knowledge/constants' import { DOCUMENT_PROCESSING_STATUSES } from '@/lib/knowledge/documents/types' import { getOperatorsForFieldType, isValidFilterValue } from '@/lib/knowledge/filters/types' import { knowledgeDocumentUploadMetadataSchema } from '@/lib/knowledge/upload-metadata' @@ -115,18 +119,26 @@ export function parseDocumentTagFiltersParam( return z.array(documentTagFilterSchema).parse(JSON.parse(value)) } +/** A text tag value that fits its index row; see {@link MAX_DOCUMENT_INDEXED_TEXT_LENGTH}. */ +const documentTagValueSchema = z + .string() + .max(MAX_DOCUMENT_INDEXED_TEXT_LENGTH, `Tag values cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters`) + export const createDocumentBodySchema = z.object({ - filename: z.string().min(1, 'Filename is required'), + filename: z + .string() + .min(1, 'Filename is required') + .max(MAX_DOCUMENT_INDEXED_TEXT_LENGTH, `Filename cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters`), fileUrl: knowledgeDocumentFileUrlSchema, fileSize: z.number().min(1, 'File size must be greater than 0'), mimeType: z.string().min(1, 'MIME type is required'), - tag1: z.string().optional(), - tag2: z.string().optional(), - tag3: z.string().optional(), - tag4: z.string().optional(), - tag5: z.string().optional(), - tag6: z.string().optional(), - tag7: z.string().optional(), + tag1: documentTagValueSchema.optional(), + tag2: documentTagValueSchema.optional(), + tag3: documentTagValueSchema.optional(), + tag4: documentTagValueSchema.optional(), + tag5: documentTagValueSchema.optional(), + tag6: documentTagValueSchema.optional(), + tag7: documentTagValueSchema.optional(), documentTagsData: z.string().optional(), }) @@ -165,7 +177,10 @@ export type SingleCreateDocumentBody = z.input ({ })) vi.mock('@/connectors/registry.server', () => ({ CONNECTOR_REGISTRY: { - fixture: { mapTags: (metadata: Record) => ({ label: metadata.label }) }, + fixture: { + mapTags: (metadata: Record) => ({ label: metadata.label, owner: metadata.owner }), + }, }, })) @@ -434,21 +436,22 @@ describe('organization source cache persistence', () => { }) describe('resolveTagMapping', () => { - it('bounds a mapped tag value that would exceed its index row limit', () => { + it('bounds a mapped tag value that would exceed its index row limit and keeps a short one intact', () => { const tags = resolveTagMapping( 'fixture', - { label: 'y'.repeat(5000) }, - { tagSlotMapping: { label: 'tag1' } } + { label: 'y'.repeat(5000), owner: 'Purchasing' }, + { tagSlotMapping: { label: 'tag1', owner: 'tag2' } } ) expect(tags?.tag1).toBe(`${'y'.repeat(512)}...`) + expect(tags?.tag2).toBe('Purchasing') }) - it('keeps a short mapped tag value intact', () => { + it('cuts by code point so a bounded value never ends in half a surrogate pair', () => { const tags = resolveTagMapping( 'fixture', - { label: 'Purchasing' }, + { label: '\u{1F600}'.repeat(600) }, { tagSlotMapping: { label: 'tag1' } } ) - expect(tags?.tag1).toBe('Purchasing') + expect(tags?.tag1).toBe(`${'\u{1F600}'.repeat(512)}...`) }) }) diff --git a/apps/sim/lib/knowledge/connectors/sync-persistence.ts b/apps/sim/lib/knowledge/connectors/sync-persistence.ts index 526861107bd..ef769c06e78 100644 --- a/apps/sim/lib/knowledge/connectors/sync-persistence.ts +++ b/apps/sim/lib/knowledge/connectors/sync-persistence.ts @@ -3,7 +3,6 @@ import { document, embedding, knowledgeBase, knowledgeConnector } from '@sim/db/ import { createLogger } from '@sim/logger' import { chunkArray } from '@sim/utils/helpers' import { generateId } from '@sim/utils/id' -import { truncate } from '@sim/utils/string' import { and, eq, exists, inArray, isNull, lt, or, sql } from 'drizzle-orm' import { getInternalApiBaseUrl } from '@/lib/core/utils/urls' import type { DbOrTx } from '@/lib/db/types' @@ -27,6 +26,7 @@ import { } from '@/lib/knowledge/documents/storage-upload' import { buildStorageKeySegment } from '@/lib/uploads/core/storage-key' import { getFileMetadataByKeys } from '@/lib/uploads/server/metadata' +import { MAX_DOCUMENT_INDEXED_TEXT_LENGTH } from '@/lib/knowledge/constants' import { CONNECTOR_REGISTRY } from '@/connectors/registry.server' import type { DocumentTags, ExternalDocument } from '@/connectors/types' @@ -175,16 +175,15 @@ export async function persistDocumentAcls( const MAX_SAFE_TITLE_LENGTH = 200 /** - * A document's filename and text tags sit under btree indexes, and Postgres refuses an index - * row past about 2.7 KB. A source title or tag value beyond that fails the row and, with it, - * every sync that lists the document again. 512 characters keeps a four-byte-per-character - * value inside the ceiling. + * Source titles and mapped tag values are untrusted machine input with no caller to refuse them, + * so they are cut to {@link MAX_DOCUMENT_INDEXED_TEXT_LENGTH} by code point, never inside a + * surrogate pair. */ -const MAX_INDEXED_TEXT_LENGTH = 512 - -/** Bounds a source-supplied value that lands in an indexed text column. */ function boundIndexedText(value: string): string { - return truncate(value, MAX_INDEXED_TEXT_LENGTH) + if (value.length <= MAX_DOCUMENT_INDEXED_TEXT_LENGTH) return value + const points = Array.from(value) + if (points.length <= MAX_DOCUMENT_INDEXED_TEXT_LENGTH) return value + return `${points.slice(0, MAX_DOCUMENT_INDEXED_TEXT_LENGTH).join('')}...` } function sanitizeStorageTitle(title: string): string { @@ -265,7 +264,7 @@ export function resolveTagMapping( for (const [semanticKey, slot] of Object.entries(mapping)) { const value = semanticTags[semanticKey] ;(result as Record)[slot] = - typeof value === 'string' ? boundIndexedText(value) : value != null ? value : null + typeof value === 'string' ? boundIndexedText(value) : (value ?? null) } return result } diff --git a/apps/sim/lib/knowledge/constants.ts b/apps/sim/lib/knowledge/constants.ts index b65aa6bd77f..f6de98cf0a0 100644 --- a/apps/sim/lib/knowledge/constants.ts +++ b/apps/sim/lib/knowledge/constants.ts @@ -3,6 +3,14 @@ import { MAX_FOLDERS_PER_WORKSPACE } from '@/lib/folders/constants' /** Max character length for a knowledge base description, enforced at every layer (UI, internal API, v1 API). */ export const KNOWLEDGE_BASE_DESCRIPTION_MAX_LENGTH = 10_000 +/** + * Max character length for a document's filename and text tag values. Both sit under btree + * indexes, and Postgres refuses an index row past about 2.7 KB (SQLSTATE 54000); 512 characters + * keeps a four-byte-per-character value inside that ceiling. Connectors truncate source titles to + * it; the document APIs reject longer input. + */ +export const MAX_DOCUMENT_INDEXED_TEXT_LENGTH = 512 + /** Hard bound for path-indexed knowledge folder trees and recursive cascades. */ export const MAX_KNOWLEDGE_FOLDERS_PER_WORKSPACE = MAX_FOLDERS_PER_WORKSPACE diff --git a/apps/sim/lib/knowledge/upload-metadata.test.ts b/apps/sim/lib/knowledge/upload-metadata.test.ts index 42cf71ed58c..c17dd77b3fe 100644 --- a/apps/sim/lib/knowledge/upload-metadata.test.ts +++ b/apps/sim/lib/knowledge/upload-metadata.test.ts @@ -7,8 +7,19 @@ import { knowledgeDocumentUploadMetadataSchema, persistedKnowledgeDocumentUploadMetadataSchema, } from '@/lib/knowledge/upload-metadata' +import { MAX_DOCUMENT_INDEXED_TEXT_LENGTH } from '@/lib/knowledge/constants' describe('knowledgeDocumentUploadMetadataSchema', () => { + it('rejects a tag value that would not fit its index row', () => { + const result = knowledgeDocumentUploadMetadataSchema.safeParse({ + tag1: 'a'.repeat(MAX_DOCUMENT_INDEXED_TEXT_LENGTH + 1), + }) + expect(result.success).toBe(false) + expect(result.error?.issues[0]?.message).toBe( + `Knowledge document tag values cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters` + ) + }) + it('rejects a recipe outside the accepted set', () => { const result = knowledgeDocumentUploadMetadataSchema.safeParse({ processingOptions: { recipe: 'totally-bogus-recipe' }, diff --git a/apps/sim/lib/knowledge/upload-metadata.ts b/apps/sim/lib/knowledge/upload-metadata.ts index 8b5c96c4ad9..6139e5464f1 100644 --- a/apps/sim/lib/knowledge/upload-metadata.ts +++ b/apps/sim/lib/knowledge/upload-metadata.ts @@ -1,5 +1,6 @@ import { z } from 'zod' import type { RecursiveRecipe } from '@/lib/chunkers/types' +import { MAX_DOCUMENT_INDEXED_TEXT_LENGTH } from '@/lib/knowledge/constants' /** * Recipes the recursive chunker implements. Mirrors `RecursiveRecipe`; the @@ -35,7 +36,10 @@ const LANGUAGE_TAG_SHAPE = /^[A-Za-z]{2,8}(?:-[A-Za-z0-9]{1,8})*$/ const knowledgeDocumentUploadTagSchema = z .string() - .max(1000, 'Knowledge document tag values cannot exceed 1000 characters') + .max( + MAX_DOCUMENT_INDEXED_TEXT_LENGTH, + `Knowledge document tag values cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters` + ) .optional() const knowledgeDocumentUploadTagShape = { From d187644c5af5cbb6d88356b8e75270de2d7e92e8 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 21 Sep 2026 20:25:51 -0700 Subject: [PATCH 4/5] fix(knowledge): cut bounded text with a shared surrogate-safe helper inside the limit and leave stored upload metadata untouched --- .../api/contracts/knowledge/documents.test.ts | 6 +++++- .../lib/api/contracts/knowledge/documents.ts | 21 +++++++++++++++---- .../connectors/sync-persistence.test.ts | 13 ++++++++---- .../knowledge/connectors/sync-persistence.ts | 19 ++++++++++------- .../sim/lib/knowledge/upload-metadata.test.ts | 11 ---------- apps/sim/lib/knowledge/upload-metadata.ts | 6 +----- packages/utils/src/string.test.ts | 17 +++++++++++++++ packages/utils/src/string.ts | 15 +++++++++++++ 8 files changed, 76 insertions(+), 32 deletions(-) diff --git a/apps/sim/lib/api/contracts/knowledge/documents.test.ts b/apps/sim/lib/api/contracts/knowledge/documents.test.ts index e6a3a52fc5b..3bbc3bc7c09 100644 --- a/apps/sim/lib/api/contracts/knowledge/documents.test.ts +++ b/apps/sim/lib/api/contracts/knowledge/documents.test.ts @@ -271,7 +271,11 @@ describe('document filename and tag bounds', () => { }) it('rejects a filename over the limit on create, upsert, and update with a descriptive message', () => { - for (const schema of [createDocumentBodySchema, upsertDocumentBodySchema, updateDocumentBodySchema]) { + for (const schema of [ + createDocumentBodySchema, + upsertDocumentBodySchema, + updateDocumentBodySchema, + ]) { const result = schema.safeParse({ ...base, filename: overLimit }) expect(result.success).toBe(false) expect(result.error?.issues[0]?.message).toBe( diff --git a/apps/sim/lib/api/contracts/knowledge/documents.ts b/apps/sim/lib/api/contracts/knowledge/documents.ts index 3d8de902eeb..cf61931c859 100644 --- a/apps/sim/lib/api/contracts/knowledge/documents.ts +++ b/apps/sim/lib/api/contracts/knowledge/documents.ts @@ -122,13 +122,19 @@ export function parseDocumentTagFiltersParam( /** A text tag value that fits its index row; see {@link MAX_DOCUMENT_INDEXED_TEXT_LENGTH}. */ const documentTagValueSchema = z .string() - .max(MAX_DOCUMENT_INDEXED_TEXT_LENGTH, `Tag values cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters`) + .max( + MAX_DOCUMENT_INDEXED_TEXT_LENGTH, + `Tag values cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters` + ) export const createDocumentBodySchema = z.object({ filename: z .string() .min(1, 'Filename is required') - .max(MAX_DOCUMENT_INDEXED_TEXT_LENGTH, `Filename cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters`), + .max( + MAX_DOCUMENT_INDEXED_TEXT_LENGTH, + `Filename cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters` + ), fileUrl: knowledgeDocumentFileUrlSchema, fileSize: z.number().min(1, 'File size must be greater than 0'), mimeType: z.string().min(1, 'MIME type is required'), @@ -180,7 +186,10 @@ export const upsertDocumentBodySchema = z.object({ filename: z .string() .min(1, 'Filename is required') - .max(MAX_DOCUMENT_INDEXED_TEXT_LENGTH, `Filename cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters`), + .max( + MAX_DOCUMENT_INDEXED_TEXT_LENGTH, + `Filename cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters` + ), fileUrl: knowledgeDocumentFileUrlSchema, fileSize: z.number().min(1, 'File size must be greater than 0'), mimeType: z.string().min(1, 'MIME type is required'), @@ -214,7 +223,11 @@ export const updateDocumentBodySchema = z.object({ filename: z .string() .min(1, 'Filename is required') - .max(MAX_DOCUMENT_INDEXED_TEXT_LENGTH, `Filename cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters`).optional(), + .max( + MAX_DOCUMENT_INDEXED_TEXT_LENGTH, + `Filename cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters` + ) + .optional(), enabled: z.boolean().optional(), chunkCount: z.number().min(0).optional(), tokenCount: z.number().min(0).optional(), diff --git a/apps/sim/lib/knowledge/connectors/sync-persistence.test.ts b/apps/sim/lib/knowledge/connectors/sync-persistence.test.ts index 2089bb3ea9c..ae29b4aabc7 100644 --- a/apps/sim/lib/knowledge/connectors/sync-persistence.test.ts +++ b/apps/sim/lib/knowledge/connectors/sync-persistence.test.ts @@ -30,7 +30,10 @@ vi.mock('@/lib/knowledge/documents/storage-cleanup', () => ({ vi.mock('@/connectors/registry.server', () => ({ CONNECTOR_REGISTRY: { fixture: { - mapTags: (metadata: Record) => ({ label: metadata.label, owner: metadata.owner }), + mapTags: (metadata: Record) => ({ + label: metadata.label, + owner: metadata.owner, + }), }, }, })) @@ -366,7 +369,8 @@ describe('persistSourceDocumentFailures', () => { priorByExternalId: new Map(), }) const [rows] = dbChainMockFns.values.mock.calls[0] as [Array<{ filename: string }>] - expect(rows[0].filename).toBe(`${'x'.repeat(512)}...`) + expect(rows[0].filename).toBe(`${'x'.repeat(509)}...`) + expect(rows[0].filename.length).toBe(512) }) it('refuses to commit a failure under a reclaimed lease', async () => { queueTableRows(schemaMock.knowledgeBase, [{ id: 'kb' }]) @@ -442,7 +446,7 @@ describe('resolveTagMapping', () => { { label: 'y'.repeat(5000), owner: 'Purchasing' }, { tagSlotMapping: { label: 'tag1', owner: 'tag2' } } ) - expect(tags?.tag1).toBe(`${'y'.repeat(512)}...`) + expect(tags?.tag1).toBe(`${'y'.repeat(509)}...`) expect(tags?.tag2).toBe('Purchasing') }) @@ -452,6 +456,7 @@ describe('resolveTagMapping', () => { { label: '\u{1F600}'.repeat(600) }, { tagSlotMapping: { label: 'tag1' } } ) - expect(tags?.tag1).toBe(`${'\u{1F600}'.repeat(512)}...`) + expect(tags?.tag1).toBe(`${'\u{1F600}'.repeat(254)}...`) + expect(tags?.tag1?.length).toBeLessThanOrEqual(512) }) }) diff --git a/apps/sim/lib/knowledge/connectors/sync-persistence.ts b/apps/sim/lib/knowledge/connectors/sync-persistence.ts index ef769c06e78..b5d2e8a221f 100644 --- a/apps/sim/lib/knowledge/connectors/sync-persistence.ts +++ b/apps/sim/lib/knowledge/connectors/sync-persistence.ts @@ -3,6 +3,7 @@ import { document, embedding, knowledgeBase, knowledgeConnector } from '@sim/db/ import { createLogger } from '@sim/logger' import { chunkArray } from '@sim/utils/helpers' import { generateId } from '@sim/utils/id' +import { truncateAtCodePoint } from '@sim/utils/string' import { and, eq, exists, inArray, isNull, lt, or, sql } from 'drizzle-orm' import { getInternalApiBaseUrl } from '@/lib/core/utils/urls' import type { DbOrTx } from '@/lib/db/types' @@ -18,6 +19,7 @@ import type { ConnectorFailureDiagnostic } from '@/lib/knowledge/connectors/conn import { resolveSourceModifiedAt } from '@/lib/knowledge/connectors/source-modified-at' import { SOURCE_CONTENT_ERROR } from '@/lib/knowledge/connectors/sync-limits' import { assertSyncLeaseHeldInTx, type SyncWriteLease } from '@/lib/knowledge/connectors/sync-lock' +import { MAX_DOCUMENT_INDEXED_TEXT_LENGTH } from '@/lib/knowledge/constants' import type { DocumentData } from '@/lib/knowledge/documents/service' import { enqueueKnowledgeStorageCleanup } from '@/lib/knowledge/documents/storage-cleanup' import { @@ -26,7 +28,6 @@ import { } from '@/lib/knowledge/documents/storage-upload' import { buildStorageKeySegment } from '@/lib/uploads/core/storage-key' import { getFileMetadataByKeys } from '@/lib/uploads/server/metadata' -import { MAX_DOCUMENT_INDEXED_TEXT_LENGTH } from '@/lib/knowledge/constants' import { CONNECTOR_REGISTRY } from '@/connectors/registry.server' import type { DocumentTags, ExternalDocument } from '@/connectors/types' @@ -174,16 +175,20 @@ export async function persistDocumentAcls( const MAX_SAFE_TITLE_LENGTH = 200 +/** The suffix a cut value carries, counted inside {@link MAX_DOCUMENT_INDEXED_TEXT_LENGTH}. */ +const INDEXED_TEXT_CUT_SUFFIX = '...' + /** * Source titles and mapped tag values are untrusted machine input with no caller to refuse them, - * so they are cut to {@link MAX_DOCUMENT_INDEXED_TEXT_LENGTH} by code point, never inside a - * surrogate pair. + * so they are cut to {@link MAX_DOCUMENT_INDEXED_TEXT_LENGTH} code units, suffix included, and + * never inside a surrogate pair. The result always passes the document APIs' own bound. */ function boundIndexedText(value: string): string { - if (value.length <= MAX_DOCUMENT_INDEXED_TEXT_LENGTH) return value - const points = Array.from(value) - if (points.length <= MAX_DOCUMENT_INDEXED_TEXT_LENGTH) return value - return `${points.slice(0, MAX_DOCUMENT_INDEXED_TEXT_LENGTH).join('')}...` + return truncateAtCodePoint( + value, + MAX_DOCUMENT_INDEXED_TEXT_LENGTH - INDEXED_TEXT_CUT_SUFFIX.length, + INDEXED_TEXT_CUT_SUFFIX + ) } function sanitizeStorageTitle(title: string): string { diff --git a/apps/sim/lib/knowledge/upload-metadata.test.ts b/apps/sim/lib/knowledge/upload-metadata.test.ts index c17dd77b3fe..42cf71ed58c 100644 --- a/apps/sim/lib/knowledge/upload-metadata.test.ts +++ b/apps/sim/lib/knowledge/upload-metadata.test.ts @@ -7,19 +7,8 @@ import { knowledgeDocumentUploadMetadataSchema, persistedKnowledgeDocumentUploadMetadataSchema, } from '@/lib/knowledge/upload-metadata' -import { MAX_DOCUMENT_INDEXED_TEXT_LENGTH } from '@/lib/knowledge/constants' describe('knowledgeDocumentUploadMetadataSchema', () => { - it('rejects a tag value that would not fit its index row', () => { - const result = knowledgeDocumentUploadMetadataSchema.safeParse({ - tag1: 'a'.repeat(MAX_DOCUMENT_INDEXED_TEXT_LENGTH + 1), - }) - expect(result.success).toBe(false) - expect(result.error?.issues[0]?.message).toBe( - `Knowledge document tag values cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters` - ) - }) - it('rejects a recipe outside the accepted set', () => { const result = knowledgeDocumentUploadMetadataSchema.safeParse({ processingOptions: { recipe: 'totally-bogus-recipe' }, diff --git a/apps/sim/lib/knowledge/upload-metadata.ts b/apps/sim/lib/knowledge/upload-metadata.ts index 6139e5464f1..8b5c96c4ad9 100644 --- a/apps/sim/lib/knowledge/upload-metadata.ts +++ b/apps/sim/lib/knowledge/upload-metadata.ts @@ -1,6 +1,5 @@ import { z } from 'zod' import type { RecursiveRecipe } from '@/lib/chunkers/types' -import { MAX_DOCUMENT_INDEXED_TEXT_LENGTH } from '@/lib/knowledge/constants' /** * Recipes the recursive chunker implements. Mirrors `RecursiveRecipe`; the @@ -36,10 +35,7 @@ const LANGUAGE_TAG_SHAPE = /^[A-Za-z]{2,8}(?:-[A-Za-z0-9]{1,8})*$/ const knowledgeDocumentUploadTagSchema = z .string() - .max( - MAX_DOCUMENT_INDEXED_TEXT_LENGTH, - `Knowledge document tag values cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters` - ) + .max(1000, 'Knowledge document tag values cannot exceed 1000 characters') .optional() const knowledgeDocumentUploadTagShape = { diff --git a/packages/utils/src/string.test.ts b/packages/utils/src/string.test.ts index 1a33db3ce04..b817c2fff8b 100644 --- a/packages/utils/src/string.test.ts +++ b/packages/utils/src/string.test.ts @@ -16,6 +16,7 @@ import { slugify, stripVersionSuffix, truncate, + truncateAtCodePoint, } from './string.js' describe('slugify', () => { @@ -304,3 +305,19 @@ describe('hasRegexMetacharacter', () => { } }) }) + +describe('truncateAtCodePoint', () => { + it('returns the input untouched when it fits', () => { + expect(truncateAtCodePoint('ab😀cd', 10)).toBe('ab😀cd') + }) + + it('cuts like truncate when the cut lands between code points', () => { + expect(truncateAtCodePoint('ab😀cd', 4)).toBe('ab😀...') + expect(truncateAtCodePoint('hello world', 8, ' …')).toBe('hello wo …') + }) + + it('moves the cut back one unit rather than splitting a surrogate pair', () => { + expect(truncateAtCodePoint('ab😀cd', 3)).toBe('ab...') + expect(truncateAtCodePoint('😀'.repeat(3), 3)).toBe('😀...') + }) +}) diff --git a/packages/utils/src/string.ts b/packages/utils/src/string.ts index 5518821940f..8cf8dbb5c33 100644 --- a/packages/utils/src/string.ts +++ b/packages/utils/src/string.ts @@ -33,6 +33,21 @@ export function truncate(str: string, sliceLength: number, suffix = '...'): stri return str.length > sliceLength ? str.slice(0, sliceLength) + suffix : str } +/** + * Like {@link truncate}, but never cuts inside a surrogate pair: when the code unit at the cut + * would split an astral character, the cut moves one unit earlier. Lengths are still counted in + * UTF-16 code units, so the result of a cut is at most `sliceLength + suffix.length` units. + * + * @example + * truncateAtCodePoint('ab😀cd', 3) // 'ab...' (the cut at 3 would split the emoji) + * truncateAtCodePoint('ab😀cd', 4) // 'ab😀...' + */ +export function truncateAtCodePoint(str: string, sliceLength: number, suffix = '...'): string { + if (str.length <= sliceLength) return str + const splitsPair = sliceLength > 0 && (str.charCodeAt(sliceLength - 1) & 0xfc00) === 0xd800 + return str.slice(0, splitsPair ? sliceLength - 1 : sliceLength) + suffix +} + /** * Lowercases `value` into the `[a-z0-9-]` charset: every run of other characters * becomes one hyphen, and leading and trailing hyphens are dropped. From 95960d552ab04261f48325ddd0d31a17519944e5 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 21 Sep 2026 20:35:14 -0700 Subject: [PATCH 5/5] fix(knowledge): leave values at the limit untouched and refuse over-long tag values on the tag-data and tag-update paths --- apps/sim/lib/knowledge/application/documents.ts | 6 ++++-- .../knowledge/connectors/sync-persistence.test.ts | 10 ++++++++++ .../lib/knowledge/connectors/sync-persistence.ts | 1 + apps/sim/lib/knowledge/documents/service.ts | 5 ++++- apps/sim/lib/knowledge/tags/utils.test.ts | 15 ++++++++++++++- apps/sim/lib/knowledge/tags/utils.ts | 12 ++++++++++++ 6 files changed, 45 insertions(+), 4 deletions(-) diff --git a/apps/sim/lib/knowledge/application/documents.ts b/apps/sim/lib/knowledge/application/documents.ts index dc584acb7ac..036aa97aea2 100644 --- a/apps/sim/lib/knowledge/application/documents.ts +++ b/apps/sim/lib/knowledge/application/documents.ts @@ -68,7 +68,7 @@ import { toKnowledgeTagFilterConditions, } from '@/lib/knowledge/tags/filter-resolution' import { getDocumentTagDefinitions } from '@/lib/knowledge/tags/service' -import { validateTagValue } from '@/lib/knowledge/tags/utils' +import { validateTagValue, validateTagValueLength } from '@/lib/knowledge/tags/utils' import { StorageService } from '@/lib/uploads' import { generateKnowledgeBaseFileKey } from '@/lib/uploads/contexts/knowledge-base/knowledge-base-file-manager' import { recordKnowledgeBaseFileOwnership } from '@/lib/uploads/server/metadata' @@ -252,7 +252,9 @@ async function resolveKnowledgeDocumentTagValueUpdates( `Tag "${definition.displayName}" requires a value; use null to clear it` ) } - const validationError = validateTagValue(definition.displayName, value, definition.fieldType) + const validationError = + validateTagValueLength(definition.displayName, value) ?? + validateTagValue(definition.displayName, value, definition.fieldType) if (validationError) { throw new OrchestrationError('validation', validationError) } diff --git a/apps/sim/lib/knowledge/connectors/sync-persistence.test.ts b/apps/sim/lib/knowledge/connectors/sync-persistence.test.ts index ae29b4aabc7..fd90b59c1d4 100644 --- a/apps/sim/lib/knowledge/connectors/sync-persistence.test.ts +++ b/apps/sim/lib/knowledge/connectors/sync-persistence.test.ts @@ -450,6 +450,16 @@ describe('resolveTagMapping', () => { expect(tags?.tag2).toBe('Purchasing') }) + it('keeps a value exactly at the limit untouched', () => { + const atLimit = 'z'.repeat(512) + const tags = resolveTagMapping( + 'fixture', + { label: atLimit }, + { tagSlotMapping: { label: 'tag1' } } + ) + expect(tags?.tag1).toBe(atLimit) + }) + it('cuts by code point so a bounded value never ends in half a surrogate pair', () => { const tags = resolveTagMapping( 'fixture', diff --git a/apps/sim/lib/knowledge/connectors/sync-persistence.ts b/apps/sim/lib/knowledge/connectors/sync-persistence.ts index b5d2e8a221f..48946ea7067 100644 --- a/apps/sim/lib/knowledge/connectors/sync-persistence.ts +++ b/apps/sim/lib/knowledge/connectors/sync-persistence.ts @@ -184,6 +184,7 @@ const INDEXED_TEXT_CUT_SUFFIX = '...' * never inside a surrogate pair. The result always passes the document APIs' own bound. */ function boundIndexedText(value: string): string { + if (value.length <= MAX_DOCUMENT_INDEXED_TEXT_LENGTH) return value return truncateAtCodePoint( value, MAX_DOCUMENT_INDEXED_TEXT_LENGTH - INDEXED_TEXT_CUT_SUFFIX.length, diff --git a/apps/sim/lib/knowledge/documents/service.ts b/apps/sim/lib/knowledge/documents/service.ts index 3d6820369cf..36732b8e036 100644 --- a/apps/sim/lib/knowledge/documents/service.ts +++ b/apps/sim/lib/knowledge/documents/service.ts @@ -179,6 +179,7 @@ import { parseNumberValue, uncompilableTagFilterError, validateTagValue, + validateTagValueLength, } from '@/lib/knowledge/tags/utils' import type { ProcessedDocumentTags } from '@/lib/knowledge/types' import { embeddingVectorValues } from '@/lib/knowledge/vector-columns' @@ -552,7 +553,9 @@ function resolveDocumentTags( const rawValue = typeof tag.value === 'string' ? tag.value.trim() : tag.value const actualFieldType = existingDef.fieldType || fieldType - const validationError = validateTagValue(tagName, String(rawValue), actualFieldType) + const validationError = + validateTagValueLength(tagName, String(rawValue)) ?? + validateTagValue(tagName, String(rawValue), actualFieldType) if (validationError) { typeErrors.push(validationError) } diff --git a/apps/sim/lib/knowledge/tags/utils.test.ts b/apps/sim/lib/knowledge/tags/utils.test.ts index d99b080eecc..078e7c563d6 100644 --- a/apps/sim/lib/knowledge/tags/utils.test.ts +++ b/apps/sim/lib/knowledge/tags/utils.test.ts @@ -2,7 +2,11 @@ * @vitest-environment node */ import { describe, expect, it } from 'vitest' -import { coerceTagFilterValue, validateTagValue } from '@/lib/knowledge/tags/utils' +import { + coerceTagFilterValue, + validateTagValue, + validateTagValueLength, +} from '@/lib/knowledge/tags/utils' describe('coerceTagFilterValue', () => { it('accepts exactly what validateTagValue accepts', () => { @@ -68,3 +72,12 @@ describe('validateTagValue', () => { expect(validateTagValue('name', 'anything', 'json')).toBeNull() }) }) + +describe('validateTagValueLength', () => { + it('accepts a value at the indexed-text limit and names the tag past it', () => { + expect(validateTagValueLength('Labels', 'a'.repeat(512))).toBeNull() + expect(validateTagValueLength('Labels', 'a'.repeat(513))).toBe( + 'Tag "Labels" cannot exceed 512 characters' + ) + }) +}) diff --git a/apps/sim/lib/knowledge/tags/utils.ts b/apps/sim/lib/knowledge/tags/utils.ts index 1d00da28889..4cd7ec4b869 100644 --- a/apps/sim/lib/knowledge/tags/utils.ts +++ b/apps/sim/lib/knowledge/tags/utils.ts @@ -1,4 +1,5 @@ import { OrchestrationError } from '@/lib/core/orchestration/types' +import { MAX_DOCUMENT_INDEXED_TEXT_LENGTH } from '@/lib/knowledge/constants' const DATE_ONLY_PATTERN = /^\d{4}-\d{2}-\d{2}$/ @@ -94,6 +95,17 @@ export function uncompilableTagFilterError(filter: { * Validate a tag value against its expected field type * Returns an error message if invalid, or null if valid */ +/** + * Text tag values sit under an index whose rows Postgres caps in size; a value past + * {@link MAX_DOCUMENT_INDEXED_TEXT_LENGTH} would fail the document write itself, so it is + * refused with a message naming the tag instead. + */ +export function validateTagValueLength(tagName: string, value: string): string | null { + return value.length > MAX_DOCUMENT_INDEXED_TEXT_LENGTH + ? `Tag "${tagName}" cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters` + : null +} + export function validateTagValue(tagName: string, value: string, fieldType: string): string | null { if (fieldType !== 'boolean' && fieldType !== 'number' && fieldType !== 'date') return null