diff --git a/apps/sim/lib/api/contracts/knowledge/documents.test.ts b/apps/sim/lib/api/contracts/knowledge/documents.test.ts index 9a12967d460..3bbc3bc7c09 100644 --- a/apps/sim/lib/api/contracts/knowledge/documents.test.ts +++ b/apps/sim/lib/api/contracts/knowledge/documents.test.ts @@ -5,11 +5,14 @@ import { describe, expect, it } from 'vitest' import { z } from 'zod' import { bulkCreateDocumentsBodySchema, + createDocumentBodySchema, documentDataSchema, listKnowledgeDocumentsQuerySchema, parseDocumentTagFiltersParam, + updateDocumentBodySchema, upsertDocumentBodySchema, } from '@/lib/api/contracts/knowledge/documents' +import { MAX_DOCUMENT_INDEXED_TEXT_LENGTH } from '@/lib/knowledge/constants' import { getDocumentIndexingStatus } from '@/lib/knowledge/documents/types' describe('document processing response compatibility', () => { @@ -255,3 +258,39 @@ describe('internal document processingOptions', () => { }) }) }) + +describe('document filename and tag bounds', () => { + const base = { fileUrl: 'https://example.com/a.txt', fileSize: 1, mimeType: 'text/plain' } + const atLimit = 'a'.repeat(MAX_DOCUMENT_INDEXED_TEXT_LENGTH) + const overLimit = `${atLimit}a` + + it('accepts a filename and tag exactly at the indexed-text limit', () => { + expect( + createDocumentBodySchema.safeParse({ ...base, filename: atLimit, tag1: atLimit }).success + ).toBe(true) + }) + + it('rejects a filename over the limit on create, upsert, and update with a descriptive message', () => { + for (const schema of [ + createDocumentBodySchema, + upsertDocumentBodySchema, + updateDocumentBodySchema, + ]) { + const result = schema.safeParse({ ...base, filename: overLimit }) + expect(result.success).toBe(false) + expect(result.error?.issues[0]?.message).toBe( + `Filename cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters` + ) + } + }) + + it('rejects a tag value over the limit on create and update', () => { + for (const schema of [createDocumentBodySchema, updateDocumentBodySchema]) { + const result = schema.safeParse({ ...base, filename: 'a.txt', tag3: overLimit }) + expect(result.success).toBe(false) + expect(result.error?.issues[0]?.message).toBe( + `Tag values cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters` + ) + } + }) +}) diff --git a/apps/sim/lib/api/contracts/knowledge/documents.ts b/apps/sim/lib/api/contracts/knowledge/documents.ts index 4b586ff6765..cf61931c859 100644 --- a/apps/sim/lib/api/contracts/knowledge/documents.ts +++ b/apps/sim/lib/api/contracts/knowledge/documents.ts @@ -15,7 +15,11 @@ import { import { privateSecretProvenanceBundleSchema } from '@/lib/api/contracts/primitives' import { defineRouteContract } from '@/lib/api/contracts/types' import { PRIVATE_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata' -import { getFieldTypeForSlot, MAX_KNOWLEDGE_DOCUMENTS_PER_CREATE } from '@/lib/knowledge/constants' +import { + getFieldTypeForSlot, + MAX_DOCUMENT_INDEXED_TEXT_LENGTH, + MAX_KNOWLEDGE_DOCUMENTS_PER_CREATE, +} from '@/lib/knowledge/constants' import { DOCUMENT_PROCESSING_STATUSES } from '@/lib/knowledge/documents/types' import { getOperatorsForFieldType, isValidFilterValue } from '@/lib/knowledge/filters/types' import { knowledgeDocumentUploadMetadataSchema } from '@/lib/knowledge/upload-metadata' @@ -115,18 +119,32 @@ export function parseDocumentTagFiltersParam( return z.array(documentTagFilterSchema).parse(JSON.parse(value)) } +/** A text tag value that fits its index row; see {@link MAX_DOCUMENT_INDEXED_TEXT_LENGTH}. */ +const documentTagValueSchema = z + .string() + .max( + MAX_DOCUMENT_INDEXED_TEXT_LENGTH, + `Tag values cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters` + ) + export const createDocumentBodySchema = z.object({ - filename: z.string().min(1, 'Filename is required'), + filename: z + .string() + .min(1, 'Filename is required') + .max( + MAX_DOCUMENT_INDEXED_TEXT_LENGTH, + `Filename cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters` + ), fileUrl: knowledgeDocumentFileUrlSchema, fileSize: z.number().min(1, 'File size must be greater than 0'), mimeType: z.string().min(1, 'MIME type is required'), - tag1: z.string().optional(), - tag2: z.string().optional(), - tag3: z.string().optional(), - tag4: z.string().optional(), - tag5: z.string().optional(), - tag6: z.string().optional(), - tag7: z.string().optional(), + tag1: documentTagValueSchema.optional(), + tag2: documentTagValueSchema.optional(), + tag3: documentTagValueSchema.optional(), + tag4: documentTagValueSchema.optional(), + tag5: documentTagValueSchema.optional(), + tag6: documentTagValueSchema.optional(), + tag7: documentTagValueSchema.optional(), documentTagsData: z.string().optional(), }) @@ -165,7 +183,13 @@ export type SingleCreateDocumentBody = z.input ({ }), isKnowledgeBaseOwnedStorageKey: (key: string) => key.startsWith('kb/'), })) -vi.mock('@/connectors/registry.server', () => ({ CONNECTOR_REGISTRY: {} })) +vi.mock('@/connectors/registry.server', () => ({ + CONNECTOR_REGISTRY: { + fixture: { + mapTags: (metadata: Record) => ({ + label: metadata.label, + owner: metadata.owner, + }), + }, + }, +})) import { MAX_ACL_TOKENS } from '@/lib/knowledge/access/tokens' import { getConnectorFailureDiagnostic } from '@/lib/knowledge/connectors/connector-error' @@ -35,6 +44,7 @@ import { addDocument, persistDocumentAcls, persistSourceDocumentFailures, + resolveTagMapping, } from '@/lib/knowledge/connectors/sync-persistence' const CONNECTOR = 'connector-1' @@ -350,6 +360,18 @@ describe('persistSourceDocumentFailures', () => { expect(JSON.stringify(dbChainMockFns.set.mock.calls)).not.toContain('private body') expect(dbChainMockFns.delete).not.toHaveBeenCalled() }) + it('bounds a source title that would exceed the filename index row limit', async () => { + leaseHeld() + const title = 'x'.repeat(5000) + await persistSourceDocumentFailures({ + ...input, + documents: [{ ...input.documents[0], title }], + priorByExternalId: new Map(), + }) + const [rows] = dbChainMockFns.values.mock.calls[0] as [Array<{ filename: string }>] + expect(rows[0].filename).toBe(`${'x'.repeat(509)}...`) + expect(rows[0].filename.length).toBe(512) + }) it('refuses to commit a failure under a reclaimed lease', async () => { queueTableRows(schemaMock.knowledgeBase, [{ id: 'kb' }]) await expect( @@ -416,3 +438,35 @@ describe('organization source cache persistence', () => { expect(mockUploadFile).not.toHaveBeenCalled() }) }) + +describe('resolveTagMapping', () => { + it('bounds a mapped tag value that would exceed its index row limit and keeps a short one intact', () => { + const tags = resolveTagMapping( + 'fixture', + { label: 'y'.repeat(5000), owner: 'Purchasing' }, + { tagSlotMapping: { label: 'tag1', owner: 'tag2' } } + ) + expect(tags?.tag1).toBe(`${'y'.repeat(509)}...`) + expect(tags?.tag2).toBe('Purchasing') + }) + + it('keeps a value exactly at the limit untouched', () => { + const atLimit = 'z'.repeat(512) + const tags = resolveTagMapping( + 'fixture', + { label: atLimit }, + { tagSlotMapping: { label: 'tag1' } } + ) + expect(tags?.tag1).toBe(atLimit) + }) + + it('cuts by code point so a bounded value never ends in half a surrogate pair', () => { + const tags = resolveTagMapping( + 'fixture', + { label: '\u{1F600}'.repeat(600) }, + { tagSlotMapping: { label: 'tag1' } } + ) + expect(tags?.tag1).toBe(`${'\u{1F600}'.repeat(254)}...`) + expect(tags?.tag1?.length).toBeLessThanOrEqual(512) + }) +}) diff --git a/apps/sim/lib/knowledge/connectors/sync-persistence.ts b/apps/sim/lib/knowledge/connectors/sync-persistence.ts index 33b53184ac2..48946ea7067 100644 --- a/apps/sim/lib/knowledge/connectors/sync-persistence.ts +++ b/apps/sim/lib/knowledge/connectors/sync-persistence.ts @@ -3,6 +3,7 @@ import { document, embedding, knowledgeBase, knowledgeConnector } from '@sim/db/ import { createLogger } from '@sim/logger' import { chunkArray } from '@sim/utils/helpers' import { generateId } from '@sim/utils/id' +import { truncateAtCodePoint } from '@sim/utils/string' import { and, eq, exists, inArray, isNull, lt, or, sql } from 'drizzle-orm' import { getInternalApiBaseUrl } from '@/lib/core/utils/urls' import type { DbOrTx } from '@/lib/db/types' @@ -18,6 +19,7 @@ import type { ConnectorFailureDiagnostic } from '@/lib/knowledge/connectors/conn import { resolveSourceModifiedAt } from '@/lib/knowledge/connectors/source-modified-at' import { SOURCE_CONTENT_ERROR } from '@/lib/knowledge/connectors/sync-limits' import { assertSyncLeaseHeldInTx, type SyncWriteLease } from '@/lib/knowledge/connectors/sync-lock' +import { MAX_DOCUMENT_INDEXED_TEXT_LENGTH } from '@/lib/knowledge/constants' import type { DocumentData } from '@/lib/knowledge/documents/service' import { enqueueKnowledgeStorageCleanup } from '@/lib/knowledge/documents/storage-cleanup' import { @@ -173,6 +175,23 @@ export async function persistDocumentAcls( const MAX_SAFE_TITLE_LENGTH = 200 +/** The suffix a cut value carries, counted inside {@link MAX_DOCUMENT_INDEXED_TEXT_LENGTH}. */ +const INDEXED_TEXT_CUT_SUFFIX = '...' + +/** + * Source titles and mapped tag values are untrusted machine input with no caller to refuse them, + * so they are cut to {@link MAX_DOCUMENT_INDEXED_TEXT_LENGTH} code units, suffix included, and + * never inside a surrogate pair. The result always passes the document APIs' own bound. + */ +function boundIndexedText(value: string): string { + if (value.length <= MAX_DOCUMENT_INDEXED_TEXT_LENGTH) return value + return truncateAtCodePoint( + value, + MAX_DOCUMENT_INDEXED_TEXT_LENGTH - INDEXED_TEXT_CUT_SUFFIX.length, + INDEXED_TEXT_CUT_SUFFIX + ) +} + function sanitizeStorageTitle(title: string): string { return title.replace(/[^a-zA-Z0-9.-]/g, '_').slice(0, MAX_SAFE_TITLE_LENGTH) } @@ -250,7 +269,8 @@ export function resolveTagMapping( const result: Partial = {} for (const [semanticKey, slot] of Object.entries(mapping)) { const value = semanticTags[semanticKey] - ;(result as Record)[slot] = value != null ? value : null + ;(result as Record)[slot] = + typeof value === 'string' ? boundIndexedText(value) : (value ?? null) } return result } @@ -278,7 +298,7 @@ function buildSkippedDocumentRow( return { id: generateId(), knowledgeBaseId, - filename: extDoc.title, + filename: boundIndexedText(extDoc.title), fileUrl: '', storageKey: null, /** No artifact was stored; a provider's reported source size is not local storage usage. */ @@ -630,7 +650,7 @@ export async function addDocument( await tx.insert(document).values({ id: documentId, knowledgeBaseId, - filename: extDoc.title, + filename: boundIndexedText(extDoc.title), fileUrl, storageKey: fileInfo.key, fileSize: artifact.bytes.length, @@ -745,7 +765,7 @@ export async function updateDocument( await tx .update(document) .set({ - filename: extDoc.title, + filename: boundIndexedText(extDoc.title), fileUrl, storageKey: fileInfo.key, fileSize: artifact.bytes.length, diff --git a/apps/sim/lib/knowledge/constants.ts b/apps/sim/lib/knowledge/constants.ts index b65aa6bd77f..f6de98cf0a0 100644 --- a/apps/sim/lib/knowledge/constants.ts +++ b/apps/sim/lib/knowledge/constants.ts @@ -3,6 +3,14 @@ import { MAX_FOLDERS_PER_WORKSPACE } from '@/lib/folders/constants' /** Max character length for a knowledge base description, enforced at every layer (UI, internal API, v1 API). */ export const KNOWLEDGE_BASE_DESCRIPTION_MAX_LENGTH = 10_000 +/** + * Max character length for a document's filename and text tag values. Both sit under btree + * indexes, and Postgres refuses an index row past about 2.7 KB (SQLSTATE 54000); 512 characters + * keeps a four-byte-per-character value inside that ceiling. Connectors truncate source titles to + * it; the document APIs reject longer input. + */ +export const MAX_DOCUMENT_INDEXED_TEXT_LENGTH = 512 + /** Hard bound for path-indexed knowledge folder trees and recursive cascades. */ export const MAX_KNOWLEDGE_FOLDERS_PER_WORKSPACE = MAX_FOLDERS_PER_WORKSPACE diff --git a/apps/sim/lib/knowledge/documents/service.ts b/apps/sim/lib/knowledge/documents/service.ts index 3d6820369cf..36732b8e036 100644 --- a/apps/sim/lib/knowledge/documents/service.ts +++ b/apps/sim/lib/knowledge/documents/service.ts @@ -179,6 +179,7 @@ import { parseNumberValue, uncompilableTagFilterError, validateTagValue, + validateTagValueLength, } from '@/lib/knowledge/tags/utils' import type { ProcessedDocumentTags } from '@/lib/knowledge/types' import { embeddingVectorValues } from '@/lib/knowledge/vector-columns' @@ -552,7 +553,9 @@ function resolveDocumentTags( const rawValue = typeof tag.value === 'string' ? tag.value.trim() : tag.value const actualFieldType = existingDef.fieldType || fieldType - const validationError = validateTagValue(tagName, String(rawValue), actualFieldType) + const validationError = + validateTagValueLength(tagName, String(rawValue)) ?? + validateTagValue(tagName, String(rawValue), actualFieldType) if (validationError) { typeErrors.push(validationError) } diff --git a/apps/sim/lib/knowledge/tags/utils.test.ts b/apps/sim/lib/knowledge/tags/utils.test.ts index d99b080eecc..078e7c563d6 100644 --- a/apps/sim/lib/knowledge/tags/utils.test.ts +++ b/apps/sim/lib/knowledge/tags/utils.test.ts @@ -2,7 +2,11 @@ * @vitest-environment node */ import { describe, expect, it } from 'vitest' -import { coerceTagFilterValue, validateTagValue } from '@/lib/knowledge/tags/utils' +import { + coerceTagFilterValue, + validateTagValue, + validateTagValueLength, +} from '@/lib/knowledge/tags/utils' describe('coerceTagFilterValue', () => { it('accepts exactly what validateTagValue accepts', () => { @@ -68,3 +72,12 @@ describe('validateTagValue', () => { expect(validateTagValue('name', 'anything', 'json')).toBeNull() }) }) + +describe('validateTagValueLength', () => { + it('accepts a value at the indexed-text limit and names the tag past it', () => { + expect(validateTagValueLength('Labels', 'a'.repeat(512))).toBeNull() + expect(validateTagValueLength('Labels', 'a'.repeat(513))).toBe( + 'Tag "Labels" cannot exceed 512 characters' + ) + }) +}) diff --git a/apps/sim/lib/knowledge/tags/utils.ts b/apps/sim/lib/knowledge/tags/utils.ts index 1d00da28889..4cd7ec4b869 100644 --- a/apps/sim/lib/knowledge/tags/utils.ts +++ b/apps/sim/lib/knowledge/tags/utils.ts @@ -1,4 +1,5 @@ import { OrchestrationError } from '@/lib/core/orchestration/types' +import { MAX_DOCUMENT_INDEXED_TEXT_LENGTH } from '@/lib/knowledge/constants' const DATE_ONLY_PATTERN = /^\d{4}-\d{2}-\d{2}$/ @@ -94,6 +95,17 @@ export function uncompilableTagFilterError(filter: { * Validate a tag value against its expected field type * Returns an error message if invalid, or null if valid */ +/** + * Text tag values sit under an index whose rows Postgres caps in size; a value past + * {@link MAX_DOCUMENT_INDEXED_TEXT_LENGTH} would fail the document write itself, so it is + * refused with a message naming the tag instead. + */ +export function validateTagValueLength(tagName: string, value: string): string | null { + return value.length > MAX_DOCUMENT_INDEXED_TEXT_LENGTH + ? `Tag "${tagName}" cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters` + : null +} + export function validateTagValue(tagName: string, value: string, fieldType: string): string | null { if (fieldType !== 'boolean' && fieldType !== 'number' && fieldType !== 'date') return null diff --git a/packages/utils/src/string.test.ts b/packages/utils/src/string.test.ts index 1a33db3ce04..b817c2fff8b 100644 --- a/packages/utils/src/string.test.ts +++ b/packages/utils/src/string.test.ts @@ -16,6 +16,7 @@ import { slugify, stripVersionSuffix, truncate, + truncateAtCodePoint, } from './string.js' describe('slugify', () => { @@ -304,3 +305,19 @@ describe('hasRegexMetacharacter', () => { } }) }) + +describe('truncateAtCodePoint', () => { + it('returns the input untouched when it fits', () => { + expect(truncateAtCodePoint('ab😀cd', 10)).toBe('ab😀cd') + }) + + it('cuts like truncate when the cut lands between code points', () => { + expect(truncateAtCodePoint('ab😀cd', 4)).toBe('ab😀...') + expect(truncateAtCodePoint('hello world', 8, ' …')).toBe('hello wo …') + }) + + it('moves the cut back one unit rather than splitting a surrogate pair', () => { + expect(truncateAtCodePoint('ab😀cd', 3)).toBe('ab...') + expect(truncateAtCodePoint('😀'.repeat(3), 3)).toBe('😀...') + }) +}) diff --git a/packages/utils/src/string.ts b/packages/utils/src/string.ts index 5518821940f..8cf8dbb5c33 100644 --- a/packages/utils/src/string.ts +++ b/packages/utils/src/string.ts @@ -33,6 +33,21 @@ export function truncate(str: string, sliceLength: number, suffix = '...'): stri return str.length > sliceLength ? str.slice(0, sliceLength) + suffix : str } +/** + * Like {@link truncate}, but never cuts inside a surrogate pair: when the code unit at the cut + * would split an astral character, the cut moves one unit earlier. Lengths are still counted in + * UTF-16 code units, so the result of a cut is at most `sliceLength + suffix.length` units. + * + * @example + * truncateAtCodePoint('ab😀cd', 3) // 'ab...' (the cut at 3 would split the emoji) + * truncateAtCodePoint('ab😀cd', 4) // 'ab😀...' + */ +export function truncateAtCodePoint(str: string, sliceLength: number, suffix = '...'): string { + if (str.length <= sliceLength) return str + const splitsPair = sliceLength > 0 && (str.charCodeAt(sliceLength - 1) & 0xfc00) === 0xd800 + return str.slice(0, splitsPair ? sliceLength - 1 : sliceLength) + suffix +} + /** * Lowercases `value` into the `[a-z0-9-]` charset: every run of other characters * becomes one hyphen, and leading and trailing hyphens are dropped.